Skip to content

Commit 55f83a5

Browse files
bbimberbbimber
authored
Merge pull request #32 from LabKey/fb_merge_discvr-20.3
Merge discvr 20.3 to develop
2 parents 9961561 + f0406ba commit 55f83a5

File tree

6 files changed

+50
-28
lines changed

6 files changed

+50
-28
lines changed

‎SequenceAnalysis/resources/queries/pipeline/Job/Sequence Jobs Without Data.qview.xml‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,14 @@
11
<customView xmlns="http://labkey.org/data/xml/queryCustomView" containerFilter="CurrentAndSubfolders">
22
<columns>
3+
<column name="RowId" />
34
<column name="Status" />
45
<column name="Description" />
5-
<column name="ActiveHostName"/>
66
<column name="Created" />
77
<column name="Provider"/>
8-
<column name="totalReadsets"/>
9-
<column name="totalAnalyses"/>
10-
<column name="totalOutputs"/>
11-
<column name="hasSequenceData"/>
8+
<column name="Folder/Path"/>
129
</columns>
1310
<filters>
14-
<filter column="hasSequenceData" operator="eq" value="false"/>
15-
<filter column="Provider" operator="in" value="sequenceOutputHandler;Sequence Pipeline;sequenceReadsetHandler"/>
16-
<filter column="Status" operator="eq" value="COMPLETE"/>
11+
<filter column="sequenceJobWithoutData" operator="eq" value="true"/>
1712
</filters>
1813
<sorts>
1914
<sort column="Created" descending="true"/>

‎SequenceAnalysis/resources/queries/pipeline/Job/Sequence Jobs.qview.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
<column name="totalReadsets"/>
99
<column name="totalAnalyses"/>
1010
<column name="totalOutputs"/>
11-
<column name="hasSequenceData"/>
11+
<column name="sequenceJobWithoutData"/>
1212
</columns>
1313
<filters>
1414
<!--<filter column="Provider" operator="neqornull" value="ETL"/>-->

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/button/CellHashingButton.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,4 +19,4 @@ public CellHashingButton()
1919
super(ModuleLoader.getInstance().getModule(SequenceAnalysisModule.class), "Generate Cell Hashing Calls", "SequenceAnalysis.window.OutputHandlerWindow.readsetButtonHandler(dataRegionName, '" + CellHashingHandler.class.getName() + "');", Arrays.asList(ClientDependency.supplierFromModuleName("ldk"), ClientDependency.supplierFromModuleName("laboratory"), ClientDependency.supplierFromPath("sequenceanalysis/window/OutputHandlerWindow.js")));
2020
setPermission(UpdatePermission.class);
2121
}
22-
}
22+
}

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/button/CiteSeqButton.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,4 +23,4 @@ public CiteSeqButton()
2323
ClientDependency.supplierFromPath("sequenceanalysis/window/OutputHandlerWindow.js")));
2424
setPermission(UpdatePermission.class);
2525
}
26-
}
26+
}

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/OrphanFilePipelineJob.java‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import org.labkey.api.pipeline.AbstractTaskFactorySettings;
1919
import org.labkey.api.pipeline.PipeRoot;
2020
import org.labkey.api.pipeline.PipelineJob;
21+
import org.labkey.api.pipeline.PipelineJobException;
2122
import org.labkey.api.pipeline.PipelineJobService;
2223
import org.labkey.api.pipeline.PipelineService;
2324
import org.labkey.api.pipeline.PipelineStatusFile;
@@ -33,11 +34,14 @@
3334
import org.labkey.api.util.PageFlowUtil;
3435
import org.labkey.api.view.ActionURL;
3536
import org.labkey.api.view.ViewBackgroundInfo;
37+
import org.labkey.api.writer.PrintWriters;
3638
import org.labkey.sequenceanalysis.SequenceAnalysisSchema;
3739
import org.labkey.sequenceanalysis.util.SequenceUtil;
3840

3941
import java.io.File;
4042
import java.io.FileFilter;
43+
import java.io.IOException;
44+
import java.io.PrintWriter;
4145
import java.net.URI;
4246
import java.net.URISyntaxException;
4347
import java.sql.ResultSet;
@@ -131,7 +135,7 @@ public boolean isJobComplete(PipelineJob job)
131135
}
132136

133137
@Override
134-
public RecordedActionSet run()
138+
public RecordedActionSet run() throws PipelineJobException
135139
{
136140
getJob().getLogger().info("## The following sections list any files or pipeline jobs that appear to be orphans, not connected to any imported readsets or sequence outputs:");
137141

@@ -205,8 +209,25 @@ public RecordedActionSet run()
205209
if (!probableDeletes.isEmpty())
206210
{
207211
StringBuilder sb = new StringBuilder();
212+
File output = new File(getJob().getLogFile().getParentFile(), "toRemove.sh");
213+
try (PrintWriter writer = PrintWriters.getPrintWriter(output))
214+
{
215+
writer.println("#!/bin/bash");
216+
writer.println("");
217+
writer.println("set -e");
218+
writer.println("set -x");
219+
writer.println("");
220+
probableDeletes.forEach(f -> writer.println("rm -Rf " + f.getPath()));
221+
}
222+
catch (IOException e)
223+
{
224+
throw new PipelineJobException(e);
225+
}
226+
208227
probableDeletes.forEach(f -> sb.append("\n").append(f.getPath()));
209-
getJob().getLogger().info("## The following files can almost certainly be deleted; however, please exercise caution:" + sb.toString());
228+
getJob().getLogger().info("## The following files can almost certainly be deleted; however, please exercise caution. Note: the file toRemove.sh has been written and can be executed to remove these:" + sb.toString());
229+
230+
210231
}
211232

212233
return new RecordedActionSet();

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/query/JobsTableCustomizer.java‎

Lines changed: 21 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import org.labkey.api.data.TableCustomizer;
77
import org.labkey.api.data.TableInfo;
88
import org.labkey.api.ldk.LDKService;
9+
import org.labkey.api.pipeline.PipelineJob;
910
import org.labkey.api.query.ExprColumn;
1011

1112
/**
@@ -58,22 +59,27 @@ public void customize(TableInfo ti)
5859
((AbstractTableInfo)ti).addColumn(newCol);
5960
}
6061

61-
String hasSequenceData = "hasSequenceData";
62-
if (ti.getColumn(hasSequenceData) == null)
62+
String sequenceJobWithoutData = "sequenceJobWithoutData";
63+
if (ti.getColumn(sequenceJobWithoutData) == null)
6364
{
64-
SQLFragment sql = new SQLFragment("(SELECT CASE " +
65-
"WHEN sum(CASE WHEN (rs.RowId IS NULL AND rd.RowId IS NULL AND sa.RowId IS NULL AND o.RowId IS NULL) THEN 0 ELSE 1 END) = 0 THEN " + ti.getSqlDialect().getBooleanFALSE() + "\n" +
66-
"WHEN sum(CASE WHEN (rs.RowId IS NULL AND rd.RowId IS NULL AND sa.RowId IS NULL AND o.RowId IS NULL) THEN 0 ELSE 1 END) IS NULL THEN " + ti.getSqlDialect().getBooleanFALSE() + "\n" +
67-
"ELSE " + ti.getSqlDialect().getBooleanTRUE() + " END as expr\n" +
68-
"FROM exp.experimentrun r\n" +
69-
"LEFT JOIN sequenceanalysis.sequence_readsets rs ON (r.RowId = rs.runid)\n" +
70-
"LEFT JOIN sequenceanalysis.readdata rd on (r.RowId = rd.runid)\n" +
71-
"LEFT JOIN sequenceanalysis.sequence_analyses sa ON (r.RowId = sa.runid)\n" +
72-
"LEFT JOIN sequenceanalysis.outputfiles o ON (r.RowId = o.runid)\n" +
73-
"WHERE (r.jobid = " + ExprColumn.STR_TABLE_ALIAS + ".RowId OR r.jobid = (SELECT p.RowId FROM pipeline.statusfiles p WHERE " + ExprColumn.STR_TABLE_ALIAS + ".jobparent = p.job))\n" +
74-
")");
75-
ExprColumn newCol = new ExprColumn(ti, hasSequenceData, sql, JdbcType.BOOLEAN, ti.getColumn("RowId"));
76-
newCol.setLabel("Has Sequence Data?");
65+
//Jobs from all exp runs connected to sequence data
66+
String runSubquery = "(SELECT DISTINCT r.jobid FROM exp.experimentrun r\n" +
67+
"WHERE\n" +
68+
"r.jobid IS NOT NULL AND (\n" +
69+
"EXISTS (SELECT rs.rowid FROM sequenceanalysis.sequence_readsets rs WHERE (r.RowId = rs.runid)) OR\n" +
70+
"EXISTS (SELECT rd.rowid FROM sequenceanalysis.readdata rd WHERE (r.RowId = rd.runid)) OR\n" +
71+
"EXISTS (SELECT sa.rowid FROM sequenceanalysis.sequence_analyses sa WHERE (r.RowId = sa.runid)) OR\n" +
72+
"EXISTS (SELECT o.runid FROM sequenceanalysis.outputfiles o WHERE (r.RowId = o.runid))\n" +
73+
"))";
74+
75+
SQLFragment sql = new SQLFragment("(CASE " +
76+
"WHEN " + ExprColumn.STR_TABLE_ALIAS + ".status IS NULL OR " + ExprColumn.STR_TABLE_ALIAS + ".status != '"+ PipelineJob.TaskStatus.complete.name().toUpperCase() + "' THEN " + ti.getSqlDialect().getBooleanFALSE() + "\n" +
77+
"WHEN " + ExprColumn.STR_TABLE_ALIAS + ".provider IS NULL OR " + ExprColumn.STR_TABLE_ALIAS + ".provider NOT IN ('sequenceOutputHandler', 'Sequence Pipeline', 'sequenceReadsetHandler') THEN " + ti.getSqlDialect().getBooleanFALSE() + "\n" +
78+
"WHEN " + ExprColumn.STR_TABLE_ALIAS + ".RowId IN " + runSubquery + " THEN " + ti.getSqlDialect().getBooleanFALSE() + "\n" +
79+
"WHEN " + ExprColumn.STR_TABLE_ALIAS + ".jobparent IS NOT NULL AND (SELECT p.RowId FROM pipeline.statusfiles p WHERE " + ExprColumn.STR_TABLE_ALIAS + ".jobparent = p.job) IN " + runSubquery + " THEN " + ti.getSqlDialect().getBooleanFALSE() + "\n" +
80+
"ELSE " + ti.getSqlDialect().getBooleanTRUE() + " END)");
81+
ExprColumn newCol = new ExprColumn(ti, sequenceJobWithoutData, sql, JdbcType.BOOLEAN, ti.getColumn("RowId"));
82+
newCol.setLabel("Sequence Job Without Data?");
7783
((AbstractTableInfo)ti).addColumn(newCol);
7884
}
7985

0 commit comments

Comments
 (0)