Skip to content

Commit 20eae1c

Browse files
bbimberbbimber
authored
Merge pull request #315 from LabKey/fb_merge_24.7_to_develop
Merge 24.7 to develop
2 parents 4352276 + bcc1703 commit 20eae1c

File tree

42 files changed

+4244
-2827
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

42 files changed

+4244
-2827
lines changed

‎SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/SequenceOutputHandler.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -198,7 +198,7 @@ default void init(JobContext ctx, List<SequenceOutputFile> inputFiles, List<Reco
198198

199199
void processFilesRemote(List<SequenceOutputFile> inputFiles, JobContext ctx) throws UnsupportedOperationException, PipelineJobException;
200200

201-
default void complete(PipelineJob job, List<SequenceOutputFile> inputs, List<SequenceOutputFile> outputsCreated, SequenceAnalysisJobSupport support) throws PipelineJobException
201+
default void complete(JobContext ctx, List<SequenceOutputFile> inputs, List<SequenceOutputFile> outputsCreated) throws PipelineJobException
202202
{
203203

204204
}

‎SequenceAnalysis/pipeline_code/sequence_tools_install.sh‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -457,12 +457,12 @@ then
457457
rm -Rf $LKTOOLS_DIR/samtools
458458
rm -Rf $LKTOOLS_DIR/bcftools
459459

460-
wget $WGET_OPTS https://github.com/samtools/samtools/releases/download/1.16.1/samtools-1.16.1.tar.bz2
461-
bunzip2 samtools-1.16.1.tar.bz2
462-
tar -xf samtools-1.16.1.tar
460+
wget $WGET_OPTS https://github.com/samtools/samtools/releases/download/1.20/samtools-1.20.tar.bz2
461+
bunzip2 samtools-1.20.tar.bz2
462+
tar -xf samtools-1.20.tar
463463
echo "Compressing TAR"
464-
bzip2 samtools-1.16.1.tar
465-
cd samtools-1.16.1
464+
bzip2 samtools-1.20.tar
465+
cd samtools-1.20
466466
./configure
467467
make
468468
install ./samtools ${LKTOOLS_DIR}/samtools
@@ -519,13 +519,13 @@ then
519519
rm -Rf $LKTOOLS_DIR/tabix
520520
rm -Rf $LKTOOLS_DIR/bgzip
521521

522-
wget $WGET_OPTS https://github.com/samtools/htslib/releases/download/1.16/htslib-1.16.tar.bz2
523-
bunzip2 htslib-1.16.tar.bz2
524-
tar -xf htslib-1.16.tar
522+
wget $WGET_OPTS https://github.com/samtools/htslib/releases/download/1.20/htslib-1.20.tar.bz2
523+
bunzip2 htslib-1.20.tar.bz2
524+
tar -xf htslib-1.20.tar
525525
echo "Compressing TAR"
526-
bzip2 htslib-1.16.tar
527-
chmod 755 htslib-1.16
528-
cd htslib-1.16
526+
bzip2 htslib-1.20.tar
527+
chmod 755 htslib-1.20
528+
cd htslib-1.20
529529
./configure
530530
make
531531

‎SequenceAnalysis/resources/queries/sequenceanalysis/alignment_summary_grouped.query.xml‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@
77
<pkColumnName>rowids</pkColumnName>
88
<columns>
99
<column columnName="rowids">
10+
<columnTitle>RowIds</columnTitle>
11+
<isHidden>true</isHidden>
1012
<isKeyField>true</isKeyField>
1113
</column>
1214
<column columnName="alleles">
@@ -50,10 +52,6 @@
5052
</conditionalFormat>
5153
</conditionalFormats>
5254
</column>
53-
<column columnName="rowids">
54-
<columnTitle>RowIds</columnTitle>
55-
<isHidden>true</isHidden>
56-
</column>
5755
<column columnName="nAlignments">
5856
<columnTitle># Alignments</columnTitle>
5957
<isHidden>true</isHidden>

‎SequenceAnalysis/resources/queries/sequenceanalysis/outputfile_categories.sql‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,19 +9,19 @@ FROM sequenceanalysis.outputfiles
99

1010
UNION ALL
1111

12-
SELECT 'VCF File'
12+
SELECT 'VCF File' as category
1313

1414
UNION ALL
1515

16-
SELECT 'BAM File'
16+
SELECT 'BAM File' as category
1717

1818
UNION ALL
1919

20-
SELECT 'BED File'
20+
SELECT 'BED File' as category
2121

2222
UNION ALL
2323

24-
SELECT 'gVCF File'
24+
SELECT 'gVCF File' as category
2525

2626
) t
2727

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/ReadDataImpl.java‎

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -201,18 +201,28 @@ public void setModifiedBy(Integer modifiedBy)
201201
_modifiedBy = modifiedBy;
202202
}
203203

204+
public File getFile1(boolean allowArchived)
205+
{
206+
return getFile(1, _fileId1, allowArchived);
207+
}
208+
204209
@Override
205210
@Transient
206211
public File getFile1()
207212
{
208-
return getFile(1, _fileId1);
213+
return getFile1(false);
209214
}
210215

211216
@Override
212217
@Transient
213218
public File getFile2()
214219
{
215-
return getFile(2, _fileId2);
220+
return getFile2(false);
221+
}
222+
223+
public File getFile2(boolean allowArchived)
224+
{
225+
return getFile(2, _fileId2, false);
216226
}
217227

218228
public void setFile(File f, int fileIdx)
@@ -248,9 +258,9 @@ public Integer getTotalReads()
248258
}
249259

250260
@Transient
251-
private File getFile(int fileIdx, Integer fileId)
261+
private File getFile(int fileIdx, Integer fileId, boolean allowArchived)
252262
{
253-
if (isArchived())
263+
if (isArchived() && !allowArchived)
254264
{
255265
return null;
256266
}
@@ -274,13 +284,16 @@ private File getFile(int fileIdx, Integer fileId)
274284
ret = d.getFile();
275285
}
276286

277-
if (ret != null && !ret.exists())
287+
if (!isArchived() && ret != null && !ret.exists())
278288
{
279289
throw new IllegalArgumentException("File does not exist: " + ret.getPath());
280290
}
281291
}
282292

283-
_cachedFiles.put(fileIdx, ret);
293+
if (!isArchived())
294+
{
295+
_cachedFiles.put(fileIdx, ret);
296+
}
284297

285298
return ret;
286299
}

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/PicardAlignmentMetricsHandler.java‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -126,9 +126,9 @@ public void processFilesOnWebserver(PipelineJob job, SequenceAnalysisJobSupport
126126
}
127127

128128
@Override
129-
public void complete(PipelineJob job, List<SequenceOutputFile> inputs, List<SequenceOutputFile> outputsCreated, SequenceAnalysisJobSupport support) throws PipelineJobException
129+
public void complete(JobContext ctx, List<SequenceOutputFile> inputs, List<SequenceOutputFile> outputsCreated) throws PipelineJobException
130130
{
131-
if (!(job instanceof SequenceOutputHandlerJob shj))
131+
if (!(ctx.getJob() instanceof SequenceOutputHandlerJob shj))
132132
{
133133
throw new IllegalStateException("Expected job to be a SequenceOutputHandlerJob");
134134
}
@@ -144,10 +144,10 @@ public void complete(PipelineJob job, List<SequenceOutputFile> inputs, List<Sequ
144144
Integer analysisId = o.getAnalysis_id();
145145
if (analysisId == null)
146146
{
147-
job.getLogger().warn("no analysis Id for file, attempting to find this job: " + o.getName());
148-
PipelineStatusFile sf = PipelineService.get().getStatusFile(job.getJobGUID());
147+
ctx.getJob().getLogger().warn("no analysis Id for file, attempting to find this job: " + o.getName());
148+
PipelineStatusFile sf = PipelineService.get().getStatusFile(ctx.getJob().getJobGUID());
149149

150-
TableSelector ts = new TableSelector(QueryService.get().getUserSchema(job.getUser(), job.getContainer(), SequenceAnalysisSchema.SCHEMA_NAME).getTable(SequenceAnalysisSchema.TABLE_ANALYSES), PageFlowUtil.set("rowid"), new SimpleFilter(FieldKey.fromString("runid/JobId"), sf.getRowId()), null);
150+
TableSelector ts = new TableSelector(QueryService.get().getUserSchema(ctx.getJob().getUser(), ctx.getJob().getContainer(), SequenceAnalysisSchema.SCHEMA_NAME).getTable(SequenceAnalysisSchema.TABLE_ANALYSES), PageFlowUtil.set("rowid"), new SimpleFilter(FieldKey.fromString("runid/JobId"), sf.getRowId()), null);
151151
if (ts.exists())
152152
{
153153
analysisId = ts.getObject(Integer.class);
@@ -160,15 +160,15 @@ public void complete(PipelineJob job, List<SequenceOutputFile> inputs, List<Sequ
160160

161161
if (o.getLibrary_id() == null)
162162
{
163-
job.getLogger().warn("no genome associated with file: " + o.getName());
163+
ctx.getJob().getLogger().warn("no genome associated with file: " + o.getName());
164164
continue;
165165
}
166166

167-
AnalysisModel m = AnalysisModelImpl.getFromDb(analysisId, job.getUser());
167+
AnalysisModel m = AnalysisModelImpl.getFromDb(analysisId, ctx.getJob().getUser());
168168
if (m != null)
169169
{
170-
job.getLogger().warn("processing analysis: " + m.getRowId());
171-
File outputDir = ((SequenceOutputHandlerJob)job).getWebserverDir(false);
170+
ctx.getJob().getLogger().warn("processing analysis: " + m.getRowId());
171+
File outputDir = ((SequenceOutputHandlerJob)ctx.getJob()).getWebserverDir(false);
172172
List<File> metricsFiles = new ArrayList<>();
173173

174174
File mf = new File(outputDir, FileUtil.getBaseName(o.getFile()) + ".summary.metrics");
@@ -191,7 +191,7 @@ else if (collectInsertSize)
191191
// This output is only created for paired data:
192192
if (o.getReadset() != null)
193193
{
194-
Readset rs = SequenceAnalysisService.get().getReadset(o.getReadset(), job.getUser());
194+
Readset rs = SequenceAnalysisService.get().getReadset(o.getReadset(), ctx.getJob().getUser());
195195
if (rs.getReadData().stream().filter(rd -> rd.getFileId2() != null).count() > 0)
196196
{
197197
throw new PipelineJobException("Missing file: " + mf2.getPath());
@@ -219,7 +219,7 @@ else if (collectWgsNonZero)
219219
throw new PipelineJobException("Missing file: " + mf4.getPath());
220220
}
221221

222-
File mf5 = new MarkDuplicatesWrapper(job.getLogger()).getMetricsFile(o.getFile());
222+
File mf5 = new MarkDuplicatesWrapper(ctx.getJob().getLogger()).getMetricsFile(o.getFile());
223223
if (mf5.exists())
224224
{
225225
metricsFiles.add(mf5);
@@ -232,23 +232,23 @@ else if (runMarkDuplicates)
232232
TableInfo ti = SequenceAnalysisManager.get().getTable(SequenceAnalysisSchema.TABLE_QUALITY_METRICS);
233233
for (File f : metricsFiles)
234234
{
235-
List<Map<String, Object>> lines = PicardMetricsUtil.processFile(f, job.getLogger());
235+
List<Map<String, Object>> lines = PicardMetricsUtil.processFile(f, ctx.getJob().getLogger());
236236
for (Map<String, Object> row : lines)
237237
{
238238
row.put("container", o.getContainer());
239-
row.put("createdby", job.getUser().getUserId());
239+
row.put("createdby", ctx.getJob().getUser().getUserId());
240240
row.put("created", new Date());
241241
row.put("readset", m.getReadset());
242242
row.put("analysis_id", m.getRowId());
243243
row.put("dataid", m.getAlignmentFile());
244244

245-
Table.insert(job.getUser(), ti, row);
245+
Table.insert(ctx.getJob().getUser(), ti, row);
246246
}
247247
}
248248
}
249249
else
250250
{
251-
job.getLogger().warn("Analysis Id " + o.getAnalysis_id() + " not found for file: " + o.getName());
251+
ctx.getJob().getLogger().warn("Analysis Id " + o.getAnalysis_id() + " not found for file: " + o.getName());
252252
}
253253
}
254254
}

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ConvertToCramHandler.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -161,7 +161,7 @@ private void checkCramAndIndex(SequenceOutputFile so) throws PipelineJobExceptio
161161
}
162162

163163
@Override
164-
public void complete(PipelineJob job, List<SequenceOutputFile> inputs, List<SequenceOutputFile> outputsCreated, SequenceAnalysisJobSupport support) throws PipelineJobException
164+
public void complete(JobContext ctx, List<SequenceOutputFile> inputs, List<SequenceOutputFile> outputsCreated) throws PipelineJobException
165165
{
166166
List<Map<String, Object>> toUpdate = new ArrayList<>();
167167
List<Map<String, Object>> oldKeys = inputs.stream().map(x -> {
@@ -175,11 +175,11 @@ public void complete(PipelineJob job, List<SequenceOutputFile> inputs, List<Sequ
175175
File cram = new File(so.getFile().getParentFile(), FileUtil.getBaseName(so.getFile()) + ".cram");
176176
checkCramAndIndex(so);
177177

178-
job.getLogger().info("Updating ExpData record with new filepath: " + cram.getPath());
178+
ctx.getJob().getLogger().info("Updating ExpData record with new filepath: " + cram.getPath());
179179
ExpData d = so.getExpData();
180180
d.setDataFileURI(cram.toURI());
181181
d.setName(cram.getName());
182-
d.save(job.getUser());
182+
d.save(ctx.getJob().getUser());
183183

184184
if (so.getName().contains(".bam"))
185185
{
@@ -194,8 +194,8 @@ public void complete(PipelineJob job, List<SequenceOutputFile> inputs, List<Sequ
194194

195195
try
196196
{
197-
Container target = job.getContainer().isWorkbook() ? job.getContainer().getParent() : job.getContainer();
198-
QueryService.get().getUserSchema(job.getUser(), target, SequenceAnalysisSchema.SCHEMA_NAME).getTable(SequenceAnalysisSchema.TABLE_OUTPUTFILES).getUpdateService().updateRows(job.getUser(), target, toUpdate, oldKeys, null, null);
197+
Container target = ctx.getJob().getContainer().isWorkbook() ? ctx.getJob().getContainer().getParent() : ctx.getJob().getContainer();
198+
QueryService.get().getUserSchema(ctx.getJob().getUser(), target, SequenceAnalysisSchema.SCHEMA_NAME).getTable(SequenceAnalysisSchema.TABLE_OUTPUTFILES).getUpdateService().updateRows(ctx.getJob().getUser(), target, toUpdate, oldKeys, null, null);
199199
}
200200
catch (QueryUpdateServiceException | InvalidKeyException | BatchValidationException | SQLException e)
201201
{

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ProcessVariantsHandler.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -722,14 +722,14 @@ public void processFilesOnWebserver(PipelineJob job, SequenceAnalysisJobSupport
722722
}
723723

724724
@Override
725-
public void complete(PipelineJob job, List<SequenceOutputFile> inputs, List<SequenceOutputFile> outputsCreated, SequenceAnalysisJobSupport support) throws PipelineJobException
725+
public void complete(JobContext ctx, List<SequenceOutputFile> inputs, List<SequenceOutputFile> outputsCreated) throws PipelineJobException
726726
{
727-
SequenceTaskHelper taskHelper = new SequenceTaskHelper(getPipelineJob(job), getPipelineJob(job).getDataDirectory());
728-
List<PipelineStepCtx<VariantProcessingStep>> providers = SequencePipelineService.get().getSteps(job, VariantProcessingStep.class);
727+
SequenceTaskHelper taskHelper = new SequenceTaskHelper(getPipelineJob(ctx.getJob()), getPipelineJob(ctx.getJob()).getDataDirectory());
728+
List<PipelineStepCtx<VariantProcessingStep>> providers = SequencePipelineService.get().getSteps(ctx.getJob(), VariantProcessingStep.class);
729729
for (PipelineStepCtx<VariantProcessingStep> stepCtx : providers)
730730
{
731731
VariantProcessingStep step = stepCtx.getProvider().create(taskHelper);
732-
step.complete(job, inputs, outputsCreated, support);
732+
step.complete(ctx.getJob(), inputs, outputsCreated, ctx.getSequenceSupport());
733733
}
734734
}
735735
}

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ReadsetInitTask.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -218,7 +218,9 @@ public RecordedActionSet run() throws PipelineJobException
218218
for (SequenceReadsetImpl rs : readsets)
219219
{
220220
getJob().getLogger().debug("caching readset: " + rs.getName() + " with " + rs.getReadData().size() + " files");
221-
getPipelineJob().getSequenceSupport().cacheReadset(rs);
221+
222+
// NOTE: allow archived data
223+
getPipelineJob().getSequenceSupport().cacheReadset(rs, true);
222224
}
223225
}
224226
catch (IOException e)
@@ -505,10 +507,10 @@ private void checkForDuplicateFileNames(List<SequenceReadsetImpl> readsets, List
505507
{
506508
Map<String, File> existingFileNames = new HashMap<>();
507509
preexistingReadData.forEach(rd -> {
508-
existingFileNames.put(rd.getFile1().getName(), rd.getFile1());
510+
existingFileNames.put(rd.getFile1(true).getName(), rd.getFile1(true));
509511
if (rd.getFile2() != null)
510512
{
511-
existingFileNames.put(rd.getFile2().getName(), rd.getFile2());
513+
existingFileNames.put(rd.getFile2(true).getName(), rd.getFile2(true));
512514
}
513515
});
514516

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ReblockGvcfHandler.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,7 @@ private File getReblockedName(File gvcf)
129129
}
130130

131131
@Override
132-
public void complete(PipelineJob job, List<SequenceOutputFile> inputs, List<SequenceOutputFile> outputsCreated, SequenceAnalysisJobSupport support) throws PipelineJobException
132+
public void complete(JobContext ctx, List<SequenceOutputFile> inputs, List<SequenceOutputFile> outputsCreated) throws PipelineJobException
133133
{
134134
List<Map<String, Object>> toUpdate = new ArrayList<>();
135135
List<Map<String, Object>> oldKeys = inputs.stream().map(x -> {
@@ -146,11 +146,11 @@ public void complete(PipelineJob job, List<SequenceOutputFile> inputs, List<Sequ
146146
throw new PipelineJobException("Unable to find file: " + reblocked.getPath());
147147
}
148148

149-
job.getLogger().info("Updating ExpData record with new filepath: " + reblocked.getPath());
149+
ctx.getJob().getLogger().info("Updating ExpData record with new filepath: " + reblocked.getPath());
150150
ExpData d = so.getExpData();
151151
d.setDataFileURI(reblocked.toURI());
152152
d.setName(reblocked.getName());
153-
d.save(job.getUser());
153+
d.save(ctx.getJob().getUser());
154154

155155
if (so.getName().contains(".g.vcf.gz"))
156156
{
@@ -165,8 +165,8 @@ public void complete(PipelineJob job, List<SequenceOutputFile> inputs, List<Sequ
165165

166166
try
167167
{
168-
Container target = job.getContainer().isWorkbook() ? job.getContainer().getParent() : job.getContainer();
169-
QueryService.get().getUserSchema(job.getUser(), target, SequenceAnalysisSchema.SCHEMA_NAME).getTable(SequenceAnalysisSchema.TABLE_OUTPUTFILES).getUpdateService().updateRows(job.getUser(), target, toUpdate, oldKeys, null, null);
168+
Container target = ctx.getJob().getContainer().isWorkbook() ? ctx.getJob().getContainer().getParent() : ctx.getJob().getContainer();
169+
QueryService.get().getUserSchema(ctx.getJob().getUser(), target, SequenceAnalysisSchema.SCHEMA_NAME).getTable(SequenceAnalysisSchema.TABLE_OUTPUTFILES).getUpdateService().updateRows(ctx.getJob().getUser(), target, toUpdate, oldKeys, null, null);
170170
}
171171
catch (QueryUpdateServiceException | InvalidKeyException | BatchValidationException | SQLException e)
172172
{

0 commit comments

Comments
 (0)