Skip to content

Commit 187b6f0

Browse files
bbimberbbimber
authored
Merge pull request #269 from LabKey/fb_merge_23.11_to_develop
Merge discvr-23.11 to develop
2 parents 3757e0f + bb660ae commit 187b6f0

File tree

46 files changed

+1706
-125
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

46 files changed

+1706
-125
lines changed

‎SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/BcftoolsRunner.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ public BcftoolsRunner(@Nullable Logger logger)
1818
super(logger);
1919
}
2020

21-
public File getBcfToolsPath()
21+
public static File getBcfToolsPath()
2222
{
2323
return SequencePipelineService.get().getExeForPackage("BCFTOOLSPATH", "bcftools");
2424
}

‎SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/TaskFileManager.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,8 @@ public interface TaskFileManager extends PipelineOutputTracker
5656

5757
boolean isDeleteIntermediateFiles();
5858

59+
public boolean performCleanupAfterEachStep();
60+
5961
boolean isCopyInputsLocally();
6062

6163
void addPicardMetricsFiles(List<PipelineStepOutput.PicardMetricsOutput> files) throws PipelineJobException;

‎SequenceAnalysis/resources/queries/sequenceanalysis/sequence_readsets/SRA Info.qview.xml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717
<column name="workbook" />
1818
<column name="sraRuns" />
1919
<column name="isArchived" />
20+
<column name="totalFiles" />
21+
<column name="readdataWithoutSra" />
2022
<column name="files" />
2123
</columns>
2224
<sorts>

‎SequenceAnalysis/resources/web/SequenceAnalysis/panel/AlignmentImportPanel.js‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,14 @@ Ext4.define('SequenceAnalysis.panel.AlignmentImportPanel', {
9494
inputValue: true,
9595
checked: true,
9696
xtype: 'checkbox'
97+
},{
98+
fieldLabel: 'Perform Cleanup After Each Step',
99+
helpPopup: 'Is selected, intermediate files from this job will be deleted after each step, instead of once at the end of the job. This can reduce the working directory size. Note: this will only apply if deleteIntermediateFiles is selected, and this is not supported across every possible pipeline type.',
100+
name: 'performCleanupAfterEachStep',
101+
inputValue: true,
102+
uncheckedValue: false,
103+
checked: true,
104+
xtype: 'checkbox'
97105
},{
98106
fieldLabel: 'Treatment of Input Files',
99107
xtype: 'combo',

‎SequenceAnalysis/resources/web/SequenceAnalysis/panel/SequenceAnalysisPanel.js‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -309,6 +309,14 @@ Ext4.define('SequenceAnalysis.panel.SequenceAnalysisPanel', {
309309
uncheckedValue: false,
310310
checked: true,
311311
xtype: 'checkbox'
312+
},{
313+
fieldLabel: 'Perform Cleanup After Each Step',
314+
helpPopup: 'Is selected, intermediate files from this job will be deleted after each step, instead of once at the end of the job. This can reduce the working directory size. Note: this will only apply if deleteIntermediateFiles is selected, and this is not supported across every possible pipeline type.',
315+
name: 'performCleanupAfterEachStep',
316+
inputValue: true,
317+
uncheckedValue: false,
318+
checked: true,
319+
xtype: 'checkbox'
312320
},{
313321
fieldLabel: 'Copy Inputs To Working Directory?',
314322
helpPopup: 'Check to copy the input files to the working directory. Depending on your environment, this may or may not help performance.',

‎SequenceAnalysis/resources/web/SequenceAnalysis/panel/VariantProcessingPanel.js‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,14 @@ Ext4.define('SequenceAnalysis.panel.VariantProcessingPanel', {
7474
inputValue: true,
7575
checked: true,
7676
xtype: 'checkbox'
77+
},{
78+
fieldLabel: 'Perform Cleanup After Each Step',
79+
helpPopup: 'Is selected, intermediate files from this job will be deleted after each step, instead of once at the end of the job. This can reduce the working directory size. Note: this will only apply if deleteIntermediateFiles is selected, and this is not supported across every possible pipeline type.',
80+
name: 'performCleanupAfterEachStep',
81+
inputValue: true,
82+
uncheckedValue: false,
83+
checked: true,
84+
xtype: 'checkbox'
7785
}, this.getSaveTemplateCfg()]
7886
};
7987
},

‎SequenceAnalysis/resources/web/SequenceAnalysis/window/ArchiveReadsetsWindow.js‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
Ext4.define('SequenceAnalysis.window.ArchiveReadsetsWindow', {
2+
extend: 'Ext.window.Window',
3+
4+
statics: {
5+
buttonHandler: function(dataRegionName){
6+
Ext4.create('SequenceAnalysis.window.ArchiveReadsetsWindow', {
7+
dataRegionName: dataRegionName,
8+
readsetIds: LABKEY.DataRegions[dataRegionName].getChecked()
9+
}).show();
10+
}
11+
},
12+
13+
initComponent: function() {
14+
Ext4.apply(this, {
15+
modal: true,
16+
title: 'Archive Readsets',
17+
width: 600,
18+
bodyStyle: 'padding: 5px;',
19+
defaults: {
20+
border: false
21+
},
22+
items: [{
23+
html: 'This helper will delete the actual FASTQ files associated with the selected readsets. It will error unless each readdata row has an SRA accession listed. You selected ' + this.readsetIds.length + ' readsets.',
24+
style: 'padding-bottom: 10px;'
25+
}],
26+
buttons: [{
27+
text: 'Submit',
28+
scope: this,
29+
handler: this.onSubmit
30+
},{
31+
text: 'Cancel',
32+
handler: function(btn){
33+
btn.up('window').close();
34+
}
35+
}]
36+
});
37+
38+
this.callParent(arguments);
39+
},
40+
41+
onSubmit: function(btn){
42+
if (!this.readsetIds.length) {
43+
Ext4.Msg.alert('Error', 'No readsets selected!');
44+
return;
45+
}
46+
47+
Ext4.Msg.wait('Saving...');
48+
LABKEY.Ajax.request({
49+
url: LABKEY.ActionURL.buildURL('sequenceanalysis', 'archiveReadsets', null),
50+
method: 'POST',
51+
jsonData: {
52+
readsetIds: this.readsetIds
53+
},
54+
scope: this,
55+
success: function(){
56+
Ext4.Msg.hide();
57+
this.close();
58+
Ext4.Msg.alert('Success', 'Readsets archived!', function(){
59+
if (this.dataRegionName){
60+
LABKEY.DataRegions[this.dataRegionName].clearSelected();
61+
}
62+
63+
LABKEY.DataRegions[this.dataRegionName].refresh();
64+
}, this);
65+
},
66+
failure: LABKEY.Utils.getCallbackWrapper(LDK.Utils.getErrorCallback())
67+
});
68+
}
69+
});

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisController.java‎

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@
7777
import org.labkey.api.exp.api.ExperimentService;
7878
import org.labkey.api.files.FileContentService;
7979
import org.labkey.api.laboratory.NavItem;
80+
import org.labkey.api.laboratory.security.LaboratoryAdminPermission;
8081
import org.labkey.api.module.Module;
8182
import org.labkey.api.module.ModuleHtmlView;
8283
import org.labkey.api.module.ModuleLoader;
@@ -3297,6 +3298,15 @@ public ApiResponse execute(CheckFileStatusForm form, BindException errors)
32973298

32983299
toolArr.put(intermediateFiles);
32993300

3301+
JSONObject performCleanupAfterEachStep = new JSONObject();
3302+
performCleanupAfterEachStep.put("name", "performCleanupAfterEachStep");
3303+
performCleanupAfterEachStep.put("defaultValue", true);
3304+
performCleanupAfterEachStep.put("label", "Perform Cleanup After Each Step");
3305+
performCleanupAfterEachStep.put("description", "Is selected, intermediate files from this job will be deleted after each step, instead of once at the end of the job. This can reduce the working directory size. Note: this will only apply if deleteIntermediateFiles is selected, and this is not supported across every possible pipeline type.");
3306+
performCleanupAfterEachStep.put("fieldXtype", "checkbox");
3307+
3308+
toolArr.put(performCleanupAfterEachStep);
3309+
33003310
ret.put("toolParameters", toolArr);
33013311

33023312
ret.put("description", handler.getDescription());
@@ -5128,4 +5138,157 @@ public void setDataFileUrl(String dataFileUrl)
51285138
_dataFileUrl = dataFileUrl;
51295139
}
51305140
}
5141+
5142+
@RequiresPermission(UpdatePermission.class)
5143+
public static class ArchiveReadsetsAction extends MutatingApiAction<ArchiveReadsetsForm>
5144+
{
5145+
@Override
5146+
public ApiResponse execute(ArchiveReadsetsForm form, BindException errors) throws Exception
5147+
{
5148+
if (form.getReadsetIds() == null || form.getReadsetIds().length == 0)
5149+
{
5150+
errors.reject(ERROR_MSG, "No readset Ids provided");
5151+
return null;
5152+
}
5153+
5154+
TableInfo readData = QueryService.get().getUserSchema(getUser(), getContainer(), SequenceAnalysisSchema.SCHEMA_NAME).getTable(SequenceAnalysisSchema.TABLE_READ_DATA);
5155+
for (int readsetId : form.getReadsetIds())
5156+
{
5157+
Readset rs = SequenceAnalysisService.get().getReadset(readsetId, getUser());
5158+
Container c = ContainerManager.getForId(rs.getContainer());
5159+
if (!getContainer().equals(c))
5160+
{
5161+
Container toTest = c.isWorkbook() ? c.getParent() : c;
5162+
if (!getContainer().equals(toTest))
5163+
{
5164+
errors.reject(ERROR_MSG, "Readset is not from this container: " + readsetId);
5165+
return null;
5166+
}
5167+
}
5168+
5169+
if (!c.hasPermission(getUser(), LaboratoryAdminPermission.class))
5170+
{
5171+
errors.reject(ERROR_MSG, "Insufficient permissions to archive readsets in the folder: " + c.getPath());
5172+
return null;
5173+
}
5174+
5175+
Set<File> toDelete = new HashSet<>();
5176+
List<Map<String, Object>> toUpdate = new ArrayList<>();
5177+
for (ReadData rd : rs.getReadData())
5178+
{
5179+
if (rd.getSra_accession() == null)
5180+
{
5181+
errors.reject(ERROR_MSG, "Cannot mark a readdata as archived that does not have an SRA accession: " + readsetId + " / " + rd.getRowid());
5182+
return null;
5183+
}
5184+
5185+
toUpdate.add(new CaseInsensitiveHashMap<>(Map.of("rowid", rd.getRowid(), "archived", true, "container", rd.getContainer())));
5186+
5187+
// File 1:
5188+
ExpData d1 = ExperimentService.get().getExpData(rd.getFileId1());
5189+
if (d1 != null)
5190+
{
5191+
File file1 = d1.getFile();
5192+
if (file1 != null && file1.exists())
5193+
{
5194+
toDelete.add(file1);
5195+
}
5196+
5197+
// find matching readdata:
5198+
SimpleFilter filter = new SimpleFilter(FieldKey.fromString("fileid1/dataFileUrl"), d1.getDataFileUrl()).addCondition(FieldKey.fromString("rowid"), rd.getRowid(), CompareType.NEQ);
5199+
TableSelector ts = new TableSelector(readData, PageFlowUtil.set("rowid", "container"), filter, null);
5200+
if (ts.exists())
5201+
{
5202+
ts.forEachResults(r -> {
5203+
toUpdate.add(new CaseInsensitiveHashMap<>(Map.of("rowid", r.getInt(FieldKey.fromString("rowid")), "archived", true, "container", r.getString(FieldKey.fromString("container")))));
5204+
});
5205+
}
5206+
}
5207+
5208+
if (rd.getFileId2() != null)
5209+
{
5210+
ExpData d2 = ExperimentService.get().getExpData(rd.getFileId2());
5211+
if (d2 != null)
5212+
{
5213+
File file2 = d2.getFile();
5214+
if (file2 != null)
5215+
{
5216+
if (file2.exists())
5217+
{
5218+
toDelete.add(file2);
5219+
}
5220+
5221+
// find matching readdata:
5222+
SimpleFilter filter = new SimpleFilter(FieldKey.fromString("fileid2/dataFileUrl"), d2.getDataFileUrl()).addCondition(FieldKey.fromString("rowid"), rd.getRowid(), CompareType.NEQ);
5223+
TableSelector ts = new TableSelector(readData, PageFlowUtil.set("rowid", "container"), filter, null);
5224+
if (ts.exists())
5225+
{
5226+
ts.forEachResults(r -> {
5227+
toUpdate.add(new CaseInsensitiveHashMap<>(Map.of("rowid", r.getInt(FieldKey.fromString("rowid")), "archived", true, "container", r.getString(FieldKey.fromString("container")))));
5228+
});
5229+
}
5230+
}
5231+
}
5232+
}
5233+
}
5234+
5235+
if (!toUpdate.isEmpty())
5236+
{
5237+
List<Map<String, Object>> keys = new ArrayList<>();
5238+
toUpdate.forEach(row -> {
5239+
5240+
keys.add(new CaseInsensitiveHashMap<>(Map.of("rowid", row.get("rowid"))));
5241+
});
5242+
5243+
try
5244+
{
5245+
readData.getUpdateService().updateRows(getUser(), getContainer(), toUpdate, keys, null, null);
5246+
}
5247+
catch (Exception e)
5248+
{
5249+
_log.error(e);
5250+
errors.reject(ERROR_MSG, "Error archiving readset: " + readsetId + ", " + e.getMessage());
5251+
return null;
5252+
}
5253+
}
5254+
5255+
if (!toDelete.isEmpty())
5256+
{
5257+
for (File f : toDelete)
5258+
{
5259+
_log.info("Deleting archived file: " + f.getPath());
5260+
f.delete();
5261+
}
5262+
}
5263+
}
5264+
5265+
return new ApiSimpleResponse("Success", true);
5266+
}
5267+
}
5268+
5269+
public static class ArchiveReadsetsForm
5270+
{
5271+
private int[] _readsetIds;
5272+
private boolean _doNotRequireSra;
5273+
5274+
public int[] getReadsetIds()
5275+
{
5276+
return _readsetIds;
5277+
}
5278+
5279+
public void setReadsetIds(int... readsetIds)
5280+
{
5281+
_readsetIds = readsetIds;
5282+
}
5283+
5284+
public boolean isDoNotRequireSra()
5285+
{
5286+
return _doNotRequireSra;
5287+
}
5288+
5289+
public void setDoNotRequireSra(boolean doNotRequireSra)
5290+
{
5291+
_doNotRequireSra = doNotRequireSra;
5292+
}
5293+
}
51315294
}

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisModule.java‎

Lines changed: 10 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,8 @@
4545
import org.labkey.sequenceanalysis.analysis.BamHaplotypeHandler;
4646
import org.labkey.sequenceanalysis.analysis.CombineStarGeneCountsHandler;
4747
import org.labkey.sequenceanalysis.analysis.CombineSubreadGeneCountsHandler;
48+
import org.labkey.sequenceanalysis.analysis.DeepVariantHandler;
49+
import org.labkey.sequenceanalysis.analysis.GLNexusHandler;
4850
import org.labkey.sequenceanalysis.analysis.GenotypeGVCFHandler;
4951
import org.labkey.sequenceanalysis.analysis.HaplotypeCallerHandler;
5052
import org.labkey.sequenceanalysis.analysis.LiftoverHandler;
@@ -58,6 +60,7 @@
5860
import org.labkey.sequenceanalysis.analysis.SbtGeneCountHandler;
5961
import org.labkey.sequenceanalysis.analysis.UnmappedSequenceBasedGenotypeHandler;
6062
import org.labkey.sequenceanalysis.button.AddSraRunButton;
63+
import org.labkey.sequenceanalysis.button.ArchiveReadsetsButton;
6164
import org.labkey.sequenceanalysis.button.ChangeReadsetStatusButton;
6265
import org.labkey.sequenceanalysis.button.ChangeReadsetStatusForAnalysesButton;
6366
import org.labkey.sequenceanalysis.button.DownloadSraButton;
@@ -77,25 +80,7 @@
7780
import org.labkey.sequenceanalysis.run.alignment.Pbmm2Wrapper;
7881
import org.labkey.sequenceanalysis.run.alignment.StarWrapper;
7982
import org.labkey.sequenceanalysis.run.alignment.VulcanWrapper;
80-
import org.labkey.sequenceanalysis.run.analysis.BamIterator;
81-
import org.labkey.sequenceanalysis.run.analysis.BcftoolsFillTagsStep;
82-
import org.labkey.sequenceanalysis.run.analysis.ExportOverlappingReadsAnalysis;
83-
import org.labkey.sequenceanalysis.run.analysis.GenrichStep;
84-
import org.labkey.sequenceanalysis.run.analysis.HaplotypeCallerAnalysis;
85-
import org.labkey.sequenceanalysis.run.analysis.ImmunoGenotypingAnalysis;
86-
import org.labkey.sequenceanalysis.run.analysis.LofreqAnalysis;
87-
import org.labkey.sequenceanalysis.run.analysis.MergeLoFreqVcfHandler;
88-
import org.labkey.sequenceanalysis.run.analysis.NextCladeHandler;
89-
import org.labkey.sequenceanalysis.run.analysis.PARalyzerAnalysis;
90-
import org.labkey.sequenceanalysis.run.analysis.PangolinHandler;
91-
import org.labkey.sequenceanalysis.run.analysis.PbsvAnalysis;
92-
import org.labkey.sequenceanalysis.run.analysis.PbsvJointCallingHandler;
93-
import org.labkey.sequenceanalysis.run.analysis.PindelAnalysis;
94-
import org.labkey.sequenceanalysis.run.analysis.SequenceBasedTypingAnalysis;
95-
import org.labkey.sequenceanalysis.run.analysis.SnpCountAnalysis;
96-
import org.labkey.sequenceanalysis.run.analysis.SubreadAnalysis;
97-
import org.labkey.sequenceanalysis.run.analysis.UnmappedReadExportHandler;
98-
import org.labkey.sequenceanalysis.run.analysis.ViralAnalysis;
83+
import org.labkey.sequenceanalysis.run.analysis.*;
9984
import org.labkey.sequenceanalysis.run.assembly.TrinityRunner;
10085
import org.labkey.sequenceanalysis.run.bampostprocessing.AddOrReplaceReadGroupsStep;
10186
import org.labkey.sequenceanalysis.run.bampostprocessing.BaseQualityScoreRecalibrator;
@@ -280,6 +265,7 @@ public static void registerPipelineSteps()
280265
SequencePipelineService.get().registerPipelineStep(new ImmunoGenotypingAnalysis.Provider());
281266
SequencePipelineService.get().registerPipelineStep(new ViralAnalysis.Provider());
282267
SequencePipelineService.get().registerPipelineStep(new HaplotypeCallerAnalysis.Provider());
268+
SequencePipelineService.get().registerPipelineStep(new DeepVariantAnalysis.Provider());
283269
SequencePipelineService.get().registerPipelineStep(new SnpCountAnalysis.Provider());
284270
SequencePipelineService.get().registerPipelineStep(new ExportOverlappingReadsAnalysis.Provider());
285271
SequencePipelineService.get().registerPipelineStep(new SubreadAnalysis.Provider());
@@ -346,6 +332,8 @@ public static void registerPipelineSteps()
346332
SequenceAnalysisService.get().registerFileHandler(new NextCladeHandler());
347333
SequenceAnalysisService.get().registerFileHandler(new ConvertToCramHandler());
348334
SequenceAnalysisService.get().registerFileHandler(new PbsvJointCallingHandler());
335+
SequenceAnalysisService.get().registerFileHandler(new DeepVariantHandler());
336+
SequenceAnalysisService.get().registerFileHandler(new GLNexusHandler());
349337

350338
SequenceAnalysisService.get().registerReadsetHandler(new MultiQCHandler());
351339
SequenceAnalysisService.get().registerReadsetHandler(new RestoreSraDataHandler());
@@ -396,9 +384,10 @@ public void doStartupAfterSpringConfig(ModuleContext moduleContext)
396384
LDKService.get().registerQueryButton(new AddSraRunButton(), SequenceAnalysisSchema.SCHEMA_NAME, SequenceAnalysisSchema.TABLE_READSETS);
397385
LDKService.get().registerQueryButton(new RunMultiQCButton(), SequenceAnalysisSchema.SCHEMA_NAME, SequenceAnalysisSchema.TABLE_READSETS);
398386
LDKService.get().registerQueryButton(new DownloadSraButton(), SequenceAnalysisSchema.SCHEMA_NAME, SequenceAnalysisSchema.TABLE_READSETS);
387+
LDKService.get().registerQueryButton(new ArchiveReadsetsButton(), SequenceAnalysisSchema.SCHEMA_NAME, SequenceAnalysisSchema.TABLE_READSETS);
399388

400-
LDKService.get().registerQueryButton(new ChangeReadsetStatusForAnalysesButton(), "sequenceanalysis", "sequence_analyses");
401-
LDKService.get().registerQueryButton(new ChangeReadsetStatusButton(), "sequenceanalysis", "sequence_readsets");
389+
LDKService.get().registerQueryButton(new ChangeReadsetStatusForAnalysesButton(), SequenceAnalysisSchema.SCHEMA_NAME, SequenceAnalysisSchema.TABLE_ANALYSES);
390+
LDKService.get().registerQueryButton(new ChangeReadsetStatusButton(), SequenceAnalysisSchema.SCHEMA_NAME, SequenceAnalysisSchema.TABLE_READSETS);
402391

403392
ExperimentService.get().registerExperimentRunTypeSource(new ExperimentRunTypeSource()
404393
{

0 commit comments

Comments
 (0)