Skip to content

Commit 0a66091

Browse files
bbimberbbimber
authored
Merge pull request #52 from LabKey/fb_merge_discvr-20.3
Merge discvr-20.3 to develop
2 parents 4c27abd + bf09d3f commit 0a66091

File tree

14 files changed

+688
-154
lines changed

14 files changed

+688
-154
lines changed

‎SequenceAnalysis/resources/queries/sequenceanalysis/readData/.qview.xml‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,13 @@
1313
<column name="description" />
1414
<column name="sra_accession" />
1515
<column name="archived" />
16-
16+
<column name="runid/JobId">
17+
<properties>
18+
<property name="columnTitle" value="Job Id"/>
19+
</properties>
20+
</column>
1721
<column name="created" />
18-
<column name="workbook" />
22+
<column name="readset/workbook" />
1923
</columns>
2024
<sorts>
2125
<sort column="readset" />

‎SequenceAnalysis/resources/web/SequenceAnalysis/panel/SequenceImportPanel.js‎

Lines changed: 200 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1331,7 +1331,7 @@ Ext4.define('SequenceAnalysis.panel.SequenceImportPanel', {
13311331
grid.getPlugin('cellediting').completeEdit( );
13321332
var s = grid.getSelectionModel().getSelection();
13331333

1334-
if (!s.length){
1334+
if (!s.length) {
13351335
Ext4.Msg.hide();
13361336
Ext4.Msg.alert('Error', 'No records selected');
13371337
return;
@@ -1344,7 +1344,8 @@ Ext4.define('SequenceAnalysis.panel.SequenceImportPanel', {
13441344
}
13451345

13461346
var originals = [];
1347-
for (var i = 0, r; r = s[i]; i++){
1347+
for (var i = 0; i < s.length; i++){
1348+
var r = s[i];
13481349
if (r.get('fileGroupId'))
13491350
originals.push(r.get('fileGroupId'));
13501351

@@ -1392,13 +1393,209 @@ Ext4.define('SequenceAnalysis.panel.SequenceImportPanel', {
13921393
grid.getView().refresh();
13931394
}, this, null, defaultVal);
13941395
}
1396+
},{
1397+
text: 'Regroup By Regex (Advanced)',
1398+
scope: this,
1399+
handler: function (btn) {
1400+
var grid = btn.up('grid');
1401+
grid.getPlugin('cellediting').completeEdit();
1402+
var s = grid.getSelectionModel().getSelection();
1403+
if (!s.length){
1404+
Ext4.Msg.hide();
1405+
Ext4.Msg.alert('Error', 'No records selected');
1406+
return;
1407+
}
1408+
1409+
Ext4.create('Ext.window.Window', {
1410+
title: 'Regroup Selected Records By RegExp',
1411+
bodyStyle: 'padding: 10px;',
1412+
width: 500,
1413+
items: [{
1414+
html: 'This helper allows you to provide a regular expression to extract the file group, and a second expression to extract platformUnit. Any file pairs with the same group will be imported as a single readset. Any files with the same non-null platform unit will be merged. Specifying different platform units can be important for read groups, and downstream operations like MarkDuplicates. Regular expressions will be tested against the first filename in each pair only. For each regex, if not match is found, the original group or platformUnit values is retained.',
1415+
border: false
1416+
},{
1417+
xtype: 'textfield',
1418+
fieldLabel: 'Group RegEx',
1419+
itemId: 'groupEx',
1420+
value: '(.*)_'
1421+
},{
1422+
xtype: 'ldk-integerfield',
1423+
fieldLabel: 'Group Index',
1424+
itemId: 'groupNum',
1425+
value: 1,
1426+
minValue: 0
1427+
},{
1428+
xtype: 'textfield',
1429+
fieldLabel: 'Platform Unit RegEx',
1430+
itemId: 'platformEx',
1431+
value: '(.*)_'
1432+
},{
1433+
xtype: 'ldk-integerfield',
1434+
fieldLabel: 'Platform Index',
1435+
itemId: 'platformNum',
1436+
value: 2,
1437+
minValue: 0
1438+
}],
1439+
buttons: [{
1440+
text: 'Test',
1441+
scope: this,
1442+
handler: function (btn) {
1443+
var groupEx = btn.up('window').down('#groupEx').getValue();
1444+
var groupNum = btn.up('window').down('#groupNum').getValue();
1445+
1446+
var platformEx = btn.up('window').down('#platformEx').getValue();
1447+
var platformNum = btn.up('window').down('#platformNum').getValue();
1448+
1449+
if (groupEx) {
1450+
groupEx = new RegExp(groupEx);
1451+
}
1452+
1453+
if (platformEx) {
1454+
platformEx = new RegExp(platformEx);
1455+
}
1456+
1457+
var rows = [];
1458+
Ext4.Array.forEach(s, function(record){
1459+
var recId = record.get('fileRecord1');
1460+
var data = this.fileNameStore.snapshot || this.fileNameStore.data;
1461+
var fileDataRecIdx = data.findIndexBy(function(r){
1462+
return r.get('id') === recId;
1463+
});
1464+
var fileDataRec = data.getAt(fileDataRecIdx);
1465+
if (!fileDataRec) {
1466+
return;
1467+
}
1468+
1469+
var val = fileDataRec.get('fileName');
1470+
var row = [val, null, null, record];
1471+
if (groupEx) {
1472+
var m = val.match(groupEx);
1473+
if (m && m.length) {
1474+
if (groupNum && groupNum < m.length) {
1475+
row[1] = m[groupNum];
1476+
}
1477+
}
1478+
}
1479+
1480+
if (platformEx) {
1481+
var m = val.match(platformEx);
1482+
if (m && m.length) {
1483+
if (platformNum && platformNum < m.length) {
1484+
row[2] = m[platformNum];
1485+
}
1486+
}
1487+
}
1488+
1489+
rows.push(row);
1490+
}, this);
1491+
1492+
rows.sort(function(a, b){
1493+
if (a[1] > b[1]) return 1;
1494+
if (b[1] > a[1]) return -1;
1495+
1496+
return 0;
1497+
});
1498+
1499+
var dataItems = [{
1500+
html: 'File'
1501+
},{
1502+
html: 'Updated Group'
1503+
},{
1504+
html: 'Updated Platform Unit'
1505+
}];
1506+
1507+
Ext4.Array.forEach(rows, function(r){
1508+
dataItems.push({
1509+
html: r[0]
1510+
});
1511+
1512+
dataItems.push({
1513+
html: r[1] || ''
1514+
});
1515+
1516+
dataItems.push({
1517+
html: r[2] || ''
1518+
});
1519+
}, this);
1520+
Ext4.create('Ext.window.Window', {
1521+
title: 'Preview',
1522+
parentWindow: btn.up('window'),
1523+
width: 800,
1524+
bodyStyle: 'padding: 5px',
1525+
defaults: {
1526+
style: 'padding: 5px;',
1527+
border: false
1528+
},
1529+
layout: {
1530+
type: 'table',
1531+
columns: 3
1532+
},
1533+
items: dataItems,
1534+
buttons: [{
1535+
text: 'Make Changes',
1536+
scope: this,
1537+
handler: function (btn) {
1538+
var win = btn.up('window');
1539+
var parentWindow = win.parentWindow;
1540+
win.close();
1541+
1542+
Ext4.Array.forEach(rows, function(row){
1543+
if (row[2]) {
1544+
row[3].set('platformUnit', row[2]);
1545+
}
1546+
1547+
if (row[1]) {
1548+
row[3].set('fileGroupId', row[1]);
1549+
}
1550+
}, this);
1551+
1552+
//clear/populate readsetStore
1553+
var fileGroupIds = [];
1554+
this.readDataStore.each(function(r){
1555+
fileGroupIds.push(r.get('fileGroupId'));
1556+
}, this);
1557+
fileGroupIds = Ext4.unique(fileGroupIds);
1558+
1559+
this.readsetStore.removeAll();
1560+
1561+
if (fileGroupIds.length){
1562+
var toAdd = [];
1563+
Ext4.Array.forEach(fileGroupIds, function(id){
1564+
toAdd.push(this.readsetStore.createModel({
1565+
fileGroupId: id
1566+
}));
1567+
}, this);
1568+
1569+
this.readsetStore.add(toAdd);
1570+
}
1571+
1572+
parentWindow.close();
1573+
}
1574+
},{
1575+
text: 'Abort',
1576+
scope: this,
1577+
handler: function (btn) {
1578+
btn.up('window').close();
1579+
}
1580+
}]
1581+
1582+
}).show();
1583+
}
1584+
},{
1585+
text: 'Close',
1586+
scope: this,
1587+
handler: function (btn) {
1588+
btn.up('window').close();
1589+
}
1590+
}]
1591+
}).show();
1592+
}
13951593
},{
13961594
text: 'Add Missing Readsets',
13971595
scope: this,
13981596
handler : function(btn) {
13991597
var grid = btn.up('grid');
14001598

1401-
14021599
var fileGroupIds = [];
14031600
this.readDataStore.each(function(r){
14041601
fileGroupIds.push(r.get('fileGroupId'));

‎SequenceAnalysis/resources/web/SequenceAnalysis/panel/VariantProcessingPanel.js‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -236,6 +236,12 @@ Ext4.define('SequenceAnalysis.panel.VariantProcessingPanel', {
236236
label: 'Allow Old RMS Mapping Data',
237237
description: 'This must be checked to allow processing of gVCFs generated by GATK3.',
238238
defaultValue: false
239+
},{
240+
fieldXtype: 'checkbox',
241+
name: 'disableFileLocking',
242+
label: 'Disable File Locking',
243+
description: 'This applies to GenomicsDB inputs only. Certain filesystems do not support file locking, including NFS and Lustre. If your data will be processed on a filesystem that does not support locking, check this.',
244+
defaultValue: true
239245
},{
240246
fieldXtype: 'checkbox',
241247
name: 'doCopyInputs',

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceNoOpDocumentParser.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,8 @@ public boolean detect(WebdavResource resource, String contentType, byte[] buf) t
4444
SequenceUtil.FILETYPE.vcf.getFileType().isType(resource.getFile()) ||
4545
SequenceUtil.FILETYPE.bam.getFileType().isType(resource.getFile()) ||
4646
RDataType.isType(resource.getFile()) ||
47-
isUnderAnalysisDir(resource);
47+
isUnderAnalysisDir(resource) ||
48+
isTileDB(resource);
4849
}
4950

5051
private boolean isUnderAnalysisDir(WebdavResource resource)
@@ -60,5 +61,10 @@ private boolean isUnderAnalysisDir(WebdavResource resource)
6061
resource.getFile().getPath().contains("/refSequenceImport/") ||
6162
resource.getFile().getPath().contains("/sequenceOutputPipeline/"));
6263
}
64+
65+
private boolean isTileDB(WebdavResource resource)
66+
{
67+
return resource.getFile() != null && resource.getFile().getPath().contains(".tbd") && resource.getFile().getPath().contains(".gbd");
68+
}
6369
}
6470

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/CombineSubreadGeneCountsHandler.java‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package org.labkey.sequenceanalysis.analysis;
22

3+
import org.apache.commons.lang3.StringUtils;
34
import org.json.JSONObject;
45
import org.labkey.api.pipeline.PipelineJob;
56
import org.labkey.api.pipeline.PipelineJobException;
@@ -48,14 +49,26 @@ protected void processOutputFiles(CountResults results, List<SequenceOutputFile>
4849
String line;
4950
while ((line = reader.readLine()) != null)
5051
{
51-
line = line.trim();
52-
if (line.startsWith("#") || line.startsWith("Geneid"))
52+
//NOTE: if gene name is null, element 0 can be empty string
53+
//line = line.trim();
54+
if (line.startsWith("#") || line.startsWith("Geneid") || line.isEmpty())
5355
{
5456
continue;
5557
}
5658

5759
String[] cells = line.split("\t");
60+
if (cells.length < 7)
61+
{
62+
throw new PipelineJobException("Line too short, was " + cells.length + ": [" + StringUtils.join(cells, "<>") + "]");
63+
}
64+
5865
String geneId = cells[0];
66+
if (geneId.isEmpty())
67+
{
68+
job.getLogger().info("Feature lacks gene ID: [" + line + "]");
69+
continue;
70+
}
71+
5972
if (OTHER_IDS.contains(geneId))
6073
{
6174
continue;

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/GenotypeGVCFHandler.java‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -301,7 +301,7 @@ private File runGenotypeGVCFs(PipelineJob job, JobContext ctx, ProcessVariantsHa
301301
if (doCopyLocal)
302302
{
303303
ctx.getLogger().info("making local copies of gVCF/GenomicsDB files prior to genotyping");
304-
filesToProcess.addAll(GenotypeGVCFsWrapper.copyVcfsLocally(inputFiles, toDelete, GenotypeGVCFHandler.getLocalCopyDir(ctx, true), ctx.getLogger(), outputVcf.exists()));
304+
filesToProcess.addAll(GenotypeGVCFsWrapper.copyVcfsLocally(ctx, inputFiles, toDelete, outputVcf.exists()));
305305
}
306306
else
307307
{
@@ -367,6 +367,12 @@ private File runGenotypeGVCFs(PipelineJob job, JobContext ctx, ProcessVariantsHa
367367
});
368368
}
369369

370+
if (ctx.getParams().optBoolean("disableFileLocking", false))
371+
{
372+
ctx.getLogger().debug("Disabling file locking for TileDB");
373+
wrapper.addToEnvironment("TILEDB_DISABLE_FILE_LOCKING", "1");
374+
}
375+
370376
wrapper.execute(genome.getSourceFastaFile(), outputVcf, toolParams, inputVcf);
371377

372378
action.addOutput(outputVcf, "VCF", outputVcf.exists(), true);
@@ -457,12 +463,11 @@ public void doWork(List<SequenceOutputFile> inputFiles, JobContext ctx) throws P
457463

458464
public static void doCopyGvcfLocally(List<SequenceOutputFile> inputFiles, JobContext ctx) throws PipelineJobException
459465
{
460-
VariantProcessingJob vpj = (VariantProcessingJob)ctx.getJob();
461466
List<File> inputVCFs = new ArrayList<>();
462467
inputFiles.forEach(f -> inputVCFs.add(f.getFile()));
463468

464469
ctx.getLogger().info("making local copies of gVCFs");
465-
GenotypeGVCFsWrapper.copyVcfsLocally(inputVCFs, new ArrayList<>(), getLocalCopyDir(ctx, true), ctx.getLogger(), false);
470+
GenotypeGVCFsWrapper.copyVcfsLocally(ctx, inputVCFs, new ArrayList<>(), false);
466471
}
467472

468473
public static File getLocalCopyDir(JobContext ctx, boolean createIfDoesntExist)

0 commit comments

Comments
 (0)