Skip to content

Commit 4f8c7c0

Browse files
bbimberbbimber
committed
Merge discvr-21.7 to develop
2 parents fdb4a6e + bb08c8e commit 4f8c7c0

File tree

10 files changed

+191
-102
lines changed

10 files changed

+191
-102
lines changed

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/GxfSorter.java‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -51,16 +51,16 @@ public File sortGff(File input, @Nullable File output) throws PipelineJobExcepti
5151
writer.println("#!/bin/bash");
5252
writer.println("set -x");
5353
writer.println("set -e");
54-
writer.println("GFF=" + input.getPath());
55-
writer.println("OUT_GFF=" + outputFile.getPath());
54+
writer.println("GFF='" + input.getPath() + "'");
55+
writer.println("OUT_GFF='" + outputFile.getPath() + "'");
5656

5757
String cat = inputIsGzip ? "zcat" : "cat";
58-
writer.println(cat + " $GFF | awk '{ if ($1 ~ \"^#\" ) print $0; else exit; }' > $OUT_GFF");
59-
writer.println("(" + cat + " $GFF | grep -v '#' | grep -v \"Parent=\" | sort -V -k1,1 -k4,4n -k5,5n; " + cat + " $GFF | grep -v '#' | grep -e \"Parent=\" | sort -V -k1,1 -k4,4n -k5,5n)| sort -V -k1,1 -k4,4n -s >> $OUT_GFF");
58+
writer.println(cat + " \"$GFF\" | grep -v -e '^$' | awk '{ if ($1 ~ \"^#\" ) print $0; else exit; }' > \"$OUT_GFF\"");
59+
writer.println("(" + cat + " \"$GFF\" | grep -v -e '^$' | grep -v '#' | grep -v \"Parent=\" | sort -V -t$'\\t' -k1,1 -k4,4n -k5,5n; " + cat + " \"$GFF\" | grep -v '#' | grep -e \"Parent=\" | sort -V -t$'\\t' -k1,1 -k4,4n -k5,5n)| sort -V -t$'\\t' -k1,1 -k4,4n -s >> \"$OUT_GFF\"");
6060

6161
if (outputIsGzip)
6262
{
63-
writer.println("bgzip -f $OUT_GFF");
63+
writer.println("bgzip -f \"$OUT_GFF\"");
6464
outputFile = new File(outputFile.getPath() + ".gz");
6565
}
6666
}
@@ -85,12 +85,12 @@ public File sortGtf(File input, @Nullable File output) throws PipelineJobExcepti
8585
writer.println("#!/bin/bash");
8686
writer.println("set -x");
8787
writer.println("set -e");
88-
writer.println("GTF=" + input.getPath());
89-
writer.println("OUT_GTF=" + outputFile.getPath());
88+
writer.println("GTF='" + input.getPath() + "'");
89+
writer.println("OUT_GTF='" + outputFile.getPath() + "'");
9090

9191
String cat = inputIsGzip ? "zcat" : "cat";
92-
writer.println(cat + " $GTF | awk '{ if ($1 ~ \"^#\" ) print $0; else exit; }' > $OUT_GTF");
93-
writer.println(cat + " $GTF | grep -v '#' | awk -v OFS='\\t' ' {");
92+
writer.println(cat + " \"$GTF\" | grep -v -e '^$' | awk '{ if ($1 ~ \"^#\" ) print $0; else exit; }' > \"$OUT_GTF\"");
93+
writer.println(cat + " \"$GTF\" | grep -v -e '^$' | grep -v '#' | awk -v OFS='\\t' ' {");
9494
writer.println("so = 3");
9595
writer.println("if (tolower($3) == \"gene\")");
9696
writer.println(" so = 1");
@@ -100,11 +100,11 @@ public File sortGtf(File input, @Nullable File output) throws PipelineJobExcepti
100100
writer.println(" so = 3");
101101
writer.println("else if (tolower($3) == \"cds\")");
102102
writer.println(" so = 4");
103-
writer.println("print so, $0 } ' | sort -V -k2,2 -k5,5n -k1,1n | cut -d$'\\t' -f2- >> $OUT_GTF");
103+
writer.println("print so, $0 } ' | sort -V -t$'\\t' -k2,2 -k5,5n -k1,1n | cut -d$'\\t' -f2- >> \"$OUT_GTF\"");
104104

105105
if (outputIsGzip)
106106
{
107-
writer.println("bgzip -f $OUT_GTF");
107+
writer.println("bgzip -f \"$OUT_GTF\"");
108108
outputFile = new File(outputFile.getPath() + ".gz");
109109
}
110110
}

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/util/SequenceUtil.java‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,7 @@ public static boolean hasMinLineCount(File f, long minLines) throws PipelineJobE
162162
}
163163
catch (IOException e)
164164
{
165-
throw new PipelineJobException(e);
165+
throw new PipelineJobException("Error processing file: " + f.getPath(), e);
166166
}
167167
}
168168

@@ -451,7 +451,12 @@ public static void sortROD(File input, Logger log, Integer startColumnIdx) throw
451451
//then sort/append the records
452452
CommandWrapper wrapper = SequencePipelineService.get().getCommandWrapper(log);
453453
String cat = isCompressed ? "zcat" : "cat";
454-
wrapper.execute(Arrays.asList("/bin/sh", "-c", cat + " '" + input.getPath() + "' | grep -v '^#' | sort -s -V -k1,1f" + (startColumnIdx == null ? "" : " -k" + startColumnIdx + "," + startColumnIdx + "n") + (isCompressed ? " | bgzip -f " : "")), ProcessBuilder.Redirect.appendTo(sorted));
454+
wrapper.execute(Arrays.asList("/bin/sh", "-c", cat + " '" + input.getPath() + "' | grep -v '^#' | sort -s -V -k1,1f" + (startColumnIdx == null ? "" : " -k" + startColumnIdx + "," + startColumnIdx + "n")), ProcessBuilder.Redirect.appendTo(sorted));
455+
456+
if (isCompressed)
457+
{
458+
sorted = bgzip(sorted, log);
459+
}
455460

456461
//replace the non-sorted output
457462
input.delete();
@@ -498,7 +503,7 @@ else if (!samples.equals(header.getGenotypeSamples()))
498503
for (File vcf : files)
499504
{
500505
String cat = vcf.getName().toLowerCase().endsWith(".gz") ? "zcat" : "cat";
501-
bashCommands.add(cat + " " + vcf.getPath() + " | grep -v '^#';");
506+
bashCommands.add(cat + " '" + vcf.getPath() + "' | grep -v '^#';");
502507
}
503508

504509
try
@@ -518,7 +523,7 @@ else if (!samples.equals(header.getGenotypeSamples()))
518523
threads = Math.max(1, threads - 1);
519524
}
520525

521-
writer.write("} | bgzip -f" + (compressionLevel == null ? "" : " --compress-level 9") + (threads == null ? "" : " --threads " + threads) + " > " + outputGzip.getPath() + "\n");
526+
writer.write("} | bgzip -f" + (compressionLevel == null ? "" : " --compress-level 9") + (threads == null ? "" : " --threads " + threads) + " > '" + outputGzip.getPath() + "'\n");
522527
}
523528

524529
SimpleScriptWrapper wrapper = new SimpleScriptWrapper(log);
@@ -555,7 +560,7 @@ public static Set<String> getContigsInVcf(File vcf) throws PipelineJobException
555560
{
556561
writer.println("#!/bin/bash");
557562
String cat = vcf.getPath().toLowerCase().endsWith(".gz") ? "zcat" : "cat";
558-
writer.println(cat + " " + vcf.getPath() + " | grep -v '#' | awk ' { print $1 } ' | sort | uniq");
563+
writer.println(cat + " '" + vcf.getPath() + "' | grep -v '#' | awk ' { print $1 } ' | sort | uniq");
559564
}
560565

561566
SimpleScriptWrapper wrapper = new SimpleScriptWrapper(null);

‎jbrowse/src/client/JBrowse/Browser/plugins/ExtendedVariantPlugin/ExtendedVariantAdapter/VcfFeature.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ export default class ExtendedVcfFeature extends VcfFeature {
1616
ID: string[]
1717
}) {
1818
if (!variant.INFO["ANN"]) {
19-
return;
19+
return(variant);
2020
}
2121

2222
let IMPACTs = new Set<String>()

‎jbrowse/src/org/labkey/jbrowse/JBrowseMaintenanceTask.java‎

Lines changed: 39 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package org.labkey.jbrowse;
22

33
import org.apache.commons.io.FileUtils;
4+
import org.apache.commons.lang3.SystemUtils;
45
import org.apache.logging.log4j.Logger;
56
import org.labkey.api.data.CompareType;
67
import org.labkey.api.data.Container;
@@ -15,20 +16,24 @@
1516
import org.labkey.api.data.TableInfo;
1617
import org.labkey.api.data.TableSelector;
1718
import org.labkey.api.ldk.LDKService;
19+
import org.labkey.api.pipeline.PipelineJobException;
1820
import org.labkey.api.query.FieldKey;
1921
import org.labkey.api.security.User;
22+
import org.labkey.api.sequenceanalysis.run.SimpleScriptWrapper;
2023
import org.labkey.api.util.PageFlowUtil;
2124
import org.labkey.api.util.SystemMaintenance.MaintenanceTask;
2225
import org.labkey.jbrowse.model.JBrowseSession;
2326
import org.labkey.jbrowse.model.JsonFile;
2427

2528
import java.io.File;
2629
import java.io.IOException;
27-
import java.util.ArrayList;
2830
import java.util.Arrays;
2931
import java.util.HashSet;
3032
import java.util.List;
33+
import java.util.Map;
3134
import java.util.Set;
35+
import java.util.function.Function;
36+
import java.util.stream.Collectors;
3237

3338
/**
3439
* User: bimber
@@ -153,6 +158,8 @@ public void run(Logger log)
153158

154159
private void processContainer(Container c, Logger log) throws IOException
155160
{
161+
log.info("processing container: " + c.getPath());
162+
156163
File jbrowseRoot = JBrowseManager.get().getBaseDir(c, false);
157164

158165
//find jsonfiles we expect to exist
@@ -162,7 +169,7 @@ private void processContainer(Container c, Logger log) throws IOException
162169
filter.addCondition(FieldKey.fromString("sequenceid"), null, CompareType.ISBLANK);
163170

164171
TableSelector ts = new TableSelector(tableJsonFiles, filter, null);
165-
List<JsonFile> rows = new ArrayList<>(ts.getArrayList(JsonFile.class));
172+
Map<String, JsonFile> rowMap = ts.getArrayList(JsonFile.class).stream().collect(Collectors.toMap(JsonFile::getObjectId, Function.identity()));
166173

167174
// Also check for genomes from this container, and any additional JsonFiles they may have:
168175
TableInfo tableGenomes = DbSchema.get(JBrowseManager.SEQUENCE_ANALYSIS, DbSchemaType.Module).getTable("reference_libraries");
@@ -176,15 +183,21 @@ private void processContainer(Container c, Logger log) throws IOException
176183
}
177184
else
178185
{
179-
for (Integer genomeId : ts2.getArrayList(Integer.class))
186+
List<Integer> genomes = ts2.getArrayList(Integer.class);
187+
log.info("total genomes in folder: " + genomes.size());
188+
for (Integer genomeId : genomes)
180189
{
181190
JBrowseSession session = JBrowseSession.getGenericGenomeSession(genomeId);
182191
for (JsonFile json : session.getJsonFiles(u, true))
183192
{
184-
expectedDirs.add(json.getBaseDir());
185-
if (!json.getBaseDir().exists())
193+
rowMap.put(json.getObjectId(), json);
194+
if (json.getBaseDir() != null)
186195
{
187-
log.error("expected jbrowse folder does not exist: " + json.getBaseDir().getPath());
196+
expectedDirs.add(json.getBaseDir());
197+
if (!json.getBaseDir().exists())
198+
{
199+
log.error("expected jbrowse folder does not exist: " + json.getBaseDir().getPath());
200+
}
188201
}
189202
}
190203
}
@@ -193,8 +206,7 @@ private void processContainer(Container c, Logger log) throws IOException
193206

194207
if (jbrowseRoot != null && jbrowseRoot.exists())
195208
{
196-
log.info("processing container: " + c.getPath());
197-
for (JsonFile json : rows)
209+
for (JsonFile json : rowMap.values())
198210
{
199211
if (json.getBaseDir() != null)
200212
{
@@ -206,14 +218,28 @@ private void processContainer(Container c, Logger log) throws IOException
206218
}
207219
}
208220

209-
log.info("expected jsonfiles: " + expectedDirs.size());
221+
log.info("expected resource folders: " + expectedDirs.size());
210222
for (String dir : Arrays.asList("tracks", "data", "references", "databases"))
211223
{
212224
File childDir = new File(jbrowseRoot, dir);
213225
if (childDir.exists())
214226
{
215227
log.info("deleting legacy jbrowse " + dir + " dir: " + childDir.getPath());
216-
FileUtils.deleteDirectory(childDir);
228+
if (SystemUtils.IS_OS_WINDOWS)
229+
{
230+
FileUtils.deleteDirectory(childDir);
231+
}
232+
else
233+
{
234+
try
235+
{
236+
new SimpleScriptWrapper(log).execute(Arrays.asList("rm", "-Rf", childDir.getPath()));
237+
}
238+
catch (PipelineJobException e)
239+
{
240+
log.error("Unable to delete directory: " + childDir.getPath(), e);
241+
}
242+
}
217243
}
218244
}
219245

@@ -231,13 +257,14 @@ private void processContainer(Container c, Logger log) throws IOException
231257
}
232258
}
233259

234-
for (JsonFile j : rows)
260+
log.info("total JsonFiles in folder: " + rowMap.size());
261+
for (JsonFile j : rowMap.values())
235262
{
236263
if (j.needsProcessing())
237264
{
238265
File expectedFile = j.getLocationOfProcessedTrack(false);
239266
boolean error = false;
240-
if (!j.isGzipped() && !expectedFile.exists())
267+
if (expectedFile != null && !expectedFile.exists())
241268
{
242269
log.error("Missing expected file: " + expectedFile.getPath());
243270
error = true;

‎jbrowse/src/org/labkey/jbrowse/model/DbBackedJsonFile.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,12 @@ protected String getSourceFileName()
7373
return getObjectId() + _suffix + ".gff.gz";
7474
}
7575

76+
@Override
77+
public boolean doIndex()
78+
{
79+
return true;
80+
}
81+
7682
@Override
7783
public File getLocationOfProcessedTrack(boolean createDir)
7884
{

‎jbrowse/src/org/labkey/jbrowse/model/JBrowseSession.java‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -217,7 +217,14 @@ public void ensureJsonFilesPrepared(User u, Logger log) throws PipelineJobExcept
217217
{
218218
for (JsonFile x : getJsonFiles(u, true))
219219
{
220-
x.prepareResource(log, false, false);
220+
try
221+
{
222+
x.prepareResource(log, false, false);
223+
}
224+
catch (Exception e)
225+
{
226+
log.error("Unable to process JsonFile: " + getObjectId(), e);
227+
}
221228
}
222229
}
223230

@@ -237,17 +244,19 @@ public JSONObject getConfigJson(User u, Logger log) throws PipelineJobException
237244

238245
JSONArray tracks = new JSONArray();
239246
List<JsonFile> jsonFiles = getJsonFiles(u, false);
247+
List<JsonFile> visibleTracks = new ArrayList<>();
240248
for (JsonFile jsonFile : jsonFiles)
241249
{
242250
JSONObject o = jsonFile.toTrackJson(u, rg, log);
243251
if (o != null)
244252
{
245253
tracks.put(o);
254+
visibleTracks.add(jsonFile);
246255
}
247256
}
248257

249258
ret.put("tracks", tracks);
250-
ret.put("defaultSession", getDefaultSessionJson(jsonFiles));
259+
ret.put("defaultSession", getDefaultSessionJson(visibleTracks));
251260

252261
return ret;
253262
}

0 commit comments

Comments
 (0)