Skip to content

Commit 2049585

Browse files
bbimberbbimber
committed
Allow genome caching to run as remote job
1 parent 490e4e9 commit 2049585

File tree

4 files changed

+174
-72
lines changed

4 files changed

+174
-72
lines changed

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisModule.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
import org.labkey.sequenceanalysis.button.RunMultiQCButton;
5050
import org.labkey.sequenceanalysis.pipeline.AlignmentAnalysisJob;
5151
import org.labkey.sequenceanalysis.pipeline.AlignmentImportJob;
52+
import org.labkey.sequenceanalysis.pipeline.CacheGenomePipelineJob;
5253
import org.labkey.sequenceanalysis.pipeline.CacheGenomeTrigger;
5354
import org.labkey.sequenceanalysis.pipeline.IlluminaImportJob;
5455
import org.labkey.sequenceanalysis.pipeline.ImportFastaSequencesPipelineJob;
@@ -407,7 +408,7 @@ public Set<ExperimentRunType> getExperimentRunTypes(@Nullable Container containe
407408
PipelineService.get().registerPipelineProvider(new ImportGenomeTrackPipelineJob.Provider(this));
408409
PipelineService.get().registerPipelineProvider(new OrphanFilePipelineProvider(this));
409410
PipelineService.get().registerPipelineProvider(new SequencePipelineProvider(this));
410-
PipelineService.get().registerPipelineProvider(new CacheGenomeTrigger.CacheGenomePipelineJobProvider(this));
411+
PipelineService.get().registerPipelineProvider(new CacheGenomePipelineJob.Provider(this));
411412

412413
LDKService.get().registerQueryButton(new ReprocessLibraryButton(), SequenceAnalysisSchema.SCHEMA_NAME, SequenceAnalysisSchema.TABLE_REF_LIBRARIES);
413414

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/CacheGenomePipelineJob.java‎

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
package org.labkey.sequenceanalysis.pipeline;
2+
3+
import org.jetbrains.annotations.NotNull;
4+
import org.labkey.api.data.Container;
5+
import org.labkey.api.files.FileUrls;
6+
import org.labkey.api.module.Module;
7+
import org.labkey.api.pipeline.AbstractTaskFactory;
8+
import org.labkey.api.pipeline.AbstractTaskFactorySettings;
9+
import org.labkey.api.pipeline.PipeRoot;
10+
import org.labkey.api.pipeline.PipelineDirectory;
11+
import org.labkey.api.pipeline.PipelineJob;
12+
import org.labkey.api.pipeline.PipelineJobException;
13+
import org.labkey.api.pipeline.PipelineProvider;
14+
import org.labkey.api.pipeline.RecordedActionSet;
15+
import org.labkey.api.security.User;
16+
import org.labkey.api.sequenceanalysis.pipeline.ReferenceGenome;
17+
import org.labkey.api.sequenceanalysis.pipeline.ReferenceGenomeManager;
18+
import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService;
19+
import org.labkey.api.util.FileType;
20+
import org.labkey.api.util.FileUtil;
21+
import org.labkey.api.util.PageFlowUtil;
22+
import org.labkey.api.util.URLHelper;
23+
import org.labkey.api.view.ViewBackgroundInfo;
24+
import org.labkey.api.view.ViewContext;
25+
26+
import java.io.File;
27+
import java.util.Arrays;
28+
import java.util.Collections;
29+
import java.util.List;
30+
import java.util.Map;
31+
32+
public class CacheGenomePipelineJob extends PipelineJob
33+
{
34+
private Map<Integer, File> _genomeMap;
35+
36+
public static class Provider extends PipelineProvider
37+
{
38+
public static final String NAME = "cacheGenomePipeline";
39+
40+
public Provider(Module owningModule)
41+
{
42+
super(NAME, owningModule);
43+
}
44+
45+
@Override
46+
public void updateFileProperties(ViewContext context, PipeRoot pr, PipelineDirectory directory, boolean includeAll)
47+
{
48+
49+
}
50+
}
51+
52+
//For serialization:
53+
protected CacheGenomePipelineJob()
54+
{
55+
56+
}
57+
58+
public CacheGenomePipelineJob(Container c, User user, PipeRoot pipeRoot, Map<Integer, File> genomeMap, File outputDir)
59+
{
60+
super(Provider.NAME, new ViewBackgroundInfo(c, user, null), pipeRoot);
61+
62+
_genomeMap = genomeMap;
63+
64+
setLogFile(new File(outputDir, FileUtil.makeFileNameWithTimestamp("cacheGenomes", "log")));
65+
66+
}
67+
68+
public Map<Integer, File> getGenomeMap()
69+
{
70+
return _genomeMap;
71+
}
72+
73+
public void setGenomeMap(Map<Integer, File> genomeMap)
74+
{
75+
_genomeMap = genomeMap;
76+
}
77+
78+
@Override
79+
public URLHelper getStatusHref()
80+
{
81+
return PageFlowUtil.urlProvider(FileUrls.class).urlBegin(getContainer());
82+
}
83+
84+
@Override
85+
public String getDescription()
86+
{
87+
return "Caches reference genomes to a remote filesystem";
88+
}
89+
90+
public static class CacheGenomesTask extends PipelineJob.Task<CacheGenomesTask.Factory>
91+
{
92+
protected CacheGenomesTask(Factory factory, PipelineJob job)
93+
{
94+
super(factory, job);
95+
}
96+
97+
public static class Factory extends AbstractTaskFactory<AbstractTaskFactorySettings, Factory>
98+
{
99+
public Factory()
100+
{
101+
super(CacheGenomesTask.class);
102+
}
103+
104+
public List<FileType> getInputTypes()
105+
{
106+
return Collections.emptyList();
107+
}
108+
109+
public String getStatusName()
110+
{
111+
return PipelineJob.TaskStatus.running.toString();
112+
}
113+
114+
public List<String> getProtocolActionNames()
115+
{
116+
return Arrays.asList("JBrowse");
117+
}
118+
119+
public PipelineJob.Task createTask(PipelineJob job)
120+
{
121+
return new CacheGenomesTask(this, job);
122+
}
123+
124+
public boolean isJobComplete(PipelineJob job)
125+
{
126+
return false;
127+
}
128+
}
129+
130+
@Override
131+
public @NotNull RecordedActionSet run() throws PipelineJobException
132+
{
133+
CacheGenomePipelineJob job = (CacheGenomePipelineJob)getJob();
134+
135+
File cacheDir = SequencePipelineService.get().getRemoteGenomeCacheDirectory();
136+
if (cacheDir == null)
137+
{
138+
return new RecordedActionSet();
139+
}
140+
141+
if (!cacheDir.exists())
142+
{
143+
cacheDir.mkdirs();
144+
}
145+
146+
for (Integer genomeId : job.getGenomeMap().keySet())
147+
{
148+
File fasta = job.getGenomeMap().get(genomeId);
149+
ReferenceGenome rg = new ReferenceGenomeImpl(fasta, null, genomeId, null);
150+
151+
ReferenceGenomeManager.get().cacheGenomeLocally(rg, getJob().getLogger());
152+
}
153+
154+
return new RecordedActionSet();
155+
}
156+
}
157+
}

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/CacheGenomeTrigger.java‎

Lines changed: 1 addition & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -2,25 +2,15 @@
22

33
import org.apache.log4j.Logger;
44
import org.labkey.api.data.Container;
5-
import org.labkey.api.files.FileUrls;
6-
import org.labkey.api.module.Module;
75
import org.labkey.api.module.ModuleLoader;
86
import org.labkey.api.pipeline.PipeRoot;
9-
import org.labkey.api.pipeline.PipelineDirectory;
10-
import org.labkey.api.pipeline.PipelineJob;
117
import org.labkey.api.pipeline.PipelineJobException;
12-
import org.labkey.api.pipeline.PipelineProvider;
138
import org.labkey.api.pipeline.PipelineService;
149
import org.labkey.api.pipeline.PipelineValidationException;
1510
import org.labkey.api.security.User;
1611
import org.labkey.api.sequenceanalysis.GenomeTrigger;
1712
import org.labkey.api.sequenceanalysis.SequenceAnalysisService;
1813
import org.labkey.api.sequenceanalysis.pipeline.ReferenceGenome;
19-
import org.labkey.api.util.FileUtil;
20-
import org.labkey.api.util.PageFlowUtil;
21-
import org.labkey.api.util.URLHelper;
22-
import org.labkey.api.view.ViewBackgroundInfo;
23-
import org.labkey.api.view.ViewContext;
2414
import org.labkey.sequenceanalysis.SequenceAnalysisModule;
2515

2616
import java.io.File;
@@ -78,7 +68,7 @@ public static void cacheGenomes(Container c, User u, Map<Integer, File> genomeMa
7868
{
7969
PipeRoot pipeRoot = PipelineService.get().findPipelineRoot(c);
8070

81-
File logFileDir = new File(pipeRoot.getRootPath(), CacheGenomePipelineJobProvider.NAME);
71+
File logFileDir = new File(pipeRoot.getRootPath(), CacheGenomePipelineJob.Provider.NAME);
8272
if (!logFileDir.exists())
8373
{
8474
logFileDir.mkdirs();
@@ -95,64 +85,4 @@ public static void cacheGenomes(Container c, User u, Map<Integer, File> genomeMa
9585
log.error(e.getMessage(), e);
9686
}
9787
}
98-
99-
public static class CacheGenomePipelineJob extends PipelineJob
100-
{
101-
private Map<Integer, File> _genomeMap;
102-
103-
//For serialization:
104-
protected CacheGenomePipelineJob()
105-
{
106-
107-
}
108-
109-
public CacheGenomePipelineJob(Container c, User user, PipeRoot pipeRoot, Map<Integer, File> genomeMap, File outputDir)
110-
{
111-
super(CacheGenomePipelineJobProvider.NAME, new ViewBackgroundInfo(c, user, null), pipeRoot);
112-
113-
_genomeMap = genomeMap;
114-
115-
setLogFile(new File(outputDir, FileUtil.makeFileNameWithTimestamp("cacheGenomes", "log")));
116-
117-
}
118-
119-
public Map<Integer, File> getGenomeMap()
120-
{
121-
return _genomeMap;
122-
}
123-
124-
public void setGenomeMap(Map<Integer, File> genomeMap)
125-
{
126-
_genomeMap = genomeMap;
127-
}
128-
129-
@Override
130-
public URLHelper getStatusHref()
131-
{
132-
return PageFlowUtil.urlProvider(FileUrls.class).urlBegin(getContainer());
133-
}
134-
135-
@Override
136-
public String getDescription()
137-
{
138-
return "Caches reference genomes to a remote filesystem";
139-
}
140-
}
141-
142-
public static class CacheGenomePipelineJobProvider extends PipelineProvider
143-
{
144-
public static final String NAME = "cacheGenomePipeline";
145-
146-
public CacheGenomePipelineJobProvider(Module owningModule)
147-
{
148-
super(NAME, owningModule);
149-
}
150-
151-
@Override
152-
public void updateFileProperties(ViewContext context, PipeRoot pr, PipelineDirectory directory, boolean includeAll)
153-
{
154-
155-
}
156-
}
157-
15888
}

‎SequenceAnalysis/webapp/WEB-INF/SequenceAnalysis/sequenceanalysisContext.xml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,12 @@
123123
<bean class="org.labkey.sequenceanalysis.pipeline.VariantProcessingScatterRemotePrepareTask$Factory">
124124
<property name="join" value="true"/>
125125
</bean>
126+
<bean class="org.labkey.sequenceanalysis.pipeline.VariantProcessingScatterRemotePrepareTask$Factory">
127+
<property name="join" value="true"/>
128+
</bean>
129+
<bean class="org.labkey.sequenceanalysis.pipeline.CacheGenomePipelineJob.CacheGenomesTask$Factory">
130+
<property name="location" value="webserver" />
131+
</bean>
126132
</list>
127133
</property>
128134
<property name="pipelines">
@@ -152,6 +158,14 @@
152158
</list>
153159
</property>
154160
</bean>
161+
<bean class="org.labkey.api.pipeline.TaskPipelineSettings">
162+
<constructor-arg type="java.lang.Class" value="org.labkey.sequenceanalysis.pipeline.CacheGenomePipelineJob"/>
163+
<property name="taskProgressionSpec">
164+
<list>
165+
<value type="java.lang.Class">org.labkey.sequenceanalysis.pipeline.CacheGenomePipelineJob.CacheGenomesTask</value>
166+
</list>
167+
</property>
168+
</bean>
155169
<bean class="org.labkey.api.pipeline.TaskPipelineSettings">
156170
<constructor-arg type="java.lang.Class" value="org.labkey.sequenceanalysis.pipeline.SequenceConcatPipelineJob" />
157171
<property name="taskProgressionSpec">

0 commit comments

Comments
 (0)