Create nimble genome using local subfolder to more easily enable caching

Author: bbimber

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/ReferenceGenomeManager.java

@@ -105,7 +105,7 @@ public void cacheGenomeLocally(ReferenceGenome genome, Logger log) throws Pipeli
         File localCacheDir = SequencePipelineService.get().getRemoteGenomeCacheDirectory();
         if (isUpToDate(genome))
         {
-            log.debug("Genome up-to-date, will not repeat rsync");
+            log.debug("Genome up-to-date, will not repeat rsync: " + genome.getGenomeId());
             genome.setWorkingFasta(new File(new File(localCacheDir, genome.getGenomeId().toString()), genome.getSourceFastaFile().getName()));
 
             return;

singlecell/src/org/labkey/singlecell/run/NimbleHelper.java

@@ -153,28 +153,18 @@ public void prepareGenome(int genomeId) throws PipelineJobException
             throw new PipelineJobException(e);
         }
 
-        File indexDir = AlignerIndexUtil.getIndexDir(rg, "nimble");
-        if (!indexDir.exists())
-        {
-            indexDir.mkdir();
-        }
+        AlignerIndexUtil.saveCachedIndex(false, getPipelineCtx(), getLocalIndexDir(genomeId, true), "nimble", rg);
+    }
 
-        for (File f : Arrays.asList(csv, fasta))
+    private File getLocalIndexDir(int genomeId, boolean createIfMissing)
+    {
+        File dir = new File(getPipelineCtx().getSourceDirectory(), "genome." + genomeId);
+        if (createIfMissing && !dir.exists())
         {
-            File cached = new File(indexDir, f.getName());
-            if (!cached.exists())
-            {
-                try
-                {
-                    getPipelineCtx().getLogger().debug("Caching file: " + cached.getPath());
-                    FileUtils.copyFile(f, cached);
-                }
-                catch (IOException e)
-                {
-                    throw new PipelineJobException(e);
-                }
-            }
+            dir.mkdir();
         }
+
+        return dir;
     }
 
     private File getGenomeCsv(int genomeId) throws PipelineJobException
@@ -196,7 +186,7 @@ private File getGenomeCsv(int genomeId, boolean forceWorkDir) throws PipelineJob
             return new File(indexDir, "genome." + genomeId + ".csv");
         }
 
-        return new File(getPipelineCtx().getSourceDirectory(), "genome." + genomeId + ".csv");
+        return checkForLegacyGenome(new File(getLocalIndexDir(genomeId, true), "genome." + genomeId + ".csv"));
     }
 
     private File getGenomeFasta(int genomeId) throws PipelineJobException
@@ -218,7 +208,37 @@ private File getGenomeFasta(int genomeId, boolean forceWorkDir) throws PipelineJ
             return new File(indexDir, "genome." + genomeId + ".fasta");
         }
 
-        return new File(getPipelineCtx().getSourceDirectory(), "genome." + genomeId + ".fasta");
+        return checkForLegacyGenome(new File(getLocalIndexDir(genomeId, true), "genome." + genomeId + ".fasta"));
+    }
+
+    // TODO: This should ultimately be removed:
+    private File checkForLegacyGenome(File fileNewLocation) throws PipelineJobException
+    {
+        if (fileNewLocation.exists())
+        {
+            return fileNewLocation;
+        }
+
+        File oldLocation = new File(fileNewLocation.getParentFile().getParentFile(), fileNewLocation.getName());
+        if (oldLocation.exists())
+        {
+            getPipelineCtx().getLogger().debug("Genome file found in old location, moving: " + oldLocation.getPath());
+            if (!fileNewLocation.getParentFile().exists())
+            {
+                fileNewLocation.getParentFile().mkdir();
+            }
+
+            try
+            {
+                FileUtils.moveFile(oldLocation, fileNewLocation);
+            }
+            catch (IOException e)
+            {
+                throw new PipelineJobException(e);
+            }
+        }
+
+        return fileNewLocation;
     }
 
     public void doNimbleAlign(File bam, PipelineStepOutput output, Readset rs, String basename) throws UnsupportedOperationException, PipelineJobException