More active deletion of temp files during SRA download

Author: bbimber

SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceAlignmentTask.java

@@ -702,6 +702,20 @@ private void alignSet(Readset rs, String basename, Map<ReadData, Pair<File, File
             _resumer.setInitialAlignmentDone(bam, alignActions, discardBam);
         }
 
+        // Because this folder can be large, delete it quickly when possible:
+        if (getCachedReadDataDir().exists())
+        {
+            try
+            {
+                getJob().getLogger().debug("Deleting cachedReadDataDir");
+                FileUtils.deleteDirectory(getCachedReadDataDir());
+            }
+            catch (IOException e)
+            {
+                throw new PipelineJobException(e);
+            }
+        }
+
         // This is a special case where the alignment does not actually generate a permanent BAM
         if (bam == null && discardBam)
         {
@@ -1892,8 +1906,34 @@ public void serializeTest() throws Exception
         }
     }
 
+    private File getCachedReadDataDir()
+    {
+        return new File(getHelper().getWorkingDirectory(), "cachedReadData");
+    }
+
     private void restoreArchivedReadDataIfNeeded(Readset rs) throws PipelineJobException
     {
+        if (_resumer.isInitialAlignmentDone())
+        {
+            getJob().getLogger().debug("Initial alignment is complete, will not attempt to download SRA if needed");
+
+            if (getCachedReadDataDir().exists())
+            {
+                getJob().getLogger().debug("Deleting existing cachedReadDataDir");
+
+                try
+                {
+                    FileUtils.deleteDirectory(getCachedReadDataDir());
+                }
+                catch (IOException e)
+                {
+                    throw new PipelineJobException(e);
+                }
+            }
+
+            return;
+        }
+
         Set<String> sraIDs = new HashSet<>();
         for (ReadData rd : rs.getReadData())
         {
@@ -1947,7 +1987,7 @@ else if (sraIDs.contains(rd.getSra_accession()))
                     }
                 }
 
-                File outDir = new File(getHelper().getWorkingDirectory(), "cachedReadData");
+                File outDir = getCachedReadDataDir();
                 getTaskFileManagerImpl().addDeferredIntermediateFile(outDir); // NOTE: this must be deferred so it remains until the end
 
                 File doneFile = new File(outDir, rd.getSra_accession() + ".done");

singlecell/src/org/labkey/singlecell/run/NimbleHelper.java

@@ -460,6 +460,21 @@ private Map<NimbleGenome, File> doAlignment(List<NimbleGenome> genomes, List<Fil
         alignArgs.add(tmpDir.getPath());
 
         boolean dockerRan = runUsingDocker(alignArgs, output, "align.all");
+
+        // Because this can be large, delete it quickly:
+        if (tmpDir.exists())
+        {
+            try
+            {
+                getPipelineCtx().getLogger().debug("Deleting nimble temp dir");
+                FileUtils.deleteDirectory(tmpDir);
+            }
+            catch (IOException e)
+            {
+                throw new PipelineJobException(e);
+            }
+        }
+
         for (NimbleGenome genome : genomes)
         {
             File alignResultsGz = new File(getPipelineCtx().getWorkingDirectory(), "alignResults." + genome.genomeId + ".txt.gz");