Bugfix genome cache pipeline

Author: bbimber

SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisMaintenanceTask.java

@@ -109,7 +109,7 @@ public void run()
                                 return;
                             }
 
-                            CacheGenomeTrigger.cacheGenomes(ContainerManager.getSharedContainer(), adminUser, genomeMap, log);
+                            CacheGenomeTrigger.cacheGenomes(ContainerManager.getSharedContainer(), adminUser, genomeMap, log, true);
                         }
                     }
                     catch (Exception e)

SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/CacheGenomePipelineJob.java

@@ -1,5 +1,6 @@
 package org.labkey.sequenceanalysis.pipeline;
 
+import org.apache.commons.lang3.StringUtils;
 import org.jetbrains.annotations.NotNull;
 import org.labkey.api.data.Container;
 import org.labkey.api.files.FileUrls;
@@ -10,8 +11,11 @@
 import org.labkey.api.pipeline.PipelineDirectory;
 import org.labkey.api.pipeline.PipelineJob;
 import org.labkey.api.pipeline.PipelineJobException;
+import org.labkey.api.pipeline.PipelineJobService;
 import org.labkey.api.pipeline.PipelineProvider;
 import org.labkey.api.pipeline.RecordedActionSet;
+import org.labkey.api.pipeline.TaskId;
+import org.labkey.api.pipeline.TaskPipeline;
 import org.labkey.api.security.User;
 import org.labkey.api.sequenceanalysis.pipeline.ReferenceGenome;
 import org.labkey.api.sequenceanalysis.pipeline.ReferenceGenomeManager;
@@ -28,10 +32,13 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
 
 public class CacheGenomePipelineJob extends PipelineJob
 {
     private Map<Integer, File> _genomeMap;
+    private boolean _deleteOtherFolders = false;
 
     public static class Provider extends PipelineProvider
     {
@@ -55,11 +62,12 @@ protected CacheGenomePipelineJob()
 
     }
 
-    public CacheGenomePipelineJob(Container c, User user, PipeRoot pipeRoot, Map<Integer, File> genomeMap, File outputDir)
+    public CacheGenomePipelineJob(Container c, User user, PipeRoot pipeRoot, Map<Integer, File> genomeMap, File outputDir, boolean deleteOtherFolders)
     {
         super(Provider.NAME, new ViewBackgroundInfo(c, user, null), pipeRoot);
 
         _genomeMap = genomeMap;
+        _deleteOtherFolders = deleteOtherFolders;
 
         setLogFile(new File(outputDir, FileUtil.makeFileNameWithTimestamp("cacheGenomes", "log")));
 
@@ -75,6 +83,16 @@ public void setGenomeMap(Map<Integer, File> genomeMap)
         _genomeMap = genomeMap;
     }
 
+    public boolean isDeleteOtherFolders()
+    {
+        return _deleteOtherFolders;
+    }
+
+    public void setDeleteOtherFolders(boolean deleteOtherFolders)
+    {
+        _deleteOtherFolders = deleteOtherFolders;
+    }
+
     @Override
     public URLHelper getStatusHref()
     {
@@ -87,6 +105,12 @@ public String getDescription()
         return "Caches reference genomes to a remote filesystem";
     }
 
+    @Override
+    public TaskPipeline getTaskPipeline()
+    {
+        return PipelineJobService.get().getTaskPipeline(new TaskId(CacheGenomePipelineJob.class));
+    }
+
     public static class CacheGenomesTask extends PipelineJob.Task<CacheGenomesTask.Factory>
     {
         protected CacheGenomesTask(Factory factory, PipelineJob job)
@@ -135,7 +159,7 @@ public boolean isJobComplete(PipelineJob job)
             File cacheDir = SequencePipelineService.get().getRemoteGenomeCacheDirectory();
             if (cacheDir == null)
             {
-                return new RecordedActionSet();
+                throw new PipelineJobException("This job should not have been initiated unless REMOTE_GENOME_CACHE_DIR is set");
             }
 
             if (!cacheDir.exists())
@@ -151,6 +175,20 @@ public boolean isJobComplete(PipelineJob job)
                 ReferenceGenomeManager.get().cacheGenomeLocally(rg, getJob().getLogger());
             }
 
+            if (job.isDeleteOtherFolders())
+            {
+                Set<String> whitelist = job.getGenomeMap().keySet().stream().map(String::valueOf).collect(Collectors.toSet());
+                File[] toDelete = cacheDir.listFiles((file) -> {
+                    return !whitelist.contains(file.getName());
+                });
+
+                if (toDelete != null && toDelete.length > 0)
+                {
+                    getJob().getLogger().info("Folders will be deleted: " + StringUtils.join(toDelete, ", "));
+                    //TODO: verify
+                }
+            }
+
             return new RecordedActionSet();
         }
     }

SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/CacheGenomeTrigger.java

@@ -11,6 +11,7 @@
 import org.labkey.api.sequenceanalysis.GenomeTrigger;
 import org.labkey.api.sequenceanalysis.SequenceAnalysisService;
 import org.labkey.api.sequenceanalysis.pipeline.ReferenceGenome;
+import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService;
 import org.labkey.sequenceanalysis.SequenceAnalysisModule;
 
 import java.io.File;
@@ -50,7 +51,7 @@ private void possiblyCache(Container c, User u, Logger log, int genomeId)
             Map<Integer, File> genomeMap = new HashMap<>();
             ReferenceGenome rg = SequenceAnalysisService.get().getReferenceGenome(genomeId, u);
             genomeMap.put(rg.getGenomeId(), rg.getSourceFastaFile());
-            cacheGenomes(c, u, genomeMap, log);
+            cacheGenomes(c, u, genomeMap, log, false);
         }
         catch (PipelineJobException e)
         {
@@ -64,8 +65,15 @@ public boolean isAvailable(Container c)
         return c.getActiveModules().contains(ModuleLoader.getInstance().getModule(SequenceAnalysisModule.NAME));
     }
 
-    public static void cacheGenomes(Container c, User u, Map<Integer, File> genomeMap, Logger log)
+    public static void cacheGenomes(Container c, User u, Map<Integer, File> genomeMap, Logger log, boolean deleteOtherFolders)
     {
+        //Dont start pipeline job unless REMOTE_GENOME_CACHE_DIR is set
+        File cacheDir = SequencePipelineService.get().getRemoteGenomeCacheDirectory();
+        if (cacheDir == null)
+        {
+            return;
+        }
+
         PipeRoot pipeRoot = PipelineService.get().findPipelineRoot(c);
 
         File logFileDir = new File(pipeRoot.getRootPath(), CacheGenomePipelineJob.Provider.NAME);
@@ -74,7 +82,7 @@ public static void cacheGenomes(Container c, User u, Map<Integer, File> genomeMa
             logFileDir.mkdirs();
         }
 
-        CacheGenomePipelineJob job = new CacheGenomePipelineJob(c, u, pipeRoot, genomeMap, logFileDir);
+        CacheGenomePipelineJob job = new CacheGenomePipelineJob(c, u, pipeRoot, genomeMap, logFileDir, deleteOtherFolders);
 
         try
         {