Bugfix genome cache maintenance task

Author: bbimber

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/ReferenceGenomeManager.java

@@ -68,13 +68,11 @@ public void markGenomeModified(ReferenceGenome genome) throws IOException
 
     public void cacheGenomeLocally(ReferenceGenome genome, Logger log) throws PipelineJobException
     {
-        File localCacheDir = SequencePipelineService.get().getRemoteGenomeCacheDirectory();
-        if (localCacheDir == null)
+        if (!SequencePipelineService.get().isRemoteGenomeCacheUsed())
         {
             return;
         }
 
-        log.info("attempting to rsync genome to local disks: " + localCacheDir.getPath());
         if (genome.isTemporaryGenome())
         {
             log.info("cannot cache custom genomes, skipping");
@@ -87,6 +85,9 @@ public void cacheGenomeLocally(ReferenceGenome genome, Logger log) throws Pipeli
             return;
         }
 
+        File localCacheDir = SequencePipelineService.get().getRemoteGenomeCacheDirectory();
+        log.info("attempting to rsync genome to local disks: " + localCacheDir.getPath());
+
         File sourceDir = genome.getSourceFastaFile().getParentFile();
 
         new SimpleScriptWrapper(log).execute(Arrays.asList(
@@ -95,6 +96,12 @@ public void cacheGenomeLocally(ReferenceGenome genome, Logger log) throws Pipeli
 
         try
         {
+            File lastUpdate = getLocalUpdateFile(genome);
+            if (!lastUpdate.exists())
+            {
+                FileUtils.touch(lastUpdate);
+            }
+
             FileUtils.touch(getRemoteSyncFile(genome.getGenomeId()));
         }
         catch (IOException e)

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/SequencePipelineService.java

@@ -78,6 +78,8 @@ static public void setInstance(SequencePipelineService instance)
 
     abstract public List<String> getJavaOpts(@Nullable Integer maxRamOverride);
 
+    abstract public boolean isRemoteGenomeCacheUsed();
+
     @Nullable
     abstract public File getRemoteGenomeCacheDirectory();
 

SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisMaintenanceTask.java

@@ -77,7 +77,7 @@ public void run(Logger log)
 
     private void possiblySubmitRemoteTask(Logger log)
     {
-        if (SequencePipelineService.get().getRemoteGenomeCacheDirectory() != null)
+        if (SequencePipelineService.get().isRemoteGenomeCacheUsed())
         {
             JobRunner jr = JobRunner.getDefault();
             jr.execute(new Runnable()
@@ -121,6 +121,10 @@ public void run()
 
             jr.waitForCompletion();
         }
+        else
+        {
+            log.debug("Genome caching not used, skipping");
+        }
     }
 
     private void verifySequenceDataPresent(Logger log)

SequenceAnalysis/src/org/labkey/sequenceanalysis/SequencePipelineServiceImpl.java

@@ -317,6 +317,13 @@ else if (maxRam != null)
         return params;
     }
 
+    @Override
+    public boolean isRemoteGenomeCacheUsed()
+    {
+        String dir = PipelineJobService.get().getConfigProperties().getSoftwarePackagePath("REMOTE_GENOME_CACHE_DIR");
+        return StringUtils.trimToNull(dir) != null;
+    }
+
     @Override
     public File getRemoteGenomeCacheDirectory()
     {

SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/CacheAlignerIndexesTask.java

@@ -164,8 +164,7 @@ public RecordedActionSet run() throws PipelineJobException
         }
 
         //also try to rsync, if enabled
-        File cacheDir = SequencePipelineService.get().getRemoteGenomeCacheDirectory();
-        if (cacheDir != null)
+        if (SequencePipelineService.get().isRemoteGenomeCacheUsed())
         {
             ReferenceGenomeManager.get().cacheGenomeLocally(referenceGenome, getJob().getLogger());
         }

SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/CacheGenomeTrigger.java

@@ -68,8 +68,7 @@ public boolean isAvailable(Container c)
     public static void cacheGenomes(Container c, User u, Map<Integer, File> genomeMap, Logger log, boolean deleteOtherFolders)
     {
         //Dont start pipeline job unless REMOTE_GENOME_CACHE_DIR is set
-        File cacheDir = SequencePipelineService.get().getRemoteGenomeCacheDirectory();
-        if (cacheDir == null)
+        if (!SequencePipelineService.get().isRemoteGenomeCacheUsed())
         {
             return;
         }

SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ReferenceGenomeImpl.java

@@ -138,8 +138,7 @@ public File getAlignerIndexDir(String name)
         {
             //if we rsync locally, these are cached w/ the same structure as the server
             //this only applies for saved reference genomes (i.e. genomeId != null)
-            File remoteDir = SequencePipelineService.get().getRemoteGenomeCacheDirectory();
-            if (remoteDir == null)
+            if (!SequencePipelineService.get().isRemoteGenomeCacheUsed())
             {
                 return new File(_workingFasta.getParentFile(), name);
             }

SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceAlignmentTask.java

@@ -408,8 +408,7 @@ private void copyReferenceResources() throws PipelineJobException
             doCopy = true;
         }
 
-        File localCachedIndexDir = SequencePipelineService.get().getRemoteGenomeCacheDirectory();
-        if (localCachedIndexDir != null)
+        if (SequencePipelineService.get().isRemoteGenomeCacheUsed())
         {
             ReferenceGenomeManager.get().cacheGenomeLocally(referenceGenome, getJob().getLogger());
         }

SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceOutputHandlerRemoteTask.java

@@ -100,8 +100,7 @@ private SequenceOutputHandlerJob getPipelineJob()
 
     public static void possiblyCacheGenomes(SequenceJob job, List<SequenceOutputFile> inputs) throws PipelineJobException
     {
-        File localCachedIndexDir = SequencePipelineService.get().getRemoteGenomeCacheDirectory();
-        if (localCachedIndexDir != null)
+        if (SequencePipelineService.get().isRemoteGenomeCacheUsed())
         {
             Set<Integer> distinctGenomes = inputs.stream().map(SequenceOutputFile::getLibrary_id).filter(Predicates.notNull()).collect(Collectors.toSet());
             for (Integer l : distinctGenomes)