Respect custom locations for sequence import

Author: bbimber

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/RefNtSequenceModel.java

@@ -418,7 +418,7 @@ public byte[] getSequenceBases()
 
     public void createFileForSequence(User u, String sequence, @Nullable File outDir) throws IOException
     {
-        File output = getExpectedSequenceFile();
+        File output = getExpectedSequenceFile(outDir);
         if (output.exists())
         {
             output.delete();
@@ -443,9 +443,9 @@ public void createFileForSequence(User u, String sequence, @Nullable File outDir
         Table.update(u, ti, this, _rowid);
     }
 
-    public File getExpectedSequenceFile() throws IllegalArgumentException
+    public File getExpectedSequenceFile(@Nullable File outDir) throws IllegalArgumentException
     {
-        return FileUtil.appendName(getHashedDir(true), _rowid + ".txt.gz");
+        return FileUtil.appendName(getSequenceFileBaseDir(outDir, true), _rowid + ".txt.gz");
     }
 
     private Container getLabKeyContainer()
@@ -553,8 +553,13 @@ public File getOffsetsFile()
         return FileUtil.appendName(d.getFile().getParentFile(), getRowid() + "_offsets.txt");
     }
 
-    private File getHashedDir(boolean create)
+    private File getSequenceFileBaseDir(@Nullable File outDir, boolean create)
     {
+        if (outDir != null)
+        {
+            return outDir;
+        }
+
         File baseDir = getBaseSequenceDir();
         String digest = Crypt.MD5.digest(String.valueOf(getRowid()));
 

SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisMaintenanceTask.java

@@ -322,8 +322,13 @@ private void processContainer(Container c, Logger log) throws IOException, Pipel
 
                 if (d.getFile().getAbsolutePath().toLowerCase().startsWith(sequenceDir.getAbsolutePath().toLowerCase()))
                 {
+                    // File existence will be verified below:
                     expectedSequences.add(d.getFile());
                 }
+                else if (!d.getFile().exists())
+                {
+                    log.error("Missing sequence file {}", d.getFile().getPath());
+                }
             });
 
             if (sequenceDir.exists())

SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisUpgradeCode.java

@@ -242,6 +242,7 @@ public void migrateSequenceDirs(final ModuleContext moduleContext)
             List<RefNtSequenceModel> nts = ts.getArrayList(RefNtSequenceModel.class);
             _log.info(nts.size() + " total sequences to migrate");
             int processed = 0;
+            int totalMigrated = 0;
             for (RefNtSequenceModel nt : nts)
             {
                 processed++;
@@ -267,11 +268,11 @@ public void migrateSequenceDirs(final ModuleContext moduleContext)
 
                 if (!RefNtSequenceModel.BASE_DIRNAME.equals(legacyFile.getParentFile().getName()))
                 {
-                    _log.error("Sequence appears to have already been migrated, this might indicate a retry after a failed move: {}", legacyFile.getPath());
+                    // NOTE: this includes sequences imported to custom locations, such as refSequenceImport pipeline jobs
                     continue;
                 }
 
-                File newLocation = nt.getExpectedSequenceFile();
+                File newLocation = nt.getExpectedSequenceFile(null);
                 if (!newLocation.getParentFile().exists())
                 {
                     newLocation.getParentFile().mkdirs();
@@ -283,11 +284,14 @@ public void migrateSequenceDirs(final ModuleContext moduleContext)
                     continue;
                 }
 
+                totalMigrated++;
                 FileUtils.copyFile(legacyFile, newLocation);
                 legacyExpData.setDataFileURI(newLocation.toURI());
                 legacyExpData.save(moduleContext.getUpgradeUser());
                 legacyFile.delete();
             }
+
+            _log.info("Total sequences migrated: {}", totalMigrated);
         }
         catch (Exception e)
         {

SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ImportFastaSequencesPipelineJob.java

@@ -12,7 +12,6 @@
 import org.labkey.api.query.DetailsURL;
 import org.labkey.api.security.User;
 import org.labkey.api.security.permissions.InsertPermission;
-import org.labkey.api.assay.AssayFileWriter;
 import org.labkey.api.util.FileType;
 import org.labkey.api.util.FileUtil;
 import org.labkey.api.view.ActionURL;
@@ -56,23 +55,22 @@ public ImportFastaSequencesPipelineJob(Container c, User user, ActionURL url, Pi
         _libraryParams = libraryParams;
 
         _outDir = createLocalDirectory(pipeRoot);
-        setLogFile(new File(_outDir, FileUtil.makeFileNameWithTimestamp("fastaImport", "log")));
+        setLogFile(FileUtil.appendName(_outDir, FileUtil.makeFileNameWithTimestamp("fastaImport", "log")));
     }
 
     public static File createLocalDirectory(PipeRoot pipeRoot) throws IOException
     {
-        File webserverOutDir = new File(pipeRoot.getRootPath(), _folderPrefix);
+        File webserverOutDir = FileUtil.appendName(pipeRoot.getRootPath(), _folderPrefix);
         if (!webserverOutDir.exists())
         {
-            webserverOutDir.mkdir();
+            FileUtil.mkdir(webserverOutDir);
         }
 
-        AssayFileWriter writer = new AssayFileWriter();
         String folderName = "SequenceImport_" + FileUtil.getTimestamp();
         webserverOutDir = FileUtil.findUniqueFileName(folderName, webserverOutDir);
         if (!webserverOutDir.exists())
         {
-            webserverOutDir.mkdirs();
+            FileUtil.mkdirs(webserverOutDir);
         }
 
         return webserverOutDir;
@@ -85,7 +83,7 @@ public String getDescription()
     }
 
     @Override
-    public TaskPipeline getTaskPipeline()
+    public TaskPipeline<?> getTaskPipeline()
     {
         return PipelineJobService.get().getTaskPipeline(new TaskId(ImportFastaSequencesPipelineJob.class));
     }

SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ImportFastaSequencesTask.java

@@ -16,6 +16,7 @@
 package org.labkey.sequenceanalysis.pipeline;
 
 import org.apache.commons.lang3.StringUtils;
+import org.jetbrains.annotations.NotNull;
 import org.labkey.api.pipeline.AbstractTaskFactory;
 import org.labkey.api.pipeline.AbstractTaskFactorySettings;
 import org.labkey.api.pipeline.PipelineJob;
@@ -71,7 +72,7 @@ public List<String> getProtocolActionNames()
         }
 
         @Override
-        public PipelineJob.Task createTask(PipelineJob job)
+        public PipelineJob.Task<?> createTask(PipelineJob job)
         {
             return new ImportFastaSequencesTask(this, job);
         }
@@ -84,7 +85,7 @@ public boolean isJobComplete(PipelineJob job)
     }
 
     @Override
-    public RecordedActionSet run() throws PipelineJobException
+    public @NotNull RecordedActionSet run() throws PipelineJobException
     {
         getJob().getLogger().info("Importing sequences from file(s): ");
         for (File f : getPipelineJob().getFastas())