Initial support for the archiving sequence data to SRA while retaining record of files

bbimber · bbimber · commit fb0a30642a2f · 2020-03-20T16:08:49.000-07:00
diff --git a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/model/ReadData.java b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/model/ReadData.java
@@ -55,4 +55,6 @@ public interface ReadData extends Serializable
     public Date getModified();
 
     public Integer getModifiedBy();
+
+    public boolean isArchived();
 }
diff --git a/SequenceAnalysis/resources/queries/sequenceanalysis/readData/.qview.xml b/SequenceAnalysis/resources/queries/sequenceanalysis/readData/.qview.xml
@@ -12,6 +12,7 @@
        <column name="date" />
        <column name="description" />
        <column name="sra_accession" />
+       <column name="archived" />
 
        <column name="created" />
        <column name="workbook" />
diff --git a/SequenceAnalysis/resources/queries/sequenceanalysis/readData/File Details.qview.xml b/SequenceAnalysis/resources/queries/sequenceanalysis/readData/File Details.qview.xml
@@ -12,6 +12,7 @@
        <column name="date" />
        <column name="description" />
        <column name="sra_accession" />
+       <column name="archived" />
 
        <column name="created" />
        <column name="workbook" />
diff --git a/SequenceAnalysis/resources/queries/sequenceanalysis/readData/With Filepath.qview.xml b/SequenceAnalysis/resources/queries/sequenceanalysis/readData/With Filepath.qview.xml
@@ -12,6 +12,7 @@
        <column name="date" />
        <column name="description" />
        <column name="sra_accession" />
+       <column name="archived" />
 
        <column name="created" />
        <column name="workbook" />
diff --git a/SequenceAnalysis/resources/schemas/dbscripts/postgresql/SequenceAnalysis-12.319-12.320.sql b/SequenceAnalysis/resources/schemas/dbscripts/postgresql/SequenceAnalysis-12.319-12.320.sql
@@ -0,0 +1,4 @@
+ALTER TABLE sequenceanalysis.barcodes ADD reverse_complement varchar(4000);
+ALTER TABLE sequenceanalysis.readdata ADD archived bool default false;
+
+UPDATE sequenceanalysis.readdata SET archived = false;
diff --git a/SequenceAnalysis/resources/schemas/dbscripts/sqlserver/SequenceAnalysis-12.319-12.320.sql b/SequenceAnalysis/resources/schemas/dbscripts/sqlserver/SequenceAnalysis-12.319-12.320.sql
@@ -0,0 +1,4 @@
+ALTER TABLE sequenceanalysis.barcodes ADD reverse_complement varchar(4000);
+ALTER TABLE sequenceanalysis.readdata ADD archived bit default 0;
+GO
+UPDATE sequenceanalysis.readdata SET archived = 0;
diff --git a/SequenceAnalysis/resources/schemas/sequenceanalysis.xml b/SequenceAnalysis/resources/schemas/sequenceanalysis.xml
@@ -14,6 +14,11 @@
                 <inputType>textarea</inputType>
                 <description>This should be the forward orientation of the sequence, as it appears in the adapter sequence</description>
             </column>
+            <column columnName="reverse_complement">
+                <nullable>false</nullable>
+                <inputType>textarea</inputType>
+                <description>The reverse complement of the sequence</description>
+            </column>
             <column columnName="group_name">
                 <nullable>false</nullable>
             </column>
@@ -1303,6 +1308,9 @@
                 <columnTitle>SRA Run</columnTitle>
                 <url>https://trace.ncbi.nlm.nih.gov/Traces/sra/?run=${sra_accession}</url>
             </column>
+            <column columnName="archived">
+                <columnTitle>Archived to SRA?</columnTitle>
+            </column>
             <column columnName="container">
             </column>
             <column columnName="createdby">
diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/ReadDataImpl.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/ReadDataImpl.java
@@ -30,6 +30,7 @@ public class ReadDataImpl implements ReadData
     private Date _modified;
     private Integer _modifiedBy;
     private Integer _runId;
+    private boolean _archived = false;
 
     private Map<Integer, File> _cachedFiles = new HashMap<>();
 
@@ -239,4 +240,15 @@ public boolean isPairedEnd()
     {
         return getFile2() != null;
     }
+
+    @Override
+    public boolean isArchived()
+    {
+        return _archived;
+    }
+
+    public void setArchived(boolean archived)
+    {
+        _archived = archived;
+    }
 }
diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisController.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisController.java
@@ -3508,6 +3508,14 @@ private JSONObject getDataJsonForReadset(SequenceOutputHandler handler, Readset
             o.put("totalReadData", rs.getReadData().size());
             for (ReadData rd : rs.getReadData())
             {
+                if (rd.isArchived())
+                {
+                    o.put("fileExists", false);
+                    o.put("isArchived", true);
+                    o.put("error", true);
+                    return o;
+                }
+
                 ExpData d = rd.getFileId1() == 0 ? null : ExperimentService.get().getExpData(rd.getFileId1());
                 if (d == null || d.getFile() == null || !d.getFile().exists())
                 {
diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisMaintenanceTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisMaintenanceTask.java
@@ -77,27 +77,47 @@ private void verifySequenceDataPresent(Logger log)
             {
                 ExpData d = ExperimentService.get().getExpData(rd.getFileId1());
                 Container c = ContainerManager.getForId(rd.getContainer());
-                if (d == null || d.getFile() == null)
+                if (!rd.isArchived())
                 {
-                    log.error("Unable to find file associated with ReadData: " + rd.getRowid() + ", " + rd.getFileId1() + " for container: " + (c == null ? rd.getContainer() : c.getPath()));
+                    if (d == null || d.getFile() == null)
+                    {
+                        log.error("Unable to find file associated with ReadData: " + rd.getRowid() + ", " + rd.getFileId1() + " for container: " + (c == null ? rd.getContainer() : c.getPath()));
+                    }
+                    else if (!d.getFile().exists())
+                    {
+                        log.error("Unable to find file associated with ReadData: " + rd.getRowid() + ", " + rd.getFileId1() + ", " + d.getFile().getPath() + " for container: " + (c == null ? rd.getContainer() : c.getPath()));
+                    }
                 }
-                else if (!d.getFile().exists())
+                else
                 {
-                    log.error("Unable to find file associated with ReadData: " + rd.getRowid() + ", " + rd.getFileId1() + ", " + d.getFile().getPath() + " for container: " + (c == null ? rd.getContainer() : c.getPath()));
+                    if (d != null && d.getFile() != null && d.getFile().exists())
+                    {
+                        log.error("ReadData marked as archived, but file exists: " + rd.getRowid() + ", " + rd.getFileId1() + ", " + d.getFile().getPath() + " for container: " + (c == null ? rd.getContainer() : c.getPath()));
+                    }
                 }
             }
 
             if (rd.getFileId2() != null)
             {
                 ExpData d = ExperimentService.get().getExpData(rd.getFileId2());
                 Container c = ContainerManager.getForId(rd.getContainer());
-                if (d == null || d.getFile() == null)
+                if (!rd.isArchived())
                 {
-                    log.error("Unable to find file associated with ReadData: " + rd.getRowid() + ", " + rd.getFileId2() + " for container: " + (c == null ? rd.getContainer() : c.getPath()));
+                    if (d == null || d.getFile() == null)
+                    {
+                        log.error("Unable to find file associated with ReadData: " + rd.getRowid() + ", " + rd.getFileId2() + " for container: " + (c == null ? rd.getContainer() : c.getPath()));
+                    }
+                    else if (!d.getFile().exists())
+                    {
+                        log.error("Unable to find file associated with ReadData: " + rd.getRowid() + ", " + rd.getFileId2() + ", " + d.getFile().getPath() + " for container: " + (c == null ? rd.getContainer() : c.getPath()));
+                    }
                 }
-                else if (!d.getFile().exists())
+                else
                 {
-                    log.error("Unable to find file associated with ReadData: " + rd.getRowid() + ", " + rd.getFileId2() + ", " + d.getFile().getPath() + " for container: " + (c == null ? rd.getContainer() : c.getPath()));
+                    if (d != null && d.getFile() != null && d.getFile().exists())
+                    {
+                        log.error("ReadData marked as archived, but file exists: " + rd.getRowid() + ", " + rd.getFileId1() + ", " + d.getFile().getPath() + " for container: " + (c == null ? rd.getContainer() : c.getPath()));
+                    }
                 }
             }
         }
diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisModule.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisModule.java
@@ -171,7 +171,7 @@ public String getName()
     @Override
     public Double getSchemaVersion()
     {
-        return 12.319;
+        return 12.320;
     }
 
     @Override
diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceReadsetImpl.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceReadsetImpl.java
@@ -407,4 +407,17 @@ public String getLegalFileName()
     {
         return getName().replaceAll("[^a-zA-Z0-9\\.\\-\\+]", "_");
     }
+
+    public boolean hasArchivedData()
+    {
+        for (ReadData rd : getReadData())
+        {
+            if (rd.isArchived())
+            {
+                return true;
+            }
+        }
+
+        return false;
+    }
 }
diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/AlignmentInitTask.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/AlignmentInitTask.java
@@ -97,6 +97,11 @@ public RecordedActionSet run() throws PipelineJobException
 
         getJob().getLogger().info("Starting to process readset: " + getPipelineJob().getReadset().getName() + " (" + getPipelineJob().getReadset().getRowId() + ")");
 
+        if (getPipelineJob().getReadset().hasArchivedData())
+        {
+            throw new PipelineJobException("The input readset has archived read data and cannot be used for new alignments");
+        }
+
         getHelper().cacheExpDatasForParams();
 
         //build reference if needed

Original file line number	Diff line number	Diff line change
`@@ -55,4 +55,6 @@ public interface ReadData extends Serializable`
`55`	`55`	`public Date getModified();`
`56`	`56`
`57`	`57`	`public Integer getModifiedBy();`
	`58`	`+`
	`59`	`+ public boolean isArchived();`
`58`	`60`	`}`
Original file line number	Diff line number	Diff line change
`@@ -77,27 +77,47 @@ private void verifySequenceDataPresent(Logger log)`
`77`	`77`	`{`
`78`	`78`	`ExpData d = ExperimentService.get().getExpData(rd.getFileId1());`
`79`	`79`	`Container c = ContainerManager.getForId(rd.getContainer());`
`80`		`- if (d == null \|\| d.getFile() == null)`
	`80`	`+ if (!rd.isArchived())`
`81`	`81`	`{`
`82`		`- log.error("Unable to find file associated with ReadData: " + rd.getRowid() + ", " + rd.getFileId1() + " for container: " + (c == null ? rd.getContainer() : c.getPath()));`
	`82`	`+ if (d == null \|\| d.getFile() == null)`
	`83`	`+ {`
	`84`	`+ log.error("Unable to find file associated with ReadData: " + rd.getRowid() + ", " + rd.getFileId1() + " for container: " + (c == null ? rd.getContainer() : c.getPath()));`
	`85`	`+ }`
	`86`	`+ else if (!d.getFile().exists())`
	`87`	`+ {`
	`88`	`+ log.error("Unable to find file associated with ReadData: " + rd.getRowid() + ", " + rd.getFileId1() + ", " + d.getFile().getPath() + " for container: " + (c == null ? rd.getContainer() : c.getPath()));`
	`89`	`+ }`
`83`	`90`	`}`
`84`		`- else if (!d.getFile().exists())`
	`91`	`+ else`
`85`	`92`	`{`
`86`		`- log.error("Unable to find file associated with ReadData: " + rd.getRowid() + ", " + rd.getFileId1() + ", " + d.getFile().getPath() + " for container: " + (c == null ? rd.getContainer() : c.getPath()));`
	`93`	`+ if (d != null && d.getFile() != null && d.getFile().exists())`
	`94`	`+ {`
	`95`	`+ log.error("ReadData marked as archived, but file exists: " + rd.getRowid() + ", " + rd.getFileId1() + ", " + d.getFile().getPath() + " for container: " + (c == null ? rd.getContainer() : c.getPath()));`
	`96`	`+ }`
`87`	`97`	`}`
`88`	`98`	`}`
`89`	`99`
`90`	`100`	`if (rd.getFileId2() != null)`
`91`	`101`	`{`
`92`	`102`	`ExpData d = ExperimentService.get().getExpData(rd.getFileId2());`
`93`	`103`	`Container c = ContainerManager.getForId(rd.getContainer());`
`94`		`- if (d == null \|\| d.getFile() == null)`
	`104`	`+ if (!rd.isArchived())`
`95`	`105`	`{`
`96`		`- log.error("Unable to find file associated with ReadData: " + rd.getRowid() + ", " + rd.getFileId2() + " for container: " + (c == null ? rd.getContainer() : c.getPath()));`
	`106`	`+ if (d == null \|\| d.getFile() == null)`
	`107`	`+ {`
	`108`	`+ log.error("Unable to find file associated with ReadData: " + rd.getRowid() + ", " + rd.getFileId2() + " for container: " + (c == null ? rd.getContainer() : c.getPath()));`
	`109`	`+ }`
	`110`	`+ else if (!d.getFile().exists())`
	`111`	`+ {`
	`112`	`+ log.error("Unable to find file associated with ReadData: " + rd.getRowid() + ", " + rd.getFileId2() + ", " + d.getFile().getPath() + " for container: " + (c == null ? rd.getContainer() : c.getPath()));`
	`113`	`+ }`
`97`	`114`	`}`
`98`		`- else if (!d.getFile().exists())`
	`115`	`+ else`
`99`	`116`	`{`
`100`		`- log.error("Unable to find file associated with ReadData: " + rd.getRowid() + ", " + rd.getFileId2() + ", " + d.getFile().getPath() + " for container: " + (c == null ? rd.getContainer() : c.getPath()));`
	`117`	`+ if (d != null && d.getFile() != null && d.getFile().exists())`
	`118`	`+ {`
	`119`	`+ log.error("ReadData marked as archived, but file exists: " + rd.getRowid() + ", " + rd.getFileId1() + ", " + d.getFile().getPath() + " for container: " + (c == null ? rd.getContainer() : c.getPath()));`
	`120`	`+ }`
`101`	`121`	`}`
`102`	`122`	`}`
`103`	`123`	`}`