Attempt to improve JBrowseTest (#92)

bbimber · web-flow · commit 184908d7ac56 · 2021-09-19T08:49:53.000-07:00
* Attempt to improve JBrowseTest
diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/util/SequenceUtil.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/util/SequenceUtil.java
@@ -455,18 +455,7 @@ public static void sortROD(File input, Logger log, Integer startColumnIdx) throw
 
         //replace the non-sorted output
         input.delete();
-        if (isCompressed)
-        {
-            File compressed = SequenceUtil.bgzip(sorted, log);
-            if (!compressed.equals(input))
-            {
-                FileUtils.moveFile(compressed, input);
-            }
-        }
-        else
-        {
-            FileUtils.moveFile(sorted, input);
-        }
+        FileUtils.moveFile(sorted, input);
         sorted.delete();
     }
 
diff --git a/jbrowse/resources/web/jbrowse/window/ModifyJsonConfigWindow.js b/jbrowse/resources/web/jbrowse/window/ModifyJsonConfigWindow.js
@@ -83,6 +83,18 @@ Ext4.define('JBrowse.window.ModifyJsonConfigWindow', {
                         handler: function (gridBtn) {
                             this.addAttribute('omitTrack', true, 'BOOLEAN');
                         }
+                    },{
+                        text: 'Add to Text Search Index?',
+                        scope: this,
+                        handler: function (gridBtn) {
+                            this.addAttribute('includeInSearch', true, 'BOOLEAN');
+                        }
+                    },{
+                        text: 'Exclude from Text Search Index?',
+                        scope: this,
+                        handler: function (gridBtn) {
+                            this.addAttribute('excludeFromSearch', true, 'BOOLEAN');
+                        }
                     },{
                         text: 'Other',
                         scope: this,
diff --git a/jbrowse/src/org/labkey/jbrowse/JBrowseMaintenanceTask.java b/jbrowse/src/org/labkey/jbrowse/JBrowseMaintenanceTask.java
@@ -15,7 +15,6 @@
 import org.labkey.api.data.TableInfo;
 import org.labkey.api.data.TableSelector;
 import org.labkey.api.ldk.LDKService;
-import org.labkey.api.pipeline.PipelineJobException;
 import org.labkey.api.query.FieldKey;
 import org.labkey.api.security.User;
 import org.labkey.api.util.PageFlowUtil;
@@ -77,6 +76,24 @@ public void run(Logger log)
         //delete JSON in output dir not associated with DB record
         try
         {
+            // TODO: ultimately remove this. This is related to the migration from JB1 to JB2
+            TableInfo jsonFiles = JBrowseSchema.getInstance().getTable(JBrowseSchema.TABLE_JSONFILES);
+            List<String> toDelete = new TableSelector(jsonFiles, PageFlowUtil.set("objectid"), new SimpleFilter(FieldKey.fromString("sequenceid"), null, CompareType.NONBLANK), null).getArrayList(String.class);
+            if (!toDelete.isEmpty())
+            {
+                log.info("deleted " + toDelete.size() + " legacy NT JsonFile records");
+                for (String key : toDelete)
+                {
+                    Table.delete(jsonFiles, key);
+
+                    int d = new SqlExecutor(JBrowseSchema.getInstance().getSchema()).execute(new SQLFragment("DELETE FROM " + JBrowseSchema.NAME + "." + JBrowseSchema.TABLE_DATABASE_MEMBERS + " WHERE jsonfile = ?", key));
+                    if (d > 0)
+                    {
+                        log.info("also deleted " + d + " orphan database member record(s) for: " + key);
+                    }
+                }
+            }
+
             //delete sessions marked as temporary
             int sessionsDeleted = Table.delete(JBrowseSchema.getInstance().getTable(JBrowseSchema.TABLE_DATABASES), new SimpleFilter(FieldKey.fromString("temporary"), true));
             if (sessionsDeleted > 0)
@@ -92,9 +109,6 @@ public void run(Logger log)
                     " WHERE " +
                     //find JSONFiles not associated with a database_member record
                     " (SELECT count(rowid) FROM " + JBrowseSchema.NAME + "." + JBrowseSchema.TABLE_DATABASE_MEMBERS + " d WHERE d.jsonfile = " + JBrowseSchema.TABLE_JSONFILES + ".objectid) = 0 AND " +
-                    //TODO: ultimately completely remove these
-                    //or library reference sequences
-                    " (SELECT count(rowid) FROM sequenceanalysis.reference_library_members d WHERE d.ref_nt_id = " + JBrowseSchema.TABLE_JSONFILES + ".sequenceid) = 0 AND " +
                     //or library tracks
                     " (SELECT count(rowid) FROM sequenceanalysis.reference_library_tracks d WHERE d.rowid = " + JBrowseSchema.TABLE_JSONFILES + ".trackid) = 0 AND " +
                     //or outputfiles
@@ -105,11 +119,8 @@ public void run(Logger log)
                 log.info("deleted " + deleted2 + " JSON files because they are not used by any sessions");
 
             //second pass at orphan JSONFiles.  Note: these might be referenced by a database_member record still.
-            List<String> toDelete = new SqlSelector(JBrowseSchema.getInstance().getSchema(), new SQLFragment("SELECT objectid FROM " + JBrowseSchema.NAME + "." + JBrowseSchema.TABLE_JSONFILES +
+            toDelete = new SqlSelector(JBrowseSchema.getInstance().getSchema(), new SQLFragment("SELECT objectid FROM " + JBrowseSchema.NAME + "." + JBrowseSchema.TABLE_JSONFILES +
                     " WHERE " +
-                    //TODO: ultimately completely remove these
-                    //reference NT sequences
-                    " (" + JBrowseSchema.TABLE_JSONFILES + ".sequenceid IS NOT NULL AND (SELECT count(rowid) FROM sequenceanalysis.ref_nt_sequences d WHERE d.rowid = " + JBrowseSchema.TABLE_JSONFILES + ".sequenceid) = 0) OR " +
                     //library tracks
                     " ( " + JBrowseSchema.TABLE_JSONFILES + ".trackid IS NOT NULL AND (SELECT count(rowid) FROM sequenceanalysis.reference_library_tracks d WHERE d.rowid = " + JBrowseSchema.TABLE_JSONFILES + ".trackid) = 0) OR " +
                     //outputfiles
@@ -119,7 +130,6 @@ public void run(Logger log)
             if (!toDelete.isEmpty())
             {
                 log.info("deleting " + toDelete.size() + " JSON files because they reference non-existent tracks, sequences or outputfiles");
-                TableInfo jsonFiles = JBrowseSchema.getInstance().getTable(JBrowseSchema.TABLE_JSONFILES);
                 for (String key : toDelete)
                 {
                     Table.delete(jsonFiles, key);
@@ -141,7 +151,7 @@ public void run(Logger log)
         }
     }
 
-    private void processContainer(Container c, Logger log) throws IOException, PipelineJobException
+    private void processContainer(Container c, Logger log) throws IOException
     {
         File jbrowseRoot = JBrowseManager.get().getBaseDir(c, false);
 
@@ -202,9 +212,8 @@ private void processContainer(Container c, Logger log) throws IOException, Pipel
                 File childDir = new File(jbrowseRoot, dir);
                 if (childDir.exists())
                 {
-                    //TODO: eventually delete these?
-                    //log.info("deleting legacy jbrowse " + dir + " dir: " + childDir.getPath());
-                    //FileUtils.deleteDirectory(childDir);
+                    log.info("deleting legacy jbrowse " + dir + " dir: " + childDir.getPath());
+                    FileUtils.deleteDirectory(childDir);
                 }
             }
 
diff --git a/jbrowse/src/org/labkey/jbrowse/model/DbBackedJsonFile.java b/jbrowse/src/org/labkey/jbrowse/model/DbBackedJsonFile.java
@@ -85,6 +85,12 @@ public File getLocationOfProcessedTrack(boolean createDir)
         return new File(trackDir, getSourceFileName());
     }
 
+    @Override
+    public boolean needsProcessing()
+    {
+        return true;
+    }
+
     @Override
     public String getLabel()
     {
diff --git a/jbrowse/src/org/labkey/jbrowse/model/JsonFile.java b/jbrowse/src/org/labkey/jbrowse/model/JsonFile.java
@@ -175,18 +175,6 @@ public File getBaseDir()
 
     private static final FileType _ft = new FileType(Arrays.asList(".bam", ".vcf"), ".vcf", FileType.gzSupportLevel.SUPPORT_GZ);
 
-    //NOTE: if the track file isnt parsed to JSON (VCF or BAM), then the file will be in getBaseDir()
-    public boolean expectDataSubdirForTrack()
-    {
-        File f = getTrackFile();
-        if (f == null)
-        {
-            throw new IllegalArgumentException("Unable to find file for JSONFile: " + _objectId);
-        }
-
-        return !_ft.isType(f);
-    }
-
     public String getLabel()
     {
         if (_label == null)
@@ -408,8 +396,9 @@ public boolean doIndex()
             return false;
         }
 
+        // The intent of this is to index GFF/GTF files that are part of the primary genome, not ad hoc resources. These files can opt-in to indexing using
         //TODO: restore this once JB2 support GTF indexing: TRACK_TYPES.gtf.getFileType().isType(sourceFilename)
-        boolean doIndex = TRACK_TYPES.gff.getFileType().isType(sourceFilename);
+        boolean doIndex = TRACK_TYPES.gff.getFileType().isType(sourceFilename) && getTrackId() != null;
         JSONObject config = getExtraTrackConfig();
         if (config != null)
         {
diff --git a/jbrowse/test/src/org/labkey/test/tests/external/labModules/JBrowseTest.java b/jbrowse/test/src/org/labkey/test/tests/external/labModules/JBrowseTest.java
@@ -15,7 +15,6 @@
  */
 package org.labkey.test.tests.external.labModules;
 
-import org.apache.velocity.util.ArrayListWrapper;
 import org.junit.Assert;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
@@ -44,7 +43,6 @@
 
 import java.io.File;
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
@@ -121,6 +119,11 @@ private By getVariantWithinTrack(String trackId, String variantText, boolean wai
     {
         trackId = "trackRenderingContainer-linearGenomeView-" + trackId;
         Locator.XPathLocator l = Locator.tagWithAttributeContaining("div", "data-testid", trackId);
+        if (waitFor)
+        {
+            waitForElement(l);
+        }
+
         l = l.append(Locator.xpath("//*[name()='text' and contains(text(), '" + variantText + "')]/..")).notHidden();
 
         if (waitFor)
@@ -145,8 +148,8 @@ private void testMessageDisplay()
     private void waitForJBrowseToLoad()
     {
         waitForElementToDisappear(Locator.tagWithText("p", "Loading..."));
-        waitForElement(Locator.tagWithClass("span", "MuiIconButton-label")); //this is the top-left icon
-        waitForElement(Locator.tagWithClassContaining("span", "MuiTypography-root"), WAIT_FOR_PAGE); //this is the icon from the track label
+        waitForElement(Locator.tagWithClass("span", "MuiIconButton-label").notHidden()); //this is the top-left icon
+        waitForElement(Locator.tagWithClassContaining("span", "MuiTypography-root").notHidden(), WAIT_FOR_PAGE); //this is the icon from the track label
 
         waitForElementToDisappear(Locator.tagWithText("div", "Loading...")); //track data
     }

Original file line number	Diff line number	Diff line change
`@@ -85,6 +85,12 @@ public File getLocationOfProcessedTrack(boolean createDir)`
`85`	`85`	`return new File(trackDir, getSourceFileName());`
`86`	`86`	`}`
`87`	`87`
	`88`	`+ @Override`
	`89`	`+ public boolean needsProcessing()`
	`90`	`+ {`
	`91`	`+ return true;`
	`92`	`+ }`
	`93`	`+`
`88`	`94`	`@Override`
`89`	`95`	`public String getLabel()`
`90`	`96`	`{`