Skip to content

Commit fb3d11c

Browse files
bbimberbbimber
authored
Expand test coverage on JBrowse full text search (#187)
* Add test coverage for JBrowse and lucene indexes
1 parent 01ae694 commit fb3d11c

File tree

7 files changed

+313
-34
lines changed

7 files changed

+313
-34
lines changed

‎SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/AbstractCommandWrapper.java‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.jetbrains.annotations.Nullable;
2323
import org.labkey.api.pipeline.PipelineJobException;
2424
import org.labkey.api.pipeline.PipelineJobService;
25+
import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService;
2526
import org.labkey.api.util.StringUtilsLabKey;
2627

2728
import java.io.BufferedReader;
@@ -287,4 +288,60 @@ protected boolean isThrowNonZeroExits()
287288
{
288289
return _throwNonZeroExits;
289290
}
291+
292+
protected File resolveFileInPath(String exe, @Nullable String packageName, boolean throwIfNotFound)
293+
{
294+
File fn;
295+
String path;
296+
297+
if (packageName != null)
298+
{
299+
path = PipelineJobService.get().getConfigProperties().getSoftwarePackagePath(packageName);
300+
if (path != null)
301+
{
302+
fn = new File(path, exe);
303+
if (fn.exists())
304+
{
305+
return fn;
306+
}
307+
}
308+
}
309+
310+
path = PipelineJobService.get().getConfigProperties().getSoftwarePackagePath(SequencePipelineService.SEQUENCE_TOOLS_PARAM);
311+
if (path != null)
312+
{
313+
fn = new File(path, exe);
314+
if (fn.exists())
315+
{
316+
return fn;
317+
}
318+
}
319+
320+
path = PipelineJobService.get().getAppProperties().getToolsDirectory();
321+
if (path != null)
322+
{
323+
fn = new File(path, exe);
324+
if (fn.exists())
325+
{
326+
return fn;
327+
}
328+
}
329+
330+
String[] paths = System.getenv("PATH").split(File.pathSeparator);
331+
for (String pathDir : paths)
332+
{
333+
fn = new File(pathDir, exe);
334+
if (fn.exists())
335+
{
336+
return fn;
337+
}
338+
}
339+
340+
if (throwIfNotFound)
341+
{
342+
throw new IllegalArgumentException("Unable to find file: "+ exe);
343+
}
344+
345+
return null;
346+
}
290347
}

‎SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/AbstractGatk4Wrapper.java‎

Lines changed: 7 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
import org.apache.logging.log4j.Logger;
66
import org.jetbrains.annotations.Nullable;
77
import org.labkey.api.pipeline.PipelineJobException;
8-
import org.labkey.api.pipeline.PipelineJobService;
98
import org.labkey.api.sequenceanalysis.SequenceAnalysisService;
109
import org.labkey.api.sequenceanalysis.pipeline.ReferenceGenome;
1110
import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService;
@@ -34,21 +33,14 @@ protected String getJarName()
3433
return "GenomeAnalysisTK4.jar";
3534
}
3635

37-
protected File getJAR()
36+
public File getJAR()
3837
{
39-
String path = PipelineJobService.get().getConfigProperties().getSoftwarePackagePath(getPackageName());
40-
if (path != null)
41-
{
42-
return new File(path);
43-
}
44-
45-
path = PipelineJobService.get().getConfigProperties().getSoftwarePackagePath(SequencePipelineService.SEQUENCE_TOOLS_PARAM);
46-
if (path == null)
47-
{
48-
path = PipelineJobService.get().getAppProperties().getToolsDirectory();
49-
}
38+
return getJAR(true);
39+
}
5040

51-
return path == null ? new File(getJarName()) : new File(path, getJarName());
41+
public File getJAR(boolean throwIfNotFound)
42+
{
43+
return resolveFileInPath(getJarName(), getPackageName(), throwIfNotFound);
5244
}
5345

5446
public void setMaxRamOverride(Integer maxRamOverride)
@@ -73,7 +65,7 @@ public void addJava8HomeToEnvironment()
7365

7466
public boolean jarExists()
7567
{
76-
return getJAR() == null || !getJAR().exists();
68+
return getJAR(false) != null;
7769
}
7870

7971
protected void ensureDictionary(File referenceFasta) throws PipelineJobException

‎SequenceAnalysis/pipeline_code/sequence_tools_install.sh‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -851,9 +851,15 @@ then
851851
rm -Rf $LKTOOLS_DIR/_preamble.py
852852
rm -Rf $LKTOOLS_DIR/cutadapt_pip
853853

854-
PIP_VERSION=`pip -V | cut -d '(' -f 2 | sed 's/python //' | cut -c 1 2>1`
854+
if [ -x "$(command -v pip3)" ];then
855+
PIP_EXE=`command -v pip3`
856+
else
857+
PIP_EXE=pip
858+
fi
855859

860+
PIP_VERSION=`$PIP_EXE -V | cut -d '(' -f 2 | sed 's/python //' | cut -c 1 2>1`
856861
if [[ $PIP_VERSION == '2' ]];then
862+
echo 'Using python 2 compatible cutadapt'
857863
wget https://pypi.python.org/packages/source/c/cutadapt/cutadapt-1.8.1.tar.gz
858864
gunzip cutadapt-1.8.1.tar.gz
859865
tar -xf cutadapt-1.8.1.tar
@@ -865,7 +871,7 @@ then
865871
cp -R ./cutadapt-1.8.1/cutadapt ${LKTOOLS_DIR}/cutadapt
866872
else
867873
CUTADAPT_BRANCH=v4.0
868-
pip install --target ${LKTOOLS_DIR}/cutadapt_pip git+https://github.com/marcelm/cutadapt.git@${CUTADAPT_BRANCH}
874+
$PIP_EXE install --target ${LKTOOLS_DIR}/cutadapt_pip git+https://github.com/marcelm/cutadapt.git@${CUTADAPT_BRANCH}
869875
ln -s ${LKTOOLS_DIR}/cutadapt_pip/bin/cutadapt ${LKTOOLS_DIR}/cutadapt
870876
fi
871877
else

‎SequenceAnalysis/test/src/org/labkey/test/tests/external/labModules/SequenceTest.java‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
import org.labkey.test.util.Ext4Helper;
4545
import org.labkey.test.util.PasswordUtil;
4646
import org.labkey.test.util.SimpleHttpResponse;
47+
import org.labkey.test.util.TestLogger;
4748
import org.labkey.test.util.TextSearcher;
4849
import org.labkey.test.util.core.webdav.WebDavUploadHelper;
4950
import org.labkey.test.util.ext4cmp.Ext4CmpRef;
@@ -97,9 +98,9 @@ protected String getProjectName()
9798

9899
private Integer _createdGenomeId = null;
99100

100-
private boolean isExternalPipelineEnabled()
101+
public static boolean isExternalPipelineEnabled(String containerPath)
101102
{
102-
SimpleHttpResponse httpResponse = WebTestHelper.getHttpResponse(WebTestHelper.getBaseURL() + "/sequenceanalysis/" + getProjectName() + "/getSequencePipelineEnabled.view");
103+
SimpleHttpResponse httpResponse = WebTestHelper.getHttpResponse(WebTestHelper.getBaseURL() + "/sequenceanalysis/" + containerPath + "/getSequencePipelineEnabled.view");
103104
Boolean sequencePipelineEnabled;
104105
try
105106
{
@@ -110,15 +111,14 @@ private boolean isExternalPipelineEnabled()
110111
{
111112
sequencePipelineEnabled = false;
112113
}
113-
log("sequencePipelineEnabled: " + sequencePipelineEnabled);
114+
115+
TestLogger.log("sequencePipelineEnabled: " + sequencePipelineEnabled);
114116

115117
if (!sequencePipelineEnabled && TestProperties.isTestRunningOnTeamCity() && WebTestHelper.getDatabaseType() == WebTestHelper.DatabaseType.PostgreSQL)
116118
{
117119
throw new IllegalStateException("When running on team city, -DsequencePipelineEnabled should be true");
118120
}
119121

120-
goToProjectHome();
121-
122122
return sequencePipelineEnabled;
123123
}
124124

@@ -139,7 +139,8 @@ public void testSteps() throws Exception
139139
analysisPanelTest();
140140
readsetPanelTest();
141141

142-
boolean sequencePipelineEnabled = isExternalPipelineEnabled();
142+
boolean sequencePipelineEnabled = isExternalPipelineEnabled(getProjectName());
143+
goToProjectHome();
143144
alignmentImportPanelTest(sequencePipelineEnabled);
144145
readsetImportTest();
145146

‎jbrowse/src/org/labkey/jbrowse/JBrowseController.java‎

Lines changed: 128 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -305,7 +305,7 @@ public void validateForm(ReprocessResourcesForm form, Errors errors)
305305
{
306306
if ((form.getJsonFiles() == null || form.getJsonFiles().length == 0) && (form.getDatabaseIds() == null || form.getDatabaseIds().length == 0))
307307
{
308-
errors.reject("Must provide a list of sessions or JSON files to re-process");
308+
errors.reject(ERROR_MSG, "Must provide a list of sessions or JSON files to re-process");
309309
}
310310
}
311311
}
@@ -813,5 +813,132 @@ private JSONObject getDemoSession(String path) throws IOException
813813
}
814814
}
815815
}
816+
817+
@RequiresPermission(AdminPermission.class)
818+
public static class LuceneQueryAction extends ReadOnlyApiAction<LuceneQueryForm>
819+
{
820+
@Override
821+
public ApiResponse execute(LuceneQueryForm form, BindException errors)
822+
{
823+
JBrowseSession session = JBrowseSession.getForId(form.getSessionId());
824+
if (session == null)
825+
{
826+
errors.reject(ERROR_MSG, "Unable to find JBrowse session: " + form.getSessionId());
827+
return null;
828+
}
829+
830+
JsonFile track = session.getTrack(getUser(), form.getTrackId());
831+
if (track == null)
832+
{
833+
errors.reject(ERROR_MSG, "Unable to find track with ID: " + form.getTrackId());
834+
return null;
835+
}
836+
837+
if (!track.shouldHaveFreeTextSearch())
838+
{
839+
errors.reject(ERROR_MSG, "This track does not support free text search: " + form.getTrackId());
840+
return null;
841+
}
842+
843+
if (!track.doExpectedSearchIndexesExist())
844+
{
845+
errors.reject(ERROR_MSG, "The lucene index has not been created for this track: " + form.getTrackId());
846+
return null;
847+
}
848+
849+
File indexPath = track.getExpectedLocationOfLuceneIndex(true);
850+
851+
// TODO: execute lucene query and get results. Below is a total guess on what the client should provide for paging:
852+
int pageSize = form.getPageSize();
853+
int offset = form.getOffset();
854+
855+
try
856+
{
857+
JSONObject results = new JSONObject();
858+
859+
//TODO: we should probably stream this
860+
return new ApiSimpleResponse(results);
861+
}
862+
catch (Exception e)
863+
{
864+
_log.error("Error in LuceneQuery", e);
865+
errors.reject(ERROR_MSG, e.getMessage());
866+
return null;
867+
}
868+
}
869+
870+
@Override
871+
public void validateForm(LuceneQueryForm form, Errors errors)
872+
{
873+
if ((form.getSearchString() == null || form.getSessionId() == null))
874+
{
875+
errors.reject(ERROR_MSG, "Must provide search string and the JBrowse session ID");
876+
}
877+
}
878+
}
879+
880+
public static class LuceneQueryForm
881+
{
882+
private String _searchString;
883+
884+
private String _sessionId;
885+
886+
// This is the GUID
887+
private String _trackId;
888+
889+
private int _pageSize = 1000;
890+
891+
private int offset = -1;
892+
893+
public String getSearchString()
894+
{
895+
return _searchString;
896+
}
897+
898+
public void setSearchString(String searchString)
899+
{
900+
_searchString = searchString;
901+
}
902+
903+
public String getSessionId()
904+
{
905+
return _sessionId;
906+
}
907+
908+
public void setSessionId(String sessionId)
909+
{
910+
_sessionId = sessionId;
911+
}
912+
913+
public int getPageSize()
914+
{
915+
return _pageSize;
916+
}
917+
918+
public void setPageSize(int pageSize)
919+
{
920+
_pageSize = pageSize;
921+
}
922+
923+
public int getOffset()
924+
{
925+
return offset;
926+
}
927+
928+
public void setOffset(int offset)
929+
{
930+
this.offset = offset;
931+
}
932+
933+
public String getTrackId()
934+
{
935+
return _trackId;
936+
}
937+
938+
public void setTrackId(String trackId)
939+
{
940+
_trackId = trackId;
941+
}
942+
}
816943
}
817944

‎jbrowse/src/org/labkey/jbrowse/model/JsonFile.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -962,6 +962,12 @@ private void prepareLuceneIndex(Logger log) throws PipelineJobException
962962
DISCVRSeqRunner runner = new DISCVRSeqRunner(log);
963963
runner.addJava8HomeToEnvironment();
964964

965+
if (!runner.jarExists())
966+
{
967+
log.error("Unable to find DISCVRSeq.jar, skiping lucene index creation");
968+
return;
969+
}
970+
965971
List<String> args = runner.getBaseArgs("VcfToLuceneIndexer");
966972
args.add("-V");
967973
args.add(getExpData().getFile().getPath());

0 commit comments

Comments
 (0)