Skip to content

Commit 19aba0b

Browse files
labkey-klumlabkey-klum
authored
Transform script refactor (#6159)
1 parent 19196e4 commit 19aba0b

28 files changed

Lines changed: 817 additions & 797 deletions

‎api/src/org/labkey/api/assay/AbstractAssayProvider.java‎

Lines changed: 19 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,16 @@
2020
import org.apache.commons.lang3.StringUtils;
2121
import org.jetbrains.annotations.NotNull;
2222
import org.jetbrains.annotations.Nullable;
23+
import org.json.JSONArray;
2324
import org.labkey.api.assay.actions.AssayRunUploadForm;
2425
import org.labkey.api.assay.actions.DesignerAction;
2526
import org.labkey.api.assay.actions.UploadWizardAction;
2627
import org.labkey.api.assay.pipeline.AssayRunAsyncContext;
2728
import org.labkey.api.assay.plate.FilterCriteria;
2829
import org.labkey.api.assay.security.DesignAssayPermission;
30+
import org.labkey.api.assay.transform.AnalysisScript;
31+
import org.labkey.api.assay.transform.DataExchangeHandler;
32+
import org.labkey.api.assay.transform.DataTransformService;
2933
import org.labkey.api.audit.AuditLogService;
3034
import org.labkey.api.data.ActionButton;
3135
import org.labkey.api.data.ButtonBar;
@@ -79,7 +83,6 @@
7983
import org.labkey.api.module.Module;
8084
import org.labkey.api.pipeline.PipeRoot;
8185
import org.labkey.api.pipeline.PipelineService;
82-
import org.labkey.api.qc.DataExchangeHandler;
8386
import org.labkey.api.query.FieldKey;
8487
import org.labkey.api.query.FilteredTable;
8588
import org.labkey.api.query.QueryService;
@@ -1243,16 +1246,15 @@ public boolean hasUsefulDetailsPage()
12431246
private static final String SCRIPT_PATH_DELIMITER = "|";
12441247

12451248
@Override
1246-
public ValidationException setValidationAndAnalysisScripts(ExpProtocol protocol, @NotNull List<File> scripts) throws ExperimentException
1249+
public ValidationException setValidationAndAnalysisScripts(ExpProtocol protocol, @NotNull List<AnalysisScript> scripts) throws ExperimentException
12471250
{
12481251
Map<String, ObjectProperty> props = new HashMap<>(protocol.getObjectProperties());
12491252
String propertyURI = ScriptType.TRANSFORM.getPropertyURI(protocol);
1250-
12511253
ValidationException validationErrors = new ValidationException();
1252-
StringBuilder sb = new StringBuilder();
1253-
String separator = "";
1254-
for (File scriptFile : scripts)
1254+
1255+
for (AnalysisScript script : scripts)
12551256
{
1257+
File scriptFile = script.getScript().toNioPathForRead().toFile();
12561258
String ext = FileUtil.getExtension(scriptFile);
12571259
if (scriptFile.isFile() && ext != null)
12581260
{
@@ -1274,10 +1276,6 @@ public ValidationException setValidationAndAnalysisScripts(ExpProtocol protocol,
12741276

12751277
validationErrors.addErrors(ParamReplacementSvc.get().validateDeprecatedReplacements(scriptText, scriptFile.getName()));
12761278
}
1277-
1278-
sb.append(separator);
1279-
sb.append(scriptFile.getAbsolutePath());
1280-
separator = SCRIPT_PATH_DELIMITER;
12811279
}
12821280
else
12831281
{
@@ -1294,20 +1292,17 @@ public ValidationException setValidationAndAnalysisScripts(ExpProtocol protocol,
12941292
if (validationErrors.getErrors().stream().anyMatch(e -> SEVERITY.ERROR == e.getSeverity()))
12951293
return validationErrors;
12961294

1297-
if (sb.length() > 0)
1295+
JSONArray json = AnalysisScript.toJson(scripts);
1296+
if (json != null)
12981297
{
12991298
ObjectProperty prop = new ObjectProperty(protocol.getLSID(), protocol.getContainer(),
1300-
propertyURI, sb.toString());
1299+
propertyURI, json.toString());
13011300
props.put(propertyURI, prop);
13021301
}
13031302
else
13041303
{
13051304
props.remove(propertyURI);
13061305
}
1307-
1308-
// Be sure to strip out any validation scripts that were stored with the legacy propertyURI. We merge and save
1309-
// them as a single list in the TRANSFORM
1310-
props.remove(ScriptType.VALIDATION.getPropertyURI(protocol));
13111306
protocol.setObjectProperties(props);
13121307

13131308
return validationErrors;
@@ -1316,7 +1311,6 @@ public ValidationException setValidationAndAnalysisScripts(ExpProtocol protocol,
13161311
/** For migrating legacy assay designs that have separate transform and validation script properties */
13171312
private enum ScriptType
13181313
{
1319-
VALIDATION("ValidationScript"),
13201314
TRANSFORM("TransformScript");
13211315

13221316
private final String _uriSuffix;
@@ -1334,26 +1328,21 @@ public String getPropertyURI(ExpProtocol protocol)
13341328

13351329
@NotNull
13361330
@Override
1337-
public List<File> getValidationAndAnalysisScripts(ExpProtocol protocol, Scope scope)
1331+
public List<AnalysisScript> getValidationAndAnalysisScripts(ExpProtocol protocol, Scope scope)
13381332
{
1339-
List<File> result = new ArrayList<>();
1333+
List<AnalysisScript> result = new ArrayList<>();
13401334
if (scope == Scope.ASSAY_DEF || scope == Scope.ALL)
13411335
{
13421336
ObjectProperty transformScripts = protocol.getObjectProperties().get(ScriptType.TRANSFORM.getPropertyURI(protocol));
13431337
if (transformScripts != null)
13441338
{
1339+
List<AnalysisScript> scripts = AnalysisScript.fromJson(transformScripts.getStringValue());
1340+
if (scripts != null)
1341+
return scripts;
1342+
1343+
// try the legacy serialization
13451344
for (String scriptPath : transformScripts.getStringValue().split("\\" + SCRIPT_PATH_DELIMITER))
1346-
{
1347-
result.add(new File(scriptPath));
1348-
}
1349-
}
1350-
ObjectProperty validationScripts = protocol.getObjectProperties().get(ScriptType.VALIDATION.getPropertyURI(protocol));
1351-
if (validationScripts != null)
1352-
{
1353-
for (String scriptPath : validationScripts.getStringValue().split("\\" + SCRIPT_PATH_DELIMITER))
1354-
{
1355-
result.add(new File(scriptPath));
1356-
}
1345+
result.add(new AnalysisScript(new File(scriptPath), Set.of(DataTransformService.TransformOperation.INSERT)));
13571346
}
13581347
}
13591348
return result;

‎api/src/org/labkey/api/assay/AssayProvider.java‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,14 @@
1616

1717
package org.labkey.api.assay;
1818

19+
import jakarta.servlet.http.HttpServletRequest;
1920
import org.fhcrc.cpas.exp.xml.ExperimentRunType;
2021
import org.jetbrains.annotations.NotNull;
2122
import org.jetbrains.annotations.Nullable;
2223
import org.labkey.api.assay.actions.AssayRunUploadForm;
2324
import org.labkey.api.assay.pipeline.AssayRunAsyncContext;
25+
import org.labkey.api.assay.transform.AnalysisScript;
26+
import org.labkey.api.assay.transform.DataExchangeHandler;
2427
import org.labkey.api.assay.plate.FilterCriteria;
2528
import org.labkey.api.data.Container;
2629
import org.labkey.api.data.ContainerFilter;
@@ -42,11 +45,10 @@
4245
import org.labkey.api.gwt.client.model.GWTPropertyDescriptor;
4346
import org.labkey.api.module.Module;
4447
import org.labkey.api.pipeline.PipelineProvider;
45-
import org.labkey.api.qc.DataExchangeHandler;
4648
import org.labkey.api.query.ValidationException;
4749
import org.labkey.api.security.User;
48-
import org.labkey.api.study.publish.PublishKey;
4950
import org.labkey.api.study.assay.ParticipantVisitResolverType;
51+
import org.labkey.api.study.publish.PublishKey;
5052
import org.labkey.api.util.Pair;
5153
import org.labkey.api.view.ActionURL;
5254
import org.labkey.api.view.HttpView;
@@ -57,7 +59,6 @@
5759
import org.springframework.web.servlet.ModelAndView;
5860
import org.springframework.web.servlet.mvc.Controller;
5961

60-
import jakarta.servlet.http.HttpServletRequest;
6162
import java.io.File;
6263
import java.io.IOException;
6364
import java.util.Collection;
@@ -239,12 +240,10 @@ enum Scope
239240
* File based QC and analysis scripts can be added to a protocol and invoked when the validate
240241
* method is called. Set to an empty list if no scripts exist.
241242
*/
242-
// TODO File->FileLike
243-
ValidationException setValidationAndAnalysisScripts(ExpProtocol protocol, @NotNull List<File> scripts) throws ExperimentException;
243+
ValidationException setValidationAndAnalysisScripts(ExpProtocol protocol, @NotNull List<AnalysisScript> scripts) throws ExperimentException;
244244

245245
@NotNull
246-
// TODO File->FileLike
247-
List<File> getValidationAndAnalysisScripts(ExpProtocol protocol, Scope scope);
246+
List<AnalysisScript> getValidationAndAnalysisScripts(ExpProtocol protocol, Scope scope);
248247

249248
void setSaveScriptFiles(ExpProtocol protocol, boolean save) throws ExperimentException;
250249
boolean isSaveScriptFiles(ExpProtocol protocol);

‎api/src/org/labkey/api/assay/AssayRunDatabaseContext.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,8 @@
2929
import org.labkey.api.exp.api.ExpRun;
3030
import org.labkey.api.exp.property.Domain;
3131
import org.labkey.api.exp.property.DomainProperty;
32-
import org.labkey.api.qc.DefaultTransformResult;
33-
import org.labkey.api.qc.TransformResult;
32+
import org.labkey.api.assay.transform.DefaultTransformResult;
33+
import org.labkey.api.assay.transform.TransformResult;
3434
import org.labkey.api.security.User;
3535
import org.labkey.api.util.NetworkDrive;
3636
import org.labkey.api.view.ActionURL;

‎api/src/org/labkey/api/assay/AssayRunUploadContext.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727
import org.labkey.api.exp.api.ExpProtocol;
2828
import org.labkey.api.exp.api.ExpRun;
2929
import org.labkey.api.exp.property.DomainProperty;
30-
import org.labkey.api.qc.TransformResult;
30+
import org.labkey.api.assay.transform.TransformResult;
3131
import org.labkey.api.security.User;
3232
import org.labkey.api.view.ActionURL;
3333
import org.labkey.api.view.HasHttpRequest;

‎api/src/org/labkey/api/assay/AssayRunUploadContextImpl.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,16 +30,15 @@
3030
import org.labkey.api.exp.api.ExperimentJSONConverter;
3131
import org.labkey.api.exp.property.Domain;
3232
import org.labkey.api.exp.property.DomainProperty;
33-
import org.labkey.api.qc.DefaultTransformResult;
34-
import org.labkey.api.qc.TransformResult;
33+
import org.labkey.api.assay.transform.DefaultTransformResult;
34+
import org.labkey.api.assay.transform.TransformResult;
3535
import org.labkey.api.security.User;
3636
import org.labkey.api.util.URIUtil;
3737
import org.labkey.api.view.ActionURL;
3838
import org.labkey.api.view.NotFoundException;
3939
import org.labkey.api.view.ViewContext;
4040
import org.labkey.vfs.FileLike;
4141

42-
import java.io.File;
4342
import java.io.IOException;
4443
import java.util.Collections;
4544
import java.util.HashMap;

‎api/src/org/labkey/api/assay/DefaultAssayRunCreator.java‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,9 @@
2727
import org.labkey.api.assay.pipeline.AssayRunAsyncContext;
2828
import org.labkey.api.assay.pipeline.AssayUploadPipelineJob;
2929
import org.labkey.api.assay.sample.AssaySampleLookupContext;
30+
import org.labkey.api.assay.transform.DataTransformService;
31+
import org.labkey.api.assay.transform.TransformDataHandler;
32+
import org.labkey.api.assay.transform.TransformResult;
3033
import org.labkey.api.data.ColumnInfo;
3134
import org.labkey.api.data.Container;
3235
import org.labkey.api.data.ContainerManager;
@@ -64,9 +67,6 @@
6467
import org.labkey.api.exp.property.ValidatorContext;
6568
import org.labkey.api.pipeline.PipelineService;
6669
import org.labkey.api.pipeline.PipelineValidationException;
67-
import org.labkey.api.qc.DataTransformer;
68-
import org.labkey.api.qc.TransformDataHandler;
69-
import org.labkey.api.qc.TransformResult;
7070
import org.labkey.api.query.BatchValidationException;
7171
import org.labkey.api.query.PropertyValidationError;
7272
import org.labkey.api.query.SimpleValidationError;
@@ -113,8 +113,7 @@ public DefaultAssayRunCreator(ProviderType provider)
113113

114114
public TransformResult transform(AssayRunUploadContext<ProviderType> context, ExpRun run) throws ValidationException
115115
{
116-
DataTransformer<ProviderType> transformer = new DefaultDataTransformer<>();
117-
return transformer.transformAndValidate(context, run);
116+
return DataTransformService.get().transformAndValidate(context, run, DataTransformService.TransformOperation.INSERT);
118117
}
119118

120119
@Override

‎api/src/org/labkey/api/assay/TsvDataHandler.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@
2828
import org.labkey.api.data.SimpleFilter;
2929
import org.labkey.api.data.Sort;
3030
import org.labkey.api.data.TSVGridWriter;
31-
import org.labkey.api.data.TSVWriter;
3231
import org.labkey.api.data.TableInfo;
3332
import org.labkey.api.data.TableSelector;
3433
import org.labkey.api.exp.ExperimentException;
@@ -38,7 +37,7 @@
3837
import org.labkey.api.exp.api.ExpData;
3938
import org.labkey.api.exp.api.ExpProtocol;
4039
import org.labkey.api.exp.api.ExpRun;
41-
import org.labkey.api.qc.TransformDataHandler;
40+
import org.labkey.api.assay.transform.TransformDataHandler;
4241
import org.labkey.api.query.FieldKey;
4342
import org.labkey.api.security.User;
4443
import org.labkey.api.util.FileType;

‎api/src/org/labkey/api/assay/actions/AssayRunUploadForm.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,8 +49,8 @@
4949
import org.labkey.api.exp.property.Domain;
5050
import org.labkey.api.exp.property.DomainProperty;
5151
import org.labkey.api.pipeline.PipelineService;
52-
import org.labkey.api.qc.DefaultTransformResult;
53-
import org.labkey.api.qc.TransformResult;
52+
import org.labkey.api.assay.transform.DefaultTransformResult;
53+
import org.labkey.api.assay.transform.TransformResult;
5454
import org.labkey.api.qc.TsvDataExchangeHandler;
5555
import org.labkey.api.query.DefaultSchema;
5656
import org.labkey.api.query.PdLookupForeignKey;

‎api/src/org/labkey/api/assay/pipeline/AssayRunAsyncContext.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,8 @@
3131
import org.labkey.api.exp.property.Domain;
3232
import org.labkey.api.exp.property.DomainProperty;
3333
import org.labkey.api.exp.property.PropertyService;
34-
import org.labkey.api.qc.DefaultTransformResult;
35-
import org.labkey.api.qc.TransformResult;
34+
import org.labkey.api.assay.transform.DefaultTransformResult;
35+
import org.labkey.api.assay.transform.TransformResult;
3636
import org.labkey.api.security.User;
3737
import org.labkey.api.security.UserManager;
3838
import org.labkey.api.view.ActionURL;

‎api/src/org/labkey/api/assay/transform/AnalysisScript.java‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
package org.labkey.api.assay.transform;
2+
3+
import org.jetbrains.annotations.Nullable;
4+
import org.json.JSONArray;
5+
import org.json.JSONException;
6+
import org.json.JSONObject;
7+
import org.labkey.vfs.FileLike;
8+
import org.labkey.vfs.FileSystemLike;
9+
10+
import java.io.File;
11+
import java.util.ArrayList;
12+
import java.util.Collection;
13+
import java.util.HashSet;
14+
import java.util.List;
15+
import java.util.Map;
16+
import java.util.Set;
17+
18+
public class AnalysisScript
19+
{
20+
FileLike _script;
21+
Set<DataTransformService.TransformOperation> _operations = new HashSet<>();
22+
23+
public AnalysisScript(File script, Set<DataTransformService.TransformOperation> operations)
24+
{
25+
_script = new FileSystemLike.Builder(script).build().getRoot();
26+
_operations = operations;
27+
}
28+
29+
private AnalysisScript(File script, List<String> operations)
30+
{
31+
_script = new FileSystemLike.Builder(script).build().getRoot();
32+
for (String op : operations)
33+
{
34+
if (op != null)
35+
_operations.add(DataTransformService.TransformOperation.valueOf(op));
36+
}
37+
38+
// default to insert only
39+
if (_operations.isEmpty())
40+
_operations.add(DataTransformService.TransformOperation.INSERT);
41+
}
42+
43+
public FileLike getScript()
44+
{
45+
return _script;
46+
}
47+
48+
public String getScriptPath()
49+
{
50+
return _script.toNioPathForRead().toString();
51+
}
52+
53+
public boolean canExecute(DataTransformService.TransformOperation operation)
54+
{
55+
return _operations.contains(operation);
56+
}
57+
58+
private static JSONObject toJson(AnalysisScript script)
59+
{
60+
JSONObject json = new JSONObject();
61+
62+
json.put("operations", script._operations);
63+
json.put("script", script._script.toNioPathForRead());
64+
65+
return json;
66+
}
67+
68+
@Nullable
69+
public static JSONArray toJson(Collection<AnalysisScript> scripts)
70+
{
71+
if (!scripts.isEmpty())
72+
{
73+
JSONArray json = new JSONArray();
74+
for (AnalysisScript script : scripts)
75+
{
76+
json.put(toJson(script));
77+
}
78+
return json;
79+
}
80+
return null;
81+
}
82+
83+
@Nullable
84+
public static List<AnalysisScript> fromJson(String jsonStr)
85+
{
86+
try
87+
{
88+
List<AnalysisScript> scripts = new ArrayList<>();
89+
JSONArray json = new JSONArray(jsonStr);
90+
for (Object o : json.toList())
91+
{
92+
if (o instanceof Map<?,?> props)
93+
{
94+
File script = new File(String.valueOf(props.get("script")));
95+
List<String> ops = (List<String>) props.get("operations");
96+
97+
scripts.add(new AnalysisScript(script, ops));
98+
}
99+
}
100+
return scripts;
101+
}
102+
catch (JSONException e)
103+
{
104+
// ignore
105+
return null;
106+
}
107+
}
108+
}

0 commit comments

Comments
 (0)