Skip to content

Commit f060b04

Browse files
labkey-jeckelslabkey-jeckels
authored
Switch to tracking NextFlow invocation counts per-job (#524)
1 parent 2f8fe93 commit f060b04

File tree

8 files changed

+129
-24
lines changed

8 files changed

+129
-24
lines changed

‎nextflow/module.properties‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,4 +5,4 @@ Description: This module provides the functionality \
55
License: Apache 2.0
66
LicenseURL: http://www.apache.org/licenses/LICENSE-2.0
77
SupportedDatabases: pgsql
8-
ManageVersion: false
8+
ManageVersion: true

‎nextflow/resources/schemas/dbscripts/nextflow.xml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
4+
* Copyright (c) 2025 LabKey Corporation
5+
*
6+
* Licensed under the Apache License, Version 2.0 (the "License");
7+
* you may not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
18+
-->
19+
<tables xsi:schemaLocation="http://labkey.org/data/xml ../../../../../schemas/tableInfo.xsd"
20+
xmlns="http://labkey.org/data/xml" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
21+
<table tableName="Job" tableDbType="TABLE">
22+
<description>Invocation counts to ensure unique NextFlow run names</description>
23+
<columns>
24+
<column columnName="JobId"/>
25+
<column columnName="InvocationCount"/>
26+
</columns>
27+
</table>
28+
</tables>

‎nextflow/resources/schemas/dbscripts/postgresql/nextflow-0.000-25.000.sql‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
/*
2+
* Copyright (c) 2025 LabKey Corporation
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
CREATE SCHEMA nextflow;
17+
18+
CREATE TABLE nextflow.Job
19+
(
20+
JobId INTEGER NOT NULL,
21+
InvocationCount INTEGER NOT NULL,
22+
CONSTRAINT PK_Job PRIMARY KEY (JobId),
23+
CONSTRAINT FK_Job_JobId FOREIGN KEY (JobId) REFERENCES pipeline.StatusFiles (RowId) ON DELETE CASCADE -- Automatically clean up when a job is deleted
24+
);

‎nextflow/src/org/labkey/nextflow/NextFlowController.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import lombok.Getter;
44
import lombok.Setter;
55
import org.apache.commons.lang3.StringUtils;
6+
import org.apache.logging.log4j.Logger;
67
import org.labkey.api.action.ApiResponse;
78
import org.labkey.api.action.ApiSimpleResponse;
89
import org.labkey.api.action.FormViewAction;
@@ -12,7 +13,6 @@
1213
import org.labkey.api.data.PropertyManager;
1314
import org.labkey.api.data.PropertyStore;
1415
import org.labkey.api.pipeline.PipeRoot;
15-
import org.labkey.api.pipeline.PipelineJob;
1616
import org.labkey.api.pipeline.PipelineProvider;
1717
import org.labkey.api.pipeline.PipelineService;
1818
import org.labkey.api.pipeline.PipelineStatusUrls;
@@ -31,6 +31,7 @@
3131
import org.labkey.api.util.Path;
3232
import org.labkey.api.util.URLHelper;
3333
import org.labkey.api.util.element.Select;
34+
import org.labkey.api.util.logging.LogHelper;
3435
import org.labkey.api.view.HtmlView;
3536
import org.labkey.api.view.JspView;
3637
import org.labkey.api.view.NavTree;
@@ -64,6 +65,8 @@ public class NextFlowController extends SpringActionController
6465
private static final DefaultActionResolver _actionResolver = new DefaultActionResolver(NextFlowController.class);
6566
public static final String NAME = "nextflow";
6667

68+
protected static final Logger LOG = LogHelper.getLogger(NextFlowPipelineJob.class, "LabKey UI and API for NextFlow usage");
69+
6770
public NextFlowController()
6871
{
6972
setActionResolver(_actionResolver);
@@ -326,8 +329,9 @@ public boolean handlePost(AnalyzeForm form, BindException errors) throws Excepti
326329
{
327330
ViewBackgroundInfo info = getViewBackgroundInfo();
328331
PipeRoot root = PipelineService.get().findPipelineRoot(info.getContainer());
329-
PipelineJob job = NextFlowPipelineJob.create(info, root, configFile.toPath(), inputFiles.stream().map(File::toPath).toList());
332+
NextFlowPipelineJob job = NextFlowPipelineJob.create(info, root, configFile.toPath(), inputFiles.stream().map(File::toPath).toList());
330333
PipelineService.get().queueJob(job);
334+
LOG.info("NextFlow job queued: {}", job.getJsonJobInfo(false));
331335
}
332336
}
333337

‎nextflow/src/org/labkey/nextflow/NextFlowManager.java‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,16 @@
33
import org.apache.commons.lang3.StringUtils;
44
import org.labkey.api.data.Container;
55
import org.labkey.api.data.CoreSchema;
6+
import org.labkey.api.data.DbSchema;
7+
import org.labkey.api.data.DbSchemaType;
68
import org.labkey.api.data.DbScope;
79
import org.labkey.api.data.PropertyManager;
10+
import org.labkey.api.data.SQLFragment;
11+
import org.labkey.api.data.SqlExecutor;
12+
import org.labkey.api.data.SqlSelector;
13+
import org.labkey.api.pipeline.PipelineService;
14+
import org.labkey.api.pipeline.PipelineStatusFile;
15+
import org.labkey.nextflow.pipeline.NextFlowPipelineJob;
816
import org.springframework.validation.BindException;
917

1018
import java.nio.file.Files;
@@ -27,6 +35,8 @@ public class NextFlowManager
2735
private static final String NEXTFLOW_S3_BUCKET_PATH = "s3BucketPath";
2836
private static final String NEXTFLOW_API_KEY = "apiKey";
2937

38+
public static final String SCHEMA_NAME = "nextflow";
39+
3040
private static final String IS_NEXTFLOW_ENABLED = "enabled";
3141

3242
private static final NextFlowManager _instance = new NextFlowManager();
@@ -158,4 +168,42 @@ public void saveEnabledState(Container container, Boolean enabled)
158168
map.save();
159169
}
160170
}
171+
172+
private DbSchema getDbSchema()
173+
{
174+
return DbSchema.get(SCHEMA_NAME, DbSchemaType.Module);
175+
}
176+
177+
private Integer getJobId(NextFlowPipelineJob job)
178+
{
179+
PipelineStatusFile file = PipelineService.get().getStatusFile(job.getJobGUID());
180+
return file == null ? null : file.getRowId();
181+
}
182+
183+
public int getInvocationCount(NextFlowPipelineJob job)
184+
{
185+
return getInvocationCount(getJobId(job));
186+
}
187+
188+
private int getInvocationCount(int jobId)
189+
{
190+
Integer result = new SqlSelector(getDbSchema(), new SQLFragment("SELECT InvocationCount FROM nextflow.Job WHERE JobId = ?", jobId)).getObject(Integer.class);
191+
return result != null ? result.intValue() : 0;
192+
}
193+
194+
public int incrementInvocationCount(NextFlowPipelineJob job)
195+
{
196+
int jobId = getJobId(job);
197+
int current = getInvocationCount(jobId);
198+
current++;
199+
if (current == 1)
200+
{
201+
new SqlExecutor(getDbSchema()).execute(new SQLFragment("INSERT INTO nextflow.Job (JobId, InvocationCount) VALUES (?, ?)", jobId, current));
202+
}
203+
else
204+
{
205+
new SqlExecutor(getDbSchema()).execute(new SQLFragment("UPDATE nextflow.Job SET InvocationCount = ? WHERE JobId = ?", current, jobId));
206+
}
207+
return current;
208+
}
161209
}

‎nextflow/src/org/labkey/nextflow/NextFlowModule.java‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package org.labkey.nextflow;
22

33
import org.jetbrains.annotations.NotNull;
4+
import org.jetbrains.annotations.Nullable;
45
import org.labkey.api.data.ContainerManager;
56
import org.labkey.api.module.ModuleContext;
67
import org.labkey.api.module.SpringModule;
@@ -40,13 +41,18 @@ protected void init()
4041
@Override
4142
public boolean hasScripts()
4243
{
43-
return false;
44+
return true;
4445
}
4546

4647
@Override
47-
public @NotNull Collection<String> getSchemaNames()
48+
public @Nullable Double getSchemaVersion()
4849
{
49-
return List.of();
50+
return 25.000;
5051
}
5152

53+
@Override
54+
public @NotNull Collection<String> getSchemaNames()
55+
{
56+
return List.of(NextFlowManager.SCHEMA_NAME);
57+
}
5258
}

‎nextflow/src/org/labkey/nextflow/pipeline/NextFlowPipelineJob.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import org.labkey.api.util.StringUtilsLabKey;
2121
import org.labkey.api.util.logging.LogHelper;
2222
import org.labkey.api.view.ViewBackgroundInfo;
23+
import org.labkey.nextflow.NextFlowManager;
2324

2425
import java.io.BufferedWriter;
2526
import java.io.File;
@@ -59,25 +60,24 @@ public NextFlowPipelineJob(ViewBackgroundInfo info, @NotNull PipeRoot root, Path
5960
super(new NextFlowProtocol(), NextFlowPipelineProvider.NAME, info, root, config.getFileName().toString(), config, inputFiles, false, false);
6061
this.config = config;
6162
setLogFile(log);
62-
LOG.info("NextFlow job queued: {}", getJsonJobInfo(null));
6363
}
6464

65-
protected JSONObject getJsonJobInfo(Long invocationCount)
65+
public JSONObject getJsonJobInfo(boolean includeInvocationCount)
6666
{
6767
JSONObject result = new JSONObject();
6868
result.put("user", getUser().getEmail());
6969
result.put("container", getContainer().getPath());
7070
result.put("filePath", getLogFilePath().getParent().toString());
71-
result.put("runName", getNextFlowRunName(invocationCount));
71+
result.put("runName", getNextFlowRunName(includeInvocationCount));
7272
result.put("configFile", getConfig().getFileName().toString());
7373
return result;
7474
}
7575

76-
protected String getNextFlowRunName(Long invocationCount)
76+
protected String getNextFlowRunName(boolean includeInvocationCount)
7777
{
7878
PipelineStatusFile file = PipelineService.get().getStatusFile(getJobGUID());
7979
String result = file == null ? "Unknown" : ("LabKeyJob" + file.getRowId());
80-
result += invocationCount == null ? "" : ("_" + invocationCount);
80+
result += includeInvocationCount ? ("_" + NextFlowManager.get().getInvocationCount(this)) : "";
8181
return result;
8282
}
8383

‎nextflow/src/org/labkey/nextflow/pipeline/NextFlowRunTask.java‎

Lines changed: 8 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,6 @@
22

33
import org.apache.logging.log4j.Logger;
44
import org.jetbrains.annotations.NotNull;
5-
import org.labkey.api.data.ContainerManager;
6-
import org.labkey.api.data.DbSequence;
7-
import org.labkey.api.data.DbSequenceManager;
85
import org.labkey.api.exp.XarFormatException;
96
import org.labkey.api.pipeline.AbstractTaskFactory;
107
import org.labkey.api.pipeline.AbstractTaskFactorySettings;
@@ -40,8 +37,6 @@ public class NextFlowRunTask extends WorkDirectoryTask<NextFlowRunTask.Factory>
4037

4138
public static final String ACTION_NAME = "NextFlow";
4239

43-
private static final DbSequence INVOCATION_SEQUENCE = DbSequenceManager.get(ContainerManager.getRoot(), NextFlowRunTask.class.getName());
44-
4540
public NextFlowRunTask(Factory factory, PipelineJob job)
4641
{
4742
super(factory, job);
@@ -54,9 +49,9 @@ public NextFlowRunTask(Factory factory, PipelineJob job)
5449

5550
// NextFlow requires a unique job name for every execution. Increment a counter to append as a suffix to
5651
// ensure uniqueness
57-
long invocationCount = INVOCATION_SEQUENCE.next();
58-
INVOCATION_SEQUENCE.sync();
59-
NextFlowPipelineJob.LOG.info("Starting to execute NextFlow: {}", getJob().getJsonJobInfo(invocationCount));
52+
NextFlowManager.get().incrementInvocationCount(getJob());
53+
54+
NextFlowPipelineJob.LOG.info("Starting to execute NextFlow: {}", getJob().getJsonJobInfo(true));
6055

6156
SecurityManager.TransformSession session = null;
6257
boolean success = false;
@@ -83,10 +78,10 @@ public NextFlowRunTask(Factory factory, PipelineJob job)
8378
File dir = getJob().getLogFile().getParentFile();
8479
getJob().runSubProcess(secretsPB, dir);
8580

86-
ProcessBuilder executionPB = new ProcessBuilder(getArgs(invocationCount));
81+
ProcessBuilder executionPB = new ProcessBuilder(getArgs());
8782
getJob().runSubProcess(executionPB, dir);
8883
log.info("Job Finished");
89-
NextFlowPipelineJob.LOG.info("Finished executing NextFlow: {}", getJob().getJsonJobInfo(invocationCount));
84+
NextFlowPipelineJob.LOG.info("Finished executing NextFlow: {}", getJob().getJsonJobInfo(true));
9085

9186
RecordedAction action = new RecordedAction(ACTION_NAME);
9287
for (Path inputFile : getJob().getInputFilePaths())
@@ -110,7 +105,7 @@ public NextFlowRunTask(Factory factory, PipelineJob job)
110105
}
111106
if (!success)
112107
{
113-
NextFlowPipelineJob.LOG.info("Failed executing NextFlow: {}", getJob().getJsonJobInfo(invocationCount));
108+
NextFlowPipelineJob.LOG.info("Failed executing NextFlow: {}", getJob().getJsonJobInfo(true));
114109
}
115110
}
116111
}
@@ -176,7 +171,7 @@ private boolean hasAwsSection(Path configFile) throws PipelineJobException
176171
}
177172

178173

179-
private @NotNull List<String> getArgs(long invocationCount) throws PipelineJobException
174+
private @NotNull List<String> getArgs() throws PipelineJobException
180175
{
181176
NextFlowConfiguration config = NextFlowManager.get().getConfiguration();
182177
Path configFile = getJob().getConfig();
@@ -201,7 +196,7 @@ private boolean hasAwsSection(Path configFile) throws PipelineJobException
201196
args.add("-c");
202197
args.add(configFile.toAbsolutePath().toString());
203198
args.add("-name");
204-
args.add(getJob().getNextFlowRunName(invocationCount));
199+
args.add(getJob().getNextFlowRunName(true));
205200
return args;
206201
}
207202

0 commit comments

Comments
 (0)