From db6458f630ceaef24ad48d11a49ee59e59545b1c Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 5 Aug 2025 10:48:42 -0700 Subject: [PATCH 01/15] Use lsid instead of objectId for ETL --- mcc/resources/etls/wnprc-datasets.xml | 43 ++------------------------- 1 file changed, 2 insertions(+), 41 deletions(-) diff --git a/mcc/resources/etls/wnprc-datasets.xml b/mcc/resources/etls/wnprc-datasets.xml index 97ecaf334..2fe8aa7b1 100644 --- a/mcc/resources/etls/wnprc-datasets.xml +++ b/mcc/resources/etls/wnprc-datasets.xml @@ -37,28 +37,6 @@ - - - - - - - - - - - - - - - - - - - - - - Copy to target @@ -84,7 +62,7 @@ date enddate title - objectid + lsid @@ -92,7 +70,7 @@ - + @@ -123,23 +101,6 @@ - - - - - - - - - - - - - - - - - From 9f86bac0fed6f71144437f470a13e1de9e349968 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 6 Aug 2025 09:11:36 -0700 Subject: [PATCH 02/15] Use Audit table for PMR ETL deletes --- PMR/resources/etls/pmr-datasets.xml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/PMR/resources/etls/pmr-datasets.xml b/PMR/resources/etls/pmr-datasets.xml index a0fc89c53..9bae7210c 100644 --- a/PMR/resources/etls/pmr-datasets.xml +++ b/PMR/resources/etls/pmr-datasets.xml @@ -148,7 +148,7 @@ - + @@ -180,7 +180,7 @@ - + @@ -188,9 +188,9 @@ - - - + + + From d79c6a7830bf4c74e7ce88e09bd441ad6aa5fcf3 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 6 Aug 2025 12:23:01 -0700 Subject: [PATCH 03/15] Add celltype column to tcr stats table --- .../schemas/dbscripts/postgresql/tcrdb-15.58-15.59.sql | 1 + .../schemas/dbscripts/sqlserver/tcrdb-15.58-15.59.sql | 1 + tcrdb/resources/schemas/tcrdb.xml | 3 +++ tcrdb/src/org/labkey/tcrdb/TCRdbModule.java | 2 +- 4 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.58-15.59.sql create mode 100644 tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.58-15.59.sql diff --git a/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.58-15.59.sql b/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.58-15.59.sql new file mode 100644 index 000000000..141403f5f --- /dev/null +++ b/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.58-15.59.sql @@ -0,0 +1 @@ +ALTER TABLE tcrdb.repertoire_stats ADD COLUMN celltype varchar(1000); \ No newline at end of file diff --git a/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.58-15.59.sql b/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.58-15.59.sql new file mode 100644 index 000000000..fe33f991d --- /dev/null +++ b/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.58-15.59.sql @@ -0,0 +1 @@ +ALTER TABLE tcrdb.repertoire_stats ADD celltype varchar(1000); \ No newline at end of file diff --git a/tcrdb/resources/schemas/tcrdb.xml b/tcrdb/resources/schemas/tcrdb.xml index b63843443..f152dc569 100644 --- a/tcrdb/resources/schemas/tcrdb.xml +++ b/tcrdb/resources/schemas/tcrdb.xml @@ -368,6 +368,9 @@ rowid + + Cell Type + Metric Name diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java b/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java index 1f4726263..26089978d 100644 --- a/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java +++ b/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java @@ -46,7 +46,7 @@ public String getName() @Override public Double getSchemaVersion() { - return 15.58; + return 15.59; } @Override From 22a436959e518602adc2911aeb21bf5483425034 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 7 Aug 2025 10:35:17 -0700 Subject: [PATCH 04/15] Add tcrdb column --- .../schemas/dbscripts/postgresql/tcrdb-15.59-15.60.sql | 1 + .../schemas/dbscripts/sqlserver/tcrdb-15.59-15.60.sql | 1 + tcrdb/resources/schemas/tcrdb.xml | 3 +++ tcrdb/src/org/labkey/tcrdb/TCRdbModule.java | 2 +- 4 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.59-15.60.sql create mode 100644 tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.59-15.60.sql diff --git a/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.59-15.60.sql b/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.59-15.60.sql new file mode 100644 index 000000000..2a024139c --- /dev/null +++ b/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.59-15.60.sql @@ -0,0 +1 @@ +ALTER TABLE tcrdb.repertoire_stats ADD COLUMN samplesize int; \ No newline at end of file diff --git a/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.59-15.60.sql b/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.59-15.60.sql new file mode 100644 index 000000000..1300ca0e0 --- /dev/null +++ b/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.59-15.60.sql @@ -0,0 +1 @@ +ALTER TABLE tcrdb.repertoire_stats ADD samplesize int; \ No newline at end of file diff --git a/tcrdb/resources/schemas/tcrdb.xml b/tcrdb/resources/schemas/tcrdb.xml index f152dc569..a8a486ce0 100644 --- a/tcrdb/resources/schemas/tcrdb.xml +++ b/tcrdb/resources/schemas/tcrdb.xml @@ -382,6 +382,9 @@ + + + Sample Size true diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java b/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java index 26089978d..39fca6481 100644 --- a/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java +++ b/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java @@ -46,7 +46,7 @@ public String getName() @Override public Double getSchemaVersion() { - return 15.59; + return 15.60; } @Override From c04c297f9e7ea45a9534c531802164433d720983 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 7 Aug 2025 17:42:45 -0700 Subject: [PATCH 05/15] Add study validation notification --- .../study/infectionAnchorDateDiscordance.sql | 5 +- .../labkey/sivstudies/SivStudiesModule.java | 3 + .../SivStudiesDataValidationNotification.java | 125 ++++++++++++++++++ .../query/SivStudiesCustomizer.java | 7 + 4 files changed, 139 insertions(+), 1 deletion(-) create mode 100644 SivStudies/src/org/labkey/sivstudies/notification/SivStudiesDataValidationNotification.java diff --git a/SivStudies/resources/queries/study/infectionAnchorDateDiscordance.sql b/SivStudies/resources/queries/study/infectionAnchorDateDiscordance.sql index 13eabe059..c19cd0943 100644 --- a/SivStudies/resources/queries/study/infectionAnchorDateDiscordance.sql +++ b/SivStudies/resources/queries/study/infectionAnchorDateDiscordance.sql @@ -6,8 +6,11 @@ SELECT FROM study.treatments t FULL JOIN studies.subjectAnchorDates t2 ON ( + t.category = 'SIV Infection' AND t2.subjectId = t.Id AND t2.eventLabel = 'SIV Infection' ) -WHERE t.category = 'SIV Infection' AND t2.date != t.date \ No newline at end of file +WHERE t2.date != t.date OR + t2.date IS NULL OR + t.date IS NULL \ No newline at end of file diff --git a/SivStudies/src/org/labkey/sivstudies/SivStudiesModule.java b/SivStudies/src/org/labkey/sivstudies/SivStudiesModule.java index 4bedeb33e..72d8b6e83 100644 --- a/SivStudies/src/org/labkey/sivstudies/SivStudiesModule.java +++ b/SivStudies/src/org/labkey/sivstudies/SivStudiesModule.java @@ -20,8 +20,10 @@ import org.jetbrains.annotations.Nullable; import org.labkey.api.data.Container; import org.labkey.api.ldk.ExtendedSimpleModule; +import org.labkey.api.ldk.notification.NotificationService; import org.labkey.api.module.ModuleContext; import org.labkey.api.studies.StudiesService; +import org.labkey.sivstudies.notification.SivStudiesDataValidationNotification; import org.labkey.sivstudies.study.ArtInitiationEventProvider; import org.labkey.sivstudies.study.SivInfectionEventProvider; @@ -56,6 +58,7 @@ public void doStartupAfterSpringConfig(ModuleContext moduleContext) { StudiesService.get().registerEventProvider(new SivInfectionEventProvider()); StudiesService.get().registerEventProvider(new ArtInitiationEventProvider()); + NotificationService.get().registerNotification(new SivStudiesDataValidationNotification()); } @Override diff --git a/SivStudies/src/org/labkey/sivstudies/notification/SivStudiesDataValidationNotification.java b/SivStudies/src/org/labkey/sivstudies/notification/SivStudiesDataValidationNotification.java new file mode 100644 index 000000000..314c662a0 --- /dev/null +++ b/SivStudies/src/org/labkey/sivstudies/notification/SivStudiesDataValidationNotification.java @@ -0,0 +1,125 @@ +package org.labkey.sivstudies.notification; + +import org.jetbrains.annotations.Nullable; +import org.labkey.api.data.Container; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.ldk.notification.AbstractNotification; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.UserSchema; +import org.labkey.api.security.User; +import org.labkey.sivstudies.SivStudiesModule; + +import java.util.Date; + +public class SivStudiesDataValidationNotification extends AbstractNotification +{ + public SivStudiesDataValidationNotification() + { + super(ModuleLoader.getInstance().getModule(SivStudiesModule.class)); + } + + @Override + public String getName() + { + return "SIV Studies Validation"; + } + + @Override + public String getCategory() + { + return "Studies"; + } + + @Override + public String getCronString() + { + return "0 0 6 * * ?"; + } + + @Override + public String getScheduleDescription() + { + return "every day at 6:00AM"; + } + + @Override + public String getDescription() + { + return "SIV Study Data Validation Alerts"; + } + + @Override + public String getEmailSubject(Container c) + { + return "SIV Study Data Validation: " + c.getName(); + } + + @Override + public @Nullable String getMessageBodyHTML(Container c, User u) + { + StringBuilder msg = new StringBuilder(); + Date now = new Date(); + + duplicateInfectionCheck(c, u, msg); + + if (!msg.isEmpty()) + { + msg.insert(0, "This email contains a series of automatic alerts about the SIV study data. It was run on: " + getDateFormat(c).format(now) + " at " + AbstractNotification._timeFormat.format(now) + ".

"); + } + + return msg.toString(); + } + + private TableInfo getTableInfo(User u, Container c, String schemaName, String queryName) + { + UserSchema us = QueryService.get().getUserSchema(u, c, schemaName); + if (us == null) + { + throw new IllegalStateException("Missing user schema: " + schemaName); + } + + TableInfo ti = us.getTable(queryName); + if (ti == null) + { + throw new IllegalStateException("Missing table: " + schemaName + "." + queryName); + } + + return ti; + } + + private void duplicateInfectionCheck(Container c, User u, StringBuilder msg) + { + String schemaName = "study"; + String queryName = "duplicateInfectionDates"; + + TableInfo ti = getTableInfo(u, c, schemaName, queryName); + + TableSelector ts = new TableSelector(ti); + long count = ts.getRowCount(); + if (count > 0) + { + msg.append("WARNING: There are ").append(count).append(" duplicate infection date records
\n"); + msg.append("

Click here to view them
\n\n"); + msg.append("

\n\n"); + } + } + + private void infectionAnchorDateDiscordance(Container c, User u, StringBuilder msg) + { + String schemaName = "study"; + String queryName = "infectionAnchorDateDiscordance"; + + TableInfo ti = getTableInfo(u, c, schemaName, queryName); + + TableSelector ts = new TableSelector(ti); + long count = ts.getRowCount(); + if (count > 0) + { + msg.append("WARNING: There are ").append(count).append(" records with discordant treatment and anchor date SIV infection records
\n"); + msg.append("

Click here to view them
\n\n"); + msg.append("

\n\n"); + } + } +} diff --git a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java index ebaeb3322..9d82243ac 100644 --- a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java +++ b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java @@ -293,6 +293,13 @@ private void appendPvlColumns(DatasetTable ds, String subjectColName, String dat newColOOR.setHidden(true); ti.addColumn(newColOOR); + + String nameHasPvl = name + "HasPvl"; + SQLFragment sqlHasPvl = new SQLFragment("(SELECT CASE WHEN count(*) > 0 THEN " + ti.getSqlDialect().getBooleanTRUE() + " ELSE " + ti.getSqlDialect().getBooleanFALSE() + " END as expr FROM studydataset." + tableName + " t WHERE t.participantid = " + ExprColumn.STR_TABLE_ALIAS + ".participantid AND t.sampletype = 'Plasma' AND t.target = 'SIV')"); + ExprColumn newColHasPvl = new ExprColumn(ti, nameHasPvl, sqlHasPvl, JdbcType.DOUBLE, subjectCol, dateCol); + newColHasPvl.setDescription("Displays whether this animal has any positive SIV PVLs"); + newColHasPvl.setLabel("Has Positive SIV PVL?"); + ti.addColumn(newColHasPvl); } } From 22c203907314c5fdc0af52ebc28df57076501f42 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 8 Aug 2025 06:18:34 -0700 Subject: [PATCH 06/15] Correct column name in ETL --- PMR/resources/etls/pmr-datasets.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/PMR/resources/etls/pmr-datasets.xml b/PMR/resources/etls/pmr-datasets.xml index 9bae7210c..2b13a3f27 100644 --- a/PMR/resources/etls/pmr-datasets.xml +++ b/PMR/resources/etls/pmr-datasets.xml @@ -188,8 +188,8 @@ - - + + From 0e99631526ed99eec36ba5855436e31b0116f667 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 8 Aug 2025 07:05:44 -0700 Subject: [PATCH 07/15] Add ETL task to verify source/dest row counts --- PMR/build.gradle | 1 + PMR/resources/etls/pmr-datasets.xml | 28 ++ PMR/resources/etls/pmr-demographics.xml | 18 +- SivStudies/resources/views/studiesAdmin.html | 3 + .../labkey/primeseq/etl/VerifyRowCount.java | 264 ++++++++++++++++++ 5 files changed, 310 insertions(+), 4 deletions(-) create mode 100644 primeseq/src/org/labkey/primeseq/etl/VerifyRowCount.java diff --git a/PMR/build.gradle b/PMR/build.gradle index dccb30067..f33b6455c 100644 --- a/PMR/build.gradle +++ b/PMR/build.gradle @@ -9,6 +9,7 @@ dependencies { BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:premiumModules:dataintegration", depProjectConfig: "published", depExtension: "module") BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:DiscvrLabKeyModules:discvrcore", depProjectConfig: "published", depExtension: "module") BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:onprcEHRModules:GeneticsCore", depProjectConfig: "published", depExtension: "module") + BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:BimberLabKeyModules:primeseq", depProjectConfig: "published", depExtension: "module") BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:DiscvrLabKeyModules:Studies", depProjectConfig: "apiJarFile") BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:premiumModules:dataintegration", depProjectConfig: "apiJarFile") diff --git a/PMR/resources/etls/pmr-datasets.xml b/PMR/resources/etls/pmr-datasets.xml index 2b13a3f27..ebb5aa1de 100644 --- a/PMR/resources/etls/pmr-datasets.xml +++ b/PMR/resources/etls/pmr-datasets.xml @@ -154,6 +154,20 @@ + + + + + + + + + + + + + + @@ -186,6 +200,20 @@ + + + + + + + + + + + + + + diff --git a/PMR/resources/etls/pmr-demographics.xml b/PMR/resources/etls/pmr-demographics.xml index 03b2bf94e..883f477ce 100644 --- a/PMR/resources/etls/pmr-demographics.xml +++ b/PMR/resources/etls/pmr-demographics.xml @@ -30,12 +30,22 @@ + + + + + + + + + + + + + + - - - - diff --git a/SivStudies/resources/views/studiesAdmin.html b/SivStudies/resources/views/studiesAdmin.html index b17398a9c..fb5e91025 100644 --- a/SivStudies/resources/views/studiesAdmin.html +++ b/SivStudies/resources/views/studiesAdmin.html @@ -24,6 +24,9 @@ items: [{ name: 'Import/Re-import Study and Reference Data', url: LABKEY.ActionURL.buildURL('sivstudies', 'importStudy.view') + },{ + name: 'Notification Admin', + url: LABKEY.ActionURL.buildURL('ldk', 'notificationAdmin.view') }] }] }] diff --git a/primeseq/src/org/labkey/primeseq/etl/VerifyRowCount.java b/primeseq/src/org/labkey/primeseq/etl/VerifyRowCount.java new file mode 100644 index 000000000..331c37dea --- /dev/null +++ b/primeseq/src/org/labkey/primeseq/etl/VerifyRowCount.java @@ -0,0 +1,264 @@ +package org.labkey.primeseq.etl; + +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.data.CompareType; +import org.labkey.api.data.Container; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.di.DataIntegrationService; +import org.labkey.api.di.TaskRefTask; +import org.labkey.api.pipeline.PipelineJob; +import org.labkey.api.pipeline.PipelineJobException; +import org.labkey.api.pipeline.RecordedActionSet; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.SimpleValidationError; +import org.labkey.api.query.UserSchema; +import org.labkey.api.query.ValidationError; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.view.ActionURL; +import org.labkey.api.writer.ContainerUser; +import org.labkey.remoteapi.CommandException; +import org.labkey.remoteapi.query.Filter; +import org.labkey.remoteapi.query.SelectRowsCommand; +import org.labkey.remoteapi.query.SelectRowsResponse; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class VerifyRowCount implements TaskRefTask +{ + protected final Map _settings = new CaseInsensitiveHashMap<>(); + protected ContainerUser _containerUser; + + private enum Settings + { + sourceRemoteSource(false), + sourceContainerPath(false), + sourceSchema(true), + sourceQuery(true), + sourceColumn(true), + sourceAdditionalFilters(false), + + destSchema(true), + destQuery(true), + destColumn(true), + destAdditionalFilters(false); + + private final boolean _isRequired; + + Settings(boolean isRequired) + { + _isRequired = isRequired; + } + + public boolean isRequired() + { + return _isRequired; + } + } + + @Override + public RecordedActionSet run(@NotNull PipelineJob job) throws PipelineJobException + { + verifyRows(job); + + return new RecordedActionSet(); + } + + @Override + public void setContainerUser(ContainerUser containerUser) + { + _containerUser = containerUser; + } + + @Override + public List preFlightCheck(Container c) + { + List errors = new ArrayList<>(); + for (String setting : getRequiredSettings()) + { + if (_settings.get(setting) == null || StringUtils.isEmpty(_settings.get(setting))) + { + errors.add(new SimpleValidationError("Missing required setting: " + setting)); + } + } + + return errors; + } + + @Override + public List getRequiredSettings() + { + return Arrays.stream(Settings.values()).filter(Settings::isRequired).map(Settings::name).toList(); + } + + @Override + public void setSettings(Map settings) + { + _settings.putAll(settings); + } + + private DataIntegrationService.RemoteConnection getRemoteDataSource(String name, Container c, Logger log) throws IllegalStateException + { + DataIntegrationService.RemoteConnection rc = DataIntegrationService.get().getRemoteConnection(name, c, log); + if (rc == null) + { + throw new IllegalStateException("Unable to find remote connection: " + name); + } + + return rc; + } + + private long getDestinationTableCount() + { + UserSchema us = QueryService.get().getUserSchema(_containerUser.getUser(), _containerUser.getContainer(), _settings.get(Settings.destSchema.name())); + if (us == null) + { + throw new IllegalStateException("Unknown schema: " + _settings.get(Settings.destSchema.name())); + } + + TableInfo ti = us.getTable(_settings.get(Settings.destQuery.name())); + if (ti == null) + { + throw new IllegalStateException("Unknown table: " + _settings.get(Settings.destQuery.name())); + } + + final SimpleFilter filter = new SimpleFilter(); + if (_settings.get(Settings.destAdditionalFilters.name()) != null) + { + if (_settings.get(Settings.destAdditionalFilters.name()) != null) + { + List additionalFilters = parseAdditionalFilters(_settings.get(Settings.destAdditionalFilters.name())); + additionalFilters.forEach(filter::addCondition); + } + } + + TableSelector ts = new TableSelector(ti, PageFlowUtil.set(_settings.get(Settings.destColumn.name())), filter, null); + + return ts.getRowCount(); + } + + private long getSourceTableCount(PipelineJob job) throws PipelineJobException + { + if (_settings.get(Settings.sourceRemoteSource.name()) != null) + { + DataIntegrationService.RemoteConnection rc = getRemoteDataSource(_settings.get(Settings.sourceRemoteSource.name()), _containerUser.getContainer(), job.getLogger()); + SelectRowsCommand sr = new SelectRowsCommand(_settings.get(Settings.sourceSchema.name()), _settings.get(Settings.sourceQuery.name())); + sr.setColumns(Collections.singletonList(_settings.get(Settings.sourceColumn.name()))); + + if (_settings.get(Settings.sourceAdditionalFilters.name()) != null) + { + List additionalFilters = parseAdditionalFilters(_settings.get(Settings.sourceAdditionalFilters.name())); + for (CompareType.AbstractCompareClause f : additionalFilters) + { + Object value; + if (f.getParamVals() == null) + { + value = null; + } + else if (f.getParamVals().length == 1) + { + value = f.getParamVals()[0]; + } + else + { + value = StringUtils.join(f.getParamVals(), ";"); + } + + Filter.Operator o = Filter.Operator.getOperatorFromUrlKey(f.getCompareType().getPreferredUrlKey()); + if (o == null) + { + throw new IllegalStateException("Unknown operator: " + f.getCompareType().getPreferredUrlKey() + ", raw filter: " + f.getCompareType().name()); + } + + sr.addFilter(new Filter(f.getFieldKey().toString(), value, o)); + } + } + + try + { + SelectRowsResponse srr = sr.execute(rc.connection, rc.remoteContainer); + + return srr.getRowCount().longValue(); + } + catch (IOException | CommandException e) + { + throw new PipelineJobException(e); + } + } + else + { + UserSchema us = QueryService.get().getUserSchema(_containerUser.getUser(), _containerUser.getContainer(), _settings.get(Settings.sourceSchema.name())); + if (us == null) + { + throw new IllegalStateException("Unknown schema: " + _settings.get(Settings.sourceSchema.name())); + } + + TableInfo ti = us.getTable(_settings.get(Settings.sourceQuery.name())); + if (ti == null) + { + throw new IllegalStateException("Unknown table: " + _settings.get(Settings.sourceQuery.name())); + } + + final SimpleFilter filter = new SimpleFilter(); + if (_settings.get(Settings.sourceAdditionalFilters.name()) != null) + { + List additionalFilters = parseAdditionalFilters(_settings.get(Settings.sourceAdditionalFilters.name())); + additionalFilters.forEach(filter::addCondition); + } + + TableSelector ts = new TableSelector(ti, PageFlowUtil.set(_settings.get(Settings.sourceColumn.name())), filter, null); + + return ts.getRowCount(); + } + } + + private List parseAdditionalFilters(String rawVal) + { + rawVal = StringUtils.trimToNull(rawVal); + if (rawVal == null) + { + return Collections.emptyList(); + } + + SimpleFilter filter = new SimpleFilter(); + String[] filters = rawVal.split(";"); + for (String queryParam : filters) + { + filter.addUrlFilters(new ActionURL().setRawQuery(queryParam), null); + } + + return filter.getClauses().stream().map(fc -> { + if (fc instanceof CompareType.AbstractCompareClause cc) + { + return cc; + } + + throw new IllegalStateException("Expected all filters to be instance CompareType.AbstractCompareClause, found: " + fc.getClass()); + }).toList(); + } + + private void verifyRows(PipelineJob job) throws PipelineJobException + { + job.getLogger().info("Verifying row count for: " + _settings.get(Settings.destSchema.name()) + "." + _settings.get(Settings.destQuery.name())); + + long source = getSourceTableCount(job); + job.getLogger().info("Source (" + _settings.get(Settings.sourceSchema.name()) + "." + _settings.get(Settings.sourceQuery.name()) + ") row count: " + source); + + long dest = getDestinationTableCount(); + job.getLogger().info("Destination (" + _settings.get(Settings.destSchema.name()) + "." + _settings.get(Settings.destQuery.name()) + ") row count: " + source); + + if (source != dest) + { + job.getLogger().error("Row counts do not match!"); + } + } +} From a93767f9c8332deea5b7be0bd3a9e077b19be6b8 Mon Sep 17 00:00:00 2001 From: bbimber Date: Sun, 10 Aug 2025 05:49:09 -0700 Subject: [PATCH 08/15] Use maxRows in SelectRowsCommand --- primeseq/src/org/labkey/primeseq/etl/VerifyRowCount.java | 1 + 1 file changed, 1 insertion(+) diff --git a/primeseq/src/org/labkey/primeseq/etl/VerifyRowCount.java b/primeseq/src/org/labkey/primeseq/etl/VerifyRowCount.java index 331c37dea..f4660bb0e 100644 --- a/primeseq/src/org/labkey/primeseq/etl/VerifyRowCount.java +++ b/primeseq/src/org/labkey/primeseq/etl/VerifyRowCount.java @@ -153,6 +153,7 @@ private long getSourceTableCount(PipelineJob job) throws PipelineJobException DataIntegrationService.RemoteConnection rc = getRemoteDataSource(_settings.get(Settings.sourceRemoteSource.name()), _containerUser.getContainer(), job.getLogger()); SelectRowsCommand sr = new SelectRowsCommand(_settings.get(Settings.sourceSchema.name()), _settings.get(Settings.sourceQuery.name())); sr.setColumns(Collections.singletonList(_settings.get(Settings.sourceColumn.name()))); + sr.setMaxRows(1); // Ensures we get rowCount if (_settings.get(Settings.sourceAdditionalFilters.name()) != null) { From 5c193294b67582691a94aa9072783bc9a3518065 Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 11 Aug 2025 10:45:21 -0700 Subject: [PATCH 09/15] Update ETL batchSize --- PMR/resources/etls/pmr-datasets.xml | 4 ++-- primeseq/src/org/labkey/primeseq/etl/VerifyRowCount.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/PMR/resources/etls/pmr-datasets.xml b/PMR/resources/etls/pmr-datasets.xml index ebb5aa1de..6b915703c 100644 --- a/PMR/resources/etls/pmr-datasets.xml +++ b/PMR/resources/etls/pmr-datasets.xml @@ -148,7 +148,7 @@ - + @@ -194,7 +194,7 @@ - + diff --git a/primeseq/src/org/labkey/primeseq/etl/VerifyRowCount.java b/primeseq/src/org/labkey/primeseq/etl/VerifyRowCount.java index f4660bb0e..95d0c2f2c 100644 --- a/primeseq/src/org/labkey/primeseq/etl/VerifyRowCount.java +++ b/primeseq/src/org/labkey/primeseq/etl/VerifyRowCount.java @@ -255,11 +255,11 @@ private void verifyRows(PipelineJob job) throws PipelineJobException job.getLogger().info("Source (" + _settings.get(Settings.sourceSchema.name()) + "." + _settings.get(Settings.sourceQuery.name()) + ") row count: " + source); long dest = getDestinationTableCount(); - job.getLogger().info("Destination (" + _settings.get(Settings.destSchema.name()) + "." + _settings.get(Settings.destQuery.name()) + ") row count: " + source); + job.getLogger().info("Destination (" + _settings.get(Settings.destSchema.name()) + "." + _settings.get(Settings.destQuery.name()) + ") row count: " + dest); if (source != dest) { - job.getLogger().error("Row counts do not match!"); + job.getLogger().error("Row counts do not match (source: {}, dest: {})!", source, dest); } } } From 61ee67821d69df85a70c4beac67903ce08817732 Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 12 Aug 2025 15:42:56 -0700 Subject: [PATCH 10/15] Simplify mGAP ETL --- mGAP/resources/etls/clinicalData.xml | 140 +++++---------------------- 1 file changed, 26 insertions(+), 114 deletions(-) diff --git a/mGAP/resources/etls/clinicalData.xml b/mGAP/resources/etls/clinicalData.xml index 1cdc49ba8..e4801d7ef 100644 --- a/mGAP/resources/etls/clinicalData.xml +++ b/mGAP/resources/etls/clinicalData.xml @@ -3,23 +3,6 @@ ClinicalData EHR Clinical Data - - - - Id - date - ageAtTime - testId - result - units - runid - objectid - - - - - - @@ -33,111 +16,40 @@ avgKinship objectid + + + - - - - Id - date - ageAtTime - tissue - organism - quantity - runid - objectid - - - + + + + + + + + - - - - - - Id - date - ageAtTime - weight - objectid - - - - - - - - - - Id - date - ageAtTime - type - tissue - units - servicerequested - collectionMethod - method - objectid - - - - - - - - - - Id - date - ageAtTime - testid - result - units - runid - objectid - - - - - - - - - - Id - date - ageAtTime - sort_order - codes - objectid - - - - - - - - - - Id - date - ageAtTime - sort_order - tissue - codes - objectid - - - - - + + + + + - + + + + + + + primaryKey + + + From 48bd615b86e6279dda1e1030e6ef082b52be3fcf Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 12 Aug 2025 15:56:12 -0700 Subject: [PATCH 11/15] Many improvements to EHR-related ETLs --- PMR/resources/etls/pmr-datasets.xml | 226 ------- PMR/resources/etls/pmrClinicalData.xml | 79 ++- PMR/resources/etls/prime-birth.xml | 66 ++ PMR/resources/etls/prime-blooddraws.xml | 52 ++ PMR/resources/etls/prime-chemistryResults.xml | 55 ++ PMR/resources/etls/prime-clinpathRuns.xml | 56 ++ ...emographics.xml => prime-demographics.xml} | 40 +- .../etls/prime-hematologyResults.xml | 55 ++ PMR/resources/etls/prime-histology.xml | 54 ++ PMR/resources/etls/prime-microbiology.xml | 55 ++ PMR/resources/etls/prime-parentage.xml | 58 ++ .../etls/prime-pathologyDiagnoses.xml | 53 ++ PMR/resources/etls/prime-weight.xml | 55 ++ .../study/datasets/datasets_manifest.xml | 25 +- .../study/datasets/datasets_metadata.xml | 607 ++++++++++++++---- PMR/src/org/labkey/pmr/PMRController.java | 4 +- mGAP/resources/etls/project_usage.xml | 27 - primeseq/resources/etls/prime-blooddraws.xml | 30 - .../resources/etls/prime-demographics.xml | 34 - primeseq/resources/etls/prime-parentage.xml | 32 - primeseq/resources/etls/prime-weight.xml | 29 - primeseq/resources/etls/prime.xml | 43 -- 22 files changed, 1182 insertions(+), 553 deletions(-) delete mode 100644 PMR/resources/etls/pmr-datasets.xml create mode 100644 PMR/resources/etls/prime-birth.xml create mode 100644 PMR/resources/etls/prime-blooddraws.xml create mode 100644 PMR/resources/etls/prime-chemistryResults.xml create mode 100644 PMR/resources/etls/prime-clinpathRuns.xml rename PMR/resources/etls/{pmr-demographics.xml => prime-demographics.xml} (52%) create mode 100644 PMR/resources/etls/prime-hematologyResults.xml create mode 100644 PMR/resources/etls/prime-histology.xml create mode 100644 PMR/resources/etls/prime-microbiology.xml create mode 100644 PMR/resources/etls/prime-parentage.xml create mode 100644 PMR/resources/etls/prime-pathologyDiagnoses.xml create mode 100644 PMR/resources/etls/prime-weight.xml delete mode 100644 mGAP/resources/etls/project_usage.xml delete mode 100644 primeseq/resources/etls/prime-blooddraws.xml delete mode 100644 primeseq/resources/etls/prime-demographics.xml delete mode 100644 primeseq/resources/etls/prime-parentage.xml delete mode 100644 primeseq/resources/etls/prime-weight.xml delete mode 100644 primeseq/resources/etls/prime.xml diff --git a/PMR/resources/etls/pmr-datasets.xml b/PMR/resources/etls/pmr-datasets.xml deleted file mode 100644 index 6b915703c..000000000 --- a/PMR/resources/etls/pmr-datasets.xml +++ /dev/null @@ -1,226 +0,0 @@ - - - PRIMe_Datasets - PRIMe Dataset Data - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Id - date - parent - relationship - method - objectid - created - modified - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Id - date - date_type - birth_condition - dam - gender - weight - wdate - type - species - geographic_origin - sire - estimated - conception - conceptualDay - objectid - created - modified - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/PMR/resources/etls/pmrClinicalData.xml b/PMR/resources/etls/pmrClinicalData.xml index 591827c32..735c44f24 100644 --- a/PMR/resources/etls/pmrClinicalData.xml +++ b/PMR/resources/etls/pmrClinicalData.xml @@ -6,18 +6,91 @@ - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/PMR/resources/etls/prime-birth.xml b/PMR/resources/etls/prime-birth.xml new file mode 100644 index 000000000..4556dba00 --- /dev/null +++ b/PMR/resources/etls/prime-birth.xml @@ -0,0 +1,66 @@ + + + PRIME_Birth_Data + PRIMe Birth Data + + + + + Id + date + date_type + birth_condition + dam + gender + weight + wdate + type + species + geographic_origin + sire + estimated + conception + conceptualDay + objectid + created + QCState/Label + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + primaryKey + + + + diff --git a/PMR/resources/etls/prime-blooddraws.xml b/PMR/resources/etls/prime-blooddraws.xml new file mode 100644 index 000000000..13abeaa15 --- /dev/null +++ b/PMR/resources/etls/prime-blooddraws.xml @@ -0,0 +1,52 @@ + + + PRIME_Blood_Data + PRIMe Blood Data + + + Copy to target + + + Id + date + reason + quantity + objectid + QCState/Label + created + modified + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + primaryKey + + + + diff --git a/PMR/resources/etls/prime-chemistryResults.xml b/PMR/resources/etls/prime-chemistryResults.xml new file mode 100644 index 000000000..0c0cd1c92 --- /dev/null +++ b/PMR/resources/etls/prime-chemistryResults.xml @@ -0,0 +1,55 @@ + + + PRIME_ChemistryResults_Data + PRIMe Chemistry Results Data + + + + + Id + date + ageAtTime + testId + result + units + runid + objectid + created + modified + QCState/Label + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + primaryKey + + + + diff --git a/PMR/resources/etls/prime-clinpathRuns.xml b/PMR/resources/etls/prime-clinpathRuns.xml new file mode 100644 index 000000000..f6afe0b27 --- /dev/null +++ b/PMR/resources/etls/prime-clinpathRuns.xml @@ -0,0 +1,56 @@ + + + PRIME_ClinpathRuns_Data + PRIMe ClinpathRuns Data + + + + + Id + date + type + tissue + units + servicerequested + collectionMethod + method + objectid + created + modified + QCState/Label + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + primaryKey + + + + diff --git a/PMR/resources/etls/pmr-demographics.xml b/PMR/resources/etls/prime-demographics.xml similarity index 52% rename from PMR/resources/etls/pmr-demographics.xml rename to PMR/resources/etls/prime-demographics.xml index 883f477ce..87fb1a758 100644 --- a/PMR/resources/etls/pmr-demographics.xml +++ b/PMR/resources/etls/prime-demographics.xml @@ -1,42 +1,44 @@ - PRIMe_Demographics + PRIME_Demographics_Data PRIMe Demographics Data - - + + Copy to target + Id date gender - species geographic_origin birth death + species status calculated_status - QCState/Label objectid + QCState/Label created modified + + + - - - - + - + - + + @@ -45,9 +47,15 @@ - - - - - + + + + + + + + primaryKey + + + diff --git a/PMR/resources/etls/prime-hematologyResults.xml b/PMR/resources/etls/prime-hematologyResults.xml new file mode 100644 index 000000000..9df7326e3 --- /dev/null +++ b/PMR/resources/etls/prime-hematologyResults.xml @@ -0,0 +1,55 @@ + + + PRIME_HematologyResults_Data + PRIMe Hematology Results Data + + + + + Id + date + ageAtTime + testId + result + units + runid + objectid + created + modified + QCState/Label + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + primaryKey + + + + diff --git a/PMR/resources/etls/prime-histology.xml b/PMR/resources/etls/prime-histology.xml new file mode 100644 index 000000000..27a21350b --- /dev/null +++ b/PMR/resources/etls/prime-histology.xml @@ -0,0 +1,54 @@ + + + PRIME_Histology_Data + PRIMe Histology Data + + + + + Id + date + ageAtTime + sort_order + tissue + codes + objectid + created + modified + QCState/Label + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + primaryKey + + + + diff --git a/PMR/resources/etls/prime-microbiology.xml b/PMR/resources/etls/prime-microbiology.xml new file mode 100644 index 000000000..194c4d83e --- /dev/null +++ b/PMR/resources/etls/prime-microbiology.xml @@ -0,0 +1,55 @@ + + + PRIME_Microbiology_Data + PRIMe Microbiology Data + + + + + Id + date + ageAtTime + tissue + organism + quantity + runid + objectid + created + modified + QCState/Label + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + primaryKey + + + + diff --git a/PMR/resources/etls/prime-parentage.xml b/PMR/resources/etls/prime-parentage.xml new file mode 100644 index 000000000..30f46b3f7 --- /dev/null +++ b/PMR/resources/etls/prime-parentage.xml @@ -0,0 +1,58 @@ + + + PRIME_Parentage_Data + PRIMe Parentage Data + + + Copy to target + + + Id + date + enddate + parent + relationship + method + objectid + QCState/Label + created + modified + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + primaryKey + + + + diff --git a/PMR/resources/etls/prime-pathologyDiagnoses.xml b/PMR/resources/etls/prime-pathologyDiagnoses.xml new file mode 100644 index 000000000..428ef6e3a --- /dev/null +++ b/PMR/resources/etls/prime-pathologyDiagnoses.xml @@ -0,0 +1,53 @@ + + + PRIME_PathologyDiagnoses_Data + PRIMe Pathology Diagnoses Data + + + + + Id + date + ageAtTime + sort_order + codes + objectid + created + modified + QCState/Label + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + primaryKey + + + + diff --git a/PMR/resources/etls/prime-weight.xml b/PMR/resources/etls/prime-weight.xml new file mode 100644 index 000000000..435679e81 --- /dev/null +++ b/PMR/resources/etls/prime-weight.xml @@ -0,0 +1,55 @@ + + + PRIME_Weight_Data + PRIMe Weight Data + + + Copy to target + + + Id + date + weight + objectid + QCState/Label + created + modified + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + primaryKey + + + + diff --git a/PMR/resources/referenceStudy/study/datasets/datasets_manifest.xml b/PMR/resources/referenceStudy/study/datasets/datasets_manifest.xml index a57ce797a..6ee27d416 100644 --- a/PMR/resources/referenceStudy/study/datasets/datasets_manifest.xml +++ b/PMR/resources/referenceStudy/study/datasets/datasets_manifest.xml @@ -28,10 +28,25 @@ - + - + + + + + + + + + + + + + + + + @@ -67,5 +82,11 @@ + + + + + + diff --git a/PMR/resources/referenceStudy/study/datasets/datasets_metadata.xml b/PMR/resources/referenceStudy/study/datasets/datasets_metadata.xml index 042e53ace..f279b1638 100644 --- a/PMR/resources/referenceStudy/study/datasets/datasets_metadata.xml +++ b/PMR/resources/referenceStudy/study/datasets/datasets_metadata.xml @@ -187,7 +187,7 @@ integer urn:ehr.labkey.org/#Project - + varchar @@ -423,114 +423,6 @@ Weight - - - - varchar - http://cpas.labkey.com/Study#ParticipantId - - ptid - - - - timestamp - http://cpas.labkey.com/Study#VisitDate - http://cpas.labkey.com/Study#VisitDate - - - timestamp - urn:ehr.labkey.org/#EndDate - - - varchar - - - integer - urn:ehr.labkey.org/#Project - - - entityid - urn:ehr.labkey.org/#ObjectId - true - - - varchar - - - varchar - - - varchar - - - double - - - varchar - - - timestamp - - - Labwork -
- - - - varchar - http://cpas.labkey.com/Study#ParticipantId - - ptid - - - - timestamp - http://cpas.labkey.com/Study#VisitDate - http://cpas.labkey.com/Study#VisitDate - - - varchar - - - double - - - varchar - - - double - - - double - - - varchar - - - varchar - - - entityid - urn:ehr.labkey.org/#ObjectId - true - - - varchar - - - integer - urn:ehr.labkey.org/#Project - - - timestamp - urn:ehr.labkey.org/#EndDate - - - varchar - - - Lab Results -
@@ -1044,4 +936,501 @@ Kinship
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + ptid + + + + timestamp + http://cpas.labkey.com/Study#VisitDate + http://cpas.labkey.com/Study#VisitDate + + + timestamp + urn:ehr.labkey.org/#EndDate + + + varchar + + + varchar + + + integer + urn:ehr.labkey.org/#Project + + + varchar + + + entityid + urn:ehr.labkey.org/#ObjectId + true + + + varchar + + + varchar + + + varchar + + + varchar + + + varchar + + + varchar + + + varchar + + + double + + + varchar + + + varchar + + + timestamp + + + varchar + + + Clinpath Runs +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + ptid + + + + timestamp + http://cpas.labkey.com/Study#VisitDate + http://cpas.labkey.com/Study#VisitDate + + + varchar + + + double + + + varchar + + + double + + + double + + + varchar + + + varchar + + + entityid + urn:ehr.labkey.org/#ObjectId + true + + + varchar + + + integer + urn:ehr.labkey.org/#Project + + + timestamp + urn:ehr.labkey.org/#EndDate + + + varchar + + + Hematology Results +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + ptid + + + + timestamp + http://cpas.labkey.com/Study#VisitDate + http://cpas.labkey.com/Study#VisitDate + + + varchar + + + entityid + urn:ehr.labkey.org/#ObjectId + true + + + varchar + + + integer + urn:ehr.labkey.org/#Project + + + timestamp + urn:ehr.labkey.org/#EndDate + + + double + + + varchar + + + varchar + + + varchar + + + varchar + + + Parasitology Results +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + ptid + + + + timestamp + http://cpas.labkey.com/Study#VisitDate + http://cpas.labkey.com/Study#VisitDate + + + varchar + + + double + + + varchar + + + double + + + double + + + double + + + varchar + + + varchar + + + entityid + urn:ehr.labkey.org/#ObjectId + true + + + integer + urn:ehr.labkey.org/#Project + + + timestamp + urn:ehr.labkey.org/#EndDate + + + varchar + + + double + + + varchar + + + double + + + double + + + varchar + + + Urinalysis Results +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + ptid + + + + timestamp + http://cpas.labkey.com/Study#VisitDate + http://cpas.labkey.com/Study#VisitDate + + + varchar + + + double + + + double + + + varchar + + + varchar + + + integer + urn:ehr.labkey.org/#Project + + + integer + + + entityid + urn:ehr.labkey.org/#ObjectId + true + + + varchar + + + timestamp + urn:ehr.labkey.org/#EndDate + + + timestamp + urn:ehr.labkey.org/#DateRequested + + + varchar + + + varchar + + + timestamp + + + Blood Draws +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + ptid + + + + timestamp + http://cpas.labkey.com/Study#VisitDate + http://cpas.labkey.com/Study#VisitDate + + + integer + + + entityid + urn:ehr.labkey.org/#ObjectId + true + + + integer + urn:ehr.labkey.org/#Project + + + integer + + + Pathology Diagnoses +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + ptid + + + + timestamp + http://cpas.labkey.com/Study#VisitDate + http://cpas.labkey.com/Study#VisitDate + + + integer + + + varchar + + + entityid + urn:ehr.labkey.org/#ObjectId + true + + + integer + urn:ehr.labkey.org/#Project + + + timestamp + urn:ehr.labkey.org/#EndDate + + + varchar + + + varchar + + + integer + + + Histology +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + ptid + + + + timestamp + http://cpas.labkey.com/Study#VisitDate + http://cpas.labkey.com/Study#VisitDate + + + varchar + + + varchar + + + varchar + + + varchar + + + integer + urn:ehr.labkey.org/#Project + + + entityid + urn:ehr.labkey.org/#ObjectId + true + + + varchar + + + Microbiology Results +
+ + + + varchar + http://cpas.labkey.com/Study#ParticipantId + + ptid + + + + timestamp + http://cpas.labkey.com/Study#VisitDate + http://cpas.labkey.com/Study#VisitDate + + + varchar + + + double + + + varchar + + + double + + + double + + + varchar + + + varchar + + + entityid + urn:ehr.labkey.org/#ObjectId + true + + + varchar + + + integer + urn:ehr.labkey.org/#Project + + + timestamp + urn:ehr.labkey.org/#EndDate + + + varchar + + + varchar + + + Chemistry Results +
diff --git a/PMR/src/org/labkey/pmr/PMRController.java b/PMR/src/org/labkey/pmr/PMRController.java index 69487b75d..8dfab686a 100644 --- a/PMR/src/org/labkey/pmr/PMRController.java +++ b/PMR/src/org/labkey/pmr/PMRController.java @@ -52,9 +52,9 @@ public static class ImportStudyAction extends ConfirmAction @Override public ModelAndView getConfirmView(Object o, BindException errors) throws Exception { - setTitle("Import mGAP Study"); + setTitle("Import PMR Study"); - return new HtmlView(HtmlString.unsafe("This will import the default mGAP study in this folder and set the EHRStudyContainer property to point to this container. Do you want to continue?")); + return new HtmlView(HtmlString.unsafe("This will import the default PMR study in this folder and set the EHRStudyContainer property to point to this container. Do you want to continue?")); } @Override diff --git a/mGAP/resources/etls/project_usage.xml b/mGAP/resources/etls/project_usage.xml deleted file mode 100644 index 67df09b66..000000000 --- a/mGAP/resources/etls/project_usage.xml +++ /dev/null @@ -1,27 +0,0 @@ - - - Project Usage - Project Usage Data - - - Copy to local table - - - subjectId - project - groupname - startdate - enddate - comment - - - - - - - - - - - - diff --git a/primeseq/resources/etls/prime-blooddraws.xml b/primeseq/resources/etls/prime-blooddraws.xml deleted file mode 100644 index 8faf28096..000000000 --- a/primeseq/resources/etls/prime-blooddraws.xml +++ /dev/null @@ -1,30 +0,0 @@ - - - PRIME_Blood_Data - PRIMe Blood Data - - - Copy to target - - - Id - date - reason - quantity - objectid - QCState - - - - - - - - - - - - - - - diff --git a/primeseq/resources/etls/prime-demographics.xml b/primeseq/resources/etls/prime-demographics.xml deleted file mode 100644 index 6485f5ee1..000000000 --- a/primeseq/resources/etls/prime-demographics.xml +++ /dev/null @@ -1,34 +0,0 @@ - - - PRIME_Demographics_Data - PRIMe Demographics Data - - - Copy to target - - - Id - date - gender - geographic_origin - birth - death - species - status - objectid - QCState - - - - - - - - - - - - - - - diff --git a/primeseq/resources/etls/prime-parentage.xml b/primeseq/resources/etls/prime-parentage.xml deleted file mode 100644 index 107683da9..000000000 --- a/primeseq/resources/etls/prime-parentage.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - PRIME_Parentage_Data - PRIMe Parentage Data - - - Copy to target - - - Id - date - parent - relationship - method - datedisabled - objectid - QCState - - - - - - - - - - - - - - - diff --git a/primeseq/resources/etls/prime-weight.xml b/primeseq/resources/etls/prime-weight.xml deleted file mode 100644 index dba86c54d..000000000 --- a/primeseq/resources/etls/prime-weight.xml +++ /dev/null @@ -1,29 +0,0 @@ - - - PRIME_Parentage_Data - PRIMe Parentage Data - - - Copy to target - - - Id - date - weight - objectid - QCState - - - - - - - - - - - - - - - diff --git a/primeseq/resources/etls/prime.xml b/primeseq/resources/etls/prime.xml deleted file mode 100644 index f84182ab3..000000000 --- a/primeseq/resources/etls/prime.xml +++ /dev/null @@ -1,43 +0,0 @@ - - - PRIME_Data - PRIMe Clinical/Demographics Data - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - From ad75704324000abd1a1a5190442ba6a432b81c26 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 13 Aug 2025 06:55:46 -0700 Subject: [PATCH 12/15] Improve mGAP QC --- .../queries/mGAP/sequenceDatasetsSource.query.xml | 10 ++++++++++ .../queries/mGAP/sequenceDatasetsSource.sql | 3 ++- mGAP/resources/schemas/mgap.xml | 1 + mGAP/resources/views/mgapDataDashboard.html | 13 +++++++++++++ 4 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 mGAP/resources/queries/mGAP/sequenceDatasetsSource.query.xml diff --git a/mGAP/resources/queries/mGAP/sequenceDatasetsSource.query.xml b/mGAP/resources/queries/mGAP/sequenceDatasetsSource.query.xml new file mode 100644 index 000000000..3f4433200 --- /dev/null +++ b/mGAP/resources/queries/mGAP/sequenceDatasetsSource.query.xml @@ -0,0 +1,10 @@ + + + + + mGAP Sequence Datasets + +
+ + + \ No newline at end of file diff --git a/mGAP/resources/queries/mGAP/sequenceDatasetsSource.sql b/mGAP/resources/queries/mGAP/sequenceDatasetsSource.sql index 0093ffe3f..e7484ab09 100644 --- a/mGAP/resources/queries/mGAP/sequenceDatasetsSource.sql +++ b/mGAP/resources/queries/mGAP/sequenceDatasetsSource.sql @@ -4,7 +4,8 @@ SELECT r.totalForwardReads as totalReads, r.sraRuns as sraAccession, r.subjectId as originalId, - (SELECT s.center FROM mgap.subjectssource s WHERE s.subjectName = am.externalAlias) as center + (SELECT s.center FROM mgap.subjectssource s WHERE s.subjectName = am.externalAlias) as center, + CASE WHEN (SELECT s.subjectName FROM mgap.subjectssource s WHERE s.subjectName = am.externalAlias) IS NULL THEN true ELSE false END as missingDemographics FROM sequenceanalysis.sequence_readsets r JOIN mgap.animalMapping am ON (r.subjectId = am.subjectname) diff --git a/mGAP/resources/schemas/mgap.xml b/mGAP/resources/schemas/mgap.xml index caed3fa92..ff56af1ae 100644 --- a/mGAP/resources/schemas/mgap.xml +++ b/mGAP/resources/schemas/mgap.xml @@ -291,6 +291,7 @@ laboratory subjects subjectname + diff --git a/mGAP/resources/views/mgapDataDashboard.html b/mGAP/resources/views/mgapDataDashboard.html index 5894d4de3..e43b7e7e5 100644 --- a/mGAP/resources/views/mgapDataDashboard.html +++ b/mGAP/resources/views/mgapDataDashboard.html @@ -81,6 +81,13 @@ queryName: 'subjectCaseMismatch', 'query.externalAlias~isnonblank': '' }) + }, { + name: 'Sequence Datasets Missing Demographics', + url: LABKEY.ActionURL.buildURL('query', 'executeQuery.view', null, { + schemaName: 'mgap', + queryName: 'sequenceDatasetsSource', + 'query.missingDemographics~eq': 'true' + }) }] },{ header: 'Prior Releases', @@ -102,6 +109,12 @@ schemaName: 'mgap', queryName: 'subjectsSource' }) + }, { + name: 'Sequence Datasets Synced to mGAP', + url: LABKEY.ActionURL.buildURL('query', 'executeQuery.view', null, { + schemaName: 'mgap', + queryName: 'sequenceDatasetsSource' + }) }] },{ header: 'Data and Processing', From ccef459965a2344307ecdd589fb7c3bffc1d4859 Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 19 Aug 2025 14:22:41 -0700 Subject: [PATCH 13/15] Add MCC column showing numAnimalsRequested --- mcc/resources/queries/mcc/requestScores/.qview.xml | 1 + mcc/src/org/labkey/mcc/MccSchema.java | 1 + mcc/src/org/labkey/mcc/query/MccRequestCustomizer.java | 8 ++++++++ 3 files changed, 10 insertions(+) diff --git a/mcc/resources/queries/mcc/requestScores/.qview.xml b/mcc/resources/queries/mcc/requestScores/.qview.xml index 3899ec233..f65fdeb51 100644 --- a/mcc/resources/queries/mcc/requestScores/.qview.xml +++ b/mcc/resources/queries/mcc/requestScores/.qview.xml @@ -7,6 +7,7 @@ + diff --git a/mcc/src/org/labkey/mcc/MccSchema.java b/mcc/src/org/labkey/mcc/MccSchema.java index 03e4cd3ed..532fed97f 100644 --- a/mcc/src/org/labkey/mcc/MccSchema.java +++ b/mcc/src/org/labkey/mcc/MccSchema.java @@ -32,6 +32,7 @@ public class MccSchema public static final String TABLE_REQUEST_REVIEWS = "requestReviews"; public static final String TABLE_REQUEST_SCORE = "requestScores"; public static final String TABLE_CENSUS = "census"; + public static final String TABLE_REQUEST_COHORTS = "requestcohorts"; public static MccSchema getInstance() { diff --git a/mcc/src/org/labkey/mcc/query/MccRequestCustomizer.java b/mcc/src/org/labkey/mcc/query/MccRequestCustomizer.java index cf0f86072..59828fc3b 100644 --- a/mcc/src/org/labkey/mcc/query/MccRequestCustomizer.java +++ b/mcc/src/org/labkey/mcc/query/MccRequestCustomizer.java @@ -63,6 +63,14 @@ public boolean isSortable() ti.addColumn(newCol2); } + if (ti.getColumn("numAnimalsRequested") == null) + { + SQLFragment sql = new SQLFragment("(SELECT SUM(rc.numberofanimals) as expr FROM " + MccSchema.TABLE_REQUEST_COHORTS + " rc WHERE rc.requestId = " + ExprColumn.STR_TABLE_ALIAS + ".requestId)"); + ExprColumn newCol = new ExprColumn(ti, "numAnimalsRequested", sql, JdbcType.INTEGER, ti.getColumn("requestId")); + newCol.setLabel("# Animals Requested"); + ti.addColumn(newCol); + } + if (ti.getColumn("transferIds") == null) { Container dataContainer = MccManager.get().getMCCContainer(ti.getContainer()); From bfaffc2f69faf13ae3d5250be765c04e6848adf9 Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 19 Aug 2025 15:28:01 -0700 Subject: [PATCH 14/15] Sql fix --- mcc/src/org/labkey/mcc/query/MccRequestCustomizer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mcc/src/org/labkey/mcc/query/MccRequestCustomizer.java b/mcc/src/org/labkey/mcc/query/MccRequestCustomizer.java index 59828fc3b..42704019f 100644 --- a/mcc/src/org/labkey/mcc/query/MccRequestCustomizer.java +++ b/mcc/src/org/labkey/mcc/query/MccRequestCustomizer.java @@ -65,7 +65,7 @@ public boolean isSortable() if (ti.getColumn("numAnimalsRequested") == null) { - SQLFragment sql = new SQLFragment("(SELECT SUM(rc.numberofanimals) as expr FROM " + MccSchema.TABLE_REQUEST_COHORTS + " rc WHERE rc.requestId = " + ExprColumn.STR_TABLE_ALIAS + ".requestId)"); + SQLFragment sql = new SQLFragment("(SELECT SUM(rc.numberofanimals) as expr FROM mcc." + MccSchema.TABLE_REQUEST_COHORTS + " rc WHERE rc.requestId = " + ExprColumn.STR_TABLE_ALIAS + ".requestId)"); ExprColumn newCol = new ExprColumn(ti, "numAnimalsRequested", sql, JdbcType.INTEGER, ti.getColumn("requestId")); newCol.setLabel("# Animals Requested"); ti.addColumn(newCol); From 550b30f92504a4124f9cb6f11b027f69f3a17c98 Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 19 Aug 2025 17:54:40 -0700 Subject: [PATCH 15/15] Add url to column --- mcc/src/org/labkey/mcc/query/MccRequestCustomizer.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mcc/src/org/labkey/mcc/query/MccRequestCustomizer.java b/mcc/src/org/labkey/mcc/query/MccRequestCustomizer.java index 42704019f..44c9b5d02 100644 --- a/mcc/src/org/labkey/mcc/query/MccRequestCustomizer.java +++ b/mcc/src/org/labkey/mcc/query/MccRequestCustomizer.java @@ -8,6 +8,7 @@ import org.labkey.api.exp.api.StorageProvisioner; import org.labkey.api.ldk.table.AbstractTableCustomizer; import org.labkey.api.ldk.table.CustomPermissionsTable; +import org.labkey.api.query.DetailsURL; import org.labkey.api.query.ExprColumn; import org.labkey.api.query.FieldKey; import org.labkey.api.study.Dataset; @@ -68,6 +69,7 @@ public boolean isSortable() SQLFragment sql = new SQLFragment("(SELECT SUM(rc.numberofanimals) as expr FROM mcc." + MccSchema.TABLE_REQUEST_COHORTS + " rc WHERE rc.requestId = " + ExprColumn.STR_TABLE_ALIAS + ".requestId)"); ExprColumn newCol = new ExprColumn(ti, "numAnimalsRequested", sql, JdbcType.INTEGER, ti.getColumn("requestId")); newCol.setLabel("# Animals Requested"); + newCol.setURL(DetailsURL.fromString("/query/executeQuery.view?schemaName=mcc&query.queryName=" + MccSchema.TABLE_REQUEST_COHORTS + "&query.requestId~eq=${requestId}")); ti.addColumn(newCol); }