From 29b56afb8739ce4e2642d98cdbd892bec9111f57 Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 30 Jun 2025 10:13:42 -0700 Subject: [PATCH 01/17] Update MCC request permission scheme to separate insert/update (#190) * Update MCC request permission scheme to separate insert/update * Update MCC request permission scheme to separate insert/update * Missed with last commit --- mcc/resources/queries/mcc/animalRequests.js | 14 +++++++++-- mcc/src/org/labkey/mcc/MccManager.java | 24 ++++++++++++++----- .../org/labkey/mcc/query/TriggerHelper.java | 23 ++++++++++++++++-- 3 files changed, 51 insertions(+), 10 deletions(-) diff --git a/mcc/resources/queries/mcc/animalRequests.js b/mcc/resources/queries/mcc/animalRequests.js index e069b428e..0a3f8ec36 100644 --- a/mcc/resources/queries/mcc/animalRequests.js +++ b/mcc/resources/queries/mcc/animalRequests.js @@ -36,9 +36,18 @@ function beforeUpsert(row, oldRow, errors) { row.status = row.status || 'Draft' - if (!triggerHelper.hasPermission(row.status)) { - errors._form = 'Insufficient permissions to update request with status: ' + row.status; + // This logic here is that the user needs update permissions on the original status, and insert permissions to the new one: + if (oldRow) { + if (oldRow.status && !triggerHelper.hasUpdatePermission(oldRow.status)) { + errors._form = 'Insufficient permissions to update request with status: ' + row.status; + } + else if (!oldRow.status) { + console.error('MCC request being submitted without a value for oldRow.status!') + } + } + if (!triggerHelper.hasInsertPermission(row.status)) { + errors._form = 'Insufficient permissions to create request with status: ' + row.status; } } @@ -72,6 +81,7 @@ function beforeDelete(row, errors){ return; } + // if (!triggerHelper.hasPermission(row.status)) { errors._form = 'Insufficient permissions to delete this request'; return; diff --git a/mcc/src/org/labkey/mcc/MccManager.java b/mcc/src/org/labkey/mcc/MccManager.java index 9ce036caa..0fe8763b9 100644 --- a/mcc/src/org/labkey/mcc/MccManager.java +++ b/mcc/src/org/labkey/mcc/MccManager.java @@ -50,26 +50,38 @@ public enum RequestStatus Submitted(2, "Submitted", MccRequestorPermission.class), RabReview(3, "RAB Review", MccRequestAdminPermission.class), PendingDecision(4, "Decision Pending", MccFinalReviewPermission.class), - Approved(5, "Approved", MccRequestAdminPermission.class), - Rejected(6, "Rejected", MccRequestAdminPermission.class), + Approved(5, "Approved", MccRequestAdminPermission.class, MccFinalReviewPermission.class), + Rejected(6, "Rejected", MccRequestAdminPermission.class, MccFinalReviewPermission.class), Processing(7, "Processing", MccRequestAdminPermission.class), Fulfilled(8, "Fulfilled", MccRequestAdminPermission.class), Withdrawn(9, "Withdrawn", MccRequestorPermission.class); int sortOrder; String label; - Class editPermission; + Class updatePermission; + Class insertPermission; RequestStatus(int sortOrder, String label, Class editPermission) + { + this(sortOrder, label, editPermission, editPermission); + } + + RequestStatus(int sortOrder, String label, Class updatePermission, Class insertPermission) { this.sortOrder = sortOrder; this.label = label; - this.editPermission = editPermission; + this.updatePermission = updatePermission; + this.insertPermission= insertPermission; + } + + public boolean canUpdate(User u, Container c) + { + return c.hasPermission(u, this.updatePermission); } - public boolean canEdit(User u, Container c) + public boolean canInsert(User u, Container c) { - return c.hasPermission(u, this.editPermission); + return c.hasPermission(u, this.insertPermission); } public String getLabel() diff --git a/mcc/src/org/labkey/mcc/query/TriggerHelper.java b/mcc/src/org/labkey/mcc/query/TriggerHelper.java index f36e999e6..9df99ae2a 100644 --- a/mcc/src/org/labkey/mcc/query/TriggerHelper.java +++ b/mcc/src/org/labkey/mcc/query/TriggerHelper.java @@ -212,11 +212,30 @@ public void cascadeDelete(String schemaName, String queryName, String keyField, } } - public boolean hasPermission(String status) + public boolean hasUpdatePermission(String status) + { + return hasPermission(status, false); + } + + public boolean hasInsertPermission(String status) + { + return hasPermission(status, true); + } + + private boolean hasPermission(String status, boolean forInsert) { try { - return MccManager.RequestStatus.resolveStatus(status).canEdit(_user, _container); + MccManager.RequestStatus s = MccManager.RequestStatus.resolveStatus(status); + if (forInsert) + { + return MccManager.RequestStatus.resolveStatus(status).canInsert(_user, _container); + } + else + { + return MccManager.RequestStatus.resolveStatus(status).canUpdate(_user, _container); + } + } catch (IllegalArgumentException e) { From 44de7eecbe6878812fc40420005bf2b93b9f652c Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 2 Jul 2025 21:07:35 -0700 Subject: [PATCH 02/17] Put ETL inside transaction --- .../sivstudies/etl/SubjectScopedSelect.java | 180 +++++++++--------- 1 file changed, 95 insertions(+), 85 deletions(-) diff --git a/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java b/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java index a36369cea..6c750b8bf 100644 --- a/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java +++ b/SivStudies/src/org/labkey/sivstudies/etl/SubjectScopedSelect.java @@ -9,6 +9,7 @@ import org.labkey.api.data.CompareType; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.DbScope; import org.labkey.api.data.SimpleFilter; import org.labkey.api.data.TableInfo; import org.labkey.api.data.TableSelector; @@ -100,7 +101,7 @@ public boolean isRequired() } } - final int BATCH_SIZE = 100; + final int BATCH_SIZE = 250; private MODE getMode() { @@ -136,127 +137,136 @@ private void checkCancelled(PipelineJob job) private void processBatch(List subjects, Logger log, PipelineJob job) { log.info("processing batch with " + subjects.size() + " subjects"); - TableInfo destinationTable = getDataDestinationTable(); + try (DbScope.Transaction t = DbScope.getLabKeyScope().ensureTransaction()) + { + TableInfo destinationTable = getDataDestinationTable(); - QueryUpdateService qus = destinationTable.getUpdateService(); - qus.setBulkLoad(true); + QueryUpdateService qus = destinationTable.getUpdateService(); + qus.setBulkLoad(true); - try - { - if (getMode() == MODE.TRUNCATE) + try { - // Find / Delete existing values: - Set keyFields = destinationTable.getColumns().stream().filter(ColumnInfo::isKeyField).collect(Collectors.toSet()); - final SimpleFilter subjectFilter = new SimpleFilter(FieldKey.fromString(_settings.get(Settings.targetSubjectColumn.name())), subjects, CompareType.IN); - if (_settings.get(Settings.targetAdditionalFilters.name()) != null) + if (getMode() == MODE.TRUNCATE) { - List additionalFilters = parseAdditionalFilters(_settings.get(Settings.targetAdditionalFilters.name())); - additionalFilters.forEach(subjectFilter::addCondition); - } + // Find / Delete existing values: + Set keyFields = destinationTable.getColumns().stream().filter(ColumnInfo::isKeyField).collect(Collectors.toSet()); + final SimpleFilter subjectFilter = new SimpleFilter(FieldKey.fromString(_settings.get(Settings.targetSubjectColumn.name())), subjects, CompareType.IN); + if (_settings.get(Settings.targetAdditionalFilters.name()) != null) + { + List additionalFilters = parseAdditionalFilters(_settings.get(Settings.targetAdditionalFilters.name())); + additionalFilters.forEach(subjectFilter::addCondition); + } - if (destinationTable.getColumn(FieldKey.fromString(_settings.get(Settings.targetSubjectColumn.name()))) == null) - { - throw new IllegalStateException("Unknown column on table " + destinationTable.getName() + ": " + _settings.get(Settings.targetSubjectColumn.name())); - } + if (destinationTable.getColumn(FieldKey.fromString(_settings.get(Settings.targetSubjectColumn.name()))) == null) + { + throw new IllegalStateException("Unknown column on table " + destinationTable.getName() + ": " + _settings.get(Settings.targetSubjectColumn.name())); + } - List> existingRows = new ArrayList<>(new TableSelector(destinationTable, keyFields, subjectFilter, null).getMapCollection()); - if (!existingRows.isEmpty()) - { - List>> batches = Lists.partition(existingRows, 5000); - log.info("deleting " + existingRows.size() + " rows in " + batches.size() + " batches"); - int i = 0; - for (List> batch : batches) + List> existingRows = new ArrayList<>(new TableSelector(destinationTable, keyFields, subjectFilter, null).getMapCollection()); + if (!existingRows.isEmpty()) { - i++; - log.info("batch " + i); - checkCancelled(job); + List>> batches = Lists.partition(existingRows, 5000); + log.info("deleting " + existingRows.size() + " rows in " + batches.size() + " batches"); + int i = 0; + for (List> batch : batches) + { + i++; + log.info("batch " + i); + checkCancelled(job); - qus.deleteRows(_containerUser.getUser(), _containerUser.getContainer(), batch, new HashMap<>(Map.of(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior, NONE, QueryUpdateService.ConfigParameters.BulkLoad, true)), null); + qus.deleteRows(_containerUser.getUser(), _containerUser.getContainer(), batch, new HashMap<>(Map.of(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior, NONE, QueryUpdateService.ConfigParameters.BulkLoad, true)), null); + t.commitAndKeepConnection(); + } + } + else + { + log.info("No rows to delete for this subject batch"); } } else { - log.info("No rows to delete for this subject batch"); + log.info("Using " + getMode().name() + " mode, source records will not be deleted"); } - } - else - { - log.info("Using " + getMode().name() + " mode, source records will not be deleted"); - } - // Query data and import - List> toImportOrUpdate = getRowsToImport(subjects, log); - if (!toImportOrUpdate.isEmpty()) - { - if (getMode() == MODE.TRUNCATE) + // Query data and import + List> toImportOrUpdate = getRowsToImport(subjects, log); + if (!toImportOrUpdate.isEmpty()) { - List>> batches = Lists.partition(toImportOrUpdate, 5000); - log.info("inserting " + toImportOrUpdate.size() + " rows in " + batches.size() + " batches"); - - int i = 0; - for (List> batch : batches) + if (getMode() == MODE.TRUNCATE) { - i++; - log.info("batch " + i); - checkCancelled(job); + List>> batches = Lists.partition(toImportOrUpdate, 5000); + log.info("inserting " + toImportOrUpdate.size() + " rows in " + batches.size() + " batches"); - BatchValidationException bve = new BatchValidationException(); - qus.insertRows(_containerUser.getUser(), _containerUser.getContainer(), batch, bve, new HashMap<>(Map.of(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior, NONE, QueryUpdateService.ConfigParameters.BulkLoad, true)), null); - if (bve.hasErrors()) + int i = 0; + for (List> batch : batches) { - throw bve; + i++; + log.info("batch " + i); + checkCancelled(job); + + BatchValidationException bve = new BatchValidationException(); + qus.insertRows(_containerUser.getUser(), _containerUser.getContainer(), batch, bve, new HashMap<>(Map.of(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior, NONE, QueryUpdateService.ConfigParameters.BulkLoad, true)), null); + if (bve.hasErrors()) + { + throw bve; + } + t.commitAndKeepConnection(); } } - } - else if (getMode() == MODE.UPDATE_ONLY) - { - List>> batches = Lists.partition(toImportOrUpdate, 5000); - log.info("updating " + toImportOrUpdate.size() + " rows in " + batches.size() + " batches"); - - int i = 0; - for (List> batch : batches) + else if (getMode() == MODE.UPDATE_ONLY) { + List>> batches = Lists.partition(toImportOrUpdate, 5000); + log.info("updating " + toImportOrUpdate.size() + " rows in " + batches.size() + " batches"); - i++; - log.info("batch " + i); - checkCancelled(job); + int i = 0; + for (List> batch : batches) + { - BatchValidationException bve = new BatchValidationException(); + i++; + log.info("batch " + i); + checkCancelled(job); - Collection keyFields = destinationTable.getPkColumnNames(); - List> keys = batch.stream().map(x -> { - Map map = new HashMap<>(); - for (String keyField : keyFields) - { - if (x.get(keyField) != null) + BatchValidationException bve = new BatchValidationException(); + + Collection keyFields = destinationTable.getPkColumnNames(); + List> keys = batch.stream().map(x -> { + Map map = new HashMap<>(); + for (String keyField : keyFields) { - map.put(keyField, x.get(keyField)); + if (x.get(keyField) != null) + { + map.put(keyField, x.get(keyField)); + } } - } - return map; - }).toList(); + return map; + }).toList(); - qus.updateRows(_containerUser.getUser(), _containerUser.getContainer(), batch, keys, bve, new HashMap<>(Map.of(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior, NONE, QueryUpdateService.ConfigParameters.BulkLoad, true)), null); - if (bve.hasErrors()) - { - throw bve; + qus.updateRows(_containerUser.getUser(), _containerUser.getContainer(), batch, keys, bve, new HashMap<>(Map.of(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior, NONE, QueryUpdateService.ConfigParameters.BulkLoad, true)), null); + if (bve.hasErrors()) + { + throw bve; + } + t.commitAndKeepConnection(); } } + else + { + throw new IllegalStateException("Unknown mode: " + getMode()); + } } else { - throw new IllegalStateException("Unknown mode: " + getMode()); + log.info("No rows to import/update for this subject batch"); } } - else + catch (SQLException | InvalidKeyException | BatchValidationException | QueryUpdateServiceException | + DuplicateKeyException e) { - log.info("No rows to import/update for this subject batch"); + throw new IllegalStateException("Error Importing/Updating Rows", e); } - } - catch (SQLException | InvalidKeyException | BatchValidationException | QueryUpdateServiceException | DuplicateKeyException e) - { - throw new IllegalStateException("Error Importing/Updating Rows", e); + + t.commit(); } } From 3a1fbe74a700cfc759d33cd215745bf320166dd9 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 3 Jul 2025 05:30:27 -0700 Subject: [PATCH 03/17] Add calculated columns --- tcrdb/src/org/labkey/tcrdb/TCRdbModule.java | 2 + .../labkey/tcrdb/TCRdbTableCustomizer.java | 38 +++++++++++++++++++ .../src/org/labkey/tcrdb/TCRdbUserSchema.java | 6 ++- 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java b/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java index d452b218b..18e7b3278 100644 --- a/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java +++ b/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java @@ -72,6 +72,8 @@ protected void doStartupAfterSpringConfig(ModuleContext moduleContext) LaboratoryService.get().registerTableCustomizer(this, TCRdbTableCustomizer.class, TCRdbSchema.SEQUENCE_ANALYSIS, "sequence_readsets"); LaboratoryService.get().registerTableCustomizer(this, TCRdbTableCustomizer.class, TCRdbSchema.SEQUENCE_ANALYSIS, "sequence_analyses"); LaboratoryService.get().registerTableCustomizer(this, TCRdbTableCustomizer.class, TCRdbSchema.NAME, TCRdbSchema.TABLE_CLONES); + LaboratoryService.get().registerTableCustomizer(this, TCRdbTableCustomizer.class, TCRdbSchema.NAME, TCRdbSchema.TABLE_CLONE_RESPONSES); + LaboratoryService.get().registerTableCustomizer(this, TCRdbTableCustomizer.class, TCRdbSchema.NAME, TCRdbSchema.TABLE_STIM_EXPERIMENTS); LaboratoryService.get().registerTableCustomizer(this, TCRdbTableCustomizer.class, TCRdbSchema.SINGLE_CELL, TCRdbSchema.TABLE_CDNAS); LDKService.get().registerQueryButton(new ChangeStatusButton(), TCRdbSchema.SINGLE_CELL, "samples"); diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbTableCustomizer.java b/tcrdb/src/org/labkey/tcrdb/TCRdbTableCustomizer.java index 1a9f00944..44226a3f6 100644 --- a/tcrdb/src/org/labkey/tcrdb/TCRdbTableCustomizer.java +++ b/tcrdb/src/org/labkey/tcrdb/TCRdbTableCustomizer.java @@ -12,6 +12,7 @@ import org.labkey.api.data.SimpleFilter; import org.labkey.api.data.Table; import org.labkey.api.data.TableInfo; +import org.labkey.api.data.WrappedColumn; import org.labkey.api.exp.api.ExpProtocol; import org.labkey.api.laboratory.LaboratoryService; import org.labkey.api.ldk.LDKService; @@ -19,6 +20,7 @@ import org.labkey.api.query.DetailsURL; import org.labkey.api.query.ExprColumn; import org.labkey.api.query.FieldKey; +import org.labkey.api.query.QueryForeignKey; import org.labkey.api.query.QueryService; import java.util.Arrays; @@ -48,6 +50,14 @@ else if (matches(ti, TCRdbSchema.NAME, TCRdbSchema.TABLE_CLONES)) { customizeClones(ti); } + else if (matches(ti, TCRdbSchema.NAME, TCRdbSchema.TABLE_CLONE_RESPONSES)) + { + customizeCloneResponses(ti); + } + else if (matches(ti, TCRdbSchema.NAME, TCRdbSchema.TABLE_STIM_EXPERIMENTS)) + { + customizeStims(ti); + } else if (ti instanceof AssayResultTable) { customizeAssayData(ti); @@ -223,6 +233,34 @@ private void addClonotypeForLocusCol(AbstractTableInfo ti, SQLFragment selectSql ti.addColumn(newCol); } + private void customizeCloneResponses(AbstractTableInfo ti) + { + if (ti.getColumn("stimId") == null) + { + WrappedColumn col = new WrappedColumn(ti.getColumn("cDNA_ID"), "stimId"); + col.setReadOnly(true); + col.setUserEditable(false); + col.setLabel("Stim Experiment"); + col.setFk(new QueryForeignKey(ti.getUserSchema(), null, ti.getUserSchema(), null, TCRdbSchema.TABLE_STIM_EXPERIMENTS, "cdna_id", "cdna_id")); + + ti.addColumn(col); + } + } + + private void customizeStims(AbstractTableInfo ti) + { + if (ti.getColumn("controlStim") == null) + { + WrappedColumn col = new WrappedColumn(ti.getColumn("controlStimId"), "controlStim"); + col.setReadOnly(true); + col.setUserEditable(false); + col.setLabel("Control Stim Info"); + col.setFk(new QueryForeignKey(ti.getUserSchema(), null, ti.getUserSchema(), null, TCRdbSchema.TABLE_STIM_EXPERIMENTS, "cdna_id", "cdna_id")); + + ti.addColumn(col); + } + } + private void customizeClones(AbstractTableInfo ti) { LDKService.get().applyNaturalSort(ti, "cloneName"); diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbUserSchema.java b/tcrdb/src/org/labkey/tcrdb/TCRdbUserSchema.java index 188fcc112..505bd957b 100644 --- a/tcrdb/src/org/labkey/tcrdb/TCRdbUserSchema.java +++ b/tcrdb/src/org/labkey/tcrdb/TCRdbUserSchema.java @@ -6,6 +6,7 @@ import org.labkey.api.data.ContainerFilter; import org.labkey.api.data.DbSchema; import org.labkey.api.data.TableInfo; +import org.labkey.api.ldk.table.ContainerScopedTable; import org.labkey.api.ldk.table.SharedDataTable; import org.labkey.api.module.Module; import org.labkey.api.query.DefaultSchema; @@ -43,9 +44,12 @@ protected TableInfo createWrappedTable(String name, @NotNull TableInfo sourceTab { if (TCRdbSchema.TABLE_MIXCR_LIBRARIES.equalsIgnoreCase(name)) { - // TODO: assert cf is null or not default? return new SharedDataTable<>(this, sourceTable).init(); } + else if (TCRdbSchema.TABLE_STIM_EXPERIMENTS.equalsIgnoreCase(name)) + { + return new ContainerScopedTable<>(this, sourceTable, null, "cdna_id").init(); + } return super.createWrappedTable(name, sourceTable, cf); } From 91be6e1e0c5a55c8a48a529aca3f8476d268c5dc Mon Sep 17 00:00:00 2001 From: bbimber Date: Sat, 5 Jul 2025 09:02:38 -0700 Subject: [PATCH 04/17] Bugfix cron string --- .../org/labkey/primeseq/notification/DiskUsageNotification.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java b/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java index 799965cf3..4ea8e2b94 100644 --- a/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java +++ b/primeseq/src/org/labkey/primeseq/notification/DiskUsageNotification.java @@ -73,7 +73,7 @@ public DateFormat getDateTimeFormat(Container c) @Override public String getCronString() { - return "0 8 * * 1 ?"; + return "0 0 8 ? * MON"; } @Override From 68456502dca81f06653bafa0dbaeec3c9abf7f4f Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 7 Jul 2025 13:46:29 -0700 Subject: [PATCH 05/17] Add SIV calculated fields --- .../study/demographics/Expanded.qview.xml | 2 ++ .../demographicsChallengeAndArt.query.xml | 22 +++++++++++++++++++ .../study/demographicsChallengeAndArt.sql | 13 +++++++++++ .../resources/views/participantView.html | 7 ++++++ .../query/SivStudiesCustomizer.java | 7 ++++++ 5 files changed, 51 insertions(+) create mode 100644 SivStudies/resources/queries/study/demographicsChallengeAndArt.query.xml create mode 100644 SivStudies/resources/queries/study/demographicsChallengeAndArt.sql create mode 100644 SivStudies/resources/views/participantView.html diff --git a/SivStudies/resources/queries/study/demographics/Expanded.qview.xml b/SivStudies/resources/queries/study/demographics/Expanded.qview.xml index 6cfb780d6..8d3c3e733 100644 --- a/SivStudies/resources/queries/study/demographics/Expanded.qview.xml +++ b/SivStudies/resources/queries/study/demographics/Expanded.qview.xml @@ -9,6 +9,8 @@ + + diff --git a/SivStudies/resources/queries/study/demographicsChallengeAndArt.query.xml b/SivStudies/resources/queries/study/demographicsChallengeAndArt.query.xml new file mode 100644 index 000000000..8e3ec5353 --- /dev/null +++ b/SivStudies/resources/queries/study/demographicsChallengeAndArt.query.xml @@ -0,0 +1,22 @@ + + + + + SIV/ART Summary + + + true + true + + + SIV Infection + + + ART + + + allInfections +
+ + + diff --git a/SivStudies/resources/queries/study/demographicsChallengeAndArt.sql b/SivStudies/resources/queries/study/demographicsChallengeAndArt.sql new file mode 100644 index 000000000..63c9f2f28 --- /dev/null +++ b/SivStudies/resources/queries/study/demographicsChallengeAndArt.sql @@ -0,0 +1,13 @@ +SELECT + t.Id, + group_concat(DISTINCT CASE + WHEN t.category = 'SIV Infection' THEN (cast(month(t.date) as varchar) || '-' || cast(dayofmonth(t.date) as varchar) || '-' || cast(year(t.date) as varchar) || ' (' || t.treatment || ')') + ELSE NULL + END, char(10)) as allInfections, + group_concat(DISTINCT CASE + WHEN t.category = 'ART' THEN (cast(month(t.date) as varchar) || '-' || cast(dayofmonth(t.date) as varchar) || '-' || cast(year(t.date) as varchar) || ' (' || t.treatment || ')') + ELSE NULL + END, char(10)) as allART, + +FROM study.treatments t +GROUP BY t.Id \ No newline at end of file diff --git a/SivStudies/resources/views/participantView.html b/SivStudies/resources/views/participantView.html new file mode 100644 index 000000000..64c6df390 --- /dev/null +++ b/SivStudies/resources/views/participantView.html @@ -0,0 +1,7 @@ + \ No newline at end of file diff --git a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java index 48b615b8a..ebaeb3322 100644 --- a/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java +++ b/SivStudies/src/org/labkey/sivstudies/query/SivStudiesCustomizer.java @@ -249,6 +249,13 @@ private void appendDemographicsColumns(AbstractTableInfo parentTable) colInfo.setLabel("Outcomes"); parentTable.addColumn(colInfo); } + + if (parentTable.getColumn("sivART") == null) + { + BaseColumnInfo colInfo = getWrappedIdCol(parentTable.getUserSchema(), "demographicsChallengeAndArt", parentTable, "sivART"); + colInfo.setLabel("SIV/ART Dates"); + parentTable.addColumn(colInfo); + } } private void appendPvlColumns(DatasetTable ds, String subjectColName, String dateColName) From 9071af59d5a2fec924eba4b49752b2edb2f9f3e8 Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 7 Jul 2025 14:02:51 -0700 Subject: [PATCH 06/17] Update URL values --- .../resources/queries/study/additionalDatatypes.query.xml | 2 +- SivStudies/resources/queries/study/demographics.query.xml | 1 - SivStudies/resources/queries/study/procedures.query.xml | 2 +- SivStudies/resources/queries/study/studyData.query.xml | 4 ++++ SivStudies/resources/queries/study/weight.query.xml | 2 +- 5 files changed, 7 insertions(+), 4 deletions(-) diff --git a/SivStudies/resources/queries/study/additionalDatatypes.query.xml b/SivStudies/resources/queries/study/additionalDatatypes.query.xml index 07aff7121..1b446dfde 100644 --- a/SivStudies/resources/queries/study/additionalDatatypes.query.xml +++ b/SivStudies/resources/queries/study/additionalDatatypes.query.xml @@ -4,7 +4,7 @@ - + diff --git a/SivStudies/resources/queries/study/demographics.query.xml b/SivStudies/resources/queries/study/demographics.query.xml index 49f10a6dd..63cf4041e 100644 --- a/SivStudies/resources/queries/study/demographics.query.xml +++ b/SivStudies/resources/queries/study/demographics.query.xml @@ -5,7 +5,6 @@ - true diff --git a/SivStudies/resources/queries/study/procedures.query.xml b/SivStudies/resources/queries/study/procedures.query.xml index 06cd4bdc6..eaa6a9606 100644 --- a/SivStudies/resources/queries/study/procedures.query.xml +++ b/SivStudies/resources/queries/study/procedures.query.xml @@ -4,7 +4,7 @@
- + diff --git a/SivStudies/resources/queries/study/studyData.query.xml b/SivStudies/resources/queries/study/studyData.query.xml index 4a1d46d73..e78b0c84f 100644 --- a/SivStudies/resources/queries/study/studyData.query.xml +++ b/SivStudies/resources/queries/study/studyData.query.xml @@ -6,6 +6,7 @@ http://cpas.labkey.com/Study#ParticipantId + laboratory/dataBrowser.view?subjectId=${Id} Date @@ -25,6 +26,9 @@ true + + true +
- + true From 4e6f690b0632f73a9c2f5494c7164eaf5245bd38 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 9 Jul 2025 11:33:31 -0700 Subject: [PATCH 07/17] Improve wording on mGAP login page --- mGAP/resources/views/login.html | 1 + 1 file changed, 1 insertion(+) diff --git a/mGAP/resources/views/login.html b/mGAP/resources/views/login.html index 9dd9f23ca..83f514341 100644 --- a/mGAP/resources/views/login.html +++ b/mGAP/resources/views/login.html @@ -14,6 +14,7 @@
+
While mGAP is a free NIH-sponsored resource, we require users register to help us track and report usage to our funders. This information is critical to demonstrate the value and impact of the resource. Please use the 'Request an Account' link below if you are not already registered. Thank you for your understanding.
Sign In
From 59dc9f07f2a4ae4048005f463b1ad919c8ca94a9 Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 9 Jul 2025 12:18:50 -0700 Subject: [PATCH 08/17] Improve wording on mGAP login page --- mGAP/resources/views/login.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mGAP/resources/views/login.html b/mGAP/resources/views/login.html index 83f514341..471911cf3 100644 --- a/mGAP/resources/views/login.html +++ b/mGAP/resources/views/login.html @@ -14,8 +14,8 @@ -
While mGAP is a free NIH-sponsored resource, we require users register to help us track and report usage to our funders. This information is critical to demonstrate the value and impact of the resource. Please use the 'Request an Account' link below if you are not already registered. Thank you for your understanding.
-
Sign In
+
Sign In / Register
+
While mGAP is a free NIH-sponsored resource, we require users register to help us track and report usage to our funders. This information is critical to demonstrate the value and impact of the resource. Please request an account if you are not already registered. Thank you for your understanding.
From 20283e50139d3421e6230cb9196c8b576564b627 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 10 Jul 2025 13:17:49 -0700 Subject: [PATCH 09/17] Change date delimiter --- .../resources/queries/study/demographicsChallengeAndArt.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SivStudies/resources/queries/study/demographicsChallengeAndArt.sql b/SivStudies/resources/queries/study/demographicsChallengeAndArt.sql index 63c9f2f28..ea898b425 100644 --- a/SivStudies/resources/queries/study/demographicsChallengeAndArt.sql +++ b/SivStudies/resources/queries/study/demographicsChallengeAndArt.sql @@ -1,11 +1,11 @@ SELECT t.Id, group_concat(DISTINCT CASE - WHEN t.category = 'SIV Infection' THEN (cast(month(t.date) as varchar) || '-' || cast(dayofmonth(t.date) as varchar) || '-' || cast(year(t.date) as varchar) || ' (' || t.treatment || ')') + WHEN t.category = 'SIV Infection' THEN (cast(month(t.date) as varchar) || '/' || cast(dayofmonth(t.date) as varchar) || '/' || cast(year(t.date) as varchar) || ' (' || t.treatment || ')') ELSE NULL END, char(10)) as allInfections, group_concat(DISTINCT CASE - WHEN t.category = 'ART' THEN (cast(month(t.date) as varchar) || '-' || cast(dayofmonth(t.date) as varchar) || '-' || cast(year(t.date) as varchar) || ' (' || t.treatment || ')') + WHEN t.category = 'ART' THEN (cast(month(t.date) as varchar) || '/' || cast(dayofmonth(t.date) as varchar) || '/' || cast(year(t.date) as varchar) || ' (' || t.treatment || ')') ELSE NULL END, char(10)) as allART, From 756e99b793daee6705047505959df0e3224823f6 Mon Sep 17 00:00:00 2001 From: bbimber Date: Thu, 10 Jul 2025 20:32:17 -0700 Subject: [PATCH 10/17] Set bulkLoad=true --- mcc/src/org/labkey/mcc/etl/NprcObservationStep.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/mcc/src/org/labkey/mcc/etl/NprcObservationStep.java b/mcc/src/org/labkey/mcc/etl/NprcObservationStep.java index 87a9f7f3c..bbfe3f086 100644 --- a/mcc/src/org/labkey/mcc/etl/NprcObservationStep.java +++ b/mcc/src/org/labkey/mcc/etl/NprcObservationStep.java @@ -21,6 +21,7 @@ import org.labkey.api.query.FieldKey; import org.labkey.api.query.InvalidKeyException; import org.labkey.api.query.QueryService; +import org.labkey.api.query.QueryUpdateService; import org.labkey.api.query.QueryUpdateServiceException; import org.labkey.api.query.UserSchema; import org.labkey.api.reader.Readers; @@ -74,6 +75,8 @@ private void processFile(PipelineJob job) throws PipelineJobException { throw new PipelineJobException("Unable to find table: clinical observations"); } + QueryUpdateService qus = clinicalObs.getUpdateService(); + qus.setBulkLoad(true); final List> toInsert = new ArrayList<>(); final List> toUpdate = new ArrayList<>(); @@ -156,7 +159,7 @@ private void processFile(PipelineJob job) throws PipelineJobException job.getLogger().info("Deleting " + toDelete.size() + " rows"); try { - clinicalObs.getUpdateService().deleteRows(_containerUser.getUser(), _containerUser.getContainer(), toDelete, null, null); + qus.deleteRows(_containerUser.getUser(), _containerUser.getContainer(), toDelete, null, null); } catch (InvalidKeyException | BatchValidationException | QueryUpdateServiceException | SQLException e) { @@ -170,7 +173,7 @@ private void processFile(PipelineJob job) throws PipelineJobException try { BatchValidationException bve = new BatchValidationException(); - clinicalObs.getUpdateService().insertRows(_containerUser.getUser(), _containerUser.getContainer(), toInsert, bve, null, null); + qus.insertRows(_containerUser.getUser(), _containerUser.getContainer(), toInsert, bve, null, null); if (bve.hasErrors()) { throw bve; @@ -192,7 +195,7 @@ private void processFile(PipelineJob job) throws PipelineJobException try { List> oldKeys = toUpdate.stream().map(x -> Map.of("objectid", x.get("objectid"))).collect(Collectors.toList()); - clinicalObs.getUpdateService().updateRows(_containerUser.getUser(), _containerUser.getContainer(), toUpdate, oldKeys, null, null); + qus.updateRows(_containerUser.getUser(), _containerUser.getContainer(), toUpdate, oldKeys, null, null); } catch (QueryUpdateServiceException | SQLException | BatchValidationException | InvalidKeyException e) { From e30ed6bb7621fdfc68bc67c4e14e5dc1f9fb5164 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 11 Jul 2025 10:13:35 -0700 Subject: [PATCH 11/17] Add tcr repertoire stats table --- .../postgresql/tcrdb-15.57-15.58.sql | 16 ++++++ .../dbscripts/sqlserver/tcrdb-15.57-15.58.sql | 16 ++++++ tcrdb/resources/schemas/tcrdb.xml | 54 +++++++++++++++++++ tcrdb/src/org/labkey/tcrdb/TCRdbModule.java | 2 +- 4 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.57-15.58.sql create mode 100644 tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.57-15.58.sql diff --git a/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.57-15.58.sql b/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.57-15.58.sql new file mode 100644 index 000000000..80b9e7d95 --- /dev/null +++ b/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.57-15.58.sql @@ -0,0 +1,16 @@ +CREATE TABLE tcrdb.repertoire_stats ( + rowid int SERIAL, + cdna_id int, + metricName varchar(1000), + value double precision, + qualValue varchar(4000), + comment varchar(4000), + + container entityid, + created timestamp, + createdby int, + modified timestamp, + modifiedby int, + + constraint PK_repertoire_stats PRIMARY KEY (rowid) +); \ No newline at end of file diff --git a/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.57-15.58.sql b/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.57-15.58.sql new file mode 100644 index 000000000..5e9c3699c --- /dev/null +++ b/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.57-15.58.sql @@ -0,0 +1,16 @@ +CREATE TABLE tcrdb.repertoire_stats ( + rowid int IDENTITY(1,1), + cdna_id int, + metricName varchar(1000), + value double precision, + qualValue varchar(4000), + comment varchar(4000), + + container entityid, + created datetime, + createdby int, + modified datetime, + modifiedby int, + + constraint PK_repertoire_stats PRIMARY KEY (rowid) +); \ No newline at end of file diff --git a/tcrdb/resources/schemas/tcrdb.xml b/tcrdb/resources/schemas/tcrdb.xml index 60a93c0ad..b63843443 100644 --- a/tcrdb/resources/schemas/tcrdb.xml +++ b/tcrdb/resources/schemas/tcrdb.xml @@ -325,6 +325,60 @@ + + + true + + + true + + + false + false + false + true + true + + + true + + + false + false + false + true + true + + +
+ + + TCR Repertoire Stats + rowid + + + Row Id + true + + + cDNA ID + + singlecell + cdna_libraries + rowid + + + + Metric Name + + + Value + + + Qualitative Value + + + true diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java b/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java index 18e7b3278..1f4726263 100644 --- a/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java +++ b/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java @@ -46,7 +46,7 @@ public String getName() @Override public Double getSchemaVersion() { - return 15.57; + return 15.58; } @Override From fb2e3a45a43921c19a3e0cbd13470e1a2162f15a Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 11 Jul 2025 10:45:25 -0700 Subject: [PATCH 12/17] Fix sql --- .../schemas/dbscripts/postgresql/tcrdb-15.57-15.58.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.57-15.58.sql b/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.57-15.58.sql index 80b9e7d95..5c5a9b3f5 100644 --- a/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.57-15.58.sql +++ b/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.57-15.58.sql @@ -1,5 +1,5 @@ CREATE TABLE tcrdb.repertoire_stats ( - rowid int SERIAL, + rowid SERIAL, cdna_id int, metricName varchar(1000), value double precision, From 578024e129e888f2f684207cda81eea1967b165e Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 23 Jul 2025 17:57:33 -0700 Subject: [PATCH 13/17] Update mGAP Query --- mGAP/resources/queries/mGAP/subjectsSource.query.xml | 1 + mGAP/resources/queries/mGAP/subjectsSource.sql | 8 +++++++- mGAP/resources/views/mgapDataDashboard.html | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/mGAP/resources/queries/mGAP/subjectsSource.query.xml b/mGAP/resources/queries/mGAP/subjectsSource.query.xml index 6185f7741..7d580fa2b 100644 --- a/mGAP/resources/queries/mGAP/subjectsSource.query.xml +++ b/mGAP/resources/queries/mGAP/subjectsSource.query.xml @@ -2,6 +2,7 @@
+ mGAP Subject/Demographics Source DatasubjectName
diff --git a/mGAP/resources/queries/mGAP/subjectsSource.sql b/mGAP/resources/queries/mGAP/subjectsSource.sql index 3a1ed394c..ce5a82461 100644 --- a/mGAP/resources/queries/mGAP/subjectsSource.sql +++ b/mGAP/resources/queries/mGAP/subjectsSource.sql @@ -10,9 +10,15 @@ SELECT WHEN s.Id IS NOT NULL THEN 'ONPRC' ELSE NULL END as center, d.status as status, - m.subjectname as originalId + m.subjectname as originalId, + p1.externalAlias as sire, + coalesce(s.Id.parents.sire, d.sire) as originalSire, + p2.externalAlias as dam, + coalesce(s.Id.parents.dam, d.dam) as originalDam, FROM mgap.animalMapping m LEFT JOIN "/Internal/PMR/".study.demographics s ON (m.subjectname = s.Id) LEFT JOIN mgap.demographics d ON (m.subjectname = d.subjectname) +LEFT JOIN mgap.animalMapping p1 ON (p1.subjectname = coalesce(s.Id.parents.sire, d.sire)) +LEFT JOIN mgap.animalMapping p2 ON (p2.subjectname = coalesce(s.Id.parents.dam, d.dam)) WHERE (s.Id IS NOT NULL OR d.subjectname IS NOT NULL) \ No newline at end of file diff --git a/mGAP/resources/views/mgapDataDashboard.html b/mGAP/resources/views/mgapDataDashboard.html index 87cf98a9c..5894d4de3 100644 --- a/mGAP/resources/views/mgapDataDashboard.html +++ b/mGAP/resources/views/mgapDataDashboard.html @@ -100,7 +100,7 @@ name: 'Subject Information Synced to mGAP', url: LABKEY.ActionURL.buildURL('query', 'executeQuery.view', null, { schemaName: 'mgap', - queryName: 'subjectDatasetsSource' + queryName: 'subjectsSource' }) }] },{ From f58e9f31777de0a89280274f554d580378ae3626 Mon Sep 17 00:00:00 2001 From: bbimber Date: Sun, 27 Jul 2025 04:37:30 -0700 Subject: [PATCH 14/17] Expand study/cohort fields --- SivStudies/resources/queries/study/demographics/.qview.xml | 2 +- .../resources/queries/study/demographics/Expanded.qview.xml | 2 +- .../resources/queries/study/demographicsProjects.query.xml | 3 +++ SivStudies/resources/queries/study/demographicsProjects.sql | 1 + 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/SivStudies/resources/queries/study/demographics/.qview.xml b/SivStudies/resources/queries/study/demographics/.qview.xml index 677d864d0..36ef67231 100644 --- a/SivStudies/resources/queries/study/demographics/.qview.xml +++ b/SivStudies/resources/queries/study/demographics/.qview.xml @@ -13,7 +13,7 @@ - + diff --git a/SivStudies/resources/queries/study/demographics/Expanded.qview.xml b/SivStudies/resources/queries/study/demographics/Expanded.qview.xml index 8d3c3e733..1598334f0 100644 --- a/SivStudies/resources/queries/study/demographics/Expanded.qview.xml +++ b/SivStudies/resources/queries/study/demographics/Expanded.qview.xml @@ -7,7 +7,7 @@ - + diff --git a/SivStudies/resources/queries/study/demographicsProjects.query.xml b/SivStudies/resources/queries/study/demographicsProjects.query.xml index fc8f02e91..ec3dddc09 100644 --- a/SivStudies/resources/queries/study/demographicsProjects.query.xml +++ b/SivStudies/resources/queries/study/demographicsProjects.query.xml @@ -14,6 +14,9 @@ All Studies + + Subgroups/Treatments + RhCMV Vaccines? diff --git a/SivStudies/resources/queries/study/demographicsProjects.sql b/SivStudies/resources/queries/study/demographicsProjects.sql index e01f2a073..853db7d13 100644 --- a/SivStudies/resources/queries/study/demographicsProjects.sql +++ b/SivStudies/resources/queries/study/demographicsProjects.sql @@ -3,6 +3,7 @@ SELECT count(s.Id) as totalProjects, group_concat(DISTINCT s.study, char(10)) as allStudies, group_concat(DISTINCT s.category, char(10)) as categories, + group_concat(DISTINCT s.subgroup, char(10)) as subgroups, GROUP_CONCAT(distinct CASE WHEN s.category = 'RhCMV-Vaccines' THEN 'Yes' ELSE null END, char(10)) as rhCmvVaccines, GROUP_CONCAT(distinct CASE WHEN s.category = 'SIV/ART' THEN 'Yes' ELSE null END, char(10)) as sivArt From 551ad881b03a1326a874b1f5079c12473729d2e8 Mon Sep 17 00:00:00 2001 From: bbimber Date: Sun, 27 Jul 2025 04:54:49 -0700 Subject: [PATCH 15/17] Expand challenge query --- .../queries/study/demographicsChallengeAndArt.query.xml | 3 +++ .../resources/queries/study/demographicsChallengeAndArt.sql | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/SivStudies/resources/queries/study/demographicsChallengeAndArt.query.xml b/SivStudies/resources/queries/study/demographicsChallengeAndArt.query.xml index 8e3ec5353..135d018b6 100644 --- a/SivStudies/resources/queries/study/demographicsChallengeAndArt.query.xml +++ b/SivStudies/resources/queries/study/demographicsChallengeAndArt.query.xml @@ -14,6 +14,9 @@ ART + + Infection Date + allInfections diff --git a/SivStudies/resources/queries/study/demographicsChallengeAndArt.sql b/SivStudies/resources/queries/study/demographicsChallengeAndArt.sql index ea898b425..6aebfaf74 100644 --- a/SivStudies/resources/queries/study/demographicsChallengeAndArt.sql +++ b/SivStudies/resources/queries/study/demographicsChallengeAndArt.sql @@ -8,6 +8,10 @@ SELECT WHEN t.category = 'ART' THEN (cast(month(t.date) as varchar) || '/' || cast(dayofmonth(t.date) as varchar) || '/' || cast(year(t.date) as varchar) || ' (' || t.treatment || ')') ELSE NULL END, char(10)) as allART, + min(CASE + WHEN t.category = 'SIV Infection' THEN t.date + ELSE NULL + END, char(10)) as infectionDate, FROM study.treatments t GROUP BY t.Id \ No newline at end of file From 14915d80219667adba76069590c4a4c48c1a59e0 Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 29 Jul 2025 06:04:39 -0700 Subject: [PATCH 16/17] Fix challenge query --- .../resources/queries/study/demographicsChallengeAndArt.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SivStudies/resources/queries/study/demographicsChallengeAndArt.sql b/SivStudies/resources/queries/study/demographicsChallengeAndArt.sql index 6aebfaf74..4e4f9fc73 100644 --- a/SivStudies/resources/queries/study/demographicsChallengeAndArt.sql +++ b/SivStudies/resources/queries/study/demographicsChallengeAndArt.sql @@ -11,7 +11,7 @@ SELECT min(CASE WHEN t.category = 'SIV Infection' THEN t.date ELSE NULL - END, char(10)) as infectionDate, + END) as infectionDate, FROM study.treatments t GROUP BY t.Id \ No newline at end of file From ab29c30d07a2d0636fbf6427cc9ac1afe4735add Mon Sep 17 00:00:00 2001 From: bbimber Date: Wed, 30 Jul 2025 10:00:09 -0700 Subject: [PATCH 17/17] Improve returnUrl logic for mGAP's login page --- mGAP/resources/views/login.html | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/mGAP/resources/views/login.html b/mGAP/resources/views/login.html index 471911cf3..ead854e20 100644 --- a/mGAP/resources/views/login.html +++ b/mGAP/resources/views/login.html @@ -1,14 +1,30 @@