From aa86413901de9b5c2f477eba7589d146890e0085 Mon Sep 17 00:00:00 2001 From: labkey-jeckels Date: Wed, 15 Oct 2025 15:16:32 -0700 Subject: [PATCH 01/15] Cache precursor metrics in a dedicated table for perf --- .../postgresql/targetedms-25.005-25.006.sql | 19 ++ resources/schemas/targetedms.xml | 10 + .../labkey/targetedms/SkylineDocImporter.java | 3 +- .../targetedms/TargetedMSDataHandler.java | 2 +- .../labkey/targetedms/TargetedMSManager.java | 63 +++-- .../labkey/targetedms/TargetedMSModule.java | 12 +- .../labkey/targetedms/TargetedMSSchema.java | 1 + .../targetedms/model/RawMetricDataSet.java | 2 +- .../targetedms/outliers/OutlierGenerator.java | 266 +++++++++++------- .../pipeline/TargetedMSImportTask.java | 2 +- .../query/QCEnabledMetricsTable.java | 6 +- .../query/QCMetricConfigurationTable.java | 6 +- 12 files changed, 248 insertions(+), 144 deletions(-) create mode 100644 resources/schemas/dbscripts/postgresql/targetedms-25.005-25.006.sql diff --git a/resources/schemas/dbscripts/postgresql/targetedms-25.005-25.006.sql b/resources/schemas/dbscripts/postgresql/targetedms-25.005-25.006.sql new file mode 100644 index 000000000..d117223e6 --- /dev/null +++ b/resources/schemas/dbscripts/postgresql/targetedms-25.005-25.006.sql @@ -0,0 +1,19 @@ +CREATE TABLE targetedms.QCMetricCache +( + Container entityid NOT NULL, + MetricId INT NOT NULL, + PrecursorChromInfoId BIGINT, + SampleFileId BIGINT NOT NULL, + MetricValue REAL, + SeriesLabel VARCHAR(200), + + CONSTRAINT PK_QCMetricCache PRIMARY KEY (Container, MetricId, PrecursorChromInfoId), + CONSTRAINT FK_QCMetricCache_PrecursorChromInfoId FOREIGN KEY (PrecursorChromInfoId) REFERENCES targetedms.PrecursorChromInfo(Id), + CONSTRAINT FK_QCMetricCache_SampleFileId FOREIGN KEY (SampleFileId) REFERENCES targetedms.SampleFile(Id), + CONSTRAINT FK_QCMetricCache_MetricIdId FOREIGN KEY (MetricId) REFERENCES targetedms.QCMetricConfiguration(Id), + CONSTRAINT FK_QCMetricCache_Container FOREIGN KEY (Container) REFERENCES core.Containers(EntityId) +); + +CREATE INDEX IDX_QCMetricCache_PrecursorChromInfoId ON targetedms.QCMetricCache(PrecursorChromInfoId); +CREATE INDEX IDX_QCMetricCache_SampleFileId ON targetedms.QCMetricCache(SampleFileId); +CREATE INDEX IDX_QCMetricCache_MetricId ON targetedms.QCMetricCache(MetricId); \ No newline at end of file diff --git a/resources/schemas/targetedms.xml b/resources/schemas/targetedms.xml index dc2deebbd..1c3fbe4ff 100644 --- a/resources/schemas/targetedms.xml +++ b/resources/schemas/targetedms.xml @@ -1926,4 +1926,14 @@ + + + + + + + + + +
diff --git a/src/org/labkey/targetedms/SkylineDocImporter.java b/src/org/labkey/targetedms/SkylineDocImporter.java index 68f3ba578..5e6f30648 100644 --- a/src/org/labkey/targetedms/SkylineDocImporter.java +++ b/src/org/labkey/targetedms/SkylineDocImporter.java @@ -254,8 +254,7 @@ public TargetedMSRun importRun(RunInfo runInfo, PipelineJob job) throws IOExcept updateRunStatus(IMPORT_SUCCEEDED, STATUS_SUCCESS); - // We may have inserted the first set of data for a given metric - TargetedMSManager.get().clearCachedEnabledQCMetrics(run.getContainer()); + TargetedMSManager.get().clearQCMetricCache(run.getContainer(), true); _progressMonitor.complete(); return TargetedMSManager.getRun(_runId); diff --git a/src/org/labkey/targetedms/TargetedMSDataHandler.java b/src/org/labkey/targetedms/TargetedMSDataHandler.java index 388730f43..8ae5208e4 100644 --- a/src/org/labkey/targetedms/TargetedMSDataHandler.java +++ b/src/org/labkey/targetedms/TargetedMSDataHandler.java @@ -294,7 +294,7 @@ public void runMoved(ExpData newData, Container container, Container targetConta } // Update the run - TargetedMSManager.moveRun(run, targetContainer, newRunLSID, newData.getRowId(), user); + TargetedMSManager.get().moveRun(run, targetContainer, newRunLSID, newData.getRowId(), user); // Delete the old entry in exp.data -- it is no longer linked to the run. ExpData oldData = ExperimentService.get().getExpData(oldDataRowID); diff --git a/src/org/labkey/targetedms/TargetedMSManager.java b/src/org/labkey/targetedms/TargetedMSManager.java index 131ae4d06..9a113ff74 100644 --- a/src/org/labkey/targetedms/TargetedMSManager.java +++ b/src/org/labkey/targetedms/TargetedMSManager.java @@ -551,6 +551,11 @@ public static TableInfo getTableQCTraceMetricValues() return getSchema().getTable(TargetedMSSchema.TABLE_QC_TRACE_METRIC_VALUES); } + public static TableInfo getTableInfoQCMetricCache() + { + return getSchema().getTable(TargetedMSSchema.TABLE_QC_METRIC_CACHE); + } + public static TableInfo getTableInfoSkylineAuditLogEntry() { return getSchema().getTable(TargetedMSSchema.TABLE_SKYLINE_AUDITLOG_ENTRY); @@ -1288,7 +1293,7 @@ public static void deleteRuns(List runIds, Container c, User user, boolean } // We may have deleted the last set of data for a given metric - containers.forEach(c2 -> TargetedMSManager.get().clearCachedEnabledQCMetrics(c2)); + containers.forEach(c2 -> TargetedMSManager.get().clearQCMetricCache(c2, true)); // Mark all the runs for deletion SQLFragment markDeleted = new SQLFragment("UPDATE " + getTableInfoRuns() + " SET ExperimentRunLSID = NULL, DataId = NULL, SkydDataId = NULL, Deleted=?, Modified=? ", Boolean.TRUE, new Date()); @@ -2354,36 +2359,28 @@ public static List getAllQCMetricConfigurations(TargetedM SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("Status"), QCMetricStatus.Disabled.toString(), CompareType.NEQ_OR_NULL); List metrics = new TableSelector(metricsTable, filter, new Sort(FieldKey.fromParts("Name"))).getArrayList(QCMetricConfiguration.class); - // We may encounter the same query to see if metrics have data more than once, so remember the values - // so we don't have to requery - Map hasDataQueries = new CaseInsensitiveHashMap<>(); + OutlierGenerator.get().cachePrecursorMetricValues(schema); + + // Identify which precursor-scoped metrics have any cached data in this container + SQLFragment existingSql = new SQLFragment("SELECT DISTINCT MetricId FROM "); + existingSql.append(getTableInfoQCMetricCache(), "c"); + existingSql.append(" WHERE c.Container = ?"); + existingSql.add(c.getEntityId()); + Set cachedMetricIds = new HashSet<>(new SqlSelector(getSchema(), existingSql).getCollection(Integer.class)); for (QCMetricConfiguration metric : metrics) { if (metric.getStatus() == null) { - if (metric.getEnabledQueryName() == null) + if (metric.isPrecursorScoped()) { - metric.setStatus(QCMetricStatus.DEFAULT); - } - else - { - boolean hasData = hasDataQueries.computeIfAbsent(metric.getEnabledQueryName(), p -> - { - TableInfo enabledQuery = schema.getTable(metric.getEnabledQueryName(), null); - if (enabledQuery != null) - { - return new TableSelector(enabledQuery).exists(); - } - _log.warn("Could not find query " + schema.getName() + "." + metric.getEnabledQueryName() + " to determine if metric " + metric.getName() + " should be enabled in container " + c.getPath()); - return false; - }); - - if (!hasData) + if (!cachedMetricIds.contains(metric.getId())) { + // Precursor-scoped metrics without cached values have no data in this container metric.setStatus(QCMetricStatus.NoData); } } + // For run-scoped metrics, do not mark as NoData here since they are not cached; leave as DEFAULT } if (metric.getStatus() == null) { @@ -2473,9 +2470,11 @@ select pci.id, COUNT(DISTINCT tci.Id) AS C FROM\s return maxCount != null ? maxCount.intValue() : 0; } - static void moveRun(TargetedMSRun run, Container newContainer, String newRunLSID, long newDataRowId, User user) + public void moveRun(TargetedMSRun run, Container newContainer, String newRunLSID, long newDataRowId, User user) { // MoveRunsTask.moveRun ensures a transaction + Container oldContainer = run.getContainer(); + SQLFragment updatePrecChromInfoSql = new SQLFragment("UPDATE "); updatePrecChromInfoSql.append(getTableInfoPrecursorChromInfo(), ""); updatePrecChromInfoSql.append(" SET container = ?").add(newContainer); @@ -2491,6 +2490,11 @@ static void moveRun(TargetedMSRun run, Container newContainer, String newRunLSID run.setDataId(newDataRowId); run.setContainer(newContainer); updateRun(run, user); + + // Clear caches for both source and destination containers + if (oldContainer != null) + clearQCMetricCache(oldContainer, true); + clearQCMetricCache(newContainer, true); } private static void addParentRunsToChain(ArrayDeque chainRowIds, Map replacedByMap, Long rowId) @@ -2929,9 +2933,22 @@ public static Map getQCFolderDateRange(Container container) return new SqlSelector(getSchema(), sql).getMap(); } - public void clearCachedEnabledQCMetrics(Container container) + /** + * Invalidate the stored QC metric value cache for this container. + * Called when underlying data or metric definitions change (e.g., Skyline import/delete, config edits). + */ + public void clearQCMetricCache(Container container, boolean clearMetricValues) { getSchema().getScope().addCommitTask(() -> _metricCache.remove(container), DbScope.CommitTaskOption.IMMEDIATE, DbScope.CommitTaskOption.POSTCOMMIT, DbScope.CommitTaskOption.POSTROLLBACK); + + if (clearMetricValues) + { + SQLFragment sql = new SQLFragment("DELETE FROM "); + sql.append(getTableInfoQCMetricCache()); + sql.append(" WHERE Container = ?"); + sql.add(container.getEntityId()); + new SqlExecutor(getSchema()).execute(sql); + } } @NotNull diff --git a/src/org/labkey/targetedms/TargetedMSModule.java b/src/org/labkey/targetedms/TargetedMSModule.java index d6224e3cf..c0b7bd85d 100644 --- a/src/org/labkey/targetedms/TargetedMSModule.java +++ b/src/org/labkey/targetedms/TargetedMSModule.java @@ -183,10 +183,12 @@ public TargetedMSModule() SKYLINE_AUDIT_LEVEL_PROPERTY.setOptions(auditOptions); SKYLINE_AUDIT_LEVEL_PROPERTY.setDefaultValue("0"); SKYLINE_AUDIT_LEVEL_PROPERTY.setCanSetPerContainer(true); - SKYLINE_AUDIT_LEVEL_PROPERTY.setDescription("Defines requirements for the integrity of the audit log uploaded together with a Skyline document. \n"+ - "0 means that no audit log is required. If the log file is present in the uploaded file it will be parsed and loaded as is.\n " + - "1 means that audit log is required and its integrity will be verified using MD5 hash-based algorithm. If log integrity verification fails the document upload will be cancelled. \n" + - "2 means that audit log is required and its integrity will be verified using RSA-encryption algorithm. If log integrity verification fails the document upload will be cancelled."); + SKYLINE_AUDIT_LEVEL_PROPERTY.setDescription(""" + Defines requirements for the integrity of the audit log uploaded together with a Skyline document.\s + 0 means that no audit log is required. If the log file is present in the uploaded file it will be parsed and loaded as is. + \ + 1 means that audit log is required and its integrity will be verified using MD5 hash-based algorithm. If log integrity verification fails the document upload will be cancelled.\s + 2 means that audit log is required and its integrity will be verified using RSA-encryption algorithm. If log integrity verification fails the document upload will be cancelled."""); SKYLINE_AUDIT_LEVEL_PROPERTY.setShowDescriptionInline(true); addModuleProperty(SKYLINE_AUDIT_LEVEL_PROPERTY); //------------------------rr @@ -229,7 +231,7 @@ public String getName() @Override public Double getSchemaVersion() { - return 25.005; + return 25.006; } @Override diff --git a/src/org/labkey/targetedms/TargetedMSSchema.java b/src/org/labkey/targetedms/TargetedMSSchema.java index 8312d35ec..748540ab4 100644 --- a/src/org/labkey/targetedms/TargetedMSSchema.java +++ b/src/org/labkey/targetedms/TargetedMSSchema.java @@ -227,6 +227,7 @@ public class TargetedMSSchema extends UserSchema public static final String TABLE_QC_METRIC_EXCLUSION = "QCMetricExclusion"; public static final String TABLE_QC_ENABLED_METRICS = "QCEnabledMetrics"; public static final String TABLE_QC_TRACE_METRIC_VALUES = "QCTraceMetricValues"; + public static final String TABLE_QC_METRIC_CACHE = "QCMetricCache"; public static final String TABLE_GUIDE_SET = "GuideSet"; diff --git a/src/org/labkey/targetedms/model/RawMetricDataSet.java b/src/org/labkey/targetedms/model/RawMetricDataSet.java index 3513e8edd..61a0727ad 100644 --- a/src/org/labkey/targetedms/model/RawMetricDataSet.java +++ b/src/org/labkey/targetedms/model/RawMetricDataSet.java @@ -181,7 +181,7 @@ public String getDataType() } } - @Nullable + @NotNull public String getSeriesLabel() { if (null != seriesLabel) diff --git a/src/org/labkey/targetedms/outliers/OutlierGenerator.java b/src/org/labkey/targetedms/outliers/OutlierGenerator.java index c0dfbe4fa..028768e64 100644 --- a/src/org/labkey/targetedms/outliers/OutlierGenerator.java +++ b/src/org/labkey/targetedms/outliers/OutlierGenerator.java @@ -18,12 +18,13 @@ import org.apache.commons.lang3.StringUtils; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import org.labkey.api.collections.IntHashMap; +import org.labkey.api.action.SpringActionController; import org.labkey.api.collections.LongHashMap; import org.labkey.api.data.Container; import org.labkey.api.data.RuntimeSQLException; import org.labkey.api.data.SQLFragment; import org.labkey.api.data.SqlSelector; +import org.labkey.api.data.SqlExecutor; import org.labkey.api.data.TableInfo; import org.labkey.api.data.TableSelector; import org.labkey.api.query.QueryService; @@ -72,7 +73,6 @@ public class OutlierGenerator { private static final OutlierGenerator INSTANCE = new OutlierGenerator(); - private static final DecimalFormat DECIMAL_FORMAT = new DecimalFormat("0.000"); private OutlierGenerator() {} @@ -81,7 +81,7 @@ public static OutlierGenerator get() return INSTANCE; } - private String getEachSeriesTypePlotDataSql(QCMetricConfiguration configuration, List annotationGroups) + private String getEachSeriesTypePlotDataSql(QCMetricConfiguration configuration) { String schemaName = "targetedms"; String queryName = configuration.getQueryName(); @@ -93,7 +93,7 @@ private String getEachSeriesTypePlotDataSql(QCMetricConfiguration configuration, { sql.append("(SELECT 0 AS PrecursorChromInfoId, SampleFileId, "); sql.append(" metric.Name AS SeriesLabel, "); - sql.append("\nvalue as MetricValue, metric, ").append(configuration.getId()).append(" AS MetricId"); + sql.append("\nvalue as MetricValue, Metric AS MetricId "); sql.append("\n FROM ").append(schemaName).append('.').append(TargetedMSManager.getTableQCTraceMetricValues().getName()); sql.append(" WHERE metric = ").append(configuration.getId()); sql.append(")"); @@ -102,51 +102,15 @@ private String getEachSeriesTypePlotDataSql(QCMetricConfiguration configuration, { sql.append("(SELECT PrecursorChromInfoId, SampleFileId, "); sql.append(" CAST(IFDEFINED(SeriesLabel) AS VARCHAR) AS SeriesLabel, "); - sql.append("\nMetricValue, 0 as metric, ").append(configuration.getId()).append(" AS MetricId"); - + sql.append("\nMetricValue, ").append(configuration.getId()).append(" AS MetricId"); sql.append("\n FROM ").append(schemaName).append('.').append(queryName); - - if (!annotationGroups.isEmpty()) - { - sql.append(" WHERE "); - StringBuilder filterClause = new StringBuilder("SampleFileId.ReplicateId IN ("); - var intersect = ""; - var selectSql = "(SELECT ReplicateId FROM targetedms.ReplicateAnnotation WHERE "; - for (AnnotationGroup annotation : annotationGroups) - { - filterClause.append(intersect) - .append(selectSql) - .append(" Name='") - .append(annotation.getName().replace("'", "''")) - .append("'"); - - - var annotationValues = annotation.getValues(); - if (!annotationValues.isEmpty()) - { - var quoteEscapedVals = annotationValues.stream().map(s -> s.replace("'", "''")).toList( ); - var vals = "'" + StringUtils.join(quoteEscapedVals, "','") + "'"; - filterClause.append(" AND Value IN (").append(vals).append(" )"); - } - filterClause.append(" ) "); - intersect = " INTERSECT "; - } - filterClause.append(") "); - sql.append(filterClause); - } - if (configuration.getTraceName() != null) - { - sql.append(" WHERE metric = ").append(configuration.getId()); - } sql.append(")"); } return sql.toString(); } /** @return LabKey SQL to fetch all the values for the specified metrics */ - private String queryContainerSampleFileRawData(List configurations, Date startDate, - Date endDate, List annotationGroups, - boolean showExcluded) + private String queryContainerSampleFileRawData(List configurations) { // Copy so that we can use our preferred sort configurations = new ArrayList<>(configurations); @@ -166,44 +130,43 @@ private String queryContainerSampleFileRawData(List confi { if (alreadyAdded.add(Pair.of(configuration.getId(), 1))) { - sql.append(sep).append(getEachSeriesTypePlotDataSql(configuration, annotationGroups)); + sql.append(sep).append(getEachSeriesTypePlotDataSql(configuration)); } sep = "\nUNION ALL\n"; } - sql.append(") X"); - sql.append("\nINNER JOIN SampleFile sf ON X.SampleFileId = sf.Id"); - if (null != startDate || null != endDate) + return sql.toString(); + } + + /** + * We cache all precursor-scoped metrics in targetedms.QCMetricCache for performance. + * Run-scoped metrics are not cached like this as they are fast enough to query directly from the backing tables + * like targetedms.SampleFile and targetedms.QCTraceMetricValues. + */ + public void cachePrecursorMetricValues(TargetedMSSchema schema) + { + SQLFragment existingSql = new SQLFragment("SELECT Container FROM ").append(TargetedMSManager.getTableInfoQCMetricCache(), "c").append(" WHERE Container = ?").add(schema.getContainer()); + if (!new SqlSelector(schema.getDbSchema(), existingSql).exists()) { - var sqlSeparator = "WHERE"; + List allMetrics = TargetedMSManager.getAllQCMetricConfigurations(schema); + List precursorMetrics = allMetrics.stream() + .filter(QCMetricConfiguration::isPrecursorScoped) + .toList(); - if (null != startDate) - { - sql.append("\n").append(sqlSeparator); - sql.append(" sf.AcquiredTime >= '"); - sql.append(startDate); - sql.append("' "); - sqlSeparator = "AND"; - } + String computeAllSql = queryContainerSampleFileRawData(precursorMetrics); + TableInfo tiAll = QueryService.get().createTable(schema, computeAllSql, null, true); - if (null != endDate) + try (var ignored = SpringActionController.ignoreSqlUpdates()) { - sql.append("\n").append(sqlSeparator); - sql.append("\n sf.AcquiredTime < TIMESTAMPADD('SQL_TSI_DAY', 1, CAST('"); - sql.append(endDate); - sql.append("' AS TIMESTAMP))"); + SQLFragment insertAll = new SQLFragment(); + insertAll.append("INSERT INTO "); + insertAll.append(TargetedMSManager.getTableInfoQCMetricCache()).append(" (Container, MetricId, PrecursorChromInfoId, SampleFileId, MetricValue, SeriesLabel) "); + insertAll.append(" SELECT ?, lk.MetricId, lk.PrecursorChromInfoId, lk.SampleFileId, lk.MetricValue, lk.SeriesLabel FROM "); + insertAll.append(tiAll, "lk"); + + new SqlExecutor(TargetedMSManager.getSchema()).execute(insertAll); } } - else - { - sql.append("\nWHERE sf.AcquiredTime IS NOT NULL"); - } - if (!showExcluded) - { - sql.append(" AND sf.Excluded = false"); - } - - return sql.toString(); } public List getRawMetricDataSets(TargetedMSSchema schema, List configurations, Date startDate, Date endDate, List annotationGroups, boolean showExcluded, boolean showExcludedPrecursors) @@ -219,55 +182,148 @@ public List getRawMetricDataSets(TargetedMSSchema schema, List sampleFiles.put(sf.getId(), sf); } - String labkeySQL = queryContainerSampleFileRawData(configurations, startDate, endDate, annotationGroups, showExcluded); + // Split configurations into cacheable (precursor-scoped) vs direct-query (run-scoped) + List runScoped = configurations.stream() + .filter(c -> !c.isPrecursorScoped()) + .toList(); + List precursorScoped = configurations.stream() + .filter(QCMetricConfiguration::isPrecursorScoped) + .toList(); - // Use strictColumnList = false to avoid a potentially expensive injected join for the Container via lookups - TableInfo ti = QueryService.get().createTable(schema, labkeySQL, null, true); - - SQLFragment sql = new SQLFragment("SELECT lk.*, pci.PrecursorId "); - sql.append(" FROM "); - sql.append(ti, "lk"); - sql.append(" LEFT OUTER JOIN "); - sql.append(TargetedMSManager.getTableInfoPrecursorChromInfo(), "pci"); - sql.append(" ON lk.PrecursorChromInfoId = pci.Id "); + cachePrecursorMetricValues(schema); + // Load precursor info and metric map + Map excludedPrecursorIds = new LongHashMap<>(); + Map precursors; try { - Map excludedPrecursorIds = new LongHashMap<>(); - Map precursors = loadPrecursors(schema, excludedPrecursorIds, showExcludedPrecursors); + precursors = loadPrecursors(schema, excludedPrecursorIds, showExcludedPrecursors); + } + catch (SQLException e) + { + throw new RuntimeSQLException(e); + } + Map metrics = new HashMap<>(); + configurations.forEach(m -> metrics.put(m.getId(), m)); - Map metrics = new IntHashMap<>(); - configurations.forEach(m -> metrics.put(m.getId(), m)); + // Read requested precursor values from the cache with all the filters + SQLFragment sql = new SQLFragment(); + sql.append("SELECT x.*, pci.PrecursorId FROM ("); - try (ResultSet rs = new SqlSelector(TargetedMSManager.getSchema(), sql).getResultSet(false)) + String separator = ""; + if (!precursorScoped.isEmpty()) + { + sql.append("SELECT c.PrecursorChromInfoId, c.SampleFileId, c.SeriesLabel, c.MetricValue, c.MetricId "); + sql.append(" FROM "); + sql.append(TargetedMSManager.getTableInfoQCMetricCache(), "c"); + sql.append(" WHERE c.Container = ?\n"); + sql.add(schema.getContainer()); + sql.append(" AND c.MetricId IN ("); + sql.append(StringUtils.repeat("?", ",", precursorScoped.size())); + sql.addAll(precursorScoped.stream().map(QCMetricConfiguration::getId).toList()); + sql.append(")"); + separator = "\nUNION ALL\n"; + } + + if (!runScoped.isEmpty()) + { + sql.append(separator); + String runScopedLabKeySql = queryContainerSampleFileRawData(runScoped); + TableInfo ti = QueryService.get().createTable(schema, runScopedLabKeySql, null, true); + sql.append("SELECT lk.* "); + sql.append(" FROM "); + sql.append(ti, "lk"); + } + + sql.append(") x "); + + sql.append(" LEFT OUTER JOIN "); + sql.append(TargetedMSManager.getTableInfoPrecursorChromInfo(), "pci"); + sql.append(" ON x.PrecursorChromInfoId = pci.Id "); + sql.append(" INNER JOIN "); + sql.append(TargetedMSManager.getTableInfoSampleFile(), "sf"); + sql.append(" ON x.SampleFileId = sf.Id "); + + if (null != startDate || null != endDate) + { + if (null != startDate) { - while (rs.next()) - { - long sampleFileId = rs.getLong("SampleFileId"); - Long precursorId = getLong(rs, "PrecursorId"); + sql.append(" AND sf.AcquiredTime >= ?"); + sql.add(startDate); + } + if (null != endDate) + { + sql.append(" AND sf.AcquiredTime < ?"); + // Add one day to be exclusive upper bound, mimicking TIMESTAMPADD('SQL_TSI_DAY', 1, endDate) + sql.add(new Date(endDate.getTime() + 24L * 60L * 60L * 1000L)); + } + } + else + { + sql.append(" AND sf.AcquiredTime IS NOT NULL"); + } + if (!showExcluded) + { + sql.append(" AND sf.ReplicateId NOT IN (SELECT ReplicateId FROM "); + sql.append(TargetedMSManager.getTableInfoQCMetricExclusion(), "x"); + sql.append(" WHERE x.MetricId IS NULL)"); + } - if (excludedPrecursorIds.containsKey(precursorId)) - continue; + if (!annotationGroups.isEmpty()) + { + sql.append(" AND sf.ReplicateId IN ("); + String intersect = ""; + for (AnnotationGroup annotation : annotationGroups) + { + sql.append(intersect).append(" SELECT ReplicateId FROM ") + .append(TargetedMSManager.getTableInfoReplicateAnnotation(), "ra").append(" WHERE ra.Name = ?"); + sql.add(annotation.getName()); - // Sample-scoped metrics won't have an associated precursor - RawMetricDataSet.PrecursorInfo precursor = null; - if (precursorId != null) + List vals = annotation.getValues(); + if (!vals.isEmpty()) + { + sql.append(" AND ra.Value IN ("); + String vsep = ""; + for (String v : vals) { - precursor = precursors.get(precursorId); - if (precursor == null) - { - throw new IllegalStateException("Could not find Precursor with Id " + precursorId); - } + sql.append(vsep).append("?"); + sql.add(v); + vsep = ","; } + sql.append(")"); + } + intersect = " INTERSECT "; + } + sql.append(")"); + } + + try (ResultSet rs = new SqlSelector(TargetedMSManager.getSchema(), sql).getResultSet(false)) + { + while (rs.next()) + { + int metricId = rs.getInt("MetricId"); + long sampleFileId = rs.getLong("SampleFileId"); + Long precursorId = getLong(rs, "PrecursorId"); - RawMetricDataSet row = new RawMetricDataSet(sampleFiles.get(sampleFileId), precursor); + if (excludedPrecursorIds.containsKey(precursorId)) + continue; - row.setMetric(metrics.get(rs.getInt("MetricId"))); // this datarow is not setting the correct metric - row.setSeriesLabel(rs.getString("SeriesLabel")); - row.setPrecursorChromInfoId(getLong(rs, "PrecursorChromInfoId")); - row.setMetricValue(getDouble(rs, "MetricValue")); - result.add(row); + RawMetricDataSet.PrecursorInfo precursor = null; + if (precursorId != null) + { + precursor = precursors.get(precursorId); + if (precursor == null) + { + throw new IllegalStateException("Could not find Precursor with Id " + precursorId); + } } + + RawMetricDataSet row = new RawMetricDataSet(sampleFiles.get(sampleFileId), precursor); + row.setMetric(metrics.get(metricId)); + row.setSeriesLabel(rs.getString("SeriesLabel")); + row.setPrecursorChromInfoId(getLong(rs, "PrecursorChromInfoId")); + row.setMetricValue(getDouble(rs, "MetricValue")); + result.add(row); } } catch (SQLException e) diff --git a/src/org/labkey/targetedms/pipeline/TargetedMSImportTask.java b/src/org/labkey/targetedms/pipeline/TargetedMSImportTask.java index 50ebc69e7..20a07cff8 100644 --- a/src/org/labkey/targetedms/pipeline/TargetedMSImportTask.java +++ b/src/org/labkey/targetedms/pipeline/TargetedMSImportTask.java @@ -83,7 +83,7 @@ public Factory() } @Override - public PipelineJob.Task createTask(PipelineJob job) + public TargetedMSImportTask createTask(PipelineJob job) { return new TargetedMSImportTask(this, job); } diff --git a/src/org/labkey/targetedms/query/QCEnabledMetricsTable.java b/src/org/labkey/targetedms/query/QCEnabledMetricsTable.java index f3bdddfdd..f87565135 100644 --- a/src/org/labkey/targetedms/query/QCEnabledMetricsTable.java +++ b/src/org/labkey/targetedms/query/QCEnabledMetricsTable.java @@ -77,7 +77,7 @@ public QueryUpdateService getUpdateService() @Override protected Map _insert(User user, Container c, Map row) throws SQLException, ValidationException { - TargetedMSManager.get().clearCachedEnabledQCMetrics(c); + TargetedMSManager.get().clearQCMetricCache(c, false); validateBounds(row); return super._insert(user, c, row); } @@ -125,7 +125,7 @@ private static void validateBounds(Map row) throws ValidationExc @Override protected Map _update(User user, Container c, Map row, Map oldRow, Object[] keys) throws SQLException, ValidationException { - TargetedMSManager.get().clearCachedEnabledQCMetrics(c); + TargetedMSManager.get().clearQCMetricCache(c, false); validateBounds(row); return super._update(user, c, row, oldRow, keys); } @@ -133,7 +133,7 @@ protected Map _update(User user, Container c, Map row) throws InvalidKeyException { - TargetedMSManager.get().clearCachedEnabledQCMetrics(c); + TargetedMSManager.get().clearQCMetricCache(c, false); super._delete(c, row); } }; diff --git a/src/org/labkey/targetedms/query/QCMetricConfigurationTable.java b/src/org/labkey/targetedms/query/QCMetricConfigurationTable.java index 78cd35ae4..5710740af 100644 --- a/src/org/labkey/targetedms/query/QCMetricConfigurationTable.java +++ b/src/org/labkey/targetedms/query/QCMetricConfigurationTable.java @@ -106,7 +106,7 @@ protected QCMetricConfigurationTableUpdateService(QCMetricConfigurationTable que protected Map insertRow(User user, Container container, Map row) throws DuplicateKeyException, ValidationException, QueryUpdateServiceException, SQLException { var insertedRow = super.insertRow(user, container, row); - TargetedMSManager.get().clearCachedEnabledQCMetrics(container); + TargetedMSManager.get().clearQCMetricCache(container, true); calculateAndInsertTraceValuesForMetric(asInteger(insertedRow.get("Id")), container, user); return insertedRow; } @@ -117,7 +117,7 @@ protected Map updateRow(User user, Container container, Map updateRow(User user, Container container, Map deleteRow(User user, Container container, Map oldRow) throws InvalidKeyException, QueryUpdateServiceException, SQLException { - TargetedMSManager.get().clearCachedEnabledQCMetrics(container); + TargetedMSManager.get().clearQCMetricCache(container, true); deleteTraceValueForMetric(asInteger(oldRow.get("id")), container); return super.deleteRow(user, container, oldRow); } From 3de5b93a56c9875f3e6d1b80c5bfe5e1c978d057 Mon Sep 17 00:00:00 2001 From: labkey-jeckels Date: Wed, 15 Oct 2025 16:38:06 -0700 Subject: [PATCH 02/15] Remove enabledQueryName --- .../model/QCMetricConfiguration.java | 14 --- ...cEnabled_IsotopologuePrecursorAccuracy.sql | 1 - ...MetricEnabled_IsotopologuePrecursorLOD.sql | 1 - ...MetricEnabled_IsotopologuePrecursorLOQ.sql | 1 - ...cEnabled_IsotopologuePrecursorRSquared.sql | 1 - .../QCMetricEnabled_isotopeDotp.sql | 1 - .../targetedms/QCMetricEnabled_lhRatio.sql | 1 - .../QCMetricEnabled_libraryDotp.sql | 1 - .../QCMetricEnabled_massErrorPrecursor.sql | 1 - .../QCMetricEnabled_massErrorTransition.sql | 1 - ...ricEnabled_precursorAndTransitionAreas.sql | 20 ----- .../QCMetricEnabled_precursorArea.sql | 20 ----- .../QCMetricEnabled_transitionArea.sql | 48 ---------- .../QCRunMetricEnabled_iRTCorrelation.sql | 1 - .../QCRunMetricEnabled_iRTIntercept.sql | 1 - .../QCRunMetricEnabled_iRTSlope.sql | 1 - .../targetedms/QCRunMetricEnabled_ticArea.sql | 17 ---- .../queries/targetedms/qcMetricsConfig.sql | 1 - .../postgresql/targetedms-25.005-25.006.sql | 4 +- resources/schemas/targetedms.xml | 1 - .../window/AddNewMetricWindow.js | 26 ------ .../labkey/targetedms/TargetedMSManager.java | 71 +++++++-------- .../targetedms/outliers/OutlierGenerator.java | 87 ++++++++++--------- .../ConfigureMetricsUIPage.java | 3 +- .../TargetedMSIsotopologueTest.java | 2 +- webapp/TargetedMS/js/QCTrendPlotPanel.js | 4 +- 26 files changed, 84 insertions(+), 246 deletions(-) delete mode 100644 resources/queries/targetedms/QCMetricEnabled_IsotopologuePrecursorAccuracy.sql delete mode 100644 resources/queries/targetedms/QCMetricEnabled_IsotopologuePrecursorLOD.sql delete mode 100644 resources/queries/targetedms/QCMetricEnabled_IsotopologuePrecursorLOQ.sql delete mode 100644 resources/queries/targetedms/QCMetricEnabled_IsotopologuePrecursorRSquared.sql delete mode 100644 resources/queries/targetedms/QCMetricEnabled_isotopeDotp.sql delete mode 100644 resources/queries/targetedms/QCMetricEnabled_lhRatio.sql delete mode 100644 resources/queries/targetedms/QCMetricEnabled_libraryDotp.sql delete mode 100644 resources/queries/targetedms/QCMetricEnabled_massErrorPrecursor.sql delete mode 100644 resources/queries/targetedms/QCMetricEnabled_massErrorTransition.sql delete mode 100644 resources/queries/targetedms/QCMetricEnabled_precursorAndTransitionAreas.sql delete mode 100644 resources/queries/targetedms/QCMetricEnabled_precursorArea.sql delete mode 100644 resources/queries/targetedms/QCMetricEnabled_transitionArea.sql delete mode 100644 resources/queries/targetedms/QCRunMetricEnabled_iRTCorrelation.sql delete mode 100644 resources/queries/targetedms/QCRunMetricEnabled_iRTIntercept.sql delete mode 100644 resources/queries/targetedms/QCRunMetricEnabled_iRTSlope.sql delete mode 100644 resources/queries/targetedms/QCRunMetricEnabled_ticArea.sql diff --git a/api-src/org/labkey/api/targetedms/model/QCMetricConfiguration.java b/api-src/org/labkey/api/targetedms/model/QCMetricConfiguration.java index 707128d44..73d10893e 100644 --- a/api-src/org/labkey/api/targetedms/model/QCMetricConfiguration.java +++ b/api-src/org/labkey/api/targetedms/model/QCMetricConfiguration.java @@ -24,7 +24,6 @@ public class QCMetricConfiguration implements Comparable private String _name; private String _queryName; private boolean _precursorScoped; - private String _enabledQueryName; private QCMetricStatus _status; private String _traceName; private Double _minTimeValue; @@ -75,16 +74,6 @@ public void setPrecursorScoped(boolean precursorScoped) _precursorScoped = precursorScoped; } - public String getEnabledQueryName() - { - return _enabledQueryName; - } - - public void setEnabledQueryName(String enabledQueryName) - { - _enabledQueryName = enabledQueryName; - } - public QCMetricStatus getStatus() { return _status; @@ -182,9 +171,6 @@ public JSONObject toJSON(){ jsonObject.put("queryName", _queryName); jsonObject.put("precursorScoped", _precursorScoped); jsonObject.put("metricStatus", getStatus() == null ? QCMetricStatus.DEFAULT.toString() : getStatus().toString()); - if (_enabledQueryName != null) { - jsonObject.put("enabledQueryName", _enabledQueryName); - } if (_traceName != null) { jsonObject.put("traceName", _traceName); } diff --git a/resources/queries/targetedms/QCMetricEnabled_IsotopologuePrecursorAccuracy.sql b/resources/queries/targetedms/QCMetricEnabled_IsotopologuePrecursorAccuracy.sql deleted file mode 100644 index f35643bdb..000000000 --- a/resources/queries/targetedms/QCMetricEnabled_IsotopologuePrecursorAccuracy.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT 1 AS E WHERE EXISTS (SELECT Id FROM targetedms.PrecursorChromInfoAnnotation WHERE Name = 'PrecursorAccuracy') \ No newline at end of file diff --git a/resources/queries/targetedms/QCMetricEnabled_IsotopologuePrecursorLOD.sql b/resources/queries/targetedms/QCMetricEnabled_IsotopologuePrecursorLOD.sql deleted file mode 100644 index 65dcf6b0a..000000000 --- a/resources/queries/targetedms/QCMetricEnabled_IsotopologuePrecursorLOD.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT 1 AS E WHERE EXISTS (SELECT Id FROM targetedms.PrecursorChromInfoAnnotation WHERE Name = 'LOD') \ No newline at end of file diff --git a/resources/queries/targetedms/QCMetricEnabled_IsotopologuePrecursorLOQ.sql b/resources/queries/targetedms/QCMetricEnabled_IsotopologuePrecursorLOQ.sql deleted file mode 100644 index c49c4f0f7..000000000 --- a/resources/queries/targetedms/QCMetricEnabled_IsotopologuePrecursorLOQ.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT 1 AS E WHERE EXISTS (SELECT Id FROM targetedms.PrecursorChromInfoAnnotation WHERE Name = 'LOQ') \ No newline at end of file diff --git a/resources/queries/targetedms/QCMetricEnabled_IsotopologuePrecursorRSquared.sql b/resources/queries/targetedms/QCMetricEnabled_IsotopologuePrecursorRSquared.sql deleted file mode 100644 index a392b3fc4..000000000 --- a/resources/queries/targetedms/QCMetricEnabled_IsotopologuePrecursorRSquared.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT 1 AS E WHERE EXISTS (SELECT Id FROM targetedms.PrecursorChromInfoAnnotation WHERE Name = 'RSquared') \ No newline at end of file diff --git a/resources/queries/targetedms/QCMetricEnabled_isotopeDotp.sql b/resources/queries/targetedms/QCMetricEnabled_isotopeDotp.sql deleted file mode 100644 index 1fbdec14c..000000000 --- a/resources/queries/targetedms/QCMetricEnabled_isotopeDotp.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT 1 AS E WHERE EXISTS (SELECT Id FROM targetedms.PrecursorChromInfo WHERE IsotopeDotp IS NOT NULL) \ No newline at end of file diff --git a/resources/queries/targetedms/QCMetricEnabled_lhRatio.sql b/resources/queries/targetedms/QCMetricEnabled_lhRatio.sql deleted file mode 100644 index dfa290189..000000000 --- a/resources/queries/targetedms/QCMetricEnabled_lhRatio.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT 1 AS E WHERE EXISTS (SELECT Id FROM targetedms.PrecursorAreaRatio) \ No newline at end of file diff --git a/resources/queries/targetedms/QCMetricEnabled_libraryDotp.sql b/resources/queries/targetedms/QCMetricEnabled_libraryDotp.sql deleted file mode 100644 index 47cb1422f..000000000 --- a/resources/queries/targetedms/QCMetricEnabled_libraryDotp.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT 1 AS E WHERE EXISTS (SELECT Id FROM targetedms.PrecursorChromInfo WHERE LibraryDotp IS NOT NULL) \ No newline at end of file diff --git a/resources/queries/targetedms/QCMetricEnabled_massErrorPrecursor.sql b/resources/queries/targetedms/QCMetricEnabled_massErrorPrecursor.sql deleted file mode 100644 index 4b935593a..000000000 --- a/resources/queries/targetedms/QCMetricEnabled_massErrorPrecursor.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT 1 AS E WHERE EXISTS (SELECT Id FROM targetedms.TransitionChromInfo WHERE MassErrorPPM IS NOT NULL AND TransitionId.Charge IS NULL) \ No newline at end of file diff --git a/resources/queries/targetedms/QCMetricEnabled_massErrorTransition.sql b/resources/queries/targetedms/QCMetricEnabled_massErrorTransition.sql deleted file mode 100644 index 9fa98891e..000000000 --- a/resources/queries/targetedms/QCMetricEnabled_massErrorTransition.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT 1 AS E WHERE EXISTS (SELECT Id FROM targetedms.TransitionChromInfo WHERE MassErrorPPM IS NOT NULL AND TransitionId.Charge IS NOT NULL) \ No newline at end of file diff --git a/resources/queries/targetedms/QCMetricEnabled_precursorAndTransitionAreas.sql b/resources/queries/targetedms/QCMetricEnabled_precursorAndTransitionAreas.sql deleted file mode 100644 index 0705cc577..000000000 --- a/resources/queries/targetedms/QCMetricEnabled_precursorAndTransitionAreas.sql +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright (c) 2016-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ --- We need both transition and precursor areas -SELECT 1 AS E -WHERE - EXISTS (SELECT E FROM QCMetricEnabled_precursorArea) - AND EXISTS (SELECT E FROM QCMetricEnabled_transitionArea) diff --git a/resources/queries/targetedms/QCMetricEnabled_precursorArea.sql b/resources/queries/targetedms/QCMetricEnabled_precursorArea.sql deleted file mode 100644 index 13474dae1..000000000 --- a/resources/queries/targetedms/QCMetricEnabled_precursorArea.sql +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright (c) 2016-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ --- We need to check both the proteomics and small molecule data. Use two separate EXISTS subqueries so we stop as --- soon as we find any data -SELECT 1 AS E WHERE -EXISTS (SELECT Id, FragmentType, Quantitative FROM Transition t WHERE FragmentType = 'precursor' AND Charge IS NULL) -OR EXISTS (SELECT Id, FragmentType, Quantitative FROM MoleculeTransition t WHERE FragmentType = 'precursor' AND Charge IS NULL) \ No newline at end of file diff --git a/resources/queries/targetedms/QCMetricEnabled_transitionArea.sql b/resources/queries/targetedms/QCMetricEnabled_transitionArea.sql deleted file mode 100644 index 34805d959..000000000 --- a/resources/queries/targetedms/QCMetricEnabled_transitionArea.sql +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright (c) 2016-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ --- Approximate the checks in GeneralTransition.isQuantitative() --- We need to check both the proteomics and small molecule data. Use two separate EXISTS subqueries so we stop as --- soon as we find any data -SELECT 1 AS E WHERE EXISTS ( -SELECT Id, FragmentType, Quantitative FROM Transition t - WHERE - (Quantitative = TRUE) OR (Quantitative IS NULL AND - (FragmentType != 'precursor' AND - t.GeneralPrecursorId.GeneralMoleculeId.PeptideGroupId.RunId IN ( - SELECT r.Id - FROM - targetedms.Runs r LEFT OUTER JOIN - targetedms.TransitionFullScanSettings tfss - ON r.Id = tfss.RunId - WHERE AcquisitionMethod IS NULL OR AcquisitionMethod != 'DDA' - )) - ) -) -OR EXISTS ( -SELECT Id, FragmentType, Quantitative FROM MoleculeTransition t -WHERE - (Quantitative = TRUE) OR (Quantitative IS NULL AND - (FragmentType != 'precursor' AND - t.GeneralPrecursorId.GeneralMoleculeId.PeptideGroupId.RunId IN ( - SELECT r.Id - FROM - targetedms.Runs r LEFT OUTER JOIN - targetedms.TransitionFullScanSettings tfss - ON r.Id = tfss.RunId - WHERE AcquisitionMethod IS NULL OR AcquisitionMethod != 'DDA' - )) - ) -) \ No newline at end of file diff --git a/resources/queries/targetedms/QCRunMetricEnabled_iRTCorrelation.sql b/resources/queries/targetedms/QCRunMetricEnabled_iRTCorrelation.sql deleted file mode 100644 index 0b3d07909..000000000 --- a/resources/queries/targetedms/QCRunMetricEnabled_iRTCorrelation.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT Id FROM targetedms.SampleFile WHERE IRTCorrelation IS NOT NULL \ No newline at end of file diff --git a/resources/queries/targetedms/QCRunMetricEnabled_iRTIntercept.sql b/resources/queries/targetedms/QCRunMetricEnabled_iRTIntercept.sql deleted file mode 100644 index 0512c072f..000000000 --- a/resources/queries/targetedms/QCRunMetricEnabled_iRTIntercept.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT Id FROM targetedms.SampleFile WHERE IRTIntercept IS NOT NULL \ No newline at end of file diff --git a/resources/queries/targetedms/QCRunMetricEnabled_iRTSlope.sql b/resources/queries/targetedms/QCRunMetricEnabled_iRTSlope.sql deleted file mode 100644 index 9a6aa8440..000000000 --- a/resources/queries/targetedms/QCRunMetricEnabled_iRTSlope.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT Id FROM targetedms.SampleFile WHERE IRTSlope IS NOT NULL \ No newline at end of file diff --git a/resources/queries/targetedms/QCRunMetricEnabled_ticArea.sql b/resources/queries/targetedms/QCRunMetricEnabled_ticArea.sql deleted file mode 100644 index 68a4ebb49..000000000 --- a/resources/queries/targetedms/QCRunMetricEnabled_ticArea.sql +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright (c) 2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -SELECT Id FROM targetedms.SampleFile WHERE TicArea IS NOT NULL diff --git a/resources/queries/targetedms/qcMetricsConfig.sql b/resources/queries/targetedms/qcMetricsConfig.sql index 2a4ff616d..d1f1248ef 100644 --- a/resources/queries/targetedms/qcMetricsConfig.sql +++ b/resources/queries/targetedms/qcMetricsConfig.sql @@ -20,7 +20,6 @@ SELECT qmc.QueryName, qmc.PrecursorScoped, qmc.Container, -- including to lock out editing pre-configured qc metrics, - qmc.EnabledQueryName, qem.Status, CASE WHEN qem.metric IS NULL THEN FALSE ELSE TRUE END AS Inserted, diff --git a/resources/schemas/dbscripts/postgresql/targetedms-25.005-25.006.sql b/resources/schemas/dbscripts/postgresql/targetedms-25.005-25.006.sql index d117223e6..4e7d0a41d 100644 --- a/resources/schemas/dbscripts/postgresql/targetedms-25.005-25.006.sql +++ b/resources/schemas/dbscripts/postgresql/targetedms-25.005-25.006.sql @@ -16,4 +16,6 @@ CREATE TABLE targetedms.QCMetricCache CREATE INDEX IDX_QCMetricCache_PrecursorChromInfoId ON targetedms.QCMetricCache(PrecursorChromInfoId); CREATE INDEX IDX_QCMetricCache_SampleFileId ON targetedms.QCMetricCache(SampleFileId); -CREATE INDEX IDX_QCMetricCache_MetricId ON targetedms.QCMetricCache(MetricId); \ No newline at end of file +CREATE INDEX IDX_QCMetricCache_MetricId ON targetedms.QCMetricCache(MetricId); + +ALTER TABLE targetedms.QCMetricConfiguration DROP COLUMN EnabledQueryName; \ No newline at end of file diff --git a/resources/schemas/targetedms.xml b/resources/schemas/targetedms.xml index 1c3fbe4ff..6e1a28c3e 100644 --- a/resources/schemas/targetedms.xml +++ b/resources/schemas/targetedms.xml @@ -1349,7 +1349,6 @@ - diff --git a/resources/web/PanoramaPremium/window/AddNewMetricWindow.js b/resources/web/PanoramaPremium/window/AddNewMetricWindow.js index 590553d2d..eba112162 100644 --- a/resources/web/PanoramaPremium/window/AddNewMetricWindow.js +++ b/resources/web/PanoramaPremium/window/AddNewMetricWindow.js @@ -37,7 +37,6 @@ Ext4.define('Panorama.Window.AddCustomMetricWindow', { schemaName: this.SCHEMA_NAME, success: function(queriesInfo) { this.queries = queriesInfo.queries; - this.enabledqueries = queriesInfo.queries; } }); @@ -49,7 +48,6 @@ Ext4.define('Panorama.Window.AddCustomMetricWindow', { this.getQueriesCombo(), this.getMetricTypeCombo(), this.getYAxisLabelField(), - this.getEnabledQueriesCombo(), this.getQueryError(), ]; }, @@ -150,28 +148,6 @@ Ext4.define('Panorama.Window.AddCustomMetricWindow', { return this.yAxisLabelField; }, - getEnabledQueriesCombo: function() { - if(!this.enabledQueriesCombo) { - var config = Ext4.apply(this.getQueriesConfig('Enabled Query', 'enabledQueryName'), { - listeners: { - scope: this, - expand: function (field, options) { - if (this.enabledqueries) { - this.enabledQueriesCombo.bindStore(this.getQueriesStore()); - } - } - } - }); - this.enabledQueriesCombo = Ext4.create('Ext.form.field.ComboBox', config); - - if(this.operation === this.update) { - this.enabledQueriesCombo.setValue(this.metric.EnabledQueryName); - } - } - - return this.enabledQueriesCombo; - }, - getMetricTypeCombo: function() { if(!this.metricTypeCombo) { var metricTypeStore = Ext4.create('Ext.data.Store', { @@ -320,8 +296,6 @@ Ext4.define('Panorama.Window.AddCustomMetricWindow', { newMetric.YAxisLabel = this.yAxisLabelField.getValue(); newMetric.PrecursorScoped = this.metricTypeCombo.getValue(); - newMetric.EnabledQueryName = this.enabledQueriesCombo.getValue(); - if(this.operation === this.update) { newMetric.id = this.metric.id; } diff --git a/src/org/labkey/targetedms/TargetedMSManager.java b/src/org/labkey/targetedms/TargetedMSManager.java index 9a113ff74..e9ef61136 100644 --- a/src/org/labkey/targetedms/TargetedMSManager.java +++ b/src/org/labkey/targetedms/TargetedMSManager.java @@ -162,7 +162,37 @@ private TargetedMSManager() * A cache to make it faster to render QC folders. A number of API calls come from the * client rendering the overview, all of which need to know the enabled configs. */ - private static final Cache> _metricCache = CacheManager.getCache(1000, TimeUnit.HOURS.toMillis(1), "Enabled QC metric configs"); + private static final Cache> _metricCache = CacheManager.getBlockingCache(1000, TimeUnit.HOURS.toMillis(1), "Enabled QC metric configs", + (c, argument) -> + { + TargetedMSSchema schema = (TargetedMSSchema) argument; + TableInfo metricsTable = schema.getTableOrThrow("qcMetricsConfig", null); + SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("Status"), QCMetricStatus.Disabled.toString(), CompareType.NEQ_OR_NULL); + List metrics = new TableSelector(metricsTable, filter, new Sort(FieldKey.fromParts("Name"))).getArrayList(QCMetricConfiguration.class); + + OutlierGenerator.get().cachePrecursorMetricValues(schema, metrics); + + // Identify which precursor-scoped metrics have any cached data in this container + SQLFragment sql = new SQLFragment("SELECT DISTINCT MetricId FROM ("); + sql.append(OutlierGenerator.get().getRawMetricSql(schema, metrics)); + sql.append(") y WHERE MetricValue IS NOT NULL"); + Set cachedMetricIds = new HashSet<>(new SqlSelector(getSchema(), sql).getCollection(Integer.class)); + + for (QCMetricConfiguration metric : metrics) + { + if (!cachedMetricIds.contains(metric.getId())) + { + metric.setStatus(QCMetricStatus.NoData); + } + else if (metric.getStatus() == null) + { + metric.setStatus(QCMetricStatus.DEFAULT); + } + } + // Ensure we get a case-insensitive sort regardless of DB collation + Collections.sort(metrics); + return Collections.unmodifiableList(metrics); + }); public static TargetedMSManager get() { @@ -2353,44 +2383,7 @@ private static Double getValue(Object o) public static List getAllQCMetricConfigurations(TargetedMSSchema schema) { - return _metricCache.get(schema.getContainer(), null, (c, argument) -> - { - TableInfo metricsTable = schema.getTableOrThrow("qcMetricsConfig", null); - SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("Status"), QCMetricStatus.Disabled.toString(), CompareType.NEQ_OR_NULL); - List metrics = new TableSelector(metricsTable, filter, new Sort(FieldKey.fromParts("Name"))).getArrayList(QCMetricConfiguration.class); - - OutlierGenerator.get().cachePrecursorMetricValues(schema); - - // Identify which precursor-scoped metrics have any cached data in this container - SQLFragment existingSql = new SQLFragment("SELECT DISTINCT MetricId FROM "); - existingSql.append(getTableInfoQCMetricCache(), "c"); - existingSql.append(" WHERE c.Container = ?"); - existingSql.add(c.getEntityId()); - Set cachedMetricIds = new HashSet<>(new SqlSelector(getSchema(), existingSql).getCollection(Integer.class)); - - for (QCMetricConfiguration metric : metrics) - { - if (metric.getStatus() == null) - { - if (metric.isPrecursorScoped()) - { - if (!cachedMetricIds.contains(metric.getId())) - { - // Precursor-scoped metrics without cached values have no data in this container - metric.setStatus(QCMetricStatus.NoData); - } - } - // For run-scoped metrics, do not mark as NoData here since they are not cached; leave as DEFAULT - } - if (metric.getStatus() == null) - { - metric.setStatus(QCMetricStatus.DEFAULT); - } - } - // Ensure we get a case-insensitive sort regardless of DB collation - Collections.sort(metrics); - return Collections.unmodifiableList(metrics); - }); + return _metricCache.get(schema.getContainer(), schema, null); } public static List getEnabledQCMetricConfigurations(TargetedMSSchema schema) { diff --git a/src/org/labkey/targetedms/outliers/OutlierGenerator.java b/src/org/labkey/targetedms/outliers/OutlierGenerator.java index 028768e64..e133136ea 100644 --- a/src/org/labkey/targetedms/outliers/OutlierGenerator.java +++ b/src/org/labkey/targetedms/outliers/OutlierGenerator.java @@ -143,12 +143,11 @@ private String queryContainerSampleFileRawData(List confi * Run-scoped metrics are not cached like this as they are fast enough to query directly from the backing tables * like targetedms.SampleFile and targetedms.QCTraceMetricValues. */ - public void cachePrecursorMetricValues(TargetedMSSchema schema) + public void cachePrecursorMetricValues(TargetedMSSchema schema, List allMetrics) { SQLFragment existingSql = new SQLFragment("SELECT Container FROM ").append(TargetedMSManager.getTableInfoQCMetricCache(), "c").append(" WHERE Container = ?").add(schema.getContainer()); if (!new SqlSelector(schema.getDbSchema(), existingSql).exists()) { - List allMetrics = TargetedMSManager.getAllQCMetricConfigurations(schema); List precursorMetrics = allMetrics.stream() .filter(QCMetricConfiguration::isPrecursorScoped) .toList(); @@ -163,7 +162,7 @@ public void cachePrecursorMetricValues(TargetedMSSchema schema) insertAll.append(TargetedMSManager.getTableInfoQCMetricCache()).append(" (Container, MetricId, PrecursorChromInfoId, SampleFileId, MetricValue, SeriesLabel) "); insertAll.append(" SELECT ?, lk.MetricId, lk.PrecursorChromInfoId, lk.SampleFileId, lk.MetricValue, lk.SeriesLabel FROM "); insertAll.append(tiAll, "lk"); - + insertAll.add(schema.getContainer()); new SqlExecutor(TargetedMSManager.getSchema()).execute(insertAll); } } @@ -182,16 +181,6 @@ public List getRawMetricDataSets(TargetedMSSchema schema, List sampleFiles.put(sf.getId(), sf); } - // Split configurations into cacheable (precursor-scoped) vs direct-query (run-scoped) - List runScoped = configurations.stream() - .filter(c -> !c.isPrecursorScoped()) - .toList(); - List precursorScoped = configurations.stream() - .filter(QCMetricConfiguration::isPrecursorScoped) - .toList(); - - cachePrecursorMetricValues(schema); - // Load precursor info and metric map Map excludedPrecursorIds = new LongHashMap<>(); Map precursors; @@ -206,37 +195,9 @@ public List getRawMetricDataSets(TargetedMSSchema schema, List Map metrics = new HashMap<>(); configurations.forEach(m -> metrics.put(m.getId(), m)); - // Read requested precursor values from the cache with all the filters - SQLFragment sql = new SQLFragment(); - sql.append("SELECT x.*, pci.PrecursorId FROM ("); - - String separator = ""; - if (!precursorScoped.isEmpty()) - { - sql.append("SELECT c.PrecursorChromInfoId, c.SampleFileId, c.SeriesLabel, c.MetricValue, c.MetricId "); - sql.append(" FROM "); - sql.append(TargetedMSManager.getTableInfoQCMetricCache(), "c"); - sql.append(" WHERE c.Container = ?\n"); - sql.add(schema.getContainer()); - sql.append(" AND c.MetricId IN ("); - sql.append(StringUtils.repeat("?", ",", precursorScoped.size())); - sql.addAll(precursorScoped.stream().map(QCMetricConfiguration::getId).toList()); - sql.append(")"); - separator = "\nUNION ALL\n"; - } - - if (!runScoped.isEmpty()) - { - sql.append(separator); - String runScopedLabKeySql = queryContainerSampleFileRawData(runScoped); - TableInfo ti = QueryService.get().createTable(schema, runScopedLabKeySql, null, true); - sql.append("SELECT lk.* "); - sql.append(" FROM "); - sql.append(ti, "lk"); - } - + SQLFragment sql = new SQLFragment("SELECT x.*, pci.PrecursorId FROM ("); + sql.append(getRawMetricSql(schema, configurations)); sql.append(") x "); - sql.append(" LEFT OUTER JOIN "); sql.append(TargetedMSManager.getTableInfoPrecursorChromInfo(), "pci"); sql.append(" ON x.PrecursorChromInfoId = pci.Id "); @@ -337,6 +298,46 @@ public List getRawMetricDataSets(TargetedMSSchema schema, List return result; } + public SQLFragment getRawMetricSql(TargetedMSSchema schema, List configurations) + { + // Split configurations into cacheable (precursor-scoped) vs direct-query (run-scoped) + List runScoped = configurations.stream() + .filter(c -> !c.isPrecursorScoped()) + .toList(); + List precursorScoped = configurations.stream() + .filter(QCMetricConfiguration::isPrecursorScoped) + .toList(); + + // Read requested precursor values from the cache with all the filters + SQLFragment sql = new SQLFragment(); + + String separator = ""; + if (!precursorScoped.isEmpty()) + { + sql.append("SELECT c.PrecursorChromInfoId, c.SampleFileId, c.SeriesLabel, c.MetricValue, c.MetricId "); + sql.append(" FROM "); + sql.append(TargetedMSManager.getTableInfoQCMetricCache(), "c"); + sql.append(" WHERE c.Container = ?\n"); + sql.add(schema.getContainer()); + sql.append(" AND c.MetricId IN ("); + sql.append(StringUtils.repeat("?", ",", precursorScoped.size())); + sql.addAll(precursorScoped.stream().map(QCMetricConfiguration::getId).toList()); + sql.append(")"); + separator = "\nUNION ALL\n"; + } + + if (!runScoped.isEmpty()) + { + sql.append(separator); + String runScopedLabKeySql = queryContainerSampleFileRawData(runScoped); + TableInfo ti = QueryService.get().createTable(schema, runScopedLabKeySql, null, true); + sql.append("SELECT lk.* "); + sql.append(" FROM "); + sql.append(ti, "lk"); + } + return sql; + } + /** * Fetch all the precursors in this folder. Loaded separately from the metric values because a given precursor will * have many metrics, so for DB query and Java memory use it's more efficient to not flatten them into a single diff --git a/test/src/org/labkey/test/pages/panoramapremium/ConfigureMetricsUIPage.java b/test/src/org/labkey/test/pages/panoramapremium/ConfigureMetricsUIPage.java index abb1555c0..483e4ef6f 100644 --- a/test/src/org/labkey/test/pages/panoramapremium/ConfigureMetricsUIPage.java +++ b/test/src/org/labkey/test/pages/panoramapremium/ConfigureMetricsUIPage.java @@ -201,8 +201,7 @@ public enum CustomMetricProperties metricName("Name", false), queryName("Metrics Query", true), yAxisLabel("Y-Axis Label", false), - metricType("Metric Type", true), - enabledQueryName("Enabled Query", true); + metricType("Metric Type", true); private final String formLabel; private final boolean isSelect; diff --git a/test/src/org/labkey/test/tests/panoramapremium/TargetedMSIsotopologueTest.java b/test/src/org/labkey/test/tests/panoramapremium/TargetedMSIsotopologueTest.java index 5f9208cf4..1934fac00 100644 --- a/test/src/org/labkey/test/tests/panoramapremium/TargetedMSIsotopologueTest.java +++ b/test/src/org/labkey/test/tests/panoramapremium/TargetedMSIsotopologueTest.java @@ -69,7 +69,7 @@ public void testIsotopologueMetric() log("Verifying that two new metric properties are added"); clickButton("Add New Custom Metric", 0); Window metricWindow = new Window.WindowFinder(getDriver()).withTitle("Add New Metric").waitFor(); - assertElementPresent(Locator.name(ConfigureMetricsUIPage.CustomMetricProperties.enabledQueryName.name())); + assertElementPresent(Locator.name(ConfigureMetricsUIPage.CustomMetricProperties.metricName.name())); } } diff --git a/webapp/TargetedMS/js/QCTrendPlotPanel.js b/webapp/TargetedMS/js/QCTrendPlotPanel.js index 8303fa15c..f22fbb0b9 100644 --- a/webapp/TargetedMS/js/QCTrendPlotPanel.js +++ b/webapp/TargetedMS/js/QCTrendPlotPanel.js @@ -1607,7 +1607,9 @@ Ext4.define('LABKEY.targetedms.QCTrendPlotPanel', { toggleGuideSetMsgDisplay : function() { var toolbarMsg = this.down('#GuideSetMessageToolBar'); - toolbarMsg.up('toolbar').setVisible(this.enableBrushing); + if (toolbarMsg) { + toolbarMsg.up('toolbar').setVisible(this.enableBrushing); + } }, highlightOutliersForClickedReplicate: function(plot, precursorInfo, replicateId) { From 07b1535f553b76d59451efe0628096bccc259a40 Mon Sep 17 00:00:00 2001 From: labkey-jeckels Date: Wed, 15 Oct 2025 17:02:01 -0700 Subject: [PATCH 03/15] Manual cache clearing option --- resources/views/configureQCMetric.html | 18 +++++++++++++++--- .../targetedms/TargetedMSController.java | 11 +++++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/resources/views/configureQCMetric.html b/resources/views/configureQCMetric.html index d0880f74f..f32b54904 100644 --- a/resources/views/configureQCMetric.html +++ b/resources/views/configureQCMetric.html @@ -57,12 +57,13 @@ qcMetricsTable += ''; }); - qcMetricsTable += '' + '
' + + qcMetricsTable += '
' + '' + '' + '' + - '' + - ''; + '' + + '' + + '
Edits to queries backing existing custom metrics require a manual cache clearing to display the updated results.

'; jQuery('#qcMetricsTable').html(qcMetricsTable); @@ -80,6 +81,17 @@ jQuery('#createNewTraceMetricButton').click(function() { LABKEY.internal.ConfigureQCMetrics.addNewMetric('trace') }); + jQuery('#clearCacheButton').click(function() { + jQuery('#qcMetricsError').text('Clearing cached metrics...'); + LABKEY.Ajax.request({ + url: LABKEY.ActionURL.buildURL('targetedms', 'clearQCMetricsCache.api'), + method: 'POST', + success: function() { + jQuery('#qcMetricsError').text('Cleared cached metrics.'); + }, + failure: LABKEY.Utils.getCallbackWrapper(LABKEY.internal.ConfigureQCMetrics.onError, this, true) + }); + }); jQuery.each(qcMetrics, function (index, row) { jQuery('#editLink' + row.id).click(function (e) { diff --git a/src/org/labkey/targetedms/TargetedMSController.java b/src/org/labkey/targetedms/TargetedMSController.java index 11b76b13f..a376752e8 100644 --- a/src/org/labkey/targetedms/TargetedMSController.java +++ b/src/org/labkey/targetedms/TargetedMSController.java @@ -8159,6 +8159,17 @@ public long getId() } } + @RequiresPermission(AdminPermission.class) + public static class ClearQCMetricsCacheAction extends MutatingApiAction + { + @Override + public Object execute(Object form, BindException errors) + { + TargetedMSManager.get().clearQCMetricCache(getContainer(), true); + return new ApiSimpleResponse("success", true); + } + } + public static class TargetedMSUrlsImpl implements TargetedMSUrls { public static TargetedMSUrlsImpl get() From 1bccab151bee209b6eaa6b65afc5d0d20c453d3b Mon Sep 17 00:00:00 2001 From: labkey-jeckels Date: Thu, 16 Oct 2025 15:27:53 -0700 Subject: [PATCH 04/15] Fix trace metric creation and selective samplefile delete --- .../labkey/targetedms/TargetedMSManager.java | 10 +++++--- .../query/QCMetricConfigurationTable.java | 24 +++++++++++++------ 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/src/org/labkey/targetedms/TargetedMSManager.java b/src/org/labkey/targetedms/TargetedMSManager.java index e9ef61136..99251c6d0 100644 --- a/src/org/labkey/targetedms/TargetedMSManager.java +++ b/src/org/labkey/targetedms/TargetedMSManager.java @@ -1637,7 +1637,7 @@ public static String getSampleFileUploadFile(long sampleFileId) /** * @return the file paths of the Skyline documents containing the given sample files */ - @Nullable + @NotNull public static List deleteSampleFilesAndDependencies(List sampleFileIds) { purgeDeletedSampleFiles(sampleFileIds); @@ -1684,6 +1684,10 @@ public static void purgeDeletedSampleFiles(List sampleFileIds) // Delete from PrecursorAreaRatio (dependent of PrecursorChromInfo) execute(getTempChromInfoIdsDependentDeleteSql(getTableInfoPrecursorAreaRatio(), "PrecursorChromInfoId", precursorChromInfoIdsTempTableName)); + // Delete from QCMetricCache (dependent of PrecursorChromInfo and SampleFile) + execute(getTempChromInfoIdsDependentDeleteSql(getTableInfoQCMetricCache(), "PrecursorChromInfoId", precursorChromInfoIdsTempTableName)); + execute(new SQLFragment("DELETE FROM ").append(getTableInfoQCMetricCache()).append(whereClause)); + // Delete from PrecursorChromInfo execute(getTempChromInfoIdsDependentDeleteSql(getTableInfoPrecursorChromInfo(), "Id", precursorChromInfoIdsTempTableName)); @@ -2891,10 +2895,10 @@ else if (traceValue != null && values[i] >= traceValue) public static List getTraceMetricConfigurations(Container container, User user) { - return getEnabledQCMetricConfigurations(new TargetedMSSchema(user, container)) + return getAllQCMetricConfigurations(new TargetedMSSchema(user, container)) .stream() .filter(qcMetricConfiguration -> qcMetricConfiguration.getTraceName() != null) - .collect(Collectors.toList()); + .toList(); } public static List getKeywords(long sequenceId) diff --git a/src/org/labkey/targetedms/query/QCMetricConfigurationTable.java b/src/org/labkey/targetedms/query/QCMetricConfigurationTable.java index 5710740af..78ab3b2d7 100644 --- a/src/org/labkey/targetedms/query/QCMetricConfigurationTable.java +++ b/src/org/labkey/targetedms/query/QCMetricConfigurationTable.java @@ -44,7 +44,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; import static org.labkey.api.exp.api.ExperimentService.asInteger; @@ -106,7 +105,6 @@ protected QCMetricConfigurationTableUpdateService(QCMetricConfigurationTable que protected Map insertRow(User user, Container container, Map row) throws DuplicateKeyException, ValidationException, QueryUpdateServiceException, SQLException { var insertedRow = super.insertRow(user, container, row); - TargetedMSManager.get().clearQCMetricCache(container, true); calculateAndInsertTraceValuesForMetric(asInteger(insertedRow.get("Id")), container, user); return insertedRow; } @@ -117,7 +115,6 @@ protected Map updateRow(User user, Container container, Map updateRow(User user, Container container, Map deleteRow(User user, Container container, Map oldRow) throws InvalidKeyException, QueryUpdateServiceException, SQLException { - TargetedMSManager.get().clearQCMetricCache(container, true); deleteTraceValueForMetric(asInteger(oldRow.get("id")), container); - return super.deleteRow(user, container, oldRow); + Map result = super.deleteRow(user, container, oldRow); + TargetedMSManager.get().clearQCMetricCache(container, true); + return result; } private void deleteTraceValueForMetric(int metricId, Container container) @@ -144,11 +142,20 @@ private void deleteTraceValueForMetric(int metricId, Container container) private void calculateAndInsertTraceValuesForMetric(int metricId, Container container, User user) { + // Make sure that we pick up the new metric + TargetedMSManager.get().clearQCMetricCache(container, true); + var qcMetricConfigurations = TargetedMSManager - .getEnabledQCMetricConfigurations(_queryTable.getUserSchema()) + .getAllQCMetricConfigurations(_queryTable.getUserSchema()) .stream() .filter(qcMetricConfiguration -> qcMetricConfiguration.getId() == metricId) - .collect(Collectors.toList()); + .toList(); + + if (qcMetricConfigurations.isEmpty()) + { + throw new IllegalStateException("No QCMetricConfiguration found for Id " + metricId); + } + var runsInContainer = TargetedMSManager.getRunsInContainer(container); for (TargetedMSRun run : runsInContainer) @@ -156,6 +163,9 @@ private void calculateAndInsertTraceValuesForMetric(int metricId, Container cont var qcTraceMetricValues = TargetedMSManager.calculateTraceMetricValues(qcMetricConfigurations, run); qcTraceMetricValues.forEach(qcTraceMetricValue -> Table.insert(user, TargetedMSManager.getTableQCTraceMetricValues(), qcTraceMetricValue)); } + + // Clear again so that we have the right state for the presence/absence of data + TargetedMSManager.get().clearQCMetricCache(container, false); } } } From b1f7a9f83fd11f52ee6f048b47c35ccad217f5c4 Mon Sep 17 00:00:00 2001 From: labkey-jeckels Date: Sat, 18 Oct 2025 07:25:25 -0700 Subject: [PATCH 05/15] Fix filtering --- src/org/labkey/targetedms/TargetedMSManager.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/org/labkey/targetedms/TargetedMSManager.java b/src/org/labkey/targetedms/TargetedMSManager.java index 99251c6d0..a71091542 100644 --- a/src/org/labkey/targetedms/TargetedMSManager.java +++ b/src/org/labkey/targetedms/TargetedMSManager.java @@ -167,8 +167,7 @@ private TargetedMSManager() { TargetedMSSchema schema = (TargetedMSSchema) argument; TableInfo metricsTable = schema.getTableOrThrow("qcMetricsConfig", null); - SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("Status"), QCMetricStatus.Disabled.toString(), CompareType.NEQ_OR_NULL); - List metrics = new TableSelector(metricsTable, filter, new Sort(FieldKey.fromParts("Name"))).getArrayList(QCMetricConfiguration.class); + List metrics = new TableSelector(metricsTable, null, new Sort(FieldKey.fromParts("Name"))).getArrayList(QCMetricConfiguration.class); OutlierGenerator.get().cachePrecursorMetricValues(schema, metrics); From 6dd9d520bacb7a36f6e95eb784c16b6adb1a2bc3 Mon Sep 17 00:00:00 2001 From: labkey-jeckels Date: Sat, 18 Oct 2025 07:33:47 -0700 Subject: [PATCH 06/15] Fix empty Pareto plot --- webapp/TargetedMS/js/ParetoPlotPanel.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapp/TargetedMS/js/ParetoPlotPanel.js b/webapp/TargetedMS/js/ParetoPlotPanel.js index fd96d4f9d..2f6d6152c 100644 --- a/webapp/TargetedMS/js/ParetoPlotPanel.js +++ b/webapp/TargetedMS/js/ParetoPlotPanel.js @@ -39,7 +39,7 @@ Ext4.define('LABKEY.targetedms.ParetoPlotPanel', { var parsed = JSON.parse(response.responseText); - if (Object.keys(parsed.sampleFiles).length === 0) { + if (!parsed.sampleFiles || Object.keys(parsed.sampleFiles).length === 0) { Ext4.get(this.plotDivId).update('
No sample files loaded yet. Import some via Skyline, AutoQC, or the Data Pipeline tab here in Panorama.
'); return; } From 36c828bab0ed56429fd5670e7ec953bb5c017bd0 Mon Sep 17 00:00:00 2001 From: labkey-jeckels Date: Sat, 18 Oct 2025 07:54:18 -0700 Subject: [PATCH 07/15] Be more selective on some metrics to match old enable/disable behavior --- .../QCMetric_massErrorTransition.sql | 3 +- .../queries/targetedms/QCMetric_peakArea.sql | 38 ++++++++++++++++++- .../targetedms/QCMetric_precursorArea.sql | 11 ++++-- .../targetedms/QCMetric_transitionArea.sql | 31 ++++++++++++++- 4 files changed, 76 insertions(+), 7 deletions(-) diff --git a/resources/queries/targetedms/QCMetric_massErrorTransition.sql b/resources/queries/targetedms/QCMetric_massErrorTransition.sql index ba627d5ce..20127dc67 100644 --- a/resources/queries/targetedms/QCMetric_massErrorTransition.sql +++ b/resources/queries/targetedms/QCMetric_massErrorTransition.sql @@ -18,7 +18,7 @@ SELECT * FROM ( pci.Id AS PrecursorChromInfoId, SampleFileId AS SampleFileId, -- Use the error from the most intense transition associated with the precursor - (SELECT COALESCE(MassErrorPPM, -1000) AS MetricValue + (SELECT COALESCE(MassErrorPPM, -100000) AS MetricValue FROM TransitionChromInfo tci WHERE TransitionId.Charge IS NOT NULL AND tci.PrecursorChromInfoId = pci.Id @@ -26,3 +26,4 @@ SELECT * FROM ( ORDER BY Area DESC, Id LIMIT 1) AS MetricValue FROM PrecursorChromInfo pci ) X WHERE MetricValue IS NOT NULL +AND MetricValue != -100000 \ No newline at end of file diff --git a/resources/queries/targetedms/QCMetric_peakArea.sql b/resources/queries/targetedms/QCMetric_peakArea.sql index f131b6931..8c1ed122c 100644 --- a/resources/queries/targetedms/QCMetric_peakArea.sql +++ b/resources/queries/targetedms/QCMetric_peakArea.sql @@ -17,4 +17,40 @@ SELECT Id AS PrecursorChromInfoId, SampleFileId AS SampleFileId, TotalArea AS MetricValue -FROM PrecursorChromInfo \ No newline at end of file +FROM PrecursorChromInfo +WHERE + -- Ensure we have precursor areas for peptides or small molecules + (EXISTS (SELECT Id, FragmentType, Quantitative FROM Transition t WHERE FragmentType = 'precursor' AND Charge IS NULL) + OR EXISTS (SELECT Id, FragmentType, Quantitative FROM MoleculeTransition t WHERE FragmentType = 'precursor' AND Charge IS NULL)) + + -- Ensure we have transition areas for peptides or small molecules +AND (EXISTS ( + SELECT Id, FragmentType, Quantitative FROM Transition t + WHERE + (Quantitative = TRUE) OR (Quantitative IS NULL AND + (FragmentType != 'precursor' AND + t.GeneralPrecursorId.GeneralMoleculeId.PeptideGroupId.RunId IN ( + SELECT r.Id + FROM + targetedms.Runs r LEFT OUTER JOIN + targetedms.TransitionFullScanSettings tfss + ON r.Id = tfss.RunId + WHERE AcquisitionMethod IS NULL OR AcquisitionMethod != 'DDA' + )) + ) + ) + OR EXISTS ( + SELECT Id, FragmentType, Quantitative FROM MoleculeTransition t + WHERE + (Quantitative = TRUE) OR (Quantitative IS NULL AND + (FragmentType != 'precursor' AND + t.GeneralPrecursorId.GeneralMoleculeId.PeptideGroupId.RunId IN ( + SELECT r.Id + FROM + targetedms.Runs r LEFT OUTER JOIN + targetedms.TransitionFullScanSettings tfss + ON r.Id = tfss.RunId + WHERE AcquisitionMethod IS NULL OR AcquisitionMethod != 'DDA' + )) + ) + )) \ No newline at end of file diff --git a/resources/queries/targetedms/QCMetric_precursorArea.sql b/resources/queries/targetedms/QCMetric_precursorArea.sql index 008b1c536..6a30810f2 100644 --- a/resources/queries/targetedms/QCMetric_precursorArea.sql +++ b/resources/queries/targetedms/QCMetric_precursorArea.sql @@ -14,7 +14,10 @@ * limitations under the License. */ SELECT - Id AS PrecursorChromInfoId, - SampleFileId AS SampleFileId, - TotalPrecursorArea AS MetricValue -FROM PrecursorChromInfo \ No newline at end of file + Id AS PrecursorChromInfoId, + SampleFileId AS SampleFileId, + TotalPrecursorArea AS MetricValue +FROM PrecursorChromInfo +WHERE + (EXISTS (SELECT Id, FragmentType, Quantitative FROM Transition t WHERE FragmentType = 'precursor' AND Charge IS NULL) + OR EXISTS (SELECT Id, FragmentType, Quantitative FROM MoleculeTransition t WHERE FragmentType = 'precursor' AND Charge IS NULL)) \ No newline at end of file diff --git a/resources/queries/targetedms/QCMetric_transitionArea.sql b/resources/queries/targetedms/QCMetric_transitionArea.sql index 0629c06ab..99c593a41 100644 --- a/resources/queries/targetedms/QCMetric_transitionArea.sql +++ b/resources/queries/targetedms/QCMetric_transitionArea.sql @@ -17,4 +17,33 @@ SELECT Id AS PrecursorChromInfoId, SampleFileId AS SampleFileId, TotalNonPrecursorArea AS MetricValue -FROM PrecursorChromInfo \ No newline at end of file +FROM PrecursorChromInfo +WHERE + EXISTS (SELECT Id, FragmentType, Quantitative + FROM Transition t + WHERE (Quantitative = TRUE) + OR (Quantitative IS NULL AND + (FragmentType != 'precursor' AND + t.GeneralPrecursorId.GeneralMoleculeId.PeptideGroupId.RunId IN ( + SELECT r.Id + FROM + targetedms.Runs r LEFT OUTER JOIN + targetedms.TransitionFullScanSettings tfss + ON r.Id = tfss.RunId + WHERE AcquisitionMethod IS NULL OR AcquisitionMethod != 'DDA' + )) + )) + OR EXISTS (SELECT Id, FragmentType, Quantitative + FROM MoleculeTransition t + WHERE (Quantitative = TRUE) + OR (Quantitative IS NULL AND + (FragmentType != 'precursor' AND + t.GeneralPrecursorId.GeneralMoleculeId.PeptideGroupId.RunId IN ( + SELECT r.Id + FROM + targetedms.Runs r LEFT OUTER JOIN + targetedms.TransitionFullScanSettings tfss + ON r.Id = tfss.RunId + WHERE AcquisitionMethod IS NULL OR AcquisitionMethod != 'DDA' + )) + )) \ No newline at end of file From 27236e49c31fb4431a82f5b60678b2558d2df4c3 Mon Sep 17 00:00:00 2001 From: labkey-jeckels Date: Sun, 19 Oct 2025 08:40:17 -0700 Subject: [PATCH 08/15] Skip precursor area - shouldn't have data --- .../labkey/test/tests/targetedms/TargetedMSQCGuideSetTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/test/src/org/labkey/test/tests/targetedms/TargetedMSQCGuideSetTest.java b/test/src/org/labkey/test/tests/targetedms/TargetedMSQCGuideSetTest.java index d81dc3de3..4dfaa65b3 100644 --- a/test/src/org/labkey/test/tests/targetedms/TargetedMSQCGuideSetTest.java +++ b/test/src/org/labkey/test/tests/targetedms/TargetedMSQCGuideSetTest.java @@ -555,7 +555,6 @@ private void verifyGuideSetSmallMoleculeStats(GuideSet gs) throws IOException, C gs.addStats(new GuideSetStats("lhRatio", 0)); gs.addStats(new GuideSetStats("transitionPrecursorRatio", 0, precursor, null, null)); gs.addStats(new GuideSetStats("transitionArea", 2, precursor, 2.4647614E7, 5061170.5265)); - gs.addStats(new GuideSetStats("precursorArea", 2, precursor, 0.0, 0.0)); validateGuideSetStats(gs); } From ad9f203faa189670f0b94e3e66d58d89535fad86 Mon Sep 17 00:00:00 2001 From: labkey-jeckels Date: Sun, 19 Oct 2025 13:23:15 -0700 Subject: [PATCH 09/15] Test fixes --- .../labkey/test/tests/targetedms/TargetedMSQCGuideSetTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/src/org/labkey/test/tests/targetedms/TargetedMSQCGuideSetTest.java b/test/src/org/labkey/test/tests/targetedms/TargetedMSQCGuideSetTest.java index 4dfaa65b3..dfb2852af 100644 --- a/test/src/org/labkey/test/tests/targetedms/TargetedMSQCGuideSetTest.java +++ b/test/src/org/labkey/test/tests/targetedms/TargetedMSQCGuideSetTest.java @@ -430,7 +430,7 @@ private void validateGuideSetStats(GuideSet gs) throws IOException, CommandExcep { for (GuideSetStats stats : gs.getStats()) { - + log("Testing guide set stats for " + stats.getMetricName()); SelectRowsCommand cmd = new SelectRowsCommand("targetedms", "GuideSetStats_" + stats.getMetricName()); cmd.setRequiredVersion(9.1); cmd.setColumns(Arrays.asList("GuideSetId", "TrainingStart", "TrainingEnd", "ReferenceEnd", "SeriesLabel", "NumRecords", "Mean", "StandardDev")); @@ -549,7 +549,6 @@ private void verifyGuideSetSmallMoleculeStats(GuideSet gs) throws IOException, C String precursor = "C16,"; gs.addStats(new GuideSetStats("retentionTime", 2, precursor, 0.7729333639144897, 9.424035327035906E-5)); - gs.addStats(new GuideSetStats("peakArea", 2, precursor, 2.4647614E7, 5061170.5265)); gs.addStats(new GuideSetStats("fwhm", 2, precursor, 0.023859419859945774, 0.0010710133238455678)); gs.addStats(new GuideSetStats("fwb", 2, precursor, 0.11544176936149597, 0.012810408164340708)); gs.addStats(new GuideSetStats("lhRatio", 0)); From 9498b4acb37ddbca6ef614e475856246f3584103 Mon Sep 17 00:00:00 2001 From: labkey-jeckels Date: Thu, 23 Oct 2025 15:30:49 -0700 Subject: [PATCH 10/15] Fix metric deletion --- .../query/QCMetricConfigurationTable.java | 3 +-- .../panoramapremium/TargetedMSQCPremiumTest.java | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/org/labkey/targetedms/query/QCMetricConfigurationTable.java b/src/org/labkey/targetedms/query/QCMetricConfigurationTable.java index 78ab3b2d7..a11e02ff3 100644 --- a/src/org/labkey/targetedms/query/QCMetricConfigurationTable.java +++ b/src/org/labkey/targetedms/query/QCMetricConfigurationTable.java @@ -123,9 +123,8 @@ protected Map updateRow(User user, Container container, Map deleteRow(User user, Container container, Map oldRow) throws InvalidKeyException, QueryUpdateServiceException, SQLException { deleteTraceValueForMetric(asInteger(oldRow.get("id")), container); - Map result = super.deleteRow(user, container, oldRow); TargetedMSManager.get().clearQCMetricCache(container, true); - return result; + return super.deleteRow(user, container, oldRow); } private void deleteTraceValueForMetric(int metricId, Container container) diff --git a/test/src/org/labkey/test/tests/panoramapremium/TargetedMSQCPremiumTest.java b/test/src/org/labkey/test/tests/panoramapremium/TargetedMSQCPremiumTest.java index b059b7c15..595bc8864 100644 --- a/test/src/org/labkey/test/tests/panoramapremium/TargetedMSQCPremiumTest.java +++ b/test/src/org/labkey/test/tests/panoramapremium/TargetedMSQCPremiumTest.java @@ -179,6 +179,20 @@ public void testAddNewMetric() log("Verifying new metric got edited"); waitForElement(Locator.linkWithText(metricName2)); + configureUI.setLeveyJennings(metricName2, "-3", "3"); + configureUI.clickSave(); + qcPlotsWebPart = qcDashboard.getQcPlotsWebPart(); + List metricOptions = getMetric1TypeOptions(qcPlotsWebPart); + assertTrue("Didn't find '" + metricName2 + "' in :" + metricOptions, metricOptions.contains(metricName2)); + goToConfigureMetricsUI().deleteMetric(metricName2); + goToDashboard(); + metricOptions = getMetric1TypeOptions(qcPlotsWebPart); + assertFalse("Found '" + metricName2 + "' in :" + metricOptions, metricOptions.contains(metricName2)); + } + + private static List getMetric1TypeOptions(QCPlotsWebPart qcPlotsWebPart) + { + return qcPlotsWebPart.getMetric1TypeOptions(); } @Test From 12993e286a03954a7c0bfbb9fed068f53a475dd9 Mon Sep 17 00:00:00 2001 From: labkey-jeckels Date: Thu, 23 Oct 2025 15:45:59 -0700 Subject: [PATCH 11/15] Improve error messaging when there's a bad metric query --- .../labkey/targetedms/TargetedMSManager.java | 47 +++++++++++-------- webapp/TargetedMS/js/QCSummaryPanel.js | 1 + 2 files changed, 28 insertions(+), 20 deletions(-) diff --git a/src/org/labkey/targetedms/TargetedMSManager.java b/src/org/labkey/targetedms/TargetedMSManager.java index a71091542..46ef668b3 100644 --- a/src/org/labkey/targetedms/TargetedMSManager.java +++ b/src/org/labkey/targetedms/TargetedMSManager.java @@ -165,32 +165,39 @@ private TargetedMSManager() private static final Cache> _metricCache = CacheManager.getBlockingCache(1000, TimeUnit.HOURS.toMillis(1), "Enabled QC metric configs", (c, argument) -> { - TargetedMSSchema schema = (TargetedMSSchema) argument; - TableInfo metricsTable = schema.getTableOrThrow("qcMetricsConfig", null); - List metrics = new TableSelector(metricsTable, null, new Sort(FieldKey.fromParts("Name"))).getArrayList(QCMetricConfiguration.class); + try + { + TargetedMSSchema schema = (TargetedMSSchema) argument; + TableInfo metricsTable = schema.getTableOrThrow("qcMetricsConfig", null); + List metrics = new TableSelector(metricsTable, null, new Sort(FieldKey.fromParts("Name"))).getArrayList(QCMetricConfiguration.class); - OutlierGenerator.get().cachePrecursorMetricValues(schema, metrics); + OutlierGenerator.get().cachePrecursorMetricValues(schema, metrics); - // Identify which precursor-scoped metrics have any cached data in this container - SQLFragment sql = new SQLFragment("SELECT DISTINCT MetricId FROM ("); - sql.append(OutlierGenerator.get().getRawMetricSql(schema, metrics)); - sql.append(") y WHERE MetricValue IS NOT NULL"); - Set cachedMetricIds = new HashSet<>(new SqlSelector(getSchema(), sql).getCollection(Integer.class)); + // Identify which precursor-scoped metrics have any cached data in this container + SQLFragment sql = new SQLFragment("SELECT DISTINCT MetricId FROM ("); + sql.append(OutlierGenerator.get().getRawMetricSql(schema, metrics)); + sql.append(") y WHERE MetricValue IS NOT NULL"); + Set cachedMetricIds = new HashSet<>(new SqlSelector(getSchema(), sql).getCollection(Integer.class)); - for (QCMetricConfiguration metric : metrics) - { - if (!cachedMetricIds.contains(metric.getId())) + for (QCMetricConfiguration metric : metrics) { - metric.setStatus(QCMetricStatus.NoData); - } - else if (metric.getStatus() == null) - { - metric.setStatus(QCMetricStatus.DEFAULT); + if (!cachedMetricIds.contains(metric.getId())) + { + metric.setStatus(QCMetricStatus.NoData); + } + else if (metric.getStatus() == null) + { + metric.setStatus(QCMetricStatus.DEFAULT); + } } + // Ensure we get a case-insensitive sort regardless of DB collation + Collections.sort(metrics); + return Collections.unmodifiableList(metrics); + } + catch (RuntimeException e) + { + throw new PanoramaBadDataException("Failed to calculate metric values. Double-check the metric configurations and the backing queries. " + (e.getMessage() == null ? "" : e.getMessage()), e); } - // Ensure we get a case-insensitive sort regardless of DB collation - Collections.sort(metrics); - return Collections.unmodifiableList(metrics); }); public static TargetedMSManager get() diff --git a/webapp/TargetedMS/js/QCSummaryPanel.js b/webapp/TargetedMS/js/QCSummaryPanel.js index 5ef7b1894..99bf9d736 100644 --- a/webapp/TargetedMS/js/QCSummaryPanel.js +++ b/webapp/TargetedMS/js/QCSummaryPanel.js @@ -98,6 +98,7 @@ Ext4.define('LABKEY.targetedms.QCSummary', { }, this, false), failure: LABKEY.Utils.getCallbackWrapper(function (response) { + this.removeAll(); this.add(Ext4.create('Ext.Component', { autoEl: 'span', cls: 'labkey-error', From 104ddf1dc455979c05380c2ecba38ca07e15f2f1 Mon Sep 17 00:00:00 2001 From: labkey-jeckels Date: Thu, 23 Oct 2025 16:05:51 -0700 Subject: [PATCH 12/15] Improve error messaging when there's a bad metric query --- resources/views/configureQCMetric.html | 5 ++++- .../targetedms/query/QCMetricConfigurationTable.java | 2 -- .../test/components/targetedms/QCPlotsWebPart.java | 5 +++++ .../test/components/targetedms/QCSummaryWebPart.java | 7 ++++++- .../panoramapremium/TargetedMSIsotopologueTest.java | 8 +------- .../tests/panoramapremium/TargetedMSPremiumTest.java | 2 +- .../TargetedMSPeptideSummaryHeatmapTest.java | 4 ++-- .../targetedms/TargetedMSQCConfigureMetricTest.java | 10 +++++----- 8 files changed, 24 insertions(+), 19 deletions(-) diff --git a/resources/views/configureQCMetric.html b/resources/views/configureQCMetric.html index f32b54904..0709dd236 100644 --- a/resources/views/configureQCMetric.html +++ b/resources/views/configureQCMetric.html @@ -4,7 +4,8 @@

Loading...
-
+
+