diff --git a/api/src/org/labkey/api/exp/query/ExpDataTable.java b/api/src/org/labkey/api/exp/query/ExpDataTable.java index b60aa86b18c..7169b39a81b 100644 --- a/api/src/org/labkey/api/exp/query/ExpDataTable.java +++ b/api/src/org/labkey/api/exp/query/ExpDataTable.java @@ -35,6 +35,7 @@ enum Column SourceProtocolApplication, SourceApplicationInput, DataFileUrl, + ReferenceCount, Run, RunApplication, RunApplicationOutput, diff --git a/api/src/org/labkey/api/exp/query/ExpSchema.java b/api/src/org/labkey/api/exp/query/ExpSchema.java index 5fd2cedf774..60eb0985840 100644 --- a/api/src/org/labkey/api/exp/query/ExpSchema.java +++ b/api/src/org/labkey/api/exp/query/ExpSchema.java @@ -47,6 +47,7 @@ import org.labkey.api.security.User; import org.labkey.api.security.permissions.InsertPermission; import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.settings.AppProps; import org.labkey.api.util.StringExpression; import org.labkey.api.view.ActionURL; import org.labkey.api.view.ViewContext; @@ -70,6 +71,7 @@ public class ExpSchema extends AbstractExpSchema public static final String SAMPLE_STATE_TYPE_TABLE = "SampleStateType"; public static final String SAMPLE_TYPE_CATEGORY_TABLE = "SampleTypeCategoryType"; public static final String MEASUREMENT_UNITS_TABLE = "MeasurementUnits"; + public static final String SAMPLE_FILES_TABLE = "UnreferencedSampleFiles"; public static final SchemaKey SCHEMA_EXP = SchemaKey.fromParts(ExpSchema.SCHEMA_NAME); public static final SchemaKey SCHEMA_EXP_DATA = SchemaKey.fromString(SCHEMA_EXP, ExpSchema.NestedSchemas.data.name()); @@ -220,6 +222,20 @@ public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFil return expSchema.setupTable(result); } }, + UnreferencedSampleFiles + { + @Override + public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) + { + return new ExpUnreferencedSampleFilesTable(expSchema, cf); + } + + @Override + public boolean includeTable() + { + return AppProps.getInstance().isOptionalFeatureEnabled(SAMPLE_FILES_TABLE); + } + }, SampleStatus { @Override diff --git a/api/src/org/labkey/api/exp/query/ExpUnreferencedSampleFilesTable.java b/api/src/org/labkey/api/exp/query/ExpUnreferencedSampleFilesTable.java new file mode 100644 index 00000000000..9409802b054 --- /dev/null +++ b/api/src/org/labkey/api/exp/query/ExpUnreferencedSampleFilesTable.java @@ -0,0 +1,114 @@ +package org.labkey.api.exp.query; + +import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.NotNull; +import org.labkey.api.data.BaseColumnInfo; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.CoreSchema; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.VirtualTable; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.files.FileContentService; +import org.labkey.api.query.FilteredTable; +import org.labkey.api.query.UserIdQueryForeignKey; +import org.labkey.api.query.UserSchema; +import org.labkey.api.query.column.BuiltInColumnTypes; + +public class ExpUnreferencedSampleFilesTable extends FilteredTable +{ + public ExpUnreferencedSampleFilesTable(@NotNull ExpSchema schema, ContainerFilter cf) + { + super(createVirtualTable(schema), schema, cf); + setDescription("Contains all sample files that are not referenced by any domain fields."); + wrapAllColumns(true); + } + + private static TableInfo createVirtualTable(@NotNull ExpSchema schema) + { + return new ExpUnreferencedSampleFilesTable.FileUnionTable(schema); + } + + private static class FileUnionTable extends VirtualTable + { + private final SQLFragment _query; + + public FileUnionTable(@NotNull ExpSchema schema) + { + super(CoreSchema.getInstance().getSchema(), ExpSchema.SAMPLE_FILES_TABLE, schema); + + FileContentService svc = FileContentService.get(); + + _query = new SQLFragment(); + if (svc == null) + return; + + SQLFragment listQuery = svc.listSampleFilesQuery(schema.getUser()); + if (StringUtils.isEmpty(listQuery)) + return; + + TableInfo expDataTable = ExperimentService.get().getTinfoData(); + TableInfo materialTable = ExperimentService.get().getTinfoMaterial(); + + _query.appendComment("", getSchema().getSqlDialect()); + + SQLFragment sampleFileSql = new SQLFragment("SELECT m.Container, if.FilePathShort \n") + .append("FROM (") + .append(svc.listSampleFilesQuery(schema.getUser())) + .append(") AS if \n") + .append("JOIN ") + .append(materialTable, "m") + .append(" ON if.SourceKey = m.RowId"); + + SQLFragment unreferencedFileSql = new SQLFragment("SELECT ed.rowId, ed.name as filename, ed.container, ed.created, ed.createdBy, ed.DataFileUrl FROM ") + .append(expDataTable, "ed") + .append(" LEFT JOIN (") + .append(sampleFileSql) + .append(" ) sf\n") + .append(" ON ed.name = sf.FilePathShort AND ed.container = sf.container\n") + .append(" WHERE ed.datafileurl LIKE ") + .appendValue("%@files/sampletype/%") + .append(" AND sf.FilePathShort IS NULL"); + + _query.append(unreferencedFileSql); + + _query.appendComment("", getSchema().getSqlDialect()); + + var rowIdCol = new BaseColumnInfo("RowId", this, JdbcType.INTEGER); + rowIdCol.setHidden(true); + rowIdCol.setKeyField(true); + addColumn(rowIdCol); + + var fileNameCol = new BaseColumnInfo("FileName", this, JdbcType.VARCHAR); + addColumn(fileNameCol); + + if (schema.getUser().hasApplicationAdminPermission()) + { + var filePathCol = new BaseColumnInfo("DataFileUrl", this, JdbcType.VARCHAR); + filePathCol.setHidden(true); + addColumn(filePathCol); + } + + var containerCol = new BaseColumnInfo("Container", this, JdbcType.VARCHAR); + containerCol.setConceptURI(BuiltInColumnTypes.CONTAINERID_CONCEPT_URI); + addColumn(containerCol); + + var createdCol = new BaseColumnInfo("Created", this, JdbcType.DATE); + addColumn(createdCol); + + var createdByCol = new BaseColumnInfo("CreatedBy", this, JdbcType.INTEGER); + createdByCol.setFk(new UserIdQueryForeignKey(getUserSchema(), true)); + addColumn(createdByCol); + } + + @NotNull + @Override + public SQLFragment getFromSQL() + { + return _query; + } + + } + +} diff --git a/api/src/org/labkey/api/files/FileContentService.java b/api/src/org/labkey/api/files/FileContentService.java index 9791ba68902..0b8b8d8019e 100644 --- a/api/src/org/labkey/api/files/FileContentService.java +++ b/api/src/org/labkey/api/files/FileContentService.java @@ -325,6 +325,8 @@ default void fireFileDeletedEvent(@NotNull Path deleted, @Nullable User user, @N */ SQLFragment listFilesQuery(@NotNull User currentUser); + SQLFragment listSampleFilesQuery(@NotNull User currentUser); + void setWebfilesEnabled(boolean enabled, User user); /** diff --git a/api/src/org/labkey/api/files/FileListener.java b/api/src/org/labkey/api/files/FileListener.java index 858d1f54306..91702ed4b94 100644 --- a/api/src/org/labkey/api/files/FileListener.java +++ b/api/src/org/labkey/api/files/FileListener.java @@ -94,4 +94,9 @@ default void fileDeleted(@NotNull Path deleted, @Nullable User user, @Nullable C * */ SQLFragment listFilesQuery(); + + @Nullable default SQLFragment listSampleFilesQuery() + { + return null; + } } diff --git a/api/src/org/labkey/api/files/TableUpdaterFileListener.java b/api/src/org/labkey/api/files/TableUpdaterFileListener.java index ce46e143c87..5b9d5dcda33 100644 --- a/api/src/org/labkey/api/files/TableUpdaterFileListener.java +++ b/api/src/org/labkey/api/files/TableUpdaterFileListener.java @@ -353,10 +353,10 @@ public Collection listFiles(@Nullable Container container) @Override public SQLFragment listFilesQuery() { - return listFilesQuery(false, null); + return listFilesQuery(false, null, false); } - public SQLFragment listFilesQuery(boolean skipCreatedModified, String filePath) + public SQLFragment listFilesQuery(boolean skipCreatedModified, CharSequence filePath, boolean extractName) { SQLFragment selectFrag = new SQLFragment(); selectFrag.append("SELECT\n"); @@ -395,6 +395,16 @@ else if (_table.getColumn("Folder") != null) selectFrag.append(" ").appendIdentifier(_pathColumn.getSelectIdentifier()).append(" AS FilePath,\n"); + if (extractName) + { + SqlDialect dialect = _table.getSchema().getSqlDialect(); + SQLFragment fileNameFrag = new SQLFragment(); + fileNameFrag.append("regexp_replace(").appendIdentifier(_pathColumn.getSelectIdentifier()).append(", "); + fileNameFrag.append(dialect.getStringHandler().quoteStringLiteral(".*/")).append(", "); + fileNameFrag.append(dialect.getStringHandler().quoteStringLiteral("")).append(")");; + selectFrag.append(" ").append(fileNameFrag).append(" AS FilePathShort,\n"); + } + if (_keyColumn != null) selectFrag.append(" ").appendIdentifier(_keyColumn.getSelectIdentifier()).append(" AS SourceKey,\n"); else @@ -408,6 +418,8 @@ else if (_table.getColumn("Folder") != null) if (StringUtils.isEmpty(filePath)) selectFrag.append(" IS NOT NULL\n"); + else if (filePath instanceof SQLFragment) + selectFrag.append(" = ").append(filePath).append("\n"); else selectFrag.append(" = ").appendStringLiteral(filePath, _table.getSchema().getSqlDialect()).append("\n"); diff --git a/experiment/src/org/labkey/experiment/ExperimentModule.java b/experiment/src/org/labkey/experiment/ExperimentModule.java index 435448ca0b7..90312720f61 100644 --- a/experiment/src/org/labkey/experiment/ExperimentModule.java +++ b/experiment/src/org/labkey/experiment/ExperimentModule.java @@ -18,6 +18,7 @@ import org.apache.commons.lang3.math.NumberUtils; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.json.JSONObject; import org.labkey.api.admin.FolderSerializationRegistry; import org.labkey.api.assay.AssayProvider; import org.labkey.api.assay.AssayService; @@ -112,6 +113,7 @@ import org.labkey.api.vocabulary.security.DesignVocabularyPermission; import org.labkey.api.webdav.WebdavResource; import org.labkey.api.webdav.WebdavService; +import org.labkey.api.writer.ContainerUser; import org.labkey.experiment.api.DataClassDomainKind; import org.labkey.experiment.api.ExpDataClassImpl; import org.labkey.experiment.api.ExpDataClassTableImpl; @@ -181,6 +183,7 @@ import static org.labkey.api.data.ColumnRenderPropertiesImpl.STORAGE_UNIQUE_ID_CONCEPT_URI; import static org.labkey.api.data.ColumnRenderPropertiesImpl.TEXT_CHOICE_CONCEPT_URI; import static org.labkey.api.exp.api.ExperimentService.MODULE_NAME; +import static org.labkey.api.exp.query.ExpSchema.SAMPLE_FILES_TABLE; public class ExperimentModule extends SpringModule { @@ -266,6 +269,9 @@ protected void init() } else { + OptionalFeatureService.get().addExperimentalFeatureFlag(SAMPLE_FILES_TABLE, "Manage Unreferenced Sample Files", + "Enable 'Unreferenced Sample Files' table to view and delete sample files that are no longer referenced by samples", false); + OptionalFeatureService.get().addExperimentalFeatureFlag(NameGenerator.EXPERIMENTAL_ALLOW_GAP_COUNTER, "Allow gap with withCounter and rootSampleCount expression", "Check this option if gaps in the count generated by withCounter or rootSampleCount name expression are allowed.", true); } @@ -1117,4 +1123,12 @@ public Collection getProvisionedSchemaNames() { return PageFlowUtil.set(DataClassDomainKind.PROVISIONED_SCHEMA_NAME, SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME); } + + @Override + public JSONObject getPageContextJson(ContainerUser context) + { + JSONObject json = super.getPageContextJson(context); + json.put(SAMPLE_FILES_TABLE, OptionalFeatureService.get().isFeatureEnabled(SAMPLE_FILES_TABLE)); + return json; + } } diff --git a/experiment/src/org/labkey/experiment/FileLinkFileListener.java b/experiment/src/org/labkey/experiment/FileLinkFileListener.java index 4a19b2b4ccb..83b1e7ff358 100644 --- a/experiment/src/org/labkey/experiment/FileLinkFileListener.java +++ b/experiment/src/org/labkey/experiment/FileLinkFileListener.java @@ -275,7 +275,7 @@ public SQLFragment listFilesQuery(boolean skipCreatedModified) return listFilesQuery(skipCreatedModified, null); } - public SQLFragment listFilesQuery(boolean skipCreatedModified, String filePath) + public SQLFragment listFilesQuery(boolean skipCreatedModified, CharSequence filePath) { final SQLFragment frag = new SQLFragment(); @@ -298,8 +298,11 @@ public SQLFragment listFilesQuery(boolean skipCreatedModified, String filePath) frag.append("WHERE\n"); if (StringUtils.isEmpty(filePath)) frag.append(" op.StringValue IS NOT NULL AND\n"); + else if (filePath instanceof SQLFragment) + frag.append(" op.StringValue = ").append(filePath).append(" AND\n"); else frag.append(" op.StringValue = ").appendStringLiteral(filePath, OntologyManager.getTinfoObject().getSqlDialect()).append(" AND\n"); + frag.append(" o.ObjectId = op.ObjectId AND\n"); frag.append(" PropertyId IN (\n"); frag.append(" SELECT PropertyId\n"); @@ -311,7 +314,26 @@ public SQLFragment listFilesQuery(boolean skipCreatedModified, String filePath) SQLFragment containerFrag = new SQLFragment("?", containerId); TableUpdaterFileListener updater = new TableUpdaterFileListener(table, pathColumn.getColumnName(), TableUpdaterFileListener.Type.filePath, null, containerFrag); frag.append("UNION").append(StringUtils.isEmpty(filePath) ? "" : " ALL" /*keep duplicate*/).append("\n"); - frag.append(updater.listFilesQuery(skipCreatedModified, filePath)); + frag.append(updater.listFilesQuery(skipCreatedModified, filePath, false)); + }); + + return frag; + } + + @Override + public SQLFragment listSampleFilesQuery() + { + final SQLFragment frag = new SQLFragment(); + + hardTableFileLinkColumns((schema, table, pathColumn, containerId, domainUri) -> { + if (PROVISIONED_SCHEMA_NAME.equalsIgnoreCase(schema.getName())) + { + SQLFragment containerFrag = new SQLFragment("?", containerId); + TableUpdaterFileListener updater = new TableUpdaterFileListener(table, pathColumn.getColumnName(), TableUpdaterFileListener.Type.filePath, "rowid", containerFrag); + if (!frag.isEmpty()) + frag.append("UNION").append("").append("\n"); + frag.append(updater.listFilesQuery(true, null, true)); + } }); return frag; diff --git a/experiment/src/org/labkey/experiment/api/ExpDataTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpDataTableImpl.java index 1896b1eb5b6..ce2db439e6e 100644 --- a/experiment/src/org/labkey/experiment/api/ExpDataTableImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpDataTableImpl.java @@ -34,12 +34,14 @@ import org.labkey.api.data.Container; import org.labkey.api.data.ContainerFilter; import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.CoreSchema; import org.labkey.api.data.ExcelWriter; import org.labkey.api.data.JdbcType; import org.labkey.api.data.MutableColumnInfo; import org.labkey.api.data.RenderContext; import org.labkey.api.data.SQLFragment; import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.SqlSelector; import org.labkey.api.exp.DomainDescriptor; import org.labkey.api.exp.OntologyManager; import org.labkey.api.exp.PropertyColumn; @@ -75,6 +77,7 @@ import org.labkey.api.security.User; import org.labkey.api.security.permissions.InsertPermission; import org.labkey.api.security.permissions.UpdatePermission; +import org.labkey.api.settings.AppProps; import org.labkey.api.util.ConfigurationException; import org.labkey.api.util.FileUtil; import org.labkey.api.util.HtmlString; @@ -84,6 +87,7 @@ import org.labkey.api.writer.HtmlWriter; import org.labkey.api.writer.MemoryVirtualFile; import org.labkey.api.writer.VirtualFile; +import org.labkey.experiment.FileLinkFileListener; import org.labkey.experiment.controllers.exp.ExperimentController; import org.labkey.experiment.lineage.LineageMethod; import org.springframework.beans.MutablePropertyValues; @@ -200,6 +204,8 @@ public List addFileColumns(boolean isFilesTable) addColumn(Column.FileExtension); addColumn(Column.WebDavUrl); addColumn(Column.WebDavUrlRelative); + if (AppProps.getInstance().isOptionalFeatureEnabled(ExpSchema.SAMPLE_FILES_TABLE)) + addColumn(getFileLinkReferenceCountColumn()); var flagCol = addColumn(Column.Flag); if (isFilesTable) flagCol.setLabel("Description"); @@ -227,6 +233,48 @@ public List addFileColumns(boolean isFilesTable) return customProps; } + // This is included in exp.data, not just exp.files because we want to be able to show a filtered view of + // sample files from our applications, and exp.files will not show subfolders + private MutableColumnInfo getFileLinkReferenceCountColumn() + { + var result = wrapColumn(Column.ReferenceCount.name(), _rootTable.getColumn("RowId")); + result.setDescription("The number of references to this file from File fields in any domain."); + result.setJdbcType(JdbcType.INTEGER); + result.setHidden(true); + result.setDisplayColumnFactory(colInfo -> new ExpDataFileColumn(colInfo) + { + private Long getCount(ExpData data) + { + if (data == null || StringUtils.isEmpty(data.getDataFileUrl()) || data.getFile() == null) + return null; + else + { + FileLinkFileListener fileListener = new FileLinkFileListener(); + SQLFragment unionSql = fileListener.listFilesQuery(true, data.getFile().getAbsolutePath()); + + return new SqlSelector(CoreSchema.getInstance().getSchema(), unionSql).getRowCount(); + } + } + + @Override + protected void renderData(HtmlWriter out, ExpData data) + { + Long val = getCount(data); + if (val == null) + out.write(""); + else + out.write(val); + } + + @Override + protected Object getJsonValue(ExpData data) + { + return getCount(data); + } + }); + return result; + } + @Override public boolean supportTableRules() // intentional override { diff --git a/filecontent/src/org/labkey/filecontent/FileContentServiceImpl.java b/filecontent/src/org/labkey/filecontent/FileContentServiceImpl.java index f3ed5d2eaff..d890353d832 100644 --- a/filecontent/src/org/labkey/filecontent/FileContentServiceImpl.java +++ b/filecontent/src/org/labkey/filecontent/FileContentServiceImpl.java @@ -1232,6 +1232,27 @@ public Map> listFiles(@NotNull Container container) return files; } + @Override + public SQLFragment listSampleFilesQuery(@NotNull User currentUser) + { + SQLFragment frag = new SQLFragment(); + String union = ""; + frag.append("("); + + for (FileListener fileListener : _fileListeners) + { + SQLFragment subselect = fileListener.listSampleFilesQuery(); + if (subselect != null && !subselect.isEmpty()) + { + frag.append(union); + frag.append(subselect); + union = "UNION\n"; + } + } + frag.append(")"); + return union.isEmpty() ? new SQLFragment() : frag; + } + @Override public SQLFragment listFilesQuery(@NotNull User currentUser) {