Skip to content

Commit b10dd3c

Browse files
bbimberbbimber
committed
Add AuditSummaryUserSchema, mainly for use in ETLs
1 parent 9c07da2 commit b10dd3c

File tree

5 files changed

+292
-7
lines changed

5 files changed

+292
-7
lines changed

‎QueryExtensions/build.gradle‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,5 +6,4 @@ plugins {
66

77
dependencies {
88
BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:DiscvrLabKeyModules:discvrcore", depProjectConfig: "published", depExtension: "module")
9-
BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:DiscvrLabKeyModules:discvrcore", depProjectConfig: "published", depExtension: "module")
109
}

‎SequenceAnalysis/build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,7 @@ dependencies {
156156
apiImplementation "org.apache.commons:commons-math3:${commonsMath3Version}"
157157
BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:LabDevKitModules:laboratory", depProjectConfig: "published", depExtension: "module")
158158
BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:LabDevKitModules:LDK", depProjectConfig: "published", depExtension: "module")
159+
BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:DiscvrLabKeyModules:discvrcore", depProjectConfig: "published", depExtension: "module")
159160
}
160161

161162

‎Studies/build.gradle‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,5 +11,7 @@ dependencies
1111

1212
BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:LabDevKitModules:laboratory", depProjectConfig: "published", depExtension: "module")
1313
BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:LabDevKitModules:LDK", depProjectConfig: "published", depExtension: "module")
14+
15+
BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:DiscvrLabKeyModules:discvrcore", depProjectConfig: "published", depExtension: "module")
1416
}
1517

‎discvrcore/src/org/labkey/discvrcore/AuditSummaryUserSchema.java‎

Lines changed: 282 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,282 @@
1+
package org.labkey.discvrcore;
2+
3+
import org.jetbrains.annotations.Nullable;
4+
import org.junit.AfterClass;
5+
import org.junit.Assert;
6+
import org.junit.BeforeClass;
7+
import org.junit.Test;
8+
import org.labkey.api.audit.AuditLogService;
9+
import org.labkey.api.collections.CaseInsensitiveHashMap;
10+
import org.labkey.api.data.ColumnInfo;
11+
import org.labkey.api.data.CompareType;
12+
import org.labkey.api.data.Container;
13+
import org.labkey.api.data.ContainerFilter;
14+
import org.labkey.api.data.ContainerManager;
15+
import org.labkey.api.data.DbSchema;
16+
import org.labkey.api.data.DbSchemaType;
17+
import org.labkey.api.data.JdbcType;
18+
import org.labkey.api.data.SQLFragment;
19+
import org.labkey.api.data.SimpleFilter;
20+
import org.labkey.api.data.TableInfo;
21+
import org.labkey.api.data.TableSelector;
22+
import org.labkey.api.exp.property.DomainProperty;
23+
import org.labkey.api.module.Module;
24+
import org.labkey.api.module.ModuleLoader;
25+
import org.labkey.api.query.BatchValidationException;
26+
import org.labkey.api.query.DefaultSchema;
27+
import org.labkey.api.query.ExprColumn;
28+
import org.labkey.api.query.FieldKey;
29+
import org.labkey.api.query.QueryForeignKey;
30+
import org.labkey.api.query.QuerySchema;
31+
import org.labkey.api.query.QueryService;
32+
import org.labkey.api.query.SimpleUserSchema;
33+
import org.labkey.api.query.UserSchema;
34+
import org.labkey.api.security.MutableSecurityPolicy;
35+
import org.labkey.api.security.SecurityManager;
36+
import org.labkey.api.security.SecurityPolicyManager;
37+
import org.labkey.api.security.User;
38+
import org.labkey.api.security.UserManager;
39+
import org.labkey.api.security.ValidEmail;
40+
import org.labkey.api.security.roles.ReaderRole;
41+
import org.labkey.api.study.Dataset;
42+
import org.labkey.api.study.Study;
43+
import org.labkey.api.study.StudyService;
44+
import org.labkey.api.study.TimepointType;
45+
import org.labkey.api.util.GUID;
46+
import org.labkey.api.util.PageFlowUtil;
47+
import org.labkey.api.util.TestContext;
48+
49+
import java.util.ArrayList;
50+
import java.util.Collections;
51+
import java.util.Date;
52+
import java.util.HashSet;
53+
import java.util.List;
54+
import java.util.Map;
55+
import java.util.Set;
56+
57+
public class AuditSummaryUserSchema extends SimpleUserSchema
58+
{
59+
public static final String NAME = "AuditSummary";
60+
public static final String QUERY_AUDIT = "QueryUpdateAuditLog";
61+
public static final String DATASET_AUDIT = "DatasetUpdateAuditLog";
62+
63+
private AuditSummaryUserSchema(User user, Container container, DbSchema schema)
64+
{
65+
super(DiscvrCoreModule.NAME, null, user, container, schema);
66+
}
67+
68+
public static void register(final Module m)
69+
{
70+
final DbSchema dbSchema = DbSchema.get(DiscvrCoreModule.NAME, DbSchemaType.Module);
71+
72+
DefaultSchema.registerProvider(DiscvrCoreModule.NAME, new DefaultSchema.SchemaProvider(m)
73+
{
74+
@Override
75+
public QuerySchema createSchema(final DefaultSchema schema, Module module)
76+
{
77+
return new AuditSummaryUserSchema(schema.getUser(), schema.getContainer(), dbSchema);
78+
}
79+
});
80+
}
81+
82+
@Override
83+
public @Nullable String getDescription()
84+
{
85+
return "Contains a summary view of audit records, without record detail, primarily for use in ETLs ";
86+
}
87+
88+
@Override
89+
public TableInfo createTable(String name, ContainerFilter cf)
90+
{
91+
if (name.equalsIgnoreCase(QUERY_AUDIT))
92+
{
93+
return generateQueryAuditSql(name, cf);
94+
}
95+
else if (name.equalsIgnoreCase(DATASET_AUDIT))
96+
{
97+
return generateDatasetAuditSql(name, cf);
98+
}
99+
100+
return super.createTable(name, cf);
101+
}
102+
103+
@Override
104+
public Set<String> getTableNames()
105+
{
106+
Set<String> tables = new HashSet<>(super.getTableNames());
107+
tables.add(QUERY_AUDIT);
108+
tables.add(DATASET_AUDIT);
109+
110+
return Collections.unmodifiableSet(tables);
111+
}
112+
113+
@Override
114+
public synchronized Set<String> getVisibleTableNames()
115+
{
116+
return getTableNames();
117+
}
118+
119+
private SimpleTable<?> generateAuditSql(String tableName, String eventType, ContainerFilter cf)
120+
{
121+
Container target = getContainer().isWorkbookOrTab() ? getContainer().getParent() : getContainer();
122+
String storageTableName = AuditLogService.get().getAuditProvider(eventType).getDomain().getStorageTableName();
123+
DbSchema auditSchema = DbSchema.get("audit", DbSchemaType.Module);
124+
TableInfo ti = auditSchema.getTable(storageTableName);
125+
126+
SimpleTable<?> st = new SimpleTable<>(this, ti, ContainerFilter.current(target));
127+
st.setName(tableName);
128+
st.init();
129+
130+
// Don't expose more information than needed:
131+
st.removeColumn(st.getColumn("CreatedBy"));
132+
st.removeColumn(st.getColumn("ImpersonatedBy"));
133+
st.removeColumn(st.getColumn("oldRecordMap"));
134+
st.removeColumn(st.getColumn("newRecordMap"));
135+
136+
ColumnInfo lsid = st.getColumn("lsid");
137+
if (lsid != null)
138+
{
139+
if (st.getSqlDialect().isPostgreSQL())
140+
{
141+
st.addColumn(new ExprColumn(st, FieldKey.fromString("primaryKey"), new SQLFragment("right(" + ExprColumn.STR_TABLE_ALIAS + ".lsid, position('.', reverse(" + ExprColumn.STR_TABLE_ALIAS + ".lsid))-1)"), JdbcType.VARCHAR, lsid));
142+
}
143+
else if (ti.getSqlDialect().isSqlServer())
144+
{
145+
st.addColumn(new ExprColumn(st, FieldKey.fromString("primaryKey"), new SQLFragment("right(" + ExprColumn.STR_TABLE_ALIAS + ".lsid, charindex('.', reverse(" + ExprColumn.STR_TABLE_ALIAS + ".lsid))-1)"), JdbcType.VARCHAR, lsid));
146+
}
147+
}
148+
149+
return st;
150+
}
151+
152+
private TableInfo generateQueryAuditSql(String tableName, ContainerFilter cf)
153+
{
154+
return generateAuditSql(tableName, "QueryUpdateAuditEvent", cf);
155+
}
156+
157+
private TableInfo generateDatasetAuditSql(String tableName, ContainerFilter cf)
158+
{
159+
SimpleTable<?> ti = generateAuditSql(tableName, "DatasetAuditEvent", cf);
160+
Container target = getContainer().isWorkbookOrTab() ? getContainer().getParent() : getContainer();
161+
162+
UserSchema us = QueryService.get().getUserSchema(getUser(), target, "study");
163+
if (us != null)
164+
{
165+
ti.getMutableColumn("DatasetId").setFk(new QueryForeignKey(us.getTable("Datasets"), "DataSetId", "Name"));
166+
}
167+
168+
return ti;
169+
}
170+
171+
public static class TestCase extends Assert
172+
{
173+
public static final String PROJECT_NAME = "QueryExtensionsTestProject";
174+
175+
@BeforeClass
176+
public static void setup() throws Exception
177+
{
178+
doCleanup();
179+
180+
Container c = getContainer(true);
181+
User u = getReaderUser(true);
182+
183+
MutableSecurityPolicy p = new MutableSecurityPolicy(c.getPolicy());
184+
p.addRoleAssignment(u, ReaderRole.class);
185+
SecurityPolicyManager.savePolicy(p, TestContext.get().getUser());
186+
}
187+
188+
private static Container getContainer(boolean createIfNeeded)
189+
{
190+
if (createIfNeeded && ContainerManager.getForPath(PROJECT_NAME) == null)
191+
{
192+
Container c = ContainerManager.createContainer(ContainerManager.getRoot(), PROJECT_NAME);
193+
c.setActiveModules(PageFlowUtil.set(ModuleLoader.getInstance().getModule("study"), ModuleLoader.getInstance().getModule(DiscvrCoreModule.NAME)));
194+
}
195+
196+
return ContainerManager.getForPath(PROJECT_NAME);
197+
}
198+
199+
private Dataset createDataset(String name, boolean isDemographics) throws Exception
200+
{
201+
Dataset d1 = StudyService.get().createDataset(getContainer(true), TestContext.get().getUser(), name, null, isDemographics);
202+
d1.setKeyManagementType(Dataset.KeyManagementType.GUID);
203+
d1.setKeyPropertyName("objectId");
204+
DomainProperty objectId1 = d1.getDomain().addProperty();
205+
objectId1.setName("objectId");
206+
objectId1.setPropertyURI(AuditSummaryUserSchema.class.getName() + ":ObjectId");
207+
d1.getDomain().save(TestContext.get().getUser());
208+
d1.save(TestContext.get().getUser());
209+
210+
return d1;
211+
}
212+
213+
@Test
214+
public void testAuditTables() throws Exception
215+
{
216+
Container c = getContainer(true);
217+
218+
// Add a study:
219+
Study s = StudyService.get().createStudy(c, TestContext.get().getUser(), "DemoStudy", TimepointType.CONTINUOUS, true);
220+
221+
Dataset d1 = createDataset("Dataset1", false);
222+
Dataset d2 = createDataset("Dataset2", true);
223+
224+
String guid1 = new GUID().toString();
225+
226+
List<Map<String, Object>> toInsert = new ArrayList<>();
227+
toInsert.add(new CaseInsensitiveHashMap<>(Map.of("ParticipantId", "P1", "Date", new Date(), "ObjectId", guid1)));
228+
TableInfo t1 = d1.getTableInfo(TestContext.get().getUser());
229+
t1.getUpdateService().insertRows(TestContext.get().getUser(), c, toInsert, new BatchValidationException(), null, null);
230+
231+
TableInfo t2 = d2.getTableInfo(TestContext.get().getUser());
232+
t2.getUpdateService().insertRows(TestContext.get().getUser(), c, toInsert, new BatchValidationException(), null, null);
233+
234+
TableInfo qa = QueryService.get().getUserSchema(TestContext.get().getUser(), c, NAME).getTable(DATASET_AUDIT);
235+
236+
TableSelector ts1 = new TableSelector(qa, PageFlowUtil.set("primaryKey"));
237+
assertEquals("Incorrect row count", ts1.getRowCount(), 2L);
238+
239+
assertEquals("Incorrect PK", new TableSelector(qa, PageFlowUtil.set("primaryKey"), new SimpleFilter(FieldKey.fromString("Comment"), "inserted", CompareType.CONTAINS).addCondition(FieldKey.fromString("DatasetId"), d1.getDatasetId()), null).getObject(String.class), guid1);
240+
assertEquals("Incorrect PK", new TableSelector(qa, PageFlowUtil.set("primaryKey"), new SimpleFilter(FieldKey.fromString("Comment"), "inserted", CompareType.CONTAINS).addCondition(FieldKey.fromString("DatasetId"), d2.getDatasetId()), null).getObject(String.class), "P1");
241+
242+
assertEquals("Incorrect PK", new TableSelector(qa, PageFlowUtil.set("primaryKey"), new SimpleFilter(FieldKey.fromString("Comment"), "inserted", CompareType.CONTAINS).addCondition(FieldKey.fromString("DatasetId/Name"), d1.getName()), null).getObject(String.class), guid1);
243+
assertEquals("Incorrect PK", new TableSelector(qa, PageFlowUtil.set("primaryKey"), new SimpleFilter(FieldKey.fromString("Comment"), "inserted", CompareType.CONTAINS).addCondition(FieldKey.fromString("DatasetId/Name"), d2.getName()), null).getObject(String.class), "P1");
244+
245+
t1.getUpdateService().deleteRows(TestContext.get().getUser(), c, Collections.singletonList(new CaseInsensitiveHashMap<>(Map.of("ObjectId", guid1))), null, null);
246+
assertEquals("Incorrect PK", new TableSelector(qa, PageFlowUtil.set("primaryKey"), new SimpleFilter(FieldKey.fromString("Comment"), "deleted", CompareType.CONTAINS), null).getObject(String.class), guid1);
247+
248+
// Ensure this works as reader:
249+
User reader = getReaderUser(true);
250+
qa = QueryService.get().getUserSchema(reader, c, NAME).getTable(DATASET_AUDIT);
251+
assertEquals("Incorrect PK", new TableSelector(qa, PageFlowUtil.set("primaryKey"), new SimpleFilter(FieldKey.fromString("Comment"), "inserted", CompareType.CONTAINS).addCondition(FieldKey.fromString("DatasetId/Name"), d1.getName()), null).getObject(String.class), guid1);
252+
assertEquals("Incorrect PK", new TableSelector(qa, PageFlowUtil.set("primaryKey"), new SimpleFilter(FieldKey.fromString("Comment"), "inserted", CompareType.CONTAINS).addCondition(FieldKey.fromString("DatasetId/Name"), d2.getName()), null).getObject(String.class), "P1");
253+
}
254+
255+
@AfterClass
256+
public static void doCleanup() throws Exception
257+
{
258+
if (getContainer(false) != null)
259+
{
260+
ContainerManager.delete(getContainer(false), TestContext.get().getUser());
261+
}
262+
263+
if (getReaderUser(false) != null)
264+
{
265+
UserManager.deleteUser(getReaderUser(false).getUserId());
266+
}
267+
}
268+
269+
private static final String READER_USER = "readerUserForTesting@myDomain.com";
270+
271+
private static User getReaderUser(boolean createIfNeeded) throws Exception
272+
{
273+
if (createIfNeeded && !UserManager.userExists(new ValidEmail(READER_USER)))
274+
{
275+
SecurityManager.NewUserStatus nus = SecurityManager.addUser(new ValidEmail(READER_USER), TestContext.get().getUser());
276+
return nus.getUser();
277+
}
278+
279+
return UserManager.getUser(new ValidEmail(READER_USER));
280+
}
281+
}
282+
}

‎discvrcore/src/org/labkey/discvrcore/DiscvrCoreModule.java‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,9 @@
1818

1919
import org.jetbrains.annotations.NotNull;
2020
import org.jetbrains.annotations.Nullable;
21-
import org.labkey.api.data.Container;
2221
import org.labkey.api.module.DefaultModule;
2322
import org.labkey.api.module.ModuleContext;
23+
import org.labkey.api.util.PageFlowUtil;
2424
import org.labkey.api.view.WebPartFactory;
2525

2626
import java.util.Collection;
@@ -60,6 +60,7 @@ protected Collection<WebPartFactory> createWebPartFactories()
6060
protected void init()
6161
{
6262
addController(DiscvrCoreController.NAME, DiscvrCoreController.class);
63+
AuditSummaryUserSchema.register(this);
6364
}
6465

6566
@Override
@@ -70,15 +71,15 @@ public void doStartup(ModuleContext moduleContext)
7071

7172
@Override
7273
@NotNull
73-
public Collection<String> getSummary(Container c)
74+
public Set<String> getSchemaNames()
7475
{
75-
return Collections.emptyList();
76+
return Collections.singleton(DiscvrCoreSchema.NAME);
7677
}
7778

79+
7880
@Override
79-
@NotNull
80-
public Set<String> getSchemaNames()
81+
public @NotNull Set<Class> getIntegrationTests()
8182
{
82-
return Collections.singleton(DiscvrCoreSchema.NAME);
83+
return PageFlowUtil.set(AuditSummaryUserSchema.TestCase.class);
8384
}
8485
}

0 commit comments

Comments
 (0)