Skip to content

Commit c54ca96

Browse files
bbimberbbimber
committed
Initial PRIMe ETLs
1 parent 12b829a commit c54ca96

File tree

5 files changed

+168
-0
lines changed

5 files changed

+168
-0
lines changed

‎primeseq/resources/etls/prime-blooddraws.xml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<etl xmlns="http://labkey.org/etl/xml" standalone="false">
3+
<name>PRIME_Blood_Data</name>
4+
<description>PRIMe Blood Data</description>
5+
<transforms>
6+
<transform type="RemoteQueryTransformStep" id="blood">
7+
<description>Copy to target</description>
8+
<source remoteSource="PRIME" schemaName="study" queryName="blood" sourceTimeout="0">
9+
<sourceColumns>
10+
<column>Id</column>
11+
<column>date</column>
12+
<column>reason</column>
13+
<column>quantity</column>
14+
<column>objectid</column>
15+
<column>QCState</column>
16+
</sourceColumns>
17+
</source>
18+
<destination schemaName="study" queryName="blood" targetOption="merge" bulkLoad="true">
19+
20+
</destination>
21+
</transform>
22+
</transforms>
23+
<incrementalFilter timestampColumnName="modified" pkColumnName="lsid" className="ModifiedSinceFilterStrategy" >
24+
<deletedRowsSource remoteSource="PRIME" schemaName="auditlog" queryName="DatasetAuditEvent" timestampColumnName="Created" sourceTimeout="0" deletedSourceKeyColumnName="lsid" targetKeyColumnName="lsid">
25+
<sourceFilters>
26+
<sourceFilter column="datasetid/Name" operator="eq" value="blood" />
27+
</sourceFilters>
28+
</deletedRowsSource>
29+
</incrementalFilter>
30+
</etl>

‎primeseq/resources/etls/prime-demographics.xml‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<etl xmlns="http://labkey.org/etl/xml" standalone="false">
3+
<name>PRIME_Demographics_Data</name>
4+
<description>PRIMe Demographics Data</description>
5+
<transforms>
6+
<transform type="RemoteQueryTransformStep" id="demographics">
7+
<description>Copy to target</description>
8+
<source remoteSource="PRIME" schemaName="study" queryName="Demographics" sourceTimeout="0">
9+
<sourceColumns>
10+
<column>Id</column>
11+
<column>date</column>
12+
<column>gender</column>
13+
<column>geographic_origin</column>
14+
<column>birth</column>
15+
<column>death</column>
16+
<column>species</column>
17+
<column>status</column>
18+
<column>objectid</column>
19+
<column>QCState</column>
20+
</sourceColumns>
21+
</source>
22+
<destination schemaName="study" queryName="Demographics" targetOption="merge" bulkLoad="true">
23+
24+
</destination>
25+
</transform>
26+
</transforms>
27+
<incrementalFilter timestampColumnName="modified" pkColumnName="lsid" className="ModifiedSinceFilterStrategy" >
28+
<deletedRowsSource remoteSource="PRIME" schemaName="auditlog" queryName="DatasetAuditEvent" timestampColumnName="Created" sourceTimeout="0" deletedSourceKeyColumnName="lsid" targetKeyColumnName="lsid">
29+
<sourceFilters>
30+
<sourceFilter column="datasetid/Name" operator="eq" value="blood" />
31+
</sourceFilters>
32+
</deletedRowsSource>
33+
</incrementalFilter>
34+
</etl>

‎primeseq/resources/etls/prime-parentage.xml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<etl xmlns="http://labkey.org/etl/xml" standalone="false">
3+
<name>PRIME_Parentage_Data</name>
4+
<description>PRIMe Parentage Data</description>
5+
<transforms>
6+
<transform type="RemoteQueryTransformStep" id="demographics">
7+
<description>Copy to target</description>
8+
<source remoteSource="PRIME" schemaName="study" queryName="Parentage" sourceTimeout="0">
9+
<sourceColumns>
10+
<column>Id</column>
11+
<column>date</column>
12+
<column>parent</column>
13+
<column>relationship</column>
14+
<column>method</column>
15+
<column>datedisabled</column>
16+
<column>objectid</column>
17+
<column>QCState</column>
18+
</sourceColumns>
19+
</source>
20+
<destination schemaName="study" queryName="Parentage" targetOption="merge" bulkLoad="true">
21+
22+
</destination>
23+
</transform>
24+
</transforms>
25+
<incrementalFilter timestampColumnName="modified" pkColumnName="lsid" className="ModifiedSinceFilterStrategy" >
26+
<deletedRowsSource remoteSource="PRIME" schemaName="auditlog" queryName="DatasetAuditEvent" timestampColumnName="Created" sourceTimeout="0" deletedSourceKeyColumnName="lsid" targetKeyColumnName="lsid">
27+
<sourceFilters>
28+
<sourceFilter column="datasetid/Name" operator="eq" value="parentage" />
29+
</sourceFilters>
30+
</deletedRowsSource>
31+
</incrementalFilter>
32+
</etl>

‎primeseq/resources/etls/prime-weight.xml‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<etl xmlns="http://labkey.org/etl/xml" standalone="false">
3+
<name>PRIME_Parentage_Data</name>
4+
<description>PRIMe Parentage Data</description>
5+
<transforms>
6+
<transform type="RemoteQueryTransformStep" id="weight">
7+
<description>Copy to target</description>
8+
<source remoteSource="PRIME" schemaName="study" queryName="weight" sourceTimeout="0">
9+
<sourceColumns>
10+
<column>Id</column>
11+
<column>date</column>
12+
<column>weight</column>
13+
<column>objectid</column>
14+
<column>QCState</column>
15+
</sourceColumns>
16+
</source>
17+
<destination schemaName="study" queryName="weight" targetOption="merge" bulkLoad="true">
18+
19+
</destination>
20+
</transform>
21+
</transforms>
22+
<incrementalFilter timestampColumnName="modified" pkColumnName="lsid" className="ModifiedSinceFilterStrategy" >
23+
<deletedRowsSource remoteSource="PRIME" schemaName="auditlog" queryName="DatasetAuditEvent" timestampColumnName="Created" sourceTimeout="0" deletedSourceKeyColumnName="lsid" targetKeyColumnName="lsid">
24+
<sourceFilters>
25+
<sourceFilter column="datasetid/Name" operator="eq" value="weight" />
26+
</sourceFilters>
27+
</deletedRowsSource>
28+
</incrementalFilter>
29+
</etl>

‎primeseq/resources/etls/prime.xml‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<etl xmlns="http://labkey.org/etl/xml">
3+
<name>PRIME_Data</name>
4+
<description>PRIMe Clinical/Demographics Data</description>
5+
<transforms>
6+
<transform id="demographics" type="TaskrefTransformStep">
7+
<taskref ref="org.labkey.di.steps.QueueJobTask">
8+
<settings>
9+
<setting name="transformId" value="{primeseq}/prime-demographics"/>
10+
</settings>
11+
</taskref>
12+
</transform>
13+
14+
<transform id="parentage" type="TaskrefTransformStep">
15+
<taskref ref="org.labkey.di.steps.QueueJobTask">
16+
<settings>
17+
<setting name="transformId" value="{primeseq}/prime-parentage"/>
18+
</settings>
19+
</taskref>
20+
</transform>
21+
22+
<transform id="weight" type="TaskrefTransformStep">
23+
<taskref ref="org.labkey.di.steps.QueueJobTask">
24+
<settings>
25+
<setting name="transformId" value="{primeseq}/prime-weight"/>
26+
</settings>
27+
</taskref>
28+
</transform>
29+
30+
<transform id="blooddraws" type="TaskrefTransformStep">
31+
<taskref ref="org.labkey.di.steps.QueueJobTask">
32+
<settings>
33+
<setting name="transformId" value="{primeseq}/prime-blooddraws"/>
34+
</settings>
35+
</taskref>
36+
</transform>
37+
38+
</transforms>
39+
40+
<schedule>
41+
<cron expression="0 0 3 * * ?"/>
42+
</schedule>
43+
</etl>

0 commit comments

Comments
 (0)