Data validation for SNPRC ETL

bbimber · bbimber · commit 3081251660d4 · 2024-04-17T08:26:03.000-07:00
diff --git a/mcc/resources/etls/snprc-datasets.xml b/mcc/resources/etls/snprc-datasets.xml
@@ -136,6 +136,10 @@
                     <column>weight</column>
                     <column>objectid</column>
                 </sourceColumns>
+                <sourceFilters>
+                    <sourceFilter column="weight" operator="gt" value="0"/>
+                    <sourceFilter column="weight" operator="isnonblank" value=""/>
+                </sourceFilters>
             </source>
             <destination schemaName="study" queryName="weight" targetOption="merge" bulkLoad="true" batchSize="20000">
                 <columnTransforms>
diff --git a/mcc/resources/queries/study/demographics.js b/mcc/resources/queries/study/demographics.js
@@ -110,6 +110,10 @@ function onUpsert(helper, scriptErrors, row, oldRow){
             idToMccAlias[row.sire] = row.sireMccAlias;
         }
     }
+
+    if (!row.date) {
+        row.date = new Date();
+    }
 }
 
 function onComplete(event, errors, helper){

Original file line number	Diff line number	Diff line change
`@@ -110,6 +110,10 @@ function onUpsert(helper, scriptErrors, row, oldRow){`
`110`	`110`	`idToMccAlias[row.sire] = row.sireMccAlias;`
`111`	`111`	`}`
`112`	`112`	`}`
	`113`	`+`
	`114`	`+ if (!row.date) {`
	`115`	`+ row.date = new Date();`
	`116`	`+ }`
`113`	`117`	`}`
`114`	`118`
`115`	`119`	`function onComplete(event, errors, helper){`