Create ETLs to transform SND Lookup and LookupSet data to CAMP tables (…

…#613)

* added ETLs for Lookups and LookupSets

* updated ETLs with incrementalFilters

* changed query for deleteRowsSource

* changes to etl code

* removed old lookup table ETLs and fixed query joins

* removed web gen files

* changed case when statement in lookup query

snprc_ehr/resources/etls/ExportSndLookupSets.xml

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<etl xmlns="http://labkey.org/etl/xml">
+    <name>Export SND LookupSets to Animal PKG_ATTRIB_LOOKUPS</name>
+    <description>Transform Lookup Sets for SND module to Animal PKG_ATTRIB_LOOKUPS data</description>
+    <transforms>
+        <transform id="step1" type="org.labkey.di.pipeline.TransformTask">
+            <description>Merge to target</description>
+            <source schemaName="snprc_ehr" queryName="ExportSndLookupSets" />
+            <destination schemaName="animalExport" queryName="PKG_ATTRIB_LOOKUPS" bulkLoad="true" targetOption="merge">
+                <alternateKeys>
+                    <column name="OBJECT_ID" />
+                </alternateKeys>
+            </destination>
+        </transform>
+    </transforms>
+    <incrementalFilter className="ModifiedSinceFilterStrategy" timestampColumnName="ENTRY_DATE_TM">
+        <deletedRowsSource schemaName="snd" queryName="DeletedLookupSets" timestampColumnName="modified" deletedSourceKeyColumnName="objectId" targetKeyColumnName="object_id"/>
+    </incrementalFilter>
+    <schedule>
+        <poll interval="60m" />
+    </schedule>
+</etl>

snprc_ehr/resources/etls/ExportSndLookups.xml

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<etl xmlns="http://labkey.org/etl/xml">
+    <name>Export SND Lookups to Animal LOOKUP_TABLE</name>
+    <description>Transform Lookup items for SND module to Animal LOOKUP_TABLE data</description>
+    <transforms>
+        <transform id="step1" type="org.labkey.di.pipeline.TransformTask">
+            <description>Merge to target</description>
+            <source schemaName="snprc_ehr" queryName="ExportSndLookups" />
+            <destination schemaName="animalExport" queryName="LOOKUP_TABLE" bulkLoad="true" targetOption="merge">
+                <alternateKeys>
+                    <column name="OBJECT_ID" />
+                </alternateKeys>
+            </destination>
+        </transform>
+    </transforms>
+    <incrementalFilter className="ModifiedSinceFilterStrategy" timestampColumnName="ENTRY_DATE_TM">
+        <deletedRowsSource schemaName="snd" queryName="DeletedLookups" timestampColumnName="modified" deletedSourceKeyColumnName="objectId" targetKeyColumnName="object_id"/>
+    </incrementalFilter>
+    <schedule>
+        <poll interval="60m" />
+    </schedule>
+</etl>

snprc_ehr/resources/queries/snprc_ehr/ExportSndLookupSets.sql

@@ -0,0 +1,14 @@
+SELECT
+    SetName as LOOKUP_KEY,
+    ObjectId as OBJECT_ID,
+    LEFT(s.Email,
+    CASE WHEN charindex('@', s.Email) = 0
+    THEN LEN(s.Email)
+    ELSE (charindex('@', s.Email) - 1) END) as USER_NAME,
+    ls.Modified as ENTRY_DATE_TM
+FROM
+    snd.LookupSets ls
+    INNER JOIN
+    core.SiteUsers s
+ON
+    ls.ModifiedBy = s.UserId

snprc_ehr/resources/queries/snprc_ehr/ExportSndLookups.sql

@@ -0,0 +1,24 @@
+SELECT
+    LookupId as LOOKUP_ID,
+    ls.SetName as LOOKUP_KEY,
+    Value as VALUE,
+    CASE
+        WHEN l.Displayable = 0 THEN 'Y' ELSE 'N' END as IS_HIDDEN,
+    l.SortOrder as ORDER_NUM,
+    l.ObjectId as OBJECT_ID,
+    'N' as DEFAULT_FLAG,
+    LEFT(s.Email,
+	CASE WHEN charindex('@', s.Email) = 0
+	THEN LEN(s.Email)
+	ELSE (charindex('@', s.Email) - 1) END) as USER_NAME,
+    l.Modified as ENTRY_DATE_TM
+FROM
+    snd.Lookups l
+INNER JOIN
+    snd.LookupSets ls
+ON
+    l.LookupSetId = ls.LookupSetId
+INNER JOIN
+    core.SiteUsers s
+ON
+    l.modifiedBy = s.UserId