From 5a8236d8905eb1c5da2f3334b590766c3897ed59 Mon Sep 17 00:00:00 2001
From: Gary Jones <jonesga@ohsu.edu>
Date: Thu, 11 Jan 2024 14:55:13 -0700
Subject: [PATCH] Update of ETL Processes for Randall Data removes old ETL and
 inserts a new one (#891)

---
 .../resources/etls/HansenQueriesToFile.xml    | 105 ------------------
 .../resources/etls/mfshtoRandalData_2024.xml  | 105 ++++++++++++++++++
 2 files changed, 105 insertions(+), 105 deletions(-)
 delete mode 100644 onprc_ehr/resources/etls/HansenQueriesToFile.xml
 create mode 100644 onprc_ehr/resources/etls/mfshtoRandalData_2024.xml

diff --git a/onprc_ehr/resources/etls/HansenQueriesToFile.xml b/onprc_ehr/resources/etls/HansenQueriesToFile.xml
deleted file mode 100644
index 640fd5763..000000000
--- a/onprc_ehr/resources/etls/HansenQueriesToFile.xml
+++ /dev/null
@@ -1,105 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-
-<etl xmlns="http://labkey.org/etl/xml" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-     xsi:schemaLocation="http://labkey.org/etl/xml ">
-    <name>mfshToRandalData</name>    <description>imports the data from mfsh animal details on ONPRC</description>
-    <description>imports the data from mfsh animal details on ONPRC</description>
-    <transforms>
-        <transform id="step1">
-            <description>Define Study Animals</description>
-            <source queryName="studyAnimals" schemaName="Randal_data"/>
-            <destination schemaName="Onprc_ehr" queryName="StudyDetails_RandalData"/>
-        </transform>
-        <transform id="step2">
-            <description>Export Query to File Share</description>
-            <source queryName="RandData_AnimalGroups" schemaName="onprc_ehr"/>
-            <destination type="file" dir="hansonFiles" fileBaseName="AnimalGroups-${TransformRunId}-${Timestamp}" fileExtension="csv" />
-        </transform>
-        <transform id="step3">
-            <description>Export Query to File Share</description>
-            <source queryName="RandData_blood" schemaName="onprc_ehr"/>
-            <destination type="file" dir="hansonFiles" fileBaseName="BloodDetails-${TransformRunId}-${Timestamp}" fileExtension="csv" />
-        </transform>
-        <transform id="step3a">
-            <description>Export Query to File Share</description>
-            <source queryName="RandData_birth" schemaName="onprc_ehr"/>
-            <destination type="file" dir="hansonFiles" fileBaseName="BirthDetails-${TransformRunId}-${Timestamp}" fileExtension="csv" />
-        </transform>
-        <transform id="step3b">
-            <description>Export Query to File Share</description>
-            <source queryName="RandData_assignment" schemaName="onprc_ehr"/>
-            <destination type="file" dir="hansonFiles" fileBaseName="assignment-${TransformRunId}-${Timestamp}" fileExtension="csv" />
-        </transform>
-        <transform id="step3c">
-            <description>Export Query to File Share</description>
-            <source queryName="RandData_arrival" schemaName="onprc_ehr"/>
-            <destination type="file" dir="hansonFiles" fileBaseName="arrival-${TransformRunId}-${Timestamp}" fileExtension="csv" />
-        </transform>
-        <transform id="step3d">
-            <description>Export Query to File Share</description>
-            <source queryName="RandData_animalgroups" schemaName="onprc_ehr"/>
-            <destination type="file" dir="hansonFiles" fileBaseName="animal_groups-${TransformRunId}-${Timestamp}" fileExtension="csv" />
-        </transform>
-        <transform id="step3e">
-            <description>Export Query to File Share</description>
-            <source queryName="RandData_weights" schemaName="onprc_ehr"/>
-            <destination type="file" dir="hansonFiles" fileBaseName="weights-${TransformRunId}-${Timestamp}" fileExtension="csv" />
-        </transform>
-        <transform id="step3f">
-            <description>Export Query to File Share</description>
-            <source queryName="RandData_Cases" schemaName="onprc_ehr"/>
-            <destination type="file" dir="hansonFiles" fileBaseName="Cases-${TransformRunId}-${Timestamp}" fileExtension="csv" />
-        </transform>
-        <transform id="step3g">
-            <description>Export Query to File Share</description>
-            <source queryName="RandData_clinremarks" schemaName="onprc_ehr"/>
-            <destination type="file" dir="hansonFiles" fileBaseName="clinremarks-${TransformRunId}-${Timestamp}" fileExtension="csv" />
-        </transform>
-        <transform id="step3h">
-            <description>Export Query to File Share</description>
-            <source queryName="RandData_demographics" schemaName="onprc_ehr"/>
-            <destination type="file" dir="hansonFiles" fileBaseName="demographics-${TransformRunId}-${Timestamp}" fileExtension="csv" />
-        </transform>
-        <transform id="step3i">
-            <description>Export Query to File Share</description>
-            <source queryName="RandData_deaths" schemaName="onprc_ehr"/>
-            <destination type="file" dir="hansonFiles" fileBaseName="deaths-${TransformRunId}-${Timestamp}" fileExtension="csv" />
-        </transform>
-        <transform id="step3j">
-            <description>Export Query to File Share</description>
-            <source queryName="RandData_departure" schemaName="onprc_ehr"/>
-            <destination type="file" dir="hansonFiles" fileBaseName="departure-${TransformRunId}-${Timestamp}" fileExtension="csv" />
-        </transform>
-        <transform id="step3k">
-            <description>Export Query to File Share</description>
-            <source queryName="RandData_drug" schemaName="onprc_ehr"/>
-            <destination type="file" dir="hansonFiles" fileBaseName="drug-${TransformRunId}-${Timestamp}" fileExtension="csv" />
-        </transform>
-        <transform id="step3l">
-            <description>Export Query to File Share</description>
-            <source queryName="RandData_flags" schemaName="onprc_ehr"/>
-            <destination type="file" dir="hansonFiles" fileBaseName="flags-${TransformRunId}-${Timestamp}" fileExtension="csv" />
-        </transform>
-        <transform id="step3m">
-            <description>Export Query to File Share</description>
-            <source queryName="RandData_Hematology" schemaName="onprc_ehr"/>
-            <destination type="file" dir="hansonFiles" fileBaseName="Hematology-${TransformRunId}-${Timestamp}" fileExtension="csv" />
-        </transform>
-        <transform id="step3n">
-            <description>Export Query to File Share</description>
-            <source queryName="RandData_Housing" schemaName="onprc_ehr"/>
-            <destination type="file" dir="hansonFiles" fileBaseName="Housing-${TransformRunId}-${Timestamp}" fileExtension="csv" />
-        </transform>
-        <transform id="step3o">
-            <description>Export Query to File Share</description>
-            <source queryName="RandData_MHCData" schemaName="onprc_ehr"/>
-            <destination type="file" dir="hansonFiles" fileBaseName="MHCData-${TransformRunId}-${Timestamp}" fileExtension="csv" />
-        </transform>
-        <transform id="step3p">
-            <description>Export Query to File Share</description>
-            <source queryName="RandData_Notes" schemaName="onprc_ehr"/>
-            <destination type="file" dir="hansonFiles" fileBaseName="Notes-${TransformRunId}-${Timestamp}" fileExtension="csv" />
-        </transform>
-    </transforms>
-
-</etl>
diff --git a/onprc_ehr/resources/etls/mfshtoRandalData_2024.xml b/onprc_ehr/resources/etls/mfshtoRandalData_2024.xml
new file mode 100644
index 000000000..5bc8a444e
--- /dev/null
+++ b/onprc_ehr/resources/etls/mfshtoRandalData_2024.xml
@@ -0,0 +1,105 @@
+<etl xmlns="http://labkey.org/etl/xml">
+    <name>mfshToRandalData_2024</name>
+    <description>imports the data from mfsh animal details on ONPRC</description>
+    <transforms>
+        <transform id="step1">
+            <description>Define Hansen Study Animals 2022</description>
+            <source queryName="studyAnimals" schemaName="Randal_data"/>
+            <destination queryName="StudyDetails_RandalData" schemaName="Onprc_ehr" targetOption="truncate"/>
+        </transform>
+        <transform id="step2">
+            <description>Export Query to File Share</description>
+            <source queryName="RandData_AnimalGroups" schemaName="onprc_ehr"/>
+            <destination dir="hansenFiles" fileBaseName="AnimalGroups-${TransformRunId}-${Timestamp}" fileExtension="csv" type="file"/>
+        </transform>
+        <transform id="step3">
+            <description>Export Query to File Share</description>
+            <source queryName="RandData_blood" schemaName="onprc_ehr"/>
+            <destination dir="hansenFiles" fileBaseName="BloodDetails-${TransformRunId}-${Timestamp}" fileExtension="csv" type="file"/>
+        </transform>
+        <transform id="step3a">
+            <description>Export Query to File Share</description>
+            <source queryName="RandData_birth" schemaName="onprc_ehr"/>
+            <destination dir="hansenFiles" fileBaseName="BirthDetails-${TransformRunId}-${Timestamp}" fileExtension="csv" type="file"/>
+        </transform>
+        <transform id="step3b">
+            <description>Export Query to File Share</description>
+            <source queryName="RandData_assignment" schemaName="onprc_ehr"/>
+            <destination dir="hansenFiles" fileBaseName="assignment-${TransformRunId}-${Timestamp}" fileExtension="csv" type="file"/>
+        </transform>
+        <transform id="step3c">
+            <description>Export Query to File Share</description>
+            <source queryName="RandData_arrival" schemaName="onprc_ehr"/>
+            <destination dir="hansenFiles" fileBaseName="arrival-${TransformRunId}-${Timestamp}" fileExtension="csv" type="file"/>
+        </transform>
+        <transform id="step3e">
+            <description>Export Query to File Share</description>
+            <source queryName="RandData_weights" schemaName="onprc_ehr"/>
+            <destination dir="hansenFiles" fileBaseName="weights-${TransformRunId}-${Timestamp}" fileExtension="csv" type="file"/>
+        </transform>
+        <transform id="step3f">
+            <description>Export Query to File Share</description>
+            <source queryName="RandData_Cases" schemaName="onprc_ehr"/>
+            <destination dir="hansenFiles" fileBaseName="Cases-${TransformRunId}-${Timestamp}" fileExtension="csv" type="file"/>
+        </transform>
+        <transform id="step3g">
+            <description>Export Query to File Share</description>
+            <source queryName="RandData_clinremarks" schemaName="onprc_ehr"/>
+            <destination dir="hansenFiles" fileBaseName="clinremarks-${TransformRunId}-${Timestamp}" fileExtension="csv" type="file"/>
+        </transform>
+        <transform id="step3h">
+            <description>Export Query to File Share</description>
+            <source queryName="RandData_demographics" schemaName="onprc_ehr"/>
+            <destination dir="hansenFiles" fileBaseName="demographics-${TransformRunId}-${Timestamp}" fileExtension="csv" type="file"/>
+        </transform>
+        <transform id="step3i">
+            <description>Export Query to File Share</description>
+            <source queryName="RandData_deaths" schemaName="onprc_ehr"/>
+            <destination dir="hansenFiles" fileBaseName="deaths-${TransformRunId}-${Timestamp}" fileExtension="csv" type="file"/>
+        </transform>
+        <transform id="step3j">
+            <description>Export Query to File Share</description>
+            <source queryName="RandData_departure" schemaName="onprc_ehr"/>
+            <destination dir="hansenFiles" fileBaseName="departure-${TransformRunId}-${Timestamp}" fileExtension="csv" type="file"/>
+        </transform>
+        <transform id="step3k">
+            <description>Export Query to File Share</description>
+            <source queryName="RandData_drug" schemaName="onprc_ehr"/>
+            <destination dir="hansenFiles" fileBaseName="drug-${TransformRunId}-${Timestamp}" fileExtension="csv" type="file"/>
+        </transform>
+        <transform id="step3l">
+            <description>Export Query to File Share</description>
+            <source queryName="RandData_flags" schemaName="onprc_ehr"/>
+            <destination dir="hansenFiles" fileBaseName="flags-${TransformRunId}-${Timestamp}" fileExtension="csv" type="file"/>
+        </transform>
+        <transform id="step3m">
+            <description>Export Query to File Share</description>
+            <source queryName="RandData_Hematology" schemaName="onprc_ehr"/>
+            <destination dir="hansenFiles" fileBaseName="Hematology-${TransformRunId}-${Timestamp}" fileExtension="csv" type="file"/>
+        </transform>
+        <transform id="step3n">
+            <description>Export Query to File Share</description>
+            <source queryName="RandData_Housing" schemaName="onprc_ehr"/>
+            <destination dir="hansenFiles" fileBaseName="Housing-${TransformRunId}-${Timestamp}" fileExtension="csv" type="file"/>
+        </transform>
+        <transform id="step3q">
+            <description>Export Query to File Share Project</description>
+            <source queryName="project" schemaName="pf_ehrData"/>
+            <destination dir="hansenFiles" fileBaseName="Projects-${TransformRunId}-${Timestamp}" fileExtension="csv" type="file"/>
+        </transform>
+        <transform id="step3r">
+            <description>Export Query to File Share Protocols</description>
+            <source queryName="protocol" schemaName="pf_ehrData"/>
+            <destination dir="hansenFiles" fileBaseName="Protocol-${TransformRunId}-${Timestamp}" fileExtension="csv" type="file"/>
+        </transform>
+        <transform id="step3s">
+            <description>Export Query to File Share</description>
+            <source queryName="RandalData_bcs" schemaName="onprc_ehr"/>
+            <destination dir="hansenFiles" fileBaseName="Notes-${TransformRunId}-${Timestamp}" fileExtension="csv" type="file"/>
+        </transform>
+    </transforms>
+    <schedule>
+        <!--5AM-->
+        <cron expression="0 0 05 * * ?"/>
+    </schedule>
+</etl>
\ No newline at end of file