Update ETL-Synthea for development updates (#185)

* Address Issue 162 improve version support (#174) * Update functions for supported OMOP versions * Update formatting for lintr issues (#175) * Update package to split intermediate steps and event table loading (#176) * Update package to support synthea 3.1.0 and 3.2.0 (#178) * Add support for loading synthea v3.1 and v3.2 * Add synthea version to source description * Address lint errors * Merge branch 'main' into develop * Add location and caresite support #177 (#181) * Add scripts and vignettes for location and care_site support * Remove redundant join from allvisittable query (#182)
OHDSI · Feb 23, 2024 · a034b71 · a034b71
1 parent ad5d50d
commit a034b71
Show file tree

Hide file tree

Showing 107 changed files with 2,825 additions and 880 deletions.
diff --git a/DESCRIPTION b/DESCRIPTION
@@ -1,12 +1,13 @@
 Package: ETLSyntheaBuilder
 Type: Package
 Title: A Builder for Converting the Synthea Data to the OMOP CDM
-Version: 1.0
+Version: 2.0
 mDate: 2021-12-04
 Author: Anthony Molinaro [aut, cre],
         Clair Blacketer [aut],
-        Frank DeFalco [aut]
-Maintainer: Anthony Molinaro <amolin19@its.jnj.com>
+        Frank DeFalco [aut],
+        Evanette Burrows [aut]
+Maintainer: Evanette Burrows <eburrow3@its.jnj.com>
 Description: ETL and Builder to convert Synthea Data to the OMOP CDM.
     Staring with csv files for an OMOP Vocabulary and csv files for Synthea,
     this package creates database tables from these csv files and maps them
@@ -27,7 +28,7 @@ Remotes:
   OHDSI/CommonDataModel
 Encoding: UTF-8
 LazyData: true
-RoxygenNote: 7.2.2
+RoxygenNote: 7.2.3
 Suggests: 
     rmarkdown,
     knitr,

diff --git a/NAMESPACE b/NAMESPACE
@@ -3,6 +3,8 @@
 export(CreateCDMIndexAndConstraintScripts)
 export(CreateCDMIndices)
 export(CreateCDMTables)
+export(CreateExtraIndices)
+export(CreateMapAndRollupTables)
 export(CreateSyntheaTables)
 export(CreateVisitRollupTables)
 export(CreateVocabMapTables)
@@ -17,7 +19,6 @@ export(LoadVocabFromSchema)
 export(TruncateEventTables)
 export(TruncateVocabTables)
 export(backupCDM)
-export(createExtraIndices)
 export(createPrunedTables)
 export(exportToSQLite)
 export(getEventConceptId)

diff --git a/R/CreateCDMIndexAndConstraintScripts.r b/R/CreateCDMIndexAndConstraintScripts.r
@@ -6,11 +6,11 @@
 #'                                     function \code{createConnectionDetails} in the
 #'                                     \code{DatabaseConnector} package.
 #' @param cdmSchema  The name of the CDM database schema.  Requires read and write permissions to this database. On SQL
-#'                                     Server, this should specifiy both the database and the schema,
+#'                                     Server, this should specify both the database and the schema,
 #'                                     so for example 'cdm_instance.dbo'.
-#' @param cdmVersion Your CDM version.  Currently "5.3.1" and "6.0.0" are supported.
+#' @param cdmVersion Your CDM version.  Currently "5.3.1" and "5.4.0" are supported.
 #' @param githubTag  An optional github tag from which to pull the DDL script.
-#'                   Currently "v5.3.1", "v5.3.1_fixes", "v6.0.0", and "v6.0.0_fixes" are supported.  The default is NULL.
+#'                   Currently "v5.3.1" and "v5.4.0" are supported.  The default is NULL.
 #'
 #' @details This function creates SQL scripts for the indices and constraints on tables in a CDM by referring to the
 #'          correct SQL DDL script in the OHDSI CommonDataModel repo. The database platform is
@@ -29,13 +29,13 @@ CreateCDMIndexAndConstraintScripts <-
   {
     supportedDbs      <-
       c("oracle", "postgresql", "pdw", "netezza", "sql server")
-    supportedVersions <- c("5.3.1", "6.0.0")
+    supportedVersions <- c("5.3.1", "5.4.0")
     supportedTags     <-
-      c("v5.3.1", "v6.0.0", "v5.3.1_fixes", "v6.0.0_fixes")
+      c("v5.3.1", "v5.4.0")
     rdbms <- tolower(connectionDetails$dbms)
 
     if (!(cdmVersion %in% supportedVersions)) {
-      stop("Unsupported CDM specified. Supported CDM versions are \"5.3.1\" and \"6.0.0\"")
+      stop("Unsupported CDM specified. Supported CDM versions are \"5.3.1\" and \"5.4.0\"")
     }
 
     if (!(rdbms %in% supportedDbs)) {
@@ -46,17 +46,17 @@ CreateCDMIndexAndConstraintScripts <-
 
     if (!is.null(githubTag) && !(githubTag %in% supportedTags)) {
       stop(
-        "Unrecognized github tag.  Supported values are \"v5.3.1\", \"v6.0.0\", \"v5.3.1_fixes\", and \"v6.0.0_fixes\""
+        "Unrecognized github tag.  Supported values are \"v5.3.1\" and \"v5.4.0\""
       )
     }
 
     if (cdmVersion == "5.3.1" &&
-        !(githubTag %in% c("v5.3.1", "v5.3.1_fixes"))) {
+          !(githubTag %in% c("v5.3.1"))) {
       stop("cdmVersion and githubTag mismatch.")
     }
 
-    if (cdmVersion == "6.0.0" &&
-        !(githubTag %in% c("v6.0.0", "v6.0.0_fixes"))) {
+    if (cdmVersion == "5.4.0" &&
+          !(githubTag %in% c("v5.4.0"))) {
       stop("cdmVersion and githubTag mismatch.")
     }
 
@@ -74,34 +74,16 @@ CreateCDMIndexAndConstraintScripts <-
             httr::GET(
               "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.3.1/Oracle/OMOP%20CDM%20oracle%20constraints.txt"
             )
-        } else if (githubTag == "v5.3.1_fixes") {
-          webResponseInd <-
-            httr::GET(
-              "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.3.1_fixes/Oracle/OMOP%20CDM%20oracle%20v5_3_1%20primary%20keys.sql"
-            )
-          webResponseCon <-
-            httr::GET(
-              "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.3.1_fixes/Oracle/OMOP%20CDM%20oracle%20v5_3_1%20constraints.sql"
-            )
         }
-      } else if (cdmVersion == "6.0.0") {
-        if (is.null(githubTag) || githubTag == "v6.0.0") {
-          webResponseInd <-
-            httr::GET(
-              "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0.0/Oracle/OMOP%20CDM%20oracle%20pk%20indexes.txt"
-            )
-          webResponseCon <-
-            httr::GET(
-              "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0.0/Oracle/OMOP%20CDM%20oracle%20constraints.txt"
-            )
-        } else if (githubTag == "v6.0.0_fixes") {
+      } else if (cdmVersion == "5.4.0") {
+        if (is.null(githubTag) || githubTag == "v5.4.0") {
           webResponseInd <-
             httr::GET(
-              "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0_fixes/Oracle/OMOP%20CDM%20oracle%20pk%20indexes.txt"
+              "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.4.0/inst/ddl/5.4/oracle/OMOPCDM_oracle_5.4_indices.sql"
             )
           webResponseCon <-
             httr::GET(
-              "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0_fixes/Oracle/OMOP%20CDM%20oracle%20constraints.txt"
+              "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.4.0/inst/ddl/5.4/oracle/OMOPCDM_oracle_5.4_constraints.sql"
             )
         }
       }
@@ -116,34 +98,16 @@ CreateCDMIndexAndConstraintScripts <-
             httr::GET(
               "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.3.1/PostgreSQL/OMOP%20CDM%20postgresql%20constraints.txt"
             )
-        } else if (githubTag == "v5.3.1_fixes") {
-          webResponseInd <-
-            httr::GET(
-              "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.3.1_fixes/PostgreSQL/OMOP%20CDM%20postgresql%20v5_3_1%20primary%20keys.sql"
-            )
-          webResponseCon <-
-            httr::GET(
-              "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.3.1_fixes/PostgreSQL/OMOP%20CDM%20postgresql%20v5_3_1%20constraints.sql"
-            )
         }
-      } else if (cdmVersion == "6.0.0") {
-        if (is.null(githubTag) || githubTag == "v6.0.0") {
-          webResponseInd <-
-            httr::GET(
-              "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0.0/PostgreSQL/OMOP%20CDM%20postgresql%20pk%20indexes.txt"
-            )
-          webResponseCon <-
-            httr::GET(
-              "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0.0/PostgreSQL/OMOP%20CDM%20postgresql%20constraints.txt"
-            )
-        } else if (githubTag == "v6.0.0_fixes") {
+      } else if (cdmVersion == "5.4.0") {
+        if (is.null(githubTag) || githubTag == "v5.4.0") {
           webResponseInd <-
             httr::GET(
-              "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0_fixes/PostgreSQL/OMOP%20CDM%20postgresql%20pk%20indexes.txt"
+              "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.4.0/inst/ddl/5.4/postgresql/OMOPCDM_postgresql_5.4_indices.sql"
             )
           webResponseCon <-
             httr::GET(
-              "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0_fixes/PostgreSQL/OMOP%20CDM%20postgresql%20constraints.txt"
+              "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.4.0/inst/ddl/5.4/postgresql/OMOPCDM_postgresql_5.4_constraints.sql"
             )
         }
       }
@@ -158,34 +122,16 @@ CreateCDMIndexAndConstraintScripts <-
             httr::GET(
               "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.3.1/ParallelDataWarehouse/OMOP%20CDM%20pdw%20constraints.txt"
             )
-        } else if (githubTag == "v5.3.1_fixes") {
-          webResponseInd <-
-            httr::GET(
-              "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.3.1_fixes/ParallelDataWarehouse/OMOP%20CDM%20pdw%20v5_3_1%20indices.sql"
-            )
-          webResponseCon <-
-            httr::GET(
-              "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.3.1_fixes/ParallelDataWarehouse/OMOP%20CDM%20pdw%20v5_3_1%20constraints.sql"
-            )
         }
-      } else if (cdmVersion == "6.0.0") {
-        if (is.null(githubTag) || githubTag == "v6.0.0") {
-          webResponseInd <-
-            httr::GET(
-              "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0.0/ParallelDataWarehouse/OMOP%20CDM%20pdw%20pk%20indexes.txt"
-            )
-          webResponseCon <-
-            httr::GET(
-              "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0.0/ParallelDataWarehouse/OMOP%20CDM%20pdw%20constraints.txt"
-            )
-        } else if (githubTag == "v6.0.0_fixes") {
+      } else if (cdmVersion == "5.4.0") {
+        if (is.null(githubTag) || githubTag == "v5.4.0") {
           webResponseInd <-
             httr::GET(
-              "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0_fixes/ParallelDataWarehouse/OMOP%20CDM%20pdw%20pk%20indexes.txt"
+              "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.4.0/inst/ddl/5.4/pdw/OMOPCDM_pdw_5.4_indices.sql"
             )
           webResponseCon <-
             httr::GET(
-              "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0_fixes/ParallelDataWarehouse/OMOP%20CDM%20pdw%20constraints.txt"
+              "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.4.0/inst/ddl/5.4/pdw/OMOPCDM_pdw_5.4_constraints.sql"
             )
         }
       }
@@ -196,22 +142,12 @@ CreateCDMIndexAndConstraintScripts <-
             httr::GET(
               "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.3.1/Netezza/OMOP%20CDM%20netezza%20primary%20keys.txt"
             )
-        } else if (githubTag == "v5.3.1_fixes") {
-          webResponseInd <-
-            httr::GET(
-              "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.3.1_fixes/Netezza/OMOP%20CDM%20netezza%20v5_3_1%20primary%20keys.sql"
-            )
         }
-      } else if (cdmVersion == "6.0.0") {
-        if (is.null(githubTag) || githubTag == "v6.0.0") {
-          webResponseInd <-
-            httr::GET(
-              "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0.0/Netezza/OMOP%20CDM%20netezza%20primary%20keys.txt"
-            )
-        } else if (githubTag == "v6.0.0_fixes") {
+      } else if (cdmVersion == "5.4.0") {
+        if (is.null(githubTag) || githubTag == "v5.4.0") {
           webResponseInd <-
             httr::GET(
-              "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0_fixes/Netezza/OMOP%20CDM%20netezza%20primary%20keys.txt"
+              "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.4.0/inst/ddl/5.4/netezza/OMOPCDM_netezza_5.4_primary_keys.sql"
             )
         }
       }
@@ -226,34 +162,16 @@ CreateCDMIndexAndConstraintScripts <-
             httr::GET(
               "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.3.1/Sql%20Server/OMOP%20CDM%20sql%20server%20constraints.txt"
             )
-        } else if (githubTag == "v5.3.1_fixes") {
-          webResponseInd <-
-            httr::GET(
-              "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.3.1_fixes/Sql%20Server/OMOP%20CDM%20sql%20server%20v5_3_1%20primary%20keys.sql"
-            )
-          webResponseCon <-
-            httr::GET(
-              "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.3.1_fixes/Sql%20Server/OMOP%20CDM%20sql%20server%20v5_3_1%20constraints.sql"
-            )
         }
-      } else if (cdmVersion == "6.0.0") {
-        if (is.null(githubTag) || githubTag == "v6.0.0") {
-          webResponseInd <-
-            httr::GET(
-              "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0.0/Sql%20Server/OMOP%20CDM%20sql%20server%20pk%20indexes.txt"
-            )
-          webResponseCon <-
-            httr::GET(
-              "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0.0/Sql%20Server/OMOP%20CDM%20sql%20server%20constraints.txt"
-            )
-        } else if (githubTag == "v6.0.0_fixes") {
+      } else if (cdmVersion == "5.4.0") {
+        if (is.null(githubTag) || githubTag == "v5.4.0") {
           webResponseInd <-
             httr::GET(
-              "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0_fixes/Sql%20Server/OMOP%20CDM%20sql%20server%20pk%20indexes.txt"
+              "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.4.0/inst/ddl/5.4/sql_server/OMOPCDM_sql_server_5.4_indices.sql"
             )
           webResponseCon <-
             httr::GET(
-              "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0_fixes/Sql%20Server/OMOP%20CDM%20sql%20server%20constraints.txt"
+              "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.4.0/inst/ddl/5.4/sql_server/OMOPCDM_sql_server_5.4_primary_keys.sql"
             )
         }
       }
@@ -267,30 +185,23 @@ CreateCDMIndexAndConstraintScripts <-
     indexDDL      <- toupper(indexDDL)
     constraintDDL <- toupper(constraintDDL)
 
-    if (githubTag == "v5.3.1_fixes") {
-      indexDDL      <-
-        SqlRender::render(sql = indexDDL, CDMDATABASESCHEMA = cdmSchema)
-      constraintDDL <-
-        SqlRender::render(sql = constraintDDL, CDMDATABASESCHEMA = cdmSchema)
-    } else {
-      indexDDL <- gsub(" ON ", " ON @CDM_SCHEMA.", indexDDL)
-      indexDDL <-
-        gsub("ALTER TABLE", "ALTER TABLE @CDM_SCHEMA.", indexDDL)
-      indexDDL <- gsub("@CDM_SCHEMA. ", "@CDM_SCHEMA.", indexDDL)
-      indexDDL <-
-        SqlRender::render(indexDDL, CDM_SCHEMA = cdmSchema)
-      indexDDL <-
-        SqlRender::translate(indexDDL, targetDialect = rdbms)
-
-      constraintDDL <-
-        gsub("ALTER TABLE", "ALTER TABLE @CDM_SCHEMA.", constraintDDL)
-      constraintDDL <-
-        gsub("@CDM_SCHEMA. ", "@CDM_SCHEMA.", constraintDDL)
-      constraintDDL <-
-        SqlRender::render(constraintDDL, CDM_SCHEMA = cdmSchema)
-      constraintDDL <-
-        SqlRender::translate(constraintDDL, targetDialect = rdbms)
-    }
+    indexDDL <- gsub(" ON ", " ON @CDM_SCHEMA.", indexDDL)
+    indexDDL <-
+      gsub("ALTER TABLE", "ALTER TABLE @CDM_SCHEMA.", indexDDL)
+    indexDDL <- gsub("@CDM_SCHEMA. ", "@CDM_SCHEMA.", indexDDL)
+    indexDDL <-
+      SqlRender::render(indexDDL, CDM_SCHEMA = cdmSchema)
+    indexDDL <-
+      SqlRender::translate(indexDDL, targetDialect = rdbms)
+
+    constraintDDL <-
+      gsub("ALTER TABLE", "ALTER TABLE @CDM_SCHEMA.", constraintDDL)
+    constraintDDL <-
+      gsub("@CDM_SCHEMA. ", "@CDM_SCHEMA.", constraintDDL)
+    constraintDDL <-
+      SqlRender::render(constraintDDL, CDM_SCHEMA = cdmSchema)
+    constraintDDL <-
+      SqlRender::translate(constraintDDL, targetDialect = rdbms)
 
     # Save the translated sql ddl to be run at a later time
     if (!dir.exists("output")) {

diff --git a/R/CreateCDMIndices.r b/R/CreateCDMIndices.r
@@ -6,7 +6,7 @@
 #'                                     function \code{createConnectionDetails} in the
 #'                                     \code{DatabaseConnector} package.
 #' @param cdmSchema  The name of the CDM database schema.  Requires read and write permissions to this database. On SQL
-#'                                     Server, this should specifiy both the database and the schema,
+#'                                     Server, this should specify both the database and the schema,
 #'                                     so for example 'cdm_instance.dbo'.
 #' @param cdmVersion Your CDM version.  Currently "5.3" and "5.4" are supported.
 #' @param outputFolder Location of the SQL scripts if sqlOnly = TRUE. Default is NULL.
@@ -25,23 +25,23 @@ CreateCDMIndices <-
            cdmVersion,
            outputFolder = NULL,
            sqlOnly = FALSE)
-{
+  {
     if (!sqlOnly) {
-
-		print("Creating Indices on CDM Tables....")
+      print("Creating Indices on CDM Tables....")
 
-		indexSQLFile <- CommonDataModel::writeIndex(
-		targetDialect     = connectionDetails$dbms,
-		cdmVersion        = cdmVersion,
-		cdmDatabaseSchema = cdmSchema,
-		outputfolder      = tempdir())
+      indexSQLFile <- CommonDataModel::writeIndex(
+        targetDialect     = connectionDetails$dbms,
+        cdmVersion        = cdmVersion,
+        cdmDatabaseSchema = cdmSchema,
+        outputfolder      = tempdir()
+      )
+
+      indexDDL <- SqlRender::readSql(paste0(tempdir(), "/", indexSQLFile))
+      conn <- DatabaseConnector::connect(connectionDetails)
+      DatabaseConnector::executeSql(conn, indexDDL)
+      DatabaseConnector::disconnect(conn)
+      print("Index Creation Complete.")
 
-		indexDDL <- SqlRender::readSql(paste0(tempdir(),"/",indexSQLFile))
-		conn <- DatabaseConnector::connect(connectionDetails)
-		DatabaseConnector::executeSql(conn,indexDDL)
-		DatabaseConnector::disconnect(conn)
-		print("Index Creation Complete.")
-
     } else {
       if (is.null(outputFolder)) {
         stop("Must specify an outputFolder location when using sqlOnly = TRUE")
@@ -56,4 +56,4 @@ CreateCDMIndices <-
         cdmDatabaseSchema = cdmSchema
       )
     }
-}
+  }