Merge pull request #160 from ljwoodley/add_project_and_instance_fields_to_rcc_job_log

Add project and instance fields to rcc job log

Author: pbchase

NAMESPACE

@@ -23,7 +23,9 @@ export(get_current_time)
 export(get_institutional_person_data)
 export(get_job_duration)
 export(get_package_scope_var)
+export(get_project_instance)
 export(get_project_life_cycle)
+export(get_project_name)
 export(get_redcap_credentials)
 export(get_redcap_db_connection)
 export(get_redcap_email_revisions)
@@ -45,6 +47,8 @@ export(scrape_user_api_tokens)
 export(send_email)
 export(set_package_scope_var)
 export(set_project_api_token)
+export(set_project_instance)
+export(set_project_name)
 export(set_script_name)
 export(set_script_run_time)
 export(set_super_api_token)

R/logging.R

@@ -29,6 +29,8 @@ build_etl_job_log_df <- function(job_duration, job_summary, level) {
   ) %>%
   dplyr::mutate(
     log_date = get_current_time(),
+    project = get_project_name(),
+    instance = get_project_instance(),
     script_name = get_script_name(),
     script_run_time = get_script_run_time(),
     job_duration = .data$job_duration,
@@ -65,6 +67,8 @@ build_formatted_df_from_result <- function(result, database_written, table_writt
     dplyr::mutate(record_level_data = purrr::pmap(.data, ~ rjson::toJSON(c(...)))) %>%
     dplyr::select(primary_key = pk_col, .data$record_level_data) %>%
     dplyr::mutate(
+      project = get_project_name(),
+      instance = get_project_instance(),
       script_name = get_script_name(),
       script_run_time = get_script_run_time(),
       log_date = get_current_time(),
@@ -130,6 +134,19 @@ get_script_run_time <- function() {
   return(redcapcustodian.env$script_run_time)
 }
 
+#' Fetches the package-scoped value of project_name
+#' @export
+get_project_name <- function() {
+  return(redcapcustodian.env$project_name)
+}
+
+#' Fetches the package-scoped value of project_instance
+#' @export
+get_project_instance <- function() {
+  return(redcapcustodian.env$project_instance)
+}
+
+
 #' Initialize the connection to the log db and set redcapcustodian.env$log_con
 #'
 #' @param drv, an object that inherits from DBIDriver (e.g. RMariaDB::MariaDB()), or an existing DBIConnection object (in order to clone an existing connection).
@@ -205,6 +222,55 @@ set_script_run_time <- function(fake_runtime = lubridate::NA_POSIXct_) {
   return(redcapcustodian.env$script_run_time)
 }
 
+#' Sets the package-scoped value of project_name
+#'
+#' @param project_name Defaults to NULL. If provided and not NULL, this value is used.
+#'                     If NULL, the function attempts to fetch the value from the environment variable.
+#' @return the package-scoped value of project_name
+
+#' @examples
+#' \dontrun{
+#' project_name <- set_project_name()
+#' project_name <- set_project_name("project_name")
+#' }
+#'
+#' @export
+set_project_name <- function(project_name = "") {
+  if (project_name == "") {
+    project_name <- Sys.getenv("PROJECT")
+  }
+
+  assign("project_name",
+         project_name,
+         envir = redcapcustodian.env)
+
+  return(redcapcustodian.env$project_name)
+}
+
+#' Sets the package-scoped value of project_instance
+#' @param project_instance Defaults to NULL. If provided and not NULL, this value is used.
+#'                     If NULL, the function attempts to fetch the value from the environment variable.
+#'
+#' @return the package-scoped value of project_instance
+#' @examples
+#' \dontrun{
+#' project_instance <- set_project_instance()
+#' project_instance <- set_project_instance("project_instance")
+#' }
+#'
+#' @export
+set_project_instance <- function(project_instance = "") {
+  if (project_instance == "") {
+    project_instance <- Sys.getenv("INSTANCE")
+  }
+
+  assign("project_instance",
+         project_instance,
+         envir = redcapcustodian.env)
+
+  return(redcapcustodian.env$project_instance)
+}
+
 #' Attempts to connect to the DB using all LOG_DB_* environment variables. Returns an empty list if a connection is established, returns an `error_list` entry otherwise.
 #'
 #' @param drv, an object that inherits from DBIDriver (e.g. RMariaDB::MariaDB()), or an existing DBIConnection object (in order to clone an existing connection).

R/utils.R

@@ -47,6 +47,8 @@ get_package_scope_var  <- function(key) {
 
 #' Initialize all etl dependencies
 #'
+#' @param project_name name passed to \code{\link{set_project_name}}
+#' @param project_instance name passed to \code{\link{set_project_instance}}
 #' @param script_name name passed to \code{\link{set_script_name}}
 #' @param fake_runtime An optional asserted script run time passed to \code{\link{set_script_run_time}}, defaults to the time this function is called
 #' @param log_db_drv, an object that inherits from DBIDriver (e.g. RMariaDB::MariaDB()), or an existing DBIConnection object (in order to clone an existing connection).
@@ -57,7 +59,9 @@ get_package_scope_var  <- function(key) {
 #'   init_etl("name_of_file")
 #' }
 #'
-init_etl <- function(script_name = "", fake_runtime = NULL, log_db_drv = RMariaDB::MariaDB()) {
+init_etl <- function(script_name = "", project_name = "", project_instance = "", fake_runtime = NULL, log_db_drv = RMariaDB::MariaDB()) {
+  set_project_name(project_name)
+  set_project_instance(project_instance)
   set_script_name(script_name)
   if (!is.null(fake_runtime)) {
     set_script_run_time(fake_runtime)

inst/schema/rcc_job_log.sql

@@ -1,13 +1,17 @@
 CREATE TABLE `rcc_job_log` (
   `id` bigint NOT NULL AUTO_INCREMENT,
   `log_date` datetime NOT NULL,
+  `project` VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `instance` VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL,
   `script_name` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL,
   `script_run_time` datetime NOT NULL,
   `job_summary_data` mediumtext DEFAULT NULL,
   `job_duration` double not null,
   `level` enum('SUCCESS','DEBUG','ERROR') CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL,
   PRIMARY KEY (`id`),
   KEY `log_date` (`log_date`),
+  KEY `project` (`project`),
+  KEY `instance` (`instance`),
   KEY `script_name` (`script_name`),
   KEY `script_run_time` (`script_run_time`),
   KEY `level` (`level`)

man/get_project_instance.Rd

@@ -1,13 +1,17 @@
 CREATE TABLE `rcc_job_log` (
   `id` bigint NOT NULL AUTO_INCREMENT,
   `log_date` datetime NOT NULL,
+  `project` VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `instance` VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL,
   `script_name` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL,
   `script_run_time` datetime NOT NULL,
   `job_summary_data` mediumtext DEFAULT NULL,
   `job_duration` double not null,
   `level` enum('SUCCESS','DEBUG','ERROR') CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL,
   PRIMARY KEY (`id`),
   KEY `log_date` (`log_date`),
+  KEY `project` (`project`),
+  KEY `instance` (`instance`),
   KEY `script_name` (`script_name`),
   KEY `script_run_time` (`script_run_time`),
   KEY `level` (`level`)

man/get_project_name.Rd

@@ -1,3 +1,4 @@
+PROJECT=study_template
 INSTANCE=Development
 TIME_ZONE=America/New_York
 

man/init_etl.Rd

@@ -53,13 +53,16 @@ If this is not your first REDCap Custodian rodeo, you might remember what you ne
 You'll need to talk to at least a REDCap or a MySQL database. You will probably want both. Rename the `example.env` to `testing.env` and configure it for development on your computer. That file is composed of five sections.
 
 ```sh
+PROJECT=study_template
 INSTANCE=Development
 TIME_ZONE=America/New_York
 ```
 
-`INSTANCE` names the REDCap system or _instance_ you'll be talking to. This file assumes you are talking to only one REDCap in your script. There are other tools for multiple-instances.
+`PROJECT` names the project, study, or git repository that owns the scripts. This is useful when reading log data from a shared log database. A unique value in the PROJECT field allows you to identify log entries specific to a project.
 
-`TIME_ZONE` should be the local timezone of your REDCap instance. Note that REDCap time facts in local time. The MariaDB driver and the Lubridate library default to UTC. That can get complicated. Be careful with time. For more details see [`stupid_date_tricks.R`](https://gist.github.com/pbchase/ed55ab5dacbcc5d8a702a9cb935cccb5)
+`INSTANCE` names the _instance_ or a project, study, or REDCap system. If scripts are deployed in development, testing, and production, a unique value in this field allows you to identify log entries from different deployments.
+
+`TIME_ZONE` should be the local timezone of your REDCap instance. Note that REDCap records time facts in local time. The MariaDB driver and the Lubridate library default to UTC. That can get complicated. Be careful with time. For more details see [`stupid_date_tricks.R`](https://gist.github.com/pbchase/ed55ab5dacbcc5d8a702a9cb935cccb5)
 
 ```sh
 # Email config