From 3a603d7038881c8025d04cba70eb59be37582789 Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Thu, 25 Jan 2024 18:41:15 +0800 Subject: [PATCH 01/25] Increment version number to 3.9.3.9000 --- DESCRIPTION | 2 +- NEWS.md | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 76625f9..39c7235 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: tarflow.iquizoo Title: Setup "targets" Workflows for "iquizoo" Data Processing -Version: 3.9.3 +Version: 3.9.3.9000 Authors@R: c( person("Liang", "Zhang", , "psychelzh@outlook.com", role = c("aut", "cre"), comment = c(ORCID = "0000-0001-9041-1150")), diff --git a/NEWS.md b/NEWS.md index 321ae48..873d038 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,5 @@ +# tarflow.iquizoo (development version) + # tarflow.iquizoo 3.9.3 ## Breaking Changes From 9618305ac491f7e20b239e2ee398aa199646f916 Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Sun, 17 Mar 2024 16:31:03 +0800 Subject: [PATCH 02/25] Add separate setup module file Signed-off-by: Liang Zhang --- R/database.R | 93 --------------------------- R/setup.R | 131 +++++++++++++++++++++++++++++++++++++++ R/targets.R | 39 ------------ man/check_source.Rd | 2 +- man/setup_option_file.Rd | 2 +- man/setup_source.Rd | 2 +- man/setup_templates.Rd | 2 +- 7 files changed, 135 insertions(+), 136 deletions(-) create mode 100644 R/setup.R diff --git a/R/database.R b/R/database.R index ef540bb..b9843d4 100644 --- a/R/database.R +++ b/R/database.R @@ -105,96 +105,3 @@ fetch_data <- function(query, project_id, game_id, ..., params = list(project_id, game_id) ) } - -#' Set data source -#' -#' @param driver The driver used. Set as an option of `"tarflow.driver"`. -#' Options are [odbc::odbc()] and [RMariaDB::MariaDB()], both of which need -#' pre-configurations. Default to first available one. -#' @param dsn The data source name of an **ODBC** database connector. See -#' [odbc::dbConnect()] for more information. Used when `driver` is set as -#' [odbc::odbc()]. -#' @param groups Section identifier in the `default.file`. See -#' [RMariaDB::MariaDB()] for more information. Used when `driver` is set as -#' [RMariaDB::MariaDB()]. -#' @return An S3 class of `tarflow.source` with the options. -#' @export -setup_source <- function(driver = getOption("tarflow.driver"), - dsn = getOption("tarflow.dsn"), - groups = getOption("tarflow.groups")) { - structure( - list( - driver = driver, - dsn = dsn, - groups = groups - ), - class = "tarflow.source" - ) -} - -#' Check if the database based on the given data source is ready -#' -#' @param source The data source from which data is fetched. See -#' [setup_source()] for details. -#' @return TRUE if the database is ready, FALSE otherwise. -#' @export -check_source <- function(source = setup_source()) { - if (!inherits(source, "tarflow.source")) { - cli::cli_abort( - "{.arg source} must be created by {.fun setup_source}.", - class = "tarflow_bad_source" - ) - } - # nocov start - if (inherits(source$driver, "OdbcDriver")) { - return(DBI::dbCanConnect(source$driver, dsn = source$dsn)) - } - # nocov end - if (inherits(source$driver, "MariaDBDriver")) { - return(DBI::dbCanConnect(source$driver, groups = source$groups)) - } - return(FALSE) -} - -# nocov start - -#' Setup MySQL database connection option file -#' -#' This function will create a MySQL option file at the given path. To ensure it -#' works, set these environment variables before calling this function: -#' - `MYSQL_HOST`: The host name of the MySQL server. -#' - `MYSQL_USER`: The user name of the MySQL server. -#' - `MYSQL_PASSWORD`: The password of the MySQL server. -#' -#' @param path The path to the option file. Default location is operating system -#' dependent. On Windows, it is `C:/my.cnf`. On other systems, it is -#' `~/.my.cnf`. -#' @param overwrite Whether to overwrite the existing option file. -#' @param quietly A logical indicates whether message should be suppressed. -#' @return NULL (invisible). -#' @export -setup_option_file <- function(path = NULL, overwrite = FALSE, quietly = FALSE) { - my_cnf_tmpl <- read_file(package_file("database", "my.cnf.tmpl")) - path <- path %||% default_file() - if (file.exists(path) && !overwrite) { - if (!quietly) { - cli::cli_alert_warning( - "Option file already exists. Use {.arg overwrite = TRUE} to overwrite.", - class = "tarflow_option_file_exists" - ) - } - return(invisible()) - } - writeLines(stringr::str_glue(my_cnf_tmpl), path) -} - -# helper functions -default_file <- function() { - if (Sys.info()["sysname"] == "Windows") { - return("C:/my.cnf") - } else { - return("~/.my.cnf") - } -} - -# nocov end diff --git a/R/setup.R b/R/setup.R new file mode 100644 index 0000000..0c184a6 --- /dev/null +++ b/R/setup.R @@ -0,0 +1,131 @@ +#' Set up templates used to fetch data +#' +#' If you want to extract data based on your own parameters, you should use this +#' function to set up your own SQL templates. Note that the SQL queries should +#' be parameterized. +#' +#' @param contents The SQL template file used to fetch contents. At least +#' `project_id` and `game_id` columns should be included in the fetched data +#' based on the template. `project_id` will be used as the only parameter in +#' `users` and `project` templates, while all three will be used in `raw_data` +#' and `scores` templates. +#' @param users The SQL template file used to fetch users. Usually you don't +#' need to change this. +#' @param raw_data The SQL template file used to fetch raw data. See +#' [fetch_data()] for details. Usually you don't need to change this. +#' @param scores The SQL template file used to fetch scores. See [fetch_data()] +#' for details. Usually you don't need to change this. +#' @param progress_hash The SQL template file used to fetch progress hash. +#' Usually you don't need to change this. +#' @return A S3 object of class `tarflow.template` with the options. +#' @export +setup_templates <- function(contents = NULL, + users = NULL, + raw_data = NULL, + scores = NULL, + progress_hash = NULL) { + structure( + list( + contents = contents %||% package_file("sql", "contents.sql"), + users = users %||% package_file("sql", "users.sql"), + raw_data = raw_data %||% package_file("sql", "raw_data.sql"), + scores = scores %||% package_file("sql", "scores.sql"), + progress_hash = progress_hash %||% + package_file("sql", "progress_hash.sql") + ), + class = "tarflow.template" + ) +} + +#' Set data source +#' +#' @param driver The driver used. Set as an option of `"tarflow.driver"`. +#' Options are [odbc::odbc()] and [RMariaDB::MariaDB()], both of which need +#' pre-configurations. Default to first available one. +#' @param dsn The data source name of an **ODBC** database connector. See +#' [odbc::dbConnect()] for more information. Used when `driver` is set as +#' [odbc::odbc()]. +#' @param groups Section identifier in the `default.file`. See +#' [RMariaDB::MariaDB()] for more information. Used when `driver` is set as +#' [RMariaDB::MariaDB()]. +#' @return An S3 class of `tarflow.source` with the options. +#' @export +setup_source <- function(driver = getOption("tarflow.driver"), + dsn = getOption("tarflow.dsn"), + groups = getOption("tarflow.groups")) { + structure( + list( + driver = driver, + dsn = dsn, + groups = groups + ), + class = "tarflow.source" + ) +} + +#' Check if the database based on the given data source is ready +#' +#' @param source The data source from which data is fetched. See +#' [setup_source()] for details. +#' @return TRUE if the database is ready, FALSE otherwise. +#' @export +check_source <- function(source = setup_source()) { + if (!inherits(source, "tarflow.source")) { + cli::cli_abort( + "{.arg source} must be created by {.fun setup_source}.", + class = "tarflow_bad_source" + ) + } + # nocov start + if (inherits(source$driver, "OdbcDriver")) { + return(DBI::dbCanConnect(source$driver, dsn = source$dsn)) + } + # nocov end + if (inherits(source$driver, "MariaDBDriver")) { + return(DBI::dbCanConnect(source$driver, groups = source$groups)) + } + return(FALSE) +} + +# nocov start + +#' Setup MySQL database connection option file +#' +#' This function will create a MySQL option file at the given path. To ensure it +#' works, set these environment variables before calling this function: +#' - `MYSQL_HOST`: The host name of the MySQL server. +#' - `MYSQL_USER`: The user name of the MySQL server. +#' - `MYSQL_PASSWORD`: The password of the MySQL server. +#' +#' @param path The path to the option file. Default location is operating system +#' dependent. On Windows, it is `C:/my.cnf`. On other systems, it is +#' `~/.my.cnf`. +#' @param overwrite Whether to overwrite the existing option file. +#' @param quietly A logical indicates whether message should be suppressed. +#' @return NULL (invisible). +#' @export +setup_option_file <- function(path = NULL, overwrite = FALSE, quietly = FALSE) { + my_cnf_tmpl <- read_file(package_file("database", "my.cnf.tmpl")) + path <- path %||% default_file() + if (file.exists(path) && !overwrite) { + if (!quietly) { + cli::cli_alert_warning( + "Option file already exists. Use {.arg overwrite = TRUE} to overwrite.", + class = "tarflow_option_file_exists" + ) + } + return(invisible()) + } + writeLines(stringr::str_glue(my_cnf_tmpl), path) +} + +# helper functions +default_file <- function() { + if (Sys.info()["sysname"] == "Windows") { + return("C:/my.cnf") + } else { + return("~/.my.cnf") + } +} + +# nocov end diff --git a/R/targets.R b/R/targets.R index e82f1f5..27ed428 100644 --- a/R/targets.R +++ b/R/targets.R @@ -102,45 +102,6 @@ tar_prep_iquizoo <- function(params, ..., ) } -#' Set up templates used to fetch data -#' -#' If you want to extract data based on your own parameters, you should use this -#' function to set up your own SQL templates. Note that the SQL queries should -#' be parameterized. -#' -#' @param contents The SQL template file used to fetch contents. At least -#' `project_id` and `game_id` columns should be included in the fetched data -#' based on the template. `project_id` will be used as the only parameter in -#' `users` and `project` templates, while all three will be used in `raw_data` -#' and `scores` templates. -#' @param users The SQL template file used to fetch users. Usually you don't -#' need to change this. -#' @param raw_data The SQL template file used to fetch raw data. See -#' [fetch_data()] for details. Usually you don't need to change this. -#' @param scores The SQL template file used to fetch scores. See [fetch_data()] -#' for details. Usually you don't need to change this. -#' @param progress_hash The SQL template file used to fetch progress hash. -#' Usually you don't need to change this. -#' @return A S3 object of class `tarflow.template` with the options. -#' @export -setup_templates <- function(contents = NULL, - users = NULL, - raw_data = NULL, - scores = NULL, - progress_hash = NULL) { - structure( - list( - contents = contents %||% package_file("sql", "contents.sql"), - users = users %||% package_file("sql", "users.sql"), - raw_data = raw_data %||% package_file("sql", "raw_data.sql"), - scores = scores %||% package_file("sql", "scores.sql"), - progress_hash = progress_hash %||% - package_file("sql", "progress_hash.sql") - ), - class = "tarflow.template" - ) -} - # helper functions tar_projects_info <- function(contents, templates, check_progress) { c( diff --git a/man/check_source.Rd b/man/check_source.Rd index 9180d8c..0114107 100644 --- a/man/check_source.Rd +++ b/man/check_source.Rd @@ -1,5 +1,5 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/database.R +% Please edit documentation in R/setup.R \name{check_source} \alias{check_source} \title{Check if the database based on the given data source is ready} diff --git a/man/setup_option_file.Rd b/man/setup_option_file.Rd index e76ddbf..af9abdd 100644 --- a/man/setup_option_file.Rd +++ b/man/setup_option_file.Rd @@ -1,5 +1,5 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/database.R +% Please edit documentation in R/setup.R \name{setup_option_file} \alias{setup_option_file} \title{Setup MySQL database connection option file} diff --git a/man/setup_source.Rd b/man/setup_source.Rd index 7a47d67..f082d09 100644 --- a/man/setup_source.Rd +++ b/man/setup_source.Rd @@ -1,5 +1,5 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/database.R +% Please edit documentation in R/setup.R \name{setup_source} \alias{setup_source} \title{Set data source} diff --git a/man/setup_templates.Rd b/man/setup_templates.Rd index 16d708a..a66961e 100644 --- a/man/setup_templates.Rd +++ b/man/setup_templates.Rd @@ -1,5 +1,5 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/targets.R +% Please edit documentation in R/setup.R \name{setup_templates} \alias{setup_templates} \title{Set up templates used to fetch data} From 4ded090301e2fcc1cc0659b6c612ee3ccf242855 Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Sun, 17 Mar 2024 16:45:46 +0800 Subject: [PATCH 03/25] Added `suffix_format` argument to `fetch_data()` What's more, `query` argument is now optional. Signed-off-by: Liang Zhang --- R/database.R | 36 +++++++++++++++++++++++++++++------- R/targets.R | 4 ++-- man/fetch_data.Rd | 37 ++++++++++++++++++++++++++++++++----- 3 files changed, 63 insertions(+), 14 deletions(-) diff --git a/R/database.R b/R/database.R index b9843d4..e31669c 100644 --- a/R/database.R +++ b/R/database.R @@ -70,25 +70,46 @@ fetch_iquizoo_mem <- function(cache = NULL) { #' Fetch data from iQuizoo database #' -#' @param query A parameterized SQL query. Note the query should also contain -#' a `glue` expression to inject the table name, i.e., `"{ table_name }"`. +#' This function is a wrapper of [fetch_iquizoo()], which is used as a helper +#' function to fetch data from the iQuizoo database. +#' +#' The data essentially means one of the two types of data: raw data or scores. +#' The raw data is the original data collected from the game, while the scores +#' are the scores calculated by the iQuizoo system. While scores can also be +#' calculated from the raw data, the pre-calculated scores are used to for some +#' quick analysis. +#' +#' The data is separated by project date, so the table name is suffixed by the +#' project date, which is automatically fetched from the database by this +#' function. You could set the format of the date suffix by `suffix_format`, +#' although currently you should not need to change it because it probably will +#' not change in the future. Finally, this suffix should be substituted into the +#' query, which should contain an expression to inject the table name, i.e., +#' `"{table_name}"`. +#' #' @param project_id The project id to be bound to the query. #' @param game_id The game id to be bound to the query. #' @param ... Further arguments passed to [fetch_iquizoo()]. #' @param what What to fetch. Can be either "raw_data" or "scores". +#' @param query A parameterized SQL query. A default query file is stored in the +#' package, which is often enough for most cases. You can also specify your +#' own query file by this argument. See details for more information. +#' @param suffix_format The format of the date suffix. See details for more +#' information. #' @return A [data.frame] contains the fetched data. #' @export -fetch_data <- function(query, project_id, game_id, ..., - what = c("raw_data", "scores")) { +fetch_data <- function(project_id, game_id, ..., + what = c("raw_data", "scores"), + query = NULL, + suffix_format = "%Y0101") { check_dots_used() what <- match.arg(what) - # the database stores data from each year into a separate table with the - # suffix of course date with the format "0101" + # data separated by project date, so we need to get the project date first suffix <- package_file("sql", "project_date.sql") |> read_file() |> fetch_iquizoo(params = project_id) |> .subset2("project_date") |> - format("%Y0101") + format(suffix_format) table_name <- paste0( switch(what, raw_data = "content_orginal_data_", @@ -96,6 +117,7 @@ fetch_data <- function(query, project_id, game_id, ..., ), suffix ) + query <- query %||% read_file(package_file("sql", paste0(what, ".sql"))) fetch_iquizoo( stringr::str_glue( query, diff --git a/R/targets.R b/R/targets.R index 27ed428..a121c4d 100644 --- a/R/targets.R +++ b/R/targets.R @@ -158,10 +158,10 @@ tar_fetch_data <- function(contents, templates, what) { progress_hash purrr::pmap( list( - query = !!read_file(templates[[what]]), project_id = project_id, game_id = game_id, - what = !!what + what = !!what, + query = !!read_file(templates[[what]]) ), fetch_data ) |> diff --git a/man/fetch_data.Rd b/man/fetch_data.Rd index a05fed2..2e65bad 100644 --- a/man/fetch_data.Rd +++ b/man/fetch_data.Rd @@ -4,12 +4,16 @@ \alias{fetch_data} \title{Fetch data from iQuizoo database} \usage{ -fetch_data(query, project_id, game_id, ..., what = c("raw_data", "scores")) +fetch_data( + project_id, + game_id, + ..., + what = c("raw_data", "scores"), + query = NULL, + suffix_format = "\%Y0101" +) } \arguments{ -\item{query}{A parameterized SQL query. Note the query should also contain -a \code{glue} expression to inject the table name, i.e., \code{"{ table_name }"}.} - \item{project_id}{The project id to be bound to the query.} \item{game_id}{The game id to be bound to the query.} @@ -17,10 +21,33 @@ a \code{glue} expression to inject the table name, i.e., \code{"{ table_name }"} \item{...}{Further arguments passed to \code{\link[=fetch_iquizoo]{fetch_iquizoo()}}.} \item{what}{What to fetch. Can be either "raw_data" or "scores".} + +\item{query}{A parameterized SQL query. A default query file is stored in the +package, which is often enough for most cases. You can also specify your +own query file by this argument. See details for more information.} + +\item{suffix_format}{The format of the date suffix. See details for more +information.} } \value{ A \link{data.frame} contains the fetched data. } \description{ -Fetch data from iQuizoo database +This function is a wrapper of \code{\link[=fetch_iquizoo]{fetch_iquizoo()}}, which is used as a helper +function to fetch data from the iQuizoo database. +} +\details{ +The data essentially means one of the two types of data: raw data or scores. +The raw data is the original data collected from the game, while the scores +are the scores calculated by the iQuizoo system. While scores can also be +calculated from the raw data, the pre-calculated scores are used to for some +quick analysis. + +The data is separated by project date, so the table name is suffixed by the +project date, which is automatically fetched from the database by this +function. You could set the format of the date suffix by \code{suffix_format}, +although currently you should not need to change it because it probably will +not change in the future. Finally, this suffix should be substituted into the +query, which should contain an expression to inject the table name, i.e., +\code{"{table_name}"}. } From d9382928a961e0fd9aff407a6fc16c7cbc754da5 Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Sun, 17 Mar 2024 16:47:55 +0800 Subject: [PATCH 04/25] NEWS bulletin Signed-off-by: Liang Zhang --- NEWS.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/NEWS.md b/NEWS.md index 873d038..f495837 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,14 @@ # tarflow.iquizoo (development version) +## Breaking Changes + +* Now `query` argument of `fetch_data()` is optional. If not specified, the default query stored in the package will be used. + +## Enhancements + +* Added `suffix_format` argument to `fetch_data()` to specify the format of suffix in the query file. This is useful when you want to use a different format of suffix in the query file. +* Enhanced the documentation of `fetch_data()`. + # tarflow.iquizoo 3.9.3 ## Breaking Changes From ad6deb4638fea896997af4d1d8ed380cd896c8c8 Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Mon, 18 Mar 2024 10:12:41 +0800 Subject: [PATCH 05/25] Remove one unused package from suggests Signed-off-by: Liang Zhang --- DESCRIPTION | 1 - 1 file changed, 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 39c7235..e3427f2 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -41,7 +41,6 @@ Suggests: roxygen2, testthat (>= 3.0.0), tibble, - tidytable, withr Remotes: psychelzh/data.iquizoo, From 096e19852a08d2abaa3ac3275f226948ef0b3502 Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Mon, 18 Mar 2024 17:34:30 +0800 Subject: [PATCH 06/25] Get rid of dplyr, tidyr and purrr These should not be necessary for this project. Fix #84 Signed-off-by: Liang Zhang --- DESCRIPTION | 6 ++-- NAMESPACE | 2 -- R/targets.R | 91 +++++++++++++++++++++++++---------------------------- R/utils.R | 9 ++++++ R/zzz.R | 2 -- 5 files changed, 54 insertions(+), 56 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index e3427f2..ababa6f 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -22,14 +22,11 @@ Imports: cli, data.iquizoo (>= 2023.10.22), DBI, - dplyr, memoise, - purrr, rlang (>= 1.0.0), stringr, tarchetypes, - targets, - tidyr + targets Suggests: bit64, covr, @@ -37,6 +34,7 @@ Suggests: lifecycle, odbc, preproc.iquizoo (>= 2.6.0), + purrr, RMariaDB (>= 1.3.1), roxygen2, testthat (>= 3.0.0), diff --git a/NAMESPACE b/NAMESPACE index d137ab8..f0971d2 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -9,6 +9,4 @@ export(setup_source) export(setup_templates) export(tar_prep_iquizoo) export(use_targets_pipeline) -import(dplyr) import(rlang) -import(tidyr) diff --git a/R/targets.R b/R/targets.R index a121c4d..6304b7c 100644 --- a/R/targets.R +++ b/R/targets.R @@ -79,11 +79,11 @@ tar_prep_iquizoo <- function(params, ..., expr(unserialize(!!serialize(contents, NULL))) ), tar_projects_info(contents, templates, check_progress), - purrr::map( + lapply( what, \(what) tar_fetch_data(contents, templates, what) ) |> - purrr::list_flatten(), + unlist(recursive = FALSE), if ("raw_data" %in% what && action_raw_data != "none") { tar_action_raw_data(contents, action_raw_data) } @@ -106,14 +106,12 @@ tar_prep_iquizoo <- function(params, ..., tar_projects_info <- function(contents, templates, check_progress) { c( tarchetypes::tar_map( - contents |> - distinct(.data$project_id) |> - mutate(project_id = as.character(.data$project_id)), + data.frame(project_id = as.character(unique(contents$project_id))), targets::tar_target_raw( "progress_hash", - expr( + bquote( fetch_iquizoo( - !!read_file(templates[["progress_hash"]]), + .(read_file(templates[["progress_hash"]])), params = list(project_id) ) ), @@ -123,10 +121,10 @@ tar_projects_info <- function(contents, templates, check_progress) { ), targets::tar_target_raw( "users", - expr( + bquote( fetch_iquizoo( - !!read_file(templates[["users"]]), - params = list(!!unique(contents$project_id)) + .(read_file(templates[["users"]])), + params = list(.(unique(contents$project_id))) ) |> unique() ), @@ -136,40 +134,36 @@ tar_projects_info <- function(contents, templates, check_progress) { } tar_fetch_data <- function(contents, templates, what) { - tarchetypes::tar_map( - contents |> - distinct(.data$project_id, .data$game_id) |> - mutate( - across(c("project_id", "game_id"), as.character) - ) |> - summarise( - progress_hash = list( - syms( - stringr::str_glue("progress_hash_{project_id}") + game_ids <- unique(as.character(contents$game_id)) + targets <- vector("list", length(game_ids)) + names(targets) <- game_ids + for (game_id in game_ids) { + project_ids <- with(contents, as.character(project_id[game_id == game_id])) + targets[[game_id]] <- targets::tar_target_raw( + paste0(what, "_", game_id), + bquote({ + .(create_hash_deps(project_ids)) + do.call( + rbind, + lapply( + .(project_ids), + \(project_id) { + fetch_data( + project_id, + .(game_id), + what = .(what), + query = .(read_file(templates[[what]])) + ) + } ) - ), - project_id = list(.data$project_id), - .by = "game_id" - ), - names = "game_id", - targets::tar_target_raw( - what, - expr({ - progress_hash - purrr::pmap( - list( - project_id = project_id, - game_id = game_id, - what = !!what, - query = !!read_file(templates[[what]]) - ), - fetch_data - ) |> - purrr::list_rbind() + ) }), packages = "tarflow.iquizoo" ) - ) + } + out <- list(targets) + names(out) <- what + out } tar_action_raw_data <- function(contents, @@ -178,12 +172,14 @@ tar_action_raw_data <- function(contents, name_parsed = "raw_data_parsed", name_indices = "indices") { if (action_raw_data == "all") action_raw_data <- c("parse", "preproc") - contents <- distinct(contents, .data$game_id) |> - mutate( - tar_data = syms(sprintf("%s_%s", name_data, game_id)), - tar_parsed = syms(sprintf("%s_%s", name_parsed, game_id)), - tar_indices = syms(sprintf("%s_%s", name_indices, game_id)) - ) + contents <- within( + unique(contents["game_id"]), + { + tar_data <- syms(sprintf("%s_%s", name_data, game_id)) + tar_parsed <- syms(sprintf("%s_%s", name_parsed, game_id)) + tar_indices <- syms(sprintf("%s_%s", name_indices, game_id)) + } + ) list( raw_data_parsed = if ("parse" %in% action_raw_data) { tarchetypes::tar_eval( @@ -202,8 +198,7 @@ tar_action_raw_data <- function(contents, preproc_data(tar_parsed, prep_fun, .input = input, .extra = extra), packages = "preproc.iquizoo" ), - contents |> - data.iquizoo::match_preproc(type = "inner") + data.iquizoo::match_preproc(contents, type = "inner") ) } ) diff --git a/R/utils.R b/R/utils.R index 5ca0994..cac462f 100644 --- a/R/utils.R +++ b/R/utils.R @@ -8,3 +8,12 @@ package_file <- function(type, file) { read_file <- function(file) { paste0(readLines(file), collapse = "\n") } + +create_hash_deps <- function(project_ids) { + as.call( + c( + quote(list), + syms(paste0("progress_hash_", project_ids)) + ) + ) +} diff --git a/R/zzz.R b/R/zzz.R index 754642d..73175b0 100644 --- a/R/zzz.R +++ b/R/zzz.R @@ -1,6 +1,4 @@ #' @import rlang -#' @import dplyr -#' @import tidyr NULL # nocov start From e53ba8320f47933567a2816ce454120ea93acc10 Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Mon, 18 Mar 2024 17:37:15 +0800 Subject: [PATCH 07/25] NEWS bulletin Signed-off-by: Liang Zhang --- NEWS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.md b/NEWS.md index f495837..57d74cc 100644 --- a/NEWS.md +++ b/NEWS.md @@ -8,6 +8,7 @@ * Added `suffix_format` argument to `fetch_data()` to specify the format of suffix in the query file. This is useful when you want to use a different format of suffix in the query file. * Enhanced the documentation of `fetch_data()`. +* Let package not depend on dplyr, tidyr and purrr packages (#84). # tarflow.iquizoo 3.9.3 From 4629be52d65ae669c790c96607f79f599cdf8f25 Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Mon, 18 Mar 2024 17:49:05 +0800 Subject: [PATCH 08/25] Use `mapply()` Signed-off-by: Liang Zhang --- R/targets.R | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/R/targets.R b/R/targets.R index 6304b7c..8b9ef21 100644 --- a/R/targets.R +++ b/R/targets.R @@ -145,16 +145,15 @@ tar_fetch_data <- function(contents, templates, what) { .(create_hash_deps(project_ids)) do.call( rbind, - lapply( + mapply( + fetch_data, .(project_ids), - \(project_id) { - fetch_data( - project_id, - .(game_id), - what = .(what), - query = .(read_file(templates[[what]])) - ) - } + .(game_id), + MoreArgs = list( + what = .(what), + query = .(read_file(templates[[what]])) + ), + SIMPLIFY = FALSE ) ) }), From 7259612e7108036e14abd220f47a515720cec6fd Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Mon, 18 Mar 2024 17:59:36 +0800 Subject: [PATCH 09/25] Use `sapply()` to add data names Signed-off-by: Liang Zhang --- R/targets.R | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/R/targets.R b/R/targets.R index 8b9ef21..8a876db 100644 --- a/R/targets.R +++ b/R/targets.R @@ -79,11 +79,11 @@ tar_prep_iquizoo <- function(params, ..., expr(unserialize(!!serialize(contents, NULL))) ), tar_projects_info(contents, templates, check_progress), - lapply( + sapply( what, - \(what) tar_fetch_data(contents, templates, what) - ) |> - unlist(recursive = FALSE), + \(what) tar_fetch_data(contents, templates, what), + simplify = FALSE + ), if ("raw_data" %in% what && action_raw_data != "none") { tar_action_raw_data(contents, action_raw_data) } @@ -160,9 +160,7 @@ tar_fetch_data <- function(contents, templates, what) { packages = "tarflow.iquizoo" ) } - out <- list(targets) - names(out) <- what - out + targets } tar_action_raw_data <- function(contents, From 1ede4501d4a43e664003dc492409450056e9668f Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Mon, 18 Mar 2024 19:06:56 +0800 Subject: [PATCH 10/25] Use `splice` in `bquote()` Signed-off-by: Liang Zhang --- R/targets.R | 33 ++++++++++++++++++--------------- R/utils.R | 9 --------- 2 files changed, 18 insertions(+), 24 deletions(-) diff --git a/R/targets.R b/R/targets.R index 8a876db..81a2ef9 100644 --- a/R/targets.R +++ b/R/targets.R @@ -141,22 +141,25 @@ tar_fetch_data <- function(contents, templates, what) { project_ids <- with(contents, as.character(project_id[game_id == game_id])) targets[[game_id]] <- targets::tar_target_raw( paste0(what, "_", game_id), - bquote({ - .(create_hash_deps(project_ids)) - do.call( - rbind, - mapply( - fetch_data, - .(project_ids), - .(game_id), - MoreArgs = list( - what = .(what), - query = .(read_file(templates[[what]])) - ), - SIMPLIFY = FALSE + bquote( + { + list(..(syms(paste0("progress_hash_", project_ids)))) + do.call( + rbind, + mapply( + fetch_data, + .(project_ids), + .(game_id), + MoreArgs = list( + what = .(what), + query = .(read_file(templates[[what]])) + ), + SIMPLIFY = FALSE + ) ) - ) - }), + }, + splice = TRUE + ), packages = "tarflow.iquizoo" ) } diff --git a/R/utils.R b/R/utils.R index cac462f..5ca0994 100644 --- a/R/utils.R +++ b/R/utils.R @@ -8,12 +8,3 @@ package_file <- function(type, file) { read_file <- function(file) { paste0(readLines(file), collapse = "\n") } - -create_hash_deps <- function(project_ids) { - as.call( - c( - quote(list), - syms(paste0("progress_hash_", project_ids)) - ) - ) -} From b95d5278f597b6133ea2cb24f9c647e4b886d71d Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Mon, 18 Mar 2024 20:00:17 +0800 Subject: [PATCH 11/25] Fix error of wrong subsetting Signed-off-by: Liang Zhang --- R/targets.R | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/R/targets.R b/R/targets.R index 81a2ef9..7a27132 100644 --- a/R/targets.R +++ b/R/targets.R @@ -138,7 +138,9 @@ tar_fetch_data <- function(contents, templates, what) { targets <- vector("list", length(game_ids)) names(targets) <- game_ids for (game_id in game_ids) { - project_ids <- with(contents, as.character(project_id[game_id == game_id])) + project_ids <- as.character( + contents$project_id[contents$game_id == game_id] + ) targets[[game_id]] <- targets::tar_target_raw( paste0(what, "_", game_id), bquote( From ad31157885c2e84e91e3280cbcdd4ccb8c73139f Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Mon, 18 Mar 2024 20:06:53 +0800 Subject: [PATCH 12/25] Do not use names Signed-off-by: Liang Zhang --- R/targets.R | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/R/targets.R b/R/targets.R index 7a27132..cbb47bc 100644 --- a/R/targets.R +++ b/R/targets.R @@ -156,7 +156,8 @@ tar_fetch_data <- function(contents, templates, what) { what = .(what), query = .(read_file(templates[[what]])) ), - SIMPLIFY = FALSE + SIMPLIFY = FALSE, + USE.NAMES = FALSE ) ) }, From 5438e945ba0a0a90405d723b0341322c2a8e8bcd Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Mon, 18 Mar 2024 20:08:34 +0800 Subject: [PATCH 13/25] Add test case for fetched data Signed-off-by: Liang Zhang --- tests/testthat/_snaps/targets.md | 184 ++++++++++++++++++++++++++++++- tests/testthat/test-targets.R | 4 +- 2 files changed, 186 insertions(+), 2 deletions(-) diff --git a/tests/testthat/_snaps/targets.md b/tests/testthat/_snaps/targets.md index 6cc92cc..2fadd7e 100644 --- a/tests/testthat/_snaps/targets.md +++ b/tests/testthat/_snaps/targets.md @@ -3,6 +3,188 @@ { "type": "character", "attributes": {}, - "value": ["contents_origin", "indices_383674715747205", "indices_383679060169477", "indices_383791726646021", "indices_383795602420485", "progress_hash_555452035072389", "raw_data_383674715747205", "raw_data_383679060169477", "raw_data_383791726646021", "raw_data_383795602420485", "raw_data_388200929063813", "raw_data_388594665001861", "raw_data_391556354638725", "raw_data_parsed_383674715747205", "raw_data_parsed_383679060169477", "raw_data_parsed_383791726646021", "raw_data_parsed_383795602420485", "raw_data_parsed_388200929063813", "raw_data_parsed_388594665001861", "raw_data_parsed_391556354638725", "scores_383674715747205", "scores_383679060169477", "scores_383791726646021", "scores_383795602420485", "scores_388200929063813", "scores_388594665001861", "scores_391556354638725", "users"] + "value": ["contents_origin", "indices_383674715747205", "indices_383679060169477", "indices_383791726646021", "indices_383795602420485", "progress_hash_555452035072389", "raw_data_383674715747205", "raw_data_383679060169477", "raw_data_383791726646021", "raw_data_383795602420485", "raw_data_388200929063813", "raw_data_388594665001861", "raw_data_391556354638725", "raw_data_parsed_383674715747205", "raw_data_parsed_383679060169477", "raw_data_parsed_383791726646021", "raw_data_parsed_383795602420485", "raw_data_parsed_388200929063813", "raw_data_parsed_388594665001861", "raw_data_parsed_391556354638725", "scores", "scores_383674715747205", "scores_383679060169477", "scores_383791726646021", "scores_383795602420485", "scores_388200929063813", "scores_388594665001861", "scores_391556354638725", "users"] + } + +--- + + { + "type": "list", + "attributes": { + "names": { + "type": "character", + "attributes": {}, + "value": ["user_id", "user_name", "user_sex", "user_dob", "organization_name", "grade_name", "class_name"] + }, + "row.names": { + "type": "integer", + "attributes": {}, + "value": [1, 2, 3, 4, 5, 6, 7] + }, + "class": { + "type": "character", + "attributes": {}, + "value": ["data.frame"] + } + }, + "value": [ + { + "type": "double", + "attributes": { + "class": { + "type": "character", + "attributes": {}, + "value": ["integer64"] + } + }, + "value": [2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309] + }, + { + "type": "character", + "attributes": {}, + "value": ["陈佳洋", "茹祎", "于胜蓝", "李荟镕", "张忱涵", "连宗鸿", "徐小琳"] + }, + { + "type": "integer", + "attributes": {}, + "value": [2, 2, 2, 2, 2, 2, 2] + }, + { + "type": "double", + "attributes": { + "class": { + "type": "character", + "attributes": {}, + "value": ["Date"] + } + }, + "value": [-25202, -25201, -25200, -25199, -25198, -25197, -25196] + }, + { + "type": "character", + "attributes": {}, + "value": ["北京师范大学", "北京师范大学", "北京师范大学", "北京师范大学", "北京师范大学", "北京师范大学", "北京师范大学"] + }, + { + "type": "character", + "attributes": {}, + "value": ["睡眠实验", "睡眠实验", "睡眠实验", "睡眠实验", "睡眠实验", "睡眠实验", "睡眠实验"] + }, + { + "type": "character", + "attributes": {}, + "value": ["行为实验1April", "行为实验1April", "行为实验1April", "行为实验1April", "行为实验1April", "行为实验1April", "行为实验1April"] + } + ] + } + +--- + + { + "type": "list", + "attributes": { + "names": { + "type": "character", + "attributes": {}, + "value": ["project_id", "user_id", "game_id", "game_version", "game_time", "ability_id", "game_stage", "game_star", "game_score_raw", "game_score_std"] + }, + "row.names": { + "type": "integer", + "attributes": {}, + "value": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49] + }, + "class": { + "type": "character", + "attributes": {}, + "value": ["data.frame"] + } + }, + "value": [ + { + "type": "double", + "attributes": { + "class": { + "type": "character", + "attributes": {}, + "value": ["integer64"] + } + }, + "value": [2.74429768e-309, 2.74429768e-309, 2.74429768e-309, 2.74429768e-309, 2.74429768e-309, 2.74429768e-309, 2.74429768e-309, 2.74429768e-309, 2.74429768e-309, 2.74429768e-309, 2.74429768e-309, 2.74429768e-309, 2.74429768e-309, 2.74429768e-309, 2.74429768e-309, 2.74429768e-309, 2.74429768e-309, 2.74429768e-309, 2.74429768e-309, 2.74429768e-309, 2.74429768e-309, 2.74429768e-309, 2.74429768e-309, 2.74429768e-309, 2.74429768e-309, 2.74429768e-309, 2.74429768e-309, 2.74429768e-309, 2.74429768e-309, 2.74429768e-309, 2.74429768e-309, 2.74429768e-309, 2.74429768e-309, 2.74429768e-309, 2.74429768e-309, 2.74429768e-309, 2.74429768e-309, 2.74429768e-309, 2.74429768e-309, 2.74429768e-309, 2.74429768e-309, 2.74429768e-309, 2.74429768e-309, 2.74429768e-309, 2.74429768e-309, 2.74429768e-309, 2.74429768e-309, 2.74429768e-309, 2.74429768e-309] + }, + { + "type": "double", + "attributes": { + "class": { + "type": "character", + "attributes": {}, + "value": ["integer64"] + } + }, + "value": [2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309, 2.0185087e-309] + }, + { + "type": "double", + "attributes": { + "class": { + "type": "character", + "attributes": {}, + "value": ["integer64"] + } + }, + "value": [1.93454543e-309, 1.93454543e-309, 1.93454543e-309, 1.93454543e-309, 1.93454543e-309, 1.93454543e-309, 1.93454543e-309, 1.91991274e-309, 1.91991274e-309, 1.91991274e-309, 1.91991274e-309, 1.91991274e-309, 1.91991274e-309, 1.91991274e-309, 1.91796743e-309, 1.91796743e-309, 1.91796743e-309, 1.91796743e-309, 1.91796743e-309, 1.91796743e-309, 1.91796743e-309, 1.89620222e-309, 1.89620222e-309, 1.89620222e-309, 1.89620222e-309, 1.89620222e-309, 1.89620222e-309, 1.89620222e-309, 1.89618307e-309, 1.89618307e-309, 1.89618307e-309, 1.89618307e-309, 1.89618307e-309, 1.89618307e-309, 1.89618307e-309, 1.89562643e-309, 1.89562643e-309, 1.89562643e-309, 1.89562643e-309, 1.89562643e-309, 1.89562643e-309, 1.89562643e-309, 1.89560496e-309, 1.89560496e-309, 1.89560496e-309, 1.89560496e-309, 1.89560496e-309, 1.89560496e-309, 1.89560496e-309] + }, + { + "type": "character", + "attributes": {}, + "value": ["", "", "", "", "", "", "", "", "", "", "", "", "", "", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0"] + }, + { + "type": "double", + "attributes": { + "tzone": { + "type": "character", + "attributes": {}, + "value": ["UTC"] + }, + "class": { + "type": "character", + "attributes": {}, + "value": ["POSIXct", "POSIXt"] + } + }, + "value": [1681983351, 1681983969, 1681985754, 1681986356, 1681987913, 1681988834, 1681989212, 1681982784, 1681983237, 1681985329, 1681985755, 1681987218, 1681988416, 1681988719, 1681982923, 1681983470, 1681985408, 1681985868, 1681987354, 1681988489, 1681988797, 1681983341, 1681983950, 1681985742, 1681986338, 1681987893, 1681988820, 1681989201, 1681983245, 1681983843, 1681985675, 1681986218, 1681987764, 1681988767, 1681989104, 1681983054, 1681983633, 1681985540, 1681986023, 1681987536, 1681988613, 1681988956, 1681982990, 1681983549, 1681985473, 1681985928, 1681987439, 1681988555, 1681988878] + }, + { + "type": "double", + "attributes": { + "class": { + "type": "character", + "attributes": {}, + "value": ["integer64"] + } + }, + "value": [1.93454615e-309, 1.93454615e-309, 1.93454615e-309, 1.93454615e-309, 1.93454615e-309, 1.93454615e-309, 1.93454615e-309, 1.91991319e-309, 1.91991319e-309, 1.91991319e-309, 1.91991319e-309, 1.91991319e-309, 1.91991319e-309, 1.91991319e-309, 1.9179678e-309, 1.9179678e-309, 1.9179678e-309, 1.9179678e-309, 1.9179678e-309, 1.9179678e-309, 1.9179678e-309, 1.89620278e-309, 1.89620278e-309, 1.89620278e-309, 1.89620278e-309, 1.89620278e-309, 1.89620278e-309, 1.89620278e-309, 1.89618415e-309, 1.89618415e-309, 1.89618415e-309, 1.89618415e-309, 1.89618415e-309, 1.89618415e-309, 1.89618415e-309, 1.89563061e-309, 1.89563061e-309, 1.89563061e-309, 1.89563061e-309, 1.89563061e-309, 1.89563061e-309, 1.89563061e-309, 1.89560665e-309, 1.89560665e-309, 1.89560665e-309, 1.89560665e-309, 1.89560665e-309, 1.89560665e-309, 1.89560665e-309] + }, + { + "type": "integer", + "attributes": {}, + "value": [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + }, + { + "type": "integer", + "attributes": {}, + "value": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + }, + { + "type": "double", + "attributes": {}, + "value": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5] + }, + { + "type": "double", + "attributes": {}, + "value": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + } + ] } diff --git a/tests/testthat/test-targets.R b/tests/testthat/test-targets.R index 74fa362..c126237 100644 --- a/tests/testthat/test-targets.R +++ b/tests/testthat/test-targets.R @@ -52,10 +52,12 @@ test_that("Workflow works", { ~organization_name, ~project_name, "北京师范大学", "4.19-4.20夜晚睡眠test" ) - tar_prep_iquizoo(params) + tar_prep_iquizoo(params, combine = "scores") }) targets::tar_make(reporter = "silent", callr_function = NULL) expect_snapshot_value(targets::tar_objects(), style = "json2") + expect_snapshot_value(targets::tar_read(users), style = "json2") + expect_snapshot_value(targets::tar_read(scores), style = "json2") }) }) From 017c5c13ba5d30c9f0c80e07a7be1f00bab5fa22 Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Tue, 19 Mar 2024 16:19:54 +0800 Subject: [PATCH 14/25] Use `.mapply()` Signed-off-by: Liang Zhang --- R/targets.R | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/R/targets.R b/R/targets.R index cbb47bc..3933604 100644 --- a/R/targets.R +++ b/R/targets.R @@ -148,16 +148,13 @@ tar_fetch_data <- function(contents, templates, what) { list(..(syms(paste0("progress_hash_", project_ids)))) do.call( rbind, - mapply( + .mapply( fetch_data, - .(project_ids), - .(game_id), + list(.(project_ids), .(game_id)), MoreArgs = list( what = .(what), query = .(read_file(templates[[what]])) - ), - SIMPLIFY = FALSE, - USE.NAMES = FALSE + ) ) ) }, From 387b88788e7901bd5ebed20d8eec9121a350d4bb Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Thu, 28 Mar 2024 10:01:05 +0800 Subject: [PATCH 15/25] Enhance organization of pkgdown site Signed-off-by: Liang Zhang --- _pkgdown.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/_pkgdown.yml b/_pkgdown.yml index 5606e05..715decd 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -13,10 +13,10 @@ reference: - fetch_iquizoo - fetch_iquizoo_mem - fetch_data - - check_source - - setup_option_file - title: "Miscellaneous Option Functions" desc: Functions to help you set up options. contents: - setup_templates - setup_source + - setup_option_file + - check_source From 2a426bbb0494f9f5171d0bd92c8bbffd06a752c2 Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Thu, 28 Mar 2024 22:09:21 +0800 Subject: [PATCH 16/25] Export all the modular targets factory functions Fix #77 Signed-off-by: Liang Zhang --- NAMESPACE | 3 ++ R/targets.R | 66 ++++++++++++++++++++++++++++++++++++------- man/tar_fetch_data.Rd | 28 ++++++++++++++++++ man/tar_prep_proj.Rd | 27 ++++++++++++++++++ man/tar_prep_raw.Rd | 34 ++++++++++++++++++++++ 5 files changed, 148 insertions(+), 10 deletions(-) create mode 100644 man/tar_fetch_data.Rd create mode 100644 man/tar_prep_proj.Rd create mode 100644 man/tar_prep_raw.Rd diff --git a/NAMESPACE b/NAMESPACE index f0971d2..c1939fc 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -7,6 +7,9 @@ export(fetch_iquizoo_mem) export(setup_option_file) export(setup_source) export(setup_templates) +export(tar_fetch_data) export(tar_prep_iquizoo) +export(tar_prep_proj) +export(tar_prep_raw) export(use_targets_pipeline) import(rlang) diff --git a/R/targets.R b/R/targets.R index 3933604..afa232a 100644 --- a/R/targets.R +++ b/R/targets.R @@ -78,14 +78,14 @@ tar_prep_iquizoo <- function(params, ..., "contents_origin", expr(unserialize(!!serialize(contents, NULL))) ), - tar_projects_info(contents, templates, check_progress), + tar_prep_proj(contents, templates, check_progress), sapply( what, \(what) tar_fetch_data(contents, templates, what), simplify = FALSE ), if ("raw_data" %in% what && action_raw_data != "none") { - tar_action_raw_data(contents, action_raw_data) + tar_prep_raw(contents, action_raw_data) } ) c( @@ -102,8 +102,24 @@ tar_prep_iquizoo <- function(params, ..., ) } -# helper functions -tar_projects_info <- function(contents, templates, check_progress) { +#' Generate a set of targets for preparing project-level data +#' +#' There are mainly two types of data to be fetched, i.e., the progress hash and +#' the user information. The former is used to check the progress of the +#' project, while the latter is used to identify the users involved in the +#' project. +#' +#' @param contents The contents structure used as the configuration of data +#' fetching. +#' @param templates The SQL template files used to fetch data. See +#' [setup_templates()] for details. +#' @param check_progress Whether to check the progress hash. Set it as `FALSE` +#' if the project is finalized. +#' @return A list of target objects. +#' @export +tar_prep_proj <- function(contents, + templates = setup_templates(), + check_progress = TRUE) { c( tarchetypes::tar_map( data.frame(project_id = as.character(unique(contents$project_id))), @@ -133,7 +149,22 @@ tar_projects_info <- function(contents, templates, check_progress) { ) } -tar_fetch_data <- function(contents, templates, what) { +#' Generate a set of targets for fetching data +#' +#' This target factory is the main part of the `tar_prep_iquizoo` function. It +#' fetches the raw data and scores for each project and task/game combination. +#' +#' @param contents The contents structure used as the configuration of data +#' fetching. +#' @param templates The SQL template files used to fetch data. See +#' [setup_templates()] for details. +#' @param what What to fetch. +#' @return A list of target objects. +#' @export +tar_fetch_data <- function(contents, + templates = setup_templates(), + what = c("raw_data", "scores")) { + what <- match.arg(what) game_ids <- unique(as.character(contents$game_id)) targets <- vector("list", length(game_ids)) names(targets) <- game_ids @@ -166,11 +197,26 @@ tar_fetch_data <- function(contents, templates, what) { targets } -tar_action_raw_data <- function(contents, - action_raw_data, - name_data = "raw_data", - name_parsed = "raw_data_parsed", - name_indices = "indices") { +#' Generate a set of targets for wrangling and pre-processing raw data +#' +#' This target factory is the main part of the `tar_prep_iquizoo` function. It +#' wrangles the raw data into a tidy format and calculates indices based on the +#' parsed data. +#' +#' @param contents The contents structure used as the configuration of data +#' fetching. +#' @param action_raw_data The action to be taken on the fetched raw data. +#' @param name_data The name of the raw data target. +#' @param name_parsed The name of the parsed data target. +#' @param name_indices The name of the indices target. +#' @return A list of target objects. +#' @export +tar_prep_raw <- function(contents, + action_raw_data = c("all", "parse", "none"), + name_data = "raw_data", + name_parsed = "raw_data_parsed", + name_indices = "indices") { + action_raw_data <- match.arg(action_raw_data) if (action_raw_data == "all") action_raw_data <- c("parse", "preproc") contents <- within( unique(contents["game_id"]), diff --git a/man/tar_fetch_data.Rd b/man/tar_fetch_data.Rd new file mode 100644 index 0000000..935b713 --- /dev/null +++ b/man/tar_fetch_data.Rd @@ -0,0 +1,28 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/targets.R +\name{tar_fetch_data} +\alias{tar_fetch_data} +\title{Generate a set of targets for fetching data} +\usage{ +tar_fetch_data( + contents, + templates = setup_templates(), + what = c("raw_data", "scores") +) +} +\arguments{ +\item{contents}{The contents structure used as the configuration of data +fetching.} + +\item{templates}{The SQL template files used to fetch data. See +\code{\link[=setup_templates]{setup_templates()}} for details.} + +\item{what}{What to fetch.} +} +\value{ +A list of target objects. +} +\description{ +This target factory is the main part of the \code{tar_prep_iquizoo} function. It +fetches the raw data and scores for each project and task/game combination. +} diff --git a/man/tar_prep_proj.Rd b/man/tar_prep_proj.Rd new file mode 100644 index 0000000..24b294f --- /dev/null +++ b/man/tar_prep_proj.Rd @@ -0,0 +1,27 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/targets.R +\name{tar_prep_proj} +\alias{tar_prep_proj} +\title{Generate a set of targets for preparing project-level data} +\usage{ +tar_prep_proj(contents, templates = setup_templates(), check_progress = TRUE) +} +\arguments{ +\item{contents}{The contents structure used as the configuration of data +fetching.} + +\item{templates}{The SQL template files used to fetch data. See +\code{\link[=setup_templates]{setup_templates()}} for details.} + +\item{check_progress}{Whether to check the progress hash. Set it as \code{FALSE} +if the project is finalized.} +} +\value{ +A list of target objects. +} +\description{ +There are mainly two types of data to be fetched, i.e., the progress hash and +the user information. The former is used to check the progress of the +project, while the latter is used to identify the users involved in the +project. +} diff --git a/man/tar_prep_raw.Rd b/man/tar_prep_raw.Rd new file mode 100644 index 0000000..bbf5214 --- /dev/null +++ b/man/tar_prep_raw.Rd @@ -0,0 +1,34 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/targets.R +\name{tar_prep_raw} +\alias{tar_prep_raw} +\title{Generate a set of targets for wrangling and pre-processing raw data} +\usage{ +tar_prep_raw( + contents, + action_raw_data = c("all", "parse", "none"), + name_data = "raw_data", + name_parsed = "raw_data_parsed", + name_indices = "indices" +) +} +\arguments{ +\item{contents}{The contents structure used as the configuration of data +fetching.} + +\item{action_raw_data}{The action to be taken on the fetched raw data.} + +\item{name_data}{The name of the raw data target.} + +\item{name_parsed}{The name of the parsed data target.} + +\item{name_indices}{The name of the indices target.} +} +\value{ +A list of target objects. +} +\description{ +This target factory is the main part of the \code{tar_prep_iquizoo} function. It +wrangles the raw data into a tidy format and calculates indices based on the +parsed data. +} From 291fe403b8862a2af1bff64f9647ec726b879606 Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Thu, 28 Mar 2024 22:12:08 +0800 Subject: [PATCH 17/25] Add missing functions Signed-off-by: Liang Zhang --- _pkgdown.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/_pkgdown.yml b/_pkgdown.yml index 715decd..4ff8f0e 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -7,6 +7,9 @@ reference: contents: - use_targets_pipeline - tar_prep_iquizoo + - tar_prep_proj + - tar_fetch_data + - tar_prep_raw - title: "Low-level Database operations" desc: Functions to help you interact with database. contents: From 5fd44f2917b0803507d3f2a2ed468ebf27fc9935 Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Fri, 29 Mar 2024 11:04:34 +0800 Subject: [PATCH 18/25] Enhance `check_progress` support Signed-off-by: Liang Zhang --- R/targets.R | 53 +++++++++++++++++++++-------------- man/tar_fetch_data.Rd | 7 ++++- tests/testthat/test-targets.R | 20 +++++++++++++ 3 files changed, 58 insertions(+), 22 deletions(-) diff --git a/R/targets.R b/R/targets.R index afa232a..7a1d835 100644 --- a/R/targets.R +++ b/R/targets.R @@ -81,7 +81,7 @@ tar_prep_iquizoo <- function(params, ..., tar_prep_proj(contents, templates, check_progress), sapply( what, - \(what) tar_fetch_data(contents, templates, what), + \(what) tar_fetch_data(contents, templates, what, check_progress), simplify = FALSE ), if ("raw_data" %in% what && action_raw_data != "none") { @@ -121,20 +121,22 @@ tar_prep_proj <- function(contents, templates = setup_templates(), check_progress = TRUE) { c( - tarchetypes::tar_map( - data.frame(project_id = as.character(unique(contents$project_id))), - targets::tar_target_raw( - "progress_hash", - bquote( - fetch_iquizoo( - .(read_file(templates[["progress_hash"]])), - params = list(project_id) - ) - ), - packages = "tarflow.iquizoo", - cue = targets::tar_cue(if (check_progress) "always") + if (check_progress) { + tarchetypes::tar_map( + data.frame(project_id = as.character(unique(contents$project_id))), + targets::tar_target_raw( + "progress_hash", + bquote( + fetch_iquizoo( + .(read_file(templates[["progress_hash"]])), + params = list(project_id) + ) + ), + packages = "tarflow.iquizoo", + cue = targets::tar_cue("always") + ) ) - ), + }, targets::tar_target_raw( "users", bquote( @@ -159,11 +161,15 @@ tar_prep_proj <- function(contents, #' @param templates The SQL template files used to fetch data. See #' [setup_templates()] for details. #' @param what What to fetch. +#' @param check_progress Whether to check the progress hash. If set as `TRUE`, +#' targets objects names as `progress_hash_{project_id}` will be checked. Set +#' it as `FALSE` if the project is finalized. #' @return A list of target objects. #' @export tar_fetch_data <- function(contents, templates = setup_templates(), - what = c("raw_data", "scores")) { + what = c("raw_data", "scores"), + check_progress = TRUE) { what <- match.arg(what) game_ids <- unique(as.character(contents$game_id)) targets <- vector("list", length(game_ids)) @@ -174,9 +180,15 @@ tar_fetch_data <- function(contents, ) targets[[game_id]] <- targets::tar_target_raw( paste0(what, "_", game_id), - bquote( - { - list(..(syms(paste0("progress_hash_", project_ids)))) + as.call(c( + quote(`{`), + if (check_progress) { + bquote( + list(..(syms(paste0("progress_hash_", project_ids)))), + splice = TRUE + ) + }, + bquote( do.call( rbind, .mapply( @@ -188,9 +200,8 @@ tar_fetch_data <- function(contents, ) ) ) - }, - splice = TRUE - ), + ) + )), packages = "tarflow.iquizoo" ) } diff --git a/man/tar_fetch_data.Rd b/man/tar_fetch_data.Rd index 935b713..bedd6b6 100644 --- a/man/tar_fetch_data.Rd +++ b/man/tar_fetch_data.Rd @@ -7,7 +7,8 @@ tar_fetch_data( contents, templates = setup_templates(), - what = c("raw_data", "scores") + what = c("raw_data", "scores"), + check_progress = TRUE ) } \arguments{ @@ -18,6 +19,10 @@ fetching.} \code{\link[=setup_templates]{setup_templates()}} for details.} \item{what}{What to fetch.} + +\item{check_progress}{Whether to check the progress hash. If set as \code{TRUE}, +targets objects names as \verb{progress_hash_\{project_id\}} will be checked. Set +it as \code{FALSE} if the project is finalized.} } \value{ A list of target objects. diff --git a/tests/testthat/test-targets.R b/tests/testthat/test-targets.R index c126237..d04473c 100644 --- a/tests/testthat/test-targets.R +++ b/tests/testthat/test-targets.R @@ -151,3 +151,23 @@ test_that("Ensure project date is used", { nrow(targets::tar_read(scores)) |> expect_gt(0) }) }) + +test_that("Ensure `check_progress = FALSE` work", { + targets::tar_dir({ + targets::tar_script({ + library(targets) + tar_prep_iquizoo( + contents = data.frame( + project_id = bit64::as.integer64(132121231360389), + game_id = bit64::as.integer64(268008982646879) + ), + what = "scores", + check_progress = FALSE + ) + }) + targets::tar_make(reporter = "silent", callr_function = NULL) + expect_true( + all(!startsWith(targets::tar_objects(), "progress_hash")) + ) + }) +}) From b39c66a9f27ebde6f6451fc56909daba96258c25 Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Fri, 29 Mar 2024 11:07:31 +0800 Subject: [PATCH 19/25] NEWS bulletin Signed-off-by: Liang Zhang --- NEWS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NEWS.md b/NEWS.md index 57d74cc..49371ee 100644 --- a/NEWS.md +++ b/NEWS.md @@ -9,6 +9,8 @@ * Added `suffix_format` argument to `fetch_data()` to specify the format of suffix in the query file. This is useful when you want to use a different format of suffix in the query file. * Enhanced the documentation of `fetch_data()`. * Let package not depend on dplyr, tidyr and purrr packages (#84). +* Exported more targets factory functions: `tar_prep_proj()`, `tar_fetch_data()`, `tar_prep_raw()`. +* Do not add `progress_hash` objects when `check_progress` is set to `FALSE` in `tar_prep_iquizoo()`. # tarflow.iquizoo 3.9.3 From d388f83eaf5931e9341ee980fffa4816fc074435 Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Fri, 29 Mar 2024 11:22:17 +0800 Subject: [PATCH 20/25] Enhance documentation Signed-off-by: Liang Zhang --- R/targets.R | 12 ++++++++---- man/tar_fetch_data.Rd | 6 ++++-- man/tar_prep_proj.Rd | 6 ++++-- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/R/targets.R b/R/targets.R index 7a1d835..4cce1d2 100644 --- a/R/targets.R +++ b/R/targets.R @@ -113,8 +113,10 @@ tar_prep_iquizoo <- function(params, ..., #' fetching. #' @param templates The SQL template files used to fetch data. See #' [setup_templates()] for details. -#' @param check_progress Whether to check the progress hash. Set it as `FALSE` -#' if the project is finalized. +#' @param check_progress Whether to check the progress hash. When set as `TRUE`, +#' a progress hash objects named as `progress_hash_{project_id}` for each +#' project will be added into the target list. Set it as `FALSE` if the +#' projects are finalized. #' @return A list of target objects. #' @export tar_prep_proj <- function(contents, @@ -162,8 +164,10 @@ tar_prep_proj <- function(contents, #' [setup_templates()] for details. #' @param what What to fetch. #' @param check_progress Whether to check the progress hash. If set as `TRUE`, -#' targets objects names as `progress_hash_{project_id}` will be checked. Set -#' it as `FALSE` if the project is finalized. +#' Before fetching the data, the progress hash objects named as +#' `progress_hash_{project_id}` will be depended on, which are typically +#' generated by [tar_prep_proj()]. If the projects are finalized, set this +#' argument as `FALSE`. #' @return A list of target objects. #' @export tar_fetch_data <- function(contents, diff --git a/man/tar_fetch_data.Rd b/man/tar_fetch_data.Rd index bedd6b6..8a52434 100644 --- a/man/tar_fetch_data.Rd +++ b/man/tar_fetch_data.Rd @@ -21,8 +21,10 @@ fetching.} \item{what}{What to fetch.} \item{check_progress}{Whether to check the progress hash. If set as \code{TRUE}, -targets objects names as \verb{progress_hash_\{project_id\}} will be checked. Set -it as \code{FALSE} if the project is finalized.} +Before fetching the data, the progress hash objects named as +\verb{progress_hash_\{project_id\}} will be depended on, which are typically +generated by \code{\link[=tar_prep_proj]{tar_prep_proj()}}. If the projects are finalized, set this +argument as \code{FALSE}.} } \value{ A list of target objects. diff --git a/man/tar_prep_proj.Rd b/man/tar_prep_proj.Rd index 24b294f..17b2101 100644 --- a/man/tar_prep_proj.Rd +++ b/man/tar_prep_proj.Rd @@ -13,8 +13,10 @@ fetching.} \item{templates}{The SQL template files used to fetch data. See \code{\link[=setup_templates]{setup_templates()}} for details.} -\item{check_progress}{Whether to check the progress hash. Set it as \code{FALSE} -if the project is finalized.} +\item{check_progress}{Whether to check the progress hash. When set as \code{TRUE}, +a progress hash objects named as \verb{progress_hash_\{project_id\}} for each +project will be added into the target list. Set it as \code{FALSE} if the +projects are finalized.} } \value{ A list of target objects. From 856ff51cfb1e2d1ae0f89ebce20df44f0a1c4419 Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Fri, 29 Mar 2024 12:14:26 +0800 Subject: [PATCH 21/25] Update CIs Signed-off-by: Liang Zhang --- .github/workflows/R-CMD-check.yaml | 3 ++- .github/workflows/lint.yaml | 2 +- .github/workflows/pkgdown.yaml | 4 ++-- .github/workflows/test-coverage.yaml | 6 +++--- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index 653aa6e..777f092 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -29,7 +29,7 @@ jobs: R_KEEP_PKG_SOURCE: yes steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: r-lib/actions/setup-pandoc@v2 @@ -51,3 +51,4 @@ jobs: MYSQL_PASSWORD: ${{ secrets.MYSQL_PASSWORD }} with: upload-snapshots: true + build_args: 'c("--no-manual","--compact-vignettes=gs+qpdf")' diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index f4c4ef2..f60d047 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -14,7 +14,7 @@ jobs: env: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: r-lib/actions/setup-r@v2 with: diff --git a/.github/workflows/pkgdown.yaml b/.github/workflows/pkgdown.yaml index ed7650c..a7276e8 100644 --- a/.github/workflows/pkgdown.yaml +++ b/.github/workflows/pkgdown.yaml @@ -22,7 +22,7 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: r-lib/actions/setup-pandoc@v2 @@ -41,7 +41,7 @@ jobs: - name: Deploy to GitHub pages 🚀 if: github.event_name != 'pull_request' - uses: JamesIves/github-pages-deploy-action@v4.4.1 + uses: JamesIves/github-pages-deploy-action@v4.5.0 with: clean: false branch: gh-pages diff --git a/.github/workflows/test-coverage.yaml b/.github/workflows/test-coverage.yaml index ce99f80..709a124 100644 --- a/.github/workflows/test-coverage.yaml +++ b/.github/workflows/test-coverage.yaml @@ -15,7 +15,7 @@ jobs: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: r-lib/actions/setup-r@v2 with: @@ -43,12 +43,12 @@ jobs: if: always() run: | ## -------------------------------------------------------------------- - find ${{ runner.temp }}/package -name 'testthat.Rout*' -exec cat '{}' \; || true + find '${{ runner.temp }}/package' -name 'testthat.Rout*' -exec cat '{}' \; || true shell: bash - name: Upload test results if: failure() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: coverage-test-failures path: ${{ runner.temp }}/package From fcac202f7889c3feb847a0a1492dd13c6d10fad0 Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Fri, 29 Mar 2024 12:40:38 +0800 Subject: [PATCH 22/25] Remove "progress_hash" global variable Signed-off-by: Liang Zhang --- R/targets.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/targets.R b/R/targets.R index 4cce1d2..fc5dd76 100644 --- a/R/targets.R +++ b/R/targets.R @@ -271,7 +271,7 @@ objects <- function() { utils::globalVariables( c( - "progress_hash", "project_id", "game_id", + "project_id", "game_id", "tar_data", "tar_parsed", "tar_indices", "wrangle_data", "preproc_data", "prep_fun", "input", "extra" From 1a8ccebf001bb24804f92081e366cc9e25ceb61c Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Fri, 29 Mar 2024 14:32:52 +0800 Subject: [PATCH 23/25] Better handle preproc.iquizoo requirements Signed-off-by: Liang Zhang --- R/targets.R | 2 ++ 1 file changed, 2 insertions(+) diff --git a/R/targets.R b/R/targets.R index fc5dd76..8ddb9e1 100644 --- a/R/targets.R +++ b/R/targets.R @@ -243,6 +243,7 @@ tar_prep_raw <- function(contents, ) list( raw_data_parsed = if ("parse" %in% action_raw_data) { + check_installed("preproc.iquizoo", "becasue required in wrangling.") tarchetypes::tar_eval( targets::tar_target( tar_parsed, @@ -253,6 +254,7 @@ tar_prep_raw <- function(contents, ) }, indices = if ("preproc" %in% action_raw_data) { + check_installed("preproc.iquizoo", "becasue required in pre-processing.") tarchetypes::tar_eval( targets::tar_target( tar_indices, From d20b3abe14b32aef556fe5c60d03c8fd90630f14 Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Fri, 29 Mar 2024 16:29:58 +0800 Subject: [PATCH 24/25] Enhance code quality Signed-off-by: Liang Zhang --- R/targets.R | 74 +++++++++++++++----------------- tests/testthat/_snaps/targets.md | 10 ++--- 2 files changed, 39 insertions(+), 45 deletions(-) diff --git a/R/targets.R b/R/targets.R index 8ddb9e1..42ffeea 100644 --- a/R/targets.R +++ b/R/targets.R @@ -175,41 +175,40 @@ tar_fetch_data <- function(contents, what = c("raw_data", "scores"), check_progress = TRUE) { what <- match.arg(what) - game_ids <- unique(as.character(contents$game_id)) - targets <- vector("list", length(game_ids)) - names(targets) <- game_ids - for (game_id in game_ids) { - project_ids <- as.character( - contents$project_id[contents$game_id == game_id] - ) - targets[[game_id]] <- targets::tar_target_raw( - paste0(what, "_", game_id), - as.call(c( - quote(`{`), - if (check_progress) { + tapply( + contents, + contents$game_id, + \(contents) { + project_ids <- as.character(unique(contents$project_id)) + game_id <- as.character(unique(contents$game_id)) + targets::tar_target_raw( + paste0(what, "_", game_id), + as.call(c( + quote(`{`), + if (check_progress) { + bquote( + list(..(syms(paste0("progress_hash_", project_ids)))), + splice = TRUE + ) + }, bquote( - list(..(syms(paste0("progress_hash_", project_ids)))), - splice = TRUE - ) - }, - bquote( - do.call( - rbind, - .mapply( - fetch_data, - list(.(project_ids), .(game_id)), - MoreArgs = list( - what = .(what), - query = .(read_file(templates[[what]])) + do.call( + rbind, + .mapply( + fetch_data, + list(.(project_ids), .(game_id)), + MoreArgs = list( + what = .(what), + query = .(read_file(templates[[what]])) + ) ) ) ) - ) - )), - packages = "tarflow.iquizoo" - ) - } - targets + )), + packages = "tarflow.iquizoo" + ) + } + ) } #' Generate a set of targets for wrangling and pre-processing raw data @@ -233,14 +232,10 @@ tar_prep_raw <- function(contents, name_indices = "indices") { action_raw_data <- match.arg(action_raw_data) if (action_raw_data == "all") action_raw_data <- c("parse", "preproc") - contents <- within( - unique(contents["game_id"]), - { - tar_data <- syms(sprintf("%s_%s", name_data, game_id)) - tar_parsed <- syms(sprintf("%s_%s", name_parsed, game_id)) - tar_indices <- syms(sprintf("%s_%s", name_indices, game_id)) - } - ) + contents <- unique(contents["game_id"]) + contents$tar_data <- syms(sprintf("%s_%s", name_data, contents$game_id)) + contents$tar_parsed <- syms(sprintf("%s_%s", name_parsed, contents$game_id)) + contents$tar_indices <- syms(sprintf("%s_%s", name_indices, contents$game_id)) list( raw_data_parsed = if ("parse" %in% action_raw_data) { check_installed("preproc.iquizoo", "becasue required in wrangling.") @@ -273,7 +268,6 @@ objects <- function() { utils::globalVariables( c( - "project_id", "game_id", "tar_data", "tar_parsed", "tar_indices", "wrangle_data", "preproc_data", "prep_fun", "input", "extra" diff --git a/tests/testthat/_snaps/targets.md b/tests/testthat/_snaps/targets.md index 2fadd7e..2c60a50 100644 --- a/tests/testthat/_snaps/targets.md +++ b/tests/testthat/_snaps/targets.md @@ -131,12 +131,12 @@ "value": ["integer64"] } }, - "value": [1.93454543e-309, 1.93454543e-309, 1.93454543e-309, 1.93454543e-309, 1.93454543e-309, 1.93454543e-309, 1.93454543e-309, 1.91991274e-309, 1.91991274e-309, 1.91991274e-309, 1.91991274e-309, 1.91991274e-309, 1.91991274e-309, 1.91991274e-309, 1.91796743e-309, 1.91796743e-309, 1.91796743e-309, 1.91796743e-309, 1.91796743e-309, 1.91796743e-309, 1.91796743e-309, 1.89620222e-309, 1.89620222e-309, 1.89620222e-309, 1.89620222e-309, 1.89620222e-309, 1.89620222e-309, 1.89620222e-309, 1.89618307e-309, 1.89618307e-309, 1.89618307e-309, 1.89618307e-309, 1.89618307e-309, 1.89618307e-309, 1.89618307e-309, 1.89562643e-309, 1.89562643e-309, 1.89562643e-309, 1.89562643e-309, 1.89562643e-309, 1.89562643e-309, 1.89562643e-309, 1.89560496e-309, 1.89560496e-309, 1.89560496e-309, 1.89560496e-309, 1.89560496e-309, 1.89560496e-309, 1.89560496e-309] + "value": [1.89560496e-309, 1.89560496e-309, 1.89560496e-309, 1.89560496e-309, 1.89560496e-309, 1.89560496e-309, 1.89560496e-309, 1.89562643e-309, 1.89562643e-309, 1.89562643e-309, 1.89562643e-309, 1.89562643e-309, 1.89562643e-309, 1.89562643e-309, 1.89618307e-309, 1.89618307e-309, 1.89618307e-309, 1.89618307e-309, 1.89618307e-309, 1.89618307e-309, 1.89618307e-309, 1.89620222e-309, 1.89620222e-309, 1.89620222e-309, 1.89620222e-309, 1.89620222e-309, 1.89620222e-309, 1.89620222e-309, 1.91796743e-309, 1.91796743e-309, 1.91796743e-309, 1.91796743e-309, 1.91796743e-309, 1.91796743e-309, 1.91796743e-309, 1.91991274e-309, 1.91991274e-309, 1.91991274e-309, 1.91991274e-309, 1.91991274e-309, 1.91991274e-309, 1.91991274e-309, 1.93454543e-309, 1.93454543e-309, 1.93454543e-309, 1.93454543e-309, 1.93454543e-309, 1.93454543e-309, 1.93454543e-309] }, { "type": "character", "attributes": {}, - "value": ["", "", "", "", "", "", "", "", "", "", "", "", "", "", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0"] + "value": ["1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "1.0.0", "", "", "", "", "", "", "", "", "", "", "", "", "", ""] }, { "type": "double", @@ -152,7 +152,7 @@ "value": ["POSIXct", "POSIXt"] } }, - "value": [1681983351, 1681983969, 1681985754, 1681986356, 1681987913, 1681988834, 1681989212, 1681982784, 1681983237, 1681985329, 1681985755, 1681987218, 1681988416, 1681988719, 1681982923, 1681983470, 1681985408, 1681985868, 1681987354, 1681988489, 1681988797, 1681983341, 1681983950, 1681985742, 1681986338, 1681987893, 1681988820, 1681989201, 1681983245, 1681983843, 1681985675, 1681986218, 1681987764, 1681988767, 1681989104, 1681983054, 1681983633, 1681985540, 1681986023, 1681987536, 1681988613, 1681988956, 1681982990, 1681983549, 1681985473, 1681985928, 1681987439, 1681988555, 1681988878] + "value": [1681982990, 1681983549, 1681985473, 1681985928, 1681987439, 1681988555, 1681988878, 1681983054, 1681983633, 1681985540, 1681986023, 1681987536, 1681988613, 1681988956, 1681983245, 1681983843, 1681985675, 1681986218, 1681987764, 1681988767, 1681989104, 1681983341, 1681983950, 1681985742, 1681986338, 1681987893, 1681988820, 1681989201, 1681982923, 1681983470, 1681985408, 1681985868, 1681987354, 1681988489, 1681988797, 1681982784, 1681983237, 1681985329, 1681985755, 1681987218, 1681988416, 1681988719, 1681983351, 1681983969, 1681985754, 1681986356, 1681987913, 1681988834, 1681989212] }, { "type": "double", @@ -163,7 +163,7 @@ "value": ["integer64"] } }, - "value": [1.93454615e-309, 1.93454615e-309, 1.93454615e-309, 1.93454615e-309, 1.93454615e-309, 1.93454615e-309, 1.93454615e-309, 1.91991319e-309, 1.91991319e-309, 1.91991319e-309, 1.91991319e-309, 1.91991319e-309, 1.91991319e-309, 1.91991319e-309, 1.9179678e-309, 1.9179678e-309, 1.9179678e-309, 1.9179678e-309, 1.9179678e-309, 1.9179678e-309, 1.9179678e-309, 1.89620278e-309, 1.89620278e-309, 1.89620278e-309, 1.89620278e-309, 1.89620278e-309, 1.89620278e-309, 1.89620278e-309, 1.89618415e-309, 1.89618415e-309, 1.89618415e-309, 1.89618415e-309, 1.89618415e-309, 1.89618415e-309, 1.89618415e-309, 1.89563061e-309, 1.89563061e-309, 1.89563061e-309, 1.89563061e-309, 1.89563061e-309, 1.89563061e-309, 1.89563061e-309, 1.89560665e-309, 1.89560665e-309, 1.89560665e-309, 1.89560665e-309, 1.89560665e-309, 1.89560665e-309, 1.89560665e-309] + "value": [1.89560665e-309, 1.89560665e-309, 1.89560665e-309, 1.89560665e-309, 1.89560665e-309, 1.89560665e-309, 1.89560665e-309, 1.89563061e-309, 1.89563061e-309, 1.89563061e-309, 1.89563061e-309, 1.89563061e-309, 1.89563061e-309, 1.89563061e-309, 1.89618415e-309, 1.89618415e-309, 1.89618415e-309, 1.89618415e-309, 1.89618415e-309, 1.89618415e-309, 1.89618415e-309, 1.89620278e-309, 1.89620278e-309, 1.89620278e-309, 1.89620278e-309, 1.89620278e-309, 1.89620278e-309, 1.89620278e-309, 1.9179678e-309, 1.9179678e-309, 1.9179678e-309, 1.9179678e-309, 1.9179678e-309, 1.9179678e-309, 1.9179678e-309, 1.91991319e-309, 1.91991319e-309, 1.91991319e-309, 1.91991319e-309, 1.91991319e-309, 1.91991319e-309, 1.91991319e-309, 1.93454615e-309, 1.93454615e-309, 1.93454615e-309, 1.93454615e-309, 1.93454615e-309, 1.93454615e-309, 1.93454615e-309] }, { "type": "integer", @@ -178,7 +178,7 @@ { "type": "double", "attributes": {}, - "value": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5] + "value": [-0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] }, { "type": "double", From 56dbf8d5225ad09d484e607dbd8f1f817feb7409 Mon Sep 17 00:00:00 2001 From: Liang Zhang Date: Fri, 29 Mar 2024 17:53:02 +0800 Subject: [PATCH 25/25] Fix compatibility issue with R older than 4.3.0 Signed-off-by: Liang Zhang --- R/targets.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/targets.R b/R/targets.R index 42ffeea..3636270 100644 --- a/R/targets.R +++ b/R/targets.R @@ -175,7 +175,7 @@ tar_fetch_data <- function(contents, what = c("raw_data", "scores"), check_progress = TRUE) { what <- match.arg(what) - tapply( + by( contents, contents$game_id, \(contents) {