From 79638f5109bfb0829409c50d72d244986bf55b9e Mon Sep 17 00:00:00 2001 From: Paul Wessel Date: Tue, 18 Jan 2022 14:56:19 -1000 Subject: [PATCH 1/2] Let GMT detect if DCW on server has been updated For those who rely on the cloud to get the latest DCW, this PR will check if the version in the cloud is fresher than theirs, and if so we delete the local files in .gmt/geography/dcw so new ones can be downloaded. --- src/gmt_dcw.c | 56 +++++++++++++++++++++++++++++++++++++++++++++ src/gmt_internals.h | 1 + src/gmt_remote.c | 45 +++++++++++++++++++++++++++--------- 3 files changed, 91 insertions(+), 11 deletions(-) diff --git a/src/gmt_dcw.c b/src/gmt_dcw.c index 590662c979b..a2542ebde31 100644 --- a/src/gmt_dcw.c +++ b/src/gmt_dcw.c @@ -89,6 +89,32 @@ static char *GMT_DCW_continents[GMT_DCW_N_CONTINENTS] = {"Africa", "Antarctica", /* Local functions only visible inside this file */ +GMT_LOCAL bool gmtdcw_need_refresh (struct GMT_CTRL *GMT, char *fpath) { + /* Obtains versions from both local and cloud versions and returns true if we must refresh */ + FILE *fp = NULL; + char this_version[GMT_LEN16] = {""}, cloud_version[GMT_LEN16] = {""}; + char path[PATH_MAX] = {""}; + int tmajor, tminor, tpatch = 0, cmajor, cminor, cpatch = 0; + int cdfid, err; + + if (gmt_nc_open (GMT, fpath, NC_NOWRITE, &cdfid)) return true; /* fpath is dcw-gmt.nc so if not found we have work to do */ + /* Get version attribute */ + gmt_M_memset (this_version, strlen (this_version), char); + if (nc_get_att_text (cdfid, NC_GLOBAL, "version", this_version)) return true; /* If we cannot read version then what file are we reading here? */ + gmt_nc_close (GMT, cdfid); /* Done with reading netCDF file */ + sprintf (path, "%s/geography/dcw/VERSION", GMT->session.USERDIR); /* Open local VERSION file which we know is fresh */ + if ((fp = fopen (path, "r")) && fscanf (fp, "%s\n", cloud_version) < 1) return false; /* Not good, cannot happen but also safety valve */ + fclose (fp); /* Done reading from VERSION */ + gmt_strrepc (this_version, '.', ' '); /* Get rid of periods */ + gmt_strrepc (cloud_version, '.', ' '); /* Get rid of periods */ + sscanf (this_version, "%d %d %d", &tmajor, &tminor, &tpatch); /* Get the version major, minor and patch numbers */ + sscanf (cloud_version, "%d %d %d", &cmajor, &cminor, &cpatch); + if (cmajor > tmajor) return true; /* Cloud major version is more recent, must refresh */ + if (cminor > tminor) return true; /* Cloud major equals this major, but cloud minor version is more recent so must refresh */ + if (cpatch > tpatch) return true; /* Cloud major and minor equals this major and minor, but cloud patch is more recent so must refresh */ + return false; /* No, we are up to date */ +} + GMT_LOCAL bool gmtdcw_get_path (struct GMT_CTRL *GMT, char *name, char *suffix, char *path) { /* This is the order of checking: * 1. Check in GMT->session.DCWDIR, if set @@ -116,7 +142,37 @@ GMT_LOCAL bool gmtdcw_get_path (struct GMT_CTRL *GMT, char *name, char *suffix, } } if (GMT->session.USERDIR) { /* Check user dir via remote download */ + int action; char remote_path[PATH_MAX] = {""}; + /* First check what the DCW version is on the server */ + sprintf (path, "%s/geography/dcw/VERSION", GMT->session.USERDIR); + action = gmtlib_refresh_file (GMT->parent, path); + if (action == GMT_NOTSET) return false; + if (action == GMT_YES_DOWNLOAD) { /* Update the VERSION file before checking version */ + sprintf (path, "%s/geography/dcw", GMT->session.USERDIR); /* Local directory destination */ + if (access (path, R_OK) && gmt_mkdir (path)) { /* Must first create the directory */ + GMT_Report (GMT->parent, GMT_MSG_ERROR, "Unable to create GMT directory : %s\n", path); + return false; + } + sprintf (path, "%s/geography/dcw/VERSION", GMT->session.USERDIR); + snprintf (remote_path, PATH_MAX, "%s/geography/dcw/VERSION", gmt_dataserver_url (GMT->parent)); /* Unique remote path */ + if (gmt_download_file (GMT, "VERSION", remote_path, path, true)) { + GMT_Report (GMT->parent, GMT_MSG_ERROR, "Unable to obtain remote file %s\n", remote_path); + return false; + } + else /* Successfully downloaded the first time */ + GMT_Report (GMT->parent, GMT_MSG_INFORMATION, "Refreshed DCW VERSION file %s\n", path); + } + /* First check if the netCDF file is here and if it needs refreshing */ + sprintf (path, "%s/geography/dcw/dcw-gmt.nc", GMT->session.USERDIR); + if (access (path, R_OK) == 0) { /* Previously downloaded, check if outdated */ + if (gmtdcw_need_refresh (GMT, path)) { /* Yep, wipe older files so new can be downloaded */ + sprintf (path, "%s/geography/dcw", GMT->session.USERDIR); + if (gmt_remove_dir (GMT->parent, path, false)) { + GMT_Report (GMT->parent, GMT_MSG_ERROR, "Unable to remove directory %s\n", path); + } + } + } sprintf (path, "%s/geography/dcw/%s%s", GMT->session.USERDIR, name, suffix); if (access (path, R_OK) == 0) { /* Previously downloaded */ GMT_Report (GMT->parent, GMT_MSG_DEBUG, "3. DCW: Read the Digital Chart of the World from %s\n", path); diff --git a/src/gmt_internals.h b/src/gmt_internals.h index dc345e624fd..b7e7f3cfd57 100644 --- a/src/gmt_internals.h +++ b/src/gmt_internals.h @@ -53,6 +53,7 @@ struct GMT_XINGS { EXTERN_MSC char *dlerror (void); #endif +EXTERN_MSC int gmtlib_refresh_file (struct GMTAPI_CTRL *API, char *file); EXTERN_MSC void gmtlib_terminate_session (); EXTERN_MSC unsigned int gmtlib_pick_in_col_number (struct GMT_CTRL *GMT, unsigned int col, unsigned int *col_pos_in); EXTERN_MSC bool gmtlib_set_do_seconds (struct GMT_CTRL *GMT, double inc); diff --git a/src/gmt_remote.c b/src/gmt_remote.c index 491ceb0a00c..ed59c60e343 100644 --- a/src/gmt_remote.c +++ b/src/gmt_remote.c @@ -807,6 +807,36 @@ GMT_LOCAL struct GMT_DATA_HASH *gmtremote_hash_load (struct GMT_CTRL *GMT, char return (L); }; +int gmtlib_refresh_file (struct GMTAPI_CTRL *API, char *file) { + /* Return various codes depending on outcome: + * If file does not exist, return 1 for downloading it + * If file exist but unable to stat, return -1 for error + * If file exist and is older than refresh_time, return 1 for re-downloading it + * If file exist and is young enough, return 0 to use as is. + */ + struct stat buf; + time_t mod_time, right_now = time (NULL); /* Unix time right now */ + + if (access (file, R_OK)) return GMT_YES_DOWNLOAD; /* Not found locally so need to download the first time */ + + if (stat (file, &buf)) { /* Should not happen, but who knows */ + GMT_Report (API, GMT_MSG_ERROR, "Unable to get information about %s via stat - abort\n", file); + return GMT_NOTSET; + } + + /* Get the file's modification (creation) time */ +#ifdef __APPLE__ + mod_time = buf.st_mtimespec.tv_sec; /* Apple even has tv_nsec for nano-seconds... */ +#else + mod_time = buf.st_mtime; +#endif + + if ((right_now - mod_time) > (GMT_DAY2SEC_I * API->GMT->current.setting.refresh_time)) + return GMT_YES_DOWNLOAD; /* Older than selected number of days; Time to get a new file */ + else + return GMT_NO_DOWNLOAD; /*OK to use this file */ +} + GMT_LOCAL int gmtremote_refresh (struct GMTAPI_CTRL *API, unsigned int index) { /* This function is called every time we are about to access a @remotefile. * It is called twice: Once for the hash table and once for the info table. @@ -825,6 +855,7 @@ GMT_LOCAL int gmtremote_refresh (struct GMTAPI_CTRL *API, unsigned int index) { * The result of this is that any file(s) that have changed will be removed * so that they must be downloaded again to get the new versions. */ + int action; struct stat buf; time_t mod_time, right_now = time (NULL); /* Unix time right now */ char indexpath[PATH_MAX] = {""}, old_indexpath[PATH_MAX] = {""}, new_indexpath[PATH_MAX] = {""}, url[PATH_MAX] = {""}; @@ -862,18 +893,10 @@ GMT_LOCAL int gmtremote_refresh (struct GMTAPI_CTRL *API, unsigned int index) { /* Here we have the existing index file and its path is in indexpath. Check how old it is */ - if (stat (indexpath, &buf)) { - GMT_Report (API, GMT_MSG_ERROR, "Unable to get information about %s - abort\n", indexpath); - return 1; - } - /* Get its modification (creation) time */ -#ifdef __APPLE__ - mod_time = buf.st_mtimespec.tv_sec; /* Apple even has tv_nsec for nano-seconds... */ -#else - mod_time = buf.st_mtime; -#endif + action = gmtlib_refresh_file (API, indexpath); + if (action == GMT_NOTSET) return 1; - if ((right_now - mod_time) > (GMT_DAY2SEC_I * GMT->current.setting.refresh_time)) { /* Older than selected number of days; Time to get a new index file */ + if (action == GMT_YES_DOWNLOAD) { /* Older than selected number of days; Time to get a new index file */ GMT_Report (API, GMT_MSG_DEBUG, "File %s older than 24 hours, get latest from server.\n", indexpath); strcpy (new_indexpath, indexpath); /* Duplicate path name */ strcat (new_indexpath, ".new"); /* Append .new to the copied path */ From 3ea3dc56e9c03a2602bebf0746cb2ed54aec8c49 Mon Sep 17 00:00:00 2001 From: Paul Wessel Date: Tue, 18 Jan 2022 16:12:38 -1000 Subject: [PATCH 2/2] clean up --- src/gmt_dcw.c | 5 ++--- src/gmt_remote.c | 5 +---- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/gmt_dcw.c b/src/gmt_dcw.c index a2542ebde31..20a5ffea652 100644 --- a/src/gmt_dcw.c +++ b/src/gmt_dcw.c @@ -95,7 +95,7 @@ GMT_LOCAL bool gmtdcw_need_refresh (struct GMT_CTRL *GMT, char *fpath) { char this_version[GMT_LEN16] = {""}, cloud_version[GMT_LEN16] = {""}; char path[PATH_MAX] = {""}; int tmajor, tminor, tpatch = 0, cmajor, cminor, cpatch = 0; - int cdfid, err; + int cdfid; if (gmt_nc_open (GMT, fpath, NC_NOWRITE, &cdfid)) return true; /* fpath is dcw-gmt.nc so if not found we have work to do */ /* Get version attribute */ @@ -146,8 +146,7 @@ GMT_LOCAL bool gmtdcw_get_path (struct GMT_CTRL *GMT, char *name, char *suffix, char remote_path[PATH_MAX] = {""}; /* First check what the DCW version is on the server */ sprintf (path, "%s/geography/dcw/VERSION", GMT->session.USERDIR); - action = gmtlib_refresh_file (GMT->parent, path); - if (action == GMT_NOTSET) return false; + if ((action = gmtlib_refresh_file (GMT->parent, path)) == GMT_NOTSET) return false; if (action == GMT_YES_DOWNLOAD) { /* Update the VERSION file before checking version */ sprintf (path, "%s/geography/dcw", GMT->session.USERDIR); /* Local directory destination */ if (access (path, R_OK) && gmt_mkdir (path)) { /* Must first create the directory */ diff --git a/src/gmt_remote.c b/src/gmt_remote.c index ed59c60e343..a6bd3d8a51c 100644 --- a/src/gmt_remote.c +++ b/src/gmt_remote.c @@ -856,8 +856,6 @@ GMT_LOCAL int gmtremote_refresh (struct GMTAPI_CTRL *API, unsigned int index) { * so that they must be downloaded again to get the new versions. */ int action; - struct stat buf; - time_t mod_time, right_now = time (NULL); /* Unix time right now */ char indexpath[PATH_MAX] = {""}, old_indexpath[PATH_MAX] = {""}, new_indexpath[PATH_MAX] = {""}, url[PATH_MAX] = {""}; const char *index_file = (index == GMT_HASH_INDEX) ? GMT_HASH_SERVER_FILE : GMT_INFO_SERVER_FILE; struct LOCFILE_FP *LF = NULL; @@ -893,8 +891,7 @@ GMT_LOCAL int gmtremote_refresh (struct GMTAPI_CTRL *API, unsigned int index) { /* Here we have the existing index file and its path is in indexpath. Check how old it is */ - action = gmtlib_refresh_file (API, indexpath); - if (action == GMT_NOTSET) return 1; + if ((action = gmtlib_refresh_file (API, indexpath)) == GMT_NOTSET) return 1; if (action == GMT_YES_DOWNLOAD) { /* Older than selected number of days; Time to get a new index file */ GMT_Report (API, GMT_MSG_DEBUG, "File %s older than 24 hours, get latest from server.\n", indexpath);