From b5307133bebfc27dfcb4b5991a2a5fbdcee6dc23 Mon Sep 17 00:00:00 2001 From: Adam Collins Date: Mon, 20 Jan 2025 14:12:53 +1000 Subject: [PATCH 1/3] #483 add sandbox like UI --- gradle.properties | 2 +- grails-app/assets/compile/spAppModules.js | 2 +- .../spApp/controller/addPointsCtrl.js | 303 ++++++++++++++++++ .../spApp/service/biocacheService.js | 22 +- .../spApp/service/layersService.js | 52 +++ .../spApp/templates/addPointsContent.tpl.htm | 165 ++++++++++ .../spApp/templates/sandBoxContent.tpl.htm | 2 +- grails-app/conf/application.yml | 8 + grails-app/conf/logback.xml | 2 +- grails-app/conf/menu-config.json | 7 +- .../spatial/portal/PortalController.groovy | 54 +++- .../org/ala/spatial/portal/UrlMappings.groovy | 3 + grails-app/i18n/messages.properties | 14 + grails-app/views/portal/index.gsp | 2 + 14 files changed, 620 insertions(+), 18 deletions(-) create mode 100644 grails-app/assets/javascripts/spApp/controller/addPointsCtrl.js create mode 100644 grails-app/assets/javascripts/spApp/templates/addPointsContent.tpl.htm diff --git a/gradle.properties b/gradle.properties index 00c1447e..0c2cf77d 100644 --- a/gradle.properties +++ b/gradle.properties @@ -10,4 +10,4 @@ webdriverBinariesVersion=2.6 #chromeDriverVersion=2.45.0 geckodriverVersion=0.24.0 seleniumSafariDriverVersion=3.14.0 -alaSecurityLibsVersion=6.2.0 +alaSecurityLibsVersion=6.3.0 diff --git a/grails-app/assets/compile/spAppModules.js b/grails-app/assets/compile/spAppModules.js index 0ae47478..8cf7beb7 100644 --- a/grails-app/assets/compile/spAppModules.js +++ b/grails-app/assets/compile/spAppModules.js @@ -1,2 +1,2 @@ /* Do not edit. This file built at compile by _Events.groovy */ -$spAppModules = ["tool-add-layer-service","tool-export-checklist-service","sandbox-service","lists-service","layers-service","http-service","colour-service","i18n-service","tool-export-sample-service","species-auto-complete-service","logger-service","menu-service","doi-service","tool-export-doi-service","layout-service","tool-add-facet-service","url-params-service","popup-service","facet-auto-complete-service","tool-export-area-service","gaz-auto-complete-service","layer-distances-service","keep-alive-service","tool-area-report-service","poi-service","bie-service","event-service","predefined-layer-lists-service","sessions-service","tool-add-species-service","tools-service","predefined-areas-service","tool-export-map-service","biocache-service","flickr-service","workflow-service","map-service","select-area-directive","facet-editor-directive","select-date-range-directive","i18n-directive","sp-percent-directive","sp-options-directive","sp-legend-directive","layer-list-upload-directive","lists-list-directive","species-auto-complete-directive","envelope-directive","select-multiple-area-directive","nearest-locality-directive","sp-menu-directive","sandbox-list-directive","http-ui-directive","draw-area-directive","workflow-annotation-directive","point-comparison-directive","species-options-directive","leaflet-quick-links-directive","doi-auto-complete-directive","google-places-directive","lifeform-select-directive","playback-directive","layer-auto-complete-directive","layer-list-select-directive","select-facet-directive","select-species-directive","workflow-item-directive","area-list-select-directive","sp-map-directive","select-layers-directive","biocache-chart-directive","gaz-auto-complete-directive","modal-iframe-instance-ctrl","species-info-ctrl","facet-ctrl","workflow-ctrl","add-d-o-i-ctrl","tabulate-ctrl","layout-ctrl","area-report-ctrl","add-w-m-s-ctrl","tool-ctrl","leaflet-map-controller","sand-box-ctrl","sessions-ctrl","create-species-list-ctrl","csv-ctrl","facet-editor-modal-ctrl","add-area-ctrl","log-ctrl","analysis-ctrl"]; \ No newline at end of file +$spAppModules = ["tool-add-layer-service","tool-export-checklist-service","sandbox-service","lists-service","layers-service","http-service","colour-service","i18n-service","tool-export-sample-service","species-auto-complete-service","logger-service","menu-service","doi-service","tool-export-doi-service","layout-service","tool-add-facet-service","url-params-service","popup-service","facet-auto-complete-service","tool-export-area-service","gaz-auto-complete-service","layer-distances-service","keep-alive-service","tool-area-report-service","poi-service","bie-service","event-service","predefined-layer-lists-service","sessions-service","tool-add-species-service","tools-service","predefined-areas-service","tool-export-map-service","biocache-service","flickr-service","workflow-service","map-service","select-area-directive","facet-editor-directive","select-date-range-directive","i18n-directive","sp-percent-directive","sp-options-directive","sp-legend-directive","layer-list-upload-directive","lists-list-directive","species-auto-complete-directive","envelope-directive","select-multiple-area-directive","nearest-locality-directive","sp-menu-directive","sandbox-list-directive","http-ui-directive","draw-area-directive","workflow-annotation-directive","point-comparison-directive","species-options-directive","leaflet-quick-links-directive","doi-auto-complete-directive","google-places-directive","lifeform-select-directive","playback-directive","layer-auto-complete-directive","layer-list-select-directive","select-facet-directive","select-species-directive","workflow-item-directive","area-list-select-directive","sp-map-directive","select-layers-directive","biocache-chart-directive","gaz-auto-complete-directive","modal-iframe-instance-ctrl","species-info-ctrl","facet-ctrl","workflow-ctrl","add-d-o-i-ctrl","tabulate-ctrl","layout-ctrl","area-report-ctrl","add-w-m-s-ctrl","tool-ctrl","leaflet-map-controller","add-points-ctrl","sand-box-ctrl","sessions-ctrl","create-species-list-ctrl","csv-ctrl","facet-editor-modal-ctrl","add-area-ctrl","log-ctrl","analysis-ctrl"]; \ No newline at end of file diff --git a/grails-app/assets/javascripts/spApp/controller/addPointsCtrl.js b/grails-app/assets/javascripts/spApp/controller/addPointsCtrl.js new file mode 100644 index 00000000..8ac0603d --- /dev/null +++ b/grails-app/assets/javascripts/spApp/controller/addPointsCtrl.js @@ -0,0 +1,303 @@ +(function (angular) { + 'use strict'; + /** + * @memberof spApp + * @ngdoc controller + * @name AddAPointsCtrl + * @description + * Add points to the map using spatial-service's sandbox services + */ + angular.module('add-points-ctrl', ['map-service', 'layers-service', 'predefined-areas-service']) + .controller('AddPointsCtrl', ['LayoutService', '$scope', 'MapService', '$timeout', 'LayersService', + '$uibModalInstance', 'PredefinedAreasService', 'BiocacheService', 'LoggerService', 'data', + function (LayoutService, $scope, MapService, $timeout, LayersService, $uibModalInstance, PredefinedAreasService, BiocacheService, LoggerService, inputData) { + + $scope.inputData = inputData; + + $scope.step = 'default'; + $scope.method = 'existing'; + + $scope.errorMsg = ''; + + $scope.datasetName = 'my dataset name'; + $scope.message = ''; + $scope.status = 'queued'; + $scope.statusUrl = ''; + $scope.dataResourceUid = ''; + + $scope.maxFileSize = $SH.maxUploadSize; + + $scope.uploadingFile = false; + $scope.uploadProgress = 0; + + $scope.file = null; + + $scope.uploadsList = []; + $scope.searchUploads = ''; + $scope.sortType = 'date'; + $scope.sortReverse = true; + + LayoutService.addToSave($scope); + + $scope.init = function () { + // get a list of all prior uploads + if ($SH.userId) { + // find existing old sandbox uploads + if ($SH.sandboxServiceUrl && $SH.sandboxUrl) { + BiocacheService.userUploads($SH.userId, $SH.sandboxServiceUrl).then(function (data) { + + if (data.totalRecords === 0) { + return; + } + + // add bs and ws to each item + var items = data.facetResults[0].fieldResult; + items.forEach(function (item) { + item.bs = $SH.sandboxServiceUrl; + item.ws = $SH.sandboxUrl; + // get dataset_name and last_load_date + BiocacheService.searchForOccurrences({ + qid: item.fq, // skip qid registration for this one-off query + bs: item.bs, + ws: item.ws + }, [], 0, 0, 'dataset_name,last_processed_date').then(function (data) { + if (data.totalRecords > 0) { + // handle facets returning in a different order + var order = data.facetResults[0].fieldName === 'dataset_name' ? 0 : 1; + item.label = data.facetResults[order === 0 ? 0 : 1].fieldResult[0].label; + item.date = data.facetResults[order === 0 ? 1 : 0].fieldResult[0].label; + + // format the date so that it is sortable. It is currently a string, e.g. "2010-11-01T00:00:00Z" + item.date = new Date(item.date).toISOString().slice(0, 10); + + item.addedToMap = false; + + item.old = true; + + $scope.uploadsList.push(item); + } + }); + }); + }); + } + + // find existing spatial-service sandbox uploads + BiocacheService.userUploads($SH.userId, $SH.sandboxSpatialServiceUrl).then(function (data) { + if (data.totalRecords === 0) { + return; + } + + // add bs and ws to each item + var items = data.facetResults[0].fieldResult; + items.forEach(function (item) { + item.bs = $SH.sandboxSpatialServiceUrl; + item.ws = $SH.sandboxSpatialUiUrl; + // get dataset_name and last_load_date + BiocacheService.searchForOccurrences({ + qid: item.fq, // skip qid registration for this one-off query + bs: item.bs, + ws: item.ws + }, [], 0, 0, 'datasetName,lastProcessedDate').then(function (data) { + if (data.totalRecords > 0) { + // handle facets returning in a different order + var order = data.facetResults[0].fieldName === 'datasetName' ? 0 : 1; + item.label = data.facetResults[order === 0 ? 0 : 1].fieldResult[0].label; + item.date = data.facetResults[order === 0 ? 1 : 0].fieldResult[0].label; + + // format the date so that it is sortable. It is currently a string, e.g. "2010-11-01T00:00:00Z" + item.date = new Date(item.date).toISOString().slice(0, 10); + + item.addedToMap = false; + + item.old = false; + + $scope.uploadsList.push(item); + } + }); + }); + }); + } + } + + $scope.uploadFile = function (newFiles) { + + if (newFiles == null || newFiles.length == 0) { + return + } + + var file = newFiles[0] + + if (file.$error) { + if (file.$errorMessages.maxSize) { + bootbox.alert($i18n(476, "The uploaded file is too large. Max file size:") + " " + Math.floor($scope.maxFileSize / 1024 / 1024) + "MB"); + return + } + } + + $scope.file = file; + + // remove file extension and add date/time + var dateTime = new Date().toLocaleString(); + var newName = file.name.replace(/\.[^/.]+$/, "") + " " + dateTime; + $scope.datasetName = newName.substring(0, 200); // limit to 200 characters + }; + + $scope.checkStatus = function() { + LayersService.getSandboxUploadStatus($scope.statusUrl).then(function (data) { + $scope.status = data.status; + $scope.message = data.message; + + if ($scope.status === 'running') { + $timeout(function () { + $scope.checkStatus(); + }, 3000); // wait 3 seconds before checking status + } else if ($scope.status === 'finished') { + // successful + } + }, function (error) { + if (!error.handled) { + $scope.status = 'error'; + if (error.data.error) { + $scope.message = error.data.error; + } else { + $scope.message = "status code: " + error.status; + } + } + }); + } + + // currently removing the auto close step, so that the user can read the last status message + $scope.addToMapAndClose = function () { + var q = { + q: ['dataResourceUid:"' + $scope.dataResourceUid + '"'], + name: $scope.datasetName, + bs: $SH.sandboxSpatialServiceUrl, + ws: $SH.sandboxSpatialUiUrl + }; + + if (!$scope.logged) { + $scope.logged = true + + LoggerService.log("Create", "Points", {query: q, name: $scope.datasetName}) + } + + BiocacheService.newLayer(q, undefined, q.name).then(function (data) { + if (data != null) { + MapService.add(data); + } + $scope.$close(); + }); + }; + + $scope.ok = function () { + if ($scope.errorMsg || $scope.status === 'error') { + $scope.$close(); + } else if ($scope.status === 'finished') { + $scope.addToMapAndClose(); + } else { + $scope.step = 'uploading'; + $scope.uploadingFile = true; + + LayersService.uploadSandboxFile($scope.file, $scope.datasetName, $scope.file.name).then(function (response) { + if (response.data.error) { + $scope.status = 'error'; + $scope.uploadingFile = false; + $scope.errorMsg = response.data.error; + return + } else { + $scope.status = 'starting'; + $scope.statusUrl = response.data.statusUrl; + $scope.message = response.data.message; + $scope.dataResourceUid = response.data.dataResourceUid; + + $timeout(function () { + $scope.checkStatus(); + }, 3000); // wait 3 seconds before checking status + } + + $scope.uploadingFile = false; + }, function (error) { + $scope.errorMsg = "Unexpected error."; + if (!error.handled) { + if (error.status == 500) { + $scope.errorMsg = "Unexpected error: the uploaded file may be broken or unrecognised."; + } else { + if (error.data.error) { + $scope.errorMsg = error.data.error; + } else { + $scope.errorMsg = $i18n(540,"An error occurred. Please try again and if the same error occurs, send an email to support@ala.org.au and include the URL to this page, the error message and what steps you performed that triggered this error."); + } + } + } + $scope.uploadingFile = false; + + }, function (evt) { + $scope.uploadProgress = parseInt(100.0 * evt.loaded / evt.total); + }); + } + } + + $scope.isDisabled = function () { + if ($scope.step === 'uploading') { + return $scope.status !== 'finished' && $scope.status !== 'error' + } else { + return $scope.file == null || $scope.uploadingFile; + } + } + + $scope.isLoggedIn = $scope.isLoggedIn = $SH.userId !== undefined && $SH.userId !== null && $SH.userId.length > 0; + $scope.isNotLoggedIn = !$scope.isLoggedIn; + + $scope.addToMap = function (item) { + item.addedToMap = true; + + BiocacheService.registerLayer(item.bs, item.ws, [item.fq], undefined, undefined, true, true, item.label).then(function (data) { + if (data != null) { + MapService.add(data); + } + }); + } + + $scope.delete = function (item) { + bootbox.confirm("Are you sure you want to delete \"" + item.label + "\?", function (result) { + if (result) { + // extract id from item.fq with the content `data_resource_uid:"39632cdd-4e1f-41d8-922a-c09a68270b2d"` + var dataResourceUid = item.fq.split('"')[1]; + + LayersService.deleteSandboxUpload(dataResourceUid).then(function (data) { + if (data != null) { + $scope.uploadsList = $scope.uploadsList.filter(function (i) { + return i.fq !== item.fq; + }); + } + }); + } + }); + } + + $scope.download = function (item) { + var url = item.ws + "/occurrences/search?q=" + item.fq; + var a = document.createElement('a'); + a.target = '_blank'; + a.href = url; + a.click(); + } + + $scope.searchFilter = function(item) { + if (!$scope.searchUploads) { + return true; + } + var searchText = $scope.searchUploads.toLowerCase(); + return item.label.toLowerCase().includes(searchText); + }; + + $scope.showFinished = function() { + return st + } + + $scope.init(); + + + }]) + +}(angular)); diff --git a/grails-app/assets/javascripts/spApp/service/biocacheService.js b/grails-app/assets/javascripts/spApp/service/biocacheService.js index 17452443..30bcd3d3 100644 --- a/grails-app/assets/javascripts/spApp/service/biocacheService.js +++ b/grails-app/assets/javascripts/spApp/service/biocacheService.js @@ -388,7 +388,7 @@ * @param {List} fqs (Optional) additional fq terms * @param {Integer} pageSize (Optional) page size (default=1) * @param {Integer} offset (Optional) offset (default=0) - * @param {Boolean} facet (Optional) include server default facets (default=false) + * @param {String} facets (Optional) comma delimited facets (flimit is -1) * @returns {Promise(Map)} search results * * @example @@ -418,8 +418,8 @@ * "activeFacetMap": {} * } */ - searchForOccurrences: function (query, fqs, pageSize, offset, facet) { - facet = facet || false; + searchForOccurrences: function (query, fqs, pageSize, offset, facets) { + facets = facets || ""; pageSize = pageSize === undefined ? 1 : pageSize; offset = offset || 0; var fqList = (fqs === undefined ? '' : '&fq=' + this.joinAndEncode(fqs)); @@ -427,13 +427,27 @@ if (response == null) { return {} } - return $http.get(query.bs + "/occurrences/search?facet=" + facet + "&pageSize=" + pageSize + "&startIndex=" + offset + "&q=" + response.qid + fqList + '&sort=id', _httpDescription('searchForOccurrences')).then(function (response) { + return $http.get(query.bs + "/occurrences/search?facets=" + facets + "&pageSize=" + pageSize + "&startIndex=" + offset + "&q=" + response.qid + fqList + '&sort=id', _httpDescription('searchForOccurrences')).then(function (response) { if (response.data !== undefined) { return response.data; } }) }) }, + /** + * Facet search for dataResourceUid for a given userId. + * + * @param userId user's id used to upload data resources + * @param sandboxServiceUrl biocache-service URL for the sandbox + * @returns {*} + */ + userUploads: function (userId, sandboxServiceUrl) { + return $http.get(sandboxServiceUrl + "/occurrences/search?facets=data_resource_uid&flimit=-1&pageSize=0&q=user_id:" + userId, _httpDescription('userUploads')).then(function (response) { + if (response.data !== undefined) { + return response.data; + } + }) + }, /** * Get facet list * @memberof BiocacheService diff --git a/grails-app/assets/javascripts/spApp/service/layersService.js b/grails-app/assets/javascripts/spApp/service/layersService.js index 9961e79d..e4154194 100644 --- a/grails-app/assets/javascripts/spApp/service/layersService.js +++ b/grails-app/assets/javascripts/spApp/service/layersService.js @@ -556,7 +556,59 @@ } } return result; + }, + /** + * Upload an sandbox file. zip (csv) or csv + * @memberof LayersService + * @param {File} file + * @param {string} datasetName name of the dataset + * @returns {Promise(Map)} in progress uploaded csv info + */ + uploadSandboxFile: function (file, datasetName) { + var uploadURL = $SH.baseUrl + "/sandbox?name=" + encodeURI(datasetName); + + file.upload = Upload.upload({ + url: uploadURL, + data: {file: file} + }); + + return file.upload; + }, + /** + * Get sandbox list + * @memberof LayersService + * @param {string} userId + * @returns {Promise(List)} list of sandbox uploads + */ + listSandbox: function (userId) { + var urlProxy = $SH.sandboxBiocacheServiceUrl + "/occurrences/search?q=userId:" + userId + "&pageSize=0&facets=raw_datasetName"; + return $http.get(urlProxy, _httpDescription('listSandbox', {})).then(function (response) { + return response.data; + }); + }, + /** + * Get sandbox upload status + * @memberof LayersService + * @param {string} statusUrl + * @returns {Promise(Map)} status of the upload + */ + getSandboxUploadStatus: function (statusUrl) { + return $http.get(statusUrl, _httpDescription('getUploadStatus')).then(function (response) { + return response.data; + }); + }, + /** + * Delete a sandbox upload, by dataResourceUid. Only applies to the spatial-service sandbox. + * @memberof LayersService + * @param {string} dataResourceUid + * @returns {Promise(Map)} status of the delete + */ + deleteSandboxUpload: function (dataResourceUid) { + return $http.delete($SH.baseUrl + "/sandbox?id=" + dataResourceUid, _httpDescription('deleteSandboxUpload')).then(function (response) { + return response.data; + }); } + } return thiz; diff --git a/grails-app/assets/javascripts/spApp/templates/addPointsContent.tpl.htm b/grails-app/assets/javascripts/spApp/templates/addPointsContent.tpl.htm new file mode 100644 index 00000000..8828aa8d --- /dev/null +++ b/grails-app/assets/javascripts/spApp/templates/addPointsContent.tpl.htm @@ -0,0 +1,165 @@ +
+ +

Import Points

+
+ +
+
+
+
+
+
+
+

Select method

+
+ +
+
+
+
+
+
+
+
+
+ +
+
+
+

Your uploads

+
+ +
+
+ + + + + + + + + + + + + + + + + + +
+
+
+
+ +
+
+
+
+
+ + Name + + + + Date + + + + Records + +
+ {{ item.label }}{{ item.date }}{{ item.count | number }} + + View Records + + +
+
+
+
+
+ +
+
+
+

Upload a CSV file with occurrences.

+
+ +
+
+ Select a CSV or zipped CSV file. Please note that for the records in a data set to be effectively discoverable and able to be mapped it needs to meet a minimum set of standards. Data needs to include, at a minimum: scientificName/vernacularName to identify the organism and eventDate, decimalLatitude and decimalLongitude to indicate where and when the occurrence happened. The data needs to have a unique identifier column - either catalogNumber or occurrenceID with a unique value for each record. The number of columns must match the number of column headers (problems with this often indicate commas and/or line breaks in the fields). Additional fields will increase the usability of the data. +
+
+
+
+ +
+ + + +
+
+
+
+
+ +
+
+
+

Update the dataset name

+
+ +
+
+ Dataset name + +
+ +
+
+
+
+ +
+
+
+

Importing

+
+ +
+
{{status}} : {{message}}
+ +
{{ errorMsg }}
+
+
+
+
+
+
+
+
+ +
+ + + +
diff --git a/grails-app/assets/javascripts/spApp/templates/sandBoxContent.tpl.htm b/grails-app/assets/javascripts/spApp/templates/sandBoxContent.tpl.htm index 4c7c2e7e..f85b6cdf 100644 --- a/grails-app/assets/javascripts/spApp/templates/sandBoxContent.tpl.htm +++ b/grails-app/assets/javascripts/spApp/templates/sandBoxContent.tpl.htm @@ -348,4 +348,4 @@

Next steps:

-
\ No newline at end of file + diff --git a/grails-app/conf/application.yml b/grails-app/conf/application.yml index e68b7fab..bb479855 100644 --- a/grails-app/conf/application.yml +++ b/grails-app/conf/application.yml @@ -225,6 +225,14 @@ sandbox: sandboxService: url: 'https://sandbox.ala.org.au/biocache-service' +# Config to use spatial-service/pipelines instead of sandbox/biocache-store for uploading points. +# - Read access to sandbox/biocache-store data is enabled when sandbox.url and sandboxService.url are set. +# - Update menu-config.json to use `open: addPoints` instead of `open: sandBox` +sandboxSpatial: + hubUrl: 'https://sandbox-test.ala.org.au/ala-hub' + serviceUrl: 'https://sandbox-test.ala.org.au/biocache-service' + + gazField: 'cl915' userObjectsField: 'cl1082' diff --git a/grails-app/conf/logback.xml b/grails-app/conf/logback.xml index 721e225c..5362b547 100644 --- a/grails-app/conf/logback.xml +++ b/grails-app/conf/logback.xml @@ -8,7 +8,7 @@ UTF-8 - '[SPATIAL-HUB] %clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(%5p) %clr(--){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m%n%wex' + %clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(%5p) %clr(%logger{39} [%file:%line]){cyan} %clr(:){faint} %m%n%wex diff --git a/grails-app/conf/menu-config.json b/grails-app/conf/menu-config.json index bed4aa93..b5d53f70 100644 --- a/grails-app/conf/menu-config.json +++ b/grails-app/conf/menu-config.json @@ -114,12 +114,7 @@ "items": [ { "name": "Points", - "open": "sandBox", - "params": { - "display": { - "size": "full" - } - } + "open": "addPoints" }, { "name": "Species List", diff --git a/grails-app/controllers/au/org/ala/spatial/portal/PortalController.groovy b/grails-app/controllers/au/org/ala/spatial/portal/PortalController.groovy index 80308642..6b57414c 100644 --- a/grails-app/controllers/au/org/ala/spatial/portal/PortalController.groovy +++ b/grails-app/controllers/au/org/ala/spatial/portal/PortalController.groovy @@ -352,16 +352,20 @@ class PortalController { } else { def json = request.JSON as Map - Map headers = [apiKey: grailsApplication.config.api_key] + Map params = [sessionId: params.sessionId] + for (def key : json.keySet()) { + if (key != 'sessionId') { + params.put(key, String.valueOf(json[key])) + } + } - def r = hubWebService.postUrl("${grailsApplication.config.layersService.url}/tasks/create?" + - "userId=${userId}&sessionId=${params.sessionId}", json, headers) + def r = webService.post("${grailsApplication.config.layersService.url}/tasks/create", null, params, ContentType.APPLICATION_JSON, false, true) if (r == null) { render [:] as JSON } else { response.status = r.statusCode - render JSON.parse(new String(r?.text ?: "{}")) as JSON + render r.resp as JSON } } } @@ -745,4 +749,46 @@ class PortalController { return result } + + def postSandboxFile() { + def userId = getValidUserId(params) + + if (!userId) { + notAuthorised() + } else { + MultipartFile mFile = ((MultipartHttpServletRequest) request).getFile('file') + + // write mFile to temporary file with the same extension + File tempFile = File.createTempFile("sandbox", mFile.originalFilename.substring(mFile.originalFilename.lastIndexOf('.'))) + mFile.transferTo(tempFile) + + String ce = grailsApplication.config.character.encoding + + String url = "${grailsApplication.config.layersService.url}/sandbox/upload?name=${URLEncoder.encode((String) params.name, ce)}" + def r = webService.postMultipart(url, null, null, [tempFile], ContentType.APPLICATION_JSON, false, true) + + if (!r) { + render [:] as JSON + } else { + render r.resp as JSON, status: String.valueOf(r.statusCode) + } + } + } + + def deleteSandboxFile(String id) { + def userId = getValidUserId(params) + + if (!userId) { + notAuthorised() + } else { + def url = "${grailsApplication.config.layersService.url}/sandbox/delete?id=${id}" + def r = webService.delete(url, null, ContentType.APPLICATION_JSON, false, true) + + if (!r) { + render [:] as JSON + } else { + render r.resp as JSON, status: String.valueOf(r.statusCode) + } + } + } } diff --git a/grails-app/controllers/au/org/ala/spatial/portal/UrlMappings.groovy b/grails-app/controllers/au/org/ala/spatial/portal/UrlMappings.groovy index 5a89b61c..e12a99fd 100644 --- a/grails-app/controllers/au/org/ala/spatial/portal/UrlMappings.groovy +++ b/grails-app/controllers/au/org/ala/spatial/portal/UrlMappings.groovy @@ -8,6 +8,9 @@ class UrlMappings { "/hub/$hub"(controller: "portal", action: "index") + "/sandbox"(controller: "portal", action: "postSandboxFile", method: "POST") + "/sandbox"(controller: "portal", action: "deleteSandboxFile", method: "DELETE") + "/$controller/$action?/$id?(.$format)?" { constraints { // apply constraints here diff --git a/grails-app/i18n/messages.properties b/grails-app/i18n/messages.properties index 16d40b62..179fde9f 100644 --- a/grails-app/i18n/messages.properties +++ b/grails-app/i18n/messages.properties @@ -584,3 +584,17 @@ Area\ Report\ -\ interactive=Area Report - interactive 544=Select mapped species. 545=Annotate your workflow 546=Export points. +550=CSV file with occurrences. +551=Select a CSV or zipped CSV file. Please note that for the records in a data set to be effectively discoverable and able to be mapped it needs to meet a minimum set of standards. Data needs to include, at a minimum: scientificName/vernacularName to identify the organism and eventDate, decimalLatitude and decimalLongitude to indicate where and when the occurrence happened. The data needs to have a unique identifier column - either catalogNumber or occurrenceID with a unique value for each record. The number of columns must match the number of column headers (problems with this often indicate commas and/or line breaks in the fields). Additional fields will increase the usability of the data. +552=Dataset name +553=Prior uploads +554=List of prior uploads +555=Update the Dataset name +556=Importing +557=Records +558=Select method +559=Previous uploads +560=Import Points +561=Your uploads +562=Update the dataset name +563=View Records diff --git a/grails-app/views/portal/index.gsp b/grails-app/views/portal/index.gsp index 446c1d36..da5be4b2 100644 --- a/grails-app/views/portal/index.gsp +++ b/grails-app/views/portal/index.gsp @@ -46,6 +46,8 @@ sandboxUiUrl: '${config.sandbox.uiUrl}', sandboxUrls: ['${config.sandbox.url}'], sandboxServiceUrls: ['${config.sandboxService.url}'], + sandboxSpatialUiUrl: '${config.sandboxSpatial.hubUrl}', + sandboxSpatialServiceUrl: '${config.sandboxSpatial.serviceUrl}', gazField: '${config.gazField}', geoserverUrl: '${config.geoserver.url}', collectionsUrl: '${config.collections.url}', From 07113bc9447eeba4c9e66eba5bf1da564efe6092 Mon Sep 17 00:00:00 2001 From: Adam Collins Date: Mon, 20 Jan 2025 14:14:03 +1000 Subject: [PATCH 2/3] version bump --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 07b2cbaf..b8ff40e8 100644 --- a/build.gradle +++ b/build.gradle @@ -12,7 +12,7 @@ buildscript { } } -version "2.1.2-SNAPSHOT" +version "2.2.0-SNAPSHOT" group "au.org.ala" apply plugin:"eclipse" From 97800a9e1b70c4475e599c8a12061c32843ae85e Mon Sep 17 00:00:00 2001 From: Adam Collins Date: Wed, 29 Jan 2025 15:00:51 +1000 Subject: [PATCH 3/3] addPoints UI changes --- _Events.groovy | 2 +- grails-app/assets/javascripts/application.js | 1 + .../spApp/controller/addPointsCtrl.js | 4 +++- .../spApp/templates/addPointsContent.tpl.htm | 10 ++++------ .../assets/stylesheets/spatial-hub-generic.css | 2 +- grails-app/assets/stylesheets/spatial-hub.css | 8 ++++++++ grails-app/i18n/messages.properties | 17 +++++++++++++++-- package-lock.json | 13 +++++++++++++ package.json | 1 + 9 files changed, 47 insertions(+), 11 deletions(-) diff --git a/_Events.groovy b/_Events.groovy index 1171a73a..fdd9cf30 100644 --- a/_Events.groovy +++ b/_Events.groovy @@ -28,7 +28,7 @@ def build(String baseDir) { 'angular-ui-bootstrap/dist/ui-bootstrap-csp.css', 'bootbox/dist/bootbox.min.js', 'jquery/dist/jquery.min.js', 'ng-file-upload/dist/ng-file-upload.js', 'ngbootbox/dist/ngBootbox.min.js', 'bootstrap/dist/', 'leaflet/dist/', 'leaflet-draw/dist/', 'leaflet-measure/dist/', 'proj4/dist/proj4.js', - 'proj4leaflet/src/proj4leaflet.js', 'lz-string/libs/lz-string.min.js'] + 'proj4leaflet/src/proj4leaflet.js', 'lz-string/libs/lz-string.min.js', 'angular-sanitize/angular-sanitize.min.js'] files.each { name -> def dst = new File(baseDir + '/grails-app/assets/node_modules/' + name) dst.getParentFile().mkdirs() diff --git a/grails-app/assets/javascripts/application.js b/grails-app/assets/javascripts/application.js index 6f061e36..66f24b31 100644 --- a/grails-app/assets/javascripts/application.js +++ b/grails-app/assets/javascripts/application.js @@ -33,6 +33,7 @@ //= require angular-slider/slider.js //= require angular-sortable/sortable.js //= require ng-file-upload/dist/ng-file-upload.min.js +//= require angular-sanitize/angular-sanitize.min.js //bootbox //= require ngbootbox/dist/ngBootbox.min.js diff --git a/grails-app/assets/javascripts/spApp/controller/addPointsCtrl.js b/grails-app/assets/javascripts/spApp/controller/addPointsCtrl.js index 8ac0603d..d83e2fe7 100644 --- a/grails-app/assets/javascripts/spApp/controller/addPointsCtrl.js +++ b/grails-app/assets/javascripts/spApp/controller/addPointsCtrl.js @@ -7,7 +7,7 @@ * @description * Add points to the map using spatial-service's sandbox services */ - angular.module('add-points-ctrl', ['map-service', 'layers-service', 'predefined-areas-service']) + angular.module('add-points-ctrl', ['map-service', 'layers-service', 'predefined-areas-service', 'ngSanitize']) .controller('AddPointsCtrl', ['LayoutService', '$scope', 'MapService', '$timeout', 'LayersService', '$uibModalInstance', 'PredefinedAreasService', 'BiocacheService', 'LoggerService', 'data', function (LayoutService, $scope, MapService, $timeout, LayersService, $uibModalInstance, PredefinedAreasService, BiocacheService, LoggerService, inputData) { @@ -37,6 +37,8 @@ $scope.sortType = 'date'; $scope.sortReverse = true; + $scope.instructions = $i18n(551, "Select a CSV or zipped CSV file."); + LayoutService.addToSave($scope); $scope.init = function () { diff --git a/grails-app/assets/javascripts/spApp/templates/addPointsContent.tpl.htm b/grails-app/assets/javascripts/spApp/templates/addPointsContent.tpl.htm index 8828aa8d..acc811ad 100644 --- a/grails-app/assets/javascripts/spApp/templates/addPointsContent.tpl.htm +++ b/grails-app/assets/javascripts/spApp/templates/addPointsContent.tpl.htm @@ -20,7 +20,7 @@

Select method

ng-change="method='upload'">Upload CSV
+ ng-change="method='existing'; file=null">Previous upload
@@ -95,13 +95,11 @@

Your uploads

-

Upload a CSV file with occurrences.

+

Upload a CSV file with occurrences

-
- Select a CSV or zipped CSV file. Please note that for the records in a data set to be effectively discoverable and able to be mapped it needs to meet a minimum set of standards. Data needs to include, at a minimum: scientificName/vernacularName to identify the organism and eventDate, decimalLatitude and decimalLongitude to indicate where and when the occurrence happened. The data needs to have a unique identifier column - either catalogNumber or occurrenceID with a unique value for each record. The number of columns must match the number of column headers (problems with this often indicate commas and/or line breaks in the fields). Additional fields will increase the usability of the data. -
+

@@ -144,7 +142,7 @@

Importing

-
{{status}} : {{message}}
+
{{message}}
{{ errorMsg }}

diff --git a/grails-app/assets/stylesheets/spatial-hub-generic.css b/grails-app/assets/stylesheets/spatial-hub-generic.css index a3a8770e..4549f29e 100644 --- a/grails-app/assets/stylesheets/spatial-hub-generic.css +++ b/grails-app/assets/stylesheets/spatial-hub-generic.css @@ -3331,4 +3331,4 @@ img.map-icon { ul.errors { padding-top: 150px -} \ No newline at end of file +} diff --git a/grails-app/assets/stylesheets/spatial-hub.css b/grails-app/assets/stylesheets/spatial-hub.css index 57087f9c..05c53f44 100644 --- a/grails-app/assets/stylesheets/spatial-hub.css +++ b/grails-app/assets/stylesheets/spatial-hub.css @@ -560,6 +560,14 @@ form.banner { left: 50%; } +.inline-spinner { + background: url(../images/spinner.gif); + background-repeat: no-repeat; + background-size: 16px 16px; + height: 16px; + width: 16px; +} + #legend { margin-right:0px; } diff --git a/grails-app/i18n/messages.properties b/grails-app/i18n/messages.properties index 179fde9f..110d64d4 100644 --- a/grails-app/i18n/messages.properties +++ b/grails-app/i18n/messages.properties @@ -584,8 +584,21 @@ Area\ Report\ -\ interactive=Area Report - interactive 544=Select mapped species. 545=Annotate your workflow 546=Export points. -550=CSV file with occurrences. -551=Select a CSV or zipped CSV file. Please note that for the records in a data set to be effectively discoverable and able to be mapped it needs to meet a minimum set of standards. Data needs to include, at a minimum: scientificName/vernacularName to identify the organism and eventDate, decimalLatitude and decimalLongitude to indicate where and when the occurrence happened. The data needs to have a unique identifier column - either catalogNumber or occurrenceID with a unique value for each record. The number of columns must match the number of column headers (problems with this often indicate commas and/or line breaks in the fields). Additional fields will increase the usability of the data. +550=CSV file with occurrences +551=

Select a CSV or zipped CSV file. Please note that for the records in a data set to be effectively discoverable and able to be mapped it needs to meet a minimum set of standards.

\ +

Data needs to include, at a minimum:

\ +
    \ +
  • scientificName/vernacularName to identify the organism
    • \ +
  • eventDate to indicate when the occurrence happened
    • \ +
  • decimalLatitude and decimalLongitude to indicate where the occurrence happened
    • \ +
  • A unique identifier column - either catalogNumber or occurrenceID, with a unique value for each record
    • \ +
\ +\ +

Also note:

\ +
    \ +
  • The number of columns must match the number of column headers (problems with this often indicate commas and/or line breaks in the fields).
    • \ +
  • Additional fields will increase the usability of the data.
    • \ +
552=Dataset name 553=Prior uploads 554=List of prior uploads diff --git a/package-lock.json b/package-lock.json index 16b862ad..4b940c51 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,7 @@ "angular-aria": "1.5.8", "angular-leaflet-directive": "0.9.5", "angular-route": "1.5.8", + "angular-sanitize": "1.5.8", "angular-touch": "1.5.8", "angular-ui-bootstrap": "^1.3.3", "bootbox": "^5.0.1", @@ -185,6 +186,13 @@ "resolved": "https://registry.npmjs.org/angular-route/-/angular-route-1.5.8.tgz", "integrity": "sha1-lkCT3n7I3FeZvVapOmzl9gCVZ0s=" }, + "node_modules/angular-sanitize": { + "version": "1.5.8", + "resolved": "https://registry.npmjs.org/angular-sanitize/-/angular-sanitize-1.5.8.tgz", + "integrity": "sha512-SCr0ekZLacVt5RzgufvfYQRYr34pJSL+lVcbBBtvXyMXNC2g9RcTxA+Uwgaq4AoxnxGiZb2WYrXMtL9nYGAo+Q==", + "deprecated": "For the actively supported Angular, see https://www.npmjs.com/package/@angular/core. AngularJS support has officially ended. For extended AngularJS support options, see https://goo.gle/angularjs-path-forward.", + "license": "MIT" + }, "node_modules/angular-template": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/angular-template/-/angular-template-2.1.4.tgz", @@ -1946,6 +1954,11 @@ "resolved": "https://registry.npmjs.org/angular-route/-/angular-route-1.5.8.tgz", "integrity": "sha1-lkCT3n7I3FeZvVapOmzl9gCVZ0s=" }, + "angular-sanitize": { + "version": "1.5.8", + "resolved": "https://registry.npmjs.org/angular-sanitize/-/angular-sanitize-1.5.8.tgz", + "integrity": "sha512-SCr0ekZLacVt5RzgufvfYQRYr34pJSL+lVcbBBtvXyMXNC2g9RcTxA+Uwgaq4AoxnxGiZb2WYrXMtL9nYGAo+Q==" + }, "angular-template": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/angular-template/-/angular-template-2.1.4.tgz", diff --git a/package.json b/package.json index 5b6a364c..785f7ba6 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,7 @@ "angular-route": "1.5.8", "angular-touch": "1.5.8", "angular-ui-bootstrap": "^1.3.3", + "angular-sanitize": "1.5.8", "bootbox": "^5.0.1", "bootstrap": "3.4.1", "chromedriver": "89.0.0",