From a8459f7dd81a15b77660894ed3029e7afb2bd6cf Mon Sep 17 00:00:00 2001 From: Adam Collins Date: Tue, 21 Jan 2025 16:04:49 +1000 Subject: [PATCH 1/2] #158 change remote layer copy to use JWT --- grails-app/conf/application.yml | 3 ++ .../ala/spatial/ManageLayersController.groovy | 46 +++++++++++++++---- .../au/org/ala/spatial/FieldService.groovy | 3 +- .../au/org/ala/spatial/LayerService.groovy | 3 +- .../ala/spatial/ManageLayersService.groovy | 4 +- .../org/ala/spatial/TaskQueueService.groovy | 7 +++ grails-app/views/manageLayers/remote.gsp | 37 +++++++++++++-- .../au/org/ala/spatial/SpatialConfig.groovy | 2 + .../groovy/au/org/ala/spatial/Util.groovy | 6 ++- .../org/ala/spatial/process/LayerCopy.groovy | 13 ++---- .../ala/spatial/process/SlaveProcess.groovy | 32 +++++++++---- .../spatial/process/StandardizeLayers.groovy | 1 - src/main/resources/processes/LayerCopy.json | 3 ++ 13 files changed, 124 insertions(+), 36 deletions(-) diff --git a/grails-app/conf/application.yml b/grails-app/conf/application.yml index f8c49d6..e49badc 100644 --- a/grails-app/conf/application.yml +++ b/grails-app/conf/application.yml @@ -474,3 +474,6 @@ sandboxSolrCollection: sandbox sandboxThreadCount: 2 pipelinesCmd: "java -Dspark.local.dir=/data/spatial-data/sandbox/tmp -Djava.io.tmpdir=/data/spatial-data/sandbox/tmp -Dlog4j.configuration=file:/data/spatial-data/modelling/la-pipelines/log4j.properties -cp /data/spatial-data/modelling/la-pipelines/pipelines-2.19.0-SNAPSHOT-shaded.jar" pipelinesConfig: "--config=/data/spatial-data/modelling/la-pipelines/la-pipelines.yaml" + +# For the copying of layers between spatial-service instances, a JWT with the following role must be provided in the `/manageLayers/remote` page. +layerCopyRole: ROLE_ADMIN diff --git a/grails-app/controllers/au/org/ala/spatial/ManageLayersController.groovy b/grails-app/controllers/au/org/ala/spatial/ManageLayersController.groovy index 9a18f0a..950172a 100644 --- a/grails-app/controllers/au/org/ala/spatial/ManageLayersController.groovy +++ b/grails-app/controllers/au/org/ala/spatial/ManageLayersController.groovy @@ -15,14 +15,15 @@ package au.org.ala.spatial +import au.ala.org.ws.security.RequireApiKey +import au.org.ala.web.AuthService import grails.converters.JSON import grails.converters.XML import org.apache.commons.io.FileUtils -import org.apache.commons.io.IOUtils import org.hibernate.criterion.CriteriaSpecification import org.json.simple.JSONObject -import org.json.simple.parser.JSONParser +import javax.servlet.http.HttpServletResponse import javax.transaction.Transactional import java.text.SimpleDateFormat @@ -37,6 +38,7 @@ class ManageLayersController { FieldService fieldService LayerService layerService + AuthService authService /** * admin only or api_key @@ -614,8 +616,9 @@ class ManageLayersController { def copy() { String spatialServiceUrl = params.spatialServiceUrl String fieldId = params.fieldId + String jwt = params.jwt - manageLayersService.updateFromRemote(spatialServiceUrl, fieldId) + manageLayersService.updateFromRemote(spatialServiceUrl, fieldId, jwt) redirect(controller: "Tasks", action: "index") } @@ -628,11 +631,11 @@ class ManageLayersController { @RequireAdmin def enable() { if (params.id.isNumber()) { - def layer = layerDao.getLayerById(params.id.toInteger(), false) + def layer = layerService.getLayerById(params.id.toInteger(), false) layer.enabled = true layerService.updateLayer(layer) } else { - def field = fieldDao.getFieldById(params.id, false) + def field = fieldService.getFieldById(params.id, false) field.enabled = true fieldService.updateField(field) } @@ -645,11 +648,22 @@ class ManageLayersController { * a layer: 'cl..._res', 'el..._res', will provide the standardized files at the requested resolution * (or next detailed) - shape files or diva grids * - * admin only or api_key, do not redirect to CAS + * Requires JWT with the configured layerCopyRole + * * @return */ - @RequireAdmin + @RequireApiKey def resource() { + // do permission check + if (!authService.userInRole(spatialConfig.layerCopyRole)) { + response.sendError(HttpServletResponse.SC_FORBIDDEN, "Forbidden, required role: " + spatialConfig.layerCopyRole) + return + } + + if (!isLayerCopyFile(params.resource)) { + response.sendError(HttpServletResponse.SC_FORBIDDEN, "Forbidden, only allowed access to layer files") + } + OutputStream outputStream = null try { outputStream = response.outputStream as OutputStream @@ -674,13 +688,27 @@ class ManageLayersController { /** * for slaves to peek at a resource on the master * - * admin only or api_key, do not redirect to CAS + * Requires JWT with the configured layerCopyRole * * @return */ - @RequireAdmin + @RequireApiKey def resourcePeek() { + // do permission check + if (!authService.userInRole(spatialConfig.layerCopyRole)) { + response.sendError(HttpServletResponse.SC_FORBIDDEN, "Forbidden, required role: " + spatialConfig.layerCopyRole) + return + } + + if (!isLayerCopyFile(params.resource)) { + response.sendError(HttpServletResponse.SC_FORBIDDEN, "Forbidden, only allowed access to layer files") + } + //write resource render fileService.info(params.resource.toString()) as JSON } + + private boolean isLayerCopyFile(String resource) { + return resource.startsWith("/layer/") || resource.startsWith("/standard_layer/") || resource.startsWith("/public/layerDistances.properties") + } } diff --git a/grails-app/services/au/org/ala/spatial/FieldService.groovy b/grails-app/services/au/org/ala/spatial/FieldService.groovy index 1f7d836..dd45feb 100644 --- a/grails-app/services/au/org/ala/spatial/FieldService.groovy +++ b/grails-app/services/au/org/ala/spatial/FieldService.groovy @@ -171,7 +171,8 @@ class FieldService { Map map = field.properties map.put('id', field.id) - Fields.executeUpdate(sql, map) + + Sql.newInstance(dataSource).executeUpdate(sql, map) } List getLayersByCriteria(String keywords) { diff --git a/grails-app/services/au/org/ala/spatial/LayerService.groovy b/grails-app/services/au/org/ala/spatial/LayerService.groovy index 9eaa93b..6f2ce54 100644 --- a/grails-app/services/au/org/ala/spatial/LayerService.groovy +++ b/grails-app/services/au/org/ala/spatial/LayerService.groovy @@ -65,7 +65,8 @@ class LayerService { void updateLayer(Layers layer) { log.debug("Updating layer metadata for " + layer.getName()) String sql = "update layers set citation_date=:citation_date, classification1=:classification1, classification2=:classification2, datalang=:datalang, description=:description, displayname=:displayname, displaypath=:displaypath, enabled=:enabled, domain=:domain, environmentalvaluemax=:environmentalvaluemax, environmentalvaluemin=:environmentalvaluemin, environmentalvalueunits=:environmentalvalueunits, extents=:extents, keywords=:keywords, licence_link=:licence_link, licence_notes=:licence_notes, licence_level=:licence_level, lookuptablepath=:lookuptablepath, maxlatitude=:maxlatitude, maxlongitude=:maxlongitude, mddatest=:mddatest, mdhrlv=:mdhrlv, metadatapath=:metadatapath, minlatitude=:minlatitude, minlongitude=:minlongitude, name=:name, notes=:notes, path=:path, path_1km=:path_1km, path_250m=:path_250m, path_orig=:path_orig, pid=:pid, respparty_role=:respparty_role, scale=:scale, source=:source, source_link=:source_link, type=:type, uid=:uid where id=:id" - Layers.executeUpdate(sql, layer) + + Sql.newInstance(dataSource).executeUpdate(sql, layer) } diff --git a/grails-app/services/au/org/ala/spatial/ManageLayersService.groovy b/grails-app/services/au/org/ala/spatial/ManageLayersService.groovy index 3f97ae7..8e61614 100644 --- a/grails-app/services/au/org/ala/spatial/ManageLayersService.groovy +++ b/grails-app/services/au/org/ala/spatial/ManageLayersService.groovy @@ -1389,7 +1389,7 @@ class ManageLayersService { * @param fieldId * @return */ - def updateFromRemote(String spatialServiceUrl, String fieldId) { + def updateFromRemote(String spatialServiceUrl, String fieldId, String jwt) { def f = JSON.parse(httpCall("GET", spatialServiceUrl + "/field/${fieldId}?pageSize=0", null, null, @@ -1429,7 +1429,7 @@ class ManageLayersService { createOrUpdateField(field, field.id + '', false) } - Map input = [layerId: layer.requestedId, fieldId: field.id, sourceUrl: spatialServiceUrl, displayPath: origDisplayPath] as Map + Map input = [layerId: layer.requestedId, fieldId: field.id, sourceUrl: spatialServiceUrl, displayPath: origDisplayPath, jwt: jwt] as Map Task task = tasksService.create("LayerCopy", UUID.randomUUID(), input).task task diff --git a/grails-app/services/au/org/ala/spatial/TaskQueueService.groovy b/grails-app/services/au/org/ala/spatial/TaskQueueService.groovy index 1deb0c4..893e685 100644 --- a/grails-app/services/au/org/ala/spatial/TaskQueueService.groovy +++ b/grails-app/services/au/org/ala/spatial/TaskQueueService.groovy @@ -193,6 +193,13 @@ class TaskQueueService { taskWrapper.task.message = 'finished' taskWrapper.task.history.put(System.currentTimeMillis() as String, "finished (id:${taskWrapper.task.id})" as String) + // map output.task to task when null to prevent error when flushing task + taskWrapper.task.output.each { + if (!it.task) { + it.task = taskWrapper.task + } + } + // flush task because it is finished Task.withTransaction { if (!taskWrapper.task.save(flush: true)) { diff --git a/grails-app/views/manageLayers/remote.gsp b/grails-app/views/manageLayers/remote.gsp index 4b90d94..5e8046a 100644 --- a/grails-app/views/manageLayers/remote.gsp +++ b/grails-app/views/manageLayers/remote.gsp @@ -18,6 +18,8 @@

Copy Layers from Remote Server

+ +
The local and remote server is the same so layers cannot be copied
@@ -44,12 +46,27 @@

-
This will copy a layer from a remote spatial-service to the local spatial-service. - +
+

Config

+ + + + + + + + + + + +
URL of the remote spatial-service
JWT for the remote service. It must have the role ${spatialConfig.layerCopyRole} and not expire before the task is finished.
+
+


+

List of layers

Layer filter