158 remote layer copy

build.gradle

@@ -28,7 +28,7 @@ ext {
     drivers = ["firefox", "chrome", "chromeHeadless"]
 }
 
-version "2.1.6-SNAPSHOT"
+version "2.2.0-SNAPSHOT"
 group "au.org.ala"
 
 apply plugin: "eclipse"

grails-app/conf/application.yml

@@ -474,3 +474,6 @@ sandboxSolrCollection: sandbox
 sandboxThreadCount: 2
 pipelinesCmd: "java -Dspark.local.dir=/data/spatial-data/sandbox/tmp -Djava.io.tmpdir=/data/spatial-data/sandbox/tmp -Dlog4j.configuration=file:/data/spatial-data/modelling/la-pipelines/log4j.properties -cp /data/spatial-data/modelling/la-pipelines/pipelines-2.19.0-SNAPSHOT-shaded.jar"
 pipelinesConfig: "--config=/data/spatial-data/modelling/la-pipelines/la-pipelines.yaml"
+
+# For the copying of layers between spatial-service instances, a JWT with the following role must be provided in the `/manageLayers/remote` page.
+layerCopyRole: ROLE_ADMIN

grails-app/controllers/au/org/ala/spatial/ManageLayersController.groovy

@@ -15,14 +15,15 @@
 
 package au.org.ala.spatial
 
+import au.ala.org.ws.security.RequireApiKey
+import au.org.ala.web.AuthService
 import grails.converters.JSON
 import grails.converters.XML
 import org.apache.commons.io.FileUtils
-import org.apache.commons.io.IOUtils
 import org.hibernate.criterion.CriteriaSpecification
 import org.json.simple.JSONObject
-import org.json.simple.parser.JSONParser
 
+import javax.servlet.http.HttpServletResponse
 import javax.transaction.Transactional
 import java.text.SimpleDateFormat
 
@@ -37,6 +38,7 @@ class ManageLayersController {
 
     FieldService fieldService
     LayerService layerService
+    AuthService authService
 
     /**
      * admin only or api_key
@@ -614,8 +616,9 @@ class ManageLayersController {
     def copy() {
         String spatialServiceUrl = params.spatialServiceUrl
         String fieldId = params.fieldId
+        String jwt = params.jwt
 
-        manageLayersService.updateFromRemote(spatialServiceUrl, fieldId)
+        manageLayersService.updateFromRemote(spatialServiceUrl, fieldId, jwt)
         redirect(controller: "Tasks", action: "index")
 
     }
@@ -628,11 +631,11 @@ class ManageLayersController {
     @RequireAdmin
     def enable() {
         if (params.id.isNumber()) {
-            def layer = layerDao.getLayerById(params.id.toInteger(), false)
+            def layer = layerService.getLayerById(params.id.toInteger(), false)
             layer.enabled = true
             layerService.updateLayer(layer)
         } else {
-            def field = fieldDao.getFieldById(params.id, false)
+            def field = fieldService.getFieldById(params.id, false)
             field.enabled = true
             fieldService.updateField(field)
         }
@@ -645,11 +648,22 @@ class ManageLayersController {
      * a layer: 'cl..._res', 'el..._res', will provide the standardized files at the requested resolution
      *          (or next detailed) - shape files or diva grids
      *
-     * admin only or api_key, do not redirect to CAS
+     * Requires JWT with the configured layerCopyRole
+     *
      * @return
      */
-    @RequireAdmin
+    @RequireApiKey
     def resource() {
+        // do permission check
+        if (!authService.userInRole(spatialConfig.layerCopyRole)) {
+            response.sendError(HttpServletResponse.SC_FORBIDDEN, "Forbidden, required role: " + spatialConfig.layerCopyRole)
+            return
+        }
+
+        if (!isLayerCopyFile(params.resource)) {
+            response.sendError(HttpServletResponse.SC_FORBIDDEN, "Forbidden, only allowed access to layer files")
+        }
+
         OutputStream outputStream = null
         try {
             outputStream = response.outputStream as OutputStream
@@ -674,13 +688,27 @@ class ManageLayersController {
     /**
      * for slaves to peek at a resource on the master
      *
-     * admin only or api_key, do not redirect to CAS
+     * Requires JWT with the configured layerCopyRole
      *
      * @return
      */
-    @RequireAdmin
+    @RequireApiKey
     def resourcePeek() {
+        // do permission check
+        if (!authService.userInRole(spatialConfig.layerCopyRole)) {
+            response.sendError(HttpServletResponse.SC_FORBIDDEN, "Forbidden, required role: " + spatialConfig.layerCopyRole)
+            return
+        }
+
+        if (!isLayerCopyFile(params.resource)) {
+            response.sendError(HttpServletResponse.SC_FORBIDDEN, "Forbidden, only allowed access to layer files")
+        }
+
         //write resource
         render fileService.info(params.resource.toString()) as JSON
     }
+
+    private boolean isLayerCopyFile(String resource) {
+        return resource.startsWith("/layer/") || resource.startsWith("/standard_layer/") || resource.startsWith("/public/layerDistances.properties")
+    }
 }

grails-app/services/au/org/ala/spatial/FieldService.groovy

@@ -171,7 +171,8 @@ class FieldService {
 
         Map map = field.properties
         map.put('id', field.id)
-        Fields.executeUpdate(sql, map)
+
+        Sql.newInstance(dataSource).executeUpdate(sql, map)
     }
 
     List<Layers> getLayersByCriteria(String keywords) {

grails-app/services/au/org/ala/spatial/LayerService.groovy

@@ -65,7 +65,8 @@ class LayerService {
     void updateLayer(Layers layer) {
         log.debug("Updating layer metadata for " + layer.getName())
         String sql = "update layers set citation_date=:citation_date, classification1=:classification1, classification2=:classification2, datalang=:datalang, description=:description, displayname=:displayname, displaypath=:displaypath, enabled=:enabled, domain=:domain, environmentalvaluemax=:environmentalvaluemax, environmentalvaluemin=:environmentalvaluemin, environmentalvalueunits=:environmentalvalueunits, extents=:extents, keywords=:keywords, licence_link=:licence_link, licence_notes=:licence_notes, licence_level=:licence_level, lookuptablepath=:lookuptablepath, maxlatitude=:maxlatitude, maxlongitude=:maxlongitude, mddatest=:mddatest, mdhrlv=:mdhrlv, metadatapath=:metadatapath, minlatitude=:minlatitude, minlongitude=:minlongitude, name=:name, notes=:notes, path=:path, path_1km=:path_1km, path_250m=:path_250m, path_orig=:path_orig, pid=:pid, respparty_role=:respparty_role, scale=:scale, source=:source, source_link=:source_link, type=:type, uid=:uid where id=:id"
-        Layers.executeUpdate(sql, layer)
+
+        Sql.newInstance(dataSource).executeUpdate(sql, layer)
     }
 
 

grails-app/services/au/org/ala/spatial/ManageLayersService.groovy

@@ -1389,7 +1389,7 @@ class ManageLayersService {
      * @param fieldId
      * @return
      */
-    def updateFromRemote(String spatialServiceUrl, String fieldId) {
+    def updateFromRemote(String spatialServiceUrl, String fieldId, String jwt) {
         def f = JSON.parse(httpCall("GET",
                 spatialServiceUrl + "/field/${fieldId}?pageSize=0",
                 null, null,
@@ -1429,7 +1429,7 @@ class ManageLayersService {
             createOrUpdateField(field, field.id + '', false)
         }
 
-        Map input = [layerId: layer.requestedId, fieldId: field.id, sourceUrl: spatialServiceUrl, displayPath: origDisplayPath] as Map
+        Map input = [layerId: layer.requestedId, fieldId: field.id, sourceUrl: spatialServiceUrl, displayPath: origDisplayPath, jwt: jwt] as Map
         Task task = tasksService.create("LayerCopy", UUID.randomUUID(), input).task
 
         task

grails-app/services/au/org/ala/spatial/TaskQueueService.groovy

@@ -193,6 +193,13 @@ class TaskQueueService {
                 taskWrapper.task.message = 'finished'
                 taskWrapper.task.history.put(System.currentTimeMillis() as String, "finished (id:${taskWrapper.task.id})" as String)
 
+                // map output.task to task when null to prevent error when flushing task
+                taskWrapper.task.output.each {
+                    if (!it.task) {
+                        it.task = taskWrapper.task
+                    }
+                }
+
                 // flush task because it is finished
                 Task.withTransaction {
                     if (!taskWrapper.task.save(flush: true)) {

grails-app/views/manageLayers/remote.gsp

@@ -18,6 +18,8 @@
     <h1>Copy Layers from Remote Server</h1>
 </div>
 
+<g:set var="spatialConfig" bean="spatialConfig"/>
+
 <g:if test="${spatialServiceUrl == localUrl}">
     <div class="col-lg-8">
         <div class="warning">The local and remote server is the same so layers cannot be copied</div>
@@ -44,12 +46,27 @@
     <br/>
     <br/>
 </g:if>
-
 <div class="container-fluid">
     This will copy a layer from a remote spatial-service to the local spatial-service.
-
+    <br/>
+    <h3>Config</h3>
+    <table>
+        <tbody>
+        <tr>
+            <td>URL of the remote spatial-service</td>
+            <td><input type="text" id="spatialServiceUrl" value="${spatialServiceUrl}" style='width:500px;margin-left:20px;'/><button onclick="refreshLayerList()">Refresh layer list</button></td>
+        </tr>
+        <tr>
+            <td>JWT for the remote service. It must have the role ${spatialConfig.layerCopyRole} and not expire before the task is finished.</td>
+            <td><textarea id="jwt" style='margin-left:20px;' cols="100" rows="10"></textarea></td>
+        </tr>
+        </tbody>
+    </table>
+</div>
+<div class="container-fluid">
     <br/>
     <br/>
+    <h3>List of layers</h3>
     Layer filter
     <select id="listSelector">
         <option value="divLocal">local only</option>
@@ -229,7 +246,14 @@
     });
 
     function copy(id) {
-        $.post("${localUrl}/manageLayers/copy?fieldId=" + id + "&spatialServiceUrl=" + encodeURIComponent("${spatialServiceUrl}"))
+        var jwt = document.getElementById('jwt').value;
+        if (jwt == "") {
+            alert("Please provide a JWT");
+            return;
+        }
+
+        $.post("${localUrl}/manageLayers/copy?fieldId=" + id + "&spatialServiceUrl=" + encodeURIComponent("${spatialServiceUrl}") +
+            "&jwt=" + encodeURIComponent(jwt));
         $('#copy' + id)[0].remove();
         $('#txtcopy' + id)[0].innerText = 'copying';
     }
@@ -256,6 +280,13 @@
         }
         location.href = downloadurl;
     }
+
+    function refreshLayerList() {
+        const remoteUrl = document.getElementById('spatialServiceUrl').value;
+        const url = new URL(window.location.href);
+        url.searchParams.set('remoteUrl', remoteUrl);
+        window.location.href = url.toString();
+    }
 </script>
 </div>
 </body>

src/main/groovy/au/org/ala/spatial/SpatialConfig.groovy

@@ -223,4 +223,6 @@ class SpatialConfig {
     String pipelinesConfig
     String sandboxSolrCollection
     Integer sandboxThreadCount
+
+    String layerCopyRole
 }

src/main/groovy/au/org/ala/spatial/Util.groovy

@@ -63,7 +63,7 @@ class Util {
         }
     }
 
-    static Map<String, Object> getStream(String url) {
+    static Map<String, Object> getStream(String url, String jwt) {
         HttpClient client = null
         HttpMethodBase call = null
         try {
@@ -76,6 +76,10 @@ class Util {
             try {
                 call = new GetMethod(url)
 
+                if (jwt) {
+                    call.addRequestHeader("Authorization", "Bearer " + jwt)
+                }
+
                 client.executeMethod(call)
             } catch (Exception e) {
                 log.error url, e

src/main/groovy/au/org/ala/spatial/process/LayerCopy.groovy

@@ -22,17 +22,14 @@ import grails.converters.JSON
 import groovy.util.logging.Slf4j
 import org.grails.web.json.JSONObject
 
-//@CompileStatic
 @Slf4j
 class LayerCopy extends SlaveProcess {
 
     void start() {
         String layerId = getInput('layerId')
         String fieldId = getInput('fieldId')
-        String  sourceUrl = getInput('sourceUrl')
-
-        //TODO: fetch default sld from geoserver
-        String displayPath = getInput('displayPath')
+        String sourceUrl = getInput('sourceUrl')
+        String sourceJwt = getInput('jwt')
 
         Fields field = getField(fieldId)
         Layers layer = getLayer(layerId)
@@ -46,7 +43,7 @@ class LayerCopy extends SlaveProcess {
         }
 
         //get layer files
-        getFile("/layer/${layer.name}", sourceUrl)
+        getFile("/layer/${layer.name}", sourceUrl, sourceJwt)
         addOutputFiles("/layer/${layer.name}", true)
 
         //get standardized files
@@ -64,13 +61,13 @@ class LayerCopy extends SlaveProcess {
         }
 
         resolutions.each { res ->
-            getFile("/standard_layer/${res}/${field.id}", sourceUrl)
+            getFile("/standard_layer/${res}/${field.id}", sourceUrl, sourceJwt)
             addOutputFiles("/standard_layer/${res}/${field.id}")
         }
 
         //get layerdistances
         taskLog("get layer distances")
-        getFile('/public/layerDistances.properties', sourceUrl)
+        getFile('/public/layerDistances.properties', sourceUrl, sourceJwt)
         JSONObject dists = JSON.parse(Util.getUrl(sourceUrl + "/layerDistances/layerdistancesJSON.json")) as JSONObject
         def distString = ''
         for (def f : getFields()) {

src/main/groovy/au/org/ala/spatial/process/SlaveProcess.groovy

@@ -88,8 +88,8 @@ class SlaveProcess {
     void stop() {}
 
 
-    def getFile(String path, String remoteSpatialServiceUrl) {
-        def remote = peekFile(path, remoteSpatialServiceUrl)
+    def getFile(String path, String remoteSpatialServiceUrl, String jwt) {
+        def remote = peekFile(path, remoteSpatialServiceUrl, jwt)
 
         //compare p list with local files
         def fetch = []
@@ -105,7 +105,7 @@ class SlaveProcess {
         if (fetch.size() < remote.size()) {
             //fetch only some
             fetch.each {
-                getFile(it, remoteSpatialServiceUrl)
+                getFile(it, remoteSpatialServiceUrl, jwt)
             }
         } else if (fetch.size() > 0) {
             //fetch all files
@@ -114,11 +114,10 @@ class SlaveProcess {
 
             try {
                 def shortpath = path.replace(spatialConfig.data.dir, '')
-                def url = remoteSpatialServiceUrl + "/master/resource?resource=" + URLEncoder.encode(shortpath, 'UTF-8') +
-                        "&api_key=" + spatialConfig.serviceKey
+                def url = remoteSpatialServiceUrl + "/master/resource?resource=" + URLEncoder.encode(shortpath, 'UTF-8')
 
                 def os = new BufferedOutputStream(new FileOutputStream(tmpFile))
-                def streamObj = Util.getStream(url)
+                def streamObj = Util.getStream(url, jwt)
                 try {
                     if (streamObj?.call) {
                         os << streamObj?.call?.getResponseBodyAsStream()
@@ -167,20 +166,33 @@ class SlaveProcess {
         }
     }
 
-    List peekFile(String path, String spatialServiceUrl = spatialConfig.spatialService.url) {
+    /**
+     * Get some info on file/s (path, exists, lastModified) from the local fs or remote spatial service.
+     *
+     * The result will include info for the file "spatialConfig.data.dir + path" if it exists, otherwise it will
+     * include info for the files "spatialConfig.data.dir + path + ".*" if they exist.
+     *
+     * @param path begins with '/' and is relative to spatialConfig.data.dir
+     * @param spatialServiceUrl specify if a remote spatial service is to be used
+     * @param jwt required if a remote spatial service is used
+     * @return
+     */
+    List peekFile(String path, String spatialServiceUrl = spatialConfig.spatialService.url, String jwt = null) {
         String shortpath = path.replace(spatialConfig.data.dir.toString(), '')
 
+        // if the spatial service is the same as the local service, use the file service
         if (spatialServiceUrl.equals(spatialConfig.grails.serverURL)) {
             return fileService.info(shortpath.toString())
         }
 
         List map = [[path: '', exists: false, lastModified: System.currentTimeMillis()]]
 
         try {
-            String url = spatialServiceUrl + "/master/resourcePeek?resource=" + URLEncoder.encode(shortpath, 'UTF-8') +
-                    "&api_key=" + spatialConfig.serviceKey
+            String url = spatialServiceUrl + "/master/resourcePeek?resource=" + URLEncoder.encode(shortpath, 'UTF-8')
+
+            // use the provided JWT authentication in the request header, and assign the JSON GET result to the map
+            map = JSON.parse(Util.urlResponse("GET", url, null, ['Authorization': 'Bearer ' + jwt])?.text) as List
 
-            map = JSON.parse(Util.getUrl(url)) as List
         } catch (err) {
             log.error "failed to get: " + path, err
         }

src/main/groovy/au/org/ala/spatial/process/StandardizeLayers.groovy

@@ -157,7 +157,6 @@ class StandardizeLayers extends SlaveProcess {
 
             //copy txt for 'a' and 'b'
             if (new File('/layer/' + l.name + '.txt').exists()) {
-                getFile('/layer/' + l.name + '.txt')
                 FileUtils.copyFile(new File(spatialConfig.data.dir.toString() + '/layer/' + l.name + '.txt'),
                         new File(spatialConfig.data.dir.toString() + '/standard_layer/' + res + '/' + f.id + '.txt'))
                 addOutput('file', '/standard_layer/' + res + '/' + f.id + '.txt')