Merge pull request #3460 from AtlasOfLivingAustralia/feature/issue3421

Feature/issue3421

Author: chrisala

.github/workflows/build.yml

@@ -20,10 +20,10 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Set up JDK 11
+      - name: Set up JDK 17
         uses: actions/setup-java@v3
         with:
-          java-version: '11'
+          java-version: '17'
           distribution: 'adopt'
 
       - name: Install nodejs

build.gradle

@@ -21,6 +21,8 @@ plugins {
     id 'jacoco'
     id 'com.williamhill.wiremock' version '0.4.1'
 }
+sourceCompatibility = '11'
+targetCompatibility = '17'
 
 version "$meritVersion"
 group "au.org.ala"
@@ -152,6 +154,17 @@ dependencies {
     implementation "org.commonmark:commonmark:0.24.0"
     implementation "com.googlecode.owasp-java-html-sanitizer:owasp-java-html-sanitizer:20240325.1"
 
+    // Used to get a token that can be used to access the BDR API hosted in Azure
+    implementation platform('com.azure:azure-sdk-bom:1.2.28')
+
+    implementation 'com.azure:azure-identity'
+    implementation 'com.azure:azure-storage-blob'
+
+    // Used to access the Cognito Identity Pool as tokens from the user pool
+    // don't have an audience claim which is required by Azure
+    implementation(platform("software.amazon.awssdk:bom:2.27.21"))
+    implementation 'software.amazon.awssdk:cognitoidentity'
+
     compileOnly "io.micronaut:micronaut-inject-groovy"
     console "org.grails:grails-console"
     profile "org.grails.profiles:web"
@@ -188,7 +201,7 @@ dependencies {
     providedCompile "io.methvin:directory-watcher:0.4.0"
 
     if (!Boolean.valueOf(inplace)) {
-        implementation "org.grails.plugins:ecodata-client-plugin:7.3-SNAPSHOT"
+        implementation "org.grails.plugins:ecodata-client-plugin:8.0-SNAPSHOT"
     }
 }
 

gradle.properties

@@ -16,4 +16,4 @@ org.gradle.jvmargs=-Xmx2048M
 seleniumVersion=3.12.0
 seleniumSafariDriverVersion=3.14.0
 snapshotCacheTime=1800
-alaSecurityLibsVersion=6.3.0
+alaSecurityLibsVersion=7.0.0

grails-app/assets/javascripts/dataSets.js

@@ -18,6 +18,8 @@ var DataSetsViewModel =function(dataSets, projectService, config) {
         return new DataSetSummary(dataSet);
     });
 
+    self.downloadProjectDataSetsUrl = config.downloadProjectDataSetsUrl;
+
     self.newDataSet = function() {
         window.location.href = config.newDataSetUrl;
     };
@@ -40,6 +42,30 @@ var DataSetsViewModel =function(dataSets, projectService, config) {
         return (report && report.name);
     }
 
+    function isDownloadableMonitorDataSet(dataSet) {
+
+        if (dataSet.collectionApp !== MONITOR_APP) {
+            return false;
+        }
+        var protocolId = dataSet.protocol;
+        var downloadableProtocols = config.downloadableProtocols || [];
+
+        var isDownloadable = downloadableProtocols.indexOf(protocolId) >= 0;
+        if (isDownloadable) {
+            var now = moment();
+            var creationDate = moment(dataSet.dateCreated);
+            var minutesToInjestDataSet = config.minutesToInjestDataSet || 1;
+            if (dataSet.progress !== ActivityProgress.planned) {
+                if (creationDate.add(minutesToInjestDataSet, 'minutes').isBefore(now)) {
+                    isDownloadable = true;
+                }
+            }
+        }
+        return isDownloadable;
+    }
+
+    self.enableProjectDataSetsDownload = _.find(dataSets, isDownloadableMonitorDataSet) != null;
+
     /** View model backing for a single row in the data set summary table */
     function DataSetSummary(dataSet) {
 
@@ -69,20 +95,24 @@ var DataSetsViewModel =function(dataSets, projectService, config) {
             });
         };
 
+        this.downloadUrl = null;
+        if (isDownloadableMonitorDataSet(dataSet)) {
+            this.downloadUrl = config.downloadDataSetUrl + '/' + dataSet.dataSetId;
+        }
+
         if (this.createdIn === MONITOR_APP) {
             if (this.progress == ActivityProgress.planned) {
                 var now = moment();
                 var creationDate = moment(dataSet.dateCreated);
+
                 if (creationDate.add(1, 'minutes').isBefore(now)) {
                     this.progress = 'sync error';
                 }
                 else {
                     this.progress = 'sync in progress';
                 }
-
             }
         }
-
     }
 };
 

grails-app/assets/javascripts/projects.js

@@ -1271,8 +1271,11 @@ function ProjectPageViewModel(project, sites, activities, userRoles, config) {
             editDataSetUrl: config.editDataSetUrl,
             deleteDataSetUrl: config.deleteDataSetUrl,
             viewDataSetUrl: config.viewDataSetUrl,
+            downloadDataSetUrl: config.downloadDataSetUrl,
+            downloadProjectDataSetsUrl: config.downloadProjectDataSetsUrl,
             returnToUrl: config.returnToUrl,
             reports: project.reports || [],
+            downloadableProtocols: config.downloadableProtocols,
             viewReportUrl: config.viewReportUrl
         };
         var projectService = new ProjectService({}, config);

grails-app/conf/application.groovy

@@ -251,6 +251,17 @@ security {
         requiredClaims = ["sub", "iat", "exp", "jti", "client_id"]
     }
 }
+bdr.api.url="https://changeMe.org.au/api"
+bdr.api.readTimeout=60000
+bdr['client-id']="changeMe"
+bdr['client-secret']="changeMe"
+bdr.discoveryUri="https://changeMe.org.au/.well-known"
+bdr.jwtScopes="read"
+bdr.azure.clientId='changeMe'
+bdr.azure.tenantId='changeMe'
+bdr.azure.apiScope='api://changeme/.default'
+//bdr.dataSet.formats=["application/geo+json","application/rdf+xml", "text/turtle", "application/ld+json", "application/n-triples"]
+bdr.dataSet.formats=["application/geo+json", "text/turtle"]
 
 webservice.jwt = true
 webservice['jwt-scopes'] = "ala/internal users/read ala/attrs ecodata/read_test ecodata/write_test"
@@ -302,6 +313,8 @@ environments {
         app.default.hub='merit'
         wiremock.port = 8018
         security.oidc.discoveryUri = "http://localhost:${wiremock.port}/cas/oidc/.well-known"
+        security.jwt.discoveryUri = "http://localhost:${wiremock.port}/cas/oidc/.well-known"
+        bdr.discoveryUri = "http://localhost:${wiremock.port}/cas/oidc/.well-known"
         security.oidc.allowUnsignedIdTokens = true
         security.oidc.clientId="oidcId"
         security.oidc.secret="oidcSecret"
@@ -333,6 +346,7 @@ environments {
         spatial.baseUrl = "http://localhost:${wiremock.port}"
         spatial.layersUrl = spatial.baseUrl + "/ws"
         grails.mail.port = 3025 // com.icegreen.greenmail.util.ServerSetupTest.SMTP.port
+
     }
     production {
         grails.logging.jul.usebridge = false

grails-app/conf/logback.xml

@@ -15,7 +15,7 @@
     <appender-ref ref="STDOUT" />
   </logger>
 
-  <logger name="org.pac4j" level="WARN">
+  <logger name="org.pac4j" level="INFO">
     <appender-ref ref="STDOUT" />
   </logger>
 

grails-app/conf/spring/resources.groovy

@@ -1,5 +1,7 @@
 import au.org.ala.merit.CheckRisksAndThreatsTask
 import au.org.ala.merit.MeritServletContextConfig
+import au.org.ala.merit.config.BdrTokenConfig
+import au.org.ala.merit.hub.HubAwareLinkGenerator
 import au.org.ala.merit.StatisticsFactory
 import au.org.ala.merit.hub.HubAwareLinkGenerator
 import au.org.ala.merit.reports.NHTOutputReportLifecycleListener
@@ -27,4 +29,5 @@ beans = {
     RegionalCapacityServicesReport(RegionalCapacityServicesReportLifecycleListener)
 
     meritServletContextConfig(MeritServletContextConfig)
+    bdrTokenConfig(BdrTokenConfig)
 }

grails-app/controllers/au/org/ala/merit/DataSetController.groovy

@@ -1,17 +1,18 @@
 package au.org.ala.merit
 
-import au.org.ala.merit.PreAuthorise
-import au.org.ala.merit.ProjectService
+
 import au.org.ala.merit.config.ProgramConfig
 import grails.converters.JSON
 import org.springframework.http.HttpStatus
 
 class DataSetController {
 
     static allowedMethods = [create:'GET', edit:'GET', save:'POST', delete:'POST']
-
+    private static final Integer DEFAULT_BDR_QUERY_LIMIT = 5000
     ProjectService projectService
     DataSetSummaryService dataSetSummaryService
+    BdrService bdrService
+    WebService webService
 
     // Note that authorization is done against a project, so the project id must be supplied to the method.
     @PreAuthorise(accessLevel = 'editor')
@@ -157,6 +158,64 @@ class DataSetController {
         render response as JSON
     }
 
+    @PreAuthorise(accessLevel = 'admin')
+    def downloadProjectDataSets(String id, String format, Integer limit) {
+        if (!id) {
+            render status: HttpStatus.NOT_FOUND
+            return
+        }
+        Map projectData = projectData(id)
+        List supportedFormats = grailsApplication.config.getProperty('bdr.dataSet.formats', List)
+        if (!format) {
+            format = supportedFormats[0]
+        }
+        if (format !in supportedFormats) {
+            render status: HttpStatus.BAD_REQUEST
+            return
+        }
+
+        bdrService.downloadProjectDataSet(id, format, projectData.project.name, response, limit ?: DEFAULT_BDR_QUERY_LIMIT)
+    }
+
+    @PreAuthorise(accessLevel = 'admin')
+    def download(String id, String dataSetId, String format, Integer limit) {
+        Map projectData = projectData(id)
+
+        List supportedFormats = grailsApplication.config.getProperty('bdr.dataSet.formats', List)
+        if (!format) {
+            format = supportedFormats[0]
+        }
+        if (format !in supportedFormats) {
+            render status: HttpStatus.BAD_REQUEST
+            return
+        }
+
+        Map dataSet = projectData.project?.custom?.dataSets?.find{it.dataSetId == dataSetId}
+
+        if (!dataSet) {
+            render status: HttpStatus.NOT_FOUND
+            return
+        }
+        else {
+            if (isMonitorDataSet(dataSet)) {
+                if (isProtocolSupportedForDownload(dataSet)) {
+                    bdrService.downloadDataSet(id, dataSet.dataSetId, dataSet.name, format, response, limit ?: DEFAULT_BDR_QUERY_LIMIT)
+                }
+            }
+            else if (dataSet.url) {
+                webService.proxyGetRequest(response, dataSet.url, false)
+            }
+        }
+    }
+
+    private static boolean isMonitorDataSet(Map dataSet) {
+        return dataSet.protocol
+    }
+
+    private static boolean isProtocolSupportedForDownload(Map dataSet) {
+        return true
+    }
+
     @PreAuthorise(accessLevel = 'editor')
     def delete(String id) {
 

grails-app/controllers/au/org/ala/merit/ProjectController.groovy

@@ -200,14 +200,16 @@ class ProjectController {
         if (datasetsVisible && project.custom?.dataSets) {
             projectService.filterDataSetSummaries(project.custom?.dataSets)
         }
+        List downloadableProtocols = downloadableProtocols()
+
         boolean showExternalIds = userService.userHasReadOnlyAccess() || userService.userIsSiteAdmin()
         def model = [overview       : [label: 'Overview', visible: true, default: true, type: 'tab', publicImages: imagesModel, displayOutcomes: false, blog: blog, hasNewsAndEvents: hasNewsAndEvents, hasProjectStories: hasProjectStories, canChangeProjectDates: canChangeProjectDates, outcomes:project.outcomes, objectives:config.program?.config?.objectives, showExternalIds:showExternalIds],
                      documents      : [label: 'Documents', visible: config.includesContent(ProgramConfig.ProjectContent.DOCUMENTS), type: 'tab', user:user, template:'docs', activityPeriodDescriptor:config.activityPeriodDescriptor ?: 'Stage'],
                      details        : [label: 'MERI Plan', default: false, disabled: !meriPlanEnabled, visible: meriPlanVisible, meriPlanVisibleToUser: meriPlanVisibleToUser, risksAndThreatsVisible: canViewRisks, announcementsVisible: true, project:project, type: 'tab', template:'viewMeriPlan', meriPlanTemplate:MERI_PLAN_TEMPLATE+'View', config:config, activityPeriodDescriptor:config.activityPeriodDescriptor ?: 'Stage'],
                      plan           : [label: 'Activities', visible: true, disabled: !user?.hasViewAccess, type: 'tab', template:'projectActivities', grantManagerSettingsVisible:user?.isCaseManager, project:project, reports: project.reports, scores: scores, risksAndThreatsVisible: risksAndThreatsVisible],
                      site           : [label: 'Sites', visible: config.includesContent(ProgramConfig.ProjectContent.SITES), disabled: !user?.hasViewAccess, editable:user?.isEditor, type: 'tab', template:'projectSites'],
                      dashboard      : [label: 'Dashboard', visible: config.includesContent(ProgramConfig.ProjectContent.DASHBOARD), disabled: !user?.hasViewAccess, type: 'tab'],
-                     datasets       : [label: 'Data set summary', visible: datasetsVisible, template: '/project/dataset/dataSets', type:'tab'],
+                     datasets       : [label: 'Data set summary', visible: datasetsVisible, template: '/project/dataset/dataSets', downloadableProtocols: downloadableProtocols, supportedFormats:bdrDataSetSupportedFormats(), type:'tab'],
                      admin          : [label: 'Admin', visible: adminTabVisible, user:user, type: 'tab', template:'projectAdmin', project:project, canChangeProjectDates: canChangeProjectDates, minimumProjectEndDate:minimumProjectEndDate, showMERIActivityWarning:true, showAnnouncementsTab: showAnnouncementsTab, showSpecies:true, meriPlanTemplate:MERI_PLAN_TEMPLATE, showMeriPlanHistory:showMeriPlanHistory, requireMeriPlanApprovalReason:Boolean.valueOf(config.supportsMeriPlanHistory),  config:config, activityPeriodDescriptor:config.activityPeriodDescriptor ?: 'Stage', canRegenerateReports: canRegenerateReports, hasSubmittedOrApprovedFinalReportInCategory: hasSubmittedOrApprovedFinalReportInCategory, canModifyMeriPlan: canModifyMeriPlan, showRequestLabels:config.supportsParatoo, outcomeStartIndex:outcomeStartIndex]]
 
         if (template == MERI_ONLY_TEMPLATE) {
@@ -274,6 +276,17 @@ class ProjectController {
         return [view: 'index', model: model]
     }
 
+    private List<String> downloadableProtocols() {
+        String BDR_DOWNLOAD_SUPPORTED_TAG = 'bdr_download_supported'
+        List<Map> forms = activityService.monitoringProtocolForms()
+        forms = forms.findAll{BDR_DOWNLOAD_SUPPORTED_TAG in it.tags}
+        forms.collect{it.externalId }
+    }
+
+    private List bdrDataSetSupportedFormats() {
+        grailsApplication.config.getProperty("bdr.dataSet.formats", List.class)
+    }
+
     private Map buildRLPTargetsModel(Map model, project){
         //Verify project.outcomes (from program config) with primaryOutcome and secondaryOutcomes in project.custom.details.outcomes
         Map primaryOutcome = project.custom?.details?.outcomes?.primaryOutcome

grails-app/controllers/au/org/ala/merit/UrlMappings.groovy

@@ -128,6 +128,11 @@ class UrlMappings {
             action = 'upload'
         }
 
+        "/dataSet/download/$id/$dataSetId" {
+            controller = 'dataSet'
+            action = 'download'
+        }
+
         "500"(view:'/error')
         "404"(view:'/404')
         "/$hub/$controller/ws/$action/$id" {

grails-app/i18n/messages.properties

@@ -368,4 +368,7 @@ report.status.published=Report approved
 report.status.pendingApproval=Report submitted
 report.status.unpublished=Report not submitted
 report.status.cancelled=Report not required
-report.status.=
+report.status.=
+
+bdr.dataSet.format.application/geo+json=GeoJSON
+bdr.dataSet.format.text/turtle=Turtle