From b551c91151be8002f423c5414fa532290463c888 Mon Sep 17 00:00:00 2001 From: temi Date: Thu, 27 Feb 2025 05:41:18 +1100 Subject: [PATCH 1/4] #1586 - supports new species list - added licence field for species list - migrated code to ECP and integrated them --- build.gradle | 2 +- .../javascripts/speciesFieldsSettings.js | 51 ++++++++++--- grails-app/conf/application.yml | 7 ++ .../biocollect/BioActivityController.groovy | 4 +- .../ProjectActivityController.groovy | 8 +- .../merit/ActivityController.groovy | 1 + .../biocollect/merit/ProxyController.groovy | 24 +++--- .../biocollect/merit/SearchController.groovy | 47 +++++------- .../biocollect/merit/SpeciesController.groovy | 1 + .../biocollect/merit/ActivityService.groovy | 1 + .../biocollect/merit/ProjectService.groovy | 1 + .../biocollect/merit/SpeciesService.groovy | 76 ++++++++----------- .../ala/biocollect/merit/WebService.groovy | 11 ++- .../views/project/csProjectTemplate.gsp | 9 ++- .../views/project/worksProjectTemplate.gsp | 9 ++- .../views/projectActivity/_addSpecies.gsp | 9 ++- grails-app/views/species/_species.gsp | 10 ++- .../merit/SpeciesServiceSpec.groovy | 24 +----- 18 files changed, 159 insertions(+), 136 deletions(-) diff --git a/build.gradle b/build.gradle index e46c9c047..3e79bd9a3 100644 --- a/build.gradle +++ b/build.gradle @@ -163,7 +163,7 @@ dependencies { implementation "io.jsonwebtoken:jjwt-api:0.11.5" if (!Boolean.valueOf(inplace)) { implementation "org.grails.plugins:ala-map-plugin:3.0.1" - implementation "org.grails.plugins:ecodata-client-plugin:7.2-SNAPSHOT" + implementation "org.grails.plugins:ecodata-client-plugin:7.3-SNAPSHOT" } testCompileOnly "org.grails:grails-test-mixins:3.3.0" diff --git a/grails-app/assets/javascripts/speciesFieldsSettings.js b/grails-app/assets/javascripts/speciesFieldsSettings.js index 09755f706..f6738128f 100644 --- a/grails-app/assets/javascripts/speciesFieldsSettings.js +++ b/grails-app/assets/javascripts/speciesFieldsSettings.js @@ -88,14 +88,43 @@ var SpeciesConstraintViewModel = function (o, fieldName) { self.transients.fieldName = ko.observable(fieldName); self.transients.bioSearch = ko.observable(fcConfig.speciesSearchUrl); self.transients.allowedListTypes = [ - {id: 'SPECIES_CHARACTERS', name: 'SPECIES_CHARACTERS'}, - {id: 'CONSERVATION_LIST', name: 'CONSERVATION_LIST'}, - {id: 'SENSITIVE_LIST', name: 'SENSITIVE_LIST'}, - {id: 'LOCAL_LIST', name: 'LOCAL_LIST'}, - {id: 'COMMON_TRAIT', name: 'COMMON_TRAIT'}, - {id: 'COMMON_HABITAT', name: 'COMMON_HABITAT'}, - {id: 'TEST', name: 'TEST'}, - {id: 'OTHER', name: 'OTHER'}]; + {id: 'SPECIES_CHARACTERS', name: 'Species characters'}, + {id: 'CONSERVATION_LIST', name: 'Conservation species list'}, + {id: 'SENSITIVE_LIST', name: 'Sensitive list'}, + {id: 'LOCAL_LIST', name: 'Local checklist'}, + {id: 'COMMON_TRAIT', name: 'Traits'}, + {id: 'TEST', name: 'Testing List'} + ]; + self.transients.allowedLicences = [ + { + "value": "CC0", + "label": "Creative Commons Zero" + }, + { + "value": "CC-BY", + "label": "Creative Commons By Attribution" + }, + { + "value": "CC-BY-NC", + "label": "Creative Commons By Attribution-Noncommercial" + }, + { + "value": "CC-BY-NC-ND", + "label": "Creative Commons By Attribution-Noncommercial-Noderivatives" + }, + { + "value": "CC-BY-NC-SA", + "label": "Creative Commons By Attribution-Noncommercial-Sharealike" + }, + { + "value": "CC-BY-ND", + "label": "Creative Commons By Attribution-Noderivatives" + }, + { + "value": "CC-BY-SA", + "label": "Creative Commons By Attribution-Sharealike" + } + ] self.transients.showAddSpeciesLists = ko.observable(false); self.transients.showExistingSpeciesLists = ko.observable(false); @@ -206,6 +235,7 @@ var SpeciesConstraintViewModel = function (o, fieldName) { var jsData = {}; jsData.listName = self.newSpeciesLists.listName(); jsData.listType = self.newSpeciesLists.listType(); + jsData.licence = self.newSpeciesLists.licence(); jsData.description = self.newSpeciesLists.description(); jsData.listItems = ""; @@ -283,6 +313,7 @@ var NewSpeciesListViewModel = function (o) { self.dataResourceUid = ko.observable(o.dataResourceUid); self.description = ko.observable(o.description); self.listType = ko.observable(o.listType); + self.licence = ko.observable(o.licence); self.allSpecies = ko.observableArray(); self.inputSpeciesViewModel = new SpeciesViewModel({}, fcConfig); @@ -309,7 +340,7 @@ var NewSpeciesListViewModel = function (o) { }; self.transients = {}; - self.transients.url = ko.observable(fcConfig.speciesListsServerUrl + "/speciesListItem/list/" + o.dataResourceUid); + self.transients.url = ko.observable(fcConfig.speciesListsServerUrl + "/" + o.dataResourceUid); }; @@ -542,7 +573,7 @@ var SpeciesList = function (o) { self.transients = {}; - self.transients.url = ko.observable(fcConfig.speciesListsServerUrl + "/speciesListItem/list/" + o.dataResourceUid); + self.transients.url = ko.observable(fcConfig.speciesListsServerUrl + "/" + o.dataResourceUid); self.transients.check = ko.observable(false); self.transients.truncatedListName = ko.computed(function () { return truncate(self.listName(), 45); diff --git a/grails-app/conf/application.yml b/grails-app/conf/application.yml index 273a0962e..287b967a4 100644 --- a/grails-app/conf/application.yml +++ b/grails-app/conf/application.yml @@ -74,8 +74,15 @@ images: lists: baseURL: "https://lists.ala.org.au" # "https://lists-test.ala.org.au/public/speciesLists" + uiBaseURL: "https://lists.ala.org.au" commonFields: ['rawScientificName', 'matchedName', 'commonName'] facetsToRemoveFromProjectFinderPage: ['projLifecycleStatus'] + apiVersion: "v1" + +listsFieldMappingV2: + matchedName: "classification.scientificName" + commonName: "classification.vernacularName" + rawScientificName: "scientificName" merit: baseURL: "https://fieldcapture-test.ala.org.au" diff --git a/grails-app/controllers/au/org/ala/biocollect/BioActivityController.groovy b/grails-app/controllers/au/org/ala/biocollect/BioActivityController.groovy index 0f8ae23e8..3daa9bcb4 100644 --- a/grails-app/controllers/au/org/ala/biocollect/BioActivityController.groovy +++ b/grails-app/controllers/au/org/ala/biocollect/BioActivityController.groovy @@ -31,9 +31,7 @@ import org.grails.web.json.JSONObject import org.springframework.context.MessageSource import org.springframework.web.multipart.MultipartFile -import static org.apache.http.HttpStatus.SC_BAD_REQUEST -import static org.apache.http.HttpStatus.SC_OK -import static org.apache.http.HttpStatus.SC_INTERNAL_SERVER_ERROR +import static org.apache.http.HttpStatus.* @SecurityScheme(name = "auth", type = SecuritySchemeType.HTTP, diff --git a/grails-app/controllers/au/org/ala/biocollect/ProjectActivityController.groovy b/grails-app/controllers/au/org/ala/biocollect/ProjectActivityController.groovy index 7008ac6bf..038a9c877 100644 --- a/grails-app/controllers/au/org/ala/biocollect/ProjectActivityController.groovy +++ b/grails-app/controllers/au/org/ala/biocollect/ProjectActivityController.groovy @@ -184,10 +184,10 @@ class ProjectActivityController { } } log.debug "values: " + (values as JSON).toString() - def response = speciesService.addSpeciesList(postBody); - def result - if (response?.resp?.druid) { - result = [status: "ok", id: response.resp.druid] + Map response = speciesService.addSpeciesList(postBody); + Map result + if (response) { + result = [status: "ok", id: response.druid] } else { result = [status: 'error', error: "Error creating new species lists, please try again later."] } diff --git a/grails-app/controllers/au/org/ala/biocollect/merit/ActivityController.groovy b/grails-app/controllers/au/org/ala/biocollect/merit/ActivityController.groovy index ee0b3d2fe..2e66dd280 100644 --- a/grails-app/controllers/au/org/ala/biocollect/merit/ActivityController.groovy +++ b/grails-app/controllers/au/org/ala/biocollect/merit/ActivityController.groovy @@ -1,6 +1,7 @@ package au.org.ala.biocollect.merit import au.org.ala.biocollect.ProjectActivityService +import au.org.ala.biocollect.merit.SpeciesService import grails.converters.JSON import org.apache.http.HttpStatus import org.apache.poi.ss.usermodel.Workbook diff --git a/grails-app/controllers/au/org/ala/biocollect/merit/ProxyController.groovy b/grails-app/controllers/au/org/ala/biocollect/merit/ProxyController.groovy index 610c7916a..9b73a6e92 100644 --- a/grails-app/controllers/au/org/ala/biocollect/merit/ProxyController.groovy +++ b/grails-app/controllers/au/org/ala/biocollect/merit/ProxyController.groovy @@ -1,11 +1,12 @@ package au.org.ala.biocollect.merit +import au.org.ala.ecodata.forms.SpeciesListService import grails.converters.JSON import org.apache.commons.io.FilenameUtils import au.org.ala.web.SSO class ProxyController { - + static responseFormats = ['json'] def webService, commonService, projectService SpeciesService speciesService @@ -18,16 +19,20 @@ class ProxyController { } def speciesLists() { - def paramString = commonService.buildUrlParamsFromMap(params) - render webService.get("${grailsApplication.config.lists.baseURL}/ws/speciesList${paramString}", false) + respond speciesListService.searchSpeciesList(params.sort, params.max, params.offset, params.guid, params.order, params.searchTerm) } def speciesList() { - render webService.get("${grailsApplication.config.lists.baseURL}/ws/speciesList?druid=${params.druid}", false) + SpeciesListService.SpeciesList speciesList = speciesListService.getSpeciesListMetadata(params.druid) + // it is possible for old species list to return list of species list when druid is not found + if (!speciesList.isValid()) + speciesList = null + + respond speciesList } def speciesItemsForList() { - render webService.get("${grailsApplication.config.lists.baseURL}/ws/speciesListItems/${params.druid}?includeKVP=true", false) + respond speciesListService.allSpeciesListItems(params.druid) } def intersect(){ @@ -43,17 +48,16 @@ class ProxyController { } def speciesProfile(String id) { - Map result = speciesService.getSpeciesDetailsForTaxonId(id); + Map result = speciesService.getSpeciesDetailsForTaxonId(id) render result } @SSO def speciesListPost() { def postBody = request.JSON - def druidParam = (postBody.druid) ? "/${postBody.druid}" : "" // URL part - def postResponse = webService.doPost("${grailsApplication.config.lists.baseURL}/ws/speciesListPost${druidParam}", postBody) - if (postResponse.resp && postResponse.resp.druid) { - def druid = postResponse.resp?.druid?:druid + def postResponse = speciesService.addSpeciesList(postBody) + String druid = postResponse?.druid + if (druid) { postBody.druid = druid def result = projectService.update(postBody.projectId, [listId: druid, listReason: postBody.reason]) diff --git a/grails-app/controllers/au/org/ala/biocollect/merit/SearchController.groovy b/grails-app/controllers/au/org/ala/biocollect/merit/SearchController.groovy index 142e298b1..ce0694c2d 100644 --- a/grails-app/controllers/au/org/ala/biocollect/merit/SearchController.groovy +++ b/grails-app/controllers/au/org/ala/biocollect/merit/SearchController.groovy @@ -1,10 +1,14 @@ package au.org.ala.biocollect.merit +import au.org.ala.ecodata.forms.SpeciesListService import grails.converters.JSON import org.apache.commons.lang.StringUtils +import org.springframework.http.HttpStatus class SearchController { + static responseFormats = ['json'] def searchService, webService, speciesService, commonService, projectActivityService + SpeciesListService speciesListService grails.core.GrailsApplication grailsApplication /** @@ -24,11 +28,11 @@ class SearchController { * @return */ def species(String q, Integer limit) { - render speciesService.searchForSpecies(q, limit, params.listId) as JSON + respond speciesService.searchForSpecies(q, limit, params.listId) } def searchSpeciesList(String sort, Integer max, Integer offset, String guid, String order, String searchTerm){ - render speciesService.searchSpeciesList(sort, max, offset, guid, order, searchTerm) as JSON + respond speciesListService.searchSpeciesList(sort, max, offset, guid, order, searchTerm) } /** @@ -54,25 +58,11 @@ class SearchController { } def getCommonKeys(){ - try { - if(params.druid){ - def resp = webService.getJson("${grailsApplication.config.lists.baseURL}/ws/listCommonKeys?druid=${params.druid}")?:[] - if(resp instanceof List){ - if(grailsApplication.config.lists.commonFields){ - resp?.addAll(grailsApplication.config.lists.commonFields) - } - - resp.sort() - render text: resp as JSON, contentType: 'application/json' - } else { - render text: resp.error, status: resp.statusCode?:HttpStatus.INTERNAL_SERVER_ERROR - } - } else { - render status: HttpStatus.BAD_REQUEST, text: 'Parameter druid is required.' - } - } catch (Exception ex){ - log.error (ex.message, ex) - render status: HttpStatus.INTERNAL_SERVER_ERROR, text: "An error occurred - ${ex.message}" + if (params.druid) { + List commonFields = speciesListService.getCommonKeys(params.druid) + respond commonFields + } else { + render status: HttpStatus.BAD_REQUEST, text: 'Parameter druid is required.' } } @@ -89,15 +79,14 @@ class SearchController { * Example: /search/getSpeciesTranslation?id=urn:lsid:biodiversity.org.au:afd.taxon:4136b6d0-b5be-45d4-8323-e96e03d94218&listId=dr8016 * */ def getSpeciesTranslation(String id, String listId) { - def items = webService.getJson("${grailsApplication.config.lists.baseURL}/ws/speciesListItems/${listId}?includeKVP=true")?:[] - def kvp = [:] - if(items instanceof List){ - items.each { - if("${it.lsid}" == "${id}") { - kvp = it.kvpValues - } + SpeciesListService.SpeciesListItem items = speciesListService.allSpeciesListItems(listId) ?: [] + List kvp = [] + items.each { + if (it.lsid == id) { + kvp = it.kvpValues } } - render kvp as JSON + + respond kvp } } diff --git a/grails-app/controllers/au/org/ala/biocollect/merit/SpeciesController.groovy b/grails-app/controllers/au/org/ala/biocollect/merit/SpeciesController.groovy index a229d522b..1e9e94259 100644 --- a/grails-app/controllers/au/org/ala/biocollect/merit/SpeciesController.groovy +++ b/grails-app/controllers/au/org/ala/biocollect/merit/SpeciesController.groovy @@ -1,5 +1,6 @@ package au.org.ala.biocollect.merit +import au.org.ala.biocollect.merit.SpeciesService import grails.converters.JSON import org.apache.http.HttpStatus diff --git a/grails-app/services/au/org/ala/biocollect/merit/ActivityService.groovy b/grails-app/services/au/org/ala/biocollect/merit/ActivityService.groovy index 7c52675d5..524828f6a 100644 --- a/grails-app/services/au/org/ala/biocollect/merit/ActivityService.groovy +++ b/grails-app/services/au/org/ala/biocollect/merit/ActivityService.groovy @@ -3,6 +3,7 @@ package au.org.ala.biocollect.merit import au.org.ala.biocollect.DateUtils import au.org.ala.biocollect.ProjectActivityService import au.org.ala.biocollect.UtilService +import au.org.ala.biocollect.merit.SpeciesService import grails.core.GrailsApplication import org.joda.time.DateTime import org.joda.time.Period diff --git a/grails-app/services/au/org/ala/biocollect/merit/ProjectService.groovy b/grails-app/services/au/org/ala/biocollect/merit/ProjectService.groovy index d5658d7aa..219d41a73 100644 --- a/grails-app/services/au/org/ala/biocollect/merit/ProjectService.groovy +++ b/grails-app/services/au/org/ala/biocollect/merit/ProjectService.groovy @@ -3,6 +3,7 @@ package au.org.ala.biocollect.merit import au.org.ala.biocollect.EmailService import au.org.ala.biocollect.OrganisationService import au.org.ala.biocollect.merit.hub.HubSettings +import au.org.ala.biocollect.merit.SpeciesService import grails.converters.JSON import org.springframework.context.MessageSource import au.org.ala.web.UserDetails diff --git a/grails-app/services/au/org/ala/biocollect/merit/SpeciesService.groovy b/grails-app/services/au/org/ala/biocollect/merit/SpeciesService.groovy index 0ee7aadbf..c7e67297b 100644 --- a/grails-app/services/au/org/ala/biocollect/merit/SpeciesService.groovy +++ b/grails-app/services/au/org/ala/biocollect/merit/SpeciesService.groovy @@ -1,9 +1,11 @@ package au.org.ala.biocollect.merit +import au.org.ala.ecodata.forms.SpeciesListService +import au.org.ala.ecodata.forms.SpeciesListService.SpeciesListItem import com.opencsv.CSVParser +import com.opencsv.CSVParserBuilder import com.opencsv.CSVReader import com.opencsv.CSVReaderBuilder -import com.opencsv.CSVParserBuilder import grails.converters.JSON import grails.plugin.cache.Cacheable @@ -17,6 +19,7 @@ class SpeciesService { static final String SCIENTIFIC_NAME_COMMON_NAME = 'SCIENTIFICNAME(COMMONNAME)' def webService, grailsApplication + SpeciesListService speciesListService def searchForSpecies(searchTerm, limit = 10, listId = null) { @@ -52,7 +55,7 @@ class SpeciesService { def searchSpeciesInLists(String searchTerm, Map speciesConfig = [:], limit = 10, offset = 0){ List druids = speciesConfig.speciesLists?.collect{it.dataResourceUid} Map fields = getSpeciesListAutocompleteLookupFields(speciesConfig) - List listResults = searchSpeciesListOnFields(searchTerm, druids, fields.fieldList, limit, offset) + List listResults = speciesListService.searchSpeciesListOnFields(searchTerm, druids, fields.fieldList, limit, offset) formatSpeciesListResultToAutocompleteFormat(listResults, fields.fieldMap) } @@ -76,11 +79,11 @@ class SpeciesService { * @return */ Map formatSpeciesListResultToAutocompleteFormat(List queryResult, Map fields){ - List autoCompleteList = queryResult?.collect { result -> + List autoCompleteList = queryResult?.collect { SpeciesListItem result -> Map searchResult = [id: result.id, guid: result.lsid, lsid: result.lsid, listId: result.dataResourceUid] - searchResult.scientificName = result[fields.scientificNameField]?: result.kvpValues?.find { it.key == fields.scientificNameField } ?.value + searchResult.scientificName = result.kvpValues?.find { it.key == fields.scientificNameField } ?.value ?: result.scientificName searchResult.scientificNameMatches = searchResult.scientificName ? [ searchResult.scientificName ] : [] - searchResult.commonName = result[fields.commonNameField]?: result.kvpValues?.find { it.key == fields.commonNameField } ?.value + searchResult.commonName = result.kvpValues?.find { it.key == fields.commonNameField } ?.value ?: result.commonName searchResult.commonNameMatches = searchResult.commonName ? [ searchResult.commonName ] : [] searchResult } @@ -120,7 +123,7 @@ class SpeciesService { * @return a JSON formatted String of the form {"autoCompleteList":[{...results...}]} */ private def filterSpeciesList(String query, String listId) { - def listContents = webService.getJson("${grailsApplication.config.lists.baseURL}/ws/speciesListItems/${listId}?q=${query}") + def listContents = speciesListService.allSpeciesListItems(listId, query) def filtered = listContents.collect({[id: it.id, listId: listId, name: it.name, commonName: it.commonName, scientificName: it.scientificName, scientificNameMatches:[it.name], guid:it.lsid]}) @@ -131,22 +134,6 @@ class SpeciesService { return results } - /** - * Executes a query on given fields in supplied data resources. - * @param query the term to search for. - * @param listId the id of the list to search. - * @return - */ - private def searchSpeciesListOnFields(String query, List listId = [], List fields = [], limit = 10, offset = 0) { - def listContents = webService.getJson("${grailsApplication.config.lists.baseURL}/ws/queryListItemOrKVP?druid=${listId.join(',')}&fields=${URLEncoder.encode(fields.join(','), "UTF-8")}&q=${URLEncoder.encode(query, "UTF-8")}&includeKVP=true&max=${limit}&offset=${offset}") - - if(listContents.hasProperty('error')){ - throw new Exception(listContents.error) - } - - return listContents - } - def searchBie(searchTerm, fq, limit) { if (!limit) { limit = 10 @@ -166,22 +153,6 @@ class SpeciesService { webService.getJson(url) } - def searchSpeciesList(String sort = 'listName', Integer max = 100, Integer offset = 0, String guid = null, String order = "asc", String searchTerm = null) { - def list - String url = "${grailsApplication.config.lists.baseURL}/ws/speciesList?sort=${sort}&max=${max}&offset=${offset}&order=${order}&q=${searchTerm?:''}" - - if (!guid) { - list = webService.getJson(url) - } else { - // Search List by species in the list - url = "${url}&items=createAlias:items&items.guid=eq:${guid}" - list = webService.getJson(url) - } - - list - - } - /** * formats a name into the specified format * if species does not match to a taxon, then mention it in name. @@ -263,14 +234,31 @@ class SpeciesService { data } - def addSpeciesList(postBody) { - webService.doPost("${grailsApplication.config.lists.baseURL}/ws/speciesList", postBody) + Map addSpeciesList (Map postBody) { + if (speciesListService.checkListAPIVersion(SpeciesListService.LIST_VERSION_V1)) { + String druid = postBody.druid ?: "" + return speciesListService.uploadSpeciesListUsingV1(postBody, druid) + } else { + List rows = getSpeciesListAsCSV(postBody.listItems as String) + return speciesListService.uploadSpeciesListUsingV2(rows, postBody.listName, postBody.description, postBody.licence, postBody.listType) + } } - def getAllSpeciesList(){ - // 1000 is the maximum that species list could give right now. - String url = "${grailsApplication.config.lists.baseURL}/ws/speciesList?sort=listName&offset=0&max=1000" - webService.getJson(url) + /** + * Convert comma seperated species list to a format resembling CSV. + * @return + */ + List getSpeciesListAsCSV(String listItems){ + List csv = [] + if (listItems) { + csv.add(["scientificName"]) + List speciesList = listItems.split(',')?.toList() + speciesList.each { + csv.add([it]) + } + } + + csv } /** diff --git a/grails-app/services/au/org/ala/biocollect/merit/WebService.groovy b/grails-app/services/au/org/ala/biocollect/merit/WebService.groovy index 09ac97f59..24cd3cb28 100644 --- a/grails-app/services/au/org/ala/biocollect/merit/WebService.groovy +++ b/grails-app/services/au/org/ala/biocollect/merit/WebService.groovy @@ -14,27 +14,26 @@ */ package au.org.ala.biocollect.merit -import groovyx.net.http.HTTPBuilder + +import au.org.ala.ws.tokens.TokenService import grails.converters.JSON +import grails.web.http.HttpHeaders +import groovyx.net.http.HTTPBuilder import groovyx.net.http.Method import org.apache.http.entity.mime.HttpMultipartMode import org.apache.http.entity.mime.MultipartEntity import org.apache.http.entity.mime.content.InputStreamBody import org.apache.http.entity.mime.content.StringBody import org.grails.web.converters.exceptions.ConverterException -import grails.web.http.HttpHeaders -import org.springframework.core.env.Environment import org.springframework.http.MediaType import org.springframework.web.multipart.MultipartFile -import au.org.ala.ws.tokens.TokenService import javax.annotation.PostConstruct import javax.servlet.http.Cookie import javax.servlet.http.HttpServletResponse import java.nio.charset.StandardCharsets -import static org.apache.http.HttpHeaders.* - +import static org.apache.http.HttpHeaders.ACCEPT /** * Helper class for invoking ecodata (and other Atlas) web services. */ diff --git a/grails-app/views/project/csProjectTemplate.gsp b/grails-app/views/project/csProjectTemplate.gsp index c1828ee49..418c19340 100644 --- a/grails-app/views/project/csProjectTemplate.gsp +++ b/grails-app/views/project/csProjectTemplate.gsp @@ -2,6 +2,13 @@ + + + + + + + @@ -71,7 +78,7 @@ getSpeciesFieldsForSurveyUrl: "${createLink(controller: 'projectActivity', action: 'ajaxGetSpeciesFieldsForSurvey')}", speciesProfileUrl: "${createLink(controller: 'proxy', action: 'speciesProfile')}", speciesListUrl: "${createLink(controller: 'search', action: 'searchSpeciesList')}", - speciesListsServerUrl: "${grailsApplication.config.lists.baseURL}", + speciesListsServerUrl: "${speciesListServerURL}", speciesSearchUrl: "${createLink(controller: 'search', action: 'species')}", searchBieUrl: "${raw(createLink(controller: 'project', action: 'searchSpecies', params: [id: project.projectId, limit: 10]))}", imageUploadUrl: "${createLink(controller: 'image', action: 'upload')}", diff --git a/grails-app/views/project/worksProjectTemplate.gsp b/grails-app/views/project/worksProjectTemplate.gsp index 5d80455a7..6fb4ace0f 100644 --- a/grails-app/views/project/worksProjectTemplate.gsp +++ b/grails-app/views/project/worksProjectTemplate.gsp @@ -2,6 +2,13 @@ <%@ page import="grails.converters.JSON" %> + + + + + + + @@ -62,7 +69,7 @@ addNewSpeciesListsUrl: "${raw(createLink(controller: 'projectActivity', action: 'ajaxAddNewSpeciesLists', params: [projectId:project.projectId]))}", speciesProfileUrl: "${createLink(controller: 'proxy', action: 'speciesProfile')}", speciesListUrl: "${createLink(controller: 'search', action: 'searchSpeciesList')}", - speciesListsServerUrl: "${grailsApplication.config.lists.baseURL}", + speciesListsServerUrl: "${speciesListServerURL}", speciesSearchUrl: "${createLink(controller: 'search', action: 'species')}", imageUploadUrl: "${createLink(controller: 'image', action: 'upload')}", bieUrl: "${grailsApplication.config.bie.baseURL}", diff --git a/grails-app/views/projectActivity/_addSpecies.gsp b/grails-app/views/projectActivity/_addSpecies.gsp index 4c49f2026..186bdc05c 100644 --- a/grails-app/views/projectActivity/_addSpecies.gsp +++ b/grails-app/views/projectActivity/_addSpecies.gsp @@ -6,17 +6,22 @@

Create new species lists

-
+
-
+
+
+ + +
diff --git a/grails-app/views/species/_species.gsp b/grails-app/views/species/_species.gsp index 81db0d13d..1fba2a6a4 100644 --- a/grails-app/views/species/_species.gsp +++ b/grails-app/views/species/_species.gsp @@ -1,5 +1,6 @@ +
@@ -10,9 +11,14 @@

Species lists can be selected to be used by this project when species information is required to be supplied as a part of activity reporting. Lists are created and managed using the ALA Species List tool. + + + + + +

- ${grailsApplication.config.lists.baseURL}/speciesListItem/list/${project.listId} - ALA Species List URL: ${listUrl} + ALA Species List URL: ${speciesListServerURL}

diff --git a/src/test/groovy/au/org/ala/biocollect/merit/SpeciesServiceSpec.groovy b/src/test/groovy/au/org/ala/biocollect/merit/SpeciesServiceSpec.groovy index f59abc4de..b2f39b258 100644 --- a/src/test/groovy/au/org/ala/biocollect/merit/SpeciesServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/biocollect/merit/SpeciesServiceSpec.groovy @@ -1,5 +1,6 @@ package au.org.ala.biocollect.merit + import grails.testing.services.ServiceUnitTest import spock.lang.Specification /* @@ -28,29 +29,6 @@ class SpeciesServiceSpec extends Specification implements ServiceUnitTest> [:] - } - - void "should call list tool with passed search term"() { - String searchTerm = 'abc' - String url = "xyz/ws/speciesList?sort=listName&max=10&offset=0&order=asc&q=abc" - - when: - service.searchSpeciesList('listName', 10, 0, null, 'asc', searchTerm) - - then: - 1 * service.webService.getJson(url) >> [:] - - } - def "formatTaxonName should format name based on displayType"() { setup: Map data = [commonName: commonName, scientificName: scientificName] From 1f6b3d54e6fdf7332af7256fc4986747abbc6040 Mon Sep 17 00:00:00 2001 From: temi Date: Thu, 6 Mar 2025 07:04:24 +1100 Subject: [PATCH 2/4] #1586 - replaced postMultipart with OkHttp3 version --- .../biocollect/BioActivityController.groovy | 14 +++-- .../biocollect/merit/ImageController.groovy | 3 +- .../biocollect/merit/SiteController.groovy | 2 +- .../biocollect/merit/ActivityService.groovy | 8 +-- .../biocollect/merit/DocumentService.groovy | 2 +- .../biocollect/merit/MetadataService.groovy | 5 +- .../biocollect/merit/ProjectService.groovy | 33 +++++----- .../ala/biocollect/merit/WebService.groovy | 61 ++----------------- 8 files changed, 39 insertions(+), 89 deletions(-) diff --git a/grails-app/controllers/au/org/ala/biocollect/BioActivityController.groovy b/grails-app/controllers/au/org/ala/biocollect/BioActivityController.groovy index 3daa9bcb4..148fc53c7 100644 --- a/grails-app/controllers/au/org/ala/biocollect/BioActivityController.groovy +++ b/grails-app/controllers/au/org/ala/biocollect/BioActivityController.groovy @@ -59,6 +59,7 @@ class BioActivityController { static int MAX_FLIMIT = 500 static allowedMethods = ['bulkDelete': 'POST', bulkRelease: 'POST', bulkEmbargo: 'POST'] + static responseFormats = ['json'] /** * Update Activity by activityId or @@ -1577,15 +1578,16 @@ class BioActivityController { } if (pActivityId && type && file) { - def content = activityService.convertExcelToOutputData(pActivityId, type, file) - def status = SC_OK - if (content.error) { - status = SC_INTERNAL_SERVER_ERROR + def result = activityService.convertExcelToOutputData(pActivityId, type, file) + if (!org.springframework.http.HttpStatus.resolve(result.statusCode).is2xxSuccessful()) { + respond (result.resp, status: result.statusCode) + } + else { + respond (result.content?.subMap('data') ?: result) } - render text: content as JSON, status: status } else { - render text: [message: "Missing required parameters - pActivityId, type & data (excel file)"] as JSON, status: SC_BAD_REQUEST + respond([message: "Missing required parameters - pActivityId, type & data (excel file)"], status: SC_BAD_REQUEST) } } diff --git a/grails-app/controllers/au/org/ala/biocollect/merit/ImageController.groovy b/grails-app/controllers/au/org/ala/biocollect/merit/ImageController.groovy index a467c8982..5985431f6 100644 --- a/grails-app/controllers/au/org/ala/biocollect/merit/ImageController.groovy +++ b/grails-app/controllers/au/org/ala/biocollect/merit/ImageController.groovy @@ -27,6 +27,7 @@ import io.swagger.v3.oas.annotations.security.SecurityRequirement import io.swagger.v3.oas.annotations.security.SecurityScheme import org.apache.commons.io.FilenameUtils import org.imgscalr.Scalr +import org.springframework.http.HttpStatus import org.springframework.web.multipart.MultipartFile import javax.imageio.ImageIO @@ -131,7 +132,7 @@ class ImageController { MultipartFile file = request.getFile('files') def result = webService.postMultipart(url, params, file, 'image') - if (result.content) { + if (HttpStatus.resolve(result.statusCode as int).is2xxSuccessful()) { def detailsUrl = "${grailsApplication.config.images.baseURL}ws/getImageInfo?id=${result.content.imageId}" def imageDetails = webService.getJson(detailsUrl) def thumbnailUrl = imageDetails.imageUrl.replace("/original", "/thumbnail") diff --git a/grails-app/controllers/au/org/ala/biocollect/merit/SiteController.groovy b/grails-app/controllers/au/org/ala/biocollect/merit/SiteController.groovy index 38ae63f2c..27dc2afe8 100644 --- a/grails-app/controllers/au/org/ala/biocollect/merit/SiteController.groovy +++ b/grails-app/controllers/au/org/ala/biocollect/merit/SiteController.groovy @@ -364,7 +364,7 @@ class SiteController { def result = siteService.uploadShapefile(f) - if (!result.error && result.content.size() > 1) { + if (org.springframework.http.HttpStatus.resolve(result.statusCode as int).is2xxSuccessful()) { def content = result.content def shapeFileId = content.remove('shp_id') def firstShape = content["0"] diff --git a/grails-app/services/au/org/ala/biocollect/merit/ActivityService.groovy b/grails-app/services/au/org/ala/biocollect/merit/ActivityService.groovy index 524828f6a..4f9ace73c 100644 --- a/grails-app/services/au/org/ala/biocollect/merit/ActivityService.groovy +++ b/grails-app/services/au/org/ala/biocollect/merit/ActivityService.groovy @@ -442,12 +442,6 @@ class ActivityService { } def convertExcelToOutputData(String id, String type, def file){ - def result = webService.postMultipart(grailsApplication.config.ecodata.service.url + "/metadata/extractOutputDataFromActivityExcelTemplate", [pActivityId: id, type: type], file, 'data', true) - if (result.error) { - return result.details - } - else { - return result.content?.subMap('data') ?: result - } + webService.postMultipart(grailsApplication.config.ecodata.service.url + "/metadata/extractOutputDataFromActivityExcelTemplate", [pActivityId: id, type: type], file, 'data') } } diff --git a/grails-app/services/au/org/ala/biocollect/merit/DocumentService.groovy b/grails-app/services/au/org/ala/biocollect/merit/DocumentService.groovy index b4afc935c..155c8ddbb 100644 --- a/grails-app/services/au/org/ala/biocollect/merit/DocumentService.groovy +++ b/grails-app/services/au/org/ala/biocollect/merit/DocumentService.groovy @@ -64,7 +64,7 @@ class DocumentService { def file = new File(grailsApplication.config.upload.images.path, document.filename) // Create a new document, supplying the file that was uploaded to the ImageController. result = createDocument(document, document.contentType, new FileInputStream(file)) - if (!result.error) { + if (org.springframework.http.HttpStatus.resolve(result.statusCode as int).is2xxSuccessful()) { file.delete() } } diff --git a/grails-app/services/au/org/ala/biocollect/merit/MetadataService.groovy b/grails-app/services/au/org/ala/biocollect/merit/MetadataService.groovy index 2b5300f88..c1000036a 100644 --- a/grails-app/services/au/org/ala/biocollect/merit/MetadataService.groovy +++ b/grails-app/services/au/org/ala/biocollect/merit/MetadataService.groovy @@ -1,5 +1,6 @@ package au.org.ala.biocollect.merit +import org.springframework.http.HttpStatus import org.springframework.web.multipart.MultipartFile import java.text.DateFormat @@ -275,8 +276,8 @@ class MetadataService { def data = webService.postMultipart(url, params, file, 'data') def result - if (data.error || data.status != SC_OK) { - result = [status:data.status, error:data.error?:'No data was found that matched the columns in this table, please check the template you used to upload the data. '] + if (!HttpStatus.resolve(data.statusCode as int).is2xxSuccessful()) { + result = [status:data.statusCode, error:data.content ?: 'No data was found that matched the columns in this table, please check the template you used to upload the data. '] } else { result = [status:SC_OK, data:data.content.data] diff --git a/grails-app/services/au/org/ala/biocollect/merit/ProjectService.groovy b/grails-app/services/au/org/ala/biocollect/merit/ProjectService.groovy index 219d41a73..72433972a 100644 --- a/grails-app/services/au/org/ala/biocollect/merit/ProjectService.groovy +++ b/grails-app/services/au/org/ala/biocollect/merit/ProjectService.groovy @@ -3,10 +3,10 @@ package au.org.ala.biocollect.merit import au.org.ala.biocollect.EmailService import au.org.ala.biocollect.OrganisationService import au.org.ala.biocollect.merit.hub.HubSettings -import au.org.ala.biocollect.merit.SpeciesService +import au.org.ala.web.UserDetails import grails.converters.JSON import org.springframework.context.MessageSource -import au.org.ala.web.UserDetails +import org.springframework.http.HttpStatus class ProjectService { @@ -510,24 +510,25 @@ class ProjectService { Map params = [projectIds:projectId,userId:userId] response = webService.postMultipart(url, params, null, null, null) - if(response.error){ - if(response.error.contains('Timed out')){ - throw new SocketTimeoutException(response.error) + if (HttpStatus.resolve(response.statusCode as int).is2xxSuccessful()) { + if (isAdmin) { + response?.content?.each { key, value -> + if (value != null) { + permissions[key] = true + } else { + permissions[key] = null; + } + } } else { - throw new Exception(response.error) + permissions = response.content; } } - - if(isAdmin){ - response?.content?.each{ key, value -> - if(value != null){ - permissions[key] = true - } else { - permissions[key] = null; - } + else { + if (response.error?.contains('Timed out')) { + throw new SocketTimeoutException(response.error) + } else { + throw new Exception(response.error) } - } else { - permissions = response.content; } permissions diff --git a/grails-app/services/au/org/ala/biocollect/merit/WebService.groovy b/grails-app/services/au/org/ala/biocollect/merit/WebService.groovy index 24cd3cb28..db2d75991 100644 --- a/grails-app/services/au/org/ala/biocollect/merit/WebService.groovy +++ b/grails-app/services/au/org/ala/biocollect/merit/WebService.groovy @@ -15,15 +15,10 @@ package au.org.ala.biocollect.merit +import au.org.ala.ecodata.forms.EcpWebService import au.org.ala.ws.tokens.TokenService import grails.converters.JSON import grails.web.http.HttpHeaders -import groovyx.net.http.HTTPBuilder -import groovyx.net.http.Method -import org.apache.http.entity.mime.HttpMultipartMode -import org.apache.http.entity.mime.MultipartEntity -import org.apache.http.entity.mime.content.InputStreamBody -import org.apache.http.entity.mime.content.StringBody import org.grails.web.converters.exceptions.ConverterException import org.springframework.http.MediaType import org.springframework.web.multipart.MultipartFile @@ -48,6 +43,7 @@ class WebService { def grailsApplication TokenService tokenService + EcpWebService ecpWebService @PostConstruct void init() { @@ -457,12 +453,10 @@ class WebService { * @param url the URL to forward to. * @param params the (string typed) HTTP parameters to be attached. * @param file the Multipart file object to forward. - * @param includeFailureDetails if true, the return value will include response body. If content type is JSON, an object will be returned in `details` property. * @return [status:, content: */ - def postMultipart(url, Map params, MultipartFile file, fileParam = 'files', boolean includeFailureDetails = false) { - - postMultipart(url, params, file.inputStream, file.contentType, file.originalFilename, fileParam, includeFailureDetails) + Map postMultipart(String url, Map params, MultipartFile file, fileParam = 'files', boolean useToken = false, boolean userToken = false) { + ecpWebService.postMultipart(url, params, file, fileParam, useToken, userToken) } /** @@ -475,50 +469,7 @@ class WebService { * @param fileParamName the name of the HTTP parameter that will be used for the post. * @return [status:, content: */ - def postMultipart(url, Map params, InputStream contentIn, contentType, originalFilename, fileParamName = 'files', boolean includeFailureDetails = false) { - - def result = [:] - def user = userService.getUser() - - HTTPBuilder builder = new HTTPBuilder(url) - - builder.request(Method.POST) { request -> - requestContentType : 'multipart/form-data' - MultipartEntity content = new MultipartEntity(HttpMultipartMode.BROWSER_COMPATIBLE) - if (contentIn) content.addPart(fileParamName, new InputStreamBody(contentIn, contentType, originalFilename?:fileParamName)) - params.each { key, value -> - if (value) { - content.addPart(key, new StringBody(value.toString())) - } - } - - if (isDomainWhitelisted(new URL(url))) { - headers."Authorization" = getAuthHeader() - } - - addHubUrlPath(headers) - - - if (user) { - headers[grailsApplication.config.app.http.header.userId] = user.userId - } - else { - log.warn("No user associated with request: ${url}") - } - request.setEntity(content) - - response.success = {resp, message -> - result.status = resp.status - result.content = message - } - - response.failure = {resp, reader -> - result.status = resp.status - result.error = "Error POSTing to ${url}" - if (includeFailureDetails) - result.details = reader - } - } - result + Map postMultipart(String url, Map params, InputStream contentIn, contentType, originalFilename, fileParamName = 'files', boolean useToken = false, boolean userToken = false) { + ecpWebService.postMultipart(url, params, contentIn, contentType, originalFilename, fileParamName, useToken, userToken) } } From 4243d02d8b5bc683848ce3afe75c63607d8c1ecf Mon Sep 17 00:00:00 2001 From: temi Date: Thu, 6 Mar 2025 12:10:58 +1100 Subject: [PATCH 3/4] #1586 - fixed missing service --- .../au/org/ala/biocollect/merit/ProxyController.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/grails-app/controllers/au/org/ala/biocollect/merit/ProxyController.groovy b/grails-app/controllers/au/org/ala/biocollect/merit/ProxyController.groovy index 9b73a6e92..34c28dd02 100644 --- a/grails-app/controllers/au/org/ala/biocollect/merit/ProxyController.groovy +++ b/grails-app/controllers/au/org/ala/biocollect/merit/ProxyController.groovy @@ -9,6 +9,7 @@ class ProxyController { static responseFormats = ['json'] def webService, commonService, projectService SpeciesService speciesService + SpeciesListService speciesListService def geojsonFromPid(String pid) { def shpUrl = "${grailsApplication.config.spatial.layersUrl}/shape/geojson/${pid}" From 4cfc92220b9304df21e9c8b1b0056bb304219fde Mon Sep 17 00:00:00 2001 From: temi Date: Thu, 6 Mar 2025 14:00:32 +1100 Subject: [PATCH 4/4] #1586 - added one more postMutipart method to WebService --- .../org/ala/biocollect/merit/WebService.groovy | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/grails-app/services/au/org/ala/biocollect/merit/WebService.groovy b/grails-app/services/au/org/ala/biocollect/merit/WebService.groovy index db2d75991..59c2e9025 100644 --- a/grails-app/services/au/org/ala/biocollect/merit/WebService.groovy +++ b/grails-app/services/au/org/ala/biocollect/merit/WebService.groovy @@ -472,4 +472,21 @@ class WebService { Map postMultipart(String url, Map params, InputStream contentIn, contentType, originalFilename, fileParamName = 'files', boolean useToken = false, boolean userToken = false) { ecpWebService.postMultipart(url, params, contentIn, contentType, originalFilename, fileParamName, useToken, userToken) } + + + /** + * Post a local file to an URL using multipart/form-data. + * @param url + * @param params + * @param file + * @param contentType + * @param originalFilename + * @param fileParamName + * @param useToken + * @param userToken + * @return + */ + Map postMultipart(String url, Map params, File file, String contentType, String originalFilename, String fileParamName = 'files', boolean useToken = false, boolean userToken = false) { + ecpWebService.postMultipart(url, params, file, contentType, originalFilename, fileParamName, useToken, userToken) + } }