Skip to content

Commit 482788b

Browse files
chrisalachrisala
authored
Merge pull request #919 from AtlasOfLivingAustralia/feature/issue3049
Monitor form sync and occurrence record creation
2 parents 9aa8a86 + fb03f14 commit 482788b

40 files changed

+8409
-1928
lines changed

‎build.gradle

+1-1
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ plugins {
2020
id "com.gorylenko.gradle-git-properties" version "2.4.1"
2121
}
2222

23-
version "4.5-SNAPSHOT"
23+
version "4.5-SPECIES-SNAPSHOT"
2424
group "au.org.ala"
2525
description "Ecodata"
2626

‎grails-app/conf/application.groovy

+95
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,9 @@ if (!google.geocode.url) {
106106
if (!temp.file.cleanup.days) {
107107
temp.file.cleanup.days = 1
108108
}
109+
if(!paratoo.location.excluded) {
110+
paratoo.location.excluded = ['location.vegetation-association-nvis']
111+
}
109112
access.expiry.maxEmails=500
110113

111114

@@ -531,6 +534,7 @@ ecodata.documentation.exampleProjectUrl = 'http://ecodata-test.ala.org.au/ws/act
531534
// Used by ParatooService to sync available protocols
532535
paratoo.core.baseUrl = 'https://dev.core-api.monitor.tern.org.au/api'
533536
paratoo.excludeInterventionProtocols = true
537+
paratoo.core.documentationUrl = '/documentation/swagger.json'
534538

535539
auth.baseUrl = 'https://auth-test.ala.org.au'
536540
userDetails.web.url = "${auth.baseUrl}/userdetails/"
@@ -1391,3 +1395,94 @@ elasticsearch {
13911395
username = 'elastic'
13921396
password = 'password'
13931397
}
1398+
1399+
// paratoo / monitor
1400+
1401+
paratoo.defaultPlotLayoutDataModels = [
1402+
[
1403+
dataType: "geoMap",
1404+
name: "plot_layout",
1405+
validate: "required"
1406+
],
1407+
[
1408+
dataType: "list",
1409+
name: "plot_visit",
1410+
validate: "required",
1411+
columns: [
1412+
[
1413+
dataType: "date",
1414+
name: "end_date",
1415+
dwcAttribute: "eventDate"
1416+
],
1417+
[
1418+
dataType: "text",
1419+
name: "visit_field_name"
1420+
],
1421+
[
1422+
dataType: "date",
1423+
name: "start_date",
1424+
dwcAttribute: "eventDate"
1425+
]
1426+
]
1427+
]
1428+
]
1429+
1430+
paratoo.defaultPlotLayoutViewModels = [
1431+
[
1432+
type: "row",
1433+
items: [
1434+
[
1435+
type: "col",
1436+
items: [
1437+
[
1438+
type: "section",
1439+
title: "Plot Visit",
1440+
preLabel: "Plot Visit",
1441+
boxed: true,
1442+
items: [
1443+
[
1444+
type: "repeat",
1445+
source: "plot_visit",
1446+
userAddedRows: false,
1447+
items: [
1448+
[
1449+
type: "row",
1450+
class: "output-section",
1451+
items: [
1452+
[
1453+
type: "col",
1454+
items: [
1455+
[
1456+
type: "date",
1457+
source: "end_date",
1458+
preLabel: "End Date"
1459+
],
1460+
[
1461+
type: "text",
1462+
source: "visit_field_name",
1463+
preLabel: "Visit Field Name"
1464+
],
1465+
[
1466+
type: "date",
1467+
source: "start_date",
1468+
preLabel: "Start Date"
1469+
]
1470+
]
1471+
]
1472+
]
1473+
]
1474+
]
1475+
]
1476+
]
1477+
],
1478+
[
1479+
type: "geoMap",
1480+
source: "plot_layout",
1481+
orientation: "vertical"
1482+
]
1483+
]
1484+
]
1485+
]
1486+
]
1487+
]
1488+

‎grails-app/controllers/au/org/ala/ecodata/AdminController.groovy

+57
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
package au.org.ala.ecodata
22

3+
import au.org.ala.ecodata.paratoo.ParatooCollection
4+
import au.org.ala.ecodata.paratoo.ParatooProject
5+
import au.org.ala.ecodata.paratoo.ParatooProtocolConfig
36
import au.org.ala.web.AlaSecured
47
import grails.converters.JSON
58
import grails.util.Environment
@@ -725,4 +728,58 @@ class AdminController {
725728
render errors as JSON
726729
}
727730

731+
/**
732+
* Re-fetch data from Paratoo. Helpful when data could not be parsed correctly the first time.
733+
*
734+
* @return
735+
*/
736+
@AlaSecured(["ROLE_ADMIN"])
737+
def reSubmitDataSet() {
738+
String projectId = params.id
739+
String dataSetId = params.dataSetId
740+
String userId = params.userId ?: userService.getCurrentUser().userId
741+
if (!projectId || !dataSetId || !userId) {
742+
render text: [message: "Bad request"] as JSON, status: HttpStatus.SC_BAD_REQUEST
743+
return
744+
}
745+
746+
ParatooCollection collection = new ParatooCollection(orgMintedUUID: dataSetId, coreProvenance: [:])
747+
List<ParatooProject> projects = paratooService.userProjects(userId)
748+
ParatooProject project = projects.find {it.project.projectId == projectId }
749+
if (project) {
750+
paratooService.submitCollection(collection, project, userId)
751+
render text: [message: "Submitted request to fetch data for dataSet $dataSetId in project $projectId by user $userId"] as JSON, status: HttpStatus.SC_OK, contentType: 'application/json'
752+
}
753+
else {
754+
render text: [message: "Project not found"] as JSON, status: HttpStatus.SC_NOT_FOUND
755+
}
756+
}
757+
758+
759+
/**
760+
* Helper function to check the form generated for a protocol during the sync operation.
761+
* Usual step is to update Paratoo config in DB. Use this function to check the form generated.
762+
* @return
763+
*/
764+
@AlaSecured(["ROLE_ADMIN"])
765+
def checkActivityFormForProtocol() {
766+
String protocolId = params.id
767+
List protocols = paratooService.getProtocolsFromParatoo()
768+
Map protocol = protocols.find { it.attributes.identifier == protocolId }
769+
if (!protocol) {
770+
render text: [message: "Protocol not found"] as JSON, status: HttpStatus.SC_NOT_FOUND, contentType: 'application/json'
771+
return
772+
}
773+
774+
Map documentation = paratooService.getParatooSwaggerDocumentation()
775+
ParatooProtocolConfig config = paratooService.getProtocolConfig(protocolId)
776+
if (!config) {
777+
render text: [message: "Protocol config not found"] as JSON, status: HttpStatus.SC_NOT_FOUND, contentType: 'application/json'
778+
return
779+
}
780+
781+
Map template = paratooService.buildTemplateForProtocol(protocol, documentation, config)
782+
render text: template as JSON, status: HttpStatus.SC_OK, contentType: 'application/json'
783+
}
784+
728785
}

‎grails-app/controllers/au/org/ala/ecodata/ParatooController.groovy

+3-9
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,7 @@
11
package au.org.ala.ecodata
22

33
import au.ala.org.ws.security.SkipApiKeyCheck
4-
import au.org.ala.ecodata.paratoo.ParatooCollection
5-
import au.org.ala.ecodata.paratoo.ParatooCollectionId
6-
import au.org.ala.ecodata.paratoo.ParatooPlotSelection
7-
import au.org.ala.ecodata.paratoo.ParatooPlotSelectionData
8-
import au.org.ala.ecodata.paratoo.ParatooProject
9-
import au.org.ala.ecodata.paratoo.ParatooToken
4+
import au.org.ala.ecodata.paratoo.*
105
import groovy.util.logging.Slf4j
116
import io.swagger.v3.oas.annotations.OpenAPIDefinition
127
import io.swagger.v3.oas.annotations.Operation
@@ -31,7 +26,6 @@ import javax.ws.rs.GET
3126
import javax.ws.rs.POST
3227
import javax.ws.rs.PUT
3328
import javax.ws.rs.Path
34-
3529
// Requiring these scopes will guarantee we can get a valid userId out of the process.
3630
@Slf4j
3731
@au.ala.org.ws.security.RequireApiKey(scopes = ["profile", "openid"])
@@ -294,10 +288,10 @@ class ParatooController {
294288
boolean hasProtocol = paratooService.protocolWriteCheck(userId, dataSet.project.id, collectionId.protocolId)
295289
if (hasProtocol) {
296290
Map result = paratooService.submitCollection(collection, dataSet.project)
297-
if (!result.error) {
291+
if (!result.updateResult.error) {
298292
respond([success: true])
299293
} else {
300-
error(HttpStatus.SC_INTERNAL_SERVER_ERROR, result.error)
294+
error(HttpStatus.SC_INTERNAL_SERVER_ERROR, result.updateResult.error)
301295
}
302296
} else {
303297
error(HttpStatus.SC_FORBIDDEN, "Project / protocol combination not available")

‎grails-app/controllers/au/org/ala/ecodata/ProjectController.groovy

+11
Original file line numberDiff line numberDiff line change
@@ -366,6 +366,17 @@ class ProjectController {
366366
}
367367
}
368368

369+
@RequireApiKey
370+
def fetchDataSetRecords (String projectId, String dataSetId) {
371+
if (projectId && dataSetId) {
372+
List records = projectService.fetchDataSetRecords(projectId, dataSetId)
373+
render text: records as JSON, contentType: 'application/json'
374+
}
375+
else {
376+
render status: 400, text: "projectId and dataSetId are required parameters"
377+
}
378+
}
379+
369380
def importProjectsFromSciStarter(){
370381
Integer count = projectService.importProjectsFromSciStarter()?:0
371382
render(text: [count: count] as JSON, contentType: 'application/json');

‎grails-app/controllers/au/org/ala/ecodata/UrlMappings.groovy

+1
Original file line numberDiff line numberDiff line change
@@ -194,6 +194,7 @@ class UrlMappings {
194194
"/ws/project/getDataCollectionWhiteList"(controller: "project"){ action = [GET:"getDataCollectionWhiteList"] }
195195
"/ws/project/getBiocollectFacets"(controller: "project"){ action = [GET:"getBiocollectFacets"] }
196196
"/ws/project/getDefaultFacets"(controller: "project", action: "getDefaultFacets")
197+
"/ws/project/$projectId/dataSet/$dataSetId/records"(controller: "project", action: "fetchDataSetRecords")
197198
"/ws/admin/initiateSpeciesRematch"(controller: "admin", action: "initiateSpeciesRematch")
198199

199200
"/ws/document/download"(controller:"document", action:"download")

‎grails-app/domain/au/org/ala/ecodata/Record.groovy

+11-1
Original file line numberDiff line numberDiff line change
@@ -14,23 +14,29 @@ class Record {
1414
activityId index: true
1515
projectActivityId index: true
1616
lastUpdated index: true
17+
dataSetId index: true
18+
outputId index: true
1719
version false
1820
}
1921

2022
ObjectId id
2123
String projectId //ID of the project within ecodata
2224
String projectActivityId
25+
String dataSetId
2326
String activityId
2427
String occurrenceID
2528
String outputSpeciesId // reference to output species outputSpeciesId.
2629
String userId
2730
String eventDate //should be a date in "yyyy-MM-dd" or "2014-11-24T04:55:48+11:00" format
31+
String scientificName
32+
String name
33+
String vernacularName
2834
Double decimalLatitude
2935
Double decimalLongitude
3036
Double generalizedDecimalLatitude
3137
Double generalizedDecimalLongitude
3238
Integer coordinateUncertaintyInMeters
33-
Integer individualCount
39+
Integer individualCount = 1
3440
Integer numberOfOrganisms
3541
Date dateCreated
3642
Date lastUpdated
@@ -66,6 +72,10 @@ class Record {
6672
outputItemId nullable: true
6773
status nullable: true
6874
outputSpeciesId nullable: true
75+
dataSetId nullable: true
76+
name nullable: true
77+
vernacularName nullable: true
78+
scientificName nullable: true
6979
}
7080

7181
String getRecordNumber(sightingsUrl){

‎grails-app/services/au/org/ala/ecodata/ElasticSearchService.groovy

+1-1
Original file line numberDiff line numberDiff line change
@@ -1317,7 +1317,7 @@ class ElasticSearchService {
13171317
* @param path
13181318
* @return
13191319
*/
1320-
List getDataFromPath(output, List path){
1320+
static List getDataFromPath(output, List path){
13211321
def temp = output
13221322
List result = []
13231323
List navigatedPath = []

‎grails-app/services/au/org/ala/ecodata/MetadataService.groovy

+17-7
Original file line numberDiff line numberDiff line change
@@ -844,13 +844,23 @@ class MetadataService {
844844
data
845845
}
846846

847-
Map autoPopulateSpeciesData(Map data){
848-
if (!data?.guid && data?.scientificName) {
849-
def result = speciesReMatchService.searchBie(data.scientificName, 10)
850-
// only if there is a single match
851-
if (result?.autoCompleteList?.size() == 1) {
852-
data.guid = result?.autoCompleteList[0]?.guid
853-
data.commonName = data.commonName ?: result?.autoCompleteList[0]?.commonName
847+
Map autoPopulateSpeciesData (Map data, int limit = 10) {
848+
String searchName = (data?.scientificName)?.trim()
849+
if (!data?.guid && (searchName)) {
850+
def result = speciesReMatchService.searchBie(searchName, limit)
851+
// find the name that exactly matches the search name
852+
def bestMatch = result?.autoCompleteList?.find {
853+
it.matchedNames?.findResult { String name ->
854+
name.equalsIgnoreCase(searchName)
855+
|| name.equalsIgnoreCase(data.name)
856+
|| name.equalsIgnoreCase(data.commonName)
857+
}
858+
}
859+
860+
if (bestMatch) {
861+
data.guid = bestMatch?.guid
862+
data.commonName = data.commonName ?: bestMatch?.commonName
863+
data.scientificName = data.scientificName ?: bestMatch?.name
854864
}
855865
}
856866

‎grails-app/services/au/org/ala/ecodata/OutputService.groovy

+12-4
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ package au.org.ala.ecodata
33
import au.org.ala.ecodata.converter.RecordConverter
44
import au.org.ala.ecodata.metadata.OutputMetadata
55

6-
import static au.org.ala.ecodata.Status.ACTIVE
76
import static au.org.ala.ecodata.Status.DELETED
87

98
class OutputService {
@@ -155,8 +154,12 @@ class OutputService {
155154
props.data = saveImages(props.data, props.name, output.outputId, props.activityId);
156155
props.data = saveAudio(props.data, props.name, output.outputId, props.activityId);
157156

158-
159-
createOrUpdateRecordsForOutput(activity, output, props)
157+
try {
158+
createOrUpdateRecordsForOutput(activity, output, props)
159+
}
160+
catch (Exception ex) {
161+
log.error("Error creating records for activity - ${activity.activityId}", ex)
162+
}
160163
commonService.updateProperties(output, props)
161164

162165
return [status: 'ok', outputId: output.outputId]
@@ -241,7 +244,12 @@ class OutputService {
241244

242245
List statusUpdate = recordService.updateRecordStatusByOutput(outputId, Status.DELETED)
243246
if (!statusUpdate) {
244-
createOrUpdateRecordsForOutput(activity, output, props)
247+
try {
248+
createOrUpdateRecordsForOutput(activity, output, props)
249+
}
250+
catch (Exception ex) {
251+
log.error("Error creating records for activity - ${activity.activityId}", ex)
252+
}
245253
commonService.updateProperties(output, props)
246254
result = [status: 'ok']
247255
} else {

0 commit comments

Comments
 (0)