Skip to content

Commit d86a97d

Browse files
charvolantcharvolant
authored
Merge pull request #239 from charvolant/issue-235
Allow strict name matching of lists
2 parents 4a51574 + cd555a0 commit d86a97d

File tree

15 files changed

+520
-316
lines changed

15 files changed

+520
-316
lines changed

‎build.gradle

+2-2
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@ dependencies {
112112
compile "org.nibor.autolink:autolink:0.5.0"
113113
compile "com.opencsv:opencsv:3.7"
114114

115-
runtime 'org.cache2k:cache2k-jcache:1.2.0.Final'
115+
runtime 'org.cache2k:cache2k-jcache:2.6.1.Final'
116116

117117
testCompile "io.micronaut:micronaut-inject-groovy"
118118
testCompile "org.grails:grails-gorm-testing-support"
@@ -136,7 +136,7 @@ dependencies {
136136
}
137137

138138
compile 'au.org.ala:ala-cas-client:3.0.0'
139-
compile ("au.org.ala.names:ala-namematching-client:1.7") {
139+
compile ('au.org.ala.names:ala-namematching-client:1.9-SNAPSHOT') {
140140
exclude group: "com.squareup.okhttp3", module: "okhttp"
141141
}
142142
compile 'au.org.ala.plugins:openapi:1.1.0'

‎grails-app/conf/application.yml

+2
Original file line numberDiff line numberDiff line change
@@ -206,6 +206,8 @@ outboundhttp:
206206

207207
namematching:
208208
serviceURL: https://namematching-ws.ala.org.au
209+
defaultLoose: false
210+
defaultStyle: STRICT
209211
dataCacheConfig:
210212
enableJmx: true
211213
entryCapacity: 20000

‎grails-app/controllers/au/org/ala/specieslist/EditorController.groovy

+9-2
Original file line numberDiff line numberDiff line change
@@ -150,12 +150,14 @@ class EditorController {
150150
// check for changed values
151151
def keys = SpeciesListKVP.executeQuery("select distinct key from SpeciesListKVP where dataResourceUid= :dataResourceUid", [dataResourceUid: sli.dataResourceUid])
152152
def kvpRemoveList = [] as Set
153+
def changed = false
153154

154155
keys.each { key ->
155156
def kvp = sli.kvpValues.find { it.key == key } // existing KVP if any
156157

157158
if (params[key] != kvp?.value) {
158159
log.debug "KVP has been changed: " + params[key] + " VS " + kvp?.value
160+
changed = true
159161
def newKvp = SpeciesListKVP.findByDataResourceUidAndKeyAndValue(sli.dataResourceUid, key, params[key])
160162

161163
if (kvp) {
@@ -189,14 +191,19 @@ class EditorController {
189191
sli.removeFromKvpValues(it)
190192
}
191193

192-
//check if rawScientificName has changed
194+
//check if name information has changed
193195
if (params.rawScientificName.trim() != sli.rawScientificName.trim()) {
194196
log.debug "rawScientificName is different: " + params.rawScientificName + " VS " + sli.rawScientificName
195197
sli.rawScientificName = params.rawScientificName
198+
changed = true
196199
// lookup guid
197-
helperService.matchNameToSpeciesListItem(sli.rawScientificName, sli)
200+
helperService.matchNameToSpeciesListItem(sli.rawScientificName, sli, sli.mylist)
198201
//sli.guid = helperService.findAcceptedLsidByScientificName(sli.rawScientificName)?: helperService.findAcceptedLsidByCommonName(sli.rawScientificName)
199202
}
203+
if (changed) {
204+
log.debug "re-matching name for ${params.rawScientificName}"
205+
helperService.matchNameToSpeciesListItem(sli.rawScientificName, sli, sli.mylist)
206+
}
200207

201208
if (!sli.validate()) {
202209
def message = "Could not update SpeciesListItem: ${sli.rawScientificName} - " + sli.errors.allErrors

‎grails-app/controllers/au/org/ala/specieslist/SpeciesListController.groovy

+22-10
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
package au.org.ala.specieslist
1616

1717
import au.org.ala.web.AuthService
18+
import au.org.ala.names.ws.api.SearchStyle
1819
import com.opencsv.CSVReader
1920
import grails.converters.JSON
2021
import grails.gorm.transactions.Transactional
@@ -30,6 +31,7 @@ class SpeciesListController {
3031
private static final String[] ACCEPTED_CONTENT_TYPES = ["text/plain", "text/csv"]
3132

3233
HelperService helperService
34+
ColumnMatchingService columnMatchingService
3335
AuthService authService
3436
BieService bieService
3537
BiocacheService biocacheService
@@ -164,6 +166,8 @@ class SpeciesListController {
164166
formParams.category,
165167
formParams.generalisation,
166168
formParams.sdsType,
169+
formParams.looseSearch== null || formParams.looseSearch.isEmpty() ? null : Boolean.parseBoolean(formParams.looseSearch),
170+
formParams.searchStyle == null || formParams.searchStyle.isEmpty() ? null : SearchStyle.valueOf(formParams.searchStyle),
167171
header.split(","),
168172
vocabs)
169173

@@ -472,6 +476,7 @@ class SpeciesListController {
472476
while (offset < totalRows) {
473477
List items
474478
List guidBatch = [], sliBatch = []
479+
Map<SpeciesList, List<SpeciesListItem>> batches = new HashMap<>()
475480
List<SpeciesListItem> searchBatch = new ArrayList<SpeciesListItem>()
476481
if (id) {
477482
items = SpeciesListItem.findAllByDataResourceUid(id, [max: BATCH_SIZE, offset: offset])
@@ -481,23 +486,30 @@ class SpeciesListController {
481486

482487
SpeciesListItem.withSession { session ->
483488
items.eachWithIndex { SpeciesListItem item, Integer i ->
489+
SpeciesList speciesList = item.mylist
490+
List<SpeciesListItem> batch = batches.get(speciesList)
491+
if (batch == null) {
492+
batch = new ArrayList<>();
493+
batches.put(speciesList, batch)
494+
}
484495
String rawName = item.rawScientificName
485-
log.debug i + ". Rematching: " + rawName
496+
log.debug i + ". Rematching: " + rawName + "/" + speciesList.dataResourceUid
486497
if (rawName && rawName.length() > 0) {
487-
searchBatch.add(item)
498+
batch.add(item)
488499
} else {
489500
item.guid = null
490501
if (!item.save(flush: true)) {
491502
log.error "Error saving item (" + rawName + "): " + item.errors()
492503
}
493504
}
494505
}
495-
496-
helperService.matchAll(searchBatch)
497-
searchBatch.each {SpeciesListItem item ->
498-
if (item.guid) {
499-
guidBatch.push(item.guid)
500-
sliBatch.push(item)
506+
batches.each { list, batch ->
507+
helperService.matchAll(batch, list)
508+
batch.each {SpeciesListItem item ->
509+
if (item.guid) {
510+
guidBatch.push(item.guid)
511+
sliBatch.push(item)
512+
}
501513
}
502514
}
503515

@@ -522,7 +534,7 @@ class SpeciesListController {
522534
private parseDataFromCSV(CSVReader csvReader, String separator) {
523535
def rawHeader = csvReader.readNext()
524536
log.debug(rawHeader.toList()?.toString())
525-
def parsedHeader = helperService.parseHeader(rawHeader) ?: helperService.parseData(rawHeader)
537+
def parsedHeader = columnMatchingService.parseHeader(rawHeader) ?: helperService.parseData(rawHeader)
526538
def processedHeader = parsedHeader.header
527539
log.debug(processedHeader?.toString())
528540
def dataRows = new ArrayList<String[]>()
@@ -531,7 +543,7 @@ class SpeciesListController {
531543
dataRows.add(helperService.parseRow(currentLine.toList()))
532544
currentLine = csvReader.readNext()
533545
}
534-
def nameColumns = helperService.speciesNameColumns + helperService.commonNameColumns
546+
def nameColumns = columnMatchingService.speciesNameMatcher.names + columnMatchingService.commonNameMatcher.names
535547
if (processedHeader.find {
536548
it == "scientific name" || it == "vernacular name" || it == "common name" || it == "ambiguous name"
537549
} && processedHeader.size() > 0) {

‎grails-app/controllers/au/org/ala/specieslist/WebServiceController.groovy

+13-4
Original file line numberDiff line numberDiff line change
@@ -282,12 +282,16 @@ class WebServiceController {
282282
dataResourceUid: sl.dataResourceUid,
283283
listName : sl.listName,
284284
dateCreated : sl.dateCreated,
285+
lastUpdated : sl.lastUpdated,
286+
lastUploaded : sl.lastUploaded,
285287
username : sl.username,
286288
fullName : sl.getFullName(),
287289
itemCount : sl.itemsCount,//SpeciesListItem.countByList(sl)
288290
isAuthoritative: (sl.isAuthoritative ?: false),
289291
isInvasive : (sl.isInvasive ?: false),
290-
isThreatened : (sl.isThreatened ?: false)
292+
isThreatened : (sl.isThreatened ?: false),
293+
looseSearch : sl.looseSearch,
294+
searchStyle : sl.searchStyle?.toString()
291295
]
292296
if (sl.listType) {
293297
retValue["listType"] = sl?.listType?.toString()
@@ -313,11 +317,13 @@ class WebServiceController {
313317
def listCounts = allLists.totalCount
314318
def retValue = [listCount: listCounts, sort: params.sort, order: params.order, max: params.max, offset: params.offset,
315319
lists : allLists.collect {
316-
[dataResourceUid: it.dataResourceUid,
320+
[
321+
dataResourceUid: it.dataResourceUid,
317322
listName : it.listName,
318323
listType : it?.listType?.toString(),
319324
dateCreated : it.dateCreated,
320325
lastUpdated : it.lastUpdated,
326+
lastUploaded : it.lastUploaded,
321327
username : it.username,
322328
fullName : it.getFullName(),
323329
itemCount : it.itemsCount,
@@ -328,9 +334,12 @@ class WebServiceController {
328334
sdsType : it.sdsType,
329335
isAuthoritative: it.isAuthoritative ?: false,
330336
isInvasive : it.isInvasive ?: false,
331-
isThreatened : it.isThreatened ?: false]
332-
}]
337+
isThreatened : it.isThreatened ?: false,
338+
looseSearch : it.looseSearch,
339+
searchStyle : it.searchStyle?.toString()
340+
]
333341

342+
}]
334343
render retValue as JSON
335344
}
336345
}

‎grails-app/domain/au/org/ala/specieslist/SpeciesList.groovy

+8-1
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@
1515

1616
package au.org.ala.specieslist
1717

18+
import au.org.ala.names.ws.api.SearchStyle
19+
1820
class SpeciesList {
1921
def authService
2022

@@ -29,6 +31,7 @@ class SpeciesList {
2931
String wkt
3032
Date dateCreated
3133
Date lastUpdated
34+
Date lastUploaded
3235
ListType listType
3336
Boolean isPrivate
3437
Boolean isSDS
@@ -42,8 +45,9 @@ class SpeciesList {
4245
String generalisation
4346
String category
4447
String sdsType
48+
Boolean looseSearch // if undefined use the server default
49+
SearchStyle searchStyle // if undefined use the server default
4550
String ownerFullName // derived by concatenating the firstName and surname fields
46-
4751
static transients = [ "fullName" ]
4852

4953
static hasMany = [items: SpeciesListItem, editors: String]
@@ -67,6 +71,9 @@ class SpeciesList {
6771
generalisation(nullable: true)
6872
authority(nullable: true)
6973
sdsType nullable: true
74+
looseSearch nullable: true
75+
searchStyle nullable: true
76+
lastUploaded nullable: true
7077
userId nullable: true
7178
ownerFullName nullable: true // derived
7279
}

‎grails-app/i18n/messages.properties

+6-1
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,9 @@ speciesList.authority.label=SDS Authority
9595
speciesList.category.label=SDS Category
9696
speciesList.generalisation.label=SDS Coordinate Generalisation
9797
speciesList.sdsType.label=SDS Type
98+
speciesList.looseSearch.label=Loose Name Search
99+
speciesList.searchStyle.label=Name Search Style
100+
speciesList.lastUploaded.label=Date last uploaded
98101

99102
upload.heading.hasList=Resubmit to an existing list
100103
upload.heading=Upload a list
@@ -371,7 +374,9 @@ public.lists.items.header01=Supplied Name
371374
public.lists.items.header02=Scientific Name (matched)
372375
public.lists.items.header03=Author (matched)
373376
public.lists.items.header04=Common Name (matched)
374-
public.lists.items.header05=Image
377+
public.lists.items.header05=Family (matched)
378+
public.lists.items.header06=Kingdom (matched)
379+
public.lists.items.header07=Image
375380
generic.lists.label.reload= - Page will be reloaded ...
376381
public.lists.items.edit.error=Required field: supplied name cannot be blank
377382

0 commit comments

Comments
 (0)