Bump 4.5.0

#279 Fixing missing supplied fields
#288 Track rematching process

Author: qifeng-bai

.travis.yml

@@ -7,8 +7,8 @@ branches:
   - master
   - develop
   - feature/grails5
-  - bug/issue-225
   - review-isprivate
+  - 279_missing_taxon
 
 before_install:
   - rm -fr $HOME/.gradle/caches

build.gradle

@@ -16,9 +16,7 @@ plugins {
     id "com.gorylenko.gradle-git-properties" version "2.4.1"
 }
 
-
-version "4.4.0"
-
+version "4.5.0"
 group "au.org.ala"
 
 apply plugin:"eclipse"
@@ -130,7 +128,7 @@ dependencies {
     implementation "org.grails.plugins:ala-auth:$alaSecurityLibsVersion"
     implementation "org.grails.plugins:ala-ws-security-plugin:$alaSecurityLibsVersion"
     implementation "org.grails.plugins:ala-ws-plugin:$alaSecurityLibsVersion"
-    runtimeOnly "org.grails.plugins:ala-bootstrap3:4.1.0"
+    runtimeOnly "org.grails.plugins:ala-bootstrap3:4.3.0"
     implementation "au.org.ala:userdetails-service-client:$alaSecurityLibsVersion"
     runtimeOnly ("org.grails.plugins:ala-admin-plugin:2.3.0") {
         exclude module: "cache"

changelogs.txt

@@ -0,0 +1,32 @@
+21 Sep 2023
+Qifeng.bai@csiro.au
+* database change:
+
+    Add a new table: matchedSpecies with fields: taxonConceptID, scientificName, scientificNameAuthorship, vernacularName, kingdom, phylum, taxonClass, taxonOrder, taxonRank, family, genus
+
+          CREATE TABLE `matched_species` (
+              `id` int NOT NULL AUTO_INCREMENT,
+              `taxon_concept_id` varchar(255) DEFAULT NULL,
+              `scientific_name` varchar(255) NOT NULL,
+              `scientific_name_authorship` varchar(255) DEFAULT NULL,
+              `vernacular_name` varchar(255) DEFAULT NULL,
+              `kingdom` varchar(255) DEFAULT NULL,
+              `phylum` varchar(255) DEFAULT NULL,
+              `taxon_class` varchar(255) DEFAULT NULL,
+              `taxon_order` varchar(255) DEFAULT NULL,
+              `family` varchar(255) DEFAULT NULL,
+              `genus` varchar(255) DEFAULT NULL,
+              `taxon_rank` varchar(255) DEFAULT NULL,
+              `version` BIGINT NOT NULL DEFAULT 0,
+              UNIQUE KEY `id_UNIQUE` (`id`)
+              )
+
+        Create an index for:
+        ALTER TABLE `specieslist`.`matched_species`
+          Add index `idx_concept_id` (`taxon_concept_id`);
+        ALTER TABLE `specieslist`.`matched_species`
+          Add index `idx_scientific_name` (`scientific_name`);
+    Link the new created matchedSpecies table to speciesListItem table:
+            ALTER TABLE `specieslist`.`species_list_item`
+            ADD COLUMN `matched_species_id` INT DEFAULT NULL
+

gradle.properties

@@ -4,4 +4,4 @@ groovyVersion=3.0.11
 org.gradle.daemon=true
 org.gradle.parallel=true
 org.gradle.jvmargs=-Dfile.encoding=UTF-8 -Xmx1024M
-alaSecurityLibsVersion=6.0.4
+alaSecurityLibsVersion=6.1.0

grails-app/conf/application.yml

@@ -261,3 +261,6 @@ genusColumns: 'genus,genericEpithet,generic epithet'
 rankColumns: 'taxonrank,rank,taxon rank,taxonomicrank,taxonomic rank,linnaean rank'
 #determnines whether a  list owner's email is visible on list info panel.
 ownerVisibleToEditor: false
+
+biocacheService:
+    baseURL: https://biocache-ws.ala.org.au/ws

grails-app/controllers/au/org/ala/specieslist/AdminController.groovy

@@ -57,11 +57,14 @@ class AdminController {
             // retrieve qualified SpeciesListItems for performance reason
             def itemsIds = queryService.getFilterSpeciesListItemsIds(params)
             def lists = queryService.getFilterListResult(params, false, itemsIds)
+            def rematchLogs = helperService.queryRematchingProcess()
             def model = [lists:lists,
                          total:lists.totalCount,
                          typeFacets: (params.listType) ? null : queryService.getTypeFacetCounts(params, false, itemsIds),
                          tagFacets: queryService.getTagFacetCounts(params, itemsIds),
-                         selectedFacets:queryService.getSelectedFacets(params)]
+                         selectedFacets:queryService.getSelectedFacets(params),
+                         rematchLogs: rematchLogs
+            ]
             if (searchTerm) {
                 params.q = searchTerm
                 model.errors = "Error: Search terms must contain at least 3 characters"

grails-app/controllers/au/org/ala/specieslist/EditorController.groovy

@@ -213,17 +213,14 @@ class EditorController {
             if (params.rawScientificName.trim() != sli.rawScientificName.trim()) {
                 log.debug "rawScientificName is different: " + params.rawScientificName + " VS " + sli.rawScientificName
                 sli.rawScientificName = params.rawScientificName
-                changed = true
                 // lookup guid
                 helperService.matchNameToSpeciesListItem(sli.rawScientificName, sli, sli.mylist)
                 //sli.guid = helperService.findAcceptedLsidByScientificName(sli.rawScientificName)?: helperService.findAcceptedLsidByCommonName(sli.rawScientificName)
-            }
-            if (changed) {
-                log.debug "re-matching name for ${params.rawScientificName}"
-                helperService.matchNameToSpeciesListItem(sli.rawScientificName, sli, sli.mylist)
+                helperService.getCommonNamesAndUpdateRecords([sli],[sli.guid])
                 sl.lastMatched = new Date()
             }
 
+
             if (!sli.validate()) {
                 def message = "Could not update SpeciesListItem: ${sli.rawScientificName} - " + sli.errors.allErrors
                 log.error message

grails-app/controllers/au/org/ala/specieslist/PublicController.groovy

@@ -39,7 +39,7 @@ class PublicController {
             searchTerm = params.q
             params.q = ""
         }
-        params.max = Math.min(params.max ? params.int('max') : 25, 1000)
+        params.max = Math.min((params.max ? params.int('max') : 25) ?: 25, 1000)
         params.sort = params.sort ?: "listName"
         if (params.isSDS){
             //to ensure backwards compatibility for a commonly used URL

grails-app/controllers/au/org/ala/specieslist/SpeciesListController.groovy

@@ -507,13 +507,24 @@ class SpeciesListController {
     /**
      * Rematches the scientific names in the supplied list
      */
-    @Transactional
     def rematch() {
+        long  beforeId = 0
         if (params.id && !params.id.startsWith("dr")) {
             params.id = SpeciesList.get(params.id)?.dataResourceUid
             log.info("Rematching for " + params.id)
         } else {
-            log.error("Rematching for ALL")
+            String msg = "Rematching for ALL"
+            if (params.beforeId) {
+                try {
+                    beforeId = Long.parseLong(params.beforeId)
+                    if ( beforeId > 0) {
+                        msg = "Continue to rematch the rest of species before id: " + beforeId
+                    }
+                }catch(Exception e){
+                }
+            }
+
+            log.warn(msg)
         }
         Integer totalRows, offset = 0;
         String id = params.id
@@ -523,66 +534,7 @@ class SpeciesListController {
             return
         }
 
-        if (id) {
-            totalRows = SpeciesListItem.countByDataResourceUid(id)
-        } else {
-            totalRows = SpeciesListItem.count();
-        }
-
-        while (offset < totalRows) {
-            List items
-            List guidBatch = [], sliBatch = []
-            Map<SpeciesList, List<SpeciesListItem>> batches = new HashMap<>()
-            List<SpeciesListItem> searchBatch = new ArrayList<SpeciesListItem>()
-            if (id) {
-                items = SpeciesListItem.findAllByDataResourceUid(id, [max: BATCH_SIZE, offset: offset])
-            } else {
-                items = SpeciesListItem.list(max: BATCH_SIZE, offset: offset)
-            }
-
-            SpeciesListItem.withSession { session ->
-                items.eachWithIndex { SpeciesListItem item, Integer i ->
-                    SpeciesList speciesList = item.mylist
-                    List<SpeciesListItem> batch = batches.get(speciesList)
-                    if (batch == null) {
-                        batch = new ArrayList<>();
-                        batches.put(speciesList, batch)
-                    }
-                    String rawName = item.rawScientificName
-                    log.debug i + ". Rematching: " + rawName + "/" + speciesList.dataResourceUid
-                    if (rawName && rawName.length() > 0) {
-                        batch.add(item)
-                    } else {
-                        item.guid = null
-                        if (!item.save(flush: true)) {
-                            log.error "Error saving item (" + rawName + "): " + item.errors()
-                        }
-                    }
-                }
-                batches.each { list, batch ->
-                    helperService.matchAll(batch, list)
-                    batch.each {SpeciesListItem item ->
-                        if (item.guid) {
-                            guidBatch.push(item.guid)
-                            sliBatch.push(item)
-                        }
-                    }
-                }
-
-                if (!guidBatch.isEmpty()) {
-                    helperService.getCommonNamesAndUpdateRecords(sliBatch, guidBatch)
-                }
-
-                session.flush()
-                session.clear()
-            }
-
-            offset += BATCH_SIZE;
-            log.info("Rematched ${offset} of ${totalRows} - ${Math.round(offset * 100 / totalRows)}% complete")
-            if (offset > totalRows) {
-                log.error("Rematched ${offset} of ${totalRows} - ${Math.round(offset * 100 / totalRows)}% complete")
-            }
-        }
+        helperService.rematch(id,beforeId)
 
         render(text: "${message(code: 'admin.lists.page.button.rematch.messages', default: 'Rematch complete')}")
     }

grails-app/controllers/au/org/ala/specieslist/SpeciesListItemController.groovy

@@ -18,6 +18,8 @@ package au.org.ala.specieslist
 
 import com.opencsv.CSVWriter
 import grails.converters.JSON
+import groovy.json.JsonBuilder
+import groovy.json.JsonOutput
 
 class SpeciesListItemController {
 
@@ -110,7 +112,7 @@ class SpeciesListItemController {
                 } else {
                     if (requestParams.message)
                         flash.message = requestParams.message
-                    requestParams.max = Math.min(requestParams.max ? requestParams.int('max') : 10, 100)
+                    requestParams.max = Math.min(requestParams.max ? requestParams.int('max') : 100, 1000)
                     requestParams.sort = requestParams.sort ?: "itemOrder"
                     requestParams.offset = requestParams.int('offset') ?: 0
                     requestParams.fetch = [kvpValues: 'select']
@@ -133,10 +135,13 @@ class SpeciesListItemController {
 
                     log.debug("Checking speciesList: " + speciesList)
                     log.debug("Checking editors: " + speciesList.editors)
+                    def speciesListItems = queryService.getSpeciesListItemsByParams(requestParams, baseQueryAndParamsForListingSLI)
+
+
                     render(view: 'list', model: [
                             speciesList: speciesList,
                             params: requestParams,
-                            results: queryService.getSpeciesListItemsByParams(requestParams, baseQueryAndParamsForListingSLI),
+                            results: speciesListItems,
                             totalCount: queryService.getTotalCountByParams(requestParams, baseQueryAndParams),
                             noMatchCount: noMatchCount,
                             distinctCount: queryService.getDistinctCountByParams(requestParams, baseQueryAndParams),

grails-app/controllers/au/org/ala/specieslist/UrlMappings.groovy

@@ -37,11 +37,16 @@ class UrlMappings {
             action = [GET:'getListItemDetails']
         }
 
+        "/ws/createItem" (controller: 'webService', action: 'createItem')
+
+        "/ws/deleteItem" (controller: 'webService', action: 'deleteItem')
+
         "/ws/queryListItemOrKVP" (controller: 'webService'){
             action = [GET:'queryListItemOrKVP']
         }
 
         //"/ws/speciesListItems" (controller: "webService", action: "getListItemDetails")
+        "/ws/findSpeciesByName" (controller: "webService", action: "findSpeciesByName")
         "/ws/speciesListItems/keys" (controller: "webService", action: "listKeys")
         "/ws/speciesListItems/byKeys" (controller: "webService", action: "listItemsByKeys")
 
@@ -66,6 +71,7 @@ class UrlMappings {
             action = [GET: 'getSpeciesListItemKvp']
         }
 
+        "/ws/rematchStatus" (controller: "webService", action: "rematchStatus")
 		"/"(controller: 'public' ,action:  'index')
 		"500"(view:'/error')
         "404"(view:'/404')