Release 4.4.0

#273 fix deletion of a list which has a large mount of KVP records
fix bugs on params and optimise the list-item query

Author: qifeng-bai

.gitignore

@@ -6,3 +6,12 @@ out
 /.slcache
 /logs
 *.iml
+grails-app/.DS_Store
+grails-app/controllers/.DS_Store
+grails-app/controllers/au/.DS_Store
+grails-app/controllers/au/org/.DS_Store
+grails-app/controllers/au/org/ala/.DS_Store
+grails-app/services/.DS_Store
+grails-app/services/au/.DS_Store
+grails-app/services/au/org/.DS_Store
+grails-app/services/au/org/ala/.DS_Store

build.gradle

@@ -16,7 +16,9 @@ plugins {
     id "com.gorylenko.gradle-git-properties" version "2.4.1"
 }
 
-version "4.1.3"
+
+version "4.4.0"
+
 group "au.org.ala"
 
 apply plugin:"eclipse"

grails-app/conf/application.yml

@@ -1,4 +1,8 @@
 ---
+
+logging:
+    config: /data/specieslist-webapp/config/logback.xml
+
 grails:
     profile: web
     codegen:
@@ -135,7 +139,7 @@ dataSource:
 environments:
     development:
         grails:
-            serverURL: "http://dev.ala.org.au:8080"
+            serverURL: "http://localhost:8080"
         dataSource:
             dbCreate: none
             url: jdbc:mysql://localhost/specieslist?autoReconnect=true&connectTimeout=0&useUnicode=true&characterEncoding=UTF-8

grails-app/conf/logback.xml

@@ -3,6 +3,9 @@
 
     <conversionRule conversionWord="clr" converterClass="org.springframework.boot.logging.logback.ColorConverter" />
     <conversionRule conversionWord="wex" converterClass="org.springframework.boot.logging.logback.WhitespaceThrowableProxyConverter" />
+    <logger name="org.hibernate" level="ERROR"/>
+    <logger name="org.grails.config" level="ERROR"/>
+
 
     <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
         <encoder>
@@ -11,7 +14,7 @@
         </encoder>
     </appender>
 
-    <root level="warn">
+    <root level="INFO">
         <appender-ref ref="STDOUT" />
     </root>
 </configuration>

grails-app/controllers/au/org/ala/specieslist/SpeciesListController.groovy

@@ -83,7 +83,9 @@ class SpeciesListController {
         def sl = SpeciesList.get(params.id)
         if(sl){
             helperService.deleteDataResourceForList(sl.dataResourceUid)
-            sl.delete(flush: true)
+            SpeciesListItem.executeUpdate("delete from SpeciesListItem where dataResourceUid = :dataResourceUid", ["dataResourceUid": sl.dataResourceUid])
+            SpeciesListKVP.executeUpdate("delete from SpeciesListKVP where dataResourceUid = :dataResourceUid", ["dataResourceUid": sl.dataResourceUid])
+            SpeciesList.executeUpdate("delete from SpeciesList where dataResourceUid = :dataResourceUid", ["dataResourceUid": sl.dataResourceUid])
         }
         redirect(action: 'list')
     }

grails-app/controllers/au/org/ala/specieslist/WebServiceController.groovy

@@ -187,42 +187,35 @@ class WebServiceController {
     def getListItemsForSpecies() {
         def guid = params.guid.replaceFirst("https:/", "https://")
         def lists = params.dr?.split(",")
-        def isBIE = params.boolean('isBIE')
-        def props = [fetch: [kvpValues: 'join', mylist: 'join']]
-
-        def results = queryService.getFilterListItemResult(props, params, guid, lists, null)
+        def queryParams = params.subMap(["max","sort","offset"])
 
+        def results = queryService.getListForSpecies(guid, params.isBIE, lists, queryParams )
         log.debug("RESULTS: " + results)
 
         // fetch lists that this user has access to view
         def hidePrivateLists = grailsApplication.config.getProperty('publicview.hidePrivateLists', Boolean, false)
         def permittedPrivateLists = queryService.visibleLists(false, hidePrivateLists)
 
-        def filteredRecords = results.findAll { !it.mylist.isPrivate || permittedPrivateLists.contains(it.dataResourceUid) }
-
-        if (isBIE) {
-            // BIE only want lists with isBIE == true
-            filteredRecords = filteredRecords.findAll { it.mylist.isBIE }
-        }
-
-        def listOfRecordMaps = filteredRecords.collect { li -> // don't output private lists
+        def listOfRecordMaps = results.findResults {
+            // don't output private lists
+            (!it.mylist.isPrivate) || permittedPrivateLists.contains(it.dataResourceUid) ?
             [
-                dataResourceUid: li.dataResourceUid,
-                guid           : li.guid,
+                dataResourceUid: it.dataResourceUid,
+                guid           : it.guid,
                 list           : [
-                    username: li.mylist.username,
-                    listName: li.mylist.listName,
-                    sds     : li.mylist.isSDS ?: false,
-                    isBIE   : li.mylist.isBIE ?: false
+                    username: it.mylist.username,
+                    listName: it.mylist.listName,
+                    sds     : it.mylist.isSDS ?: false,
+                    isBIE   : it.mylist.isBIE ?: false
                 ],
-                kvpValues      : li.kvpValues.collect { kvp ->
+                kvpValues      : it.kvpValues.collect { kvp ->
                     [
                         key       : kvp.key,
                         value     : kvp.value,
                         vocabValue: kvp.vocabValue
                     ]
                 }
-            ]
+            ] : null
         }
         render listOfRecordMaps as JSON
     }
@@ -1337,5 +1330,10 @@ class WebServiceController {
         return isAllowed
     }
 
+    def handleException(final Exception e ) {
+        log.error(e.message)
+        return {error: e.message}
+    }
+
 
 }

grails-app/services/au/org/ala/specieslist/HelperService.groovy

@@ -28,6 +28,7 @@ import org.grails.web.json.JSONArray
 import org.nibor.autolink.*
 import org.springframework.context.MessageSource
 import org.springframework.context.i18n.LocaleContextHolder
+import org.apache.http.util.EntityUtils
 
 import javax.annotation.PostConstruct
 
@@ -112,7 +113,8 @@ class HelperService {
                     requestContentType = ContentType.JSON
                     headers."Authorization" = "${grailsApplication.config.registryApiKey}"
                     response.success = { resp ->
-                        log.info(resp?.toString())
+                        def result = (resp.getEntity() != null ? EntityUtils.toString(resp.getEntity()) : "")
+                        log.info("${drId} has been deleted from ${grailsApplication.config.collectory.baseURL} with ${result}")
                     }
                     response.failure = { resp ->
                         log.error("Delete request for ${drId} failed with status ${resp.status}")

grails-app/services/au/org/ala/specieslist/QueryService.groovy

@@ -15,7 +15,7 @@
 
 package au.org.ala.specieslist
 
-
+import groovy.time.*
 import org.hibernate.Criteria
 import org.hibernate.criterion.CriteriaQuery
 import org.hibernate.criterion.Order
@@ -180,9 +180,11 @@ class QueryService {
             if (query){
                 selectedFacets << [query: key, facet: query]
             } else if (key == LIST_TYPE){
-                def cleanedVaue = value.replaceAll("eq:", "")
-                query = listTyoeFacets.get(cleanedVaue)
-                selectedFacets << [query: key, facet: query]
+                if (value) {
+                    def cleanedValue = value.replaceAll("eq:", "")
+                    query = listTyoeFacets.get(cleanedValue)
+                    selectedFacets << [query: key, facet: query]
+                }
             } else if (key == WKT){
                 query = WKT_QUERY
                 selectedFacets << [query: WKT, facet: [label:'spatialBounds.list.label']]
@@ -445,8 +447,38 @@ class QueryService {
         order = order ?: ASC
         c.order(new Order(sort, ASC.equalsIgnoreCase(order)))
     }
+    /*
+    * Retrieves the species list items by given guid.
+    *
+    * @param queryParams : only supports: max, offset, sort
+    * @param guid
+    * @param isBIE
+    * @param lists: data resource ids
+    * @return
+    */
+    def getListForSpecies(guid, isBIE, lists, queryParams ) {
+        def speciesListProperties = getSpeciesListProperties()
+        def c = SpeciesListItem.createCriteria()
+
+        def results = c.list(queryParams) {
+            eq(GUID, guid)
+            if (isBIE) {
+                mylist {
+                    eq("isBIE", isBIE.toBoolean())
+                }
+            }
+            if (lists) {
+                'in'(DATA_RESOURCE_UID, lists)
+            }
+        }
+        return results
+    }
 
     /**
+     * @Todo
+     * Fix bug when param: max is set
+     * @See issue #271
+     *
      * retrieves the species list items that obey the supplied filters.
      *
      * When a distinct field is provided the values of the field are returned rather than a SpeciesListItem
@@ -461,6 +493,9 @@ class QueryService {
         def speciesListProperties = getSpeciesListProperties()
         def c = SpeciesListItem.createCriteria()
 
+        //params – pagination parameters (max, offset, etc...) closure – The closure to execute
+        //sort, max, offset
+        // max parameter does not work, @see #271
         c.list(props += params) {
             //set the results transformer so that we don't get duplicate records because of
             // the 1:many relationship between a list item and KVP
@@ -513,6 +548,7 @@ class QueryService {
                     }
                 }
             }
+
         }
     }
 
@@ -793,12 +829,14 @@ class QueryService {
                 queryParameters.qCommonName = '%'+q+'%'
                 queryParameters.qRawScientificName = '%'+q+'%'
             }
-
+            def timeStart = new Date()
             def results = SpeciesListItem.executeQuery("select kvp.key, kvp.value, kvp.vocabValue, count(sli) as cnt from SpeciesListItem as sli " +
                     "join sli.kvpValues  as kvp where sli.dataResourceUid = :druid ${ids ? 'and sli.id in (:ids)' : ''} " +
                     "${q ? 'and (sli.matchedName like :qMatchedName or sli.commonName like :qCommonName or sli.rawScientificName like :qRawScientificName) ' : ''} " +
                     "group by kvp.key, kvp.value, kvp.vocabValue, kvp.itemOrder, kvp.key order by kvp.itemOrder, kvp.key, cnt desc",
                     queryParameters)
+            def timeStop = new Date()
+            log.info("Query KVP of " + fqs + "took " + TimeCategory.minus(timeStop, timeStart))
 
             //obtain the families from the common list facets
             def commonResults = SpeciesListItem.executeQuery("select sli.family, count(sli) as cnt from SpeciesListItem sli " +
@@ -816,15 +854,15 @@ class QueryService {
         } else {
             def qParam = '%'+q+'%'
             def queryParameters = q ? [dataResourceUid: id, matchedName: qParam, commonName: qParam, rawScientificName: qParam] : [dataResourceUid: id]
-
+            def timeStart = new Date()
             def results = SpeciesListItem.executeQuery('select kvp.key, kvp.value, kvp.vocabValue, count(sli) as cnt from SpeciesListItem as sli ' +
                     'join sli.kvpValues as kvp where sli.dataResourceUid = :dataResourceUid ' +
                     "${q ? 'and (sli.matchedName like :matchedName or sli.commonName like :commonName or sli.rawScientificName like :rawScientificName) ' : ''} " +
                     'group by kvp.key, kvp.value, kvp.vocabValue, kvp.itemOrder order by kvp.itemOrder, kvp.key, cnt desc',
                     queryParameters)
-
+            def timeStop = new Date()
+            log.info("Query KVP of " + id + "took " + TimeCategory.minus(timeStop, timeStart))
             properties = results.findAll{it[1].length()<maxLengthForFacet}.groupBy{it[0]}.findAll{it.value.size()>1 }
-
             //obtain the families from the common list facets
             def commonResults = SpeciesListItem.executeQuery('select family, count(*) as cnt from SpeciesListItem ' +
                     'where family is not null AND dataResourceUid = :dataResourceUid ' +

grails-app/views/_speciesList.gsp

@@ -48,8 +48,6 @@
                     var url = "${request.contextPath}"+"/speciesList/"+action+ "/"+listId;
                     //console.log("Dialog ACTION ITEMS",listId, url)
                     $.post(url, function(data){
-                        //alert('Value returned from service: '  + data.uid);
-                        alert('${message(code:'admin.lists.actions.button.message.ok', default:'Action was successful!')}');
                         window.location.reload()
                     }).error(function(jqXHR, textStatus, error) {
                         alert("An error occurred: " + error + " - " + jqXHR.responseText);

grails-app/views/speciesListItem/list.gsp

@@ -225,7 +225,7 @@
                 var url = "${request.contextPath}"+"/speciesList/delete/" + listId;
 
                 $.post(url, function(data){
-                    window.location.reload()
+                    window.location.replace("${request.contextPath}"+"/speciesList/list");
                 }).error(function(jqXHR, textStatus, error) {
                     alert("An error occurred: " + error + " - " + jqXHR.responseText);
                     $(modal).modal('hide');