Merge pull request #341 from AtlasOfLivingAustralia/259_email_template_design

259 email template design

Author: qifeng-bai

grails-app/conf/application.yml

@@ -179,7 +179,8 @@ userdetails:
 useSpeciesListsAlerts: true
 useSpatialAlerts: true
 useBlogsAlerts: true
-useCitizenScienceAlerts: true
+useCitizenScienceAlerts: false
+
 biosecurity:
     cronExpression: '0 0 11 ? * THU'
     query:

grails-app/controllers/au/org/ala/alerts/AdminController.groovy

@@ -741,6 +741,27 @@ class AdminController {
         }
     }
 
+    /**
+     * Reset the previous / current results stored in QueryResult
+     * @return
+     */
+    @AlaSecured(value = ['ROLE_ADMIN'], anyRole = true)
+    @Transactional
+    def resetQueryResult() {
+        try{
+            def id = params.id.toInteger()
+            if(id){
+                queryResultService.reset(id)
+                render([status: 0, message: "Query result has been reset"] as JSON)
+            } else {
+                render([status: 1, message: "Missing ID"] as JSON)
+            }
+        } catch (Exception e) {
+            render([status: 1, message: "Error in resetting query result: ${e.message}"] as JSON)
+        }
+    }
+
+
     @AlaSecured(value = ['ROLE_ADMIN', 'ROLE_BIOSECURITY_ADMIN'], anyRole = true, redirectController = 'notification', redirectAction = 'myAlerts', message = "You don't have permission to view that page.")
     def listBiosecurityAuditCSV() {
         def result = [:]

grails-app/controllers/au/org/ala/alerts/WebserviceController.groovy

@@ -39,6 +39,7 @@ import static io.swagger.v3.oas.annotations.enums.ParameterIn.HEADER
 class WebserviceController {
 
     def queryService
+    def queryResultService
     def userService
     def notificationService
     def biosecurityService
@@ -364,6 +365,7 @@ class WebserviceController {
         redirectIfSupplied(params)
     }
 
+
     @Operation(
             method = "POST",
             tags = "alerts",

grails-app/domain/au/org/ala/alerts/QueryResult.groovy

@@ -1,7 +1,5 @@
 package au.org.ala.alerts
 
-import com.jayway.jsonpath.JsonPath
-
 import java.text.SimpleDateFormat
 import java.util.zip.GZIPInputStream
 import java.util.zip.GZIPOutputStream
@@ -75,7 +73,7 @@ class QueryResult {
     }
 
     String toString() {
-        "${frequency?.name}: ${lastChanged ? 'Changed' : 'No Change'} on ${lastChecked}"
+        "${frequency?.name}: ${query?.name} : ${id} ${lastChanged ? 'Changed' : 'No Change'} on ${lastChecked}"
     }
 
     Map brief() {
@@ -129,7 +127,6 @@ class QueryResult {
     }
 
     byte[] compress(String json) {
-        //store the last result from the webservice call
         if (json) {
             ByteArrayOutputStream bout = new ByteArrayOutputStream()
             GZIPOutputStream gzout = new GZIPOutputStream(bout)

grails-app/init/au/org/ala/alerts/BootStrap.groovy

@@ -4,7 +4,6 @@ import ala.postie.BiosecurityQueriesJob
 
 class BootStrap {
 
-    javax.sql.DataSource dataSource
     def grailsApplication
     def messageSource
     def siteLocale
@@ -119,7 +118,7 @@ class BootStrap {
 
         title = messageSource.getMessage("query.citizen.records.title", null, siteLocale)
         descr = messageSource.getMessage("query.citizen.records.descr", null, siteLocale)
-        if (grailsApplication.config.useCitizenScienceAlerts.toBoolean() &&
+        if (grailsApplication.config.useCitizenScienceAlerts?.toBoolean() &&
                 Query.findAllByName(title).isEmpty()) {
             Query newCitizenScienceRecords = (new Query([
                     baseUrl: grailsApplication.config.biocacheService.baseURL,
@@ -142,7 +141,7 @@ class BootStrap {
 
         title = messageSource.getMessage("query.citizen.records.imgs.title", null, siteLocale)
         descr = messageSource.getMessage("query.citizen.records.imgs.descr", null, siteLocale)
-        if (grailsApplication.config.useCitizenScienceAlerts.toBoolean() &&
+        if (grailsApplication.config.useCitizenScienceAlerts?.toBoolean() &&
                 Query.findAllByName(title).isEmpty()) {
             Query newCitizenScienceRecordsWithImages = (new Query([
                     baseUrl: grailsApplication.config.biocacheService.baseURL,

grails-app/jobs/ala/postie/HourlyQueriesJob.groovy

@@ -1,5 +1,8 @@
 package ala.postie
 
+import grails.plugins.schwartz.monitor.listener.QuartzJobListener
+import org.quartz.JobExecutionContext
+
 class HourlyQueriesJob {
 
     static triggers = {
@@ -11,6 +14,6 @@ class HourlyQueriesJob {
     def execute() {
         log.info("****** Starting hourly update ****** " + new Date())
         notificationService.execQueryForFrequency('hourly')
-        log.info("****** Finished hourly update ******" + new Date())
+        log.info("****** Finished hourly update ****** " + new Date())
     }
 }

grails-app/services/au/org/ala/alerts/AnnotationService.groovy

@@ -17,10 +17,9 @@ package au.org.ala.alerts
 
 import com.jayway.jsonpath.JsonPath
 import com.jayway.jsonpath.PathNotFoundException
-import grails.converters.JSON
 import org.grails.web.json.JSONArray
 import org.grails.web.json.JSONObject
-
+import groovy.json.JsonOutput
 import java.text.SimpleDateFormat
 
 /**
@@ -67,11 +66,10 @@ class AnnotationService {
     }
 
     /**
-     * If fireNotZero property is true, diff method will not be called
+     * If fireNotZero property is true, this method will not be called
      *
-     * for normal alerts, comparing occurrence uuid is enough to show the difference.
-     * for my annotation alerts, same occurrence record could exist in both result but have different assertions.
-     * so comparing occurrence uuid is not enough, we need to compare 50001/50002/50003 sections inside each occurrence record
+     * This method compare the difference of Annotations between previous and last result.
+     * for annotation query, it returns the records that have new annotations after the given DATE. So it does not require to call this method.
 
      * return a list of records that their annotations have been changed or deleted
      * @param String previous
@@ -99,9 +97,9 @@ class AnnotationService {
             def record = it.value
             def previousRecord = oldRecordsMap.get(record.uuid)
             if (previousRecord) {
-                String currentAssertions = JSON.stringify(filterAssertions(record.user_assertions))
-                String previousAssertions = JSON.stringify(filterAssertions(previousRecord.user_assertions))
-                currentAssertions || previousAssertions
+                String currentAssertions = JsonOutput.toJson(record.user_assertions)
+                String previousAssertions = JsonOutput.toJson(previousRecord.user_assertions)
+                currentAssertions != previousAssertions
             } else {
                 true
             }

grails-app/services/au/org/ala/alerts/DiffService.groovy

@@ -68,21 +68,23 @@ class DiffService {
         // The following check is determined by the last propertyValue, since it overwrites the previous one
 
         queryResult.propertyValues.each { pv ->
-            log.debug("[QUERY " + queryResult.query.id + "] " +
-                    " Has changed check:" + pv.propertyPath.name
-                    + ", value:" + pv.currentValue
-                    + ", previous:" + pv.previousValue
-                    + ", fireWhenNotZero:" + pv.propertyPath.fireWhenNotZero
-                    + ", fireWhenChange:" + pv.propertyPath.fireWhenChange
-            )
-
-            // Two different types of queries: Biocache and Blog/News
-            // Biocache: totalRecords and last_loaded_records
-            // Blog/News: last_blog_id
-            if (pv.propertyPath.fireWhenNotZero) {
-                changed = pv.currentValue?.toInteger() ?: 0 > 0
-            } else if (pv.propertyPath.fireWhenChange) {
-                changed = pv.previousValue != pv.currentValue
+            if (pv) {
+                log.debug("[QUERY " + queryResult.query.id + "] " +
+                        " Has changed check:" + pv.propertyPath.name
+                        + ", value:" + pv.currentValue
+                        + ", previous:" + pv.previousValue
+                        + ", fireWhenNotZero:" + pv.propertyPath.fireWhenNotZero
+                        + ", fireWhenChange:" + pv.propertyPath.fireWhenChange
+                )
+
+                // Two different types of queries: Biocache and Blog/News
+                // Biocache: totalRecords and last_loaded_records
+                // Blog/News: last_blog_id
+                if (pv.propertyPath.fireWhenNotZero) {
+                    changed = pv.currentValue?.toInteger() ?: 0 > 0
+                } else if (pv?.propertyPath.fireWhenChange) {
+                    changed = pv.previousValue != pv.currentValue
+                }
             }
         }
 

grails-app/services/au/org/ala/alerts/ImageService.groovy

@@ -33,10 +33,12 @@ class ImageService {
         def dataResourceInfos = collectionService.findResourceByUids(collectionUrl, drIds)
         drIds.each { id ->
             def dataResourceInfo = dataResourceInfos[id]
-            groupedByDataResource[id].each { record ->
-                record["dataResourceInfo"] = [:]
-                record["dataResourceInfo"].lastUpdated = dataResourceInfo.lastUpdated
-                record["dataResourceInfo"].alaPublicUrl = dataResourceInfo.alaPublicUrl
+            if (dataResourceInfo) {
+                groupedByDataResource[id].each { record ->
+                    record["dataResourceInfo"] = [:]
+                    record["dataResourceInfo"].lastUpdated = dataResourceInfo.lastUpdated
+                    record["dataResourceInfo"].alaPublicUrl = dataResourceInfo.alaPublicUrl
+                }
             }
         }
         return groupedByDataResource

grails-app/services/au/org/ala/alerts/MyAnnotationService.groovy

@@ -46,15 +46,26 @@ class MyAnnotationService{
         }catch (PathNotFoundException e){
             log.warn("Current result is empty or doesn't have any records containing a field ${recordJsonPath} defined in recordJsonPath")
         }
-        // if an occurrence record doesn't exist in previous result (added) or has different open_assertions or verified_assertions or corrected_assertions than previous (changed).
+        // if an occurrence record doesn't exist in previous result (added) or has different verified_assertions than previous (changed).
         def records = curRecordsMap.findAll {
             def record = it.value
-            !oldRecordsMap.containsKey(record.uuid) ||
-                    record.open_assertions != oldRecordsMap.get(record.uuid).open_assertions ||
-                    record.verified_assertions != oldRecordsMap.get(record.uuid).verified_assertions ||
-                    record.corrected_assertions != oldRecordsMap.get(record.uuid).corrected_assertions
-        }.values()
+            // if the record has verified assertions and it is a list - some legacy records may have a String instead of a list
+            if (!record.verified_assertions?.isEmpty() && record.verified_assertions instanceof List) {
+                if (oldRecordsMap.containsKey(record.uuid)) {
+                    if (oldRecordsMap.get(record.uuid).verified_assertions?.isEmpty()) {
+                        return true
+                    } else {
+                        return record.verified_assertions.collect { it.uuid }.join(',') != oldRecordsMap.get(record.uuid).verified_assertions.collect { it.uuid }.join(',')
+                    }
+                } else {
+                    return true
+                }
+            } else {
+                //No verified assertions in current records
+                return false
+            }
 
+        }.values()
 
         //if an occurrence record exists in previous result but not in current, it means the annotation is deleted.
         //We need to add these records as a 'modified' record
@@ -88,28 +99,14 @@ class MyAnnotationService{
                         JSONArray assertions = assertionsData.json as JSONArray
                         occurrence.put('user_assertions', assertions)
 
-                        def (origUserAssertions, openAssertions, verifiedAssertions, correctedAssertions) = filterMyAssertions(assertions, userId)
-                        // only include record has at least 1 (50001/50002/50003) assertion
+                        def verifiedAssertions = findVerifiedAssertions(assertions, userId)
+                        // only include record has at least 1 (50001/50002/50003) assertions that VERIFY the user's assertions
                         // They will be used for diffService (records that will be included in alert email)
-                        if (!openAssertions.isEmpty() || !verifiedAssertions.isEmpty() || !correctedAssertions.isEmpty()) {
-
-                            // find the open/verfied/corrected annotations which COMMENTed on all the assertions created by the users
-                            def processedAssertionIds = openAssertions.collect { it.uuid } + verifiedAssertions.collect { it.uuid } + correctedAssertions.collect { it.uuid }
-                            def processedAssertions = assertions.findAll{
-                                processedAssertionIds.contains(it.relatedUuid)
-                            }
-                            occurrence.put('processed_assertions', processedAssertions)
-
-                            // Those open/verified/corrected assertions will be used to retrieve diff (records that will be included in alert email)
-                            openAssertions.sort { it.uuid }
-                            verifiedAssertions.sort { it.uuid }
-                            correctedAssertions.sort { it.uuid }
-                            occurrence.put('open_assertions', openAssertions.collect { it.uuid }.join(','))
-                            occurrence.put('verified_assertions', verifiedAssertions.collect { it.uuid }.join(','))
-                            occurrence.put('corrected_assertions', correctedAssertions.collect { it.uuid }.join(','))
+                        if ( !verifiedAssertions.isEmpty()) {
+                            occurrence.put('verified_assertions', verifiedAssertions)
+                            reconstructedOccurrences.push(occurrence)
                         }
                     }
-                    reconstructedOccurrences.push(occurrence)
                 }
             }
             reconstructedOccurrences.sort { it.uuid }
@@ -123,34 +120,30 @@ class MyAnnotationService{
 
     /**
      * Search the assertions which the USER has made
-     * return those have been open-issued, verified or corrected
+     * return assertions which verify or comment the user's assertions
      * @param assertions
      * @param userId
      * @return
      */
-
-    private static def filterMyAssertions(JSONArray assertions, String userId) {
-        def origUserAssertions = []
-        def openAssertions = []
-        def verifiedAssertions = []
-        def correctedAssertions = []
+    private static def findVerifiedAssertions(JSONArray assertions, String userId) {
+        def sortedAssertions = []
         if (assertions) {
             // all the original user assertions (issues users flagged)
-            origUserAssertions = assertions.findAll { it.uuid && !it.relatedUuid && it.userId == userId }
-
-            // all the 50001 (open issue) assertions (could belong to userId or other users)
-            def openIssueIds = assertions.findAll { it.uuid && it.relatedUuid && it.code == 50000 && it.qaStatus == 50001 }.collect { it.relatedUuid }
-
-            // all the 50002 (verified) assertions (could belong to userId or other users)
-            def verifiedIds = assertions.findAll { it.uuid && it.relatedUuid && it.code == 50000 && it.qaStatus == 50002 }.collect { it.relatedUuid }
-
-            // all the 50003 (corrected) assertions (could belong to userId or other users)
-            def correctedIds = assertions.findAll { it.uuid && it.relatedUuid && it.code == 50000 && it.qaStatus == 50003 }.collect { it.relatedUuid }
+            def origUserAssertions = assertions.findAll { it.uuid && !it.relatedUuid && it.userId == userId }
+            // Find assertions which commented on the original user assertions
+            def myOriginalUuids = origUserAssertions*.uuid
+            def verifiedAssertions = myOriginalUuids.collectMany { uuid ->
+                assertions.findAll { it.relatedUuid == uuid && ( it.qaStatus == 50001 || it.qaStatus == 50002 || it.qaStatus == 50003) }
+            }
 
-            openAssertions = origUserAssertions.findAll { openIssueIds.contains(it.uuid) }
-            verifiedAssertions = origUserAssertions.findAll { verifiedIds.contains(it.uuid) }
-            correctedAssertions = origUserAssertions.findAll { correctedIds.contains(it.uuid) }
+            try {
+                sortedAssertions = verifiedAssertions.sort { a, b ->
+                    Date.parse("yyyy-MM-dd'T'HH:mm:ssX", b.created) <=> Date.parse("yyyy-MM-dd'T'HH:mm:ssX", a.created)
+                }
+            } catch (Exception e) {
+                sortedAssertions = verifiedAssertions
+            }
         }
-        return [origUserAssertions, openAssertions, verifiedAssertions, correctedAssertions,]
+        return  sortedAssertions
     }
 }

grails-app/services/au/org/ala/alerts/NotificationService.groovy

@@ -3,10 +3,8 @@ package au.org.ala.alerts
 import com.jayway.jsonpath.JsonPath
 import grails.gorm.transactions.NotTransactional
 import grails.converters.JSON
-import org.apache.commons.io.IOUtils
 import org.apache.commons.lang.time.DateUtils
 import org.grails.web.json.JSONArray
-import org.grails.web.json.JSONElement
 import org.grails.web.json.JSONObject
 
 import javax.transaction.Transactional
@@ -164,12 +162,17 @@ class NotificationService {
             def duration = TimeCategory.minus(endTime, startTime)
             qr.addLog("Time cost: ${duration}")
             if(!dryRun) {
-                QueryResult.withTransaction {
-                    if (!qr.save(validate: true, flush: true)) {
-                        qr.errors.allErrors.each {
-                            log.error(it)
+                try {
+                    QueryResult.withTransaction {
+                        if (!qr.save(validate: true)) {
+                            qr.errors.allErrors.each {
+                                log.error(it)
+                            }
                         }
                     }
+                } catch (Exception e) {
+                    //todo check why sometimes scheduler cannot save the queryresult
+                    log.error("An unexpected error occurred in saving queryresult: ${qr}", e)
                 }
             }
             else {
@@ -646,10 +649,9 @@ class NotificationService {
         log.debug("Checking frequency : " + frequencyName)
         Date now = new Date()
         Frequency frequency = Frequency.findByName(frequencyName)
-        execQueryForFrequency(frequency, sendEmails)
-        //update the frequency last checked
-        frequency = Frequency.findByName(frequencyName)
         if (frequency) {
+            execQueryForFrequency(frequency, sendEmails)
+            //update the frequency last checked
             frequency.lastChecked = now
             Frequency.withTransaction {
                 if (!frequency.save(validate: true, flush: true)) {