#380 Quartz simulation

Author: qifeng-bai

grails-app/controllers/au/org/ala/alerts/AdminController.groovy

@@ -735,10 +735,13 @@ class AdminController {
      * Only send emails in Development environment
      * And NO database updates
      */
-    def triggerHourlyQueries() {
-        log.info("****** Simulating hourly jobs, NO Database updates****** " + new Date())
-        def logs = notificationService.execQueryForFrequency('hourly', Environment.current == Environment.DEVELOPMENT, true)
-        log.info("****** End hourly job simulation ****** " + new Date())
+    def triggerQueriesByFrequency(String frequency) {
+        def allowedFrequencies = ['hourly', 'weekly', 'daily', 'monthly']
+        def jobFrequency = (frequency in allowedFrequencies) ? frequency : 'daily'
+
+        log.info("****** Simulating ${jobFrequency} jobs, NO Database updates****** " + new Date())
+        def logs = notificationService.execQueryForFrequency(jobFrequency, Environment.current == Environment.DEVELOPMENT, true)
+        log.info("****** End ${jobFrequency} job simulation ****** " + new Date())
         render(logs.sort { [it.succeeded ? 1 : 0, it.hasChanged ? 0 : 1] } as JSON)
     }
 
@@ -748,6 +751,7 @@ class AdminController {
      * Run a task to execute the query for a specific frequency
      *
      */
+    @Deprecated //Use triggerHourlyQueries instead
     def dryRunAllQueriesForFrequency(){
         def freq = params.frequency
         Frequency frequency = Frequency.findByName(freq)

grails-app/controllers/au/org/ala/alerts/NotificationController.groovy

@@ -129,7 +129,7 @@ class NotificationController {
         // Since the previous and current results are loaded from database,
         // the diffService.getRecordChanges() should be called to get the new records
         queryResult.newRecords = diffService.getRecordChanges(queryResult)
-
+        queryResult.succeeded = true
         boolean hasChanged =  queryResult.newRecords.size() > 0
         if (hasChanged != queryResult.hasChanged) {
             log.error("Warning: Calculated hasChanged flag is not consistent with result in database. Calculated: ${hasChanged}, in database: ${queryResult.hasChanged}")

grails-app/services/au/org/ala/alerts/DiffService.groovy

@@ -124,7 +124,7 @@ class DiffService {
                 log.warn "queryId: " + queryResult.query.id + ", queryResult:" + queryResult.id + " last or previous objects contains HTML and not JSON"
             }
         } catch (Exception ex) {
-            log.error("queryId: ${queryResult.query.id}, Runtime error: ${ex.getMessage()}")
+            log.error("Diff Exception: [${queryResult.query.id}, ${queryResult.query?.name}]: Runtime error: ${ex.getMessage()}")
         }
 
         return records
@@ -312,7 +312,7 @@ class DiffService {
             ids1 = JsonPath.read(last, fullRecordJsonPath)
             ids2 = JsonPath.read(previous, fullRecordJsonPath)
         } catch (PathNotFoundException e){
-            log.info("it's not an error. Result doesn't contain any records since the returned json does not contain any 'recordJsonPath', if result is empty.")
+            log.debug("it's not an error. Result doesn't contain any records since the returned json does not contain any 'recordJsonPath', if result is empty.")
         }
 
         // pull together the records that have been added

grails-app/services/au/org/ala/alerts/MyAnnotationService.groovy

@@ -39,15 +39,17 @@ class MyAnnotationService{
         try {
             oldRecordsMap = JsonPath.read(previous, recordJsonPath).collectEntries { [(it.uuid): it] }
         }catch (PathNotFoundException e){
-            log.warn("Previous result is empty or doesn't have any records containing a field ${recordJsonPath} defined in recordJsonPath")
+            log.debug("Previous result is empty or doesn't have any records containing a field ${recordJsonPath} defined in recordJsonPath")
         }
         try {
             curRecordsMap = JsonPath.read(last, recordJsonPath).collectEntries { [(it.uuid): it] }
         }catch (PathNotFoundException e){
-            log.warn("Current result is empty or doesn't have any records containing a field ${recordJsonPath} defined in recordJsonPath")
+            log.debug("Current result is empty or doesn't have any records containing a field ${recordJsonPath} defined in recordJsonPath")
         }
         // if an occurrence record doesn't exist in previous result (added) or has different verified_assertions than previous (changed).
-        def records = curRecordsMap.findAll {
+
+        // NOTE: values of a linkedHashMap is immutable, so we need to create a new list if we will add new records after
+        Collection records = curRecordsMap.findAll {
             def record = it.value
             // if the record has verified assertions and it is a list - some legacy records may have a String instead of a list
             if (!record.verified_assertions?.isEmpty() && record.verified_assertions instanceof List) {
@@ -67,11 +69,23 @@ class MyAnnotationService{
 
         }.values()
 
+        // make a copy of the records list to make it mutable
+        def mutableRecords = new ArrayList(records)
         //if an occurrence record exists in previous result but not in current, it means the annotation is deleted.
         //We need to add these records as a 'modified' record
-        records.addAll(oldRecordsMap.findAll { !curRecordsMap.containsKey(it.value.uuid) }.values())
+        //mutableRecords.addAll(oldRecordsMap.findAll { !curRecordsMap.containsKey(it.value.uuid) }.values())
+
+        def missingEntries = oldRecordsMap.findAll { entry ->
+            def record = entry.value  // Get the value from the map entry
+            !curRecordsMap.containsKey(record.uuid) // Check if the uuid is missing in curRecordsMap
+        }
+        Collection missingRecords = missingEntries.values()
+        if (missingRecords.size() > 0) {
+            log.debug("Found ${missingRecords.size()} missing records: ${missingRecords.collect { it.uuid }}")
+            mutableRecords.addAll(missingRecords)
+        }
 
-        return records
+        return mutableRecords
     }
 
     /**

grails-app/services/au/org/ala/alerts/NotificationService.groovy

@@ -102,22 +102,19 @@ class NotificationService {
         } finally {
             Date endTime = new Date()
             def duration = TimeCategory.minus(endTime, startTime)
+            String msg = "${qr.succeeded ? (qr.hasChanged ? qr.newRecords.size() + ' new records found' : 'Completed - No new records') : 'Aborted'}. [${query.id}, ${query.name}, ${frequency.name}]. Time cost: ${duration}"
+            qr.addLog(msg)
+            log.info(msg)
 
             if(!dryRun ){
                 if (qr.succeeded) {
-                    String msg = "Completed [${query.id}, ${query.name}, ${frequency.name}]. Time cost: ${duration}"
-                    qr.addLog(msg)
-                    log.info(msg)
-
                     QueryResult.withTransaction {
                         if (!qr.save(validate: true)) {
                             qr.errors.allErrors.each {
                                 log.error(it)
                             }
                         }
                     }
-                } else {
-                    log.info("Aborted. [${query.id}, ${query.name}, ${frequency.name}]. Time cost: ${duration}")
                 }
             }
             else {
@@ -563,7 +560,7 @@ class NotificationService {
      * @return logs for each query
      */
     def execQueryForFrequency(String frequencyName, boolean sendEmails = true, boolean dryRun = false) {
-        log.info("Checking frequency : ${frequencyName}, emails ${sendEmails} dryRun ${dryRun}")
+        log.info("Checking frequency : ${frequencyName}, emails ${sendEmails}, dryRun ${dryRun}")
         def logs = []
         Date now = new Date()
         Frequency frequency = Frequency.findByName(frequencyName)

grails-app/views/admin/index.gsp

@@ -35,11 +35,19 @@
                 View each alert type with counts for users</g:link> - View the list of all available custom and default alerts with user subscription count.</li>
 
             <li><g:link controller="admin" action="deleteOrphanAlerts">Delete orphaned queries</g:link> - Remove queries no longer associated with Alert Notification/Subscription.</li>
-            <li><g:link controller="admin" action="dryRunAllQueriesForFrequency" params="[frequency: 'daily']" target="_blank">Debug daily alerts</g:link> - Dry-run of daily alerts to determine if emails will be triggered on the next schedule. </li>
+%{--            <li><g:link controller="admin" action="dryRunAllQueriesForFrequency" params="[frequency: 'daily']" target="_blank">Debug daily alerts</g:link> - Dry-run of daily alerts to determine if emails will be triggered on the next schedule. </li>--}%
             <li class="admin"><a class="btn btn-info" href="${request.contextPath}/admin/query">Debug and Test</a> - For testers and developers</li>
-            <li class="admin"><a class="btn btn-info" href="${g.createLink(controller: 'admin', action: 'triggerHourlyQueries')}" target="_blank">
-                Simulating an Hourly Scheduler Job
-            </a> Will NOT update the database, and emails will ONLY be sent in the development environment.</li>
+            <li class="admin">
+                Simulating a
+                <select id="frequencySimulated" class="form-select">
+                    <option value="hourly">Hourly</option>
+                    <option value="daily" selected>Daily</option>
+                    <option value="weekly">Weekly</option>
+                    <option value="monthly">Monthly</option>
+                </select>
+                Scheduled Job <a class="btn btn-info" id="simulatedFrequencyLink" href="${g.createLink(controller: 'admin', action: 'triggerQueriesByFrequency', params: [frequency: 'daily'])}" target="_blank">Run</a>
+                - Will NOT update the database, and emails will ONLY be sent in the Development environment.
+            </li>
         </ul>
         <h4>Fix empty/invalid notification_token</h4>
         <ul>
@@ -87,9 +95,16 @@
             </plugin:isAvailable>
         </ul>
     </div>
-
-
-
 </div>
+
+<script>
+    $(document).ready(function() {
+        $('#frequencySimulated').change(function() {
+            let selectedValue = $(this).val();
+            let queryLink = $('#simulatedFrequencyLink');
+            queryLink.attr('href', "${g.createLink(controller: 'admin', action: 'triggerQueriesByFrequency')}?frequency=" + selectedValue);
+        });
+    });
+</script>
 </body>
 </html>

grails-app/views/admin/query.gsp

@@ -47,14 +47,6 @@
 
 </head>
 <body>
-    <div>
-        <div class="panel panel-default">
-            <div class="panel-heading">Dry run tests. No DB updates, No emails sent </div>
-            <div class="panel-body">
-                <a class="btn btn-info" href = "/admin/dryRunAllQueriesForFrequency?frequency=daily" target="_blank">Dry run daily tasks</a>
-            </div>
-        </div>
-     </div>
     <div>
         <ul class="nav nav-tabs" id="myTab" role="tablist">
             <g:each var="queryType" in="${queries.keySet()}" status="i">

grails-app/views/email/myAnnotations.gsp

@@ -40,7 +40,7 @@
                 <g:each status="i" in="${records}" var="oc">
                     <tr>
                         <td style="padding: 20px;background-color: white;font-family: 'Roboto', sans-serif;font-size: 14px;line-height: 1.5;">
-                        <g:set var="occurrencelink" value="${query.baseUrlForUI}/occurrences/${oc.uuid}"></g:set>
+                        <g:set var="occurrencelink" value="${query.baseUrlForUI}/occurrences/${oc.uuid}#userAnnotationsList"></g:set>
                         <g:set var="assertionlink" value="${query.baseUrl}/occurrences/${oc.uuid}/assertions"></g:set>
                         <table style="width: 100%">
                             <tr style="vertical-align: top;">