Skip to content

Commit d3170cf

Browse files
qifeng-baiqifeng-bai
committedJan 22, 2025
#360 Fixed totalRecords for different queries
1 parent a6252f0 commit d3170cf

File tree

13 files changed

+88
-65
lines changed

13 files changed

+88
-65
lines changed
 

‎build.gradle

+1-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
* rights and limitations under the License.
1212
*/
1313
buildscript {
14-
version "4.4.1-SNAPSHOT"
14+
version "4.4.2-SNAPSHOT"
1515
group "au.org.ala"
1616
}
1717

‎grails-app/controllers/au/org/ala/alerts/AdminController.groovy

+2-2
Original file line numberDiff line numberDiff line change
@@ -372,7 +372,7 @@ class AdminController {
372372
qr.previousCheck = qr.lastChecked
373373
qr.lastChecked = since
374374
query.lastChecked = since
375-
def records = notificationService.retrieveRecordForQuery(qr.query, qr)
375+
def records = notificationService.collectUpdatedRecords(qr)
376376

377377
String urlPrefix = "${grailsApplication.config.getProperty("grails.serverURL")}${grailsApplication.config.getProperty('security.cas.contextPath', '')}"
378378
def localeSubject = messageSource.getMessage("emailservice.update.subject", [query.name] as Object[], siteLocale)
@@ -603,7 +603,7 @@ class AdminController {
603603
def recipient =
604604
[email: currentUser.email, userUnsubToken: currentUser.unsubscribeToken, notificationUnsubToken: '']
605605
emailService.sendGroupNotification(qs, fre, [recipient])
606-
def results = ["hasChanged": hasChanged, "records": records, "recipient": currentUser.email, details: qs.brief()]
606+
def results = ["hasChanged": hasChanged, "totalRecords": qs.totalRecords, "records": records, "recipient": currentUser.email, details: qs.brief()]
607607
render results as JSON
608608
} else {
609609
render([status: 1, message: "Cannot find query: ${id}"] as JSON)

‎grails-app/controllers/au/org/ala/alerts/NotificationController.groovy

+17-2
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,6 @@ class NotificationController {
122122
* Debug the algorithm used to detect changes in the latest result / previous result
123123
*
124124
*/
125-
126125
def evaluateChangeDetectionAlgorithm = {
127126
def query = Query.get(params.queryId)
128127

@@ -142,7 +141,23 @@ class NotificationController {
142141
queryResult.lastResult = queryResult.compress(lastResult)
143142

144143
def records = notificationService.collectUpdatedRecords(queryResult)
145-
def results = ["hasChanged": hasChanged, "brief": queryResult.brief(), "records": records]
144+
145+
def emailSent = false
146+
if (params.emailMe) {
147+
User currentUser = userService.getUser()
148+
if (currentUser) {
149+
def recipient =
150+
[email: currentUser.email, userUnsubToken: currentUser.unsubscribeToken, notificationUnsubToken: '']
151+
//Pseudo Frequency
152+
Frequency fre = Frequency.findByName("weekly")
153+
emailService.sendGroupNotification(queryResult, fre, [recipient])
154+
emailSent = true
155+
} else {
156+
log.warn("No user found to send email to.")
157+
}
158+
}
159+
160+
def results = ["hasChanged": hasChanged, emailSent: emailSent, totalRecords: queryResult.totalRecords, "brief": queryResult.brief(), "records": records]
146161
render results as JSON
147162
}
148163

‎grails-app/domain/au/org/ala/alerts/QueryResult.groovy

+5
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,11 @@ class QueryResult {
1818
byte[] previousResult
1919
String logs
2020
transient boolean succeed = true
21+
/**
22+
* Store the total number of records from Biocache queries, e.g. occurrences, images, etc.
23+
* However, some queries, e.g. myAnnotation, lists, their own diffService need to update this value.
24+
*/
25+
transient int totalRecords = 0
2126

2227
String[] getLog() {
2328
return logs ? logs.split("\n") : []

‎grails-app/i18n/messages.properties

+2-2
Original file line numberDiff line numberDiff line change
@@ -146,8 +146,8 @@ my.alerts.my.custom.alerts=My custom alerts
146146
my.alerts.delete.label=Delete
147147
my.alerts.enable.to.email=Enable an alert to have emails sent to your email address <b>{0}</b>
148148
my.alerts.data.resource.desc=Data resource pages e.g. <a href="{0}">{1}</a> for alerts on new records or annotations.
149-
my.alerts.species.desc=Species pages e.g. <a href="{0}">{1}</a> for alerts on new records or annotations.
150-
my.alerts.region.desc=Region pages e.g. <a href="{0}">{1}</a> for alerts on new records or annotations.
149+
my.alerts.species.desc=Species pages e.g. <a href="{0}">{1}</a> for alerts on new records.
150+
my.alerts.region.desc=Region pages e.g. <a href="{0}">{1}</a> for alerts on new records.
151151
my.alerts.new.record.desc=Any <a href="{0}">{1}</a> for alerts on new records or annotations.
152152
my.alerts.look.for.btn=Look for the <a class="btn btn-default" href="javascript:void(0);" disabled="true"><i class="glyphicon glyphicon-bell"></i> Alerts</a> button.
153153
my.alerts.problem.retry="There was a problem updating your alert frequency. Please try again later."

‎grails-app/services/au/org/ala/alerts/DiffService.groovy

+1-2
Original file line numberDiff line numberDiff line change
@@ -215,11 +215,10 @@ class DiffService {
215215
records = dataResourceService.diff(queryResult)
216216
} else if ( queryService.isBiocacheImages(queryResult.query)) {
217217
records = imageService.diff(queryResult)
218-
} else if ( queryService.isBiocacheImages(queryResult.query)) {
219-
records = datasetService.diff(queryResult)
220218
} else {
221219
records = findNewRecordsById(previous, last, queryResult.query.recordJsonPath, queryResult.query.idJsonPath)
222220
}
221+
queryResult.totalRecords = records.size()
223222
} else {
224223
log.warn "queryId: " + queryResult.query.id + ", queryResult:" + queryResult.id + " last or previous objects contains HTML and not JSON"
225224
}

‎grails-app/services/au/org/ala/alerts/EmailService.groovy

+5-4
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ class EmailService {
6868
* @return
6969
*/
7070
def Map generateEmailModel(Query query, String frequency, QueryResult queryResult) {
71-
def records = notificationService.retrieveRecordForQuery(query, queryResult)
71+
def records = notificationService.collectUpdatedRecords(query, queryResult)
7272
def totalRecords = queryService.totalNumberWhenNotZeroPropertyEnabled(queryResult)
7373
[
7474
title : query.name,
@@ -93,12 +93,12 @@ class EmailService {
9393

9494
log.debug("Using email template: " + query.emailTemplate)
9595

96-
def records = notificationService.retrieveRecordForQuery(query, queryResult)
96+
def records = notificationService.collectUpdatedRecords(queryResult)
9797

98-
int totalRecords = records.size()
98+
int totalRecords = queryResult.totalRecords
9999
int maxRecords = grailsApplication.config.getProperty("biosecurity.query.maxRecords", Integer, 500)
100100

101-
if (queryResult.hasChanged || Environment.current == Environment.DEVELOPMENT ) {
101+
if (totalRecords > 0 || Environment.current == Environment.DEVELOPMENT ) {
102102
if (grailsApplication.config.getProperty("mail.enabled", Boolean, false)) {
103103
def emails = recipients.collect { it.email }
104104
log.info "Sending emails for ${query.name} to ${emails.size() <= 2 ? emails.join('; ') : emails.take(2).join('; ') + ' and ' + emails.size() + ' other users.'}"
@@ -126,6 +126,7 @@ class EmailService {
126126
}
127127
}
128128

129+
129130
void sendGroupEmail(Query query, subsetOfAddresses, QueryResult queryResult, records, Frequency frequency, int totalRecords, String userUnsubToken, String notificationUnsubToken) {
130131
String urlPrefix = "${grailsApplication.config.security.cas.appServerName}${grailsApplication.config.getProperty('security.cas.contextPath', '')}"
131132
def localeSubject = messageSource.getMessage("emailservice.update.subject", [query.name] as Object[], siteLocale)

‎grails-app/services/au/org/ala/alerts/NotificationService.groovy

+20-30
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ import org.hibernate.FlushMode
1414

1515
class NotificationService {
1616

17-
int PAGING_MAX = 1000
17+
int PAGING_MAX = 500
1818
def sessionFactory
1919
def httpService
2020
def emailService
@@ -33,20 +33,6 @@ class NotificationService {
3333
qr
3434
}
3535

36-
def retrieveRecordForQuery(query, queryResult) {
37-
if (query.recordJsonPath) {
38-
// return all of the new records if query is configured to fire on a non-zero value OR if previous value does not exist.
39-
if (queryService.firesWhenNotZero(query)) {
40-
diffService.getNewRecords(queryResult)
41-
// return diff of new and old records for all other cases
42-
} else {
43-
diffService.getNewRecordsFromDiff(queryResult)
44-
}
45-
} else {
46-
[]
47-
}
48-
}
49-
5036
/**
5137
* CORE method of searching if there are new records for a given query and frequency.
5238
* EXCEPT BioSecurity Query which is handled elsewhere.
@@ -60,7 +46,6 @@ class NotificationService {
6046
*/
6147

6248
QueryResult executeQuery(Query query, Frequency frequency, boolean runLastCheck=false, boolean dryRun=false, Date checkDate= new Date()) {
63-
6449
def session = sessionFactory.currentSession
6550
session.setFlushMode(FlushMode.MANUAL) // Set the flush mode to MANUAL
6651

@@ -80,15 +65,13 @@ class NotificationService {
8065
try {
8166
def processedJson
8267

83-
if (!urlString.contains("___MAX___")) {
84-
// queries without paging
85-
processedJson = processQuery(query, urlString)
86-
} else {
87-
// queries with paging
68+
if (urlString.contains("___MAX___")) {
8869
int max = PAGING_MAX
8970
int offset = 0
9071
def result = []
9172
boolean finished = false
73+
// todo : should be an independent method for Lists
74+
// NOTES: the method only works for Lists
9275
def allLists = []
9376

9477
while (!finished) {
@@ -130,9 +113,14 @@ class NotificationService {
130113
json.lists = JSON.parse((allLists as JSON).toString()) as JSONArray
131114
processedJson = json.toString()
132115
}
116+
} else {
117+
processedJson = processQuery(query, urlString)
133118
}
134119

135-
//update the stored properties
120+
// todo:
121+
// Review: Duplicated with the process of identifying record changes after
122+
// It is only for updating the stored properties
123+
// Considering remove
136124
refreshProperties(qr, processedJson)
137125

138126
// set check time
@@ -141,7 +129,7 @@ class NotificationService {
141129
qr.previousResult = qr.lastResult
142130
qr.lastResult = qr.compress(processedJson)
143131
qr.lastChecked = checkDate
144-
//todo: review this algorithm for all queries
132+
//todo: review if it is necessary to store
145133
qr.hasChanged = diffService.hasChanged(qr)
146134
qr.queryUrlUsed = urlString
147135
qr.queryUrlUIUsed = urlStringForUI
@@ -170,7 +158,6 @@ class NotificationService {
170158
}
171159
}
172160
} catch (Exception e) {
173-
//todo check why sometimes scheduler cannot save the queryresult
174161
log.error("An unexpected error occurred in saving queryresult: ${qr}", e)
175162
}
176163
}
@@ -223,7 +210,7 @@ class NotificationService {
223210
processedJson = processQueryBiosecurity(query, since, to)
224211
}
225212
} else {
226-
// queries with paging
213+
// It is works on species lists request
227214
int max = PAGING_MAX
228215
int offset = 0
229216
def allLists = []
@@ -584,7 +571,7 @@ class NotificationService {
584571
// these query and queryResult read/write methods are called by the scheduled jobs
585572
Integer totalRecords = null
586573
QueryResult queryResult = QueryResult.findByQueryAndFrequency(query, frequency)
587-
def records = retrieveRecordForQuery(query, queryResult)
574+
def records = collectUpdatedRecords(queryResult)
588575
Integer fireWhenNotZero = queryService.totalNumberWhenNotZeroPropertyEnabled(queryResult)
589576
totalRecords = records.size()
590577

@@ -875,17 +862,20 @@ class NotificationService {
875862

876863
/**
877864
* Copied from EmailService
878-
* Todo : Not full correct, need to be fixed
865+
* Todo : Not full correct, need to be checked
866+
* Different queries may need different methods to calculate the total number of records
879867
* @param queryResult
880868
* @return
881869
*/
882-
//@NotTransactional
883870
def collectUpdatedRecords(queryResult) {
884871
if (queryResult.query?.recordJsonPath) {
885872
// return all of the new records if query is configured to fire on a non-zero value OR if previous value does not exist.
886873
if (queryService.firesWhenNotZero(queryResult.query)) {
887-
diffService.getNewRecords(queryResult)
888-
// return diff of new and old records for all other cases
874+
def records = diffService.getNewRecords(queryResult)
875+
//Some queries, like biocache,has a pageSize, so it only returns a subset of the total records + totalRecords
876+
def jsonResult = JSON.parse( queryResult.decompress(queryResult.lastResult)) as JSONObject
877+
queryResult.totalRecords = jsonResult.totalRecords !=0 ? jsonResult.totalRecords: records.size()
878+
return records
889879
} else {
890880
diffService.getNewRecordsFromDiff(queryResult)
891881
}

‎grails-app/services/au/org/ala/alerts/UserService.groovy

+10-10
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,6 @@ import grails.plugin.cache.Cacheable
1919
import grails.util.Holders
2020
import grails.util.Environment
2121

22-
23-
import grails.gorm.transactions.Transactional
24-
25-
@Transactional
2622
class UserService {
2723

2824
static transactional = true
@@ -191,9 +187,11 @@ class UserService {
191187
def notificationInstance = new Notification()
192188
notificationInstance.query = Query.findByName(messageSource.getMessage("query.ala.blog.title", null, siteLocale))
193189
notificationInstance.user = user
194-
if (!notificationInstance.save(flush: true)) {
195-
notificationInstance.errors.allErrors.each {
196-
log.error(it)
190+
Notification.withTransaction {
191+
if (!notificationInstance.save(flush: true)) {
192+
notificationInstance.errors.allErrors.each {
193+
log.error(it)
194+
}
197195
}
198196
}
199197
}
@@ -239,9 +237,11 @@ class UserService {
239237
if (userDetails?.userId && userDetails?.email) {
240238
log.debug "User is not in user table - creating new record for " + userDetails
241239
user = new User([email: userDetails.email, userId: userDetails.userId, locked: userDetails.locked, frequency: Frequency.findByName("weekly")])
242-
if (!user.save(flush: true, failOnError: true)) {
243-
user.errors.allErrors.each {
244-
log.error(it)
240+
User.withTransaction {
241+
if (!user.save(flush: true, failOnError: true)) {
242+
user.errors.allErrors.each {
243+
log.error(it)
244+
}
245245
}
246246
}
247247
}

‎grails-app/views/admin/query.gsp

+5-5
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@
9090
<p><i class="fa fa-info-circle" aria-hidden="true" title="The URL MAY be used to build a UI link for this query."></i> <i>ref. ${query.baseUrlForUI}</i></p>
9191
</div>
9292
<i class="fa fa-cog" aria-hidden="true"></i> <i><b>JSON record path:</b>${query.recordJsonPath} <b>JSON ID path:</b>${query.idJsonPath}</i>
93-
%{-- <div>--}%
93+
%{-- <div>--}%
9494
%{-- <g:if test="${query.notifications}">--}%
9595
%{-- <g:each var="notification" in="${query.notifications}">--}%
9696
%{-- <li>${notification.user?.email}</li> --}%
@@ -140,14 +140,14 @@
140140
<div style="text-align: right;">
141141
<hr>
142142
<div style="padding: 5px;">
143-
<label>Evaluate the new record discovery algorithm by comparing the latest and previous results in database </label><g:link class="btn btn-info" controller="notification" action="evaluateChangeDetectionAlgorithm" params="[queryId: query.id, queryResultId: queryResult.id]" target="_blank">
144-
Evaluate
143+
<label>Evaluate the new record discovery algorithm by comparing the latest and previous results in database </label><g:link class="btn btn-info" controller="notification" action="evaluateChangeDetectionAlgorithm" params="[queryId: query.id, queryResultId: queryResult.id, emailMe:true]" target="_blank">
144+
Evaluate & email me
145145
</g:link>
146146
</div>
147147
<g:if test="${queryType != 'biosecurity'}">
148148
<div style="padding: 5px;">
149-
<label>Get the latest records, compare with the current result in the database, Email me the results </label><g:link class="btn btn-info" controller="admin" action="emailMeLastCheck" params="[queryId: query.id, frequency: queryResult.frequency?.name]" target="_blank">
150-
Email me
149+
<label>Collect the latest records, compare with the current result in the database, Email me the results </label><g:link class="btn btn-info" controller="admin" action="emailMeLastCheck" params="[queryId: query.id, frequency: queryResult.frequency?.name]" target="_blank">
150+
Collect & Email me
151151
</g:link>
152152
</div>
153153
<div style="padding: 5px;">

‎grails-app/views/email/biocache.gsp

+15-3
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,15 @@
1212
a {
1313
color: #003A70;
1414
}
15+
16+
.info-button {
17+
border: 1pt solid #C44D34;
18+
text-decoration: none;
19+
font-size: 14px;
20+
padding: 10px 15px 10px 15px;
21+
color: black;
22+
border-radius: 8px;
23+
}
1524
</style>
1625
<body style="background-color: #f4f4f4;margin: 0;padding: 0;font-family: 'Roboto', sans-serif;font-size: 16px;line-height: 1.5;">
1726
<table style="width: 100%; background-color: #f4f4f4;border-spacing: 0;border-collapse: collapse;">
@@ -35,7 +44,10 @@
3544
</tr>
3645
<tr>
3746
<td style="background-color: #E8E8E8;color: #000;padding: 40px 30px 40px 30px;text-align: center;font-family: 'Roboto', sans-serif;font-size: 22px;line-height: 1.5;">
38-
<div> ${totalRecords} occurrence ${totalRecords == 1 ? 'record' : 'records'} ${totalRecords == 1 ? 'has' : 'have'} been updated
47+
<div> <g:formatNumber number="${totalRecords}" format="###,###" /> occurrence ${totalRecords == 1 ? 'record' : 'records'} ${totalRecords == 1 ? 'has' : 'have'} been added
48+
</div>
49+
<div>
50+
<a class="btn info-button" href="${moreInfo}">View all new records</a>
3951
</div>
4052
</td>
4153
</tr>
@@ -61,8 +73,8 @@
6173
<g:if test="${oc.family}">
6274
Family: ${oc.family}<br/>
6375
</g:if>
64-
<g:if test="${oc.dataProviderName}">
65-
Source: ${oc.dataProviderName}
76+
<g:if test="${oc.dataResourceName}">
77+
Source: ${oc.dataResourceName}
6678
</g:if>
6779
</p>
6880
</td>

0 commit comments

Comments
 (0)