#238 ( biosecurity) add fq into csv and other csv operations

qifeng-bai · qifeng-bai · commit 358caec1e964 · 2024-08-22T16:11:58.000+10:00
#265 add download full csv, csv deletion function and admin links to myAnnotation and other small changes
diff --git a/grails-app/assets/stylesheets/alerts.css b/grails-app/assets/stylesheets/alerts.css
@@ -108,4 +108,10 @@ a.btn-ala {
     color: #fff; /* White text */
 }
 
+.row-fluid {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+}
+
 
diff --git a/grails-app/controllers/au/org/ala/alerts/AdminController.groovy b/grails-app/controllers/au/org/ala/alerts/AdminController.groovy
@@ -731,7 +731,9 @@ class AdminController {
 
         // Get a list of all CSV files in the folder
         String mergedCSVFile = biosecurityCSVService.aggregateCSVFiles(folderName)
-
+        if (folderName == "/" || folderName.isEmpty()) {
+            folderName = "biosecurity_alerts"
+        }
         def saveToFile = folderName +".csv"
         response.contentType = 'application/octet-stream'
         response.setHeader('Content-Disposition', "attachment; filename=\"${saveToFile}\"")
@@ -776,4 +778,25 @@ class AdminController {
             render(status: 200, text: "QueryResult not found")
         }
     }
+
+    @AlaSecured(value = ['ROLE_ADMIN', 'ROLE_BIOSECURITY_ADMIN'], anyRole = true)
+    def deleteBiosecurityAuditCSV(String filename) {
+        Map message = [status : 1, message: ""]
+        def BASE_DIRECTORY = grailsApplication.config.biosecurity.csv.local.directory
+        def file = new File(BASE_DIRECTORY, filename)
+        if (!file.exists() || file.isDirectory()) {
+            message['status'] = 1
+            message['message'] = "File not found"
+        } else {
+            if (file.renameTo(new File(BASE_DIRECTORY,  filename +'_deleted'))){
+                message['status'] = 0
+                message['message'] = "The file has been temporarily deleted. You can contact the system administrator to recover it."
+            } else {
+                message['status'] = 1
+                message['message'] = "File deletion failed"
+            }
+        }
+
+        render(status: 200, contentType: 'application/json', text: message as JSON)
+    }
 }
diff --git a/grails-app/controllers/au/org/ala/alerts/UrlMappings.groovy b/grails-app/controllers/au/org/ala/alerts/UrlMappings.groovy
@@ -31,6 +31,7 @@ class UrlMappings {
         "/admin/deleteQuery"(controller: 'admin', action: 'deleteQuery')
         "/biosecurity/csv"(controller: 'admin', action: 'listBiosecurityAuditCSV')
         "/biosecurity/csv/download"(controller: 'admin', action: 'downloadBiosecurityAuditCSV')
+        "/biosecurity/csv/delete"(controller: 'admin', action: 'deleteBiosecurityAuditCSV')
         "/biosecurity/csv/aggregate"(controller: 'admin', action: 'aggregateBiosecurityAuditCSV')
 
 
diff --git a/grails-app/services/au/org/ala/alerts/BiosecurityCSVService.groovy b/grails-app/services/au/org/ala/alerts/BiosecurityCSVService.groovy
@@ -75,14 +75,15 @@ class BiosecurityCSVService {
 
         def tempFilePath = Files.createTempFile(outputFile, ".csv")
         def tempFile = tempFilePath.toFile()
-        String rawHeader = "recordID:uuid, recordLink:occurrenceLink, scientificName,taxonConceptID,decimalLatitude,decimalLongitude,eventDate,occurrenceStatus,dataResourceName,multimedia,media_id:image," +
+        // example of rawHeader
+        // recordID:uuid  recordID is the header name, uuid is the property in the record
+        String rawHeader = "recordID:uuid, recordLink:occurrenceLink, scientificName,taxonConceptID,decimalLatitude,decimalLongitude,eventDate,occurrenceStatus,dataResourceName,multimedia,mediaId:image," +
                 "vernacularName,taxonConceptID_new,kingdom,phylum,class:classs,order,family,genus,species,subspecies," +
                 "firstLoadedDate:firstLoaded,basisOfRecord,match," +
-                "search_term,correct_name:scientificName,provided_name:providedName,common_name:vernacularName,state:stateProvince,lga,shape,list_id:listId,list_name:listName,cw_state,shape_feature,creator:collector," +
-                "license,mimetype,width,height," +
-                "image_url:smallImageUrl," + // TBC , multiple image urls
-                "date_sent:dateSent,"+
-                "cl"
+                "searchTerm:search_term,correct name:scientificName,provided name:providedName,common name:vernacularName,state:stateProvince,lga layer,lga,fq,list id:listId,list name:listName, listLink:listLink, cw_state,shape feature:shape_feature,creator:collector," +
+                "license,mimetype," +
+                "image url:smallImageUrl," + // TBC , multiple image urls
+                "date sent:dateSent"
                 //"fq, kvs"
         if (grailsApplication.config.biosecurity.csv.headers) {
             rawHeader = grailsApplication.config.biosecurity.csv.headers
@@ -92,7 +93,6 @@ class BiosecurityCSVService {
         def fields = []
         def headersAndFields = rawHeader.split(',')
         headersAndFields.each { entry ->
-
             def parts = entry.trim().split(':', 2)  // Split on ':' with a limit of 2 parts
             headers << parts[0]  // Add the part before ':' to the first array
             if (parts.size() > 1) {
@@ -107,36 +107,14 @@ class BiosecurityCSVService {
             writer.write(headers.join(",")+ "\n")
             records.each { record ->
                 def values = fields.collect { field ->
-                    def value = ""
-                    if(record.containsKey(field)) {
-                        value = record[field]
-                        //if value is a list, convert it to a string. e.g. collectors, images
-                        if (value instanceof List) {
-                            value = "\"${value.join(";")}\""  // Join the list with ';' and wrap it in double quotes
-                        } else {
-                            value = value.toString()
-                            switch (field) {
-                                case "eventDate":
-                                    if (value) {
-                                        value = new SimpleDateFormat("dd/MM/yyyy hh:mm:ss").format(value.toLong())
-                                    }
-                                    break
-                                default:
-                                    if (value instanceof List) {
-                                        value = "\"${value.join(";")}\""  // Join the list with ';' and wrap it in double quotes
-                                    } else {
-                                        value = value.toString()
-                                    }
-                                    break
-                            }
-                        }
-                    } else {
-                        //special cases
-                        if(field == "lga") {
+                    def value = record[field]
+
+                    switch (field) {
+                        case "lga":
                             //read from cl (context layer)
                             def cls = record["cl"]
-                            //LGA2023
-                            def layerId= grailsApplication.config.biosecurity.csv.lga ?:"LGA2023"
+                            //LGA2023 is the default layer id
+                            def layerId =  grailsApplication.config.getProperty('biosecurity.csv.lga', 'LGA2023')
                             if(cls) {
                                 String matched = cls.find {
                                     def (k, v) = it.split(':') // Split the string into key and value
@@ -145,8 +123,30 @@ class BiosecurityCSVService {
                                 //assure return "" if matched is null
                                 value = matched?.split(':')?.with { it.size() > 1 ? it[1] : "" } ?: ""
                             }
-                        }
+                            break
+                        case "lga layer":
+                            value = grailsApplication.config.biosecurity.csv.lga ?:"LGA2023"
+                            break
+                        case "eventDate":
+                            if (value) {
+                                value = new SimpleDateFormat("dd/MM/yyyy hh:mm:ss").format(value.toLong())
+                            } else {
+                                value = ""
+                            }
+                            break
+                        default:
+                            if (record.containsKey(field)) {
+                                if (value instanceof List) {
+                                    value = "\"${value.join(";")}\""  // Join the list with ';' and wrap it in double quotes
+                                } else {
+                                    value = value.toString()
+                                }
+                            } else {
+                                value = ""
+                            }
+                            break
                     }
+
                     return value
                 }
                 writer.write(values.join(","))
@@ -165,9 +165,10 @@ class BiosecurityCSVService {
     String aggregateCSVFiles(String folderName) {
         def BASE_DIRECTORY = grailsApplication.config.biosecurity.csv.local.directory
         def folder = new File(BASE_DIRECTORY, folderName)
-        Collection<File> csvFiles = folder.listFiles().findAll { it.isFile() && it.name.endsWith('.csv') }
+        Collection<File> csvFiles = []
+        collectCsvFiles(folder, csvFiles)
 
-        log.info("Aggregate CSV files into one file")
+        log.info("Aggregate ${csvFiles.size()} CSV files under ${folder} into one file")
         def tempFilePath = Files.createTempFile("merged_", ".csv")
         def tempFile = tempFilePath.toFile()
         tempFile.withWriter { writer ->
@@ -189,6 +190,21 @@ class BiosecurityCSVService {
         return tempFile.absolutePath
     }
 
+    /**
+     * Collect CSV files from the folder and its subfolders
+     * @param folder
+     * @param collectedFiles
+     */
+    private void collectCsvFiles(File folder, Collection<File> collectedFiles) {
+        folder.listFiles().each { file ->
+            if (file.isDirectory()) {
+                collectCsvFiles(file, collectedFiles) // Recursively collect CSV files from subfolders
+            } else if (file.isFile() && file.name.endsWith('.csv')) {
+                collectedFiles.add(file)
+            }
+        }
+    }
+
     /**
      * Move file from source to destination
      * @param source
@@ -219,13 +235,14 @@ class BiosecurityCSVService {
      * @param dir
      * @return Key value pair of folder and files
      * */
+
     private List<Map> listFilesRecursively(File dir) {
         def BASE_DIRECTORY = grailsApplication.config.biosecurity.csv.local.directory
         def rootDir = new File(BASE_DIRECTORY)
         def foldersAndFiles = rootDir.listFiles().findAll { it.isDirectory() }.collect { folder ->
             [
                     name: folder.name,
-                    files: folder.listFiles().findAll { it.isFile() }.collect { file -> file.name }
+                    files: folder.listFiles().findAll { it.isFile() && it.name.endsWith('.csv') }.collect { file -> file.name }
             ]
         }
         return foldersAndFiles
diff --git a/grails-app/services/au/org/ala/alerts/BiosecurityService.groovy b/grails-app/services/au/org/ala/alerts/BiosecurityService.groovy
@@ -181,6 +181,7 @@ class BiosecurityService {
                 record["dateSent"] = new SimpleDateFormat("dd/MM/yyyy").format(to)
                 record["listName"] = query.name
                 record["listId"] = drId
+                record["listLink"] = grailsApplication.config.getProperty('lists.baseURL') + "/speciesListItem/list/" + drId
                 return record
             }
 
@@ -245,6 +246,7 @@ class BiosecurityService {
                     if (listItem.kvpValues?.size()>0) {
                         //Do not join, let CSV generate handle it
                         occurrence['kvs'] = listItem.kvpValues.collect { kv -> "${kv.key}:${kv.value}" }
+                        occurrence['fq'] = listItem.kvpValues?.find { it.key == 'fq' }?.value
                     }
                     fetchExtraInfo(occurrence.uuid, occurrence)
                 }
diff --git a/grails-app/services/au/org/ala/alerts/EmailService.groovy b/grails-app/services/au/org/ala/alerts/EmailService.groovy
@@ -106,7 +106,7 @@ class EmailService {
         int totalRecords = records.size()
         int maxRecords = grailsApplication.config.getProperty("biosecurity.query.maxRecords", Integer, 500)
 
-        if (queryResult.hasChanged || Environment.current == Environment.DEVELOPMENT || Environment.current == Environment.TEST) {
+        if (queryResult.hasChanged || Environment.current == Environment.DEVELOPMENT ) {
             if (grailsApplication.config.getProperty("mail.enabled", Boolean, false)) {
                 def emails = recipients.collect { it.email }
                 log.info "Sending emails for ${query.name} to ${emails.size() <= 2 ? emails.join('; ') : emails.take(2).join('; ') + ' and ' + emails.size() + ' other users.'}"
diff --git a/grails-app/services/au/org/ala/alerts/UserService.groovy b/grails-app/services/au/org/ala/alerts/UserService.groovy
@@ -20,7 +20,7 @@ import grails.converters.JSON
 import grails.plugin.cache.Cacheable
 import grails.util.Holders
 import grails.util.Environment
-import au.org.ala.alerts.Query
+
 
 import grails.gorm.transactions.Transactional
 
@@ -36,7 +36,6 @@ class UserService {
     def getUserAlertsConfig(User user) {
 
         log.debug('getUserAlertsConfig - Viewing my alerts :  ' + user)
-
         //enabled alerts
         def notificationInstanceList = Notification.findAllByUser(user)
 
diff --git a/grails-app/views/admin/biosecurity.gsp b/grails-app/views/admin/biosecurity.gsp
@@ -397,13 +397,13 @@
                 </div>
                 <p></p>
                 <div class="row" style="text-align: right">
-                    <div class="col-sm-10" >CSV files generated for each Biosecurity Alert [Experimental purpose]</div>
+                    <div class="col-sm-10" >Download CSV list of all occurrences from all biosecurity alerts sent (scheduled and manual)</div>
                     <div class="col-sm-2" >
-                        <a class="btn btn-info" href="${createLink(controller: 'admin', action: 'listBiosecurityAuditCSV')}" target="_blank">CSV Auditing</a>
+                        <a class="btn btn-info" href="${createLink(controller: 'admin', action: 'listBiosecurityAuditCSV')}" target="_blank">Reporting</a>
                     </div>
                 </div>
                 <p></p>
-                <div>
+%{--                <div>
                 <g:if test="${queries}">
                     <form target="_blank" action="${request.contextPath}/admin/csvAllBiosecurity" method="post">
                         <div class="row" style="text-align: right">
@@ -416,7 +416,7 @@
                         </div>
                     </form>
                 </g:if>
-                </div>
+                </div>--}%
 
             </div>
         </div>
diff --git a/grails-app/views/admin/biosecurityCSV.gsp b/grails-app/views/admin/biosecurityCSV.gsp
@@ -1,7 +1,7 @@
 <!DOCTYPE html>
 <html lang="en">
 <head>
-    <title>Biosecurity Audit CSV </title>
+    <title>Biosecurity Alerts Reporting</title>
     <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
     <meta name="layout" content="${grailsApplication.config.skin.layout}"/>
     <meta name="breadcrumb" content="BioSecurity CSV"/>
@@ -30,23 +30,60 @@
                 icon.toggleClass('fa-folder fa-folder-open-o');
             });
         });
+
+        function deleteFile(filename) {
+            $.ajax({
+                url: "${createLink(controller: 'admin', action: 'deleteBiosecurityAuditCSV')}",
+                type: 'POST',
+                data: {
+                    filename: filename
+                },
+                success: function(response) {
+                    // Assuming the response is a JSON object with a message
+                    alert(response.message);
+                    location.reload();
+                },
+                error: function(xhr, status, error) {
+                    alert("Error: " + xhr.responseText);
+                }
+            });
+        }
     </script>
 </head>
 <body>
+    <div>
+        <h2>All Biosecurity Alerts Data</h2>
+        Download a comprehensive CSV file detailing all occurrence records from every biosecurity alert sent. This includes both scheduled and manually triggered emails
+        <br/>
+        <a class="btn btn-primary " href="${createLink(controller: 'admin', action: 'aggregateBiosecurityAuditCSV', params: [folderName:'/'])}">
+        <i class="fa fa-cloud-download" aria-hidden="true" ></i>  Download Full CSV Report
+        </a>
+        <hr>
+    </div>
+
     <g:if test="${status == 0}">
-        <g:each in="${foldersAndFiles}" var="folder">
-            <div class="folder" data-folder="${folder.name}">
-                <i class="fa fa-folder folder-icon folder" aria-hidden="true"></i> ${folder.name}
-                <a href="${createLink(controller: 'admin', action: 'aggregateBiosecurityAuditCSV', params: [folderName:folder.name])}">
-                    <i class="fa fa-cloud-download" aria-hidden="true" title="Download as one CSV file for the date."></i>
-                </a>
-            </div>
-            <div class="file-list" id="files-${folder.name}">
-                <g:each in="${folder.files}" var="file">
-                    <div><a href="${createLink(controller: 'admin', action: 'downloadBiosecurityAuditCSV', params: [filename:folder.name +'/' + file])}"><i class="fa fa-download" aria-hidden="true"></i>  ${file}</a></div>
-                </g:each>
-            </div>
-        </g:each>
+        <div>
+            <h2>Individual Biosecurity Alerts Data</h2>
+            Download individual CSV files for each biosecurity alert email, detailing all occurrence records. Files are sorted by the date the alert was sent.
+            <g:each in="${foldersAndFiles}" var="folder">
+                <div class="folder" data-folder="${folder.name}">
+                    <i class="fa fa-folder folder-icon folder" aria-hidden="true"></i> ${folder.name}
+                    <a href="${createLink(controller: 'admin', action: 'aggregateBiosecurityAuditCSV', params: [folderName:folder.name])}">
+                        <i class="fa fa-cloud-download" aria-hidden="true" title="Download as one CSV file for the date."></i>
+                    </a>
+                </div>
+                <div class="file-list" id="files-${folder.name}">
+                    <g:each in="${folder.files}" var="file">
+                        <div>
+                            <a href="${createLink(controller: 'admin', action: 'downloadBiosecurityAuditCSV', params: [filename:folder.name +'/' + file])}"><i class="fa fa-download" aria-hidden="true"></i>  ${file}</a>
+                            <a href="#" onclick="deleteFile('${folder.name}/${file}'); return false;">
+                                <i class="fa fa-trash-o" aria-hidden="true"></i>
+                            </a>
+                        </div>
+                    </g:each>
+                </div>
+            </g:each>
+        </div>
     </g:if>
     <g:else>
         ${message}
diff --git a/grails-app/views/notification/myAlerts.gsp b/grails-app/views/notification/myAlerts.gsp
@@ -11,15 +11,25 @@
         <meta name="breadcrumbParent" content="${grailsApplication.config.userDetails.web.url}/myprofile, ${message(code:"my.alerts.breadcrumb.parent")}" />
         <g:set var="userPrefix" value="${adminUser ? user.email : message(code:"my.alerts.my") }"/>
         <title><g:message code="my.alerts.title" args="[userPrefix]" /> | ${grailsApplication.config.skin.orgNameLong}</title>
-
         <asset:stylesheet href="alerts.css"/>
     </head>
     <body>
       <div id="content">
           <header id="page-header">
-            <div class="inner row-fluid">
-              <h1><g:message code="my.alerts.h1" args="[userPrefix]" /></h1>
-            </div>
+              <div class="inner row-fluid">
+                  <div class="content">
+                      <h1><g:message code="my.alerts.h1" args="[userPrefix]" /></h1>
+                  </div>
+                  <div>
+                      <% if (request.isUserInRole("ROLE_ADMIN")) { %>
+                      <a href="${createLink(controller: 'admin', action: 'index')}" class="btn btn-primary">Admin</a>
+                      <% } %>
+
+                      <% if (request.isUserInRole("ROLE_BIOSECURITY_ADMIN")) { %>
+                      <a href="${createLink(controller: 'admin', action: 'biosecurity')}" class="btn btn-primary">Biosecurity Admin</a>
+                      <% } %>
+                  </div>
+              </div>
           </header>
           <g:if test="${flash.message}">
               <div class="alert alert-info">${flash.message}</div>

Original file line number	Diff line number	Diff line change
`@@ -108,4 +108,10 @@ a.btn-ala {`
`108`	`108`	`color: #fff; /* White text */`
`109`	`109`	`}`
`110`	`110`
	`111`	`+.row-fluid {`
	`112`	`+ display: flex;`
	`113`	`+ justify-content: space-between;`
	`114`	`+ align-items: center;`
	`115`	`+}`
	`116`	`+`
`111`	`117`
Original file line number	Diff line number	Diff line change
`@@ -181,6 +181,7 @@ class BiosecurityService {`
`181`	`181`	`record["dateSent"] = new SimpleDateFormat("dd/MM/yyyy").format(to)`
`182`	`182`	`record["listName"] = query.name`
`183`	`183`	`record["listId"] = drId`
	`184`	`+ record["listLink"] = grailsApplication.config.getProperty('lists.baseURL') + "/speciesListItem/list/" + drId`
`184`	`185`	`return record`
`185`	`186`	`}`
`186`	`187`
`@@ -245,6 +246,7 @@ class BiosecurityService {`
`245`	`246`	`if (listItem.kvpValues?.size()>0) {`
`246`	`247`	`//Do not join, let CSV generate handle it`
`247`	`248`	`occurrence['kvs'] = listItem.kvpValues.collect { kv -> "${kv.key}:${kv.value}" }`
	`249`	`+ occurrence['fq'] = listItem.kvpValues?.find { it.key == 'fq' }?.value`
`248`	`250`	`}`
`249`	`251`	`fetchExtraInfo(occurrence.uuid, occurrence)`
`250`	`252`	`}`