Added test cases for #357

patkyn · patkyn · commit f347ad4bc5cc · 2019-10-02T15:58:27.000+10:00
diff --git a/grails-app/services/au/org/ala/volunteer/ExportService.groovy b/grails-app/services/au/org/ala/volunteer/ExportService.groovy
@@ -6,6 +6,7 @@ import com.google.common.base.Stopwatch
 import grails.transaction.Transactional
 
 import java.util.concurrent.TimeUnit
+import java.util.concurrent.atomic.AtomicInteger
 import java.util.regex.Pattern
 import java.util.zip.ZipOutputStream
 import java.util.zip.ZipEntry
@@ -107,7 +108,13 @@ class ExportService {
 
         Map results = [:]
         if (project.requiredNumberOfTranscriptions > 1) {
-            results = valuesMap
+            if (valuesMap) {
+                results = valuesMap
+            } else {
+                // return empty map to allow the export of task fields even though there are no transcription fields
+                // this behaviour is consistent with the single transcription task
+                results = [(EMPTY_TRANSCRIPTIONID): [:]]
+            }
         }
         else {
             results << getTranscribedAndUploadedFields(task, valuesMap)
@@ -195,6 +202,7 @@ class ExportService {
 
         int threadPoolSize = grailsApplication.config.exportCSVThreadPoolSize ?: THREAD_POOL
         GParsPool.withPool threadPoolSize, {
+            final AtomicInteger numberOfTasks = new AtomicInteger(0)
             taskList.eachParallel { Task task ->
                 def sw2 = Stopwatch.createUnstarted()
                 def sw3 = Stopwatch.createUnstarted()
@@ -229,8 +237,12 @@ class ExportService {
                     def elapsed = sw2.elapsed(MILLISECONDS)
                     if (elapsed > 50) log.debug("Got column values in {}ms", elapsed)
                     writer.writeNext(values as String[])
+                    //valuesList.add(values as String[])
                 }
+                numberOfTasks.addAndGet(1)
+                //writer.writeAll(valuesList)
             }
+            log.info ("Wrote {} tasks", numberOfTasks.toString())
         }
 
         log.debug("Wrote all tasks in {}ms", sw.elapsed(MILLISECONDS))
@@ -292,6 +304,7 @@ class ExportService {
 
         int threadPoolSize = grailsApplication.config.exportCSVThreadPoolSize ?: THREAD_POOL
         GParsPool.withPool threadPoolSize, {
+            final AtomicInteger numberOfTasks = new AtomicInteger(0)
             taskList.eachParallel { task ->
                 Map toExport = getTranscriptionsToExport(project, task, valueMap[task.id])
                 toExport.each { transcriptionId, transcriptionValueMap ->
@@ -309,7 +322,9 @@ class ExportService {
                         writer.writeNext(values)
                     }
                 }
+                numberOfTasks.addAndGet(1)
             }
+            log.info ("Wrote {} tasks", numberOfTasks.toString())
         }
         writer.flush();
         zipStream.closeEntry();
diff --git a/grails-app/services/au/org/ala/volunteer/TaskService.groovy b/grails-app/services/au/org/ala/volunteer/TaskService.groovy
@@ -146,22 +146,38 @@ class TaskService {
         Task.findAllByProjectAndIsFullyTranscribed(project, true, params)
     }
 
+    /***
+     * Obtain Fully Transcribed tasks and the corresponding transcriptions for the project (eager fetching)
+     * Note: if there are 2000 fully transcribed tasks and 4000 transcriptions (2 transcriptions per task), this should return 2000 rows of tasks.
+     */
     List getFullyTranscribedTasksAndTranscriptions(Project projectInstance, Map params) {
-        Task.createCriteria().list (params) {
-            eq 'project', projectInstance
-            eq 'isFullyTranscribed', true
-            fetchMode 'transcriptions', FetchMode.JOIN
-        }
+        Task.executeQuery("""
+                        select t from Task t
+                        left outer join fetch t.transcriptions
+                        where t.project = :projectInstance
+                        and t.isFullyTranscribed = true
+                        order by t.id
+                    """, [projectInstance: projectInstance], params)
     }
 
-    List getValidTranscribedTasks(Project project, Map params) {
-        Task.createCriteria().list (params) {
-            eq 'project', project
-            eq 'isValid', true
-            fetchMode 'transcriptions', FetchMode.JOIN
-        }
+    /***
+     * Obtain Fully Validated tasks and the corresponding transcriptions for the project (eager fetching)
+     * Note: if there are 2000 validated tasks and 4000 transcriptions (2 transcriptions per task), this should return 2000 rows of tasks.
+     */
+    List getValidTranscribedTasks(Project projectInstance, Map params) {
+        Task.executeQuery("""
+                        select t from Task t
+                        left outer join fetch t.transcriptions
+                        where t.project = :projectInstance
+                        and t.isValid = true
+                        order by t.id
+                    """, [projectInstance: projectInstance], params)
     }
 
+    /***
+     * Obtain all tasks and the corresponding transcriptions for the project (eager fetching)
+     * Note: if there are 2000 tasks and 4000 transcriptions (2 transcriptions per task), this should return 2000 rows of tasks.
+     */
     List getAllTasksAndTranscriptionsIfExists(Project projectInstance, Map params) {
         Task.executeQuery("""
                         select t from Task t
diff --git a/grails-app/views/task/exportOptionsFragment.gsp b/grails-app/views/task/exportOptionsFragment.gsp
@@ -35,7 +35,7 @@
     $("#btnExportTasks").click(function (e) {
         e.preventDefault();
         var format = $("input:radio[name='optionsExport']:checked").val();
-        var url = "${createLink(controller:'project', action:'exportCSV', id: projectId, params:[validated: exportCriteria == 'validated', transcribed: exportCriteria=='transcribed'])}&exportFormat=" + format;
+        var url = "${createLink(controller:'project', action:'exportCSV', id: projectId, params:[validated: exportCriteria == 'validated', transcribed: exportCriteria=='transcribed']).encodeAsJavaScript()}&exportFormat=" + format;
         window.location = url;
     });
 
diff --git a/src/test/groovy/au/org/ala/volunteer/ExportServiceSpec.groovy b/src/test/groovy/au/org/ala/volunteer/ExportServiceSpec.groovy
@@ -27,6 +27,8 @@ class ExportServiceSpec extends Specification {
     def setup() {
         fieldService = Mock(FieldService)
         service.fieldService = fieldService
+        grailsApplication.config.exportCSVThreadPoolSize = 1
+        service.grailsApplication = grailsApplication
         taskService = Mock(TaskService)
         service.taskService = taskService
         response = new GrailsMockHttpServletResponse()
@@ -48,8 +50,8 @@ class ExportServiceSpec extends Specification {
         mockDomain(Project, [project])
     }
 
-    private Task createTask() {
-        Task task = new Task(transcriptions: new HashSet(), project:project)
+    private Task createTask(String externalIdentifier = '') {
+        Task task = new Task(transcriptions: new HashSet(), project:project, externalIdentifier: externalIdentifier)
         project.tasks.add(task)
         mockDomain(Task, [task])
 
@@ -71,6 +73,189 @@ class ExportServiceSpec extends Specification {
         fields
     }
 
+    def "Test non parrallel writes is working for larger tasks"() {
+        setup:
+        project.transcriptionsPerTask = 2
+        String userA = 'userA'
+        String userB = 'userB'
+
+        List allFields = new ArrayList()
+
+        int numOfTasks = 100
+        for (int i= 1; i <= numOfTasks; i++) {
+            Task task = createTask("image${i}.jpq")
+            List fields1 = transcribeTask(task, [[name:"scientificName", value:"Magpie"], [name:"individualCount", value:"10"]], userA)
+            allFields.addAll(fields1)
+            List fields2 = transcribeTask(task, [[name:"scientificName", value:"Crow"], [name:"individualCount", value:"5"]], userB)
+            allFields.addAll(fields2)
+        }
+
+        List<Task> taskList = project.tasks as List
+        List<String> fieldNames = taskOrTranscriptionFields + ["scientificName", "individualCount"]
+
+        when:
+        service.export_default(project, taskList, fieldNames, allFields, response)
+        List results = new CSVMapReader(new StringReader(response.text)).readAll()
+
+        then:
+        1 * fieldService.getMaxRecordIndexByFieldForProject(project) >> [['scientificName', 0], ['individualCount', 0]]
+        1 * taskService.getUserMapFromTaskList(taskList) >> [(userA):[displayName:userA], (userB):[displayName:userB]]
+
+        and:
+        results.size() == 200
+        for (int i= 1; i <= numOfTasks; i++) {
+            results.findAll { it.externalIdentifier == "image${i}.jpq" }.size() == 2
+        }
+        results.findAll{it.transcriberID == userA}.size() == numOfTasks
+        results.findAll{it.transcriberID == userB}.size() == numOfTasks
+    }
+
+    def "Test parrallel writes is working for larger tasks"() {
+        setup:
+        grailsApplication.config.exportCSVThreadPoolSize = 10
+        service.grailsApplication = grailsApplication
+
+        project.transcriptionsPerTask = 2
+        String userA = 'userA'
+        String userB = 'userB'
+
+        List allFields = new ArrayList()
+
+        int numOfTasks = 100
+        for (int i= 1; i <= numOfTasks; i++) {
+            Task task = createTask("image${i}.jpq")
+            List fields1 = transcribeTask(task, [[name:"scientificName", value:"Magpie"], [name:"individualCount", value:"10"]], userA)
+            allFields.addAll(fields1)
+            List fields2 = transcribeTask(task, [[name:"scientificName", value:"Crow"], [name:"individualCount", value:"5"]], userB)
+            allFields.addAll(fields2)
+        }
+
+        List<Task> taskList = project.tasks as List
+        List<String> fieldNames = taskOrTranscriptionFields + ["scientificName", "individualCount"]
+
+        when:
+        service.export_default(project, taskList, fieldNames, allFields, response)
+        List results = new CSVMapReader(new StringReader(response.text)).readAll()
+
+        then:
+        1 * fieldService.getMaxRecordIndexByFieldForProject(project) >> [['scientificName', 0], ['individualCount', 0]]
+        1 * taskService.getUserMapFromTaskList(taskList) >> [(userA):[displayName:userA], (userB):[displayName:userB]]
+
+        and:
+        results.size() == 200
+        for (int i= 1; i <= numOfTasks; i++) {
+            results.findAll { it.externalIdentifier == "image${i}.jpq" }.size() == 2
+        }
+        results.findAll{it.transcriberID == userA}.size() == 100
+        results.findAll{it.transcriberID == userB}.size() == 100
+    }
+
+    def "All project transcription tasks data can be exported in CSV form for multiple transcription project"() {
+        setup:
+        project.transcriptionsPerTask = 2
+        String userA = 'userA'
+        String userB = 'userB'
+        Task birdTask = createTask()
+        Task kangarooTask = createTask()
+        List birdFields1 = transcribeTask(birdTask, [[name:"scientificName", value:"Magpie"], [name:"individualCount", value:"10"]], userA)
+        List birdFields2 = transcribeTask(birdTask, [[name:"scientificName", value:"Crow"], [name:"individualCount", value:"5"]], userB)
+        List kangarooFields1 = transcribeTask(kangarooTask, [[name:"scientificName", value:"Red Kangaroo"], [name:"individualCount", value:"2"]], userA)
+        List kangarooFields2 = transcribeTask(kangarooTask, [[name:"scientificName", value:"Red Kangaroo"], [name:"individualCount", value:"2"]], userB)
+
+        List<Task> taskList = project.tasks as List
+        List<String> fieldNames = taskOrTranscriptionFields + ["scientificName", "individualCount"]
+
+        when:
+        service.export_default(project, taskList, fieldNames, birdFields1 + birdFields2 + kangarooFields1 + kangarooFields2, response)
+        List results = new CSVMapReader(new StringReader(response.text)).readAll()
+
+        then:
+        1 * fieldService.getMaxRecordIndexByFieldForProject(project) >> [['scientificName', 0], ['individualCount', 0]]
+        1 * taskService.getUserMapFromTaskList(taskList) >> [(userA):[displayName:userA], (userB):[displayName:userB]]
+
+        and:
+        results.size() == 4 // not counting headers
+        results.findAll{it.transcriberID == userA && it.taskID == birdTask.id.toString() && it.scientificName == 'Magpie'}.size() == 1
+        results.findAll{it.transcriberID == userA && it.taskID == kangarooTask.id.toString() && it.scientificName == 'Red Kangaroo'}.size() == 1
+        results.findAll{it.transcriberID == userB && it.taskID == birdTask.id.toString() && it.scientificName == 'Crow'}.size() == 1
+        results.findAll{it.transcriberID == userB && it.taskID == kangarooTask.id.toString() && it.scientificName == 'Red Kangaroo'}.size() == 1
+    }
+
+    def "Partially transcribed project tasks data can be exported in CSV form for multiple transcription project"() {
+        setup:
+        project.transcriptionsPerTask = 2
+        String userA = 'userA'
+        String userB = 'userB'
+        Task birdTask = createTask()
+        Task kangarooTask = createTask()
+        List birdFields1 = transcribeTask(birdTask, [[name:"scientificName", value:"Magpie"], [name:"individualCount", value:"10"]], userA)
+        List kangarooFields1 = transcribeTask(kangarooTask, [[name:"scientificName", value:"Red Kangaroo"], [name:"individualCount", value:"2"]], userB)
+
+        List<Task> taskList = project.tasks as List
+        List<String> fieldNames = taskOrTranscriptionFields + ["scientificName", "individualCount"]
+
+        when:
+        service.export_default(project, taskList, fieldNames, birdFields1 + kangarooFields1, response)
+        List results = new CSVMapReader(new StringReader(response.text)).readAll()
+
+        then:
+        1 * fieldService.getMaxRecordIndexByFieldForProject(project) >> [['scientificName', 0], ['individualCount', 0]]
+        1 * taskService.getUserMapFromTaskList(taskList) >> [(userA):[displayName:userA], (userB):[displayName:userB]]
+
+        and:
+        results.size() == 2 //not counting headers
+        results.findAll{it.transcriberID == userA && it.taskID == birdTask.id.toString() && it.scientificName == 'Magpie'}.size() == 1
+        results.findAll{it.transcriberID == userB && it.taskID == kangarooTask.id.toString()  && it.scientificName == 'Red Kangaroo'}.size() == 1
+    }
+
+    def "For multiple transcription project with tasks that have not been transcribed, data can be exported in CSV"() {
+        setup:
+        project.transcriptionsPerTask = 2
+        String macpieImage = 'macpieImage.jpg'
+        String kangarooImage = 'kangarooImage.jpg'
+        Task birdTask = createTask(macpieImage)
+        Task kangarooTask = createTask(kangarooImage)
+
+        List<Task> taskList = project.tasks as List
+        List<String> fieldNames = taskOrTranscriptionFields
+
+        when:
+        service.export_default(project, taskList, fieldNames, [], response)
+        List results = new CSVMapReader(new StringReader(response.text)).readAll()
+
+        then:
+        1 * fieldService.getMaxRecordIndexByFieldForProject(project) >> []
+        1 * taskService.getUserMapFromTaskList(taskList) >> [:]
+
+        and:
+        results.size() == 2
+        results.find{it.taskID == kangarooTask.id.toString()}.externalIdentifier == kangarooImage
+        results.find{it.taskID == birdTask.id.toString()}.externalIdentifier == macpieImage
+    }
+
+    def "For single transcription project with tasks that have not been transcribed, data can be exported in CSV"() {
+        setup:
+        project.transcriptionsPerTask = 1
+        String macpieImage = 'macpieImage.jpg'
+        Task birdTask = createTask(macpieImage)
+
+        List<Task> taskList = project.tasks as List
+        List<String> fieldNames = taskOrTranscriptionFields
+
+        when:
+        service.export_default(project, taskList, fieldNames, [], response)
+        List results = new CSVMapReader(new StringReader(response.text)).readAll()
+
+        then:
+        1 * fieldService.getMaxRecordIndexByFieldForProject(project) >> []
+        1 * taskService.getUserMapFromTaskList(taskList) >> [:]
+
+        and:
+        results.size() == 1
+        results[0].externalIdentifier == macpieImage
+    }
+
+
     def "All project task data can be exported in CSV form for single transcription projects"() {
         setup:
         String userId = '1234'
