diff --git a/grails-app/controllers/au/org/ala/merit/AdminController.groovy b/grails-app/controllers/au/org/ala/merit/AdminController.groovy index a8d54a126..f1c5f23ab 100644 --- a/grails-app/controllers/au/org/ala/merit/AdminController.groovy +++ b/grails-app/controllers/au/org/ala/merit/AdminController.groovy @@ -418,7 +418,7 @@ class AdminController { session.status = status def fileIn = new FileInputStream(file) try { - def result = importService.gmsImport(fileIn, status.projects, preview, update) + def result = importService.projectImport(fileIn, status.projects, preview, update) status.finished = true status.error = result.error } diff --git a/grails-app/services/au/org/ala/merit/ImportService.groovy b/grails-app/services/au/org/ala/merit/ImportService.groovy index f8b3cc60c..ca9e4a0fb 100644 --- a/grails-app/services/au/org/ala/merit/ImportService.groovy +++ b/grails-app/services/au/org/ala/merit/ImportService.groovy @@ -297,7 +297,7 @@ class ImportService { organisations += metadataService.organisationList()?.list } - Map gmsImport(InputStream csv, List status, Boolean preview, Boolean update, String charEncoding = 'Cp1252') { + Map projectImport(InputStream csv, List status, Boolean preview, Boolean update, String charEncoding = 'Cp1252') { Map programs = [:].withDefault{name -> programService.getByName(name) @@ -307,8 +307,7 @@ class ImportService { mu?.managementUnitId } refreshOrganisationList() - def mapper = new GmsMapper(metadataService.activitiesModel(), metadataService.programsModel(), organisations, abnLookupService, metadataService.getOutputTargetScores(), programs, managementUnits) - + def mapper = new GmsMapper(metadataService.activitiesModel(), metadataService.programsModel(), organisations, abnLookupService, metadataService.getOutputTargetScores(), programs, managementUnits, false, update) def action = preview?{rows -> mapProjectRows(rows, status, mapper, update)}:{rows -> importAll(rows, status, mapper, update)} Map result = [:] @@ -360,7 +359,7 @@ class ImportService { def mapProjectRows(projectRows, List status, GmsMapper mapper, Boolean update) { - Map mappingResults = mapper.mapProject(projectRows) + Map mappingResults = mapper.mapProject(projectRows, update) String grantId = mappingResults.project.grantId String externalId = mappingResults.project.externalId @@ -380,7 +379,7 @@ class ImportService { void importAll(projectRows, List status, GmsMapper mapper, Boolean update) { - def projectDetails = mapper.mapProject(projectRows) + def projectDetails = mapper.mapProject(projectRows, update) def grantId = projectDetails.project.grantId?:'' def externalId = projectDetails.project.externalId?:'' @@ -394,9 +393,6 @@ class ImportService { def editorEmail = projectDetails.project.remove('editorEmail') def editorEmail2 = projectDetails.project.remove('editorEmail2') - //When projects are loaded into MERIT via CSV upload, they are given a status of "Application". - projectDetails.project.status ?: 'application' - // Create the organisation first so we can link it to the project. if (projectDetails.organisation) { Map orgCreationResult = organisationService.update(null, projectDetails.organisation) @@ -409,7 +405,7 @@ class ImportService { } } - def result = importProject(projectDetails.project, update) // Do not overwrite existing projects because of the impacts to sites / activities etc. + def result = importProject(projectDetails.project, update) if (result.project == 'existing' && !update) { status << [grantId:grantId, externalId:externalId, success:false, errors:['Project already exists in MERIT, skipping']] diff --git a/grails-app/views/admin/import.gsp b/grails-app/views/admin/import.gsp index fad01a314..5df7ef77e 100644 --- a/grails-app/views/admin/import.gsp +++ b/grails-app/views/admin/import.gsp @@ -34,9 +34,7 @@

- Please note this function will replace all project information for each project.
- It is designed for use with grants hub data and to fix newly loaded projects. Please test changes - to existing MERIT projects in the staging system before running the update in production. + Please test changes to existing MERIT projects in the staging system before running the update in production.

diff --git a/src/integration-test/groovy/au/org/ala/fieldcapture/ImportProjectsSpec.groovy b/src/integration-test/groovy/au/org/ala/fieldcapture/ImportProjectsSpec.groovy index f83d2a156..12fef4bc9 100644 --- a/src/integration-test/groovy/au/org/ala/fieldcapture/ImportProjectsSpec.groovy +++ b/src/integration-test/groovy/au/org/ala/fieldcapture/ImportProjectsSpec.groovy @@ -49,6 +49,7 @@ class ImportProjectsSpec extends StubbedCasSpec { and: List rows2 = projectResults() rows2.size() == 2 + rows2[1].success == 'Yes' when: to Organisation, 'test_organisation' @@ -75,6 +76,8 @@ class ImportProjectsSpec extends StubbedCasSpec { and: "The data is relevant to the projects loaded" List rows = projectResults() rows.size() == 3 + rows[1].success == 'Yes' + rows[2].success == 'Yes' when: importProjects() @@ -125,6 +128,7 @@ class ImportProjectsSpec extends StubbedCasSpec { and: List rows2 = projectResults() rows2.size() == 2 + rows2[1].success == 'Yes' when: "We navigate to the program page to find the new imported project, then open it" to ProgramPage, 'configurable_meri_plan' @@ -152,4 +156,59 @@ class ImportProjectsSpec extends StubbedCasSpec { adminContent.meriPlan.budget[0].budgetAmounts()*.value() == ["20000", "10000"] } + + def "Projects can be updated via import with only the fields provided in the spreadsheet being updated"() { + + setup: + File csv = new File(getClass().getResource("/grants-hub-update-data.csv").toURI()) + loginAsMeritAdmin(browser) + + when: + to ProjectImport + checkUpdateCheckbox() + attachFile(csv) + + then: "The projects are validated and the validation results are displayed" + waitFor{validateComplete()} + + and: "The data is relevant to the projects loaded" + projectResults().size() == 2 + + when: + importProjects() + + then: + waitFor{loadComplete()} + and: + List rows2 = projectResults() + rows2.size() == 2 + rows2[1].success == 'Yes' + + when: "We navigate to the program page to find the new imported project, then open it" + to ProgramPage, 'configurable_meri_plan' + openProjectByGrantId('cep-1') + + then: + at RlpProjectPage + + when: + displayOverview() + + then: + overview.program.text() == "Configurable MERI Plan Program" + overview.projectId.text() == "cep-1" + overview.status.text().equalsIgnoreCase("Active") + overview.externalIds*.text() == ["1234"] + overview.description.text() == "Grants project description - updated" + + when: + openMeriPlanEditTab() + + then: + adminContent.meriPlan.budget.size() == 1 + adminContent.meriPlan.budget[0].description.value() == "Project funding" + adminContent.meriPlan.budget[0].budgetAmounts()*.value() == ["1", "2"] + + } + } diff --git a/src/integration-test/groovy/pages/ProjectImport.groovy b/src/integration-test/groovy/pages/ProjectImport.groovy index 67aa11223..ab7cde215 100644 --- a/src/integration-test/groovy/pages/ProjectImport.groovy +++ b/src/integration-test/groovy/pages/ProjectImport.groovy @@ -1,6 +1,7 @@ package pages import geb.Page +import geb.module.Checkbox class ProjectImport extends Page { @@ -12,6 +13,7 @@ class ProjectImport extends Page { fileInput { $('#fileUpload') } importButton { $('button[data-bind*=doImport]')} progressSummary { $('span[data-bind*=progressSummary]') } + updateCheckbox { $('#update').module(Checkbox) } } def attachFile(File file) { @@ -22,7 +24,13 @@ class ProjectImport extends Page { importButton.click() } + def checkUpdateCheckbox() { + updateCheckbox.check() + } + List projectResults() { + + List columns = ["grantId", "externalId", "success", "errors", "messages"] List rows = [] def progressTable = $('table.table') progressTable.find("tbody tr").each { @@ -30,7 +38,11 @@ class ProjectImport extends Page { it.find("td").each { col -> cols << col.text() } - rows << cols + Map row = [:] + columns.eachWithIndex{ col, index -> + row[col] = cols[index] + } + rows << row } rows diff --git a/src/integration-test/resources/grants-hub-update-data.csv b/src/integration-test/resources/grants-hub-update-data.csv new file mode 100644 index 000000000..66c5c3b95 --- /dev/null +++ b/src/integration-test/resources/grants-hub-update-data.csv @@ -0,0 +1,2 @@ +APP_ID,EXTERNAL_ID,APP_NM,APP_DESC,PROGRAM_NM,ABN,ORG_ID,START_DT,FINISH_DT,ORDER_NO,FUNDING,AUTHORISEDP_EMAIL,GRANT_MGR_EMAIL,GRANT_MGR_EMAIL_2,APPLICANT_EMAIL,ADMIN_EMAIL,EDITOR_EMAIL,EDITOR_EMAIL_2,TAGS,PROJECT_STATUS,MERI_PLAN_STATUS,FUNDING_TYPE,ORIGIN_SYSTEM,FINANCIAL_YEAR_FUNDING_DESCRIPTION,FUNDING_21_22,FUNDING_22_23 +cep-1,,,Grants project description - updated,,,,,,,,,,,,,,,,Active,,,,Project funding,1,2 \ No newline at end of file diff --git a/src/main/groovy/au/org/ala/merit/GmsMapper.groovy b/src/main/groovy/au/org/ala/merit/GmsMapper.groovy index 1e58bd763..b3580394b 100644 --- a/src/main/groovy/au/org/ala/merit/GmsMapper.groovy +++ b/src/main/groovy/au/org/ala/merit/GmsMapper.groovy @@ -2,7 +2,7 @@ package au.org.ala.merit import au.com.bytecode.opencsv.CSVWriter import org.apache.commons.lang.WordUtils -import org.apache.commons.validator.EmailValidator +import org.apache.commons.validator.routines.EmailValidator import java.text.DecimalFormat import java.text.SimpleDateFormat @@ -95,6 +95,7 @@ class GmsMapper { def geographicInfoMapping = [ NATIONWIDE:[name:'nationwide', type:'boolean', description:'If true, this project does not have a primary state'], + STATEWIDE:[name:'statewide', type:'boolean', description:'If true, this project does not have a primary electorate'], PRIMARY_STATE:[name:'primaryState', type:'string', description:'The primary state to be manually assigned to this project'], PRIMARY_ELECTORATE:[name:'primaryElectorate', type:'string', description:'The primary electorate to be manually assigned to this project'], OTHER_ELECTORATES:[name:'otherElectorates', type:'list', description:'Other electorates to be manually assigned to this project. Enter as a comma separated list'], @@ -135,6 +136,8 @@ class GmsMapper { ] private boolean includeProgress + /** If the mapping is being done to update a project, mandatory fields aren't required as they won't be updated if absent */ + private boolean mapForUpdate = false public GmsMapper() { this.activitiesModel = [] @@ -144,9 +147,10 @@ class GmsMapper { this.programs = [:] this.managementUnits = [:] includeProgress = false + mapForUpdate = false } - GmsMapper(activitiesModel, programModel, organisations, abnLookup, List scores, Map programs = [:], Map managementUnits = [:], includeProgress = false) { + GmsMapper(activitiesModel, programModel, organisations, abnLookup, List scores, Map programs = [:], Map managementUnits = [:], includeProgress = false, mapForUpdate = false) { this.activitiesModel = activitiesModel this.programModel = programModel this.includeProgress = includeProgress @@ -155,6 +159,7 @@ class GmsMapper { this.programs = programs this.managementUnits = managementUnits this.abnLookupService = abnLookup + this.mapForUpdate = mapForUpdate } /** Creates a CSV file in the format required to import projects into MERIT with additional instructions */ @@ -222,29 +227,21 @@ class GmsMapper { return errors } - def mapProject(projectRows) { + def mapProject(projectRows, boolean update = false) { List errors = [] List messages = [] - Map result = gmsToMerit(projectRows[0], projectMapping) // All project rows have the project details. + Map result = csvToProjectProperties(projectRows[0], projectMapping, update) // All project rows have the project details. def project = result.mappedData - project.projectType = 'works' - project.isMERIT = true + if (!update) { + // Set some default project properties for the MERIT hub + project.projectType = 'works' + project.isMERIT = true + } - mapGeographicInfo(projectRows[0], project, errors) + mapGeographicInfo(projectRows[0], project, update, errors) - - String programName = project.associatedSubProgram ?: project.associatedProgram - Map program = programs[programName] - String programId = program?.programId - if (programId) { - project.remove('associatedProgram') - project.remove('associatedSubProgram') - project.programId = programId - } - else { - errors << "Program ${programName} doesn't match an existing MERIT programme" - } + mapProgram(project, update, errors) if (project.managementUnitName) { String managementUnitName = project.remove('managementUnitName') @@ -254,10 +251,9 @@ class GmsMapper { } } - Map organisation = mapOrganisation(project, program, errors, messages) + Map organisation = mapOrganisation(project, update, errors, messages) errors.addAll(result.errors) - project.planStatus = project.planStatus ?: 'not approved' mapRisks(projectRows, project, errors) @@ -270,10 +266,51 @@ class GmsMapper { project.custom = [details:meriPlan] } + if (update) { + Map flatProperties = flattenMap(project) + Set propertiesToUpdate = flatProperties.keySet() + propertiesToUpdate.remove('grantId') // This identifies the project and won't be updated. + messages << "Update: ${propertiesToUpdate}" + + } + [project:project, sites:sites, activities:activities, errors:errors, messages:messages, organisation:organisation] } + private Map flattenMap(Map map, String separator = '.') { + map.collectEntries { k, v -> + if (v instanceof Map) { + flattenMap(v, separator).collectEntries { k1, v1 -> + String newKey = k+separator+k1 + [(newKey): v1] + } + } else { + [(k): v] + } + } + } + + private Map mapProgram(LinkedHashMap project, boolean update, ArrayList errors) { + String programName = project.associatedSubProgram ?: project.associatedProgram + if (!programName) { + if (!update) { + errors << "Please supply a program name (PROGRAM_NM) or sub-program name (ROUND_NM) for the project" + } + return + } + Map program = programs[programName] + String programId = program?.programId + if (programId) { + project.remove('associatedProgram') + project.remove('associatedSubProgram') + project.programId = programId + } else { + errors << "Program ${programName} doesn't match an existing MERIT programme" + } + program + } + private Map findExistingOrganisation(String organisationId, String abn) { organisations.find{ (abn && (it.abn == abn)) || @@ -281,7 +318,7 @@ class GmsMapper { } - private Map mapOrganisation(Map project, Map program, List errors, List messages) { + private Map mapOrganisation(Map project, boolean update, List errors, List messages) { Map organisation Map abnLookup String error = null @@ -339,7 +376,7 @@ class GmsMapper { project.associatedOrgs = [ [organisationId:organisation.organisationId, name: contractName ?: organisation.name, description:description]] } - } else { + } else if (!update) { error = "Please supply an organisationId (ORG_ID) or ABN (ABN) for the project" } if (error) { @@ -395,7 +432,7 @@ class GmsMapper { def count = siteRows.size() siteRows.eachWithIndex {siteRow, i -> - def siteResult = gmsToMerit(siteRow, siteMapping) + def siteResult = csvToProjectProperties(siteRow, siteMapping) def site = siteResult.mappedData errors.addAll(siteResult.errors) @@ -432,7 +469,7 @@ class GmsMapper { def activityRows = projectRows.findAll{it[DATA_TYPE_COLUMN] == ACTIVITY_DATA_TYPE && it[DATA_SUB_TYPE_COLUMN] == ACTIVITY_DATA_SUB_TYPE} def activities = [] activityRows.eachWithIndex { activityRow, i -> - def activityResult = gmsToMerit(activityRow, activityMapping) + def activityResult = csvToProjectProperties(activityRow, activityMapping) def mappedActivity = activityResult.mappedData errors.addAll(activityResult.errors) @@ -526,7 +563,7 @@ class GmsMapper { def risks = [] riskRows.eachWithIndex { riskRow, i -> - def result = gmsToMerit(riskRow, riskMapping) + def result = csvToProjectProperties(riskRow, riskMapping) errors.addAll(result.errors) def risk = [ "threat" : "", @@ -546,19 +583,19 @@ class GmsMapper { } } - private void mapGeographicInfo(Map rowData, Map project, List errors) { + private void mapGeographicInfo(Map rowData, Map project, boolean update, List errors) { - Map result = gmsToMerit(rowData, geographicInfoMapping) + Map result = csvToProjectProperties(rowData, geographicInfoMapping, update) if (result.mappedData) { project.geographicInfo = result.mappedData } - errors.addAll(errors) + errors.addAll(result.errors) } private def mapTarget(rowMap) { def errors = [] - def map = gmsToMerit(rowMap, outputTargetColumnMapping) + def map = csvToProjectProperties(rowMap, outputTargetColumnMapping) def target = map.mappedData errors.addAll(map.errors) @@ -594,7 +631,7 @@ class GmsMapper { [mappedData:result, errors: errors] } - private def gmsToMerit(Map rowMap, Map mapping) { + private def csvToProjectProperties(Map rowMap, Map mapping, boolean update = false) { def result = [:] def errors = [] mapping.each { Map.Entry e -> @@ -605,16 +642,20 @@ class GmsMapper { String propertyName = columnMapping.name if (columnMapping.multipleColumnsSupported) { List values = findMultiValue(mappingColumnName, rowMap) - if (!result[propertyName]) { + values = values.findAll{it} + if (!result[propertyName] && (values || !update)) { result[propertyName] = [] } - values = values.findAll{it} - result[propertyName].addAll(values.collect{convertByType(it, columnMapping)}.findAll{it}) + if (values || !update) { + result[propertyName].addAll(values.collect{convertByType(it, columnMapping, update)}.findAll{it}) + } } else { - result[propertyName] = convertByType(rowMap[mappingColumnName], columnMapping) + def value = rowMap[mappingColumnName]?.trim() + if (value || !update) { // If we are doing an update, ignore empty values. + result[propertyName] = convertByType(rowMap[mappingColumnName], columnMapping, update) + } } - } catch (Exception ex) { errors << "Error converting value: ${rowMap[mappingColumnName]} from row ${rowMap.index} column: ${mappingColumnName}, ${ex.getMessage()}" @@ -627,22 +668,25 @@ class GmsMapper { private List findMultiValue(String mappingColumnName, Map rowMap) { rowMap.findAll { String key, def value -> key.startsWith(mappingColumnName) - }.collect{it.value} + }.collect{it.value?.trim()} } - private def convertByType(String value, Map mapping) { + private def convertByType(String value, Map mapping, boolean update) { String type = mapping.type value = value?value.trim():'' def result = null switch (type) { case 'date': - result = convertDate(value, mapping.mandatory) + result = convertDate(value, mapping.mandatory && !update) break case 'decimal': result = convertDecimal(value) break case 'string': - result = value ?: (mapping['default']?:value) + result = value + if (!result && !update) { + result = mapping['default']?:value + } break case 'url': URI.create(value) // validation purposes only @@ -669,6 +713,9 @@ class GmsMapper { case 'list': result = value?.split(',').collect{it.trim()}.findAll{it} break + case 'boolean': + result = value?.toBoolean() // accept "y"/"true"/"1" as true + break default: throw new IllegalArgumentException("Unsupported type: ${type}") } diff --git a/src/main/scripts/releases/4.3/adhoc/migrateProjectOverrideSettings.js b/src/main/scripts/releases/4.3/adhoc/migrateProjectOverrideSettings.js new file mode 100644 index 000000000..cbc51e720 --- /dev/null +++ b/src/main/scripts/releases/4.3/adhoc/migrateProjectOverrideSettings.js @@ -0,0 +1,20 @@ +load('../../../utils/audit.js'); +let projects = db.project.find({$or:[{'geographicInfo.primaryState':{$ne:null}}, {'geographicInfo.primaryElectorate':{$ne:null}}]}); +while (projects.hasNext()) { + let project = projects.next(); + let changed = false; + if (project.geographicInfo.primaryState && !project.geographicInfo.overridePrimaryState) { + project.geographicInfo.overridePrimaryState = true; + changed = true; + } + if (project.geographicInfo.primaryElectorate && !project.geographicInfo.overridePrimaryElectorate) { + project.geographicInfo.overridePrimaryElectorate = true; + changed = true; + } + if (changed) { + print("Updating project " + project.projectId); + db.project.updateOne({projectId: project.projectId}, {$set: {geographicInfo: project.geographicInfo}}); + audit(project, project.projectId, 'au.org.ala.ecodata.Project', "system"); + } + +} \ No newline at end of file diff --git a/src/test/groovy/au/org/ala/merit/GmsMapperSpec.groovy b/src/test/groovy/au/org/ala/merit/GmsMapperSpec.groovy index 2ecd6d5e6..258ecc811 100644 --- a/src/test/groovy/au/org/ala/merit/GmsMapperSpec.groovy +++ b/src/test/groovy/au/org/ala/merit/GmsMapperSpec.groovy @@ -30,7 +30,7 @@ class GmsMapperSpec extends Specification{ activitiesModel = JSON.parse(new InputStreamReader(getClass().getResourceAsStream('/activities-model.json'))) Map programModel = [programs:[[name:'Green Army']]] List organisations = [[ organisationId: "123", name:'Test org 1', abn:'12345678901'], [organisationId:'2', name:"Org 2", abn:""]] - gmsMapper = new GmsMapper(activitiesModel, programModel, organisations, abnLookupService, scores, programs) + gmsMapper = new GmsMapper(activitiesModel, programModel, organisations, abnLookupService, scores, programs, [:], false, false) } /** @@ -412,4 +412,19 @@ class GmsMapperSpec extends Specification{ [idType:'GRANT_AWARD', externalId: 'g1'], [idType:'GRANT_AWARD', externalId: 'g2']] !result.errors } + + def "The GMSMapper can update multi-column values such as the externalIds"() { + when: + Map project = [APP_ID:'g1', PROGRAM_NM:"Green Army", ORG_TRADING_NAME:'Test org 1', ABN:'12345678901', FUNDING_TYPE:"RLP", START_DT:'2019/07/01', FINISH_DT:'2020/07/01'] + Map idData = [ORDER_NO:'o1', ORDER_NO_2:'o2', WORK_ORDER_ID:'w1', GRANT_AWARD_ID:'g1', GRANT_AWARD_ID_2:'g2', TECH_ONE_ID:'t1', TECH_ONE_ID_2:'t2'] + project += idData + Map result = gmsMapper.mapProject([project], true) + + then: + result.project.externalIds == [[idType:'INTERNAL_ORDER_NUMBER', externalId: 'o1'], [idType:'INTERNAL_ORDER_NUMBER', externalId: 'o2'], + [idType:'TECH_ONE_CODE', externalId: 't1'], [idType:'TECH_ONE_CODE', externalId: 't2'], + [idType:'WORK_ORDER', externalId: 'w1'], + [idType:'GRANT_AWARD', externalId: 'g1'], [idType:'GRANT_AWARD', externalId: 'g2']] + !result.errors + } } diff --git a/src/test/groovy/au/org/ala/merit/ImportServiceSpec.groovy b/src/test/groovy/au/org/ala/merit/ImportServiceSpec.groovy index 298ee7847..e453557a0 100644 --- a/src/test/groovy/au/org/ala/merit/ImportServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/merit/ImportServiceSpec.groovy @@ -114,7 +114,7 @@ class ImportServiceSpec extends Specification implements ServiceUnitTest> [list:[[name:"Test Organisation 2", organisationId:'org2Id', abn:'12345678901']]] @@ -190,7 +190,7 @@ class ImportServiceSpec extends Specification implements ServiceUnitTest> [list:[[name:"Test Organisation 2", organisationId:'org2Id', abn:'12345678901']]] @@ -230,7 +230,7 @@ class ImportServiceSpec extends Specification implements ServiceUnitTest