Merge branch 'develop'

Author: qifeng-bai

README.md

@@ -44,5 +44,15 @@ Change logs
 DB schema update:
 ``` ALTER TABLE alerts.query_result MODIFY logs TEXT NULL; ```
 
-### 4.2.0 Release
+### 4.3.0 Release
 No DB changes
+
+
+### 4.4.0 Release
+Apply new templates for alerts
+
+#### Change logs
+Database update: Check release/4.4.0-release.sql
+
+
+

build.gradle

@@ -11,7 +11,7 @@
  * rights and limitations under the License.
  */
 buildscript {
-    version "4.2.4"
+    version "4.4.0-SNAPSHOT"
     group "au.org.ala"
 }
 
@@ -54,8 +54,8 @@ bootWar {
 repositories {
     repositories {
         mavenLocal()
-        maven { url "https://nexus.ala.org.au/content/groups/public/" }
         mavenCentral()
+        maven { url "https://nexus.ala.org.au/content/groups/public/" }
         maven { url "https://repo.grails.org/grails/core/" }
     }
     mavenCentral()
@@ -127,6 +127,8 @@ dependencies {
     implementation "org.grails.plugins:ala-ws-plugin:$alaSecurityLibsVersion"
     implementation "au.org.ala:userdetails-service-client:$alaSecurityLibsVersion"
     implementation "org.grails.plugins:ala-admin-plugin:2.3.0"
+    implementation "au.org.ala.plugins:openapi:1.3.0"
+    implementation "org.grails.plugins:aws-sdk-s3:2.4.8"
 
     // for ContentType
     implementation "org.apache.httpcomponents:httpcore:4.4.16"
@@ -136,7 +138,7 @@ dependencies {
     //Using AWS SES as email provider
     implementation 'org.grails.plugins:mail:4.0.0'
     implementation 'au.org.ala:ala-mail:1.0.0-SNAPSHOT'
-    implementation group: 'com.amazonaws', name: 'aws-java-sdk-sts', version: "1.12.759"
+    implementation 'com.amazonaws:aws-java-sdk-sts:1.12.759'
 
     implementation 'org.grails.plugins:schwartz-monitor:2.0.1.ALA-SNAPSHOT'
     implementation 'org.grails.plugins:quartz:2.0.13'
@@ -154,13 +156,14 @@ dependencies {
     testRuntimeOnly("org.seleniumhq.selenium:selenium-firefox-driver:4.10.0")
     testRuntimeOnly("org.seleniumhq.selenium:selenium-safari-driver:4.10.0")
     runtimeOnly 'commons-lang:commons-lang:2.6'
-    implementation 'commons-io:commons-io:2.11.0'
+    implementation 'commons-codec:commons-codec:1.17.1'
+    implementation 'commons-io:commons-io:2.17.0'
+    implementation 'org.apache.commons:commons-csv:1.12.0'
     runtimeOnly 'org.hamcrest:hamcrest-core:2.2'
     runtimeOnly 'org.hamcrest:hamcrest-library:2.2'
     runtimeOnly 'xalan:xalan:2.7.3'
     implementation 'com.jayway.jsonpath:json-path:2.7.0'
     implementation 'com.jayway.jsonpath:json-path-assert:2.7.0'
-    implementation 'au.org.ala.plugins:openapi:1.3.0'
 }
 //
 //bootRun {

cicd/deploy/config.ini cicd/backend/config.ini

@@ -1,22 +1,25 @@
 [DEFAULT]
-PRODUCT_COMPONENT = deploy
+PRODUCT_COMPONENT = backend
 PIPELINE_STACK_NAME = ala-${PRODUCT_NAME}-${PRODUCT_COMPONENT}-pipeline-${ENVIRONMENT}
-BASE_STACK_NAME = ala-${PRODUCT_NAME}-base-${ENVIRONMENT}
 DATABASE_STACK_NAME = ala-${PRODUCT_NAME}-database-${ENVIRONMENT}
-AUTO_DEPLOY = false
+HELM_RELEASE_NAME = ala-alerts-${CLEAN_BRANCH}
+DOMAIN_NAME = ${PRODUCT_NAME}.${HOSTED_ZONE}
+AUTO_DEPLOY = true
 SLACK_DEPLOY_NOTIFICATION = false
 SLACK_ALERT_CHANNEL = deployments
 
 [development]
 # code pipeline
 PIPELINE_STACK_NAME = ala-${PRODUCT_NAME}-${PRODUCT_COMPONENT}-pipeline-${CLEAN_BRANCH}
-BASE_STACK_NAME = ala-${PRODUCT_NAME}-base-${CLEAN_BRANCH}
-DATABASE_STACK_NAME = ala-${PRODUCT_NAME}-database-${CLEAN_BRANCH}
+; Update the database stack name if creating a new database stack for feature branch
+DATABASE_STACK_NAME = ala-${PRODUCT_NAME}-database-develop
 SLACK_DEPLOY_NOTIFICATION = true
 SLACK_ALERT_CHANNEL = zabbix-alerts
-AUTO_DEPLOY = false
+DOMAIN_NAME = ${PRODUCT_NAME}-${CLEAN_BRANCH}.${HOSTED_ZONE}
 
 [testing]
+SLACK_DEPLOY_NOTIFICATION = true
+SLACK_ALERT_CHANNEL = zabbix-alerts
 
 [staging]
 

cicd/build/pipeline/build_alerts_buildspec.yaml cicd/backend/pipeline/build_alerts_buildspec.yaml

@@ -4,52 +4,51 @@ env:
   shell: bash
   variables:
     JAVA_TOOL_OPTIONS: -Dhttps.protocols=TLSv1.2
+  exported-variables:
+    - BUILD_TAG
 
 phases:
   install:
+    runtime-versions:
+      java: corretto11
     commands:
       - echo Installing dependencies...
       - apt update -y
         && cat /etc/lsb-release
-        && apt-get -q -y install openjdk-11-jdk
-        && apt-get -q -y install unzip
-        && apt-get -q -y install zip
-        && apt-get -q -y install curl
         && curl -s "https://get.sdkman.io" | bash
         && chmod a+x /root/.sdkman/bin/sdkman-init.sh
         && source /root/.sdkman/bin/sdkman-init.sh && sdk install grails 6.0.0 /root/.sdkman/candidates/grails/6.0.0
-        && apt-get install -y apt-transport-https ca-certificates curl software-properties-common
+        && apt-get install -y apt-transport-https ca-certificates software-properties-common
         && curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
         && add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
         && apt-get update
         && apt-get install -y docker-ce
+      - java -version
       - ./gradlew clean
       - ./gradlew assemble
 
   build:
     commands:
       - echo Build started on $(date)
       - echo Building the project...
-      - ./gradlew build
+      - ./gradlew build --stacktrace
       - ./gradlew bootWar
 
   post_build:
     commands:
       - echo Post-build phase...
       - echo Build completed on $(date)
-#      - aws s3 rm s3://ala-alerts-builds/devbuilds/$CLEAN_BRANCH/ --recursive
-#      - aws s3 cp build/libs/*.war s3://ala-alerts-builds/devbuilds/$CLEAN_BRANCH/
       - echo Logging in to Amazon ECR...
       - aws ecr get-login-password | docker login -u AWS --password-stdin "https://$AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com"
       - rm -rf build/libs/*-plain.war
       - echo Building the Docker image...
       - docker build -t ala-alerts .
       - echo Tagging the Docker image...
-      - export REPO=$AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/$ALERTS_ECR_REPOSITORY_NAME
-      - echo REPO=$REPO
+      - BUILD_TAG=$CLEAN_BRANCH-$CODEBUILD_BUILD_NUMBER
+      - echo build tag is $BUILD_TAG
       - echo tagging
-      - docker tag ala-alerts:latest $REPO:$CLEAN_BRANCH-$CODEBUILD_BUILD_NUMBER
-      - docker tag ala-alerts:latest $REPO:latest
+      - docker tag ala-alerts:latest $ECR_REPO:$BUILD_TAG
+      - docker tag ala-alerts:latest $ECR_REPO:latest
       - echo Pushing the Docker image to Amazon ECR...
-      - docker push $REPO:$CLEAN_BRANCH-$CODEBUILD_BUILD_NUMBER
-      - docker push $REPO:latest
+      - docker push $ECR_REPO:$BUILD_TAG
+      - docker push $ECR_REPO:latest

cicd/backend/pipeline/deploy_alerts_buildspec.yaml

@@ -0,0 +1,97 @@
+version: 0.2
+
+env:
+  shell: bash
+  variables:
+    JAVA_TOOL_OPTIONS: -Dhttps.protocols=TLSv1.2
+  secrets-manager:
+    DB_PASSWORD: $ALERTS_SECRET_NAME:db-password
+
+phases:
+  install:
+    commands:
+      - echo Installing dependencies...
+      - cat /etc/lsb-release
+      - apt update -y
+        && apt-get -q -y install openjdk-11-jdk
+        && pip3 install jinja2
+        && curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
+        && chmod 700 get_helm.sh
+        && ./get_helm.sh
+
+  build:
+    commands:
+      - echo Build started on $(date)
+      - wget https://raw.githubusercontent.com/AtlasOfLivingAustralia/ala-install/refs/heads/master/ansible/roles/alerts/templates/alerts-config.properties
+      - aws eks --region ap-southeast-2 update-kubeconfig --name $EKS_CLUSTER_NAME
+      - |
+        USER_POOL_ID=$(aws cloudformation describe-stacks --stack-name $COGNITO_STACK_NAME \
+          --query "Stacks[0].Outputs[?OutputKey=='UserPoolId'].OutputValue" --output text)
+      - |
+        CLIENT_ID=$(aws cloudformation describe-stacks --stack-name $BASE_STACK_NAME \
+          --query "Stacks[0].Outputs[?OutputKey=='AlertsAppClient'].OutputValue" --output text)
+      - |
+        CLIENT_SECRET=$(aws cognito-idp describe-user-pool-client --user-pool-id $USER_POOL_ID \
+          --client-id $CLIENT_ID --query "UserPoolClient.ClientSecret" --output text)
+      - |
+        SERVER_TO_SERVER_CLIENT_ID=$(aws cloudformation describe-stacks --stack-name $BASE_STACK_NAME \
+          --query "Stacks[0].Outputs[?OutputKey=='AlertsServerToServerAppClient'].OutputValue" --output text)
+      - |
+        SERVER_TO_SERVER_CLIENT_SECRET=$(aws cognito-idp describe-user-pool-client --user-pool-id $USER_POOL_ID \
+          --client-id $SERVER_TO_SERVER_CLIENT_ID --query "UserPoolClient.ClientSecret" --output text)
+      - kubectl config set-context --current --namespace=alerts
+      - |
+        python3 cicd/gen_app_config.py -t alerts-config.properties \
+          -v $CODEBUILD_SRC_DIR_AnsibleInventoriesSourceArtifact/alerts/alerts-feature-branch \
+          -b $CLEAN_BRANCH \
+          -o alerts-config.properties
+      - sed -i "s/dataSource.password=/dataSource.password=$DB_PASSWORD/" alerts-config.properties
+      - sed -i "s/security.oidc.clientId=/security.oidc.clientId=$CLIENT_ID/" alerts-config.properties
+      - sed -i "s/security.oidc.secret=/security.oidc.secret=$CLIENT_SECRET/" alerts-config.properties
+      - sed -i "s/webservice.client-id=/webservice.client-id=$SERVER_TO_SERVER_CLIENT_ID/" alerts-config.properties
+      - sed -i "s/webservice.client-secret=/webservice.client-secret=$SERVER_TO_SERVER_CLIENT_SECRET/" alerts-config.properties
+      - sed -i "s/mysql-rds\/alerts/mysql-rds-$CLEAN_BRANCH\/alertsdevelopment/" alerts-config.properties
+      - sed -i "s/ala-alerts-test/alerts-$CLEAN_BRANCH/" alerts-config.properties
+      - sed -i '1i server.port=8000' alerts-config.properties
+      - kubectl delete secret alerts-config-secret-$CLEAN_BRANCH || true
+      - kubectl create secret generic alerts-config-secret-$CLEAN_BRANCH --from-file=alerts-config.properties
+      - |
+        ALERTS_ROLE_ARN=$(aws cloudformation describe-stacks --stack-name $BASE_STACK_NAME \
+          --query "Stacks[0].Outputs[?OutputKey=='AlertsRoleArn'].OutputValue" --output text)
+      - printf "$(kubectl config current-context)\n"
+      - |
+        eksctl create iamserviceaccount --name alerts-service-account-$CLEAN_BRANCH \
+          --namespace alerts --cluster $EKS_CLUSTER_NAME --attach-role-arn $ALERTS_ROLE_ARN \
+          --approve --override-existing-serviceaccounts \
+          --tags "product=alerts,environment=development,branch=${CLEAN_BRANCH}"
+      - cd helm
+      - |
+        export CERTIFICATE_ARN=$(aws acm list-certificates --region ap-southeast-2 \
+          --query "CertificateSummaryList[?DomainName=='*.$HOSTED_ZONE' && Status=='ISSUED'].CertificateArn" --output text)
+      - BUILD_TAG=$CLEAN_BRANCH-$CODEBUILD_BUILD_NUMBER
+      - env
+      - |
+        helm upgrade --install $HELM_RELEASE_NAME . -n alerts \
+          --set image.repository=$ECR_REPO \
+          --set image.tag=$BUILD_TAG \
+          --set ingress.hostname=$DOMAIN_NAME \
+          --set rds.externalName=$DB_WRITE_ENDPOINT \
+          --set ingress.certificateArn=$CERTIFICATE_ARN \
+          --set secret.name=alerts-config-secret-$CLEAN_BRANCH \
+          --set serviceAccount.name=alerts-service-account-$CLEAN_BRANCH \
+          --set rds.name=mysql-rds-$CLEAN_BRANCH
+      - kubectl rollout restart deployment $HELM_RELEASE_NAME
+  post_build:
+    commands:
+      - echo Post-build phase...
+      - echo Build completed on $(date)
+
+artifacts:
+  base-directory: $CODEBUILD_SRC_DIR
+  files:
+    - '**/*'
+  secondary-artifacts:
+    AnsibleInventoriesSourceArtifact:
+      base-directory: $CODEBUILD_SRC_DIR_AnsibleInventoriesSourceArtifact
+      files:
+        - '**/*'

cicd/build/pipeline/deploy_notification_buildspec.yaml cicd/backend/pipeline/deploy_notification_buildspec.yaml

@@ -139,15 +139,16 @@ aws cloudformation deploy \
     --parameter-overrides \
         pAutoDeploy=$AUTO_DEPLOY \
         pBootstrapStackName=$BOOTSTRAP_STACK_NAME \
+        pBaseStackName=$BASE_STACK_NAME \
         pBucketsStackName=$BUCKETS_STACK_NAME \
         pCleanBranch=$clean_branch \
-        pEksClusterName=$EKS_CLUSTER_NAME \
         pEnvironment=$environment \
         pGitHubBranch=$branch \
         pGitHubOwner=$GITHUB_OWNER \
         pGitHubRepositoryName=$GITHUB_REPO_NAME \
         pPipelineFingerprint=$PIPELINE_MD5 \
         pProductComponent=$PRODUCT_COMPONENT \
         pProductName=$PRODUCT_NAME \
+        pDomainName=$DOMAIN_NAME \
         pRestartExecutionOnUpdate=$RESTART_PIPELINE_ON_UPDATE \
 

cicd/deploy/pipeline/deploy_pipeline.sh cicd/backend/pipeline/deploy_pipeline.sh

@@ -10,15 +10,21 @@ env:
     DEBIAN_FRONTEND: "noninteractive"
   exported-variables:
     - CODEBUILD_BUILD_NUMBER
-    - BASE_STACK_FILE_PFIX
     - PRODUCT_NAME
     - PRODUCT_COMPONENT
     - SLACK_DEPLOY_NOTIFICATION
     - SLACK_ALERT_CHANNEL
     - ALERTS_SECRET_NAME
-    - ALERTS_ECR_REPOSITORY_NAME
+    - BASE_STACK_FILE_PFIX
+    - BASE_STACK_NAME
+    - CODEBUILD_BUILD_NUMBER
+    - COGNITO_STACK_NAME
     - DB_READ_ENDPOINT
     - DB_WRITE_ENDPOINT
+    - DOMAIN_NAME
+    - EKS_CLUSTER_NAME
+    - HELM_RELEASE_NAME
+    - HOSTED_ZONE
 
 phases:
 
@@ -59,25 +65,36 @@ phases:
       - cicd/gen_env_vars.py --env $ENVIRONMENT --clean-branch $CLEAN_BRANCH --conf cicd/$PRODUCT_COMPONENT/config.ini > env.txt
       - echo loading config..
       - set -a ; source env.txt ; set +a
+      # import stack export from the base cloudformation stack
+      - echo importing stack output...
+      - ALERTS_SECRETS=$(aws cloudformation describe-stacks --stack-name $BASE_STACK_NAME --query "Stacks[0].Outputs[?OutputKey=='AlertsSecrets'].OutputValue" --output text)
+      - echo ALERTS_SECRETS=$ALERTS_SECRETS
       - ALERTS_SECRET_NAME_ARN=$(aws cloudformation describe-stacks --stack-name $BASE_STACK_NAME --query "Stacks[0].Outputs[?OutputKey=='AlertsSecret'].OutputValue" --output text)
       - ALERTS_SECRET_NAME=$(echo $ALERTS_SECRET_NAME_ARN | cut -d':' -f7 | rev | cut -d'-' -f2- | rev)
-      - ALERTS_ECR_REPOSITORY_NAME=$(aws cloudformation describe-stacks --stack-name $BASE_STACK_NAME --query "Stacks[0].Outputs[?OutputKey=='AlertsRepositoryName'].OutputValue" --output text)
+      - echo ALERTS_SECRET_NAME=$ALERTS_SECRET_NAME
+      - |
+        if [[ "$ENVIRONMENT" == "development" ]]; then
+          ALERTS_SECRET_NAME="alerts-develop"
+        fi
+      - echo ALERTS_SECRET_NAME=$ALERTS_SECRET_NAME
       - DB_READ_ENDPOINT=$(aws cloudformation describe-stacks --stack-name $DATABASE_STACK_NAME --query "Stacks[0].Outputs[?OutputKey=='ReadEndpoint'].OutputValue" --output text)
       - DB_WRITE_ENDPOINT=$(aws cloudformation describe-stacks --stack-name $DATABASE_STACK_NAME --query "Stacks[0].Outputs[?OutputKey=='WriteEndpoint'].OutputValue" --output text)
       - echo ALERTS_SECRET_NAME=$ALERTS_SECRET_NAME
-      - echo ALERTS_ECR_REPOSITORY_NAME=$ALERTS_ECR_REPOSITORY_NAME
       - echo DB_READ_ENDPOINT=$DB_READ_ENDPOINT
       - echo DB_WRITE_ENDPOINT=$DB_WRITE_ENDPOINT
+      - export EKS_CLUSTER_NAME=$(aws cloudformation list-exports --query "Exports[?Name=='$REGOLITH_STACK_NAME-ClusterName'].Value" --output text)
+      - echo EKS_CLUSTER_NAME=$EKS_CLUSTER_NAME
     finally:
       - #echo This always runs
 
   build:
     commands:
       - echo Entered the build phase...
+      # package templates and generate the template configs
     finally:
       - #echo This always runs
 
 
   post_build:
     commands:
-      - #echo Entered the post_build phase...
+      - #echo Entered the post_build phase...