Merge pull request #245 from ministryofjustice/DBA-667

Dba 667

Author: bill-buchan

.github/workflows/oracle-db-backup.yml

@@ -0,0 +1,12 @@
+[
+    {
+        "TargetEnvironment":"delius-core-dev",
+        "TargetHost":"delius_primarydb",
+        "CronSchedule":"10 14 * * TUE"
+    },
+    {
+        "TargetEnvironment":"delius-core-test",
+        "TargetHost":"delius_primarydb",
+        "CronSchedule":"10 16 * * TUE"
+    }
+]

.github/workflows/oracle-db-delete-dbids-not-in-use-schedule.json

@@ -0,0 +1,62 @@
+name: "Oracle: Schedule Delete DBID Backups Not In Use"
+on:
+  push:
+    branches:
+      - "DBA-667"
+  schedule:
+    - cron: '10 14 * * TUE'
+    - cron: '10 15 * * TUE'
+    - cron: '10 16 * * TUE'
+jobs:
+  prepare-run-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      scheduled_matrix: ${{ steps.filter-validate-schedule.outputs.scheduling_matrix }}
+    steps:
+      - name: Checkout Delete DBIDs Backups Schedule
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout-cone-mode: false
+          sparse-checkout: |
+             .github/workflows/oracle-db-delete-dbids-not-in-use-schedule.json
+          path: operations
+          ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.SourceCodeVersion || 'main' }}
+          fetch-depth: 0
+
+      - name: Filter Validate Schedule
+        id: filter-validate-schedule
+        run: |
+             SCHEDULED_JSON=$(jq '[.[] | select (.CronSchedule=="${{ github.event.schedule }}") | {"TargetEnvironment","TargetHost"}]' operations/.github/workflows/oracle-db-delete-dbids-not-in-use-schedule.json | jq '{include: .}')
+             echo "scheduling_matrix="$(echo ${SCHEDULED_JSON} | sed 's/ //g') >> $GITHUB_OUTPUT
+
+  report-schedule-delete-dbids-not-in-use:
+    needs: prepare-run-matrix
+    runs-on: ubuntu-latest
+    if: ${{ needs.prepare-run-matrix.outputs.scheduled_matrix != '{"include":[]}' }}
+    steps:
+      - name: Report Deletion of DBIDs Scheduled to Run
+        id: report-run
+        run: |
+              echo "Running Deletion of Unused DBID Backups for these targets: ${{ needs.prepare-run-matrix.outputs.scheduled_matrix }}"
+ 
+  report-no-scheduled-delete-dbids-not-in-use:
+    needs: prepare-run-matrix
+    runs-on: ubuntu-latest
+    if: ${{ needs.prepare-run-matrix.outputs.scheduled_matrix == '{"include":[]}' }}
+    steps:
+        - name: Report Nothing to Do
+          id: report-no-run
+          run: |
+              echo "No targets scheduled for DBID backup deletion."
+
+  delete-dbids-not-in-use:
+    needs: prepare-run-matrix
+    if: ${{ needs.prepare-run-matrix.outputs.scheduled_matrix != '{"include":[]}' }}
+    strategy:
+      matrix: ${{fromJson(needs.prepare-run-matrix.outputs.scheduled_matrix)}}
+    name: Delete DBID Backups Not In Use
+    uses:
+      ministryofjustice/hmpps-delius-operational-automation/.github/workflows/oracle-db-delete-dbids-not-in-use.yml@main
+    with:
+      TargetEnvironment: ${{ matrix.TargetEnvironment }}
+      TargetHost: ${{ matrix.TargetHost }}

.github/workflows/oracle-db-delete-dbids-not-in-use.yml

@@ -24,15 +24,6 @@
 
 - block:
     - include_tasks: fetch_inventory_files.yml
-  delegate_to: localhost
-  become: no
-
-# Copy inventory file from Ansible Controller to same directory on the target Host
-- name: Copy Inventory File to Target Host for Further Processing
-  copy:
-    src: "{{ inventory_file }}"
-    dest: "{{ inventory_file }}"
-    owner: oracle
 
 - name: Get Passwords
   include_tasks: get_facts.yml

.github/workflows/oracle-db-schedule-delete-dbids-not-in-use.yml

@@ -1,5 +1,5 @@
 - name: Delete DBIDs Not Longer in Use
-  hosts: '{{ rman_target | default("localhost", true) }}'
+  hosts: "{{ rman_target }}"
   gather_facts: yes
   become: yes
   become_user: oracle

playbooks/oracle_backup/check_defunct_backups.yml

@@ -88,17 +88,18 @@ function generate_jwt()
 {
 # Get a JSON Web Token to authenicate against the HMPPS Bot.
 # The HMPPS bot can provide exchange this for a GitHub Token for action GitHub workflows.
-
 BOT_APP_ID=$(aws ssm get-parameter --name "/github/hmpps_bot_app_id" --query "Parameter.Value" --with-decryption --output text)
 BOT_PRIVATE_KEY=$(aws ssm get-parameter --name "/github/hmpps_bot_priv_key" --query "Parameter.Value" --with-decryption --output text)
 
-# Define expiry time for JWT
+# Define expiry time for JWT - we will be using it immediately so just use a 10 minute expiry time.
 NOW=$(date +%s)
-INITIAL=$((${NOW} - 60)) # Issues 60 seconds in the past
+INITIAL=$((${NOW} - 60)) # Issues 60 seconds in the past (avoid time jitter problem)
 EXPIRY=$((${NOW} + 600)) # Expires 10 minutes in the future
 
+# This function is used to apply Base64 encoding for the token to allow it to be passed on.
 b64enc() { openssl base64 | tr -d '=' | tr '/+' '_-' | tr -d '\n'; }
 
+# The JWT requires a Header, Payload and Signature as defined here.
 HEADER_JSON='{
     "typ":"JWT",
     "alg":"RS256"
@@ -111,7 +112,7 @@ PAYLOAD_JSON='{
     "exp":'"${EXPIRY}"',
     "iss":'"${BOT_APP_ID}"'
 }'
-# Payload encode
+# Payload encode in Base64
 PAYLOAD=$( echo -n "${PAYLOAD_JSON}" | b64enc )
 
 # Signature
@@ -138,23 +139,33 @@ printf '%s\n' "$GITHUB_TOKEN"
 
 function github_repository_dispatch()
 {
+# Because this script is intended to run asynchronously and may be called by a GitHub Workflow, we use
+# GitHub Repository Dispatch events to call back to the Workflow to allow it to continue.  This is a 
+# workaround to avoid two issues:
+# (1) Timeout of GitHub actions lasting over 6 hours.
+# (2) Billing costs associated with the GitHub hosted runner actively waiting whilst the backup runs.
+#
+# We supply 2 parameters to this function:
+#  EVENT_TYPE is a user-defined event to pass to the GitHub repository.   The backup worflow is triggered
+#  for either oracle-db-backup-sucess or oracle-db-backup-failure events.   These are the only 2 which
+#  should be used.
+#  JSON_PAYLOAD is the JSON originally passed to the script using the -j switch.  This allows the
+#  workflow to continue where it left off because this JSON contains the name of the environment, host
+#  and period of the backup, along with any associated parameters.
 EVENT_TYPE=$1
 JSON_PAYLOAD=$2
 GITHUB_TOKEN_VALUE=$(get_github_token | jq -r '.token')
+# We set the Phase in the JSON payload corresponding to whether the backup has succeeded or failed.
+# This is informational only - it is GitHub event type (oracle-db-backup-success/failure) which 
+# determines what the workflow does next.
 if [[ "$EVENT_TYPE" == "oracle-db-backup-success" ]]; then
     JSON_PAYLOAD=$(echo $JSON_PAYLOAD | jq -r '.Phase = "Backup Succeeded"')
 else
     JSON_PAYLOAD=$(echo $JSON_PAYLOAD | jq -r '.Phase = "Backup Failed"')
 fi
-info "Running Repository Dispatch:${EVENT_TYPE}:${JSON_PAYLOAD}:${GITHUB_TOKEN_VALUE}"
 JSON_DATA="{\"event_type\": \"${EVENT_TYPE}\",\"client_payload\":${JSON_PAYLOAD}}"
-info "JD1: $JSON_DATA"
-JSON_DATA=$(echo $JSON_DATA | jq @json)
-info "JD2: $JSON_DATA"
-cat <<EOCURL | tee -a /tmp/eocurl.txt
-curl -X POST -H "Accept: application/vnd.github+json" -H "Authorization: token ${GITHUB_TOKEN_VALUE}"  --data ${JSON_DATA} ${REPOSITORY_DISPATCH}
-EOCURL
-curl -X POST -H "Accept: application/vnd.github+json" -H "Authorization: token ${GITHUB_TOKEN_VALUE}"  --data ${JSON_DATA} ${REPOSITORY_DISPATCH}
+info "Posting repository dispatch event"
+curl -X POST -H "Accept: application/vnd.github+json" -H "Authorization: token ${GITHUB_TOKEN_VALUE}"  --data-raw "${JSON_DATA}" ${REPOSITORY_DISPATCH}
 RC=$?
 if [[ $RC -ne 0 ]]; then
       # We cannot use the error function for dispatch failures as it contains its own dispatch call   
@@ -184,17 +195,14 @@ update_ssm_parameter () {
   MESSAGE=$2
   info "Updating SSM Parameter ${SSM_PARAMETER} Message to ${MESSAGE}"
   SSM_VALUE=$(aws ssm get-parameter --name "${SSM_PARAMETER}" --query "Parameter.Value" --output text)
-  info "I: $NEW_SSM_VALUE"
   NEW_SSM_VALUE=$(echo ${SSM_VALUE} | jq -r --arg MESSAGE "$MESSAGE" '.Message=$MESSAGE')
-  info "B: $NEW_SSM_VALUE"
   if [[ "$STATUS" == "Success" ]]; then
      NEW_SSM_VALUE=$(echo ${NEW_SSM_VALUE} | jq -r '.Phase = "Backup Succeeded"')
   elif [[ "$STATUS" == "Running" ]]; then
      NEW_SSM_VALUE=$(echo ${NEW_SSM_VALUE} | jq -r '.Phase = "Backup In Progress"') 
   else
      NEW_SSM_VALUE=$(echo ${NEW_SSM_VALUE} | jq -r '.Phase = "Backup Failed"')
   fi
-  info "A: $NEW_SSM_VALUE"
   aws ssm put-parameter --name "${SSM_PARAMETER}" --type String --overwrite --value "${NEW_SSM_VALUE}" 1>&2
 }
 
@@ -739,6 +747,21 @@ info "Trace File          = $TRACE_FILE"
 info "Archivelog Range    = $ARCHIVELOGS"
 info "Specific Datafiles  = $DATAFILES"
 
+if [[ ! -z "$REPOSITORY_DISPATCH" ]]; then
+   REPOSITORY_DISPATCH="https://api.github.com/repos/${REPOSITORY_DISPATCH}/dispatches"
+   info "GitHub Actions Repository Dispatch Events will be sent to : $REPOSITORY_DISPATCH"
+fi
+
+if [[ ! -z "$JSON_INPUTS" ]]; then
+   # The JSON Inputs are used to record the parameters originally passed to GitHub
+   # actions to start the backup job.   These are only used for actioning a repository
+   # dispatch event to indicate the end of the backup job run.  They do NOT
+   # override the command line options passed to the script.
+   JSON_INPUTS=$(echo $JSON_INPUTS | base64 --decode )
+elif [[ ! -z "$REPOSITORY_DISPATCH" ]]; then
+   error "JSON inputs must be supplied using the -j option if Repository Dispatch Events are requested."
+fi
+
 validate user
 info "Execute $THISUSER bash profile"
 . $HOME/.bash_profile
@@ -784,23 +807,6 @@ if [[ ! -z "$SSM_PARAMETER" ]]; then
    update_ssm_parameter "Running" "Running: $0 $*"
 fi
 
-if [[ ! -z "$REPOSITORY_DISPATCH" ]]; then
-   REPOSITORY_DISPATCH="https://api.github.com/repos/${REPOSITORY_DISPATCH}/dispatches"
-   info "GitHub Actions Repository Dispatch Events will be sent to : $REPOSITORY_DISPATCH"
-fi
-
-if [[ ! -z "$JSON_INPUTS" ]]; then
-   # The JSON Inputs are used to record the parameters originally passed to GitHub
-   # actions to start the backup job.   These are only used for actioning a repository
-   # dispatch event to indicate the end of the backup job run.  They do NOT
-   # override the command line options passed to the script.
-   JSON_INPUTS=$(echo $JSON_INPUTS | base64 --decode )
-   info "Original JSON Inputs to GitHub Action: $JSON_INPUTS"
-elif [[ ! -z "$REPOSITORY_DISPATCH" ]]; then
-   error "JSON inputs must be supplied using the -j option if Repository Dispatch Events are requested."
-fi
-
-touch $RMANCMDFILE
 info "Create rman tags and format"
 create_tag_format
 info "Generating rman command file"