Merge branch 'main' into DBA-834

Author: bill-buchan

.github/workflows/oracle-db-password-rotation-schedule.json

@@ -7,6 +7,10 @@
         "TargetEnvironment":"delius-mis-dev",
         "CronSchedule":"00 12 * * TUE"
     },
+    {
+        "TargetEnvironment":"delius-mis-stage",
+        "CronSchedule":"00 14 * * THU"
+    },
     {
         "TargetEnvironment":"delius-core-test",
         "CronSchedule":"00 14 * * TUE"

.github/workflows/oracle-db-password-rotation-schedule.yml

@@ -4,6 +4,7 @@ on:
     - cron: '00 12 * * TUE'
     - cron: '00 14 * * TUE'
     - cron: '00 10 * * THU'
+    - cron: '00 14 * * THU'
 
 jobs:
   prepare-run-matrix:

helm/migrate-db/Chart.yaml

@@ -0,0 +1,5 @@
+apiVersion: v2
+appVersion: 0.1
+version: 0.0.1
+description: A Helm chart for deploying a job to migrate DB data
+name: migrate-db

helm/migrate-db/templates/config-map.yaml

@@ -0,0 +1,25 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: migrate-db-script
+data:
+  entrypoint.sh: |-
+    #!/bin/bash
+    set -e
+    echo "${SRC_DB_HOST}:5432:${SRC_DB_NAME}:${SRC_DB_USER}:${SRC_DB_PASS}" > ~/.pgpass
+    echo "${DST_DB_HOST}:5432:${DST_DB_NAME}:${DST_DB_USER}:${DST_DB_PASS}" >> ~/.pgpass
+    chmod 0600 ~/.pgpass
+    chown job:job ~/.pgpass
+    set -x
+
+    # Dump the source database
+    pg_dump --jobs=4 --host="$SRC_DB_HOST" --username="$SRC_DB_USER" --dbname="$SRC_DB_NAME" --no-owner --no-privileges --verbose --format=directory --file=/home/job/db-dump
+
+    psql --host="$DST_DB_HOST" --username="$DST_DB_USER" --dbname="$DST_DB_NAME" -c "drop schema if exists public cascade;"
+
+    psql --host="$DST_DB_HOST" --username="$DST_DB_USER" --dbname="$DST_DB_NAME" -c "create schema public;"
+
+    # Restore the source database dump to the destination database
+    pg_restore --jobs=4 --host="$DST_DB_HOST" --username="$DST_DB_USER" --dbname="$DST_DB_NAME" --no-owner --no-privileges --verbose /home/job/db-dump
+    rm -rv /home/job/db-dump ~/.pgpass

helm/migrate-db/templates/job.yaml

@@ -0,0 +1,84 @@
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: migrate-db
+spec:
+  template:
+    spec:
+      containers:
+        - name: migrate-db
+          image: ghcr.io/ministryofjustice/hmpps-delius-alfresco-db-utils:latest # move this image to this repo
+          imagePullPolicy: IfNotPresent
+          resources:
+            limits:
+              cpu: 4
+              memory: 2Gi
+          command:
+            - /bin/entrypoint.sh
+          env:
+            - name: SRC_DB_NAME
+              valueFrom:
+                secretKeyRef:
+                  name: legacy-rds-instance-{{ .Values.component }}
+                  key: DATABASE_NAME
+            - name: SRC_DB_USER
+              valueFrom:
+                secretKeyRef:
+                  name: legacy-rds-instance-{{ .Values.component }}
+                  key: DATABASE_USERNAME
+            - name: SRC_DB_PASS
+              valueFrom:
+                secretKeyRef:
+                  name: legacy-rds-instance-{{ .Values.component }}
+                  key: DATABASE_PASSWORD
+            - name: SRC_DB_HOST
+              valueFrom:
+                secretKeyRef:
+                  name: legacy-rds-instance-{{ .Values.component }}
+                  key: RDS_INSTANCE_ADDRESS
+            - name: DST_DB_NAME
+              valueFrom:
+                secretKeyRef:
+                  name: rds-instance-output-{{ .Values.component }}
+                  key: DATABASE_NAME
+            - name: DST_DB_USER
+              valueFrom:
+                secretKeyRef:
+                  name: rds-instance-output-{{ .Values.component }}
+                  key: DATABASE_USERNAME
+            - name: DST_DB_PASS
+              valueFrom:
+                secretKeyRef:
+                  name: rds-instance-output-{{ .Values.component }}
+                  key: DATABASE_PASSWORD
+            - name: DST_DB_HOST
+              valueFrom:
+                secretKeyRef:
+                  name: rds-instance-output-{{ .Values.component }}
+                  key: RDS_INSTANCE_ADDRESS
+          volumeMounts:
+            - name: migrate-db-script
+              mountPath: /bin/entrypoint.sh
+              readOnly: true
+              subPath: entrypoint.sh
+          securityContext:
+            allowPrivilegeEscalation: false
+            privileged: false
+            readOnlyRootFilesystem: false
+            runAsNonRoot: true
+            runAsUser: 999
+            capabilities:
+              drop:
+                - ALL
+            seccompProfile:
+              type: RuntimeDefault
+      serviceAccount: hmpps-migration-{{ .Values.environment }}
+      serviceAccountName: hmpps-migration-{{ .Values.environment }}
+      restartPolicy: Never
+      volumes:
+        - name: migrate-db-script
+          configMap:
+            name: migrate-db-script
+            defaultMode: 0755
+  backoffLimit: 0

playbooks/delius_audit_replication_schema/delius_audit_replication_schema/files/disable_user_update.sh

@@ -28,10 +28,22 @@ END LOOP;
 END;
 /
 
+-- Incoming USER_ INSERTs may contain STAFF_IDs which may or may not exist in stage
+-- or pre-prod; we allow the DELIUS_USER_SUPPORT audit control check if they exist
+-- so that we can set any invalid STAFF_IDs to NULL and allow the rest of the USER_
+-- record to be created.
+GRANT SELECT ON delius_app_schema.staff TO delius_user_support;
+
+-- We do not wish to generate CDC records for LAST_ACCESSED_DATETIME as user access
+-- times in the repository database have no baring on those on the client.  We
+-- use the Unilink-supplied PKG_TRIGGERSUPPORT package to temporarily disable
+-- CDC capture on USER_ if no personal details have been changed.
+GRANT EXECUTE ON delius_app_schema.pkg_triggersupport TO delius_user_support; 
+
 -- Additionally we add a trigger to enforce the same restrictions for the Application Schema itself.
 -- This trigger is also used to workaround a difference between NLS Date formats used in DMS
 -- and those used by the Delius application.
-CREATE OR REPLACE TRIGGER delius_user_support.audit_control_on_user_
+create or replace TRIGGER delius_user_support.audit_control_on_user_
 FOR delete OR update OR insert ON delius_app_schema.user_
 COMPOUND TRIGGER
 
@@ -51,6 +63,7 @@ COMPOUND TRIGGER
   END BEFORE STATEMENT;
 
   BEFORE EACH ROW IS
+     l_staff_id_exists INTEGER;
   BEGIN
         IF USER = 'DELIUS_AUDIT_DMS_POOL'
         THEN
@@ -69,10 +82,10 @@ COMPOUND TRIGGER
             */
             EXECUTE IMMEDIATE 'ALTER SESSION SET nls_date_format = ''YYYY-MM-DD HH24:MI:SS''';
             EXECUTE IMMEDIATE 'ALTER SESSION SET nls_timestamp_format = ''YYYY-MM-DD HH24:MI:SS.FF9''';
-            
-                        
+
+
             /*
-            There are multiple reasons why the Delius application may update the 
+            There are multiple reasons why the Delius application may update the
             LAST_UPDATED_USER_ID and LAST_UPDATED_DATETIME columns, but not all of these involve
             attributes which are replicated to the client database.   This can cause confusion
             if these attributes are updated but nothing appears to have changed.
@@ -94,13 +107,35 @@ COMPOUND TRIGGER
                     -- did not impact data that was replicated.
                     :new.last_updated_user_id  := :old.last_updated_user_id;
                     :new.last_updated_datetime := :old.last_updated_datetime;
+                    -- Also do not update the row version as no actual change has been made
+                    :new.row_version := :old.row_version;
+                    -- Disable CDC for the user record if no actual personal data change
+                    -- (This is to prevent last accessed datetime records from production
+                    -- triggering CDC even when we suppress the change in the trigger).
+                    delius_app_schema.pkg_triggersupport.procSetCDCFlag(FALSE);
                 END IF;
                 -- Staff IDs in stage and pre-prod are independent of those in production
                 -- since these are not relevant to audit, and it may be necessary for
                 -- some users to have staff records in stage and pre-prod without having
                 -- corresponding records in production.  Therefore we prevent any overwriting
                 -- of the staff ID                
                 :new.staff_id := :old.staff_id;
+                -- We ignore changes to LAST_ACCESSED_DATETIME as this is the time the user was
+                -- accessed in the repository which is not relevant to the local database
+                :new.last_accessed_datetime := :old.last_accessed_datetime;
+            ELSIF INSERTING THEN
+               -- Also, not all Staff IDs from production will exist in stage & pre-prod
+               -- so it we are attempting to insert a new user with a non-existent staff_id
+               -- we simply set it to NULL
+               SELECT CASE WHEN EXISTS (
+                  SELECT 1
+                  FROM delius_app_schema.staff
+                  WHERE staff_id = :new.staff_id
+                ) THEN 1 ELSE 0 END
+                INTO l_staff_id_exists FROM DUAL;
+                IF l_staff_id_exists = 0 THEN
+                   :new.staff_id := NULL;
+                END IF;
             END IF;
         END IF;
 
@@ -132,6 +167,8 @@ COMPOUND TRIGGER
   BEGIN
    IF USER = 'DELIUS_AUDIT_DMS_POOL'
      THEN
+        -- Reset CDC Flag to default value
+        delius_app_schema.pkg_triggersupport.procSetCDCFlag(NULL);
         EXECUTE IMMEDIATE 'ALTER SESSION SET nls_date_format = ''YYYY-MM-DD HH24:MI:SS''';
         EXECUTE IMMEDIATE 'ALTER SESSION SET nls_timestamp_format = ''YYYY-MM-DD HH24:MI:SS.FF9''';
      END IF;