Merge pull request #622 from neicnordic/rabbitmq_consumer_timeout

Rabbitmq consumer timeout

.github/integration/rabbitmq-federation.yml

@@ -39,8 +39,9 @@ services:
 
   federation_test:
     command:
-      - /bin/sh
-      - /tests/rabbitmq/10_federation_test.sh
+      - "/bin/sh"
+      - "/tests/run_scripts.sh"
+      - "/tests/rabbitmq"
     container_name: federation
     depends_on:
       certfixer:
@@ -49,7 +50,7 @@ services:
         condition: service_started
       rabbitmq:
         condition: service_healthy
-    image: alpine:latest
+    image: debian:stable-slim
     profiles:
       - federation
     volumes:
@@ -98,6 +99,7 @@ services:
       - RABBITMQ_SERVER_CERT=/etc/rabbitmq/ssl/mq.crt
       - RABBITMQ_SERVER_KEY=/etc/rabbitmq/ssl/mq.key
       - RABBITMQ_SERVER_VERIFY=verify_none
+      - RABBITMQ_CONSUMER_TIMEOUT=1000
     healthcheck:
       test:
         [

.github/integration/tests/rabbitmq/10_federation_test.sh

@@ -8,14 +8,17 @@ if [ -f /.dockerenv ]; then
 	CEGA="cegamq:15671"
 fi
 
-if [ ! "$(command -v jq)" ]; then
-	if [ "$(id -u)" != 0 ]; then
-		echo "jq is missing, unable to install it"
-		exit 1
-	fi
-
-	apk add --no-cache curl jq
-fi
+for t in curl jq; do
+    if [ ! "$(command -v $t)" ]; then
+        if [ "$(id -u)" != 0 ]; then
+            echo "$t is missing, unable to install it"
+            exit 1
+        fi
+
+        apt-get -o DPkg::Lock::Timeout=60 update >/dev/null
+        apt-get -o DPkg::Lock::Timeout=60 install -y "$t" >/dev/null
+    fi
+done
 
 RETRY_TIMES=0
 until curl -s --cacert /tmp/certs/ca.crt -u guest:guest "https://$MQHOST/api/federation-links" | jq -r '.[].status' | grep running; do

.github/integration/tests/rabbitmq/20_configuration_test.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+set -ex
+
+MQHOST="localhost:15672"
+if [ -f /.dockerenv ]; then
+	MQHOST="mq:15671"
+fi
+
+timeout=$(curl -s --cacert /tmp/certs/ca.crt -u guest:guest "https://$MQHOST/api/queues/sda/from_cega" | jq '."consumer_details"[]."consumer_timeout"')
+if [ "$timeout" -ne 1000 ]; then
+    echo "active timeout is wrong, expected: 1000, actual: $timeout"
+    exit 1
+fi
+
+echo "configuration test completed successfully"

charts/sda-mq/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v2
 name: sda-mq
-version: 0.7.4
+version: 0.7.5
 appVersion: v0.2.67
 kubeVersion: '>= 1.26.0'
 description: RabbitMQ component for Sensitive Data Archive (SDA) installation

charts/sda-mq/README.md

@@ -34,6 +34,7 @@ Parameter | Description | Default
 `global.shovel.user` | Username to federated server |`""`
 `global.shovel.vhost` | Vhost on federated sever to connect to |`""`
 `externalPkiService.tlsPath` | If an external PKI service is used, this is the path where the certifiates are placed | `""`
+`extraConfig.consumer_timeout` | message handling timeout in milliseconds | `""`
 `rbacEnabled` | Use role based access control. |`true`
 `revisionHistory` | Number of revisions to keep for the option to rollback a deployment | `3`
 `updateStrategyType` | Update strategy type. | `RollingUpdate`

charts/sda-mq/templates/statefulset.yaml

@@ -101,6 +101,10 @@ spec:
           value: {{ template "verifyPeer" . }}
         {{- end }}
       {{- end }}
+      {{- if .Values.extraConfig.consumer_timeout }}
+        - name: RABBITMQ_CONSUMER_TIMEOUT
+          value: {{ .Values.extraConfig.consumer_timeout }}
+      {{- end }}
       {{- if .Values.global.vhost }}
         - name: MQ_VHOST
           value: {{ .Values.global.vhost | quote }}

charts/sda-mq/values.yaml

@@ -31,6 +31,9 @@ global:
     user: ""
     vhost: ""
 
+# timeout for acking a message, in milliseconds
+extraConfig:
+  consumer_timeout: ""
 
 # If an external PKI infrastructure is used to supply certificates set this to true
 externalPkiService:

rabbitmq/docker-entrypoint.sh

@@ -52,7 +52,8 @@ fi
 cat >/var/lib/rabbitmq/advanced.config<<-EOF
 [
 	{rabbit, [
-		{default_consumer_prefetch, {false,100}}
+		{consumer_timeout, ${RABBITMQ_CONSUMER_TIMEOUT:-14400000}},
+		{default_consumer_prefetch, {false,1}}
 		]
 	}
 ].