Merge pull request #176 from neicnordic/latest_from_upstream

Latest from upstream

.github/integration/tests/postgres/30_ingest_queries.sh

@@ -1,31 +1,36 @@
 #!/bin/sh
 set -eou pipefail
 
-result=$(psql -U postgres -h postgres -d sda -At -c "SELECT submission_file_path,submission_user from sda.files;")
-path=$(echo "$result" | cut -d '|' -f1)
-user=$(echo "$result" | cut -d '|' -f2)
-
 export PGPASSWORD=ingest
+user="test-user"
+corrID="33d29907-c565-4a90-98b4-e31b992ab376"
 
 ## insert file
-resp=$(psql -U ingest -h postgres -d sda -At -c  "UPDATE local_ega.main SET status = 'IN_INGESTION' WHERE local_ega.main.submission_file_path = '$path' AND local_ega.main.submission_user = '$user' AND local_ega.main.status = 'INIT' RETURNING id;")
-if [ "$(echo "$resp" | tr -d '\n')" != "1UPDATE 1" ]; then
+fileID=$(psql -U ingest -h postgres -d sda -At -c "SELECT sda.register_file('inbox/test-file.c4gh', '$user');")
+if [ -z "$fileID" ]; then
+    echo "register_file failed"
+    exit 1
+fi
+
+resp=$(psql -U ingest -h postgres -d sda -At -c "INSERT INTO sda.file_event_log(file_id, event, correlation_id, user_id, message) VALUES('$fileID', 'submitted', '$corrID', '$user', '{}');")
+if [ "$(echo "$resp" | tr -d '\n')" != "INSERT 0 1" ]; then
     echo "insert file failed"
     exit 1
 fi
 
 ## store header
-resp=$(psql -U ingest -h postgres -d sda -At -c "UPDATE local_ega.files SET header = '637279707434676801000000010000006c00000000000000' WHERE id = 1;")
+resp=$(psql -U ingest -h postgres -d sda -At -c "UPDATE sda.files SET header = '637279707434676801000000010000006c00000000000000' WHERE id = '$fileID';")
 if [ "$resp" != "UPDATE 1" ]; then
     echo "store header failed"
     exit 1
 fi
+
 ## set archived
 archive_path=d853c51b-6aed-4243-b427-177f5e588857
 size="2035150"
 checksum="f03775a50feea74c579d459fdbeb27adafd543b87f6692703543a6ebe7daa1ff"
-resp=$(psql -U ingest -h postgres -d sda -At -c "UPDATE local_ega.files SET status = 'ARCHIVED', archive_path = '$archive_path', archive_filesize = '$size', inbox_file_checksum = '$checksum', inbox_file_checksum_type = 'SHA256' WHERE id = 1;")
-if [ "$resp" != "UPDATE 1" ]; then
+resp=$(psql -U ingest -h postgres -d sda -At -c "SELECT sda.set_archived('$fileID', '$corrID', '$archive_path', '$size', '$checksum', 'SHA256');")
+if [ "$resp" != "" ]; then
     echo "mark file archived failed"
     exit 1
 fi

.github/integration/tests/postgres/40_verify_queries.sh

@@ -2,22 +2,30 @@
 set -eou pipefail
 
 export PGPASSWORD=verify
+corrID="33d29907-c565-4a90-98b4-e31b992ab376"
+fileID=$(psql -U verify -h postgres -d sda -At -c "SELECT DISTINCT file_id from sda.file_event_log WHERE correlation_id = '$corrID';")
+
+## get file status
+status=$(psql -U verify -h postgres -d sda -At -c "SELECT event from sda.file_event_log WHERE correlation_id = '$corrID' ORDER BY id DESC LIMIT 1;")
+if [ "$status" = "" ]; then
+    echo "get file status failed"
+    exit 1
+fi
 
 ## get file header
 header="637279707434676801000000010000006c00000000000000"
-dbheader=$(psql -U verify -h postgres -d sda -At -c "SELECT header from local_ega.files WHERE id = 1;")
+dbheader=$(psql -U verify -h postgres -d sda -At -c "SELECT header from sda.files WHERE id = '$fileID';")
 if [ "$dbheader" != "$header" ]; then
     echo "wrong header recieved"
     exit 1
 fi
 
 ## mark file as 'COMPLETED'
 archive_checksum="64e56b0d245b819c116b5f1ad296632019490b57eeaebb419a5317e24a153852"
-archive_size="2035150"
 decrypted_size="2034254"
 decrypted_checksum="febee6829a05772eea93c647e38bf5cc5bf33d1bcd0ea7d7bdd03225d84d2553"
-resp=$(psql -U verify -h postgres -d sda -At -c "UPDATE local_ega.files SET status = 'COMPLETED', archive_filesize = '$archive_size', archive_file_checksum = '$archive_checksum', archive_file_checksum_type = 'SHA256', decrypted_file_size = '$decrypted_size', decrypted_file_checksum = '$decrypted_checksum', decrypted_file_checksum_type = 'SHA256' WHERE id = 1;")
-if [ "$resp" != "UPDATE 1" ]; then
-    echo "mark file ready failed"
+resp=$(psql -U verify -h postgres -d sda -At -c "SELECT sda.set_verified('$fileID', '$corrID', '$archive_checksum', 'SHA256', '$decrypted_size', '$decrypted_checksum', 'SHA256')")
+if [ "$resp" != "" ]; then
+    echo "set_verified failed"
     exit 1
 fi

charts/sda-db/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v2
 name: sda-db
-version: "0.5.1"
+version: "0.5.2"
 description: Database component for Sensitive Data Archive (SDA) installation
 home: https://neic-sda.readthedocs.io
 icon: https://neic.no/assets/images/logo.png

charts/sda-db/values.yaml

@@ -26,7 +26,7 @@ extraSecurityContext: {}
 
 image:
   repository: ghcr.io/neicnordic/sda-db
-  tag: v2.0.7
+  tag: v2.1.10
   pullPolicy: IfNotPresent
 
 # utilize network isolation

charts/sda-mq/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v2
 name: sda-mq
-version: "0.4.5"
+version: "0.4.6"
 description: RabbitMQ component for Sensitive Data Archive (SDA) installation
 home: https://neic-sda.readthedocs.io
 icon: https://neic.no/assets/images/logo.png

charts/sda-mq/values.yaml

@@ -35,7 +35,7 @@ extraSecurityContext: {}
 
 image:
   repository: ghcr.io/neicnordic/sda-mq
-  tag: v1.4.20
+  tag: v1.4.38
   pullPolicy: Always
 
 # utilize network isolation

charts/sda-svc/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v2
 name: sda-svc
-version: "0.18.6"
+version: "0.18.9"
 kubeVersion: ">= 1.19.0-0"
 description: Components for Sensitive Data Archive (SDA) installation
 home: https://neic-sda.readthedocs.io

charts/sda-svc/README.md

@@ -75,7 +75,11 @@ Parameter | Description | Default
 `global.auth.jwtAlg` | Key type to sign the JWT, available options are RS265 & ES256, Must match the key type |`"ES256"`
 `global.auth.jwtKey` | Private key used to sign the JWT. |`""`
 `global.auth.jwtPub` | Public key ues to verify the JWT. |`""`
+`global.auth.resignJWT` | Resign the LS-AAI JWTs. |`true`
 `global.auth.useTLS` | Run a TLS secured server. |`true`
+`global.auth.corsOrigins` | Domain name allowed for cross-domain requests. |`""`
+`global.auth.corsMethods` | Allowed cross-domain request methods. |`""`
+`global.auth.corsCreds` | Include credentials in cross-domain requests. |`false`
 `global.broker.host` | Domain name or IP address to the message broker. |`""`
 `global.broker.exchange` | Exchange to publish messages to. |`""`
 `global.broker.port` | Port for the message broker. |`5671`
@@ -84,6 +88,7 @@ Parameter | Description | Default
 `global.broker.password` | Shared password to the message broker. |`/`
 `global.broker.username` | Shared user to the message broker. |`/`
 `global.broker.backupRoutingKey` | routing key used to send messages to backup service |`""`
+`global.broker.prefetchCount` | Number of messages to retrieve from the broker at the time, setting this to `1` will create a round-robin behavior between consumers |`2`
 `global.cega.host` | Full URI to the EGA user authentication service. |`""`
 `global.cega.user` | Username for the EGA user authentication service. |`""`
 `global.cega.password` | Password for the EGA user authentication service. |`""`
@@ -117,7 +122,6 @@ Parameter | Description | Default
 `global.download.trusted.iss` | Array of trusted OIDC endpoints | ``
 `global.download.trusted.iss[iss]` | URI to the OIDC service | `https://login.elixir-czech.org/oidc/`
 `global.download.trusted.iss[jku]` | The URI to the OIDCs jwk endpoint | `https://login.elixir-czech.org/oidc/jwk`
-
 `global.elixir.oidcdHost` | URL to the OIDc service. | `"https://login.elixir-czech.org/oidc/"`
 `global.elixir.jwkPath` | Public key path on the OIDC host. | `jwk`
 `global.inbox.servicePort` | The port that the inbox is accessible via. | `2222`

charts/sda-svc/templates/auth-deploy.yaml

@@ -107,6 +107,16 @@ spec:
           value: "{{ .Values.global.elixir.provider }}"
         - name: ELIXIR_SCOPE
           value: "ga4gh_passport_v1"
+        - name: ELIXIR_JWKPATH
+          value: {{ .Values.global.elixir.jwkPath | quote }}
+        {{- if .Values.global.auth.corsOrigins }}
+        - name: CORS_ORIGINS
+          value: {{ .Values.global.auth.corsOrigins | quote }}
+        - name: CORS_METHODS
+          value: {{ .Values.global.auth.corsMethods | quote }}
+        - name: CORS_CREDENTIALS
+          value: {{ .Values.global.auth.corsCreds | quote }}
+        {{- end }}
         {{- if or ( eq "federated" .Values.global.schemaType) ( eq "" .Values.global.schemaType) }}
         - name: CEGA_AUTHURL
           value: {{ .Values.global.cega.host | quote }}
@@ -121,6 +131,8 @@ spec:
           value: "{{ template "jwtPath" . }}/{{ .Values.global.auth.jwtKey }}"
         - name: JWTSIGNATUREALG
           value: {{ .Values.global.auth.jwtAlg }}
+        - name: RESIGNJWT
+          value: {{ .Values.global.auth.resignJwt | quote }}
         {{- if .Values.global.tls.enabled}}
         - name: SERVER_CERT
           value: {{ template "tlsPath" . }}/tls.crt

charts/sda-svc/templates/backup-deploy.yaml

@@ -133,6 +133,8 @@ spec:
           value: {{ required "A valid MQ host is required" .Values.global.broker.host | quote }}
         - name: BROKER_PORT
           value: {{ .Values.global.broker.port | quote }}
+        - name: BROKER_PREFETCHCOUNT
+          value: {{ .Values.global.broker.prefetchCount | quote }}
         - name: BROKER_ROUTINGERROR
           value: {{ .Values.global.broker.routingError }}
         - name: BROKER_ROUTINGKEY

charts/sda-svc/templates/finalize-deploy.yaml

@@ -76,6 +76,8 @@ spec:
           value: {{ required "A valid MQ host is required" .Values.global.broker.host | quote }}
         - name: BROKER_PORT
           value: {{ .Values.global.broker.port | quote }}
+        - name: BROKER_PREFETCHCOUNT
+          value: {{ .Values.global.broker.prefetchCount | quote }}
         - name: BROKER_ROUTINGERROR
           value: {{ .Values.global.broker.routingError }}
         - name: BROKER_ROUTINGKEY

charts/sda-svc/templates/ingest-deploy.yaml

@@ -115,6 +115,8 @@ spec:
           value: {{ required "A valid MQ host is required" .Values.global.broker.host | quote }}
         - name: BROKER_PORT
           value: {{ .Values.global.broker.port | quote }}
+        - name: BROKER_PREFETCHCOUNT
+          value: {{ .Values.global.broker.prefetchCount | quote }}
         - name: BROKER_ROUTINGERROR
           value: {{ .Values.global.broker.routingError }}
         - name: BROKER_ROUTINGKEY

charts/sda-svc/templates/mapper-deploy.yaml

@@ -76,6 +76,8 @@ spec:
           value: {{ required "A valid MQ host is required" .Values.global.broker.host | quote }}
         - name: BROKER_PORT
           value: {{ .Values.global.broker.port | quote }}
+        - name: BROKER_PREFETCHCOUNT
+          value: {{ .Values.global.broker.prefetchCount | quote }}
         - name: BROKER_ROUTINGERROR
           value: {{ .Values.global.broker.routingError }}
         - name: BROKER_VHOST

charts/sda-svc/templates/s3-inbox-ingress.yaml

@@ -17,7 +17,8 @@ metadata:
     nginx.ingress.kubernetes.io/rewrite-target: "/"
     nginx.ingress.kubernetes.io/backend-protocol: "{{ ternary "HTTPS" "HTTP" .Values.global.tls.enabled }}"
     nginx.ingress.kubernetes.io/proxy-body-size: 2000m
-    nginx.ingress.kubernetes.io/proxy-request-buffering: "off"
+    nginx.ingress.kubernetes.io/proxy-read-timeout: 300s
+    nginx.ingress.kubernetes.io/proxy-request-buffering: "on"
     {{- end }}
     {{- if .Values.global.ingress.clusterIssuer }}
     cert-manager.io/cluster-issuer: {{ .Values.global.ingress.clusterIssuer | quote }}

charts/sda-svc/templates/verify-deploy.yaml

@@ -135,6 +135,8 @@ spec:
           value: {{ required "A valid MQ host is required" .Values.global.broker.host | quote }}
         - name: BROKER_PORT
           value: {{ .Values.global.broker.port | quote }}
+        - name: BROKER_PREFETCHCOUNT
+          value: {{ .Values.global.broker.prefetchCount | quote }}
         - name: BROKER_QUEUE
           value: "archived"
         - name: BROKER_ROUTINGERROR

charts/sda-svc/values.yaml

@@ -137,9 +137,9 @@ global:
     copyHeader: false
 
   auth:
-    # @param elixirID, client ID to the Elixir OIDC for the service endpoint 
+    # @param elixirID, client ID to the Elixir OIDC for the service endpoint
     elixirID:
-    # @param elixirSecret, client secret to the Elixir OIDC for the service endpoint 
+    # @param elixirSecret, client secret to the Elixir OIDC for the service endpoint
     elixirSecret:
     # @param jwtSecret, name of the secret holding the jwt signing key
     jwtSecret:
@@ -149,6 +149,14 @@ global:
     jwtKey:
     # @param jwtPub, name of the public signing key
     jwtPub:
+    # @param resignJwt, if true (or empty) the jwt will be resigned with the jwtKey
+    resignJwt: true
+    # @param corsOrigins, domain name of allowed origin for cross-domain requests
+    corsOrigins: ""
+    # @param corsMethods, allowed methods for cross-domain requests
+    corsMethods: ""
+    # @param corsCreds, allow credentials in the request, cors is disabled if false
+    corsCreds: false
 
   broker:
     durable: true
@@ -162,9 +170,10 @@ global:
     username: ""
     verifyPeer: true
     vhost: "/"
+    prefetchCount: 2
 
   cega:
-    ## @param host, URI to CEGA NSS server users endpoint 
+    ## @param host, URI to CEGA NSS server users endpoint
     host: ""
     ## @param user, usernamen for accessing the CEGA NSS host
     user: ""
@@ -258,7 +267,7 @@ credentials:
   doa:
     dbUser: ""
     dbPassword: ""
-  
+
   download:
     dbUser: ""
     dbPassword: ""
@@ -310,7 +319,7 @@ auth:
   name: auth
   replicaCount: 2
   repository: ghcr.io/neicnordic/sda-auth
-  imageTag: v0.6.54
+  imageTag: v0.7.6
   imagePullPolicy: IfNotPresent
   resources:
     requests:
@@ -331,7 +340,7 @@ backup:
   deploy: false
   replicaCount: 1
   repository: ghcr.io/neicnordic/sda-pipeline
-  imageTag: v0.3.94
+  imageTag: v0.4.27
   imagePullPolicy: IfNotPresent
   resources:
     requests:
@@ -350,7 +359,7 @@ backup:
 doa:
   name: doa
   repository: ghcr.io/neicnordic/sda-doa
-  imageTag: v1.6.46
+  imageTag: v1.6.62
   imagePullPolicy: IfNotPresent
   replicaCount: 2
   resources:
@@ -374,7 +383,7 @@ download:
   name: download
   replicaCount: 2
   repository: ghcr.io/neicnordic/sda-download
-  imageTag: v1.6.117
+  imageTag: v1.9.14
   imagePullPolicy: IfNotPresent
   resources:
     requests:
@@ -394,7 +403,7 @@ finalize:
   name: finalize
   replicaCount: 1
   repository: ghcr.io/neicnordic/sda-pipeline
-  imageTag: v0.3.94
+  imageTag: v0.4.27
   imagePullPolicy: IfNotPresent
   resources:
     requests:
@@ -414,7 +423,7 @@ ingest:
   name: ingest
   replicaCount: 1
   repository: ghcr.io/neicnordic/sda-pipeline
-  imageTag: v0.3.94
+  imageTag: v0.4.27
   imagePullPolicy: IfNotPresent
   resources:
     requests:
@@ -435,7 +444,7 @@ intercept:
   name: ingest
   replicaCount: 1
   repository: ghcr.io/neicnordic/sda-pipeline
-  imageTag: v0.3.94
+  imageTag: v0.4.27
   imagePullPolicy: IfNotPresent
   resources:
     requests:
@@ -454,7 +463,7 @@ intercept:
 mapper:
   replicaCount: 1
   repository: ghcr.io/neicnordic/sda-pipeline
-  imageTag: v0.3.94
+  imageTag: v0.4.27
   imagePullPolicy: IfNotPresent
   resources:
     requests:
@@ -473,7 +482,7 @@ mapper:
 s3Inbox:
   name: s3Inbox
   repository: ghcr.io/neicnordic/sda-s3proxy
-  imageTag: v0.1.153
+  imageTag: v0.2.38
   imagePullPolicy: IfNotPresent
   replicaCount: 2
   resources:
@@ -494,7 +503,7 @@ s3Inbox:
 sftpInbox:
   name: sftpInbox
   repository: ghcr.io/neicnordic/sda-inbox-sftp
-  imageTag: v1.10.4
+  imageTag: v1.12.16
   imagePullPolicy: IfNotPresent
   replicaCount: 2
   resources:
@@ -527,7 +536,7 @@ releasetest:
 verify:
   replicaCount: 1
   repository: ghcr.io/neicnordic/sda-pipeline
-  imageTag: v0.3.94
+  imageTag: v0.4.27
   imagePullPolicy: IfNotPresent
   resources:
     requests:

postgresql/initdb.d/01_main.sql

@@ -20,7 +20,9 @@ VALUES (0, now(), 'Created with version'),
        (3, now(), 'Reorganized out views/tables'),
        (4, now(), 'Refactored schema'),
        (5, now(), 'Add field for correlation ids'),
-       (6, now(), 'Add created_at field to datasets');
+       (6, now(), 'Add created_at field to datasets'),
+       (7, now(), 'Add permissions to mapper to files'),
+       (8, now(), 'Add ingestion functions');
 
 -- Datasets are used to group files, and permissions are set on the dataset
 -- level