Merge pull request #499 from neicnordic/charts_sync_services

[Charts] sync services

charts/sda-db/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v2
 name: sda-db
-version: 0.7.2
-appVersion: v0.2.13
+version: 0.7.3
+appVersion: v0.2.21
 description: Database component for Sensitive Data Archive (SDA) installation
 home: https://neic-sda.readthedocs.io
 icon: https://neic.no/assets/images/logo.png

charts/sda-mq/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v2
 name: sda-mq
-version: 0.6.2
-appVersion: v0.2.13
+version: 0.6.3
+appVersion: v0.2.21
 description: RabbitMQ component for Sensitive Data Archive (SDA) installation
 home: https://neic-sda.readthedocs.io
 icon: https://neic.no/assets/images/logo.png

charts/sda-mq/templates/ingress.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.global.ingress.hostname }}
+{{- if .Values.global.ingress.hostName }}
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:

charts/sda-mq/values.yaml

@@ -4,7 +4,7 @@ global:
   ingress:
     # extra annotations for the ingress
     annotations: {}
-    hostname: ""
+    hostName: ""
     ingressClassName: "nginx"
     issuer: ""
     clusterIssuer: ""

charts/sda-svc/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v2
 name: sda-svc
-version: 0.21.2
-appVersion: v0.2.13
+version: 0.21.3
+appVersion: v0.2.21
 kubeVersion: '>= 1.19.0-0'
 description: Components for Sensitive Data Archive (SDA) installation
 home: https://neic-sda.readthedocs.io

charts/sda-svc/README.md

@@ -62,7 +62,7 @@ Parameter | Description | Default
 `global.archive.volumePath` | Path to the mounted `posix` volume. |`/archive`
 `global.archive.nfsServer` | URL or IP address to a NFS server. |`""`
 `global.archive.nfsPath` | Path on the NFS server for the archive. |`""`
-`global.backupArchive.storageType` | Storage type for the backup of the data archive, available options are `s3` and `posix`. |`s3`
+`global.backupArchive.storageType` | Storage type for the backup of the data archive, available options are `s3` and `posix`. |`null`
 `global.backupArchive.s3Url` | URL to S3 backup archive instance. |`""`
 `global.backupArchive.s3Bucket` | S3 backup archive bucket. |`""`
 `global.backupArchive.s3Region` | S3 backup archive region. |`us-east-1`
@@ -135,13 +135,26 @@ Parameter | Description | Default
 `global.inbox.nfsPath` | Path on the NFS server for the inbox. |`""`
 `global.inbox.existingClaim` | Existing volume to use for the `posix` inbox. | `""`
 `global.inbox.s3Url` | URL to S3 inbox instance. |`""`
+`global.inbox.s3Port` | Port that the S3 inbox is available on. |`443`
 `global.inbox.s3Bucket` | S3 inbox bucket. |`""`
 `global.inbox.s3Region` | S3 inbox region. |`""`
 `global.inbox.s3ChunkSize` | S3 chunk size in MB. |`15`
 `global.inbox.s3AccessKey` | Access key to S3 inbox . |`null`
 `global.inbox.s3SecretKey` | Secret key to S3 inbox. |`null`
 `global.inbox.s3CaFile` | CA certificate to use if the S3 inbox is internal. |`null`
 `global.inbox.s3ReadyPath` | Endpoint to verify that the inbox is respondig. |`""`
+`global.sync.api.password` | Password for authenticating to the syncAPI server | `null`
+`global.sync.api.user` | User for authenticating to the syncAPI server | `null`
+`global.sync.centerPrefix` | Prefix for locally generated datasets | `null`
+`global.sync.destination.storageType` | Storage type for the sync destination, currently only supports S3 | `s3`
+`global.sync.destination.s3Accesskey` | Access key to S3 sync destination | `null`
+`global.sync.destination.s3Bucket` | sync destination bucket | `null`
+`global.sync.destination.s3Port` | Port that the S3 sync destination instance is available on | `443`
+`global.sync.destination.s3Secretkey` | Secret key to S3 sync destination | `null`
+`global.sync.destination.s3url` | URL to S3 sync destination instance. | `null`
+`global.sync.remote.host` | URL to the remote syncAPI host | `null`
+`global.sync.remote.password` | Password for connecting to the remote syncAPI host | `null`
+`global.sync.remote.user` | Username for connecting to the remote syncAPI host | `null`
 `global.tls.enabled` | Use TLS for all connections. |`true`
 `global.tls.issuer` | Issuer for TLS certificate creation. |`""`
 `global.tls.clusterIssuer` | ClusterIssuer for TLS certificate creation. |`""`
@@ -152,31 +165,33 @@ If no shared credentials for the message broker and database are used these shou
 
 Parameter | Description | Default
 --------- | ----------- | -------
-`credentials.sync.dbUser` | Databse user for sync | `""`
-`credentials.sync.dbPassword` | Database password for sync | `""`
-`credentials.sync.mqUser` | Broker user for sync | `""`
-`credentials.sync.mqPassword` | Broker password for sync | `""`
-`credentials.doa.dbUser` | Databse user for doa | `""`
+`credentials.doa.dbUser` | Database user for doa | `""`
 `credentials.doa.dbPassword` | Database password for doa| `""`
-`credentials.download.dbUser` | Databse user for download | `""`
+`credentials.download.dbUser` | Database user for download | `""`
 `credentials.download.dbPassword` | Database password for download| `""`
-`credentials.finalize.dbUser` | Databse user for finalize | `""`
+`credentials.finalize.dbUser` | Database user for finalize | `""`
 `credentials.finalize.dbPassword` | Database password for finalize | `""`
 `credentials.finalize.mqUser` | Broker user for finalize | `""`
 `credentials.finalize.mqPassword` | Broker password for finalize | `""`
 `credentials.inbox.mqUser` | Broker user for inbox | `""`
 `credentials.inbox.mqPassword` | Broker password for inbox | `""`
-`credentials.ingest.dbUser` | Databse user for ingest | `""`
+`credentials.ingest.dbUser` | Database user for ingest | `""`
 `credentials.ingest.dbPassword` | Database password for ingest | `""`
 `credentials.ingest.mqUser` | Broker user for ingest  | `""`
 `credentials.ingest.mqPassword` | Broker password for ingest | `""`
 `credentials.intercept.mqUser` | Broker user for intercept  | `""`
 `credentials.intercept.mqPassword` | Broker password for intercept | `""`
-`credentials.test.dbUser` | Databse user for test | `""`
+`credentials.sync.dbUser` | Database user for sync | `""`
+`credentials.sync.dbPassword` | Database password for sync | `""`
+`credentials.sync.mqUser` | Broker user for sync | `""`
+`credentials.sync.mqPassword` | Broker password for sync | `""`
+`credentials.syncapi.mqUser` | Broker user for sync | `""`
+`credentials.syncapi.mqPassword` | Broker password for sync | `""`
+`credentials.test.dbUser` | Database user for test | `""`
 `credentials.test.dbPassword` | Database password for test | `""`
 `credentials.test.mqUser` | Broker user for test | `""`
 `credentials.test.mqPassword` | Broker password for test | `""`
-`credentials.verify.dbUser` | Databse user for verify | `""`
+`credentials.verify.dbUser` | Database user for verify | `""`
 `credentials.verify.dbPassword` | Database password for verify | `""`
 `credentials.verify.mqUser` | Broker user for verify | `""`
 `credentials.verify.mqPassword` | Broker password for verify | `""`
@@ -246,6 +261,18 @@ Parameter | Description | Default
 `sftpInbox.resources.requests.cpu` | CPU request for sftpInbox container. |`100m`
 `sftpInbox.resources.limits.memory` | Memory limit for sftpInbox container. |`256Mi`
 `sftpInbox.resources.limits.cpu` | CPU limit for sftpInbox container. |`250m`
+`sync.replicaCount`| desired number of sync containers | `1`
+`sync.annotations` | Specific annotation for the sync pod | `{}`
+`sync.resources.requests.memory` | Memory request for sync container. |`128Mi`
+`sync.resources.requests.cpu` | CPU request for sync container. |`100m`
+`sync.resources.limits.memory` | Memory limit for sync container. |`512Mi`
+`sync.resources.limits.cpu` | CPU limit for sync container. |`500m`
+`syncAPI.replicaCount`| desired number of syncAPI containers | `1`
+`syncAPI.annotations` | Specific annotation for the syncAPI pod | `{}`
+`syncAPI.resources.requests.memory` | Memory request for syncAPI container. |`64Mi`
+`syncAPI.resources.requests.cpu` | CPU request for syncAPI container. |`100m`
+`syncAPI.resources.limits.memory` | Memory limit for syncAPI container. |`256Mi`
+`syncAPI.resources.limits.cpu` | CPU limit for syncAPI container. |`500m`
 `verify.replicaCount`| desired number of verify containers | `1`
 `verify.annotations` | Specific annotation for the verify pod | `{}`
 `verify.resources.requests.memory` | Memory request for verify container. |`128Mi`

charts/sda-svc/templates/_helpers.yaml

@@ -289,14 +289,6 @@ Create chart name and version as used by the chart label.
   {{- end -}}
 {{- end -}}
 
-{{- define "S3ArchiveURL" -}}
-  {{- if .Values.global.inbox.s3Port }}
-    {{- printf "%s:%v" .Values.global.inbox.s3Url .Values.global.inbox.s3Port }}
-  {{- else }}
-    {{- printf "%s" .Values.global.inbox.s3Url }}
-  {{- end }}
-{{- end -}}
-
 {{- define "TLSissuer" -}}
     {{- if and .Values.global.tls.clusterIssuer .Values.global.tls.issuer }}
         {{- fail "Only one of global.tls.issuer or global.tls.clusterIssuer should be set" }}

charts/sda-svc/templates/auth-deploy.yaml

@@ -123,6 +123,7 @@ spec:
         - name: CEGA_AUTHURL
           value: {{ .Values.global.cega.host | quote }}
         {{- end }}
+      {{- if .Values.global.auth.resignJwt }}
         - name: JWTISSUER
         {{- if .Values.global.tls.enabled }}
           value: "https://{{ .Values.global.ingress.hostName.auth }}"
@@ -133,6 +134,7 @@ spec:
           value: "{{ template "jwtPath" . }}/{{ .Values.global.auth.jwtKey }}"
         - name: JWTSIGNATUREALG
           value: {{ .Values.global.auth.jwtAlg }}
+      {{- end }}
         - name: RESIGNJWT
           value: {{ .Values.global.auth.resignJwt | quote }}
         {{- if .Values.global.tls.enabled}}
@@ -178,12 +180,12 @@ spec:
         - name: tls
           mountPath: {{ template "tlsPath" . }}
       {{- end }}
-      {{- if not .Values.global.vaultSecrets }}
+      {{- if and (.Values.global.auth.resignJwt) (not .Values.global.vaultSecrets) }}
         - name: jwt
           mountPath: {{ template "jwtPath" . }}
       {{- end }}
       volumes:
-      {{- if not .Values.global.vaultSecrets }}
+      {{- if and (.Values.global.auth.resignJwt) (not .Values.global.vaultSecrets) }}
         - name: jwt
           projected:
             defaultMode: 0440

charts/sda-svc/templates/finalize-deploy.yaml

@@ -116,8 +116,6 @@ spec:
           value: "{{ .Values.global.backupArchive.volumePath }}"
   {{- end }}
 {{- end }}
-        - name: BROKER_DURABLE
-          value: {{ .Values.global.broker.durable | quote }}
         - name: BROKER_EXCHANGE
           value: {{ default "sda" .Values.global.broker.exchange }}
         - name: BROKER_QUEUE

charts/sda-svc/templates/inbox-service.yaml

@@ -1,4 +1,5 @@
 {{- if or (or (eq "all" .Values.global.deploymentType) (eq "external" .Values.global.deploymentType) ) (not .Values.global.deploymentType) }}
+{{- if ne "" .Values.global.inbox.storageType }}
 apiVersion: v1
 kind: Service
 metadata:
@@ -14,3 +15,4 @@ spec:
   selector:
     app: {{ template "sda.fullname" . }}-inbox
 {{- end }}
+{{- end }}

charts/sda-svc/templates/ingest-deploy.yaml

@@ -105,8 +105,6 @@ spec:
         - name: BROKER_VERIFYPEER
           value: {{ .Values.global.broker.verifyPeer | quote }}
       {{- end }}
-        - name: BROKER_DURABLE
-          value: {{ .Values.global.broker.durable | quote }}
         - name: BROKER_EXCHANGE
           value: {{ default "sda" .Values.global.broker.exchange }}
         - name: BROKER_QUEUE

charts/sda-svc/templates/intercept-certificate.yaml

@@ -1,3 +1,4 @@
+{{- if eq "federated" .Values.global.schemaType }}
 {{- if .Values.global.tls.enabled }}
 {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }}
 apiVersion: cert-manager.io/v1
@@ -36,3 +37,4 @@ spec:
     group: cert-manager.io
 {{- end -}}
 {{- end -}}
+{{- end -}}

charts/sda-svc/templates/intercept-deploy.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.intercept.deploy}}
+{{- if eq "federated" .Values.global.schemaType }}
 {{- if or (or (eq "all" .Values.global.deploymentType) (eq "internal" .Values.global.deploymentType) ) (not .Values.global.deploymentType) }}
 apiVersion: apps/v1
 kind: Deployment
@@ -82,8 +82,6 @@ spec:
         - name: BROKER_VERIFYPEER
           value: {{ .Values.global.broker.verifyPeer | quote }}
       {{- end }}
-        - name: BROKER_DURABLE
-          value: "true"
         - name: BROKER_EXCHANGE
           value: {{ default "sda" .Values.global.broker.exchange | quote}}
         - name: BROKER_HOST

charts/sda-svc/templates/intercept-secrets.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.intercept.deploy}}
+{{- if eq "federated" .Values.global.schemaType }}
 {{- if or (or (eq "all" .Values.global.deploymentType) (eq "internal" .Values.global.deploymentType) ) (not .Values.global.deploymentType) }}
 {{- if not .Values.global.vaultSecrets }}
 ---
@@ -9,7 +9,7 @@ metadata:
 type: Opaque
 data:
   mqPassword: {{ required "MQ password is required" (include "mqPassInterceptor" .) | b64enc }}
-  mqUser: {{( required "MQ user is required" include "mqUserInterceptor" .) | b64enc }}
+  mqUser: {{ required "MQ user is required" (include "mqUserInterceptor" .) | b64enc }}
 {{- end }}
 {{- end }}
 {{- end }}

charts/sda-svc/templates/mapper-deploy.yaml

@@ -66,8 +66,6 @@ spec:
         securityContext:
           allowPrivilegeEscalation: false
         env:
-        - name: BROKER_DURABLE
-          value: {{ .Values.global.broker.durable | quote }}
         - name: BROKER_EXCHANGE
           value: {{ default "sda" .Values.global.broker.exchange }}
         - name: BROKER_QUEUE

charts/sda-svc/templates/s3-inbox-deploy.yaml

@@ -188,8 +188,10 @@ spec:
         {{- if.Values.global.auth.jwtPub }}
         - name: SERVER_JWTPUBKEYPATH
           value: {{ include "jwtPath" . }}
-        {{- else }}
-          {{ fail "The global.auth.jwtPub is required for s3Inbox" }}
+        {{- end }}
+        {{- if not .Values.global.auth.resignJwt }}
+        - name: SERVER_JWTPUBKEYURL
+          value: {{ .Values.global.oidc.provider }}{{ .Values.global.oidc.jwkPath }}
         {{- end }}
       {{- if .Values.global.log.format }}
         - name: LOG_FORMAT

charts/sda-svc/templates/sync-certificate.yaml

@@ -1,3 +1,4 @@
+{{- if eq "isolated" .Values.global.schemaType }}
 {{- if ne "" .Values.global.sync.remote.host }}
 {{- if .Values.global.tls.enabled }}
 {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }}
@@ -38,3 +39,4 @@ spec:
 {{- end -}}
 {{- end -}}
 {{- end -}}
+{{- end -}}

charts/sda-svc/templates/sync-deploy.yaml

@@ -1,3 +1,4 @@
+{{- if eq "isolated" .Values.global.schemaType }}
 {{- if ne "" .Values.global.sync.remote.host }}
 {{- if or (or (eq "all" .Values.global.deploymentType) (eq "internal" .Values.global.deploymentType) ) (not .Values.global.deploymentType) }}
 apiVersion: apps/v1
@@ -91,6 +92,8 @@ spec:
         - name: ARCHIVE_LOCATION
           value: "{{ .Values.global.archive.volumePath }}"
   {{- end }}
+        - name: SYNC_CENTERPREFIX
+          value: {{ .Values.global.sync.centerPrefix}}
         - name: SYNC_DESTINATION_TYPE
   {{- if eq "s3" .Values.global.sync.destination.storageType }}
           value: "s3"
@@ -113,22 +116,18 @@ spec:
   {{- end }}
         - name: C4GH_FILEPATH
           value: "{{ template "c4ghPath" . }}/{{ .Values.global.c4gh.keyFile }}"
-        - name: C4GH_SYNCPUBKEY
+        - name: C4GH_SYNCPUBKEYPATH
           value: "{{ template "c4ghPath" . }}/{{ .Values.global.c4gh.syncPubKey }}"
-        - name: BROKER_DURABLE
-          value: {{ .Values.global.broker.durable | quote }}
         - name: BROKER_EXCHANGE
           value: {{ default "sda" .Values.global.broker.exchange }}
         - name: BROKER_QUEUE
-          value: {{ .Values.global.broker.syncRoutingKey }}
+          value: {{ default "mapping_stream" .Values.global.sync.brokerQueue }}
         - name: BROKER_HOST
           value: {{ required "A valid MQ host is required" .Values.global.broker.host | quote }}
         - name: BROKER_PORT
           value: {{ .Values.global.broker.port | quote }}
         - name: BROKER_PREFETCHCOUNT
           value: {{ .Values.global.broker.prefetchCount | quote }}
-        - name: BROKER_ROUTINGKEY
-          value: "completed"
         - name: BROKER_VHOST
           value: {{ .Values.global.broker.vhost | quote }}
         - name: BROKER_SERVERNAME
@@ -158,7 +157,7 @@ spec:
     {{- end }}
   {{- end }}
         - name: DB_DATABASE
-          value: {{ default "lega" .Values.global.db.name | quote }}
+          value: {{ default "sda" .Values.global.db.name | quote }}
         - name: DB_HOST
           value: {{ required "A valid DB host is required" .Values.global.db.host | quote }}
         - name: DB_PORT
@@ -175,6 +174,8 @@ spec:
       {{- end }}
         - name: SCHEMA_TYPE
           value: {{ default "isolated" .Values.global.schemaType }}
+        - name: SYNC_REMOTE_HOST
+          value: {{ .Values.global.sync.remote.host }}
   {{- if not .Values.global.vaultSecrets }}
     {{- if eq "s3" .Values.global.archive.storageType }}
         - name: ARCHIVE_ACCESSKEY
@@ -192,12 +193,12 @@ spec:
         - name: SYNC_DESTINATION_ACCESSKEY
           valueFrom:
             secretKeyRef:
-              name: {{ template "sda.fullname" . }}-keys
+              name: {{ template "sda.fullname" . }}-sync
               key: s3AccessKey
         - name: SYNC_DESTINATION_SECRETKEY
           valueFrom:
             secretKeyRef:
-              name: {{ template "sda.fullname" . }}-keys
+              name: {{ template "sda.fullname" . }}-sync
               key: s3SecretKey
     {{- end }}
         - name: BROKER_PASSWORD
@@ -210,6 +211,11 @@ spec:
               secretKeyRef:
                 name: {{ template "sda.fullname" . }}-sync
                 key: mqUser
+        - name: C4GH_PASSPHRASE
+          valueFrom:
+            secretKeyRef:
+              name: {{ required "A secret for the c4gh key is required" .Values.global.c4gh.secretName }}
+              key: passphrase
         - name: DB_PASSWORD
           valueFrom:
               secretKeyRef:
@@ -242,12 +248,12 @@ spec:
             mountPath: {{ template "c4ghPath" . }}
       {{- end }}
   {{- if eq "posix" .Values.global.archive.storageType }}
-        - name: archive
-          mountPath: {{ .Values.global.archive.volumePath | quote }}
+          - name: archive
+            mountPath: {{ .Values.global.archive.volumePath | quote }}
   {{- end }}
   {{- if and (not .Values.global.pkiService) .Values.global.tls.enabled }}
-        - name: tls
-          mountPath: {{ template "tlsPath" . }}
+          - name: tls
+            mountPath: {{ template "tlsPath" . }}
   {{- end }}
       volumes:
   {{- if and (not .Values.global.pkiService) .Values.global.tls.enabled }}
@@ -293,3 +299,4 @@ spec:
       restartPolicy: Always
 {{- end }}
 {{- end }}
+{{- end }}