DRAFT: Add new instructions API

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.6.0
+current_version = 1.7.0
 commit = False
 tag = False
 

docker-compose-dev.yml

@@ -39,6 +39,7 @@ services:
       SERVER_DOMAIN: ${SERVER_DOMAIN:-localmaeher.dev.pvarki.fi}
       API_PORT: ${NGINX_HTTPS_PORT:-4439}
       VITE_ASSET_SET: ${VITE_ASSET_SET:-neutral}
+      RELEASE_TAG: "localdev"
     networks:
       - intranet
     ports:  # REMINDER Do not expose these in production

docker-compose-local.yml

@@ -71,12 +71,12 @@ x-keycloakinit_users_env: &keycloakinit_users_env
   KEYCLOAK_PASSWORD: *kcadminpass # pragma: allowlist secret
 
 x-takbuilds: &takbuildinfo
-  image: &takimage "pvarki/takserver:${TAK_RELEASE:-5.2-RELEASE-30}${DOCKER_TAG_EXTRA:-}"
+  image: &takimage "pvarki/takserver:${TAK_RELEASE:-5.3-RELEASE-24}${DOCKER_TAG_EXTRA:-}"
   build:
     context: ./takserver
     dockerfile: Dockerfile
     args:
-      TAK_RELEASE: ${TAK_RELEASE:-5.2-RELEASE-30}
+      TAK_RELEASE: ${TAK_RELEASE:-5.3-RELEASE-24}
 
 x-nginxbuilds: &nginxbuildinfo
   image: pvarki/nginx:1.25${DOCKER_TAG_EXTRA:-}
@@ -120,10 +120,13 @@ services:
       target: production
     environment:
       MW_DOMAIN: *serverdomain
-      MW_PRODUCTS: "tak,kc,fake"
+      MW_PRODUCTS: "tak,kc,fake,bl"
       MW_RASENMAEHER__API_PORT: *apiport
+      MW_RASENMAEHER__USER_PORT: *apiport
       MW_FAKE__API_PORT: *productport
+      MW_FAKE__USER_PORT: *productport
       MW_TAK__API_PORT: *takapiport
+      MW_TAK__USER_PORT: 8443
       CAROOT: "/data/persistent/mkcert"
       MW_LE_EMAIL: "notusedwithmkcert@example.com"
       MW_LE_TEST: "true"
@@ -399,6 +402,10 @@ services:
       RM_KC_USERNAME: *kcadminuser
       RM_KC_PASSWORD: *kcadminpass  # pragma: allowlist secret
       RM_KC_REALM: *kc_realm
+      UVICORN_LOG_LEVEL: "debug"
+      RM_LOG_LEVEL: "DEBUG"
+      RM_LOG_LEVEL_INT: "10"
+      RELEASE_TAG: "local"
     networks:
       - apinet
       - kcnet
@@ -441,6 +448,7 @@ services:
       target: production
       args:
         VITE_ASSET_SET: ${VITE_ASSET_SET:-neutral}
+        RELEASE_TAG: "local"
     volumes:
       - rmui_files:/deliver
 
@@ -748,7 +756,7 @@ services:
     restart: unless-stopped
 
   takrmapi:
-    image: pvarki/takrmapi:local${DOCKER_TAG_EXTRA:-}-tak${TAK_RELEASE:-5.2-RELEASE-30}
+    image: pvarki/takrmapi:local${DOCKER_TAG_EXTRA:-}-tak${TAK_RELEASE:-5.3-RELEASE-24}
     build:
       context: ./takintegration
       dockerfile: Dockerfile
@@ -758,12 +766,15 @@ services:
     network_mode: "service:takconfig"
     environment:
       LOG_CONSOLE_FORMATTER: "local"
+      UVICORN_LOG_LEVEL: "debug"
+      TI_LOG_LEVEL: "10"
     volumes:
       - ca_public:/ca_public
       - le_certs:/le_certs
       - kraftwerk_shared_tak:/pvarki
       - takrmapi_data:/data/persistent
       - tak_data:/opt/tak/data
+      - tak_www_static:/www_static
     depends_on:
       rmnginx:
         condition: service_healthy
@@ -785,6 +796,7 @@ services:
       - nginx_templates:/nginx_templates
       - ca_public:/ca_public
       - le_certs:/le_certs
+      - tak_www_static:/www_static
     environment:
       NGINX_HOST: *takdomain
       NGINX_HTTPS_PORT: *takapiport
@@ -794,7 +806,7 @@ services:
       CFSSL_OCSP_BIND_PORT: *oscpport
       NGINX_OCSP_UPSTREAM: *ocsphost
       DNS_RESOLVER_IP: *dnsresolver
-      NGINX_TEMPLATE_DIR: "templates_productapi"
+      NGINX_TEMPLATE_DIR: "templates_rasenmaeher_takapi"
     networks:
       - taknet
       - intranet
@@ -851,3 +863,4 @@ volumes:
   takrmapi_data:
   rmui_files:
   nginx_templates:
+  tak_www_static:

docker-compose.yml

@@ -68,15 +68,15 @@ x-keycloakinit_users_env: &keycloakinit_users_env
   KEYCLOAK_PASSWORD: *kcadminpass # pragma: allowlist secret
 
 x-takbuilds: &takbuildinfo
-  image: &takimage "pvarki/takserver:${TAK_RELEASE:-5.2-RELEASE-30}-d${RELEASE_TAG:-1.6.0}${DOCKER_TAG_EXTRA:-}"
+  image: &takimage "pvarki/takserver:${TAK_RELEASE:-5.3-RELEASE-24}-d${RELEASE_TAG:-1.7.0}${DOCKER_TAG_EXTRA:-}"
   build:
     context: ./takserver
     dockerfile: Dockerfile
     args:
-      TAK_RELEASE: ${TAK_RELEASE:-5.2-RELEASE-30}
+      TAK_RELEASE: ${TAK_RELEASE:-5.3-RELEASE-24}
 
 x-nginxbuilds: &nginxbuildinfo
-  image: pvarki/nginx:1.25-d${RELEASE_TAG:-1.6.0}${DOCKER_TAG_EXTRA:-}
+  image: pvarki/nginx:1.25-d${RELEASE_TAG:-1.7.0}${DOCKER_TAG_EXTRA:-}
   build:
     context: ./nginx
     dockerfile: Dockerfile
@@ -115,20 +115,24 @@ x-takserver_env: &takserver_env
 
 services:
   miniwerk:
-    image: pvarki/miniwerk:1.1.0-d${RELEASE_TAG:-1.6.0}${DOCKER_TAG_EXTRA:-}
+    image: pvarki/miniwerk:1.3.0-d${RELEASE_TAG:-1.7.0}${DOCKER_TAG_EXTRA:-}
     build:
       context: ./miniwerk
       dockerfile: Dockerfile
       target: production
     environment:
       MW_DOMAIN: *serverdomain
       MW_RASENMAEHER__API_PORT: *apiport
+      MW_RASENMAEHER__USER_PORT: *apiport
       MW_FAKE__API_PORT: *productport
+      MW_FAKE__USER_PORT: *productport
       MW_TAK__API_PORT: *takapiport
+      MW_TAK__USER_PORT: 8443
       MW_LE_EMAIL: ${MW_LE_EMAIL?LE contact email must be defined}
       MW_LE_TEST: ${MW_LE_TEST:-true}  # see example_env.sh
       MW_MKCERT: ${MW_MKCERT:-false}  # When LetEncrypt cannot be used set to "true"
       MW_KEYTYPE: "rsa"
+      #MW_PRODUCTS: "tak,kc,bl"
       MW_PRODUCTS: "tak,kc"
     volumes:
       - kraftwerk_shared_fake:/pvarkishares/fake
@@ -141,7 +145,7 @@ services:
       - "80:80"
 
   cfssl:
-    image: pvarki/cfssl:api-1.2.0-d${RELEASE_TAG:-1.6.0}${DOCKER_TAG_EXTRA:-}
+    image: pvarki/cfssl:api-1.2.0-d${RELEASE_TAG:-1.7.0}${DOCKER_TAG_EXTRA:-}
     build:
       context: ./cfssl
       dockerfile: Dockerfile
@@ -164,7 +168,7 @@ services:
     restart: unless-stopped
 
   ocsp:
-    image: pvarki/cfssl:ocsp-1.2.0-d${RELEASE_TAG:-1.6.0}${DOCKER_TAG_EXTRA:-}
+    image: pvarki/cfssl:ocsp-1.2.0-d${RELEASE_TAG:-1.7.0}${DOCKER_TAG_EXTRA:-}
     build:
       context: ./cfssl
       dockerfile: Dockerfile
@@ -190,7 +194,7 @@ services:
     restart: unless-stopped
 
   ocsprest:
-    image: pvarki/cfssl:ocsprest-1.0.4-d${RELEASE_TAG:-1.6.0}${DOCKER_TAG_EXTRA:-}
+    image: pvarki/cfssl:ocsprest-1.0.4-d${RELEASE_TAG:-1.7.0}${DOCKER_TAG_EXTRA:-}
     build:
       context: ./cfssl
       dockerfile: Dockerfile
@@ -250,7 +254,7 @@ services:
         condition: service_completed_successfully
 
   openldap:
-    image: pvarki/openldap:1.0.0-d${RELEASE_TAG:-1.6.0}${DOCKER_TAG_EXTRA:-}
+    image: pvarki/openldap:1.0.0-d${RELEASE_TAG:-1.7.0}${DOCKER_TAG_EXTRA:-}
     build:
       context: ./keycloak/openldap
       dockerfile: Dockerfile
@@ -360,7 +364,7 @@ services:
         condition: service_healthy
 
   rmapi:
-    image: pvarki/rmapi:1.5.0-d${RELEASE_TAG:-1.6.0}${DOCKER_TAG_EXTRA:-}
+    image: pvarki/rmapi:1.6.0-d${RELEASE_TAG:-1.7.0}${DOCKER_TAG_EXTRA:-}
     build:
       context: ./api
       dockerfile: Dockerfile
@@ -381,6 +385,8 @@ services:
       RM_KC_USERNAME: *kcadminuser
       RM_KC_PASSWORD: *kcadminpass  # pragma: allowlist secret
       RM_KC_REALM: *kc_realm
+      RM_LOG_LEVEL: "INFO"
+      RELEASE_TAG: ${RELEASE_TAG:-1.7.0}
     networks:
       - apinet
       - kcnet
@@ -414,18 +420,19 @@ services:
     restart: unless-stopped
 
   rmui:
-    image: pvarki/rmui:1.3.0-${VITE_ASSET_SET:-neutral}-d${RELEASE_TAG:-1.6.0}${DOCKER_TAG_EXTRA:-}
+    image: pvarki/rmui:1.3.0-${VITE_ASSET_SET:-neutral}-d${RELEASE_TAG:-1.7.0}${DOCKER_TAG_EXTRA:-}
     build:
       context: ./ui
       dockerfile: Dockerfile
       target: production
       args:
         VITE_ASSET_SET: ${VITE_ASSET_SET:-neutral}
+        RELEASE_TAG: ${RELEASE_TAG:-1.7.0}
     volumes:
       - rmui_files:/deliver
 
   nginx_templates:
-    image: pvarki/nginx:templates-d${RELEASE_TAG:-1.6.0}${DOCKER_TAG_EXTRA:-}
+    image: pvarki/nginx:templates-d${RELEASE_TAG:-1.7.0}${DOCKER_TAG_EXTRA:-}
     build:
       context: ./nginx
       dockerfile: Dockerfile
@@ -484,7 +491,7 @@ services:
     restart: unless-stopped
 
   kwinit:  # Mostly to make sure it's built
-    image: pvarki/kw_product_init:1.0.0-d${RELEASE_TAG:-1.6.0}${DOCKER_TAG_EXTRA:-}
+    image: pvarki/kw_product_init:1.0.0-d${RELEASE_TAG:-1.7.0}${DOCKER_TAG_EXTRA:-}
     build:
       context: ./kw_product_init
       dockerfile: Dockerfile
@@ -648,13 +655,16 @@ services:
     restart: unless-stopped
 
   takrmapi:
-    image: pvarki/takrmapi:1.3.0-tak${TAK_RELEASE:-5.2-RELEASE-30}-d${RELEASE_TAG:-1.6.0}${DOCKER_TAG_EXTRA:-}
+    image: pvarki/takrmapi:1.4.1-tak${TAK_RELEASE:-5.3-RELEASE-24}-d${RELEASE_TAG:-1.7.0}${DOCKER_TAG_EXTRA:-}
     build:
       context: ./takintegration
       dockerfile: Dockerfile
       target: production
       args:
         TAKSERVER_IMAGE: *takimage
+    environment:
+      UVICORN_LOG_LEVEL: "info"
+      TI_LOG_LEVEL: "20"
     labels:
         - "autoheal=true"
     network_mode: "service:takconfig"
@@ -664,6 +674,7 @@ services:
       - kraftwerk_shared_tak:/pvarki
       - tak_data:/opt/tak/data
       - takrmapi_data:/data/persistent
+      - tak_www_static:/www_static
     depends_on:
       rmnginx:
         condition: service_healthy
@@ -685,6 +696,7 @@ services:
       - nginx_templates:/nginx_templates
       - ca_public:/ca_public
       - le_certs:/le_certs
+      - tak_www_static:/www_static
     environment:
       NGINX_HOST: *takdomain
       NGINX_HTTPS_PORT: *takapiport
@@ -694,7 +706,7 @@ services:
       CFSSL_OCSP_BIND_PORT: *oscpport
       NGINX_OCSP_UPSTREAM: *ocsphost
       DNS_RESOLVER_IP: *dnsresolver
-      NGINX_TEMPLATE_DIR: "templates_productapi"
+      NGINX_TEMPLATE_DIR: "templates_rasenmaeher_takapi"
     networks:
       - taknet
       - intranet
@@ -764,3 +776,4 @@ volumes:
   takrmapi_data:
   rmui_files:
   nginx_templates:
+  tak_www_static:

nginx/templates_rasenmaeher/default.conf.template

@@ -109,7 +109,7 @@ server {
 
     location /api {
         if ($ssl_client_verify != SUCCESS) {
-            return 302 https://${NGINX_HOST}:${NGINX_HTTPS_PORT}/error?code=mtls_fail;
+            return 302 https://${NGINX_HOST}:${NGINX_HTTPS_PORT}/error?code=mtls_fail&exta=$ssl_client_verify;
         }
         proxy_pass  http://${NGINX_UPSTREAM}:${NGINX_UPSTREAM_PORT}/api;
         proxy_redirect                      off;
@@ -123,14 +123,14 @@ server {
     }
 
     # Even though users sees code 400 the code is 495 http://nginx.org/en/docs/http/ngx_http_ssl_module.html#errors
-    error_page 495 https://${NGINX_HOST}:${NGINX_HTTPS_PORT}/error?code=mtls_fail;
+    error_page 495 =302 https://${NGINX_HOST}:${NGINX_HTTPS_PORT}/error?code=mtls_fail&exta=$ssl_client_verify;
 
     location / {
         if ($redir_uri != "") {
             return 301 $redir_uri$request_uri;
         }
         if ($ssl_client_verify != SUCCESS) {
-            return 302 https://${NGINX_HOST}:${NGINX_HTTPS_PORT}/error?code=mtls_fail;
+            return 302 https://${NGINX_HOST}:${NGINX_HTTPS_PORT}/error?code=mtls_fail&exta=$ssl_client_verify;
         }
         index index.html;
         root /rmui_files;

nginx/templates_rasenmaeher_takapi/default.conf.template

@@ -0,0 +1,43 @@
+ssl_certificate /le_certs/${NGINX_CERT_NAME}/fullchain.pem;
+ssl_certificate_key /le_certs/${NGINX_CERT_NAME}/privkey.pem;
+
+include /etc/nginx/includes/le_common_settings.conf;
+
+server {
+    server_name ${NGINX_HOST};
+
+    # HTTPS configuration
+    listen ${NGINX_HTTPS_PORT} ssl;
+
+    ssl_client_certificate /ca_public/ca_chain.pem;
+    ssl_verify_client      on;
+    ssl_ocsp leaf;
+    ssl_ocsp_responder http://${NGINX_OCSP_UPSTREAM}:${CFSSL_OCSP_BIND_PORT};
+    resolver ${DNS_RESOLVER_IP} ipv6=off;
+    #ssl_crl /ca_public/crl.pem;
+    ssl_verify_depth 3;
+
+
+    # This volume may only have content that EVERYONE with proper client certificate can view
+    location /content/static {
+        if ($ssl_client_verify != SUCCESS) {
+            return 401;
+        }
+        root /www_static;
+    }
+
+    location / {
+        if ($ssl_client_verify != SUCCESS) {
+            return 401;
+        }
+        proxy_pass  http://${NGINX_UPSTREAM}:${NGINX_UPSTREAM_PORT};
+        proxy_redirect                      off;
+        proxy_set_header  Host              $http_host;
+        proxy_set_header  X-Real-IP         $remote_addr;
+        proxy_set_header  X-Forwarded-For   $proxy_add_x_forwarded_for;
+        proxy_set_header  X-Forwarded-Proto $scheme;
+        proxy_read_timeout                  900;
+        proxy_set_header  X-ClientCert-DN   $ssl_client_s_dn;
+        proxy_set_header  X-ClientCert-Serial   ssl_client_serial;
+    }
+}

tests/conftest.py

@@ -22,7 +22,7 @@
 LOGGER = logging.getLogger(__name__)
 CA_PATH = Path(__file__).parent / "testcas"
 JWT_PATH = Path(__file__).parent / "testjwts"
-DEFAULT_TIMEOUT = aiohttp.ClientTimeout(total=15.0)
+DEFAULT_TIMEOUT = aiohttp.ClientTimeout(total=25.0)
 OPENAPI_VER = "3.1.0"
 API = os.environ.get("RM_API_BASE", "https://localmaeher.dev.pvarki.fi:4439/api")  # pylint: disable=E1101
 VER = "v1"

tests/requirements.txt

@@ -14,3 +14,4 @@ pytest-asyncio>=0.23,<1.0.0
 libpvarki @ git+https://github.com/pvarki/python-libpvarki.git@1.9.1
 bump2version>=1.0.1,<2.0.0
 pendulum>=3.0.0,<4.0.0
+flaky>=3.8.1,<4.0.0