feat(reports): reports sidecar, configurable caching (#104)

* fix archived recording deletion

* add expiration date to recording metadata, do not show such 'temporary' recordings in API query responses

* refactor to add evaluation predicate to interface

* refactor to extract separate reports in-memory caching layer

* report more accurate status code on failure

* implement remote sidecar report generator

* add switch for configuring reports sidecar

* proactive cache invalidation on recording deletion

* tolerate non-connectable target when removing active recording record

* log hibernate SQL statements

* fix concurrent modification exception on target loss

* optionally log full MessagingServer WebSocket message JSON when log level is debug

pom.xml

@@ -32,10 +32,11 @@
     <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
 
     <io.cryostat.core.version>2.27.0</io.cryostat.core.version>
-    
+
     <org.apache.commons.codec.version>1.16.0</org.apache.commons.codec.version>
     <org.apache.commons.io.version>2.13.0</org.apache.commons.io.version>
     <org.apache.httpcomponents.version>4.5.14</org.apache.httpcomponents.version>
+    <org.apache.httpcomponents.version>5.2.1</org.apache.httpcomponents.version>
     <org.apache.commons.lang3.version>3.13.0</org.apache.commons.lang3.version>
     <org.apache.commons.validator.version>1.7</org.apache.commons.validator.version>
     <org.projectnessie.cel.bom.version>0.3.21</org.projectnessie.cel.bom.version>
@@ -133,7 +134,7 @@
     </dependency>
     <dependency>
       <groupId>io.quarkus</groupId>
-    <artifactId>quarkus-hibernate-validator</artifactId>
+      <artifactId>quarkus-hibernate-validator</artifactId>
     </dependency>
     <dependency>
       <groupId>io.quarkus</groupId>
@@ -163,8 +164,8 @@
       <version>${org.apache.commons.io.version}</version>
     </dependency>
     <dependency>
-      <groupId>org.apache.httpcomponents</groupId>
-      <artifactId>httpclient</artifactId>
+      <groupId>org.apache.httpcomponents.client5</groupId>
+      <artifactId>httpclient5</artifactId>
       <version>${org.apache.httpcomponents.version}</version>
     </dependency>
     <dependency>

smoketest.bash

@@ -26,6 +26,7 @@ display_usage() {
     echo -e "\t-p\t\t\t\t\t\tDisable auth Proxy."
     echo -e "\t-s [minio|seaweed|cloudserver|localstack]\tS3 implementation to spin up (default \"minio\")."
     echo -e "\t-g\t\t\t\t\t\tinclude Grafana dashboard and jfr-datasource in deployment."
+    echo -e "\t-r\t\t\t\t\t\tconfigure a cryostat-Reports sidecar instance"
     echo -e "\t-t\t\t\t\t\t\tinclude sample applications for Testing."
     echo -e "\t-V\t\t\t\t\t\tdo not discard data storage Volumes on exit."
     echo -e "\t-X\t\t\t\t\t\tdeploy additional development aid tools."
@@ -35,7 +36,7 @@ display_usage() {
 
 s3=minio
 ce=podman
-while getopts "hs:pgtOVXcb" opt; do
+while getopts "hs:prgtOVXcb" opt; do
     case $opt in
         h)
             display_usage
@@ -68,6 +69,9 @@ while getopts "hs:pgtOVXcb" opt; do
         b)
             OPEN_TABS=true
             ;;
+        r)
+            FILES+=('./smoketest/compose/reports.yml')
+            ;;
         *)
             display_usage
             exit 1

smoketest/compose/cryostat.yml

@@ -27,6 +27,7 @@ services:
     environment:
       QUARKUS_HTTP_HOST: "cryostat"
       QUARKUS_HTTP_PORT: ${CRYOSTAT_HTTP_PORT}
+      QUARKUS_HIBERNATE_ORM_LOG_SQL: "true"
       CRYOSTAT_DISCOVERY_PODMAN_ENABLED: "true"
       CRYOSTAT_DISCOVERY_JDP_ENABLED: "true"
       JAVA_OPTS_APPEND: "-XX:+FlightRecorder -XX:StartFlightRecording=name=onstart,settings=default,disk=true,maxage=5m -Dcom.sun.management.jmxremote.autodiscovery=true -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9091 -Dcom.sun.management.jmxremote.rmi.port=9091 -Djava.rmi.server.hostname=127.0.0.1 -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.local.only=false"

smoketest/compose/reports.yml

@@ -0,0 +1,24 @@
+version: "3"
+services:
+  cryostat:
+    environment:
+      - CRYOSTAT_SERVICES_REPORTS_URL=http://reports:10001
+  reports:
+    image: ${CRYOSTAT_REPORTS_IMAGE:-quay.io/cryostat/cryostat-reports:latest}
+    hostname: reports
+    restart: unless-stopped
+    deploy:
+      resources:
+        limits:
+          cpus: '0.5'
+          memory: 512m
+    expose:
+      - "10001"
+    labels:
+      kompose.service.expose: "reports"
+      io.cryostat.discovery: "true"
+      io.cryostat.jmxHost: "reports"
+      io.cryostat.jmxPort: "11224"
+    environment:
+      JAVA_OPTS_APPEND: "-Dcom.sun.management.jmxremote.autodiscovery=true -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=11224 -Dcom.sun.management.jmxremote.rmi.port=11224 -Djava.rmi.server.hostname=reports -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.local.only=false"
+      QUARKUS_HTTP_PORT: 10001

src/main/java/io/cryostat/ConfigProperties.java

@@ -20,6 +20,16 @@ public class ConfigProperties {
     public static final String AWS_OBJECT_EXPIRATION_LABELS =
             "storage.buckets.archives.expiration-label";
 
+    public static final String REPORTS_SIDECAR_URL = "cryostat.services.reports.url";
+    public static final String MEMORY_CACHE_ENABLED =
+            "cryostat.services.reports.memory-cache.enabled";
+    public static final String STORAGE_CACHE_ENABLED =
+            "cryostat.services.reports.storage-cache.enabled";
+    public static final String ARCHIVED_REPORTS_STORAGE_CACHE_NAME =
+            "cryostat.services.reports.storage-cache.name";
+    public static final String ARCHIVED_REPORTS_EXPIRY_DURATION =
+            "cryostat.services.reports.storage-cache.expiry-duration";
+
     public static final String GRAFANA_DASHBOARD_URL = "grafana-dashboard.url";
     public static final String GRAFANA_DASHBOARD_EXT_URL = "grafana-dashboard-ext.url";
     public static final String GRAFANA_DATASOURCE_URL = "grafana-datasource.url";

src/main/java/io/cryostat/ExceptionMappers.java

@@ -20,53 +20,65 @@
 import io.cryostat.targets.TargetConnectionManager;
 
 import io.netty.handler.codec.http.HttpResponseStatus;
+import jakarta.inject.Inject;
 import jakarta.persistence.NoResultException;
 import org.hibernate.exception.ConstraintViolationException;
+import org.jboss.logging.Logger;
 import org.jboss.resteasy.reactive.RestResponse;
 import org.jboss.resteasy.reactive.server.ServerExceptionMapper;
 import org.projectnessie.cel.tools.ScriptException;
 import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
 
 public class ExceptionMappers {
+
+    @Inject Logger logger;
+
     @ServerExceptionMapper
     public RestResponse<Void> mapNoResultException(NoResultException ex) {
         return RestResponse.notFound();
     }
 
     @ServerExceptionMapper
     public RestResponse<Void> mapConstraintViolationException(ConstraintViolationException ex) {
+        logger.warn(ex);
         return RestResponse.status(HttpResponseStatus.BAD_REQUEST.code());
     }
 
     @ServerExceptionMapper
     public RestResponse<Void> mapValidationException(jakarta.validation.ValidationException ex) {
+        logger.warn(ex);
         return RestResponse.status(HttpResponseStatus.BAD_REQUEST.code());
     }
 
     @ServerExceptionMapper
     public RestResponse<Void> mapScriptException(ScriptException ex) {
+        logger.warn(ex);
         return RestResponse.status(HttpResponseStatus.BAD_REQUEST.code());
     }
 
     @ServerExceptionMapper
     public RestResponse<Void> mapNoSuchKeyException(NoSuchKeyException ex) {
+        logger.warn(ex);
         return RestResponse.status(HttpResponseStatus.NOT_FOUND.code());
     }
 
     @ServerExceptionMapper
-    public RestResponse<Void> mapIllegalArgumentException(IllegalArgumentException exception) {
+    public RestResponse<Void> mapIllegalArgumentException(IllegalArgumentException ex) {
+        logger.warn(ex);
         return RestResponse.status(HttpResponseStatus.BAD_REQUEST.code());
     }
 
     @ServerExceptionMapper
-    public RestResponse<Void> mapJmxConnectionException(ConnectionException exception) {
+    public RestResponse<Void> mapJmxConnectionException(ConnectionException ex) {
+        logger.warn(ex);
         return RestResponse.status(HttpResponseStatus.BAD_GATEWAY.code());
     }
 
     @ServerExceptionMapper
     public RestResponse<Void> mapFlightRecorderException(
-            org.openjdk.jmc.rjmx.services.jfr.FlightRecorderException exception) {
-        if (TargetConnectionManager.isJmxAuthFailure(exception)) {
+            org.openjdk.jmc.rjmx.services.jfr.FlightRecorderException ex) {
+        logger.warn(ex);
+        if (TargetConnectionManager.isJmxAuthFailure(ex)) {
             return RestResponse.status(HttpResponseStatus.FORBIDDEN.code());
         }
         return RestResponse.status(HttpResponseStatus.BAD_GATEWAY.code());

src/main/java/io/cryostat/MessageCodecs.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright The Cryostat Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.cryostat;
+
+import io.cryostat.recordings.Recordings.LinkedRecordingDescriptor;
+
+import io.quarkus.runtime.StartupEvent;
+import io.quarkus.vertx.LocalEventBusCodec;
+import io.vertx.core.eventbus.EventBus;
+import jakarta.enterprise.event.Observes;
+import jakarta.inject.Inject;
+
+public class MessageCodecs {
+
+    @Inject EventBus bus;
+
+    void onStart(@Observes StartupEvent evt) {
+        bus.registerDefaultCodec(LinkedRecordingDescriptor.class, new LocalEventBusCodec<>());
+    }
+}

src/main/java/io/cryostat/Producers.java

@@ -16,6 +16,7 @@
 package io.cryostat;
 
 import java.net.URI;
+import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ForkJoinPool;
 import java.util.concurrent.ScheduledExecutorService;
@@ -73,6 +74,13 @@ public static Base64 produceBase64Url() {
         return new Base64(0, null, true);
     }
 
+    @Produces
+    @ApplicationScoped
+    @DefaultBean
+    public static ExecutorService produceExecutorService() {
+        return ForkJoinPool.commonPool();
+    }
+
     @Produces
     @ApplicationScoped
     @DefaultBean

src/main/java/io/cryostat/discovery/ContainerDiscovery.java

@@ -369,8 +369,8 @@ public void handleContainerEvent(ContainerSpec desc, EventKind evtKind) {
             } else {
                 realm.children.remove(t.discoveryNode);
             }
-            t.delete();
             realm.persist();
+            t.delete();
         }
     }
 

src/main/java/io/cryostat/discovery/JDPDiscovery.java

@@ -150,8 +150,8 @@ public synchronized void handleJdpEvent(JvmDiscoveryEvent evt) {
             case LOST:
                 Target t = Target.getTargetByConnectUrl(connectUrl);
                 realm.children.remove(t.discoveryNode);
-                t.delete();
                 realm.persist();
+                t.delete();
                 break;
             default:
                 logger.warnv("Unknown JVM discovery event {0}", evt.getEventKind());

src/main/java/io/cryostat/expressions/MatchExpressionEvaluator.java

@@ -36,7 +36,6 @@
 import io.quarkus.cache.CaffeineCache;
 import io.quarkus.cache.CompositeCacheKey;
 import io.quarkus.vertx.ConsumeEvent;
-import io.smallrye.common.annotation.Blocking;
 import jakarta.annotation.Nullable;
 import jakarta.enterprise.context.ApplicationScoped;
 import jakarta.inject.Inject;
@@ -60,8 +59,7 @@ public class MatchExpressionEvaluator {
     @Inject Logger logger;
     @Inject CacheManager cacheManager;
 
-    @Blocking
-    @ConsumeEvent(MatchExpression.EXPRESSION_ADDRESS)
+    @ConsumeEvent(value = MatchExpression.EXPRESSION_ADDRESS, blocking = true)
     void onMessage(ExpressionEvent event) {
         switch (event.category()) {
             case CREATED: