From 1b4e7b7ea17e67da8b5f6e46fadc909130d014c4 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 18 Jan 2025 00:04:40 +0000 Subject: [PATCH] Added chart versions: airlock/microgateway: - 4.4.3 airlock/microgateway-cni: - 4.4.3 kubecost/cost-analyzer: - 2.5.3 --- assets/airlock/microgateway-4.4.3.tgz | Bin 0 -> 82253 bytes assets/airlock/microgateway-cni-4.4.3.tgz | Bin 0 -> 11475 bytes assets/kubecost/cost-analyzer-2.5.3.tgz | Bin 0 -> 142171 bytes .../microgateway-cni/4.4.3/.helmignore | 27 + .../airlock/microgateway-cni/4.4.3/Chart.yaml | 43 + .../airlock/microgateway-cni/4.4.3/README.md | 138 + .../4.4.3}/gke-values.yaml | 0 .../4.4.3}/openshift-values.yaml | 0 .../4.4.3}/questions.yml | 0 .../4.4.3/templates/NOTES.txt | 15 + .../4.4.3/templates/_helpers.tpl | 101 + .../4.4.3/templates/clusterrole.yaml | 25 + .../4.4.3}/templates/clusterrolebinding.yaml | 0 .../4.4.3}/templates/configmap.yaml | 0 .../4.4.3/templates/daemonset.yaml | 138 + .../network-attachment-definition.yaml | 0 .../4.4.3}/templates/scc-role.yaml | 0 .../4.4.3}/templates/scc-rolebinding.yaml | 0 .../4.4.3}/templates/serviceaccount.yaml | 0 .../4.4.3/templates/tests/rbac.yaml | 64 + .../4.4.3/templates/tests/test-install.yaml | 103 + .../microgateway-cni/4.4.3/values.schema.json | 233 + .../microgateway-cni/4.4.3/values.yaml | 94 + charts/airlock/microgateway/4.3.3/.helmignore | 3 +- charts/airlock/microgateway/4.3.3/Chart.yaml | 15 +- charts/airlock/microgateway/4.3.3/README.md | 147 +- .../{4.4.0 => 4.3.3}/app-readme.md | 0 ...cesscontrols.microgateway.airlock.com.yaml | 124 + ...ntsecurities.microgateway.airlock.com.yaml | 139 + .../denyrules.microgateway.airlock.com.yaml | 1804 ++++++ ...nvoyclusters.microgateway.airlock.com.yaml | 58 + ...nfigurations.microgateway.airlock.com.yaml | 185 + ...yhttpfilters.microgateway.airlock.com.yaml | 58 + .../graphqls.microgateway.airlock.com.yaml | 88 + ...aderrewrites.microgateway.airlock.com.yaml | 759 +++ ...propagations.microgateway.airlock.com.yaml | 108 + .../crds/limits.microgateway.airlock.com.yaml | 651 ++ ...idcproviders.microgateway.airlock.com.yaml | 305 + ...lyingparties.microgateway.airlock.com.yaml | 224 + .../openapis.microgateway.airlock.com.yaml | 167 + .../parsers.microgateway.airlock.com.yaml | 358 ++ ...disproviders.microgateway.airlock.com.yaml | 159 + ...ionhandlings.microgateway.airlock.com.yaml | 77 + ...ecargateways.microgateway.airlock.com.yaml | 758 +++ .../telemetries.microgateway.airlock.com.yaml | 96 + .../4.3.3/dashboards/blockLogs.json | 510 ++ .../4.3.3/dashboards/blockMetrics.json | 758 +++ .../4.3.3/dashboards/license.json | 521 ++ .../4.3.3/dashboards/overview.json | 1138 ++++ .../microgateway/4.3.3/templates/NOTES.txt | 48 +- .../microgateway/4.3.3/templates/_helpers.tpl | 122 +- .../templates/operator/_operator_helpers.tpl | 0 .../4.3.3/templates/operator/_rbac.gen.tpl | 237 + .../templates/operator/_webhooks.gen.tpl | 339 + .../4.3.3/templates/operator/configmap.yaml | 394 ++ .../operator/dashboard-configmap.yaml | 0 .../templates/operator/deployment.yaml | 0 .../templates/operator/manager-role.yaml | 0 .../operator/manager-rolebinding.yaml | 0 .../templates/operator/metrics-service.yaml | 0 .../templates/operator/mutating-webhook.yaml | 0 .../templates/operator/podmonitor.yaml | 0 .../templates/operator/role.yaml | 0 .../templates/operator/rolebinding.yaml | 0 .../templates/operator/selfsigned-issuer.yaml | 0 .../templates/operator/serviceaccount.yaml | 0 .../templates/operator/servicemonitor.yaml | 0 .../operator/serving-certificate.yaml | 0 .../operator/validating-webhook.yaml | 0 .../templates/operator/webhook-service.yaml | 0 .../templates/operator/xds-service.yaml | 0 .../4.3.3/templates/tests/rbac.yaml | 135 +- .../templates/tests/service.yaml | 0 .../templates/tests/statefulset.yaml | 0 .../4.3.3/templates/tests/test-install.yaml | 258 +- .../microgateway/4.3.3/values.schema.json | 491 +- charts/airlock/microgateway/4.3.3/values.yaml | 266 +- charts/airlock/microgateway/4.4.0/.helmignore | 3 +- charts/airlock/microgateway/4.4.0/Chart.yaml | 15 +- charts/airlock/microgateway/4.4.0/README.md | 153 +- .../microgateway/4.4.0/gke-values.yaml | 4 + .../microgateway/4.4.0/openshift-values.yaml | 15 + .../airlock/microgateway/4.4.0/questions.yml | 18 + .../microgateway/4.4.0/templates/NOTES.txt | 62 +- .../microgateway/4.4.0/templates/_helpers.tpl | 122 +- .../templates/clusterrole.yaml | 0 .../4.4.0/templates/clusterrolebinding.yaml | 20 + .../4.4.0/templates/configmap.yaml | 22 + .../{4.3.3 => 4.4.0}/templates/daemonset.yaml | 0 .../network-attachment-definition.yaml | 13 + .../4.4.0/templates/scc-role.yaml | 22 + .../4.4.0/templates/scc-rolebinding.yaml | 20 + .../4.4.0/templates/serviceaccount.yaml | 13 + .../4.4.0/templates/tests/rbac.yaml | 135 +- .../4.4.0/templates/tests/test-install.yaml | 258 +- .../microgateway/4.4.0/values.schema.json | 521 +- charts/airlock/microgateway/4.4.0/values.yaml | 290 +- charts/airlock/microgateway/4.4.3/.helmignore | 28 + charts/airlock/microgateway/4.4.3/Chart.yaml | 44 + charts/airlock/microgateway/4.4.3/README.md | 186 + .../airlock/microgateway/4.4.3/app-readme.md | 28 + ...cesscontrols.microgateway.airlock.com.yaml | 2 +- ...ntsecurities.microgateway.airlock.com.yaml | 2 +- ...ritypolicies.microgateway.airlock.com.yaml | 2 +- .../denyrules.microgateway.airlock.com.yaml | 2 +- ...nvoyclusters.microgateway.airlock.com.yaml | 2 +- ...nfigurations.microgateway.airlock.com.yaml | 2 +- ...yhttpfilters.microgateway.airlock.com.yaml | 2 +- .../graphqls.microgateway.airlock.com.yaml | 2 +- ...aderrewrites.microgateway.airlock.com.yaml | 2 +- ...propagations.microgateway.airlock.com.yaml | 2 +- .../crds/jwks.microgateway.airlock.com.yaml | 2 +- .../crds/limits.microgateway.airlock.com.yaml | 2 +- ...idcproviders.microgateway.airlock.com.yaml | 2 +- ...lyingparties.microgateway.airlock.com.yaml | 2 +- .../openapis.microgateway.airlock.com.yaml | 2 +- .../parsers.microgateway.airlock.com.yaml | 2 +- ...disproviders.microgateway.airlock.com.yaml | 2 +- ...ionhandlings.microgateway.airlock.com.yaml | 2 +- ...ecargateways.microgateway.airlock.com.yaml | 2 +- .../telemetries.microgateway.airlock.com.yaml | 2 +- .../dashboards/blockLogs.json | 0 .../dashboards/blockMetrics.json | 0 .../dashboards/headerLogs.json | 0 .../{4.4.0 => 4.4.3}/dashboards/license.json | 0 .../dashboards/logOnlyLogs.json | 0 .../dashboards/logOnlyMetrics.json | 0 .../{4.4.0 => 4.4.3}/dashboards/overview.json | 0 .../microgateway/4.4.3/templates/NOTES.txt | 61 + .../microgateway/4.4.3/templates/_helpers.tpl | 153 + .../templates/operator/_operator_helpers.tpl | 42 + .../templates/operator/_rbac.gen.tpl | 0 .../templates/operator/_webhooks.gen.tpl | 0 .../templates/operator/configmap.yaml | 1 + .../operator/dashboard-configmap.yaml | 28 + .../4.4.3/templates/operator/deployment.yaml | 143 + .../templates/operator/manager-role.yaml | 33 + .../operator/manager-rolebinding.yaml | 45 + .../templates/operator/metrics-service.yaml | 47 + .../templates/operator/mutating-webhook.yaml | 28 + .../4.4.3/templates/operator/podmonitor.yaml | 27 + .../4.4.3/templates/operator/role.yaml | 45 + .../4.4.3/templates/operator/rolebinding.yaml | 20 + .../templates/operator/selfsigned-issuer.yaml | 13 + .../templates/operator/serviceaccount.yaml | 13 + .../templates/operator/servicemonitor.yaml | 60 + .../operator/serving-certificate.yaml | 19 + .../operator/validating-webhook.yaml | 28 + .../templates/operator/webhook-service.yaml | 23 + .../4.4.3/templates/operator/xds-service.yaml | 24 + .../4.4.3/templates/tests/rbac.yaml | 143 + .../4.4.3/templates/tests/service.yaml | 23 + .../4.4.3/templates/tests/statefulset.yaml | 56 + .../4.4.3/templates/tests/test-install.yaml | 227 + .../microgateway/4.4.3/values.schema.json | 572 ++ charts/airlock/microgateway/4.4.3/values.yaml | 237 + .../kubecost/cost-analyzer/2.5.3/Chart.yaml | 13 + charts/kubecost/cost-analyzer/2.5.3/README.md | 110 + .../cost-analyzer/2.5.3/app-readme.md | 25 + .../2.5.3/ci/aggregator-values.yaml | 17 + .../federatedetl-primary-netcosts-values.yaml | 35 + .../2.5.3/ci/statefulsets-cc.yaml | 46 + .../2.5.3/crds/cluster-turndown-crd.yaml | 78 + .../cost-analyzer/2.5.3/custom-pricing.csv | 7 + .../2.5.3/grafana-dashboards/README.md | 45 + .../grafana-dashboards/attached-disks.json | 549 ++ .../grafana-dashboards/cluster-metrics.json | 1683 +++++ .../cluster-utilization.json | 3196 +++++++++ .../deployment-utilization.json | 1386 ++++ .../aggregator-dashboard.json | 668 ++ .../multi-cluster-container-stats.json | 787 +++ .../multi-cluster-disk-usage.json | 571 ++ .../multi-cluster-network-transfer-data.json | 685 ++ .../kubernetes-resource-efficiency.json | 408 ++ .../label-cost-utilization.json | 1146 ++++ .../namespace-utilization.json | 1175 ++++ .../network-cloud-services.json | 408 ++ .../networkCosts-metrics.json | 672 ++ .../grafana-dashboards/node-utilization.json | 1389 ++++ .../pod-utilization-multi-cluster.json | 788 +++ .../grafana-dashboards/pod-utilization.json | 860 +++ .../grafana-dashboards/prom-benchmark.json | 5691 +++++++++++++++++ .../workload-metrics-aggregator.json | 988 +++ .../grafana-dashboards/workload-metrics.json | 893 +++ .../cost-analyzer/2.5.3/questions.yaml | 187 + .../create-admission-controller-tls.sh | 29 + .../cost-analyzer/2.5.3/templates/NOTES.txt | 31 + .../2.5.3/templates/_helpers.tpl | 1558 +++++ .../aggregator-cloud-cost-deployment.yaml | 173 + ...aggregator-cloud-cost-service-account.yaml | 23 + .../aggregator-cloud-cost-service.yaml | 17 + .../2.5.3/templates/aggregator-service.yaml | 28 + .../templates/aggregator-servicemonitor.yaml | 31 + .../templates/aggregator-statefulset.yaml | 222 + .../templates/alibaba-service-key-secret.yaml | 20 + .../templates/aws-service-key-secret.yaml | 20 + .../awsstore-deployment-template.yaml | 56 + .../awsstore-service-account-template.yaml | 15 + .../templates/azure-service-key-secret.yaml | 24 + .../templates/cloud-integration-secret.yaml | 16 + ...st-analyzer-account-mapping-configmap.yaml | 13 + .../cost-analyzer-alerts-configmap.yaml | 11 + ...cost-analyzer-asset-reports-configmap.yaml | 14 + ...analyzer-cloud-cost-reports-configmap.yaml | 13 + ...nalyzer-cluster-role-binding-template.yaml | 56 + ...alyzer-cluster-role-template-readonly.yaml | 26 + .../cost-analyzer-cluster-role-template.yaml | 108 + .../cost-analyzer-config-map-template.yaml | 37 + .../cost-analyzer-deployment-template.yaml | 1241 ++++ ...analyzer-frontend-config-map-template.yaml | 1512 +++++ .../cost-analyzer-ingestion-configmap.yaml | 14 + .../cost-analyzer-ingress-template.yaml | 56 + ...-analyzer-metrics-config-map-template.yaml | 13 + ...zer-network-costs-config-map-template.yaml | 16 + ...zer-network-costs-podmonitor-template.yaml | 32 + ...alyzer-network-costs-service-template.yaml | 34 + .../cost-analyzer-network-costs-template.yaml | 161 + .../cost-analyzer-networks-costs-ocp-scc.yaml | 30 + .../templates/cost-analyzer-ocp-route.yaml | 25 + ...ost-analyzer-oidc-config-map-template.yaml | 49 + .../cost-analyzer-pkey-configmap.yaml | 23 + .../cost-analyzer-pricing-configmap.yaml | 141 + ...cost-analyzer-prometheusrule-template.yaml | 22 + .../templates/cost-analyzer-pvc-template.yaml | 33 + ...ost-analyzer-saml-config-map-template.yaml | 14 + ...cost-analyzer-saved-reports-configmap.yaml | 13 + .../cost-analyzer-server-configmap.yaml | 73 + ...ost-analyzer-service-account-template.yaml | 13 + .../cost-analyzer-service-template.yaml | 66 + ...cost-analyzer-servicemonitor-template.yaml | 34 + .../cost-analyzer-smtp-configmap.yaml | 12 + .../templates/diagnostics-deployment.yaml | 183 + .../2.5.3/templates/diagnostics-service.yaml | 20 + .../2.5.3/templates/etl-utils-deployment.yaml | 129 + .../2.5.3/templates/etl-utils-service.yaml | 17 + .../external-grafana-config-map-template.yaml | 11 + .../2.5.3/templates/extra-manifests.yaml | 8 + .../templates/federated-store-secret.yaml | 12 + .../templates/forecasting-deployment.yaml | 154 + .../2.5.3/templates/forecasting-service.yaml | 17 + .../frontend-deployment-template.yaml | 222 + .../templates/frontend-service-template.yaml | 53 + .../gcpstore-config-map-template.yaml | 61 + .../grafana/grafana-clusterrole.yaml | 24 + .../grafana/grafana-clusterrolebinding.yaml | 24 + .../grafana-configmap-dashboard-provider.yaml | 28 + .../templates/grafana/grafana-configmap.yaml | 90 + .../grafana-dashboard-attached-disks.yaml | 21 + ...na-dashboard-cluster-metrics-template.yaml | 21 + ...ashboard-cluster-utilization-template.yaml | 21 + ...board-deployment-utilization-template.yaml | 21 + ...bernetes-resource-efficiency-template.yaml | 21 + ...board-label-cost-utilization-template.yaml | 21 + ...hboard-namespace-utilization-template.yaml | 21 + ...afana-dashboard-network-cloud-sevices.yaml | 21 + .../grafana-dashboard-network-costs.yaml | 21 + ...a-dashboard-node-utilization-template.yaml | 21 + ...shboard-pod-utilization-multi-cluster.yaml | 21 + ...na-dashboard-pod-utilization-template.yaml | 21 + ...dashboard-prometheus-metrics-template.yaml | 21 + ...grafana-dashboard-workload-aggregator.yaml | 21 + .../grafana-dashboard-workload-metrics.yaml | 21 + .../grafana-dashboards-json-configmap.yaml | 24 + .../grafana/grafana-datasource-template.yaml | 38 + .../templates/grafana/grafana-deployment.yaml | 291 + .../2.5.3/templates/grafana/grafana-pvc.yaml | 26 + .../templates/grafana/grafana-secret.yaml | 19 + .../templates/grafana/grafana-service.yaml | 51 + .../grafana/grafana-serviceaccount.yaml | 13 + .../2.5.3/templates/install-plugins.yaml | 43 + ...tegrations-postgres-queries-configmap.yaml | 14 + .../integrations-postgres-secret.yaml | 19 + .../integrations-turbonomic-secret.yaml | 19 + ...admission-controller-service-template.yaml | 15 + ...ubecost-admission-controller-template.yaml | 30 + .../kubecost-cluster-context-switcher.yaml | 14 + ...ost-cluster-controller-actions-config.yaml | 56 + .../kubecost-cluster-controller-template.yaml | 310 + .../kubecost-oidc-secret-template.yaml | 16 + .../kubecost-priority-class-template.yaml | 15 + .../kubecost-saml-secret-template.yaml | 12 + .../mimir-proxy-configmap-template.yaml | 21 + .../mimir-proxy-deployment-template.yaml | 57 + .../mimir-proxy-service-template.yaml | 18 + .../monitoring-role-binding-template.yaml | 17 + .../templates/monitoring-role-template.yaml | 19 + ...network-costs-servicemonitor-template.yaml | 32 + .../2.5.3/templates/plugins-config.yaml | 13 + .../prometheus-alertmanager-configmap.yaml | 21 + .../prometheus-alertmanager-deployment.yaml | 160 + .../prometheus-alertmanager-ingress.yaml | 41 + ...prometheus-alertmanager-networkpolicy.yaml | 22 + .../prometheus-alertmanager-pdb.yaml | 16 + .../prometheus-alertmanager-pvc.yaml | 35 + ...metheus-alertmanager-service-headless.yaml | 33 + .../prometheus-alertmanager-service.yaml | 55 + ...rometheus-alertmanager-serviceaccount.yaml | 11 + .../prometheus-alertmanager-statefulset.yaml | 167 + .../prometheus-node-exporter-daemonset.yaml | 147 + .../prometheus-node-exporter-ocp-scc.yaml | 29 + .../prometheus-node-exporter-service.yaml | 47 + ...ometheus-node-exporter-serviceaccount.yaml | 11 + .../prometheus-server-clusterrole.yaml | 39 + .../prometheus-server-clusterrolebinding.yaml | 18 + .../prometheus-server-configmap.yaml | 100 + .../prometheus-server-deployment.yaml | 268 + .../prometheus/prometheus-server-ingress.yaml | 45 + .../prometheus/prometheus-server-pdb.yaml | 15 + .../prometheus/prometheus-server-pvc.yaml | 37 + .../prometheus-server-service-headless.yaml | 29 + .../prometheus/prometheus-server-service.yaml | 62 + .../prometheus-server-serviceaccount.yaml | 17 + .../prometheus-server-statefulset.yaml | 241 + .../prometheus/prometheus-server-vpa.yaml | 22 + ...ations-allowlists-config-map-template.yaml | 13 + .../2.5.3/templates/tests/_helpers.tpl | 5 + .../2.5.3/templates/tests/basic-health.yaml | 48 + .../cost-analyzer/2.5.3/values-amp.yaml | 20 + .../2.5.3/values-custom-pricing.yaml | 17 + .../2.5.3/values-eks-cost-monitoring.yaml | 44 + .../values-openshift-cluster-prometheus.yaml | 26 + .../cost-analyzer/2.5.3/values-openshift.yaml | 8 + .../values-savings-rec-allowlist-aws.yaml | 790 +++ .../values-savings-rec-allowlist-azure.yaml | 283 + .../values-savings-rec-allowlist-gcp.yaml | 76 + .../2.5.3/values-windows-node-affinity.yaml | 25 + .../kubecost/cost-analyzer/2.5.3/values.yaml | 2419 +++++++ index.yaml | 114 +- 328 files changed, 53863 insertions(+), 1581 deletions(-) create mode 100644 assets/airlock/microgateway-4.4.3.tgz create mode 100644 assets/airlock/microgateway-cni-4.4.3.tgz create mode 100644 assets/kubecost/cost-analyzer-2.5.3.tgz create mode 100644 charts/airlock/microgateway-cni/4.4.3/.helmignore create mode 100644 charts/airlock/microgateway-cni/4.4.3/Chart.yaml create mode 100644 charts/airlock/microgateway-cni/4.4.3/README.md rename charts/airlock/{microgateway/4.3.3 => microgateway-cni/4.4.3}/gke-values.yaml (100%) rename charts/airlock/{microgateway/4.3.3 => microgateway-cni/4.4.3}/openshift-values.yaml (100%) rename charts/airlock/{microgateway/4.3.3 => microgateway-cni/4.4.3}/questions.yml (100%) create mode 100644 charts/airlock/microgateway-cni/4.4.3/templates/NOTES.txt create mode 100644 charts/airlock/microgateway-cni/4.4.3/templates/_helpers.tpl create mode 100644 charts/airlock/microgateway-cni/4.4.3/templates/clusterrole.yaml rename charts/airlock/{microgateway/4.3.3 => microgateway-cni/4.4.3}/templates/clusterrolebinding.yaml (100%) rename charts/airlock/{microgateway/4.3.3 => microgateway-cni/4.4.3}/templates/configmap.yaml (100%) create mode 100644 charts/airlock/microgateway-cni/4.4.3/templates/daemonset.yaml rename charts/airlock/{microgateway/4.3.3 => microgateway-cni/4.4.3}/templates/network-attachment-definition.yaml (100%) rename charts/airlock/{microgateway/4.3.3 => microgateway-cni/4.4.3}/templates/scc-role.yaml (100%) rename charts/airlock/{microgateway/4.3.3 => microgateway-cni/4.4.3}/templates/scc-rolebinding.yaml (100%) rename charts/airlock/{microgateway/4.3.3 => microgateway-cni/4.4.3}/templates/serviceaccount.yaml (100%) create mode 100644 charts/airlock/microgateway-cni/4.4.3/templates/tests/rbac.yaml create mode 100644 charts/airlock/microgateway-cni/4.4.3/templates/tests/test-install.yaml create mode 100644 charts/airlock/microgateway-cni/4.4.3/values.schema.json create mode 100644 charts/airlock/microgateway-cni/4.4.3/values.yaml rename charts/airlock/microgateway/{4.4.0 => 4.3.3}/app-readme.md (100%) create mode 100644 charts/airlock/microgateway/4.3.3/crds/accesscontrols.microgateway.airlock.com.yaml create mode 100644 charts/airlock/microgateway/4.3.3/crds/contentsecurities.microgateway.airlock.com.yaml create mode 100644 charts/airlock/microgateway/4.3.3/crds/denyrules.microgateway.airlock.com.yaml create mode 100644 charts/airlock/microgateway/4.3.3/crds/envoyclusters.microgateway.airlock.com.yaml create mode 100644 charts/airlock/microgateway/4.3.3/crds/envoyconfigurations.microgateway.airlock.com.yaml create mode 100644 charts/airlock/microgateway/4.3.3/crds/envoyhttpfilters.microgateway.airlock.com.yaml create mode 100644 charts/airlock/microgateway/4.3.3/crds/graphqls.microgateway.airlock.com.yaml create mode 100644 charts/airlock/microgateway/4.3.3/crds/headerrewrites.microgateway.airlock.com.yaml create mode 100644 charts/airlock/microgateway/4.3.3/crds/identitypropagations.microgateway.airlock.com.yaml create mode 100644 charts/airlock/microgateway/4.3.3/crds/limits.microgateway.airlock.com.yaml create mode 100644 charts/airlock/microgateway/4.3.3/crds/oidcproviders.microgateway.airlock.com.yaml create mode 100644 charts/airlock/microgateway/4.3.3/crds/oidcrelyingparties.microgateway.airlock.com.yaml create mode 100644 charts/airlock/microgateway/4.3.3/crds/openapis.microgateway.airlock.com.yaml create mode 100644 charts/airlock/microgateway/4.3.3/crds/parsers.microgateway.airlock.com.yaml create mode 100644 charts/airlock/microgateway/4.3.3/crds/redisproviders.microgateway.airlock.com.yaml create mode 100644 charts/airlock/microgateway/4.3.3/crds/sessionhandlings.microgateway.airlock.com.yaml create mode 100644 charts/airlock/microgateway/4.3.3/crds/sidecargateways.microgateway.airlock.com.yaml create mode 100644 charts/airlock/microgateway/4.3.3/crds/telemetries.microgateway.airlock.com.yaml create mode 100644 charts/airlock/microgateway/4.3.3/dashboards/blockLogs.json create mode 100644 charts/airlock/microgateway/4.3.3/dashboards/blockMetrics.json create mode 100644 charts/airlock/microgateway/4.3.3/dashboards/license.json create mode 100644 charts/airlock/microgateway/4.3.3/dashboards/overview.json rename charts/airlock/microgateway/{4.4.0 => 4.3.3}/templates/operator/_operator_helpers.tpl (100%) create mode 100644 charts/airlock/microgateway/4.3.3/templates/operator/_rbac.gen.tpl create mode 100644 charts/airlock/microgateway/4.3.3/templates/operator/_webhooks.gen.tpl create mode 100644 charts/airlock/microgateway/4.3.3/templates/operator/configmap.yaml rename charts/airlock/microgateway/{4.4.0 => 4.3.3}/templates/operator/dashboard-configmap.yaml (100%) rename charts/airlock/microgateway/{4.4.0 => 4.3.3}/templates/operator/deployment.yaml (100%) rename charts/airlock/microgateway/{4.4.0 => 4.3.3}/templates/operator/manager-role.yaml (100%) rename charts/airlock/microgateway/{4.4.0 => 4.3.3}/templates/operator/manager-rolebinding.yaml (100%) rename charts/airlock/microgateway/{4.4.0 => 4.3.3}/templates/operator/metrics-service.yaml (100%) rename charts/airlock/microgateway/{4.4.0 => 4.3.3}/templates/operator/mutating-webhook.yaml (100%) rename charts/airlock/microgateway/{4.4.0 => 4.3.3}/templates/operator/podmonitor.yaml (100%) rename charts/airlock/microgateway/{4.4.0 => 4.3.3}/templates/operator/role.yaml (100%) rename charts/airlock/microgateway/{4.4.0 => 4.3.3}/templates/operator/rolebinding.yaml (100%) rename charts/airlock/microgateway/{4.4.0 => 4.3.3}/templates/operator/selfsigned-issuer.yaml (100%) rename charts/airlock/microgateway/{4.4.0 => 4.3.3}/templates/operator/serviceaccount.yaml (100%) rename charts/airlock/microgateway/{4.4.0 => 4.3.3}/templates/operator/servicemonitor.yaml (100%) rename charts/airlock/microgateway/{4.4.0 => 4.3.3}/templates/operator/serving-certificate.yaml (100%) rename charts/airlock/microgateway/{4.4.0 => 4.3.3}/templates/operator/validating-webhook.yaml (100%) rename charts/airlock/microgateway/{4.4.0 => 4.3.3}/templates/operator/webhook-service.yaml (100%) rename charts/airlock/microgateway/{4.4.0 => 4.3.3}/templates/operator/xds-service.yaml (100%) rename charts/airlock/microgateway/{4.4.0 => 4.3.3}/templates/tests/service.yaml (100%) rename charts/airlock/microgateway/{4.4.0 => 4.3.3}/templates/tests/statefulset.yaml (100%) create mode 100644 charts/airlock/microgateway/4.4.0/gke-values.yaml create mode 100644 charts/airlock/microgateway/4.4.0/openshift-values.yaml create mode 100644 charts/airlock/microgateway/4.4.0/questions.yml rename charts/airlock/microgateway/{4.3.3 => 4.4.0}/templates/clusterrole.yaml (100%) create mode 100644 charts/airlock/microgateway/4.4.0/templates/clusterrolebinding.yaml create mode 100644 charts/airlock/microgateway/4.4.0/templates/configmap.yaml rename charts/airlock/microgateway/{4.3.3 => 4.4.0}/templates/daemonset.yaml (100%) create mode 100644 charts/airlock/microgateway/4.4.0/templates/network-attachment-definition.yaml create mode 100644 charts/airlock/microgateway/4.4.0/templates/scc-role.yaml create mode 100644 charts/airlock/microgateway/4.4.0/templates/scc-rolebinding.yaml create mode 100644 charts/airlock/microgateway/4.4.0/templates/serviceaccount.yaml create mode 100644 charts/airlock/microgateway/4.4.3/.helmignore create mode 100644 charts/airlock/microgateway/4.4.3/Chart.yaml create mode 100644 charts/airlock/microgateway/4.4.3/README.md create mode 100644 charts/airlock/microgateway/4.4.3/app-readme.md rename charts/airlock/microgateway/{4.4.0 => 4.4.3}/crds/accesscontrols.microgateway.airlock.com.yaml (99%) rename charts/airlock/microgateway/{4.4.0 => 4.4.3}/crds/contentsecurities.microgateway.airlock.com.yaml (99%) rename charts/airlock/microgateway/{4.4.0 => 4.4.3}/crds/contentsecuritypolicies.microgateway.airlock.com.yaml (99%) rename charts/airlock/microgateway/{4.4.0 => 4.4.3}/crds/denyrules.microgateway.airlock.com.yaml (99%) rename charts/airlock/microgateway/{4.4.0 => 4.4.3}/crds/envoyclusters.microgateway.airlock.com.yaml (98%) rename charts/airlock/microgateway/{4.4.0 => 4.4.3}/crds/envoyconfigurations.microgateway.airlock.com.yaml (99%) rename charts/airlock/microgateway/{4.4.0 => 4.4.3}/crds/envoyhttpfilters.microgateway.airlock.com.yaml (98%) rename charts/airlock/microgateway/{4.4.0 => 4.4.3}/crds/graphqls.microgateway.airlock.com.yaml (98%) rename charts/airlock/microgateway/{4.4.0 => 4.4.3}/crds/headerrewrites.microgateway.airlock.com.yaml (99%) rename charts/airlock/microgateway/{4.4.0 => 4.4.3}/crds/identitypropagations.microgateway.airlock.com.yaml (99%) rename charts/airlock/microgateway/{4.4.0 => 4.4.3}/crds/jwks.microgateway.airlock.com.yaml (99%) rename charts/airlock/microgateway/{4.4.0 => 4.4.3}/crds/limits.microgateway.airlock.com.yaml (99%) rename charts/airlock/microgateway/{4.4.0 => 4.4.3}/crds/oidcproviders.microgateway.airlock.com.yaml (99%) rename charts/airlock/microgateway/{4.4.0 => 4.4.3}/crds/oidcrelyingparties.microgateway.airlock.com.yaml (99%) rename charts/airlock/microgateway/{4.4.0 => 4.4.3}/crds/openapis.microgateway.airlock.com.yaml (99%) rename charts/airlock/microgateway/{4.4.0 => 4.4.3}/crds/parsers.microgateway.airlock.com.yaml (99%) rename charts/airlock/microgateway/{4.4.0 => 4.4.3}/crds/redisproviders.microgateway.airlock.com.yaml (99%) rename charts/airlock/microgateway/{4.4.0 => 4.4.3}/crds/sessionhandlings.microgateway.airlock.com.yaml (99%) rename charts/airlock/microgateway/{4.4.0 => 4.4.3}/crds/sidecargateways.microgateway.airlock.com.yaml (99%) rename charts/airlock/microgateway/{4.4.0 => 4.4.3}/crds/telemetries.microgateway.airlock.com.yaml (99%) rename charts/airlock/microgateway/{4.4.0 => 4.4.3}/dashboards/blockLogs.json (100%) rename charts/airlock/microgateway/{4.4.0 => 4.4.3}/dashboards/blockMetrics.json (100%) rename charts/airlock/microgateway/{4.4.0 => 4.4.3}/dashboards/headerLogs.json (100%) rename charts/airlock/microgateway/{4.4.0 => 4.4.3}/dashboards/license.json (100%) rename charts/airlock/microgateway/{4.4.0 => 4.4.3}/dashboards/logOnlyLogs.json (100%) rename charts/airlock/microgateway/{4.4.0 => 4.4.3}/dashboards/logOnlyMetrics.json (100%) rename charts/airlock/microgateway/{4.4.0 => 4.4.3}/dashboards/overview.json (100%) create mode 100644 charts/airlock/microgateway/4.4.3/templates/NOTES.txt create mode 100644 charts/airlock/microgateway/4.4.3/templates/_helpers.tpl create mode 100644 charts/airlock/microgateway/4.4.3/templates/operator/_operator_helpers.tpl rename charts/airlock/microgateway/{4.4.0 => 4.4.3}/templates/operator/_rbac.gen.tpl (100%) rename charts/airlock/microgateway/{4.4.0 => 4.4.3}/templates/operator/_webhooks.gen.tpl (100%) rename charts/airlock/microgateway/{4.4.0 => 4.4.3}/templates/operator/configmap.yaml (99%) create mode 100644 charts/airlock/microgateway/4.4.3/templates/operator/dashboard-configmap.yaml create mode 100644 charts/airlock/microgateway/4.4.3/templates/operator/deployment.yaml create mode 100644 charts/airlock/microgateway/4.4.3/templates/operator/manager-role.yaml create mode 100644 charts/airlock/microgateway/4.4.3/templates/operator/manager-rolebinding.yaml create mode 100644 charts/airlock/microgateway/4.4.3/templates/operator/metrics-service.yaml create mode 100644 charts/airlock/microgateway/4.4.3/templates/operator/mutating-webhook.yaml create mode 100644 charts/airlock/microgateway/4.4.3/templates/operator/podmonitor.yaml create mode 100644 charts/airlock/microgateway/4.4.3/templates/operator/role.yaml create mode 100644 charts/airlock/microgateway/4.4.3/templates/operator/rolebinding.yaml create mode 100644 charts/airlock/microgateway/4.4.3/templates/operator/selfsigned-issuer.yaml create mode 100644 charts/airlock/microgateway/4.4.3/templates/operator/serviceaccount.yaml create mode 100644 charts/airlock/microgateway/4.4.3/templates/operator/servicemonitor.yaml create mode 100644 charts/airlock/microgateway/4.4.3/templates/operator/serving-certificate.yaml create mode 100644 charts/airlock/microgateway/4.4.3/templates/operator/validating-webhook.yaml create mode 100644 charts/airlock/microgateway/4.4.3/templates/operator/webhook-service.yaml create mode 100644 charts/airlock/microgateway/4.4.3/templates/operator/xds-service.yaml create mode 100644 charts/airlock/microgateway/4.4.3/templates/tests/rbac.yaml create mode 100644 charts/airlock/microgateway/4.4.3/templates/tests/service.yaml create mode 100644 charts/airlock/microgateway/4.4.3/templates/tests/statefulset.yaml create mode 100644 charts/airlock/microgateway/4.4.3/templates/tests/test-install.yaml create mode 100644 charts/airlock/microgateway/4.4.3/values.schema.json create mode 100644 charts/airlock/microgateway/4.4.3/values.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/Chart.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/README.md create mode 100644 charts/kubecost/cost-analyzer/2.5.3/app-readme.md create mode 100644 charts/kubecost/cost-analyzer/2.5.3/ci/aggregator-values.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/ci/federatedetl-primary-netcosts-values.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/ci/statefulsets-cc.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/crds/cluster-turndown-crd.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/custom-pricing.csv create mode 100644 charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/README.md create mode 100644 charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/attached-disks.json create mode 100644 charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/cluster-metrics.json create mode 100644 charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/cluster-utilization.json create mode 100644 charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/deployment-utilization.json create mode 100644 charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/grafana-templates/aggregator-dashboard.json create mode 100644 charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/grafana-templates/multi-cluster-container-stats.json create mode 100644 charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/grafana-templates/multi-cluster-disk-usage.json create mode 100644 charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/grafana-templates/multi-cluster-network-transfer-data.json create mode 100644 charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/kubernetes-resource-efficiency.json create mode 100644 charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/label-cost-utilization.json create mode 100644 charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/namespace-utilization.json create mode 100644 charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/network-cloud-services.json create mode 100644 charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/networkCosts-metrics.json create mode 100644 charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/node-utilization.json create mode 100644 charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/pod-utilization-multi-cluster.json create mode 100644 charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/pod-utilization.json create mode 100644 charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/prom-benchmark.json create mode 100644 charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/workload-metrics-aggregator.json create mode 100644 charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/workload-metrics.json create mode 100644 charts/kubecost/cost-analyzer/2.5.3/questions.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/scripts/create-admission-controller-tls.sh create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/NOTES.txt create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/_helpers.tpl create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/aggregator-cloud-cost-deployment.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/aggregator-cloud-cost-service-account.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/aggregator-cloud-cost-service.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/aggregator-service.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/aggregator-servicemonitor.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/aggregator-statefulset.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/alibaba-service-key-secret.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/aws-service-key-secret.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/awsstore-deployment-template.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/awsstore-service-account-template.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/azure-service-key-secret.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/cloud-integration-secret.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-account-mapping-configmap.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-alerts-configmap.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-asset-reports-configmap.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-cloud-cost-reports-configmap.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-cluster-role-binding-template.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-cluster-role-template-readonly.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-cluster-role-template.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-config-map-template.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-deployment-template.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-frontend-config-map-template.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-ingestion-configmap.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-ingress-template.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-metrics-config-map-template.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-network-costs-config-map-template.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-network-costs-podmonitor-template.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-network-costs-service-template.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-network-costs-template.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-networks-costs-ocp-scc.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-ocp-route.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-oidc-config-map-template.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-pkey-configmap.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-pricing-configmap.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-prometheusrule-template.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-pvc-template.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-saml-config-map-template.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-saved-reports-configmap.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-server-configmap.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-service-account-template.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-service-template.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-servicemonitor-template.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-smtp-configmap.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/diagnostics-deployment.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/diagnostics-service.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/etl-utils-deployment.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/etl-utils-service.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/external-grafana-config-map-template.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/extra-manifests.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/federated-store-secret.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/forecasting-deployment.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/forecasting-service.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/frontend-deployment-template.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/frontend-service-template.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/gcpstore-config-map-template.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-clusterrole.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-clusterrolebinding.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-configmap-dashboard-provider.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-configmap.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-attached-disks.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-cluster-metrics-template.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-cluster-utilization-template.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-deployment-utilization-template.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-kubernetes-resource-efficiency-template.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-label-cost-utilization-template.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-namespace-utilization-template.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-network-cloud-sevices.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-network-costs.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-node-utilization-template.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-pod-utilization-multi-cluster.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-pod-utilization-template.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-prometheus-metrics-template.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-workload-aggregator.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-workload-metrics.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboards-json-configmap.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-datasource-template.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-deployment.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-pvc.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-secret.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-service.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-serviceaccount.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/install-plugins.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/integrations-postgres-queries-configmap.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/integrations-postgres-secret.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/integrations-turbonomic-secret.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/kubecost-admission-controller-service-template.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/kubecost-admission-controller-template.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/kubecost-cluster-context-switcher.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/kubecost-cluster-controller-actions-config.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/kubecost-cluster-controller-template.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/kubecost-oidc-secret-template.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/kubecost-priority-class-template.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/kubecost-saml-secret-template.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/mimir-proxy-configmap-template.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/mimir-proxy-deployment-template.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/mimir-proxy-service-template.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/monitoring-role-binding-template.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/monitoring-role-template.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/network-costs-servicemonitor-template.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/plugins-config.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-alertmanager-configmap.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-alertmanager-deployment.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-alertmanager-ingress.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-alertmanager-networkpolicy.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-alertmanager-pdb.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-alertmanager-pvc.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-alertmanager-service-headless.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-alertmanager-service.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-alertmanager-serviceaccount.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-alertmanager-statefulset.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-node-exporter-daemonset.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-node-exporter-ocp-scc.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-node-exporter-service.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-node-exporter-serviceaccount.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-server-clusterrole.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-server-clusterrolebinding.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-server-configmap.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-server-deployment.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-server-ingress.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-server-pdb.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-server-pvc.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-server-service-headless.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-server-service.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-server-serviceaccount.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-server-statefulset.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-server-vpa.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/savings-recommendations-allowlists-config-map-template.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/tests/_helpers.tpl create mode 100644 charts/kubecost/cost-analyzer/2.5.3/templates/tests/basic-health.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/values-amp.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/values-custom-pricing.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/values-eks-cost-monitoring.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/values-openshift-cluster-prometheus.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/values-openshift.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/values-savings-rec-allowlist-aws.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/values-savings-rec-allowlist-azure.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/values-savings-rec-allowlist-gcp.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/values-windows-node-affinity.yaml create mode 100644 charts/kubecost/cost-analyzer/2.5.3/values.yaml diff --git a/assets/airlock/microgateway-4.4.3.tgz b/assets/airlock/microgateway-4.4.3.tgz new file mode 100644 index 0000000000000000000000000000000000000000..72a11c8a8d30a8692b105800bed6f6a19975dbe5 GIT binary patch literal 82253 zcmV)RK(oIeiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PMYKf7>{cFut$#DKJbjwlfxWSw6JunQ=Sm9zT=A8@s#z`y^Qh zB0&jlieLkv9CecCv;Tj%Nbu5S#YrdTi^L*GDosJ>p?hH=C z|JdZyZnxX}dwcT#cDr5tzuW6||D&^au-EPHAN2PR{-fRL9dvsC0oohIqUK2$hx9+% zw{EN0xj)GRqlj?GF^Slq0RW^N2hii=Wawhj@*qbeLT3ZOke5)*XXP>>_E5yo06agt ztoj;VvjOW zs_T(SE62Ng?p|wgBhTQFRl$x!ICF8tI8A=%61Z-tg`+9vs5PG9qhPi--23eh?;qHU zuP6!-rHCHqJG>Z$9|=t+RX7CVL#~4j06fTHNJg#)IS-K>6C@hM$RB_hQXU~%SVU00BrsJxZ`&FZoApusw_@Xh#*6{6~_^Ugb-maiBZJHIN+k# z93)_je=*HXyA2q}>CZiP&+RpQ#5{^)E?=F1f1z-q+5iEez(;XNW;hxFK1S6ojEQL6 zQ2FNOCP!lw<@l(9J<(2pLo{f$AY+KLmUzc<#auQWHLlU@hESgk8cp!)$&cb+Gz2Gc z9AZy$G58TvbOXas7C1StC;=lMoSdJ5m=caW$@)$3^4F6KQCN4U(ad+K2`CwgI!AOy66zt@kf}Dpaz@!xF;|7pVl)6S zBV}Wn=bJArS^peQPKHilm>{-s17N=X@Amh4`vv{K*V(_<|9A2{0L>;i2}5$j0GT36 zv2TQ_n5TdfkT3*!^Gj7zsemOxLI#2)4Anal0E)B}a2w)%HP~lx zh(abnK;M`Dg`r*3Ba;bOc!p(C<1X3 zhN6F16k=`z7YYx+>$i;umQ^sBy+R|*IGz0)%~YWmaDo~QPwEl?WShT^5g#Kenqt>D zmgHkF!h8n2G4ig#%@})Q5K}Zo5odCQpv0t=#3Kq*nvXMui6a1k7bc7&3T`kT1Cnqb za^zQ=DyOfWFjpdUIYzo!h$piDx?zw+=zWYl0b_{9aEb|)N)0oXAO`$I4!G3X(;kQj z2QcAe0y*|z7|uY3tY}U53Q`1MND=gBq9G<4!jbHbMqgs3=#OYX=tMw(z>&Sxw9te= zj_89RqSkb$q9X$kz>uK^(W)>d{EK8VL{xF4R4z;!8lr7e7Cezwhp<)!XkG4wvn(J1=b=^(o6+uyB}D`B z4+S@4Xn{9jm=ih!0VNYndoi|RUakNhSfB>L`8^TFG^y0vO+yE#hz*A$)3$=i;b;IH zsm!gy7ymlx_V-oWz{exRc#872<`KaUkOPN}VYk0O@D7d;>IH}W!~JeB>>ci-Bj4Ze z4v*TM;ePv|v)}Q%2Z!y0px^2G2f@%s9%}cIsN^4b`#opJ)$JRB#3Z(o{g@sPUzJf9 z4bryTUeGye`zYwN`$7Mp+ZlR6x3}kYJNt+GongPzZ+i!O!y)wc4%>T22M66wyVFJR zsDH5Mc?X`=b0JeeOhW9I*LNOm4P7~x*6cASF062&3!?x3? z#k#urGV-sqS`hkyQ|x8kh~sc(Af;1UnAg%&R?@&&vj*U=0*+8GDDnXmMmZErD;ZU& zNg=vSEyGXBrd7fhgm{9vRt*3EFHQ!a({8sX=FJ364LyzuJyxa!`n8LVcyMguBEk#1X)pNj8yh5fxODpe5hfZP?V;C~y*r zX6BQ=1_LPQnrodV5g-u|7zkaJB4M>6HFX{b7z9}A+$9ij!OsjyHp5X6CMfcd(%Rk_ z`H2wYqCEpK?eICIaDq6Z6?(=aQGiDYO_@Qxuq1gZj4e+Hgk%IlG(}pFhGg_i{2BlU zM*&gO=Tw8%ie!W$O$MR)ta)yHr-rYd% zjZ4h2B{NJjl{9DBzhxX+nHGM~oJ=@W_}4{hq+kFhdSN zTH2-z6MW%;ch8fM;-l`%;safrC43kj?-0F#7q(RfVB6#K|G256Bk?^M$#5qNhJ zhSvK$rg~~8OxP`kG%qD#6{x*w$T5PheMQ^ z2htnU{;$SLl4Bxqm6J8sLf&uM1;YK38Y4nCIoTl`JCblIaUkVJVwt`ghfWY3dN{-! za!GGZxB2r%^3pCCCLA!1r3bACOgms2tG(sFnyVfTjYYE;#F=s6@6$%aA zv~CE!#?h!L#=1#{zafvamgE=KdXTbMQ()iJ^S;RtZ%R&X>MjVQMNdk_b=nY44MaQO zosXFd7SCDZ3c%C%!dqu%HR*v`4EX@A(QE)vG{uxeqSf7$8wcP8;b+k!Q2f6HyMY*D%kC;#FR4<3Lo$`aONY>_Fnok{tIP)}M6)@b zg)=nG{j{<9zBMNxNFvYJeI=V|0c`|t8uuszs{8kxL|6*~mU}a|XRuH`b6G@>ChDa& zax-*QkR=5>&m*(Y0!sGTmYrT{8UW|NGS4rKCN%ZuzgHT!|4yeA_(>D`PhgbU z)_M$1&d;jpptyb>K#D@CIrVZ&h$u70=BrE;L1hD)i4`HPXR~{KDChjNV{wq+9WDBl z4zhw^3b?|{FVXrm#X?!Kdc`s|WHjcdA!JP0J1~MHWgJL(m+A`{{FBo&wTU2zA(z3& z0KYeN5b()Fn3M171jjqg8!2KhN=rdGIiufs0>frjtAU~sj?iG)c-T`}{4+(viojQi zg{gkrVI&Op+Wk(j*Y$htPVcbO8@BtwLBG=*4iEQ{zYlxe_K`R2wcEX*GaPn{98rBhC$cw@B647w4vv9`ujb9xbGddgU$gu=pe5T5!yRELVMlAgP_+v z4B$b(dldBhJ=i|*yv_zAVatVg<_-6ki1_p?VmF=c;i7;~sm@;8gnLeWAB#p|n2$85 z@l;FeLi-6@hhZOiP$>=drR)s|!oWsu!vvNNei(t&CJ0{ic>$#E?V)N0>KrthladixBr9*KN(UP%02C zVNsBQ-iW$Bnzp<##L*~$)NiTwmX&gRIiUO2gaX8OViowE zA?ozu{;=QKA9jX*$LqBFe!CMKb^54}`a{_Fy?yV{>mMGpyWRb6uNU+Ms%Rg2{owHH zm>~JV|02sl{j#F(EnEyLRT0*NN$6};1k{;KS1>&^p`~0(NYg|^_Hkqyi2 z5g+-28D$(J^bs{9lhy<#MrCBepB{bD^=I2Z&m}h=}D1n!0TBJn^UXm1vf&H}lxF9f?#zq>i0H#iLC+CNPqso*8XqGBPI`lYjT zKmSve|INHHnn3qAMxu?hF+cyi*XgwP3i;ptz0Ur9{`Xxx9~uBS57me}1K^ldNHO?L z%?N~!T0VsVZ*|+GHdxGM~jr)KsgG>Rn7pj3~#5IRo%P3N^=0=uY2S^@YzB zjg%4bX;;-LCW0?p&rSok>pY|=5Op53d?eGbgl*lrkPx23xC)4sm9#!2>a!};Et1Li z6=aj|i@D3WuNJ%^At8cMW{8={<)^fXObVadjb$-XgL5&By$XLwDV$~KV~!>T%xijD z-&f5jg?HuQ-pzlV;OH5OMttlHK*##@DIoW6K#)uZI=Ye? znA2hXSIRBbOqLn6Wka>(k=46?b@qO}J67pc6_r}f2eF*kO5dd3q?_+^)w)jgs-?yo zDUp2XAc;=kyoHm$8K2qq^j3Fp^up`9N}(|QvRuA-YM}z?JUXS{Yt=efsB}PieuKFx z9;~e9L1A83Hd(JWK(*QjMePCBm#j(AI?V&>O1cbA*G zwqu}g)4ToWE&mKIi<>-RlN<5;o#IcK0+{?Aze z`D}4Ko&NGUbX+mLX*Ja(#!2ShoI_lSyZmKEj}XmGWWq`%uvQIu=gkgg0l#VGl^)Z)rSB@Aqar=)7SL4M?0i^dy6Xbf)L#md3h!#JfwrXIQ@t2& z9T2mAI#6jwEZf(8t0+ic%kDsf{_PMkb z{VqxiyZD-|d>xcEqCUr*IR>atQBGO2r2J9{PBr`zwbyH;ikx#=x}83Hf058~h*!gT zon@}3n7=KWMka~1)=qiz#{b)+f7I9|6Zrn;2%4>6cZ^3_&_%Q2!hMcYGlIRfXK$&D z-KDqX#p0#qY1!=t#nA?9#AQJhnZb(Q8>sSM*Y9dRLV3GxMiuEPx3uu3^-xc^jfC^_ z6qTJQVXy{)g~QwU^d0Py`gWh({x5TuEo;4adHM9h+E{S^*M7IxzTf}#g`UeX zjIP0qBytC4c8Nsg=1;fr;X@PP0Jy)Xor38Zaan4itHW*J)2Bw`--r1F^7K)ZW}Btk zcwwEfi9lU(tWP`gk|E@E$e9B=UGOUcDDp@`;RwmDaRNA0I?T5uQR+S$ytm|G2NB^Zam7%- zlU*@g65Tw^)4Bn)sEB7e|$EYFv3s#6T9AW3>^ zqT2xJ>8f_2^n6vDCap0FC#}X~aI(xUXw0%>()4i-*4dDHQ}xz|55Rq;&IWU{1B5?) zYKae;7tECh@HgaF1;GCi!6Ain?UA9sl{R*kz)PbaozF|(^ge1zx$XDVk zWGIzo56!Jg@-fS(Ab18h~@rnKcYF2{(XfKa4P(U(xo(XbO3C3c&B zdT5a)$%Q-;nq(-Um9_sVAn%B$T4X96WX{nniE2V^Fwbnl<3XUH! z&}=e9r-(L%`~l5oK&XeBLV1PQtV)wEg13_c zST}(#2|(|WmvE#RU-V&X7hq|CW(Rbo=BhsK)4H-^>yfaqn7o*^c*npN)7KEl1M*$) z6iY7fF-wn?mP8X4q-~5FoykVgk%K*sizX(n}gLo?ym2Gf$bwJGt-0 z0Ww@L&$N^Fv+0QurI-O>1Sx*g%mzMva=)}+^u0Va{{Ph&g)ySc^itb#9#0GH&Bn1UOA1Po9Rz!QPSlz`ziBFkwik zG|TIeAOeg+ed?V&u2M;={hqG7txt@9^5R027g~zN$lGZcJVFx5qf62IIK&?2fF(hI z-wS!s*^@w^$L0C@k6T*K$d#iy8%qUln^{qqgxts->1`P$VkgOfkE-jU)8bD&xlm|0 zO$QXjo229^iKd8h8Lp5gu;#-HD$cA<08EQe6aW?FUA8kSIlZJ+^=Yp~*{n63^N9WP zk+3hV18$2|Qd3rd<8U+w)}RE-E}yfBs{!O|FH#sv95?wZi6q&7~5^G_Ix@@c*wgFfeJf@DQhG;xW^`Ir zK7!v9!cn28WhnZpl;&`z02$8FM-f(65NR#M6@FdrC63B3&!sc_=~FZJG6QZ4@|7EEgO4A< zs;EKV?^NK|t__7O9*ca$&m!kG`d#0uwkT+5M&GFzpq-pK^)MazU#!uc4}Bi9xOwmt zMk53s#*mM9!NV&cpu2KXxIcwtsM6V6kKGxZgigsGdWMW`ee&GF*plk_!zSVm)gr%lMJMe;Z%i$bo!CCv}dNMob|<+mW}JN z6i1bzFC2g|WdBApiF8WcbT53Vl1*ugi$-FwYzVFbAV#1@_142)ll?7}cFKXiVgpL3 zosvl`to-uSu%Uq(Sk=TX^F^_|miBd@ss^g~zV4l1KMNaWG7N>hRCB2U#2klfb%B-* zhp9`^2)%zUh2^{d-EMz;-Dw`Zed9mgdE?gq+WyC&xxM|5LH6q78~^cZ*!<(9`G1>l zU$>h_=5JZ}jk|2&oyR*n{}46)wrzj-n*@@3BTGJf2i!HtkXzQW#JaHeHznG)Wh6ZH zj)Y~tPy;Nl){9lcr#~N zTlwo%>gyWKcELlnKq3@aSqn&*#{kB$z+L^^&U~$7;cE#n8p_VxkVXY=*5-{01j`61 z84l7#53ns!a?XSFYbIKUO%*$&B#;1|=*4%q&U;0QRo4sg}# z!A@CenkW+b-NMj$`W`c0pv%Z6oH zJj^kmF7l;Ay_HW*{NIqZb!Gm##frd!{J-vAx08?mcf0L=?>_#27teBWRTUj2G`lce zc(^1|7f3JXf2Qk^#fapBU{6-u6>gUSlk=Z*v5yqTm0}+$dbSC?5V?FTO9!MD)Ja=n zg+Pm289A|TZ(b*+Q`_1K{VPU<-0BFIx)^97%&NgG14|ifVBHs78(S?dUFn100Mu7` za{>~U-E5h*yD|;!1L|JXh1F}yyZQo{!Xnh7?4a_zad#Fr71@C}E9h++b@U(3YLIH8 zj)j}S&80cg2LO7fj3j=juJ5t0)>(B7bhT#^k_^I$R&t@N|J}B$Ug`YCkGYpR)sXGD zk006{`qBdVj-Q44pAMntj!<-qOn^oC|Lxv>LI1Zq`}g|)P97PbI{EqX9mAEbp5AvGM7gH zO@-xCgsjB7{t>SRF(!Tqo{vJ5!mC7EHKP7jN4u`qAJKX%z!41bABdI^(V$lY$2jCk z2~-8fOM_LZ@$<=kcQn&#ay~j6!k{XiT{$siblbePwo>1262Mr_v{G{_D|NDPE^Len z)o83b**{CmbEeO7JfE7!>Hr=bjphurF6@Pnu`&xT>~(ENZ-GRf02SX;fna^BuT>~N z<~*h(;pldPWu5+&QAkH$Z;}{MSTO}wn`1d+*V|EX+gB%|>5DndOLf01pT(5Ka0HDk z5}$s%xn?PSCYYNa1p8i0$rSrpQ{o*(;S5Kk7*eZqBt{X8F-w_%vaA+GK4wKw`W&k< zjQkKsBlDW05J}Hs$=S4#ajY4dTXFa*?W@7IGZgP$x!w}ryp+&RU0H7!dVtO1%+1>>51~)W1M%l&L*0iZ z88I!rh!yXY*TjpT&!2pC%=f3{gB*O*s7EHvX$OY!7&&STjGoQz{z*RAa-UzQ7W@o#DLttF4|X4FgTI!g_HzSg(I z10{NVH=Izhn71rDR8;+zc%o*R^Sj{;y_UNaD~Ph+(%9?1e>Y>DF4iwK+N$)oG~Q|d zznc-aGcuMPce~EFH2THtl<$fOtQ4H3!-U0p-x@QN(vrR(j!?N*OEZEh{!NX4mVNf! zjJis(TY8mImj0H;JWDbBZiYOUN4f0C=PG?m3{c7{{ciZ6kUG08ClsoEOT18#O#J*C3tA%O7u0IJiM^QX?mp>ZDQw=tBuWIO7z?SE}zf{HpXmD+)T@ z1GnwA-H!E9v~eX5^%+=A`LhU4uy<9Gd!Em9-?A3umO~g`1&Dj&D~>0KBzyomZI%`> zaIRpCuhJt~f9a(K#CuIMRWjFs7+|H8ptf6hmfXsJ=TiYW^4`y$2F;}IWudIp{ReEG8QSv23hiYej5BjFW^ z3&yc|%rv$zy}^-BZmt;qLkvTE@34QcUt5oHir^$)XTRU?m1_v5^ho3-l%k01ib8?4 z+l8`ndv493fJ!Db&lFRh!0>8J7_aDTH3w&pPbPW(va4=A-l}-7wh~v@P>so>Vk^Gu z)b^^XAamvFYXhCbP}#VRz}8@b_?Y-r=&MKFTX8iPAMv2~(BlKJ_0!YKYG9HYbr7_) zO}*c%+(|y#+LHEA1utJ!Omw@GYCf7tC~CM8nc}>fKjp#N)81-POz~HC+XhE-RFY}w zbBfOHBGFkRJi&?r^z%`4LxL0(g=ujVvS)!v1uKYEa0}52))KE^B@qi2idm3~+IoU5 z6u9cCs>QI?qXqSsLj^1n&*m|^2$z7jP+*tQTr9e?XIUYhJqyLTdFG0B z^QifCrBmt#U`y`XXRVkL(Fv5WMZH*CL4XBHLOrXi3iT`!Z|d2++Ayi=MfnWr&})fdFw-}}scxc&0{>8q2=m#?l~oIHPeaei|8bjM{= z&ow(DLgK-&vYDI??)451azo%>rQ<(Wfs}Y9iM1eC+@kf~xd!pFN(HUPj7v3;gKzG0 zyE4kTVrnd_DlPSRjZh3jJgt;9GKw~Z;Q;jZ+f|Y_<^8_J`)M#i@re!CmIlRg%;llo zM!gJJjntLx2#Wu#$KWz#<{F(mktP!oUGY#xYQ*OviNvQEQlan523p9j^c57@fTWR? z%V!tU&Q-6v7JR=_EwdGyL)obuNupqVn=| zuIPAuPwUCN0-@_k-OMl4E@6o*qk&H@&rVm@ud7%?U!~TqTB5LqpmMPS+-uD*qcx?` zUO-O0<8YJd9x9O?`ug9@@aRul5z=mBuvym7a)65`kE|8?K~ZD+@s2aP4e#Q=4o z#W2;ffIYG50P8Y~L7Jt)Uu3#8m>BGAfVKd{~EFG9ghoN~YbSLV7?vi7m zI_Z|mxucK_VR+?}n@9%~bfiGD5OWH1uFRW`oEy%PbEAxc)wb2Hm^Dw*FIgyuotkB4 z^9Y2Y$MF@Frn>Vi2QRmaH6I0?L#zfT^ z`_ACe`L4sq6hU9Sl5e{Un)EhX5ZeP^bt8@aB-yE3^rAN~<*Hl(7+v|z*aQo?t zpUz%9l|G-Ir}BDc``g*`lb>=Q&yz4Tc~T!fqzMVCWp%x?8>eN=Rb7ydEM|0?I;Em1 zOc5u0Z`mZp3lGlGJQ!&P(7+$Em4o)TXMhTbKwi z=LLH+747OKQcDYe%hU7PD!HZYrK@!3 zR1xN1gcv)gWSH84!j}BWj(JqBDaI8zF>j0nt^aj-X;9KD5>HSFXBYB{TPwLBMidjj z^jhbVUXCeZV-ot=7naE5nyfFTacz&Lu+)2jGX67ku(xIhS^f@W)1}2Zb91gyw$t49 zv&$ppJ;^^aLW+@WE zYJ1L-Dn6${bn{I~4V|^?{>F@xv7$c8L7T6P>g(NWsJSR)8ft->5Ij*Ys;e)yb^dOP(GK@CT~8tP|0D`2C#yHCmUQyTl@9e2y0mI~@$~Z7m#_YP^~=e#vnS~~Vo5!f zi!H07G-|&X%xcw?M(`2+CUX$^7IVq<3)I8D!aThL-n@Uf|hguF2UTQP}7TY^R& zXih;h%p$nT+y#F%>_#ZS07sw$x}f);z|UePEJP@_gWxy-^!VYnZUN_7_#L$EHjdiu z-VXTdU%_kekb>i5AP@p?{}b>r%0;^ctY!qZE-4HG?8#kyR0V_pC*VZ!CwOl0CwMCD z7=RftcZY0&zXRzgazrOMl0gaKA=)jy#~fsl=>Y3LeRXhch74L8n~3lfqc_nj#5kl} z<_^ci2Qc7>0u-@?s+3~L!I&`qLUDrt;9~}dA@b!t)llB;2{|5ysJi9T7iZ}$^q2u0 zF%Cmfov7RFg(L62iB1s}d@r;;rbI4PFff5L07FIqMZYJQA^?3~UQq15x|iOT?qlZXdL7i{8_B!(k$JufJ!g)wL(Hh7TIQu{%oI)oZC95whEck zxUI137{QRIiA3s`RIfka$UiYlJA=dap#r2cpiG6KT}me`io$9R=^c)yuu_IcdwWV* zWfxMV3lM5oUZy^kGjeKV@}(BZ1K_lD+1C63U>5N%j?`SBOY4OUzOAO_!d}W1avE9p zQ(Bw2lNvr7_ELM86wHuhab`3{VNU6!JF_%Dg=F+h6ta0!^8dq9jq01AV;V+CATJrW^i{Bt0n<1X91b{!ZWp_rG=y4vPDKyLy8kD2=_yLjVkp!S*$pv8uYJ6ayatBb1*AvtcKN^)M|epo|dD?F@)@uXeN9Y;1l@bGDQDA^7yEnazn0A1vHHf{@%Gt)KFnoF9IsUCs~7+mWl3 zh&`%G3y&xaU<9)-@}3My-b}!&tH13Qnx8%?y&ywl;3N1wAskgIv&vo6%c2#K&_^`e znY~HGsxYb92|h=I?^VDyTqRz`6h}M&&LgIxusJ?4K)OJntzBqQYx|-5BMuRBe?lB= z8yFd#T4iuMJD^C3^q7e`f1CNgQ2wVD_4)@u%$NV23i;pOKj_@c|GRi>VF}}ywNi=s zBpvw;o&xbrAOFB&B*VMrR^VzsNPj&gNyKX~OUtSo880}69F25%j_F^O$Z?7%b8#$s zNfnnQtw3%A>sbZab@ai=l!5W}Q%;frha$11BQi$bHA^OX87s$aN+^PduJKjfA0R{L zkdJ{=cWRBZBN1a`*zNDLWReMY0IriE@_6Wq|7nCcVlE~vonO~XUFiWRZedIZZIUql z9MKUf4_{1to4slo&%$T1%UGe%^3!DlQ>N5dHG9TH*$oWbwg+bF7(Y}J-C7f~7f!`* zvoLyG70t3@JyW}q5?yvqmW8y=BVH)jWX#;DOxd#gXIiU?*5AJ2>R^5q3Ki?ww{O0@ z++EdlZbwwH(c;Q#-P-EHRot5P@{73}Yt+sxO4nspC0{y)N~D{&fRnFPv2X+GR*6AX z&I`!$UR0F!kNhVYTTR2X8ojEe#g2VcNMU`kYi}T~UN8jNhT(#c&8>`Atc|CEUmJ)H z5jWMLDK=&m`BF7`SqCMk?|rs5#58)nYOT(~TGb2Ki-udKIjuX|a|XM!++f#hl+s!U zs6KA5R>^eQbM<8H0ub2bB2XR)Yfb98N;cN%GxHV>D=MYuDu&?@de@ia85yyc(NjuE z#*^hlAH`dC+76$@t?NV9%uke-%%hD)O%GAdO8hV0S}!ZIdddxvn44twR){?mG1Mvn zu1rAErB`oojx}H!On2>i#jOG?87e??;pCfKZqaBRB$k9&jn2}`P3vH`I`C>Fms+=4 z2f@_<=aE}#;p-OTxfI}9DJ2(`Uy$e(Av`w~bQb)H=ZU_&SkX%_*G~O)ke5b3QRt!b z8nIsz?5Q<#A@kZTv0nVL{F+&XSg%(o5%EhahXB}bKU2e;gyvvbi;mgL&(tgo0->}m zi_pBeRf9^cl(s_Ul!Ds2U@KJZMn5M$8i2i=C8!Fo%BF=QMe_=*X<;)ryS1oX79id| z>RO-FzDig;^PIc&K18w)h+qtO?UrI{PeSE74gh(Kgt|pH%!X5ZEJmg z(z0auo4f&~A^aVM!9_0G&{)UknH#WJCdJ_EwtzM}3BChEXuYX$>p764>>aZD!)jKq z$uqpS^6sCSXQBP4Hs_c_R@MpuEsFp2_x6kNpZ(6g{dXr%WwNvuS{2%7{xP`OS=L`F z$!}h0T(fX7V`+s1-{rEtjJ#eZ^J`NnhiRfj6TmnCldZz&!7u<lU;ixB|A`pmXk^yf)-eDUuK#rRiu(Uxzkjd)@8Zej|C?)SDH>s!W?to#d(-|VgEnSyca8-AQgXf|`aDFraRUXha3mbTrU#n=C6jvWT5D6sPS#o>*tCL_ z*7TRl3u$;nzm&m|C6^o~d43TT-2orvU3UVD6H1}XR%i+;Y0T1gK7w?sl-}G>6H!=d z(|A;Ty`th;eTR5Wi*nb+)Aun|c}iAZ#Wk7@Kz=hqb;`48cL2_^gH2?L^D1eQR_E^b zaDR`_BK;o|U!TPMx%U6HJA1uOdH>(uz5c(8r?CE`x3lO3O*ba~b3OXmT99orXx_ll z$n^+Cgz0l)w5@QS*1z4hYpL=SZeN#sS5L2SBVmTm?t|4RC>Ujl)(bZbn%>vDL(&|w zJW>g$uC%hF{2yj>hgXnVIfYO2_fs?hhjfH^`S{z)8`sQvuF~bK*dYClKa1r5ZSwxi zvq=8;_X_@B=U}gYfBwgvJj<;AH@|2Wmz4cF#3v<@@H4;VZR9$sO9jQ$TBdzyu#%sp zJy9l?cmiW3+NOvOt*;}LJdVHy=fp@ zcdf)+4&B#*bv=3I4HcD07Ft)${Yg~GGV*`jyf5=ClKbyO1YzFk^E;U3>Y3o$ZulC60}YjXo381_YaEeKi$sWef;N6 zp4^GQg;k7HnxDy$-=rWfvDG&%%1y9E^!h^fdH_Cr9fED~{Ff3MDha)w3AkwfA9RZG z-%h`CZ~xuNQ)mC>HFKRMSWMk9w!m#9#hlFUFD@aLNTAUKgLjcgvjHn<$@zb)HJ}Ca zzt`U{<$w0?^FQw9Sup=^wfb|BgYj*gBDqA0-=Ay+=#I~WoR|FYL8U)9%O}`5yH+B^ z6N1J=cBO*m&&NUvAP0{O>;M&az0fN%~uM>NVE)gm4!o%R9nDTzzzwqI(}>X^d)-2F4p-8D0i zzxRoFjnG%RxV)78r}GSPfIV|g)hcZ)ivRcaivC}BuXpeN-O005{O2?q{LKb|ayhJD z+`Iei;@Xdx?2TUBox7>Uen~mveh0|?4v=MbfJ}=V zbH4}Veh&pKBSVsOIl=lA}?A`DGxRYn2^8da6 zEuX&fXOsOS8he)LSA%23es;;xe8F$ZeQhVI+{g7c^VGYxnQeZ{&3}vK|3R-? zIRCe^f6%*^|9A2f@_*#GyC{Og5czAR``L0G#&O|dO^-}s5}}9>fNZL!*ph)mD;kzT zTP&N{7*gcV5lE_eXVner5(j5V4OmRexW%TLdy+3l@=ffhSr@}>15NGrncFq6CO_RX z(mIT!cIcKikTz=%HE5K!ls3(=3A!}1ilmFga!2YqU1{cli;h{FR>lVWBt9Rx~2F;g-OqTX1twWDlxfz&K`=aI#D7Z$b5%b(vL2y%N%lLym$ZWJphz z{Hqr8pp;V^WTGMTu2JN#&OLE$_bR&<6Q5asw#-GC`lsk-k}Ci?hqAHFcedGfZUsgz zb~P#bJwc4$4wPC$#3Zu8EglGKJNmM7dhW1PP1(A%@%ojwHT2VhOCi2@ZB@H2?ridA zh>hd6(pKX2_4sGA!t1UoyJyFGcC59Rss&FvawYLo5O|y23-vudbK^g^4gBk8QT)H( z?Um1e>fOhG?&7&s{HJ1pU`=mD!CThOKQmi9gXP5?t@6E;uUa+ykUV9t{;ypS8=tQ@7{mqCtqDl&x*^UYYdN14IM zv4p(S_X(!*NUFuuUaEp1@1NBo{=S~M_P;HdZ{YyUxBt6)`|WPg{@>fXxBu_tSug%` zPW+9T{F=Z1d?x==#?RMf&sUg~V-!wYHg1gxxmF}Ay>6<7!)8oE?9B#Xhyp@UQ+|@` zf%1e{ZF>l12q{0;WnZAF+~1uyM@)z%ZBE-I&JgnmPH;<%lE=e=V1rW_!yyha$B4QA zPKHSHqx&5GCf63-`K53qOR4dwQS}>B^65z_9+^yFlm`{@| zG`4sP@0`X#vuXV3<2GnEIi4Vr@Zo#loC1s4?lE1 z6+b(T^aT!5ii`x8F3ltLj^rj-FfS7$5<7Av~*0k5;m3D9V2B&NMyczQOQLVPXpPfBo0f zmp@uTK9`Np@Hv^7a|XbfcAQ=NtC3jFAXf_jM{ln! zz6ZdmiFP>UwI0P6*5A3d#au17n}r9Q0eD?x{K9Vl97+Cl=W9Mid$$a|5F~Jxh!8CZ z515mVz(>Hz|6wF5Pw)Kiw~eYVpSE&r0Gx;a%;+vp?*sTaTYR}0;}C(@;2{Rhkb|Q( zcnkjeSMZnRt)UIfg_4c$6Jy!2{{;B>;r2HETW4oSe2B`LT%j1;wUGE)#8xU`SI@Xt z^)jFT7esL|1AGiQ$dFBINZp8-^=YfFduhZ32Np94hcUHKKr>vzXRAhDZSw!`1kstZ zLWj4&M=+u&2F>4V2d3a*DsgxGaJ#w>wZm(<;m04`^l&Z(?i2V3e)}DK1n}k>Y<-9+ zjyP}taKOX%r>&jL+80j12lYJw@0^#qhZX``kJuK#3`B&37*Z~L9zI6te)%n?Hh2K$ zkr=t)k^uA`c?m~gYmCB)H0ic>WgU~Cq3?-IDDnZ0W#oYEW!Xo_<#UQSpGoPQojJF&lZ{w9sREpvT~2wqRlngB#%V|wJpKR9 zE(;Y)vsC!R!Zj0aTF$*q+2cvJdkt^`h6qpu{aNLlq$CVU!gY9SuE@$_k4J50sTpgt z+}ei~7N(u#{H;sX93Y-2IKtnmBW{UCD)M`~psyg^R%gp(F&XMT(6L`mz#oW`G|5T4 zKoLtQ0uhlx0stujF-1(UiO}!j#}t!<2`kdL)L&0TIg6uwS)4kRI4w0F478Y}&Q@mb zC}T%xW?@Nup@?QFf;prJUS~m_5C_^G8*$JBZ%gIX)Ho-;l=*UEZ`tu7^QW?)x>@8V zGoLc{)bkwV-EJ=uF%Andb*?AOp(KM}Do+|Loy3OWSUJaD@lqW)8#ae{BC# zF3=tySD!dRo9?FnX=}#;$6!nJYpY_a%$-{8+PRe*M%|XM zvEXZXjMfpFy)3U=;dIV~8K;juJk;?R3u> zjm>62=y8hmaczLTqPNGFQhZhMt$n_@)JTyGszJ{~F-JZa&ZH+&A)ia5p=NKtXmunQ$%lWPbd-1cZ|Y`3U+R?jRpJv+WY&(^Z)ib-TU*u@8o#^^xb7L!lA}q2+Rl!6CC<7 zNdS7+aD zV~PU&9{EZb|7FJoFQaeVgxB8V|s!JV+J%a(V%LOj*Mn zVcwGeD)bxf@DJLO|C$%$QA_+|ezIxQ$^Z##CW%3SL&O@7U3L>U9=k($-FWQsiTImP zJZe1te~ky=7f7YpI(zbzHQbmIp*|We_7QBU;*|W>aHmWt3E9M}Ck$LCjFX8uPh zH)Yu-jsij_l2>8)55{L9Iu1g}o9y=lQnUm9^7S85C6RVI-(5SaV8(fF?z zdpx1&?+pO{OZ(On9qiTC&NO-Ty0@6@vV$lMs)hS9mn$S1+VYo+6h`N6zvXhobVA zK{>+|PeuTS{MfN;I0BQqLIXhY``Dr4jRS75&&S73x8HXD-uSOp$_I_-#1}fv%D724 zBM7EMlOgi`Y)#HbQxuYTf+7xX2)zzMawGXMAYn*u7~zX>B>TGz-=VW;E` z5EO&J031OaF|NHmwyPtjFl2HrPq8$3AQbn{`LnvkSlxo!}@FofJ(7N8s!np!b}r%WxZy!Ot?zs*!yfP036-*Z~%t#z2U#5qOqB zz&wg$F0IE`Kb)NI3NTkN;p0Dc!T-7>(#;t z`9O#pX;(u9NX|lq1@etUDaKF^L5W)>9sbY7%NK&Zm?VQ}To#Uzr*32wv_?V~dU6f} zLcvcI#^e8frtVJGp(;h8#*c)`+07)k!Z1u*+XYjMZX}A2AHVjC2eeCmzd%`f{u+H<$9(10QnO>g*r%4-UGAhuwC6&mHqgxbyh2 zEBG5cAzmVSld)2Wqifb^$e4~wfd~TZVHg5GUzQ8>i&(6P-_GO5KnX}OTz=wlaD#>n zbEM(d0m$09xw$DsJaYv_Pr>CFseXYW5*#G2$Oa_gvgHY3Tux=GleP0571%K4smV-3 z8R!!8vr+^#8jr#2pAeVDE<}0cuQ_P@?ZfuoLHqDvud4=4fWDY9jwaxQdSfiuJE7=y z;E(qE?F_sf*I5znKRAi8cl`XPUz_T*x!$1!c}Yn!3=ta>B8ajkw*8~MUc24jJ2*VZ zuvJoiX#u~}XgmP_onWD{#K?eAf_>Cz2=)+CiYA`YftpLCoJ&71~>K&B2Zw+sb}DQ>%z? zw;zxphd63hqOaE)B^dj<&Qehj8P_t0(c zG#Y0SAk;@x=%0k4YKv<|5)dKeOQoQeLWQ)I0^ddk{0~XM1kM1$GL&wN0BGq210c{= zlQ51(NC?t6q23q@*s0>sgjgOfbM z;08&yS1Mad(Flg&%w9)zr6u)Jj#xEy^td)UE_hnsZ)sZOdL>+<$1=Q>qs#!*Dvo9f z&YB?g>LV+@quM&pBQUQejq7sl=)yaVuB+BllxdEmA$c!3g+bvJNJXYcUl`O?eDCu2 zT;(W-LLdEITE(w3B2Af_c0-#LOz?s)McSq;3^5zLF(FK0EuFR$cRq`e;tG{xmt(2M zyLaz~kc}HMTT+^izpc#Qe8MXA^v{G)H3~}I5KRv-gCn`5@Am##pWn8JgRdT#ZG8E$@-5jY`0Awtg z2>+>mWaYREPe^CPX+0xcS82RpDwm(d(05$3P^03bf{#qYuHa`wX_=3w6C)x!3Dhis zdxdY++wPb&P8tfFY57?;RjrL7bLyMM)J^qZgd#+7%|<`NP)&LJb5qdfxQ=oKywWt! zZABZ3%nFkarWh)5$6@3{>MJL-2PViH!w9pDVp5dkShcEei>O3AJZQCkPvA^wFkMp3iB=v?#^lnu-7>_&n3_a(;gwLqSCR0pw4;*iqqK$H-M+o%flJ) z2{O3^GPkS@%Zrz!qMZ$18|C{G?jkB{AJA2IJiuG83V`waz`#Wp(6rLp9rhy z$1r&>{v2x?0aHM3qD)$;unbH%5sr#y5i}j5RP@P-oWF&<1T>nxC_ zqHG<47-kuiyC9b7QhFfdn7oej)rH68A;vPpgClSJ2Z@l2=!itFH*V>oOXttNlwte>67k0lv@Jo#@Xh)&T63*ME> z6NGVPut}>GlSo)MKy8U=G(LiVqZ#-JE@v?UAHfrKcBK4EN8U&TJ~lo!#iRZ;|K4!PBe1`6WkIP3T6T7g3o>B(!p<@r{b4=`VF|HoY*vnx z0s9dQ2?+%xDYyi%Uv+{ICqNX}E`_AO8lIam_Qo>5U+%ilvIj?_R(`^dt)OruoCGCD z6!(&Z`_PV0=Y8W>PnZe`sU{(wFtx-a8wE*(-p49%I7DMO#nNx`0cK3dTR)M(c{xL- zJt_6(PAt_UVOlE023xewWRNT49RO73$<6b3MYZu&)z zoJ>9h9|7Z30R0HwIlACgTF?iJYuQeb z>6C@nYIv$$Q8`3A5-~Z6?i90tgg&C#1e!xbRkTFF?C>v>ux=PvRgi|XU?XYFXP5{F znyido1?j3dzl5d6#cE!#X`sVeoXFXvxzrH)=d&_;%3Pm6-E|=t<%Y;2j%mQch7*7% zLadjXlyf2y@<-;eJtLzEG=z0njfLottf)4acLZU7 zZ_n>JJ1%%>j#mI2j`SLfT5v;ER1oN>^A7WrI1647ey-%a?9REsTb-h)SKu*IovVOK zQA`--gchI{e5zK}{w2}anIxJ8Meb2ldf(8{UdylA-c*O1AWIG@=}=hBiUPWkT=mpK zt1#(anN29>EQQoal)g|Q^TEhy8hG;~$ikH18*(&n_XC<3J2i*!*i&GaB=nqN8pP z?H}#?-eK44wcGtR?Cg2#O2%8ayOdnkZE6GSzMN?bnPE|{y z7YaR);wNLT&4OgA=|oAa&XcGm7&Wrh3GoDT<_tbKUYs}sr`w-6yUqkn2%R|tr_+6o zolm=tUfQw>7RrfYw-waOktaPx00se;Zg^R8p32;5kd`PmPs+z7xZr1oq?E-`5GE+{ zl&QnKG4hizmE&pi+QSLS=oy-#Fh4dp3Wyw^G|FeE_6buh6qO1fH6c#U&s?2W6e3y{ zjpwyZoiJ9ABLH$XnfXu({dQv$TJEovL& z^sEtMonGvj=|a8SA|o5f2P7X2&i7@ZoNs$U=cw(YpwsRL{ey02=mp*0p4aW{AMSUC z{Z7B_9qbK<(Azs~?;RZ+bUW=%7r~?c!Jg+GtS#rggCm4`!D0V!zZ(pDhx_Qr_xHQQ zqjqPw-#+NPtmG299m@>1Rv+}Bq>oSa&7Itz7DJ5y-!m~)cmzZNl4hnbv`B;xX{Ala*~v|5wN z5K(1e>!V{#Yn3Xg0Qa`;QQAhGO3iO$bm1C9E!xT7*gU_Dg+m7m;LM8K5a}T;%A7(2 z?#l%(@PjHooN23tcjig#gkcS=4RM=hXRJlhv%Q@Rd?#SIO zk$3=JSI4+iDvk5@-Hr>+BH#&{nqkW0SYj+S_lzNK;gBgz^m_tBBi3>N($a<^Oz52j z{Jd-O7GilGs@XYWO-SP+7BkFCEVmKP%~*3D1G56>zCt5N{je@eOprF+)~>wU5r)31Q~-wz3bhNt~O#hBy?7QIs2?GE&}lhA|}N$;n6JicHeJzf}>6!^-+Ha`@XmD9eVx4gLb#O-|h8+{$AJbp?&D}gTt?Tfu@2M z8*@=BV6Ww-YB)_eGA-ZASG`c+fW&J7zNZlqNFvw97R%Mb!50H9kVIUpss>P!+1+_h zK1kj#mrwFL*lHnb0g8*nBJRnFZ5DQ1T-*)!!kj};QS>88C~OEh$Iy9^L(=u!K`x3A z{3gB_3dw=XiF^<{g$9NHtq9_ngu=lv@8z9`ez zi+)*>N}4npvPUlQ>NQkUImP1_pMf*PsX7i+%sZmqeM%!15{RKA4|7(9aVac)yvgUI zD5;OaseA+~UP3^^G?A6B0T?1Wpa%s}!i2M-rosZ(=DT^J(BpSBiU06_f3yF^;eY&Y zm;YV9PR`y{yo*K=c;(~a5RvrM zh-DNnpm%*pK8Wk#HR$#J@Zxt6j0qZMPs>X-MGOx0wdp2D38}pZHas1dz;pd;z5X<~Y;R_%8rZD@1k5(u|L~CgKI3SCGSEq!%!xKaqEM#hzi+(c7=Za&~Slo6@|0oO1B;k_546?o>842P0 zH@7n%fJ*Bp+}CFoEtL|qZY^a_m4~yA)SyE?$Uu<|&PKVeW5UspVKkR_D?Mg~-)9E< zuing80Qel$0~w#6IW=^(S;*H>9t1~)Y35m+20?_O;?etajo*s*AL$DhFA5(JVL}r( z3PlhX**ev1yL?`mKB$#eVj0_Qw(W_CD=cJ#B+Rua9|+Bl<$WY%Lq4Y(> zuGktpNLWPZu85MeOYx`&bD;-zVT^lXGNIVu{>1-HDhkYWEU`~YLWNJ_JX&rRKc|XC zm7;n|veZ8$K0eo69~@$?VS;xd_22oS@(oQndEH<9kGVd_#dC zC{-P=BqBwz(49q$MU&Ukr={$;0)QEsln${NWa>Cm8LUC- zHjA-_(r{R*9(ID*I4U{H_H*J8VSny?G|cGg;_NiPO^PHHSEw|qQvIOq2CfHxgMujz zN7ss6rDd!{q&iS;llTzO+GI*wi$S~k`*9*$u_AB9s3Ds_g!X`VJk zQvau}8}Xi389fCtPVX4OsmP0_9>WcXtrq$u2rLGFlUB-2Ax&4HJ%xlpi_cRUH8)qrvK;sU1emU73N+p*ADOWdu2!In3s+T^$LcNx+oUFcax+7(|aI)XA1M zc>%-|(}P=>&nd52Js~K91d8U(sA?O!YD-ybXsgvoL0j6|g1T}LY*XuLmdHwVP@@Ct@g#r1K~EC`}U8e0LlYzokxh(3E%6`sVVaAA{J~$VGo}P)S zrOb=!;HR-a5)u4(gg)vQUbc%ZAx&Bk4FUQHJ+`A>N$dcc48@k{LV!NT0roJP!H-hpaJ$J%HA768{N@$bRT?KdP(bq}uB{MJFfqMZC5g1H0Ia6Nbf-Zd8|LjwdWnHsr zlO$kR)=;-jOoSUUc=bmrf{hACWYL+!a08HYdMXsZ6}=ZY(BT}_Q@@o{J@wI1y&^1t z6IS}y3~V*R5_($7vsR~A4bXmY77ATgC1$Ay1PC@My z)V1Ig)TZDR)J{QN2~I)n6x2>Z-O=;u6x2>Z-E9na3Tgw`Dx7c%YTy*q`R<>qg4&={ znXrUSuSpcckeXSEGdmYzcj1RRUhbK)(G*Wq zER*6R(l|F^)#jR2?IoLL_`s!U2XD|&#a0v2Y}p>`z%Hv$yxSOf5f+C)f!nio16b}LwL%R{zf4GK}{&1|AerdeRGzP3J=fl@8Ws7^9QX&R?YJ?2fxGC0vcWic;hL89YGN?ethnWPd@vcLAsZhy*hdR>;zPr zZ$4cVQhrL)Q0{Z%2QZ*fDk%riC^t(Py32z>Fo50%K#)vCoO?AdY|hu}^(BCXL8%CK zvkyPPtNR$T@p~A@I28E-qJ&*THogbDpzuleCqBMjlx2}wv@D3=TbXt%Z^lWLlCm+X z6^~-Imi1WJ?BbrkvhxGgiE5x4>9XGhTUpBrLp8UK3GHq)i6OyM@k$`%)Hx^`+I9fU z8-TNO$Z>EKicreZJPB2Xeh-E(1tq{DWQVDu$=?2EZ*^<0=h!F<+uC#->v=(KZ8>Yd z&Fiv~b#8}jEiUc1=viA=I=8QuR+IMIy6F~@=WiG39B$g$JK7`Xv$kz?Y+qLx)Wp<2 zVKD?l*Ukd!Nq@*f$UN<{`TQh`m3ZAbUf-SmDsa61bKjvho^CDu+dARZ*2b63$x{M4 zcXpgSWe855O2NrfPM%7k)AE#IL1|cMGBz1A97QHW#u+bXFy`%h8K+@p?J(s9M6QoJ zUf6u-?{%GWtD0z1z7tC=j4JU+Py8xH&y`r+jO9;znsOP1a?ju(5X|94#_g>@czsT0 zGdnTbPK0hp+&4yaUJH@8Ch(TT-9l7ux1H+{kvp*yHsV^lt$4O#S||3;LM&^yeVZ(f zJ#SI06ZmH%dbLBqXDf1bVE>He8&mPhOtdoWRwmo9k|TJ9Dj|Y@vvJKcdQ(u>zZXp= zQ3&4N-CgVMn>O9VGnlXPH|KW-rQV*OoEa3mzPdGO?fN^TQYUw(Zw<@gF?4Zl6=+vh zDHl~_gW%$NH80{k?2{W9j!YzlVhGubvm05lFz$=%heM7$7uOF*q6{FS&<>E%Dew1x zd(_`Q{IdV1zkl%M?!nQ$j8u7lu(!8=><_*?KHfd}^4@@RFj6g44?EAiRt#E=`PZL+ znH}`6KmWqZ|N8SUg<=2t^DibRM81PQ7He-%-0F0g#^nIULN_q9xdVId)!gemr#zIZQ>J{?@mzHYiq#kLi(WH84y)?7)3!{;gx6~7tt_{<*h=Imk`Xawtl`$AIj{; zoE7g+;UqBVV{dP7@96MQ{JFQcH}mKI;r`+NC;NwA93Fgr^u^~#UwpE6aCq?f!6#sE z4fFpbp$w8w_7?7Ic5)j6Fvi~_LdDGhV~jpB6jBitL?!Qk*njZ>hyF1*O(=^dH|lVo z@mpPZXT8X$+J|8n$>0x)Wl#y|14Mcw6!t$P12jl*;3Fa$*IhsC_4kkZhc5sK-~a`x zGK^zUgM^47F&YtA04N;nac&7kfBjTavWl>bok<TR9~8CBT$@TlV!GrjkO(K|)|qG9mGmi?v)1pZ(5T zr(VRV{UslC3_aw(06?vl8gwrgo8+2^9OkD{kWBKb>>l`$M&UK&(nw#UMBgJwV8b1r zAj;q*&RsHc15QS0?&9$w(?7~W5Bo5P$57vz)ElD-%=(UE6rNmPeE;Tl=EE#gT3oxC zKweQSgQ2hph}A(N1iXs210K;JAAL+~;>wb|D}k=~z^qDC$Q{6$E4)c0<};Qxi_hHT zZ^&26B%=~x3J8h`q9|0sS!4oL9E{t(cOeU1Ek0$*9d1T&fi%3;xb z!>59vb4ehH`Kc@s1TI2w3MVKyg|x)vu;RVrdqjJDLyUJXe_hR2DB;~T-LKs4+clCA z(4cjIF}{}35-UPCnkd>%t}n7i3dDt=06oBvr9qNlI$0H@kyoy89k>{RB$TcBJ6eVX z6futd6hOXId}-uYibtW0diEHR4|1U<5oOhD$wd_GmFFL6($tIyjK}}^t|o4xsfO~K zw52q3VqxXf_u>ksN4{Fne#+G|SwUbB_i2$JjfUDz11(Oo0-`qPVxuNOu_H^srK_~1 za$Hj19WHegmuf!`alp!5U`){KYf-yE+$j=?)nvO`lYAnD6>x^aDY!`jM0a#*If)_s zUpVxliPWM?W};j>mR%A$bJGJPI7%phj6v@Mh#}+b;|AUzJ_@H|b(V1I&vI>*pZm0g zJNesq^5q+{8}vvptHd}HY4MLM8N zPB&1F3G%ZQ)p3Y12Ty<)F(i~9;ns+8ts5vu4Efn1#)ctA#V2R~+ZJfD|4p@hc>yb6 zz5Va#^S#5P8T;SopYI(y``R& zK8F=GT~%P17H82sb{3C93?1Ih{IwhN)ERkcH=9m@CGM@tgbEz)&A}$)USpAJb1Sew z$tN(of+d(i6HGTT!`f^YSg|?wIjpPc+5$&K7N;V$20CDYnaelSfoFh=&OM;e zLk%L;8uWk#A}!yr2OJ`83?kKjW{`{!yFtU!MKnRMyR-$d<@v>JOin4# z2RCSl2ns#q=c?%(xGFg-}b7< zJQzq_OS%klHYyLZ(DcF(aPdQF*>u`o0F~_lL{(3haX3Z#4K4eeiWez(gz9bjs-{RJs0fRqJZFa-%0GWMC5c zC=f@V3Y9H@d{^TD%?@}65G2%#QRT={N&9}}C6iS1Jd8*|^IlF~8|jC%>C|wdIUM;$ z#OuKlHD&M~MM~5PHxtwe{HO`a!RK!(izzn#82s;3j`um0LofBQ`T z^!ksl{-b;asv_E&!8C{X#a@|J6Qb`L-5XZJ^CpIg4l z=kT$YS($qeFu;D^_&q;`38GZE%)Csm1d0CYQp?>*FRS>0B;b31f?*#9!7H`{Sf4N3 z?|=U?tNXgC|EzmSoAjkxX%nwPu(IKZJ^uagUsg6&GQK1K*;HIGY>WsR^1Ca}H2v+(6S~xml+VnI5&}o%5jkqkL0t%mjLmyt*dAq&C38qV z6Kj(fDJr_OJ&ONhVVe{TJw#bVTJxqSx&dhz4KQWWDlLD`ah$Ibn!q5KYPNyCmMjt^ z<@5+r8hIEp9k4IsN_I5%sU_5_p?DTinjjiWUa+)r1if)O$UgY9M6QgJR%lCQ+Kd0n{FtkoYNFB!1^yr8@h}{ zcbxkjM-kOl@zSOOG-TTQK(VPxkR|j#%7+;OJmHcDZ)%1mmsNSY@^5L^cr<2OItb7M z3P8oqJ&v(A<^#%PAxVVO+Ic2rcSC{Yp3&c^J1bltdB~S=FtPoK6 z;7{g{5*wqb5Cu7wMbV@!t~^Otq9XO%aI8 zTR6-xwPN6qRS&ZXZ}_3}#uz=w)t3iPlNPzxdQ}vKq>Ln9h3a0c{{R7gA22D(K`?=_ z_V)3jNgPqk5cq(mJ75BR1Txt|cXWe>75M0vE~1DAKEPq+F*)HZ6`mv$-qZe3jxs#r zAm9T|y+x2;Sn)nM2>}w3_bNAFf?!DV$4Uj~OHU!cr(rnFx~&-Ih4MV20rmtBJxqdN zryR*IuOvkqCPW0oQs`3Qxh{hNN-Po8J&q&33ZkV5SrTSFarEKK&5t$=s>yE@ikT4< zi2csa4NW-Jrv*YRG44?d`y^GClr}A+9&(u-eH6m=4#2}a)rN9`_!z!|-Uk%=ckgbm z)wLCtUq+cTUwSuY0}``lQ8q9!`dg{FQ(On~!Js@U8ZSFLj+N+XN>8P%I>>P@i4F|N zxx?9;8dSzS%kcubiwbnjb0k{Lp(sC1`3-<@8mvw7@N!fN=lIDOL=OnyDhmZajfhOd zWYo1X#)v@Tji>YIkv1%Mi<2LEL$lGrJO!M=O*NJz3=qvcz69e@(ZLu#AOOJ+4~R^E0HY}4 zqy^F_1QExkM@&&AQeK2v61$HA+vXH(JH^{xMdIa)k1_E9KIOe_x9L%f9;I&mmB|TSSH6n^;^|I-TWPPLrdVXwGc&SKr$$&i= z!vPLNgsB}M=pYs%lDJdQT=^vQ5ecT%DzTZ9Qn|v#?Jm2ja-2g^7D{g(aWbpUiq@G! zFpy14Tucd+t`0DxC>R#h1TaI8^2RR|P4XZ`J4M+-o$6u6S}Uzzo))H^H{(X;xI09I zXonF4nIO;3Y5`IvSfu=(OFIxGZQ ze5%Ohr_wH?2mZS1a1`gR^~pM4Nidb&M&Q#=!7C{NN;|^q%EPQ7wb!0T6$&>EhPDo{ zHbWUOcT%c1n1MVBf0KIvR$bTzVarT#8LBp+l%V^pve*a4?OLKz|#*SfGw3Gvzx zJ>^v>gtKhQn9k_6dMa3)SAR~;teU{aTW(3#O<<1E&CY^$NLpnIHAd8x5EA1Juvj&r zzFqdxNu6+R{p1g`i&%2?sP}urY5_Byqk4@Oc`%soL;2b?jWtuIri4We#uMg7c|)ZP z3RS9ak3^Jaoi_9`h?3BULdOSAdCgun+*DyDy_Q5`7-8H^r5mA%Nw2nsu2fbLFqi@w zO%N0LCKOkdSJtO0XMU3l@hmC`b4`x7nm~?~^58`Fdn3)6LbWS}h}0438zM{*BznL; zpKL=1BNQTnJ)JF2zVZ?ZOG)?i?cr6eXXD=gPBA&h=wpls7GyIx&f)qW8XCx9g?N4l zffCyC0$_#b7(w9Em_&Z!A@XTMKhj3Lqfk%!Y-uk?{!Mu03tSaqm%uiD?S#_SEJxRb zH{mpoA5mUw*0_?U3juRNdSNXZ+W*vxLYhnv`7~V%CGJo(vjXWE$~|3(5+>H9p_lxM zMFfv<2t~u`)6#P9Kcz|JWTH;%q=8pSMMWP`Cc4qOCR7-O@LZi&V#DUslSx!Xa#&yM zce~9;VinikYS@lSCVD9hhvX>h8nU}JuQ^+%FH9+Qi9Y0osLqqrCyMv`DuI)DgdP)) z7Y!YxM=zLy0i?*+Iji`DOL*wUQ~zge{hp~PT^v$Pk{Exg2@L{a0YWtjWDT-hrk~7Z z)lv%snN$=h248liF9xw>uc~$W8l)$BZkb>6^?j-BfJ((KDG3)V@A*#M=QQFC?Pnr1 z7@#qHz!9OjsR*(tw^zuqlDrGbnKB}La!vG`f+$X1d|OBDH$i;VnqTn+t*rH~1N#L%+! z(vcHZeYw|Tcg~)a$L`{msk^_4Vk+{^tIqBQc$FXi>xShwxOwU~NBr2l^bIS&_$4yu zo^kSD(n&_J6#BGFK5$m@ZyMfg+p0A|W~8-q8l!TZg0#K|2$DuNTzi3oarC&0hbeW> zk?Tej{#v;R#tGpEn-59rtYMgUIma~h2|zxeiRU3ohe=T7!jwS}g`@Np!wjB7&qFbz z089TNVR4zU_n7F0Kt}JyokbA8T0N$?StT%d>5LTbIpGqFB#WG&H{o{5mF<9tfN?|_ z|D_KuhG3sB+(9a^WTTN2%q?U?A1NQGMGTHJIXki(t#+d+-H~STO&NH^$iv8F)OuX{ z`Z6H{l|%KBGBBl+ZL&gYNEyHLkCM5qEw9t%;gG#KXkrubB$*t8qt8Eo^LZ1WD2tlt zM)*k_)!suiBect8^z!t3M!^JipF(}9B*$6jdu1eVLe_{2M&ILti`fw68xbd^n*d*2QLbxCv0tS zl41Z!RiVuxIi3h)C8b55mI-8Lq0E+1Exmz-lE4y4tE(!lGL4x&W5 z$Yfh=xQj}Y2lXolIrpZ@IU4-t;ESU#m$j_XK!$DqhF?~&E;gbD+ZCSgAr~oBEMXSG zca)XU=G`1#KSJ?hQt82y*ks8$`-4NuVCYHb-_p)FTPBHAFa`SuJ7A!8x3BylGvfW% zkALfzObJu)+nt&5VhT8xM8kZH;c6(O;3cFJrSma8i#TLQhfVmUrSLr?%xadyf$|?; zHaaP<^z`b1{DK$}IGI3(y)149$sb$E1d59K32(YhFR3Em76@{JKq2;0z%cC6K_7A>>y_vB<-n zZYy`wz$|vIA*E7ePTnDW2~>PuLG3Ka-2UEy0n(br0n725C%ylMy?^}emHNwG@3%kf z9RKaJ+&>$wjTQv9{2}x#6iz{rze}-dM=%f35ZsXj?SQXgz(wotLUF4%Ei4L?Nz))W zm-{Jiyveme6cqiNlvE>J)`E(HHp{6kr8&Z!iwF(G1p$Up)K_A1--{-@H4&qWkiVEq zgGb~c!@^e!iaoop|M)70!GHT}FGJt{-#&Z&$K1EqOG0MTAbInAazh2hXf>3Q1iYb; zJUb1A_qWU1^x-e4}zG6Ic~| z#Vb5gSjG#3nYYD)3HPH3#NloX*;t37mokjJ&e5EAXHEMv@eIHtk)DIu0~dK&&Ao0g zZ8rC=JPM1C4eRtT=gM6Ve5z?qh?us(_Oj+bt!?E_sQx*RmkQN#D42q?tM?}tm*CUg zPr>!c-P;}Y^hW*6ISBr9gBO?dXw0Rf5L&yM3a@ks{bFVnKk`J@Afc7qjUEtr!02&T zIau`giucqLA(6j@?*2ynQSbPlA&)ZhrV$8oOCjXiM}h?#sP{ZyQ?WWIqzOUQ&#oot z5CJUGm$dFPi{pXGELsc3EI=?6QcEgoOJB~4XVVgbC>$XILgaA@Nov@qWyWw?nr`lY zcC%aPe|G<;*Z<-F_wWDb-~R8*SFb<&e>?y2`{Td;zXf7sH3yP{4bcsFnc`6{Ew)(f zsA~FN7jP$izWKsvEagr0glO`jwsxDOZE1^&w{=O}x(xIA-+UBKh2QQ18Q+RfoB!=_ z|7h>c=QH`=4n99PaQWZX0o$AZO*Ykx?Q2WomPU~mzh#Nr(%zh>##^1p9S zO12qUWIiZAO0!L=WDA^Yl$H{mZh_+_>)}xF(4N3}v6}KFLw~LV^}>PAeh^?f1`vd3 z1Q~unKxfC3;cdArf>MoPOrkL!$mqQ)@q)C#???s`DsNBD>58W#!X;Uaeg_01`cMQg zQ&E!#eDvB1MUZmy5H_TW2-?ND+(lpvLq9+uj;M-u1q0!s%;o$CM0Y?CJt~!CIx`uN zEYkTb0RLZaL}1AAFk3i)K62q;E&|F^1POPE@KSJz@LVFit|h|5!$O>(EE!%##nUGr zHql1a?li_4y-^oT?#{{eJmjP@3-GG9yiLYCaZ_EG<+}Symdy`}FHL&cRo!srqv4Pd zd1Ggj^R$Y5BrB*`YE(CV{52Pd5d%cDd~q{HgdV&*%0&d zRgMDc_5*{dd`C-Ue3`+_3&hNI7)eJ@daUpKGsqx#C2gP*_xPy-9+aF7l;vIn5r$I~ z$hjW9$~IO-w^MEK+f_J0zz_1ZGK*64qc-8ER@cYzG*@*D`4K~*9^mUHBg+ZPIX_0j z!o;8mJu*9N9bJT?hCnK&?nNNqX|pA*c)D`KEyaSu%f8Z05~ooJ5{+G(&j@6>98HSa z8|x+Lb#SRu$*;OK`?ee%n?e&Vgm$?&`06$ zgkWRfsxruv1@sYmYzM>y4e`evK$D^PQ}z*&Y*@CGAP*KkDf3|L1Tha{a!-L4=e*-0 zZI@9`W^keuA>58u10IDDL0njCv#lx+n9$KIh7}j-IF!8wP+U>d=Z(8V@ZjziT!Op1 zySuvvch|w4KyVH2?(Xgm!DT0T-miAQeQURBshXL4&YhaNQ|At+yZ^uLc4B{O|IAo6 zuWc<~ONXXD-H9(ypb6`L;{@|5TlEXav<9Pvk)swIStHnx=yp}Adu|0%G$=hyZ5 zvg!A^kBLw~^*x0-{pWY1MP>Y-)-%qri=+7%$SE&XfYFM$*fuMTZMsNJUP#>a@Pn}$ zB6lYiPeH0bTpQ%!aP)h0LO$rCg&-Y8+tnOL&=rUnu@2!M4>YCcIF#T!Z|rCT?y%g; zzWs#Pmltd%*L&7qn_7X*PK(2IZp~Pna%v4}ndPz3Z zMeqr*43%8ETd&KK!z%1tV-2~-G`*pgMCKVvNf#E0ZO=EQH&jb`Z2~;xBqy362L1*Kb^Y`ER*))eJGRzYA*65O}`$(IYBs$YmLb8a$EQcEW*q zzZ6Cu10=6lqFC{cl!xzT+07q$J6x%|4Ht1P3;A0<_h|iSyE@e+#nUIBZJi^U3gGFF zd!X5LhX{PL#MV;R?m%V#;QdfD2LblwCfRM^hHD==*A&FdJfD|?bg-h7UPe3YDbH?e zE|k02pO=%nP=P3QbF!i=1vZXR=HXg}0<22*IeXM!L0C|JPYXiN1Fr0}TmW!o^RhuM zAddg8Y=9fszm=W-x3c7*t93%BCpA(>G1eOqwb0_+l?*>z~^IyjP%=CezFSo`+&6%x{6FZW~tX`=3_3m)J>zyozx5$1t@bbBYR2zLMnAR z-i+lg{-MmxSuBhSnC+qd3YaB^mOI(Dz;loaQ~P4Hmkwk572C-XWz-jov%q{4-e#SA z2-2F^hY!qyXXkhJb_aI(I%lAF6#Ui_0K`b{rwc(wn9jly>Z0e(#;;zE@j%#Z|aAHz7NCPt_6U zpP1UJ9GE$eHV9>v^W->cvLiz0v~aZ*UV?rxq?5BZ84hxCf8vn?EDB*PXs33k^^ZF{ zxBn==6rlu<(R#`ZmvE<#67>A+$oZbTv}gh*pE$z2Y%NIUFg$(y9GYw4kWnL`F==fC z1y~9uw~4r1^OSY57tdY_zRUC&lp?B;HJ|a6<*bhQwbO8ySr$Uy^P}odW)0PNX%rPt zjF3JRC_;aAFL7%|)mB-|tRvR9-(;uoLD=L7OizJxeE4+y ztrf-+Js|lCXRVntb8-?XZND%_+nWoF1bgM4C0D5E>3Yo|OP7-|iVq`^dWOZ5u+heA zqeVf2m`KpEF6X4P+Lq4ag0G}t@b#7q`R2AA+XKJeEF0T|ym zDZ%FyZeSJRX?5M3DoE1KUSItNaC5Szw}B;ak+f^9Btq0H8C13owHn;WepK!GZfRXx zO25D*H#$jUK6WM(AGh|@Bzr;XWhx%AhlW^GJ$$v!foNuVb38``7gK_zk86CZ?r{$ssP-F$ z?_9^%e)bQA@32OD!wyuew>Z@rV_=_;c|o3kd%8oI9muhOd?naGbgGD zJtwLFy<(W`LG4oYj2}o>k7j48Fqt95Cy=@P z8O75#YhPV0E4|s_Z>|CS`lo|6$G_{*=RkGzHa<#EV19MBveA3JFS>v%aj3him#UW_ zZsq9ct-q&P?#v~PN#Dt`>qq>)dtADbyPQOf@^|;VNt=5#4=w)makSKXVr*_MssG*a zyjpXtu1+Ba;XEeG$Y)nFaNx*F_x|v*+OWUz>K-ck3>=QeWf>|ut|N@y?wM)azf_%o z+yiGB;<79aUDsuz@T!CP?Lr3@fOGV~ckXnpXX1=`?Y7`xe&exILnuZhwH*D9drL(z zA&MN1a=xU$2`(WEPzstHpFL^k`%j)FS@3 zn%#=PujYv7E!GA@hTzf+c*~Bf-H~fh5BMB42W^3TM~Iaz_6Add;MNTI%TBx9k$BK< z_*^yHqk?#<)m52Gasy-1&4wwL9vwj|-+3>hLP1!>u89m!kCGcFfW1Rv!Rbh@9(o28 zzCi-(62Drm>rQqS@oMHXPv(`YY4qb*($bT8z;)8?-8>7?XG<$P`qFbkW^;~JAmM3a ze9m1g6pP;2n>hXYvhl+y(8i)xkLV}2jg6*`=}TSNK-xk<%SIi;_uO$7o0Vxwc#3fm(D!@cV!uOp)iW)oIdI6jX3X zPdKH~FrOf-yB5w(UfaJF+=6`eX!Hdm`Sxa;5kmRN4kxzY3R57tarb!e&>>c|v0p%u z#IH!ZCcs~Qw~9GsvBwgqGP1dw$9ftnla_8g_2$93W^WrDa zQpHFtLX*|z`;%ie6i|d$4Wog*ph-3)lAK$~>c9wws34ezs%Yd#O>HZ1Vwj5=LVwzS zr^1c$fHhS-z=%3^P*k!0(?Ny5t4IQn$Y=bsKP@7|oDWw3YFt2GiBw)$vaqW51I!F9 zi7ShAwg4aqMgQDb?b3n6E-pTM^Gqoj`6bnxDPMvTUyr@C+UXQ-{ih% zZcSoa>oBg&`gEVYgg}WB0RqN}E@AU0P{j0y{>~yLVyB1wLX$d9mTxoQne1A92D9T_ zV4m(wCSy*o!#$v6VW?YuwS_JcjfGX|78nSh%?r=qQb%qh3BrrT-RjMADn$8vdSvS9 zf@Ha*{`lWOx6cxI3bP3UxebFGN8-f!>+YW=A&*I>W!O1Uhh7&xPD7xj{MW+92Oj-3 zx&Kxf&_|Felb%vS%?WkTKl*9G6*!L*DcRa`}_ zUvhgP?h0y##s24lvQccEz${gRn7Vh*x8S6v)mUjx>Ql!4B#$0}Rr;wpOVrBpJ@f+@ zs3|siCWle9Qo3@mQaa1fsue}bvZda|?LhST?7U6jMjYhzpN&{Se;h&2d}R}RW*R5` zsCz_)V%vm*lJ%e)Cp|lGWuywvA2WOXbbk@J*(YiLXR~+uX&ORkiScW;ovzn>4KGdf z+t)ip15aG21pD*H_vniW!s&)$wH<9X`tTjXX_ebVqt^W#p2i?$TF%B^Ud~1#R-1`f z?*EL1Pyt#!3&i`NV7olSbgrHbjGrHVDd^sQI` zv3%f)v%$#~ZnMFOI^aT50F~xm*B^ zotcG*Uve(=LFl^K$EPAo_?$BfGwj%RA;>AM9v*mTEm0+T`1^|J!eO1@=TzT0i3sRmyFG^ zAS|s;1lIpv^MhSt~J<)8rE zJfL$p5Aq)e>RZ5=($)CV9q_(K2YQo98BW{dOOrl8=kc$%W#{T-4fLZ&s{Zrl)`hwt zHjfNk`6`?WoG{N#bx82ys_iKa zm{`SOg<-E<^>R3ks1(oI8W=UUOBT2Ty*@#r^uwrVc)fh39j?qU-8bk*r|ALz^gy{Q zK$IvGboQn_R#f)@$+AehYc0%SUf*lSN-YdUcnoP@{`jzYyHSe6U{|B#%>aUIwC7#h zjmL}$TFCar#zrGSEHzp_Q${-BCc)F;+hoE0+FcTB)b5gcGkcZ3zq#tm{^fm@=yUy@ z`;q$*<`=nY2F59Q+fWx*Uw21WPiw`!%ftP77nwz99kqbq?sCEBR7Xlt&XL(4WCSMy z{$B=1*^uQ1-8t%#Zp{B0*+#r=*l}{zMlf>KyZ1Dr$84M29i#Xz9$g&T)k4!`M#sBt zA)FqC6{%6db%qX(9XVL@1P>#K2;%H$8iwqpSmqpcfxSKqzX*}Il=O52@FQ3P!i9K1 z{)qcJ_;&$Yzkht}q`h;zzyDx#M~=O`^1OL{{n63!ZVbegZ*DGo(jGDK6H~0py>LA# zx{T1Qkh(tJ#?Hz7-k-7bK7e*Cwagyyxt1%UFIt%jNPy_9%0;jkh+dnkK6hZq4PNOVmH@k_&s-H7jBj*GqZQr+wqr)oLl%0Lt16H#fSq|UaSY3 zJvU^(9rN|!+$3HVpqjg-He;0xRZ$ujyHE&c+iS;Kg6&0WAu9iT*10L)H;)9i$BqLfwKO3bB>@vJd(vhh3I~P& zQ271}tZqX>KTiP0lf8oHbmN{}utJb$=HbysQS#s49{Kx4g1H|GB>mwi0}Tnifq&v* zVY^cu9UUtI$XoaXzIFaj13Nu`(P|)JP3aA7kF?eP3~^rNaS4VtO7v`Eztd?(*-Q*# z{KiM3vhEG++Dj@-{E2np|CIRP*zj8OxeJc;ZtRe$fGKN-5y-iRAATL}$1I5O5-%vn zXNWC^a2Vw02%0G(7$9LE+{<4Y`vU@+pQONX<4>QS|7V&16IcKUl0qI$ALpg{eX=@A zPkBIZpB@31w~o{gI+8En=6)f)hP`tzssLyWLN6Gf-lvAhk|&1p>Hs!sQ(sxwA|>RU z&Qp+deUhVmCc~U*!btrN-G`f5Nnm@v?fV$%8-L&0^FfX6<>3`f?vebMO*->s z5ps4wXo6T@Y$~n{(@=F56hdyFKj;w{q(Cf^vX1AmXI=74T8x>}SfuORPwOJqr$-MD zr5XMD7qw0vQl#P+8o(TPE^Gz}6H$wwi7HK#4o8U*e~7PgFPuPtF?=wi4(47jU@)BX zfP{YNtf6y4LfrTXq8(}_gWewv-SU&@c;Jf-5tSA!-VaAnB$4&ow9!hy6Ohr7xGaey zQ!N*J%>U@eGY;iVMQMOHUPn)Yg_J#&f7>delcVNQ?S+U+vTW6S8-^pBJ+(U7&Ap6p zJMH4?>KJp7KhIh(sI(Y?JK5N!jp`#-azvaH|4Mbg2NNdKi3d|NY?g=sEaSVRKrN%e zwsl>ewdu_ERWfA9y^dr= zmQ#;KPB&FA&u;x;ow+|l#|Veudb%O_(POy&;9F zV)pL1Uw*Cs$K947HuE?nhJ?swtN-UoX&wJ;l)OXOs2`$|0BsK-A%YHGn6SoCsAoc* zvXn^7Bq#(=Qm>wmgvn%eZS)7kp8*561&fdO(p;1LZ?U$XgR!a z0pP+rC50FfXuw{l4LHgYLZOs+{IEUNuN+xjco?1#wrn94UxjZd+5+QZ+FIdCSMF9q zt$xWf0O73|XP;R%G2a8_Do-DY_~`&*!tZb&f*FiKlPDt8w>^J@vONZ)6>svLu`t&& zL^v_Q;;urw;UpD=I>-FUXS7IMaq*F0?c+>vLB|P82fRr8i6?wp6mgIS!5R@J;C@4* zzXSsG`VU%xNmf_-AN5msms3@@@{B!Q-5^RXE-+sFPicUhr?oi$F2;=bhnqdtU+a5t zjA=}_GXt7?gPm{Z^{kTxPWhVxALpzNlo|@LezW?|iXDTfua4BX6Oa!#s6EKTQyz;h z&d+uC`l~T*nHlITjr2T}G+&rla3w;{5;?i6*@;yUQ!Lzi5Q$9PA>fGBPr)lrWDUP= z3XXzwNx87meYuP(jybuA>|X<~6{7=Vie7GEuxSz#V^FyFfJVHpUVOC*q6#6M*n6S27wk)8V83k&gy((w?r*tRy+^5u$>ZXxAf6 zB{?G1nvl=1aLjuQuL=6vebEUtX0TwX)IUA9zV2z9w>iY;ZzuCRJ0$?JBUny8vIa2O)8r#$SF=$E4DEQ)x zEv@g-gcOE@%Igjeo}y(!(y~&w9f}C(Sv8~%fxj=cW9D!TOfUS&?_!C6S9R60t+7ImP)@-IWEgl-F|^FrhS7%4?Yj1N%5&wxv+(g zc31^bl)BMa_g8gCD^Mk|sshntjSRKr*mM|ZB+rkNU^*CQpsZjEX8tOk(j1CUfP{ma z(rH(@HPlO7$Up>r_x2gy?eX?#z$Nbw$#(sz ze@;05@=4gviC61u>+{LsL-zS>7rLe1HF@*wwa)t1@v;(Mtxw}!C6*Z? zqg*lv@Ksv#a8=CgV*Q$X{$v6fE5E74-{F`{;a(X9z}gv` zow@o^a;HRp5in~Ob(c{jAROePF#W)q-f>_h=9-06=-?g!RRLvS6rE=Bb5cwLKeWiH z!Hf;x9vTpUNV>ZKy~+9qGk$2BFWP>3dUR)pO&t}5O#~u@VAL5B9otzlDM@uQ++t7= z{ZSkPf{&Oqa@vh?A&3^~$TSH#Foq|1`vgKn<?=@+?~f<52kWWZ#m zNqcl$^nx(U)IOa(8a@ZG?9KMNd_N9{Pk_t3_Un_m|Dn+`a5M5|z3gV3>PIBuk&Dnk z>$pYeA^$I3K^1&oQKsNsCi7h)&PD||Ir{y1A?-rUL=jMlif(t$fWd1}akVDpM)8=; zd^}BtVrX>iT~65KTVgmtG-WXqz%k~+wbYgdJ5BhTRC!q$&P_dA_G|#p9G6GGV7H@; zmMgTyaDo|jg1Cf_BNJJ~%nyAL#~oO?ZVktoctNq-FG_{N%8hu0PLo!4^@ljus)bbX z_TmUu*hRmY)M?8;GK+5#rIq-Z?-7+KvCWUmS|bm-oRsM2DdmbaqKFNb(OXL(M!p?S zvRg-%!uhvA8~oPt%Wz(28AU7zj0{v9q+1j5_Z{QC^s}uRrQ)1Wr@c-Em1{qBkf#`I)=Xq@ zA96U%tfQ#ho9CtrCLPLTG(s{e$j8`s=KwB!3Tn*7&WHsAQBb-s_2G4Mt4CMgSlctI zayuHv?Ie7e&4SgL;m}h+#LP%;K#>GRIHiiGa%ahv9>B4OK-ovujK%lKDY6n064xqt z8`I4E7ZaAo4r8?z=oi?#>RszjTdC7aD=^D-3MCO>OVG-RU`Sa41)0zOAO)Oaj**xY zX>h}lCK8xSB*3U0_gs@(r)gKb&jv1@)b{j_VDwOwj6j{q)xls&Ho&Y>E_1owBV_EE+(!G zGJ!^E1Qfke71)`}41Jhcv5GdZ#PM|B8wwbwuRw zc^dcBvUSzfB+*da!w>ESC1+O^QpNRub%{yP$=_ARaHwp}Cxwd50m<2_lc`G?M1+06 zSHG{MD0@|(rD`XL&arGdoL-Gg!NO_Ezg4s|EgG6OEkZYxI;JU7J>nvP^|0xi=v>34 zz)j8YuFFl#$PLsk7dJRo#5C&3RWX*NrYXUz4jQd>$$b$kb^jI9(!RXN=gr(Y_qriO zr)Tr~HxZJ3DAl@UyDjMpLukvC-u}h4pY~Wn({5F^g*6x8zl%OGghOmGLqJ28D%p^2 z!9gZrcphF~2-(Nk&s_24;&~)P^yciRcGhBbS3Qldll#kJ!Q0C7@4Mg5QE;W|X=z8L z?7+DR@=zwwiBmu+=H8b0Uc)+XkhH;pZ}Z{-u!p?b@OUs@h1vf1a@1C9Wl~oS;YZK zz74jHPW2IFW0x1$>=9@Y5K?#oU*=>bpxN}vJ{r)mIK@Slf%e0=TV zJG~8zwO^o6^Q+Oow{jMe@_DmRk~Hqg{gmOQ>@`%6|*SRZWx zoasoM=2HEY9UXpHRt6jH^PJeXX^mgZ1Tj0y;|@y*+VIo~U^1HaV0)X;Y_>$yG4&bt z8b}s@ozs|lES*rr_&VHB2@yArXoHfV)RT3U3Fa|Qo)~h{pc+mWa8zQwq1i|YubmEC zI54*o%O05+=kYt1OTyv#R+B#@zw@>}nV3|E6DJl;H7WjF#OCwtXi%~xUl5E7r0|yv&KNPXd7Vi}@=Hytatj%uMHk=6e9zpP zFY1pEY_95?Wu=x`02a?8UR+c&kgvtT6E#M_83g-R1K`ZZIej)-6C9-$Uyj|QPum3e zX)1UF!bjuiwZ1&hdQvX{!wf_Yjx~&aZ9^0+m>!>`QdqAQ4HFjqkj#j5x$f?S z%!m@z9j`B2#0Eq=1YI{dyz1w2o5^acaps!g+QB{8RSF*$EHE8OSZy75Ps?Pye*6)V zo#F_~q8OoONYnqLP@*myq=z0gXG z4V#Pw&0IiTm+GcX-)XApH%=Z-Q%!CEgqGW@p6%4o96aEr`X>}zmS*Qo02os*8Eb<=im%1%0?>ltxx9!;eKuRp{OU?QW&JlISRHA+N2RS8zz%KqFw5ld6vo83H{); zJ)4{{^9t+>AVw1kC7fR37ELoWW6r|2Z2~+={;8QyRoWJhGkor)0$fAn$B>2!g&Lx; z>63s%YzM%{vRr&o4%W8smWCr>^4eb`9nkDC^;Ea%J7Uz9zhnW& z906ppO{%M<{FEUPzl*UNUv5kwR!_;ZX&o7{@$|`RG5i^Uy(mUwX=re!=2O!|(cT2r zE}FuCXVI=*4`v4XN7@?oSIUf?qKZa!j~%>QZFYjijO{uI_zdQ`FKyp+f%X<+;dDuQ za1%(>T1F72=pG^yxU1il_KJp0N={P=;hCVO=(LvVd1YyQq1tiDF&RGPPSG|Qz-l>2 z*XAm+C7#qMY#n8dhnV}M4y4{`@e1rLyJYQRJj$3Jyg?d_wW?%I&gsVSAYRkPnaU$~BiQ=c{J)!# zk@Vlq*b(i*ND90eFHKGU-Hd0ZMW%`63*_lesJ?K(7&#DOI^8oVxx6TgI^H9B;cRV$ z`5PqvnWXK!y{{&|7LcT^ty%-CG#ixf6-g%B)E;ZDGp- zu`1rxI9FQbTo}}1EM3G>9+NH)p)sz#&@Y86oa0#%)6ugz%jrm8KY6z!#$;^T&_xO3 z5XQJ<+iXJh#S&gOXS{Xu=&Cx9(7sueYiPuemF+G3M#fVYIoMV}Chi6sd!C4=Nx@zx zV`Y*yJnmDb1_5xy4jH^KU9IlZ3A>>E!3Bg%|p`+RC-<1IBOHL^*6u#0M$@w0F)fj`Z) ze%_e+TKVBQrq|Ueh#5|}v5>P2X$ef3kqwAYPB}2Dz6=pV{-kgktt6?Uliff+@(+wq z*WFiV%fQHvQqRD=t0hy&QKvB*F}=LQZ%sHH&~bzE86d~^51CWx&I_ZC$O`-95Y8w3 za5WsNJe7-5k=hPHUZ_2!_*Ro)VD~e^%;+Ax3gyIQsW@>o*dr??`973q#e{6Vn}@Qx&%p!j(FRBj!C6 z?*m{Ex&uVwFiUV$?@yfoda5!C+b@|XSGLc1x1ccr_yfy7?8$uR%#8YUu`m$?Na9{mLC zpdc7_7K97OY+wiWBD+8#Mv4BEZfa;*C_gsTTV@2$Z=5hFDN*=iO{vXr&Sj6u!B1Y7 z_+^E;cn92Z{&u|bf^o38Btj^jFEOpu(GYp@x2_O22skjL*GfL(WL@0J`i1z}@H~a~ zQhW5%d;$Y6{wis=8*F0gwTky0i&@uc&z)qqU2oFzcTjKI^1ePJtwU*sBEr1*o%>@U zD_)of)?k!B^<#Y)PlUn=;b|SbZ$I8n4|lVo#v18+oF&pNCW8d3qa&s{rnMWS)2O^# z1>+!CMO2r?t6)nlMUZT_qW^kYp#(Q_{dvD9#tfH*WbLXly)h-l=)`SHw^+uP=%O-7 z=2*jXc_MrYpIRe>bN}N^UAg|g`{cikTC0>Rps@AXA`&jNytB*V7I~-?qiugDKtV0q ziv&D%?$F>7uhaBlWEsT5OP}|msR#Xg(Oz^WZfg5V)f2LOx*6McE%%Sb?B!{_misG{ zZ=GWTz@7-k0Z8iN2bdJDOH4P)I z)D|dqxyG5Dp-TpnnZru}d@k!hwTpi!WNHPdcI`O-NA22i*3sDIaxjeeIr_V%|IcEwqk+<4?DB0IyGQuD~ z%esa+kYC2Qtmr&Zg9lpii1?kp%F&XXWIr}hBQ0l8Kx@7O?aW-7(JY;y)=;a5U)aoC zc;1+4(!nUff=pd5IJS~l)v+zu`ETPvILPn5wR(1MUtaCFJd~UIh=%Xhm(wtgTO(@L z9OROFro;Lq z;4Fx%FE$YfdxUB4+5;QNwX{OPkVMR?jZ+yf@3Ek!2_K=2WV;v+(HKt=q%UYX)98ET z-3x-%l7X2bN=J$PGODc#ymnW1_7N97D_X-qJ|wX1EjZNv8U)}3qZ^?bq}tqFlrs}oEQNy?>1gwl`| z!4){ux7G=(hZHH`ZJZNa)f=I9^<%9A#qLh58I175u6G=8rD-4qVp+0(W6*&T6|Y-uM+A%9=!~XGFU*Y~ z?9gr5&GKirx2e3G(A#5w{Fmmz7EH+XU#Q`fyKK9k&nwY)z;VfpfH-*JYB~@A_*+wOQn5s6usGZeZ z-_PzvZeRj*j`q2wfrcZW&4C=_;cN{=Y3O9@H@eg1Z4+Dg8Y~&b0JcC#%yeh5pB(qFIiceNyV7p^65_NnQHus70lN%b1|T4nj5BS)nI9w z9XL&9;<{>ATEIe^4w@anBcRPTcKRPad-T*c4Ggi85)oH}Q}561Y(D5&vPCyHJ#LMB z4Al8tA?O)~Wu9mz+cbqLu0;w1VuMr$ROMVr;DFK7ng|$&d_I&~ju;|uew$e_E@tVF zgIBZ?@bT9!(Ctq#5#&geYmKMd<==IY1pXf|pE*ecFq8u|^uNQuc)4z(?kY#k;JkgF zFZTr|_4c9JgUxJrAbUmQ=?U)-7l%AwDbAYV5!)nOqy{CpKukXcgAB=h^(-2s228Zo z8Nsmz$2>&ph6)2##5p#nZn_k}fyRX6CjQu3Nh){=YlA#9^tf7aElgJ}0nGxmv-%b| zi!3K4-r+AlY!vKyBF;NkscT`l0ofB` z2c#Z`S9q|I94ua8ykkLlp+g4sJXE}q{hx@r&fN_vlY)h{{Mn~xXJh>%GRN`C8*LBJyBu=%|KmcU`8NF&M zqyp+&(3sG3)J#pK^H>(q{hsmQw&}L@NW0V;GN*QVHzgwxigcy70J*}$`FM6|HMTnQ z4V{g3T)H(TlJ6M4u2LjY-s*qzgY8ga&~}L(ctQaJUR9i(%s$_VqGx>yNk+*Kd^`I^ zSiB*gBT6kzaB_g^sXx7)7fGSH`yIRvi zXNf8rc1(WLu8F$9=f>O^M07Y@4(hN!=&wCqbngo^R$k}$YeKgIn=TLHZP(2_1J5BF zTIV&m&eaekt2T_O4z&8faao1P!vTW#$N(6CU}eO=pO%YAcW;p=EDyCRfbB4A3Te~R z>O5PV#K1uqmF=Pln!WYPAd#jvVEa}&(IGiu6U-h#Uo7aJ@Rq7JY!b*%@1$iX#xKp| zxk?dtfa|2ON(+AOr2dP>{y;gzYuV$q37%>POiCpHo@xd`yS|ViU69@ek)Xu@3U~x$ zM<7}M(cX5DtTp5Ct!;pK1{i%%z_V=PdIkR0WBsK`2_Wu)TP?-(kEUg>eZ;J#g~8ww#x`7N`CP>hq3geDi!=FT`9cM zRgJTQ`4=NAPwGYmQ#v4Jq+<9t$`n2pWxG+2mBiDrMc=>smn_=7HVg0lOBNOUBa1{g z|B^-cmb9Cz5lyLmwIhlxp>M%zp!4d-d9o-h|qD65Br4U>Epx$e{a zM|rxPg1|@@meMBg(6-eb{4ed0T9%0)NjDoY=O%AB$M=(suF@J>VzG#B;udF-+xIli z(HPhx}BY_2dDLErK|hL-Z{_DmCkp3YwbSt&bL6s*w*Q0 zI&)aN0$4~Qct)9eiFhn-YHdB9!=nn8-@ujuHS!+OyFH%S7%93A6 zI5U2%6f|E_hZyGgZlgpSJ*oG4o_Nr62F`9kia?81o7F4|1(OHi^SAQ1dC4w{gxv53 z`V?~KzAoU#)8YkXfk5a$=F|7owSeC;9V;9vMpHK$yp$yZ@n!bb04~UaO~{WyFs?1g zlp7~ozj$_C!Cvq9n|v}>+c1HpB@k(Wq zF3T0$osbTBaTN83m_l&hcRNT2$XE1GA_8mhHM_fbP<^bgtz|ts15*i)x0Bxc7?F_q z1zvXV3K{uVcsSp7kFU{dmgcWni2i&1y4GN3F+v=;^5t7({aOB4H^K*fpYHr>R-6+A zIrF!1xUdB*^!_|#oc$xHP{V!$1{(A+|5FLOJwLGkPe@c&7%GVK zgW5VuOEnQ1s}kL>56G|fdS%zb-EeFA4$O(jvSna#f~H(6ZF)g$TYwU}j#-@WIHxMc z`LbX2iV@Xsa4$_2{X8-UD{()9yYqbP3P1+(6^@fJJwZYKkjTj~JmVW36MsU2SiR5V zd$nq4E(I%_IpIQ%kl8Pt1OhokJz<8#hi{4Zd_f`{?jAu@Ka$==X`1&Xzm;nA!HLZ7!fvm4RIH^gVF+1rwS z3=|A>5k;!0T$Jy^uH@tT?ku1L3px#0lA&2djJP3BgqCM2rHhE1c$59)GD6rA>bqb4 zeoi*>1uB0>xxykd{3WVbynpIjeRX3Tul&UdY+p1<63vKx<4XesG8!go+gTwr#<>#j zso-u5voL`)%xVbpWyDDEDb;Hiaiqz_ZBt5G4d;*4tO5CAhss4O;S;NDAUg4@6NpZL zdx_BK0ha%N>_oz$(vcU&i(|0x$jvZ@X8^?ySTi=`fPdfw(K1Pe{`ize1cEDSkTR!W z4|NgwV$GMUuUn+6dw53e-p^+iokFn6#SvZyZd+18+r>wtp$q{~Zr%Yzg8K|F0aEpW znCMvMrgxS8D=6LIp>J<}!7-#kpTzzuR}9H@iWFt3;2_b>t72;LWj4tduWCje-?edM z6GTCLO(49`Txp>VSx^+}pHqars%iGi{n#nqjfA4HL$`T-!b`eyLmh(n$g_%kgOJPE zzZ94I0Z}Hc;@|I>vgVhSZ{cQYJmGn{kcZQ7e7LgH$jpG-YTxvym`omc+xl8F+WGU) zg>z+!Vk^X1vycJkFfaP2Y)nzYM9xylNYtHKvfDqsM4dantuqZ%)Hav{gw9v<3wVvjI zs;G047V|6YYO9B4TgtL0S9)z>H&RvVPzSqwNvPDt}2_Gt}NvmZVL-k-IfE7)1D?{W|6UVIvM!^VTE{aBb+7#Ex-hlW;uLZ8DQy$WpT z=>`cD2F5enN1Y;>ga z4iJa)Vq;sXZQUe_X1LiwNGVfkLt=OqX))Rq*?gg2&&zM^R9&J9uc*sz*Z*EE!hzavK-l8c)PWZ#|e}v zrQ5N?HJ*efQwc540}0cMf6DBZTL?@k?0t(B`(&-fb1g}AY}kK zfRYdt`5rBNa9w)1AIOOVvCLX49fVoo@(1wpuwSP;&J^{yO{uA_G`VfWW{y6OXXGd! z(UHLEl_gQTONIi5*N4pV!+}r-p}usN2ZZ%H{0GrbxDX6O-d}ZoxA-z{9=FHIAxskR zW>ZUlqF+c31z)Z?cb4yf>Q=YI%h}YY|MwY)Py}z6A&*KUCyh#s5M>g#*%tv;H7$*UzUaQv7U#ao9*SjpmPZWYpKk|Sz=tmERm(E-QE0>+%|=9G!B|n z5El~&wDluw3rL$H;$YxBI?RF-&C3Ji!6jy9cJ97YSNV6QmXz4fVCd|FHFfOv`F8}c ztnTPX;VqD+5XjamG@xv?(=vIs@*u3|g-PH>wp+2;*xN8Uq0@%J9o4=Cq5h*u5?jrWd`J36$P;fkJ3Jo~gVZ@8 zmX5AeYu1+a!>zpBttF_X=CuS#wmh=-hp7_`zmZxXq}759*-v~#nvz)7>*sZPr^yJ{)`IB zlsDV@aWCRi;91y@4djC7(;i-ebF>28VUV~My5PX?{!DQg{0^pJtv7t5wOE(6?M>l|p zg-s6(1K?fZM9R+TOAX^UE)`8b`MEW)`fct}m4Jjf6ATk{>4BP@B8_qq23TUI48%sC z|9}l5PN6{xR)&V{DWghbjuclS;g$T7X61qr0WYkNXnO^jEeR2JJ`wn>BHZG6u+8VS zevWqB`kPOrsw1H?C(L-sG?e_5%Im-qqfrtlLMbG`r{4yjywU*-(YJR?ZyoGo>)_(~ z#GT`4d;b<;y|d%xas#KO&F4{n`=|5u#>F$v)5|H;diK_)u8v>*eK*4_^V@y@F&I7n z^xU_opJbfA$DUMX13dcAP{n3KetG<|Fdr)E(EAaGIz>)agfgJ}&~kp=D!H zqf-<}bN(mj!#_jj?zv-0*oS>1ppPSyU zDM36q2}oyKghdI|7e`jkzB$ug+&ki@gHGn; z^AivPF+E4fyltz-K8dvVT$GAI%z>&U3paV_cpc=Nu^(a9W)E|O#=dBb)Z$8MRe{os zc54lJ+s3`&CyV}~%(gf$b5epiq1lZWtFS658yEgzqGue(d(CHGj;bZ=hiDr6ZwuEG zj=`|4_|Kf}nhl0wFFUM?ZeVZ}D-bRq*kBJjXlp}fyr$jq7>w#&?37)}ISPm+7b6sp zV6%iucp!YJdwltjZo%y2-`u$Hf3S~{R{A|m!I=58bejLSnrl}F+gmq$;i4M0;~DJk z=lGC6+%-gfu~ja zG!U+?P7}ZEeyCAAD6w!{$2=s6Vrni*-OVcBIi{sdfg1Q=piwz06gr8POhhZn;I{=6 zEzh=$Mt3dTeaE`+DA1ORx`%3pgBi#wKh!^>CcOoP+ada{f=moI7Yq@k!H9+f?R&2P z_-&6U=9;4>xg?H2x1^SxXP6gA-AXVk7lkbIC+>hlojUYDg+>qEDJ&dra zCyC4SFy+17|KRN#d^G3&w`X>+JGO1xwv8Rzwr$(I$J()N+qP}{H_tck-|#l+$;oNc zr0Gd}dg&AJh~*?>k>i@@-i?T-A$)0ra;Y*)OHr9`t)wB+{AOe;vG^3;cJ? z$BKEAaxM8Ve5Q9_v7xBg9WCPNd2QI@_TAdF;X`HLhUH3SUskf*z8TZ?UjwG=f1T&R z?a0cZO^c1#=wi+2_Bw6^PB&@H=6$H{n>Ve0|BS@H_}*>h6dOFlO(;m3=Nfgo%96V^ z9V^iaID=_Rk~${XI*6^L{vrUr`-Kjc)_mb17t4pp_zVYZ)*C#ugbtt?`;YZ$6FH4Ih#udi72b3zA9!w{_! zvP(>)V)lzN?-; zMle0s!pgA8EK`%bB)nB3WeP=qM#$Xx^kci;z`z;OWY=8Cf%Bt@;@1p#dqCxnh)885 z5)K%NAxh?Z`qs`&O3;Z;*cg*KQ|L^hsB0zjFb_qoGEbQ^HuqDMJ(74&J`DPSM!~Z&b`r zmcTZ+usqB1A)J;rmh*I>SX*-@ACArlSk9YUbN=_>GG+gpH8(9*oRLA$W#qZg!c8BC z0_G-_bbVUp&ZL=hXVR5ZNtBgzg6EMlXO(b1bWI+X<;}dT88(kgV|T(QI}E3lF?6L+ zbnD|vrv8*v!ew5vgTInm4W1@MoFo$*+LIlVok>TAZ?rlNcnjH%Hup4*RvRwl6VN0s zkc9iHKK`cvx;jW%UB#Wr8qSu_U-}V|Q843?DBjRuqFn7i+A%SSF(C~bgw6#rS#O-~ zF1+YjxRJBa|3G9PuI7Di3slR(+B#OJSDhIjrOWO98XYSReBjP6`lWrb88(@vU6BY^ zovQA5w;o*Q@mEzdwErp%T}jEYmBy+Pb)c~{xS=c#91t_41JZeDIaCpZ1?%hA!@E~(Tl z006I(=aj3%-lwVAcAe}Prb_20KRt6Ltc-*xkv4h1e3Xw2B5u zDchMp93E@e+Nrgb@9O%QwUy7Sm*a8&Z-vkP(8{!H6?f2D2jZ(l@Wwi4bx zSN0|HYgE>}t$(1sw|d^MN)?J=qbC=&Za-}voU|Y_BYPlcZC2Y34%VgR?VYu;veysM zSVg(44G01leV~??zXqGcUq%M99y{bwBzVz@2r}Sfw-)qB>lVg1UND4&nZjQda9Obo z%RsGZAk5f=a%Pi|M5z;dPr*IR#tTAOM7n>LHKrt5J^U$*uo+-gh=)IemFghz(PBg0 zkQstA33b?$QF^~t1>n4H2O(f)(R2$v_mm1n8cvzxSjX{Za^*UukRfAuIJq`|69o1X zJe|4qAcqhb7@>VkkG=bT%xZrgkB!NlZ&v$!4UfG8K62<(zFXfyU-$9K%c-dNyq}NO z@8|n-ZqM-e;>{e&zTONE$(&y5jfu+3zdtl(hHctXXMq?31P9vFg0#olBJy1XM)ECq zIfHlAqxWh@uaH|ObB(2~K^}Bi^hy`??YlQhO z!u|{7V%5&&AFNcfBUaB$x~!usuFbB=4vG5mwr7R;!Za*3f+VNwUSXaAA~(Z(sS=_W zE~be#5H;F3wWg9$cg%nEuDSh%E-?locfUttdE*7|K;GIv$7w6Cga#_CiPHR*mL9t$7Py}l`~Bi3iG2Wla` z(ED82*u&HSnJUe%MdN*|Y7&Lpq^1NdSO+&7p=oyUs(p*nT#I6-UMJpInVee0qh>Q*$*amh-yBD-anhE zr{-6CR~O*@dGP4zXvq|7ovxWp_oBnA_WrK^ak8>9J9@h3jV4w*B`h?~yQ%A@`7Y2C zA58Z4Im~|4dk4876CNR*w%hfU4j!{lxtK0CA#N6*zk*A!kZh_=AP5AHVYTJ;VbR0Zu!QX3dHR$k*hT4Q`c7ZI9Ak?WTGEozWJYa2d^EXK zHt}X()p)2y!Zt3>Mb6pPNDjzE|Md|rj$wAH4W=+Oc&q*h88#9IOp^B#bZLRQ#E#cb zK?Ob#ez}+{Q?_KS%OnVh_{(|xjKTu{_PefErR9W!z*6s$=H8$i{Q}3hGrc_t!$!5XV5&%65wU7{QTp~aB zHUOu(1*m!g80&m}K6ir%3SAKRjm8CZ`+ocWJkyDZim|j0=n#Lm%dd z9pin9|GnCrk#L|2IG_Ib+#gnk8!Jp$_?pZvaaQ0m0AI{pl?FOvJ2*Fwt!H9nHVz}o zZ))!7C3@2Wps~E|g_+c5?I|3RPsNFefqtVlPYo`J*FuK~+9VB#>g5)`u>ghhEkA)y z0SVo;(G!-TjwAcCC6$S+gO$Ov^>1@?rVE3H)*mPQ-4#Gq9Ljk=fdrf8z;!0LA7U2< zsO=uxMYKE63*WVD4Sd5Da~N#-b|@nzaBbSxf1bB}?A%|${aMg--P@xWZJkFJDen5dGRB~`irnXnB#iwqma>OL%q*3Ao zN5XJE55889S(smz`XRuY*u(qK&vQGHn#q`XhfqSYM03(m`m9P`b2B$O4lQ{3Qnee+ z(3k2y1~@+q5=qI8>dAz4nPfz(Q+>Zit{S2dRvH@?>w)WryG zn}%s(i4dunKHG-{N~ha^!o3JF1}ttKPkZVvV5m}CtGFl#ZD#1JYo7%@Ag&@Uww7i$ z>kUK_zB)z#xZ&)(z++$w>41cULSEFo<3UE?a|7WG6mk5GrL+hD_?XGp+k3YssSZXa zYl@=K6hHG6V9xlasjR@HAEIM<=@>TqKu$0MdrHiD(oq9}I<74^QjH9!rNJEB!~KUo z72)5E6O@tx65TO!D!V`1Z-jDVTgQSeJG`gN1EtnHrh$^A8#mY#ZprtV_MZ5l*!S8F z>aT5d;#A3m{QRUpGtXgzb^ervHrGFU@rlnfUCFTuh0ul$d&KSY|Wla~a1p0$g-r$R2>+OLlSgmKD!c==S zb(6Ka`d$nFo{V}h=Lq>d0aw^GXB z_cK!q`Cb>g`?U}?4d~oS6k&m-g$nvDrGYB8CY=|ZwBS+vHKU7@EQU@a{*mGBn5FT{ zRRCiLSYm^Ymdji4%6e11Ern0{RQbC|4ryCI=2U46$cXM3D#tmdU`~!1us|{)td0EpAR=FbncvqmFub8!S$Y z6X6#A`U0?KuoGIb6CqPW+B*1beB*+zq-HtiB*1S5eRd|Yjc#mEwRLEZIh{$~V@F0O zi2cf3_%i>A&@R+6aS3Q>X*rHE=asCHXO*Z);^yBW=wr-qIDYl-$>#{@F?h}Azo$;r zGe2Wa7of#Bq4F1n> zoH%c$m;B9Bk|QchmYR=rjVijqmB|B|F`YFv%3D;EE5OIKFrH`uu9^U?PwIn(&17q# zC+W52JVfKcYHK`-+mINiAd{95_c0#st@^nwlg_0_ZOPHJU*&23g=($J-OdfwHP@eE zffkoU7jvbu*I?L|$pr+n4m+-m5b@zV{EzHqCv+zaz8Kf~bLc^o_m{Hvx_79_P8j(x zW?n>P7(oTCUWM&x{Ta=$(z;=?c>8NYXExtBAw%($s9i`^X;%dws#9B?uSRgv0D1N|BSUF&St#-wq11O{Pl z;)igmc+%5(8@JLBobRT?IgT{}5bQi&1={y20>WO_F`dFpH?6-taiMr^i+N~SwR>LP z*}R(Z3)%fuRd8pbHFebJz$4dXGs?M&oen=}bC=l^C(+Jb*;Yke&E;A=%}*$wFiRr{ zXsB)vqn$1?E@KH?I;%%QbxJ&tm~b4uiT)Cf_90-K7{P?^O#Fo({sMRj`i*g*MS(0(D8uUVXF|RtcevXKAr|uUNPgc(Z(dJnTm{~JKx{BB6J}M z0QcZJcOZpAFaphMzScRkn zHHMN|gT$~Azf2c_^ygINw-U4oP=w4sTs@no`dEy+K0VSWCemA7ljVrz5o+Eq3YcS1 zJl>)3`i#=X5T{qqb4DpV=S-HV$HU&AFg--nYItrU7&i)k?D6%B9+Y0&kojS(fCyEcPz@{2|5zw}3pSFv zRf>@zN%Er;qj@j2j|jxMBDZZ!d8gL^XFkWQ*fbgoy; z(NrgZls8wc5khUS8Tr4#IoJCkp*#{B@Q>cFRq0I?+|5*2_4rj}5JiRU=n{d}#D>A= zAamQA#ic;c4Qmh+;HUV@As_`vfqE}}sV@lfYH=ftn7`$C zd8L;E(Po&N;1B*I?H-t)APrYi{m;>Q>RU7BO&TZb^5h8q$2f_PcsB+=Z1#1zH?JJE@qg=#lIpKnGyVjJPwP3&J{- zqSoLr!~1AM&yO<-vC(sKK@!>w5nA5g{uF0h7iy0i-1uUs!mz6cX?p|N1QjShXyx3J z!ab6$4A8iG=^n(n8ms)QxVzLy)BDt94kP$*>Al9wLG`d>_@Wc?--%QrPhZN%?^%55 z);!mwb8&d-USB`wqWQYkPD1@&&RDxf27Qr!JVmaP=NbD2T!M!d0Bkx;jAx4AuY_VE zhl9e-lL_V~kJ9vpUNN=7(xBHzlV%4+x)A`eUb%(i2QfT^GI`*)=aAg2_xJ6eMr}c$ zhTpEYHUr#h=5aI5;vU!&mq?paw2nqupa--B6Cw4G+k^{SS-EtC&=|CPA31iUsg74K zxhSLGmhm!{kf3~=)`f|JH%gKolAqC|`C*``@z=UsXWUh7E4Z9>GsVG22BWUIqj52e zw*!jr+E5{bz;2p~#jHdK_DF3rTCkyJ`5xA)OGFd>EUCWn{G=Tk3mK+*t5q5Eby?l< z!ZxdOr6}Q9XTH_l)y0dvi1(rfj70^ticb#7R`%*=y7(kmSGEWxt77V3oC$l^OD%eIty~1yNrS*es|rp`>Q(XN2Daec7nm$rsUA27 zcjx^!cY2pn8-XZS^+XQ1lQ7XdE0Nt-#ci(;SZEEm5tA*!4)h&aA7f!3nxhw1dEuf&0CN-8`u z$_i=H%d4g?{06#lAKwulPX(s-XV*?WbLnNl+o4_$)Z^^c=GD`a^86#kRU@g2*cl3x z1Q%CT=|`xYdZZt~x22jk{9F95i(OQekuw|tD&EE#wZv+5NNf4$A0f&XV~nv8^?x*6 z&PKV_^TpDXTZ&3IdhpP>t{AQ~t>PmY*9i;qa8kkPczWfNNze6*CK}eo|7&s^)+RoS z&Q&tQrOGFN`UTs!&jOic?aY!@+LDxo$pp*mB==YC^#yC|MTi;PQW+zD#q$cCSrCzcp4nPhlzALng$0 zREL79zr3Gc9UrBA*|RFYK3iw)c#$nlT&t5&TnZ$Wq*{y0s=K+kG@7atoO&uX^0V&8T?VRw(hozUi!0-eMjR8>r{y{p0X>@7y%Zvl#jk#;Gxy(>`B)znt za8I^UA}@$mD$)jc-!a_T;eEgQjotT*PLR@}F6l0yO$_LwZ%oJiF3 z)lM$SUYrlL#L7iXW#un|k#~OprsUldgs(7#&OHhUC*UnAT=4>&4y9M69fCv9jsivR zq7=uE0o!C)A1IB-qy0taywiDQEQCW~|5JEW)=pu+KxMOKWwTjjacji?X0nUEwW5Pp zEN?1#ik&C?hRDf{lvbNIX51lmxOnnCYRE5Ko)6H|k$zy$8QJA0kY;j8;bX>=o(f z5y}AY-TN$l-tm~`IVe^uq!}%oy4pkX1f`j{Fn|6if3E?|A=RFpWH%GPf#QI$%C=Y` zP~Z<_=PJR%ZgmOJ> z03W;6QwD~v$VyKCXz*(@CuNe%4icORKL{%i_Mo7%N&()94x*D3vQC!@y;GN+q_O9d zwneNdO>De}%VB`ip@*Yxn7yu#tqzo0L#0q{@ng}42%8C1C{R&8d7d=<4f(AGMv6Vh z(i65=Kv~=~AzI|{Zc~*;--)W7vW6i zO(rRxtA3|~>U|IFXj(LI5t7%qnpJ`#bgbDchnlaFfYEZiea$9~G%mEJ65CL@{h@64 z9i6V=4(>ug$=kNqS!R@cl0Z=4O)08Rt)16)sp}#;;){(vzd$0N{#aZ-L$_W_lUkn) z;@Ay5A|B)Lqts9thW^=9-&JSSxk(q4<7&I)2)=&Gi3y|xpcE&tmi+BW@M&je^!}VK2xQ!RBEi@E>$3| zxw@MWB2XGAIy}?yKBz5o^%i4yMCy(FwSXzFv5o^1luCzu291c##$g)Hx{asL?~d)Ek-1r`< zg#mR9cL|rX_iL{{pcu6L?v!PhK*&xJBU(FhSG{blrJRu(vVo}$KoktyydzR-wYb)x zizMJo+MWg~S4I)HRvIIVL`jJWNtj9+OiSvaZ71GazlJZjzVNYi z;kl}MV@@KYG2b&s59TNoHkNJ<7F0vWG8itgx{MMbDtD;mw&Y}U7DmlslibA4U+5EK zwHbsk*d@|q(^zYuNceGaluMHqO+zScxDBInbz$Ze8PAsQ2D5zAhOFhI9Rerip)5BW z?t2*is>k2UdH@M~-t+VPdXHWnxUvOHP+|i0?7@ae6ChI>p;x6ffQKI=r5`c4t^h7Y z!O43AUA7xSTEj-b9JZBwfGOh;x^pkBNet>U4-}|*SpfbGl=C{rZ0e(k3p6tA)p|(( z^*VPV+Lr{*hE%NP4>p-PIJ)No9DMY{iz;y0Wxv+FBy0%*0bcKnPI@3mOuMLM!QqfD zOuIHzUvhp1&DLN=&3*azU`TiCbqc0c|&Ro!ri-rFDWycSqEyxkuiXa~LvOHiufq*${UO+-P*I zaHrAk%`d3xtTbFnZXy=l{{MeT#j3N)OpEiCYzWBVg*qyZPw5V|;;e8}87;4Dewcr6 z1@R<0IXO8QS1X7LJRO#8PQA^c$FPmMcvq8T-Q?9iZ%x;8Z>L-(Yo5pN)i`H(*tmEA zJkE}q@27(UyX_(=Te<1jEx-fL7CYbidh=!CJ-BOsaheKo@nnjYv^+cQmUw>eZ@s=2 zmT{r-{bn+U^8Mqn0-e@k8wjWtVs1>=pyH}O|6>r(c$U`w{&0&zh3iXIjHL+OhflQ11Cfs5rs_OydKQWOm zMmRl2B*}wlUN1Gbi?X(c@8er4wublP$C+GsykTc}yjE4!wijcytbz%OC$vjTT&o_YQVdAOslOpqPgJ5R+I0`p&} z`w8%bm^;%g!R9UH+$Lx}LVo?icB91ODP3HrqRKw^KPGVoaq6H@4e%oU`U?DUS(X8n zw;775XNYQiSYlCCSP%zU*DYi>*VB@=l!Oi^{sQNypDCTLdh% zF{z7|X@j-w!f^={szSX@Zb%mw~+_YVji#^z~@oPzZc0lWSmFP&#%y*|7F?D)f*Z18aqLGN=*XK=lt$Jip)->b9}{X(z&o|e#QG)hCP13r7ZZEn&VFi z`^q}|9$ZVXhE3GB7Q(N_kBCXwgiZ9z#S;?FKOf|SVp@InIU#_$LmuV)D%AiAm;vY5 z_+V8K)>;!ru+9$?%O)qB{tfvWDmC^89mO+g>X2zFJ?JkRcZe5*DZN<|z5yghcX7o+ zT~2-Tzp-E(1{#_q1Uz@>U>1s_`fakxUqWoxWHO^{cMMO8gjCQ&vzOo6luw#Yc#vX4-E!TE6zi5k z_7+{~vS=pZi7b~U9iPJ{#FC^#RY9+LgRfl}fN-LA&Hdis5>)1);^H})luYalp(qQk z`V;K$fGSEVbxybyj^2fK^pz}g2Q)UG=_En_UsSBi%gIo$S?(jL!>6S3HD!sItz@aa zeH{tsg%(f?3@}nI`i(4TrVF=0%~_m$X(>u2md#mh2L;a&N@O1wODnAt9eDDJdASoL zXK0I#$cYUFtz4vgWN!7v3it7a$@ zd9N}|K>raI!NJ1W=_8KX3~cE~-pT!nYu$Nm2yym&;nc=aHPH_$ja-&6$aG_A(kk_f zo}G61cRC=q@UXuoj4ObnsyW6STbAaYLAS8Y#!WBDSt~_vNzED95JO&)5OLwk^h?S| zNo#V`_IaUmkt~!WWa^-c5u)9!|?ZNSDA%**|+&v1S}&K z>Rja@vN5W*!lnj<%*RBxiBPc`FE2<2uZx#Vq^lQUgcavo5Q*@hP1fkgtKgA@Iuk0B z+Cf*b#TQqjkB4&^fir6^^;Bj&H+vuO5!tfac-Nmall~vvpXmRx(UI)X*|&v$ab472 z(NXn)b~YfM>X!GwW+jL|CQg1{hogu4i&I*V7C`A+&7l_ll%tEPPUVk84B)-yXw$P5IN}= zX?Z0X@#b8v5bxYf3FSOO1DN1VnD9y4oaibov*F-ex2#R8U6!0=$zh%gczk_&^^d1h zL`3?FqbQN8RFV;=8_e&P%sJ_~^YN&F7xpdt7W#fY1>3td3VhP-^M3UL)WO5?h2Z7* zXX_CTuBW`&bmybS*T?D38ZK)F{xaa6;Ybwnm(Bn>;a*fQaEZi~DFMWzT#QE!9Vgek z8N8mY@AQ_ItHsucY%KI`C?K?0KPA(mBj#&M0x_K`ElK{bkzEp}$DqRaZ5TPhZ^+^b z+&`5-Qq5*~Ob6kuxza;~XWbK{q&h_PX}=XQr|eyLhaf`2esIyOmY&QcB(xJt<={>P z!rhy;4eUrLQz|1OFP|+4tyEY)*UA8v&x&p47*App!iCOq!Tq_O`E}Dy93}& zLKEW4C>LyHwEaOYM5zFx@^DJ`$BIrw@snEnonxgI!qe89j}*IcB?ZznQjj{~5{Q z1GWBwi1whA0|4}6kgKN8j@+zF=zB~aX}!c*6ln7K`laSd(NzaoHb9zQ1_f5T!}&ql zG<2*?6r8ttU_#I%MBJt6`C4Dg(WC$s>+I=>D{yy)_!7d?@3sIKFdRI@SC7>CEdzFc zNIFkUmDQ4&pR?@hYmrOwj^c|d*3G~?wIr3CQ1d^o zP~txAd008^Oo@bNLr6*MIH0Mq9oQopuDCxTB49ioN9Vi-FOWYmVB!W84zd&Le2H?C zJm(o#%!FF)`Tg{#pDeJy61S}IxAOd>W=e<+9rj0Ymxq3eEF8kfs@-^3F$mQbZe#2i znz{L(sCk%x&$J5(7Ol2Dj`_zj8N$l27k<Za!HkRu7^RZfyHCe0{An_U$)Rno2#@QI+IRH_S7}+>pIWfzDwU#uH9`f zuaSge<&a-m#`se4xBbnDJWGmt-~GII_)OF;5J2$BPu{3YgW)VdT>!Qz!$w)6x53!Q#rU(d}Uitbm6jVcMm3Fw5%@*zQdv(Z3Dhf^+PGl`hP7_jR=Hu@s@}Z8)nIuIPZ5PKTr%%ElUmT7h4PPw`NG( zMG~7a{{;g?Gpm%PI(VU4P{9gl!OKI&O&o#RE*qe+ZJ5poHmfjOP+2Z3uWOEOmE%+z z1Ex$&O2`MwLpu}vP`NoBBdL~fBq`&l{K)FkzY|`X98QK2v3HGi>3}bDXhsl+F&29O z{!BsK^#{49I$!4^8$Yr*o9I8EiSlM@3P0r#FQ|Y+Fl06VwXsol$h_u$V*+wHH7&<> zd=;Qf9U+P*LMd@}FM}`h)b#0!7f3!w->QoU>9qgyz(^jsei!r6+sCI(LHJaI7tt&7-Ymc?*9Ejg zxuJe`y)=htcuF(Q6Q)el#m8|qb1{1a3;qQMsToUOldiGRvr~qVq?yTvc03{cE&_>~ zg1`yP=62Va3HvHw+p}}4E~c6ocb>FSS>d9@iv^^<5If>uS_9!K|V@nWzM9d|({^MicvMoI;9GYDEEX6W`H zC8r#OhXP~n{N->1LspS(bJhM(?{be|&D!zUj$)XeqFIj^4b&? zpAZWjBRuB>CP`|&e_Cp_>d3^lzBZ2FvU9o-CHQj4xW-*ikQ|O1l(A>{F2_2}m(N3Coe&Rv1#t1F)Tv%q=_Glo~)o{{hfT+9+mjlVFA{MG4 zh|!Bm7JPx__-T`EhpyLk9`J}|1kZjr)CQ_rgTRD1+O`7~MU+TNx;D!3GE+QOLx(GqK< z)Kx2UNv<`X2sDEyl9CLfidT%ReMasgJo_k*plk2SPVei|!Jf|Nz0+~AxXaGcDY5gM zJsyeplJTr;CFi@ZFZmmhkF=l#vfb`&#$_o>)%qY*g)T-*Q55E2V8K}GFwRK3 zf4UMOT<@a#s3IB4lBS(dFE&qMj5u`FQr!*Y79=#_xs%ME@x%M}0FxY*d1=3qp+BO= zQ*nauJ{m1)UMd@$>6R`9QJ&0>1|c%kFJh=3MY3R~A&s{9;j9Ybs27wG4`%aB1Xd?0 zfi&p0&O2$xQE~{y$}d)UVncJ-97cmw3{+aIN5JT zq^SIO7ri%+onR}W$CzYJrbW)UfM?TBwDPt%(P4dLUc!9nT6?xsO4Q}KWkJGy&tM?o zwmfkT5O%sP34{tz7(t)$THy>);G({UJasT~g+c+mMB(p5C}%*L6B&0&n9e4eYxk ztCa+Zt)HI3?qvGTQKhAcp&F(;)^8GueQlxF<#5%@BCB9_Z*9WQM<@AWE$|}DMN!B1@(*`MxUe0iN+c>f!i8##(6k9`wILF}^{ zNq0N5`R)!GEOa4UUH}QfH=gM3qa~+w|2-+_g!JM%yF=pW0C#rp@+ZJqEW-loO|z=c zn07W@BO|Ij>X#IKk`=ZI=;64gv#NkGg7ruiCDeK3R3Cuy{YY8rRZcAM0UG!p*uR(K zAk3k#JY4oH>3(G42oESnl#e{t56m=3)Bqz;5FsKbqFNdRrxeJ)?ot8i1oBQGopv(& z=as;?Fo)e|R7oT_j_nr{sbG)&!A?RP19BpSp?muf-f`^xnlZeP;RV{8Z~0dp&jhwc zNapBk**1DtsNT9qjRaj1iv`w``WiqI>Inp5+S2TV*pW=dzhlDg0I9n<7G(#XmTU4v zKInOIftKKj16=a@w*GKb-vjNPI<}8OiOEiI_;RTZ#LI1UQI#$hda5vVOSyUJ(X5kw zgC?ytJ}VB}R_bQHW3^tC3G9hT>GHxBY#0H0;=g(zx+b9YtYg@_iky&hr#W6uFYJ_> z{fV(xr>Q9Z6xlPMxI5i>5ts9sn%%#$bPY_uGa>e6{tdibkFuEBugnoo>y_SH1}puU zl0AARE9oJ3$zFN$@A>l}$I7vo0TReTYMps<*%U%b!d^9Za$S_5R~XT0Z)#Gzs3Os) zO7U*GK0HMJu*WTcugbLZ(@>h_@f0w#{MzJfAqUzmZDI5Ca@IQD+r8hM(sy7f1evL+ zq^Zk5PQASn-MyNjNMV8C*PUEk7B@??r+~gpvxccD;<<~x5=)h-TN)W9+LTJtgY|=e zkxvJ+EkRE-()myKb~_JO_oSOR0ck;<6X&w1k$!UYy&gh%CJHejXEdA`g7^uh*95&~ zifeg`6-%q?5P~D;*PeS4TO78O)A{Qdx3%eqMo1r*rs%+-C%{A~+D(ZauhBCj(bNk0 z@JVrGn3(rL;MaNQ)lno4eafSjdwKFL<5vhP6)rYlFvnIek~+|(k}{fG5HqL1)$=f` z_)(5u&mz|N#1e>OZMBPz*kodtQiA&Hx#OL1b zN~hnDqJ6-L^Hba8^;ocnYk!7*sjEhZf{bAw6cvXvZHd-h{-NW^`uXz#75VrilI5WM&+r60FGG8b|%FY-RS}&yOyLv-o_g_Ui+R@q|-NY)+MuKoJs%!PmEsj|UW3!GWonngEvL zg_2e_U@_o3K_U;(6UYR5-C9AI+Q~SI&2pR`6}d*CroYc~>KtBpuiy-9*h&BacCde#mx1wKbM3ut7D4eafR52A=fKz>%qKv6V)@-AVzHI;uE@Z1d`1(W z9}R;KJ}yzDg(Sy;i;(PfnNcTI9Dwj;kapWX)~IwGQ;6e|>RqaU#GJY zb%Uw3=GR336{u``s+meFMmvSXWS;qA9z<432U0$ejQq&I)=Hkg*(GaKM`iZW;2RI~ z_wrm*>eF;+`w2@Ovk^pVH@%owrnMXsg_?@1T=k;nl>|C1)TaZ~9cGqMtxcT;_DKx)xcKAut?wPSnnZv z0t>PpyC5`>c>pBzq-VlH)TDzTdFGf%O%Fd>;GSaSS@|W&q80e)# z!WcB+N5;jg_G$m(xMP1~=z41p43!3%M`HVLCpdzY@kCeYzQ#%hgBpQL! zw;00P;*UEgiZhJ%H9gi^hcl%b!>K9(Z+Eldr;?&%g;!B2#rj5yy+irI?7IeU4o@XH z^UL-2iF+VC>3)9!(uM;yLNz0o#I7CnNo81cHcY!m*~ij6b~vh;h)harNomfDS0a)V za>rlUDju50*Aa7^zQ|N7Z#|O7Sh)l{K^Be6jW!? zkuci0@$UiyR38gRJPV3+$}hgB8o^q1dpi}T>5D_BMMLy8Vsf9x4O|Wu*v2KH zC=kiqN06E?jj-b6t@7ioFo(LM^g$c@8+#$jRhD_Crke>Ww|>RgJqF*))LxHUber}l z<%(MdY=$LnCaZ*S`skCB<|%4r%7xs*eR|+0{2{EbZ!%2D>Y>7&;S9f(IHP}|le)rt z4GA&N-V*Ub=amk^KeQt%gCXKc??|s1f(!nISKdW4lKMyZCl;~M6Dr&|ub$M8o7+_W z*S3;FOr;#aa74(wn?XzF5kAP?aOC$~ZIWb?B}c1)7`(X*O)9KRX+v6J)k_bdmPG)2 zDA78@rN2G1e<4+h9HqybbD{*le+=ayHmq6U9#H~9D%@CpIkbR|@xQS(o&y^+x`9>Q zJ(8EK@xR(C-4U+J65VgD3YeP2HWAr|h^9K&DbZ3`J4M#9P$W|ONLeNmAfDJp9K zYH^|ES(qSa&+xhEefw1Q}IZt`(d4L#M zCq@?N(zcqeuI~2gR?2r(b@lPrrkZbS?dR&y)A;2i8W-J<@9o3N+tJe7@tvv0r<`b+ zWRJior;_fY@?!^A++XRVvyS}w7yIA|sj%>-tdT#l7U@ZV!8(()B9myEdInN(F!`di-g&vT#5DQt57WrEuS7hHZS+;EEw3TRVj&7E z#GFs@ngCzNbmP*jfdMDThIq9-cv`Z~4!)b|b|pVs!m*@?Pn4C7^>L&c#tK&33=UK@mKJ zqWX&yQ=d@-#DN?4_qWuE8ZK!8^qfSxS_0t#<-M>!kw1Fe6OYXH)+vNw7XXYzO6ook z+;`cvPY5WSH}OdBwM*C~TtJ&gvI;&C9FddorMbn~rYakRKqm+>AqoMVTsM{lXw*yk z<39@;VmWg%xcoTA|Lf{2V=C#wCUJLn_lvtb zgS)%CyUW19AcMQR>%}kb4uiW5Zi5bP%lqwaHrb?qoTQUZI^F3$RrOTW$ELJ4ozKio z1!5-82>I!ZyWd4E>A_VI=&H6k%BCI4kT6Rhg;F{W?EW4^1V%`y9` z7(!U!qttGRjc3D3=>S*;$|rSzsCdV(RFG&=$}%!mxY#7kds*lgC-4G23K+_Xq(=Yg zl^QH-3&60C2eta$ecwPK(fNik!p!-(q?l;-Sb>67 z`fpD>K7cwu(o<9M`+PN^{NZwc)^V5EZr`iJ?<94F4SQ#{#I%L)H|LTiaC_0&?JK=8 zb8&UQIW8Pb{x->J3l`W`dHH)z?A~diByB#$Ub;j3wvcbrUrdpG)`Fq_s=1osj}C?I z*F!fYecmmrrmczr@MPwa;%cn&wDG)n+})X%RYAJF?{o|BqxG_KA|>GNSeMWaOOvF1 z2!<-!9vd1r6v5voH>pfUGztx-n9Xi;QJ27FTK1uJahf%O9%IeRaB|kAn18mo82dkv zRQu_o`Mqy6&KQIzx+usiUaZ}7w{Qf`^CE&1GX+zg1Nikr1E{O5{mNe$Y1Dae1V$LL z57?C&jT!0}5MG}1-`i-px_=lf>!}T~kNFqNfbtCsol5FPI=(uw^H(l|W3IS%2<4kS zxPhM*7s0{@`{3bI$G>60@0sIIBOiyvS**;fT0b4TSJzg@)<3zujGXG)9(p_JdgbNL z3=B-zW5u`Ephbo?p8O*;1jN1mUWCqweeON|^_I?UxiGhF$NU|CNGa>l-|WG=m7iBHrH}%a9u!K? zkvnJCWu^J!DC|qQ78SRjK}HI@tgj?u#;WK4PhtMxuMW%nC< zqw9?{b`CX<-2Ug>@|q$)+f#UDvKSfo7xaytj3=~{rt1G-ymEy@K@S@ugoo%ZXWte5 ztc**U)15eDfn%y$f@ekLkmTO6)pbVuLner^v#OIFoRxL771|~y1R7bQ;6mJDp8dj4 zTQder@zA*{;H<=+i#UR$=DmCL4|Hqg;Ff#m4xAx?b9)nab8}>e9v>hKw-zEl5zC|z zd-rbe_iivCt1>g0eA)}%?>NTqoKMfXb}7C`)CX*m2dSYLg!rN4uxex~Z5!cFSy$u! zZo=QlKH9;J@#N5y?`hb0(B|Ehx#_8MNmk3nILSNMd34{YTNtLb@ay12wIeGwWhr4~ zu|&&^<;eA6#hvCj7R7{7j??NmhBm|c z|B3wVD8AGgdS1ae{U*4ALs)p74VE;S6#s`+MN(gs%KfY7w;x-&3|;n)ivmaEhH4xV zuu>LLJ7h@5t123u{`S|K2`zBSvCSa3w6QRtA-cDG8oeuL+&3!>OJvy?Bo;JfEAD}r z)*!CyFHXIn<~)7ZA8JY!n7Govl=~f=MD{j9Ri^soPa&C)K(2@=0wCKnF*Z$$>6Aw$ znF{Y4wMhKA8O1b8E&>zKA)3a8e^}D7Ctv;y?qOn-^STdB)r&oZH`Z6=ajl0g{1m)v=uGrp$AFs}a2{f0=Nc>YU$iU`yYeyO5Q8}F=s#Fe3{ z>{me8TS}fUWYhHCJAbs2%*L}y12)?s*ydA2oh4syyQtH&yvXDNTWo922hyCo; z)6b3%Z0SkVy8+FblDyeDI$Q)4?avn`--~jh=2svgp6u_CK!qxo zEn?5o6PZHXJIaD?RMz}`zdL@7y(WKG7#*NcFCX7cQJm=`=DVJ@ zW9&LH;A}OYpbH8)i+FhjA#M-Y|JcjbZj`}E59u4QBVDNmZl@!$1`$8=mR!WPPWbKZ zG-mOr+e*9(eKUG_3H&oHkDG{2a}sFH1fqR?`Sjm;-nhES??j57ogRJ6jo*#07u%c) zf5`8eP4~DGI0GWlUeO!3Jmx+gpq{o+L&7oBsNsLU1WJb{5(K;4SD%Ovh1TzQQ{en7 zVnsnx1C?bZhNj}@kd$Dmv8xlspQqCa^ghX z7*Crm=PM1v>Nu+apQ>1m!rZ1JNBUq~hpgBl{s+ij$ld-iw&#!w!ctV)9oHG-nL_E| ztT}2aZz1TY9xIFyUln^L9#Cl%$qj9)z5P{GQ@H6XSHfGZA29)>Cze9~o6P_maS91r zdrUA5RM!+^AoEloU!enyUqGW&fKi z$~&M66~gVb#9C>qA-u&jvhS*xcjn}fu`2IqjCXRNRrAlnl6lIs(~WCDM9_!~ed)MW zQat=YdJ$T0cShjVfY?)0UD@8BgTSuMHjU)fuci3+2QJo2o8?)VPCxT%MfN@tU=^Ch zt_qNzd20`1z%Z3T8ch7A_Lv#Bn52GVXhZ90=1Eq(So)Jo%oC+1F6&1F|J6)f|M%&_ zq{Ss`k30c1IKkWla=Kg4!F`z8bfs0i8WmkpaeOiW# zv?G?gAP?*h_c9=wV%hf}t@cNF|6@f;yLK*?z-j;tem;g47`Q1IY{Ue#92>`m#y@sl6&POw?tk#IR#x#@kB7B)TTFFpdub#&Oi(;D{ZJgIM~n&2@}Yd=-`Vo2#z~B)%BI3j zEFn8wb#{ozD#DOG(`s`rjkM~tHrL+}+G>)|@|eBcS7stig@J{ThOg`(*XOHDs@#OM zrT+jB+F zGp|J8T9g4-yg~$2daIxag3dWNp&Rb$*yUx5Clp2rWPhFR?4aTJ3Jdqm zfSR*47ds{>l*jig3Dpq}Dtg+Pc)H?2h;p1myFPom%B`KQ4S0L}LQk>WAFs@EO;1nVuB4;$L3+iu*0BB;@#?nSo zMyQ^aX#5U$(2@-F?P2VCpa$6guDmV+WY4btC?hCR8@KuU3g zOFo=2sT^^ocB%+HjLJ&tF)C_0r|FSDeRw&R1WvR}SuMPq^$Bsjo}h2BYe!69;tBEj}krsG@a= zkzY`bb-7M^==f0BuRTff0A#B9U&JgO5vl>uHa{Xi8Aw)b8NF<1ljd)LXMt5`8Gz(& zwbGZ`>28*TFbB(;w| zZqCm6{u5&nJ1)jszq?B>XJY9Ij(0Kd!2k}Q&a*#=c1cK`_ygs(J6KXq(m~}LlFNbW z-pWQAI8ucIE@oV^hGc+TRQ$Niju0DOk$<5sQZ(wn_6JAGnr)L0WJ_VQca7Cxso){R z*T^x%p3iT;)_aoD%+8wWZw7!jkd@Uw1ft3vhBC%-oHY)LgD!_u((Ub7{jAU$*Z&IRrg_|a@6ptR}$gzUcWk=h%fuWh)wF| z$*Y`nC=kzQxU&HC;nB>u2y5(>Lg7PRM!s;}lL1t!riz*6t!Gcs_Db^>n>Jlhw+>$V zipNv5r!FT%SNA8*=XP<)PvJP$d^^QZL~@84*rJB{aJUTqn|w8m*cIBI{%x?)TDJF~ zSdyygy;&*wL2IpsVoe`_`;}g#?`dNK{-R$I&Gm?7mK|BQA-?QQx`>dBnI1k!gmJ)k zBh^Q-Wb*b8k2(2TVR<%-({qU4tcw9uW5L_+Xc7(I(5{LLIOdS(3U1<20ZT4JWGiHQPInMkf-ORph zJTw>UZpri*eC|;Tjc8hX@fEqD{rJ@L_jbnaHoHzs);EoZnONEIQVyV$RxFkCFGhZA zJBLKRx^68I9Peh$(7_K#HU|Q>sS|HiUAKmq) z(h_5MF9nPwOl=>yG9UMGw4V`E^Usf$Vf4Au>O&phc{@5=)>o3bYJw*l>N_=(K{X)! zo*BDLuJquw^~pO!`OL;@nq$n&imR}3E%pqajIQrIT(G12Y?s@?sCCL%(Lhxk2(5~2 zZ5VT%lidww{wJ_Tj+6b!^Z*Yq-L3_H=%=K4{XH&{SjgI>G+(N5f$Wk>ioW-sCa4YG zD-)o~*st^&Rg8fEI*+$lyymC^fQAK_(oRC^$WQ>rEBHa)R$WKp*j(-wLBPSxG~WaB z;h&&y`+SU>jIP5yybXt&8l$qpveK;^sBGvEk8%hn@PYyHEr&m3B@? zLz_r_*PtKl)jQ?os8;wRJ?YWPC`~8C?}`FEZdd`*g%raI%Ne{p1z)OUzO0AM(6&j# z60lXQxNcQx*JjrNa7?S>mS5W69KA7Su8iYu8?B7nma8f*xo3L_lw|kXDLodOfG8%k z`ty3AefEyi{dRBH@b~#6%^AY<(pk8qTx;T1Bo_T2xiYKhaj;(S zP2=>Z2X3kEM>zyQ+RcvScQ$KU_FZM)x1`Ajs?KZ1$&&QUP1u5ve6tBG-FZnjH>EHa}{9Thde#T1q@BF}M~F4rR+jJ}0p+r}{Wqb%x{N@%k` zB|T~A%FgEM(_eDr5_M&N^#n<3ZcZ!9Q10a%ljkUqg2y^I?7+wJ7AJ)JgnISIf5AfY z$LZ|LesG_6KX3N6!>*O9$18%|i>TTA^Se9sYnX6zzxeC(EVYM`hZjoYx_2;2jL_l* z=>{ErW8vH3c@@#!life{%gaaVx=c|pAG<5|=I+NqB)4C{k6|_?eT^q9g)!AIU3i_0 zilv5z2~V;`7mLk9KeKKtNu!V~lG2X9i!QE$qvXpUF-aW%jVc9wmx~;C-YNh=IwK=8 zBYHLLML~7lxrbj$vvM*Zlj(c8S#a&5<;{S-EfF)1kNbbv?f#F?BW-K>kP+?|RNF}X zf@)jDr#Ig5NoZ!Ft1DNCwT-S|zoZnVrLRX0 z4EZ)d4?b^fMKOH!g}sK1g>oUA%7d`;<8-d?UzLVJx3MD3Z(@{b(J+zR#;eQvB|@`#E{*~ zbyk>C@Xu6Q>})j5e4(%wqyJFY^XwN2%e)b461AOelfB=oK45HtD5+s!$!CPh_L+me}BCag;9iD#?q-I9H=QB`I~?oT|2q)dKE^OOiu%i~>4IrGp4tyR`t6ccVl zEo%Sfs!UbS4#vd-!?8F!>oz{Y&92mGy!Y;p`x^tw?L|cRmwAz087q?hYEtLbcm+t* z5_Q_R3aw`Z+^v_Gf(@-49>P2_*qO-u8b~BkX-dbbh zZQwJSI0bMo*y!3QM@26GolZ17R`@(xH;W+R@bZA2ztDibwb-6Ovt4gK`BtyXk&b;c zVJQ&)XBB^MRC9)_;$$r*y(VmY;TN3qV3dF(FYr{3usWmG19dNNU1#`Takv@wRYJK| zm|61)!6cmWh7TJMNm+hx8V7}b5`R;a#}UA>iTh%4Bte>p|4+zSI730%olwEbaAL2Q z)7wDsshau2-h6*}nc!t3xo-x)Nw|I48Ll*`VPq*PK$0 zhwFuE4)lS$+lF_we)$?$p#mJE^Hf-dxg(& z6_l{zArn;(ekBAIs#Btn-G~kSfoqvFzA~*OvRrl z!DKqR8+rPTuzRQG3CYWgTfeNlL{J)2wj@j z;j>Iy?Xw($&1EY4Ld^&ka=&80kO#3Z>O%)`Sn=m~fb}^p`=swi7vY@-9)86a|4M)8 z_{Hpps?}#ZTte%um1o42oDL2IxdcPqVtQbP+MLb7(@=gn7c!aO=5aC+bMau+Oo zf&0KjBoBjeHwOzuM61$wUm`R7J@UM9^-L`kbwnh(E=lkbLmew?f zn3&Qye~OD->}Wv%&?5!cleFR5o1NuY5gl>`&M=g|_-k^7+M5pJ_%b|!n@+fQZ@mrK zH#qE>C*?WvUfsq4`yHL-f|8=i>W#;2@gen}s}5~OG;W+?g1q6ddVeep17{;nd+`5- zH!f&cn!$ZRyg|_2D$GlM^$^3w`y4)7sMdC{FzqVakpNUzw60un)SpdRM!)d>gu zjt||Bu1eSIOjm{dh+SgIugUR|-dOAz=w*UK8AMNZQ$U_AB!HtikwH61fdT%a3HniE zS0;%J>RY;O$o-P$E>;GzJlbI)JwiZkpW0guQ%HpZ8#Z+%8PpwPjwkub z#J%S^+VG4v!?%3NP=>?HsUw*2`es+RO$J4--wt6vMuYLVT~fxaxN1-5}wDF0|zu#fKbT{QH6$~+cx?8 z7Y|&MgTq%CLkDy8W~?QQp!TFn$b1LUEeWN;v=S|VwxeDmThO49MUB%`sk>KUg6m3; zftwkTH(N_Xu5!lZpwG2X^Z}!OywH_ucj@CcW0Cu}6pUdO!?aX%YO5KP_v%sdwsY~# z2Kfx1BPm%6wfd;gT~es^2-#6&)}hqJ!v|BaO^I5LtE`6e`Vz!kvFa?j z0UVdC<8xuM`kWRnp>Y$5JS4V8#r}4`-Mj)4NQ!_jJunTu2zj9#l1UY1aJ5WzM;4^+ zF4gT{{RN*}2J|SttB_Z`M1{W!96YOMogyqa6j;H-bhA*J{s~9A>;gLE%h9sGMhhZk zqZd9lCut68MF-grv%Y8AJM3$Any^Abhw6XiO1WDpq5*zGq+w8!SPkTYm_y)yCnKa^gnm^Qx$ z>=o26J-sNCmkOClRT3ZX4N4W}3%=d`NoyRvNKaDl{QlE?jW&8OwP|iC4mlGlGP9TP zuj@HlQGfu(kP=^-t*8JYfec{~@mRm&XWL?2cIlvPz`2jNceJw#oRsrApzi^pp?1q5S&pw1e^udC8BA8n5G^+R+m}(W@ljBIQbkwz6?5x?Q)<%X!@QrY z5w7Czf4R8vZ!*8}P}h@%wyq8HFD7|2A5SeX(1~y?Dry+KJ-Au+Ercv>LMQ9yCguEF zXL@~D3_v)1gyZf$`4zjexiN(2#zMDYY>w%@JjX>%=G8-E%pqe+>B>s-wdemJ%NmwF z*KddYUAJ577KRUjol3uruUm;mi=1QQn{zjMPC-odmyH^xrIG&JPFtY*7N@QJ_1cY8 zq31|{96Dx#zx@X%b=ZZ5;~n=r#%`AgkTM3LsRk(khv$kh#FvF2JL|Po*a<7jwgcSc z{D@O8@Y5bgxy8;uHv&G^Z=ZS{hU^B%I)8rn(^^| z0I5G|kulYW0~8xNV+t*>qVEsDgiK>Q^`7o-i~xAUHdTu@La7HRiGIc^o?lRaT7Uh1 zL&pRSLaLS$GJ5HwGVzF>%s}HLR%jh}-lh2@3OPCRRECJqpjZ z8`m@8g28UKtiGXxuKUy=v;oyLMgbM*RnG-+Q`&_fI_MqNc=tM!#5*b@4=xP0GO(nW zjLt+j#DJc%piu;_yp zR94qauSO@8nZU9CA;JkS{v*OEIz>$WWl<)pQK^O?nztj5{j95LYduyD4t!qPLK_@AT(FQcH=o*OrJ|8tIpo-Wh0FBbX#WokQ2#RyFd7|IkqGzK9!G*9RUr9$Zd#P=B(YH? zXBweJl>qh2_%JkrMKo5PFfTJR-V4_Nl4f1&>0M+$ITh;;wx>;sAm_r1WKZak7U_P-_m7kVCgc`6XZe`Ey&rWnL5Hid~e zKE(f#`3Ed4l4@ba9oX0zZ(VuYymHBpNRWag6J%{4kg3@>^0Y8CqL1!)Ayb}plTf2V zML&s6+;qpkk&BpkoGsX^$q*&6rra<{{^|pZ#5r!QQ`7JLIp3%|J3%eJFu#%IyP!Vb zsfMYcz~Jr(L`Ud_Cj&7)>!ec~+~#-G#l>6bowFH(me*R;NPgAq|4(?foVDF$Jsy4?C z+r8K=E351{#KeRj8BkbYDK#a}mjYlr?4qH5t4T;7bBlZAa1gM4z8^|87u=_m>b}$R zTFFOK$RdIZ6}lX$#a9|$lG>;yo@#^Uuh-BQ<`7awMg5}P+))SFx*wA!^^+%d7hVyl zQzXtt13enaGS;bE5>?`p!HTl1&-(A-W06Yh|8ZQw zVZnXYO7JSr%!L>r;{dzy-gR008?U$O0K=hB+(Xl2vQ~G-7+|h;NX4J{7$85~@{}AA48MJE9?QKI`V3Z<;8o0={e9 zPalSH`s)m{zigLLo>)w#mVq*Zs{JyyWPUlK=9^UWU2N;HSF)_&&bGl10h!f9E0lzZ zpKHmA%!0NsTL>^hH|kx`1U*yC>jLuPNf!pTR|-5oSF@d~5KNn>^&}fd^B)@JE}LHa zE~#S6n?x66x+L$-D%aU+8tV42$Pm9%>o0Q;jVCoyXm-XRTbi;ok<&&@nCa~WNZ5%_ zPoL^hzeFzesb%(;rKKj^psiUsvjG(<2GX82>?$V%HLkdp7WC&4KH{{s^uXn(7OO6^ zEb;3C#_j&($~B)xcdNTTGFx_;oFjT;ubjt=7SpFx$XoW<*&tTmOJ8p$=`bym_S+)+ zUbu6I1+XyxWX>E81@-m^;MBM;b8PrIwlE!YY`(k?>R+#0h2>>kBvrQc549;Al3fXE zntd<2ug2#|VPey&DTFzMi0+q2CD>p<6IpklQVggnvQ7ADVK=}HIgW*~=N3P)WdPhP z#()}fiaThSk}SUa!`hdpR{jgy8NA^)UsaN$HvM5mzYb&rI=u`n_PJ(ys79xTVZ?Ej zm5)_0*Eop)FFe!2C{6~dR>KP&=Dd-Eq+Fh&m+=Ya*n6G`4d#je_SRd2B%8~#>AC09 zLGXJ7)Zlgwy~)zEF)ge#t@XmQv2{6d`ll!9io>%pv##9Ear96&v&FWkQ%vWJY9KuR zEXK*5v)6e?qm!>bAOkQA&8f6fdbItELo>BDTw(})aEzBun^#g)c@Lv)F*%;b6~I@$ zzgVd@pGGbXsM}(u?+3#W=_?^U_MlEGnE2!8nA$rGeqiz|RPaOsFsb2|&r4Qk z9aM?TGVjPZ8O_Y^$R0;9Os^aIH9=lU1IvIK(b2^TSjfy>bMV6|YfbT1U6oQuhT%Vt zV;5p4OuNxiIA-rD7`a)1!m3lDny3=iSjbXQJQwAvZqbRA(Z=+*HC~hbBuIDE?!m$0 zCjR*+tD1SK>|h%ISCs(PNW@MxN=(?xV396eRSt-g!o{Kw^b0yK1%bT;^QV43C9{ij zQsOD5wSJ1Jq$EL%9Tnjh8$w}(Omo%_S_{1tFQ}0ozj14_tH}`r2-!s2 zb5;`W**+(tqAXi?zOtC>+xdK3xo<&0f4BGFFxa9*2;E5tNrz*nX4VMn0|*@( z-u|*EIMaE^RN3DdakF61GE0)3GaX*!7Ec!8S;ye@gYPDk47kHltqoR$ow_XT=8^rF zNNGMQ`b1+xOMi_Ff5k&C<8=g(Z!c6u+On31I78dI#yrbU1Pn1FuHj(6C!%(VOX_NV zpt9M$%G@;6soNbuI;(F45dV<^6S73m4hmSTb^sx+`G&kw71qyU=?RDJ%dmd z%ShPxDl)^6aj-avwE&VMvuCP&!09f@$VSIA?ANnjHkL-S8;q#BYjO+=;T5c8AVzem z&Lhh+rk6Bv#D3uo3tZpusWf#|av-rbbAdMpF-Cc*MyW~FOzAIn#>N|koooQ`)UEo| z;6-lzoD0RExgor!t#OM>o0?AOde!4EWbSQ^LQ@B!=K6zdcWdCH+Z4Hplso+eZGU8oB9!lrs}m^=g6~*!ey!F-ld!Dd&tg7-##qjOjVUoZ{J%u8cjo zWREArKBuB?5As*AKqA*c`OXAOKzFU|mgd8Uu}d%-)j-#HVyOa#uR1?bzUOSo zMR$O@@Eo}16M$M=AE?|-IT=Q2^TQ*pu?T$3V^;9j5j4oUt;z}8K+54n#uH*od(+7D z>KpMTsSS6;`fw7u(FCM^tfJ@jQt`iF=AGio36taG4rK1t;Y{5A5%P}ZZ<5wSsD{Tc z0anzvrE2+{QntrG52GabkHp_(3`V~`P^Won=jkdLgxdegIV+04^^ZM>a`kI#kT+@0 z(z4p!i>KntBXf!o)#r^^B#*m^aclM-t7<>kuv+Jd+8||G}&n{?{tiAbP<$UCsQXVAt-27f6PXCT~ zIW?a3T&_QrdY#t(5Z~75G{SZCtrj)zdy!2|l8*LIcg*aAOD)KA(NP2-H60cp5__1= zr$)?%Q{tI?=8+-p(2jnJUn5*iV7R+n1%lz~{o(|x2Ggl9hKMCY^sHgWwfJLhRD@!< zSQxY2-Pw3j6z^K8$FSH5-S)z+T|f5+@hj0;Va<4sx-)|fHr^fEI+zPQOGuD;`ML=* zYkC}Z^%0$3*EKyvTpvbVPP_LinPu^nAU6=7IHEh@RXT{^&{(ufK4Y$MZrPCLO`Bib z;m4eh@WIMc)ljeyFRze*aLz!mkdP3c*HitYE8p9S)Lt+DtY&9WE%!;)KqaTgA35&@ z@Lr`Y&%Szj=pav%Dij4x0UiK!8GkM?KX(6BvXv8gmSC_1rSkW8Jkdfkqo2PJ4&XVA z&LG3P%EaiyeR!E}jz-s)eso!(GO{eroY8fn{+KF`5D0-5Kl13?9w28|?AMP8AM-Ft zN4apb3>)#mN{g)|dL0Y<6`YN5%>K%vgQJ^ac)damkXieR6^NEIAX zlzp6%-f*IH_7C}tw$h0Y8rs8snvwxcU#JD%)>Rt2Lh)1}?SM_53&oUGB&&VsN^R_C z22O+)J5x8hg11?BfDhGGjKa1|RVj^PrJ zGJGo$0ra`0)5^esrBiytZ0dg6d*j7?0Ly6p4D%?@#F62eosd$Tx7%C9*bRr~rtCqM zKl)j}unrTGIl159I~_C&<}(L2uzw*qTf$3Qf$51SC`{*OyB{+j|QFH0)o2tPJv-jo9rXyM7{6>Sry(6C$u}<#E*55XI-w_gk$G08IFKzZt4 zOp=01SvlQuvh#gi7+3GmMS5T$MO-`P{YM9-mJfwB292|tSeUQI+Y;0f9V9H)YH56l z0GX(GS&{G7mMS&hKxtWW4k)QDaW2pRfBuG*%40**-?xVd{zpu_C0o-B7i$KCORef$ z11dg{8i5m`3L18vv<&905sB`GTrFt?3Z4NdN~wFndi78F z+-$%)NV|?|{M`3Lg+db;^@<*=f8Ed=jD;^vmpPTdl69;=W-fnQ-X6|{wzV#9^FKGG z(tqiCpF}oN8*0z%R9*6YfS5uzdgSITPTOi>X6$W%U0;ny1c#C|lQwwr<+8 zieJwJ6YKKBa#$mShC0yY{`uoM{)>D&CA#r-SQeZ_OSOX4jgj zrh#coE`Dv^!9r~s)>mFIlp(HciC@GM&Az>#SR<&%qY(B;f( zqc~4-Kuk@nro+(uhE=_~3(d*gvuXl1NNiRbJA%#9Ie;(oa5I&_G?1%>p0Dc?QFoRfOT8ayo)2TjL2UV+ZAyV9$T?pgj7A)k%Dq# zo^bFNlwLv>6+fN-gS7hjdYvY=ro~(EJ_H3)qtwbVHHfU)D{}quoFuxFQG{cW@-bw2 zI5D_=)h$V_`%5|6`2tog*2}hh0pL@5fOY*Bs^5q`aVwzu;*rUwI@fCg>v(Py;qL|g zC8!7G{OR%T&qF0Mhso*!|1D4Gb0vU^<*h+08JC;*ER1lo?YOb@c23qT?bFP&~QPx;+BkDt|JcEMzo!i3F7Lq$j3~V z$SQDm3RA_BDfaNZnz2>T`#pcLb;~vFjOj#q*c_#PQ8(N6OQgFKvEfYG?RYe54WRLV z#q1+{*X`;k$To?canUh-%Z;c11@RrAMMb(xo55q*!BP!+lAxj zkTV5+Sq7SR-Nd`9*jeEbvAO#hOw~9%S>e>43bg#CSCX^FTULYrJ{r9SYHwgK4nkS? zK#y~+xDw|@>oIn&rIl~Ys%QTmXgeId=0z*>GS=5< zu{`{3gCo8oCc$MZ)(fTK(eqw^B7eL!boPpFM){ofgI{-FDA?x~^jagW!&_);*}F`K$&XoPAB)}r!eZri zYiaHIRPn{mbS6dl>5n8PBKdc=KB#Vd>C}V2e020|N2#3S{pzOwbPW}D)Oz$MZMpT= zA>x4&7atcF1eQDO*&iTfAo?I-ed(mtJVYdVEuxCc77Dc&BKln# zEUiyw-0qU&jiMq-OABU)feT&(rG_b7)I zG8`IS1v?@Oc@`ijt+?WWP-EJR zcPr|?7WW*7%8x6Z?r&}%J7 zl5P|x6Q$xN(yoRnc0`2ekc>dhv{U+=1vfz=u^a}g#^;S~73)l3{ Z&BHqB;P+rKFtE?h^ID%ZFc=iD{{iwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PKBxf7`Z_==qzUVvf?ivArfG+wvniO}nm>wB9C;H_kS{y*b?& zB8L)c3Sa@yjvD)Z_WQ$wPf@b`XwsgVeYyK`I3&izRq!WdUXQO@MR10X`tn1pJa4Fk%%AyGKuVmbhcVJ4`W&X!4Nh)m)D zULC$|`^wJb04_ZM03R5c-~jfiNO*V#uPV$3&v=+kkXd9*aez!D0~jln%0agq@lXcD zRt-Wv=~j8S+YkEP>l=AVhO!NInviKgnN%YCRT0;L)LQ?uu-sJhFN7A(Tvmf+>EiXbD3YseOY9D^UWMbQamUnf1(S!-g$nlf1VE~ zxsdwsf+h+DWh2jrXLx>s;pe`+1;38738%21fD;Dpd*05VHc zF4TWi2kvnEyY^^9Xz&_LGEPNZ{uv1dVbfs-+YZhfE^q zsS!y8Jcdq_WjcWo=-0NkodT8wOC-culGt}V27yio1fG814)!G(Vj?vN5=G{}B&pU6 z`DDVGEj55oR}Pud(7CQ^rD$3Ia6(3SoF&N#h60t$yKpf^25FWgx_?v`l7UAXg$MBd zXYWzPGMG%?;D|~kra$7;7CIsm^t?xKl47Xmce!{bGflXbwB=SJ2?A3tsp4V^vEUQ9 z7}Id9@zrQ91r;hV2GIF6)7BAhRl$Y>>g_8s%3IT*6d4VmW7Kg`@Z^X6&FvlA97J@4 zQWXfUP-c*fxtI_I9XTeO+dG5ZjebAg9g^Wz)DJgz`g_qXZa?Ywp2Qp57;bNpjlEuP zhiq)_g}Yl@aTv#&y=aT{$##E8`hC(_4NU7fLrH_D89wPq9_bPmn?cd;-qw@7Xfxgn zH;3E11jA@E-rmBEo$cMdCm8pJo10I_6C~T)xHa6{4a4CM_V#dVC+zLS{oQb<=(FYr zNO?lTsWDm9n97piy6ea0;{XohBd(4GN@VIf?mgy_47`*_tz6qGD@B#2jJ$D@phZFn z8kX&)X)@J6^tc#4VN#JKK>;CmMnd2ff;2D;^lOGvnco0mnhjv1XYJkzP7Ggd^m?zT zZTW~tc!CLrN)x7~!Dj>~oSmQo%7BXqO)<`rsWbhGlGelnUJPppj7M53S)~z;mNtC^ zLxoZ^HI+6MrJmj0qJO6`iTzt2o3N zGefS~(SjO)5*g+#t-MZ=ooEAcb?H|16wtyoPI>ANom9|sn&1edPT^)6w!)7}|1Xu$ zmS~C9$-#l9#{goINc8Gds11=E^DK!#WK6S7*$_;4gaN$Dq=F#=GBQjYS&*Gm!I}0% z^G-Oj0ky8ig`i3+d+iX&oJYr8DpFRl0{UFs?wNGVJVFhDw9ZIuL6C zP?=K$BgkZKG32HID4e7TQ3ht=ZC*oA0!OGWxH#KaiiBf5uFufOp)R1u)6C*jjp0?6 zs7xA8t+2W`F|QXuX&I44Z63Ut9ier{Y9V3zZ6+Nz8a90u#9f;ap;(K?ATg(B88GjIeyrx6Odx^7yv0Hq=!FuTBgMSa}_nj~Of7*sTH{be0wW)v1t%`wq< zg>RWgMdy(YE+X1LgZ)l7n*s?7onLLvdOL&)$7~ux!D%n~2ogNUWR65@ z!dj8heNXu4r7k=G%3_|wct(Y>Ij2qxawrykXGaK>G(tTZxj9i`tCT1^Hy||{VyL~N zj78vFrKlP^*StFAnSdV)`xYYYYtmt6zl)kKa5JKTCOHM(r%754s~1jF-)=uONaxvEMis`!wp=RW+%^W$J!QPPo%C zCKJ$5PG{{YXNs2&RNXd}tA$zW>=|P{t=E7SIz+nHxeXP z-Ob)+&)?YWy7JnF?5Jsqx+Wj~f}yhSv~hq!QIwqld@{n{sRnb4x0U)@m9kOCyuMnu zb(%roW$RT#z|}fm35<1}N8PA!LuB{FKzK#cHXs!r`PPt{;o!Dz5FyC4v!1`E3MbWa z?Syt;TFC9bXyP^A&5~}t{&hmxOJpN8?hIg~^69GheN|~Y*Ug;)yq{^gc=2MA=e}de_t5#GdPPuSZ31J$yx^^wtoHsjO|}-FVh2R6~`wZZ(fLF59RWl5j!U zf;Z=|TgT45O%}|d3^`Abuudx%)dG9VL+~1q+PN`DY;JpAgGuMHXdYszutpiXY$03? z;k-&cn)ik1O9aR>g5<=sL~DZ$F0D#`nI^7!@I=`zCBV1#MHe zC!_U9?n0|d%;%aeS7)pavTV|reV_~(H72(*)HO5hnSfQN^77|76J8!GxI2<%G^B}5 z9)wtoOCE2y4@`#Hj3uD`(B``fB3Sbs3csAUc?8th%%(XLl8@t_JOBf&eE=t1r-uG@J&-;VGc z=godgDj!Fy_Aafr4sm0EZlSicBJ!^O)X~d_=dRC8{5M}^klmx#Z=at8>Z7`A8`tdr z+34@I#D8}m;=iBcc{?WT45mCYYb(wgq--=dMKtidW3%4_kV(7r>XvEE(urL@cwAYZ zvIgm6vkFsvZ>`tICNZs8$hAv6i`Ll!?*$hSXTmHOX%9tAj9Ia6a>Zzb0(5oj&Mmol z=+h^l%vj0LafSSuy&W2a+bPk30IseA@9h{N9FtU`fX64OvJ}37!%AAgW3by+hA3go zFFlwd(Ki4o{$pFVZh zyyqWNt(}YA87>#f?5+#EweHoG_vw=l##36PN=zUY{IWR75|xXP$rKHKvdha=#}O_; ziHwD?vt@qL$w?N+^dofq4meoIBK^}d=pJx(jzR%pZUaF*k<9u_K$+rY5rHvSC*UR> zvfi@kkv5ri9U4R6HZ0_?^G~UuOvTXoH~Am`COZ&R_F*)um_8*E4LC(c3t$aqU{%W( zfaU|J*REA$Q88L|3fuf#QjB4oCCLx&T=jA;O*k{;l$KGbgbiDl zgi32H?=UX(xoObtQjl)PsLeK3HpL^)w>7zt#df5dut^?lDS2NHt7G+MBqnF+x!-ef z*zV022R-8M_gAJfgQv^OTZv;q77HYJV`*mV=6Kd#-VnKx3w^_gHCVRMWguGKZ2wu~ z*vjx~UdiWjBJB`6zOtnfGQ{L2!)&URW8CapmF#t^;SvO<2?-H8|K5Slht4fj-ekg= zMOoeeWn01GtgWFyj=QEC16ay9bX^WCQcF~jF7oFBdwESO5tb_Z#nx{9jihN%--uJ} z;4RIsN|j6aHRB2!RGNh{A&iVL@`qCkDCazv2$t73SC zbWqcba$#y9wS6(Qd}5p3d?U(LL*Ga9=AL@7z-_x}*k?;{PdUK_tMh=f%A8T10`g`$ z?nh@S0Bv%%jKL;PaB_~Kq=vu|etbn#IEK^zUFluE-|(OO{Lg4@^`F7~zgE5-_$w>l z4$4=T|BTk&6aTk;|Nr_wzwh}^^53TLKZ9ipudc1GeyeNz`%3l0zZ)R6H>TuQrx4Uk zG=FJQ)cpDiZ|He(1%HK(5*c=u8U2!|GwOJC6;`xozEW5cdIoDNR<%37%ID?5Q&^D( zc1xYsD#0uA3m4iC|MaQyk}@tzI#uoAVy1?Uace88v}yk8-(T7v``ph0|36GJsZa=> z;N6acxGw(N+nMqIcOU%!&+{}U3^62OK26x!xIC4|d?T3}b3uQzJJ`$e`G)xzq$(nQFj@u(c`%r31datpk_9D2tFO=v!cOOup(ySW~WX zES9a9+k)+2-8?0-#O`&JifXF2^pv_}uzavHc!|8MN< z)b;=7#@566KcDCM-SodZt>P~`4e$itU~JA*bko2B3VHyu6X;#&B{LHQcN39?UEcy{ zx^sgD+C0u6pF1kM;hP?}-LL;Ji~K>KMf%^E4HJ^y^)R?=^FOwl=fCZ4?erh?|L1sW z$8HvC+>HDylHQy4UPeLu>2!Qf7#ES$@?tj34&c(akeaESw;*1b_i0n~4FPA3`#d!} zF9ug-&SAnwFKB|t#3Xrky5~f66FyolJ*19}Zl}bkjH1>vQT6*`JBaRd1KA=g2(GSf zgN0vd;JS?cPoVgJO3s=%%oHyDAeWc>mzbp%=>NzZqb2dqr-5Iy|GT%lwcFJH{oc-l z{{I|LRsWMTm8C6ToTBzH2`rifWQyi2K3O?f5MV-7IDT2_$+CT!T|X@w-?5G766#v2 z432(gdZM3jjNzHgCeEg-4d9B6j(-)r%}-bRjz}|EK&N2?c2*5OHZfmj6R6~z7?rG& z#>$#o^YM#QBJCPpxp-GiS+R#!Wm3IDF)DKx>YGRAk7Nazx#PD7jWiiIiuOfP8?d{u z-%L9-CM-6-z+G|9xv#aE;_S+zesKb&w|tv3R4T@@LMb1J7*+nZ?|eVuBi|gK)cDe3 z6ciTB)m3#f1+w!dSgV5$jt)N@J%9V|^_w3*9K1eyark|$7}!+g7lKa)jSt`w#$J*1 zN1Qq@z4@HsbTDJtwUoPyLTvBUcZWwGo*lkv>&*bVV=h(o{GLU$xeiS2Y^7E}KmPZ3 z&nMgozV{=xG))bb0 z%#nRrdMx-5>rnAysY3I|9mq917(R68buKTE>LzOMV?vWm;M=i4Ip#?;fbH6Mv;B}H z&oCj=6D<#s9KcqsNQ#2;XwFAPC&)9^`c)u_C_^d#AU)q$)UBP~(*4R03O=ys1lRF3 zrw$+In@OL`kR)#3M1_$nrvW!4IwUC>(u6A7hy`sc0*VrKylp7I9 zh?GYLie(yKPY`}6sG=dsEiZva1?EJy+g!b3{M&^oo@KOl_FKeyEo?Wtm~bDf5H+Uy zUXT!v>-uGTo~7K}2wy^+xyj)!mh()z)xGj%vF17Tw{B1I#GQm)Gu1JUF;oLM;wRx4 zqpT)wJk?dW7{K$7R4Q4iXH4P)$5#7M#F^9@Yq`p0qt8ELn5iY&w31&#UCV-ztGco^ zlGYw*wA&0DPDQTNiep4uO_-*lxJ~m#ob%?*H&u%dI7!vi9O3!t8lx-MAlJr!7R3MD z$-_P=4uJ5>vl4C+6TD{qr?=aRId7E3~+DZ`hwY_IM9V>I>vwhM|A=Os{Rx8j)(jlM;-(P^Y|4M_XA%uBi{1 z#Hv2|(tP>we4b|s{eQRQ&t>vIcN^=!o4x+S`7fX6`Q7yYz2<-Z6{dmPC5+xRQ?yFi zGHG;f%IJM0j6NiS{(+uF`ri(q+qQZKFW@@=e`kApM*sIWAN2p{c$V5Q+?)amMc>d5 zx776ug1TQ))z4<4G^j(C$x8VH`1p&S1@dnWQtg^4=xz~!Yvg}xqqjYi|My`3eV(Tw z|MrTb{65s%%l)mgL5h=me_yGeJzJwi0<{&k;2;Wvs^|Ou{hG#q4YJ*bse2bnEp6xC zrIB0m#$T5y=7hmIBu!068gZ>6J6*QL#B_<3rd;OmhzOAImu(U&cSTYzO3h6 zR`V@ykZHL@j%8bV+x+CV*3`E4T*|pwl(YGbOXW4rOH<1GSn1W0$&K?ek6IwsvV?t* zWB0=k-H*$H8VY>OJojfzXtJ_e*(!?Rm_z5e5L^u4t4|x1G%5S3@%Q8U?b`_-$-(2I zh~shd8hkcafmDpg8*e|ox&l9(li`jZsVXv@ukudkN29&^{@uaR;Q?oHjRYZ4&@o+y zOBe}E!T%NEvD-5otletU@Z%;DG={$y^btHsc&5N6iGlCCOzWpRz2A{;{{S_{GA-9! zK5xde*lAcGmS2glCCOY5e?irlvOFU^?_Esu?pbCz7-M)w*$6J&ZPFULxeI&>7ep3} zQ{3`osPZ2C7&24ZIL>{Z-1 z>R?ju;nsUu|5w{@dGXtpD_Pb{@`u_$<#OIJUR$Nl+Z@ zxa%32!7!sqq!n#`olf9Ag12KTL1t;ng_0o0m?SVt_|P7BK-p*=1Zqd`9F53V-V+vi zkAUIGD$bQuU`#(^WEWfiz8ZjbGJrEvQ=Jd<$v?78%URPbLgR9-+MBwbrv@L#;(T!uWtuID|1A269f#;9^( z?ymdqoFp`|e*PC7{~7sz^L+o=tLMQay1zCS%m0@7-H`uYZ~H<1KgaWE?nQ5&_xJZj zEJNzQna!J6lNozhTc0>AM9zP4*FFu{nDryZmGP)I-I2^YQ$$EdBF zh={Y*s$D8YC{aD_8GWQb9Ok#=U(7$L%QOn5p$rr?5$vpC@wh96^| zB>Y0cGdw@R@O3I7C4xkBG?78OKvQWTrYK^~tT9#j1r!p&T}rFlkpUbkDc! z>7bEOf@0(o78$PxqXKq8-WZ7<1PNrM-C_l1TjQBrhl?>y5J)18ET2e&H4PO z1Y*}Hz185u2hV%-=n=edJ@VFCbQxs|M|l`UQ%TYYLn45(U#u20!G?d28?n z&rwL!t~kkB)Pml**W3YscoLB9EJ|6pB%`9Lx?{%sxp^W9s} zKanZNB;FbvSlO;sD=(K{@7J z6XllJZa?X7^?KX=-M!rsTdU=lR`9FZSNyMx>KRLq42&`wp=T}_wwLUNb`Er0V&*9X z0>3uv1n>+Mp^32~3iH^6NL{VMyjWX%ZzHBR7Y!?-#{!M%VE9n?dW=a$&hiy+4YW(O z{_MKQY+u2Us_dLk8q+^xg8l*Y(NPYmBwSzG`}XdU`q}b{Y1OmTeXh z5!t3amwQk|#pgi+pP6MG-!6*-59y%W{WT*~?Si>l4x8D!etxIZV`HY6RiAvlu>rWT z{8%h1f}gZxUgVd#J)MWQ>;DWH^q7803VV~YoTHE@nN)xuwQojMG6fSGKo(-nGi>VcMJIv4IVGlYnN9_#XL~X@->4lc;tr*e zr?4Us;eE{@y6ptM*WcUP*xBoCZP^LjxC)V(M&)YYd7A;Oyg{wckwqA-!vF9LLc-u2 z1&!UEvi1&trfqy@F4Ky|VrQvSBL}+i(*mXDiE0UbPnF0J zd|zNDbDePyYQE^y4RA~NNZxDLOj_3c_w6S7ZJBPgX@Ec4NVVg5;}Ki7tH0yO>HiNn zliHLs!I!(Ji&-z?$*Q*9OiGIOq}jAC@+brs)>1xku@w`zId+b79&kB~VahWYlXGp+ zr-W)(H$*0J9a4LVxefSPGcm1K*XlY%Da+m{49CB5h5?EZXF)jbx}v@-x*AwS4hnxh zM z^{e7QqR0i7$}JmJVD1F*Svc;)H6R)}9+x1M&@Rm-oOY^da2*Y@NX5_|kicfAO==rO zsaw%*DYp*Im`fGdC1bM|tY=K5Cf93-HsPg+bl2hO|Nh-n65bEmc z@u}`#6F)uHP13r&mO5!`*;fmR>(c40!}zaES^e`Sq@BIG<TFX8lkr(!5{e%AD@ zl+)5|tzD&QS7Bji{W7l1n#0s+^9hfN0Bc=vs`-Ys@y)Xhb;IvmVx!`P>(dY%B7wlE zt8<#aU2)Ne^*kiuawN)VU`#@+4U}A6cfiVwz!F{oM)%+3%xQ41%;|U<%sOB&3N-Bt z(`Hq(0%dS&BSfbrmu}s0DC4GaXNxzIHffQwS)*8Synl^cno$g`Uh)x`ivnu*qL-Q8 zCgAL%EETAgW7%tP8E0tVo>*m!Ht=qXo4&wX(-pJQB{Vrh&g@jDXC@+|oVROJy0h#n z(I<^qzGXvyKDHiJH6%;m$gr@H*9*gEHg(aIRYtHfR(M_OMQUT-RGtnkFekPBcFd#4 zG9LyTOK$bOoZqGAQyOVQ3{rvTCK_hs!1O`4D08qHE0gZX8Kjv=xx~PttF^%}r=-;c z;h1xolZ&LO;HjWm&MV{&VE>%ZL|c4jdC;uq2^_#_c@^y`T+k$Om(ZH#HBkL!-A3)y zn5ZWGH9pSwN~Tbw&je1Hkplx(6EH;)b1^Z(ZI_~KDsZJK!(DkM4)bc9B8Rp8tERR()oPxhkCcr)A_ZtoHdqs@4G z3paMQclVxP+#7CgJ|Rz#Y;WV%aBnvZhdbEY!>ygLw-fhw!yU{2KkRRA@7Tt`HKq+N zjo6BtB5afx@^*JO`u%u!NQPTcKiu5u??t<~{iNS}5^ro{xV=d>_IkY?vaz`r?rv?x zVH|JvqAk)V+x;Qw_ep0pfShm=6d5^jaFQ5RyI;9Sphkkj_=u}xffAWA{W#7{w8E6M z8iGm(qXJG~$|W^(X$@S%kC^cwOcRHOMimgT;FEm1a3ot8-k3p0_q~RP@lGm@=>iUP zrC#+Q>dMUab5XBaz%iR4v4*3YGEf|wEq$WvB@8*&>iN_-WYr0bER5V7ys5;FT+;u# zqnpzfnRJtnnOkQg=eieK_?AVvJhrt)UN>)cYO}tc#qv2H7j}e7XVhyRJ>m)na2S^x z67_d!9RLEM^%IP@x?aS;3VX;>VPR{AySSk*D=m}>nHo)H^fit1NxF&zPl}1wxzGW$ zEHj<$NjEd3Nj4c$Ho6#fEon`>a+vU8_q-qUf=#wct=J8mKe8%eIl7!KY4` zW}QK2qc`cScP2RDVru^Ey`r5f!!>Ul5VLJqdt(LvH3gx>xY$Z;@mO%o)v{c!+o&I^ z(hB-oE+7pkvolRAT%&AeHPex^#AdN`gBBQt&rM*rwCs)M*O;vXv3iV37$HO9GJJ9! zmz#~n6@3=D`lk9t^K4{25K@WP)VVx&x+@C_vm|kHr+M7k_VvEXRntv3l^G<~YXA## zS?dXHD%%dgr3q!ew4q7wr#w~YeUv90xTvMc0h0^!VDh-JtFc;$u;j72(88Vbqzhye z)}FTp&oJbZ)SB!jkZ7~jkc}uqG(%2hZtqXDD;fnw%9-wuV}S_S)TXX!Fewo#U%HX# zvc~eQ2oiXvYy`%w?HK!Qwi(Y^hhNJq{pSBX+Su*?eZ9UC0+mkVVZl~1Y6yBRi zZJlC8@D|ys=Y9Qm-!ClBll>pG!^~kNqOq|<W09(d2D*BO=a>T3-d<|Hk{8z*t~h zCd9T3ia`#2o2cfGvthUMO)ZQ0mU+zgmlEs<@IiOJOj%BuO$5Ge((>HW<9*UJCq&dmM)+gqCt z=f8cHrxb2gPI)>{(kK zUH-&wPH;zaL}hZv06KSDB{xb=>o;VP%)g(_Sp+9$^}M>wdhC{2O(45i6dxkQsKRl5 zJ^cRGz$^Hfu5{)Ok=1Rt86vle_FbOAaleDJbbiPmetcT8Kb9Px%G@=??XH7i#Bjz0(+pIa`{kMcXZH0Pf%(6RmzEx!SR3M2a=<{=5>zzl|Zt~}D(J+zZ tVvX7)jW(CDHH8dURpRZzU)nNxcpjdI=l6d8e*gdg|NpZQl#KxB008_1r-c9j literal 0 HcmV?d00001 diff --git a/assets/kubecost/cost-analyzer-2.5.3.tgz b/assets/kubecost/cost-analyzer-2.5.3.tgz new file mode 100644 index 0000000000000000000000000000000000000000..7fa189a3b2d18aae401686b627a0abc6d8a38928 GIT binary patch literal 142171 zcmV(=K-s?^iwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PMZ%cH_8}Fgm~W6o`~bS0xrDyX?B?>+~G8?W*>4`7)NPdy;%P zDu{$6#3aE0pj7H4Yt3Vv@Ac+M&cfaR2vU^nx}+!HTJdDMY!ckKZEWoOj)l^Fn$vXi z6O)6}C6zkb&@BCzFZeSY4u{W=kKy0paM=8J`26t2zZ@RFI6gXg{^I1t;lB(IkB5gx z|3ZdeauB} zA$^k5jE%^5B4Y(zFnCBvDbo>IYF((&V6a}Vqx*8k3|d98$U20?RMS)}qL^x(G6OYc zrWekV5h61+4TnkAyqLd^~MVUY+>K7dzo3G|PWuw7>D#f%K zm}hEG8%0#)i%#3dy=zn%KA&SWrlOz#LQg%lanOI8A*A%x|0mKqwMWywtU{^%vFV0og5$jOcWh$K|g1Hv`o zisV8Q0VkHG5&3b+@(SFXrivYeU#@wY!Y4B`N~LLUZlw92_Phk@gs~U+N*eUgCp7i}Z+|xl^ zDlM|US$$Y`GeV@-qd}cU^{XA~;|50wo6|DYky^!Ohu8<26)7V$NhDK>h&iFbzM8Se zBF|Z@i53yL$m6t3m}#L!EK>5sc1wix^La`x7P*itF$h+}2^cQV6^l#BCimRXZ9d=F z9X8{+m_x&ZYFqqP$4eI9!}8s|oe*>5n9){a1>imM=aR|Igvk|;*|${jcwFk`gvFBS zE1=fNOu=8Ihn?t@#FA0Xh<(0bIg>oD8us&QIQ{-yimY$0hu^-NoZnuJFVAZaZ^o0! zkJq;LqF}jN^11GG2w6xvr#an08!qPVT))9~ z?S~pZPDDm|ezkqs_H7@#(AvJX0QTdhikHlE&^376HBihi9m7>(ltTB)%H(0%tqFgKq*9yooj~QWkQV5=Fi&c2! zA6$wKTmt7XRV4XqLs$)sq>_f%W$0K$-ZCn4k{L=z zwLjs88xkptfNzQxLM+T5A{%5CoIxQ)QpS4V=H4&LS<2(;ZwAv@#D&sAvId#}Zs7Re z0DN)^Oxh*Q>4GK19x2$An~(-9vI6hETb8kG#w5^srb$_ttx(Kxs5E1mNrlT{aSb-A zjadYoe~;H{w8)Cc4)V8Vf8X0%l&op4fz&HT-dx@g#nO2{VMQu7%Nz+wSTpdUg^G`)1>g`Ud{6+k$F) z=;Dl3%tR7yaX|c_LtsFF;G1)veGKAy5@P?Jz&y_FsrR=#aBxeZ^yuaA<;lPd`fka> z3%f}$%wQ8T+xR&}@I_41Q~?o~3%RBe5XaITa}M0}kg5@7xS^QBS#qxp_co)-Qolu3 z;5B1q$i_j&hps8aY_Qz0tgyy%u&uGNvVu zmED-jn{Y%oWt!fIl*gM9xtL!GeIuD-me>+2BQLT7??n5w5K@nb*#`GOia)VfUnC>) zyX$wi)0^Ate?LFHn_irSHYwk5eZLjpI>=uaYln6yzOZJ&d>l4U}(=~{BVg|3ep z#wQ@^@MN$S@?I4*W@_-Y|NSEQ=I@BX6TbN2_@)(m$Co$6j{+xz>++*n7CY}s140Yc zuO<+cs=sDR_m3h93r5!ph~v&V(t;0GhlASE+iBOH`3XBlqh*|mFILCyj7hctMS~xa zecmH9j+s)^d$t*o@#*RLWHSB!{C5V`ew>goyd?K*(>`=&uu&V81es-XB==08O|k3i|R6L zKfzg^mB7gf^t>?e|g+#Zgd5a9Y4VXv%_P4f?e6@#jaJ;BF7sx1>P6lQ)N)JTA zB-KKy{-Wdw8=yQm2m}o-au9yaBO&=u@MO?4oR}!hRQh4#e9@O|#iU}MLRWj<%2xtz z_Q<$Wm73j~fgFrGY0h7Wm&nOw%-kt(m{Z6$yCTc^4(f_3c#m!G-iKJYvqz%TtW zFC<^_lr5~R)3{dvN=06O34c#LsX!!@cBptz+{?(^3Y%STfqO(;v3C>fR<*weL^6~J znwWEnH-T{ngLAlpxUbt23$L{#2(}eCpFKU;6Cz%q*P4e!kXVI z`a+dj3z>{au}*-T^vL+9QZjOBHM|dAt34wt%qx1I8kCMQ9!sIb95pg5@0V&oe*!!; z|Lt1^@_;GBDsY;rkeNZk-Uq_oUva5Rn)WjqFL}=JD$i8wW&R0#;Nf8B752-o$CsCr zo9X5G-R;HcWO{M+=6rH@aeXyCzdF0QzPP&k2EGN<2(Zk4_8;%gZ+{mwMU1D3*_nsA zy&fr8r4?FmFhQ8b9OE)lJl}XkoWfrXG+hja(P1>~8!@{-JnkP3`$xm0ya&R8kF$Stjb%{)yx%YkkE<=5lizq>u3 zUXHKEZ_dxA7iZ^JcNcfRo1VVCIKMKS>^IeCcW`9)es*_$HMXDOG4q%DIAzk%?Z)YA zRfba2)|{DK7@`tarY+!f%-d6xl-)Dp)hzFvC98I3UJYM?L~(8P&HgjIQYR^m?|)>o zr4aY;LQ|M|pdw{nqd1JmObzaw-E;7*|Ls3_{OsxYQr$6{skW{^n4f5kZz3_D^B6bW zKAIiv>~L_A$0FnT!ovNOk->hQXOySGA`{XllEu6*`1Ah+7eTEBO%jeUZ}+yvj-Zc8 zIWzLnh@66^o7lg7Q^bgHAU$$MdAfmUW6CC?Y{io|gPJ^O)PN~V?~O-AHoxCPAFISh@X#eyxuQi$TRGa(xzB0mAFU2}%@>JxSnqAmEe6eCu z@`R~vG$A1&>iMtx2zv*%1TBtsHc>zS@iaf#Kj7i45Z~$Jn`rNzzqh~iMb38P{Vp6n z8V;kEe+T@15O?jUTQN77kp-3rhK58V{uY|VYE@gRTX$l~nz8$ILrOTgPYYUgysIdUh+FD<13l)27I#XsMv?4(53p>nqUb)TwCI;D{Q^M!99#z=oi9z&C zNLHI?U=o7w&NyAG<(@z$tf(@REZQADjGUQOpr(^Rah ziD^Mlq3H~pVb2!Ym+)CIFrv_=yRE~s=G}($F`+A#kXvR5KvMzGDuO8UL0|e0;BWi= zgBp3mB*A(pf)GLjq&c-tYyIxR%JBhQz$~tfNZYl-ZEg6(8nZ%V=xUMBP4(QJfA;NW zM7o{;R1Lt)d4|4uBhrnjW(y$$2t0uSC}u{kgD=fr=8G~bk#q*V#`Y=mi(;;5-~c33 zA_ZflZ;lj5jp_%trk80E4GXRY_RBskc&GZdPe7U7wOm5?6QK1BUOQ!E!ByXIJ51@m z;y>|x(O(@k`Xt0@HbVC=$diZy`D+Xw<(rkg$3u@@1(3@TKZ|Fa{*F8~|{O|V%`0%Jde06v9Y;^K!bn@!=yYR*eTOW@wOP*}w3y_I+Y;pcIEZ1Bz zJA=Ix7+dPAs`bkiXOe13lx>b|{EW7GE*ZEvUS^$jrm zhkvt8dv}svw@jVtxD?Xhn-0lmXB+DR`OCZpXOUB(ex8*J*AD$FOxK@$DQ;o!$E+YY zWHnc@*Idc1E_$`qw671lM#2$NGF6B?0kagvM&?)SCh%9U=3y9EXF5-)rbOhCVSEaa zj9Y9}J;G_$uwYAJ*36Ly%$jKELRlvMYHB2OloYBn#^3DfF;oOzR(`Gc}d z!V>&>aiHT$4(qh*i`!+0*a%BYT)=4gssQ=T(9X`2La?>Ei}d9GrbT3InP zNI4s2_*62QT<7V=boZL4Ol_2AS+Ec>E$ED=Tyy4+Mj}N~o#FA@x45ep=!B;TdN-5P zF|k1<=H^gD0Jb8!ayW}d!w@D8|jAUTvv+&C=C#KzA*i3 z1J$CYTRCXXK4|CZt-P~F9a!WLVSBx6J2?y=*&VMv^Vk(~NsrmGM{Lh7Ox=U4F+DY+ zEV_h@NKE@?myD$5{Fm>R3|(Wz9Q|5dU$$H5o;_QyabKYLCf<_w7TZV!s?Ixvq zE@Y+%SA>EGDUzlia z>#Olw^n;rVLk!?tp%oMzz45NL^F_@Qx_i@_+p3|udCUk+4Yk+=!88_wTK4{1gGiH@7l63G>(B5pd(x51lDv!Y#Wg~q_>PF z4-V`W9MoL&A)~c z@jO^uPhcxn&QcP9L0)bQMkG$~D1=l!|OLm=u4N zP~fu@$68n0Q06I|$tBy6gw1)*EHf?Ec~mhLZJf_emU$mDvWFc(?hfw_#E-<^JK&{u zdNDXXBU*@5k(A#vGL{QLPS5PE%;RMD{Sq>cApxDwS#0ju5H^TIo>rL!hLFKJQ{e{c z7wMiC6PlZ4TC(_FwMogQ4XnR{h6DD2E6ohahIlyax!N0;BeGm7YU;f&a?KVvls3P& zwm8Jv2jQQ%Y`T;)k&BGSyOz=4TD#fsI}ar6!NQBZcT%@oA%rIkh%I1PWvi!V|V2tf_KP5LchDXc3X|+2zI6 z#pLdGe0P0&KqeP==TmrgK$woBr?yArELwVecl*{GUaIXOw=@MXJUqRqz$b+JsD3o5vO4w`l3zTJUFj1~(IwIY}qh;5P{X4NHb6~A=jXaDj z9V6=z!<`-%fAU>pC3@uc{6F4Z+@7CBgnby1Y!kU*MLuT5d(1U}-GKkyZeS5^|8{1Em2#d8`uPA zV#N&RCf4uOn#h-Q#mGI|kgj>BqCY8-cdM(uXGO458fr>4xv}d)v@?P_t05|Api!U_ z)B>D+D|kSYfh{ycb5!mtWpG_ZTRrVq38O+ zpcHj_@cP=mh4rj`JTKF<0Rh>rZ9py%rUtAAt&%CMo_Pxf{9^aE1M@2ipcpb|hMsUS zKzHxAmJxXYK(-%3>j@7fwme*2HF-F|wJXei6cB9vlSE0lx** z5Mdc2NZ01SVorEw&*#)+bjFT13 zW0sJD7vMq#(5!N$kUVDUfH1>>6p|~vv)qDr%zK_MPSwgxVvB!q^k`(&;(#aS9v}ZW z=~|s32IX~2)n`=czBR$WyS*3@^~_8=?#F7?hfpZn-4v1!&?r_ba1bLan(_pa5Sb5A zSWzpC+&itV-Vw0?`Us$pGKf{j0&-PeFXPL%vG`o*&{Oh)fnL+j%%PpcUEzL^Uq7=+>P zz$rN(gj?@mYir>zW>#GoYq$ASl?9V9)b7?2NJbOGZalOKuHE)RSa!lj>9T}~92Wl5SL<#NTg@K! zY6;gD>AA+-c*17Z#1ppqrGVrTN21z@rCG`kn4}4VSe&tjKe7YdYTL16I_(4UsPbU7d*e+W{x~k8xEj~ zNMG`JNtQHEQik-1%u@)1NVtlPAh-!&{I0druT%xkNbF`=;N;ZMyMt=`>)Z$yAbI>? zc)8I1qNcxJ;ag1L_8-mVVQ{~8s=P1!kI`QtHn|%&5|?7apw7LAMiCxMSQOsrVdzd zn-$Hnc9n>@OmN!Hg6R3di!VrFR%URt6|LIKlzVKiBq^?Dr%$#|M>fT2nGOj5ZStTEO3P=Ju%_Knr{@u{3orXc znR+qFe`51un+?ESo>9eY(lpSKKEWg!Psyy*K{&T*YV~@Z9xedU>9WWqHlPX!tR@CraA}d7Ba!{*=qaSs= zs_(T+4%Z=D89dSQe+k#&h!_M(Ss@Z|!=jfW0p(EjxbL2FMdDIQG&urA^1=Q{BF`(v zS#uzROM1(IZ&V|4WIAw!%zlB8J3d88*JitO(>U~5?z)|aQ-(BGuD(rf-EmW6@S7>o zgU|3#F?xxy4>Yz47HH=LX!1m|1y@>b4)DzZ(RA_DbDLfn3qB)u(9!mwM(A6}K4?kL z^A%>`bQpT|=Hlu40)&LlgasHI(~O|efRF_yn}SgImI7# zoo=w{OwxPUzqNASw0c?O%$;p6E=n*Ld0SP+G9k^0J2|=JU5`+PSWUE~nk_cgSKV~X zbEeFpoi!7e26>LT<}^KHDcww1Eb>H+2)b<-OmdOs-XX@TtNl9vN>THzt)PI5lNf9+eefpk05S*gr_=5r?1RPmn}A>`!n z=>C5-T0sLs$nnm%?(3tI=l*?4<$_Huw~maqhjmPzAKQjYp0FyMIQ0K16`Pl-Tkk^h zjLMD6lWu+V4Dp?KQpI!hLuN_snvHx%*RojBoF%uFQ?u`G>mBADBq9&{@7sOf9P(6kvo;2kAjR4DJcu3C8L>g2~@wf#=mc(uf3i6n@FvUgc+u*H~cfBi4hl(p*UUejm7#;cy63vNlv9cg)Pce_(LO=5ucWZU=^24KVftJAu_IQ_Zcn zZ;0CM>jzmJ6HfQe`qgmw0>=qYPkqEu4X6u8s{)et!hG@U&~Qm!thltXq2=5JGz)+B-bfq<(e2K?aV5U&*(;t$g9Jhqgjele&TS3yb!l#&Y+1kv%Zsy(wO$s;xiTt3_NZBBH2rbU&rV8H%X!p5T=lz8+R}Iw^ z7vVb?R&5o`BQ+w&ui&fKHHZBwGfqyZmoY$1R0$d^ox2zn?Bv#Ny*Zd+*@(wPB2qFd z<9p^}|6sjdv&34!l@^jNkPsp?2>#5q3gj3x$6$dULP4!-`b@~&!RbR2sK6CH((8U- z^&jM!?CK0skuUn1$*gZj+izMcdeulK*aRgaX9sJH)2`jtD!3)GVww^7AUb5lDey|L zw3aZKsaC>R&xS$-ArzpAGVk$~HE>X?ovAc>hKX?k&H=&#eDrzWeJ-G)l!Z_j@bizd z!hy_*Nzc#!b9{O8)+sqKsAclZ{)tc>K_eD3LlL32QDz|6q*E!1UH)G6tbW(~CIK6P zzG?oA*(Ie(0nwi^XvM6c2D~cT0=U)nqV)$^52tet3{Y5_q!3DR>lI<(&^$;>Zv#l( zh3khACa&*(y%D8!E^X#o?)r9N!g|{dAJy9R2xO*hU3hr2W9{t%Pc4_@y}erA*&$T# z{Z&|dn$ZtaY1mK0PYnI=-kjdF4(`qA#u&c&s!(e+hmkHj7B*nd?649@LxJ z3KmnIn8r~QZ8rcO(!?!rx&j;PNz3--@UoG znoNH~nM3fqtQ@OUV9hM>Ey>!~uq$I#uB^Xy=`(aUx}Iw*vRYaN`&Y+cpzmlfbBk=@X## z3Brl%Awxa4V6H|lp5wm+zjtkmsp1-lwcE(E@!k0B`ptB5ad-X=P>QWjXu%!1ZGJVr zxiCWU&JT;-AKu`H!FND&s_-3Wg2qyGV7s@nivq9Sd;w9}^=v-&TNF8-@Q6jAl{T2J zo;q`X&1V$icaXKv6f$mdy#)3&M^?nfFTCY8!vfTqz^GY=>!J{;SZr)G=1(GL2bOJm z3Xpxu1Vdl!fZbrs99%4jTZwIJ4Fnuj0~wL7`K}Q(;=1XZZVue28jWyVyWai;nYlW< z*4<`Bv6+E1)yfNa-m^+v`HamvBXTS$l^fOn#aDa_S1Cnnrm|z5W@!Ptxu6ngs zU#DWB4(!-%stkN&P}V+%D*_@PQ(M32M@E*SsQi{Q`X?a?&&=*H^wOZ3;84ILQ3;X% z!Y+4Q+tKr;$)8jx7z))F6PUa;ua0VeuCY?LLoi#KS| ze3Ys8Xt>AL3%4w!@u^NOe)iIT)(by8I(crs{7h@`u+2tmpKcpzkA}m~vj~4mcc7mg z%LU%+uxr=8PV>0p3!ECa-94yTT4`5ur=|-*O&a<{war-A_#c<-% zUf9C2-*43rR!gzAITkT<2588@&X_<==3==~+<9jrM5;B7mpF7osFl4MSg~XolQeZA z;Ztke$r(%hq9t#xDYr4*@nmv-H@&zzef#e0eERnKbo_RDb}{*WdU`$C zS^*ol+99OW@ipWEACc~FAdP3&?coh%PlQeE^$64HJIzy@1PTg<>W#V6LYJ-CCUky5 z<2!_6OPVVP#W57F*F#In>d;JjbySN-G&C8Sr_=>P!F}#cTHOKDX_q8iT1L&lLlO4C zVM}-5rQEQkZMN<1^?4efN}^^u9cUn){fyi616Z@H)9ZEI61MtzHH4tNOw%o+g86J^ ze>0dhVJV9tVJtk-B4x4`JHEeU8PjR0vGj&B_O+&3r1QBU59rO(=@F=sT^$ylFC;f)${=rbc8hh2`nE+S>pH-i1 zH`J->(>M!5zQbku;=l{4mu3&UgYF-#`d_lI6v?Z~fmjOK*^)7U50SqRiR$;snxV5K z&9QIdYYSUPO7wjvpcxGaX|HMY*+0h%` zbbmBUeI>N7gbOaxUNA+Ac*bb1>SAdX?zH9wONc0;XiOmt7Yia3UQ2dRr-=I9e%JEb zUN%oD&vu2lLp(Ev0N9WKh)S(0mrxbf2uZ87tJ}n@WA%~{U=Fx#T}>`-yv6+0@Ri|P zG?fdcZ^B0g;qU2*tp^d+e~ZMP~+vHhK?oj)+@QLK6E1>YLJbs#U4^OU9`gYjU?0`gtyaT! zThH+q;&oJgc8XR!FXe;7#_eQWOKKSy>~5j9B!S|RVDd-JN~2=*c5=q8^9#MZWJ6&9 zT9uy_xf;iR1``a%17=|BI~C?ZMf0Ea6iDaT;?!3NyrhMXsSJHHyjQ*Bl_yYv-(~Z$ znz%mslbB8G**xq#mm(K(>Ih1hU>5*QxzBfXnAP8CmptJ4PK+CFJ@&t>L1gB(Z)Ook%IQ@|xAtil9K})1 z-lauhwf(&w{MFDP`;eUOfmyGX7Fb8zs@Y>&mDc7tDBZ!C6;36(szJ{_Uq^v%aL&yf z1EA~G|1nH17aG04jW86sK5p6E32VK!I>We>gGEvL0xE-9DrSS^_}SredVF{?n?tE@!@m&^5yXO=-KdO9v{--t7nuR4=~@>KxwL3zqXEp;q&2e>U-yuUvfqsTwD}Wg+5!1EqUSO4V?X>jYWd{f_|qrRuj_~I*MEJ)lc$b(bX**S zaa+~W#Tl{`87%oM2UzZ@~l{h@O^_UEt=ifgb){JEqXLy@bM3x0Jy z(Zz_Yj-r#3$ki7w_LVAkT9wmE4`;KH6q3#Phbj{m)~gKv2GyqZ1OT4;bxOm#ea&X} zdhMrt&f-mMP2#wPL6HP}5Y|24WVjfJd85zZRM?89C1#jx`L2PS6`iY&MWYUFkDQ0< zeJhir;?A8;dv1ez{k{ecKD%W!@sBF|i|rk(dr?Lq_N}sm9d5l=sAC0R6a1+FUgTT{ zcv;)1Eqz$yP2=HrLP1H3wh#OvZF`AQSL03W1T}Z!YG@azi^(qkk5;uYX%R;K9!vT* zR0R>Q!mK=5DrIW-22j5netu*1yRLPmG)3)Mc;0*MCw^Pm-)a+RXzyWYB01wqmSB|o z7HTl>U7w78c$a%^zC)ZI@a0LFGI`}y3~frtcG~AWH!np}=hzJ{1CC$kF?(PmYRW{P zCp8OdrUXJF6!bxUPs_dr7Uf zX%!-A;8v(Zso!q>0byP@sNnKn_n5Xq&_%H(TB$%ILkm|fa9)MqqF zgG08fgO)D5+U&;0F4?(-2J(x*GVS7#>n(gf#b+%qUvHVI+geRZz{a2zE@{nn-Zqk2 zgT_E=?{b6H|4fZlu$C>6L<_Q39AUm`XBYOA&{isel& zP`4^O3zxYDfjQtKPJ-hhZvraduKJ6ckdTKKfLikhOar@)!nVbNQ7`30f!{Mu;V+R}&!IGAS#ykki(jr%gOf#AB99H#>q%pgxyN^5@4ntnsvC&}U z_i;ql=Y%}*0c{NgHR8~#7Z?L&QyTRgRBxGvASgx3f^e-MOMP`gVIYpp3M+Z+Y9N9~ zfdw@8F1zGOX|tqKa`OW?_8o-!<@39@fz zj9}wpX0RH^Ga@8;b2^E_9QFPt>-B}IqlFl|S;n_@0O zSg#)XMtr~Z7DA|+%8FIZ>|tVZU1A!)7+ zKgqK)^F-WPc`;HX_FubZX^7>jg_#4z^e$p8|@2A3w&SgX0*0$^`^wQRl*HkB8x5G{c&)Hg`ZA zRxq@?Ju<6hIeq_d0z7mO%*Yoxn9?8@(gLn}h`_XVdrQ__Ixolo--ute`kOt6tQL(D z?$Fq9#@3i9J8jn1wp3^zWqhu|+^gdupR1^`q5MIAD8YV&{$}fAr@f*MAoA+_~jnGPS&6Op>hv=Mn*Xfg{>8ua^Bd(XeqnLNMaCkCGvdZrQA^yjmd1V#ZgE(*JmCSsttJ zn)s`(R!kSt@=n*3gUG2G&ntEMNfwBf2ykZ?aIzTH!RODG)@{gEcvWF7XPa8zutGQ2 zlRH~4|K07|4yjCMx6JI7FWC~fuI)Udx%F_h5c(28*q60C{GnUCGCc>+5U_8;cG6H{*mkDcL*Y2y=!Z@pZ7K>G}vzM5a%$e@a7Y_ZB)BX z+69VGg4n^Ef3l^mSMiY_$9{esdo*glhQAEU`Y;Fb2z_vM?khVo7Xvg6H){XwV#I@Wy}|59=eUdlgiyY|5tdY8eSIN1-=D3(y@c zx7WkfM73@SEnZZW`_`5Z2eFa{kGDU>kF7Io{jqsblir7&Kq?s9u{5$4KyY`NL$ZWG zi6U~V)3X^cU24q$AY4k2b=;@Xto49C_s&hidvz%r=Rdx=u(*tO;nM&#W)6ZQp{hdt z7$AX;Pzj&U87K=IK@Ya?V4wlqU_#ax$=3pc2s^66HLy)YUh~p)#s-A=9vWq|c<(&s zAn8~fRxz&c_E(kDS|4If`!P*cTnYJAC)iG(_5d;^dgsv!HhDT+8B(0bVi98z3K^ z=%&h3z+nbf}KRT>X8utH)d zZviszT(GaBy-@z5NNLP4kx-o$2d_qCKnp%t9fFji2ERNU9Us0r9zc5GMtp@m6D}qd z6L++Dq{l)sp;|?1s1KYx<4{^Vu3Q*&1>f46}dS%{{Jf z@4&GFA=2Z!=>vPSJx?>G>2%kmyvAo57v&V{zfBc7d`uy2?O&@Fm=R@aqrIkz>FI2v znF^ZN(4(o<^-Uq??=~dY4AMnpTtC##fh%Zy4^`2nC<>N54ELcA?38fc&B>S*iRWsn zmr`h*vSfCI$ z{Va1=1&d)?P||im1QZ4J)E**PMmZQ0!RZUAfl>FkQapDu+m`EioL% zugUv1)8hLPxrU;m*m=b?`SQ2aR(lq6o6l)KP{R_uA3<^Opiq~e6x_@)LkdvkpJ;*f zJ*s~7iIlZUweWcI`Y@^TjZ5iqqw#ZWrJfIW`Xu^w z%~|8G&9p<9Go z zpZSmmaIvdTtLd!XZGqmaME?Y%!6&Fo&raW5PG4V4&(43CzPWifeRp^9_ODginr$i?Jb}dK_^OJFv2u?G6MVnb$iga?RI&CZkPbIk=Jxl_?{Vr1 z8cbdBv9Nam9Hl)E0m#~O4+@JMJfcZ+G50s#Uez7#-Bcdc=#gXa%^wjBr-ciSXNfuV%Hxg8L>3G~taHn%!k^nn;*K`cVcQ^pzd;@hx5QO!!@ zo@PGcXlkCfG}uL`ealqPWNPDWWdn+%&JfIzGd*Pxqqn0KRp*fydmHtgJf7i6+!xk_3IQKCtTfIqgX7mB4wH-yWebmX)oKhPn)m8Z>z)w z(}V#(1*UO)xqvfL=DPZGYc_Tpzb%a@Y?q++<`ha?w7j@v+1TAQ?XO=;w&Ts6nL+q! z?`;h&B+a%*5ASP_&9)Y+IOI)=Rjq|%JeBopcgHaZ(%ZiOLks3#%i=*JCV2Gv04piV zq9vH>^yXbC%9knIVhY)ZLXLY1bNB6+ZvUPqYbj9<9di)tO2vomy{{sJ=-t#>9 z#c#W3z_m}9b_-pIoR@Lyq2`U_K13=@eWt%fga)g$91QP%Vh{w7I+fksVz zOI7JFDS;Q(3>#W|R4t=MkCe(V1LoCtgK?O37P9SDrUMu1_gQpZ(m4HW=|WQ4|X zDZ%DJsMeCtO3fTpTQbdOrGkVrz9eTS$b0?RqBk^I8L=U$msB6rwD^s3>-OELD>1Yc zMi>Nnh9RDdSVmk7+@KBz?KT7W&V>sfzKEWOWg>ROeYVARwnuhiaxP!rhIc#!F}0tF4Jox{jt+ zTVMnoGO!DDeAydW3Fp#z<3Q%M1=FP9@jXjOSs2;_#je`&KD#8+ld!H)MGK(53jjwz zxWCO4D3k0wE$B?F*mf}w(%oCEq8lb#1FI>zE1|{Ejl-;mH6G(66Qb8VW@fC-JqpX2 z-AOv1^O(F9i-j2{bYM$?ruG?Bvk(P2Aqg{BVr#MZ4f5I2dW3<;)3YJTcwXA#CmOr3 zJy`o3w0`lQ$4v99=L6lny;`nS}4iOde zbp&{V{)H(?O7O|5HAv(nH8j`> zIci@4LI?@zyGMhZuK0o)Z0_qN^UwPwA7GYte|dUpSmUP}6;EyHQj8kO#EMyOBlFZ8 zEk_feq^VC)WlNl%42K!GUSURn6pH`B1|_UuIhfIXLTmn0m1o%%L*?4p@qLa#^g1+a zNNqD5jit*i^;u7bP8@V#kA|0gs{pA>>*21>(=A_Fdjx9d=fDs zy~xS$Mea|FBGgx4wMNZ>w6R50JKtIVwHpCC(()Y2TRZ0lp7+h4fpZIaD?~A)@x9@` z3~UA;buY$OV{&n`I>t&)H0|G%vJi^7cI1{VxYBad4Wzm5;n9m|X#Q{TvKw>)fZp=_ zp1g%ZtPaMcW^gp$KYtY+og4#%!{;H08_8FO(evG7Rv4a>+t;W4!&iqdYjB2s80O{@sF_^God@Doq2g z+)Xa>O(q-*8%v>-|J-b_J0OtoRJtg%1yKAM(`&|Zd&|Hf2=UOvq4qn&{vLQ{7@$KY z63aFRsD(Ya8p8JMnz4P!^~gouce-YR>Rv3dX9k8N06rv#G@ovCutco`yl}9-%w}0& zcKvtnnQh?VVA()mw!mSn0$ceTH~s$|uTo8+PJQM)HCL8(mtt{gxVR~uYbK`+5D7Po ze!mU?cYVg@7-!hWLiv894;nBGCWY%#cb<>TSAdw=QXr^23OP z4k5dx>NBeJVU@*vZ_==n zu6n%+A-Khb-aC}!{{(N>OKP8 z(Idc}M2AN&qr;P^-WXmURm?=EwPtrCjb|-GQCl*lTJE7;38e~=IK@Dlk_TWX=*U3R zTI$Ljk6The?X5LllN|#!k}J+<=wKm|@PUWbT5Wg9XsVZ|OBUZl>4fd^wq~CT)5D*6 zXPLTGU>d!1l}^7KWBpQ|lWz@^XRz^CVU>H$ne6m>9Ve1iP)8k~_fuhNs{OT!!?C>< zlEoC&K6CrKBJ)ziNjEJDsx`zq&8T%{H20kW<==}L1m%I;;LGfH=2ZuyW_9R} zFS+Y0(l^R{`FwHaJ>E$x&zG4+ zk9t4Q4hic=MB7#KcPs56Yq*)~YG}^-s|VF|9c1+_W9wN7jMOnAUfTfcgKNm#ty1jK za7&DRHbgnrOX*?z!;|d~4&QJb?!<7hRU{HwQOWWC*gIAzsLeJ5j4CK7mvED@eC74c zAg~(%<`6UK>O5N-Ur3%&$M{Y1S#HXm1O?j-F14X z-1P)&?2D^9R6(rRV1Y3*qwzh<@mRZ$I?W7jA6xuB19{s@GK0A*xzT4j9R!7G1L|B2 z6DpD0qK21DFNG0Z(LIYPvJ~sal&~dBB64ag3y4bX39!0~=ZlmTBI%MRmYJ!SP`nYz zQ_>Cmu)5@l3tq(&TzQ*3Md6_9Vh+0z#s!s%1rN<}(ykr@L1zalH)uO>>j70GoJJ0h ztS_zTQq0g?t(k<);;0%9mu5u#-2P!MhoGGA_K-%8Bk-?|ac+*hTsj?`9)Cc5! z?F_vSrW&qK$HlmV`d^Xv^~I}g`TK1vl)Y?_Re;hUjamr-Dmsn=ikd)R>Jrt74C|)w zIY4`81R-OxL*HL5aB#2vot6Dc(*@5L=eDNW#MM&snVZeQL@l}*V%N6s!p9KLYE7ks zHPuYc#X9u`+6WI84FzI$L4rs=f!d>`Su^WOB;!3dFp)SXur;hb1#myn}GD z4?IKD>B|qO9pgVSXHkPWDotiQU$`)Rh-&XOiUvj2MhL|l4{TN`+5Wwu5s57)(<5VV zAi&okOTrj{1^MoYLvkOQ>BpyL9W}i@0iHHvbFH$ z$tbAQ+=?8g2dXJncH)YtA~mc=t_3D15XHuEk*2L-KA>EQVMYKTjxvSfM4;xR837M2 zud+-QrTgvz@f#O!h-F*ohid$W@p+Q*9BSkE z5#;UCvf$f<{KtYIfY>p`$ za%HYpE0~A1s4Q_Lno+}x=4E#F?Wrg>H-0Zhs*fp_K{B!n z3G5`!GK>jDD(2$mf(!uMy9;+kJ%z*>LAj?2Sr06S*|07JrGoIbtq9y#%2}Npmrj0V z+3(2m39!Z}#wD{`Mw2_7kSCz6GsUt`4^P#cQUym*%}bv{-gN`GSmf}YnyaykN}?~p;8%?dy4PphM%ZJW*AKC*Mwb)`cSk19G8j%3#tKK`&%SkL z4O`mDUqnRA=M@`pB{UoDJjDCZ5)L@Zqz^AG8}e-nj=nS_s-1f))lTpG_qK9Cwb-w61b`e#SV}avWd3%oIuvP zy3*9~Js=#<3=||c*ZHXdQRz;r=sfGxIYCz4mGiGvqw0o4-&M>5LtohX7Vl|P_L1NO zp3S}_@S~T=ZEsgl=r#10J+F}*I*nv+TPOMR>cFt<&xGl~3h;nJw0`%Xo1Cjp*Fn3g zjVidlJfHqBzP%WK`}VwplLf+UWNVSJDg4p3V;Cn16kfFEUs&<>>kdGL%E?}vda@TP zC_A-!&ZG~{%`j?kPiyNakl&58=%tdx$ls!X#VODy#}*0gZ~14CY-e zd$od4vvSdKb}w|AJGP~(@ELj|-?Y4`Irh1iIOj7oInri5j~&}-WgUNWqQs{(DI|+o z!t&U92ey?5sr6WMIJ7EeU%xc|GJ|-iY*MX`j&rIVV{$Gmsnezvv-U&V?y8PMemz5t z_r9~Sr5UfUQl%u{2c)S>w?}4&@vuHAQbo8EA;W6bRt49vsfEM45sBB)?;BrDk6(j; zhYUlmAEqH#_$i!Y3m7P3_*hs$tsVq;3Uwlt*+I6hFlYd#RJhC~XDdePGFL>t4N5*i z#y1d8?g@%hY=4Yen_mMULo znrv(IqngUoH5*6CCHZYlIp6;#=#WEpM>E5UH^NMh_33WG3$^$=AYz|=t7{Dm&ekBf zT|2FL&9+{&+gGjo6gcl&su}Ws_BmK!euV1HzzDhKWxoBegrRw9oUtt5ybWe!!I7 zBZ9@VI!%A1z@Me&y>c1aWyu+*i(DwpV@00Kn5Iun5s=6Cp8U$L@l!Hm8<8gn%$JXk$d9NShT8TOIJ)rVXuHpbL)C5N*Q$fz?0dn?FoX+2wqyzZ&-aWK)hFM9 zTCXvIn^G;Ug52WLSE@Re-?20!KY%v3zKTQ`T2mGxCy?o;ETT?TE7Y71OSsHwQ(W^E z){2oCnhy<18G_O?P6YyENrcSR>{dzt~UpS*BMyL&e1BYX=rM(2K_9!L zmtOehb!+k1;Eu9&Fby+-S-g>@yyc6fCKLV>hQubm-CZdSubg)*QdZx6R#>qk`97IA z3K~A7_<3}A_=uR|U5>vWFKbsR{QbH(r`%Srx26=+@PcbWEli8khcNS)rQf_S`)L+B4OT)x^5q&~AdI2(49jk=Fps3daD^HP+GSi6|Rs&Rl-7Sg-QieL91t=vSb8?%p*8jXf? z3u9{JHGnNeN^4J-)Sb7@Yd;C(zL_iRI+cye|SFg@S%JgI_GtLKI~Jq>L-D8^D9 zFO7VFDvhdnWYycSz8>7g8F+$OKXNW5OWBI%+Lt?OVNT6NoXE`Q)m^^!Q{4{kb&g@S z8W_*35W4ESkO236<@z(9i?3f5utys0*9$MRmdk)N7IG9wNIRNcxy!M^2os1aBQ%A4 zN+5Y^fzIQwd$_UITL6QUt7o|SHgws8gX%?_TpKLI;NU7Bh!&xvV|R7b^^wniIQ%2H zmYGfPu$(I-B`TvfMTwZZfy_Be6YzZ%ieXxjCPPqPx_)EfT3=jM3C+z55CVh0D}v)5 zR(LljplB*mRtxA(xza2F$$R68%JnViGZzeeovC)IW;`@7gGO7OOc?5AtxK}1*DtEt znf2UFJZQnyz>cv`3+_L|XQ#E=SBSd!W<>s4L$MHFyL1+Jf78ItZiSoON5N|#tpR#? z^sEK03v+xN*kG7*A)4TRF{)XdJ{IDP%I-mTMt=&v^c>|**8i~oY*?`!VW-<{$4QOl z4{zW9sLW3z(mj5;3;`MT7(u|_wDx>y^)M}+-;OW8MCTo>`U0+0(h0>Ea~lU9q?R&N=7}Xc zEO{Exf0X30%-oW3=~Yb5098w;CC#mA?o(IyMzTejQb}f1ai3ELRC2e>G#jU>SI^K}xJ`HAZ!u)hdo0hmneyrlVIco*9;JhqNSTG;TLAb|f>?wq!I==FG|k!FOcN zC^DsQ#X=Dp$4sd~W%)rjQnb`Ww=6a6V{{b|;y|t6N|b9u`Iagk zlerYRW_dzD>!>m2y^ta)V?B0{!jRiroWj4;6;-k1g(g_ixI^%lTR(l*75e6GKX=&~ zd^dO0SGwAW67JA!#u5w`d4s7J-lSqiQ!?S25z`*h5YYJEx(JGS5I?-53vS(qtUU!P z<(VFWfN3$GGimtR9WX}ZP5(dLPp8w&YiFI=y$h4?DM$r}HB5svZkV6R8yVcK?R3c* z0=Ly6s~TQD!cVC(V-RHJa`6Sd!D+W3RU;3djihZV2fcORFzXOI(mRZ)9g)MK+8ub0 z$rEmU`7)=Qm0M4mnGf4df%TE>7|YJl>>f|sh$@ThC8mJ*pB@%G=->yr3PF#A-bY*7=FsT; zl_KAkd6Ke(ypeQHb86n&TAcPCF!IoBLsndIIK-C7d-|R+7|a5{AAGWncSoL5^pxsb z@E!XvUS(Bv6$oK-lk{Y+7Z$p!!e9JnWXWt8atKK+XM)P)Qvg-Ba12Jv>Yl3mpR}|g z+uo{=GVIuWYnYBOvUkgt(*XN%5VtKu4aX9@Hze2$XUuLe2exG-ewBX(&5Z#awU?z# zN6k~x4?>($u4H_Oo=d$@3>*5@J&K;4M8ifq%`D%T=XJ+z-X?M#Qift>nqC1Rx z?vju0NcZq5V0QTHJ;!%8&xe2fY?u2z`0?$sd0KMn=pJ$Q&BMd!S%kXZVz+72Y=U_` z!6L<@&MrfDyA9NzZ08LCx22WT&<)K|3Ak;1Lstsj2qs3rA{9A}$d6w2=TZ_tNM6+h zu2ZXK1s4Ge<;=IZXkV?klqrl+4a~3AYS}CCfz>KRGTzQivh}3q#)Txn>Krd_Iu(+< z#QlCS?^M$W{J0D%bxX02nu(i%A{AfQs|C1R*;oUh98Smu8gnTU7Wnw}$t|M^L|puowG*xB_(dx5mjy@y8sh>Z`>{Bb+)KBy6duatK)t=CzAMwq)5 zby9@qB%_5lC&VIOF{xc{0#49<@DW_Km9eZ0Rge{3D)o8x9KmxkFm`Yif?*RF$(S@`r$A#Ac7v!u~CCPij|(!{0cnXD2y&{1n~H zI*udtN628<2y^zNeCCC2>*xz8DHh7L5Q`fB7~FeX5bT?d=kC1__W!f@@9%Bf$ig_j zKl`u1qx8IqdrbKnC)Im;zON!HjyATXmE?5$-R`48A}q0{NS2`NsBZrE&x65*APL@N z#c8|YIo((!Fc<)X!C)|#7TPD#EH0>h5^|1I%8JUf1)1GWZv1eUo$NS-Y!c)I@ZHN06$k&_+U1G;W z{czq?x>N0>uAkG@qzfNyKJ$DDJP!$)hK>h0CO}8*-+M}5DxAEUrYM&9p2D`zT3b=l zz6bATlCq3sGEdvH)qyKx7pcG~v6rKsbeM^u<{3bm><-|D^EG9qIz(@7jxh@n5;SRqJGBgT{Q=G?yQbV`}KE@H_0hVl^Ns_#R=28(G)9a%%6?-EaHomXLCiUW^ z&jx!S{1eQBBLkap%VvmaH9Ce+Age%vj3;#FFQ+^%GaH_!JlrmqzK9Jd4x*4TQ^l2n z;TfH-JPA-Y}0mwxbM24b5OQBVH6@7Zfi4xIkxb&D+Qj=z3g&+0fzsAoX+0pS*YS@P_|HI>*g`wSR^n1z`-8E=t)>&qnxX&Rr}lVe2=Ubn!C-$gkuC(anhmV@MH3x34Lpa5Ce<*7#zCZpoVN z&qx^Yu(AAb0V-)wuG7HxAiGGXji#WsZYIK1N;FEW%z3Mv-QAPv!uM~5PR@#4wgb*E zSBkgLHxHfm z-RmB7lXYUyAC2Xe`;PRT5riiKe?i0@)&&ybJxCtHuS6lNf{e)qL`%RDvoJ+kb@Mua zw(~6O^z=++Mv~d^;I71_z6|uIdOVT}vw-aY!puy~nYh#jfY-gIbTWcs4iy2Dm3E#t zrbtHsQ!QNgnD`m^Hh>e!w|8QX?*VU2IS@RPMqolgC_YxJ!rvL|s3V^H)&-bhZFxrH#Djn30ZkOFc z4QU0~ZtO@svhEu`fV; zaJa1;gwBjmzFF8J!4FXSrEe3LedU&=JjSo)Y#8|hpvZ?uRjP=k??np;I45);0kpvd z7iHlCk~{Z%Z$j*}JpoVYIC%QPq=~Z=JC@6402`rjkxe!QEQPIP&aelDL}(CR9Unkw zDq-pc5SysY_@l=#Qn+xNsBG9+5RyR9@O&9;mK$P&z(ke~5I%&}!;)A0Y|3o@GuxvO*h&*%AC-dcdbWxoblrB9NMNCs_1veRCRcJX=~F z_{@e^n7X*m4VkTHE=hh6+T!l)XcYJne47U=QD+{k5^qFdMI+dCrg)0Qq5nNutCPL) zYkc1#OWf1Ey80Jmy}*yoaD;~jhWIPD{U&t)#owZQo&Y|eJDEk~h@sSy@k!cUF)WiqcG#nlspS(PM_4+%oa{{h^>wz;O)uzJd4OkY} zy9=KVG1&V8p4=!L%2hhvEMZQm=NCY}$Jg+D7K}7mWj+fmAu&BeSK<;u2~moF$e@+; zS)kN)w`84!s20P_Uk0m)Sa%M&Ct-YOxFzf6FVy$6rg)BFR{J@m3z7aeMGa1!IT(Mx zali(GkpcS;l66n{JGf|Q&S$|yz91i&1TeWfoE#jsJKw#XygKQ;Jv^Jde0ln^eYEpT z-c?-UAaK07JO(J!Cx$FLk&DdReyA(_;#++}ow*0D{3>a8oG}#wmy#f&1-lndq%N*a z8t_Z*0_?!0zOWzigG~Y)MvVKa&o&7n`B^Ym^-cV4C@j=*Ruv)Mo%|dde#z|Ps+7rG zvBBf)=xNILBRm&z?8PuGaS#{MLJ`ShF$bC+F0pA8;vvdN=*-2CjH}eoNNR;e-YIy51_L)So$V z;&ewvP1c^!2 z8<~MY?zA%ar6)#58dfD#8)~-{@Gh~(gH|)~R`xUq{eXlbni{5*2zDX^NO$0pt#Lg% zV<3Ep`Ho#>%g<_EkIwc``}%ATy%}EYp-%g34|U$X-9w$LANSDN`}cdOdp+Dk=Whmk zsCPEpL%plv9{T&>$31k_f4_$Y7gu{|)P2vTjo(%w5nGp(Zmvh%O+yOnN0v*X2*N3g z^qsaeb^d|S2z6JiSLl0m#qhx%wlVr1b=uAN%A1|mh}XP#Zk>SG4!`qezxFR28cqD0 zNpIAj93P&ZPGF!ycI>l-P6Kcpt)^Tz?f4!WL9a|^lkXWx@9?Y!6 z?@n#|*q+T^+Q)J2j_VLUH#bM*=Je*3ZM{0YnOY~WX0NU3tJ72KwS9Vw55Kdg2ZyiX zdSl{YUhmb>4Zb4E*7^=f)BJvum?PLGapy1iKBzQMO3F`#F0 zf+Ly5gdbkO43^ucN8sm7yji#oB4RgDpPkq59ZKQ^m714^kdE#gELr}0vW}&y7^yy! z0nSz-Tk$}qmm{nF{~uZh2M5Pu*RUNb?i_)J0}{@N7eVZ7@kI~*FC;6d>5;h~!SX#z z!Vn6P_`BF*gYcLYsc?A^u*zY}-T3RBLujGx5KyB-P5B*6pmClb2C z3Q?&Z;Re$2^!3q{Hb-)uFgPnuEfrCgo5HA8mx2@X5r>!y${CJ|(N+?pwDf~wvE$*M zw4YG{vDsd1(KU_8@~1tii}mJ?_wlDadB*r@Pk3EH`lh&fMq3MlZP7U2XA^^i!C1`J zszu;qY|5*r7zTi(o0E7+>S}d zD)g^^qi5s5*3la?1XV^n7!N56I9{CuEU#6Ln)Rz41nwG<*b`8EMy@j>9wq6sR?G?zqkbjANgclOZ(u-f$y%L` zhXf9Evm}$yZfW`2wB?00Xy6z$TZ6{Wzy6KnArGi=OV$lIa>*V5mspUHG|(qlfAJyN z6S9&ri`_#ww#pNE+ZTLUtBIHriJH7+fnce^{CJ=GOCp?z_5&O(_EXrh_3{F?Pme}y zw)`6nn_YB9X0FV{4~B49X7J>;4tzEkMlxL(G*4{wg3Ox`Izzytu~`)fcU71Lf|e@^ z-Ka5cnpQOSyyQ2AGW|6}Z$*X1L+p2>%ke0cK$bQJ-okof(9$a@wiKUA zD^i4~{vG)eOGqL3xWxlBCHk_(#gji&T*koR)c3l=u0n>?M6q#4>>y7fEQ(7z47Rz% z+1mH{iECe!VT)lu)`!XW(CS#-y-EM*$VUq%dSCYtaS6`=!^R43l-12K+a+Q@Z~(<> zX~8`;YgB2`b(65e5*cBH}?TjTHDIl=ON)4xqCttoGCC0{HWtEm)NsoqdNKV+MBu9SrS{t z+fIH5^+L~!$Xwh)UiZ>P-VoRA&3rHYehg+Ox(v=Qm4=CU9<D#j z$7)JpiLmfpTdJ+OknP07r3bbZwqBwdEKi~J7GaVF?}-*%|M4@Y3Jn^tmY)eUlp0gBX{C(8F z5?4vEQhhJxLE-{Zq$WvKRE|lR{x(t*Q98t|JAem!!w|J}ZQy&_XrX^;KCcf6|JO$P&afdb4ckAACeY4A z>+vL0e9r`q>(v;jA}FGi5rVv1EQ6uS8KJ&uPe&YgV5yfPclMIpN+M?P^%ds33dpqR0F8<=_u|)dKfp1^KEkf!H!ZoGWy)$mR#lZ! z7a&4d4L#<>cFVY!w@g=hXucv`6IE0r)nyZx7XExw2H zs~x5DZlMN$bGXOFL|6#$%kZ6NGWo5F6(p27LX`t~DP)j&x&y%aaI;EhD|8$S%8coK?&&LrzWD2voPBr^U(hzm~4%1AyFMTmzCQTds`_883M zMY5uZ#0NtX?mXjP-@!-T@Xg9~d(!W*g|Hhq9@)QrO|bVI{e$g;mel?r+-5S zTWGuVS-?|Tj|dINgk{Av-9v^N#_@S*~e(;!;uw)5GaUy+B8rTS)V>73ulh9 zpl$0c$n4fwp4`&>kLGi6e);(Rj)c6@%gapq+hrE8T9hZaOoCbcRW z9udkrzX+SV;hT165c(h2ZwL-aIA(%q0Z`l_VMIm8pp%KhQ2HCOuMGS!Lapynlly}; z#p@;*?EJ38l;a!vkt|7C4NWVkbnOKA{DsXiA8O=n777JlddeS0HwC zC$4|g{0O=F&=UXRhiDbdLu?a%JrRWD4x}iR{|&NN@%~X!!z1_7b&rZ~{N{NYveN8x zd(cB~@a&d&b`uS_R)>Km_uSq=U1cRB=10EtLB)tJeLq@gzPg*cZ^)u|M>{)^WR1zz zpo1S0Y_IpgGGK;1d9lrAz`F6ndmKW~l&&epd&U*eEuMA}8i?FPMP$X%-(k)&@pRd| z#TJqK4-ZPrLyg3~)08p(7#)53#GoaJh*8#fLA*UA{2akWM>sXXrVr9Vwi?&k-IFPr zhWhMMo-q8*J{H{}PjG*jYrJIGU)<F` zzV0=5cHV)08|j`khzG6t|Dg|zVEk2x#-YDr=cR@3gTC{81;$KKek9^Zt0CQXc5-?1 zvE})ZGb3icoMB(un*P!o)Ys>;K6&!rWI^13gtQq2?ql01l>ZJ-Umd+XP04?!Cr3wL z<-ezR9v=3;MR(4!#T~vKm#~BF_e(Y`vju6PZ}<7WzW?pch=AX|A4WKlR(MKod@wu1 z+;?+{9Zx&a0|drz#LY|to0E1ofH5Hitmy{5kaB|_BW1TKE@P#)>n9%Ii`5B2$BS-K zxw#cYBEsAFA$pMmS_MeijT?t|+HY=V&0iYYmX=tChF^Goq-3FN{&FKYEp2%QUOuR% zFrunIIONwNu=-?WajTAbtyVJley;wcrmot&YDI6bV%g;PA7ByPzX1=tB`^CwRw-O{e9A@SB{i9fX;cjj6Ar{xEcvlN+6=uckH zL;8s9nJAl$Ib6*n*Z1Z!damQ48+_-7QW{S7R@|^-ifrEld7hok*if2GOu>Z0N7Pvc zuCy#j%tgX%wpp7X4^TpTQAn04J%?;*|6A0HV02NmfMm#EW?A%n@tX-U z1FLfEt;JX^dDs9f0y~QHhOk=UPDSiEyMwsDxWzgEGzRBp5pk)pAnvkB7yA$>vz1JN zY#lZa4_-G98@q6QK>a-wl4uorV0ICPV0TO*^9K(h$eyU|Br-gsd=g<#G z=&seNTT7`0!wdtNa1b`_uo4g$J=ci2D;ZO|H~gJq+!n`8-WZ+$_Q zL9_<2O(!Btx|?7GA;eYu#_??OaZfcKmsW)pP@&YNy2R4pHKcOA36?)`O;tBpN_eB{ z<>E2m=^hj^hDs>&0J&9}Upe3@^cI-DCu_a=B(*eBxC(6H8&qKE!cdmLkhoStPH!a{ zBak&m1J{XA!)he(ki!G5Ob%K=X_Ld2%Jn@tY85u;+@N`cUbw_Fw!DjuGGn3RsUc{p z-kcoiGYDAbqN5O^WO+xzj=v1pQg8h4_lM1c=7Dv9{^yAQ(MYwP#ZVVY*v{A1J!TX+ z>+A`b@r}O%g&5zr6r*bX79 zRIFkQ=4i`q4BO#0lw?`rPVN-t?@e#B@6vCQzZ3frrwl{jBetUW|Hwec2S=yR;BRg& z8CkDiA08c^9=v+3@m3}!DKRO6*nhiC@SodNrvob^yfQlZQ8*Fdh@If1EuNX3f7HdL zc=uQ8R1tJA4v^JEKY2ntqJ(T=DvdPK{K*sEP<$8l)(GOV@%%R-_g4=+swg%jc(y?P zD&oQ!!YZcO|I@KDNfq&UCk~brFWuc{IK{JgyyLIFW6#oG_1Tc8MO8rz|86t#KOF6* zNGzVa(WuXN>*1Sr2aHHL@FobpB;^5g)cgqYEp_G|#3}S5Jsc!>BbcdeDfy{ zB)B-HS41e6*?>>X9Z-7`&$f7gJCTG(toiGnPCE<@QC*rjq+%&CL1{`{hun#7FzjVC z(Zhp-{{o9*96{6@7z4%al90YM^I`M!1CMou?X|YHxNqLimc`RB!8@4m@1t-!7S!a&xh}69vL=#=Aq{1l4Jjk!;MSK~Sr>|-5rVijKBF;ajWreL0^++>D z;%q9^)Vmq}%a=k=_cTaO|DHZHXKz$c+t>)nZfwrMlc&?bw?`sX36J%^Yf{W`l#96N zAPpzMPz5`U@GU_!BHStn{3XLXar#s%VK5pfZJo~F7bI~#u`jtjo@2+0Z-+7+l1JK* zEd4uTH?#T-({>~$H%`m6+jI%GN%$7lA_Xnnou> z!6rYUns6n?nX$BzgUSE>cb0diS?2z?JDrTzo`@8z{3k(U-b7u#NAU3Bv6JSMXr9x; zV)yT({?#6L5%;4-bEmD6?YiqdIO{2?C-fsC!lKSd7%>Uh@oeYLu~*oY5^vxKD;M^% zm1hea%;SZYIDphbpjq47N%<6tWQt96%@4^;GN-pZN{?`=^*t4WF}n9@mY6 z3Qd*EX}<7OlgA;7k3vPPApxPdRed^(if@H6WG8dL&Qb`YdmO?c$GJg+51Qr{wXe=l z_x*^+2c*ka~`(|WyMm=PQ&K*xq!Ddj|L}zC)wx<^#zHul6-ufQB2+5p$yu{IL zff~(k$t`Vu+dwax9USnG;0~cpc1V*qPQRmHSAIlx(QXXue0+2Szzp|SN5|Gd1LeYN zo;#7cagz`SAFlkU`_YjX#r%R--50}E@jxbyjd(5-g2o1acxb6+RXQ+P#RC340`M=S)k1pF+?ei`gbcdtfXxzOTqYwRy z>&xz_i7vWK_`U28yXaka*xi2CfO~3@TWSgKN*U(uSr8J?u~H`@bZ4h{VSFGVgz`|) zgdH2A0TC4fBTgQL=B_`5NN@~8lZ2CWg~(Oad`5Pa)ZFO0?%1oNDH5v3_^NP6X>GY? z>y@p~hohMcl1rP~NXj9xe2+AcQHieZDYyr3!j4F3Yq)0tY%7uhV&UYAetrakKZ7I- zdP?crr>?M^IDQJF269I{nYok4u1+K^)Tji^hv6#iL~MMw~>a|7Eb z>?>9{#WM?X-itdd9Kz+(E%A7;MiQB=jS4(-@?jQ~caz+~6U_RriAPjCrytQQfUjBu{66u`>345IfyB8}{cDZV=08QnSo zZ$a8=b$a`qGkueWY}dC=Fv6Y_tznX=;8so44(C4Vobgkp$~7zX*E>o9&+VnTqY>D4 z2=DO~6<+INv$|P%v#Liq-2qJ_noZ9nF$7Q>vu>ve%Eb{oY|`e;nb~TWD=w8DI<=|7 z9b!UaXb*ZqN}$bmm@*k)Unnn-W?072ZGDdtqJk`FFeoe0dE}2rVjp3nA{v*fb!j8T zgULeV)@i#<^8P8Cd7TUc>g_T?mwqvI)Gi`ar5x_znhKC#(B5!a;D1P8E%nHYbvh6D=>}i zvV@%sOF-LX2^Bn3ix@){N$5X^D^xg7Zpac+nQtMcb16e8BjU~Zf(f1H+0|P<9@|>F zsLS1ERr)tgt_!6zv^)V*$Yt&$*T2s$hX&FKQP9Mk&PXZ3ea6~mAi3Xb$S1PzemT&#%rqb zZrDi*_%E8LzKXfQLk5jM$wD^vUl}9 zs6Y-$VocD(qB=crC@&AnDZ)cMw_$CXcqwuKKU-x4mgQR*|#xan2Oy`_L@Bkh- zz>3R2SP2nA0uGyUkXcC?$qW6CqHyKS(CIP!856c;K6)CwSIsRayhS!`WhT=EsJ|q7Ok$w+8;7_eUWbff9f>^B zFQ)vwa&|RhWg)9gKxVtmo!*TKCksH7vjDn4MZXe1DDd#tyBQLQIgCbvMPftS9funj>cTHjY+(;A*?JUEnubRXlIcv~Am$@BkQA~YfY*O1oO=R-8|HkNKw~)tRB=?EU(Mj$!^`|P{Q9u)w+DrJ3;D@@@ zE7Fz709s{yJ1fZLGKsio@mB$;Qlav#p+c3%aGe;u#lfd1vo_oL>oxQka)gvQJm=L$ zM5))`0%<1Dsa~`enHE>v60z!yJOL&BnVU22!wGs5bb0kve*I&~uWI`=Oc2U$vdPGR z)x=`s)PF{))F6m%fWmgQ28E+*l2|jtYm_Mq!5SpbLa4f6S4<5?G15d3$<545WUuF$ zG)0vEvh7L~7x^c0xk$2Dlc$5xtVh(+*JS;?$d|i$P(5zEpIpBw_zm$#qY{vu1)ERE zW#eFQq{`OR@vc`fr}KH?L`_H=DY4QT+Mw@6Fe;8hA&AdE9{bTCptR&z9NRT=gR+jY z;;hr6w42jxKr1%O{t)?Db14a26;Z|A0!&mB+qwoR4pUeS9M>kBfL5w5ran0qMci3a zQew}R(U=jYoWz9Ila~VPz|!-D7PiQQuP%8dgG>JoIYuC=)PFZnWAX?MaSCTz4QU=c ziGxmsbV_X6rtZQj#z6q&#dXR}$KU5-wrx$50|9iT|O`=shaP)u<53j zx0Xg+-greKEhQF00VFB9C&>1_=MlPxphE=eMNj7T=`b$UOKLtj_LKM|_?A(8K;kf56=b5IbbL~kK8CHG;E zP{DwkCCPlz;EPRI=7x6T`6`8)K6=%jjgnB}1)G26C7ZpH-O706z^$N`+^iqzo9?Z2 z!UQzScZX!Q3L(?;jpIfnl=94MMuLcEFAC%(0C)g+$nkV z+k_1La!cmnCZZ6t6E-Bhk>2T!M;)7><{b(pChi*DhYt7~c)n*bh;htvwn2!dnjyrD z#>nF#8<#lPaVV0?AYy}n=lcFFY$4b~&xOtObMT*Fl^q_!i^+gsm=0A~cO7pzQ`f`4&`qVllqpd_s};m)^T}dj2i{)<*nn{{To@zN?O~JU!hoI ziGyTt+Xvy~Ejriw9!25G7_j<<3<=E}Xo&G>B!es(V16&EWFpy_{3*2tH4w%YLKv&Y z!>0!N9ip9sI>JygH>s4Ej2E^ui%{dPq52j!82DWiux}<1YMMlBuGqu?B1K)wC}Imh zWhbwi`vcId>SvNiFgCzLx3f1>Ov&j95-F3>FVy!GHH(`NR*7eiXY4>`w%BnAzCoUE zi^vshWkrx!Z45~;1J@xoWGqrF5F%v{RB4~>F z2Z(W$0MWwlDDwYKK`l5vJt5xA%)P&V)U)%3%$*u!mh&C=AT|c3C$fKd@YK zN8Ii8gG4x~>r5 zCc@dUQyG#`Gu6`X-HMEoWQ3CnJv59+or$|*C+zOWzz?I6J~H+!3s#x9MER$kxD7j1 zf8r9=pLXH~E4o1EBF%MC7lRdDh;-OhrcE#Yw6nHRkLu5vUi)dMYy{?$gKNJ<~ zXS1rLY^qa*o(t=JCZc{)N;VvHpNgC}Yq}0im$+ESOvg7O$aF#yS4YuOj+!)FZzkQy z$~kOYJtdk|@zp4o)-|tf#az_&biLgL+c+N9@ry&01+sq0Nv zMij0#Pr3-U%_%a$&7GaC-0hx+_y&9U4{)(N=a{5#b;awiF^}Z<6w12kLb<|iZow@` zvueuHjs2%-d29Rj%7Lor(P=JTi3et>N#lGua5rOjRe*h79;fFf-7=#~%wEF(2`HL` zW6>NK@LGtQcBV|BOMG-CD4El?<9iYLC_?zsQx8paR8Qx=wJC!=xomUvF2$qhozQrWxIs%IK2fF->+OQ7?xn(!Tc z`qa`&WT8AHt>_yzxg)M0@SHHhd9)n_rk3Zku<7_J+xOwV$bn)L*YceCB68O)SR_#< z5@%)HgV49FhX?dc4l0>x%(PK}Wqs(yBoGp8J078Q5c*S+%zvHy#;JG_MZq}{B#j^i zaSQD)2zH~zZ}O!dMlJN+!NCc#cmf>ZDVW2i0V%xpPYnsLvhjZFO z|7zUhh=e!(z1#Tr4xePo$v|@VXckyFa0C@7t~N-L1A+{W{>*n3d7AcX9iHa^SS>XX+EO$8E z*S|h|i9AOV<76OtJSPJ2V!ZcB7mX;WcOb9$pbJWFI9=LxRC&%h%PryAQ}(@nwAqC37oHjZ%J0odICN$*!&(Xyv< z(n@Aat=xTkHMi|-PkTB9Sy=X|_rEA3ntr-bR$1bl*1(a+etEu$eMvUTemZZ26gbXz*;IkwV zX1mrXca~%ctIl*4R7xS0=rvwNi;DDXDsJjvIK{IxnI_r{eV3HTW6)aOh!+3U(D5C6 zR`4kgN%pOp^ul*sK|1DxNB*L~$5gc|t zXon2gaAH}^p(tE=-07GiC!&x@Gw^K$s<|AQJMUDLXy~!~Q?;{Iw%yDbYZK~-R^in5 z{H2q-#NrY#&%}!y?3PVvTq%ne@$Om~u^``csWh*uNI{klA)9(OBO7!su1DkUaMC*y z9RsJ<5|4uNl(dzmrB1HgVtn6*{<5Xqb@^Oe;@~}556MkSa|>2M12(~dKeuF^L1SBD zu%fE1$$o=j|LnRmp1kk=IJxX!UyUb&_V``yT<2tRjB(UZES%#yBJcKlJ-QqZCY}D( z+ur#jdd;w_+FvDIETcdXj%l~lP`hjoCRgptZZ2NAz|1(jm`CO`a~e1GR?D+D9Y0tP za2Pqj3U|>#W&p3QFDGYjCY}D^$4Tei_0{`P>EIPNqB|g!kD>wQFPO|pn8&Nf!*-{4 zbw0TsUL+B1utR-of8X_I*j@NEY8{>&oxa}3fwLclct&WWNd6Qwl%<{o)Bl2+tM2#$ zu$v5q-M77eraCVaSH)et=wF{r2E+b`-dT4zX%BjgS)>59d%xk)`s|#2LN`sJU?U;ggi-8Mj_LC|?;fCiy6V^T+Z(1SQ3pz2^BaNs~*N{7aI zDtUZ29>&0f_#T=a0oSH6SaaW>yW}}VQ^$1$@aFSv354zQ^I`YAJ?;-D?X$CCcQne8 zn-hoJB57R>U`&4B9UQ!lr$bBKW-fbDG<7}3*v=>2tM;3VZfXN>MB$1wOvQ)SSCih= zxI6sNzR0X?B=X$+3>L2KAZHfE`H23_afO|NU-gD?s^yf9^c0E zA`tH-otIEe$eKoE2^q`tCimTVJYZD*&nyxqD2#2uU)Q0V2F1QjAPM}VK{2dGCbWux z(IUjC3A2r$UG?iA=J`BfdAas^ET*Ji)MIy2FbTzwG`lbAxF2>}kF?h&Hy`DMw zMA)-&XrDQhn+Ka$zPjHkJX7?p&KWQ3UsXL=6oW0+yp?^INL}>LCl}oh-K>mJz@J=y zegQ!qQozpMOhAYjzhnQ@I*i!UA;I=|!PrXX7^1=dVb~jYH*4UB(1}QS3!VN|=XyBo zUX3UEj9p)i>owQ$z1b=Zi5KiYz^OS1IC^#zl+QRCqJP39K)?9Jq| zd)XiUIJxLu_G%)lL$7oOEJGIty9&;HLN~`e(@-(``gU{H?p^#Sc!?p|#)0_p~PDC}#lDV~#IBF%gLavKkOmvZ{qLc8dX@;RcB0>+`U88{)z_UDIPn@}gL2$aI^I`jK`>LHIu@(gJ6Y%0Fe(2Mu zgm7E`4yxZf>r_5`LgCT--hdqhdsp4bhxSG949K(WRLJk2Zk?d-xsFH1{w?u7VAruR z&;DNFteD%s?7UZ(!Dqhfk{MTTrH?2X0U)lot5;-)mDz%1k0||%i*ARF&PdX5r`>ti z%{^zN#;h1!-l&{c!w9nZGo_rb_VdwKFXL-=K+!qExc|O;H5vCVyZ!4d0;ZABAXejT z+|q$?6FURdUG&fU*W<~sd)6CvJL7U9Ho&TYK!K?Rki+mzd(&mEE8FvJd+M6naICtc zLH}yR7;b0y<6vBlTPb_g{>*iW5!W2lvB?ZWTC;oy8M2%b+a|WfE5$d=FBJP@fQYggZywX) zG4(QYVaW)j7oU;yI$Bom@1y=zh5Jz)Er^HTz)c3*Ju}(|zJ9brd&%IK%2%`SF<*x& zbhHLb?^17kS37n}TR-vDvYjw9YoI4evV~<}=&F31#^;-g2<;~*EQwd*Q<-~M1?%7Y z1@ZX&@sy6G_P^Dve`6N*rR;!{?YgI`v!N* zx0_vaO{I3n_TL+V0LbFVsL zH2!hj9sW4^p?7uG|DpcR-?7Y^zph+cP@bhRs=RlQUgA17UVXqrS>O zcjMwY#B%fIj`vZx94v8=jaEddpU8k^6bva|x|$cPe)F}PNyYY{1!@8|t-(!29EuBq ztdl62pP!6Js*4sR4hl3~Ic{V*9vTz8r0BYbhHu)P|3Pbig_bKCiGe|EjF>3FzQP}# zA3?w`=Vpztw?^~OUj?+ea}NJ9@dQl0Ai5XGE-SF~mA)ka$Pni%oZ=av@)^cf)K39m zEt(vaIh^EID`6|++`uTj$RfoDBaIV(n@rW0H|OzJW7az3?CW%%dQ$^N62i7a zvLWFMY8BR6YBr8A2hdT~3kc&go;jatsWMEucf^Zmdu2OOw>Eb&11K1@RD~zzPI;LI z)7A}E`e^gQY+eyxfhR0E)vR{M7fn1TUepJx^r!CRd=_*JM|@EHYw_AXqLBQ$aza88 zMzaNoP9oXq5)NNH0t7#$2l9Sa`cUY1^AN%@UVF`*0T0lqa{r6usgbW@USyR8O~_Np zp)}Emkk-!s$k-aTKa1B8gxG+5ysm3u&$d*{@@-bL@C@6wWj_7y4kP)c9}?tv zH*jC;_}&g1Y?uo+oDHcgeo#E_U+ORYEq@wQXZ{Z+o4?_Mb%rC%{$uA?Zf{fKLzZ73 zefrectI*sa^nW3<&6^YTt2Q@g>tW-@ptiB8fTPhns|H&YS7u85iMOVI@5FD?!dl@9 z>MY3YHqMS-#i1a77FEhout3)$N9L{E=59*1%)bQuZ55K-<^Og4S6%;V`NM!J1`sd% z9<1b2LOG(fas{isj%5rSHkQegp5x zIuE!Sy8(_CH6SPIVVxv|OoZQhCZ-f+1oIo6 zyKOg-;C1q}3g=SaBh5xaIUw%*Gj2oW9ons4A>%9*_v?x{(Sn4ju6cf#>Y77k&-T!C z6`@2s;C##H$k6X}m%Z%Wa4zle=n0HYL*I3^?B5#6tFuJ9@UBicW*Pe-FJ!eQU00H= zZ~TzVFoeD-!ci;sl-LOsX%S{do$6EKm~yi;;kuEl=4V2C|JxmeI{q?Paj}K5G`d=% zDW=3mK6md5iD#3L*kBSsya@R>VqrKl)RBQjC@g=z^k{SE+kMqyd_08?c0?mu@_Qb< zu=PE19Wd}|VAq-ADYiuDZ!3;H2e&p2D05GXzcaR9SRx}WWlQKMYD+|4x0W~vn7pNQ z2)Pwp#(XN2qm)EeNWdtgym)NNQKJqNV5nQt9U`yjO$hUB*Qv84QRvKQohlxQ?)~r< z`cbQ*7U!!{`GC!R=>P>hHbZ$qh=o?8d5Z6dU45c>$ejx1mQi3TCAGwsQ0Ek*I-dn_ z8Cp_`gW8uM)5}DjSL7v@_8~q;S?O3_ChW}>((OskX^Gv^nsJjIi{3lYY(c`3qI_qy zF!%frco^ycrHR0N*-=r&fFg2>EuE#q$jkqzQK6w6#dnT3r&dU2{&Gn?TZ{sBUH_gj z#-c@{SgB&+t?yAIX0hV#kTcp?+xCrQ>8ou5GsAijfxtO?2d=o0SKG29IBE6Wf{bSQ@_F z+(GENCyrZE)5}3Sl-mic0?3+}0etU73oWRrW)EOIWF~xVrCm_wWGt`c{}dLwNqt}A zC+{b(oz=XX=14QDfV{0*?t*yoxoS)fm98CXf7(bf!4V zj*R)-+R>tKYEJ6PY)i|r#WzMzI?5ON&-2)$nzg*z%QVNS<qpkTw40EE{BB1}t+j;DHe#8ab)=oiNaj|O6WOQ>0s0JIk$Fx8LGHZbYMBq9zOk3y> zoEBV2-@wOziKE%#LYt7>DUtO|lX5EjA)*jRWWMH*qIE!8Xh`_2(+*N3BwE&}j6H4w z3B<^=87FXH@|MACF0Uw@_&gIGBF$6=2Z0e@UD7I;mDlMqiLg8=GAnL9@a=Y7{;^FX zrA?FO$S_u#T$$2_HDz5%Rw&omh0XGfA1-Or4~Rz>&Q0ceA;)>9$PvZ_mFD`yyhd3w zsVt`H`kP}{50kQX5DHgbn_g2Aw$S0h!C~?}SmLEV+!)?wW<4ZK+KMtRSsaygZpK{3 zZ%X6X)J$~2&_6}ePim5ns(TNKSqMqgO58jOV`|n~3!NMtBt8}&%8PoVAsDxOm5y<} zIlwH5WgAL)8f5Dy8AZ;kDlyY?WH7@l<@qn=O;5?drEZ1G@nDY=ktJ=V%wQ}YKHO|F zkt!=4T1yJ+3~3Wdww6RXkY+i%+aC6!x=%cqY3khn7M&3qI9jgSJsH#zju2TF$Y!GX zvCMGmP=2!UmjRBPspC4)ns3iM#IqIn?92rU<|K%=>p~te1dGh(p^}-H9xTX`gxK8! z5#II*M@fkPF={i?j59uQ+5_*v|3y@o3h1(a;5#N(#}%cF3MTP z4{2{7j*!XS$4FrV{xjW_Vh;}w3go9x6LI}fz#*!YueGynKE+Qvarvo$%IEQ~JdIiJ zU{tias}Vt&YisU-ur{2kr#Q!RB4uDkQ>U^QCI?|N%df@X(XMJgN#K$VlbEBHML$V5 zVZ6{nuu&|R*t1*cUk$ow?4gFTHx>U*6fave{+)lAE^a>eJeWPHzC7NCkHBJ6jx1(5opF9E^Z%7G-j zPE*|Tp+aQkVpb7^j?V~ZlRDW=rAFOkN95~{iz&_BUCQBP513}i6(u+Ao=d{XBe`lX z$~3AmF?zmDMnbC@!_eQhYUtP=iA?0XBupxL5z<~hAu9OhhV3gF!OAxqBBA~HrLWum za8JGaUuiBVO#Lce0vF%^9vr?rK1tpGo*W$?eZBvEiswJ;{&(l$&lO?m!4R}Gq5EzM zpQ=iXHLEsH-^r@$*ioWr^()x|&HPhc&So~7)QDxSG&8qgMK_%0ZRO9q*=)rB>lpu( z^Z$d>R|hHne{_2A)&A#69^I^^`X*pIlzVRQkP``>k!C1$dB+!F8uC&(X`QfvDS|MA z;Km1yj>rrwmLFuRUe*-^uB5YW*L)JbLwY{XfO? z#n->q&Szi%x)!LKO;@pUL#_ok*S4#2-C`TC3JfVRa-W$aotQb&i9T#~(w&M9%T{sV zaL1l>yRXgdzRoJcs^}O%$IXD$H)i{jiFK8q>imD{dk!1K`rAMW|35lB&B%X;U(f$f z^Q2gwnCnY1*D)1bat}sbA*}OxK^%@ZZ_fOX_!I(lq>qyVta(c?@wIJR_&#boC>@)8 z)8F*&P#;va-GX>_;5%BtZO%xOOx+PFn3Dkx%P?|MiJK9kvk(VF-jEKya{+JfA>GZF>dHw~SD*Jz|@!jkUSiJuq9UdO1&wqy}N2g!+|EGBJY|9{oVJiF# zXh7QhL#&*B#MME*JJ3iJ9g$My^B+oF`05c<&m*YV1VEOPdy`TR)YBnsxT#g}IO4W6 z9Tz)GX;DIXI<>TWkvQkZT-BaRe*h?x$5%#gM4nGCI3gYk0`P@}{$+_6+rSkR6R5h>O5I zq$Oab3xDN`$k7WN5{MfO(HtOdw}J*7h?6*j8auC}$VW@;fOQaJa~u#h(K7;^(OCwe ze@ApM!V>$1$nj3@MC6|-wxMCm^=G$^H|I_6Hqj_@Tz8KZ1~Y603nPyZo3-Bn1F1$F zGl=&9y@*{g?=i&h*k#2rd`7&7|5w3_7C3rN(ftBP+~PbSp(Pd);`V*Wp*(u_jUP4> z^qFJX{RG4rw=8Z%w@aaqYdk@v+i@d|*F-_l8g7IJcwgNJtGW^9yQNjp2-k7ptEwq3 zb>@3W2NtaE1G!tNa0-Xr`W<_m-{jW3CLiT9nIX7RFU71ZQrS(>%1Q||!gI&dP4<-y zdBV1qI=_(%XX!*`tWE&z3MeC4FTKjXA(6`i<8zY~Vg16xCRO!UsOvbG(`cSQKvnSh z7WRIa*XE{PpI==~|6;DDDs^(?T2r8qEevXEEOKZt&ZyK#^&_(c)8p`vi2I=ee`_Lh z*NaFPgbpP_;EW@Ji*T_U%ur3T&vwJON@0I1HJ}Ul@AqVF%DGad7N3>0_mXmJ;3eZo znI!~J&qFxIBL@v+cA^r#PBC&O4=1iy&nr{lUtKzUbQuL@0|%iPKf{nQxEl=?RLmWw z?ALM+Ey|>e;?E}kJX5h^7fcaGk&6zSjb=eB)jDcUnQuZCFI_7xY|TixAq7-wwj%Zg z>HIUu$7L;^Ren?4OB$_+x;$)-3TZ@^vbIGm#myG%nJ|WU6)hgM`~e3BMQdN%8clIV zi$qia7f$i4$YT~@g}zI4*Y+(ri&L3cvtZ_HI|}=bJuCTK)L$)V@b){*fE)L8{c?e0 zSjl%CiWc)p-XGT?Ui2v1D`=|PoYeN!-YVvm+_Seug5p%I$0%;f7{!I&=`xP-EF?B9 z>21nhu~OTNcTF)R8vkfCb}ODw>O$V&kfJ!4F}!oxnI$ufB7St$bzIzf2HIwH5Us+g z@A*rokPF47fEI`sIoK_ui1Lw)!Pe+Sp84R$uJ#)_*w=D%uHf8Uqqo0cGAE%~Y&Eto zDp1EJGaPm*x>~2YZPL1ab-&)E?pOC~Q@8#APS;=D)%tTfSbsJ3`g1+G_Fu&*W@@!j z=KphenvVZ{bnw;w`)Qt>4JD~+L;0B^(t!@a9a)%rCsZBP>I<@q0hD+t!}%y{AjVx* zHQL>RD`8?AD=`P-&Nlr3|GFYOGzvuq&{k~4WJxt_3;k~K)&yUb*_|fb(qeK;))S{{ z_Q9OtQ&|laaK4tcHa2WvfGNHA!#LYveo8IsZfl1CwkMrA#x z0`r+Qyrvr&%MfQ+;p50~!fnq^C72YbC5B!5@C^NZMI%IALIQN-hvbfgiCaAxk{C6h zOzeFD)Dq6Qhlb;Ez|2Pd`nf~wOARYhr@ay^SiJm{2 z{*RH}Xf@@@2OQt)o&ASt0IA;js_H-EsKp9Es^+87QY|6Wz(G)}i4K@0CV?Z=2!vj+ zHY8o4sKJYfhrOr^N9sZX%2pn%N zJVll`sI(oFp8qra|K#P%ul)Z>o^8wj5F(6I%Oxk4TxPH0+!$DVBSmqIFT~>szSl<@6vBW`qf%HWdJy3i8dap7_&;KOSO~v(3DT%C* zK*0FgY@l@iKRHbA|Hp@iU)TRrJm&Qe^F8F#ZP@$`rSbxU*XIu!sTtTr+~gmHS?Ein zlxHM{EK~#hJXbgMxmicG^{=IX+JFg^uK!o*_5bqd<=6H9B#(Jzt6WE=JRn{2k$gb0 zfegiEc|oe_KL|g_E|pTIkQu`&jA2XjUv>RM))W49N{_|ABKJ`m3Oz-#ZA!-F{Rr>1z55nMESRw9&uIj5m4+xwB7Yz{vlQM|oH!{%e2=HSXXre{UId}TDeQ54YD{=V%)mK}S6A^Yb( z@z|@?l6c&DYhO*mMA`ICBx&uRG>`Td(bCYG3CnlAncKhsHl3sYj~03au#x9NnO`9Ie8a zrpB+&Q$LU3|CKL+OZfls>0w&`d;IG3EB}9zM~kOfX$w#s<1f7hY)ucqx038eA{cv$ zt*5pSjitv7U1R5xnP)bRJm`rnqbhX%_li5%rW8M$pQrz53LDd&Uw;#ZP?Ek);$Lh& z)ep!L7XK8zieJbEeqH$y)?Y*RE;gj>q#Cld6jE(_Upf&)yVROh8LfEq@0p98W&URk zd%1)(tYuQ+?MXE_5CdiyQY}klj6)oSLVJhF()SX@WWqqQO=PEP8JGwl^u1sFX^evU zv7LchMLwNjSLOgRU;yjQ_af{DzAej3`q&B*0p!KXYd`R9+o9nK98abzd%lgqM;bZs zRBTGup(BwgHghqhWQ%?0OV-@FX!=ekmCdDp^F0;6BCX6_Vh_(rXvtlXC#G-h?-vY=*18U( z84p&|JU;9Ty_+?&@#td|A083`nZGo?nu~|E8GhN$8Z0d(N;kjSUQVmod7UqVbVzLt z9VU*IAi@B|HAYaOE6QtFn9~;G$F`0i*ZPt^x_t&>V+^*k1~X{ZqGn$EbFaH6!?DI% zt1!J94*9Ff=mawMZ;1+OqjE67ot)8^O)HkvDI%$MFc{~}K{`R{Q- z01M^6!;@FX$LaXr$1hL5`hPsh^N^E{3D1ca?Iz!GhbI2v2?GE9)#Cjna$lucVmxF8 zpPFuN>!o%6FwqDfH2@{eSF2ba*)V6H({O}(Qr%?SgzYm1}D8Ab~AZ45Fmuh0h#K(rn>aH^$_q(}qejY!%0& z%t?BlsZ>4H{+8L-BUo55rnzx4YsA{zFrp_yvjv&m($#Wb_$#EMDe#vaQAo3u(!+A* z=r9T_BozY26!^w*eq4I~!joRT1pP(W)x)H$9fZP_*QVE$ge`PFAVJxA8rh9 zGqWBNCPYR9(37`N+r7tYs!+*>Sl!f~4k2+})5_xv$-CRYUCkY@IQb$kV%d%!T9hn< zXnp2{E%abq{srdUs?AENJk52Bb9+jZE2Uwtly$ziU1UxsoyanGOV)en8;%ibeP3-p z2K;BLDV3)B;Xy(C^l8HXGNrASiL0xX&*AIUk|9=~;-{UsN_kFm=5b52m~QFMGmT(g z?WzRjhKUm;EFND@!(&Scf+c)`QCW{Fr5~No0v^D#fFM`XDV%d_Wb9>)&*W&E{g8YhJDDmS)0^;)cC5aZk-9XJ< z)L39f6^-3p6Z+oV@jgO#tB#SeMh+hp3k9k)GgXjU8RG1eEK->ul#%~P2nVyYbQ1UX z4u@8_^2}3)vgK!ZwjiaY^HKG~oHeTEcEsj z8pD!A)Ro)KO2w!G$F9;^mZA!IAEmCv7H|~?SZ)8xMyEZZzbEV6n6uWZ z#iuGQXefDymyClaKThOHg_nx)$Uy^{`K{o#F>)qhgfrZ{GWj!AK74c;1!V)r%6yI? zV{kVbEU1_}OxdqnB;}GW&1Zo*`Mey<`7E$L69wkBT|AgI0LyG<+O9D3m%%C`;KfEl zOI;XBB5h;^aWPA&(92b%Osn>M&{XK_ylKfsR^KkAm!M%%-qa51z12RMf_6V{F=b+; z=3{P(D7(GMD90mb!Kfu;4%C_>CYKb}r`{lUR*5saVaVTYj(SnuCmzeqNE0mKscBdg zOOu7Fif{Q9V;#o^c*!2&RkX-^U!t;OYowr9wMA9{pt_=9<5z|`Rmbuud#O=Uyyzqn zxwP`gAS3@tv{Xx}Db@(9Hwz=2KRwXcRBqlFt>U5Q-Jg75{4S+m+3x zsi2i=9c>bC7OugIf}OGp#4NmYt++6#Egpe$S+fOB}_nT97aY3mmW#*0W8?s09NR` zq_Rv?{Stw}-Z}^J9eY;txv0PFd(F#*>K$gljfyY*a)4u4=`I$E7V}BoAIRPFDB3G% zs@j~?zI)s%=9R7+w?%^FV-(^K{N7(L_6&8|7hw|cg6<{&q|7N2K&l(FC{3pF7nNZ+ z)7D~f3Eqmqpe$X)G+hX#OLbiAEHf5)Sy*yPSn;$l1m3TisVs+KR-vciRkUV(NQd0W z08VJd%ZulX)YcHKFoX7V@S$69;UANYJ^iCmj6Lmcfo2R)wn;T|-2M!NqvM)3nPiXS zIZ?-uJ-OiG+ib>UxY)+?c}V6s@yWL8|mys`z9>58tW{>P$z8i*>VegQzG~HhUM}e>0 zTt5n(CHfuxy7D7ZMAb=_1ONsr*X1U|hD98IEZ}T~Ud4m@G3oI6PwW?>xar`vd{ z;q!DnOmcfO4Fx;o^UO<+g{%;D_{)&dcn^Lrn1f>64iwET!Nuk`$xX6;|Qr3`5D39qFY=}}V zHVOu-k*XGofMzXpgI!9}`=r8QmM^Uu0s|B<8gnFp`7#)U{>Qb!QaLLQnCdxMIU*mu zWUWn@PiSG)20i%k%etLB$R&ND#4h^b6TQk1qz~2H-pgsh0G$$ zvA-{a!%Oz;6tJ`exuy|W@+!vTEAj4<-QpzkZaf}L2E+b8e@qbX9fO+8jIFn@h$4>O zb?9a~$pNz>lYo`kGGK=BXfu>)1)S+-NSA@W>eolv^Wondh@M{`Q!_;4I)KzLCW^Ty zNoh-QuYn)tMl(q+i{Ft7;r5?9x` zBO5lcAlQu-<&9J}7&b&{(Fd72^ShJu^k%s=b><&Vsxs+x;Z1mRRTx{B3M&}eR}u$v zMX`#&69SvHfdr|I;M2*C*-g!|7@;LznWs4`G_cAcnJ0~*@~ZQKSpW+^aS+O@&Wcqf z>Jai9goef-o-ht8;cEn998?CGae&0DmKm&&2CDE^d`~S{{#KT~=kwZhTLu=JBLTF} zyI14MMZeR&m<;;Eap``N)6!#vldsaqS7}72ow6-Jqd_=*b-lZ6`4ne2)zR4KLFtOG zytmi!3CwQq^I6z*{C%l{IozMm0&5xU6EE72l|%ayK?ykKgsz*+!vHamT{==87N_Rc47dly~ff(Ipuq)?$DhDcr} z)=_B#Iz4!OkYBlb)#;yg&nEA?hVgtG`pdLnPo>|7NMV(Hvgv}AnUV@OuWXg%xT-H^ zyLX1X_Ir7As!_{ zDuEak=>q&4o}p}k!w60NF!b*!a-As&vFonUP3SMRsxtZ4+=&*eDdhcX|F#OrKE^gT znS-UcKXv`-{t`RhKEE#B2l(kyaX7pyM0lFRumAA|^wkrXgSS zM&s^P@9I34sa8T?$oK}exooI+*1a0{#y?KRKMpn@CWnb(PP87c12g%NU+Y3?tSAZF zlsa>d*o~b1tbI1=T=W>rHE~)%J|w@coRHY4fwA2I4RP2O; zy+J3$RCKz-aqn%f(;jyxgZB7c^@-__Pz1Rqa3S^?`e$ya4o&9@pKKcY@4Hu%VfXE@ zJ9;!WqU{h z7taW4SPj&eG>VV0U-cqi^&+3=mWK7VqUx}8@6U)B5b`_nXTIA)Mu-;3{H@-KNZT}6n_7DQ5E;*;Y!k%Qec;^Os)%zV$LE%Y+kH4-|$ ztv(*6qNc1Mfv(nx!^HQmffNXSFN2?cTdp_vd{oo4CzR zrS;f*Gz(G`jj}xcyMu${N?g0ZH2#algiH2S)-LYucL$ZbYap2d(|1_OtQDGn&5(PU zV4PX)k~bSeIoWu8P0Y69+)LkFicjvzW?Jdq>5^$nc8f1+*OhQdptnX$} zBPwK}u}Y0Cb7tBy%Mt%Ft^u0IC~rDFcS-+K-h}Rjnlm_AxcS48Ga!FRHwJ~&)5h{d z_2PLJ5W_x++zLD&CS?r+Psow!{^7+0vAgLH*pU8O^nSIsmad7-y5xTM2xm0|JO zc|$tJR!)(!&Y5Fv74_T90V@<=aY->hwer0Fytt({P_jBBW2s5LBHMg?ou92$dKPJr{d$a zE6UomYPlv11s&$qQ>F;ri_6J}_QiE~lslHWp@$qdB4Ip<9EYv)CCD?+rNAW)iq>`e zqC1qf*O%=<)?CFz9p&IFD!+7bwFJ?@uzz;lsf}utCMxdT%P6RX>F9DisEcMP@QU*u zR70^nsEuJ6=z7N|aX;+eM3-dghwBSx>Eudn=l#p>9HDv?sa-s=k*zM zc@x!;U6LqtX4MAyvO6C3I<<$lN)siAs$sW%HtAno{FvL=3$EagWfN@sU;z<_I2J`f zDZg8=V_tU0@4D9`Fh=bTC*7;FLBDrpWTPhkA1&T*o)gd}sqxc~OzN~G(Sod~3_DMR zD}q_J2QD=j_CNH_y2D9((91Y##UgdP_ZuFq&;I$3!TIF!-F0{UokxbRXym)Up1KFC zU;erLN|PG~gn?d#rIUT1n&=iSkGGU%U8u6sGEjqKi5 zp~1}d_NcD#&>{Uh5{8bQJ#@#L^f}{w{SV#Yuy@v-T=YiyXOC)qO0`_4Ta`@|;ysi* z^NIVjH<&uJb|PIuPQTCIOy0Cdz0Rb4J$^U29(9L#Vv$)~D0Udm&(&ES@d63Lh_1P3 zgCG7ReHye!qaXUiv+dIa!66C9{w?v!#R4{X-S)6Mwu052d3@FK~w5{YZIkx5`Mm;nZZ!CKAEuMtPV9?a8U`o5LeTw1%iRA1>dt23{#~hbh~1U8F;?R%h%>uw`^_3m42g zfw}Cp21mWCa6_TlNH97@6}7_XRW7o8OE_6X+=#4Qc2BihaPMj$R->69t3qXkWRmXp zs_HtF++q_^P&xeAs}!bmSl@7O5Bc8bh$4RF6I$8@_wZwHaCmgo>h-vur`8}ABx7d9 zRXkjY^Jvt@Bf%cphF{wEKM5+8YQ1Pg$MK zZ>feR*i%0bZJRJo0J%U$zel&U^@U^QsUyA4;ZbYBxOwZI>p%m|g5lzc|Dkt2xN7?S z(jSNw=-3B$a@F)-C_3cH-=+75TkF7&fdAtm2|o1B!EF=x);}m6Y(hhKVad`Hr z@9|}3JbWX-p$)BjR$`=ugQVMScm30a(~GyWBZ4R;G&*c9i-$OXUIZeEWA8FkPkD@< z=$Os|(z@t(KMjt18s(-E#6)`C{AGW4_x1kvFWYFh@$+jq{Mp=VyxxXGw7UbZ&Hq2y z`DOR@_O@?1sJX~y^uC5keQ$erZ+EAu^pYqyTb}N6#7XP8)jjODjt9MdyW8?@MjDdo z??u$(mn={!cv=Yx#4c%ue;f(R<)uXWVEI4-YL30Ep+b&*S~sh_)_*rsZ++i5iPYhC|raDMp5KsrtQ{j%jr?aQt> zAv!<3xa?>6L1kh%fuB2}b>R0YqW8yBk*@w6+f#;A=r(tEv++GXJU#p53(FAA)sPWC_wKR-lvVYG7|6Fm)$fL-rL#Hct@r*frs}>MWo$d@>c5@^iqC1&G0Keo$d7x`=wRwIM1Tn#0)aGrKPHV4^t+-Rc?f<*<%Ce#<*Gp zXkC=6c>)CI2fA7ebY-lo*&+t#gu9v*XwJCrT$LT_x#$Wxoh#B#TxNrm!-9*W~y61g2Ush!;7QV@gQiMOHzPc z0KpJJHl+xV;Sf<|g6n(m!6%`xCXMhda=|^B0v~|{aBRDHG-fU!L$4I&)v1jba-%RX zVjbnqn86iEx5ho3g7&eSfE$qUZj9`pX&&o}`#bp`p=|y@OyE`x4tG`YeI02-mDCr&Tq0UD;BUgcJCM*ws69om1}1 zl%@gdR0ic(DYwnBT-~XDDd>uQt@={QW8J69o67CeAs+nn;%~5vdyL%dWL!Y0`OtRDC&fv0}e`Sax zf=%b#zjt`qJ=5aTNe8Faf`pFKC4?!!sVEUd#?ClW4We%kF1x1%QP2Y|aqknmZ_h3- zJHe@{&IR-A@^Zi#mVFkT9+-+oW;_t!kUVrwk534lK<&+pi^KC4H_x=Y=Z6LPnFY7- zq~O=a`y%wCp3nVverVtR>3OStSy-J(wlMqT4J~pRv*E1$wtd-;EOCV=TL$bB2o|_} z%ZDTP0$Hp9P7V&JzkMDnQR1?E3}kNYClHy&w4x{?-n!w66J zz)c2yW{BF*8d%(_5jX-5u(=fvk(h?y7d<@=uO?`<`^c)AJIZFjbXJd^{0`05pZTvf z%!8wNxMzh1%~`I^d6*mpZF37)#+m84Y)0p%s9d&}ZLQ=fb4@7Q;puB1YEN{6`v~EY zZ`Kv=SsE&Mk&d{mb&G7~9!^dDJw<+Ju65PA==TPmlF=_~OGvbnVM-`<+#h*0pS2Eq zijed%Dt3{Dpo?l*4xt||-?VsdLkE{)@VL*4cCfaOwWz75@em`+)XZP#;44I3OzZ=2 z+f)?bl|{gUAXEm|O|m;&(gAjP_ou=LL(JQdC_C}-u;1#Qv_F>L3;6SaN8k+xtBK*r zCNASnD#1Y2u=e=$@a;vr$ER@j@?vm$AugDnw~ux5NVgHK{{)Vp?O{TvHi#W4MhF3H zi_}YDFo!6xFSm+#YGdL!+~*besjiw;esF5OkfVvi?q@jUPORcZDlUVf$-vt(W2NL# zIXdE^5n5m~`n|0TEu7*fzaQ)TQ7mFWYfM~L0^A}+q})KKdwPD@)tiXpON-1@M>sf? z;%KYiy2!MWWHm$)L$>O;Fr#0CZtL(k^PX&e2ZO7#g_>=jot<914YR+L>nPzDI!|qe z=q=Q$W3lnkYIO$hPJ8|KOgy|n$cY5rJM4+DLjCmp_y$EHnCP1~gd5*GWKG1^0;-5& z)A!^N;Wc`8fKQ2{e3?F-z!6F+PE|#&yyCT05}L#FwVTo{19qiq}x7M&rs9O41=AXT;(G&rR$as$|U*|Q)$zw7xGZg%tTT)AAc_N zoG9ns2$>IqQkhB82$T=A&L+u>J#xIZE-s7<8)Mu5h?KEM5ix z#0(z+uVBrrXU&LGf%82?QbFyYF-D@j1Wx`a_rB-ANOm`+mB?N*gj zB~Kq@R#xmqj}fkG%^ByqPzsX}>D(+PdA*)HaZeFQ>MhMoxqI^Z^9`nD(c!Mj>X95m z`Ds0L(Br8+l5h*<_RxV&<@YEw@bftP&1GuQV7_3sW+iNykE=OJTMDeo^DI73n6>5M zA$X(Oxy83za5SQ51R0@<4VR1Uk%b&$RRd^xh9>s}}>R7dp^{fRB; zYB0qN`qiManp#C)#C+y$Vxk_hkim%LXd=IjXMj}XoBO#;dyH5}Tt|?Gg!FQ;t-;~Y zTSlY-61-V-uN-2L(S6VHmiQxLyNp5{%O|BRG9xC^&mT5V_7j^_^x zK&Kj%`iJK3>)q|W{nvZ1n?E$R_qLmRKY+$E z2rzXSq(3y~ZY$ZjZ{*1}cSWm%tiZ^c2vC2HWNV zI0_E1p9P2Ho@7%PXzbwZD)UH7_zN98OQ}C`A{k^dA-1^O6<2w7Jvbp69s=C&`vCmI zVUE6FXW%l(umJ$Ee-*0Gk3d{H;-3Ms4Ps(@1crq2V_^@j@5}DTyfFd0cM$ik-)r1l zbE1y%XpCqrY=(mm+niu~U?F?N#$eBzMA9dRlFt~K0DX2aMJBLGmD`EDAL8WMq4?I* z6X2AMrHUMfa2{0rVf*Pz&8%FmyxIR&(U zEmPm<1lo9rT&6EU&7rnG)OEGrZf&zRajQdOGjjbGGSo=lL3g6*?j5?WLdU7*qxgBu zCYBZv`Mrlt9T66PLD(?22qP6*9y)`OQMM32EZ3>&9Ow zxXj;AAiXikogIJcSP&;(rI7Lj!NegpvRUc_C9YdF$pm7%>Tq+HKQX6{%P4{q9#!`t zDd|!Yed?lP;Rki{w;4NJ5gvJ-lMkF22avGv>LQie0V5Fh6elm zF`%xH$M4eJ&@nT!84ZqNB@|(J74jr*$V<>?NGLL(D}pR(1cy`xJQO|)n~D1}Iwb_e zUZ^1h%>h}iz!(zhH$0Q1h^j|JCJUcAX`t${!uLcT21$$VTzl9OW_uI6Vy75+N=0f~ zSn;t3mV{zC?P7P%aTPT@n|>GJJ@2?GKmt{VbB9$-#qduW7R*k`@*SRKBn%&=)yBO! z85?3Uqh=BGqa<7BBZaEtCFO?F@lxTuU&vLsoR9cJbmLwKp&1?L!PZu(S}D0TnOY17 zxtFUF@{cy|Wdg;|drA)b*Xe;)X4FsA7bIreRjFakJ|kuzl8Ph+B%b~V@|lcW0^0#U z_ZvjbQ9_vT;<~%Li(ZT15B&eNF`HPwMn~t+GzVh@O+>*FH{1C82~e3#T~?*&R)h{f zm0wfm?b*(lJ&s~JBBEEhM9qHyBRxhvT~R)qs^&N_S&)vEFc z-za$$zT@#h2M!z|u-DiHb%W=p9qcuBeFBE<1cAnWOBIsXC72-Yq(*h4os|^1GG<-R zi#w6QJ@~l_N9eakMS*KQff}5$@qnVgr^scX9$9&VhwtzP!F6sxhC|hN9&_r}cPwt! znyNA`Vj;rBwnd~}P^sjU1%Fh#2wK(C4gk+7nh=Ht&@^d&wG%?`AT^Qc{Q}QL5sz%2 z2gBf#_bU!bV*hTV>G9SgBW%YDF?v53Vhg$8`IKV*zdESPvn*nCngza7Q<4)ALth*T z(|@&p@1PgpMP%uG5yyifHl;S$ZZtqeK)wIBU6ET?O_2Pdv(qRvc*>9=u`w-hWk5o+ z)rTXVe)zejQboP1R)Y|(@af7*>~tQiCB0UQ6n7nD;Ng9T5=IOXca(KqYv6<*#3R!~ z3DA)srFQcHR0C0lhuH9zJ@1LT%hqp*Ziv#>VqaAfA>j(7{RhGUpNr z3aoER_sC#9aiHsR2i1c*M{kXnMis~SQAUzy0+u1VF<^cioPgLF0#K9;JY=J=0y~XH zOoNyEWSfZ=7T*X(@bMdw2b!Sj8V@y4beqBV*%=8ROVn0wHJaPXKz#{uA#EMPsqsJH zzw0kO)yyP!_CNl{dCQwBT9r?v!UY-(kyK0-_5w>4|7eBoKkYB?1wHyFpJ#}Az z7jIkrSDs^GgK(A(^Q=d0d#*W^xY>6KqWg^q3$oR%P*P zK;j;2Dgae3Os{aEwMDh*1YDWFqnIIY==|3~dhoog{j~KQ)SoxOHAV0yPmH9FXjK)| zV!HWrqqw-$6jqB(ZQ&;qY_rL=GGa}sqk(Aiw%gdv)kQOk2cP4fgk`47Le61wJOKkN zBjv#>QAT%I`k;&{e<3vl-YYhUm#+vajFtrge@5K*N_>ml=Xh_pujGCinWG#O+|c$7 z4hn^VaIFS~4wBGbp;f)?o_a`dWoa;__}5J3*`ue&jc;-wkSSZ^k&qG(+3Q4>H(7j0 z_(kgaL(!~Y(^(sU!UmuKg*9fc64E>3GVq-Da7^U=rl=ouoT;c#csv=1DY7hhUr4{!jh-|5i%1Z{p$b>00Nm6DGM!E@MkXVL1v+ zx~=sc@{g^!$gQj)m0fs??U7se4_JEm3${mB+bgHVXz<){ETF{7?uB#kU1JXVUVefi zt^V1{vM!0}8uQF1CVETB)LDJ5rBPjDzENb5$pqOVo3_gh7I$aZUBL}|KFaIE$N%{$ z$;gp24Z~x*$B02}xi4!!ZLKUDRR}+CKF96=(vkbz``?=L^CO`*7J=p^!xsL+3$8jx z6)3M8Pu1>V*k)+tN9V4wskf^l6P)c6lM>cZy*foM{(=H>N0(GqS8NwS{<|L>g?Q^u zT?g4_C>=Ybly}0Du~milhNB*3Cgc`nZz2T)a%C9hGQ)rz-V~lO|O)ylA`FGFN ztPA=|y{jABV63{z6~}hnw5{}%RmIp_4W@;dtut4+H(lvN(IqoTLUO|^<>BheF-u^% z;TGeGH&&fl9+Degm3BOT8fLAB(|Q)Smg3%^Zf-5<&X3jHy2~J&uHyWN;kSky^0yx= z#~B~5_2oz%UPId?HZlY2>dG=nf$D~D5;0cZaYc{v8uM{#8y3V9WL^<#I$2@6U=qml1fFMOUp&D=~Ce(uvk@Z#lQk>NO9d!McOmKZ4jFDHsCBPy@BGj@Jczz-1>hF3Hj|>$}IgOxWkqOt`QL2M5c#u zqFPXk$EXsKq$b#PJq=kj100Hm`14kJwEqwIZxtr$h^8wu-s`#k4&T)1PRpL_sX1D~ zgtCKhI99I;ny)i_Ufs08MtmZCsRY%LcS&b35+*fjM2nn1+@|T7KEr0 z?eCVK#gxi#M0asF%1e?G--7m%Ms~xyZ$*DeBfH_m^1>C#v=0q~OzjoiJtpv5dk(EQgVvBgCc2)E>Vz6&=i9Q!$0?I`WPi{g;ylYh zb`5zqLhR7NNBoB3%JE7@ZVh?kt}<`jb>fYB;+zJH^@_46jpBFDDCHztbv~t0Tthw~ z8(xcG6lGMsaw#39Hr9~B#?)nG!ZTWyjnP#yb7rEUHK+9~cXDSZ_&Q&ubtt*TCZcVJ z;R*gyHVaNUqN^JFZfTyqgXmCVP7U?}mD%oWB!1s52aB+XqU$NPR>avr#UmfV zHD{6o@rqnWRP)HgZ(UjB$MsxE_V~E1IcvOSo6r@x9Hr)vkK)=hid*Jan^7Fab!U`` z-IeE*kK?+siW3Ce42>xNT)%}DHX&P0rfIQVNA^YMIHk8x1@^NN<&zDqH?hlahGwPq z_r9x%j0XT(h%$g|69~%z&$10}x5KpAluZ4k*^!|IxQN~&n!&c)?K+3NBIX6jMkLVs zZdsdR$^ZNmW+)GtQh!1X5Gq7xfoh&Fxym2p~&0M3$D~%V~otHg;q1xZh-3ZW*pJ)mTwOaRW2F7X6c1v zY@#-G#?VG)SCgs6C;5QgI|#ghj<^x$?Oy(JiTxsuy@!;K_lrZrK(1RoA~vJMsvcSv zxvRD*9%1_hc=6K<0IfTC@0J8`j@X!(?hEkZZL9wZbS}OBwR`bG8jcBg;Z zzUZmpdt|{w{fJXjL?WX1P{gLx2HTAWc=1qCR;e6-3Zqk0`T8P_0J2;(Bg`SRTn*4T zv-8+~9NrDtFkGfZE?cC+3t(~rS_$?(E80Uwk!`xi!ce3Z2;piLys3*yiIEYazYAU< z|CAK?7AW(DYSnmn2oTpcI+UTLjp>2i?lt@quNTB-W zP&|S3J}i?MYd&N`!AqMk@DfcNcApw0=N=DgLu4Wf88Ul}P&h)uGrxxniWu11Dpf0c z%yVuZ31&-Uza{!mIRM^#Dlq;*;aQg3BA;_JO}RyIO;yH3jz#V#$Yv_$J|$zN&DX~< zvH=?-a4t_^Aj{I8h_?%{3+UA5yEDw|NMO8Ht71nM22FScFmpjDVXzC}EyNZ##zV#% zbs!r{Q2t-)dZsG`i;vd-tAPvRO=j##3TT0g5w<^bD8N^tusxCh+6bA*tj*9%=mH2n z{On4=jzfOF0(S_wV=}eOpk2VmkO@o};Z1$`IRskMicMWlziVkZr3!tE^j~*84CrsK*T4dLNEO0nQt3TRNs-wOGqv@jiOI>NS=jET?s9z%Lf?3T;$1Tv%rrzxffD|7^ZxLSy6 z0qY9tiK(3SaZ;v>=rJ8b%C5x0nFnaQwpZIx6&#Z(weF80gI&as&HVBY58!2TOL-Ws z#5KgMcf6vvuSCaF;RWNpa<{bO8qd_uu~#qF%rU&zwxY1aJv@Ym4P1(Xm#G#zG3mIK zG$m#=rnK$O*klS;R>L0xYVieho@Nt}vKs9oe%NZB+Af3ke7#M<%4)dh5Ozc+6KI== z&e3QLPF91LwqZd$LFN^)rjxlkngo^AEWf>S8abA^m#1^Itbmr)_#0$dr$#t2XK672 zk<|=e2j^&m*XIqS;K$8}DQ)TNGx;=4`uW4=S*!P+KLs)z5ZVL-vE zaE_for*;n~R@I zpzzid6H@aSyCQ|%*RRi@gMF3g0Ngg8-C)~10HVe@be>HRgPvl2@C*POPS6361XLwC zCf!PCV!-h@NE7H>z=^+S^Z>aIGz7F-7g-3psCE&&7NsnBjV$+o3n|{ifHtvtQ8UP7 zLTsLD!D}A;18i)Y$Yx;IgA*X1?7I-P*%VU|%#*KUWKBeo!0Zyr&8j3rt6#qs;(y`! zPrZF%@V&(3&!7K0jplZ8{x_OCoB6+tCt;lP866I#^rz5f7ZiCuoLUy2UkAVG=80Ft zn|R(F3G4YeBZI1yiri{UAnEq3Tc9$A=fp&zB#Hrz(2+n$1;lRcnQT+afRX4@0!}rq zzi>D{#X69UU9X2pN;>>QRZ+Zu?;u?b{;isjo;G54gx+9#RJ-}vtzl9R<>{%YtCpq_ z^Oi0ggWBAoJK0gugj|e~PjF)C!GuO~Ele0DQ}F>yFAfT{Bemi>$T-kV5W|A58`0E% zeH6Sskp1$MbrR!9(F6-|A4=zgh|HM z)M|~@#)=l=6Ef!7Q029J_z9RlbofA!q?EaLAH62*a`X4BKcaLiL%s50OeQ^%F@*X0L*ZzfrwL|9jQ67*>bJ< zQ%wJxAV#s_`kHoi0-Z9dVBY?3f3K0y|MnU?+Z+9F8IQ)rF}@d3se_tbi$7B2igWGq z;v+BG;>Fr|@(rGkP3*R}v2f)aCm)Iq#_4VtExi)h%JjLd}uU=NYx8O5%O;VjqZ0y9YZaRLFC!t@g=Uj7b6P_R&?fS=K1*0hGUZVgglwCN zuy%M`Q1e_UEMXwO1AlkUcc*+GUX9)U9TS^~RK%{hfZ)H!Hbe9lS_hyx2}}tFg%0Wq z54Z!+Yy^JZVrO3Wb)~TM7)hR-7ZeM?2*EntrP-)`=w_J8}^yBqnxjAt?OU#Zsl7cJDHxa15f zCX0i!NJ7;Z`vW&-30BKWN+yVn(bVm6skEgq7-x}2)ey%oQ$nk1{R33mW8OlVKtGV_ z(fb-Ebs=!WQ^a%uA=Dopz)N9o)hsKNZAtgX{jxiaMsrIq|uavR(k$>wj~nnL7X9ZSL=F*8gQZi?aX7 z8US=0H`)f_(CA)(h`&pp(HDS|-R0a3-)t-5zP}>+L)(WgjefqDOnM&xl|zKj zS%;X1QTfDWKm`we)?66Wwv)%}Ia;$~6rF84rdlXFCFh)$Sv?F!cM35)fg{wJT9$|k zAmNhylJ?1Xo)09_EIm1B$NUBoD5iKaCN8@WeUI5?eBJlZ;4X(3c|tl}ydwKggkc9H z^ULglKgN9YH+bo3oNdiG;AG?OsubI!iVT5&((B6nbBf)GJBLMJ)F^U@i+TR)uU`+c z%Q84p46%ZP9(`sM9@0^4a6me84>WT(8%X}J&q5{~upMGy<32^T=u-~;;xM#akl`Kj z>xV_R$y61EDtvj6OR=baM6qTgwbK<3vTqLxF>@@Ai4E06f7kBy2c2&FNaE|Q2SZQF z091olr7~dm;@nvdk_TCjFE6`wlnXdP1^h6O;Ql zY5LXf_Lcpb$NZ3Nsoy<3IXOKVoVDM+J-v84Xk8q>Icpu4ZkJIw9Ae{)j7HcVDKPMa zsuF{<_S?Z(>#B8DoQD<}ouOM~6gQ2+<(_?_1?A5)MFzTM>eL&9TRI7 zn$*aYonQ;)cCdZf?_8GPU``tvE*5*0(JzaQkH*Nj=}>ZwN~vV&%EfiuTYkIm+!K3A z?X#5V4F%_{LN?g#dZnrkB}S%d5o@q+BbI_=532^TZAPJSqpvo^7L%gzN)8o?FD3bD zq+*e!nvQHm_0VG@g@wwkLyM4$j48$J{)pHNeU3T8Iuzex3yqK&(`1Wo6MAvTv+Wm> zK$7}EM$bYsaX?FdtfY=ofvix^XzmRjK3-=5k$vib2`56_|J~4KVZJ}k(6;vo0u@Ft}o28crc_%Kevk&VL%s zy=Kz?Z@00(@&8!L^AN>beOWt#4!p(|W*E7(!_KLHqT|-yK^Ig49jrziWV>TLWa6G- zJ)!*^5!+=HVw<^@t>|KSV9B;18xlHkB{1Gtl0~LeOLgG1m2^4LVHP@~=v<|^=e>Kmz{Fj61r4h;`xv2#c3-&0TI8J}YE-xK&e%S*Y7|NfIF|9AJ3 z^1r#ex4Hknl&6IJ7Xu(E{JY-Uf^t5T-$e?G=5!)oiHtArIZEY3yBwh4>sK+>7kJZM zd#tPN_rfo$r*=fSD0t#l026nwul{pN2;+fQ0^z`WF0S?d zQw>ZZq3NW8g1DbCOKO5#QP{o8)J3PqeR6|rRh8(gWP0^O4aeaa7DhHZJ@%kfl3;?5 z@sNK?mVU-OeCZL-Eh;=rRHti|D6WUHMI{{HBPD@C^0*@pbS$Ds5?0)lUV&3KzU-bU z;rQR;Kq?j)VY}-`$ktU&7D~V{v1^c9MDKO=dxa0w#4yaLZhas;Iaq^O(rYr{iedo@ABYfUk{lA)?AlvqNS5afP7QVjW6 zFGZl2k)~ICT0=aEgA89O;PP*xgOxZbV#N6C6?lH@5hDO6s(?h48HosgWe|b4?BRBs ziukE1{~zhEB-K;Jqr$;k{iW%rUOc~prjcE%*rCK^&2EyNlVs_ zg^|-OF(rRL^>i-%6qNrI8`vJrxCG?f|JvKzOT~X}?r!$~OL+>*|2c(_Jbq$Ah=2H| zgpl7Wp3t!)@Q-B3Ox5FC?qXmJzSZ^sgn3c|d8qhQDB`plWF;XT=jrEB=Q2R@sB>8e zDIJ)903$tqxnsdNx(;Cyp8lUR;~Ic_G<>^<=K-$W!}FQ(^uRnSO(i`2Kj)(9qvHEE z;{V$@c>ngX$LZVnIGqRA0Nmr@D>B}O_=-Pg$JYnrV8{d>#qWfyojbDJ5x)qiY@9f z5VMlshvGZH@S_D8oA7-Zlw@Et<29ue*~a}5F(V2|?+WTYF(9+z0>4ZtdV?(s+oLX@ z2LVRuFNN?@0-OuG-t^ivD0UbocEm)*r{egfK+c8WQFL?_p%%O>j+6vuE_C|HhADJ{ zm&MVMz|4hCdpJb&cuGChJb_gFQ*qQ%AZHfPQeGbqE3-_l8#A8cW|sDCG$IT`hGH3p zJWSGG3gM&#IOp{%<-F50qA>=Xv&&)_RMmFpu$(DFsjE^Z$^C2Y$ zsl4+_zAu4R4A^{lC3Y6lyAt@!FBv!(Zvb3FH$kD`Z3%oNK=WbL_w?8}I{Z}$TmlgD zA@Y&X8}4ZoM<#q-0wEdHtVm6ux8v@>dqfYr7d}eymxVBjS5Y~57sF@dOk-L=@OL33 zq{_3>BLERo0b(ypVWLw2R^M5v==pCD7nn}1;(@>YC(r+9doOkWb9Z}_|6wUlV!A|( zR`~Bo(V<0T3VxliaKRc+`j=U}JGD@5fmxG7#DJ7a5D|A#f1*5@dU_O32$5H@Sgh+T zxC(|xK)3bL19E>p^+fqjpMvuL)|fH(^AAPx|LmmvKbyOo`~ORMl9I&}rY=J^6Svdl z7|^l{W3pUy1H^)OqKX`yt|3K__9b+p2#1w)JeuQf(JCZ3PLaV%gsFX)E7x%f{3F~W zg@%D#_neq0O#jnG(ELa-Lv5Si#mwWqe|XPBJc)MXH>oVi1?oRMNXdo65T5?|_TN(! zCO*y^@F62Y2Zc;T@D~*0Ozd`32x4cu)_9A9MLDs!Wx19<`ur~@|6Mq-9_{{TV{d0a zb^mjFbN_!SPhvWFj8-V%-s>gZ|MZ$#q5Ge_=i0QcKDm~jwRLe%(fNOi%qm4ZSYc@| z@Vxn-*8g^Q8k_q+OL-oYK_NK5yMo;caiAzQr*sHC#pRKfb{CggToIS?*v!xi3Khj@ zNMEh|Sef;oZ|Y~g0hn|Dqp`b}jQ_CH*u;NW&Qr)-E{OmswVuu&R@UnZBSCY@f zbLcqX4G=fqb&yO=-*M2dIq%&R=s0OX108221Wd8h3>6^@U%ymTp5;D@3Jkfk5gl03 zL67u8oUhCeb=DIg6=a6sL`FCIdv8cr=2sk^5CK8M7}YIC7%&Q@vQF-p{d6soGd%C&aEv z)eY5`d3R^IUD5q6?7$aU8F;Pe?_#LO>K8@ssf{&{@D#WIVFOkD80w2Y|J&b9`TuSF zKbG?(5Bg$(?rS25mDZ-;iW?S)h!uT#egic#Q0QPy@v{Z4~mn{U+LQrzeN_)H} z%7n?|Wcki#JktXBrP0{&U&sj;{MV_~FVL2G@Dq^|yc0M9 zlyVl7>}evmNTPugKih+p6T}lW9>ZRSqclhf;Twm}5gQY;0%}{JGKL5H!(*+eenlXqQck;Z!&sG^fjZA#>=0A`1(a}%f zBs_*Eh)h`z8N@bSC0MadJn&=Ohz5d;iSg=6;umRAE7npyOr{`Nq$*iT|MPFyEcpJ{ zcC(Sn|FPTL_t5^wo6u*jbVP7tHma6_PD6ePZu zRz}s{JOD2MbP;DRwnx$}iqxkAnFR$i9p|-ARV~xm+YE~qEN(nsx z{V@VBq8cxFS{TFtBcX77Apv<&1062yz|=7zL%=0~jUnUtV^Njyo}pf5_@eh-GVELv zv{E7WS*hQ3mAFft+S+Nxx}|NRP`x}f66cHBE+-EVO4(*pP&GKu(@L;KrO2_c0o?=8 zEWJA^IzsjX7d|m;ec;l__8u$N&WtTdd$;oAF@t_7xUrpR>d-l>b%&UTQTchB2EZ62 zb|0vG|%vtM}8qGuW~Q|#ID#|W?7I_`Ovz9_paK~t{I9hGNx2mW*Pb{SrZZ; z4wT(32V1{Y_XDmbuk|egHw~;}ri!#T#jMU3-gC=?8ziTS^72=Kg}l@%E{(h3m?11# z_2Vh{JC3j*@bj}BNN>XU%I7=^{O)!m_A$yT8?#@P=#`u`HHKgmNmnzUf`0k&6wGT0 zGp~64C-7sxecV0(6L^DwbNgT5r@B-h0IpLVV|oA{g4YJI8N@cC64!!;q){&cfz+GI z|B@wNCL;&uD?H^I;@`p7uLn{I zJ+y~-UBc1CaJZ&f?6c;lU^Wr;;afm!r1k3Kpu8QqqF$Ck6Ay$PWR zf#Pb~@V~}o+e4Jxw-~rc7tf`?u>gg6okhMU9QVY1Yv1s*dZs$u7 ze+hmZ)boJnpqk`@SfkIq^*b>*vep;xnRs2 z#W|}@#4K&2(m9p)Ap15=e=lUo{sU-a}iv}tbXCfCu~Wv}1r4o;5~y}E@w zq^ulDY6iPRktv&wa8;l~v-RhMlF32hLtr;Oht3Cd-$lc80m;rV502vDxnDJCYUmSm z?xc0x>K^u6$AezK-E9qy+7~CMZ`F)T4%&J`=i`P^Wh=+0hi@<1z5eM@Z_vHG7@S@V zj@rHc;Jkg@I#V|mC3(mO_u=wQi^Dqk{rEyV;L12%7Hse0@VwRQ93Hjwqg-PP%5I?3 zJv~3{e$qk}mUY!y;7~k)G~p_g!;DtHbnWGj5Je2xETzU|_G{2>9UiwY&gSW$ zCmxAy-4|-MeRg(w@pf=}(QkFH4mG1q*HOab{GX;y|1-u%tJN92JN0JMEM2@o$T<%G zeuuq@Rj8jw?X$DiQGf8Rb#^|uIy}2<^=4?^AeMy;_6}JS@ij-Ty7w+alMW@HbBEo# ze!ny5blZRE$)bWfX3XKgE~%FOUYUX86Et-(u;$}OLoLI)?Hg{pr?1r~^WF^m|*IspBnPTme65$04bTWr@9 z0?_O|ik=eK#0*`QK+G>X?#PtHZdNxe4H}GCo=PSkRsbNqTnZA<_u7?g_kkbpY|THi z%TSI9pTP15&R+ zs+A@4zD62M#&xoiC;53dyf$8?OM7(kA2F*sW!ReE`>R0w&;8{6k7i?U6aRN9PeRc~ z%)0cruDjSVs~b@WchOQa1UySJhDEv$J&oT^*!`#W)CmzO;HKj;Z)qO+8nIXmx=6&D zDDA}q9fz;|)J6=sHB9PR!_Q7zJJSHL?J{VqMr5L1iB`UUSD~WrK04=;)R1OlXcJf5 zj}{3ueXUR7OW`e7*SyT*Ifs-&OtE8MniI`;??ba#Z>KERnZomX4u6^0+F00CrbsQ- z8TZCJ=NtFVc@pOwmG$wFOs)H|57@>GL+cn>@IJh8-;4_)2T@GS@atae;}BwNN>P7I zkvk@qc>uN>a&-&eqLuGPzQivRd>f#pZP*~b)-&a&h~nrP%L^;jm_ z5HI>5=KHiv=G*_~J^d+=|7$yW|Ff~Py|Mo-j_DC{-ay{Ddl<4Om70xjF{4&tBx1@4`a_#?mYI>~OqnrN;dwf+G z`yV6zOQW&3oy`B;+-dA@=KnID?7O?Ay{NoF@=R#RZ3q56Re4k;hSn1)0jdi;rO&az zKj3}}@%G9HEhr^Ncu3=q2_8P0`P z|4bwbXxrBP7Fd+Ckc_~h{e@)M>*+EiQ{bt+hGbl3bsQ4+)Gm2kt%Z3HNr)}RQAh=F zX+A+E-G8($M5T=LvOf@A zS0aX>1KAk3V=}c&aE<&s{8Q>R%yk2A`JdV*q8`}fWj&BMBagSd+&CE;lFl3~rZA9< zu~a@(OoqP(Go1XZg?o)gqoB`5rhX84XOz8pST>x=Bl<#=$#9}`E%5o3wvMNt(byOwZGD+V}LDFrg^msbpGvR(?X_L910X^1%hs^!3=_z2F$!#L85 zV__Wbf;boln!`Zl=4CGDs|C7WY0%5US0;o*F`+S^qymN0L$XMvINrhQBMZ82%45F( zfb`m=23f?2Qw^$SQ8537R%uzbr<>gNa-H2`rkq9lyNMTx+d8Vvh1gbve;(Rld{S?h z2gKqJunTNtAlHTT9$X^>PF)1Z(A$bcTT$PAwo3>*!4?u>RKf#z)y8E4?*VaywZ!rO z0~mtqT-89I0BBj{4s`C>nAm_3Zj7j!h#`Xk$VV(NG35i}{uQ_zV`I$QF^CzVW=ttX zHnZ*lMHWUbAoeS8hs3@IUKi&so`D7+3mkxIyme0E6V~f?&p#^F90C`v86LT6a#`wlEu}M#9oka)xHh221b*&K z=?IAlpk=|rWex$eqM<3k zg0k2w{&0HLkEg%y1L{Z4nxj66RU~hbLd=e^{h8m8kZObx)#Fy?j2Zx7h9+*(DqR)f z(s`9ystO40*8X}HFQOj&sbg9rohu>e+%2S4I<(xNV>xaw4W zRZtvVv@R~e9fC`OLvVKsPSD`)8r%n$;O_43E`vJ(f(LhZx0yTtx%WKW*Qu`Vu3o!l z@AcVIr!cGil6P1cR4(~TRkT&9{~5ly-f3XHcDsB=xh_{M;}&Um67*Uaz?(|Pq41KENCX`Bq{ z0}Yj0)8X!HR|)Cm=2q&aHpGD;9_Pzjs9JTV0|RQ}ZQKJqwjchoLk@GE%T^mYo2{?G z9v22c9^RAI8JxR?C*E)NJdc!)6W3L`%x|fNvk_Vg^miwn-oTs;eBGJ>WD6&>$xs7| zjd@?fsnhIIlA5OH>ywZ3-%FjL%?>+f{L3&$0a(lX&gv^P|7+)*LN!qnU&M_lhI#bN zEVg0|OwyTG@Bi*jV3nd%L!co$+-#I0xZKjtP-GHvrsD?tBQ=#fQwdjIO2rK3C>$_R z)VFR86(yQE-O&_5n;1m(@&_1)=W~(fj4tcdK68|sjOd4cR4h1auAlt-`k`Wd>hS24 z%5T~^fHkq~&MSxhr;ZKLgbI^fIiB?Ddc2FkB=eMpQsB?jEWKJ=y;_0w{{e*O@dUeD zMT^RL)&~T)cSi=jj~cCA*-6KB=Uqj zvikpEG^*d4e*S+T&50_OvlatG;i0=w&Dc@Kr7b>e3+CQ>v}Tx1cAHLIe@H)j+9!e` zNm6lbbNMPecsx6xm0k`ew+g)D{P5i(;s3=@{Kz$m7~Zql9`uHk&CvaNC~IZE3VTbS zLaZZN(?QpSxfJZ?{f|=cnjtY*f}HONFg270`m57a%r=%vmEi(57lR7viO2Y7{U1MLi4 z$;^^d2lQynuI+Y=ZC5P%`9z>VOp`s12wDM9t>Ga`Lsu113gjaXLO`!UV!S77-w47l zsnZEq*$&aD<5eRP!H5$1PC1KQL7xz%lsyqP8T-)PNV zfA~_FCHBGBh#z+ClsgKXL#=r(+P*le>A_LoVPMUcddp zpg5zm6|!KSbi(}2QagPZEm*<)Fj13%dio*YiEkQEmP&17WQ7(#jgY(hjd0$Ga-V^$ zHCVz9f)^z`EkonxL5Ddtcs?@iQo*3R>T{Qx@qZ3RF6N2wwn^m);?<{s<%aG*@Qkx4 zT%$xaAyoSnsJPCtw^1I*QkIQE$Bx-6^)&z&S}=nGM)pl40r97xDB$^4= z&N%R@;!c5WpA%MTmKMaG_%)C>X;@LT-H2n@KAKG5?)?eZYwsXefj$8JiTMaOw3TQ| z7db*=ocWO^LF6vNPmzMFLFjjJWOIiwl=1!Ngu3M`11s)2{&`yzVy;0dwI*&VnopCa zm}@yom_l3zG9NRv*4F{kv* z&_aFD>dZBZ0{1YS&VvYc$^kou=h4zRC$bMl*hp;CNCD7P@v?ah)0jP>nrPYogyC3l zXDxYuG5O=+jdM(e_imMU>TzIfzugI@?}-G-+5OfE8ku7*d@>5UpqB^=Y1mt|qvS`z zx``wQk)kQiDa8^Z7y}2H$N%)=Hi>#xj2#tluWj{U_cAJ4^V}@(|2do!(fKfxQd{%gb1Zdx0Ei2?x z-~RRL`ULV1`FVg>zV79Nb`@TRst&^8&_kx_1A=f=m<#hrc>PTqGfy(aK!!fiZ8F1Y zfqE~%d4(}>&d2o2mUC`dBG_?=BGaBIUDo7v@bOJgYQ17?Wyh`ZlZi}IFJ9zcx>)VCv<)4IsF2LN`^%;x3h zSMaiG?r0*INY1LGDdbA!XBu|c?lUu$sTg8LfNukzHx#JiH6;3&)j0-q{cFcGGQfh5Hp`zk zvbEoW=i9QH4v2LV?C4mRumH}=dUlleK1>Cd#D$`K@5#u9N^9qhK)2Qu9?gt(s^@(M zIDH3T*#%Ej$!W%{FktzPatlL&1$}0vyC0V_W@PJsfE!{M#kN3rAzh%?2ay7od<;Bo zt6RBu7kFlj`Yp-hK~~|QI>Ua)9Yce;<|N>~nhf3eF|HSjdHDUG^3V5jDhxkE2O~D3 zcV&Ml3y>S~?H9BfuR4GA;VjnrSz+dV{)}ZvmqEcTJ-<-W%itm9v;o_9*MN}5yKdSb zNDk#Fj=nw!}cj8-;jS41E~COY-#=`OBZ#2QgWa?l{NEr?g}1^c}Vy8s*Z?rK7$h+v0orfdfsBmNmKD;16{sNocgDv^yV zj=7`AplrdH6*(&E5=>P^Zif2V)J#+Hd=pm=HP;sG(~~v@2DSlvR>91B$i-)qX~_LO zcq@9XG+4cOrzQVL`m}X{+5PGdeo{rm1T#rhvhBB{!YsBEK5=81d@>7IVS>AbT)(AIht+~5;lS0tt0px_$^ES{#%GZ~o)PTvpJZxlWei** zsLTXzLY%iC{dwj_;p3F7gYhXqXU46d`~7IJY~qYr4(w=0`v4v}i+MMkxEs0^`K&;! z4P0t8VAhZue@4YTqhE^>JU9UGR(#5X+Ft{>tKI_V*2f^iQ^<`|f~}iys~=SJNpMOu zVT3}a#p69K;=%t#PpKUw>{haL4)<>m=o5F`0(?YH!1L=v0R2CA3%4R{>CbpwF~qap ztWf>meqh8pGX|0kS*_h`iUiy)t z1VI`LtXFF&;)FbJ$kT?H9A0;nA)nX_0s&ojHt>P}_Zi%UfF3D;32!paTjd1g>JLZ> z(b-IQ0qiY==p-E}wxYe@86{QFmE*fe+MQs-`wWRf*050xL9m^enuYmTcysII(EBa; z*CqJUZdZiqK%4mmnD9&-DN^)USJ~C7+Vjj2{Eh)Tg@*>6zzBXyIs(>txbxfs=x5X> z0dij#ZOw=wOPGiF2DrH^!c90K3wB+Mxn{kEyaz-F{}V2Ak;02%TplXn3Y-Mt-`g6% zOeg@|?!+3E2xOXjzO6|KN%?#iv(}x%gDRC7Q zn5R~Rc!%7}ztn6>FcCm~$&vf_cB{sFpD6`{P8sBQ_2ojOtM2KE`xZbnGpP3h5DnzS zI=RQ>I7BbktU{ULz9;ehV2HbrZ1)`AZ46M5oPjN1w50OR&HH`a!9akuU13h+^LbQ1 zIn4(<{-8kN5k9Xi^8Ax{?pIZJII34|+^#L3q=|n8akX_AEruIrZa&pz?e7@3-*9=a z6$SZj0bL!C;sBd~<6vO640F$5Hvj8XF8uGIM)MTL-f}-_sQT?*s)Y5bp%L+fiE~+Z zd@Oi*JLbPB*#CgdKfd3p(6oIr`uaC@L!mBVd0ecy2kGD6ws>95jPfr!TLF1ZxZkbK z+pFa)H`NmQ-Zu7tJ@#SD_g_az@$pD>;HXQyvtz&&Ss0mIKWlz>y~_>j6?%-#6gU zE-LA0O*rHIFJ80W8K(Mv7+hr>mL&rG}XTy~UHD?t$gk7GkhdeC9P@ z04@{SgMuL3EO-CpQ2liTFOBQ&Qbe$wM|3#Til`kPLw29BZP+F*Lhf=u+|bKOby(3L zRoV^ua0o0Ug=i#P?=;DKrGg@v$lLmk3TkC!>g+RwB2471LU{WkGFfg`gfW7Gs6x|2xDMfl7xH$aP9cq$vc*}7t(92;xvA&uX2j)( z)c9pTP)!%#8DT_aKgJI8LL!AZKKYF0M<5^4P-;^9^;pnyhTvLvi|^!}B;txS+3SK7 zkU%hy0{kSC-5*qlS96Y#p zHy_n+uG}NRS77EG|J>&$`FrJm7~K11XY^0wM+gw6*h;BhKc^F!vUn8evU28U=Mkq> zi-GRCA%M+6dTQCkMQW@opud@ms>JI3G1$~3c65GwaUxac=@=sl_8LNeJ1+BPs7bU* z;r$|du1ktWW6nWORnk+7@sF`6MgP+)_1jWWE~Y;HnOJ7M1^|;D`Ez9ZZVg>R%QbUj z!kIf=hcx2My}-p!u@9EDq=a_NKF@Kchyq0lNC+Uv&^ER?GSa)(P=2BOoqMTtKuD|s z?T5e_YWT{Cf)0NWPn1m{7Rp?4SLl7CnLvN6&acC)zsZ9q3ecLaKV&l?O_a(^aZ#i=5hK#IRbS>@_Gho+V5$Uamg5nq$AQBx=s^XLVCP`TPvf zzY+5{6U&b84SPBjr!_Za_8nmve2jpF28V23%pvHqowi|z!>~CzUWws^Z;<~dLP)es zUEDoFl!~QmY76}&_}lUP5Dyi^ywvO>I|muLMN99s);6G!VSJX@V~maZ&lvP&f z(s&xT@o-w#0$T93vzScFkbm71>b=7(_Kq3A?*}thl|(_#nb6ZP9=CFUJORPtq!Mam zxy~%G(kVr5#mSV>Zue*Ee=$Wd=eoVBsg5v1hAQ2pz|#GdOhl?5lzt<3Im~)e2f&~2 zmbx&v`v4b#3~)h5PfsMcIoYz{2`GNTlstQ-MiTWV{vY)vwNtYbPW*3omZta=0So6#g0*>8z1ZZoy3yvAISJp;_&^(_P{PwAhU zgQlZ+Y4sg-k4mu;fEJ{&2jWW?JXP$56F)ivV3Q1C|3@>(fAOrJY=C=`qGu=&X+lO6 zSdl`c#B~|7lkC!~(jooryG(@aCN2hs@9r&1YX|IGQ2JQanWA1+efO&E+i+WJ>+Vrn zAc3sGaJor)_dbv?qvwafXFZCbCkUh72)2O>B*!YS zP=OmRWs^6&mmggi$w=-NZ_st->__$EqF$H4n%21Z`N8?VJPokxLe|6(utRe4ZoE0bX}2_`j+kke+391S^j`O=FJ54-N8sgmzX`ML2A+72|-upxnF^-ywo7~cqE%Bk<14QFd zrgYYJ#ELcs^+ZJLut*T6Yn-uV(2evjkOd~?s89z!#0l#81a)nGM#TPRH}&alxDF_7 z*hZy=XsRHN3Dsp})1@}52I%>62>4(P5H>}~q^OS3vl6@m7#cu4Q3w~4{zr}AwYxO6 z%Ndm%xNmoUQ7aL^+h3u zn`8wZbr~AkoxFRg_HxchnGztgwkmk7$a)J#ZK7{%Y7?sm{gESOo*kx8mHl|psA64idy9Uh93`DhC34JKG9UrOHsb(EDkoAB@XU9A) zYWRoDseiAZZmCJKIm?>^=<301iidzTzn&)G960r+yr^<)y$*goPaX{w&YH``Xxi*G z&yXV;MguWI64{c9vZ6Dk_G>cYzJUt$FU5ZnfeBAkY~lfvy(2QtM7o2{cK~iy2yq{m zvs{<-nD=pb`0hv_x8Iu<-JVVl*Sb4$Sp9jEsVj`-R0=ehqU|fwC^eQSB{PLGWDc>` znUd=UQ*ls=l8I{oA>ObNr_lz8}`$;@t>4IK}0XDCM-e&Zk8S%GCSdSO;7v>3-^i;0fs#&O&&U(17c3f z9Df6J`9O5$ zomj8#=tnW+e;9=;oHOXewCXmnxwx(?aZVbJsjK!oXgjrxI!k3|O{Po}ce3zP1kT7X z6c{F@$l@lyyZk00M%||OeAcp0S*H)pykZ3!? zP4mWtav_Ge7|y-|4S(b%;0Q&pqgdP|Y7}`d_*Y?}+>&*d4JOg`^a5nmvL2hKOFsW8 z3k;Q?I#HnD8P|$-o!3GS=ho9r*52jf+vMg!-W9+xThINa4MCvOy7x-oJ+8ARYHv%# z;NoMIclAWS7jS1e*m-){+$yq>Zofk1p#^Z;I2(ZnJv;a!6Qi>UZ?5bAb9A4QZhO1= z%h%<=f3?#k0zTc!cI`^@Aop)m+Du0~y1JBczwsILe)0kDSNeE0nmPc1d~G0CVg1iO zhx=w%?-+>wb_s8BOV2$o4G#x4CjC_-n~!Cz1p0S-FKKZ&N4rnJ&b&_AjVG&pJy+c& zCE%)1N4uSY!GkhO?G|VNnA4V769@Q7aD3Rf3OqH)G34GjBfW#|?kN12jC$ITKD%2y zSR{5C9RiR6!AK&x-L~MHBf zjvk%gK8Wv^vu^Teuih-qOc3?m1bW&I-5uX-Uf&*B9qU9|Ti0ZOD|xT9SD^Q^%H=I? zu&Co!+>Y71PaF6s{0!g6w3+2nm@|i!{BrE^q3g-;PwUo|-j?s?8EMx3`|ZAvoq+;q zebd&jktN^j9mIO4|5VIF^fYzRvxe~m

hLzEkEAx$fdcY+U&|MsqSta!~IuL5%2ea=g_8kzR?+M^&wySodax4nZ6`>SVV zPyZK7L>P~^3h^a;sMPbf3_;EfnAc{@V zFBWW0OF2`)5H*NU4{i4zAoO9s%YztvlW@>FZF^SlVgGtPa$%6Gdo}g3b8%B5A06zG zHx9Wz~`zNsX{6D|t^IkrI=e_NFllGhCEHz>fXgQ51^~JKzkJsztX(#Up zad5dOZL~jY?!wa3pU=y~2(&8Dp4$yt>j9lj9*lIQ)tG(=c5d`Q?h9Dv#sGB+uW+!2 zkB$s)Pp-|y>s_|9g5OPY3`p9SwBPUzvw=;U*$IaYT!NjxCs$cZU94l98Qod&#P=3f zr21rexggNx{q8${wAB7|Yu^>_ z2gABaO8CZeR>#^3@?Y+JxATA5TM+yP?(lWl>5p1UgAhUHy?hRZX=$`ahMPS>MKvPP zQSw&jjzMBJ6+;_FUuZW&;4;4!Z9~L0LT>WurNhA}j`(j`<(E_vcSVtfK&+r^n-n z_s!$o!HxCZ{aseD$6njC!<`*i|E|U!zU*y~wd+~?ZhCTOt!M^P7C_eH>Op^)*RNT& z+AKqmjlpHkzqHt!H*&+xpI6cVKmVH}L*~2TvovPU4)7o8qcm0nt`9AOJTPGn*w5Sb z_3*x~X-UBO@o6~kN|?B&Xoi*lC`Y81(8keqdDqs2b}94FxA6L-dUU@{|YRE}pX5n*rfAj^Zv zv9Hu&6Bbg%dJ>;#dNE%$0`ckE_S(d0=fhLedCC=T>ZWPbmF~eWZb#=f33#CW@X(+z`aXb9DyZq3D zJLNBY@_F$|ShOb#M!MOPba-?ahYoRD{sFr2%HEELCDl&dX3y78M}#M$zJ#4bO(LOd zP0Rzs-;LY!;6A=R3VsOqsB8XS%Dc;i2P>9+&<_A=rl%bsUhf|L7ck3_4kxJSN*Mn% zWH#fg8Z2>-R9ZBZP{SX-pL-^Rlk;X=Wsmdv3#T$tprRC+?dOLV5cf|nxIf`Y_oS1~ z4)sDvu!Y(D_%JNDL$Y|#vl4v`HOSk4=D%|M4Gq~2j8T~N5ZuPV`a-MoX_x-fE7EIV z_~+(w$=B!-2;sZmKBw z3qn!GzQxH^@;a^GWoYZV8Aa1b>(Rxhh)dULSCy9{Fg^l+-=qzc3)w~e@7qFU-}ew+ zN4~udzwy1EFkrwgCzEH6iJzCXS|{9@lIF8D`cZWGK!TVkL`|&+c%gQ_wSRr z`xKbvbZn@8BF|t}QC-$sS_rjqpi;2gJ}}6H`I?tPFrc3fVDYve`_?CF*1dzBytq6AMzVaASUWMpR=V*d%?QA+Z*mi=)m)mBmIJ2Q;R2E;Sk z=9nho{|QAK#ne?d3VG@%o32oxGA|J-gn39=!07yL_(eeIT~s9% z0lSB(B%bd#G@tNJt?A`Z4koRc8sc{aiFvM>X0n}o%M~dKdzlO}l-h98R5X1h%RZo@ zkZO+a#M}fDC6)O{GJb$panyp&J%*<^;^J9%SQGA@vRF))JH7Up)aBcHYokY^IjxIT z%Gcx^MUgYuTrERpeObsxZXptU9Lc@aD_rB39Ld^K(IJh)CsUeyn)(G=lZ=r_Pa>1H z?x{A`BlhtN6;X!B8=-dTSAj2GGn1%CJg}@8QaL&2=bK_)c384_-tXRyfM00E^EvQp zLQ7muPm9B=FQAjf9fQVi=92BX9O58M_~QPHCJ9-*u-2YkXEqXkt)E#v=kL4so>vc9 zdv>{pEX_nikB`a*J^nt;0N>gY?xOmL#y`FOAGeF3#XsZYRf8KjcZ18LT3uytK5wH( z;R1Dk{(g}T_GEVb)g_H{_?}?_x?HSUX00>0FuVj#m;=1Ub^dJQeWsS5IMf<+->Nvt ztnc!9I2w-nXxhlpdMnfFs+;*!7i(~J0qo>oY0u_csL9bH~lA^wDDCKF?sdhb{c!JDA|Kf_gbRr4QBgC1{M z0(GSqZ|(KpBfaB=5|I;~W2)R?WELT4lsp?HkNgQ=%1OS0(Cb;}&2(Ihm#G zJruJ!LdD+x($wR8sHdA0q!m-b|CX&hYNk)oD9=;HOX`p*QvzjB@pqC2zjyNX?l0Pm zo&y!7lQw3el75DQUIt{j7nN{6aKg()1yU73Z%+s+AsNAtxThj8&q^SmBAM9mSZVj2 z*xtK>zkaIlHzIy_nb5`fz%jzW+QwkRlHwe|f!1F22xgWq*b|=_oo)QucOxp2519Sa zKRaNEc`O#Bu)T7AJpONp?>aypnDq;!_$QnFC}o=+VeT$Zy1PK#tUf` zXd*)zR$$;NM0tMdp!o=t@4()Uc;r@T@@60sIJx7MvJ~~T9Fp)G`qiJTwV2(XIgofG z?z}faXRLeQ>a-fls{ON^vcFim^en?nOp_+CD}q6V7d+^a$3H(aMFaEu5Nttv0G?kh ze+Q&GeqL0&(d3l z2~Y9=0AOU6NbA5GG5hb1j?I-2;!MjVaekA9_@L(9W~zLkk383!N3XL%)2CkEy^u{+ zB9~59P2l4C{_q5rJAWnij)~0&EkJA1Me0s?8&cjXJIC>EFp^p(~PHy+H_P@-`>G z^XvrY@ODgJ%#qQotRTM^c>hbO{Tpl717pK_?9n`~{1ZD?xcq})6~xM59~ET-2j?}F zRxvectgv0yaFkk_5E<;OxH5*wke23_k^ok9tNAgGjMST z^bJghe?QyCR?HiNS8kNk|4pzqnJVhK>C*Pm6zEy3vT&$VFs#GDqF0DxHDMT+LAheR^(5l zj*_2aqwLs+{A35oqe9s8e#K1`SqbKRO81yhil+h8sLR!38yI?B8VtV;SewU>Rx)*{ zne(fe2fC03?^d{{HX5s=iQ?2`#&I46OBy==h{_NUNSom6{Lq_I2Xt3ezvtn-eN>@> zY-#UAnz^$yKy@X0B7yIQxsbt4Ns*+hO~Kz6^hRk~o-GOc#XIq&dbXc#yURLYUrlH% zm)_eF0DGRxZm-!P0-}_-rFaPNcfbv~VSv7&Vcj3dVuld&o-$FZ|1gV;($Isd^$ogJ zIZ;O?(6JIw7kl#Z%eP=GR&2*gwu&r0{%Tk-Fud@GIh7HQ$2+x6del?z z`>Y_vyoSe;TJ)$YaW-5~JEAf#u6!v=+(Aslh}{t*$hnJuDLJ7tlqH*ZU(l=(WDEPw zR==D*e3UX=kweAJUC&&0q$Kn&Rlr_zE2DeDv(v`2NsA&A-_p8Afy&^=pJw`tERl`s zn%ETb{gf6^m*%%VwIR=$uHV<-5eqGSux=Ky259s^Pdk>ZK=Clal32BDwg_ALse{uUg+w>51-;M0|J^~Wvwj${5@ z)r;@Y)PMhwsjFL2$&X1;zn&r|FfL_09KpNO$0M4KCzalQDq%%GG&I?zWb`(1sx5fz z6(RNO5F1`kArA*%bxiwn)Rg_zbi0O2W#*z%%g@E+X2CXWb*kJ0Eq3cz!6p@1ye8bW z18dKzD0ShSptcHA8~tJ2!NbNo$zrYRh3k|FdoEXVZMomDE%`dQ5)4?f%6?k(O8gH@ zgWwEUHzkQ5^SjN)Uee`C6(q8(I)zlS#P?&LdK4Hx?OGlJMvAPK62DBg?jM|OPwa4* zXUfG-I@QlK!`tTTZUw(R@>;X7x<#x|d_nEp8-s%+iV}bBhr<5Fj-l+zInTx@rnD~C zmkc`*Ch!qNvYz-wp*rCs{NNe+UzIgHXiOgH|*Xh7t}&h zM^RU9@dhA$;QF|b{Ky~=4_qn%34NuLE;rWiM-W~+;wI84Y$?UghxuAkC$ z^Vi29@!e#TsRBt~d*4)J=n<)6%;cOrR$j-h$$koHw9;74{?aa`A1r~TaQ&|t@z{$ zO`o~N;f;J z39;(S@}!+}!jeN6$JP{@G&?YirP0O0?r7DsmTOBF%3yNT#ym5H(|T^H6_Ybf$WJav z8x0qh;$^88{@1QkK#8YduE|iTJUUu5-4{03qq2tWn@aK;1*CIh^`ukx!*OS|l2Cl-ZHLo~NtGy!XlU`E${^XYyZG@( zX+L{pGOq#3UEEln)Pr#p%y z)Y#n0M0LsIUDx}jyaa}9F+mKCp}i()ZtCHeKdA!kkfJD6oAAN-yHoH^Eq%} z2(g8W{!BKf#NXJ>giqq-ZSK=3_vu{Y;orFl1msJ`!Q49gf|JyHicK*dwe{$^= z88%o=zxuingKSt8Kf2wn6w9HMzQY#J5 zBn9j0ww7-nX!|(v5{fqtK9v3o^5|KsI0vLTE3$&+e+FL|@4_>+4M+8n3|aJ0P#*BP z#wP-bMLu@mec74aGjN#E5H;BIv&$~0m{*`M9=a<#@qMPL!J{MM`v4zPY9g>@PgjG% zI0SB*p9B->^Y35WQN>|$q!2uuRO;nRkGXyooU4#Z*09xNo+?B?c1Awp$9VtFlCa~A zf(DD_yq`Xw(N~gzJ3mKa`R8w*xa=rX;}?GAS&@9rx@nm!>th-0E$r6afnxzxEo*~} z@l8d01p~+kEYD%Q461L07jOnI3m1Gk`HWP?lz9>R2L%8a$z8>&?fz<#A;Sf#yT3x4 zL{M3#9~gt;7+9P&f>K6vuUnTi%iwAEDzzH$Wc#HSI4u-MPC9|*!j>Qn8*NFdrZ#ja ztrR6I1xK?ODywNx^B695bvr{$l*X)75>}d!$A_3ifcviuY+6uciMYS=a6s}Yl9?pF zR-=EoOm5R7c^qfeH`}jTkoB2P=UC}%E=6w4h1+^>1;SSD@ zVfBk>8Bo|G1uA#S11FZ(OkXKajF*9b0Tm2GS*GBl>etc2*}@SUC;kAW(i^dz(AC%s zI{I!7jhrO2!}RYWuqru_VSiYi_ZM{jOp#e?=y|weTq`J(c z4oAW9GNR+F2)NsjwI&b==E<6709~opmgIpeez_!G>M&`2-&vxZYH}6X5tx($(IduQ zZ7kU5zObSfAu%99Wr%=viO*`xKy5L9yxl;MDU3UL;Hm`jnO)XPyH}Cwm1y=j|76~n zO4t#>@DCBmDVb%qxRpVy#*c63Bgt}bXlT%};Evl?eOB7hn@q@h@wPnb=3z%$x00X7U*m+w)7pu8nGDe`&%k z%%|D}WNHx*P+q+xx1Tn|v&Pw9fB2ZD#z~Z({Hw+|F6`2VZ+WK=(h=6#`}+!_$ZY>;9IJLbS#v%AwMc(EjWiACjjlnM&Y3& z1TdO-tDX`YPvcD6`tDSE@O5y%D7*A;47n_nqlw<0d(MGE1plao8DU`jD9T(fna|Zp z3Q^YbRC_qbcGHUnX#LoV%%=|p1%1jPsq1IIPrie%SQ6vH_ZTsR4sG`Ja62< zDlH`|1R^6AkYxt_<-Gdx?}y45uHlhZS@c1W_$w$;6Q4wHETub7GENTfcQWY9sMvPq ztc;~xuk?cfwApwO1kHC)*AZxf1p7<`x$xlssWCrtV47KL%oWq#GS|Tgz_viIY|mPp z5>x)tX5vDOKW2{q8J`Z1)1E5paF($%1v!|5zBtGt`Q$+=of}tGFV#9&HNkqOC?#$( zPsCO@>!Vv+!fxWuoZwYLZtghg$W|i!8P}2cb$KjA(lH^j(wC9X|G14R$R<^equ`Wf zaAYamR&2JezBinXrU??9_3z|Sk3CUS_NsM;Y{(zQl*Q;GVRu&v}y)FATFr5 z(-T@hW4cJ)TDQhR?aBV1D?&hhJN!aJb*3a`2WHE6o5RX~m>=h#O^G_8^W;$i&M&1hA zlkpnkV}ZO?m1=OAGXlrx%4)4*RUI(QlO23F*Mz!O0aF`JJ~Z`EJ_^GpWRLBt_d}l$ zR+NM_3uIm2LhR|1c&Ll9HuwcsHxT@+u$r+4`qV~mFzQ3cLY}Q~h0xAPoy9N(JdPS@ zlul|$eH#DLvI@~UaVaBthhA)R>+sLB!qwWcwQ*aqQAx@Vf6Gqr+2!+_ayD2DBv=t9 zE8xO;FgAK&{M)Des_cAe0HjrmJd1F_KFCOPPWX=Cv5{U}^-YU?Q^_cl`o zl!gYSMFAGbnO1*qyR=9FThlM!-UB&?0rO;UQ&`}d_|5E_tqFRYGP^4#om{D5fNrv? z$HZ4)DZhd0-c8#by-wY&hZJK?1I-CfZoI0dfiq1FJ-BWhuL%H-arU@u(D(BIqEj`9RB@< z(GSxr>{i!w|Do6$EuQ?t-WVHfBwVk3p-^J;$HrZql*?VTF?U{T!u?$B$NKOIV+V>j z{!$|y_*RMHZURg;lb6zkiaQS0w4^=(&s{V#LL$e{!jr*N)@!iZ2;XPlqEQX2;dJsV zt>PwwDpWL&oIKzI?sxf#3H-feUrh~~#=)4{1yaGfJ3 zhwTjHrjfM>;y%cKMi6fLaHQH5Xq3`rY2?~2#V>Q26}z2P#JV!${@v5R2+ zs6_(ezfeEQ!!CbN^AMqoOHhx>Xz`Gt3$340Tr-h*um!_`579m(tLqcd$jPFY=cf~U zEgZ+D$Ky&WX*sAWQUzfwks6+D%zW$>xn10r6h>7-=#y%Dk$&Si5l;=($*@~WLi>%v z218}da$`E#K_FP@3Cq4;Cjj=Jx>I`pY6Tx*pnz~>L8W#$mOqt@9D)H?vof?|GXo>Q zlF{|2t?q&3=_vwKs29^{JMgD0_?DpAb>#OQU(YAOe%o?!~e`gMS%3FygpKxX=%oWxE@F1N3r1;Iq zhFJ!e7;p!J@-ZGl@^QwF>sm=1OOmbOD++HOoYNAqL;Xj8K#RAeqB!|$u4MHn9^S-} z!K2oI9`Zu3e;porl!CE7D;d8a_kWcI535MW!VP%xp6X%;7 zo5exhZo=TDiBG-$g8$<#y$MR^M1XZ~#K<%!tPGN52PkxLk&B-#r1V9#$(x@4AAdW# z_gGUx+!!8P-LpvnFUunI2D1a~a%9oe3qDpIgEN`I1Dl_lJ!%gZYyGn4K20M&fw-1{ z26ekdU)atIRt{MZ*=?=w(dSi;Ia5RF{RvbIeXE{`0F;j%JV2sBP-$WD4chJVH~G6B z^yeQ3DNr@rAB%XyjasSzsr1{T1!S_%6di>+#Y8}Kw99MPn0NsV2zPTksx|BOfH!vY zc<|S3j*Bfj;cR8>k?SF$%koFb$X8>nUnq6;5`YJG@6bP^Yl*?%W2{uv*r*wczOsIa z*|I?~v)(WDI^fR*m?IXE-Tt-zi)-l}o25f+0q)}e+WBN#diQ_$I;ZH&x}aOfwmY_s zH@0otwr$&X(y?v3< znbtRN3q^MCe+NfBZBB?NgbIgH8kR_O04Yt_`}Y04!8?SxVhq@IPT7j6jZnGWX?Lk1?C5fc*}&Qbr0 zrOJu9Cta8qC$6y)HA0L2?sopxQlXb20QiTY%U=G9dVFm)5CP4?s}UjfA&H` zUC88iNwwH?dBsElFGsQKynvmsz-BqrlR(zgISNHPEj`q=XkpiM`@Q_6=3J#o2zkds zFH@Q~g%jYO(nraHC8Vg^-gS%r)XbLTDgUZo1X!D>Vzyn zi$XOl!yUYu*s@kE@?+Cull4YA0h{)rKuYZWyOzW_FnQ^!dZ2KYJG+}Nbz#bkTB&?P zOf03!l&~G3@jV6C_EGb5RO8F=B~lwVq3r? zYjh3VH~>rKWAeVuh9`(u5DV({smzSF@1V}RWkoWrlDtW8T#`|qLDRPCoz=Pf7^(iq zk*aZ43NSk?7?=I>4EAdF*wZLC+|7gJje&ynUvO;p*r)a)Ntbge$`uVyN$tI}JL(V0 z)To8iqQ>RBH_!b;Qi7VS9Hg6x0}IhgcCT0YN`#s9%xvxcGa8x{RSWn9%>ZlGvKb_N z`c7(|9LZcp=sm+Pt8^i31CGx{;1vOfHWEUG!_$P!FYQx}(sqNj)N#t68hLFwYCal!;$-@Srg(Ojr# zpiDV51*Prux`n!-u1|#U%3(4M3XK;$8a1&o8b^0>Ue>Ci=})80>EfWO;iF zcgYrd-d%UAJl+`ppb6>6R;qiOVAYz<=}g}w;=BxDL1O}O9Ho-Ypn?GEB3mm6!>v;h zf;?QJ0yiHPeVhE(>vY*d%V?vat4VO;ka$J_cm*E%AR)8`WnF zycPwh2Zm#`ZK9K(0D5BdN?*If@gOvkP@+lm-!+<3*W`(=d5|Y6RMoK1=;U?F*rkVz zMrR*?G_xUMO+kPr+BB|CO0IE-i}#g2WYayPBjYcDNocTmXIEhNYyFII4zF`zw4&ly zJ>O_}1}o@19|{B%&=e;(W6uN3by;T5N@c8#Q?zlR)ShK&QbAF2_Vn*;Wgr9vTzRPhq99-ZvL&S$(QfH@Z%-++Mvl)gD)@iQp>9~^3 zgBc){O&S9nPl-kDB1K{&w&N}4&@X-VFf3$+Vdgg^%hFY3dR9S?68yEPz5I&WIrol^ z6|z{2{1Uq+X|*g%!;|u89mml`sd`i}(-w5i%15vET1}4g>_JP4i&$A<3^mPc5hfJt zf44*MBJe~%LH%D&G;xw+Zf)p#X&nFZ@oDjDWY$H)Ony*|{G7^J(K#uNXrx_iR_S|N zM6;0R`w7E?rnZaoa$%6I3JCl+lsN$@$tsU(|5)G{Yr|4uim^cEsg11`)h(Ud4*ThJ zMh~>9s!oJFW|eOxjQmteAP2vlr3CR((Ts9Rt@71__?ZR$-C+dJxMmR9%`sx3G9VyB3owgKtf`<;zY#wme3@b09xsB8DIC%h$;SSAK~Zrn;6Bf zZ`&T-!ym7+tNNY4p^?v5&Marjm9B<-`mW~pl9A}0(W8dRsLzaNzI+ou(fzT*Od+fA z(~$}0Oio)z!d&-KN3_ngGCI3@+7T7zwi==@U0i>L{ zgI8mAvmiVt$BmA$7Xot>OGCrbCQ2UMqN=~0nN`KXYy0^4m33x`MQ4v(>bf_zge{<-}|mcz1jxD~L z*Od3q#41>?4Qx{ZX0jeDMxzJUJ{%KaIbY=50WQn=i0=`domV$mvR~5e##bNLmyfY_ zEL7hc(^6jyhhB5X-&PmjO)t@qSReX!v(=G6jTRqZlEA+Y3dqLr^3}`@$%5l4s+vi& z!6(Mu5qidhW!-kTj*zF!S%q2&!e}UKjOTsmo)i%21A)JJL({K|7O_%bPQF>piO134 zVUZD4?ShQ3r6*(+z&{woD`8&r!9VgF>D_UMacr=!V^9#)qC2wQdoQu!9J;P&QGH3h>c+h4 zmN_FYvZkk(aOu>#Hi6y+QQ>x=>$s0R@zn3!`(?3Wk)94+9e}6cRSYx{A6j6h6LuXiJ1Z z@w{N#HVw{z$co(_D_ZlN^4E(yC(?K{QyI1)1&}8PL;p33??h+LEYzCM z`j|DX1aOc&3Ae1q!t)ES4FuAQk>7bKV%sIAPN$(- zp|vqkX%1XM%O@sU8XO!>|Fri(jBh0N;NhUirhS#)!P)>-dQRAl5gP|^`c(@582c=u zBNaMU#OG?lw@=kWe;fguHst&MHg?S$sUP~IsR^_zf{61(j(ePD1`(u+B27(meXB@A zM?Yee+hwkMkl3nyxsO3gycGIfYYIDNm7K2`epuWz7k*01Qos8Tn@}byyjF>^`5URY z9;G8Xc0RQ2M&Fv)F5=!Q?9O^$QxMSf+PCKYc3E~@OBGQgrHb8)oo0PmN7RwOH3`#C zLwD;@jqN+#(f70S2?uL`9zZw_>1#vezgKijGKpav(K=vnz1>$=4{Tn<*NR*o8r&{8 z7SY{N+t9!%vRp10W-UC{@ekqw&-CP$KA0%B8tu54$mo8ZoWtAjPJ`wIp#0# zI3UBY4L40@rKcX=NX*gQVnx+j*Fi%=42bNw6W^nhxY{jb`Kwoj^R*1MpGxwR{ zas72j;^Ru)Ek-`mqxQ$y@>ZGPgVT5-?(YSdkgNk&EoKE5$5 ze8Ml5l*@yH|KU_=@{+Q%j;3sTGgaz^K`BDz$qB1`TYnlFN|oR*X({>t7Z;H_rVTP< z9eY*;x8S#~L|};}O}OCwjXjzD*5~ER!^N4k=29*G$@==n^9_h81l1$n9(w}93Ot^t zx+yz>!+$Qj-M*V>I9~G;ut@2fJoSy0PO?$kZ2FLUxXr>6Ga|~BhkNK zA6T)lr?T$*AuWLJ`pLhwy~e$6AM+F6 z9@kap{7`|6IV$)ZKLQ*qtp(VEYmXh@m%}j%mDPmHmX_@7Y>Rkl_8Dt=b>L>;M&E-D zeDZR>n}W!b1#81#Sy?Vu@wiyuG2^m1k|VSks}W2y0UeWUc!xn!3PPv`t)4zX-XPx; zb&jf|@mnOuM8pQs0OR0nq7>33U|R#Oi;+mwdTdHt03HbzQMNG3jb8*$g~%e4L}gFh z7^cb)?tU`zaN4y^S>Da0kPzb-VQgxuGbE#kk)}Zmk?}XR?z|@aJA&>; zJELipva#vmVwt?jIKY6=fwS!VF+oHFt9)Z&+ZErv0-uQUCeSFLGh7EUSwp&%SyhN>7JdAE)LOs7uL7AwMM<6}S2$1!Bc1={8y7e#v_n z{919TfdpL!{_N2HQi2Y6k9Z+0 zaA$$wenBNlPvabHg|-VDhZvo69LCHDK)wr)qgvXB(8i$CVg<`AxaK4Q3XB{bchcdS z5}XZgWkL(GK(SQ<+1T%&F6psBEZH5&IM9|Ukrn76fTuE_3gy&gXKV;rfvRXm11DU!E23DQN`~^(^umklvPE!3%DbMD_kLJG` zUH_i3|B>==_`DW4KSr@frZrK9?Auw~kPyE*khFL_CSKjS_n4Y$ze8G$sj>P9e?esA zk#YD>3wKRax7jPFMcJ?IuEq~?B^OC|lg;|Ex5fuFpZRe$-|V`3h;N&#c0qHrK6V@j zT&~DS2ze3cI-c^S-)%cG#0r}Q3-~ryF<0&0(s(&5^31o$d$}TetX=h=S~acjD4kZJ zY9?n62j&>@2}#O=6|f$Lftba8Lvg*A`xISJpN#NFc9!ZYP=b#>x3aXOwhNac@xDaica)NAe((>FqMIX~de@ z>Zg|l+l?_uhxs<-DXyxKo1h(YL@Ysn_Xfe&Q&+RZH^YV+j&O@PO(&I1MTjBjrigi% zW|d>j3DFGGQRXn!hA~8wseN2Gg93Ra|L+R7$E!2L$*lHY*|RcYb%HkF&K~#LD;=R` zgf?a&%Yx%4+MFOlC?hj&4kA9`+AuhUn-hI|W&hgz_(jLMXpruw+1M`RjW8;?8MN~$ zDZ;GA2kT45E$GH`KpE#17tM3R{ZDJw$;+BV!snpoa-#Es?PPCs+~^{X_OApE$<|P)<>6t6(Q#q&6Il~6`jaeJLDKd^qGdiCL^F&x z8yn<~Tpcl8=ymllha|^#LkK3G6XKh-(`k*J=A~@mNs9In95*h+*@aCnESsoGcE~oP zS1)W0SdKcX6m1HrK>mzPDF^FFCz^p)0wFtip4D(bj`kvn`Y~DA>)Zq8&#U#*sR|-j zkeXG>#mEgUDm8oo3+7Gp4=fT$MJEo>c|R>ic-WA6ibVb^{>N7Rk9ZUVkz$px5OE~O z2CAyO=VaM0lkLZE5GwJAN1d172zsBqO2buWEM}0LKIWpRw&Or8%)Ue6{4mECZd&Akk0|ettRqLpadej#)%3Q6Gs)+=|6ui&(}@LgzS`{9J*cBQ~1jK9sL_ z478ig0WbO{m)=I^{3yB2MBo^l^E8BuWAS%9XB96>%?v%u!H~X(Gidq?eh>+eE*!$w zI(J^`jnU6RPu`8$>^EQjN83+P=t~`~@LQ^2m64hEJ7s|GND6@1$J{YOxy{{ow3lWG zJ}gd))NrhUad(8H;kk(-$`p^Px9nSQM{fsBvxVd~JypqL7s-6(>2l~O-TuqO!TNBX zq~6#Mp?%<%#vDiwvKQxNlcwJZqW>qp5WWGqaRAJ0XWsq~TR#GlOI4 zhR5*_>W$0qaV>dnTZA=gYp}IBOeqiOCn~X)K=8;g+Bbl3AR^yQ*~@|PW)A2rZp@(w zmUl2bBQnx@0NU3wK!#B$q#w8_1_4s+AEiM9lWxAcf=8&&+y2GlZJFxTJVeAbco>1l zIGU@D>4={b?}4(z*mh$>pxOWpJ}$-tjXfCqOT|p*I8IysVw+B_I-b4G#7><#=HK%S|l(B$FIoc3)Gi- z6onhUAW#u->XGTq-5QgP@7RHc{&j? zwP@OCQ3B{}OhC2X@n!oW7bCQ8d|1pbs7rKAfD=z`?yh()%-N=9ot;rUPaT`pF8ovQ z)cgC>6@9OYzB*{#$drh#(|LAP%;1ZEuHE3bzp-<1)OS+~jR*s`Vw-&(5TS~;A{5$? z6NvK)5wU>uog0nJVS{l=|w zS`Ko}$t+>(kQV+rANCDPYXV4@{@5+(6Fn*&kJ`gkRdmz+=VH}okBMlaSsCy}u_MJp%{R+?pODNBkr@_p8veA~BcC6nfRu83I9m)=h0>Ihq7*Fpfg?6ExalXLmp$XKb-z3%IAhUCa*@_X%9) zXA0&n|Hc9}Is@NoT5XLlJsmCD=+&GyQKcdkW4>AS(_hA$8xAg&8AI4i7%RxoyPj@A zEeaKN6ph^y7!{|bisNrX&4g9SCJdsk4(@Jh zmcf);$c9ZGCw-fHfibXc@WnU_NZN9t?Xs}4N;AYXQ!_l;_Fk)hV<3TI{;8TjpQLC% z0I*JS#TzwaRB-MUcW#KVncIH=`9yxV${OKZHH7@JxB7_lw@Uo)!Etv*D_#E)#K)6)lKvpy*9b;f{cT|5 zgWa!?Y&Fri5T?;Gd(m5OLS2y31CXkV80F%$e$u$^ChUnhriXe|%BP?(7AHKMH9r33 z7(T1ku1te}-FI($u*yFS(OKl$8#Y^cyR@u^YSsKY>bKPL`frvL>G6H}& zE$bp6L1el+doX2Ewk!Nxk`_jw#ynAia?Asp`GDV~{)1xfaYfP9v=nGJf=M7VW+A8Kmu+o<(5#NgURLv~0 zm@nD8;nm4JoP-i@GXH+-u<8qYNTwJ0xXm(=wIj8KnOtP&aiqgoVDPDLUuXYAP1v=U z-hK>O$WW%+nMf`|o(OWnRfb0|(-;!?6oe0S>7=`YWO>Sl)0kL&3ahPu;KGCAD^7EAl}$(P@5@d)L$GR}WnAL!=}GMiN$^ZM{{E$1w)kkE_YK7d}sMOL3E4z!Y8bL55j1o)#?*3!`;S=A&3>_zgo z2luL`wiWww>6<|~r1D-r2X$FiB8(T(i(FGgDq-;cdwSaBtM5oG_1m5*J>{Fkg>f^Q zD?d}Y7oWsFW`257>g9;~M0OD9wh9P!4%TKdD)==#WY9%RCmDziQP3Hs@~UINfoHBA z0T*@7ofg#--fpqhb4Hod18N7E8=noh)faL&71jn8870s6UaQ8z>7rbrh%>|(&72LP z?XPudC%7;sTXrp#!QgTX(n=s_ zs{)_qutgqZiEh4`p~!9{CPi*wocb7!(bql*(3G zoyq!$ppgCYbvi`k9ebuT(d!r=l2b8ylYQiD;)M@JS@K0(FNyEw(%ng#hF$B&ChToT zDQxuUDN*~aCvy7iu|`h5;yOxOA&{Eu>og*SRHC&_5j6b~=#Z0O%#XY70~v+KohCK! zfGB(1Zv&?_b+X-aECQN;->R4>SZLF)+Hai7@yPk$Zcx|M{;Pp}m$Ul}-y9?>~t1P<}IOGN99fTi500B#<4 z+^I1;mt);>b)H_$L$Mm4F%f;rnY<0(+7)B(7+;~;>E^-U`vh)-Wrttp3$iU3< z(bYiomD;^%SI1Ih7H+Th`JyaU`iB)TNO_XU*e@d=AJrK+YqaqYLa6#v1EmZyG%dwSS-F__)_D+HdB3r*qf{U-rsg`TyZ-;jm))?-Hy zm1F}E$B^3+#f6W+1KV$5HwH6CN*59$*6XkaSt-zHN{!+o-med2X=5Kcr8|(iv&4lNu0ty{ktUCBa0R!(DW; zsg_vi>?{OJ8uMroqvT3P-+Z4X>tmttr?LuWag>=#OPa}#^x55x?6WX@e6B$TC9X+D zkcg>lTYhdbr;y~Nb9Gs2v%K3z5XgJD0fe7TDKcJ%m@Z*AwYq`S$_f;bh21KCmKZ#-d)SlbONt#0MZByGV&3^4R2Y zP*+*I7}-d&mY3MHl;X5il2$6tFUZWBor^qZ#$Jn{ii+L|>p~WoT7-q&L?M`>3ky%e z*3!y)$H`*?AA)o3>xGI+VSfx;;Y-q$s_mZOfTzWyy5P;q%a>hKrM3TzkF|1ymlD{p zWyQLQNbme|g`#mC7jY4t0#G)&1pg&0_8`;S)9|uFZ+uGBA6iT~{=UB+ zbD6rkRy8PM0?d7TeH9+XZ}Pn=XoIi%d=<8eeMi~$pJ48-^!4$B_dt7~@ax3JS>{!y z?r|%&lJ6t`{2GQN4n<=tu*MjB=*8s`3U-^It7(yq`te1`d_J+&Pu`zf>J*Lp{MApr z=I~1lnS2A@!pSigD-!RS@PDlyM}OdxuLRU6+_KEyHn$Wt&?u>^dDIPh@9x7~t9o4X z-Q-2W@dQd^D4*c|{3Kf(}xB>^ceP*?!%@faXg z2U6{x2+f@YgBCxM1UggIjj>2ls zG1a7nwKv(NXM>|=P=q%nwIU#M|L{47t!_IIVM#C{Tk7z*= zlBJ@6xfo1CN_$BigG(!kHS*PEwL?kU5*-ptVIIj)Y)6JKK~YTypUJ$z>Os?F`Z989 zri}{M@$My}P#dW=L;E7Z?Arm3*z&Tj(zUncscKthKd*U=6+4*NK#H`*mU-A<^;o57K%=DuctVPYwJ)Z`FBx#WLYhFw61@ z5T4@<2e}*NKFu0HZ4ROF&cFV)Zgp1PhI?jV+eUu2nlz^e)GGSKp)wy+i)b@<*SPLC z=v%Xu65zdx(~RD>)IPXNf6ZG?%7@)<+n!#hIf%VtwFVZ6S$?3W5u^S~fSxLCUO6PV zg{W_}oP2~oFRH(W-4YJT{bf9Ww|fL}asaK1@yacvx60tR{c$|96Cq!-&H)mTeen!0 zwDPAk3|arxZ#zzxtWM~=j}g1TV1gDQpySKG#;vWRH}Tkf+ZYIE5HI;A!3-o7sc z{!XmRc<9>pUfnToXJ^}z{U?s@G&cRv)4V<>SBCzQXhj&}Arblv2)8j{wsjxW<(zpF z(g%`pY@}TkGu)b+B0N^>wor}Rv3yBS9@i@Mac?!ouZj+Ak_q{tR`ys-?N3OfPUoRu zcDsUruBXiijxUV0dVpA<@f`RrZhQjji&7bMo>ts52=)r-EcAl-!-r@4PyCS5Y=Kq$ zy0-S{TUaAYX>iZhnhto)(=xS&eQl80{#qLI4ngMmc|EGf5T9}-?E6*lJ#iA?@}f-0-9U< zn2C|KJl0_YMhS~)u^WI%*`B5L&)wofxJ|BwuHy4GP2zjpGa!WB5SJv!ES+=dhO%f- zY=F_MTv9<Syo)=bJ&pP3t5ulG>=eM zI&`aLbiv4wuYv2u?VG_K)aE@Go>(M~jmtI(|3t`V&|=%$r}~fcm|Hv4vsUA?RRUzMYI;Z$pyDp$O5*L!4}%P1%|{ zxILauTGyGoMs?1LR~x*>Tw4XMC6CIeoZ!T*BBfp?OxcoMUC_F!*oR?u>92`F6=nQv zM#80N`0L&7D(>x`Z5Zv|8hUEjnRa$A`3pRcVS<3 z-)a-?%A&2O6DOEKBE{>@xjIHaeu>=G6yDsFlvKyqEu~6oRkqmMj!ndSg(j{bNT=B) zSCSBo);Dx3Bt}s&k>4m-JdbJqgmN11bbcekA{;*hr*^S%LlN|p9uI;%*9fv+sP6B6 zdY%_0&K)7a23hJ&Vuz_wBM0fSK{|9hPla|e&SXEGox#%V5M3`+{z%Nv-Hv_6zIr=K+RLvX7NfSGNjP)AfR2H(fQvNS2@ak=|GbP7oqLd7)4 z>^B*vT4 zq6tT9Muf6es)kuGiz*qv;eLC9xpidhS?ram;wlO~_{RhBCu9_MJbP~r@n6a;-$ z0$GAflmV9<$062u?pRcPPVFgzo1N@Xj^&+ism|#hf3xCH->wwq>2gi^Btv`YlT#^FWE<9Cak2XOehQ%<|n9Dx50?6j*mg$Y0Vv_z; z%-M(f(L6}=7XmR32L=q(P^<>?c`0Q;oLhrw|7*)Cj+aN>2}S;Z=V*nqSe4b<9Go;@ zT{-Eaya2JprGdUNI+!syvUe%^6HsObCX0fhbcIULVY84XCXstY?XmoKzeEcgcK(rp zeMi=n?|EvM2oae}O%uO`F*O&0Fjf);OSPTytsRd3>`z=Y$JMD?I|ZrQZXV3DOsw|| zeOzA3#J=WUvpu}Xuo(nCKk7-s&Pc_MQ7E0ClucnMav8Vyg@&ojYQ6(p!^n+5DAHj3 z6k$loa?i4sUK4%%6Z>kKT0tDT>o{r|E}hHzQ2#yva~vwQB=le_Sdx8oO z%2hXPm1>^G%Z9k9wx)b@BQ933hy@)w`qWoYh}6t@z*&LC6CtXM=tVU?U=-ae^b~L` zXhr>;c=dRSm@hSK7SmU{r=xjc?j}W(x#A!Ush%RSn03I)#({6KoWP;=(-Q&vLFcBK zEXo6YEFT}Z1ZNoZ_-{PUf<@=p^8s`&$ufSW!3ID~)b_u2>0o10xM$(6n6LTINPPyWelteol`9VFizUxgh=t89mLx)f-moX%Ur_6C z*H3rT1#w&sJ^y_hEFvF6Zx3zbAR6-ubetZ`*+il8VvO!%*us2I7~Xi>G9$%e{nbZ) z1F*kNx&HGfJW#@-TrHH(0fK>0Fbiz3t#hRVu9{3B@^8^#kEV_$bIF!l_>#=l@-hqC zFvV$JI0pVn2!Gc%Ol+}K-MEPVc#B?~sRt&k@4^A8ap6j(c4esEW+lekiPx&*WmV77WV)GD=3?bTAs zUl|x(^MVJiBng?bHi#NsC9^3DU4<%uT65sNz?L33pOj?BbSJqLfTt8HKU@QoFQSfq zQUmkwWDV(+4FkjM_!tmS05H-92c^N{FlQschGWgB&^B8Cb5L}El)u;Zk8~0X7!DR? zP#_o31eQ9nOX zx4nqfByI1jQfVqgFj6(N(PqEB93OY)?Y;B7-6mdao1|gu(s}`8vOyT__vm}z$nCBw zWyU}yfH5mMYN*GLNgqa55C`6do8_ zN1zO*yfU*RUVILI(&gTaxR!fLn6!n9aziqK@RHx09Y8z5pN!_jFSC&E$p6(_G#v7n z4oZbdN8%YY1Q&-0*;_h^_+1r9`GA0BeI*o+Ir8S9(=$E5H(UNhYd>I>sI-cFOk0l;7}~N3;KOb zIYN+)G&rTqMTv~?^>vvV3k$lU)Wm9H%y-xS?*EB#L>Fa`hL%FaYfY8TTb)cb$J*E% z=j*YavEFQ+R>sayj~oV4wVKB^TP|eNanQtBGqt7dtXAVLGrSBZ)1umKmN$)7LEip^xR$wz1UaEVU(*ivCJ##%}QkIgou3(cHqSZCxK)O6%yftZiMM zUXokXQI$?O=%EcAXLDoVIkz#w;Tj8us)|5OC53y@(#im2o&Rb~;mcIBa7sBMM(n-- z?xX(mAojaD~tWCm!j_g0l6NaA;w0N4J&<`Er^$XF&sN8I?5XK>c;Tc)k*FPhpY|8Wz@uHW-TTwiSR=N{3 zg-a=ZE(ZjSS>-gJTS@*dS$C+XI$|L2$U9$Ht-$efs$PC(>Hm*tOhOS4I%cv^?f4oo z#7@py9S=X&^B#Xp0QNNr;(Aer`=Gq?ZZ|dG29w!Wd*6QyeSM#DzwW*1+CNz>*wT6J zy{kKWlVspxJevz!K7hSjhQPlN#`(V}8#VYhJ>DlPAL>|MQH)vB+NOrS`L`_~`^o!|h)2y?x4e@LNT zR1GoGxocR_`hTEk8G@{;bC?%i_rVa%-}5u5F;4f3InM9{)kL8>NTp>WaNj~-8kL}E}S{ro)lG8 znT9`25#8#=%7^HuO(l`z2ob$ty`W*^P6JTu4gsrKJ8R?Oq*^~WC{!~UOM=7YvKdgt zBNJ1>BZlDhpYXg{I3=TM6`a*^bE*7T^X%N4JjktG&k`?aeN`curTTcDlko86^DVWi zo~1JvKnS++zP+_ze*uOXu z6Y_}UjfzYVPlyNWJGP%`@{*wu3m@k9E=%~h?FZULfmy{fl8Xi+(nw-eu1xb+FSfwl zmW#a=Kj6Q3C%5X_a$ai}?8)E4vi>Q}!_czrCr5pItdi!L!KDAhDI%bpsW zWOl`EjRl(!8L=c?oXz=hyK1X_jjaF75MDzAC!0FrZS{IeZNUQDC z18dE%Jz_cRCmfZ-ZSr$Q*vz0{<=1_;nZVQ>{8bisr2oasn3NrSL8hJz%5PrHMp{xq zY>+~2eaoY2>+sPCM%VJR4nP3PIcUOiMDo$qvYGSuh}iMkRg;=`T_!`rJe_i8nNx*m zJi2z{4T0^`@c8(YLom(I?VtwWkJnW&yq__rXFJ_izca}5q>wKGkKwVJde|Y@(b9ii z;!ftA`K29f-41*ql9e&uXhj7&;5GLH>yFuc|MGdN-&&x4^_gUg#!(_VL+~fphyaJ~ zM%moaqWfOXo4z}Zn~>Xn2T+CdFTG+AG82VzcF(tzrc)}3nJS~;S{>z&?h=vERs(En z#+yf)rXk4fzA~K&)}s{7<2ffa;@1G#hDRyxx3Sg!aCGWw1w)KP7;?e1>A}C%!^-E>0At-KVB`mJyDX3w;35Hv3^|sJptkJTs4~CZj7VrJT zpYJbrAF>g^FM3W=@#|8RP&tE`zg<8NA0O_MLK0hg7B|mnG{vxjQH-B^G;U$DAX-4SQNn>=(M+vgt0cyj$olRI{vuQ^)&5 zVbO7j>K`g6UXg@LRG`4v0!TaD_!n1D6tJI;{tO*{Lns1=FX3iphii<@szTu1O)5+Z zgsIicc{KQ8&68T_B#p`!p~CCke(2l?-61@LpKY}>EruVmi0SQEFbimgu3f~$ z9x()1VDE7Zr9(qKkxY&zBZ`UA2|V2t6JgW8Qi24S0Si6P7pSl?qIs;sCIzgCuIEs9 zsS#`p0y-=p(_yk1y3S%?(M61paSPYj<)fsz^qWmx{JXP~1oib^Cu97b$7)}S_VY4e zF_V+cobbh0^hJWRo#3~BKNu}h+)0R>4n`_8gN$eI%>eYbiBHh4ph!d$IQDiJC|m_N zLyp#8#2F_hN50WQ=^{Ms5|B9J5vbsFF?_}CV&9J;xl5&g#};6;Y^=u7s`}=cxJZ(+ zKxrkRC~re1odM3jg~T$|+%PLB6$JElAQ&TzVYfiZ?~n>(j3_vE3riqKe*`nG>Z-)O zRA}s9c4`}y&K-RO99JnL67s`VE%dur=!Ds|kwk@f&Lh;&V-o<3IP$EAR zIAmc+!;=3p`Bw3DkkPanRvm_`2j%sy=TrILg8mh=6FX!38C!`Y+M<0M2ShCy$;~c> z?RoWP$UUc6#(V|HYgjxMDW}4Kh1RJMO86Fo2$sTr9SY1XtBjGR2A!A(a8$y(ycQ?H z$rJ%|6WG+J9fEMs?Y!=`aG<1lSU^(I;{T%27$!Cb9s}zn96aKJp}|0EhVdBJ{i)py zT=#CNMG1GS5RpPvymD}f(FHOK4`2#tfeS2jzK*zluXSb<8li`2(uciz`gG79=tnju zpb;fj1rR!$h11;-B;57xwQ@LtjQ{^tvS9S9zp9@z>{*x#{tQ*;Zn#)Kc$rNh%?2os z?q{H73Gk(_ z(6YWBpzdt05$5-EL-|(`S;Q!*2q8RTE}%>r!4N+jc=#-TmM)75Ly1E%;&S0_l7do$ zQUCWvH%G6q3!rau$NX&wXA8x34#s<3qkr&rWBs-)QZs3Afba9zPHsU&D|m7tsxU zo6>6+e;+4T86ud(5sJ~;gT%s6LPJOmBF|K0Jvi@3B$ftFhpJY@Mu*%QVhr>W*)?Z~|1*ZBo%K00NZNn1#r?kz!{*)s5(3-u&%|O#89LH8_NjbmTm%cgSSe)XeL;P6>77S8iQF;W2CtWQL zp(S9sFx#i-+zWAy{7&QRQ)}A^4Y}WR*ZQzs$WKNw){hvK0DC2CAtGMq2OreeSnsG< zMOu+)*n(BX?treAs^AR#?~{RHYel{Z1;=3+{ewEw-%s=mFL4U%(|;7tUS$@93CG?w zI7r*)6EOOwb;lfO@mr*JWWpyz%1x&=$Yb})?}%Z8b%VYnt(tJ_0?T54E0*KDCkTg= zn}gapL34>9Sn()Ugblh9s!Z#sCgrQ-HyG#w(;fojUOguM+`*U$gwqlUR%xZ{6SN6% zRN=>TQrrpTTwg-4h!+nK)aKjvb1l>KI|k*O|9&AhcQX8*$3~12|Hi`w$_aaaW`rVS z4!jB7-lthU=}LvY zbE){`g71X<@LV4rlMrfb+GgQ_!2Ka%({Q@_fhU-J$&@_Xjw@*uYRQ`#wpcs0!H(*0 zgg$NNrZ7TFH;6BXFyVkw;yO{-s8t0;Z;7M5p2XJ3qk2WrGD-fR*ge_p`~GD6c~ain z2I=?h>&?0@$6(J1U~<7dnFX9C-&?qr>l(R`Oe2|bunP5t$Er9!Ou@$EMP*sf)h)W8 z{}j_921@Z3%_~LxuQWlM->=)oCnI(L2dh9-zde6^b!DKGjZMvk|9|%0y}4~1OCO$p z?N5PaXJ_rSWs8!0OHI$O96R09wQ=H!lb-60vMQK_ByK2DO;WzJU(aWMi?|U0L5jLd zHdNIei-bsA4sf1>bMTxbK@lFbplDcS&8c>FQ;kjoz&fdy_kFfE8hJr(Wt$`zq)0|T zon5X%IvdU7J)8*xwCZWWlS$-wY2qpj`fsTg0YAJgAM>K|3r2$*6fI)>WG-1?#!tLO zL-0P6QBbi=esps&oU0yPGqf`g?PnFpR~}a4%P1OF0-Pru(5v6rKA{=fdCl7Mb(^)= z1~%h6ukp=U!mA$NVq$twZKRo7kC>*FW#NaLu6>g`(na>6OUF{B6T^7nPsaSyZ+)k6aFmmCgy9qp;eOzNfFK#Sd|U25oC^g6;d)6FE(H z!Og>tcxa2bI7ku~{zul#c!(VANypp6gp;&d8=ms7Qm%lSEryJ(ZD$L8&3*Li*N-I* z4uvoPur0)b==MdR>9jwop+fmRFg#Nsudl{ zYdEu)F&bkAL-YxQb?)p5r+=k6KKFV2cG7ul!g#91n6!hOY*q^9!EX9ggiSclV!N6K z+-$@wJ)d-z;^<7U#t6>9WdoOs`tgT%A12 zH0^u?#0(I77G0f1KCjkf()y=31iP3+Vwn$elw07{qDAb$EskXcU(&c2=SE4dsJc3p_KvxYDrWNZ3h*;Msk=AxQ- z)1&b2SsaL5;913L<8`X7R;9Cww!|%{Kdb0_z2GeVj3roiYH%5c8S!d(U9hoQ7`tpv zI+~M?n*6fU9rRn$F$Y?o`EA(GFRku!dO#B!4rbd#NqfMAMdr7)J8s0Wty3k9@^V^FsxuQ{o9Rr?IXzi)e)T0br+LPn<`H5X z)RCR{xtkJM-=rfx2FX@LvW?^Ek!*9VST_t5HBb~3YiM}eqlg4!QM?^5!H{%i|Bs*A zmgoA2gAu$ZGr*^sXQzXnz_*pYaz+}Z*%b8E7l5VKDCg=&ZCTjfQgraNFU#?@E>aC| z{7GA!&IWH7is@{^!4O7X90nllVs9JIDGdV`VACR9ICRJQrag0O*K_*460278ut>jG zVNbT}t#L|DICFa<>4{|?xs|tF=X&He9bQcqQUInuf$>AvLC}Gk2~`01w5uIOU!PUG(duK^^+mUdYi2vUHRofmYWtdVIqQV8W0)1*@S#)(8`aoS z=CvSgYldy&G<1szY<xEa`<)gv}M%fYL!o7?6##cT2ogp|J`W}f=!*GRc4BgVAH0mHNn1091`8xvy<57fM z5ZHF#JmuJ2SwXZwR z5BBGJq91>q6zrNkDzisr29(}i>pd#7uCL)X)qZ15*Y!rLFG@_#b~8%?yEU1FEMi-r zXq@%-)ubWP0-5|+yHzwzhoRd^4Yl2NK=}9h1&M>7u=2M9LZ%;~p${FjBU0|Czr#nc zo_6=z0ccz7L2YXf;0v#8{^u}s`QJPHKkl|I%978S_dz>2p!pW&lICTphWAYT4s#0o z7P|6ae%W$X(~Xr?<6iWH_scx-HUD7P zmOj-u9hYNq3uD#MuA7)+&P1!u|KL8jxyYtK2T@&~{LEyKGoF1NB%L)B9$FIOf!OFu{39M|| z2388!4Xj)wth7N5Zzo_>NwL8t8l2ri&ri+dT)IoT{h*Y`bT7HG>i`>OJQ$_uLuy{@ zxAl!$q@Qs|l<=mhQH=N(8z1&^E=9Odz$hJmqFQOajzJwYw2iBL4C-{8mJ-5OjwMa; zOpZZaq5%)VR}BOJ%vmUQ*PMka8KZ)+d|p0C6KC?w26X-6Ow3uRbqg|C?OCW)Ij`a@ z)DMUClcQEca|0bhr;gTlcqcsSq>T;TKR#U90jQNGg5Hev-JFDa9w(t@bkmp$#cs z%a?^svDH&kQ9dzLYB;LK;vQ&Ym4%NAvExJ3+;h^a?C{X87;b_|cH6-LIT1Be1oX-e zzAWYZQ03hbIjCpzdv}s!MJWQ6jQFt2GDu+pk56Zpi!26M=IMh?8fB4JB#XJ zDOSn09kcp!IpzUPGH>tKn;WueI`?)LbLpjNF)X|=J?B*oQlSG}p z4;|m`tYPh(454_pNW%&!W)Ef#;+f*u5&ld5&hx@JqOR_v}fs0 zW{RFZog{c#T8N)bDhP9Ui*>3wyfyXVE#rG#>rEu%dwo&EZG5j%x#u;w#`pR>$JO{= zpW&dctMAo#=holvI?E|X<6EacSaT2Tf@m$kPCLjlwP?{l$*?0L#h?D&w)S7gvAh48 zd#$uT6^?W40r3gD?Eui!KVyE!QUXO@ub=Fw&lpql+!2%xeE{8gp0!a~`XJs+{ZFKN zt6tpn|Lve1gi-qcX$R#$p5vfQ)2R6l%6j-fn523sXDl$Puiw-KnDP{2yf7gq@cW}T?#|Z~RA0^1831#j2;M?vl2qQ55 zC$w;>YF|-=Q3bWDY2RJsC1}DvQGY{AFQ-e$%PT?=y`~r+b#>vz-+>YoWTLF-r!ck& zGx@5+XF{xA3}(p3Ay;DUI4wgA4TBhj>jp6_8ZiK^S?ZM{1~6NVn@x5jc_1?ALJl~r1op(9w;VBa($XCgwAh&t zsV)f`41Jh*K~HFr<$G}=jq9UcwJO~oBOf`5R@{XHV~ys1{rITx%ox30G~&!fD$g6f z$8>)6MV9RV4)9~s4swPg`DTdob>xH3*2ngGi`ah@y#j6^y(cb9oFIE$5qp|ves1^)7&%H zZbX~=IsWym&G_b&BPz2EE5jMV!GcFq)2_0XRT|CQGuLi3>-$G1M{1_Rtkv}P^mDXy zP=|{jRc-sqTUcpiHP2ncv8}g?7hiIB%8||Uu_!~U#5^s0l=CeuZEM9bR-d|dL%eSv z7Y=ZGZ`$F#z8F8WYMHr@YQAwL&8sw`>J!&+Jd4-j^xl-?`K}isz*>18YT`=RH2 z$D7dnDfGaqQbpF<4%>K?CyjL|1&!Ku>1$K|YKsWhI+VWI^4DfvzANqCHKcDgk3IdC zHQXDZj&72dzg2$Ck^CC@tzti?Uqx7F9lLT{vKr>b;m;87%v(26co*{{X+oG48b-*0 z335RVocp6VK@p)ak#o+k`jLJmT#d^`L*s4`hK0ji1@{~0eJe$bNOTp zE`Orvpcla|3}DTLasMWSksBYlXy}Iz0~91Rp+EQkFAjsM@2J=7^`lk`|68xu<$vvZ ztI<0Ai`_bIH4cxC50CA?)Emu4qwyC|Uu@IKXA~ze`b&N0cd3orkWae2SuhB~1hUnD zta$1kuHb>b8F_y4YoIt7C6}>PtASsLFo9Z4ob4#UvGO^;Wna&C<%9wDbz{eP?*{JF z_dvt8_dw(Da1Yp`3yG_=v#dGbui(r_QKHSi#HF0O+qc;wgqI*SC-Y?(cu5$=;J-&V zC<;)5VvtRb73hJ1 zXb=W5O8-tDho96P`RSE?+&&ArUZQlb)eBI9Jy|D*Cnw}e1c&{M015*N;xKQJKNA!| z+GzXmQ%IIne22GP7`$icv7~eXf4+k?pt8`QD4dQILaD z^fvMOqVI&esb=6Jgkw2yh6?IFJJ<%SVV;&6!`!TCkz5&zer_16SGC?ykQt8T~9{r z?BdS?U*e%amuL+ml>ZIBR`?U#{7X12>2{9bM+#dUx3@kVoGCX#{)3Xfi-$1!?vv;b zL3+EM@5^eZY3~8sKG_5I$!WSW*eBBB(H-dwFdU2{K8HM}k*-yzP3Zx2l8Wx%Y#QLD z=7SA7vVze$vn5_0l<~arPLI%OjgU<8M?x|aFw2P zfwL8&O{8P7vh6lD)I>Z_BHOzOO1183y%y=L)(;&JK>9-os3oTq zhfo*EmeC-3LC;5NZ)d;5>y9{MPsACE@0QQl5K~M3RB^KM-H>tf6&L)#_OIw3jc7C) z{6Pw7C}a#MsRBw)W&|Y-14;}iF`%R}C?OHE88mT*Bg)+$BPR@87DBoxB=YpNfzPQ@ z_%VP^-wP0~h7(38eysA07l1F$ufF8Kg#i%H7C;!RPys90M>B$jrU45EEEup*1uX3B z>~MmkY+Iad5=3a|dk%~rvs;9BZ7fp2+B=NEElRcQPJMS5{0OpF37hQMON5K_c?;j# zXYFxO>_y1!5F*Y_?DUa4@{#+)JHFa3|OTgy~_(aFTVC4z1vgVc!ESHfwSI5f5 zM9xbB%kp`)IYQ+^B)mAR+*Gh~OTo&jv+uABD%bN+xn-cTfyxFdSKL=yZ(rX!Yj>Sy zkHr}@ktRPhnj9ES{=77KYer3eXf(Oe29o3{R{+nCSvp3&S!b4&S7rc&La#AIFVwC8Tl&Ddk8?HW0sNnoU1vkuc zKjQ&yJCxXB_Zi_2W5^o}G6M!_%o@>lYA}ewAO?fX3xk;8`lZ7UP2%_L^=h{FrEHJ9 zJfKpP4osU2tSA*&K_ak;xh9&^0tk{StS)I~xjBV;+O`!%-53{oeo~^S6&)>fktTf@ z$=Y%jf7VmlX;X^{%(5B=ymy0P@}T)qdhdTw6l!jfWMs;-w3*1PC~f7nmS|6Z#y0j` z^e>uBOjNE}`XSyN<<zk=MTF%KDg#wTsnGO^(8|)FO}NkQRrQgp(dXU(_Gp0k z9^Z)neiC=~U#EA&j<8>!J)K3BUwyooGJ?Ij+nwZPrLQRX-Q+5k@?Pe7k7wbAP!%&C zfH&aru^`F-JUxN`FTKs0mtK(HNEj>W{K>|4A{BpBA8S4QlGLXMUs>>hOF@x28MaYc9){Sha45OL~UVaST1)>}Q)YD_o~ zm19u#BcSP0(x)@^XD;2`gx!f(SFt|}`tV%aht0}h^{dRg{WCEJC0jMUe%<%FIU|;% zTTZ_M{tB4r3lLY6JkT1%pHLk7Bg*8OQ~27%f=sVBRPD@@3Ob{~$cKq{i*i|N3la)y z(g|8OBj-EODhjSB$TG-w8&njcg)RBKgIN{xIR_v0;wNoo5dmUp738=8a!;QckfJlW z)*PSZF6+_=7EREyT!p-7Zp1E0&Pjo1!IowmOzel&c(%a%g*V^WpnqO}gn@8Q8 z9BcE~r@5Ix3>Ei#$7>!yPxB#EUdVCbmTiPh-t)j?8M0JU_Fe;?Xx_c}!t>Tnfa}QyU+-y!*!b zpq~OvpHFqZ5==kV_E?j8z9id&8V>e&GvRbnE1}sauo4=}wi0q)jyahLYh(-96WIb% z?7*tbfYpL_yIu#c$*!=ElFn`Dj|K!8#GO%$T#DJNQPQ9&_TmHu33-g;LvK*gu})6V zg9D(gi35kw@sfw-)@B*LeQhl{AS?vb&+#whEzL64S;aL;@n45QFNag1U>D-3=;_-Jsf0C-0rs@};V8u=!4d8EC%J(C3Ie8$n$AxruTM6uC<8Gx(hZDxE>ntI0GrPo6 z&hz&tXfRAAKj>A-vJHKo9f-1aq{9{&V3Y)!DpwCG9@OyI~=mC=3>jOKSn4@ zK|Xm|hs^h#>;Q>;vW|<5EnV#)`Jhlb1mC^km&m_<2prv#NtQ;jBNEudgB<^p)yR_X z=Bl1v(-)>|Cuy5-V-%y$qzK`%~%I9{H90b)-u(qrwv zep%9!nov^<&Uh2+1xe9(c$ACZU~=T5E*$wuO+k06Q00ng#f$A?DuOUhoN|Y7r;{f^ zxF?XRVH6Hf(nq6sA+p0pSE!tX)l+lpiC|@KkR58y4p4`8wML6shU}1ZcxOq!8+y)n zyiZ4eO{?Ewdjz8SW*rKVEaI+$)fB(jgY54^;cr{~(yWVj1pBoi{B66W$=_HTyaDP6 z!!q_Q;)9RG2a9jXIw}1k?1vt^VtrX%r0)AO#Cwm{O%&e6D5BPnkPY9IsM z2LgUN$j$%@W)Wca5PseWZ>C!rE_ze{c0hJ5G}34OVU>A z{is&giyi3yj$m}1z$m#41MGsLz=Q0ue&##B?6T@-vwZIseNahwtRW1LU%U@Ge$07W zwUC3CqQfnNbGcLGUs+ub`R;ibbiJPb#T328#iHoYan|3)Mc6j;5c()dP|fjS9D7b) z5;axK-ZP353~WkX1n<50Z5X*Iy4F%5X@78zQmBJmJ!OFYCuy6RBQ?k5qxr#mb{cM6 z!7Cp+Xh0CT1xHC(_JbQ3o%vobpylf&McoMAU9){!OoCHVxa;};M-rGd=CQp9x*ie& z@6>aEO8Iozpq`oHVZ@^xHV`G$A_ z`ZblXVdz8ZAHo-5RMt~ziGQiVkD7AW0Z$284ok|V%Nx4z;w18Jn44UmXR8`3(D4ukRr?i(1c>f|!jiox91Zi1WKGF|D8^H@H*-m{**2zf?3H1+Tk^(< zR!T~5mUm%j>sf8-hBw%~SvtAn1S#x4x=(q308yn_y$GRT%%A~rw?S}d=|3f7H)-V} zA#|of)lTZ=6)U{^ZKA;+1)gx$pc2*#rC^*=yr7F2<2U8n5M&6zo6()RN^;blH!(vqUrkho>fh9 zM;A05fPxqVsQ{a{gvP7z_ACBP4h!O!b@mdV zP$Yp^&Pvo}E}*B=DN)3Pqp{Go13(VV8Y4E}RIy=OW7kO-!5#wtN;}O`OHm0(yaA%o zO}s!sn1(T;N~)N>#~4xS4~!8-3mYTK7*WOTFh*2aJ&h4HW#xZpf6RFS>Rl!C&G&Yj4kyh)8!u0vv zjFD3qycwIP?@c@!bZ}C?cGM1LGpSyP2kZRmi?OMIv8jwrWo#;AQyH6zJ;B&i#-`F& z&G@ErH_Cid_R)C5t6>Z;V|cC1@X{s*fw$1};f+tJw;v0Weq@H#{*R>wRaIN6DAS)% zWYy-@`OE=>XRLdT*=5WwV|E#{tGFS?>@sH8_*WXUD|?VJyTs4)HoFS_T~+L^rm?$> z-L)=9ugX#$@e^3)02Yw!sSh+m%qP-m;B79$?>L)kTX?>}9V{^F=F)C7X{uT|%LcxZ z&#Zwji*&K{l1#_U33l^YH%Q-CWVY1PNecx^h^56IqZHFu<0eo>GcC-uN-I#FS(@J} zW0>ZJ3Mi%d{ZeXaUP@0f&2N`eP0PhN8N9B9mXun!nI;9xMPFMF)b#?7TVf)!{S0B=PkW`ybmi$%KnPoPw(v^K zFj)%5=8x#%GZ~%&d$b^3z~VX;NJ=PM>wbfg2XB0YIse~b-CmQfRTGqj&v}ZzG-kV! znLu6%_&1rCS0b^;S41LvTQpW=E_?YA8K1p)mC>|Y{2Eo@|4r2(IOF36E+uWrrYcjI zoos|VvR)O4*lCYjcL;sQgu?jc+fS=zur{ArK2z|&+=oFZ z_py>sq^fH22k;PLx0rs0-`oFP=nt8K&x}5Amc2@U#Ek7Z02W)uB}cdLVt|B z2x%Hv#5zxQGxC>Nhy0@6$QDY=uJ1SZZD}3Py-(K%USaEle7m(T-mmZ1%ioK6+tpVJ zR$T6tpS=MB>P_LD>Hz3636x|x)aDH+2Bu~GCu%k}ZkbUkJiQ|I){V_tZl$gB8&s*f*E%54A@#IQKy2xu=$x~a=J6p*kTk3_atFT zU22J9e`J{lvc%_@+i)N2TP*e_J{8W;LT@2WsVpmp62IX%UtyttP`8&g&I4HH@yprr zWiQZs@ru26%D#4Cw=zFmZXV-1*Qo@Tao;2HJCo{AvROKQle7#|fSja`aei@^*8tw{c#6-f;2c*wY6rMU z)Nb#1?oVy&75!G`_u8YLvRA{DfC@c77E_Vje{_&5w*{4a0c(-YXig9!%kUTG}Zs3QU*48B}Q z@CVAXCP71NXy|6)`iec?gNk$+T-lU>w>=zvE1z@hM2NHj?*yPYiHWEc&IVTgr?%w` zM{R2_RT1fzFhUbdmg2dRpDJ`}kXfM2fNTP!@?RwW^N_ZPSt`o2i+sNZCjqhrHeaE| zHx<@3re2|`clqw}z39(YNaL>hsdRUm7B2jM6w5Vl#ocFcJB z_2W95SQlRW9YhGafF!dw#+6*i7+1!)(znVOSH`${e#TYK|5jvO9UJS)SXcCZiFMVD zI|NurD0cdAON~PLU%b~%B3T5luobKi>rK1&Evy%&R|kHCL|unUGL@iVqd4Q8{9lpJn05@yugVNjMc+_PB-pw9Mhq2hmu?> zk(h-+Z=e{rgISLMJ?U_I+V}%aDoH6ov{>iVgWY7`PCPWrh&R( z7aI(mHb$TcwdjRgPh=h|pbV7>Lp5Zf3Q?#)5=vgIp$9FYs}UF-#Rag8z>tfeL==-J zA1vkQEMgnvgfsLt;w$vrf?7_&BA!5P-9?ePPGI1|$SvUV6ZpNa3qMf)F5!b-XeCK+#qbja73Sz~2D$22(yDVR?WtfSPx9Kfm}D%Ko0Chd(*P(LP$|l{9)~7mI3sv z+s zI$H)FzC?IPQY;j~!&#spiH)2DjLrp%t{oJuEeVQ#(c~95h}0lbz{kKKQiDhhA~lF~ z#fVhhuRPTQ(nAACUjQJrK`jpxy&!Z^-04OLEdi9$w~>=K^L^da7<64XJ-rX1iwxA% zNxXrYynun425PP$)HJT{O?P$YHZu#bW7A;A7l0j?xS6>wn~BTvYvb%A6|fx-ldW{B znV(rE4=y|9%pW}Uxn^mmARjpi(qaaJq4dGXKwLC)QQC^+&iyR3HYRx7X{`eMjBo$O zS|Z-bPCePu)rn6%X+oLRrKrNuys?qe1q;piWoEGl-6uVWXU@6La?XAAQc~pBDoR4| zq}L|%maA0N%~`1C)Y5en!{r=I#{X71<#{~mDbLaaormHPPII_ZINWLNNvAr`k_0+^ z#AbDYcn%zXZn=JKfFTpnE%)lwZJ9|bKx|JiCC z+O~ZDvuWFQ%bfpQ#>brhl9e{R^SYlT!}!g? zfg3vUKHI6-cf!E|Mzg+OgMnLfuxDu$p#$jNrVux~g!|@z>5=r;v3!q!g*7{Q>V&=z zhjD&yqf)h#EZmv)}&5jhy&vXG32tHH&9D|(z^O!=PC=_)u8kv{!EYGjNBj8AFn-4RDGv3<tfUas*YV_m$im1n>& zg1W^>7Y@)TjI4wgq!<4u43KbXDoAX6980x@_*l(RK(z7!<|QDTW-}2>g!G58n~dFL z>?UJ38N12YO~!6oq0^|qa5^!D)5|rSCWOUP`3OX0%-j%@0b^q~%_JSu+fC0&JT}B= zN}FBPWa1~hvJ>X?U~q(PJt;5iWaK8ZQ*L1 zBjSm>SQfd;>h#(~@)|vSCLJwP9~R5;QoBo(_>7=JlKFSoc&u&8JXnUncRfErQBD9f zqQ}ID{9w^_q5*b;*L}*0Qh^{&&tyZe!`m{+o^j zU_2P-VVuO^k}{?43A|)%&={K)E?FChB{QlUym$RvxXQ-zf2Ca$1t>vrEka}^tD$b! zb37C{4>HkL!Ka-6t8sXI)ROtX8m+_QW5fTojL)OtYI@FGO>-noYLLy_&7@4K$e8pA z{V_tRlBf|TZ-Fa`sCNJ(1V+IqMlOg@s;G&y81AH!pqtH%rx~B1WkaKBo`0-N^9bu; zwZw)-%b1{;wGN6onp8y<9HgEtk+d>zVVryjam1V`WhQP$?4%XO&ESM7aA<{BG@?jGlr-vt84`~VHY=t0WlQn;~K5`oU1M}EP+iFf|vr?z$V@j|`Sww`u( zaT98jH34t84@0+;wT0Q4n65X4V~_00Pi>1dv~BIBbNZ(?p3-Hp^9yY3t_k<4Ce$xh zB($cfHd&F_Z%S-i$diW=!_r7iY#yWyM4=L4qmc3`f+CiIEcq?hJtnd!QK6feMh01bOdL z6bryKt(z3hI>S*JcwJm$F@9|apdA>yMxwZ#lbCOoAZ&smbAI(@Vk@RN^Xf6V%z(2C zBbug+pI6x0Qc|yOZCPTFzs+LIaX7Xbs!opKmxZd2jsM1igxSDq zg6&%=?wfteJq}hfxhUzxo33DW9Hqivr-oI01G&^YF|)mO$`U7Ht|- zBRG>DpYyQlyl1jZVy!b$fEz6kp?Ku0BAyKlMSUL*&aB-1C*?}c^U23G9wNtsez61C zxOqWQ*Hm%y59Xn!RjS`eXR=lE{|2K*gughJ;iu=k6BM$`$k0I%Yk@shcVtsk9*&R9P zqjpx#Lxq8 z>|yY>*GVT$Ck5lU9DZs)97X;^CsiX~?fT%`E_rzF!NbtaKbKfq5yD{x)G;`K_nEto z?nRJtukb(EmJ&>~w)Mo@v%5#0(Y6G}GIr^eof=+30rBh3{*SwDEB&_H4tBw7aNMlR zGS%gdy0im2)G;O%!DPXB7a_MRCUTakKO%R2hE9UlM){<)>Lmn^k09+_@bpw*Bc7G} z{{}4h%(4#!RWCUP&&7SaOlKu9SvmLfTNooZm8i6yIMe~8Qc1u|qXklVj17V@2*x0o zDG|)koDj^BK`;iv%niZN-Pa7nkj+;G%76o4*BdSQpQ-Up1P2D+2t}R+zM&^L))?X# z^kUG9K`&FH7rQYZ_|mWqd@=B4diaw5L)1KqVlPfmklcp;Xn-6adV`J^bi>Cu2_x7; z_+;IG{M5GgU&pb#|2ot7x}WX=@n-Ccz^@n5Xn2WIP&<9A+OZ(bt zwZcG&Kpi+7ra#KSk+wz?!6p5h!w8Iq;5DeHFi1g*`2VlLe!T@!X!m3KzbS_^t(Fp_6!a-IN0FeX>st$oN#d6;9!G;r@+BK zfPa^N@IpVP%)kHrZw|qXOLO)uzqI;4y!&vG_B{o-=O=r>cq8n8VR zu5Jgi4xK_Nmg={vYAa8y^1RiP*&xLqqqjg&*Q}@BYF5-+$3|~$nBK~5*`(JiGURy! z*h7TGE8~U1+cen;Z9-{>6;F1SxZ{PLei$cd(JZ&$UOND7D>Z}2E|0Lq)3)?K=C}EN z&n^(PCV8-`I&*EFd5=onppvSdZB0q_oU;OM%&LID#KN-m+_kTI=GJImqkZvbB}qNT zsbBSYL6Q13@pqf1ey1`Uc>)3;WgvKomZO|o;CZVrqos_NqMxXdn zMxrJgRG#LGSw*UmSV9KPG>vEz5(`RWS9LT-;PRB;B3y}cRf`2A)z3*O8k-J(jJ5T*l~j2Co66C1RY*r3IP`^-m2vvVd>*sjp`AW8fdbHY!2 z;|4}?F59^a$?&QT#O)A_kbH_L3*PoLtgad3aW3%zyw_dz0v$JOAM+e4%#+xoqA9Iiz@9{v zfbU9&5YmfxYW_K$_DIz-RvmoJ7eS6*fj}Qc$!7*Iy@n`qQgwjyuE^bwGvK8H zF^a`A8RU9?v?3I^D8kIPR@V=Oc|vCIM@c_tH;2$cI;qF`>~e2OQO^*$@4X;j;rUog zWt)7M3v`0W5RvwHVb+e2Oo*6XvWrh+kV%Pu6nEFutMNo}l!R1Ozbc=P&j>a1Ku#Xy zV}`O|C?jnqCtbp(Sq3wL2?Eo)y2}!|Wk)CYy*1nDL8h}*sTxn9+jxHY^dXY z+BA6-!`@pMqul5cykjXONkZMm&e^G6t9?OEVjl9n34N~@sL;v)?mXbm?Nv;JxEoCa zxeerAU&uYyt8Sp0G=0|_b(_(gb~Kj*ysP90l0mfwcRv|~y@m)|bUqr8T?u3p|IHeGR67r(45HE`mUWCg}I=leb<91O!Cm4>31W+mqP<_P&HUAceBJ+KBcvqIvn%teY^acrZ%4C<^ z7vIYN)+W)0!{HuC=UHlRP{H2=R2E5E*gcSr+aAdJaS!0eQ4>X~&)hOOAQigK=Ev<>G!mJ zEq;g-G~o7$Wat#-vHM4E<**3{UyFf!fd`MT) zc;<;FWj1#CgDcMtp)T{79TUsHy%zFsqda-m-~aLd zkx91m^?!f*_Ih{!_2jGk=1aNS$ps>xE2i!8NiF4Z@+VBD>H=86cs;iTF3V@kE62tL z|63)kdZSf6ullxF{SMoQ7546q z|8?2_^{nR6jlM||&703!e#Yki=QhgJ>Ho*%|37Tlji!?Szj1urH2MFR@p;_dlGFb` zZf|YT{q?WgTU%S{_pPlg=fAy2E{Je0m+c>yV4402w8EE&8Wb)F`TdL1`{Qy%?-gbC z$3=+t5tHd}ZEZbmZ*6_s-g?^G-jZ`RSZ9Ej1W5?`$RC0`FX@YGAAo@im=7Cd)X(WF zg{1tnJytJ3Nea+Tj*gr7yOf2$o?c3q(Np|KmW35cQKOGM|9F`OMqd`C~u`$}B(8W5t?a@xzVmJjy^WaNqkKo6uV<^LGQlJ|K$+h=q zdfTqo|C3(jaZ6M-`lL>*O66LKL!Lj)7+O)&C(@ydl4A%_n3EvtFJhTzVuLD&hF8x9<#X1%elLYI$+aNLVmm|f-%NZXK3hDPgf4;;40fA#%S zHXYdg!}=buj~ja+Emz+^Jr)Z5;Mj+beM|7i7XtP7liYDz^A2<;LaH+2vx!rO`az19szd547rOfs-mDH^+g>`Wo9^;D@&b zeXZ}+U^KYF`}|U`y2zoYGE2Jj{OSw%&*#~x^!)N?+(&j{SD($Ht7+t5X-~h*PX6o_ zt{eIeCSkB@2a`C9vhJma0bgb>Q&B#jgD6*Bi!mSii|*}mS8kL%UKwf~e|1?WG-Ud(to$u0nny{)Oh8NY> zS|XM?pv%fu6FB#w(1aJ&`6A`da{4@7^qx~!+Y=$oIrMRF{Jz$+>flxGwro!X)2pTY zE9LIuiYHIw^<{}C&gXa8Y;5YoXRCRfS1weC&vEXrAz>S2mKzNEsS76t(Ye+3N2Bpgm&-1G< zfC*tLF~iUW_;em})2~S%0c;hA-7I@ONKHU|%ZG6S`r#;I764{Bu&V(7Mk_=9f2NH4}lwSCqhJ9vn@=s;BP8lq7%AojBHKsX0)R)J%|gr*Q9sJq;(K{ZUT8!{4Oa z^N>y(k6A;@7Q~#=o_3}D1QRc0ivwRrl8`PGJDFjVe6X{KMXc#XmTOQb2K(Wp@ zp%U~Mu{;y_*s(*rRKbyWc@Zq~BfSFBy_bvN)l0mPRZj;~m-}u(j2@)Er$>v$6%}84 zk8W8#w&?>_tr-~>Zt7I!=e`#M9CZWIw#T6FrSQpfpf7La#Q+8X-Xd)DLMIC27(n00 zvjBJjNYV%K^b74m5wJ1s#0HMW1 z9oI8;tn{QtLmy=t+HPnyw0$Bxp^sAh%P}3n zo+8h8gJJTJzx+Qa3MWMZ7|CC&XqJk(bj5SgzlfF>R|>pl@tO)(FZj0Lc!;5F9+n}D zMsqfMrs%haeD@eJ@%^(taKA9!t3`a^BHeH=DKLQ!*tOqp4 z5nsI%G)zZg8f;)yac`uOkS=;!r*aMi4B}4+6W!ybR)hjY=%ArP2hM0P@?qlLT>VpY zCT$n44ac@^cWm2M2OT@<*tU~9?%1|%+qP|6U!HfZe{K9*wO6%KW7N3noX2s_yh%aG z)FEWHU&*XTq*vTU*a7znyb%?Mi4d%Oco&SqmrJU*Dw2`7wkX+Ck+e05&i^K<>k(8!gb?2Tz zcF0wz{&*rpsMSp2R$ekKz0PVsR57Nuto@F5q8n$&OvW=xt4@X#v`mMjl4^ANA8J}U z?a2M^Wxw<~Q1tpUKuu2<>MA~;NaM!_IhVii!E+mBIh|hHo@aAQ)i?n?mApBZsF%qV z!?6v$9q>f9yy|w7f;#St><)wy;uwNDroTfGeS%jetG^l=Q&~46=QeXOtPTn;!#O-*()i?5+Z0HH8i?;j2c+{ney>m-V zF!VijvfWHI?zZ*`IPw1!T4hBIF*f_&aTohN&?33U7xdZ0q`fw}3n%RSh5%`u>|(Nk9<-cxU{l zIkKf3U-#TXxzlJcCeM4Cyi|2){=^Y(`^ExWq+F+0Y3cM9A$Dt%-);oA)DGYDW0IS~ zS(803+27#^<8nh4jZohf)9(C|!k|!4whfC_DKw zfANaJY#KnH@HdrJU!A!Tg@IBTI);HJy+27x&zuYQ$aC8SP6ZkW5DLIoG=|(Xu#yyhNQR0SPX2p`Dg7S}mR|m^f6gOcHLrR>1Xy}TO}2rb z21d+dwgq)4q&LfX!?&S$x1l$~$ZvWdnCqrIF79k2?Lk#UfbRd%v+C8Eq4`VMGZVI`P@c;~X~< zkR6Aw?IER2ezh0hbeGSSn{WF4`xbaVSs-5iVS;0jB&hx2W7nLyLb)p<7AQ-u2kh*c zv~qtMvBR<+TDXXkV0VKeDA<=^Hn`^uV+{p6hSq&QXY3(pticfz%U><$`9Ne!oNc#C z8;Jb_upU4NpS+2kzNm>O;^E(b5OFDMil{Hi@X|Z9GyFeUm8WV z|9F$Hn2`=!W+KrOspEOC+2A1Wq>bS@M)?G<5DTPr08vAmmNYtti-I+H%QF7_CS9H1 zX~s`b5jHwrkS?VVuKU>dcs zxJ8&cVM=VTT=7jsmbv#Op)IQ}-!E9)PPn-C!Hpnv4qF?>N&tOMvX4PlM( zfjB+yR4hftCRz70IZZrktN%+{m@Bc+uK0NlJA%H0GrR-$%yQHfDxKqDv9R(7aNR^7 zN2tdCbb|`4Q!~OWNfG=uGirwcrbYa6{IbIcafvMKMcfeiLj?p1SW%&Wu#j?+HnX}*jT zO8#!Ja}_siJ$_?0Aa4m$oGMI2@0Xrst+7qwe^?Rar*bI;(ta-Ohhz^x`b}~8O^^Pg#2Q}tm=7dqGE6`Rlr69@)Q(SnlFc^}NPLQs!KT^;+c(}n z_PfUz!PnxIFX^_qgKz4e&@W?5D$PBKF5hqzwy4mxslr3)Dz{3IVuZBX;wbsw+10!zO0y^yp{FB^yw^eq&LARLH9 z>oV)a!oVF^U?rjX#>|!p<6e94Nbpi?7-Xv&T0?8hOG&j&fJT!Fx~Rs*w1u_0$cf4} zre@u+*5J`sglG1lv$fy7w4S|3YkC#nLs;p9SmjhB=Y|P%!qLl%-03lX45O;;Fq;-p z4c2ec#7Dv8scQTCLicET6mcp$mKJ!t?)BuR$duQ}r2E6|re;!ZKEGpqvO|CAbbh{G zJiL4yH-{EKHb=c#INm#C?yq;(LeTNRzHNCyUcl8upNpUUE(Z116X4zdd;o{hpSWY_ z1a1*4E~S-_I?V%SPRu>GAxHh3r?A3FdvRpvnEJVQ7Ox;vI!IWK!sWL??< zVLE@qf#qHbF^HUWT~kycwfvANz#&yM@~TL>;Tu-X;V>!Avly_eUlD+)CQuqV$71Hy2Ep1r=gFQ9oKyThP)=ar)F+cDrC zUk`7ZL2;9lJD!hFZ!%y2FOQ=;vAX&D7G|xS1bLQlRsSRYM(0b~<2Pli;YVBX z|9FW}-vWsHWL2}95#f2?!-L<=f8gIG!@q6u8q+Age6a~pzY!_IzlZrr8oj$XdI$tE zdZk>2>)hQh(W@kajg71MtYNC25jTuwdA5+vjo>(2L*z5FkHN|VYRS9 zLWTfHoPaj9usErHr~?Wf)F4do)2)pdI@Tvs-o~B(aR*y4zi#Vbbof*xEdzb0{}!*- zno56xP#}2?aUCn71>!d`!FOQIT%I{4Sv-}J2C&JZ&5lQtD60{niW~QZ$zshGg(Alt z45SI)N0QWCGQmvIE84-rk@UT&(C8b6QABj7kcp1KW)SVOe8{=l*fG+nX{ zliAb5;Cr&YXVD8h{Lz2*-B523;Z&1hu>KE108us#wU`d0nq1=vt!9f=@@1II?KW^j zB{-^J{p-@2EB#Bj79ZU8JnE7ieL0ZYpHHJw5-C^1Hm(HX*|#vWB_u`f_A_H&v>y3v zm#o1bR`O_7pdZNJ%?UM!3?fh|!w(nCZUxSvw-VC+XFtwfYf!)U3_=yM2k2VwHpaE= zoBv-qhLc5<_=cE2csk+l;YZO**0ygaW(AUyTVo4^T zYLr#7ky^!?VCFL%e9}M4m}|_#H+O*UfDV;5jrlYy{;6B zODecs8$`^cDlUdie@)qaksz|?Frh|PlIQSzm|i_o%;s)siuO3lIqBZ4X-w^Kmzup~ zY;>iDmo6oczBo!JL)>EKKnVGg#K~O%k{usfR3JgnfrlnLi@d})Zc`N z{>AYAax!w`OZRi?=_H@`<@+z$vtIju$sXW-e2WxQbReh3bruCs9Z7}b6@Egt8_+ZN zTXZgX9GOmaU^+N5R!pT}U~O&9V{L2gt*P^|p}CVhR(TdiqZ5d{lp9LCE2dk9R`u~g zxvc?zR{%v6yJ~ACG;}y>X?IjA-`y*IEB``OuerIot8g28Rb|b;<(n3g+xuY^H^=VZ z&2!=Md{cMu)Uz&HG{~}tNF(Tvxmdk0{;;d1wS*WouTtzh-~sGvnD5>Jc`2{0&-DP&rW`yS$#sh3PSAqP1Wy9SsWh-3CvQYZD1dMIKn*e4BMfSCDT&SRxuJ_Q zAXxhz#3M>4++Hz;2#nVlXn1G9&@wnZnei=pm-Okq$VjHVr(>r9#ru4c>GLx4@hIra zJ-hR}*Fy!0R26K*{k7Wo<0cM>Ix zR~^64(abkI`&nmS0KXrwW)eL?XA^Bcq{f34#i01e`D~P21HU(n`icjoiAtK(1uv)H z&k(`~RH(bb`ES_m#gI>VWUN!<$GL>$Pf>;NE&N9_&kiih0va-5BS$xADRnVZ<3C0Ko}Klq*$TCv+OGY`yq(0B9_K=*4X)y&*Ixo}{5a zA*pE5%MbR^k!OAB@1*;^FytTTcF*io7+PgI+C8wyQr@NcSgiGUnZIY!ZIa|&e^EVL zm)&=7wR$}?7k^EH5%|&#N#Dlzg0t=8*xk35z1_9eIB#36Os5F4>e?&BGtvoEA31#r zUakcHKWS)9ano;*^jY~WseWDeitYjXPKJhdzhlxRljBdlSC;WOr5?-SW|Wc8z9Ty$H&(O!C!_2 zw%{Oa_pI7XLalHvl(}){Xe8)wE2YL!7<|LMZs(9Fpd;pLkv49$w9aS`@n#8J01V>_QekPdk2{cz9Dj^DT?{ zuZ`LZWRjR{mRmJsi7+xrhR46W7M z^o7+7vpp3g0@@R~)u(Y&Oy1UN$T$7%g^x{|Vm?`pdbY4j+ZySuc@n>V#qFuJIi_{H z{lG!`TGU(mpKIoi-IrGZKOzAsghTw9i8vzytbX#SBZ@t0MSz@FRia=nq@X$7gNu(I<6%yJrr5 z1Klo`>AHu>zHdlwwA`4$=BE|8#MDdETXDB9`xD9AByY8;WdjM^>I8)r%y@VTj&6krV*IQhL}U2&r~Pm zpH5jO%dvkd?uy(KbdVefG_v1y;ckdcV`2Wlz3FBI#T^S0La74f;EHoj3UPnDKg_nO zEd3X8LjEZ+;1+v{_HHYqp6y8rhJH7KBr7=*a=Ri&(i$>4iClS zjAHY~edyG+!|zIsySpsuhPAN(kcS#2csEBHZUHw=yOVpD5QS?agx~Dq*624cLL$- z2k~SE>vj4@cIs5J>f}PJ8g(Gd?GZzkUT&-+tdmkm%8tQKZc`=o`gubtDb;!&P8m77 zKSTFh?(9Yn`02Z+-$KYP1&QTwJ{m5aeRs6=_{BZ=B?UcN8JH;m>m}1)-6QqCES8IO z^x95jj3{0dez`0B_f7sQ$CBh0Lba~|RZQMvnZroE#}A18rPrrYMl2tPu*mg)gH#fk zg^2?SijPKk;lL+cKB!cyooB5C_2Rc{#>(UpSI*3t>OJnSw?i zGEgfiQ0u>%f-oc}BY@Tw34@S*6LRg*jMzp6R*X(GybIHcN#RQ2e;MsM@8WwijwYTO zuy&h0B!g1=vSnlU{-ASe7zTBgGTxIkR1V{`=L4TLyqt1`H#-tH^`LnVDn?A#D%fECm*E>G^`2xRcoLjAt*B8yNW27=-rlR;3SQ2#g*@h+ zEv4}7h(b`MLwp|~l=)VahP~U*u|xyX*tcV|$>mA!yL&V9p?VV(fq0;R28FTL62j6E z#M2FJ-*--0b2Z9qaz3&(z*venDsNew=?&mpEvnB)qYRRga^P?mJ{MRI+8dX>5Y*rm z4=|UOj2A?vM+#w-`A)#~%)$1GCG?IZ^dT>-qy^`MjimyTo}+Sa@MqrWnTxJah1Lo9 zLp};R8WC8`3VR8A{Z~R1dxo*=W(q~R-k9@+qrn5eNN1JV2EUF`^-8BU?SsKn@SD@v zcjcqnG*izAr?mI0#y&p6w%{+dAd;zS@BWKO^oz*LJ`UX%W?;iGeKmx8@{C>l?J!9s z)w&gw4jETa{R5vC*>ud~?zq7BBCn4CQ8~l(iIJ^9&%6ud8Z-w&3ziA%=<1f56phT2FIN4>H#~|^_*-;9_C32xzhlu%o z#R|RWKI8f=ar%+9B-$2f~^xHI0?*2!xWd?RmP%IBfs-54;u3!tm49 z^UY_E8sMa(q#-+rM0r_XJ7Kd5fql(dem;os;48t{H@ZUKPjp(p;Lj_C!dXekHi}Ly zhe8)JToe3V;K|V7o^-vLhgg~#>kKv>D>@kYXdZ`@XqIaX^3aMH$T)TAt@^%;vgH{# z3$(M(XCUex-fIoFV;xglmrJYAzW5k>ofRsyd_LNkmqV*&(wCc8Y%E9VV!Gx=NDf`b z#Sbu=rj~7+LWw;l_=lrr+dm>U>#TRBQ8>dTZ4c)#+NEb-3xJP0YF$9{Q1H(yY->Do4vhE71DyR%iNIU=Hj1W7-qhiEn`Y35aKv zI#Eg;rY=0!$&4n?>HGJx2xk;Yg7fF6*0q9*Lfd13YA>uI8abgjKZf_Fd4b$F8-D$d zyPrk30E+}zcq&jo^(jx`!r1O#PdESsrf8sS%M^3}AASFXLbgj}#Vfy>-um|j;`Pcd z;pfiJk-bKZBc{ejRfcTlHM;WFONY@GQa7hkdm3>7n4!|7sdeyx_L=0Vre^(j)^ARc zQzqI;!{}ydW1&R$l?V(-w?K|zhdo?Mb1c}m6z>H!sPUM^NTedvsYHEK@-64MYrfLK zaa|50eP8N$a7zw3A4 zbEob`-;7hEt0$y4u`&^2H2m!8HYhp23;;<6O^S29$kX7!`%11KWce9!?2o8!XA~rR zsGN7#=QZrvA4|qY8RUxXiZbd%uXqFu9~Ix%@bLe)&xSL!tHdKq*4PFQg}2`ZaNAJYU5Kf3a9fDJhYJVOw!%_21)~e>TXQe#fF+#nlc^MS6oag znVT|eFCSvESDTDB=*KD-v0dkana{6AG+fD-@VzXCHi5R1m8Q0XmD=8%4MqDmshcX{ zlwlcf$aWfg`hGQB7xgs{)%;?;px@T<`FQX=9)DbAxMtrNN}q|AA=h!mP~(Sb@A*W? zBu`$mx1>3MMLQH;SBjfcxc+)Q+; zQQ)xr$f2p`vVG8U!YDY^8ybYR+xGr^KXRxi(vKVp-qs;9d&6mZHpr-WU5wgW$%d08?1fA#m+DM@C%J6{KqXHckwGU&XTY8sJsJ$4Hj$e$^Rxn zJ&?c;@(tZbkCivludV$*7u1O0|IY;_5{2_Cz&!iucwDe_>9nyDX`SI-4H;tH(3qd$ zcgHTug&xWXY?Um%UK%x4?X-0%?=j|OqsV8%#vW=qe?)8T8e;X~Ktxw4@yBPdBLOpd zjLn*l;=${6qMFHl@RgP|F+jql8amTYGV)a5w#s7v5}bb_@zPzpJ6&s!omX#blDE>L z9SqX&)3Nh;4Bj`-p847Hkuf5+rdtpFNGn%~!tkr_fd9Qjnr9Bq$9_WJ;)A zjv({g*EA8GW6cExKVpTCfjLy*LEJw;fvAyC`%?;6tf&HGk-U0K#t!~VbC#Qe5AVO^ zTc0KP6 z$!KSA@C8ANxb+gIcwzBQg}V4-lJ19%lZgzYCU26TT`Ml+IzfFS=}qhLQ!1fReFU{@ zLk()?#l@nB6B3&l|6(&Sr(N^(Kw({@PZ8BdPomH>I*CR%_&R%qx%KkU#}C#e?)gVZ ziKI)zQP4_agTr(tGYXgeOv~MWni7ob1=4xulym*lVT&Ddg;h%}F0McZVoIMN1*cuj znEh|hF-^+x3c!}C-ju37E+%F)JvSLgOk8EIl6(52zJ|z z?MY&QV(KE3{KrhlP2I8v9rYd>RG8|gfL;lvp74Lay4oE0i;NtRRO5`Fb3aW&t53+g zkfzIn3_(sQo2O>J96{aI(s-{?Y~qGQQ)kLA7`MF=dx3+^NXmv!Vw@_@7ZHyp&T(&X zjA)?e|5rXOf1z=5Rvpxr$SWadht>-F8Gd5f{tQ3iMUoZQB8rrT{>lbTw~3ta6b%jJ zJ!hmuQyzX-4!!*H9lH4@Sh@`9`?e%xv+DnD2gK}tvwd{m?|(Or+U%7<`5oe6(F9dZ zfp$>i^-!D<{i=shf}9p8+9?c5PfrPOk9+>hma==yh78Ig0cW1bmjWus5bj+mztj%p z+njrl$w~X{9(yw-S9Ux#*lwE9o*3-Otr>q}s7JMWHE@L1Vw}+~ESxAuyva7Csy2kB zqUi}>BIh5+QhCFq4W)B!UHV5nkQvF;*W^EL{!hJ>QTv&)F6}>S|wqbhFX-(CX%!r2%bA5D03Uy7&Hgvox z9!};$yP_bGI}-zxyEr$1jWvBbeZCSKUVIu4gFl*cs`xn8kqT*GWMc2}kP&yP^}X_`Y<78^u~h+E zV;me}X>IyRF*o9=gEgfH+h42V$9+^C*sU20+*Wb9bgVW8@LX4JuEAF2x|s;c3}JQn z=_!(cT%Y&Hy=9i0@kLCJf&yG|A}N0!!naSh&~=$<0>)n#UH-@rsoH>_KHPfZdpT#l z6f%&ZUIRZ5F2!NJWY{Ig=xCA#dd551;7QDguEWFOo@y0E5%$G!|(mf;8ALX?E26^_8~ZZJnvY*J>kd!0%~0Q~DtrGk&vVKd2_ zJeJJl@Vi`Y*g?kys?fb$)Qr*dUW>8|NA(fYZePz-cnda)I}ebIZ~X9|o?PFa<2D3r zXEfiQ_b($i-3&h=HrAJGxf*CUk^x9#T1oqo|A@CV$fHb}z9Cxv)yVLaBtKI}4y67L zh(M4{z#iFWmi09dLwMZwCL|`+lLxRvFXJ5qHgnohHw*Q8Vsi z|58>+Y^GICTp0SDSpdmVc3sSwT>B!oMZ+1q@Nrd5BPLTlPwsWUK}P88*vR+~?nJY3 zgK2U?zbH?)+Xq(ARY}W9X&KNGDb;wQg=MDs3)~_Ms${giFZWt#&vUSGTGtaaKf??O zcJW65s3~S}^xiv{RPPYpmb+R#BCv2@=fnL=2Zq02f9ip?u|5~9SHhx>S8}$MnZRUZ zeM}>ibI7*(Tg@{8N>~jsDs(uYnGfE4N_Kdoi01#O01p&CJ|@WhOpIxB1G3b0mbDxl zYFQx+TS@|Tbh^LuX9MD{jRY8;>teuQlV7j(8IC7M+QXxtFTJ&1EZ!EJ79?uTO=Hhr znZgx!@}RUJ|8oE|(Dra%sGxORSQ`7CP@#FU9c{x=S@B4oJ24M%S(mB@JLFO;x1^ji z)^7uBUS526Cl#!Bb=TKOrwAqx9%;1Nw=ggK-qZ=>gqEN`8w729Xy*^v z>SUVYH%)l{kL-tfd%Yo5gE*hB0C~RIQdz1?kGszd`?o^SD;F7c%FC`KeZCCaMQ^^e z1L+ncO~!@V)RO%aL`&z?uTmxIL3+^{%rq8`srlwGIJ%bHPRlA@th3af#kK1@g2?^Q znPsy$yu6*SrE38#kHcBZ&1ugJ$!WjAXxG;hZm|PqD!vKM#t0CG9JHsD*Z=241=mwb zmIOslBOF#w@W-*@#)OJmJrJ>|57K5YnxyTx-r>?xF+d2qTm<@Q7EeDwM}VEPcPB{G z9%S9u?q=ZpXG;Y)@}0g z>K0HtxaQ^{??58XepBy7Qka#Y_fyb|>RxRpY$mUkeg73k#c~e$e!QRHsFfjfKp`p2 zDEo-&0{A07O0cOfORjibh%ML*vUBVncU^g#&*#hy;x z9}2fHOi_E@7=1K`0s8V*1oAEOf4UW|hsyJL>J%FtqV*T35avmnAR;5=EoIPj8cN)h zf_9+5sM+tZsj}l37n~ig)1sv}KL*gsCHOz3S?v}%zxPMu9Z5xrCLxKv^le=eN!FyZ7b;e(@;nA9m?-X#3_D4hhdjvy1lLM1o zE&zw4ZH0uOtAwMY-m|Ki!_hDNeJu+e%s&n;KRu;IX$H|O7rcH3J7C?)VkM!n)07CA ziMzqhW;0)1uU#n{j5hw)e|~qGP>8y{;qFKt<}BB(59vkadc9Zod5b5;>X!> zUf8ge!%@F%aLP26tMqdK3yz}; z6~Sy%`qvB~(NjP;@W#c+^BcAFGr;5fvPfUQKD1RI4G#g+qpDwD|Mu<3NIw@zE4O5S zwe*;J6JA)CO?WChH1jXy{fxeGVBB%->B%5#r&{s#<69K^?Ig$j8Gx+o%o9*sBF^cGCV+S87&?NhbOpu?YV_oy?1T-cDcxb9#{e$_n zJ13T~+2+*K9#7>GQ=9z}ViZI8j%C(2@c6&Sex^1Hyi~^?{Y$>DC@=9o7`NAfokO|7 zaSyQpzOU0c;(U?m=WUW7;SF$aQJ|ui_@a$i?JU{T1Ix%T`34St@YV!ELs4zw+2^z> zZZixmAyik=X)y-!fV1-G;k~CzJu1VcKm8#^JvH7}H-5CAO=xkvlOhM;cSg~eiN++86 z!S5i1p&>}%Hs5GV<_h~uk{K8comuh*n^TTV~!kb5jGaB}B`K>{y2PQHZR) zYcE04_%Q=*p0As(b$S`oHkFx4%K=v0MF=Y=k+f(Lv^}};mpb+;_RJGNF#Sv;6QQ9p zgJ9mT$qa1mDrft22(ee2uY3nc_Kg{k#6A`G?zo0^LqI`gN3aCOg>iCMs~jtu-{TLR zmQ1u_1V-#l=;U;ru7+`uIM!#j<|~69`WQX^vApW(S3IHz%QK9HJ~Z8(@)sx4a0??? z@kB`@LWdRVIse8KjS|Zhet!AdlUOJD`40AV{(E|B-K9C|*K6Q&BR{zKX{Nj>t)@Z} z-t^_Fy%l+;V}cWfeA~2Du{vm6Y!GgMD(FA_c8>SeZ%88Q070dA6D>rh2NI@8GDRSm zV7=e|RGA!R#TRLrhSn?^(~7Ha?ln`2uTya740~Qh3*%cJspMqb>@{3ga$Yh zoU7}|Qh1vuCTjqE|I2egSdyGGlK8DY+GHPF(~Om;;6n6{v5C-i9#-eS$(50s48D;X zH!cd5ghK?gM{9Ru2dCE6Q-?1>7+Q*42VS>&PMoMN$lpm#P?@C|)-M0v5LrAa2Hx`+ z;=+$1f%+9k7c_cPm55ndrSNVdS+AucZAvf4()0c1& z+PN-{YSP~!)`gf=Ax10YGR_6-Ts+rfC%gkby$$O;*E?M)gPKSX9Cu_8OIuLw?$lZ` z?Ge0l&n$?FePz8^JEElmvUK@FF0T_m88Jqmo2yc|x}?Dy`3Z zI9xAUL2aKQx6snkY{gfSRIC@P@u)~ouL6{}HT?!noAaPO>IL}Rglwmzsvy7~B{Q-?{@>m}#n96_Q^H-kCPa)Gj;8aCu;o(6a z?}FA<-d`gGw*{G}?u0iX5jv^mX4g}uH?;1a1Lp3;%b$yO*sGdE>YPdW<)}RIc%5H zKGCLi9>ney#qEg<=i@8V&Zt8~=%RtS(fre{1}>9Ul9(67uZ#xfnYcMp8b{#;r;TnX zL+n2;+9R;HlaVbw+D z6ACYoWmT$idYiwV=+jX0V2um9CbEjLDqpBBPHS8(D8IO4iAfg2cOs_lkQ}8503bmMRz@URQ-HQCBYtFk9Ng4pu%0UHi6>6&PiN`-W_mo^);!f$)C!sj3aZhn^Nl?;HR9^K!WE%r`h(%j(`&# zzSbOPttlmHr(>>Kr=7~3wqVekgls0JKtctdOPhT2$$m_-uK4Jv+GR}qXZnTMp~Mi|3781`&( z70~P(q+i=1ccw~T?oN<5V7WvQS|_!3W79=!?u&Qf?QQ3$BP7fBF`uhHnaOjz8YfXA zqWrryg{=@@b)wMSbYSb8lLXnDGbaqn2MGrlKb#rwnn4`XU=5;G)lEi>OTRd@u;rv$ z(kE5hFl~+_&usK7lUa#QGN5ASEohSy2gAc^6M zg9*p=kbI>00;WoD$QD|4zdc_i5J5O_aC9i?l!^}-bNc;UAF82!cZ$aXUQR>1(?w-| zdD+xpNqjygYY%sOm;YRgRfn5xfAtJo_m$nd`{u`-hH>d1`Yk>~Rp1q+)YrZ$?r^8< zwer*s8bo+$0Ef%e~|5h^v=hGBT&%+1t{=H`9 z02IqJuwttTh`&CxiiHHRqpoi=CgcJDK5I#=)94kiM8voaoxmz=)r@Fk%}%(3w3 zOs4aEmqYgzkEPA4KAHqFudFG>mUC(p z=mCAQAqRN`-U~T)U&UoUPJGl@))zC@4fp2#^@jS{)QgMX(N=ajApefx2|9Na#r(T` zl@5&F(VoeWe~jPr$h?tnKGF}Ip93QNHTm*Y(>$$aXKk<1MErHpyx23T30OEaw2`QY zXbp#uawVQ*krE2b__lL#W+jHGz*U7mrPkE##m#{?XIA)Je4s$#7~Ts@)!a8NY_32D zgVQasq3cRsYyE%=r}0%Z?EI;# z0R6?SFgs+s8th9oWc!8%dJM3O^^PMFKkSJ9ME4Z(7WBu?Nq<@1E|apVoT@ za#thdS>S)6W}Ox}Yea^4zI0+mhPgL$jY6XddB-{lcx`yxE_QDD`uBfl#%ulTVkJ?Q z*&!dpd6dLY`Rt%P=%>OV<{!30sL2h5utKP~#M7A1zYcab>+gL&mXoQ9|B!x_qH@2% zCm3Yj2v0IlaDx2^SL@9&C&!D3K+z14m<5rI7_&beGYZ4lQBjRhnkC7Iz|jBQ092ia zGV#}072BoJb77#y964Idgd_@xHxfuT66jRi5!yD* z+JJ#3xU0meRuGnmktVp;rCQJqD^b`GtlzK!4<~={2CEV%ZWbg3FqX^w2P%1%c?5vb zYpM#9Ig9cF7$ah3W~KiFR@hHfM0!?8U@Fx2C_q;4m1fR{7lmPH(SWBw3Z?S>h=~9d zgN<;TVrRqx)P~6$g`xi!Z2xaTFGPU|WCM^0TO)%U%QgeESzFu== z3~|^SMw;P)611*#>z}#-ms0GV$~l*RQ|h=^`QWjHbOE1(5o5GOp_%wDTgVqyb61c& z&v-842t3#1E^5-opZLJ!lro#7{LNk)BfIyX1IOOfe@_BC?1;RNh1wo7c^$Ae4V*$} z@x^`30b3XRP?DitJ1#_<{ttXWr@+xtCpz2q&61Aj6*g*(SHv|_m0z7bPaNT*oFiTL zvG_=mq09KXu5O#Vk5ShW?mp@*J_kG84tCxWyB^eDO9n+VC~n)gj4TauH(q7Uqr}I- zBds&)R0>pAGxt;ZX2&-!K-}%8KnM~>tX*5jweFGK{4XsXFE$fc58Dnqnc(}h zDIZDo`t0Vts(`Z{i!%I)2HMQAse*)5+fa7-2Cg)t<-t>9pRbo&zBZx{Wf{NIy3q=z z0&(`#k0Ji-u@x#L0`}h%jyz0rcOuy9ttssy*qgZkSI&R-D8uu>l)h78VV6eO%bTBF zm#WODcyIoVv!AyagC$ALPO6);ktN!;t@)VU^ZA(gs*Z|uo%Xh;CQE^T`5+hZR6X5F zn!cdddN|?`(0#5;kidt`q7ZPn&0iR;xXn*;c3rczO!%x;oL6p;_4`AoUp^FPW^&mc z*J1wrqBwFT;DcNz;ILXxU2&|Rts#@4JKpq44Gfshkd?~fIzx{!Rk04f&7;2c9i*U` zBxK7;AOgNMZ0A^*O!4}Sjvv3m^MFRgb5QRW_RZ<}T}s9Jx9gvPWz++QmP)CpM#L!U z1L#Z>-SDEGrR7}0&&gXU-|ihAgNb|E!FAM3d^)?C$M{<4li%q1-RYcNONO!`lhA_8 z?Yd>YUP*rVC69_5VdaA2I`%eZ>+cIvfShB?1@jk8|f#yS=6Ao%m)+qdcncIx=7 zd3tvZV?4rT)Qf2~OGz079@Ev+4|v>e`mL))SzrN(RzbwpU<>|FuzPa$iq0-T$PA7j zF1_6pjytkf6V8#HV9SsSX^8{7ZECw)fg+Fvf!!Lfr0D{iCb2PG0w;fF*1VPw53N&x z>I^l8xng593dNwQYsZnXF=KkNcL6^_6HiOIm`JL=_9e1aY^r{lDb;Alw0YLXg?qSC zK7Wxt%adW;+genTy)jI8M+PT)a17Ss=_6op0h~Vqk&^}DG8Eg$GWZySaJ(|bbib9w z=xGGds&inZv$(}%i~K$K2x-~yqVBM^2|Eug?Yh zh*H*Zg=L9ELrdGnV- zB)AEzyQn;rxQr{L^|P(;VO6vayjtbe8;QTl<*Oa6lC@xoV+gv6G((4ibHM-Z^`}ce6+i z8c==$v-?MXP9k4c*+v0R11^gYKZTe-&{MC;(*m!iH+`i590Vs6ftsq8N#0*tbW7L&aMZ!6{*Gsj&O8=JW#o|Zb_wMXU@})^JLmjL{6JZ^%mn)J^ZM#ZikkpN$B2Vis zlpbL#K8Og%BX-6COOZF#bXm)AYFZUJ$s)DnOe-h0Bbf8HAgk^8#N;y50?R8Z|M^A6 zwz5WU?6!&_CFu%`t#sw<$hyDt%_>jFwV3!IYU%Lsvdxc?6yx9gb6+-Rn3o|w68)WY zm)AwvtoJbhTMHBmk=h1~&YddV8eN$!raKaa)8Ch`+}+p=!SwCn|Kc<=2X!L=%=4tn z>?h#RozVRsuw}VwovY=P$Cv76W9=&5^DnQ1CJ)owHYjI@c)rdw+cduGHwyCOI0*K3 z-Xbr%Df`F>_LIYg%mBvc!QI~j-C)R1mf^_Mf8BVGi~;*$%iq*_cEQ{@QR4aQrzPLG zQ6aa!&5Je-UrQGc+?Dw@KdbdVwDK1%9=Lr)qTn08ZCQ|wzM6+J&zW(XDFwJ5%Q@ll zvCQVJsC@{h%#peg;Mvp#$%E0w>fH*R&@m&Olo$S<{C2=y{7Zf=3RR>th5-{O-ASdI z<=g0L!8@RU3t6&gMUb;JJL~W7R6n@GdJXkZEqUmVF;m8qO=$9Wl{50hn#_Rd9cEHd$V?WSH0)I(~EI)Rk~_8#8}-Fs&o!flKCCM_k}7P&C3#Q3MbnY+OhBzl2` zc@Q|u6{bxW{{4+aK<6QQm(Ep(4gewW zND#(inT~9G9^C_@-TAx0@zb^c%eBvD}062B`I>GzH?t zqbg{z?T?#bS-FP%Fe*#;23ea&!f(wZOZrg+HcURD{#s4@vtNv+*fkcOnw;fmL3CQh zvvrL)ntP|PeZMM`}zAWa>d|Be*XL@ZYM zLg(0{<|a-`L@b_z=HRSqpkqxh&3*}4tYP3o&OhN5tYS!MXZovEy{5*sL$AIfV5&hVHV*>^4A32M-quKW$6gv3*4u zDGF4>-H{c^`0EDbp!JMD9m5sH2|!7Cq2Drq;zV;Hlh}`wb27M#J&JOTiIgEt#PiIL zsJCy`V_KIa6Sv-^ewevc`ggf2O~=++`e1nJ_i53%Nw=0} zu$VDlVo8%}LA~FrE{3u2<>T<6zW(%QSG}d@htJAWwIH6Bn@!z%$@jyO`Bd-JM5H*P zF^#^!vK&E>$hPT`j9@s=!pNLnlguB4XgeOtw}AWv%@d>tZJyqb@z|dQU>?Qn-k!qx z1adZVe(4=cH~{G^6~6(M=g#fLm7kKRvX{1&j3ulsKIa)oE`9GY`iA;Lh{fitu5gB` zRn~?U>0Pc;nq|l7^0yWL!c!J4azEtvftZ9&_^6PUCX`(N$U5m2VwLKx8(i1r{1{^G z?q(S8ZG=Zzm+f0>tR5BGq@Fef#s&D9*)7sXDxzlicihaG>74yqbmUH?6isY+`E7jp8$EWhR-xKjwQZBA^h85U z*0+tdp_bFmjSTE}dn&sT!%N{qUz)D1U|WxUoIFgQ{7%j<9Rm(sNGD{a6LDD;VtT0q zlYBqWrQ}D$GC(J7<1WK}QJ;CP1_BjBBhd|hXr~En#mY7Pv23Y>C{@~Q!S~h!N-g26 zlQ_+SZzq16?<@08tR^^6XeJJ5+gJ^98~GY=U>)mk4Un9e@j2Zz1l53dY)##I8^?3EWFh9*`4xP4ir?u(A=B<9i zLx?`#apH|(F68j2at%OwADGi?JiHHC89C#~gd8UCK{WH}Goi|zAwNMs(R~Sbt)55SPcL4 zR*mD5aUM}LXAUvtRXQmgdsRIA1Lje8DgVQ>DB_W0&2Q+_sF4oCUbpP`^uX3N??N ze;dEX)3T-U&9cJfylUsk`?XKAU<73hde&!V-PXC^=1ctGl#&1}}K;ucP5lB6F>r%h)#bp2K@fUo4 zV8J?nVvoR#o_hZ3QSj~PwT^N*~9=`{x7^HVj z;Qu14fKW;w*#8t5L`**BZ#WHGzI6GP?ZDBeZl~ASdgxRx7&(Lad-kPIC zh{Q!$7=D;*e9EM2IIY_1Q-WU^cX2xE1wD2(w$3wIzG7$bcOvhwad#O}GTh)F*FJ4S zeRpXrU6nH0IH=f#TqvCbl?33Oz3C#nK8`(-Hj=#Ed-L3==brvJ%FbtP{klg#P#T>!?L5wk@ZYKo>3j-Jt-T+ag?p zt`1s!gdX$rC+q&)+nHC^72?n5?oo)xp0iuA7*Y_57@_L>qw$&Vi8I_(WJPwp0a>-S z*Y;CzMwD&jcf8lBI=OFRM%d-%0iQL*gQa)GjTpH=n!fdrR)hw^$Yck05~ zU^VDL@A&>M^imupX}LVthnH+iAD?wNFy3)K3(~7sa(V!v4>%?&h*oA2JXpxQG{`B7 z*6@Ege?WL+{4qeV-BiTI+eRUlfgRhY*cc?*b89*61*;cJ-xz4K!um_~jgdjBLA24SN~&fjxTT4~I`sPL4Cd(~CA!6# zzxi{*UA#3_HUkoOQ8-uA0cf?48$Vw!K=xuQt4LWXT#k}+O9{#yP!wb9 z_GTGtDv@n3do5zRLc&@fkU?2{N4taThdJ}BkIQZ-8O4FEdgg((ywUaS`meBW{H>1u$js8A@dfZxu|grfhS|bx?@@$w!}eoTu2LC=wz@^?0NZKe`Kx-!=bu7F zC0>p^2SaU0Snr!(+F>DqEsHKcpbr{wUFUu@Vi&a}B}D!71Yz{~SUHJM=ADO5 zP0<$T>-~^mW2|V*B!R*2@2NXR)^$U(Lk8e^F^~xms%AjD$Y7}gXCdT1=4|Z?*>#fh zvW0a;EG|Celo>KHyPC~RbIlpPOslRa<%WDuRijQiwS0kmf&KI3hAdV1kjAr`RO=G# zA-OYeO(Gl)ITs!IhSb9T6OiEVVi-~AOo{W$icD0xVI1069hd-PQtfabNmyYW9jQ4t zhkNCd=0H6muZS|BLc6QCD}A!)Dj^!nKpUrfhi)|IMJ-EA^rSg5*CS70{xOX+8TDgE z3~`#&HHd;sWf*~?d~}IN#9`20C>@S0k$U=T_7y^2sx6igi)Ku#?cG&o%k2ynbS;^Y zf!fTK-D&cwss`uaH0M$Bx_;VzqN37RUDTh+Wv(ah%V+(s(GY_?L7jZ?D zc~~iUNR&|{`@4smJEgcy&7&TY?awoM%ootdE3$+ghYb{#SOB(u1q8F$i9lhem7)?xT3)>0%)9OLqa|cJrM{ z`zMP#NOxUH`EVfct}YhhzECkJEaxm7tB`UQ;Kc?xl!7bO*$PD!(tUEni!hcW<3bd1 z4^;0$h=Qj@=JK8lUcatAG`0=p+H+9RA>HXle##ps2r%u`w6XvFfGsOiRn)i{+d zYBNP1vq7VY%b>f9)oXL01@@G@u;vf@U%MWKvOZuXij9=M zb~#>ic;GLf1d8$QQKn)gV#SJap74kE@twnrQ;mE;KF5ah3xlC38kM)Xz&7x6r$RZA z!ub|X+lZ!MO1Qj+)nwk*jy7ROn6y|=S7YP=sy{f@hL0VFhOhrju+B#9lZHOt1sET-PbTO-6 zrD2bY3a>OYd~A^UkyDsRLNG#ThtqY0Nk8GK~5GM0A5l9jPx-Eu<9xT{NAS|%t0LUnyB>7 zH)-b#Hl|;`sOlBt3whz4Z^q79Sxi5@{i}(ws}=&pe#@q#c2=vWjM3Pw=9uPqu`T`w zzqO-FHqsdREY19V16hv8JcJI#ChRdQnulq!~W8N=d##o3U zWu2s0xxiwbwZ{Zr{DPNEWC}9O@31Z1?NZUyTVCi2*^6zkcchqzA z%og;9{$pB0cp5AVdQ%HaynpO!Yz{;fxS+Lxr?U}bhrt+**F_DYD&AoKID{_Ct>4Ng zatgzvjl*Uod^45CTT>xS-lL|(Su^5aB!+?ifsDEyZQ=7_hrBd2m=bR-~GC^Lom$A`)`sabJCBbD+^c9JCXH)aWcX66NUWqYQa4Q>f1Nlj8DIRFt+o0-N! zL@*S{0*LV1%sdw&R-i!UFMu+CWd8$T|AH+1zp_D1QY-oY%KkA#SZ!u{e*;*_0^XHO@m}{{gHgQ;G9XeJFVK zSL1nZzS>O6)5i8#WjU_~%dxfqa;pF-16Ct=uUp|+6}1QO!KX?cr=NE{d)Ik8J1tG! z2C@!7Tws8C&KmHv!`8&@OW#3|KN$8g2mXey{nXxZZFwEk&mO zC-#`%mdP3$giQZ>fc(Ldk2#??e6LRJF;|w!nj3^HOx)J~hDwaU0!l}Hp&Hr4+IB&P z_Tv*IGeF0z3#M2l3w?uj!}6_8qd_VU^}1@K>>y+ikO=ci6pri)*?E>`fGQCtQUs3d zH?s2@&A=b#ADH;d`~m04&I2?9Y>6=KB5>K|?_a*nh;oQ2&U5 zzd4`;G)KFNvt#!5I{%=MO^Dkp!eQax-^)W15Q!~HX=|3CS^0-?lzk~@UBt^J?M|9E6=uXF!R4y_zoha_(9|BMP+ z;D1K-uR!St&Pa4KOyVIj&M+YhWVZpVgCMFYHtDQkD1l+-NIKLb?=--_85G|WOyUl& zO6UX6cZ_5=AHPuvO!J?Fm&wE@IJRmUpCuEDIGDH~BI+_X-62a1E_w@X2*||=OsHFN zcbF;P9-u#~ zeH!gl#EB%cZpX6QYlX>}oH(=9YeP2kbjMP@f8?Nz6S>bDAWAvApR9%@7k_{@dKe1; z;BP|B!ECqrz4I3VcJP{9uPnde*!*Z@EX^#t>k51G3jdu0Fb;&FwnkqAfR-@m&~4eL z;n&AVl{eI#Hw4Vv-2t2Lvhz2DOP+)6f5L{gnVT)x^qGn{Ih%5uE;#8qEjj8qiWvEI z2sU+%I-U?bHrb!A@DZPqrMgxVw&afOW` zlfd7tYjDk|;$N;WP|jdrxGdp?CkxsWFTdICDtwyZ$Qdj1-r%lZ8}k_N5aVu$CAl49 zoMD`1tkZbkUs?Ak3)m27_1Ici@|QNeRfU2yw=Iq7=a7#-+hcA?Mu8X(>-<@^e14a! zeiB4_m>X8RmI;F(DB;(1I%wR;x{-LQv)iXJ4(DR zzjefd?Mse!9DV_XJDn=G`-(yy-;}_$_?39IxR;zm+jSG161LfF%@+G>$KGJDtqQc+ z^sg1kA?gqzY#Xo?e!Cs57kAVs>Doo&L;lvm>-acU=F&X@*6ybcM+aRJ2_YW5lp)0b^qY2IIV8EqVb!zhpMWWTY2s7 zuo;;Ty5mW=pAK_^`hB+?j^5P`>B{#%=u!lmfH5JuXKN}S4G^JaU|{<> zB)*rgWr+O?WbOo1cWLsL9daud@aY=TE9_g~3sI=yFWQY3$XhhPYJ$g3q$&m}*e`%| znQ5l#m2m2+Y3HhyaAI}wX$nhqS;pS(pH$64l){;~Bo%1l0sfsOW6}D>JxzpJL(6Pt zJwLR`sl9s_J7iogz%`0PL86B@IDRBlgq922gXE3e4gw-?u74YkPYwsWSDQ$7xsif5 z*W~}iX5}B z%`N}fZ3Sq$_p?34JW!}t@ToPW$+(Z~fqne-;O;RqS6upPuafjxz0}eeSCdm&>y7TgaqQSt3Njw<6hc>N2&S#X zuD#5#pwwoZ8xFngF}==&mI&DqUlbEcDs7g2zHs{bw03>8@Pi;#2xWSlusR^yLUl(F zPkT!7X!7$Fx1yJ#Hy$_6m~Flm7w|J5Qj?eO2XOUf*LMHOm z6iM$C(JPfDVz6KrwX~$UR)bCey$E=Z&Pt`sGG5NcxH1E5L|3jfu5S6dVPW=FNT`8| z{%W|l{1?J2NyU`yH>JdV7KvnqoXVY_g5X4G2Vrrz5+-d3kZ~v|DR2qGf6UnKG1waQ zGRARXpg;rMdqfdmV-bW*kQe9UK@S^{j9KKCBH$__O+lBFF;g~d6kQsoH%q86bT;ee zZyRLGBG^-cp^I_$+G<1(uB->d<0)Dx7A2P?qhU$!`+RcR1Mu)&>!x|=i-Q8Fj!vE{ z;IEWkhLL0MJcg#yoF!#h%O1H=t>U7(jE;_-2#-wgx8_*uWlNfw#d1xi+}7!ZKC`dd z2wwIj2?9n5UE}kuI;{$H?J5%yG3v23w{16tDaMILHMve5L?k2QMsj_$q@HYLhKWR< zMJcl)m0I0|$mYGcNtu}tnm``xq-yV27O^QSCP`fB7lu{Y@d6Q|kpfpeDn4bsWSl@w`4bGr9s(C^S~nj0 zqqqDs;?%2QPfJq*YR1QwI&TV@$VW=i`77T&INxxa9~yHsC& zMX3(J8F?2oQ}L0+w2kwNWdmi((Y*cD6*_bs9E-tK|DxqktT#YUcXVu7+cl(g2YDQS zqi6m^svz#1HA-EB&)XWhwfo9K-`B18xkhX4+8yPq;mSMOE3tPMlP%S@Uq6Vc0^!^T zMNTipQEdx)St}Zhj`V|=l$-VF&65tsbv}PMOEAMjM2H+6V>r&DQ&cIg3C_Z3xuBR$ zUNW>j(*gfdWS%v~^**heCdnTl_GfIS;yDO}6S@3kvAh!M z5>gJ`#3kBY(`;8$6QkrIslBDN&|I1))U)EOd>M1Lu(Xcl@A9$+bv zJP$Kr2y0P(Wm3$+QeSYc+f;tOj~#v->PT~MBO66S8Qr2+@RO8&F%#)hc*Ovs54u`B zvZ0q$GOpG>tkY)Jt)8=-7+-g}#&2v_{zr)?A>*p0WH`4QvvQX|8qJQ^eDy z$Bmse@4Fg$5-uHnT=S#<@*KBg4eUySCqv{q$( zDxC&XF4TO{^)QZqAIev6R4L7xeDP1NX~14tU;d(Bj8rABCd!c$-Za+{Mppsik=0u4Gp=Sh_T#L9*TuVM>ahN-Nqovay#YDC@1NPrN8Jv8No-}Q8h(u+_^K{xzjGgeYevyK0E^ixGji2e z{5=b|X@cTP*c)z^04D;qjRPJWX2Iu=C+Z*CfFdv2Yd18{iOu;8c++I?!9tSf9a^$Q zU4aZWLBGHYjL#KI-jb~j=-V?{wBSfwp`g_ATrH99=MO%1PQgL}saIej;@P}aQwx02 fp>G|Jodu7bTenY-;;=9<5Xip7O8^Wb9?bs(QUAbU literal 0 HcmV?d00001 diff --git a/charts/airlock/microgateway-cni/4.4.3/.helmignore b/charts/airlock/microgateway-cni/4.4.3/.helmignore new file mode 100644 index 0000000000..8561d28926 --- /dev/null +++ b/charts/airlock/microgateway-cni/4.4.3/.helmignore @@ -0,0 +1,27 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ + +# Helm unit tests +/tests +/validation diff --git a/charts/airlock/microgateway-cni/4.4.3/Chart.yaml b/charts/airlock/microgateway-cni/4.4.3/Chart.yaml new file mode 100644 index 0000000000..a447b27850 --- /dev/null +++ b/charts/airlock/microgateway-cni/4.4.3/Chart.yaml @@ -0,0 +1,43 @@ +annotations: + artifacthub.io/category: security + artifacthub.io/license: MIT + artifacthub.io/links: | + - name: Airlock Microgateway Documentation + url: https://docs.airlock.com/microgateway/4.4/ + - name: Airlock Microgateway Labs + url: https://play.instruqt.com/airlock/invite/hyi9fy4b4jzc?icp_referrer=artifacthub.io + - name: Airlock Microgateway Forum + url: https://forum.airlock.com/ + catalog.cattle.io/certified: partner + catalog.cattle.io/display-name: Airlock Microgateway CNI + catalog.cattle.io/kube-version: '>=1.25.0-0' + catalog.cattle.io/release-name: "" + charts.openshift.io/name: Airlock Microgateway CNI +apiVersion: v2 +appVersion: 4.4.3 +description: A Helm chart for deploying the Airlock Microgateway CNI plugin +home: https://www.airlock.com/en/microgateway +icon: file://assets/icons/microgateway-cni.svg +keywords: +- WAF +- Web Application Firewall +- WAAP +- Web Application and API protection +- OWASP +- Airlock +- Microgateway +- Security +- Filtering +- DevSecOps +- shift left +- CNI +kubeVersion: '>=1.25.0-0' +maintainers: +- email: support@airlock.com + name: Airlock + url: https://www.airlock.com/ +name: microgateway-cni +sources: +- https://github.com/airlock/microgateway +type: application +version: 4.4.3 diff --git a/charts/airlock/microgateway-cni/4.4.3/README.md b/charts/airlock/microgateway-cni/4.4.3/README.md new file mode 100644 index 0000000000..a1ceb88106 --- /dev/null +++ b/charts/airlock/microgateway-cni/4.4.3/README.md @@ -0,0 +1,138 @@ +# Airlock Microgateway CNI + +![Version: 4.4.3](https://img.shields.io/badge/Version-4.4.3-informational?style=flat-square) ![AppVersion: 4.4.3](https://img.shields.io/badge/AppVersion-4.4.3-informational?style=flat-square) + +*Airlock Microgateway is a Kubernetes native WAAP (Web Application and API Protection) solution to protect microservices.* + + + + + Microgateway + + +Modern application security is embedded in the development workflow and follows DevSecOps paradigms. Airlock Microgateway is the perfect fit for these requirements. It is a lightweight alternative to the Airlock Gateway appliance, optimized for Kubernetes environments. Airlock Microgateway protects your applications and microservices with the tried-and-tested Airlock security features against attacks, while also providing a high degree of scalability. +__This Helm chart is part of Airlock Microgateway. See our [GitHub repo](https://github.com/airlock/microgateway/tree/4.4.3).__ + +### Features +* Kubernetes native integration with sidecar injection and Gateway API support +* Reverse proxy functionality with request routing rules, TLS termination and remote IP extraction +* Using native Envoy HTTP filters like Lua scripting, RBAC, ext_authz, JWT authentication +* Content security filters for protecting against known attacks (OWASP Top 10) +* Access control using OpenID Connect to allow only authenticated users to access the protected services +* API security features like JSON parsing, OpenAPI specification enforcement or GraphQL schema validation + +For a list of all features, view the **[comparison of the community and premium edition](https://docs.airlock.com/microgateway/latest/#data/1675772882054.html)**. + +## Documentation and links + +Check the official documentation at **[docs.airlock.com](https://docs.airlock.com/microgateway/latest/)** or the product website at **[airlock.com/microgateway](https://www.airlock.com/en/microgateway)**. The links below point out the most interesting documentation sites when starting with Airlock Microgateway. + +* [Getting Started](https://docs.airlock.com/microgateway/latest/#data/1660804708742.html) +* [System Architecture](https://docs.airlock.com/microgateway/latest/#data/1660804709650.html) +* [Installation](https://docs.airlock.com/microgateway/latest/?topic=MGW-00000138) +* [Troubleshooting](https://docs.airlock.com/microgateway/latest/#data/1659430054787.html) +* [GitHub](https://github.com/airlock/microgateway) + +# Quick start guide + +The instructions below provide a quick start guide. Detailed information are provided in the **[manual](https://docs.airlock.com/microgateway/latest/)**. + +## Prerequisites +* [helm](https://helm.sh/docs/intro/install/) (>= v3.8.0) + +## Deploy Airlock Microgateway CNI +1. Install the CNI Plugin with Helm. + > **Note**: Certain environments such as OpenShift or GKE require non-default configurations when installing the CNI plugin. For the most common setups, values files are provided in the [chart folder](/deploy/charts/airlock-microgateway-cni). + ```bash + # Standard setup + helm install airlock-microgateway-cni -n kube-system oci://quay.io/airlockcharts/microgateway-cni --version '4.4.3' + kubectl -n kube-system rollout status daemonset -l app.kubernetes.io/instance=airlock-microgateway-cni + ``` + ```bash + # GKE setup + helm install airlock-microgateway-cni -n kube-system oci://quay.io/airlockcharts/microgateway-cni --version '4.4.3' -f https://raw.githubusercontent.com/airlock/microgateway/4.4.3/deploy/charts/airlock-microgateway-cni/gke-values.yaml + kubectl -n kube-system rollout status daemonset -l app.kubernetes.io/instance=airlock-microgateway-cni + ``` + ```bash + # OpenShift setup + helm install airlock-microgateway-cni -n openshift-operators oci://quay.io/airlockcharts/microgateway-cni --version '4.4.3' -f https://raw.githubusercontent.com/airlock/microgateway/4.4.3/deploy/charts/airlock-microgateway-cni/openshift-values.yaml + kubectl -n openshift-operators rollout status daemonset -l app.kubernetes.io/instance=airlock-microgateway-cni + ``` + > **Important:** On OpenShift, all pods which should be protected by Airlock Microgateway must explicitly reference the Airlock Microgateway CNI NetworkAttachmentDefinition via the annotation `k8s.v1.cni.cncf.io/networks` (see [documentation](https://docs.airlock.com/microgateway/latest/#data/1658483168033.html) for details). + +2. (Recommended) You can verify the correctness of the installation with `helm test`. + ```bash + # Standard and GKE setup + helm upgrade airlock-microgateway-cni -n kube-system --set tests.enabled=true --reuse-values oci://quay.io/airlockcharts/microgateway-cni --version '4.4.3' + helm test airlock-microgateway-cni -n kube-system --logs + helm upgrade airlock-microgateway-cni -n kube-system --set tests.enabled=false --reuse-values oci://quay.io/airlockcharts/microgateway-cni --version '4.4.3' + ``` + ```bash + # OpenShift setup + helm upgrade airlock-microgateway-cni -n openshift-operators --set tests.enabled=true --reuse-values oci://quay.io/airlockcharts/microgateway-cni --version '4.4.3' + helm test airlock-microgateway-cni -n openshift-operators --logs + helm upgrade airlock-microgateway-cni -n openshift-operators --set tests.enabled=false --reuse-values oci://quay.io/airlockcharts/microgateway-cni --version '4.4.3' + ``` + + Consult our [documentation](https://docs.airlock.com/microgateway/latest/?topic=MGW-00000139) in case of any installation error. + +## Support + +### Premium support +If you have a paid license, please follow the [premium support process](https://techzone.ergon.ch/support-process). + +### Community support +For the community edition, check our **[Airlock community forum](https://forum.airlock.com/)** for FAQs or register to post your question. +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| affinity | object | `{}` | Custom affinity for the DaemonSet to only deploy the CNI plugin on specific nodes. | +| commonAnnotations | object | `{}` | Annotations to add to all resources. | +| commonLabels | object | `{}` | Labels to add to all resources. | +| config.cniBinDir | string | `"/opt/cni/bin"` | Directory where the CNI plugin binaries reside on the host. This path can either be found in the documentation of your Kubernetes distribution or CNI provider. It can also be queried by running the command `crictl info -o go-template --template '{{.config.cni.binDir}}'` on your Kubernetes node. | +| config.cniNetDir | string | `"/etc/cni/net.d"` | Directory where the CNI config files reside on the host. This path can either be found in the documentation of your Kubernetes distribution or CNI provider. It can also be queried by running the command `crictl info -o go-template --template '{{.config.cni.confDir}}'` on your Kubernetes node. | +| config.excludeNamespaces | list | `["kube-system"]` | Namespaces for which this CNI plugin should not apply any modifications. | +| config.installMode | string | `"chained"` | Whether to install the CNI plugin as a `chained` plugin (default, required with most interface CNI providers), as a `standalone` plugin (required for use with Multus CNI, e.g. on OpenShift) or in `manual` mode, where no CNI network configuration is written. | +| config.logLevel | string | `"info"` | Log level for the CNI installer and plugin. | +| config.repairMode | string | `"none"` | Specifies the repair mode There is a race condition regarding the installation of the CNI Plugin and creation of Pods when starting a Node. This would cause Pods to be unprotected, because the CNI did not reconfigure the Pod's network. The Airlock Microgateway Network Validator prevents this and causes the Pod to fail on purpose. Pods can be repaired by choosing the appropriate repair mode. Available options are: `deletePods` will delete failing Pods, such that the CNI Plugin can correctly configure them `none` will not perform any action for failing Pods | +| fullnameOverride | string | `""` | Allows overriding the name to use as full name of resources. | +| image.digest | string | `"sha256:8398d2f2c2b57aecd2f53e165789ef0b229a9ea55e3b87ccb6e08e36c06f47c6"` | SHA256 image digest to pull (in the format "sha256:7144f7bab3d4c2648d7e59409f15ec52a18006a128c733fcff20d3a4a54ba44a"). Overrides tag when specified. | +| image.pullPolicy | string | `"IfNotPresent"` | Pull policy for this image. | +| image.repository | string | `"quay.io/airlock/microgateway-cni"` | Image repository from which to pull the Airlock Microgateway CNI image. | +| image.tag | string | `"4.4.3"` | Image tag to pull. | +| imagePullSecrets | list | `[]` | ImagePullSecrets to use when pulling images. | +| multusNetworkAttachmentDefinition.create | bool | `false` | Whether a NetworkAttachmentDefinition CR should be created, which can be used for applying the CNI plugin to Pods. | +| multusNetworkAttachmentDefinition.namespace | string | `"default"` | Namespace in which the NetworkAttachmentDefinition is deployed. Note: If namespace is set to a custom value, referencing the created NetworkAttachmentDefinition from other namespaces may not work if Multus namespace isolation is enabled. https://github.com/k8snetworkplumbingwg/multus-cni/blob/v4.0.2/docs/configuration.md#namespace-isolation | +| nameOverride | string | `""` | Allows overriding the name to use instead of "microgateway-cni". | +| nodeSelector | object | `{"kubernetes.io/os":"linux"}` | NodeSelector to apply to the CNI DaemonSet in order to only deploy the CNI plugin on specific nodes. | +| podAnnotations | object | `{}` | Annotations to add to all Pods. | +| podLabels | object | `{}` | Labels to add to all Pods. | +| privileged | bool | `false` | Whether the DaemonSet should run in privileged mode. Must be enabled for environments which require it for writing files to the host (e.g. OpenShift). | +| rbac.create | bool | `true` | Whether to create RBAC resources which are required for the CNI plugin to function. | +| rbac.createSCCRole | OpenShift | `false` | Whether to create RBAC resources which allow the CNI installer to use the "privileged" security context constraint. | +| resources | object | `{"requests":{"cpu":"10m","memory":"100Mi"}}` | Resource restrictions to apply to the CNI installer container. | +| serviceAccount.annotations | object | `{}` | Annotations to add to the ServiceAccount. | +| serviceAccount.create | bool | `true` | Whether a ServiceAccount should be created. | +| serviceAccount.name | string | `""` | Name of the ServiceAccount to use. If not set and create is true, a name is generated using the fullname template. | +| tests.enabled | bool | `false` | Whether additional resources required for running `helm test` should be created (e.g. Roles and ServiceAccounts). If set to false, `helm test` will not run any tests. | + +## License +View the [detailed license terms](https://www.airlock.com/en/airlock-license) for the software contained in this image. +* Decompiling or reverse engineering is not permitted. +* Using any of the deny rules or parts of these filter patterns outside of the image is not permitted. + +Airlock® is a security innovation by [ergon](https://www.ergon.ch/en) + + + + + + + Airlock Secure Access Hub + + diff --git a/charts/airlock/microgateway/4.3.3/gke-values.yaml b/charts/airlock/microgateway-cni/4.4.3/gke-values.yaml similarity index 100% rename from charts/airlock/microgateway/4.3.3/gke-values.yaml rename to charts/airlock/microgateway-cni/4.4.3/gke-values.yaml diff --git a/charts/airlock/microgateway/4.3.3/openshift-values.yaml b/charts/airlock/microgateway-cni/4.4.3/openshift-values.yaml similarity index 100% rename from charts/airlock/microgateway/4.3.3/openshift-values.yaml rename to charts/airlock/microgateway-cni/4.4.3/openshift-values.yaml diff --git a/charts/airlock/microgateway/4.3.3/questions.yml b/charts/airlock/microgateway-cni/4.4.3/questions.yml similarity index 100% rename from charts/airlock/microgateway/4.3.3/questions.yml rename to charts/airlock/microgateway-cni/4.4.3/questions.yml diff --git a/charts/airlock/microgateway-cni/4.4.3/templates/NOTES.txt b/charts/airlock/microgateway-cni/4.4.3/templates/NOTES.txt new file mode 100644 index 0000000000..bb94ff521e --- /dev/null +++ b/charts/airlock/microgateway-cni/4.4.3/templates/NOTES.txt @@ -0,0 +1,15 @@ +Thank you for installing Airlock Microgateway CNI. + +Please ensure that the helm values'.config.cniNetDir' and '.config.cniBinDir' are configured for your Kubernetes distribution. +For further information, consider our manual https://docs.airlock.com/microgateway/{{ include "airlock-microgateway-cni.docsVersion" . }}. +The chapter 'Setup > Installation' describes how to set those settings correctly. + +Further information: +* Documentation: https://docs.airlock.com/microgateway/{{ include "airlock-microgateway-cni.docsVersion" . }} +* Airlock Microgateway Labs: https://play.instruqt.com/airlock/invite/hyi9fy4b4jzc?icp_referrer=helm + +Next steps: +* Install Airlock Microgateway (if not done already) + https://artifacthub.io/packages/helm/airlock-microgateway/microgateway + +Your release version is {{ .Chart.Version }}. \ No newline at end of file diff --git a/charts/airlock/microgateway-cni/4.4.3/templates/_helpers.tpl b/charts/airlock/microgateway-cni/4.4.3/templates/_helpers.tpl new file mode 100644 index 0000000000..996491a873 --- /dev/null +++ b/charts/airlock/microgateway-cni/4.4.3/templates/_helpers.tpl @@ -0,0 +1,101 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "airlock-microgateway-cni.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Convert an image configuration object into an image ref string. +*/}} +{{- define "airlock-microgateway-cni.image" -}} + {{- if .digest -}} + {{- printf "%s@%s" .repository .digest -}} + {{- else if .tag -}} + {{- printf "%s:%s" .repository .tag -}} + {{- else -}} + {{- printf "%s" .repository -}} + {{- end -}} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 50 chars because some Kubernetes name fields are limited to 63 chars (by the DNS naming spec) +and the longest suffix is 13 characters. +If release name contains chart name it will be used as a full name. +*/}} +{{- define "airlock-microgateway-cni.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 50 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 50 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 50 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "airlock-microgateway-cni.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "airlock-microgateway-cni.labels" -}} +helm.sh/chart: {{ include "airlock-microgateway-cni.chart" . }} +{{ include "airlock-microgateway-cni.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- with .Values.commonLabels }} +{{ toYaml .}} +{{- end }} +{{- end }} + +{{/* +Common labels without component +*/}} +{{- define "airlock-microgateway-cni.labelsWithoutComponent" -}} +{{- $labels := fromYaml (include "airlock-microgateway-cni.labels" .) -}} +{{ unset $labels "app.kubernetes.io/component" | toYaml }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "airlock-microgateway-cni.selectorLabels" -}} +app.kubernetes.io/component: cni-plugin-installer +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/name: {{ include "airlock-microgateway-cni.name" . }} +{{- end }} + +{{/* +Create the name of the service account to use for the CNI Plugin +*/}} +{{- define "airlock-microgateway-cni.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "airlock-microgateway-cni.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{- define "airlock-microgateway-cni.isSemver" -}} +{{- regexMatch `^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$` . -}} +{{- end -}} + +{{- define "airlock-microgateway-cni.docsVersion" -}} +{{- if and (eq "true" (include "airlock-microgateway-cni.isSemver" .Chart.AppVersion)) (not (contains "-" .Chart.AppVersion)) -}} + {{- $version := (semver .Chart.AppVersion) -}} + {{- $version.Major }}.{{ $version.Minor -}} +{{- else -}} + {{- print "latest" -}} +{{- end -}} +{{- end -}} diff --git a/charts/airlock/microgateway-cni/4.4.3/templates/clusterrole.yaml b/charts/airlock/microgateway-cni/4.4.3/templates/clusterrole.yaml new file mode 100644 index 0000000000..7412ab5135 --- /dev/null +++ b/charts/airlock/microgateway-cni/4.4.3/templates/clusterrole.yaml @@ -0,0 +1,25 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "airlock-microgateway-cni.fullname" . }} + labels: + {{- include "airlock-microgateway-cni.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch + - patch + {{- if eq .Values.config.repairMode "deletePods" }} + - delete + {{- end }} +{{- end -}} diff --git a/charts/airlock/microgateway/4.3.3/templates/clusterrolebinding.yaml b/charts/airlock/microgateway-cni/4.4.3/templates/clusterrolebinding.yaml similarity index 100% rename from charts/airlock/microgateway/4.3.3/templates/clusterrolebinding.yaml rename to charts/airlock/microgateway-cni/4.4.3/templates/clusterrolebinding.yaml diff --git a/charts/airlock/microgateway/4.3.3/templates/configmap.yaml b/charts/airlock/microgateway-cni/4.4.3/templates/configmap.yaml similarity index 100% rename from charts/airlock/microgateway/4.3.3/templates/configmap.yaml rename to charts/airlock/microgateway-cni/4.4.3/templates/configmap.yaml diff --git a/charts/airlock/microgateway-cni/4.4.3/templates/daemonset.yaml b/charts/airlock/microgateway-cni/4.4.3/templates/daemonset.yaml new file mode 100644 index 0000000000..fcb5846ca8 --- /dev/null +++ b/charts/airlock/microgateway-cni/4.4.3/templates/daemonset.yaml @@ -0,0 +1,138 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "airlock-microgateway-cni.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "airlock-microgateway-cni.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "airlock-microgateway-cni.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + kubectl.kubernetes.io/default-container: cni-installer + {{- with mustMerge .Values.podAnnotations .Values.commonAnnotations}} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "airlock-microgateway-cni.labels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - args: + - --log-level + - "{{ .Values.config.logLevel }}" + env: + - name: CNI_NETWORK_CONFIG + valueFrom: + configMapKeyRef: + key: plugin-conf.json + name: {{ include "airlock-microgateway-cni.fullname" . }} + - name: CNI_BIN_DIR + value: /host/opt/cni/bin + - name: CNI_NET_DIR + value: /host/etc/cni/net.d + - name: KUBECONFIG_FILE_NAME + value: "{{ include "airlock-microgateway-cni.fullname" . }}-kubeconfig" + - name: INSTALL_MODE + value: {{ .Values.config.installMode }} + - name: REPAIR_MODE + value: {{ .Values.config.repairMode }} + - name: KUBERNETES_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + image: {{ include "airlock-microgateway-cni.image" .Values.image }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + name: cni-installer + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + startupProbe: + exec: + command: + - /cni-installer + - probe + failureThreshold: 5 + initialDelaySeconds: 3 + periodSeconds: 3 + timeoutSeconds: 3 + readinessProbe: + exec: + command: + - /cni-installer + - probe + failureThreshold: 1 + periodSeconds: 60 + timeoutSeconds: 3 + securityContext: + allowPrivilegeEscalation: {{ .Values.privileged }} + capabilities: + drop: + - ALL + privileged: {{ .Values.privileged }} + readOnlyRootFilesystem: true + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + seccompProfile: + type: RuntimeDefault + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + - mountPath: /host/etc/cni/net.d + name: cni-net-dir + - mountPath: /run/cni-installer + name: cni-installer-status + hostNetwork: true + priorityClassName: system-node-critical + restartPolicy: Always + securityContext: + fsGroup: 0 + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + serviceAccountName: {{ include "airlock-microgateway-cni.serviceAccountName" . }} + terminationGracePeriodSeconds: 5 + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + tolerations: + - effect: NoSchedule + operator: Exists + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + volumes: + - hostPath: + path: "{{ .Values.config.cniBinDir }}" + type: Directory + name: cni-bin-dir + - hostPath: + path: "{{ .Values.config.cniNetDir }}" + type: Directory + name: cni-net-dir + - emptyDir: {} + name: cni-installer-status diff --git a/charts/airlock/microgateway/4.3.3/templates/network-attachment-definition.yaml b/charts/airlock/microgateway-cni/4.4.3/templates/network-attachment-definition.yaml similarity index 100% rename from charts/airlock/microgateway/4.3.3/templates/network-attachment-definition.yaml rename to charts/airlock/microgateway-cni/4.4.3/templates/network-attachment-definition.yaml diff --git a/charts/airlock/microgateway/4.3.3/templates/scc-role.yaml b/charts/airlock/microgateway-cni/4.4.3/templates/scc-role.yaml similarity index 100% rename from charts/airlock/microgateway/4.3.3/templates/scc-role.yaml rename to charts/airlock/microgateway-cni/4.4.3/templates/scc-role.yaml diff --git a/charts/airlock/microgateway/4.3.3/templates/scc-rolebinding.yaml b/charts/airlock/microgateway-cni/4.4.3/templates/scc-rolebinding.yaml similarity index 100% rename from charts/airlock/microgateway/4.3.3/templates/scc-rolebinding.yaml rename to charts/airlock/microgateway-cni/4.4.3/templates/scc-rolebinding.yaml diff --git a/charts/airlock/microgateway/4.3.3/templates/serviceaccount.yaml b/charts/airlock/microgateway-cni/4.4.3/templates/serviceaccount.yaml similarity index 100% rename from charts/airlock/microgateway/4.3.3/templates/serviceaccount.yaml rename to charts/airlock/microgateway-cni/4.4.3/templates/serviceaccount.yaml diff --git a/charts/airlock/microgateway-cni/4.4.3/templates/tests/rbac.yaml b/charts/airlock/microgateway-cni/4.4.3/templates/tests/rbac.yaml new file mode 100644 index 0000000000..744799333f --- /dev/null +++ b/charts/airlock/microgateway-cni/4.4.3/templates/tests/rbac.yaml @@ -0,0 +1,64 @@ +{{- if .Values.tests.enabled -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "{{ include "airlock-microgateway-cni.fullname" . }}-tests" + namespace: {{ .Release.Namespace }} + labels: + {{- include "airlock-microgateway-cni.labelsWithoutComponent" . | nindent 4 }} + app.kubernetes.io/component: tests +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: "{{ include "airlock-microgateway-cni.fullname" . }}-tests" + namespace: {{ .Release.Namespace }} + labels: + {{- include "airlock-microgateway-cni.labelsWithoutComponent" . | nindent 4 }} + app.kubernetes.io/component: tests +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: "{{ include "airlock-microgateway-cni.fullname" . }}-tests" +subjects: +- kind: ServiceAccount + name: "{{ include "airlock-microgateway-cni.fullname" . }}-tests" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: "{{ include "airlock-microgateway-cni.fullname" . }}-tests" + namespace: {{ .Release.Namespace }} + labels: + {{- include "airlock-microgateway-cni.labelsWithoutComponent" . | nindent 4 }} + app.kubernetes.io/component: tests +rules: +- apiGroups: + - "apps" + resources: + - daemonsets + resourceNames: + - {{ include "airlock-microgateway-cni.fullname" . }} + verbs: + - get + - watch + - list +- apiGroups: + - "" + resources: + - pods + - pods/log + verbs: + - get + - list +{{- if .Values.rbac.createSCCRole }} +- apiGroups: + - security.openshift.io + resourceNames: + - privileged + resources: + - securitycontextconstraints + verbs: + - use +{{- end -}} +{{- end -}} diff --git a/charts/airlock/microgateway-cni/4.4.3/templates/tests/test-install.yaml b/charts/airlock/microgateway-cni/4.4.3/templates/tests/test-install.yaml new file mode 100644 index 0000000000..12d8c8de78 --- /dev/null +++ b/charts/airlock/microgateway-cni/4.4.3/templates/tests/test-install.yaml @@ -0,0 +1,103 @@ +{{- if .Values.tests.enabled -}} +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "airlock-microgateway-cni.fullname" . }}-test-install" + namespace: {{ .Release.Namespace }} + labels: + {{- include "airlock-microgateway-cni.labelsWithoutComponent" . | nindent 4 }} + app.kubernetes.io/component: test-install + annotations: + helm.sh/hook: test + helm.sh/hook-delete-policy: before-hook-creation +spec: + restartPolicy: Never + containers: + - name: test + image: "bitnami/kubectl:{{ .Capabilities.KubeVersion.Major }}.{{ .Capabilities.KubeVersion.Minor }}" + securityContext: + allowPrivilegeEscalation: {{ .Values.privileged }} + capabilities: + drop: + - ALL + privileged: {{ .Values.privileged }} + readOnlyRootFilesystem: true + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + readOnly: true + - mountPath: /host/etc/cni/net.d + name: cni-net-dir + readOnly: true + command: + - sh + - -c + - | + set -eu + + fail() { + echo "Error: ${1}" + echo "" + echo 'CNI installer logs:' + kubectl logs -n {{ .Release.Namespace }} daemonsets/{{ include "airlock-microgateway-cni.fullname" .}} -c cni-installer + exit 1 + } + + containsMGWCNIConf() { + cat "${1}" | grep -qe '"type":.*"{{ include "airlock-microgateway-cni.fullname" . }}"' + } + + if ! kubectl rollout status --timeout=60s -n {{ .Release.Namespace }} daemonsets/{{ include "airlock-microgateway-cni.fullname" .}}; then + fail 'CNI DaemonSet rollout did not complete within timeout' + fi + + echo "Checking whether CNI binary was installed" + if ! [ -f "/host/opt/cni/bin/{{ include "airlock-microgateway-cni.fullname" . }}" ]; then + fail 'CNI binary was not installed' + fi + + echo "Checking whether CNI kubeconfig was installed" + if ! [ -f "/host/etc/cni/net.d/{{ include "airlock-microgateway-cni.fullname" . }}-kubeconfig" ]; then + fail 'CNI kubeconfig was not created' + fi + + echo "Checking whether CNI configuration was written" + case {{ .Values.config.installMode }} in + "chained") + for file in "/host/etc/cni/net.d/"*.conflist; do + if containsMGWCNIConf "${file}"; then + echo "Success" + exit 0 + fi + done + ;; + "standalone") + if containsMGWCNIConf "/host/etc/cni/net.d/{{ include "airlock-microgateway-cni.fullname" . }}.conflist"; then + echo "Success" + exit 0 + fi + ;; + "manual") + echo "- Skipping because we are in 'manual' install mode" + echo "Success" + exit 0 + ;; + esac + + fail 'Configuration for plugin "{{ include "airlock-microgateway-cni.fullname" . }}" was not found' + serviceAccountName: "{{ include "airlock-microgateway-cni.fullname" . }}-tests" + volumes: + - hostPath: + path: "{{ .Values.config.cniBinDir }}" + type: Directory + name: cni-bin-dir + - hostPath: + path: "{{ .Values.config.cniNetDir }}" + type: Directory + name: cni-net-dir +{{- end -}} diff --git a/charts/airlock/microgateway-cni/4.4.3/values.schema.json b/charts/airlock/microgateway-cni/4.4.3/values.schema.json new file mode 100644 index 0000000000..c2cf207348 --- /dev/null +++ b/charts/airlock/microgateway-cni/4.4.3/values.schema.json @@ -0,0 +1,233 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nameOverride": { + "type": "string" + }, + "fullnameOverride": { + "type": "string" + }, + "commonLabels": { + "$ref": "#/definitions/StringMap" + }, + "commonAnnotations": { + "$ref": "#/definitions/StringMap" + }, + "imagePullSecrets": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "name" + ], + "additionalProperties": true + } + }, + "image": { + "$ref": "#/definitions/Image" + }, + "podAnnotations": { + "$ref": "#/definitions/StringMap" + }, + "podLabels": { + "$ref": "#/definitions/StringMap" + }, + "resources": { + "type": "object" + }, + "nodeSelector": { + "$ref": "#/definitions/StringMap" + }, + "affinity": { + "type": "object" + }, + "rbac": { + "type": "object", + "properties": { + "create": { + "type": "boolean" + }, + "createSCCRole": { + "type": "boolean" + } + }, + "required": [ + "create", + "createSCCRole" + ], + "additionalProperties": false + }, + "privileged": { + "type": "boolean" + }, + "serviceAccount": { + "type": "object", + "properties": { + "create": { + "type": "boolean" + }, + "annotations": { + "$ref": "#/definitions/StringMap" + }, + "name": { + "type": "string" + } + }, + "required": [ + "annotations", + "create", + "name" + ], + "additionalProperties": false + }, + "multusNetworkAttachmentDefinition": { + "type": "object", + "properties": { + "create": { + "type": "boolean" + }, + "namespace": { + "type": "string" + } + }, + "required": [ + "create", + "namespace" + ], + "additionalProperties": false + }, + "config": { + "type": "object", + "properties": { + "installMode": { + "type": "string", + "enum": [ + "chained", + "standalone", + "manual" + ] + }, + "repairMode": { + "type": "string", + "enum": [ + "deletePods", + "none" + ] + }, + "logLevel": { + "type": "string", + "enum": [ + "debug", + "info", + "warn", + "error" + ] + }, + "cniNetDir": { + "type": "string", + "minLength": 1 + }, + "cniBinDir": { + "type": "string", + "minLength": 1 + }, + "excludeNamespaces": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "cniBinDir", + "cniNetDir", + "excludeNamespaces", + "installMode", + "repairMode", + "logLevel" + ], + "additionalProperties": false + }, + "tests": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + } + }, + "required": [ + "enabled" + ], + "additionalProperties": false + }, + "global": { + "type": "object" + } + }, + "required": [ + "affinity", + "commonAnnotations", + "commonLabels", + "config", + "fullnameOverride", + "image", + "imagePullSecrets", + "multusNetworkAttachmentDefinition", + "nameOverride", + "nodeSelector", + "podAnnotations", + "podLabels", + "privileged", + "rbac", + "resources", + "serviceAccount", + "tests" + ], + "additionalProperties": false, + "definitions": { + "StringMap": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "Image": { + "type": "object", + "properties": { + "repository": { + "type": "string", + "minLength": 1 + }, + "tag": { + "type": "string" + }, + "digest": { + "type": "string", + "pattern": "^$|^sha256:[a-f0-9]{64}$" + }, + "pullPolicy": { + "type": "string", + "enum": [ + "Always", + "IfNotPresent", + "Never" + ] + } + }, + "required": [ + "digest", + "pullPolicy", + "repository", + "tag" + ], + "additionalProperties": false + } + } +} diff --git a/charts/airlock/microgateway-cni/4.4.3/values.yaml b/charts/airlock/microgateway-cni/4.4.3/values.yaml new file mode 100644 index 0000000000..08d4a1e3a8 --- /dev/null +++ b/charts/airlock/microgateway-cni/4.4.3/values.yaml @@ -0,0 +1,94 @@ +# -- Allows overriding the name to use instead of "microgateway-cni". +nameOverride: "" +# -- Allows overriding the name to use as full name of resources. +fullnameOverride: "" +# -- Labels to add to all resources. +commonLabels: {} +# -- Annotations to add to all resources. +commonAnnotations: {} +# -- ImagePullSecrets to use when pulling images. +imagePullSecrets: [] +# - name: myRegistryKeySecretName + +# Specifies the Airlock Microgateway CNI image. +image: + # -- Image repository from which to pull the Airlock Microgateway CNI image. + repository: "quay.io/airlock/microgateway-cni" + # -- Image tag to pull. + tag: "4.4.3" + # -- SHA256 image digest to pull (in the format "sha256:7144f7bab3d4c2648d7e59409f15ec52a18006a128c733fcff20d3a4a54ba44a"). + # Overrides tag when specified. + digest: "sha256:8398d2f2c2b57aecd2f53e165789ef0b229a9ea55e3b87ccb6e08e36c06f47c6" + # -- Pull policy for this image. + pullPolicy: IfNotPresent +# -- Annotations to add to all Pods. +podAnnotations: {} +# -- Labels to add to all Pods. +podLabels: {} +# -- Resource restrictions to apply to the CNI installer container. +resources: + requests: + cpu: 10m + memory: 100Mi +# -- NodeSelector to apply to the CNI DaemonSet in order to only deploy the CNI plugin on specific nodes. +nodeSelector: + kubernetes.io/os: linux +# -- Custom affinity for the DaemonSet to only deploy the CNI plugin on specific nodes. +affinity: {} +# Configures the generation of RBAC Roles and RoleBindings. +rbac: + # -- Whether to create RBAC resources which are required for the CNI plugin to function. + create: true + # -- (OpenShift) Whether to create RBAC resources which allow the CNI installer to use the "privileged" security context constraint. + createSCCRole: false +# -- Whether the DaemonSet should run in privileged mode. Must be enabled for environments which require it for writing files to the host (e.g. OpenShift). +privileged: false +# Configures the generation of the ServiceAccount. +serviceAccount: + # -- Whether a ServiceAccount should be created. + create: true + # -- Annotations to add to the ServiceAccount. + annotations: {} + # -- Name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template. + name: "" +# Configures the generation of a NetworkAttachmentDefinition for use with Multus CNI (OpenShift) +multusNetworkAttachmentDefinition: + # -- Whether a NetworkAttachmentDefinition CR should be created, which can be used for applying the CNI plugin to Pods. + create: false + # -- Namespace in which the NetworkAttachmentDefinition is deployed. + # Note: If namespace is set to a custom value, referencing the created NetworkAttachmentDefinition from other namespaces + # may not work if Multus namespace isolation is enabled. https://github.com/k8snetworkplumbingwg/multus-cni/blob/v4.0.2/docs/configuration.md#namespace-isolation + namespace: default +# Parameters for the CNI installer configuration. +config: + # -- Whether to install the CNI plugin as a `chained` plugin (default, required with most interface CNI providers), + # as a `standalone` plugin (required for use with Multus CNI, e.g. on OpenShift) + # or in `manual` mode, where no CNI network configuration is written. + installMode: "chained" + # -- Specifies the repair mode + # There is a race condition regarding the installation of the CNI Plugin and creation of Pods when starting a Node. + # This would cause Pods to be unprotected, because the CNI did not reconfigure the Pod's network. + # The Airlock Microgateway Network Validator prevents this and causes the Pod to fail on purpose. + # Pods can be repaired by choosing the appropriate repair mode. + # Available options are: + # `deletePods` will delete failing Pods, such that the CNI Plugin can correctly configure them + # `none` will not perform any action for failing Pods + repairMode: "none" + # -- Log level for the CNI installer and plugin. + logLevel: info + # -- Directory where the CNI config files reside on the host. + # This path can either be found in the documentation of your Kubernetes distribution or CNI provider. + # It can also be queried by running the command `crictl info -o go-template --template '{{.config.cni.confDir}}'` on your Kubernetes node. + cniNetDir: "/etc/cni/net.d" + # -- Directory where the CNI plugin binaries reside on the host. + # This path can either be found in the documentation of your Kubernetes distribution or CNI provider. + # It can also be queried by running the command `crictl info -o go-template --template '{{.config.cni.binDir}}'` on your Kubernetes node. + cniBinDir: "/opt/cni/bin" + # -- Namespaces for which this CNI plugin should not apply any modifications. + excludeNamespaces: + - kube-system +tests: + # -- Whether additional resources required for running `helm test` should be created (e.g. Roles and ServiceAccounts). + # If set to false, `helm test` will not run any tests. + enabled: false diff --git a/charts/airlock/microgateway/4.3.3/.helmignore b/charts/airlock/microgateway/4.3.3/.helmignore index 8561d28926..101ff5ac56 100644 --- a/charts/airlock/microgateway/4.3.3/.helmignore +++ b/charts/airlock/microgateway/4.3.3/.helmignore @@ -21,7 +21,8 @@ .idea/ *.tmproj .vscode/ - +# CRDs kustomization.yaml +/crds/kustomization.yaml # Helm unit tests /tests /validation diff --git a/charts/airlock/microgateway/4.3.3/Chart.yaml b/charts/airlock/microgateway/4.3.3/Chart.yaml index f22c19bb68..c168f9d775 100644 --- a/charts/airlock/microgateway/4.3.3/Chart.yaml +++ b/charts/airlock/microgateway/4.3.3/Chart.yaml @@ -9,15 +9,15 @@ annotations: - name: Airlock Microgateway Forum url: https://forum.airlock.com/ catalog.cattle.io/certified: partner - catalog.cattle.io/display-name: Airlock Microgateway CNI + catalog.cattle.io/display-name: Airlock Microgateway catalog.cattle.io/kube-version: '>=1.25.0-0' - catalog.cattle.io/release-name: microgateway-cni - charts.openshift.io/name: Airlock Microgateway CNI + catalog.cattle.io/release-name: microgateway + charts.openshift.io/name: Airlock Microgateway apiVersion: v2 appVersion: 4.3.3 -description: A Helm chart for deploying the Airlock Microgateway CNI plugin +description: A Helm chart for deploying the Airlock Microgateway home: https://www.airlock.com/en/microgateway -icon: file://assets/icons/microgateway-cni.svg +icon: file://assets/icons/microgateway.svg keywords: - WAF - Web Application Firewall @@ -30,13 +30,14 @@ keywords: - Filtering - DevSecOps - shift left -- CNI +- control plane +- Operator kubeVersion: '>=1.25.0-0' maintainers: - email: support@airlock.com name: Airlock url: https://www.airlock.com/ -name: microgateway-cni +name: microgateway sources: - https://github.com/airlock/microgateway type: application diff --git a/charts/airlock/microgateway/4.3.3/README.md b/charts/airlock/microgateway/4.3.3/README.md index 685c4f1f84..c98085da19 100644 --- a/charts/airlock/microgateway/4.3.3/README.md +++ b/charts/airlock/microgateway/4.3.3/README.md @@ -1,4 +1,4 @@ -# Airlock Microgateway CNI +# Airlock Microgateway ![Version: 4.3.3](https://img.shields.io/badge/Version-4.3.3-informational?style=flat-square) ![AppVersion: 4.3.3](https://img.shields.io/badge/AppVersion-4.3.3-informational?style=flat-square) @@ -40,43 +40,58 @@ Check the official documentation at **[docs.airlock.com](https://docs.airlock.co The instructions below provide a quick start guide. Detailed information are provided in the **[manual](https://docs.airlock.com/microgateway/latest/)**. ## Prerequisites +* [Airlock Microgateway CNI](https://artifacthub.io/packages/helm/airlock-microgateway-cni/microgateway-cni) +* [Airlock Microgateway License](#obtain-airlock-microgateway-license) +* [cert-manager](https://cert-manager.io/) * [helm](https://helm.sh/docs/intro/install/) (>= v3.8.0) -## Deploy Airlock Microgateway CNI -1. Install the CNI Plugin with Helm. - > **Note**: Certain environments such as OpenShift or GKE require non-default configurations when installing the CNI plugin. For the most common setups, values files are provided in the [chart folder](/deploy/charts/airlock-microgateway-cni). - ```bash - # Standard setup - helm install airlock-microgateway-cni -n kube-system oci://quay.io/airlockcharts/microgateway-cni --version '4.3.3' - kubectl -n kube-system rollout status daemonset -l app.kubernetes.io/instance=airlock-microgateway-cni - ``` - ```bash - # GKE setup - helm install airlock-microgateway-cni -n kube-system oci://quay.io/airlockcharts/microgateway-cni --version '4.3.3' -f https://raw.githubusercontent.com/airlock/microgateway/4.3.3/deploy/charts/airlock-microgateway-cni/gke-values.yaml - kubectl -n kube-system rollout status daemonset -l app.kubernetes.io/instance=airlock-microgateway-cni - ``` +In order to use Airlock Microgateway you need a license and the cert-manager. You may either request a community license free of charge or purchase a premium license. +For an easy start in non-production environments, you may deploy the same cert-manager we are using internally for testing. +### Obtain Airlock Microgateway License +1. Either request a community or premium license + * Community license: [airlock.com/microgateway-community](https://airlock.com/en/microgateway-community) + * Premium license: [airlock.com/microgateway-premium](https://airlock.com/en/microgateway-premium) +2. Check your inbox and save the license file microgateway-license.txt locally. + +> See [Community vs. Premium editions in detail](https://docs.airlock.com/microgateway/latest/#data/1675772882054.html) to choose the right license type. +### Deploy cert-manager +```bash +helm repo add jetstack https://charts.jetstack.io +helm install cert-manager jetstack/cert-manager --version '1.15.1' -n cert-manager --create-namespace --set crds.enabled=true --wait +``` + +## Deploy Airlock Microgateway Operator + +> This guide assumes a microgateway-license.txt file is present in the working directory. + +1. Install CRDs and Operator. ```bash - # OpenShift setup - helm install airlock-microgateway-cni -n openshift-operators oci://quay.io/airlockcharts/microgateway-cni --version '4.3.3' -f https://raw.githubusercontent.com/airlock/microgateway/4.3.3/deploy/charts/airlock-microgateway-cni/openshift-values.yaml - kubectl -n openshift-operators rollout status daemonset -l app.kubernetes.io/instance=airlock-microgateway-cni + # Create namespace + kubectl create namespace airlock-microgateway-system + + # Install License + kubectl -n airlock-microgateway-system create secret generic airlock-microgateway-license --from-file=microgateway-license.txt + + # Install Operator (CRDs are included via the standard Helm 3 mechanism, i.e. Helm will handle initial installation but not upgrades) + helm install airlock-microgateway -n airlock-microgateway-system oci://quay.io/airlockcharts/microgateway --version '4.3.3' --wait ``` - **Important:** On OpenShift, all pods which should be protected by Airlock Microgateway must explicitly reference the Airlock Microgateway CNI NetworkAttachmentDefinition via the annotation `k8s.v1.cni.cncf.io/networks` (see [documentation](https://docs.airlock.com/microgateway/latest/#data/1658483168033.html) for details). 2. (Recommended) You can verify the correctness of the installation with `helm test`. ```bash - # Standard and GKE setup - helm upgrade airlock-microgateway-cni -n kube-system --set tests.enabled=true --reuse-values oci://quay.io/airlockcharts/microgateway-cni --version '4.3.3' - helm test airlock-microgateway-cni -n kube-system --logs - helm upgrade airlock-microgateway-cni -n kube-system --set tests.enabled=false --reuse-values oci://quay.io/airlockcharts/microgateway-cni --version '4.3.3' - ``` - ```bash - # OpenShift setup - helm upgrade airlock-microgateway-cni -n openshift-operators --set tests.enabled=true --reuse-values oci://quay.io/airlockcharts/microgateway-cni --version '4.3.3' - helm test airlock-microgateway-cni -n openshift-operators --logs - helm upgrade airlock-microgateway-cni -n openshift-operators --set tests.enabled=false --reuse-values oci://quay.io/airlockcharts/microgateway-cni --version '4.3.3' + helm upgrade airlock-microgateway -n airlock-microgateway-system --set tests.enabled=true --reuse-values oci://quay.io/airlockcharts/microgateway --version '4.3.3' + helm test airlock-microgateway -n airlock-microgateway-system --logs + helm upgrade airlock-microgateway -n airlock-microgateway-system --set tests.enabled=false --reuse-values oci://quay.io/airlockcharts/microgateway --version '4.3.3' ``` - Consult our [documentation](https://docs.airlock.com/microgateway/latest/#data/1699611533587.html) in case of any installation error. +### Upgrading CRDs + +The `helm install/upgrade` command currently does not support upgrading CRDs that already exist in the cluster. +CRDs should instead be manually upgraded before upgrading the Operator itself via the following command: +```bash +kubectl apply -k https://github.com/airlock/microgateway/deploy/charts/airlock-microgateway/crds/?ref=4.3.3 --server-side --force-conflicts +``` + +**Note**: Certain GitOps solutions such as e.g. Argo CD or Flux CD have their own mechanisms for automatically upgrading CRDs included with Helm charts. ## Support @@ -89,33 +104,61 @@ For the community edition, check our **[Airlock community forum](https://forum.a | Key | Type | Default | Description | |-----|------|---------|-------------| -| affinity | object | `{}` | Custom affinity for the DaemonSet to only deploy the CNI plugin on specific nodes. | | commonAnnotations | object | `{}` | Annotations to add to all resources. | | commonLabels | object | `{}` | Labels to add to all resources. | -| config.cniBinDir | string | `"/opt/cni/bin"` | Directory where the CNI plugin binaries reside on the host. This path can either be found in the documentation of your Kubernetes distribution or CNI provider. It can also be queried by running the command `crictl info -o go-template --template '{{.config.cni.binDir}}'` on your Kubernetes node. | -| config.cniNetDir | string | `"/etc/cni/net.d"` | Directory where the CNI config files reside on the host. This path can either be found in the documentation of your Kubernetes distribution or CNI provider. It can also be queried by running the command `crictl info -o go-template --template '{{.config.cni.confDir}}'` on your Kubernetes node. | -| config.excludeNamespaces | list | `["kube-system"]` | Namespaces for which this CNI plugin should not apply any modifications. | -| config.installMode | string | `"chained"` | Whether to install the CNI plugin as a `chained` plugin (default, required with most interface CNI providers), as a `standalone` plugin (required for use with Multus CNI, e.g. on OpenShift) or in `manual` mode, where no CNI network configuration is written. | -| config.logLevel | string | `"info"` | Log level for the CNI installer and plugin. | +| crds.skipVersionCheck | bool | `false` | Whether to skip the sanity check which prevents installing/upgrading the helm chart in a cluster with outdated Airlock Microgateway CRDs. The check aims to prevent unexpected behavior and issues due to Helm v3 not automatically upgrading CRDs which are already present in the cluster when performing a "helm install/upgrade". | +| dashboards.config.grafana.dashboardLabel.name | string | `"grafana_dashboard"` | Name of the label that lets Grafana identify ConfigMaps that represent dashboards. | +| dashboards.config.grafana.dashboardLabel.value | string | `"1"` | Value of the label that lets Grafana identify ConfigMaps that represent dashboards. | +| dashboards.config.grafana.folderAnnotation.name | string | `"grafana_folder"` | Name of the annotation containing the folder name to file dashboards into. | +| dashboards.config.grafana.folderAnnotation.value | string | `"Airlock Microgateway"` | Name of the folder dashboards are filed into within the Grafana UI. | +| dashboards.create | bool | `false` | Whether to create any ConfigMaps containing Grafana dashboards to import. | +| dashboards.instances.blockLogs.create | bool | `true` | Whether to create the block logs dashboard. | +| dashboards.instances.blockMetrics.create | bool | `true` | Whether to create the block metrics dashboard. | +| dashboards.instances.license.create | bool | `true` | Whether to create the license dashboard. | +| dashboards.instances.overview.create | bool | `true` | Whether to create the overview dashboard. | +| engine.image.digest | string | `"sha256:3c0ebee0b560c8699723bfa433cd601b04b190c384e031d3789b83287fab7a9b"` | SHA256 image digest to pull (in the format "sha256:a3051f42d3013813b05f7513bb86ed6a3209cb3003f1bb2f7b72df249aa544d3"). Overrides tag when specified. | +| engine.image.pullPolicy | string | `"IfNotPresent"` | Pull policy for this image. | +| engine.image.repository | string | `"quay.io/airlock/microgateway-engine"` | Image repository from which to pull the Airlock Microgateway Engine image. | +| engine.image.tag | string | `"4.3.3"` | Image tag to pull. | +| engine.resources | object | `{}` | Resource restrictions to apply to the Airlock Microgateway Engine container. | +| engine.sidecar.podMonitor.create | bool | `false` | Whether to create a PodMonitor resource for monitoring. | +| engine.sidecar.podMonitor.labels | object | `{}` | Labels to add to the PodMonitor. | | fullnameOverride | string | `""` | Allows overriding the name to use as full name of resources. | -| image.digest | string | `"sha256:16317b9a8430059c15175673ad53e31d9e882a1d1af6576214eb1534d8ea6937"` | SHA256 image digest to pull (in the format "sha256:7144f7bab3d4c2648d7e59409f15ec52a18006a128c733fcff20d3a4a54ba44a"). Overrides tag when specified. | -| image.pullPolicy | string | `"IfNotPresent"` | Pull policy for this image. | -| image.repository | string | `"quay.io/airlock/microgateway-cni"` | Image repository from which to pull the Airlock Microgateway CNI image. | -| image.tag | string | `"4.3.3"` | Image tag to pull. | | imagePullSecrets | list | `[]` | ImagePullSecrets to use when pulling images. | -| multusNetworkAttachmentDefinition.create | bool | `false` | Whether a NetworkAttachmentDefinition CR should be created, which can be used for applying the CNI plugin to Pods. | -| multusNetworkAttachmentDefinition.namespace | string | `"default"` | Namespace in which the NetworkAttachmentDefinition is deployed. Note: If namespace is set to a custom value, referencing the created NetworkAttachmentDefinition from other namespaces may not work if Multus namespace isolation is enabled. https://github.com/k8snetworkplumbingwg/multus-cni/blob/v4.0.2/docs/configuration.md#namespace-isolation | -| nameOverride | string | `""` | Allows overriding the name to use instead of "microgateway-cni". | -| nodeSelector | object | `{"kubernetes.io/os":"linux"}` | NodeSelector to apply to the CNI DaemonSet in order to only deploy the CNI plugin on specific nodes. | -| podAnnotations | object | `{}` | Annotations to add to all Pods. | -| podLabels | object | `{}` | Labels to add to all Pods. | -| privileged | bool | `false` | Whether the DaemonSet should run in privileged mode. Must be enabled for environments which require it for writing files to the host (e.g. OpenShift). | -| rbac.create | bool | `true` | Whether to create RBAC resources which are required for the CNI plugin to function. | -| rbac.createSCCRole | OpenShift | `false` | Whether to create RBAC resources which allow the CNI installer to use the "privileged" security context constraint. | -| resources | object | `{"requests":{"cpu":"10m","memory":"100Mi"}}` | Resource restrictions to apply to the CNI installer container. | -| serviceAccount.annotations | object | `{}` | Annotations to add to the ServiceAccount. | -| serviceAccount.create | bool | `true` | Whether a ServiceAccount should be created. | -| serviceAccount.name | string | `""` | Name of the ServiceAccount to use. If not set and create is true, a name is generated using the fullname template. | +| license.secretName | string | `"airlock-microgateway-license"` | Name of the secret containing the "microgateway-license.txt" key. | +| nameOverride | string | `""` | Allows overriding the name to use instead of "microgateway". | +| networkValidator.image.digest | string | `"sha256:6051975a14c51b9d3b525a06004d62a4d323c08ca58e3468343095a55a42fff2"` | SHA256 image digest to pull (in the format "sha256:6051975a14c51b9d3b525a06004d62a4d323c08ca58e3468343095a55a42fff2"). Overrides tag when specified. | +| networkValidator.image.pullPolicy | string | `"IfNotPresent"` | Pull policy for this image. | +| networkValidator.image.repository | string | `"cgr.dev/chainguard/netcat"` | Image repository from which to pull the netcat image for the Airlock Microgateway Network Validator init-container. | +| networkValidator.image.tag | string | `""` | Image tag to pull. | +| operator.affinity | object | `{}` | Custom affinity to apply to the operator Deployment. Used to influence the scheduling. | +| operator.config.logLevel | string | `"info"` | Operator application log level. | +| operator.image.digest | string | `"sha256:6d3ebca355de0a67f0bf5f088a15b9410564e500033d3e1f534a2f49a05bf4c3"` | SHA256 image digest to pull (in the format "sha256:c79ee3f85862fb386e9dd62b901b607161d27807f512d7fbdece05e9ee3d7c63"). Overrides tag when specified. | +| operator.image.pullPolicy | string | `"IfNotPresent"` | Pull policy for this image. | +| operator.image.repository | string | `"quay.io/airlock/microgateway-operator"` | Image repository from which to pull the Airlock Microgateway Operator image. | +| operator.image.tag | string | `"4.3.3"` | Image tag to pull. | +| operator.nodeSelector | object | `{}` | Custom nodeSelector to apply to the operator Deployment in order to constrain its Pods to certain nodes. | +| operator.podAnnotations | object | `{}` | Annotations to add to all Pods. | +| operator.podLabels | object | `{}` | Labels to add to all Pods. | +| operator.rbac.create | bool | `true` | Whether to create RBAC resources which are required for the Airlock Microgateway Operator to function. | +| operator.replicaCount | int | `2` | Number of replicas for the operator Deployment. | +| operator.resources | object | `{}` | Resource restrictions to apply to the operator container. | +| operator.serviceAccount.annotations | object | `{}` | Annotations to add to the ServiceAccount. | +| operator.serviceAccount.create | bool | `true` | Whether a ServiceAccount should be created. | +| operator.serviceAccount.name | string | `""` | Name of the ServiceAccount to use. If not set and create is true, a name is generated using the fullname template. | +| operator.serviceAnnotations | object | `{}` | Annotations to add to the Service. | +| operator.serviceLabels | object | `{}` | Labels to add to the Service. | +| operator.serviceMonitor.create | bool | `false` | Whether to create a ServiceMonitor resource for monitoring. | +| operator.serviceMonitor.labels | object | `{}` | Labels to add to the ServiceMonitor. | +| operator.tolerations | list | `[]` | Custom tolerations to apply to the operator Deployment in order to allow its Pods to run on tainted nodes. | +| operator.updateStrategy | object | `{"type":"RollingUpdate"}` | Specifies the operator update strategy. | +| operator.watchNamespaceSelector | object | `{}` | Allows to dynamically select watch namespaces of the operator and the scope of the webhooks based on a Namespace label selector. It is able to detect and reconcile resources in all namespaces that match the label selector automatically, even for new namespaces, without restarting the operator. This facilitates a dynamic `MultiNamespace` installation mode, but still requires cluster-scoped permissions (i.e., ClusterRoles and ClusterRoleBindings). An `AllNamespaces` installation or the usage of the `watchNamespaces` requires the `watchNamespaceSelector` to be empty. Please note that this feature requires a Premium license. | +| operator.watchNamespaces | list | `[]` | Allows to restrict the operator to specific namespaces, depending on your needs. For a `OwnNamespace` or `SingleNamespace` installation the list may only contain one namespace (e.g., `watchNamespaces: ["airlock-microgateway-system"]`). In case of the `OwnNamespace` installation mode the specified namespace should be equal to the installation namespace. For a static `MultiNamespace` installation, the complete list of namespaces must be provided in the `watchNamespaces`. An `AllNamespaces` installation or the usage of the `watchNamespaceSelector` requires the `watchNamespaces` to be empty. Regardless of the installation modes supported by `watchNamespaces`, RBAC is created only namespace-scoped (using Roles and RoleBindings) in the respective namespaces. Please note that this feature requires a Premium license. | +| sessionAgent.image.digest | string | `"sha256:994bf4117adb74da4e05c22ffc168d9844bc68efa6a7fb96d73e849d1ef67b56"` | SHA256 image digest to pull (in the format "sha256:a3051f42d3013813b05f7513bb86ed6a3209cb3003f1bb2f7b72df249aa544d3"). Overrides tag when specified. | +| sessionAgent.image.pullPolicy | string | `"IfNotPresent"` | Pull policy for this image. | +| sessionAgent.image.repository | string | `"quay.io/airlock/microgateway-session-agent"` | Image repository from which to pull the Airlock Microgateway Session Agent image. | +| sessionAgent.image.tag | string | `"4.3.3"` | Image tag to pull. | +| sessionAgent.resources | object | `{}` | Resource restrictions to apply to the Airlock Microgateway Session Agent container. | | tests.enabled | bool | `false` | Whether additional resources required for running `helm test` should be created (e.g. Roles and ServiceAccounts). If set to false, `helm test` will not run any tests. | ## License diff --git a/charts/airlock/microgateway/4.4.0/app-readme.md b/charts/airlock/microgateway/4.3.3/app-readme.md similarity index 100% rename from charts/airlock/microgateway/4.4.0/app-readme.md rename to charts/airlock/microgateway/4.3.3/app-readme.md diff --git a/charts/airlock/microgateway/4.3.3/crds/accesscontrols.microgateway.airlock.com.yaml b/charts/airlock/microgateway/4.3.3/crds/accesscontrols.microgateway.airlock.com.yaml new file mode 100644 index 0000000000..b6f1ab384b --- /dev/null +++ b/charts/airlock/microgateway/4.3.3/crds/accesscontrols.microgateway.airlock.com.yaml @@ -0,0 +1,124 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + labels: + app.kubernetes.io/name: airlock-microgateway-operator + app.kubernetes.io/version: 4.3.3 + name: accesscontrols.microgateway.airlock.com +spec: + group: microgateway.airlock.com + names: + categories: + - airlock-microgateway + kind: AccessControl + listKind: AccessControlList + plural: accesscontrols + singular: accesscontrol + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: AccessControl specifies the options to perform access control with a Microgateway Engine container. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Specifies how the Airlock Microgateway Engine performs access control. + properties: + policies: + description: Policies configures access control policies. + items: + properties: + authorization: + description: Authorization configures how requests are authorized. An empty object value {} disables authorization. + properties: + authentication: + description: Authentication specifies that clients need to be authenticated with the provided method. + properties: + oidc: + description: OIDC configures client authentication using OpenID Connect. + properties: + oidcRelyingPartyRef: + description: OIDCRelyingPartyRef configures how the Airlock Microgateway Engine interacts with the OpenID provider. + properties: + name: + description: Name of the resource + minLength: 1 + type: string + required: + - name + type: object + required: + - oidcRelyingPartyRef + type: object + type: object + type: object + identityPropagation: + description: IdentityPropagation configures how the authenticated user's identity is communicated to the protected application. + properties: + actions: + description: Actions specifies the propagation actions. + items: + properties: + identityPropagationRef: + description: IdentityPropagationRef selects an IdentityPropagation to apply. + properties: + name: + description: Name of the resource + minLength: 1 + type: string + required: + - name + type: object + required: + - identityPropagationRef + type: object + type: array + onFailure: + description: |- + OnFailure configures what should happen, if an identity propagation fails. Meaning of the possible values: + _Pass_: The request should be forwarded to the upstream, without including the information from the failed identity propagations. + enum: + - Pass + type: string + required: + - actions + - onFailure + type: object + required: + - authorization + type: object + maxItems: 1 + minItems: 1 + type: array + required: + - policies + type: object + required: + - spec + type: object + served: true + storage: true + subresources: {} diff --git a/charts/airlock/microgateway/4.3.3/crds/contentsecurities.microgateway.airlock.com.yaml b/charts/airlock/microgateway/4.3.3/crds/contentsecurities.microgateway.airlock.com.yaml new file mode 100644 index 0000000000..05e059f8a0 --- /dev/null +++ b/charts/airlock/microgateway/4.3.3/crds/contentsecurities.microgateway.airlock.com.yaml @@ -0,0 +1,139 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + labels: + app.kubernetes.io/name: airlock-microgateway-operator + app.kubernetes.io/version: 4.3.3 + name: contentsecurities.microgateway.airlock.com +spec: + group: microgateway.airlock.com + names: + categories: + - airlock-microgateway + kind: ContentSecurity + listKind: ContentSecurityList + plural: contentsecurities + singular: contentsecurity + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: ContentSecurity specifies the options to secure an upstream web application with a Microgateway Engine container. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Specifies the options to secure an upstream web application with a Microgateway Engine container. + properties: + apiProtection: + description: |- + APIProtection defines the relevant configurations to protect APIs. + If undefined, default settings are applied, designed to work with most upstream web application services. + properties: + graphQLRef: + description: |- + GraphQLRef selects the relevant GraphQL configuration resource. + If undefined, default settings are applied, designed to work with most upstream web application services. + properties: + name: + description: Name of the resource + minLength: 1 + type: string + required: + - name + type: object + openAPIRef: + description: |- + OpenAPIRef selects the relevant OpenAPI configuration resource. + If undefined, default settings are applied, designed to work with most upstream web application services. + properties: + name: + description: Name of the resource + minLength: 1 + type: string + required: + - name + type: object + type: object + filter: + description: |- + Filter defines the set of filters, e.g. Airlock Deny Rules, to be applied to incoming requests + to protect against various attack patterns. + If undefined, default settings are applied, designed to work with most upstream web application services. + properties: + denyRulesRef: + description: |- + DenyRulesRef selects the relevant DenyRules configuration resource. + If undefined, default settings are applied, designed to work with most upstream web application services. + properties: + name: + description: Name of the resource + minLength: 1 + type: string + required: + - name + type: object + type: object + headerRewritesRef: + description: |- + HeaderRewritesRef selects the relevant HeaderRewrites. + If undefined, default settings are applied, designed to work with most upstream web application services. + properties: + name: + description: Name of the resource + minLength: 1 + type: string + required: + - name + type: object + limitsRef: + description: |- + LimitsRef selects the relevant Limits configuration resource. + If undefined, default settings are applied, designed to work with most upstream web application services. + properties: + name: + description: Name of the resource + minLength: 1 + type: string + required: + - name + type: object + parserRef: + description: |- + ParserRef selects the relevant Parser configuration resource. + If undefined, default settings are applied, designed to work with most upstream web application services. + properties: + name: + description: Name of the resource + minLength: 1 + type: string + required: + - name + type: object + type: object + type: object + served: true + storage: true + subresources: {} diff --git a/charts/airlock/microgateway/4.3.3/crds/denyrules.microgateway.airlock.com.yaml b/charts/airlock/microgateway/4.3.3/crds/denyrules.microgateway.airlock.com.yaml new file mode 100644 index 0000000000..fddaa375d6 --- /dev/null +++ b/charts/airlock/microgateway/4.3.3/crds/denyrules.microgateway.airlock.com.yaml @@ -0,0 +1,1804 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + labels: + app.kubernetes.io/name: airlock-microgateway-operator + app.kubernetes.io/version: 4.3.3 + name: denyrules.microgateway.airlock.com +spec: + group: microgateway.airlock.com + names: + categories: + - airlock-microgateway + kind: DenyRules + listKind: DenyRulesList + plural: denyrules + singular: denyrules + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + DenyRules configures request filtering using Airlock built-in and custom deny rules. + Deny rules establish a negative security model. They define prohibited patterns which, when a match is found in a request, lead to it being blocked from reaching the upstream web application. + To handle possible false positives, lower the security level or define fine-granular deny rule exceptions + If undefined, default settings are applied, designed to work with most upstream web application services. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Specification of the desired deny rules behavior. + properties: + request: + description: Request configures deny rules for downstream requests. + properties: + builtIn: + description: BuiltIn configures the built-in deny rules. + properties: + exceptions: + description: Exceptions allows to define exceptions for specific requests and deny rules. + items: + description: |- + DenyRulesException defines an exception for deny rules. Exceptions may be defined by any or a combination of the following elements: blockedData (the request data causing a block) or requestConditions (properties of a request without taking into consideration the reason why a request has been blocked). + At least one of blockedData and requestConditions must be set. + properties: + blockedData: + description: BlockedData defines an exception based on the request data causing the block. + properties: + graphQL: + description: |- + GraphQL defines an exception based on a blocked GraphQL query. + Only one of parameter, header, path, pathSegment, json or graphQL can be set. + properties: + argument: + description: |- + Argument defines an argument of a field of the GraphQL query. + At least one of field, argument and value must be set. + properties: + matcher: + description: StringMatcher defines the way to match a string. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + ignoreCase: + default: false + description: IgnoreCase indicates whether the matching should be case-insensitive. In case of a regex match, the regex gets wrapped with a group `(?i:...)`. + type: boolean + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + field: + description: |- + Field defines a field of the GraphQL query. + At least one of field, argument and value must be set. + properties: + matcher: + description: StringMatcher defines the way to match a string. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + ignoreCase: + default: false + description: IgnoreCase indicates whether the matching should be case-insensitive. In case of a regex match, the regex gets wrapped with a group `(?i:...)`. + type: boolean + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + value: + description: |- + Value defines the value of an argument of the GraphQL query. + At least one of field, argument and value must be set. + properties: + matcher: + description: StringMatcher defines the way to match a string. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + ignoreCase: + default: false + description: IgnoreCase indicates whether the matching should be case-insensitive. In case of a regex match, the regex gets wrapped with a group `(?i:...)`. + type: boolean + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + type: object + header: + description: |- + Header defines an exception based on a blocked header. + Only one of parameter, header, path, pathSegment, json or graphQL can be set. + properties: + name: + description: Name defines the name of a header. + properties: + matcher: + description: Matcher defines the way to match a string. In comparison to a normal StringMatcher, a value is always matched ignoring the case and can't be inverted. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + value: + description: Value defines the value of a header. + properties: + matcher: + description: StringMatcher defines the way to match a string. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + ignoreCase: + default: false + description: IgnoreCase indicates whether the matching should be case-insensitive. In case of a regex match, the regex gets wrapped with a group `(?i:...)`. + type: boolean + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + type: object + json: + description: |- + JSON defines an exception based on a blocked JSON property. + Only one of parameter, header, path, pathSegment, json or graphQL can be set. + properties: + jsonPath: + description: |- + JSONPath defines the JSONPath pattern to match the path within the JSON. + Expressions in JSONPath i.e. `?(expr)` are not supported. + minLength: 1 + type: string + key: + description: |- + Key defines the key of the JSON property. + At most one of key and value can be set. + properties: + matcher: + description: StringMatcher defines the way to match a string. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + ignoreCase: + default: false + description: IgnoreCase indicates whether the matching should be case-insensitive. In case of a regex match, the regex gets wrapped with a group `(?i:...)`. + type: boolean + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + value: + description: |- + Value defines the value of the JSON property. + At most one of key and value can be set. + properties: + matcher: + description: StringMatcher defines the way to match a string. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + ignoreCase: + default: false + description: IgnoreCase indicates whether the matching should be case-insensitive. In case of a regex match, the regex gets wrapped with a group `(?i:...)`. + type: boolean + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + type: object + parameter: + description: |- + Parameter defines an exception based on a blocked parameter. + Only one of parameter, header, path, pathSegment, json or graphQL can be set. + properties: + name: + description: Name defines the name of a parameter. + properties: + matcher: + description: StringMatcher defines the way to match a string. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + ignoreCase: + default: false + description: IgnoreCase indicates whether the matching should be case-insensitive. In case of a regex match, the regex gets wrapped with a group `(?i:...)`. + type: boolean + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + source: + default: Any + description: Source defines the source of the parameter. + enum: + - Query + - Post + - Any + type: string + value: + description: Value defines the value of a parameter. + properties: + matcher: + description: StringMatcher defines the way to match a string. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + ignoreCase: + default: false + description: IgnoreCase indicates whether the matching should be case-insensitive. In case of a regex match, the regex gets wrapped with a group `(?i:...)`. + type: boolean + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + type: object + path: + description: |- + Path defines an exception based on the blocked path. + Only one of parameter, header, path, pathSegment, json or graphQL can be set. + properties: + matcher: + description: StringMatcher defines the way to match a string. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + ignoreCase: + default: false + description: IgnoreCase indicates whether the matching should be case-insensitive. In case of a regex match, the regex gets wrapped with a group `(?i:...)`. + type: boolean + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + pathSegment: + description: |- + PathSegment defines an exception based on a blocked path segment. + Only one of parameter, header, path, pathSegment, json or graphQL can be set. + properties: + segments: + description: Segments defines the position of a segment within the path. + properties: + index: + description: Index specifies an exact path segment position by index (0-based). + minimum: 0 + type: integer + type: object + value: + description: Value defines the value of a path segment. + properties: + matcher: + description: StringMatcher defines the way to match a string. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + ignoreCase: + default: false + description: IgnoreCase indicates whether the matching should be case-insensitive. In case of a regex match, the regex gets wrapped with a group `(?i:...)`. + type: boolean + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + type: object + type: object + requestConditions: + description: RequestConditions defines an exception based on a property of a request without taking into consideration the reason why a request has been blocked. + properties: + header: + description: Header defines the matching headers of a request. + properties: + name: + description: Name defines the name of a header. + properties: + matcher: + description: Matcher defines the way to match a string. In comparison to a normal StringMatcher, a value is always matched ignoring the case and can't be inverted. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + value: + description: Value defines the value of a header. + properties: + matcher: + description: StringMatcher defines the way to match a string. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + ignoreCase: + default: false + description: IgnoreCase indicates whether the matching should be case-insensitive. In case of a regex match, the regex gets wrapped with a group `(?i:...)`. + type: boolean + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + type: object + invert: + default: false + description: Invert indicates whether the request condition should be inverted. + type: boolean + mediaType: + description: MediaType defines the matching media type from the content-type header of a request. + properties: + matcher: + description: |- + NonInvertableCaseInsensitiveStringMatcher defines the way to match a string. + In comparison to a normal StringMatcher, a value is always matched ignoring the case and can't be inverted. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + method: + description: Method defines the matching methods of a request. + items: + description: Method defines common HTTP methods. + enum: + - GET + - HEAD + - POST + - PUT + - PATCH + - DELETE + - CONNECT + - OPTIONS + - TRACE + type: string + type: array + path: + description: Path defines the matching path of a request. + properties: + matcher: + description: StringMatcher defines the way to match a string. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + ignoreCase: + default: false + description: IgnoreCase indicates whether the matching should be case-insensitive. In case of a regex match, the regex gets wrapped with a group `(?i:...)`. + type: boolean + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + remoteIP: + description: RemoteIP defines the matching remote IPs of a request. + properties: + cidrRanges: + description: CIDRRanges defines the IPv4 or IPv6 CIDR ranges, e.g. ``196.148.3.128/26`` or ``2001:db8::/28``. + items: + description: CIDRRange defines an IPv4 or IPv6 CIDR range, e.g. “196.148.3.128/26“ or “2001:db8::/28“. + format: cidr + type: string + minItems: 1 + type: array + invert: + default: false + description: Invert indicates whether the match should be inverted. + type: boolean + required: + - cidrRanges + type: object + type: object + ruleKeys: + description: RuleKeys restricts the exception to a set of deny rules. + items: + description: |- + A deny rule name can be any of the following values: + ENCODING | + EXPLOIT | + HPP | + HTML | + IDOR | + LDAP | + NOSQL | + OGNL | + PHP | + PROTOCOL | + SANITY | + SCANNING | + SQL | + TEMPLATE | + UNIXCMD | + WINCMD | + XSS + enum: + - ENCODING + - EXPLOIT + - HPP + - HTML + - IDOR + - LDAP + - NOSQL + - OGNL + - PHP + - PROTOCOL + - SANITY + - SCANNING + - SQL + - TEMPLATE + - UNIXCMD + - WINCMD + - XSS + type: string + minItems: 1 + type: array + type: object + type: array + overrides: + description: Overrides allows to override the builtIn settings for specific deny rules. + items: + description: DenyRulesOverride allows to override the builtIn settings for specific deny rules. + properties: + conditions: + description: Conditions select which built-in deny rules' settings will be adjusted. + properties: + ruleKeys: + description: RuleKeys is a list of built-in deny rule names. + items: + description: |- + A deny rule name can be any of the following values: + ENCODING | + EXPLOIT | + HPP | + HTML | + IDOR | + LDAP | + NOSQL | + OGNL | + PHP | + PROTOCOL | + SANITY | + SCANNING | + SQL | + TEMPLATE | + UNIXCMD | + WINCMD | + XSS + enum: + - ENCODING + - EXPLOIT + - HPP + - HTML + - IDOR + - LDAP + - NOSQL + - OGNL + - PHP + - PROTOCOL + - SANITY + - SCANNING + - SQL + - TEMPLATE + - UNIXCMD + - WINCMD + - XSS + type: string + minItems: 1 + type: array + types: + description: Types defines the type of attributes the override should be applied on. If Types are defined without any RuleKeys the override is applied to all deny rules. + items: + description: |- + A deny rule override type name can be any of the following values: + Header | + Parameter | + Path | + JSON | + GraphQL + enum: + - Header + - Parameter + - Path + - PathSegment + - JSON + - GraphQL + type: string + minItems: 0 + type: array + type: object + settings: + description: Settings override the corresponding properties for the selected rules. + properties: + level: + description: Level specifies the filter strength. + enum: + - Unfiltered + - Basic + - Standard + - Strict + type: string + threatHandlingMode: + description: ThreatHandlingMode specifies how threats should be handled. + enum: + - Block + - LogOnly + type: string + type: object + type: object + type: array + settings: + description: Settings contains the keys which will be adjusted. + properties: + level: + default: Standard + description: Level represents a set of deny rules with different filter strengths. + enum: + - Unfiltered + - Basic + - Standard + - Strict + type: string + threatHandlingMode: + default: Block + description: ThreatHandlingMode specifies how threats should be handled when a deny rule matches. + enum: + - Block + - LogOnly + type: string + type: object + type: object + custom: + description: Custom allows configuring additional deny rules. + properties: + rules: + description: Rules defines list of additional deny rules. + items: + properties: + blockData: + description: BlockData specifies the request data which should cause a block. + properties: + graphQL: + description: |- + GraphQL specifies to block requests containing a matching GraphQL property. + At least one of field, argument and value must be set. + properties: + argument: + description: |- + Argument defines an argument of a field of the GraphQL query. + At least one of field, argument and value must be set. + properties: + matcher: + description: StringMatcher defines the way to match a string. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + ignoreCase: + default: false + description: IgnoreCase indicates whether the matching should be case-insensitive. In case of a regex match, the regex gets wrapped with a group `(?i:...)`. + type: boolean + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + field: + description: |- + Field defines a field of the GraphQL query. + At least one of field, argument and value must be set. + properties: + matcher: + description: StringMatcher defines the way to match a string. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + ignoreCase: + default: false + description: IgnoreCase indicates whether the matching should be case-insensitive. In case of a regex match, the regex gets wrapped with a group `(?i:...)`. + type: boolean + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + value: + description: |- + Value defines the value of an argument of the GraphQL query. + At least one of field, argument and value must be set. + properties: + matcher: + description: StringMatcher defines the way to match a string. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + ignoreCase: + default: false + description: IgnoreCase indicates whether the matching should be case-insensitive. In case of a regex match, the regex gets wrapped with a group `(?i:...)`. + type: boolean + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + type: object + header: + description: |- + Header specifies to block requests containing a matching header. + Only one of parameter, header, path, pathSegment or json can be set. + properties: + name: + description: Name defines the name of a header. + properties: + matcher: + description: |- + NonInvertableCaseInsensitiveStringMatcher defines the way to match a string. + In comparison to a normal StringMatcher, a value is always matched ignoring the case and can't be inverted. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + value: + description: Value defines the value of a header. + properties: + matcher: + description: StringMatcher defines the way to match a string. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + ignoreCase: + default: false + description: IgnoreCase indicates whether the matching should be case-insensitive. In case of a regex match, the regex gets wrapped with a group `(?i:...)`. + type: boolean + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + type: object + json: + description: |- + JSON specifies to block requests containing a matching JSON property in the body. + Only one of parameter, header, path, pathSegment or json can be set. + properties: + key: + description: Key defines the key of a JSON object. + properties: + matcher: + description: StringMatcher defines the way to match a string. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + ignoreCase: + default: false + description: IgnoreCase indicates whether the matching should be case-insensitive. In case of a regex match, the regex gets wrapped with a group `(?i:...)`. + type: boolean + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + value: + description: Value defines the value of a JSON object. + properties: + matcher: + description: StringMatcher defines the way to match a string. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + ignoreCase: + default: false + description: IgnoreCase indicates whether the matching should be case-insensitive. In case of a regex match, the regex gets wrapped with a group `(?i:...)`. + type: boolean + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + type: object + parameter: + description: |- + Parameter specifies to block requests containing a matching parameter. + Only one of parameter, header, path, pathSegment or json can be set. + properties: + name: + description: Name defines the name of a parameter. + properties: + matcher: + description: StringMatcher defines the way to match a string. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + ignoreCase: + default: false + description: IgnoreCase indicates whether the matching should be case-insensitive. In case of a regex match, the regex gets wrapped with a group `(?i:...)`. + type: boolean + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + value: + description: Value defines the value of a parameter. + properties: + matcher: + description: StringMatcher defines the way to match a string. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + ignoreCase: + default: false + description: IgnoreCase indicates whether the matching should be case-insensitive. In case of a regex match, the regex gets wrapped with a group `(?i:...)`. + type: boolean + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + type: object + path: + description: |- + Path specifies to block requests with a matching path. + Only one of parameter, header, path, pathSegment or json can be set. + properties: + matcher: + description: Matcher specifies which path to block. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + ignoreCase: + default: false + description: IgnoreCase indicates whether the matching should be case-insensitive. In case of a regex match, the regex gets wrapped with a group `(?i:...)`. + type: boolean + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + pathSegment: + description: |- + PathSegment specifies to block requests containing a matching path segment. + Only one of parameter, header, path, pathSegment or json can be set. + properties: + segments: + description: |- + Segments restricts which path segments are filtered by this rule. + If not specified, all segments of a path are filtered. + properties: + index: + description: Index restricts the rule to the path segment at this index (0-based). + minimum: 0 + type: integer + type: object + value: + description: Value specifies which path segment values to block. + properties: + matcher: + description: StringMatcher defines the way to match a string. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + ignoreCase: + default: false + description: IgnoreCase indicates whether the matching should be case-insensitive. In case of a regex match, the regex gets wrapped with a group `(?i:...)`. + type: boolean + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + required: + - value + type: object + type: object + requestConditions: + description: RequestConditions defines additional request properties which must be matched in order for this rule to apply. + properties: + header: + description: Header defines the matching headers of a request. + properties: + name: + description: Name defines the name of a header. + properties: + matcher: + description: Matcher defines the way to match a string. In comparison to a normal StringMatcher, a value is always matched ignoring the case and can't be inverted. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + value: + description: Value defines the value of a header. + properties: + matcher: + description: StringMatcher defines the way to match a string. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + ignoreCase: + default: false + description: IgnoreCase indicates whether the matching should be case-insensitive. In case of a regex match, the regex gets wrapped with a group `(?i:...)`. + type: boolean + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + type: object + invert: + default: false + description: Invert indicates whether the request condition should be inverted. + type: boolean + mediaType: + description: MediaType defines the matching media type from the content-type header of a request. + properties: + matcher: + description: |- + NonInvertableCaseInsensitiveStringMatcher defines the way to match a string. + In comparison to a normal StringMatcher, a value is always matched ignoring the case and can't be inverted. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + method: + description: Method defines the matching methods of a request. + items: + description: Method defines common HTTP methods. + enum: + - GET + - HEAD + - POST + - PUT + - PATCH + - DELETE + - CONNECT + - OPTIONS + - TRACE + type: string + type: array + path: + description: Path defines the matching path of a request. + properties: + matcher: + description: StringMatcher defines the way to match a string. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + ignoreCase: + default: false + description: IgnoreCase indicates whether the matching should be case-insensitive. In case of a regex match, the regex gets wrapped with a group `(?i:...)`. + type: boolean + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + remoteIP: + description: RemoteIP defines the matching remote IPs of a request. + properties: + cidrRanges: + description: CIDRRanges defines the IPv4 or IPv6 CIDR ranges, e.g. ``196.148.3.128/26`` or ``2001:db8::/28``. + items: + description: CIDRRange defines an IPv4 or IPv6 CIDR range, e.g. “196.148.3.128/26“ or “2001:db8::/28“. + format: cidr + type: string + minItems: 1 + type: array + invert: + default: false + description: Invert indicates whether the match should be inverted. + type: boolean + required: + - cidrRanges + type: object + type: object + ruleKey: + description: RuleKey defines a technical key for the deny rule. Must be unique. + minLength: 1 + pattern: ^[A-Z][A-Z0-9_]*$ + type: string + threatHandlingMode: + default: Block + description: ThreatHandlingMode specifies how threats should be handled when a deny rule matches. + enum: + - Block + - LogOnly + type: string + required: + - blockData + - ruleKey + type: object + type: array + x-kubernetes-list-map-keys: + - ruleKey + x-kubernetes-list-type: map + type: object + type: object + type: object + type: object + served: true + storage: true + subresources: {} diff --git a/charts/airlock/microgateway/4.3.3/crds/envoyclusters.microgateway.airlock.com.yaml b/charts/airlock/microgateway/4.3.3/crds/envoyclusters.microgateway.airlock.com.yaml new file mode 100644 index 0000000000..bb564f9429 --- /dev/null +++ b/charts/airlock/microgateway/4.3.3/crds/envoyclusters.microgateway.airlock.com.yaml @@ -0,0 +1,58 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + labels: + app.kubernetes.io/name: airlock-microgateway-operator + app.kubernetes.io/version: 4.3.3 + name: envoyclusters.microgateway.airlock.com +spec: + group: microgateway.airlock.com + names: + categories: + - airlock-microgateway + kind: EnvoyCluster + listKind: EnvoyClusterList + plural: envoyclusters + singular: envoycluster + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: EnvoyCluster is an additional Envoy Cluster resource which is added to those defined by the Airlock Microgateway. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Specification of the desired additional Envoy cluster. + properties: + value: + description: Value defines the Envoy Cluster which is added to those configured by the Airlock Microgateway. + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + type: object + served: true + storage: true + subresources: {} diff --git a/charts/airlock/microgateway/4.3.3/crds/envoyconfigurations.microgateway.airlock.com.yaml b/charts/airlock/microgateway/4.3.3/crds/envoyconfigurations.microgateway.airlock.com.yaml new file mode 100644 index 0000000000..b6147ae08e --- /dev/null +++ b/charts/airlock/microgateway/4.3.3/crds/envoyconfigurations.microgateway.airlock.com.yaml @@ -0,0 +1,185 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + labels: + app.kubernetes.io/name: airlock-microgateway-operator + app.kubernetes.io/version: 4.3.3 + name: envoyconfigurations.microgateway.airlock.com +spec: + group: microgateway.airlock.com + names: + categories: + - airlock-microgateway + kind: EnvoyConfiguration + listKind: EnvoyConfigurationList + plural: envoyconfigurations + singular: envoyconfiguration + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.status + name: Status + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + EnvoyConfiguration is the Schema for the envoyconfigurations API + {{% notice warning %}} EnvoyConfiguration resources may contain sensitive information and thus RBAC permissions should be granted with care. {{% /notice %}} + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: EnvoyConfigurationSpec defines the desired state of EnvoyConfiguration + properties: + envoyResources: + properties: + clusters: + items: + type: object + x-kubernetes-preserve-unknown-fields: true + type: array + endpoints: + items: + type: object + x-kubernetes-preserve-unknown-fields: true + type: array + extensions: + items: + type: object + x-kubernetes-preserve-unknown-fields: true + type: array + listeners: + items: + type: object + x-kubernetes-preserve-unknown-fields: true + type: array + routes: + items: + type: object + x-kubernetes-preserve-unknown-fields: true + type: array + runtimes: + items: + type: object + x-kubernetes-preserve-unknown-fields: true + type: array + scopedRoutes: + items: + type: object + x-kubernetes-preserve-unknown-fields: true + type: array + secrets: + items: + type: object + x-kubernetes-preserve-unknown-fields: true + type: array + type: object + envoyResourcesRaw: + description: |- + EnvoyResourcesRaw defines the desired state for each resource type. The resources are stored as zstd compressed JSON bytes. + For debugging purposes, the resources can be inspected with the following command: `kubectl get envoyconfiguration -ojsonpath='{.spec.envoyResourcesRaw}' | base64 -d | zstd -d | jq` + format: byte + type: string + nodeID: + description: '**Deprecated:** This field is now ignored as NodeID is always derived from the resource name.' + type: string + type: object + status: + description: EnvoyConfigurationStatus defines the observed state of EnvoyConfiguration + properties: + conditions: + items: + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + format: date-time + type: string + message: + description: A human-readable message indicating details about the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of EnvoyConfiguration condition. + type: string + required: + - status + - type + type: object + type: array + status: + type: string + xds: + properties: + resourceTypes: + additionalProperties: + description: XdsResourceTypeSyncStatus defines the sync status of xDS for a specific resource type + properties: + errorMessage: + description: ErrorMessage defines an optional message why the currently served resources of this resource type are rejected by the client. + type: string + resources: + additionalProperties: + description: XdsResourceStatus defines the status of xDS for a specific resource + properties: + version: + description: Version defines the version which is currently served for this resource. + type: string + required: + - version + type: object + description: Resources defines the resources which are currently served for this resource type. + type: object + status: + description: Status defines the current sync status of this resource type. + type: string + version: + description: Version defines the version which is currently served for this resource type. + type: string + required: + - resources + - status + - version + type: object + description: ResourceTypes defines the sync statuses for each resource type. + type: object + version: + description: Version defines the version of the underlying xDS snapshot. + type: integer + required: + - version + type: object + required: + - status + - xds + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/airlock/microgateway/4.3.3/crds/envoyhttpfilters.microgateway.airlock.com.yaml b/charts/airlock/microgateway/4.3.3/crds/envoyhttpfilters.microgateway.airlock.com.yaml new file mode 100644 index 0000000000..c5eaad3642 --- /dev/null +++ b/charts/airlock/microgateway/4.3.3/crds/envoyhttpfilters.microgateway.airlock.com.yaml @@ -0,0 +1,58 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + labels: + app.kubernetes.io/name: airlock-microgateway-operator + app.kubernetes.io/version: 4.3.3 + name: envoyhttpfilters.microgateway.airlock.com +spec: + group: microgateway.airlock.com + names: + categories: + - airlock-microgateway + kind: EnvoyHTTPFilter + listKind: EnvoyHTTPFilterList + plural: envoyhttpfilters + singular: envoyhttpfilter + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: EnvoyHTTPFilter is an additional Envoy HTTP Filter resource which is added to those defined by the Airlock Microgateway. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Specification of the desired additional Envoy HTTP filter. + properties: + value: + description: Value defines the HTTP filter which is added to those configured by the Airlock Microgateway. + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + type: object + served: true + storage: true + subresources: {} diff --git a/charts/airlock/microgateway/4.3.3/crds/graphqls.microgateway.airlock.com.yaml b/charts/airlock/microgateway/4.3.3/crds/graphqls.microgateway.airlock.com.yaml new file mode 100644 index 0000000000..1d9cb3b94f --- /dev/null +++ b/charts/airlock/microgateway/4.3.3/crds/graphqls.microgateway.airlock.com.yaml @@ -0,0 +1,88 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + labels: + app.kubernetes.io/name: airlock-microgateway-operator + app.kubernetes.io/version: 4.3.3 + name: graphqls.microgateway.airlock.com +spec: + group: microgateway.airlock.com + names: + categories: + - airlock-microgateway + kind: GraphQL + listKind: GraphQLList + plural: graphqls + singular: graphql + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: GraphQL contains the configuration for the GraphQL specification. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Specification of the desired GraphQL specification. + properties: + settings: + description: Settings defines the settings to configure GraphQL. + properties: + allowIntrospection: + default: true + description: AllowIntrospection specifies if the introspection system is exposed. + type: boolean + allowMutations: + default: true + description: AllowMutations specifies if mutations are allowed. + type: boolean + schema: + description: Specifies the GraphQL schema. + properties: + source: + description: Source specifies the GraphQL schema to be enforced. + properties: + configMapRef: + description: ConfigMapRef references the configmap by its name containing the well-known key 'schema.graphql'. + properties: + name: + description: Name of the resource + minLength: 1 + type: string + required: + - name + type: object + type: object + required: + - source + type: object + threatHandlingMode: + default: Block + description: ThreatHandlingMode specifies how threats should be handled. + enum: + - Block + - LogOnly + type: string + type: object + type: object + type: object + served: true + storage: true diff --git a/charts/airlock/microgateway/4.3.3/crds/headerrewrites.microgateway.airlock.com.yaml b/charts/airlock/microgateway/4.3.3/crds/headerrewrites.microgateway.airlock.com.yaml new file mode 100644 index 0000000000..a9f832a2bb --- /dev/null +++ b/charts/airlock/microgateway/4.3.3/crds/headerrewrites.microgateway.airlock.com.yaml @@ -0,0 +1,759 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + labels: + app.kubernetes.io/name: airlock-microgateway-operator + app.kubernetes.io/version: 4.3.3 + name: headerrewrites.microgateway.airlock.com +spec: + group: microgateway.airlock.com + names: + categories: + - airlock-microgateway + kind: HeaderRewrites + listKind: HeaderRewritesList + plural: headerrewrites + singular: headerrewrites + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: HeaderRewrites is the Schema for the headerrewrites API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Specification of the desired header rewriting behavior. + properties: + request: + description: Request defines manipulations on upstream request headers. + properties: + add: + description: Add defines which request headers will be added before forwarding to the upstream. + properties: + custom: + description: |- + Custom allows configuring additional upstream request headers. + Add selected headers. + items: + properties: + headers: + description: Headers to add. + items: + description: HeaderRewritesHeader specifies a header with a particular value + properties: + name: + description: Name defines the name of a header. + minLength: 1 + type: string + value: + description: Value defines the value of a header. + type: string + required: + - name + - value + type: object + minItems: 1 + type: array + mode: + default: AddIfAbsent + description: Mode defines the header addition strategy. + enum: + - AddIfAbsent + - OverwriteOrAdd + type: string + name: + description: Name describing the configured operation. + minLength: 1 + type: string + required: + - headers + - name + type: object + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + allow: + description: |- + Allow defines which request headers will be forwarded to the upstream. + This can either be allHeaders or matchingHeaders. + Default: matchingHeaders: {...} + properties: + allHeaders: + description: AllHeaders specifies that all request headers should be forwarded. + type: object + matchingHeaders: + description: MatchingHeaders specifies which request headers should be forwarded. + properties: + builtIn: + description: BuiltIn allows configuring a set of predefined upstream request headers. + properties: + standardHeaders: + default: true + description: StandardHeaders defines whether the request headers which are forwarded to the upstream will be restricted to a set of common request headers. + type: boolean + type: object + custom: + description: Custom allows configuring additional upstream request headers. + items: + properties: + headers: + description: Headers to allow. + items: + description: |- + HeaderMatcher defines a matcher for an HTTP header. + At least one of name and value must be set. + properties: + name: + description: Name defines the name of a header. + properties: + matcher: + description: Matcher defines the way to match a string. In comparison to a normal StringMatcher, a value is always matched ignoring the case and can't be inverted. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + value: + description: Value defines the value of a header. + properties: + matcher: + description: StringMatcher defines the way to match a string. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + ignoreCase: + default: false + description: IgnoreCase indicates whether the matching should be case-insensitive. In case of a regex match, the regex gets wrapped with a group `(?i:...)`. + type: boolean + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + type: object + minItems: 1 + type: array + name: + description: Name describing the configured operation. Must be unique. + minLength: 1 + type: string + required: + - headers + - name + type: object + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + type: object + remove: + description: Remove defines which request headers will be removed before forwarding to the upstream. + properties: + builtIn: + description: BuiltIn allows configuring a set of predefined upstream request headers. + properties: + alternativeForwardedHeaders: + default: true + description: |- + AlternativeForwardedHeaders removes downstream request headers which could potentially + be abused to alter the upstream's view of the remote connection. + type: boolean + type: object + custom: + description: Custom allows configuring additional upstream request headers. + items: + properties: + headers: + description: Headers to remove. + items: + description: |- + HeaderMatcher defines a matcher for an HTTP header. + At least one of name and value must be set. + properties: + name: + description: Name defines the name of a header. + properties: + matcher: + description: Matcher defines the way to match a string. In comparison to a normal StringMatcher, a value is always matched ignoring the case and can't be inverted. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + value: + description: Value defines the value of a header. + properties: + matcher: + description: StringMatcher defines the way to match a string. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + ignoreCase: + default: false + description: IgnoreCase indicates whether the matching should be case-insensitive. In case of a regex match, the regex gets wrapped with a group `(?i:...)`. + type: boolean + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + type: object + minItems: 1 + type: array + name: + description: Name describing the configured operation. Must be unique. + minLength: 1 + type: string + required: + - headers + - name + type: object + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + type: object + response: + description: Response defines manipulations on upstream response headers. + properties: + add: + description: Add defines which response headers will be added before forwarding to the downstream. + properties: + builtIn: + description: BuiltIn allows configuring a set of predefined upstream response headers. + properties: + csp: + default: true + description: |- + CSP sets a content security policy which allows only same-origin requests except for images + if the 'Content-Security-Policy' header is not set by the upstream. + type: boolean + featurePolicy: + default: false + description: |- + FeaturePolicy sets a feature policy which prevents cross-origin use of several browser features + if the 'Feature-Policy' header is not set by the upstream. + **Deprecated:** Use permissionsPolicy instead. + type: boolean + hsts: + default: true + description: HSTS enforces the use of HTTPS if the 'Strict-Transport-Security' header is not already set by the upstream. + type: boolean + hstsPreload: + default: false + description: HSTSPreload enforces the use of HTTPS including for subdomains and enables HSTS preload. + type: boolean + permissionsPolicy: + default: true + description: |- + PermissionsPolicy sets a permissions policy which prevents cross-origin use of several browser features + if the 'Permissions-Policy' header is not set by the upstream. + type: boolean + referrerPolicy: + default: true + description: |- + ReferrerPolicy ensures that no 'Referer' header is sent for cross-origin requests + if the 'Referrer-Policy' header is not set by the upstream. + type: boolean + xContentTypeOptions: + default: true + description: XContentTypeOptions sets 'X-Content-Type-Options' to 'nosniff' if it is not set by the upstream. + type: boolean + xFrameOptions: + default: true + description: XFrameOptions sets 'X-Frame-Options' to SAMEORIGIN if it is not set by the upstream. + type: boolean + type: object + custom: + description: Custom allows configuring additional upstream response headers. + items: + properties: + headers: + description: Headers to add. + items: + description: HeaderRewritesHeader specifies a header with a particular value + properties: + name: + description: Name defines the name of a header. + minLength: 1 + type: string + value: + description: Value defines the value of a header. + type: string + required: + - name + - value + type: object + minItems: 1 + type: array + mode: + default: AddIfAbsent + description: Mode defines the header addition strategy. + enum: + - AddIfAbsent + - OverwriteOrAdd + type: string + name: + description: Name describing the configured operation. + minLength: 1 + type: string + required: + - headers + - name + type: object + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + allow: + description: |- + Allow defines which response headers will be forwarded to the downstream. + This can either be allHeaders or matchingHeaders. + Default: allHeaders: {} + properties: + allHeaders: + description: AllHeaders specifies that all response headers should be forwarded. + type: object + matchingHeaders: + description: MatchingHeaders specifies which response headers should be forwarded. + properties: + builtIn: + description: BuiltIn allows configuring a set of predefined upstream response header. + properties: + standardHeaders: + default: false + description: StandardHeaders defines whether the response headers which are forwarded to the downstream will be restricted to a set of common response headers. + type: boolean + type: object + custom: + description: Custom allows configuring additional upstream response headers. + items: + properties: + headers: + description: Headers to allow. + items: + description: |- + HeaderMatcher defines a matcher for an HTTP header. + At least one of name and value must be set. + properties: + name: + description: Name defines the name of a header. + properties: + matcher: + description: Matcher defines the way to match a string. In comparison to a normal StringMatcher, a value is always matched ignoring the case and can't be inverted. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + value: + description: Value defines the value of a header. + properties: + matcher: + description: StringMatcher defines the way to match a string. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + ignoreCase: + default: false + description: IgnoreCase indicates whether the matching should be case-insensitive. In case of a regex match, the regex gets wrapped with a group `(?i:...)`. + type: boolean + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + type: object + minItems: 1 + type: array + name: + description: Name describing the configured operation. Must be unique. + minLength: 1 + type: string + required: + - headers + - name + type: object + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + type: object + remove: + description: Remove defines which response headers will be removed before forwarding to the downstream. + properties: + builtIn: + description: BuiltIn allows configuring a set of predefined upstream response headers. + properties: + auth: + description: Auth defines the categories of headers concerning authentication. + properties: + basic: + default: false + description: Basic removes upstream response headers that advise clients to authenticate with Basic Authentication. + type: boolean + negotiate: + default: true + description: Negotiate removes upstream response headers that advise clients to authenticate with Negotiate. + type: boolean + ntlm: + default: true + description: |- + NTLM removes upstream response headers that advise clients to authenticate with NTLM. + By default, these headers are removed, because NTLM pass-through is not supported. + type: boolean + type: object + informationLeakage: + description: InformationLeakage defines the categories of headers concerning information leakage. + properties: + application: + default: true + description: Application removes upstream response headers that leak information about the deployed software. + type: boolean + server: + default: true + description: Server removes upstream response headers that leak information about the server. + type: boolean + type: object + permissiveCors: + default: true + description: PermissiveCORS removes upstream response headers for CORS (Cross-Origin Resource Sharing) which have no restrictions and therefore reduce client-side security. + type: boolean + type: object + custom: + description: Custom allows configuring additional upstream response headers. + items: + properties: + headers: + description: Headers to remove. + items: + description: |- + HeaderMatcher defines a matcher for an HTTP header. + At least one of name and value must be set. + properties: + name: + description: Name defines the name of a header. + properties: + matcher: + description: Matcher defines the way to match a string. In comparison to a normal StringMatcher, a value is always matched ignoring the case and can't be inverted. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + value: + description: Value defines the value of a header. + properties: + matcher: + description: StringMatcher defines the way to match a string. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + ignoreCase: + default: false + description: IgnoreCase indicates whether the matching should be case-insensitive. In case of a regex match, the regex gets wrapped with a group `(?i:...)`. + type: boolean + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + type: object + minItems: 1 + type: array + name: + description: Name describing the configured remove operation. Must be unique. + minLength: 1 + type: string + required: + - headers + - name + type: object + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + type: object + settings: + description: Settings configures the HeaderRewrites filter. + properties: + operationalMode: + default: Production + description: OperationalMode defines the behavior of the filter. In integration mode more information is logged about the requests and responses. + enum: + - Production + - Integration + type: string + type: object + type: object + type: object + served: true + storage: true diff --git a/charts/airlock/microgateway/4.3.3/crds/identitypropagations.microgateway.airlock.com.yaml b/charts/airlock/microgateway/4.3.3/crds/identitypropagations.microgateway.airlock.com.yaml new file mode 100644 index 0000000000..4610fe8b85 --- /dev/null +++ b/charts/airlock/microgateway/4.3.3/crds/identitypropagations.microgateway.airlock.com.yaml @@ -0,0 +1,108 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + labels: + app.kubernetes.io/name: airlock-microgateway-operator + app.kubernetes.io/version: 4.3.3 + name: identitypropagations.microgateway.airlock.com +spec: + group: microgateway.airlock.com + names: + categories: + - airlock-microgateway + kind: IdentityPropagation + listKind: IdentityPropagationList + plural: identitypropagations + singular: identitypropagation + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: IdentityPropagation specifies the desired identity propagation. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Specification of the desired identity propagation. + properties: + header: + description: Header configures identity propagation via a request header. + properties: + name: + description: Name of the header to set. + minLength: 1 + type: string + value: + description: Value to propagate to the application. + properties: + source: + description: Source from which to extract the value. + properties: + metadata: + description: Metadata specifies to extract a value from an Envoy dynamic filter metadata key. + properties: + key: + description: Key specifies the metadata key from which to load the value, e.g. `some_payload.aud`. + minLength: 1 + type: string + namespace: + description: Namespace specifies the metadata namespace within which the lookup should be performed, e.g. `envoy.filters.http.jwt_authn`. + minLength: 1 + type: string + required: + - key + - namespace + type: object + oidc: + description: OIDC specifies to extract a value from the result of an OpenID Connect flow. + properties: + idToken: + description: IDToken specifies to extract the value from the OpenID Connect ID Token. + properties: + claim: + description: Claim selects the JWT claim from which to extract the value. + minLength: 1 + type: string + required: + - claim + type: object + required: + - idToken + type: object + type: object + required: + - source + type: object + required: + - name + - value + type: object + type: object + required: + - spec + type: object + served: true + storage: true + subresources: {} diff --git a/charts/airlock/microgateway/4.3.3/crds/limits.microgateway.airlock.com.yaml b/charts/airlock/microgateway/4.3.3/crds/limits.microgateway.airlock.com.yaml new file mode 100644 index 0000000000..727b024968 --- /dev/null +++ b/charts/airlock/microgateway/4.3.3/crds/limits.microgateway.airlock.com.yaml @@ -0,0 +1,651 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + labels: + app.kubernetes.io/name: airlock-microgateway-operator + app.kubernetes.io/version: 4.3.3 + name: limits.microgateway.airlock.com +spec: + group: microgateway.airlock.com + names: + categories: + - airlock-microgateway + kind: Limits + listKind: LimitsList + plural: limits + singular: limits + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: Limits contains the configuration for limits. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Specification of the desired limits behavior. + properties: + request: + description: Request defines the limits for requests. + properties: + limited: + description: Limited enables limits on request scope. + properties: + exceptions: + description: Exceptions defines limit exceptions. + items: + description: LimitsException defines an exception for limits. + properties: + length: + description: Length defines an exception for length limits based on the data element exceeding the limit. + properties: + graphQL: + description: GraphQL defines a field, argument or value length limit exception for a GraphQL query. + properties: + argument: + description: |- + Argument restricts the exception to GraphQL queries with a matching argument of a field. + At least one of field, argument and value must be set. + properties: + matcher: + description: StringMatcher defines the way to match a string. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + ignoreCase: + default: false + description: IgnoreCase indicates whether the matching should be case-insensitive. In case of a regex match, the regex gets wrapped with a group `(?i:...)`. + type: boolean + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + field: + description: |- + Field restricts the exception to GraphQL queries with a matching field. + At least one of field, argument and value must be set. + properties: + matcher: + description: StringMatcher defines the way to match a string. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + ignoreCase: + default: false + description: IgnoreCase indicates whether the matching should be case-insensitive. In case of a regex match, the regex gets wrapped with a group `(?i:...)`. + type: boolean + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + value: + description: |- + Value restricts the exception to GraphQL queries with a matching argument value. + At least one of field, argument and value must be set. + properties: + matcher: + description: StringMatcher defines the way to match a string. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + ignoreCase: + default: false + description: IgnoreCase indicates whether the matching should be case-insensitive. In case of a regex match, the regex gets wrapped with a group `(?i:...)`. + type: boolean + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + type: object + json: + description: JSON defines a key and value length limit exception for a JSON property. + properties: + jsonPath: + description: |- + JSONPath restricts the exception to JSON properties with a matching JSONPath. + Expressions in JSONPath i.e. `?(expr)` are not supported. + minLength: 1 + type: string + required: + - jsonPath + type: object + parameter: + description: Parameter defines a name and value length limit exception for a parameter. + properties: + name: + description: Name restricts the exception to parameters with a matching name. + properties: + matcher: + description: StringMatcher defines the way to match a string. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + ignoreCase: + default: false + description: IgnoreCase indicates whether the matching should be case-insensitive. In case of a regex match, the regex gets wrapped with a group `(?i:...)`. + type: boolean + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + source: + default: Any + description: Source restricts the exception to parameters of this kind. + enum: + - Query + - Post + - Any + type: string + required: + - name + type: object + type: object + requestConditions: + description: RequestConditions defines additional request properties which must be matched in order for this exception to apply. + properties: + header: + description: Header defines the matching headers of a request. + properties: + name: + description: Name defines the name of a header. + properties: + matcher: + description: Matcher defines the way to match a string. In comparison to a normal StringMatcher, a value is always matched ignoring the case and can't be inverted. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + value: + description: Value defines the value of a header. + properties: + matcher: + description: StringMatcher defines the way to match a string. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + ignoreCase: + default: false + description: IgnoreCase indicates whether the matching should be case-insensitive. In case of a regex match, the regex gets wrapped with a group `(?i:...)`. + type: boolean + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + type: object + invert: + default: false + description: Invert indicates whether the request condition should be inverted. + type: boolean + mediaType: + description: MediaType defines the matching media type from the content-type header of a request. + properties: + matcher: + description: |- + NonInvertableCaseInsensitiveStringMatcher defines the way to match a string. + In comparison to a normal StringMatcher, a value is always matched ignoring the case and can't be inverted. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + method: + description: Method defines the matching methods of a request. + items: + description: Method defines common HTTP methods. + enum: + - GET + - HEAD + - POST + - PUT + - PATCH + - DELETE + - CONNECT + - OPTIONS + - TRACE + type: string + type: array + path: + description: Path defines the matching path of a request. + properties: + matcher: + description: StringMatcher defines the way to match a string. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + ignoreCase: + default: false + description: IgnoreCase indicates whether the matching should be case-insensitive. In case of a regex match, the regex gets wrapped with a group `(?i:...)`. + type: boolean + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + remoteIP: + description: RemoteIP defines the matching remote IPs of a request. + properties: + cidrRanges: + description: CIDRRanges defines the IPv4 or IPv6 CIDR ranges, e.g. ``196.148.3.128/26`` or ``2001:db8::/28``. + items: + description: CIDRRange defines an IPv4 or IPv6 CIDR range, e.g. “196.148.3.128/26“ or “2001:db8::/28“. + format: cidr + type: string + minItems: 1 + type: array + invert: + default: false + description: Invert indicates whether the match should be inverted. + type: boolean + required: + - cidrRanges + type: object + type: object + type: object + type: array + general: + description: General defines general request limits. + properties: + bodySize: + anyOf: + - type: integer + - type: string + default: 100Mi + description: BodySize limits the total size of the request body. It specifies the number of bytes (0 = unlimited). This limit is effective for any request not processed by one of the content parsers (e.g. json) as configured in the Parser CRD. **Note** This limit does not apply to WebSocket or gRPC traffic. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + pathLength: + anyOf: + - type: integer + - type: string + default: 1Ki + description: PathLength defines the maximum path length for all requests (parsed and unparsed). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + graphQL: + description: GraphQL defines the limits for GraphQL requests. + properties: + nestingDepth: + default: 10 + description: NestingDepth defines the maximum depth of nesting for GraphQL objects. + format: int64 + type: integer + querySize: + anyOf: + - type: integer + - type: string + default: 1Ki + description: QuerySize defines the maximum size for GraphQL queries. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + valueLength: + anyOf: + - type: integer + - type: string + default: "256" + description: ValueLength defines the maximum length for GraphQL values. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + json: + description: JSON defines the limits for JSON requests. + properties: + bodySize: + anyOf: + - type: integer + - type: string + default: 100Ki + description: BodySize limits the total size of the JSON request body. It specifies the number of bytes (0 = unlimited). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + elementCount: + default: 10000 + description: ElementCount defines the maximum number of keys and array items in the whole JSON document (recursive). + format: int64 + type: integer + keyCount: + default: 250 + description: KeyCount defines the maximum number of keys of a single JSON object (non-recursive). + format: int64 + type: integer + keyLength: + anyOf: + - type: integer + - type: string + default: "128" + description: KeyLength defines the maximum length for JSON keys. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + nestingDepth: + default: 100 + description: NestingDepth defines the maximum depth of nesting for JSON objects and JSON arrays. + format: int64 + type: integer + valueLength: + anyOf: + - type: integer + - type: string + default: 8Ki + description: ValueLength defines the maximum length for JSON values. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + multipart: + description: Multipart defines the limits for Multipart requests. + properties: + bodySize: + anyOf: + - type: integer + - type: string + default: 100Mi + description: BodySize limits the total size of the Multipart request body. It specifies the number of bytes (0 = unlimited). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + parameter: + description: Parameter defines the limits for request parameters. + properties: + bodySize: + anyOf: + - type: integer + - type: string + default: 100Ki + description: BodySize limits the total size of the form data body. It specifies the number of bytes (0 = unlimited). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + count: + default: 128 + description: Count defines the maximum number of request parameters. + format: int64 + type: integer + nameLength: + anyOf: + - type: integer + - type: string + default: "128" + description: NameLength defines the maximum length for parameter names. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + valueLength: + anyOf: + - type: integer + - type: string + default: 8Ki + description: ValueLength defines the maximum length for parameter values. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + unlimited: + description: Unlimited disables all limits on request scope. + type: object + type: object + settings: + description: Settings configures the limits filter. + properties: + threatHandlingMode: + default: Block + description: ThreatHandlingMode specifies how threats should be handled when a limit hits. + enum: + - Block + - LogOnly + type: string + type: object + type: object + type: object + served: true + storage: true diff --git a/charts/airlock/microgateway/4.3.3/crds/oidcproviders.microgateway.airlock.com.yaml b/charts/airlock/microgateway/4.3.3/crds/oidcproviders.microgateway.airlock.com.yaml new file mode 100644 index 0000000000..74acbf4da9 --- /dev/null +++ b/charts/airlock/microgateway/4.3.3/crds/oidcproviders.microgateway.airlock.com.yaml @@ -0,0 +1,305 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + labels: + app.kubernetes.io/name: airlock-microgateway-operator + app.kubernetes.io/version: 4.3.3 + name: oidcproviders.microgateway.airlock.com +spec: + group: microgateway.airlock.com + names: + categories: + - airlock-microgateway + kind: OIDCProvider + listKind: OIDCProviderList + plural: oidcproviders + singular: oidcprovider + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + OIDCProvider specifies an OpenID Provider (OP). + + + {{% notice warning %}} The OIDC feature is currently in an experimental state. + + + We encourage you to try it out and give feedback, but be aware that we do not recommend using it in a production environment yet, as security has not yet been hardened. + In particular, the current implementation has the following limitations, which we intend to address in future Microgateway releases: + - The state parameter is guessable. + - Sessions are always shared across all Microgateway Engines using the same Redis instance. + I.e. if application A and B (with different SidecarGateways) have the same Redis instance configured in their SessionHandling CR, users which are logged into application A + may be able to access authenticated routes on application B, even if their OIDCRelyingParty configuration differs. + + + {{% /notice %}} + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Specification of an OpenID Provider. + properties: + static: + description: Static configures an OpenID Provider by explicitly specifying all endpoints. + properties: + endpoints: + description: Endpoints specifies the OpenID Provider endpoints. + properties: + authorization: + description: Authorization specifies the endpoint to which the authorization request is sent. + properties: + uri: + description: URI specifies the endpoint address. + format: uri + minLength: 1 + pattern: ^(http|https)://.*$ + type: string + required: + - uri + type: object + token: + description: Token configures the endpoint from which the access, ID and refresh tokens are obtained. + properties: + tls: + description: TLS defines TLS settings. + properties: + certificateVerification: + description: CertificateVerification specifies how the certificate presented by the server is verified. + properties: + custom: + description: |- + Custom explicitly specifies how the server certificate should be verified. + Typical use cases include specifying a custom CA and SAN match when working with self-signed certificates or pinning a specific public key. + properties: + allowedSANs: + description: |- + AllowedSANs is a list of matchers to verify the Subject Alternative name. If specified, it will verify that the + Subject Alternative Name of the presented certificate matches one of the specified matchers. The matching uses “any” semantics, + that is to say, the SAN is verified if at least one matcher is matched. + AllowedSANs requires trustedCA to be set. + items: + description: |- + TLSValidationContextSANMatcher is a list of matchers to verify the Subject Alternative name. If specified, it will verify that the + Subject Alternative Name of the presented certificate matches one of the specified matchers. + properties: + matcher: + description: Matcher defines the string matcher for the SAN value. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + ignoreCase: + default: false + description: IgnoreCase indicates whether the matching should be case-insensitive. In case of a regex match, the regex gets wrapped with a group `(?i:...)`. + type: boolean + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + sanType: + description: SanType defines the type of SAN matcher. + enum: + - DNS + - Email + - URI + - IPAddress + type: string + required: + - matcher + - sanType + type: object + minItems: 1 + type: array + certificatePinning: + description: |- + CertificatePinning defines constraints the presented certificate must fulfill. + If more than one constraint is configured only one must be satisfied. + At least one of allowedSPKIs and allowedHashes must be set. + properties: + allowedHashes: + description: |- + AllowedHashes is a list of hex-encoded SHA-256 hashes. + If specified, it will verify that the SHA-256 of the DER-encoded presented certificate matches one of the specified values. + items: + type: string + minItems: 1 + type: array + allowedSPKIs: + description: |- + AllowedSPKIs is a list of base64-encoded SHA-256 hashes. + If specified, it will verify that the SHA-256 of the DER-encoded Subject Public Key Information (SPKI) of the presented certificate matches one of the specified values. + items: + type: string + minItems: 1 + type: array + type: object + crl: + description: CRL defines the Certificate Revocation List (CRL) settings. + properties: + lists: + description: Lists defines the list of secretRefs containing Certificate Revocation Lists. + items: + properties: + secretRef: + description: SecretRef defines the reference to a secret containing one or more CRL's (in PEM format) under the key 'ca.crl'. + properties: + name: + description: Name of the resource + minLength: 1 + type: string + required: + - name + type: object + required: + - secretRef + type: object + minItems: 1 + type: array + validationMode: + default: VerifyChain + description: ValidationMode defines whether only the leaf certificate or also the CA certs should be checked. + enum: + - VerifyLeafCertOnly + - VerifyChain + type: string + type: object + trustedCA: + description: TrustedCA defines which CA certificates are trusted. + properties: + certificates: + description: Certificates defines the list of secretRefs containing trusted CA certificates. + items: + properties: + secretRef: + description: SecretRef defines the reference to a secret containing one or more CA certificates under the key 'ca.crt'. + properties: + name: + description: Name of the resource + minLength: 1 + type: string + required: + - name + type: object + required: + - secretRef + type: object + minItems: 1 + type: array + verificationDepth: + default: 1 + description: |- + VerificationDepth specifies the hops in the certificate chain at which validation is performed. + 1 means that either the leaf or the signing CA must be in the set of trusted certificates. + format: int32 + type: integer + required: + - certificates + type: object + type: object + disabled: + description: |- + Disabled specifies to trust any certificate without verification. + THIS IS INSECURE AND SHOULD ONLY BE USED FOR TESTING. + type: object + publicCAs: + description: PublicCAs specifies to only accept certificates with a SAN matching "uri" and which are signed by a CA which is either directly or indirectly trusted by any of the root CA certificates shipped with the Airlock Microgateway Engine's base image. + type: object + type: object + ciphers: + description: Ciphers defines a list of the supported TLS cipher suites. For details on cipher list refer to the envoy documentation on cipher_suites in common tls configuration. + items: + type: string + minItems: 1 + type: array + protocol: + description: Protocol defines the supported TLS protocol versions. + properties: + maximum: + description: Maximum supported TLS version. + enum: + - TLSv1_0 + - TLSv1_1 + - TLSv1_2 + - TLSv1_3 + type: string + minimum: + description: Minimum supported TLS version. + enum: + - TLSv1_0 + - TLSv1_1 + - TLSv1_2 + - TLSv1_3 + type: string + type: object + type: object + uri: + description: URI specifies the endpoint address. + format: uri + minLength: 1 + pattern: ^(http|https)://.*$ + type: string + required: + - uri + type: object + required: + - authorization + - token + type: object + required: + - endpoints + type: object + type: object + required: + - spec + type: object + served: true + storage: true + subresources: {} diff --git a/charts/airlock/microgateway/4.3.3/crds/oidcrelyingparties.microgateway.airlock.com.yaml b/charts/airlock/microgateway/4.3.3/crds/oidcrelyingparties.microgateway.airlock.com.yaml new file mode 100644 index 0000000000..baa26ebcc4 --- /dev/null +++ b/charts/airlock/microgateway/4.3.3/crds/oidcrelyingparties.microgateway.airlock.com.yaml @@ -0,0 +1,224 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + labels: + app.kubernetes.io/name: airlock-microgateway-operator + app.kubernetes.io/version: 4.3.3 + name: oidcrelyingparties.microgateway.airlock.com +spec: + group: microgateway.airlock.com + names: + categories: + - airlock-microgateway + kind: OIDCRelyingParty + listKind: OIDCRelyingPartyList + plural: oidcrelyingparties + singular: oidcrelyingparty + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + OIDCRelyingParty specifies how the Airlock Microgateway Engine interacts with an OpenID Provider (OP). + + + {{% notice warning %}} The OIDC feature is currently in an experimental state. + + + We encourage you to try it out and give feedback, but be aware that we do not recommend using it in a production environment yet, as security has not yet been hardened. + In particular, the current implementation has the following limitations, which we intend to address in future Microgateway releases: + - The state parameter is guessable. + - Sessions are always shared across all Microgateway Engines using the same Redis instance. + I.e. if application A and B (with different SidecarGateways) have the same Redis instance configured in their SessionHandling CR, users which are logged into application A + may be able to access authenticated routes on application B, even if their OIDCRelyingParty configuration differs. + + + {{% /notice %}} + {{% notice info %}} The OIDC feature requires SessionHandling to be configured in the SidecarGateway. {{% /notice %}} + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Specification of the OIDC Relying Party configuration. + properties: + clientID: + description: ClientID specifies the OIDCRelyingParty "client_id". + minLength: 1 + type: string + credentials: + description: Credentials used for client authentication on the back-channel with the authorization server. + properties: + clientSecret: + description: ClientSecret authenticates with the client password issued by the OpenID Provider (OP). + properties: + method: + default: BasicAuth + description: Method specifies in which format the client secret is sent with the authorization request. + enum: + - BasicAuth + - FormURLEncoded + type: string + secretRef: + description: SecretRef specifies the kubernetes secret containing the client password with key "client.secret". + properties: + name: + description: Name of the resource + minLength: 1 + type: string + required: + - name + type: object + required: + - secretRef + type: object + required: + - clientSecret + type: object + oidcProviderRef: + description: OIDCProviderRef selects the OpenID Provider (OP) used to authenticate users. + properties: + name: + description: Name of the resource + minLength: 1 + type: string + required: + - name + type: object + pathMapping: + description: PathMapping configures the action matching. + properties: + logoutPath: + description: LogoutPath specifies which request paths should initiate a logout. + properties: + matcher: + description: StringMatcher defines the way to match a string. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + ignoreCase: + default: false + description: IgnoreCase indicates whether the matching should be case-insensitive. In case of a regex match, the regex gets wrapped with a group `(?i:...)`. + type: boolean + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + redirectPath: + description: RedirectPath specifies which request paths should be interpreted as a response from the authorization endpoint. + properties: + matcher: + description: StringMatcher defines the way to match a string. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + ignoreCase: + default: false + description: IgnoreCase indicates whether the matching should be case-insensitive. In case of a regex match, the regex gets wrapped with a group `(?i:...)`. + type: boolean + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + required: + - logoutPath + - redirectPath + type: object + redirectURI: + description: |- + RedirectURI configures the "redirect_uri" parameter included in the authorization request. + May contain envoy command operators, e.g. '%REQ(:x-forwarded-proto)%://%REQ(:authority)%/callback'. + minLength: 1 + type: string + required: + - clientID + - credentials + - oidcProviderRef + - pathMapping + - redirectURI + type: object + required: + - spec + type: object + served: true + storage: true + subresources: {} diff --git a/charts/airlock/microgateway/4.3.3/crds/openapis.microgateway.airlock.com.yaml b/charts/airlock/microgateway/4.3.3/crds/openapis.microgateway.airlock.com.yaml new file mode 100644 index 0000000000..1c09287100 --- /dev/null +++ b/charts/airlock/microgateway/4.3.3/crds/openapis.microgateway.airlock.com.yaml @@ -0,0 +1,167 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + labels: + app.kubernetes.io/name: airlock-microgateway-operator + app.kubernetes.io/version: 4.3.3 + name: openapis.microgateway.airlock.com +spec: + group: microgateway.airlock.com + names: + categories: + - airlock-microgateway + kind: OpenAPI + listKind: OpenAPIList + plural: openapis + singular: openapi + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: OpenAPI contains the configuration for the OpenAPI specification. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Specification of the desired OpenAPI specification. + properties: + response: + description: Response defines the validation behaviour for responses. + properties: + secured: + description: Secured enables response checking. + properties: + validation: + default: Lax + description: Validation defines the validation mode for responses. + enum: + - Lax + - Strict + type: string + type: object + unsecured: + description: Unsecured disables response checking. + type: object + type: object + settings: + description: Settings defines the settings to configure OpenAPI specification enforcement. + properties: + logging: + description: Logging specifies the access log behavior. + properties: + maxFailedSubvalidations: + default: 10 + description: MaxFailedSubvalidations defines the maximum number of failed subvalidations being logged. + format: int64 + type: integer + type: object + schema: + description: Schema configures the OpenAPI specification. + properties: + source: + description: Source specifies the OpenAPI specification to be enforced. + properties: + configMapRef: + description: ConfigMapRef references the configmap by its name containing the well-known key 'openapi.json'. + properties: + name: + description: Name of the resource + minLength: 1 + type: string + required: + - name + type: object + type: object + required: + - source + type: object + threatHandlingMode: + default: Block + description: ThreatHandlingMode specifies how threats should be handled. + enum: + - Block + - LogOnly + type: string + validation: + description: Validation specifies the patterns for the validation behavior. + properties: + authentication: + description: Authentication defines the settings for the authentication scheme. + properties: + oAuth2: + description: OAuth2 specifies the OAuth2 parameters. + properties: + allowedParameters: + description: AllowedParameters specifies the allowed parameters for the authentication scheme. + properties: + builtIn: + description: BuiltIn allows configuring a set of predefined allowed parameters. + properties: + standardParameters: + default: true + description: StandardParameters defines whether the allowed parameters should be expanded by the set of common parameters. + type: boolean + type: object + custom: + description: Custom allows configuring additional allowed parameters. + items: + minLength: 1 + type: string + minItems: 1 + type: array + type: object + type: object + oidc: + description: Oidc specifies the OIDC parameters. + properties: + allowedParameters: + description: AllowedParameters specifies the allowed parameters for the authentication scheme. + properties: + builtIn: + description: BuiltIn allows configuring a set of predefined allowed parameters. + properties: + standardParameters: + default: true + description: StandardParameters defines whether the allowed parameters should be expanded by the set of common parameters. + type: boolean + type: object + custom: + description: Custom allows configuring additional allowed parameters. + items: + minLength: 1 + type: string + minItems: 1 + type: array + type: object + type: object + type: object + type: object + required: + - schema + type: object + required: + - settings + type: object + required: + - spec + type: object + served: true + storage: true diff --git a/charts/airlock/microgateway/4.3.3/crds/parsers.microgateway.airlock.com.yaml b/charts/airlock/microgateway/4.3.3/crds/parsers.microgateway.airlock.com.yaml new file mode 100644 index 0000000000..db60b6c845 --- /dev/null +++ b/charts/airlock/microgateway/4.3.3/crds/parsers.microgateway.airlock.com.yaml @@ -0,0 +1,358 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + labels: + app.kubernetes.io/name: airlock-microgateway-operator + app.kubernetes.io/version: 4.3.3 + name: parsers.microgateway.airlock.com +spec: + group: microgateway.airlock.com + names: + categories: + - airlock-microgateway + kind: Parser + listKind: ParserList + plural: parsers + singular: parser + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: Parser contains the configuration for content parsers (default and custom). + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Specification of the desired parser behavior. + properties: + request: + description: Request defines the parsing for downstream requests. + properties: + custom: + description: Custom allows configuring additional rules for parser selection. + properties: + rules: + description: |- + Rules defines a custom set prepended before built-in rules of enabled request parsers. + Disable all built-in parsers to overrule them completely. + items: + properties: + action: + description: |- + Action specifies what should happen when a request condition matches. + Only one of parse or skip can be set. + properties: + parse: + description: Parse activates the configured parser. + properties: + form: + description: Form activates the Form parser. + type: object + json: + description: JSON activates the JSON parser. + type: object + multipart: + description: Multipart activates the multipart parser. + type: object + type: object + skip: + description: Skip disables any content parsing + type: object + type: object + requestConditions: + description: RequestConditions defines additional request properties which must be matched in order for this rule to apply. + properties: + header: + description: Header defines the matching headers of a request. + properties: + name: + description: Name defines the name of a header. + properties: + matcher: + description: Matcher defines the way to match a string. In comparison to a normal StringMatcher, a value is always matched ignoring the case and can't be inverted. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + value: + description: Value defines the value of a header. + properties: + matcher: + description: StringMatcher defines the way to match a string. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + ignoreCase: + default: false + description: IgnoreCase indicates whether the matching should be case-insensitive. In case of a regex match, the regex gets wrapped with a group `(?i:...)`. + type: boolean + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + type: object + invert: + default: false + description: Invert indicates whether the request condition should be inverted. + type: boolean + mediaType: + description: MediaType defines the matching media type from the content-type header of a request. + properties: + matcher: + description: |- + NonInvertableCaseInsensitiveStringMatcher defines the way to match a string. + In comparison to a normal StringMatcher, a value is always matched ignoring the case and can't be inverted. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + method: + description: Method defines the matching methods of a request. + items: + description: Method defines common HTTP methods. + enum: + - GET + - HEAD + - POST + - PUT + - PATCH + - DELETE + - CONNECT + - OPTIONS + - TRACE + type: string + type: array + path: + description: Path defines the matching path of a request. + properties: + matcher: + description: StringMatcher defines the way to match a string. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + ignoreCase: + default: false + description: IgnoreCase indicates whether the matching should be case-insensitive. In case of a regex match, the regex gets wrapped with a group `(?i:...)`. + type: boolean + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + required: + - matcher + type: object + remoteIP: + description: RemoteIP defines the matching remote IPs of a request. + properties: + cidrRanges: + description: CIDRRanges defines the IPv4 or IPv6 CIDR ranges, e.g. ``196.148.3.128/26`` or ``2001:db8::/28``. + items: + description: CIDRRange defines an IPv4 or IPv6 CIDR range, e.g. “196.148.3.128/26“ or “2001:db8::/28“. + format: cidr + type: string + minItems: 1 + type: array + invert: + default: false + description: Invert indicates whether the match should be inverted. + type: boolean + required: + - cidrRanges + type: object + type: object + required: + - action + - requestConditions + type: object + type: array + type: object + defaultContentType: + default: application/x-www-form-urlencoded + description: DefaultContentType specifies the content-type header which should be injected into the request before parser selection if it is not already present and the request has a body. + minLength: 1 + type: string + parsers: + description: Parsers defines the configuration for the available content parsers. + properties: + form: + description: Form defines the configuration for the form parser. + properties: + enable: + default: true + description: Enable defines whether form payloads are inspected. + type: boolean + mediaTypePattern: + default: .*urlencoded.* + description: MediaTypePattern is a regex specifying the media types for which the request body should be treated as form arguments. + minLength: 1 + type: string + type: object + json: + description: JSON defines the configuration for the JSON parser. + properties: + enable: + default: true + description: Enable defines whether json payloads are inspected. + type: boolean + mediaTypePattern: + default: .*json.* + description: MediaTypePattern is a regex specifying the media types for which the request body should be treated as JSON. + minLength: 1 + type: string + type: object + multipart: + description: Multipart defines the configuration for the multipart parser. + properties: + enable: + default: true + description: Enable defines whether multipart payloads are inspected. + type: boolean + mediaTypePattern: + default: .*multipart.* + description: MediaTypePattern is a regex specifying the media types for which the request body should be treated as a multipart payload. + minLength: 1 + type: string + type: object + type: object + type: object + type: object + type: object + served: true + storage: true diff --git a/charts/airlock/microgateway/4.3.3/crds/redisproviders.microgateway.airlock.com.yaml b/charts/airlock/microgateway/4.3.3/crds/redisproviders.microgateway.airlock.com.yaml new file mode 100644 index 0000000000..8c662a2d08 --- /dev/null +++ b/charts/airlock/microgateway/4.3.3/crds/redisproviders.microgateway.airlock.com.yaml @@ -0,0 +1,159 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + labels: + app.kubernetes.io/name: airlock-microgateway-operator + app.kubernetes.io/version: 4.3.3 + name: redisproviders.microgateway.airlock.com +spec: + group: microgateway.airlock.com + names: + categories: + - airlock-microgateway + kind: RedisProvider + listKind: RedisProviderList + plural: redisproviders + singular: redisprovider + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: RedisProvider contains a client configuration for connecting to a Redis database. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Specification of a Redis database client configuration. + properties: + auth: + description: Auth specifies the Redis credentials. + properties: + password: + description: Password specifies the Redis password. + properties: + secretRef: + description: SecretRef selects the secret containing the Redis password under the key 'redis.password'. + properties: + name: + description: Name of the resource + minLength: 1 + type: string + required: + - name + type: object + required: + - secretRef + type: object + username: + default: default + description: Username specifies the Redis username to authenticate with. + minLength: 1 + pattern: ^[^\s]+$ + type: string + required: + - password + type: object + mode: + description: Mode configures the redis deployment mode. + properties: + standalone: + description: Standalone specifies the standalone Redis instance to connect to. + properties: + host: + description: Host specifies the IP or hostname. + minLength: 1 + pattern: ^(\d{1,3}(\.\d{1,3}){3}|([0-9a-fA-F]{1,4}|:)+(:\d{1,3}(\.\d{1,3}){3})?|[a-z0-9\-]+(\.[a-z0-9\-]+)*)$ + type: string + port: + default: 6379 + description: Port specifies the port. + maximum: 65535 + minimum: 1 + type: integer + required: + - host + type: object + type: object + timeouts: + description: Timeouts specifies the timeouts when interacting with the Redis endpoint. + properties: + connect: + default: 5s + description: Connect specifies the timeout for establishing a connection. + type: string + maxDuration: + default: 2s + description: MaxDuration specifies the response timeout. + type: string + type: object + tls: + description: TLS defines TLS settings. If not specified, TLS is disabled i.e. unencrypted TCP is used when connecting to the Redis instance. + properties: + certificateVerification: + description: CertificateVerification specifies how the certificate presented by the server is verified. + properties: + custom: + description: Custom explicitly specifies how the server certificate should be verified. + properties: + trustedCA: + description: TrustedCA defines which CA certificates are trusted. + properties: + certificates: + description: Certificates defines the list of secretRefs containing trusted CA certificates. + items: + properties: + secretRef: + description: SecretRef defines the reference to a secret containing one or more CA certificates under the key 'ca.crt'. + properties: + name: + description: Name of the resource + minLength: 1 + type: string + required: + - name + type: object + required: + - secretRef + type: object + minItems: 1 + type: array + required: + - certificates + type: object + required: + - trustedCA + type: object + disabled: + description: 'Disabled specifies to trust any certificate without verification. THIS IS INSECURE AND SHOULD ONLY BE USED FOR TESTING. Note: This setting currently also disables TLS SNI.' + type: object + publicCAs: + description: PublicCAs specifies to only accept certificates with a SAN matching the host and which are signed by a CA which is either directly or indirectly trusted by any of the root CA certificates shipped with the Airlock Microgateway Session Agent’s base image. + type: object + type: object + type: object + required: + - mode + type: object + required: + - spec + type: object + served: true + storage: true diff --git a/charts/airlock/microgateway/4.3.3/crds/sessionhandlings.microgateway.airlock.com.yaml b/charts/airlock/microgateway/4.3.3/crds/sessionhandlings.microgateway.airlock.com.yaml new file mode 100644 index 0000000000..72747df777 --- /dev/null +++ b/charts/airlock/microgateway/4.3.3/crds/sessionhandlings.microgateway.airlock.com.yaml @@ -0,0 +1,77 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + labels: + app.kubernetes.io/name: airlock-microgateway-operator + app.kubernetes.io/version: 4.3.3 + name: sessionhandlings.microgateway.airlock.com +spec: + group: microgateway.airlock.com + names: + categories: + - airlock-microgateway + kind: SessionHandling + listKind: SessionHandlingList + plural: sessionhandlings + singular: sessionhandling + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + SessionHandling contains the configuration for session handling. + + + {{% notice warning %}} The Session Handling feature (required for OIDC) is currently in an experimental state. + + + We encourage you to try it out and give feedback, but be aware that we do not recommend using it in a production environment yet, as high-availability Redis configurations (e.g. Sentinel/Cluster) are not yet supported. + {{% /notice %}} + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Specification of the desired session handling behavior. + properties: + persistence: + description: Persistence configures where to store the session state. + properties: + redisProviderRef: + description: RedisProviderRef specifies to cache session information in the provided Redis instance. + properties: + name: + description: Name of the resource + minLength: 1 + type: string + required: + - name + type: object + required: + - redisProviderRef + type: object + required: + - persistence + type: object + required: + - spec + type: object + served: true + storage: true diff --git a/charts/airlock/microgateway/4.3.3/crds/sidecargateways.microgateway.airlock.com.yaml b/charts/airlock/microgateway/4.3.3/crds/sidecargateways.microgateway.airlock.com.yaml new file mode 100644 index 0000000000..6e1c04a489 --- /dev/null +++ b/charts/airlock/microgateway/4.3.3/crds/sidecargateways.microgateway.airlock.com.yaml @@ -0,0 +1,758 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + labels: + app.kubernetes.io/name: airlock-microgateway-operator + app.kubernetes.io/version: 4.3.3 + name: sidecargateways.microgateway.airlock.com +spec: + group: microgateway.airlock.com + names: + categories: + - airlock-microgateway + kind: SidecarGateway + listKind: SidecarGatewayList + plural: sidecargateways + singular: sidecargateway + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.status + name: Status + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: SidecarGateway contains the configuration how to configure the Airlock Microgateway Engine when used as Sidecar Container within the Pod of an application. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Specification of the desired sidecar gateway behavior. + properties: + applications: + description: Applications defines applications which run on different ports. + items: + properties: + containerPort: + default: 8080 + description: |- + ContainerPort refers to the container port. + This must be a valid port number, 0 < x < 65536. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + downstream: + description: Downstream defines the downstream configuration for this application + properties: + protocol: + description: |- + Protocol defines the exposed HTTP protocol version. At most one of http1, http2 and auto can be set. + Default: auto: {} + properties: + auto: + description: Auto specifies that the protocol should be inferred. + properties: + http2: + description: HTTP2 specifies the settings for when HTTP/2 is inferred. + properties: + allowConnect: + default: false + description: Allows proxying Websocket and other upgrades over H2 connect. + type: boolean + type: object + type: object + http1: + description: HTTP1 specifies that the client is assumed to speak HTTP/1.1. + type: object + http2: + description: HTTP2 specifies that the client is assumed to speak HTTP/2. + properties: + allowConnect: + default: false + description: Allows proxying Websocket and other upgrades over H2 connect. + type: boolean + type: object + type: object + remoteIP: + description: |- + RemoteIP defines how the remote IP of a client is propagated. + Default: xff: {...} + properties: + connectionIP: + description: ConnectionIP configures to use the source IP address of the direct downstream connection. + type: object + customHeader: + description: CustomHeader specifies to use a custom header for remote IP extraction. + properties: + headerName: + description: HeaderName specifies the name of the custom header containing the remote IP. + minLength: 1 + type: string + required: + default: true + description: Required specifies if the custom header is required. If true and not available the request will be rejected with 403. + type: boolean + required: + - headerName + type: object + xff: + description: XFF configures to use the standard 'X-Forwarded-For' header for IP extraction. + properties: + numTrustedHops: + default: 1 + description: NumTrustedHops specifies to extract the client's originating IP from the nth rightmost entry in the X-Forwarded-For header. With the default value of 1, the IP is extracted from the rightmost entry. + format: int32 + minimum: 1 + type: integer + type: object + type: object + requestNormalizations: + description: RequestNormalizations defines a set of normalization actions which are applied to the request before route matching. + properties: + mergeSlashes: + default: true + description: MergeSlashes ensures that adjacent slashes in the path are merged into one. + type: boolean + normalizePath: + default: true + description: NormalizePath ensures normalization according to RFC 3986 without case normalization. + type: boolean + type: object + restrictions: + description: Restrictions defines restrictions for downstream. + properties: + http: + description: HTTP defines limits for the HTTP protocol. + properties: + headersLength: + anyOf: + - type: integer + - type: string + default: 60Ki + description: HeadersLength defines maximum size of all request headers combined. Requests that exceed this limit will receive a 431 response. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + timeouts: + description: Timeouts defines timeouts for downstream + properties: + http: + description: HTTP defines the settings for HTTP timeouts. + properties: + idle: + default: 5m + description: |- + Idle defines the settings for the idle timeout when no data is sent or received. + A value of 0 will completely disable the timeout. + Default: 5m + type: string + maxDuration: + default: 5m + description: |- + MaxDuration defines the total duration for a HTTP request/response stream. + A value of 0 will completely disable the timeout. + Default: 5m + type: string + requestHeaders: + default: 10s + description: |- + RequestHeaders defines the duration before all request headers must be received. + A value of 0 will completely disable the timeout. + Default: 10s + type: string + type: object + type: object + tls: + description: TLS defines the TLS settings. + properties: + ciphers: + description: Ciphers defines a list of the supported TLS cipher suites. For details on cipher list refer to the envoy documentation on cipher_suites in common tls configuration. + items: + type: string + minItems: 1 + type: array + clientCertificate: + description: |- + ClientCertificate defines the TLS settings for verification of client certificates. + At most one of ignored, optional and required can be set. + Default: ignored: {} + properties: + ignored: + description: Ignored disables verification of the client certificate. + type: object + optional: + description: |- + Optional enables verification of the client certificate if one is presented. + In this mode only trustedCA and crl settings can be configured since certificatePinning and allowedSANs require a client certificate. + properties: + crl: + description: CRL defines the Certificate Revocation List (CRL) settings. + properties: + lists: + description: Lists defines the list of secretRefs containing Certificate Revocation Lists. + items: + properties: + secretRef: + description: SecretRef defines the reference to a secret containing one or more CRL's (in PEM format) under the key 'ca.crl'. + properties: + name: + description: Name of the resource + minLength: 1 + type: string + required: + - name + type: object + required: + - secretRef + type: object + minItems: 1 + type: array + validationMode: + default: VerifyChain + description: ValidationMode defines whether only the leaf certificate or also the CA certs should be checked. + enum: + - VerifyLeafCertOnly + - VerifyChain + type: string + type: object + trustedCA: + description: TrustedCA defines which CA certificates are trusted. + properties: + certificates: + description: Certificates defines the list of secretRefs containing trusted CA certificates. + items: + properties: + secretRef: + description: SecretRef defines the reference to a secret containing one or more CA certificates under the key 'ca.crt'. + properties: + name: + description: Name of the resource + minLength: 1 + type: string + required: + - name + type: object + required: + - secretRef + type: object + minItems: 1 + type: array + verificationDepth: + default: 1 + description: |- + VerificationDepth specifies the hops in the certificate chain at which validation is performed. + 1 means that either the leaf or the signing CA must be in the set of trusted certificates. + format: int32 + type: integer + required: + - certificates + type: object + required: + - trustedCA + type: object + required: + description: |- + Required contains settings for client certificate verification. A client must present a valid certificate. + At least one of trustedCA and certificatePinning must be set. + properties: + allowedSANs: + description: |- + AllowedSANs is a list of matchers to verify the Subject Alternative name. If specified, it will verify that the + Subject Alternative Name of the presented certificate matches one of the specified matchers. The matching uses “any” semantics, + that is to say, the SAN is verified if at least one matcher is matched. + AllowedSANs requires trustedCA to be set. + items: + description: |- + TLSValidationContextSANMatcher is a list of matchers to verify the Subject Alternative name. If specified, it will verify that the + Subject Alternative Name of the presented certificate matches one of the specified matchers. + properties: + matcher: + description: Matcher defines the string matcher for the SAN value. + properties: + contains: + description: |- + Contains defines a substring match on the substring specified here. Empty contains match is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + exact: + description: |- + Exact defines an explicit match on the string specified here. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + ignoreCase: + default: false + description: IgnoreCase indicates whether the matching should be case-insensitive. In case of a regex match, the regex gets wrapped with a group `(?i:...)`. + type: boolean + prefix: + description: |- + Prefix defines a prefix match on the prefix specified here. Empty prefix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + regex: + description: |- + Regex defines a regex match on the regular expression specified here. Google's [RE2 regex engine](https://github.com/google/re2/wiki/Syntax) is used. + The regex matches only single-line by default, even with ".*". To match a multi-line string prepend (?s) to your regex. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + suffix: + description: |- + Suffix defines a suffix match on the suffix specified here. Empty suffix is not allowed, please use regex instead. + Only one of exact, prefix, suffix, regex or contains can be set. + minLength: 1 + type: string + type: object + sanType: + description: SanType defines the type of SAN matcher. + enum: + - DNS + - Email + - URI + - IPAddress + type: string + required: + - matcher + - sanType + type: object + minItems: 1 + type: array + certificatePinning: + description: |- + CertificatePinning defines the constraints a client certificate must fulfill. + If more than one constraint is configured only one must be satisfied. + At least one of allowedSPKIs and allowedHashes must be set. + properties: + allowedHashes: + description: |- + AllowedHashes is a list of hex-encoded SHA-256 hashes. + If specified, it will verify that the SHA-256 of the DER-encoded presented certificate matches one of the specified values. + items: + type: string + minItems: 1 + type: array + allowedSPKIs: + description: |- + AllowedSPKIs is a list of base64-encoded SHA-256 hashes. + If specified, it will verify that the SHA-256 of the DER-encoded Subject Public Key Information (SPKI) of the presented certificate matches one of the specified values. + items: + type: string + minItems: 1 + type: array + type: object + crl: + description: CRL defines the Certificate Revocation List (CRL) settings. + properties: + lists: + description: Lists defines the list of secretRefs containing Certificate Revocation Lists. + items: + properties: + secretRef: + description: SecretRef defines the reference to a secret containing one or more CRL's (in PEM format) under the key 'ca.crl'. + properties: + name: + description: Name of the resource + minLength: 1 + type: string + required: + - name + type: object + required: + - secretRef + type: object + minItems: 1 + type: array + validationMode: + default: VerifyChain + description: ValidationMode defines whether only the leaf certificate or also the CA certs should be checked. + enum: + - VerifyLeafCertOnly + - VerifyChain + type: string + type: object + trustedCA: + description: TrustedCA defines which CA certificates are trusted. + properties: + certificates: + description: Certificates defines the list of secretRefs containing trusted CA certificates. + items: + properties: + secretRef: + description: SecretRef defines the reference to a secret containing one or more CA certificates under the key 'ca.crt'. + properties: + name: + description: Name of the resource + minLength: 1 + type: string + required: + - name + type: object + required: + - secretRef + type: object + minItems: 1 + type: array + verificationDepth: + default: 1 + description: |- + VerificationDepth specifies the hops in the certificate chain at which validation is performed. + 1 means that either the leaf or the signing CA must be in the set of trusted certificates. + format: int32 + type: integer + required: + - certificates + type: object + type: object + type: object + enable: + default: false + description: Enable defines if the downstream connection is encrypted. + type: boolean + protocol: + description: Protocol defines the supported TLS protocol versions. + properties: + maximum: + description: Maximum supported TLS version. + enum: + - TLSv1_0 + - TLSv1_1 + - TLSv1_2 + - TLSv1_3 + type: string + minimum: + description: Minimum supported TLS version. + enum: + - TLSv1_0 + - TLSv1_1 + - TLSv1_2 + - TLSv1_3 + type: string + type: object + secretRef: + description: SecretRef defines the reference to the TLS server certificate (secret of type kubernetes.io/tls). + properties: + name: + description: Name of the resource + minLength: 1 + type: string + required: + - name + type: object + xfcc: + description: |- + XFCC defines the handling of X-Forwarded-Client-Cert header. Meaning of the possible values: + _Sanitize_: Do not send the XFCC header to the next hop. This is the default value. + _ForwardOnly_: When the client connection is mTLS (Mutual TLS), forward the XFCC header in the request. + _AppendAndForward_: When the client connection is mTLS, append the client certificate information to the request’s XFCC header and forward it. + _SanitizeAndSet_: When the client connection is mTLS, reset the XFCC header with the client certificate information and send it to the next hop. + _AlwaysForwardOnly_: Always forward the XFCC header in the request, regardless of whether the client connection is mTLS. + Note: When forwarding the XFCC header in the request you might have to adjust the header length restrictions (See sidecargateway.spec.applications.downstream.restrictions.http) + enum: + - Sanitize + - ForwardOnly + - AppendAndForward + - SanitizeAndSet + - AlwaysForwardOnly + type: string + type: object + type: object + envoyHTTPFilterRefs: + description: EnvoyHTTPFilterRefs selects the relevant EnvoyHTTPFilters. + properties: + prepend: + description: Prepend selects the relevant EnvoyHTTPFilters which are added before those configured by the Airlock Microgateway. + items: + properties: + name: + description: Name of the resource + minLength: 1 + type: string + required: + - name + type: object + type: array + type: object + routes: + description: Routes defines the security configurations for different paths. The first matching route (from top to bottom) applies. + items: + description: |- + SidecarGatewayApplicationRoute defines the security configurations for different paths. + At most one of secured and unsecured can be set. + Default: secured: {...} + properties: + pathPrefix: + default: / + description: PathPrefix defines the path prefix used during route selection. + minLength: 1 + type: string + secured: + description: Secured enables WAF processing for this route. + properties: + accessControlRef: + description: |- + AccessControlRef selects the relevant AccessControl configuration resource. + If undefined, Airlock Microgateway does not perform any access control. + properties: + name: + description: Name of the resource + minLength: 1 + type: string + required: + - name + type: object + contentSecurityRef: + description: |- + ContentSecurityRef selects the relevant ContentSecurity configuration resource. + If undefined, default settings are applied, designed to work with most upstream web application services. + properties: + name: + description: Name of the resource + minLength: 1 + type: string + required: + - name + type: object + type: object + unsecured: + description: |- + Unsecured disables all WAF functionality and therefore protection for this route. + WARNING: Using this setting when the application is exposed to untrusted downstream traffic is highly discouraged. + type: object + type: object + type: array + x-kubernetes-list-map-keys: + - pathPrefix + x-kubernetes-list-type: map + telemetryRef: + description: |- + TelemetryRef selects the relevant Telemetry configuration resource. + If undefined, default settings are applied, designed to work with most upstream web application services. + properties: + name: + description: Name of the resource + minLength: 1 + type: string + required: + - name + type: object + upstream: + description: Upstream defines the upstream configuration for this application + properties: + protocol: + description: |- + Protocol defines HTTP protocol version used to communicate with the upstream. At most one of http1, http2 and auto can be set. + Default: auto: {} + properties: + auto: + description: Auto specifies to negotiate the protocol with TLS ALPN (if TLS is enabled) or, as a fallback, use the same protocol that is used by the downstream connection. + properties: + http2: + description: HTTP2 specifies the settings for when HTTP/2 is inferred. + properties: + allowConnect: + default: false + description: Allows proxying Websocket and other upgrades over H2 connect. + type: boolean + type: object + type: object + http1: + description: HTTP1 specifies to use HTTP/1.1. + type: object + http2: + description: HTTP2 specifies to use HTTP/2. + properties: + allowConnect: + default: false + description: Allows proxying Websocket and other upgrades over H2 connect. + type: boolean + type: object + type: object + timeouts: + description: Timeouts defines the timeout settings. + properties: + http: + description: HTTP defines the settings for HTTP timeouts. + properties: + idle: + description: |- + Timeout defines the settings for http timeouts. If this setting is not specified, the value of applications[].downstream.timeouts.http.idle is inherited. + A value of 0 will completely disable the timeout. + type: string + maxDuration: + default: 15s + description: |- + MaxDuration defines the total duration for a HTTP request/response stream. + Default: 15s + type: string + type: object + type: object + tls: + description: TLS defines the TLS settings. + properties: + ciphers: + description: Ciphers defines a list of the supported TLS cipher suites. For details on cipher list refer to the envoy documentation on cipher_suites in common tls configuration. + items: + type: string + minItems: 1 + type: array + enable: + default: false + description: Enable defines if the upstream connection is encrypted. + type: boolean + protocol: + description: Protocol defines the supported TLS protocol versions. + properties: + maximum: + description: Maximum supported TLS version. + enum: + - TLSv1_0 + - TLSv1_1 + - TLSv1_2 + - TLSv1_3 + type: string + minimum: + description: Minimum supported TLS version. + enum: + - TLSv1_0 + - TLSv1_1 + - TLSv1_2 + - TLSv1_3 + type: string + type: object + type: object + type: object + type: object + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - containerPort + x-kubernetes-list-type: map + envoyClusterRefs: + description: EnvoyClusterRefs selects the relevant EnvoyClusters. + items: + properties: + name: + description: Name of the resource + minLength: 1 + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + podSelector: + description: PodSelector defines to which Pods the configuration will be applied to. + properties: + matchLabels: + additionalProperties: + type: string + description: MatchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels. + type: object + type: object + sessionHandlingRef: + description: SessionHandlingRef selects the SessionHandling configuration to apply. + properties: + name: + description: Name of the resource + minLength: 1 + type: string + required: + - name + type: object + required: + - applications + type: object + status: + description: Most recently observed status of the SidecarGateway which is populated by the system. This data is read-only and may not be up to date. + properties: + conditions: + items: + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + format: date-time + type: string + message: + description: A human-readable message indicating details about the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of SidecarGateway condition. + type: string + required: + - status + - type + type: object + type: array + pods: + items: + properties: + envoyConfig: + description: EnvoyConfig indicates the name of the EnvoyConfig CR for the Pod. + type: string + name: + description: Name indicates the name of a Pod selected by the SidecarGateway. + type: string + sessionAgentSecret: + type: string + required: + - name + type: object + type: array + status: + type: string + unmanagedPods: + items: + properties: + managedBy: + description: ManagedBy indicates the Airlock Microgateway Operator instance which manages this Pod. + type: string + name: + description: Name indicates the name of a Pod selected by the SidecarGateway. + type: string + sessionAgentSecret: + type: string + required: + - name + type: object + type: array + required: + - status + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/airlock/microgateway/4.3.3/crds/telemetries.microgateway.airlock.com.yaml b/charts/airlock/microgateway/4.3.3/crds/telemetries.microgateway.airlock.com.yaml new file mode 100644 index 0000000000..3262cb1f0b --- /dev/null +++ b/charts/airlock/microgateway/4.3.3/crds/telemetries.microgateway.airlock.com.yaml @@ -0,0 +1,96 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + labels: + app.kubernetes.io/name: airlock-microgateway-operator + app.kubernetes.io/version: 4.3.3 + name: telemetries.microgateway.airlock.com +spec: + group: microgateway.airlock.com + names: + categories: + - airlock-microgateway + kind: Telemetry + listKind: TelemetryList + plural: telemetries + singular: telemetry + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: Telemetry contains the configuration for telemetry (logging, metrics & tracing). + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Specification of the desired telemetry behavior. + properties: + correlation: + description: Correlation defines the correlation aspects of Telemetry. + properties: + idSource: + description: IDSource specifies how an external correlation ID should be obtained for a request. If not specified, no correlation ID will be logged. + properties: + header: + description: Header specifies to extract the correlation ID from a request header. If the header is absent from a request, no correlation ID will be logged. + properties: + name: + default: X-Correlation-Id + description: Name of the header (case-insensitive) from which to extract the correlation ID. + minLength: 1 + type: string + type: object + required: + - header + type: object + request: + description: Request defines the request related correlation settings of Telemetry. + properties: + allowDownstreamRequestID: + default: true + description: AllowDownstreamRequestID defines whether trace sampling will consider a provided x-request-id. + type: boolean + alterRequestID: + default: true + description: AlterRequestID defines whether to alter the UUID to reflect the trace sampling decision. If disabled no modification to the UUID will be performed, this may break tracing in the upstream. + type: boolean + type: object + type: object + logging: + description: Logging defines the logging aspects of Telemetry. + properties: + accessLog: + description: AccessLog defines the access log settings of Telemetry. + properties: + format: + description: Format defines the Access Log format of the sidecar. + properties: + json: + description: JSON defines the Access Log format as JSON. + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + type: object + type: object + type: object + type: object + served: true + storage: true diff --git a/charts/airlock/microgateway/4.3.3/dashboards/blockLogs.json b/charts/airlock/microgateway/4.3.3/dashboards/blockLogs.json new file mode 100644 index 0000000000..ef0ce6d624 --- /dev/null +++ b/charts/airlock/microgateway/4.3.3/dashboards/blockLogs.json @@ -0,0 +1,510 @@ +{ + "__inputs": [ + { + "name": "DS_LOKI", + "label": "Loki", + "description": "", + "type": "datasource", + "pluginId": "loki", + "pluginName": "Loki" + }, + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "10.2.0" + }, + { + "type": "datasource", + "id": "loki", + "name": "Loki", + "version": "1.0.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Blocked requests by Airlock Microgateway retrieved from corresponding access logs.\n\nThe dashboard can be filtered by namespace and block type. Column filters on the table allow for even a more granular filtering of the logs.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "airlock-microgateway" + ], + "targetBlank": true, + "title": "Airlock Microgateway", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "panels": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "text", + "mode": "fixed" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Namespace" + }, + "properties": [ + { + "id": "custom.width", + "value": 221 + }, + { + "id": "custom.filterable" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Timestamp" + }, + "properties": [ + { + "id": "custom.width", + "value": 214 + }, + { + "id": "unit", + "value": "dateTimeAsIso" + }, + { + "id": "custom.filterable" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Method" + }, + "properties": [ + { + "id": "custom.width", + "value": 89 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Client IP" + }, + "properties": [ + { + "id": "custom.width", + "value": 138 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Request ID" + }, + "properties": [ + { + "id": "custom.width", + "value": 328 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Block Type" + }, + "properties": [ + { + "id": "custom.width", + "value": 116 + }, + { + "id": "custom.filterable", + "value": false + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Request Size" + }, + "properties": [ + { + "id": "custom.width", + "value": 126 + }, + { + "id": "unit", + "value": "bytes" + }, + { + "id": "custom.align", + "value": "right" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Attack Type" + }, + "properties": [ + { + "id": "custom.width", + "value": 217 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Application" + }, + "properties": [ + { + "id": "custom.width", + "value": 207 + } + ] + } + ] + }, + "gridPos": { + "h": 27, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": true, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "editorMode": "code", + "expr": "{container=\"airlock-microgateway-engine\", namespace=~\"${namespace:regex}\"} |= \"airlock_request_blocked_deny_rule\" |= \"envoy.access\"\n| json http_method=\"http.request.method\", url=\"url.path\", request_size=\"http.request.bytes\", client_ip=\"network.forwarded_ip\", request_id=\"http.request.id\", details=\"airlock.deny_rules.matches\"\n| label_format block_type=\"deny_rules\", attack_type=`{{ range $q := fromJson .details }} {{ if eq $q.threat_handling_mode \"block\" }} {{ $q.rule_key }} {{ end }} {{ end }}` | block_type=~\"${blockType:regex}\"", + "hide": false, + "queryType": "range", + "refId": "Deny Rule Blocks" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "editorMode": "code", + "expr": "{container=\"airlock-microgateway-engine\", namespace=~\"${namespace:regex}\"} |= \"airlock_request_blocked_limit\" |= \"envoy.access\"\n| json http_method=\"http.request.method\", url=\"url.path\", request_size=\"http.request.bytes\", client_ip=\"network.forwarded_ip\", request_id=\"http.request.id\", details=\"airlock.limits.matches\"\n| label_format block_type=\"limits\", attack_type=`{{ range $q := fromJson .details }} {{ if eq $q.threat_handling_mode \"block\" }} {{ $q.rule }} {{ end }} {{ end }}` | block_type=~\"${blockType:regex}\"", + "hide": false, + "queryType": "range", + "refId": "Limit Blocks" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "editorMode": "code", + "expr": "{container=\"airlock-microgateway-engine\", namespace=~\"${namespace:regex}\"} |= \"airlock_request_blocked_openapi\" |= \"envoy.access\"\n| json http_method=\"http.request.method\", url=\"url.path\", request_size=\"http.request.bytes\", client_ip=\"network.forwarded_ip\", request_id=\"http.request.id\", reference=\"airlock.openapi.reference\", constraint=\"airlock.openapi.request.failed_validation.constraint\", position=\"airlock.openapi.request.failed_validation.position\", message=\"airlock.openapi.request.failed_validation.message\"\n| label_format block_type=\"openapi\", attack_type=\"openapi\", details=`{{.reference }}: {{.constraint }} at {{ .position }} ({{ .message }})` | block_type=~\"${blockType:regex}\"", + "hide": false, + "queryType": "range", + "refId": "OpenAPI Blocks" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "editorMode": "code", + "expr": "{container=\"airlock-microgateway-engine\", namespace=~\"${namespace:regex}\"} |= \"airlock_request_blocked_parser\" |= \"envoy.access\"\n| json http_method=\"http.request.method\", url=\"url.path\", request_size=\"http.request.bytes\", client_ip=\"network.forwarded_ip\", request_id=\"http.request.id\", attack_type=\"airlock.parser\", failed_check=\"airlock.parser.matches[0].failed_check\", message=\"airlock.parser.matches[0].message\"\n| label_format block_type=\"parsing\", attack_type=\"parsing\", details=`{{.failed_check}}: {{.message}}` | block_type=~\"${blockType:regex}\"", + "hide": false, + "queryType": "range", + "refId": "Parser Blocks" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "editorMode": "code", + "expr": "{container=\"airlock-microgateway-engine\", namespace=~\"${namespace:regex}\"} |= \"airlock_request_blocked_graphql\" |= \"envoy.access\"\n| json http_method=\"http.request.method\", url=\"url.path\", request_size=\"http.request.bytes\", client_ip=\"network.forwarded_ip\", request_id=\"http.request.id\", reference=\"airlock.graphql.reference\", message=\"airlock.graphql.request.failed_validation.message\"\n| label_format block_type=\"graphql\", attack_type=\"graphql\", details=`{{ .reference }}: {{ .message }}` | block_type=~\"${blockType:regex}\"", + "hide": false, + "queryType": "range", + "refId": "GraphQL Blocks" + } + ], + "title": "Blocked Request logs", + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "extractFields", + "options": { + "format": "json", + "source": "labels" + } + }, + { + "id": "filterFieldsByName", + "options": { + "byVariable": false, + "include": { + "names": [ + "Time", + "attack_type", + "block_type", + "client_ip", + "details", + "http_method", + "namespace", + "request_id", + "request_size", + "url", + "pod" + ] + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Line": true, + "id": true, + "labelTypes": true, + "labels": true, + "tsNs": false + }, + "includeByName": {}, + "indexByName": { + "Time": 0, + "attack_type": 7, + "block_type": 6, + "client_ip": 9, + "details": 8, + "http_method": 3, + "namespace": 1, + "pod": 2, + "request_id": 10, + "request_size": 5, + "url": 4 + }, + "renameByName": { + "Time": "Timestamp", + "attack_type": "Attack Type", + "block_type": "Block Type", + "client_ip": "Client IP", + "details": "Details", + "http_method": "Method", + "namespace": "Namespace", + "pod": "Pod", + "request_id": "Request ID", + "request_size": "Request Size", + "tsNs": "", + "url": "Path" + } + } + } + ], + "type": "table" + } + ], + "schemaVersion": 39, + "tags": [ + "airlock-microgateway" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Loki", + "value": "P8E80F9AEF21F6940" + }, + "hide": 2, + "includeAll": false, + "label": "DS_LOKI", + "multi": false, + "name": "DS_LOKI", + "options": [], + "query": "loki", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(microgateway_license_http_rq_total,namespace)", + "hide": 0, + "includeAll": true, + "label": "Application Namespace", + "multi": true, + "name": "namespace", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(microgateway_license_http_rq_total,namespace)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "type": "query" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(microgateway_http_downstream_rq_threats_blocked_total,block_type)", + "hide": 0, + "includeAll": true, + "label": "Block Type", + "multi": true, + "name": "blockType", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(microgateway_http_downstream_rq_threats_blocked_total,block_type)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "PBFA97CFB590B2093" + }, + "hide": 2, + "includeAll": false, + "label": "DS_PROMETHEUS", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timeRangeUpdatedDuringEditOrView": false, + "timepicker": {}, + "timezone": "browser", + "title": "Airlock Microgateway Blocked Request Logs", + "uid": "adnyzcvwnyadcc", + "version": 3, + "weekStart": "" +} \ No newline at end of file diff --git a/charts/airlock/microgateway/4.3.3/dashboards/blockMetrics.json b/charts/airlock/microgateway/4.3.3/dashboards/blockMetrics.json new file mode 100644 index 0000000000..ba383d22e8 --- /dev/null +++ b/charts/airlock/microgateway/4.3.3/dashboards/blockMetrics.json @@ -0,0 +1,758 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "panel", + "id": "barchart", + "name": "Bar chart", + "version": "" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "10.2.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Metrics on requests blocked by Airlock Microgateway.\n\nDashboard can be filtered by namespaces as well as block types.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "airlock-microgateway" + ], + "targetBlank": true, + "title": "Airlock Microgateway", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "panels": [ + { + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 6, + "title": "Airlock Microgateway Block Metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total number of requests processed by Airlock Microgateway.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "text", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "round(sum(increase(microgateway_license_http_rq_total{namespace=~\"${namespace:regex}\", job=~\"${operator_namespace.regex}/.*-engine\"}[$__range])))", + "format": "time_series", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "Processed Requests", + "range": false, + "refId": "A", + "useBackend": false + } + ], + "title": "Requests", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Ratio of blocked requests vs. processed requests by Airlock Microgateway.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "text", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "nan", + "result": { + "index": 0, + "text": "n/a" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(microgateway_http_downstream_rq_threats_blocked_total{block_type=~\"${blockType:regex}\", namespace=~\"${namespace:regex}\", job=~\"${operator_namespace.regex}/.*-engine\"}[$__range])) / sum(increase(microgateway_license_http_rq_total{namespace=~\"${namespace:regex}\", job=~\"${operator_namespace.regex}/.*-engine\"}[$__range]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "Blocked Requests (%)", + "range": false, + "refId": "A", + "useBackend": false + } + ], + "title": "% Blocked Requests", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Requests per second processed by Airlock Microgateway along with the corresponding block rate.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "left", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "% Blocks" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + }, + { + "id": "max", + "value": 1 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Requests per second" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "custom.fillOpacity", + "value": 25 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 20, + "x": 0, + "y": 5 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "timezone": [ + "" + ], + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(microgateway_license_http_rq_total{namespace=~\"${namespace:regex}\", job=~\"${operator_namespace.regex}/.*-engine\"}[1m]))", + "instant": false, + "legendFormat": "Requests per second", + "range": true, + "refId": "Requests per Second" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(microgateway_http_downstream_rq_threats_blocked_total{block_type=~\"${blockType:regex}\", namespace=~\"${namespace:regex}\", job=~\"${operator_namespace.regex}/.*-engine\"}[1m])) / sum(rate(microgateway_license_http_rq_total{namespace=~\"${namespace:regex}\", job=~\"${operator_namespace.regex}/.*-engine\"}[1m]))", + "hide": false, + "instant": false, + "legendFormat": "% Blocks", + "range": true, + "refId": "Blocks" + } + ], + "title": "Requests vs. % Blocks", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Blocked requests by block type.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "super-light-orange", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisGridShow": true, + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 0, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 10, + "x": 0, + "y": 15 + }, + "id": 4, + "options": { + "barRadius": 0, + "barWidth": 0.8, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "orientation": "horizontal", + "showValue": "never", + "stacking": "none", + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "asc" + }, + "xField": "block_type", + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "10.4.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "round(sum by (block_type) (increase(microgateway_http_downstream_rq_threats_blocked_total{block_type=~\"${blockType:regex}\", namespace=~\"${namespace:regex}\", job=~\"${operator_namespace.regex}/.*-engine\"}[$__range])))", + "format": "time_series", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Block Type", + "transformations": [ + { + "id": "reduce", + "options": { + "includeTimeField": false, + "labelsToFields": true, + "mode": "seriesToRows", + "reducers": [ + "sum" + ] + } + } + ], + "type": "barchart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Blocked requests by attack type, which are subsets of the various block types.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "light-orange", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 10, + "x": 10, + "y": 15 + }, + "id": 5, + "options": { + "barRadius": 0, + "barWidth": 0.8, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "orientation": "horizontal", + "showValue": "never", + "stacking": "none", + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xField": "attack_type", + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "10.4.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "round(sum by (attack_type) (increase(microgateway_http_downstream_rq_threats_blocked_total{block_type=~\"${blockType:regex}\", namespace=~\"${namespace:regex}\", job=~\"${operator_namespace.regex}/.*-engine\"}[$__range])))", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Attack Type", + "transformations": [ + { + "id": "reduce", + "options": { + "labelsToFields": true, + "reducers": [ + "sum" + ] + } + } + ], + "type": "barchart" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [ + "airlock-microgateway" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "PBFA97CFB590B2093" + }, + "hide": 2, + "includeAll": false, + "label": "Datasource Prometheus", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "Loki", + "value": "P8E80F9AEF21F6940" + }, + "hide": 2, + "includeAll": false, + "label": "DS_LOKI", + "multi": false, + "name": "DS_LOKI", + "options": [], + "query": "loki", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(microgateway_license_valid,namespace)", + "hide": 0, + "includeAll": true, + "label": "Operator Namespace", + "multi": true, + "name": "operator_namespace", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(microgateway_license_valid,namespace)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": ".*", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(microgateway_license_http_rq_total,namespace)", + "hide": 0, + "includeAll": true, + "label": "Application Namespace", + "multi": true, + "name": "namespace", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(microgateway_license_http_rq_total,namespace)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "type": "query" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(microgateway_http_downstream_rq_threats_blocked_total,block_type)", + "hide": 0, + "includeAll": true, + "label": "Block Type", + "multi": true, + "name": "blockType", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(microgateway_http_downstream_rq_threats_blocked_total,block_type)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "type": "query" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timeRangeUpdatedDuringEditOrView": false, + "timepicker": { + "hidden": false + }, + "timezone": "browser", + "title": "Airlock Microgateway Block Metrics", + "uid": "ddnqoczu7qvb4cdd3dd", + "version": 3, + "weekStart": "" +} \ No newline at end of file diff --git a/charts/airlock/microgateway/4.3.3/dashboards/license.json b/charts/airlock/microgateway/4.3.3/dashboards/license.json new file mode 100644 index 0000000000..b9d5777e23 --- /dev/null +++ b/charts/airlock/microgateway/4.3.3/dashboards/license.json @@ -0,0 +1,521 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "10.2.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "airlock-microgateway" + ], + "targetBlank": true, + "title": "Airlock Microgateway", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "License status of Airlock Microgateway.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 1, + "text": "Invalid" + }, + "1": { + "color": "green", + "index": 0, + "text": "Valid" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "min(microgateway_license_valid{namespace=~\"${operator_namespace.regex}\"})", + "instant": true, + "legendFormat": "License Status", + "range": false, + "refId": "Licenses" + } + ], + "title": "License Status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Expiry date of the Airlock Microgateway license associated with the selected operator.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "text", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "time: L" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 3, + "y": 0 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "min(microgateway_license_expiry_timestamp_seconds{namespace=~\"${operator_namespace.regex}\"})*1000", + "instant": true, + "legendFormat": "Expiry Date (MM/DD/YYYY)", + "range": false, + "refId": "A" + } + ], + "title": "License Expiry Date", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of licensed requests for applications protected by Airlock Microgateway.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "text", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 7, + "y": 0 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(microgateway_license_max_rq_count_per_month{namespace=~\"${operator_namespace.regex}\"})", + "instant": true, + "legendFormat": "Licensed Requests", + "range": false, + "refId": "A" + } + ], + "title": "Licensed Requests", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Estimated number of requests protected by Airlock Microgateway over 30 days based on the last 7 days.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "text", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 11, + "y": 0 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(microgateway_license_http_rq_total{job=~\"${operator_namespace.regex}/.*-engine\"}[7d]))/7*30", + "instant": true, + "legendFormat": "Estimated Requests", + "range": false, + "refId": "A" + } + ], + "title": "Requests over 30 days (estimated)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of requests per week processed by Airlock Microgateway.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 16, + "x": 0, + "y": 4 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(avg_over_time(increase(microgateway_license_http_rq_total{job=~\"${operator_namespace.regex}/.*-engine\"}[7d])[2m:30s]))", + "instant": false, + "legendFormat": "# Requests per week", + "range": true, + "refId": "A" + } + ], + "title": "Processed Requests per week", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "tags": [ + "airlock-microgateway" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "PBFA97CFB590B2093" + }, + "hide": 2, + "includeAll": false, + "label": "DS_PROMETHEUS", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(microgateway_license_valid,namespace)", + "description": "", + "hide": 0, + "includeAll": false, + "label": "Operator Namespace", + "multi": false, + "name": "operator_namespace", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(microgateway_license_valid,namespace)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-7d", + "to": "now" + }, + "timeRangeUpdatedDuringEditOrView": false, + "timepicker": {}, + "timezone": "browser", + "title": "Airlock Microgateway License", + "uid": "cdpq79bzrr01se", + "version": 2, + "weekStart": "" +} \ No newline at end of file diff --git a/charts/airlock/microgateway/4.3.3/dashboards/overview.json b/charts/airlock/microgateway/4.3.3/dashboards/overview.json new file mode 100644 index 0000000000..0942766217 --- /dev/null +++ b/charts/airlock/microgateway/4.3.3/dashboards/overview.json @@ -0,0 +1,1138 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "10.2.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "airlock-microgateway" + ], + "targetBlank": true, + "title": "Airlock Microgateway", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "panels": [ + { + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 3, + "title": "Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of pods that are protected by Airlock Microgateway.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 11, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(microgateway_sidecars{namespace=~\"${operator_namespace.regex}\"})", + "instant": true, + "legendFormat": "Protected Pods", + "range": false, + "refId": "A" + } + ], + "title": "Protected Pods", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total number of requests processed by Airlock Microgateway.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "text", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "round(sum(increase(microgateway_license_http_rq_total{namespace=~\"${namespace:regex}\", job=~\"${operator_namespace.regex}/.*-engine\"}[$__range])))", + "format": "time_series", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "Processed Requests", + "range": false, + "refId": "A", + "useBackend": false + } + ], + "title": "Requests", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Ratio of blocked requests vs. processed requests by Airlock Microgateway.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "text", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "nan", + "result": { + "index": 0, + "text": "n/a" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 6, + "y": 1 + }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(microgateway_http_downstream_rq_threats_blocked_total{namespace=~\"${namespace:regex}\", job=~\"${operator_namespace.regex}/.*-engine\"}[$__range])) / sum(increase(microgateway_license_http_rq_total{namespace=~\"${namespace:regex}\", job=~\"${operator_namespace.regex}/.*-engine\"}[$__range]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "Blocked Requests (%)", + "range": false, + "refId": "A", + "useBackend": false + } + ], + "title": "% Blocked Requests", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "License status of Airlock Microgateway.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 1, + "text": "Invalid" + }, + "1": { + "color": "green", + "index": 0, + "text": "Valid" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 9, + "y": 1 + }, + "id": 10, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "min(microgateway_license_valid{namespace=~\"${operator_namespace.regex}\"})", + "instant": true, + "legendFormat": "License Status", + "range": false, + "refId": "Licenses" + } + ], + "title": "License", + "type": "stat" + }, + { + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 2, + "title": "Blocks", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Requests per second processed by Airlock Microgateway along with the corresponding block rate.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "left", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "% Blocks" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + }, + { + "id": "max", + "value": 1 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Requests per second" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "custom.fillOpacity", + "value": 25 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 6 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "timezone": [ + "" + ], + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(microgateway_license_http_rq_total{namespace=~\"${namespace:regex}\", job=~\"${operator_namespace.regex}/.*-engine\"}[1m]))", + "instant": false, + "legendFormat": "Requests per second", + "range": true, + "refId": "Requests per Second" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(microgateway_http_downstream_rq_threats_blocked_total{namespace=~\"${namespace:regex}\", job=~\"${operator_namespace.regex}/.*-engine\"}[1m])) / sum(rate(microgateway_license_http_rq_total{namespace=~\"${namespace:regex}\", job=~\"${operator_namespace.regex}/.*-engine\"}[1m]))", + "hide": false, + "instant": false, + "legendFormat": "% Blocks", + "range": true, + "refId": "Blocks" + } + ], + "title": "Requests vs. % Blocks", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Requests blocked by Airlock Microgateway categorized by their corresponding type.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "barAlignment": 0, + "drawStyle": "line", + "gradientMode": "none", + "hideValue": false, + "lineInterpolation": "linear", + "lineStyle": { + "dash": [ + 10, + 10 + ], + "fill": "solid" + }, + "showPoints": "never", + "spanNulls": false, + "type": "sparkline" + }, + "inspect": false + }, + "displayName": "Block Type", + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "block_type" + }, + "properties": [ + { + "id": "custom.width", + "value": 153 + }, + { + "id": "custom.cellOptions", + "value": { + "type": "auto" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Trend #Block Types" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 7, + "options": { + "cellHeight": "lg", + "footer": { + "countRows": false, + "enablePagination": false, + "fields": [ + "Value" + ], + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": false, + "sortBy": [ + { + "desc": true, + "displayName": "block_type" + } + ] + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (block_type) (increase(microgateway_http_downstream_rq_threats_blocked_total{namespace=~\"${namespace:regex}\", job=~\"${operator_namespace.regex}/.*-engine\"}[1m] offset -1m))/(60000/$__interval_ms)", + "format": "time_series", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "Block Types" + } + ], + "title": "Blocked Requests by Type", + "transformations": [ + { + "id": "timeSeriesTable", + "options": { + "A": { + "timeField": "Time" + }, + "Block Types": { + "stat": "sum", + "timeField": "Time" + } + } + } + ], + "type": "table" + }, + { + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 1, + "title": "Latency", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Percentiles of the application downstream latency over one minute.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "25th Percentile" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "super-light-purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "50th Percentile" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "95th Percentile" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-purple", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.25, sum(rate(envoy_http_downstream_rq_time_bucket{envoy_http_conn_manager_prefix=\"http\", namespace=~\"${namespace:regex}\", job=~\"${operator_namespace.regex}/.*-engine\"}[1m])) by (le))", + "instant": false, + "legendFormat": "25th Percentile", + "range": true, + "refId": "25th Percentile" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum(rate(envoy_http_downstream_rq_time_bucket{envoy_http_conn_manager_prefix=\"http\", namespace=~\"${namespace:regex}\", job=~\"${operator_namespace.regex}/.*-engine\"}[1m])) by (le))", + "hide": false, + "instant": false, + "legendFormat": "50th Percentile", + "range": true, + "refId": "50th Percentile" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(envoy_http_downstream_rq_time_bucket{envoy_http_conn_manager_prefix=\"http\", namespace=~\"${namespace:regex}\", job=~\"${operator_namespace.regex}/.*-engine\"}[1m])) by (le))", + "hide": false, + "instant": false, + "legendFormat": "95th Percentile", + "range": true, + "refId": "95th Percentile" + } + ], + "title": "Application Downstream Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Percentiles of the Airlock Microgateway processing time over one minute.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "25th Percentile" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "super-light-purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "50th Percentile" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "95th Percentile" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-purple", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.25, sum(rate(microgateway_rq_processing_time_ms_bucket{namespace=~\"${namespace:regex}\", job=~\"${operator_namespace.regex}/.*-engine\"}[1m])) by (le))", + "instant": false, + "legendFormat": "25th Percentile", + "range": true, + "refId": "0.25 Percentile" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum(rate(microgateway_rq_processing_time_ms_bucket{namespace=~\"${namespace:regex}\", job=~\"${operator_namespace.regex}/.*-engine\"}[1m])) by (le))", + "hide": false, + "instant": false, + "legendFormat": "50th Percentile", + "range": true, + "refId": "0.5 Percentile" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(microgateway_rq_processing_time_ms_bucket{namespace=~\"${namespace:regex}\", job=~\"${operator_namespace.regex}/.*-engine\"}[1m])) by (le))", + "hide": false, + "instant": false, + "legendFormat": "95th Percentile", + "range": true, + "refId": "0.95 Percentile" + } + ], + "title": "Airlock Microgateway Processing Time", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [ + "airlock-microgateway" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "PBFA97CFB590B2093" + }, + "hide": 2, + "includeAll": false, + "label": "DS_PROMETHEUS", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(microgateway_license_valid,namespace)", + "hide": 0, + "includeAll": true, + "label": "Operator Namespace", + "multi": true, + "name": "operator_namespace", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(microgateway_license_valid,namespace)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": ".*", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(microgateway_license_http_rq_total,namespace)", + "hide": 0, + "includeAll": true, + "label": "Application Namespace", + "multi": true, + "name": "namespace", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(microgateway_license_http_rq_total,namespace)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timeRangeUpdatedDuringEditOrView": false, + "timepicker": {}, + "timezone": "browser", + "title": "Airlock Microgateway Overview", + "uid": "fdp5jb8fnrmyoa", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/charts/airlock/microgateway/4.3.3/templates/NOTES.txt b/charts/airlock/microgateway/4.3.3/templates/NOTES.txt index bb94ff521e..6e5ce218ae 100644 --- a/charts/airlock/microgateway/4.3.3/templates/NOTES.txt +++ b/charts/airlock/microgateway/4.3.3/templates/NOTES.txt @@ -1,15 +1,47 @@ -Thank you for installing Airlock Microgateway CNI. +Thank you for installing Airlock Microgateway. -Please ensure that the helm values'.config.cniNetDir' and '.config.cniBinDir' are configured for your Kubernetes distribution. -For further information, consider our manual https://docs.airlock.com/microgateway/{{ include "airlock-microgateway-cni.docsVersion" . }}. -The chapter 'Setup > Installation' describes how to set those settings correctly. +Please ensure the following prerequisites are fulfilled: +* Cert-Manager is installed. + https://cert-manager.io/docs/installation/helm/ +* Airlock Microgateway CNI is also installed on the cluster. + https://artifacthub.io/packages/helm/airlock-microgateway-cni/microgateway-cni +* A valid Airlock Microgateway license is deployed in the Kubernetes secret 'airlock-microgateway-license'. + * Get a free Community license: https://airlock.com/en/microgateway-community + * Order a Premium license: https://airlock.com/en/microgateway-premium Further information: -* Documentation: https://docs.airlock.com/microgateway/{{ include "airlock-microgateway-cni.docsVersion" . }} +* Documentation: https://docs.airlock.com/microgateway/{{ include "airlock-microgateway.docsVersion" . }} +* CRD API reference documentation: https://docs.airlock.com/microgateway/{{ include "airlock-microgateway.docsVersion" . }}/api/crds * Airlock Microgateway Labs: https://play.instruqt.com/airlock/invite/hyi9fy4b4jzc?icp_referrer=helm +{{- if .Values.crds.skipVersionCheck }} -Next steps: -* Install Airlock Microgateway (if not done already) - https://artifacthub.io/packages/helm/airlock-microgateway/microgateway +Warning: CRD version check skipped +{{- else -}} +{{- $outdatedCRDs := (include "airlock-microgateway.outdatedCRDs" .) -}} +{{- if $outdatedCRDs -}} + {{- fail (printf ` + +Helm does not automatically upgrade CRDs from the chart's 'crds/' directory during 'helm install/upgrade'. +Therefore, the CRDs must be manually upgraded with the following command before deploying this chart: + +kubectl apply -k https://github.com/airlock/microgateway/deploy/charts/airlock-microgateway/crds/?ref=%s --server-side --force-conflicts + +If you are not using the helm install/upgrade command and instead rely on some other mechanism which is able to upgrade CRDs for deploying this chart, you can suppress this error by setting the helm value 'crds.skipVersionCheck=true'.` + .Chart.AppVersion) + -}} +{{- end -}} +{{- end -}} +{{- if .Values.tests.enabled -}} + {{- if .Values.operator.watchNamespaces -}} + {{- if not (has .Release.Namespace .Values.operator.watchNamespaces) -}} + {{- fail (printf ` + +To execute 'helm test', it is necessary that the release namespace '%s' is part of the operator's watch scope. Either disable the tests or ensure that the release namespace is added to watch namspace list ('operator.watchNamespaces') in the helm values. +` + .Release.Namespace) + -}} + {{- end -}} + {{- end -}} +{{- end }} Your release version is {{ .Chart.Version }}. \ No newline at end of file diff --git a/charts/airlock/microgateway/4.3.3/templates/_helpers.tpl b/charts/airlock/microgateway/4.3.3/templates/_helpers.tpl index 996491a873..733ba96486 100644 --- a/charts/airlock/microgateway/4.3.3/templates/_helpers.tpl +++ b/charts/airlock/microgateway/4.3.3/templates/_helpers.tpl @@ -1,14 +1,16 @@ {{/* Expand the name of the chart. +We truncate at 49 chars because some Kubernetes name fields are limited to 63 chars (by the DNS naming spec) +and the longest explicit suffix is 14 characters. */}} -{{- define "airlock-microgateway-cni.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- define "airlock-microgateway.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 49 | trimSuffix "-" }} {{- end }} {{/* Convert an image configuration object into an image ref string. */}} -{{- define "airlock-microgateway-cni.image" -}} +{{- define "airlock-microgateway.image" -}} {{- if .digest -}} {{- printf "%s@%s" .repository .digest -}} {{- else if .tag -}} @@ -20,19 +22,19 @@ Convert an image configuration object into an image ref string. {{/* Create a default fully qualified app name. -We truncate at 50 chars because some Kubernetes name fields are limited to 63 chars (by the DNS naming spec) -and the longest suffix is 13 characters. +We truncate at 36 chars because some Kubernetes name fields are limited to 63 chars (by the DNS naming spec) +and the longest implicit suffix is 27 characters. If release name contains chart name it will be used as a full name. */}} -{{- define "airlock-microgateway-cni.fullname" -}} +{{- define "airlock-microgateway.fullname" -}} {{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 50 | trimSuffix "-" }} +{{- .Values.fullnameOverride | trunc 36 | trimSuffix "-" }} {{- else }} {{- $name := default .Chart.Name .Values.nameOverride }} {{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 50 | trimSuffix "-" }} +{{- .Release.Name | trunc 36 | trimSuffix "-" }} {{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 50 | trimSuffix "-" }} +{{- printf "%s-%s" .Release.Name $name | trunc 36 | trimSuffix "-" }} {{- end }} {{- end }} {{- end }} @@ -40,62 +42,112 @@ If release name contains chart name it will be used as a full name. {{/* Create chart name and version as used by the chart label. */}} -{{- define "airlock-microgateway-cni.chart" -}} +{{- define "airlock-microgateway.chart" -}} {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} {{- end }} {{/* Common labels */}} -{{- define "airlock-microgateway-cni.labels" -}} -helm.sh/chart: {{ include "airlock-microgateway-cni.chart" . }} -{{ include "airlock-microgateway-cni.selectorLabels" . }} +{{- define "airlock-microgateway.sharedLabels" -}} +helm.sh/chart: {{ include "airlock-microgateway.chart" . }} {{- if .Chart.AppVersion }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end }} app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/part-of: {{ .Chart.Name }} {{- with .Values.commonLabels }} {{ toYaml .}} {{- end }} {{- end }} {{/* -Common labels without component +Common Selector labels */}} -{{- define "airlock-microgateway-cni.labelsWithoutComponent" -}} -{{- $labels := fromYaml (include "airlock-microgateway-cni.labels" .) -}} -{{ unset $labels "app.kubernetes.io/component" | toYaml }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "airlock-microgateway-cni.selectorLabels" -}} -app.kubernetes.io/component: cni-plugin-installer +{{- define "airlock-microgateway.sharedSelectorLabels" -}} app.kubernetes.io/instance: {{ .Release.Name }} -app.kubernetes.io/name: {{ include "airlock-microgateway-cni.name" . }} {{- end }} {{/* -Create the name of the service account to use for the CNI Plugin +Restricted Container Security Context */}} -{{- define "airlock-microgateway-cni.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "airlock-microgateway-cni.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} +{{- define "airlock-microgateway.restrictedSecurityContext" -}} +allowPrivilegeEscalation: false +privileged: false +runAsNonRoot: true +capabilities: + drop: ["ALL"] +readOnlyRootFilesystem: true +seccompProfile: + type: RuntimeDefault {{- end }} -{{- define "airlock-microgateway-cni.isSemver" -}} +{{/* Precondition: May only be used if AppVersion is isSemver */}} +{{- define "airlock-microgateway.supportedCRDVersionPattern" -}} +{{- $version := (semver .Chart.AppVersion) -}} +{{- if $version.Prerelease -}} +>= {{ $version.Major }}.{{ $version.Minor }}.{{ $version.Patch }}-{{ $version.Prerelease }} +{{- else -}} +>= {{ $version.Major }}.{{ $version.Minor }}.0 || >= {{ $version.Major }}.{{ $version.Minor }}.{{ add1 $version.Patch }}-0 +{{- end -}} +{{- end -}} + +{{- define "airlock-microgateway.outdatedCRDs" -}} +{{- if (eq "true" (include "airlock-microgateway.isSemver" .Chart.AppVersion)) -}} + {{- $supportedVersion := (include "airlock-microgateway.supportedCRDVersionPattern" .) -}} + {{- range $path, $_ := .Files.Glob "crds/*.yaml" -}} + {{- $api := ($.Files.Get $path | fromYaml).metadata.name -}} + {{- $crd := (lookup "apiextensions.k8s.io/v1" "CustomResourceDefinition" "" $api) -}} + {{- $isOutdated := false -}} + {{- if $crd -}} + {{/* If CRD is already present in the cluster, it must have the minimum supported version */}} + {{- $isOutdated = true -}} + {{- if hasKey $crd.metadata "labels" -}} + {{- $crdVersion := get $crd.metadata.labels "app.kubernetes.io/version" -}} + {{- if (eq "true" (include "airlock-microgateway.isSemver" $crdVersion)) -}} + {{- if (semverCompare $supportedVersion $crdVersion) }} + {{- $isOutdated = false -}} + {{- end }} + {{- end -}} + {{- end -}} + {{- end -}} + {{- if $isOutdated }} +{{ base $path }} + {{- end }} + {{- end -}} +{{- end -}} +{{- end -}} + +{{- define "airlock-microgateway.isSemver" -}} {{- regexMatch `^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$` . -}} {{- end -}} -{{- define "airlock-microgateway-cni.docsVersion" -}} -{{- if and (eq "true" (include "airlock-microgateway-cni.isSemver" .Chart.AppVersion)) (not (contains "-" .Chart.AppVersion)) -}} +{{- define "airlock-microgateway.docsVersion" -}} +{{- if and (eq "true" (include "airlock-microgateway.isSemver" .Chart.AppVersion)) (not (contains "-" .Chart.AppVersion)) -}} {{- $version := (semver .Chart.AppVersion) -}} {{- $version.Major }}.{{ $version.Minor -}} {{- else -}} {{- print "latest" -}} {{- end -}} {{- end -}} + +{{- define "airlock-microgateway.watchNamespaceSelector.labelQuery" -}} +{{- $list := list -}} +{{- with .matchLabels -}} + {{- range $key, $value := . -}} + {{- $list = append $list (printf "%s=%s" $key $value) -}} + {{- end -}} +{{- end -}} +{{- with .matchExpressions -}} + {{- range . -}} + {{- if has .operator (list "In" "NotIn") -}} + {{- $list = append $list (printf "%s %s (%s)" .key (lower .operator) (join "," .values)) -}} + {{- else if eq .operator "Exists" -}} + {{- $list = append $list .key -}} + {{- else if eq .operator "DoesNotExist" -}} + {{- $list = append $list (printf "!%s" .key) -}} + {{- end -}} + {{- end -}} +{{- end -}} +{{- join "," $list -}} +{{- end -}} \ No newline at end of file diff --git a/charts/airlock/microgateway/4.4.0/templates/operator/_operator_helpers.tpl b/charts/airlock/microgateway/4.3.3/templates/operator/_operator_helpers.tpl similarity index 100% rename from charts/airlock/microgateway/4.4.0/templates/operator/_operator_helpers.tpl rename to charts/airlock/microgateway/4.3.3/templates/operator/_operator_helpers.tpl diff --git a/charts/airlock/microgateway/4.3.3/templates/operator/_rbac.gen.tpl b/charts/airlock/microgateway/4.3.3/templates/operator/_rbac.gen.tpl new file mode 100644 index 0000000000..83b314cbcf --- /dev/null +++ b/charts/airlock/microgateway/4.3.3/templates/operator/_rbac.gen.tpl @@ -0,0 +1,237 @@ +{{/* AUTOGENERATED FILE DO NOT EDIT */}} + +{{/* +Operator rbac permission rules +*/}} +{{- define "airlock-microgateway-operator.rbacRules" -}} +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - pods + verbs: + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - pods/finalizers + verbs: + - update +- apiGroups: + - "" + resources: + - pods/status + verbs: + - patch + - update +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - delete + - get + - list + - update + - watch +- apiGroups: + - microgateway.airlock.com + resources: + - accesscontrols + verbs: + - get + - list + - watch +- apiGroups: + - microgateway.airlock.com + resources: + - contentsecurities + verbs: + - get + - list + - watch +- apiGroups: + - microgateway.airlock.com + resources: + - denyrules + verbs: + - get + - list + - watch +- apiGroups: + - microgateway.airlock.com + resources: + - envoyclusters + verbs: + - get + - list + - watch +- apiGroups: + - microgateway.airlock.com + resources: + - envoyconfigurations + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - microgateway.airlock.com + resources: + - envoyconfigurations/status + verbs: + - get + - patch + - update +- apiGroups: + - microgateway.airlock.com + resources: + - envoyhttpfilters + verbs: + - get + - list + - watch +- apiGroups: + - microgateway.airlock.com + resources: + - graphqls + verbs: + - get + - list + - watch +- apiGroups: + - microgateway.airlock.com + resources: + - headerrewrites + verbs: + - get + - list + - watch +- apiGroups: + - microgateway.airlock.com + resources: + - identitypropagations + verbs: + - get + - list + - watch +- apiGroups: + - microgateway.airlock.com + resources: + - limits + verbs: + - get + - list + - watch +- apiGroups: + - microgateway.airlock.com + resources: + - oidcproviders + verbs: + - get + - list + - watch +- apiGroups: + - microgateway.airlock.com + resources: + - oidcrelyingparties + verbs: + - get + - list + - watch +- apiGroups: + - microgateway.airlock.com + resources: + - openapis + verbs: + - get + - list + - watch +- apiGroups: + - microgateway.airlock.com + resources: + - parsers + verbs: + - get + - list + - watch +- apiGroups: + - microgateway.airlock.com + resources: + - redisproviders + verbs: + - get + - list + - watch +- apiGroups: + - microgateway.airlock.com + resources: + - sessionhandlings + verbs: + - get + - list + - watch +- apiGroups: + - microgateway.airlock.com + resources: + - sidecargateways + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - microgateway.airlock.com + resources: + - sidecargateways/finalizers + verbs: + - update +- apiGroups: + - microgateway.airlock.com + resources: + - sidecargateways/status + verbs: + - get + - patch + - update +- apiGroups: + - microgateway.airlock.com + resources: + - telemetries + verbs: + - get + - list + - watch +{{- end }} diff --git a/charts/airlock/microgateway/4.3.3/templates/operator/_webhooks.gen.tpl b/charts/airlock/microgateway/4.3.3/templates/operator/_webhooks.gen.tpl new file mode 100644 index 0000000000..02e3048904 --- /dev/null +++ b/charts/airlock/microgateway/4.3.3/templates/operator/_webhooks.gen.tpl @@ -0,0 +1,339 @@ +{{/* AUTOGENERATED FILE DO NOT EDIT */}} + +{{/* +Operator mutating webhooks +*/}} +{{- define "airlock-microgateway-operator.mutatingWebhooks" -}} +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: airlock-microgateway-operator-webhook + namespace: '{{ .Release.Namespace }}' + path: /mutate-v1-pod + failurePolicy: Fail + name: mutate-pod.microgateway.airlock.com + reinvocationPolicy: IfNeeded + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + resources: + - pods + sideEffects: None + objectSelector: + matchLabels: + sidecar.microgateway.airlock.com/inject: "true" +{{- end }} + +{{/* +Operator validating webhooks +*/}} +{{- define "airlock-microgateway-operator.validatingWebhooks" -}} +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: airlock-microgateway-operator-webhook + namespace: '{{ .Release.Namespace }}' + path: /validate-v1-pod + failurePolicy: Fail + name: validate-pod.microgateway.airlock.com + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - pods + sideEffects: None + objectSelector: + matchLabels: + sidecar.microgateway.airlock.com/inject: "true" +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: airlock-microgateway-operator-webhook + namespace: '{{ .Release.Namespace }}' + path: /validate-microgateway-airlock-com-v1alpha1-accesscontrol + failurePolicy: Fail + name: validate-accesscontrol.microgateway.airlock.com + rules: + - apiGroups: + - microgateway.airlock.com + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - accesscontrols + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: airlock-microgateway-operator-webhook + namespace: '{{ .Release.Namespace }}' + path: /validate-microgateway-airlock-com-v1alpha1-denyrules + failurePolicy: Fail + name: validate-denyrules.microgateway.airlock.com + rules: + - apiGroups: + - microgateway.airlock.com + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - denyrules + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: airlock-microgateway-operator-webhook + namespace: '{{ .Release.Namespace }}' + path: /validate-microgateway-airlock-com-v1alpha1-envoycluster + failurePolicy: Fail + name: validate-envoycluster.microgateway.airlock.com + rules: + - apiGroups: + - microgateway.airlock.com + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - envoyclusters + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: airlock-microgateway-operator-webhook + namespace: '{{ .Release.Namespace }}' + path: /validate-microgateway-airlock-com-v1alpha1-envoyhttpfilter + failurePolicy: Fail + name: validate-envoyhttpfilter.microgateway.airlock.com + rules: + - apiGroups: + - microgateway.airlock.com + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - envoyhttpfilters + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: airlock-microgateway-operator-webhook + namespace: '{{ .Release.Namespace }}' + path: /validate-microgateway-airlock-com-v1alpha1-graphql + failurePolicy: Fail + name: validate-graphql.microgateway.airlock.com + rules: + - apiGroups: + - microgateway.airlock.com + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - graphqls + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: airlock-microgateway-operator-webhook + namespace: '{{ .Release.Namespace }}' + path: /validate-microgateway-airlock-com-v1alpha1-headerrewrites + failurePolicy: Fail + name: validate-headerrewrites.microgateway.airlock.com + rules: + - apiGroups: + - microgateway.airlock.com + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - headerrewrites + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: airlock-microgateway-operator-webhook + namespace: '{{ .Release.Namespace }}' + path: /validate-microgateway-airlock-com-v1alpha1-identitypropagation + failurePolicy: Fail + name: validate-identitypropagation.microgateway.airlock.com + rules: + - apiGroups: + - microgateway.airlock.com + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - identitypropagations + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: airlock-microgateway-operator-webhook + namespace: '{{ .Release.Namespace }}' + path: /validate-microgateway-airlock-com-v1alpha1-limits + failurePolicy: Fail + name: validate-limits.microgateway.airlock.com + rules: + - apiGroups: + - microgateway.airlock.com + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - limits + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: airlock-microgateway-operator-webhook + namespace: '{{ .Release.Namespace }}' + path: /validate-microgateway-airlock-com-v1alpha1-oidcprovider + failurePolicy: Fail + name: validate-oidcprovider.microgateway.airlock.com + rules: + - apiGroups: + - microgateway.airlock.com + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - oidcproviders + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: airlock-microgateway-operator-webhook + namespace: '{{ .Release.Namespace }}' + path: /validate-microgateway-airlock-com-v1alpha1-oidcrelyingparty + failurePolicy: Fail + name: validate-oidcrelyingparty.microgateway.airlock.com + rules: + - apiGroups: + - microgateway.airlock.com + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - oidcrelyingparties + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: airlock-microgateway-operator-webhook + namespace: '{{ .Release.Namespace }}' + path: /validate-microgateway-airlock-com-v1alpha1-openapi + failurePolicy: Fail + name: validate-openapi.microgateway.airlock.com + rules: + - apiGroups: + - microgateway.airlock.com + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - openapis + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: airlock-microgateway-operator-webhook + namespace: '{{ .Release.Namespace }}' + path: /validate-microgateway-airlock-com-v1alpha1-parser + failurePolicy: Fail + name: validate-parser.microgateway.airlock.com + rules: + - apiGroups: + - microgateway.airlock.com + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - parsers + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: airlock-microgateway-operator-webhook + namespace: '{{ .Release.Namespace }}' + path: /validate-microgateway-airlock-com-v1alpha1-redisprovider + failurePolicy: Fail + name: validate-redisprovider.microgateway.airlock.com + rules: + - apiGroups: + - microgateway.airlock.com + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - redisproviders + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: airlock-microgateway-operator-webhook + namespace: '{{ .Release.Namespace }}' + path: /validate-microgateway-airlock-com-v1alpha1-sidecargateway + failurePolicy: Fail + name: validate-sidecargateway.microgateway.airlock.com + rules: + - apiGroups: + - microgateway.airlock.com + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - sidecargateways + sideEffects: None +{{- end }} diff --git a/charts/airlock/microgateway/4.3.3/templates/operator/configmap.yaml b/charts/airlock/microgateway/4.3.3/templates/operator/configmap.yaml new file mode 100644 index 0000000000..95e52d7df1 --- /dev/null +++ b/charts/airlock/microgateway/4.3.3/templates/operator/configmap.yaml @@ -0,0 +1,394 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "airlock-microgateway.operator.fullname" . }}-config + namespace: {{ .Release.Namespace }} + labels: + {{- include "airlock-microgateway.operator.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: + engine_bootstrap_config_template.yaml: | + # Base configuration, admin interface on port 19000 + admin: + address: + socket_address: + address: 127.0.0.1 + port_value: 19000 + dynamic_resources: + cds_config: + initial_fetch_timeout: 10s + resource_api_version: V3 + api_config_source: + api_type: GRPC + transport_api_version: V3 + grpc_services: + - envoy_grpc: + cluster_name: xds_cluster + set_node_on_first_message_only: true + # Prevent Envoy Node from overloading the xDS server due to rejected configuration when using xDS SotW gRPC + rate_limit_settings: + max_tokens: 5 + fill_rate: 0.2 + lds_config: + resource_api_version: V3 + initial_fetch_timeout: 10s + api_config_source: + api_type: GRPC + transport_api_version: V3 + grpc_services: + - envoy_grpc: + cluster_name: xds_cluster + set_node_on_first_message_only: true + # Prevent Envoy Node from overloading the xDS server due to rejected configuration when using xDS SotW gRPC + rate_limit_settings: + max_tokens: 5 + fill_rate: 0.2 + static_resources: + listeners: + - name: probe + address: + socket_address: + address: 0.0.0.0 + port_value: 19001 + filter_chains: + - filters: + - name: http_connection_manager + typed_config: + '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: probe + codec_type: AUTO + http2_protocol_options: + initial_connection_window_size: 1048576 + initial_stream_window_size: 65536 + max_concurrent_streams: 100 + route_config: + name: probe + virtual_hosts: + - name: probe + domains: + - '*' + routes: + - name: ready + match: + path: /ready + headers: + - name: ':method' + string_match: + exact: 'GET' + route: + cluster: airlock_microgateway_engine_admin + http_filters: + - name: envoy.filters.http.router + typed_config: + '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + - name: metrics + address: + socket_address: + address: 0.0.0.0 + port_value: 19002 + filter_chains: + - filters: + - name: http_connection_manager + typed_config: + '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: metrics + codec_type: AUTO + http2_protocol_options: + initial_connection_window_size: 1048576 + initial_stream_window_size: 65536 + max_concurrent_streams: 100 + route_config: + name: metrics + virtual_hosts: + - name: metrics + domains: + - '*' + routes: + - name: metrics + match: + path: /metrics + headers: + - name: ':method' + string_match: + exact: 'GET' + route: + prefix_rewrite: '/stats/prometheus' + cluster: airlock_microgateway_engine_admin + http_filters: + - name: envoy.filters.http.router + typed_config: + '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + clusters: + - name: xds_cluster + connect_timeout: 1s + type: STRICT_DNS + load_assignment: + cluster_name: xds_cluster + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: airlock-microgateway-operator-xds.$(OPERATOR_NAMESPACE).svc.cluster.local + port_value: 13377 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + '@type': type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicit_http_config: + http2_protocol_options: + connection_keepalive: + interval: 360s + timeout: 5s + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + '@type': type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_minimum_protocol_version: TLSv1_3 + tls_maximum_protocol_version: TLSv1_3 + validation_context_sds_secret_config: + name: validation_context_sds + sds_config: + resource_api_version: V3 + path_config_source: + path: /etc/envoy/validation_context_sds_secret.yaml + watched_directory: + path: /etc/envoy/ + tls_certificate_sds_secret_configs: + - name: tls_certificate_sds + sds_config: + resource_api_version: V3 + path_config_source: + path: /etc/envoy/tls_certificate_sds_secret.yaml + watched_directory: + path: /etc/envoy/ + - name: airlock_microgateway_engine_admin + connect_timeout: 1s + type: STATIC + load_assignment: + cluster_name: airlock_microgateway_engine_admin + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 19000 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + '@type': type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicit_http_config: + http2_protocol_options: + connection_keepalive: + interval: 360s + timeout: 5s + stats_config: + stats_tags: + - tag_name: "block_type" + regex: "\\.(block_type\\.([^.]+))" + - tag_name: "attack_type" + regex: "\\.(attack_type\\.([^.]+))" + - tag_name: "envoy_cluster_name" + regex: "\\.(cluster\\.([^.]+))" + - tag_name: "version" + regex: "\\.(version\\.([^.]+))" + use_all_default_tags: true + overload_manager: + resource_monitors: + - name: "envoy.resource_monitors.global_downstream_max_connections" + typed_config: + "@type": type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig + max_active_downstream_connections: 50000 + bootstrap_extensions: + - name: airlock.bootstrap.engine_build_info + typed_config: + '@type': type.googleapis.com/airlock.extensions.bootstrap.stats.v1alpha.Stats + application_log_config: + log_format: + text_format: '{"@timestamp":"%Y-%m-%dT%T.%e%z","log":{"logger":"%n","level":"%l","origin":{"file":{"name":"%g","line":%#},"function":"%!"}},"event":{"module":"envoy","dataset":"envoy.application"},"process":{"pid":%P,"thread":{"id":%t}},"ecs":{"version":"8.5"},"message":"%j"}' + engine_container_template.yaml: | + name: "$(ENGINE_NAME)" + image: "$(ENGINE_IMAGE)" + imagePullPolicy: {{ .Values.engine.image.pullPolicy }} + args: + - "--config-path" + - "/etc/envoy/bootstrap_config.yaml" + - "--base-id" + - "$(BASE_ID)" + - "--file-flush-interval-msec" + - '1000' + - "--drain-time-s" + - '60' + - "--service-node" + - "$(POD_NAME).$(POD_NAMESPACE)" + - "--service-cluster" + - "$(APP_NAME).$(POD_NAMESPACE)" + - "--log-path" + - "/dev/stdout" + - "--log-level" + - "$(LOG_LEVEL)" + volumeMounts: + - name: airlock-microgateway-bootstrap-secret-volume + mountPath: /etc/envoy + readOnly: true + env: + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + ports: + - containerPort: 13378 + protocol: TCP + - containerPort: 19001 + protocol: TCP + - containerPort: 19002 + protocol: TCP + livenessProbe: + httpGet: + path: /ready + port: 19001 + scheme: HTTP + initialDelaySeconds: 5 + periodSeconds: 5 + failureThreshold: 5 + successThreshold: 1 + timeoutSeconds: 1 + readinessProbe: + httpGet: + path: /ready + port: 19001 + scheme: HTTP + initialDelaySeconds: 5 + periodSeconds: 5 + failureThreshold: 3 + successThreshold: 1 + timeoutSeconds: 1 + securityContext: + {{- include "airlock-microgateway.restrictedSecurityContext" . | nindent 6 }} + runAsUser: $(SECURITYCONTEXT_UID) + {{- with .Values.engine.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + session_agent_container_template.yaml: | + name: "$(SESSION_AGENT_NAME)" + image: "$(SESSION_AGENT_IMAGE)" + imagePullPolicy: {{ .Values.sessionAgent.image.pullPolicy }} + args: + - "--port" + - "19004" + - "--config-path" + - "/etc/microgateway-session-agent/config.json" + volumeMounts: + - name: airlock-microgateway-session-agent-volume + mountPath: /etc/microgateway-session-agent + readOnly: true + env: + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + ports: + - containerPort: 19004 + livenessProbe: + {{- if (semverCompare ">=1.27 || >=1.27.1-0" .Capabilities.KubeVersion.Version)}} + grpc: + port: 19004 + {{- else }} + tcpSocket: + port: 19004 + {{- end }} + initialDelaySeconds: 5 + periodSeconds: 5 + failureThreshold: 5 + successThreshold: 1 + timeoutSeconds: 5 + readinessProbe: + {{- if (semverCompare ">=1.27 || >=1.27.1-0" .Capabilities.KubeVersion.Version)}} + grpc: + port: 19004 + {{- else }} + tcpSocket: + port: 19004 + {{- end }} + initialDelaySeconds: 5 + periodSeconds: 5 + failureThreshold: 3 + successThreshold: 1 + timeoutSeconds: 5 + securityContext: + {{- include "airlock-microgateway.restrictedSecurityContext" . | nindent 6 }} + runAsUser: $(SECURITYCONTEXT_UID) + {{- with .Values.sessionAgent.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + network_validator_container_template.yaml: | + name: "$(NETWORK_VALIDATOR_NAME)" + image: "$(NETWORK_VALIDATOR_IMAGE)" + imagePullPolicy: {{ .Values.networkValidator.image.pullPolicy }} + command: ["/bin/sh", "-c"] + args: + - |- + echo 'pong' | nc -v -l 127.0.0.1 13378 & + for i in 1 2 3; do + sleep 1s + if r=$(echo 'ping' | nc -v -q 0 127.0.0.1 19003) && [ $r == pong ]; then + echo -n 'Traffic redirection to Airlock Microgateway Engine is working.' > /dev/termination-log + exit 0 + fi + done + echo -en 'Traffic redirection to Airlock Microgateway Engine is not working.\nRestart the pod after ensuring that hostNetwork is disabled and a compatible Airlock Microgateway CNI version is installed on the node.\nCertain environments may also require additional configuration (see docs.airlock.com for more information).' > /dev/termination-log + exit 1 + securityContext: + {{- include "airlock-microgateway.restrictedSecurityContext" . | nindent 6 }} + runAsUser: $(SECURITYCONTEXT_UID) + operator_config.yaml: | + apiVersion: config.airlock.com/v1alpha1 + kind: OperatorConfig + health: + healthProbeBindAddress: :8081 + metrics: + bindAddress: 0.0.0.0:8080 + webhook: + port: 9443 + deployment: + sidecar: + engineContainerTemplate: "/sidecar/engine_container_template.yaml" + networkValidatorContainerTemplate: "/sidecar/network_validator_container_template.yaml" + sessionAgentContainerTemplate: "/sidecar/session_agent_container_template.yaml" + engine: + bootstrapConfigTemplate: "/engine_bootstrap_config_template.yaml" + log: + level: {{ .Values.operator.config.logLevel }} + {{- with $.Values.operator.watchNamespaceSelector }} + namespaces: + selector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $.Values.operator.watchNamespaces }} + namespaces: + list: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/charts/airlock/microgateway/4.4.0/templates/operator/dashboard-configmap.yaml b/charts/airlock/microgateway/4.3.3/templates/operator/dashboard-configmap.yaml similarity index 100% rename from charts/airlock/microgateway/4.4.0/templates/operator/dashboard-configmap.yaml rename to charts/airlock/microgateway/4.3.3/templates/operator/dashboard-configmap.yaml diff --git a/charts/airlock/microgateway/4.4.0/templates/operator/deployment.yaml b/charts/airlock/microgateway/4.3.3/templates/operator/deployment.yaml similarity index 100% rename from charts/airlock/microgateway/4.4.0/templates/operator/deployment.yaml rename to charts/airlock/microgateway/4.3.3/templates/operator/deployment.yaml diff --git a/charts/airlock/microgateway/4.4.0/templates/operator/manager-role.yaml b/charts/airlock/microgateway/4.3.3/templates/operator/manager-role.yaml similarity index 100% rename from charts/airlock/microgateway/4.4.0/templates/operator/manager-role.yaml rename to charts/airlock/microgateway/4.3.3/templates/operator/manager-role.yaml diff --git a/charts/airlock/microgateway/4.4.0/templates/operator/manager-rolebinding.yaml b/charts/airlock/microgateway/4.3.3/templates/operator/manager-rolebinding.yaml similarity index 100% rename from charts/airlock/microgateway/4.4.0/templates/operator/manager-rolebinding.yaml rename to charts/airlock/microgateway/4.3.3/templates/operator/manager-rolebinding.yaml diff --git a/charts/airlock/microgateway/4.4.0/templates/operator/metrics-service.yaml b/charts/airlock/microgateway/4.3.3/templates/operator/metrics-service.yaml similarity index 100% rename from charts/airlock/microgateway/4.4.0/templates/operator/metrics-service.yaml rename to charts/airlock/microgateway/4.3.3/templates/operator/metrics-service.yaml diff --git a/charts/airlock/microgateway/4.4.0/templates/operator/mutating-webhook.yaml b/charts/airlock/microgateway/4.3.3/templates/operator/mutating-webhook.yaml similarity index 100% rename from charts/airlock/microgateway/4.4.0/templates/operator/mutating-webhook.yaml rename to charts/airlock/microgateway/4.3.3/templates/operator/mutating-webhook.yaml diff --git a/charts/airlock/microgateway/4.4.0/templates/operator/podmonitor.yaml b/charts/airlock/microgateway/4.3.3/templates/operator/podmonitor.yaml similarity index 100% rename from charts/airlock/microgateway/4.4.0/templates/operator/podmonitor.yaml rename to charts/airlock/microgateway/4.3.3/templates/operator/podmonitor.yaml diff --git a/charts/airlock/microgateway/4.4.0/templates/operator/role.yaml b/charts/airlock/microgateway/4.3.3/templates/operator/role.yaml similarity index 100% rename from charts/airlock/microgateway/4.4.0/templates/operator/role.yaml rename to charts/airlock/microgateway/4.3.3/templates/operator/role.yaml diff --git a/charts/airlock/microgateway/4.4.0/templates/operator/rolebinding.yaml b/charts/airlock/microgateway/4.3.3/templates/operator/rolebinding.yaml similarity index 100% rename from charts/airlock/microgateway/4.4.0/templates/operator/rolebinding.yaml rename to charts/airlock/microgateway/4.3.3/templates/operator/rolebinding.yaml diff --git a/charts/airlock/microgateway/4.4.0/templates/operator/selfsigned-issuer.yaml b/charts/airlock/microgateway/4.3.3/templates/operator/selfsigned-issuer.yaml similarity index 100% rename from charts/airlock/microgateway/4.4.0/templates/operator/selfsigned-issuer.yaml rename to charts/airlock/microgateway/4.3.3/templates/operator/selfsigned-issuer.yaml diff --git a/charts/airlock/microgateway/4.4.0/templates/operator/serviceaccount.yaml b/charts/airlock/microgateway/4.3.3/templates/operator/serviceaccount.yaml similarity index 100% rename from charts/airlock/microgateway/4.4.0/templates/operator/serviceaccount.yaml rename to charts/airlock/microgateway/4.3.3/templates/operator/serviceaccount.yaml diff --git a/charts/airlock/microgateway/4.4.0/templates/operator/servicemonitor.yaml b/charts/airlock/microgateway/4.3.3/templates/operator/servicemonitor.yaml similarity index 100% rename from charts/airlock/microgateway/4.4.0/templates/operator/servicemonitor.yaml rename to charts/airlock/microgateway/4.3.3/templates/operator/servicemonitor.yaml diff --git a/charts/airlock/microgateway/4.4.0/templates/operator/serving-certificate.yaml b/charts/airlock/microgateway/4.3.3/templates/operator/serving-certificate.yaml similarity index 100% rename from charts/airlock/microgateway/4.4.0/templates/operator/serving-certificate.yaml rename to charts/airlock/microgateway/4.3.3/templates/operator/serving-certificate.yaml diff --git a/charts/airlock/microgateway/4.4.0/templates/operator/validating-webhook.yaml b/charts/airlock/microgateway/4.3.3/templates/operator/validating-webhook.yaml similarity index 100% rename from charts/airlock/microgateway/4.4.0/templates/operator/validating-webhook.yaml rename to charts/airlock/microgateway/4.3.3/templates/operator/validating-webhook.yaml diff --git a/charts/airlock/microgateway/4.4.0/templates/operator/webhook-service.yaml b/charts/airlock/microgateway/4.3.3/templates/operator/webhook-service.yaml similarity index 100% rename from charts/airlock/microgateway/4.4.0/templates/operator/webhook-service.yaml rename to charts/airlock/microgateway/4.3.3/templates/operator/webhook-service.yaml diff --git a/charts/airlock/microgateway/4.4.0/templates/operator/xds-service.yaml b/charts/airlock/microgateway/4.3.3/templates/operator/xds-service.yaml similarity index 100% rename from charts/airlock/microgateway/4.4.0/templates/operator/xds-service.yaml rename to charts/airlock/microgateway/4.3.3/templates/operator/xds-service.yaml diff --git a/charts/airlock/microgateway/4.3.3/templates/tests/rbac.yaml b/charts/airlock/microgateway/4.3.3/templates/tests/rbac.yaml index 744799333f..93bd4cd1bd 100644 --- a/charts/airlock/microgateway/4.3.3/templates/tests/rbac.yaml +++ b/charts/airlock/microgateway/4.3.3/templates/tests/rbac.yaml @@ -2,63 +2,142 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: "{{ include "airlock-microgateway-cni.fullname" . }}-tests" - namespace: {{ .Release.Namespace }} labels: - {{- include "airlock-microgateway-cni.labelsWithoutComponent" . | nindent 4 }} app.kubernetes.io/component: tests + app.kubernetes.io/name: {{ include "airlock-microgateway.name" . }}-tests + {{- include "airlock-microgateway.sharedLabels" . | nindent 4 }} + name: "{{ include "airlock-microgateway.fullname" . }}-tests" + namespace: {{ .Release.Namespace }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: "{{ include "airlock-microgateway-cni.fullname" . }}-tests" - namespace: {{ .Release.Namespace }} labels: - {{- include "airlock-microgateway-cni.labelsWithoutComponent" . | nindent 4 }} app.kubernetes.io/component: tests + app.kubernetes.io/name: {{ include "airlock-microgateway.name" . }}-tests + {{- include "airlock-microgateway.sharedLabels" . | nindent 4 }} + name: "{{ include "airlock-microgateway.fullname" . }}-tests" + namespace: {{ .Release.Namespace }} roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: "{{ include "airlock-microgateway-cni.fullname" . }}-tests" + name: "{{ include "airlock-microgateway.fullname" . }}-tests" subjects: - kind: ServiceAccount - name: "{{ include "airlock-microgateway-cni.fullname" . }}-tests" + name: "{{ include "airlock-microgateway.fullname" . }}-tests" --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - name: "{{ include "airlock-microgateway-cni.fullname" . }}-tests" - namespace: {{ .Release.Namespace }} labels: - {{- include "airlock-microgateway-cni.labelsWithoutComponent" . | nindent 4 }} app.kubernetes.io/component: tests + app.kubernetes.io/name: {{ include "airlock-microgateway.name" . }}-tests + {{- include "airlock-microgateway.sharedLabels" . | nindent 4 }} + name: "{{ include "airlock-microgateway.fullname" . }}-tests" + namespace: {{ .Release.Namespace }} rules: - apiGroups: - - "apps" + - microgateway.airlock.com resources: - - daemonsets + - sidecargateways resourceNames: - - {{ include "airlock-microgateway-cni.fullname" . }} + - "{{ include "airlock-microgateway.fullname" . }}-test-sidecargateway" verbs: - - get - - watch - - list + - get + - list + - watch + - delete - apiGroups: - - "" + - microgateway.airlock.com resources: - - pods - - pods/log + - sidecargateways verbs: - - get - - list -{{- if .Values.rbac.createSCCRole }} + - create - apiGroups: - - security.openshift.io + - "" + resources: + - events + verbs: + - list +- apiGroups: + - "apps" + resources: + - deployments resourceNames: - - privileged + - "{{ include "airlock-microgateway.operator.fullname" . }}" + verbs: + - get + - list + - watch +- apiGroups: + - "apps" resources: - - securitycontextconstraints + - statefulsets + - statefulsets/scale + resourceNames: + - "{{ include "airlock-microgateway.fullname" . }}-test-backend" verbs: - - use -{{- end -}} + - get + - list + - watch + - patch +- apiGroups: + - "" + resources: + - pods + - pods/log + - pods/status + - pods/attach + resourceNames: + - "{{ include "airlock-microgateway.fullname" . }}-test-backend-0" + - "{{ include "airlock-microgateway.fullname" . }}-test-valid-request" + - "{{ include "airlock-microgateway.fullname" . }}-test-injection-request" + verbs: + - get + - list + - create + - watch + - delete +- apiGroups: + - "" + resources: + - pods + verbs: + - create +{{- if .Values.operator.watchNamespaceSelector }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/component: tests + app.kubernetes.io/name: {{ include "airlock-microgateway.name" . }}-tests + {{- include "airlock-microgateway.sharedLabels" . | nindent 4 }} + name: "{{ include "airlock-microgateway.fullname" . }}-tests-{{ .Release.Namespace }}" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: "{{ include "airlock-microgateway.fullname" . }}-tests-{{ .Release.Namespace }}" +subjects: + - kind: ServiceAccount + name: "{{ include "airlock-microgateway.fullname" . }}-tests" + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: tests + app.kubernetes.io/name: {{ include "airlock-microgateway.name" . }}-tests + {{- include "airlock-microgateway.sharedLabels" . | nindent 4 }} + name: "{{ include "airlock-microgateway.fullname" . }}-tests-{{ .Release.Namespace }}" +rules: +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - list +{{- end }} {{- end -}} diff --git a/charts/airlock/microgateway/4.4.0/templates/tests/service.yaml b/charts/airlock/microgateway/4.3.3/templates/tests/service.yaml similarity index 100% rename from charts/airlock/microgateway/4.4.0/templates/tests/service.yaml rename to charts/airlock/microgateway/4.3.3/templates/tests/service.yaml diff --git a/charts/airlock/microgateway/4.4.0/templates/tests/statefulset.yaml b/charts/airlock/microgateway/4.3.3/templates/tests/statefulset.yaml similarity index 100% rename from charts/airlock/microgateway/4.4.0/templates/tests/statefulset.yaml rename to charts/airlock/microgateway/4.3.3/templates/tests/statefulset.yaml diff --git a/charts/airlock/microgateway/4.3.3/templates/tests/test-install.yaml b/charts/airlock/microgateway/4.3.3/templates/tests/test-install.yaml index 12d8c8de78..ab82abea73 100644 --- a/charts/airlock/microgateway/4.3.3/templates/tests/test-install.yaml +++ b/charts/airlock/microgateway/4.3.3/templates/tests/test-install.yaml @@ -2,11 +2,14 @@ apiVersion: v1 kind: Pod metadata: - name: "{{ include "airlock-microgateway-cni.fullname" . }}-test-install" + name: "{{ include "airlock-microgateway.fullname" . }}-test-install" namespace: {{ .Release.Namespace }} labels: - {{- include "airlock-microgateway-cni.labelsWithoutComponent" . | nindent 4 }} app.kubernetes.io/component: test-install + app.kubernetes.io/name: {{ include "airlock-microgateway.name" . }}-tests + sidecar.istio.io/inject: "false" + {{- include "airlock-microgateway.sharedLabels" . | nindent 4 }} + {{- include "airlock-microgateway.sharedSelectorLabels" . | nindent 4 }} annotations: helm.sh/hook: test helm.sh/hook-delete-policy: before-hook-creation @@ -16,88 +19,209 @@ spec: - name: test image: "bitnami/kubectl:{{ .Capabilities.KubeVersion.Major }}.{{ .Capabilities.KubeVersion.Minor }}" securityContext: - allowPrivilegeEscalation: {{ .Values.privileged }} - capabilities: - drop: - - ALL - privileged: {{ .Values.privileged }} - readOnlyRootFilesystem: true - runAsGroup: 0 - runAsNonRoot: false - runAsUser: 0 - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /host/opt/cni/bin - name: cni-bin-dir - readOnly: true - - mountPath: /host/etc/cni/net.d - name: cni-net-dir - readOnly: true + {{- include "airlock-microgateway.restrictedSecurityContext" . | nindent 6 }} command: - sh - -c - | set -eu + clean_up() { + echo "" + echo "### Clean up test resources" + kubectl delete --ignore-not-found=true -n {{ .Release.Namespace }} sidecargateways.microgateway.airlock.com {{ include "airlock-microgateway.fullname" . }}-test-sidecargateway || true + echo "" + echo "### Scale down '{{ include "airlock-microgateway.fullname" . }}-test-backend'" + kubectl scale -n {{ .Release.Namespace }} statefulset/{{ include "airlock-microgateway.fullname" . }}-test-backend --replicas=0 --timeout=60s + sleep 3s + echo "" + } + fail() { - echo "Error: ${1}" echo "" - echo 'CNI installer logs:' - kubectl logs -n {{ .Release.Namespace }} daemonsets/{{ include "airlock-microgateway-cni.fullname" .}} -c cni-installer + echo "### Error: ${1}" + echo "" + + if kubectl get -n {{ .Release.Namespace }} sidecargateway.microgateway.airlock.com/{{ include "airlock-microgateway.fullname" . }}-test-sidecargateway >/dev/null 2>&1; then + echo "" + echo 'Microgateway Sidecargateway status:' + kubectl get -n {{ .Release.Namespace }} sidecargateway.microgateway.airlock.com/{{ include "airlock-microgateway.fullname" . }}-test-sidecargateway -o jsonpath-as-json='{.status}' || true + echo "" + echo "" + fi + + if kubectl get -n {{ .Release.Namespace }} pods/{{ include "airlock-microgateway.fullname" . }}-test-backend-0 >/dev/null 2>&1; then + echo "Pod '{{ include "airlock-microgateway.fullname" . }}-test-backend-0':" + kubectl describe -n {{ .Release.Namespace }} pods/{{ include "airlock-microgateway.fullname" . }}-test-backend-0 || true + echo "" + echo "" + echo 'Logs of Nginx container:' + kubectl logs -n {{ .Release.Namespace }} pods/{{ include "airlock-microgateway.fullname" . }}-test-backend-0 -c nginx --tail 5 || true + echo "" + echo "" + # Wait for engine logs + sleep 10s + echo 'Logs of Microgateway Engine container:' + kubectl logs -n {{ .Release.Namespace }} pods/{{ include "airlock-microgateway.fullname" . }}-test-backend-0 -c airlock-microgateway-engine --tail 5 || true + fi + exit 1 } - containsMGWCNIConf() { - cat "${1}" | grep -qe '"type":.*"{{ include "airlock-microgateway-cni.fullname" . }}"' + create_sidecargateway() { + # create SidecarGateway resource for testing purposes + kubectl delete --ignore-not-found=true -n {{ .Release.Namespace }} sidecargateways.microgateway.airlock.com {{ include "airlock-microgateway.fullname" . }}-test-sidecargateway || true + kubectl apply -f - </dev/null 2>&1; do sleep 1s; i=$((i+1)); done + kubectl logs -f -n {{ .Release.Namespace }} {{ include "airlock-microgateway.fullname" . }}-test-valid-request + kubectl delete pod --ignore-not-found=true -n {{ .Release.Namespace }} {{ include "airlock-microgateway.fullname" . }}-test-valid-request } - if ! kubectl rollout status --timeout=60s -n {{ .Release.Namespace }} daemonsets/{{ include "airlock-microgateway-cni.fullname" .}}; then - fail 'CNI DaemonSet rollout did not complete within timeout' + {{- if .Values.operator.watchNamespaceSelector }} + echo "### Verify that Namespace Selector matches Namespace '{{ .Release.Namespace }}'" + if ! kubectl get namespace -l '{{ include "airlock-microgateway.watchNamespaceSelector.labelQuery" .Values.operator.watchNamespaceSelector }}' | grep -q {{ .Release.Namespace }}; then + labels=$(kubectl get namespace {{ .Release.Namespace }} -o jsonpath={.metadata.labels} | jq | awk '{print " " $0}') + fail {{printf `"Operator namespace '%s' is not part of the operator's watch scope. To execute 'helm test', the selector configured in the helm value 'operator.watchNamespaceSelector' must match the namespace's labels:\n* Current selector:\n%s\n\n* Current labels:\n$labels\n###"` + .Release.Namespace + (replace "\"" "\\\"" (replace "\n" "\\n" (.Values.operator.watchNamespaceSelector | toPrettyJson | indent 2))) + }} + fi + echo "" + {{- end }} + + trap clean_up EXIT + echo "" + + echo "### Waiting for Microgateway Operator Deployments to be ready" + if ! kubectl rollout status -n {{ .Release.Namespace }} --timeout=90s \ + deployments/{{ include "airlock-microgateway.operator.fullname" . }}; then + fail 'Timout occurred' + fi + echo "" + + echo "### Scale '{{ include "airlock-microgateway.fullname" . }}-test-backend' to '1' replica" + # scale to zero replicas to ensure no pods are present from previous runs + kubectl scale -n {{ .Release.Namespace }} statefulset/{{ include "airlock-microgateway.fullname" . }}-test-backend --replicas=0 --timeout=10s + kubectl scale -n {{ .Release.Namespace }} statefulset/{{ include "airlock-microgateway.fullname" . }}-test-backend --replicas=1 --timeout=10s + echo "" + + echo "### Waiting for backend pod" + i=0 + while true; do + if kubectl get -n {{ .Release.Namespace }} pods/{{ include "airlock-microgateway.fullname" . }}-test-backend-0; then + break + elif [ $i -gt 3 ]; then + fail 'Pod not ready' + fi + sleep 2s + i=$((i+1)) + done + + echo "### Checking Microgateway Engine sidecar container was injected" + if ! kubectl get -n {{ .Release.Namespace }} pods/{{ include "airlock-microgateway.fullname" . }}-test-backend-0 -o jsonpath='{.spec.containers[?(@.name=="airlock-microgateway-engine")]}' | grep -q "airlock-microgateway-engine"; then + fail 'Microgateway Engine sidecar container not injected' + fi + echo "True" + echo "" + + echo "### Checking for valid license" + i=0 + while true; do + if [ "$(kubectl get -n {{ .Release.Namespace }} pods/{{ include "airlock-microgateway.fullname" . }}-test-backend-0 -o jsonpath='{.metadata.labels.sidecar\.microgateway\.airlock\.com/licensed}')" = 'true' ]; then + break + elif [ $i -gt 30 ]; then + fail 'Microgateway license is missing or invalid' + fi + sleep 2s + i=$((i+1)) + done + echo "True" + echo "" + + echo "### Create SidecarGateway resource for testing" + if ! create_sidecargateway ; then + fail 'Creation of SidecarGateway resource failed' + fi + echo "" + + echo "### Waiting for '{{ include "airlock-microgateway.fullname" . }}-test-backend' to be ready" + if ! kubectl rollout status -n {{ .Release.Namespace }} statefulset/{{ include "airlock-microgateway.fullname" . }}-test-backend --timeout=90s; then + fail 'Timout occurred' + fi + echo "" + + echo "### Waiting for 'engine-config-valid' condition" + if ! kubectl wait -n {{ .Release.Namespace }} pods --field-selector=metadata.name={{ include "airlock-microgateway.fullname" . }}-test-backend-0 --timeout=90s --for=condition=microgateway.airlock.com/engine-config-valid=True; then + fail 'Configuration was never accepted by the Microgateway Engine' fi + sleep 5s + echo "" + echo "" - echo "Checking whether CNI binary was installed" - if ! [ -f "/host/opt/cni/bin/{{ include "airlock-microgateway-cni.fullname" . }}" ]; then - fail 'CNI binary was not installed' + echo "### Checking whether a valid request is successful and returns HTTP status code '200'" + out=$(curl -vsS --retry 3 --retry-connrefused --connect-timeout 10 "http://{{ include "airlock-microgateway.fullname" . }}-test-service:8080/" || true) + echo "Response:" + echo "${out}" + if ! echo "${out}" | grep -q "200 OK"; then + fail 'A valid request was not successful' fi + echo "" + echo "" - echo "Checking whether CNI kubeconfig was installed" - if ! [ -f "/host/etc/cni/net.d/{{ include "airlock-microgateway-cni.fullname" . }}-kubeconfig" ]; then - fail 'CNI kubeconfig was not created' + echo "### Checking whether a request with an injection attack is blocked and returns HTTP status code '400'" + out=$(curl -vsS --retry 3 --retry-connrefused --connect-timeout 10 "http://{{ include "airlock-microgateway.fullname" . }}-test-service:8080/?token='%20UnION%20all%20select%20A" || true) + echo "Response:" + echo "${out}" + if ! echo "${out}" | grep -q "400 Bad Request"; then + fail 'A malicious request was not blocked' fi + echo "" + echo "" - echo "Checking whether CNI configuration was written" - case {{ .Values.config.installMode }} in - "chained") - for file in "/host/etc/cni/net.d/"*.conflist; do - if containsMGWCNIConf "${file}"; then - echo "Success" - exit 0 - fi - done - ;; - "standalone") - if containsMGWCNIConf "/host/etc/cni/net.d/{{ include "airlock-microgateway-cni.fullname" . }}.conflist"; then - echo "Success" - exit 0 - fi - ;; - "manual") - echo "- Skipping because we are in 'manual' install mode" - echo "Success" - exit 0 - ;; - esac - - fail 'Configuration for plugin "{{ include "airlock-microgateway-cni.fullname" . }}" was not found' - serviceAccountName: "{{ include "airlock-microgateway-cni.fullname" . }}-tests" - volumes: - - hostPath: - path: "{{ .Values.config.cniBinDir }}" - type: Directory - name: cni-bin-dir - - hostPath: - path: "{{ .Values.config.cniNetDir }}" - type: Directory - name: cni-net-dir + echo "### Installation of '{{ include "airlock-microgateway.fullname" . }}' succeeded" + exit 0 + serviceAccountName: "{{ include "airlock-microgateway.fullname" . }}-tests" {{- end -}} diff --git a/charts/airlock/microgateway/4.3.3/values.schema.json b/charts/airlock/microgateway/4.3.3/values.schema.json index e087bd7004..173d6b084c 100644 --- a/charts/airlock/microgateway/4.3.3/values.schema.json +++ b/charts/airlock/microgateway/4.3.3/values.schema.json @@ -14,6 +14,15 @@ "commonAnnotations": { "$ref": "#/definitions/StringMap" }, + "crds": { + "type": "object", + "properties": { + "skipVersionCheck": { + "type": "boolean" + } + }, + "additionalProperties": false + }, "imagePullSecrets": { "type": "array", "items": { @@ -30,120 +39,304 @@ "additionalProperties": true } }, - "image": { - "$ref": "#/definitions/Image" - }, - "podAnnotations": { - "$ref": "#/definitions/StringMap" - }, - "podLabels": { - "$ref": "#/definitions/StringMap" - }, - "resources": { - "type": "object" - }, - "nodeSelector": { - "$ref": "#/definitions/StringMap" - }, - "affinity": { - "type": "object" - }, - "rbac": { + "operator": { "type": "object", "properties": { - "create": { - "type": "boolean" + "replicaCount": { + "type": "integer", + "minimum": 0 }, - "createSCCRole": { - "type": "boolean" + "updateStrategy": { + "$ref": "#/definitions/UpdateStrategy" + }, + "image": { + "$ref": "#/definitions/Image" + }, + "podAnnotations": { + "$ref": "#/definitions/StringMap" + }, + "podLabels": { + "$ref": "#/definitions/StringMap" + }, + "serviceAnnotations": { + "$ref": "#/definitions/StringMap" + }, + "serviceLabels": { + "$ref": "#/definitions/StringMap" + }, + "resources": { + "type": "object" + }, + "nodeSelector": { + "$ref": "#/definitions/StringMap" + }, + "tolerations": { + "type": "array", + "items": { + "type": "object" + } + }, + "affinity": { + "type": "object" + }, + "config": { + "type": "object", + "properties": { + "logLevel": { + "type": "string", + "enum": [ + "debug", + "info", + "warn", + "error" + ] + } + }, + "required": [ + "logLevel" + ], + "additionalProperties": false + }, + "serviceAccount": { + "type": "object", + "properties": { + "create": { + "type": "boolean" + }, + "annotations": { + "$ref": "#/definitions/StringMap" + }, + "name": { + "type": "string" + } + }, + "required": [ + "annotations", + "create", + "name" + ], + "additionalProperties": false + }, + "watchNamespaces": { + "type": "array", + "items": { + "type": "string" + } + }, + "watchNamespaceSelector": { + "$ref": "#/definitions/LabelSelector" + }, + "rbac": { + "type": "object", + "properties": { + "create": { + "type": "boolean" + } + }, + "required": [ + "create" + ], + "additionalProperties": false + }, + "serviceMonitor": { + "type": "object", + "properties": { + "create": { + "type": "boolean" + }, + "labels": { + "$ref": "#/definitions/StringMap" + } + }, + "required": [ + "create" + ], + "additionalProperties": false } }, + "oneOf": [ + { + "properties": { + "watchNamespaces": { + "minItems": 1 + }, + "watchNamespaceSelector": { + "additionalProperties": false + } + } + }, + { + "properties": { + "watchNamespaces": { + "maxItems": 0 + }, + "watchNamespaceSelector": { + "$ref": "#/definitions/LabelSelector" + } + } + } + ], "required": [ - "create", - "createSCCRole" + "affinity", + "config", + "image", + "updateStrategy", + "nodeSelector", + "podAnnotations", + "podLabels", + "rbac", + "replicaCount", + "resources", + "serviceAccount", + "serviceAnnotations", + "serviceLabels", + "serviceMonitor", + "tolerations" ], "additionalProperties": false }, - "privileged": { - "type": "boolean" - }, - "serviceAccount": { + "engine": { "type": "object", "properties": { - "create": { - "type": "boolean" + "image": { + "$ref": "#/definitions/Image" }, - "annotations": { - "$ref": "#/definitions/StringMap" + "resources": { + "type": "object" }, - "name": { - "type": "string" + "sidecar": { + "type": "object", + "properties":{ + "podMonitor": { + "type": "object", + "properties": { + "create": { + "type": "boolean" + }, + "labels": { + "$ref": "#/definitions/StringMap" + } + }, + "required": [ + "create" + ], + "additionalProperties": false + } + }, + "required": [ + "podMonitor" + ], + "additionalProperties": false } }, "required": [ - "annotations", - "create", - "name" + "image", + "resources", + "sidecar" ], "additionalProperties": false }, - "multusNetworkAttachmentDefinition": { + "networkValidator": { "type": "object", "properties": { - "create": { - "type": "boolean" - }, - "namespace": { - "type": "string" + "image": { + "$ref": "#/definitions/Image" } }, "required": [ - "create", - "namespace" + "image" ], "additionalProperties": false }, - "config": { + "sessionAgent": { "type": "object", "properties": { - "installMode": { - "type": "string", - "enum": [ - "chained", - "standalone", - "manual" - ] + "image": { + "$ref": "#/definitions/Image" }, - "logLevel": { - "type": "string", - "enum": [ - "debug", - "info", - "warn", - "error" - ] - }, - "cniNetDir": { + "resources": { + "type": "object" + } + }, + "required": [ + "image", + "resources" + ], + "additionalProperties": false + }, + "license": { + "type": "object", + "properties": { + "secretName": { "type": "string", "minLength": 1 + } + }, + "required": [ + "secretName" + ], + "additionalProperties": false + }, + "dashboards": { + "type": "object", + "properties" : { + "create": { + "type": "boolean" }, - "cniBinDir": { - "type": "string", - "minLength": 1 + "config": { + "type": "object", + "properties": { + "grafana": { + "type": "object", + "properties": { + "folderAnnotation": { + "$ref": "#/definitions/NameValuePair" + }, + "dashboardLabel": { + "$ref": "#/definitions/NameValuePair" + } + }, + "required": [ + "folderAnnotation", + "dashboardLabel" + ], + "additionalProperties": false + } + }, + "required": [ + "grafana" + ], + "additionalProperties": false }, - "excludeNamespaces": { - "type": "array", - "items": { - "type": "string" - } + "instances": { + "type": "object", + "properties": { + "overview": { + "$ref": "#/definitions/DashboardInstance" + }, + "license" : { + "$ref": "#/definitions/DashboardInstance" + }, + "blockMetrics" : { + "$ref": "#/definitions/DashboardInstance" + }, + "blockLogs" : { + "$ref": "#/definitions/DashboardInstance" + } + }, + "required": [ + "overview", + "license", + "blockMetrics", + "blockLogs" + ], + "additionalProperties": false } }, "required": [ - "cniBinDir", - "cniNetDir", - "excludeNamespaces", - "installMode", - "logLevel" + "create", + "config", + "instances" ], "additionalProperties": false }, @@ -164,22 +357,18 @@ } }, "required": [ - "affinity", "commonAnnotations", "commonLabels", - "config", + "crds", + "engine", "fullnameOverride", - "image", "imagePullSecrets", - "multusNetworkAttachmentDefinition", + "license", "nameOverride", - "nodeSelector", - "podAnnotations", - "podLabels", - "privileged", - "rbac", - "resources", - "serviceAccount", + "operator", + "networkValidator", + "sessionAgent", + "dashboards", "tests" ], "additionalProperties": false, @@ -220,6 +409,132 @@ "tag" ], "additionalProperties": false + }, + "LabelSelector": { + "type": "object", + "properties": { + "matchExpressions": { + "type": "array", + "items": { + "type": "object", + "required": [ + "key", + "operator" + ], + "properties": { + "key": { + "type": "string" + }, + "operator": { + "type": "string" + }, + "values": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "additionalProperties": false + } + }, + "matchLabels": { + "$ref": "#/definitions/StringMap" + } + }, + "additionalProperties": false + }, + "UpdateStrategy": { + "type": "object", + "oneOf" : [ + { + "properties": { + "type": { + "$ref": "#/definitions/RecreateType" + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + { + "properties": { + "type": { + "$ref": "#/definitions/RollingUpdateType" + }, + "rollingUpdate": { + "$ref": "#/definitions/RollingUpdate" + } + }, + "required": [ + "type" + ], + "additionalProperties": false + } + ] + }, + "RecreateType": { + "type": "string", + "enum": [ + "Recreate" + ] + }, + "RollingUpdateType": { + "type": "string", + "enum": [ + "RollingUpdate" + ] + }, + "RollingUpdate": { + "type": "object", + "properties": { + "maxSurge": { + "type": ["integer", "string"], + "minimum": 0, + "pattern": "^\\d+%?$" + }, + "maxUnavailable": { + "type": ["integer", "string"], + "minimum": 0, + "pattern": "^\\d+%?$" + } + }, + "anyOf": [ + {"required": ["maxSurge"]}, + {"required": ["maxUnavailable"]} + ], + "additionalProperties": false + }, + "DashboardInstance" : { + "type" : "object", + "properties" : { + "create" : { + "type" : "boolean" + } + }, + "required" : [ + "create" + ], + "additionalProperties": false + }, + "NameValuePair" : { + "type" : "object", + "properties" : { + "name" : { + "type": "string", + "minLength": 1 + }, + "value" : { + "type" : "string", + "minLength": 1 + } + }, + "required" : [ + "name", + "value" + ], + "additionalProperties": false } } } diff --git a/charts/airlock/microgateway/4.3.3/values.yaml b/charts/airlock/microgateway/4.3.3/values.yaml index 3dc707bae4..03fc87d215 100644 --- a/charts/airlock/microgateway/4.3.3/values.yaml +++ b/charts/airlock/microgateway/4.3.3/values.yaml @@ -1,4 +1,4 @@ -# -- Allows overriding the name to use instead of "microgateway-cni". +# -- Allows overriding the name to use instead of "microgateway". nameOverride: "" # -- Allows overriding the name to use as full name of resources. fullnameOverride: "" @@ -10,75 +10,203 @@ commonAnnotations: {} imagePullSecrets: [] # - name: myRegistryKeySecretName -# Specifies the Airlock Microgateway CNI image. -image: - # -- Image repository from which to pull the Airlock Microgateway CNI image. - repository: "quay.io/airlock/microgateway-cni" - # -- Image tag to pull. - tag: "4.3.3" - # -- SHA256 image digest to pull (in the format "sha256:7144f7bab3d4c2648d7e59409f15ec52a18006a128c733fcff20d3a4a54ba44a"). - # Overrides tag when specified. - digest: "sha256:16317b9a8430059c15175673ad53e31d9e882a1d1af6576214eb1534d8ea6937" - # -- Pull policy for this image. - pullPolicy: IfNotPresent -# -- Annotations to add to all Pods. -podAnnotations: {} -# -- Labels to add to all Pods. -podLabels: {} -# -- Resource restrictions to apply to the CNI installer container. -resources: - requests: - cpu: 10m - memory: 100Mi -# -- NodeSelector to apply to the CNI DaemonSet in order to only deploy the CNI plugin on specific nodes. -nodeSelector: - kubernetes.io/os: linux -# -- Custom affinity for the DaemonSet to only deploy the CNI plugin on specific nodes. -affinity: {} -# Configures the generation of RBAC Roles and RoleBindings. -rbac: - # -- Whether to create RBAC resources which are required for the CNI plugin to function. - create: true - # -- (OpenShift) Whether to create RBAC resources which allow the CNI installer to use the "privileged" security context constraint. - createSCCRole: false -# -- Whether the DaemonSet should run in privileged mode. Must be enabled for environments which require it for writing files to the host (e.g. OpenShift). -privileged: false -# Configures the generation of the ServiceAccount. -serviceAccount: - # -- Whether a ServiceAccount should be created. - create: true - # -- Annotations to add to the ServiceAccount. - annotations: {} - # -- Name of the ServiceAccount to use. - # If not set and create is true, a name is generated using the fullname template. - name: "" -# Configures the generation of a NetworkAttachmentDefinition for use with Multus CNI (OpenShift) -multusNetworkAttachmentDefinition: - # -- Whether a NetworkAttachmentDefinition CR should be created, which can be used for applying the CNI plugin to Pods. +crds: + # -- Whether to skip the sanity check which prevents installing/upgrading the helm chart in a cluster with outdated Airlock Microgateway CRDs. + # The check aims to prevent unexpected behavior and issues due to Helm v3 not automatically upgrading CRDs which are already present in the cluster + # when performing a "helm install/upgrade". + skipVersionCheck: false +operator: + # -- Number of replicas for the operator Deployment. + replicaCount: 2 + # -- Specifies the operator update strategy. + updateStrategy: + type: RollingUpdate + # Specifies the Airlock Microgateway Operator image. + image: + # -- Image repository from which to pull the Airlock Microgateway Operator image. + repository: "quay.io/airlock/microgateway-operator" + # -- Image tag to pull. + tag: "4.3.3" + # -- SHA256 image digest to pull (in the format "sha256:c79ee3f85862fb386e9dd62b901b607161d27807f512d7fbdece05e9ee3d7c63"). + # Overrides tag when specified. + digest: "sha256:6d3ebca355de0a67f0bf5f088a15b9410564e500033d3e1f534a2f49a05bf4c3" + # -- Pull policy for this image. + pullPolicy: IfNotPresent + # -- Annotations to add to all Pods. + podAnnotations: {} + # -- Labels to add to all Pods. + podLabels: {} + # -- Annotations to add to the Service. + serviceAnnotations: {} + # prometheus.io/scrape: "true" + # prometheus.io/port: "8080" + + # -- Labels to add to the Service. + serviceLabels: {} + # -- Resource restrictions to apply to the operator container. + resources: {} + # We recommend at least the following resource specification. + # limits: + # cpu: 1000m + # memory: 512Mi + # requests: + # cpu: 100m + # memory: 512Mi + + # -- Custom nodeSelector to apply to the operator Deployment in order to constrain its Pods to certain nodes. + nodeSelector: {} + # -- Custom tolerations to apply to the operator Deployment in order to allow its Pods to run on tainted nodes. + tolerations: [] + # -- Custom affinity to apply to the operator Deployment. Used to influence the scheduling. + affinity: {} + # Parameters for the operator configuration. + config: + # -- Operator application log level. + logLevel: "info" + # Configures the generation of the ServiceAccount. + serviceAccount: + # -- Whether a ServiceAccount should be created. + create: true + # -- Annotations to add to the ServiceAccount. + annotations: {} + # -- Name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template. + name: "" + # -- Allows to restrict the operator to specific namespaces, depending on your needs. + # For a `OwnNamespace` or `SingleNamespace` installation the list may only contain one namespace (e.g., `watchNamespaces: ["airlock-microgateway-system"]`). + # In case of the `OwnNamespace` installation mode the specified namespace should be equal to the installation namespace. + # For a static `MultiNamespace` installation, the complete list of namespaces must be provided in the `watchNamespaces`. + # An `AllNamespaces` installation or the usage of the `watchNamespaceSelector` requires the `watchNamespaces` to be empty. + # Regardless of the installation modes supported by `watchNamespaces`, RBAC is created only namespace-scoped (using Roles and RoleBindings) in the respective namespaces. + # Please note that this feature requires a Premium license. + watchNamespaces: [] + # -- Allows to dynamically select watch namespaces of the operator and the scope of the webhooks based on a Namespace label selector. + # It is able to detect and reconcile resources in all namespaces that match the label selector automatically, even for new namespaces, without restarting the operator. + # This facilitates a dynamic `MultiNamespace` installation mode, but still requires cluster-scoped permissions (i.e., ClusterRoles and ClusterRoleBindings). + # An `AllNamespaces` installation or the usage of the `watchNamespaces` requires the `watchNamespaceSelector` to be empty. + # Please note that this feature requires a Premium license. + watchNamespaceSelector: {} + # For further examples, see: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#resources-that-support-set-based-requirements. + # matchLabels: + # microgateway.airlock.com/enable: "true" + # matchExpressions: + # - { key: environment, operator: NotIn, values: [dev] } + + # Configures the generation of Role and RoleBinding as well as ClusterRoles and ClusterRoleBinding pairs for the ServiceAccount specified above. + rbac: + # -- Whether to create RBAC resources which are required for the Airlock Microgateway Operator to function. + create: true + # Configures the generation of a Prometheus Operator ServiceMonitor. + serviceMonitor: + # -- Whether to create a ServiceMonitor resource for monitoring. + create: false + # -- Labels to add to the ServiceMonitor. + labels: {} + # release: "" +engine: + # Specifies the Airlock Microgateway Engine image. + image: + # -- Image repository from which to pull the Airlock Microgateway Engine image. + repository: "quay.io/airlock/microgateway-engine" + # -- Image tag to pull. + tag: "4.3.3" + # -- SHA256 image digest to pull (in the format "sha256:a3051f42d3013813b05f7513bb86ed6a3209cb3003f1bb2f7b72df249aa544d3"). + # Overrides tag when specified. + digest: "sha256:3c0ebee0b560c8699723bfa433cd601b04b190c384e031d3789b83287fab7a9b" + # -- Pull policy for this image. + pullPolicy: IfNotPresent + # -- Resource restrictions to apply to the Airlock Microgateway Engine container. + resources: {} + # We recommend at least the following resource specification. + # limits: + # cpu: 500m + # memory: 128Mi + # requests: + # cpu: 10m + # memory: 40Mi + + # Additional configuration when deployed as a sidecar. + sidecar: + # Configures the generation of a Prometheus Operator PodMonitor. + podMonitor: + # -- Whether to create a PodMonitor resource for monitoring. + create: false + # -- Labels to add to the PodMonitor. + labels: {} + # release: "" +networkValidator: + # Specifies the Airlock Microgateway Network Validator image to be injected as an init-container. + image: + # -- Image repository from which to pull the netcat image for the Airlock Microgateway Network Validator init-container. + repository: "cgr.dev/chainguard/netcat" + # -- Image tag to pull. + tag: "" + # -- SHA256 image digest to pull (in the format "sha256:6051975a14c51b9d3b525a06004d62a4d323c08ca58e3468343095a55a42fff2"). + # Overrides tag when specified. + digest: "sha256:6051975a14c51b9d3b525a06004d62a4d323c08ca58e3468343095a55a42fff2" + # -- Pull policy for this image. + pullPolicy: IfNotPresent +sessionAgent: + # Specifies the Airlock Microgateway Session Agent image. + image: + # -- Image repository from which to pull the Airlock Microgateway Session Agent image. + repository: "quay.io/airlock/microgateway-session-agent" + # -- Image tag to pull. + tag: "4.3.3" + # -- SHA256 image digest to pull (in the format "sha256:a3051f42d3013813b05f7513bb86ed6a3209cb3003f1bb2f7b72df249aa544d3"). + # Overrides tag when specified. + digest: "sha256:994bf4117adb74da4e05c22ffc168d9844bc68efa6a7fb96d73e849d1ef67b56" + # -- Pull policy for this image. + pullPolicy: IfNotPresent + # -- Resource restrictions to apply to the Airlock Microgateway Session Agent container. + resources: {} + # We recommend at least the following resource specification. + # limits: + # cpu: 150m + # memory: 32Mi + # requests: + # cpu: 10m + # memory: 8Mi +license: + # -- Name of the secret containing the "microgateway-license.txt" key. + secretName: "airlock-microgateway-license" +# Creates dashboards in the form of ConfigMaps that can be imported +# by Grafana using its sidecar setup. +dashboards: + # -- Whether to create any ConfigMaps containing Grafana dashboards to import. create: false - # -- Namespace in which the NetworkAttachmentDefinition is deployed. - # Note: If namespace is set to a custom value, referencing the created NetworkAttachmentDefinition from other namespaces - # may not work if Multus namespace isolation is enabled. https://github.com/k8snetworkplumbingwg/multus-cni/blob/v4.0.2/docs/configuration.md#namespace-isolation - namespace: default -# Parameters for the CNI installer configuration. -config: - # -- Whether to install the CNI plugin as a `chained` plugin (default, required with most interface CNI providers), - # as a `standalone` plugin (required for use with Multus CNI, e.g. on OpenShift) - # or in `manual` mode, where no CNI network configuration is written. - installMode: "chained" - # -- Log level for the CNI installer and plugin. - logLevel: info - # -- Directory where the CNI config files reside on the host. - # This path can either be found in the documentation of your Kubernetes distribution or CNI provider. - # It can also be queried by running the command `crictl info -o go-template --template '{{.config.cni.confDir}}'` on your Kubernetes node. - cniNetDir: "/etc/cni/net.d" - # -- Directory where the CNI plugin binaries reside on the host. - # This path can either be found in the documentation of your Kubernetes distribution or CNI provider. - # It can also be queried by running the command `crictl info -o go-template --template '{{.config.cni.binDir}}'` on your Kubernetes node. - cniBinDir: "/opt/cni/bin" - # -- Namespaces for which this CNI plugin should not apply any modifications. - excludeNamespaces: - - kube-system + config: + # Configures the necessary label and annotations along with their values + # to enable Grafana to correctly identify the ConfigMaps containing + # dashboards and file them within a dedicated folder in the dashboard overview. + # These settings need to match the Grafana sidecar configuration. + grafana: + folderAnnotation: + # -- Name of the annotation containing the folder name to file dashboards into. + name: "grafana_folder" + # -- Name of the folder dashboards are filed into within the Grafana UI. + value: "Airlock Microgateway" + dashboardLabel: + # -- Name of the label that lets Grafana identify ConfigMaps that represent dashboards. + name: "grafana_dashboard" + # -- Value of the label that lets Grafana identify ConfigMaps that represent dashboards. + value: "1" + instances: + # Available dashboard instances that can be individually created/deployed. + overview: + # -- Whether to create the overview dashboard. + create: true + license: + # -- Whether to create the license dashboard. + create: true + blockMetrics: + # -- Whether to create the block metrics dashboard. + create: true + blockLogs: + # -- Whether to create the block logs dashboard. + create: true +# Check whether the installation of the Airlock Microgateway Helm Chart was successful. +# Requires a secret with a valid Airlock Microgateway license key already to be present. tests: # -- Whether additional resources required for running `helm test` should be created (e.g. Roles and ServiceAccounts). # If set to false, `helm test` will not run any tests. diff --git a/charts/airlock/microgateway/4.4.0/.helmignore b/charts/airlock/microgateway/4.4.0/.helmignore index 101ff5ac56..8561d28926 100644 --- a/charts/airlock/microgateway/4.4.0/.helmignore +++ b/charts/airlock/microgateway/4.4.0/.helmignore @@ -21,8 +21,7 @@ .idea/ *.tmproj .vscode/ -# CRDs kustomization.yaml -/crds/kustomization.yaml + # Helm unit tests /tests /validation diff --git a/charts/airlock/microgateway/4.4.0/Chart.yaml b/charts/airlock/microgateway/4.4.0/Chart.yaml index 25d5570c2b..883012b58a 100644 --- a/charts/airlock/microgateway/4.4.0/Chart.yaml +++ b/charts/airlock/microgateway/4.4.0/Chart.yaml @@ -9,15 +9,15 @@ annotations: - name: Airlock Microgateway Forum url: https://forum.airlock.com/ catalog.cattle.io/certified: partner - catalog.cattle.io/display-name: Airlock Microgateway + catalog.cattle.io/display-name: Airlock Microgateway CNI catalog.cattle.io/kube-version: '>=1.25.0-0' - catalog.cattle.io/release-name: microgateway - charts.openshift.io/name: Airlock Microgateway + catalog.cattle.io/release-name: microgateway-cni + charts.openshift.io/name: Airlock Microgateway CNI apiVersion: v2 appVersion: 4.4.0 -description: A Helm chart for deploying the Airlock Microgateway +description: A Helm chart for deploying the Airlock Microgateway CNI plugin home: https://www.airlock.com/en/microgateway -icon: file://assets/icons/microgateway.svg +icon: file://assets/icons/microgateway-cni.svg keywords: - WAF - Web Application Firewall @@ -30,14 +30,13 @@ keywords: - Filtering - DevSecOps - shift left -- control plane -- Operator +- CNI kubeVersion: '>=1.25.0-0' maintainers: - email: support@airlock.com name: Airlock url: https://www.airlock.com/ -name: microgateway +name: microgateway-cni sources: - https://github.com/airlock/microgateway type: application diff --git a/charts/airlock/microgateway/4.4.0/README.md b/charts/airlock/microgateway/4.4.0/README.md index b6f66cdcda..f72f069cf7 100644 --- a/charts/airlock/microgateway/4.4.0/README.md +++ b/charts/airlock/microgateway/4.4.0/README.md @@ -1,4 +1,4 @@ -# Airlock Microgateway +# Airlock Microgateway CNI ![Version: 4.4.0](https://img.shields.io/badge/Version-4.4.0-informational?style=flat-square) ![AppVersion: 4.4.0](https://img.shields.io/badge/AppVersion-4.4.0-informational?style=flat-square) @@ -40,58 +40,43 @@ Check the official documentation at **[docs.airlock.com](https://docs.airlock.co The instructions below provide a quick start guide. Detailed information are provided in the **[manual](https://docs.airlock.com/microgateway/latest/)**. ## Prerequisites -* (Recommended) [Airlock Microgateway CNI](https://artifacthub.io/packages/helm/airlock-microgateway-cni/microgateway-cni) (Required for [data plane mode sidecar](https://docs.airlock.com/microgateway/latest/?topic=MGW-00000137)) -* [Airlock Microgateway License](#obtain-airlock-microgateway-license) -* [cert-manager](https://cert-manager.io/) * [helm](https://helm.sh/docs/intro/install/) (>= v3.8.0) -In order to use Airlock Microgateway you need a license and the cert-manager. You may either request a community license free of charge or purchase a premium license. -For an easy start in non-production environments, you may deploy the same cert-manager we are using internally for testing. -### Obtain Airlock Microgateway License -1. Either request a community or premium license - * Community license: [airlock.com/microgateway-community](https://airlock.com/en/microgateway-community) - * Premium license: [airlock.com/microgateway-premium](https://airlock.com/en/microgateway-premium) -2. Check your inbox and save the license file microgateway-license.txt locally. - -> See [Community vs. Premium editions in detail](https://docs.airlock.com/microgateway/latest/#data/1675772882054.html) to choose the right license type. -### Deploy cert-manager -```bash -helm repo add jetstack https://charts.jetstack.io -helm install cert-manager jetstack/cert-manager --version 'v1.16.1' -n cert-manager --create-namespace --set crds.enabled=true --wait -``` - -## Deploy Airlock Microgateway Operator - -> This guide assumes a microgateway-license.txt file is present in the working directory. - -1. Install CRDs and Operator. +## Deploy Airlock Microgateway CNI +1. Install the CNI Plugin with Helm. + > **Note**: Certain environments such as OpenShift or GKE require non-default configurations when installing the CNI plugin. For the most common setups, values files are provided in the [chart folder](/deploy/charts/airlock-microgateway-cni). ```bash - # Create namespace - kubectl create namespace airlock-microgateway-system - - # Install License - kubectl -n airlock-microgateway-system create secret generic airlock-microgateway-license --from-file=microgateway-license.txt - - # Install Operator (CRDs are included via the standard Helm 3 mechanism, i.e. Helm will handle initial installation but not upgrades) - helm install airlock-microgateway -n airlock-microgateway-system oci://quay.io/airlockcharts/microgateway --version '4.4.0' --wait + # Standard setup + helm install airlock-microgateway-cni -n kube-system oci://quay.io/airlockcharts/microgateway-cni --version '4.4.0' + kubectl -n kube-system rollout status daemonset -l app.kubernetes.io/instance=airlock-microgateway-cni + ``` + ```bash + # GKE setup + helm install airlock-microgateway-cni -n kube-system oci://quay.io/airlockcharts/microgateway-cni --version '4.4.0' -f https://raw.githubusercontent.com/airlock/microgateway/4.4.0/deploy/charts/airlock-microgateway-cni/gke-values.yaml + kubectl -n kube-system rollout status daemonset -l app.kubernetes.io/instance=airlock-microgateway-cni + ``` + ```bash + # OpenShift setup + helm install airlock-microgateway-cni -n openshift-operators oci://quay.io/airlockcharts/microgateway-cni --version '4.4.0' -f https://raw.githubusercontent.com/airlock/microgateway/4.4.0/deploy/charts/airlock-microgateway-cni/openshift-values.yaml + kubectl -n openshift-operators rollout status daemonset -l app.kubernetes.io/instance=airlock-microgateway-cni ``` + > **Important:** On OpenShift, all pods which should be protected by Airlock Microgateway must explicitly reference the Airlock Microgateway CNI NetworkAttachmentDefinition via the annotation `k8s.v1.cni.cncf.io/networks` (see [documentation](https://docs.airlock.com/microgateway/latest/#data/1658483168033.html) for details). 2. (Recommended) You can verify the correctness of the installation with `helm test`. ```bash - helm upgrade airlock-microgateway -n airlock-microgateway-system --set tests.enabled=true --reuse-values oci://quay.io/airlockcharts/microgateway --version '4.4.0' - helm test airlock-microgateway -n airlock-microgateway-system --logs - helm upgrade airlock-microgateway -n airlock-microgateway-system --set tests.enabled=false --reuse-values oci://quay.io/airlockcharts/microgateway --version '4.4.0' + # Standard and GKE setup + helm upgrade airlock-microgateway-cni -n kube-system --set tests.enabled=true --reuse-values oci://quay.io/airlockcharts/microgateway-cni --version '4.4.0' + helm test airlock-microgateway-cni -n kube-system --logs + helm upgrade airlock-microgateway-cni -n kube-system --set tests.enabled=false --reuse-values oci://quay.io/airlockcharts/microgateway-cni --version '4.4.0' + ``` + ```bash + # OpenShift setup + helm upgrade airlock-microgateway-cni -n openshift-operators --set tests.enabled=true --reuse-values oci://quay.io/airlockcharts/microgateway-cni --version '4.4.0' + helm test airlock-microgateway-cni -n openshift-operators --logs + helm upgrade airlock-microgateway-cni -n openshift-operators --set tests.enabled=false --reuse-values oci://quay.io/airlockcharts/microgateway-cni --version '4.4.0' ``` -### Upgrading CRDs - -The `helm install/upgrade` command currently does not support upgrading CRDs that already exist in the cluster. -CRDs should instead be manually upgraded before upgrading the Operator itself via the following command: -```bash -kubectl apply -k https://github.com/airlock/microgateway/deploy/charts/airlock-microgateway/crds/?ref=4.4.0 --server-side --force-conflicts -``` - -**Note**: Certain GitOps solutions such as e.g. Argo CD or Flux CD have their own mechanisms for automatically upgrading CRDs included with Helm charts. + Consult our [documentation](https://docs.airlock.com/microgateway/latest/#data/1699611533587.html) in case of any installation error. ## Support @@ -104,67 +89,33 @@ For the community edition, check our **[Airlock community forum](https://forum.a | Key | Type | Default | Description | |-----|------|---------|-------------| +| affinity | object | `{}` | Custom affinity for the DaemonSet to only deploy the CNI plugin on specific nodes. | | commonAnnotations | object | `{}` | Annotations to add to all resources. | | commonLabels | object | `{}` | Labels to add to all resources. | -| crds.skipVersionCheck | bool | `false` | Whether to skip the sanity check which prevents installing/upgrading the helm chart in a cluster with outdated Airlock Microgateway CRDs. The check aims to prevent unexpected behavior and issues due to Helm v3 not automatically upgrading CRDs which are already present in the cluster when performing a "helm install/upgrade". | -| dashboards.config.grafana.dashboardLabel.name | string | `"grafana_dashboard"` | Name of the label that lets Grafana identify ConfigMaps that represent dashboards. | -| dashboards.config.grafana.dashboardLabel.value | string | `"1"` | Value of the label that lets Grafana identify ConfigMaps that represent dashboards. | -| dashboards.config.grafana.folderAnnotation.name | string | `"grafana_folder"` | Name of the annotation containing the folder name to file dashboards into. | -| dashboards.config.grafana.folderAnnotation.value | string | `"Airlock Microgateway"` | Name of the folder dashboards are filed into within the Grafana UI. | -| dashboards.create | bool | `false` | Whether to create any ConfigMaps containing Grafana dashboards to import. | -| dashboards.instances.blockLogs.create | bool | `true` | Whether to create the block logs dashboard. | -| dashboards.instances.blockMetrics.create | bool | `true` | Whether to create the block metrics dashboard. | -| dashboards.instances.headerLogs.create | bool | `true` | Whether to create the header rewrite logs dashboard. | -| dashboards.instances.license.create | bool | `true` | Whether to create the license dashboard. | -| dashboards.instances.logOnlyLogs.create | bool | `true` | Whether to create the log only logs dashboard. | -| dashboards.instances.logOnlyMetrics.create | bool | `true` | Whether to create the log only metrics dashboard | -| dashboards.instances.overview.create | bool | `true` | Whether to create the overview dashboard. | -| engine.image.digest | string | `"sha256:c29adf07e7536b72447ea694d0e19fe19235306c26d412a9abc43e4dd99b84c8"` | SHA256 image digest to pull (in the format "sha256:a3051f42d3013813b05f7513bb86ed6a3209cb3003f1bb2f7b72df249aa544d3"). Overrides tag when specified. | -| engine.image.pullPolicy | string | `"IfNotPresent"` | Pull policy for this image. | -| engine.image.repository | string | `"quay.io/airlock/microgateway-engine"` | Image repository from which to pull the Airlock Microgateway Engine image. | -| engine.image.tag | string | `"4.4.0"` | Image tag to pull. | -| engine.resources | object | `{}` | Resource restrictions to apply to the Airlock Microgateway Engine container. | -| engine.sidecar.podMonitor.create | bool | `false` | Whether to create a PodMonitor resource for monitoring. | -| engine.sidecar.podMonitor.labels | object | `{}` | Labels to add to the PodMonitor. | +| config.cniBinDir | string | `"/opt/cni/bin"` | Directory where the CNI plugin binaries reside on the host. This path can either be found in the documentation of your Kubernetes distribution or CNI provider. It can also be queried by running the command `crictl info -o go-template --template '{{.config.cni.binDir}}'` on your Kubernetes node. | +| config.cniNetDir | string | `"/etc/cni/net.d"` | Directory where the CNI config files reside on the host. This path can either be found in the documentation of your Kubernetes distribution or CNI provider. It can also be queried by running the command `crictl info -o go-template --template '{{.config.cni.confDir}}'` on your Kubernetes node. | +| config.excludeNamespaces | list | `["kube-system"]` | Namespaces for which this CNI plugin should not apply any modifications. | +| config.installMode | string | `"chained"` | Whether to install the CNI plugin as a `chained` plugin (default, required with most interface CNI providers), as a `standalone` plugin (required for use with Multus CNI, e.g. on OpenShift) or in `manual` mode, where no CNI network configuration is written. | +| config.logLevel | string | `"info"` | Log level for the CNI installer and plugin. | | fullnameOverride | string | `""` | Allows overriding the name to use as full name of resources. | +| image.digest | string | `"sha256:e9d711dfe75d515ad8bc5ba5e668e7a26c063bd6a291305aac458c2cbd3945f2"` | SHA256 image digest to pull (in the format "sha256:7144f7bab3d4c2648d7e59409f15ec52a18006a128c733fcff20d3a4a54ba44a"). Overrides tag when specified. | +| image.pullPolicy | string | `"IfNotPresent"` | Pull policy for this image. | +| image.repository | string | `"quay.io/airlock/microgateway-cni"` | Image repository from which to pull the Airlock Microgateway CNI image. | +| image.tag | string | `"4.4.0"` | Image tag to pull. | | imagePullSecrets | list | `[]` | ImagePullSecrets to use when pulling images. | -| license.secretName | string | `"airlock-microgateway-license"` | Name of the secret containing the "microgateway-license.txt" key. | -| nameOverride | string | `""` | Allows overriding the name to use instead of "microgateway". | -| networkValidator.image.digest | string | `"sha256:05585644690678ae6453ab12e3a5f899e7be5ab70f56c6bf1c4484d3b53587d2"` | SHA256 image digest to pull (in the format "sha256:05585644690678ae6453ab12e3a5f899e7be5ab70f56c6bf1c4484d3b53587d2"). Overrides tag when specified. | -| networkValidator.image.pullPolicy | string | `"IfNotPresent"` | Pull policy for this image. | -| networkValidator.image.repository | string | `"cgr.dev/chainguard/netcat"` | Image repository from which to pull the netcat image for the Airlock Microgateway Network Validator init-container. | -| networkValidator.image.tag | string | `""` | Image tag to pull. | -| networkValidator.resources | object | `{"limits":{"cpu":"25m","memory":"12Mi"},"requests":{"cpu":"5m","memory":"1Mi"}}` | Resource restrictions to apply to the Airlock Microgateway Network Validator init-container. | -| operator.affinity | object | `{}` | Custom affinity to apply to the operator Deployment. Used to influence the scheduling. | -| operator.config.logLevel | string | `"info"` | Operator application log level. | -| operator.gatewayAPI.controllerName | string | `"microgateway.airlock.com/gatewayclass-controller"` | Controller name referred in the GatewayClasses managed by this operator. The value must be a path prefixed by the domain `microgateway.airlock.com`. | -| operator.gatewayAPI.enabled | bool | `false` | Whether to enable the Kubernetes Gateway API related controllers. Requires that the gateway.networking.k8s.io/v1 resources are installed on the cluster. | -| operator.image.digest | string | `"sha256:80cbae58ad9badd9395fa09a7b0576561821121b8353146bbd6efa2240ab5d97"` | SHA256 image digest to pull (in the format "sha256:c79ee3f85862fb386e9dd62b901b607161d27807f512d7fbdece05e9ee3d7c63"). Overrides tag when specified. | -| operator.image.pullPolicy | string | `"IfNotPresent"` | Pull policy for this image. | -| operator.image.repository | string | `"quay.io/airlock/microgateway-operator"` | Image repository from which to pull the Airlock Microgateway Operator image. | -| operator.image.tag | string | `"4.4.0"` | Image tag to pull. | -| operator.nodeSelector | object | `{}` | Custom nodeSelector to apply to the operator Deployment in order to constrain its Pods to certain nodes. | -| operator.podAnnotations | object | `{}` | Annotations to add to all Pods. | -| operator.podLabels | object | `{}` | Labels to add to all Pods. | -| operator.rbac.create | bool | `true` | Whether to create RBAC resources which are required for the Airlock Microgateway Operator to function. | -| operator.replicaCount | int | `2` | Number of replicas for the operator Deployment. | -| operator.resources | object | `{}` | Resource restrictions to apply to the operator container. | -| operator.serviceAccount.annotations | object | `{}` | Annotations to add to the ServiceAccount. | -| operator.serviceAccount.create | bool | `true` | Whether a ServiceAccount should be created. | -| operator.serviceAccount.name | string | `""` | Name of the ServiceAccount to use. If not set and create is true, a name is generated using the fullname template. | -| operator.serviceAnnotations | object | `{}` | Annotations to add to the Service. | -| operator.serviceLabels | object | `{}` | Labels to add to the Service. | -| operator.serviceMonitor.create | bool | `false` | Whether to create a ServiceMonitor resource for monitoring. | -| operator.serviceMonitor.labels | object | `{}` | Labels to add to the ServiceMonitor. | -| operator.tolerations | list | `[]` | Custom tolerations to apply to the operator Deployment in order to allow its Pods to run on tainted nodes. | -| operator.updateStrategy | object | `{"type":"RollingUpdate"}` | Specifies the operator update strategy. | -| operator.watchNamespaceSelector | object | `{}` | Allows to dynamically select watch namespaces of the operator and the scope of the webhooks based on a Namespace label selector. It is able to detect and reconcile resources in all namespaces that match the label selector automatically, even for new namespaces, without restarting the operator. This facilitates a dynamic `MultiNamespace` installation mode, but still requires cluster-scoped permissions (i.e., ClusterRoles and ClusterRoleBindings). An `AllNamespaces` installation or the usage of the `watchNamespaces` requires the `watchNamespaceSelector` to be empty. Please note that this feature requires a Premium license. | -| operator.watchNamespaces | list | `[]` | Allows to restrict the operator to specific namespaces, depending on your needs. For a `OwnNamespace` or `SingleNamespace` installation the list may only contain one namespace (e.g., `watchNamespaces: ["airlock-microgateway-system"]`). In case of the `OwnNamespace` installation mode the specified namespace should be equal to the installation namespace. For a static `MultiNamespace` installation, the complete list of namespaces must be provided in the `watchNamespaces`. An `AllNamespaces` installation or the usage of the `watchNamespaceSelector` requires the `watchNamespaces` to be empty. Regardless of the installation modes supported by `watchNamespaces`, RBAC is created only namespace-scoped (using Roles and RoleBindings) in the respective namespaces. Please note that this feature requires a Premium license. | -| sessionAgent.image.digest | string | `"sha256:fbb90f2a52bb1b19cca6c5c133e80331153c019ec905db052c250fedbb09c3bc"` | SHA256 image digest to pull (in the format "sha256:a3051f42d3013813b05f7513bb86ed6a3209cb3003f1bb2f7b72df249aa544d3"). Overrides tag when specified. | -| sessionAgent.image.pullPolicy | string | `"IfNotPresent"` | Pull policy for this image. | -| sessionAgent.image.repository | string | `"quay.io/airlock/microgateway-session-agent"` | Image repository from which to pull the Airlock Microgateway Session Agent image. | -| sessionAgent.image.tag | string | `"4.4.0"` | Image tag to pull. | -| sessionAgent.resources | object | `{}` | Resource restrictions to apply to the Airlock Microgateway Session Agent container. | +| multusNetworkAttachmentDefinition.create | bool | `false` | Whether a NetworkAttachmentDefinition CR should be created, which can be used for applying the CNI plugin to Pods. | +| multusNetworkAttachmentDefinition.namespace | string | `"default"` | Namespace in which the NetworkAttachmentDefinition is deployed. Note: If namespace is set to a custom value, referencing the created NetworkAttachmentDefinition from other namespaces may not work if Multus namespace isolation is enabled. https://github.com/k8snetworkplumbingwg/multus-cni/blob/v4.0.2/docs/configuration.md#namespace-isolation | +| nameOverride | string | `""` | Allows overriding the name to use instead of "microgateway-cni". | +| nodeSelector | object | `{"kubernetes.io/os":"linux"}` | NodeSelector to apply to the CNI DaemonSet in order to only deploy the CNI plugin on specific nodes. | +| podAnnotations | object | `{}` | Annotations to add to all Pods. | +| podLabels | object | `{}` | Labels to add to all Pods. | +| privileged | bool | `false` | Whether the DaemonSet should run in privileged mode. Must be enabled for environments which require it for writing files to the host (e.g. OpenShift). | +| rbac.create | bool | `true` | Whether to create RBAC resources which are required for the CNI plugin to function. | +| rbac.createSCCRole | OpenShift | `false` | Whether to create RBAC resources which allow the CNI installer to use the "privileged" security context constraint. | +| resources | object | `{"requests":{"cpu":"10m","memory":"100Mi"}}` | Resource restrictions to apply to the CNI installer container. | +| serviceAccount.annotations | object | `{}` | Annotations to add to the ServiceAccount. | +| serviceAccount.create | bool | `true` | Whether a ServiceAccount should be created. | +| serviceAccount.name | string | `""` | Name of the ServiceAccount to use. If not set and create is true, a name is generated using the fullname template. | | tests.enabled | bool | `false` | Whether additional resources required for running `helm test` should be created (e.g. Roles and ServiceAccounts). If set to false, `helm test` will not run any tests. | ## License diff --git a/charts/airlock/microgateway/4.4.0/gke-values.yaml b/charts/airlock/microgateway/4.4.0/gke-values.yaml new file mode 100644 index 0000000000..d6d5c21d14 --- /dev/null +++ b/charts/airlock/microgateway/4.4.0/gke-values.yaml @@ -0,0 +1,4 @@ +# values for deploying on GKE + +config: + cniBinDir: "/home/kubernetes/bin" diff --git a/charts/airlock/microgateway/4.4.0/openshift-values.yaml b/charts/airlock/microgateway/4.4.0/openshift-values.yaml new file mode 100644 index 0000000000..3b1d6cccde --- /dev/null +++ b/charts/airlock/microgateway/4.4.0/openshift-values.yaml @@ -0,0 +1,15 @@ +# values for deploying on OpenShift + +rbac: + createSCCRole: true + +privileged: true + +multusNetworkAttachmentDefinition: + create: true + namespace: default + +config: + installMode: "standalone" + cniNetDir: "/etc/cni/multus/net.d" + cniBinDir: "/var/lib/cni/bin" diff --git a/charts/airlock/microgateway/4.4.0/questions.yml b/charts/airlock/microgateway/4.4.0/questions.yml new file mode 100644 index 0000000000..73ed44d646 --- /dev/null +++ b/charts/airlock/microgateway/4.4.0/questions.yml @@ -0,0 +1,18 @@ +questions: + - variable: config.cniNetDir + required: true + type: string + label: CNI Network Configuration Directory + group: "CNI Settings" + description: "Directory where the CNI config files reside on the host. This value depends on the kubernetes distribution and interface CNI Provider used. It can be fetched by running `crictl info -o go-template --template '{{.config.cni.confDir}}'` on your kubernetes host." + - variable: config.cniBinDir + required: true + type: string + label: CNI Plugin Binaries Directory + group: "CNI Settings" + description: "Directory where the CNI plugin binaries reside on the host. This value depends on the kubernetes distribution and interface CNI Provider used. It can be fetched by running `crictl info -o go-template --template '{{.config.cni.binDir}}'` on your kubernetes host." + - variable: config.installMode + required: true + label: CNI Plugin Installation Mode + group: "CNI Settings" + description: "Whether to install the CNI plugin as a `chained` plugin (default, required with most interface CNI providers) as a `standalone` plugin (required for use with Multus CNI, e.g. on OpenShift) or in `manual` mode, where no CNI network configuration is written. Please refer to the CNI installation documentation (https://github.com/airlock/microgateway?tab=readme-ov-file#deploy-airlock-microgateway-cni) to correctly setup the CNI Plugin for your environment." diff --git a/charts/airlock/microgateway/4.4.0/templates/NOTES.txt b/charts/airlock/microgateway/4.4.0/templates/NOTES.txt index a607483f9c..bb94ff521e 100644 --- a/charts/airlock/microgateway/4.4.0/templates/NOTES.txt +++ b/charts/airlock/microgateway/4.4.0/templates/NOTES.txt @@ -1,61 +1,15 @@ -Thank you for installing Airlock Microgateway. -{{- if .Values.operator.gatewayAPI.enabled }} +Thank you for installing Airlock Microgateway CNI. -K8s Gateway API support enabled. -Note that the K8s Gateway API support is an incubating Airlock Microgateway feature. We encourage you to try the installation and configuration for testing and evaluation. Your feedback is welcome. - - {{- if or .Values.operator.watchNamespaces .Values.operator.watchNamespaceSelector -}} - {{- fail ` - -K8s Gateway API is only supported using the 'AllNamespaces' installation mode type, ensure that 'operator.watchNamespaces' and 'operator.watchNamespaceSelector' are not configured. -` - -}} - {{- end -}} -{{- end }} - -Please ensure the following prerequisites are fulfilled: -* cert-manager is installed. - https://cert-manager.io/docs/installation/helm/ -* A valid Airlock Microgateway license is deployed in the Kubernetes secret '{{ .Release.Namespace }}/{{ .Values.license.secretName }}' - * Get a free Community license: https://airlock.com/en/microgateway-community - * Order a Premium license: https://airlock.com/en/microgateway-premium -* Airlock Microgateway CNI is installed on the cluster, when running data plane mode sidecar - https://artifacthub.io/packages/helm/airlock-microgateway-cni/microgateway-cni. - For more information about data plane modes, see https://docs.airlock.com/microgateway/{{ include "airlock-microgateway.docsVersion" . }}/#data/1660804709650.html +Please ensure that the helm values'.config.cniNetDir' and '.config.cniBinDir' are configured for your Kubernetes distribution. +For further information, consider our manual https://docs.airlock.com/microgateway/{{ include "airlock-microgateway-cni.docsVersion" . }}. +The chapter 'Setup > Installation' describes how to set those settings correctly. Further information: -* Documentation: https://docs.airlock.com/microgateway/{{ include "airlock-microgateway.docsVersion" . }} -* CRD API reference documentation: https://docs.airlock.com/microgateway/{{ include "airlock-microgateway.docsVersion" . }}/api/crds +* Documentation: https://docs.airlock.com/microgateway/{{ include "airlock-microgateway-cni.docsVersion" . }} * Airlock Microgateway Labs: https://play.instruqt.com/airlock/invite/hyi9fy4b4jzc?icp_referrer=helm -{{- if .Values.crds.skipVersionCheck }} - -Warning: CRD version check skipped -{{- else -}} -{{- $outdatedCRDs := (include "airlock-microgateway.outdatedCRDs" .) -}} -{{- if $outdatedCRDs -}} - {{- fail (printf ` - -Helm does not automatically upgrade CRDs from the chart's 'crds/' directory during 'helm install/upgrade'. -Therefore, the CRDs must be manually upgraded with the following command before deploying this chart: - -kubectl apply -k https://github.com/airlock/microgateway/deploy/charts/airlock-microgateway/crds/?ref=%s --server-side --force-conflicts - -If you are not using the helm install/upgrade command and instead rely on some other mechanism which is able to upgrade CRDs for deploying this chart, you can suppress this error by setting the helm value 'crds.skipVersionCheck=true'.` - .Chart.AppVersion) - -}} -{{- end -}} -{{- end -}} -{{- if .Values.tests.enabled -}} - {{- if .Values.operator.watchNamespaces -}} - {{- if not (has .Release.Namespace .Values.operator.watchNamespaces) -}} - {{- fail (printf ` -To execute 'helm test', it is necessary that the release namespace '%s' is part of the operator's watch scope. Either disable the tests or ensure that the release namespace is added to watch namspace list ('operator.watchNamespaces') in the helm values. -` - .Release.Namespace) - -}} - {{- end -}} - {{- end -}} -{{- end }} +Next steps: +* Install Airlock Microgateway (if not done already) + https://artifacthub.io/packages/helm/airlock-microgateway/microgateway Your release version is {{ .Chart.Version }}. \ No newline at end of file diff --git a/charts/airlock/microgateway/4.4.0/templates/_helpers.tpl b/charts/airlock/microgateway/4.4.0/templates/_helpers.tpl index 733ba96486..996491a873 100644 --- a/charts/airlock/microgateway/4.4.0/templates/_helpers.tpl +++ b/charts/airlock/microgateway/4.4.0/templates/_helpers.tpl @@ -1,16 +1,14 @@ {{/* Expand the name of the chart. -We truncate at 49 chars because some Kubernetes name fields are limited to 63 chars (by the DNS naming spec) -and the longest explicit suffix is 14 characters. */}} -{{- define "airlock-microgateway.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 49 | trimSuffix "-" }} +{{- define "airlock-microgateway-cni.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} {{- end }} {{/* Convert an image configuration object into an image ref string. */}} -{{- define "airlock-microgateway.image" -}} +{{- define "airlock-microgateway-cni.image" -}} {{- if .digest -}} {{- printf "%s@%s" .repository .digest -}} {{- else if .tag -}} @@ -22,19 +20,19 @@ Convert an image configuration object into an image ref string. {{/* Create a default fully qualified app name. -We truncate at 36 chars because some Kubernetes name fields are limited to 63 chars (by the DNS naming spec) -and the longest implicit suffix is 27 characters. +We truncate at 50 chars because some Kubernetes name fields are limited to 63 chars (by the DNS naming spec) +and the longest suffix is 13 characters. If release name contains chart name it will be used as a full name. */}} -{{- define "airlock-microgateway.fullname" -}} +{{- define "airlock-microgateway-cni.fullname" -}} {{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 36 | trimSuffix "-" }} +{{- .Values.fullnameOverride | trunc 50 | trimSuffix "-" }} {{- else }} {{- $name := default .Chart.Name .Values.nameOverride }} {{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 36 | trimSuffix "-" }} +{{- .Release.Name | trunc 50 | trimSuffix "-" }} {{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 36 | trimSuffix "-" }} +{{- printf "%s-%s" .Release.Name $name | trunc 50 | trimSuffix "-" }} {{- end }} {{- end }} {{- end }} @@ -42,112 +40,62 @@ If release name contains chart name it will be used as a full name. {{/* Create chart name and version as used by the chart label. */}} -{{- define "airlock-microgateway.chart" -}} +{{- define "airlock-microgateway-cni.chart" -}} {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} {{- end }} {{/* Common labels */}} -{{- define "airlock-microgateway.sharedLabels" -}} -helm.sh/chart: {{ include "airlock-microgateway.chart" . }} +{{- define "airlock-microgateway-cni.labels" -}} +helm.sh/chart: {{ include "airlock-microgateway-cni.chart" . }} +{{ include "airlock-microgateway-cni.selectorLabels" . }} {{- if .Chart.AppVersion }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end }} app.kubernetes.io/managed-by: {{ .Release.Service }} -app.kubernetes.io/part-of: {{ .Chart.Name }} {{- with .Values.commonLabels }} {{ toYaml .}} {{- end }} {{- end }} {{/* -Common Selector labels +Common labels without component */}} -{{- define "airlock-microgateway.sharedSelectorLabels" -}} -app.kubernetes.io/instance: {{ .Release.Name }} +{{- define "airlock-microgateway-cni.labelsWithoutComponent" -}} +{{- $labels := fromYaml (include "airlock-microgateway-cni.labels" .) -}} +{{ unset $labels "app.kubernetes.io/component" | toYaml }} {{- end }} {{/* -Restricted Container Security Context +Selector labels */}} -{{- define "airlock-microgateway.restrictedSecurityContext" -}} -allowPrivilegeEscalation: false -privileged: false -runAsNonRoot: true -capabilities: - drop: ["ALL"] -readOnlyRootFilesystem: true -seccompProfile: - type: RuntimeDefault +{{- define "airlock-microgateway-cni.selectorLabels" -}} +app.kubernetes.io/component: cni-plugin-installer +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/name: {{ include "airlock-microgateway-cni.name" . }} {{- end }} -{{/* Precondition: May only be used if AppVersion is isSemver */}} -{{- define "airlock-microgateway.supportedCRDVersionPattern" -}} -{{- $version := (semver .Chart.AppVersion) -}} -{{- if $version.Prerelease -}} ->= {{ $version.Major }}.{{ $version.Minor }}.{{ $version.Patch }}-{{ $version.Prerelease }} -{{- else -}} ->= {{ $version.Major }}.{{ $version.Minor }}.0 || >= {{ $version.Major }}.{{ $version.Minor }}.{{ add1 $version.Patch }}-0 -{{- end -}} -{{- end -}} - -{{- define "airlock-microgateway.outdatedCRDs" -}} -{{- if (eq "true" (include "airlock-microgateway.isSemver" .Chart.AppVersion)) -}} - {{- $supportedVersion := (include "airlock-microgateway.supportedCRDVersionPattern" .) -}} - {{- range $path, $_ := .Files.Glob "crds/*.yaml" -}} - {{- $api := ($.Files.Get $path | fromYaml).metadata.name -}} - {{- $crd := (lookup "apiextensions.k8s.io/v1" "CustomResourceDefinition" "" $api) -}} - {{- $isOutdated := false -}} - {{- if $crd -}} - {{/* If CRD is already present in the cluster, it must have the minimum supported version */}} - {{- $isOutdated = true -}} - {{- if hasKey $crd.metadata "labels" -}} - {{- $crdVersion := get $crd.metadata.labels "app.kubernetes.io/version" -}} - {{- if (eq "true" (include "airlock-microgateway.isSemver" $crdVersion)) -}} - {{- if (semverCompare $supportedVersion $crdVersion) }} - {{- $isOutdated = false -}} - {{- end }} - {{- end -}} - {{- end -}} - {{- end -}} - {{- if $isOutdated }} -{{ base $path }} - {{- end }} - {{- end -}} -{{- end -}} -{{- end -}} +{{/* +Create the name of the service account to use for the CNI Plugin +*/}} +{{- define "airlock-microgateway-cni.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "airlock-microgateway-cni.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} -{{- define "airlock-microgateway.isSemver" -}} +{{- define "airlock-microgateway-cni.isSemver" -}} {{- regexMatch `^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$` . -}} {{- end -}} -{{- define "airlock-microgateway.docsVersion" -}} -{{- if and (eq "true" (include "airlock-microgateway.isSemver" .Chart.AppVersion)) (not (contains "-" .Chart.AppVersion)) -}} +{{- define "airlock-microgateway-cni.docsVersion" -}} +{{- if and (eq "true" (include "airlock-microgateway-cni.isSemver" .Chart.AppVersion)) (not (contains "-" .Chart.AppVersion)) -}} {{- $version := (semver .Chart.AppVersion) -}} {{- $version.Major }}.{{ $version.Minor -}} {{- else -}} {{- print "latest" -}} {{- end -}} {{- end -}} - -{{- define "airlock-microgateway.watchNamespaceSelector.labelQuery" -}} -{{- $list := list -}} -{{- with .matchLabels -}} - {{- range $key, $value := . -}} - {{- $list = append $list (printf "%s=%s" $key $value) -}} - {{- end -}} -{{- end -}} -{{- with .matchExpressions -}} - {{- range . -}} - {{- if has .operator (list "In" "NotIn") -}} - {{- $list = append $list (printf "%s %s (%s)" .key (lower .operator) (join "," .values)) -}} - {{- else if eq .operator "Exists" -}} - {{- $list = append $list .key -}} - {{- else if eq .operator "DoesNotExist" -}} - {{- $list = append $list (printf "!%s" .key) -}} - {{- end -}} - {{- end -}} -{{- end -}} -{{- join "," $list -}} -{{- end -}} \ No newline at end of file diff --git a/charts/airlock/microgateway/4.3.3/templates/clusterrole.yaml b/charts/airlock/microgateway/4.4.0/templates/clusterrole.yaml similarity index 100% rename from charts/airlock/microgateway/4.3.3/templates/clusterrole.yaml rename to charts/airlock/microgateway/4.4.0/templates/clusterrole.yaml diff --git a/charts/airlock/microgateway/4.4.0/templates/clusterrolebinding.yaml b/charts/airlock/microgateway/4.4.0/templates/clusterrolebinding.yaml new file mode 100644 index 0000000000..04f87cb0fa --- /dev/null +++ b/charts/airlock/microgateway/4.4.0/templates/clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "airlock-microgateway-cni.fullname" . }} + labels: + {{- include "airlock-microgateway-cni.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "airlock-microgateway-cni.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ include "airlock-microgateway-cni.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end -}} diff --git a/charts/airlock/microgateway/4.4.0/templates/configmap.yaml b/charts/airlock/microgateway/4.4.0/templates/configmap.yaml new file mode 100644 index 0000000000..b880116ef9 --- /dev/null +++ b/charts/airlock/microgateway/4.4.0/templates/configmap.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "airlock-microgateway-cni.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "airlock-microgateway-cni.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: + plugin-conf.json: |- + { + "type": "{{ include "airlock-microgateway-cni.fullname" . }}", + "debug": {{ eq .Values.config.logLevel "debug" }}, + "logFilePath": "/var/log/{{ include "airlock-microgateway-cni.fullname" . }}.log", + "kubernetes": { + "kubeconfig": "{{ .Values.config.cniNetDir }}/{{ include "airlock-microgateway-cni.fullname" . }}-kubeconfig", + "excludeNamespaces": {{ toJson .Values.config.excludeNamespaces }} + } + } diff --git a/charts/airlock/microgateway/4.3.3/templates/daemonset.yaml b/charts/airlock/microgateway/4.4.0/templates/daemonset.yaml similarity index 100% rename from charts/airlock/microgateway/4.3.3/templates/daemonset.yaml rename to charts/airlock/microgateway/4.4.0/templates/daemonset.yaml diff --git a/charts/airlock/microgateway/4.4.0/templates/network-attachment-definition.yaml b/charts/airlock/microgateway/4.4.0/templates/network-attachment-definition.yaml new file mode 100644 index 0000000000..5d657e309c --- /dev/null +++ b/charts/airlock/microgateway/4.4.0/templates/network-attachment-definition.yaml @@ -0,0 +1,13 @@ +{{- if .Values.multusNetworkAttachmentDefinition.create -}} +apiVersion: "k8s.cni.cncf.io/v1" +kind: NetworkAttachmentDefinition +metadata: + name: {{ include "airlock-microgateway-cni.fullname" . }} + namespace: {{ .Values.multusNetworkAttachmentDefinition.namespace }} + labels: + {{- include "airlock-microgateway-cni.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end -}} diff --git a/charts/airlock/microgateway/4.4.0/templates/scc-role.yaml b/charts/airlock/microgateway/4.4.0/templates/scc-role.yaml new file mode 100644 index 0000000000..8627486928 --- /dev/null +++ b/charts/airlock/microgateway/4.4.0/templates/scc-role.yaml @@ -0,0 +1,22 @@ +{{- if .Values.rbac.createSCCRole -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "airlock-microgateway-cni.fullname" . }}-privileged + namespace: {{ .Release.Namespace }} + labels: + {{- include "airlock-microgateway-cni.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +rules: +- apiGroups: + - security.openshift.io + resourceNames: + - privileged + resources: + - securitycontextconstraints + verbs: + - use +{{- end -}} \ No newline at end of file diff --git a/charts/airlock/microgateway/4.4.0/templates/scc-rolebinding.yaml b/charts/airlock/microgateway/4.4.0/templates/scc-rolebinding.yaml new file mode 100644 index 0000000000..ebd02982c0 --- /dev/null +++ b/charts/airlock/microgateway/4.4.0/templates/scc-rolebinding.yaml @@ -0,0 +1,20 @@ +{{- if .Values.rbac.createSCCRole -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "airlock-microgateway-cni.fullname" . }}-privileged + namespace: {{ .Release.Namespace }} + labels: + {{- include "airlock-microgateway-cni.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "airlock-microgateway-cni.fullname" . }}-privileged +subjects: +- kind: ServiceAccount + name: {{ include "airlock-microgateway-cni.serviceAccountName" . }} +{{- end -}} diff --git a/charts/airlock/microgateway/4.4.0/templates/serviceaccount.yaml b/charts/airlock/microgateway/4.4.0/templates/serviceaccount.yaml new file mode 100644 index 0000000000..3dc8d58eae --- /dev/null +++ b/charts/airlock/microgateway/4.4.0/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "airlock-microgateway-cni.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "airlock-microgateway-cni.labels" . | nindent 4 }} + {{- with mustMerge .Values.serviceAccount.annotations .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end -}} diff --git a/charts/airlock/microgateway/4.4.0/templates/tests/rbac.yaml b/charts/airlock/microgateway/4.4.0/templates/tests/rbac.yaml index 93bd4cd1bd..744799333f 100644 --- a/charts/airlock/microgateway/4.4.0/templates/tests/rbac.yaml +++ b/charts/airlock/microgateway/4.4.0/templates/tests/rbac.yaml @@ -2,142 +2,63 @@ apiVersion: v1 kind: ServiceAccount metadata: + name: "{{ include "airlock-microgateway-cni.fullname" . }}-tests" + namespace: {{ .Release.Namespace }} labels: + {{- include "airlock-microgateway-cni.labelsWithoutComponent" . | nindent 4 }} app.kubernetes.io/component: tests - app.kubernetes.io/name: {{ include "airlock-microgateway.name" . }}-tests - {{- include "airlock-microgateway.sharedLabels" . | nindent 4 }} - name: "{{ include "airlock-microgateway.fullname" . }}-tests" - namespace: {{ .Release.Namespace }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: + name: "{{ include "airlock-microgateway-cni.fullname" . }}-tests" + namespace: {{ .Release.Namespace }} labels: + {{- include "airlock-microgateway-cni.labelsWithoutComponent" . | nindent 4 }} app.kubernetes.io/component: tests - app.kubernetes.io/name: {{ include "airlock-microgateway.name" . }}-tests - {{- include "airlock-microgateway.sharedLabels" . | nindent 4 }} - name: "{{ include "airlock-microgateway.fullname" . }}-tests" - namespace: {{ .Release.Namespace }} roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: "{{ include "airlock-microgateway.fullname" . }}-tests" + name: "{{ include "airlock-microgateway-cni.fullname" . }}-tests" subjects: - kind: ServiceAccount - name: "{{ include "airlock-microgateway.fullname" . }}-tests" + name: "{{ include "airlock-microgateway-cni.fullname" . }}-tests" --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: + name: "{{ include "airlock-microgateway-cni.fullname" . }}-tests" + namespace: {{ .Release.Namespace }} labels: + {{- include "airlock-microgateway-cni.labelsWithoutComponent" . | nindent 4 }} app.kubernetes.io/component: tests - app.kubernetes.io/name: {{ include "airlock-microgateway.name" . }}-tests - {{- include "airlock-microgateway.sharedLabels" . | nindent 4 }} - name: "{{ include "airlock-microgateway.fullname" . }}-tests" - namespace: {{ .Release.Namespace }} rules: - apiGroups: - - microgateway.airlock.com + - "apps" resources: - - sidecargateways + - daemonsets resourceNames: - - "{{ include "airlock-microgateway.fullname" . }}-test-sidecargateway" + - {{ include "airlock-microgateway-cni.fullname" . }} verbs: - - get - - list - - watch - - delete + - get + - watch + - list - apiGroups: - - microgateway.airlock.com + - "" resources: - - sidecargateways + - pods + - pods/log verbs: - - create + - get + - list +{{- if .Values.rbac.createSCCRole }} - apiGroups: - - "" - resources: - - events - verbs: - - list -- apiGroups: - - "apps" - resources: - - deployments + - security.openshift.io resourceNames: - - "{{ include "airlock-microgateway.operator.fullname" . }}" - verbs: - - get - - list - - watch -- apiGroups: - - "apps" - resources: - - statefulsets - - statefulsets/scale - resourceNames: - - "{{ include "airlock-microgateway.fullname" . }}-test-backend" - verbs: - - get - - list - - watch - - patch -- apiGroups: - - "" - resources: - - pods - - pods/log - - pods/status - - pods/attach - resourceNames: - - "{{ include "airlock-microgateway.fullname" . }}-test-backend-0" - - "{{ include "airlock-microgateway.fullname" . }}-test-valid-request" - - "{{ include "airlock-microgateway.fullname" . }}-test-injection-request" - verbs: - - get - - list - - create - - watch - - delete -- apiGroups: - - "" + - privileged resources: - - pods + - securitycontextconstraints verbs: - - create -{{- if .Values.operator.watchNamespaceSelector }} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/component: tests - app.kubernetes.io/name: {{ include "airlock-microgateway.name" . }}-tests - {{- include "airlock-microgateway.sharedLabels" . | nindent 4 }} - name: "{{ include "airlock-microgateway.fullname" . }}-tests-{{ .Release.Namespace }}" -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: "{{ include "airlock-microgateway.fullname" . }}-tests-{{ .Release.Namespace }}" -subjects: - - kind: ServiceAccount - name: "{{ include "airlock-microgateway.fullname" . }}-tests" - namespace: {{ .Release.Namespace }} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/component: tests - app.kubernetes.io/name: {{ include "airlock-microgateway.name" . }}-tests - {{- include "airlock-microgateway.sharedLabels" . | nindent 4 }} - name: "{{ include "airlock-microgateway.fullname" . }}-tests-{{ .Release.Namespace }}" -rules: -- apiGroups: - - "" - resources: - - namespaces - verbs: - - get - - list -{{- end }} + - use +{{- end -}} {{- end -}} diff --git a/charts/airlock/microgateway/4.4.0/templates/tests/test-install.yaml b/charts/airlock/microgateway/4.4.0/templates/tests/test-install.yaml index 721ae2b82e..12d8c8de78 100644 --- a/charts/airlock/microgateway/4.4.0/templates/tests/test-install.yaml +++ b/charts/airlock/microgateway/4.4.0/templates/tests/test-install.yaml @@ -2,14 +2,11 @@ apiVersion: v1 kind: Pod metadata: - name: "{{ include "airlock-microgateway.fullname" . }}-test-install" + name: "{{ include "airlock-microgateway-cni.fullname" . }}-test-install" namespace: {{ .Release.Namespace }} labels: + {{- include "airlock-microgateway-cni.labelsWithoutComponent" . | nindent 4 }} app.kubernetes.io/component: test-install - app.kubernetes.io/name: {{ include "airlock-microgateway.name" . }}-tests - sidecar.istio.io/inject: "false" - {{- include "airlock-microgateway.sharedLabels" . | nindent 4 }} - {{- include "airlock-microgateway.sharedSelectorLabels" . | nindent 4 }} annotations: helm.sh/hook: test helm.sh/hook-delete-policy: before-hook-creation @@ -19,209 +16,88 @@ spec: - name: test image: "bitnami/kubectl:{{ .Capabilities.KubeVersion.Major }}.{{ .Capabilities.KubeVersion.Minor }}" securityContext: - {{- include "airlock-microgateway.restrictedSecurityContext" . | nindent 6 }} + allowPrivilegeEscalation: {{ .Values.privileged }} + capabilities: + drop: + - ALL + privileged: {{ .Values.privileged }} + readOnlyRootFilesystem: true + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + readOnly: true + - mountPath: /host/etc/cni/net.d + name: cni-net-dir + readOnly: true command: - sh - -c - | set -eu - clean_up() { - echo "" - echo "### Clean up test resources" - kubectl delete --ignore-not-found=true -n {{ .Release.Namespace }} sidecargateways.microgateway.airlock.com {{ include "airlock-microgateway.fullname" . }}-test-sidecargateway || true - echo "" - echo "### Scale down '{{ include "airlock-microgateway.fullname" . }}-test-backend'" - kubectl scale -n {{ .Release.Namespace }} statefulset/{{ include "airlock-microgateway.fullname" . }}-test-backend --replicas=0 --timeout=60s - sleep 3s - echo "" - } - fail() { + echo "Error: ${1}" echo "" - echo "### Error: ${1}" - echo "" - - if kubectl get -n {{ .Release.Namespace }} sidecargateway.microgateway.airlock.com/{{ include "airlock-microgateway.fullname" . }}-test-sidecargateway >/dev/null 2>&1; then - echo "" - echo 'Microgateway Sidecargateway status:' - kubectl get -n {{ .Release.Namespace }} sidecargateway.microgateway.airlock.com/{{ include "airlock-microgateway.fullname" . }}-test-sidecargateway -o jsonpath-as-json='{.status}' || true - echo "" - echo "" - fi - - if kubectl get -n {{ .Release.Namespace }} pods/{{ include "airlock-microgateway.fullname" . }}-test-backend-0 >/dev/null 2>&1; then - echo "Pod '{{ include "airlock-microgateway.fullname" . }}-test-backend-0':" - kubectl describe -n {{ .Release.Namespace }} pods/{{ include "airlock-microgateway.fullname" . }}-test-backend-0 || true - echo "" - echo "" - echo 'Logs of Nginx container:' - kubectl logs -n {{ .Release.Namespace }} pods/{{ include "airlock-microgateway.fullname" . }}-test-backend-0 -c nginx --tail 5 || true - echo "" - echo "" - # Wait for engine logs - sleep 10s - echo 'Logs of Microgateway Engine container:' - kubectl logs -n {{ .Release.Namespace }} pods/{{ include "airlock-microgateway.fullname" . }}-test-backend-0 -c airlock-microgateway-engine --tail 5 || true - fi - + echo 'CNI installer logs:' + kubectl logs -n {{ .Release.Namespace }} daemonsets/{{ include "airlock-microgateway-cni.fullname" .}} -c cni-installer exit 1 } - create_sidecargateway() { - # create SidecarGateway resource for testing purposes - kubectl delete --ignore-not-found=true -n {{ .Release.Namespace }} sidecargateways.microgateway.airlock.com {{ include "airlock-microgateway.fullname" . }}-test-sidecargateway || true - kubectl apply -f - </dev/null 2>&1; do sleep 1s; i=$((i+1)); done - kubectl logs -f -n {{ .Release.Namespace }} {{ include "airlock-microgateway.fullname" . }}-test-valid-request - kubectl delete pod --ignore-not-found=true -n {{ .Release.Namespace }} {{ include "airlock-microgateway.fullname" . }}-test-valid-request + containsMGWCNIConf() { + cat "${1}" | grep -qe '"type":.*"{{ include "airlock-microgateway-cni.fullname" . }}"' } - {{- if .Values.operator.watchNamespaceSelector }} - echo "### Verify that Namespace Selector matches Namespace '{{ .Release.Namespace }}'" - if ! kubectl get namespace -l '{{ include "airlock-microgateway.watchNamespaceSelector.labelQuery" .Values.operator.watchNamespaceSelector }}' | grep -q {{ .Release.Namespace }}; then - labels=$(kubectl get namespace {{ .Release.Namespace }} -o jsonpath={.metadata.labels} | jq | awk '{print " " $0}') - fail {{printf `"Operator namespace '%s' is not part of the operator's watch scope. To execute 'helm test', the selector configured in the helm value 'operator.watchNamespaceSelector' must match the namespace's labels:\n* Current selector:\n%s\n\n* Current labels:\n$labels\n###"` - .Release.Namespace - (replace "\"" "\\\"" (replace "\n" "\\n" (.Values.operator.watchNamespaceSelector | toPrettyJson | indent 2))) - }} - fi - echo "" - {{- end }} - - trap clean_up EXIT - echo "" - - echo "### Waiting for Microgateway Operator Deployments to be ready" - if ! kubectl rollout status -n {{ .Release.Namespace }} --timeout=90s \ - deployments/{{ include "airlock-microgateway.operator.fullname" . }}; then - fail 'Timeout occurred' - fi - echo "" - - echo "### Scale '{{ include "airlock-microgateway.fullname" . }}-test-backend' to '1' replica" - # scale to zero replicas to ensure no pods are present from previous runs - kubectl scale -n {{ .Release.Namespace }} statefulset/{{ include "airlock-microgateway.fullname" . }}-test-backend --replicas=0 --timeout=10s - kubectl scale -n {{ .Release.Namespace }} statefulset/{{ include "airlock-microgateway.fullname" . }}-test-backend --replicas=1 --timeout=10s - echo "" - - echo "### Waiting for backend pod" - i=0 - while true; do - if kubectl get -n {{ .Release.Namespace }} pods/{{ include "airlock-microgateway.fullname" . }}-test-backend-0; then - break - elif [ $i -gt 3 ]; then - fail 'Pod not ready' - fi - sleep 2s - i=$((i+1)) - done - - echo "### Checking Microgateway Engine sidecar container was injected" - if ! kubectl get -n {{ .Release.Namespace }} pods/{{ include "airlock-microgateway.fullname" . }}-test-backend-0 -o jsonpath='{.spec.containers[?(@.name=="airlock-microgateway-engine")]}' | grep -q "airlock-microgateway-engine"; then - fail 'Microgateway Engine sidecar container not injected' - fi - echo "True" - echo "" - - echo "### Checking for valid license" - i=0 - while true; do - if [ "$(kubectl get -n {{ .Release.Namespace }} pods/{{ include "airlock-microgateway.fullname" . }}-test-backend-0 -o jsonpath='{.metadata.labels.sidecar\.microgateway\.airlock\.com/licensed}')" = 'true' ]; then - break - elif [ $i -gt 30 ]; then - fail 'Microgateway license is missing or invalid' - fi - sleep 2s - i=$((i+1)) - done - echo "True" - echo "" - - echo "### Create SidecarGateway resource for testing" - if ! create_sidecargateway ; then - fail 'Creation of SidecarGateway resource failed' - fi - echo "" - - echo "### Waiting for '{{ include "airlock-microgateway.fullname" . }}-test-backend' to be ready" - if ! kubectl rollout status -n {{ .Release.Namespace }} statefulset/{{ include "airlock-microgateway.fullname" . }}-test-backend --timeout=90s; then - fail 'Timeout occurred' - fi - echo "" - - echo "### Waiting for 'engine-config-valid' condition" - if ! kubectl wait -n {{ .Release.Namespace }} pods --field-selector=metadata.name={{ include "airlock-microgateway.fullname" . }}-test-backend-0 --timeout=90s --for=condition=microgateway.airlock.com/engine-config-valid=True; then - fail 'Configuration was never accepted by the Microgateway Engine' + if ! kubectl rollout status --timeout=60s -n {{ .Release.Namespace }} daemonsets/{{ include "airlock-microgateway-cni.fullname" .}}; then + fail 'CNI DaemonSet rollout did not complete within timeout' fi - sleep 5s - echo "" - echo "" - echo "### Checking whether a valid request is successful and returns HTTP status code '200'" - out=$(curl -vsS --retry 3 --retry-connrefused --connect-timeout 10 "http://{{ include "airlock-microgateway.fullname" . }}-test-service:8080/" || true) - echo "Response:" - echo "${out}" - if ! echo "${out}" | grep -q "200 OK"; then - fail 'A valid request was not successful' + echo "Checking whether CNI binary was installed" + if ! [ -f "/host/opt/cni/bin/{{ include "airlock-microgateway-cni.fullname" . }}" ]; then + fail 'CNI binary was not installed' fi - echo "" - echo "" - echo "### Checking whether a request with an injection attack is blocked and returns HTTP status code '400'" - out=$(curl -vsS --retry 3 --retry-connrefused --connect-timeout 10 "http://{{ include "airlock-microgateway.fullname" . }}-test-service:8080/?token='%20UnION%20all%20select%20A" || true) - echo "Response:" - echo "${out}" - if ! echo "${out}" | grep -q "400 Bad Request"; then - fail 'A malicious request was not blocked' + echo "Checking whether CNI kubeconfig was installed" + if ! [ -f "/host/etc/cni/net.d/{{ include "airlock-microgateway-cni.fullname" . }}-kubeconfig" ]; then + fail 'CNI kubeconfig was not created' fi - echo "" - echo "" - echo "### Installation of '{{ include "airlock-microgateway.fullname" . }}' succeeded" - exit 0 - serviceAccountName: "{{ include "airlock-microgateway.fullname" . }}-tests" + echo "Checking whether CNI configuration was written" + case {{ .Values.config.installMode }} in + "chained") + for file in "/host/etc/cni/net.d/"*.conflist; do + if containsMGWCNIConf "${file}"; then + echo "Success" + exit 0 + fi + done + ;; + "standalone") + if containsMGWCNIConf "/host/etc/cni/net.d/{{ include "airlock-microgateway-cni.fullname" . }}.conflist"; then + echo "Success" + exit 0 + fi + ;; + "manual") + echo "- Skipping because we are in 'manual' install mode" + echo "Success" + exit 0 + ;; + esac + + fail 'Configuration for plugin "{{ include "airlock-microgateway-cni.fullname" . }}" was not found' + serviceAccountName: "{{ include "airlock-microgateway-cni.fullname" . }}-tests" + volumes: + - hostPath: + path: "{{ .Values.config.cniBinDir }}" + type: Directory + name: cni-bin-dir + - hostPath: + path: "{{ .Values.config.cniNetDir }}" + type: Directory + name: cni-net-dir {{- end -}} diff --git a/charts/airlock/microgateway/4.4.0/values.schema.json b/charts/airlock/microgateway/4.4.0/values.schema.json index 05c7d77175..e087bd7004 100644 --- a/charts/airlock/microgateway/4.4.0/values.schema.json +++ b/charts/airlock/microgateway/4.4.0/values.schema.json @@ -14,15 +14,6 @@ "commonAnnotations": { "$ref": "#/definitions/StringMap" }, - "crds": { - "type": "object", - "properties": { - "skipVersionCheck": { - "type": "boolean" - } - }, - "additionalProperties": false - }, "imagePullSecrets": { "type": "array", "items": { @@ -39,336 +30,120 @@ "additionalProperties": true } }, - "operator": { + "image": { + "$ref": "#/definitions/Image" + }, + "podAnnotations": { + "$ref": "#/definitions/StringMap" + }, + "podLabels": { + "$ref": "#/definitions/StringMap" + }, + "resources": { + "type": "object" + }, + "nodeSelector": { + "$ref": "#/definitions/StringMap" + }, + "affinity": { + "type": "object" + }, + "rbac": { "type": "object", "properties": { - "replicaCount": { - "type": "integer", - "minimum": 0 - }, - "updateStrategy": { - "$ref": "#/definitions/UpdateStrategy" - }, - "image": { - "$ref": "#/definitions/Image" - }, - "podAnnotations": { - "$ref": "#/definitions/StringMap" - }, - "podLabels": { - "$ref": "#/definitions/StringMap" - }, - "serviceAnnotations": { - "$ref": "#/definitions/StringMap" - }, - "serviceLabels": { - "$ref": "#/definitions/StringMap" - }, - "resources": { - "type": "object" - }, - "nodeSelector": { - "$ref": "#/definitions/StringMap" - }, - "tolerations": { - "type": "array", - "items": { - "type": "object" - } - }, - "affinity": { - "type": "object" - }, - "config": { - "type": "object", - "properties": { - "logLevel": { - "type": "string", - "enum": [ - "debug", - "info", - "warn", - "error" - ] - } - }, - "required": [ - "logLevel" - ], - "additionalProperties": false - }, - "serviceAccount": { - "type": "object", - "properties": { - "create": { - "type": "boolean" - }, - "annotations": { - "$ref": "#/definitions/StringMap" - }, - "name": { - "type": "string" - } - }, - "required": [ - "annotations", - "create", - "name" - ], - "additionalProperties": false - }, - "watchNamespaces": { - "type": "array", - "items": { - "type": "string" - } - }, - "watchNamespaceSelector": { - "$ref": "#/definitions/LabelSelector" - }, - "rbac": { - "type": "object", - "properties": { - "create": { - "type": "boolean" - } - }, - "required": [ - "create" - ], - "additionalProperties": false - }, - "serviceMonitor": { - "type": "object", - "properties": { - "create": { - "type": "boolean" - }, - "labels": { - "$ref": "#/definitions/StringMap" - } - }, - "required": [ - "create" - ], - "additionalProperties": false + "create": { + "type": "boolean" }, - "gatewayAPI": { - "type": "object", - "properties": { - "enabled": { - "type": "boolean" - }, - "controllerName" : { - "type": "string", - "pattern": "^microgateway\\.airlock\\.com\/[A-Za-z0-9\/\\-._~%!$&'()*+,;=:]+$" - } - }, - "required": [ - "enabled" - ], - "additionalProperties": false + "createSCCRole": { + "type": "boolean" } }, - "oneOf": [ - { - "properties": { - "watchNamespaces": { - "minItems": 1 - }, - "watchNamespaceSelector": { - "additionalProperties": false - } - } - }, - { - "properties": { - "watchNamespaces": { - "maxItems": 0 - }, - "watchNamespaceSelector": { - "$ref": "#/definitions/LabelSelector" - } - } - } - ], "required": [ - "affinity", - "config", - "image", - "updateStrategy", - "nodeSelector", - "podAnnotations", - "podLabels", - "rbac", - "replicaCount", - "resources", - "serviceAccount", - "serviceAnnotations", - "serviceLabels", - "serviceMonitor", - "tolerations" + "create", + "createSCCRole" ], "additionalProperties": false }, - "engine": { + "privileged": { + "type": "boolean" + }, + "serviceAccount": { "type": "object", "properties": { - "image": { - "$ref": "#/definitions/Image" + "create": { + "type": "boolean" }, - "resources": { - "type": "object" + "annotations": { + "$ref": "#/definitions/StringMap" }, - "sidecar": { - "type": "object", - "properties":{ - "podMonitor": { - "type": "object", - "properties": { - "create": { - "type": "boolean" - }, - "labels": { - "$ref": "#/definitions/StringMap" - } - }, - "required": [ - "create" - ], - "additionalProperties": false - } - }, - "required": [ - "podMonitor" - ], - "additionalProperties": false + "name": { + "type": "string" } }, "required": [ - "image", - "resources", - "sidecar" + "annotations", + "create", + "name" ], "additionalProperties": false }, - "networkValidator": { + "multusNetworkAttachmentDefinition": { "type": "object", "properties": { - "image": { - "$ref": "#/definitions/Image" + "create": { + "type": "boolean" }, - "resources": { - "type": "object" + "namespace": { + "type": "string" } }, "required": [ - "image", - "resources" + "create", + "namespace" ], "additionalProperties": false }, - "sessionAgent": { + "config": { "type": "object", "properties": { - "image": { - "$ref": "#/definitions/Image" + "installMode": { + "type": "string", + "enum": [ + "chained", + "standalone", + "manual" + ] }, - "resources": { - "type": "object" - } - }, - "required": [ - "image", - "resources" - ], - "additionalProperties": false - }, - "license": { - "type": "object", - "properties": { - "secretName": { + "logLevel": { + "type": "string", + "enum": [ + "debug", + "info", + "warn", + "error" + ] + }, + "cniNetDir": { "type": "string", "minLength": 1 - } - }, - "required": [ - "secretName" - ], - "additionalProperties": false - }, - "dashboards": { - "type": "object", - "properties" : { - "create": { - "type": "boolean" }, - "config": { - "type": "object", - "properties": { - "grafana": { - "type": "object", - "properties": { - "folderAnnotation": { - "$ref": "#/definitions/NameValuePair" - }, - "dashboardLabel": { - "$ref": "#/definitions/NameValuePair" - } - }, - "required": [ - "folderAnnotation", - "dashboardLabel" - ], - "additionalProperties": false - } - }, - "required": [ - "grafana" - ], - "additionalProperties": false + "cniBinDir": { + "type": "string", + "minLength": 1 }, - "instances": { - "type": "object", - "properties": { - "overview": { - "$ref": "#/definitions/DashboardInstance" - }, - "license" : { - "$ref": "#/definitions/DashboardInstance" - }, - "blockMetrics" : { - "$ref": "#/definitions/DashboardInstance" - }, - "blockLogs" : { - "$ref": "#/definitions/DashboardInstance" - }, - "headerLogs" : { - "$ref": "#/definitions/DashboardInstance" - }, - "logOnlyMetrics" : { - "$ref": "#/definitions/DashboardInstance" - }, - "logOnlyLogs" : { - "$ref": "#/definitions/DashboardInstance" - } - }, - "required": [ - "overview", - "license", - "blockMetrics", - "blockLogs", - "headerLogs", - "logOnlyMetrics", - "logOnlyLogs" - ], - "additionalProperties": false + "excludeNamespaces": { + "type": "array", + "items": { + "type": "string" + } } }, "required": [ - "create", - "config", - "instances" + "cniBinDir", + "cniNetDir", + "excludeNamespaces", + "installMode", + "logLevel" ], "additionalProperties": false }, @@ -389,18 +164,22 @@ } }, "required": [ + "affinity", "commonAnnotations", "commonLabels", - "crds", - "engine", + "config", "fullnameOverride", + "image", "imagePullSecrets", - "license", + "multusNetworkAttachmentDefinition", "nameOverride", - "operator", - "networkValidator", - "sessionAgent", - "dashboards", + "nodeSelector", + "podAnnotations", + "podLabels", + "privileged", + "rbac", + "resources", + "serviceAccount", "tests" ], "additionalProperties": false, @@ -441,132 +220,6 @@ "tag" ], "additionalProperties": false - }, - "LabelSelector": { - "type": "object", - "properties": { - "matchExpressions": { - "type": "array", - "items": { - "type": "object", - "required": [ - "key", - "operator" - ], - "properties": { - "key": { - "type": "string" - }, - "operator": { - "type": "string" - }, - "values": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "additionalProperties": false - } - }, - "matchLabels": { - "$ref": "#/definitions/StringMap" - } - }, - "additionalProperties": false - }, - "UpdateStrategy": { - "type": "object", - "oneOf" : [ - { - "properties": { - "type": { - "$ref": "#/definitions/RecreateType" - } - }, - "required": [ - "type" - ], - "additionalProperties": false - }, - { - "properties": { - "type": { - "$ref": "#/definitions/RollingUpdateType" - }, - "rollingUpdate": { - "$ref": "#/definitions/RollingUpdate" - } - }, - "required": [ - "type" - ], - "additionalProperties": false - } - ] - }, - "RecreateType": { - "type": "string", - "enum": [ - "Recreate" - ] - }, - "RollingUpdateType": { - "type": "string", - "enum": [ - "RollingUpdate" - ] - }, - "RollingUpdate": { - "type": "object", - "properties": { - "maxSurge": { - "type": ["integer", "string"], - "minimum": 0, - "pattern": "^\\d+%?$" - }, - "maxUnavailable": { - "type": ["integer", "string"], - "minimum": 0, - "pattern": "^\\d+%?$" - } - }, - "anyOf": [ - {"required": ["maxSurge"]}, - {"required": ["maxUnavailable"]} - ], - "additionalProperties": false - }, - "DashboardInstance" : { - "type" : "object", - "properties" : { - "create" : { - "type" : "boolean" - } - }, - "required" : [ - "create" - ], - "additionalProperties": false - }, - "NameValuePair" : { - "type" : "object", - "properties" : { - "name" : { - "type": "string", - "minLength": 1 - }, - "value" : { - "type" : "string", - "minLength": 1 - } - }, - "required" : [ - "name", - "value" - ], - "additionalProperties": false } } } diff --git a/charts/airlock/microgateway/4.4.0/values.yaml b/charts/airlock/microgateway/4.4.0/values.yaml index 7d35047169..1b6d8d3b67 100644 --- a/charts/airlock/microgateway/4.4.0/values.yaml +++ b/charts/airlock/microgateway/4.4.0/values.yaml @@ -1,4 +1,4 @@ -# -- Allows overriding the name to use instead of "microgateway". +# -- Allows overriding the name to use instead of "microgateway-cni". nameOverride: "" # -- Allows overriding the name to use as full name of resources. fullnameOverride: "" @@ -10,227 +10,75 @@ commonAnnotations: {} imagePullSecrets: [] # - name: myRegistryKeySecretName -crds: - # -- Whether to skip the sanity check which prevents installing/upgrading the helm chart in a cluster with outdated Airlock Microgateway CRDs. - # The check aims to prevent unexpected behavior and issues due to Helm v3 not automatically upgrading CRDs which are already present in the cluster - # when performing a "helm install/upgrade". - skipVersionCheck: false -operator: - # -- Number of replicas for the operator Deployment. - replicaCount: 2 - # -- Specifies the operator update strategy. - updateStrategy: - type: RollingUpdate - # Specifies the Airlock Microgateway Operator image. - image: - # -- Image repository from which to pull the Airlock Microgateway Operator image. - repository: "quay.io/airlock/microgateway-operator" - # -- Image tag to pull. - tag: "4.4.0" - # -- SHA256 image digest to pull (in the format "sha256:c79ee3f85862fb386e9dd62b901b607161d27807f512d7fbdece05e9ee3d7c63"). - # Overrides tag when specified. - digest: "sha256:80cbae58ad9badd9395fa09a7b0576561821121b8353146bbd6efa2240ab5d97" - # -- Pull policy for this image. - pullPolicy: IfNotPresent - # -- Annotations to add to all Pods. - podAnnotations: {} - # -- Labels to add to all Pods. - podLabels: {} - # -- Annotations to add to the Service. - serviceAnnotations: {} - # prometheus.io/scrape: "true" - # prometheus.io/port: "8080" - - # -- Labels to add to the Service. - serviceLabels: {} - # -- Resource restrictions to apply to the operator container. - resources: {} - # We recommend at least the following resource specification. - # limits: - # cpu: 1000m - # memory: 512Mi - # requests: - # cpu: 100m - # memory: 512Mi - - # -- Custom nodeSelector to apply to the operator Deployment in order to constrain its Pods to certain nodes. - nodeSelector: {} - # -- Custom tolerations to apply to the operator Deployment in order to allow its Pods to run on tainted nodes. - tolerations: [] - # -- Custom affinity to apply to the operator Deployment. Used to influence the scheduling. - affinity: {} - # Parameters for the operator configuration. - config: - # -- Operator application log level. - logLevel: "info" - # Configures the generation of the ServiceAccount. - serviceAccount: - # -- Whether a ServiceAccount should be created. - create: true - # -- Annotations to add to the ServiceAccount. - annotations: {} - # -- Name of the ServiceAccount to use. - # If not set and create is true, a name is generated using the fullname template. - name: "" - # -- Allows to restrict the operator to specific namespaces, depending on your needs. - # For a `OwnNamespace` or `SingleNamespace` installation the list may only contain one namespace (e.g., `watchNamespaces: ["airlock-microgateway-system"]`). - # In case of the `OwnNamespace` installation mode the specified namespace should be equal to the installation namespace. - # For a static `MultiNamespace` installation, the complete list of namespaces must be provided in the `watchNamespaces`. - # An `AllNamespaces` installation or the usage of the `watchNamespaceSelector` requires the `watchNamespaces` to be empty. - # Regardless of the installation modes supported by `watchNamespaces`, RBAC is created only namespace-scoped (using Roles and RoleBindings) in the respective namespaces. - # Please note that this feature requires a Premium license. - watchNamespaces: [] - # -- Allows to dynamically select watch namespaces of the operator and the scope of the webhooks based on a Namespace label selector. - # It is able to detect and reconcile resources in all namespaces that match the label selector automatically, even for new namespaces, without restarting the operator. - # This facilitates a dynamic `MultiNamespace` installation mode, but still requires cluster-scoped permissions (i.e., ClusterRoles and ClusterRoleBindings). - # An `AllNamespaces` installation or the usage of the `watchNamespaces` requires the `watchNamespaceSelector` to be empty. - # Please note that this feature requires a Premium license. - watchNamespaceSelector: {} - # For further examples, see: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#resources-that-support-set-based-requirements. - # matchLabels: - # microgateway.airlock.com/enable: "true" - # matchExpressions: - # - { key: environment, operator: NotIn, values: [dev] } - - # Configures the generation of Role and RoleBinding as well as ClusterRoles and ClusterRoleBinding pairs for the ServiceAccount specified above. - rbac: - # -- Whether to create RBAC resources which are required for the Airlock Microgateway Operator to function. - create: true - # Configures the generation of a Prometheus Operator ServiceMonitor. - serviceMonitor: - # -- Whether to create a ServiceMonitor resource for monitoring. - create: false - # -- Labels to add to the ServiceMonitor. - labels: {} - # release: "" - # Configures the Kubernetes Gateway API integration. - gatewayAPI: - # -- Whether to enable the Kubernetes Gateway API related controllers. - # Requires that the gateway.networking.k8s.io/v1 resources are installed on the cluster. - enabled: false - # -- Controller name referred in the GatewayClasses managed by this operator. The value must be a path prefixed by the domain `microgateway.airlock.com`. - controllerName: microgateway.airlock.com/gatewayclass-controller -engine: - # Specifies the Airlock Microgateway Engine image. - image: - # -- Image repository from which to pull the Airlock Microgateway Engine image. - repository: "quay.io/airlock/microgateway-engine" - # -- Image tag to pull. - tag: "4.4.0" - # -- SHA256 image digest to pull (in the format "sha256:a3051f42d3013813b05f7513bb86ed6a3209cb3003f1bb2f7b72df249aa544d3"). - # Overrides tag when specified. - digest: "sha256:c29adf07e7536b72447ea694d0e19fe19235306c26d412a9abc43e4dd99b84c8" - # -- Pull policy for this image. - pullPolicy: IfNotPresent - # -- Resource restrictions to apply to the Airlock Microgateway Engine container. - resources: {} - # We recommend at least the following resource specification. - # limits: - # cpu: 500m - # memory: 128Mi - # requests: - # cpu: 10m - # memory: 40Mi - - # Additional configuration when deployed as a sidecar. - sidecar: - # Configures the generation of a Prometheus Operator PodMonitor. - podMonitor: - # -- Whether to create a PodMonitor resource for monitoring. - create: false - # -- Labels to add to the PodMonitor. - labels: {} - # release: "" -networkValidator: - # Specifies the Airlock Microgateway Network Validator image to be injected as an init-container. - image: - # -- Image repository from which to pull the netcat image for the Airlock Microgateway Network Validator init-container. - repository: "cgr.dev/chainguard/netcat" - # -- Image tag to pull. - tag: "" - # -- SHA256 image digest to pull (in the format "sha256:05585644690678ae6453ab12e3a5f899e7be5ab70f56c6bf1c4484d3b53587d2"). - # Overrides tag when specified. - digest: "sha256:05585644690678ae6453ab12e3a5f899e7be5ab70f56c6bf1c4484d3b53587d2" - # -- Pull policy for this image. - pullPolicy: IfNotPresent - # -- Resource restrictions to apply to the Airlock Microgateway Network Validator init-container. - resources: - limits: - cpu: 25m - memory: 12Mi - requests: - cpu: 5m - memory: 1Mi -sessionAgent: - # Specifies the Airlock Microgateway Session Agent image. - image: - # -- Image repository from which to pull the Airlock Microgateway Session Agent image. - repository: "quay.io/airlock/microgateway-session-agent" - # -- Image tag to pull. - tag: "4.4.0" - # -- SHA256 image digest to pull (in the format "sha256:a3051f42d3013813b05f7513bb86ed6a3209cb3003f1bb2f7b72df249aa544d3"). - # Overrides tag when specified. - digest: "sha256:fbb90f2a52bb1b19cca6c5c133e80331153c019ec905db052c250fedbb09c3bc" - # -- Pull policy for this image. - pullPolicy: IfNotPresent - # -- Resource restrictions to apply to the Airlock Microgateway Session Agent container. - resources: {} - # We recommend at least the following resource specification. - # limits: - # cpu: 150m - # memory: 32Mi - # requests: - # cpu: 10m - # memory: 8Mi -license: - # -- Name of the secret containing the "microgateway-license.txt" key. - secretName: "airlock-microgateway-license" -# Creates dashboards in the form of ConfigMaps that can be imported -# by Grafana using its sidecar setup. -dashboards: - # -- Whether to create any ConfigMaps containing Grafana dashboards to import. +# Specifies the Airlock Microgateway CNI image. +image: + # -- Image repository from which to pull the Airlock Microgateway CNI image. + repository: "quay.io/airlock/microgateway-cni" + # -- Image tag to pull. + tag: "4.4.0" + # -- SHA256 image digest to pull (in the format "sha256:7144f7bab3d4c2648d7e59409f15ec52a18006a128c733fcff20d3a4a54ba44a"). + # Overrides tag when specified. + digest: "sha256:e9d711dfe75d515ad8bc5ba5e668e7a26c063bd6a291305aac458c2cbd3945f2" + # -- Pull policy for this image. + pullPolicy: IfNotPresent +# -- Annotations to add to all Pods. +podAnnotations: {} +# -- Labels to add to all Pods. +podLabels: {} +# -- Resource restrictions to apply to the CNI installer container. +resources: + requests: + cpu: 10m + memory: 100Mi +# -- NodeSelector to apply to the CNI DaemonSet in order to only deploy the CNI plugin on specific nodes. +nodeSelector: + kubernetes.io/os: linux +# -- Custom affinity for the DaemonSet to only deploy the CNI plugin on specific nodes. +affinity: {} +# Configures the generation of RBAC Roles and RoleBindings. +rbac: + # -- Whether to create RBAC resources which are required for the CNI plugin to function. + create: true + # -- (OpenShift) Whether to create RBAC resources which allow the CNI installer to use the "privileged" security context constraint. + createSCCRole: false +# -- Whether the DaemonSet should run in privileged mode. Must be enabled for environments which require it for writing files to the host (e.g. OpenShift). +privileged: false +# Configures the generation of the ServiceAccount. +serviceAccount: + # -- Whether a ServiceAccount should be created. + create: true + # -- Annotations to add to the ServiceAccount. + annotations: {} + # -- Name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template. + name: "" +# Configures the generation of a NetworkAttachmentDefinition for use with Multus CNI (OpenShift) +multusNetworkAttachmentDefinition: + # -- Whether a NetworkAttachmentDefinition CR should be created, which can be used for applying the CNI plugin to Pods. create: false - config: - # Configures the necessary label and annotations along with their values - # to enable Grafana to correctly identify the ConfigMaps containing - # dashboards and file them within a dedicated folder in the dashboard overview. - # These settings need to match the Grafana sidecar configuration. - grafana: - folderAnnotation: - # -- Name of the annotation containing the folder name to file dashboards into. - name: "grafana_folder" - # -- Name of the folder dashboards are filed into within the Grafana UI. - value: "Airlock Microgateway" - dashboardLabel: - # -- Name of the label that lets Grafana identify ConfigMaps that represent dashboards. - name: "grafana_dashboard" - # -- Value of the label that lets Grafana identify ConfigMaps that represent dashboards. - value: "1" - instances: - # Available dashboard instances that can be individually created/deployed. - overview: - # -- Whether to create the overview dashboard. - create: true - license: - # -- Whether to create the license dashboard. - create: true - blockMetrics: - # -- Whether to create the block metrics dashboard. - create: true - blockLogs: - # -- Whether to create the block logs dashboard. - create: true - headerLogs: - # -- Whether to create the header rewrite logs dashboard. - create: true - logOnlyMetrics: - # -- Whether to create the log only metrics dashboard - create: true - logOnlyLogs: - # -- Whether to create the log only logs dashboard. - create: true -# Check whether the installation of the Airlock Microgateway Helm Chart was successful. -# Requires a secret with a valid Airlock Microgateway license key already to be present. + # -- Namespace in which the NetworkAttachmentDefinition is deployed. + # Note: If namespace is set to a custom value, referencing the created NetworkAttachmentDefinition from other namespaces + # may not work if Multus namespace isolation is enabled. https://github.com/k8snetworkplumbingwg/multus-cni/blob/v4.0.2/docs/configuration.md#namespace-isolation + namespace: default +# Parameters for the CNI installer configuration. +config: + # -- Whether to install the CNI plugin as a `chained` plugin (default, required with most interface CNI providers), + # as a `standalone` plugin (required for use with Multus CNI, e.g. on OpenShift) + # or in `manual` mode, where no CNI network configuration is written. + installMode: "chained" + # -- Log level for the CNI installer and plugin. + logLevel: info + # -- Directory where the CNI config files reside on the host. + # This path can either be found in the documentation of your Kubernetes distribution or CNI provider. + # It can also be queried by running the command `crictl info -o go-template --template '{{.config.cni.confDir}}'` on your Kubernetes node. + cniNetDir: "/etc/cni/net.d" + # -- Directory where the CNI plugin binaries reside on the host. + # This path can either be found in the documentation of your Kubernetes distribution or CNI provider. + # It can also be queried by running the command `crictl info -o go-template --template '{{.config.cni.binDir}}'` on your Kubernetes node. + cniBinDir: "/opt/cni/bin" + # -- Namespaces for which this CNI plugin should not apply any modifications. + excludeNamespaces: + - kube-system tests: # -- Whether additional resources required for running `helm test` should be created (e.g. Roles and ServiceAccounts). # If set to false, `helm test` will not run any tests. diff --git a/charts/airlock/microgateway/4.4.3/.helmignore b/charts/airlock/microgateway/4.4.3/.helmignore new file mode 100644 index 0000000000..101ff5ac56 --- /dev/null +++ b/charts/airlock/microgateway/4.4.3/.helmignore @@ -0,0 +1,28 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ +# CRDs kustomization.yaml +/crds/kustomization.yaml +# Helm unit tests +/tests +/validation diff --git a/charts/airlock/microgateway/4.4.3/Chart.yaml b/charts/airlock/microgateway/4.4.3/Chart.yaml new file mode 100644 index 0000000000..3d0dd4d0a5 --- /dev/null +++ b/charts/airlock/microgateway/4.4.3/Chart.yaml @@ -0,0 +1,44 @@ +annotations: + artifacthub.io/category: security + artifacthub.io/license: MIT + artifacthub.io/links: | + - name: Airlock Microgateway Documentation + url: https://docs.airlock.com/microgateway/4.4/ + - name: Airlock Microgateway Labs + url: https://play.instruqt.com/airlock/invite/hyi9fy4b4jzc?icp_referrer=artifacthub.io + - name: Airlock Microgateway Forum + url: https://forum.airlock.com/ + catalog.cattle.io/certified: partner + catalog.cattle.io/display-name: Airlock Microgateway + catalog.cattle.io/kube-version: '>=1.25.0-0' + catalog.cattle.io/release-name: "" + charts.openshift.io/name: Airlock Microgateway +apiVersion: v2 +appVersion: 4.4.3 +description: A Helm chart for deploying the Airlock Microgateway +home: https://www.airlock.com/en/microgateway +icon: file://assets/icons/microgateway.svg +keywords: +- WAF +- Web Application Firewall +- WAAP +- Web Application and API protection +- OWASP +- Airlock +- Microgateway +- Security +- Filtering +- DevSecOps +- shift left +- control plane +- Operator +kubeVersion: '>=1.25.0-0' +maintainers: +- email: support@airlock.com + name: Airlock + url: https://www.airlock.com/ +name: microgateway +sources: +- https://github.com/airlock/microgateway +type: application +version: 4.4.3 diff --git a/charts/airlock/microgateway/4.4.3/README.md b/charts/airlock/microgateway/4.4.3/README.md new file mode 100644 index 0000000000..8a9a3d3b66 --- /dev/null +++ b/charts/airlock/microgateway/4.4.3/README.md @@ -0,0 +1,186 @@ +# Airlock Microgateway + +![Version: 4.4.3](https://img.shields.io/badge/Version-4.4.3-informational?style=flat-square) ![AppVersion: 4.4.3](https://img.shields.io/badge/AppVersion-4.4.3-informational?style=flat-square) + +*Airlock Microgateway is a Kubernetes native WAAP (Web Application and API Protection) solution to protect microservices.* + + + + + Microgateway + + +Modern application security is embedded in the development workflow and follows DevSecOps paradigms. Airlock Microgateway is the perfect fit for these requirements. It is a lightweight alternative to the Airlock Gateway appliance, optimized for Kubernetes environments. Airlock Microgateway protects your applications and microservices with the tried-and-tested Airlock security features against attacks, while also providing a high degree of scalability. +__This Helm chart is part of Airlock Microgateway. See our [GitHub repo](https://github.com/airlock/microgateway/tree/4.4.3).__ + +### Features +* Kubernetes native integration with sidecar injection and Gateway API support +* Reverse proxy functionality with request routing rules, TLS termination and remote IP extraction +* Using native Envoy HTTP filters like Lua scripting, RBAC, ext_authz, JWT authentication +* Content security filters for protecting against known attacks (OWASP Top 10) +* Access control using OpenID Connect to allow only authenticated users to access the protected services +* API security features like JSON parsing, OpenAPI specification enforcement or GraphQL schema validation + +For a list of all features, view the **[comparison of the community and premium edition](https://docs.airlock.com/microgateway/latest/#data/1675772882054.html)**. + +## Documentation and links + +Check the official documentation at **[docs.airlock.com](https://docs.airlock.com/microgateway/latest/)** or the product website at **[airlock.com/microgateway](https://www.airlock.com/en/microgateway)**. The links below point out the most interesting documentation sites when starting with Airlock Microgateway. + +* [Getting Started](https://docs.airlock.com/microgateway/latest/#data/1660804708742.html) +* [System Architecture](https://docs.airlock.com/microgateway/latest/#data/1660804709650.html) +* [Installation](https://docs.airlock.com/microgateway/latest/?topic=MGW-00000138) +* [Troubleshooting](https://docs.airlock.com/microgateway/latest/#data/1659430054787.html) +* [GitHub](https://github.com/airlock/microgateway) + +# Quick start guide + +The instructions below provide a quick start guide. Detailed information are provided in the **[manual](https://docs.airlock.com/microgateway/latest/)**. + +## Prerequisites +* (Recommended) [Airlock Microgateway CNI](https://artifacthub.io/packages/helm/airlock-microgateway-cni/microgateway-cni) (Required for [data plane mode sidecar](https://docs.airlock.com/microgateway/latest/?topic=MGW-00000137)) +* [Airlock Microgateway License](#obtain-airlock-microgateway-license) +* [cert-manager](https://cert-manager.io/) +* [helm](https://helm.sh/docs/intro/install/) (>= v3.8.0) + +In order to use Airlock Microgateway you need a license and the cert-manager. You may either request a community license free of charge or purchase a premium license. +For an easy start in non-production environments, you may deploy the same cert-manager we are using internally for testing. +### Obtain Airlock Microgateway License +1. Either request a community or premium license + * Community license: [airlock.com/microgateway-community](https://airlock.com/en/microgateway-community) + * Premium license: [airlock.com/microgateway-premium](https://airlock.com/en/microgateway-premium) +2. Check your inbox and save the license file microgateway-license.txt locally. + +> See [Community vs. Premium editions in detail](https://docs.airlock.com/microgateway/latest/#data/1675772882054.html) to choose the right license type. +### Deploy cert-manager +```bash +helm repo add jetstack https://charts.jetstack.io +helm install cert-manager jetstack/cert-manager --version 'v1.16.1' -n cert-manager --create-namespace --set crds.enabled=true --wait +``` + +## Deploy Airlock Microgateway Operator + +> This guide assumes a microgateway-license.txt file is present in the working directory. + +1. Install CRDs and Operator. + ```bash + # Create namespace + kubectl create namespace airlock-microgateway-system + + # Install License + kubectl -n airlock-microgateway-system create secret generic airlock-microgateway-license --from-file=microgateway-license.txt + + # Install Operator (CRDs are included via the standard Helm 3 mechanism, i.e. Helm will handle initial installation but not upgrades) + helm install airlock-microgateway -n airlock-microgateway-system oci://quay.io/airlockcharts/microgateway --version '4.4.3' --wait + ``` + +2. (Recommended) You can verify the correctness of the installation with `helm test`. + ```bash + helm upgrade airlock-microgateway -n airlock-microgateway-system --set tests.enabled=true --reuse-values oci://quay.io/airlockcharts/microgateway --version '4.4.3' + helm test airlock-microgateway -n airlock-microgateway-system --logs + helm upgrade airlock-microgateway -n airlock-microgateway-system --set tests.enabled=false --reuse-values oci://quay.io/airlockcharts/microgateway --version '4.4.3' + ``` + +### Upgrading CRDs + +The `helm install/upgrade` command currently does not support upgrading CRDs that already exist in the cluster. +CRDs should instead be manually upgraded before upgrading the Operator itself via the following command: +```bash +kubectl apply -k https://github.com/airlock/microgateway/deploy/charts/airlock-microgateway/crds/?ref=4.4.3 --server-side --force-conflicts +``` + +**Note**: Certain GitOps solutions such as e.g. Argo CD or Flux CD have their own mechanisms for automatically upgrading CRDs included with Helm charts. + +## Support + +### Premium support +If you have a paid license, please follow the [premium support process](https://techzone.ergon.ch/support-process). + +### Community support +For the community edition, check our **[Airlock community forum](https://forum.airlock.com/)** for FAQs or register to post your question. +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| commonAnnotations | object | `{}` | Annotations to add to all resources. | +| commonLabels | object | `{}` | Labels to add to all resources. | +| crds.skipVersionCheck | bool | `false` | Whether to skip the sanity check which prevents installing/upgrading the helm chart in a cluster with outdated Airlock Microgateway CRDs. The check aims to prevent unexpected behavior and issues due to Helm v3 not automatically upgrading CRDs which are already present in the cluster when performing a "helm install/upgrade". | +| dashboards.config.grafana.dashboardLabel.name | string | `"grafana_dashboard"` | Name of the label that lets Grafana identify ConfigMaps that represent dashboards. | +| dashboards.config.grafana.dashboardLabel.value | string | `"1"` | Value of the label that lets Grafana identify ConfigMaps that represent dashboards. | +| dashboards.config.grafana.folderAnnotation.name | string | `"grafana_folder"` | Name of the annotation containing the folder name to file dashboards into. | +| dashboards.config.grafana.folderAnnotation.value | string | `"Airlock Microgateway"` | Name of the folder dashboards are filed into within the Grafana UI. | +| dashboards.create | bool | `false` | Whether to create any ConfigMaps containing Grafana dashboards to import. | +| dashboards.instances.blockLogs.create | bool | `true` | Whether to create the block logs dashboard. | +| dashboards.instances.blockMetrics.create | bool | `true` | Whether to create the block metrics dashboard. | +| dashboards.instances.headerLogs.create | bool | `true` | Whether to create the header rewrite logs dashboard. | +| dashboards.instances.license.create | bool | `true` | Whether to create the license dashboard. | +| dashboards.instances.logOnlyLogs.create | bool | `true` | Whether to create the log only logs dashboard. | +| dashboards.instances.logOnlyMetrics.create | bool | `true` | Whether to create the log only metrics dashboard | +| dashboards.instances.overview.create | bool | `true` | Whether to create the overview dashboard. | +| engine.image.digest | string | `"sha256:bf2d56de0f0acc1563db6c80f17e71ec5aee489e4287f328fa7529f553a07cc1"` | SHA256 image digest to pull (in the format "sha256:a3051f42d3013813b05f7513bb86ed6a3209cb3003f1bb2f7b72df249aa544d3"). Overrides tag when specified. | +| engine.image.pullPolicy | string | `"IfNotPresent"` | Pull policy for this image. | +| engine.image.repository | string | `"quay.io/airlock/microgateway-engine"` | Image repository from which to pull the Airlock Microgateway Engine image. | +| engine.image.tag | string | `"4.4.3"` | Image tag to pull. | +| engine.resources | object | `{}` | Resource restrictions to apply to the Airlock Microgateway Engine container. | +| engine.sidecar.podMonitor.create | bool | `false` | Whether to create a PodMonitor resource for monitoring. | +| engine.sidecar.podMonitor.labels | object | `{}` | Labels to add to the PodMonitor. | +| fullnameOverride | string | `""` | Allows overriding the name to use as full name of resources. | +| imagePullSecrets | list | `[]` | ImagePullSecrets to use when pulling images. | +| license.secretName | string | `"airlock-microgateway-license"` | Name of the secret containing the "microgateway-license.txt" key. | +| nameOverride | string | `""` | Allows overriding the name to use instead of "microgateway". | +| networkValidator.image.digest | string | `"sha256:7ef657ce316ce9d86f90c1dc99702d1190877c6ac2e923e696dc82c30050a14c"` | SHA256 image digest to pull (in the format "sha256:7ef657ce316ce9d86f90c1dc99702d1190877c6ac2e923e696dc82c30050a14c"). Overrides tag when specified. | +| networkValidator.image.pullPolicy | string | `"IfNotPresent"` | Pull policy for this image. | +| networkValidator.image.repository | string | `"cgr.dev/chainguard/netcat"` | Image repository from which to pull the netcat image for the Airlock Microgateway Network Validator init-container. | +| networkValidator.image.tag | string | `""` | Image tag to pull. | +| networkValidator.resources | object | `{"limits":{"cpu":"25m","memory":"12Mi"},"requests":{"cpu":"5m","memory":"1Mi"}}` | Resource restrictions to apply to the Airlock Microgateway Network Validator init-container. | +| operator.affinity | object | `{}` | Custom affinity to apply to the operator Deployment. Used to influence the scheduling. | +| operator.config.logLevel | string | `"info"` | Operator application log level. | +| operator.gatewayAPI.controllerName | string | `"microgateway.airlock.com/gatewayclass-controller"` | Controller name referred in the GatewayClasses managed by this operator. The value must be a path prefixed by the domain `microgateway.airlock.com`. | +| operator.gatewayAPI.enabled | bool | `false` | Whether to enable the Kubernetes Gateway API related controllers. Requires that the gateway.networking.k8s.io/v1 resources are installed on the cluster. | +| operator.image.digest | string | `"sha256:0cf190def105f5721bcf234c216861b5150c74bbac480497721012ea9574cc7c"` | SHA256 image digest to pull (in the format "sha256:c79ee3f85862fb386e9dd62b901b607161d27807f512d7fbdece05e9ee3d7c63"). Overrides tag when specified. | +| operator.image.pullPolicy | string | `"IfNotPresent"` | Pull policy for this image. | +| operator.image.repository | string | `"quay.io/airlock/microgateway-operator"` | Image repository from which to pull the Airlock Microgateway Operator image. | +| operator.image.tag | string | `"4.4.3"` | Image tag to pull. | +| operator.nodeSelector | object | `{}` | Custom nodeSelector to apply to the operator Deployment in order to constrain its Pods to certain nodes. | +| operator.podAnnotations | object | `{}` | Annotations to add to all Pods. | +| operator.podLabels | object | `{}` | Labels to add to all Pods. | +| operator.rbac.create | bool | `true` | Whether to create RBAC resources which are required for the Airlock Microgateway Operator to function. | +| operator.replicaCount | int | `2` | Number of replicas for the operator Deployment. | +| operator.resources | object | `{}` | Resource restrictions to apply to the operator container. | +| operator.serviceAccount.annotations | object | `{}` | Annotations to add to the ServiceAccount. | +| operator.serviceAccount.create | bool | `true` | Whether a ServiceAccount should be created. | +| operator.serviceAccount.name | string | `""` | Name of the ServiceAccount to use. If not set and create is true, a name is generated using the fullname template. | +| operator.serviceAnnotations | object | `{}` | Annotations to add to the Service. | +| operator.serviceLabels | object | `{}` | Labels to add to the Service. | +| operator.serviceMonitor.create | bool | `false` | Whether to create a ServiceMonitor resource for monitoring. | +| operator.serviceMonitor.labels | object | `{}` | Labels to add to the ServiceMonitor. | +| operator.tolerations | list | `[]` | Custom tolerations to apply to the operator Deployment in order to allow its Pods to run on tainted nodes. | +| operator.updateStrategy | object | `{"type":"RollingUpdate"}` | Specifies the operator update strategy. | +| operator.watchNamespaceSelector | object | `{}` | Allows to dynamically select watch namespaces of the operator and the scope of the webhooks based on a Namespace label selector. It is able to detect and reconcile resources in all namespaces that match the label selector automatically, even for new namespaces, without restarting the operator. This facilitates a dynamic `MultiNamespace` installation mode, but still requires cluster-scoped permissions (i.e., ClusterRoles and ClusterRoleBindings). An `AllNamespaces` installation or the usage of the `watchNamespaces` requires the `watchNamespaceSelector` to be empty. Please note that this feature requires a Premium license. | +| operator.watchNamespaces | list | `[]` | Allows to restrict the operator to specific namespaces, depending on your needs. For a `OwnNamespace` or `SingleNamespace` installation the list may only contain one namespace (e.g., `watchNamespaces: ["airlock-microgateway-system"]`). In case of the `OwnNamespace` installation mode the specified namespace should be equal to the installation namespace. For a static `MultiNamespace` installation, the complete list of namespaces must be provided in the `watchNamespaces`. An `AllNamespaces` installation or the usage of the `watchNamespaceSelector` requires the `watchNamespaces` to be empty. Regardless of the installation modes supported by `watchNamespaces`, RBAC is created only namespace-scoped (using Roles and RoleBindings) in the respective namespaces. Please note that this feature requires a Premium license. | +| sessionAgent.image.digest | string | `"sha256:d1be15a6b516b1bd1c105d01f915e5e5ba5dc6c8c5870226233f542d3e6ac5f8"` | SHA256 image digest to pull (in the format "sha256:a3051f42d3013813b05f7513bb86ed6a3209cb3003f1bb2f7b72df249aa544d3"). Overrides tag when specified. | +| sessionAgent.image.pullPolicy | string | `"IfNotPresent"` | Pull policy for this image. | +| sessionAgent.image.repository | string | `"quay.io/airlock/microgateway-session-agent"` | Image repository from which to pull the Airlock Microgateway Session Agent image. | +| sessionAgent.image.tag | string | `"4.4.3"` | Image tag to pull. | +| sessionAgent.resources | object | `{}` | Resource restrictions to apply to the Airlock Microgateway Session Agent container. | +| tests.enabled | bool | `false` | Whether additional resources required for running `helm test` should be created (e.g. Roles and ServiceAccounts). If set to false, `helm test` will not run any tests. | + +## License +View the [detailed license terms](https://www.airlock.com/en/airlock-license) for the software contained in this image. +* Decompiling or reverse engineering is not permitted. +* Using any of the deny rules or parts of these filter patterns outside of the image is not permitted. + +Airlock® is a security innovation by [ergon](https://www.ergon.ch/en) + + + + + + + Airlock Secure Access Hub + + diff --git a/charts/airlock/microgateway/4.4.3/app-readme.md b/charts/airlock/microgateway/4.4.3/app-readme.md new file mode 100644 index 0000000000..e32cac0259 --- /dev/null +++ b/charts/airlock/microgateway/4.4.3/app-readme.md @@ -0,0 +1,28 @@ +# Airlock Microgateway + +*Airlock Microgateway is a Kubernetes native WAAP (Web Application and API Protection) solution to protect microservices.* + +## Features +* Kubernetes native integration with its Operator, Custom Resource Definitions, hot-reload, automatic sidecar injection. +* Reverse proxy functionality with request routing rules, TLS termination and remote IP extraction +* Using native Envoy HTTP filters like Lua scripting, RBAC, ext_authz, JWT authentication +* Content security filters for protecting against known attacks (OWASP Top 10) +* Access control to allow only authenticated users to access the protected services +* API security features like JSON parsing or OpenAPI specification enforcement + +For a list of all features, view the **[comparison of the community and premium edition](https://docs.airlock.com/microgateway/latest/#data/1675772882054.html)**. + +## Requirements +* [Airlock Microgateway CNI Helm Chart](https://artifacthub.io/packages/helm/airlock-microgateway-cni/microgateway-cni) (Also available as Rancher Chart) +* [Airlock Microgateway License](https://github.com/airlock/microgateway?tab=readme-ov-file#obtain-airlock-microgateway-license) (After obtaining the license install it according to the [documentation](https://github.com/airlock/microgateway?tab=readme-ov-file#deploy-airlock-microgateway-operator)) +* [cert-manager](https://cert-manager.io/docs/installation/) + +## Documentation and links + +Check the official documentation at **[docs.airlock.com](https://docs.airlock.com/microgateway/latest/)** or the product website at **[airlock.com/microgateway](https://www.airlock.com/en/microgateway)**. The links below point out the most interesting documentation sites when starting with Airlock Microgateway. + +* [Getting Started](https://docs.airlock.com/microgateway/latest/#data/1660804708742.html) +* [System Architecture](https://docs.airlock.com/microgateway/latest/#data/1660804709650.html) +* [Installation](https://docs.airlock.com/microgateway/latest/#data/1660804708637.html) +* [Troubleshooting](https://docs.airlock.com/microgateway/latest/#data/1659430054787.html) +* [GitHub](https://github.com/airlock/microgateway) \ No newline at end of file diff --git a/charts/airlock/microgateway/4.4.0/crds/accesscontrols.microgateway.airlock.com.yaml b/charts/airlock/microgateway/4.4.3/crds/accesscontrols.microgateway.airlock.com.yaml similarity index 99% rename from charts/airlock/microgateway/4.4.0/crds/accesscontrols.microgateway.airlock.com.yaml rename to charts/airlock/microgateway/4.4.3/crds/accesscontrols.microgateway.airlock.com.yaml index 9e6c7047d2..5e33e1e9b7 100644 --- a/charts/airlock/microgateway/4.4.0/crds/accesscontrols.microgateway.airlock.com.yaml +++ b/charts/airlock/microgateway/4.4.3/crds/accesscontrols.microgateway.airlock.com.yaml @@ -5,7 +5,7 @@ metadata: controller-gen.kubebuilder.io/version: v0.16.4 labels: app.kubernetes.io/name: airlock-microgateway-operator - app.kubernetes.io/version: 4.4.0 + app.kubernetes.io/version: 4.4.3 name: accesscontrols.microgateway.airlock.com spec: group: microgateway.airlock.com diff --git a/charts/airlock/microgateway/4.4.0/crds/contentsecurities.microgateway.airlock.com.yaml b/charts/airlock/microgateway/4.4.3/crds/contentsecurities.microgateway.airlock.com.yaml similarity index 99% rename from charts/airlock/microgateway/4.4.0/crds/contentsecurities.microgateway.airlock.com.yaml rename to charts/airlock/microgateway/4.4.3/crds/contentsecurities.microgateway.airlock.com.yaml index 7fa610819c..3c6ffa2339 100644 --- a/charts/airlock/microgateway/4.4.0/crds/contentsecurities.microgateway.airlock.com.yaml +++ b/charts/airlock/microgateway/4.4.3/crds/contentsecurities.microgateway.airlock.com.yaml @@ -5,7 +5,7 @@ metadata: controller-gen.kubebuilder.io/version: v0.16.4 labels: app.kubernetes.io/name: airlock-microgateway-operator - app.kubernetes.io/version: 4.4.0 + app.kubernetes.io/version: 4.4.3 name: contentsecurities.microgateway.airlock.com spec: group: microgateway.airlock.com diff --git a/charts/airlock/microgateway/4.4.0/crds/contentsecuritypolicies.microgateway.airlock.com.yaml b/charts/airlock/microgateway/4.4.3/crds/contentsecuritypolicies.microgateway.airlock.com.yaml similarity index 99% rename from charts/airlock/microgateway/4.4.0/crds/contentsecuritypolicies.microgateway.airlock.com.yaml rename to charts/airlock/microgateway/4.4.3/crds/contentsecuritypolicies.microgateway.airlock.com.yaml index be0e8cfb87..27eb780b3c 100644 --- a/charts/airlock/microgateway/4.4.0/crds/contentsecuritypolicies.microgateway.airlock.com.yaml +++ b/charts/airlock/microgateway/4.4.3/crds/contentsecuritypolicies.microgateway.airlock.com.yaml @@ -5,7 +5,7 @@ metadata: controller-gen.kubebuilder.io/version: v0.16.4 labels: app.kubernetes.io/name: airlock-microgateway-operator - app.kubernetes.io/version: 4.4.0 + app.kubernetes.io/version: 4.4.3 gateway.networking.k8s.io/policy: direct name: contentsecuritypolicies.microgateway.airlock.com spec: diff --git a/charts/airlock/microgateway/4.4.0/crds/denyrules.microgateway.airlock.com.yaml b/charts/airlock/microgateway/4.4.3/crds/denyrules.microgateway.airlock.com.yaml similarity index 99% rename from charts/airlock/microgateway/4.4.0/crds/denyrules.microgateway.airlock.com.yaml rename to charts/airlock/microgateway/4.4.3/crds/denyrules.microgateway.airlock.com.yaml index ac120c80a7..85fb3d243d 100644 --- a/charts/airlock/microgateway/4.4.0/crds/denyrules.microgateway.airlock.com.yaml +++ b/charts/airlock/microgateway/4.4.3/crds/denyrules.microgateway.airlock.com.yaml @@ -5,7 +5,7 @@ metadata: controller-gen.kubebuilder.io/version: v0.16.4 labels: app.kubernetes.io/name: airlock-microgateway-operator - app.kubernetes.io/version: 4.4.0 + app.kubernetes.io/version: 4.4.3 name: denyrules.microgateway.airlock.com spec: group: microgateway.airlock.com diff --git a/charts/airlock/microgateway/4.4.0/crds/envoyclusters.microgateway.airlock.com.yaml b/charts/airlock/microgateway/4.4.3/crds/envoyclusters.microgateway.airlock.com.yaml similarity index 98% rename from charts/airlock/microgateway/4.4.0/crds/envoyclusters.microgateway.airlock.com.yaml rename to charts/airlock/microgateway/4.4.3/crds/envoyclusters.microgateway.airlock.com.yaml index 5d82a25476..95dd90fad4 100644 --- a/charts/airlock/microgateway/4.4.0/crds/envoyclusters.microgateway.airlock.com.yaml +++ b/charts/airlock/microgateway/4.4.3/crds/envoyclusters.microgateway.airlock.com.yaml @@ -5,7 +5,7 @@ metadata: controller-gen.kubebuilder.io/version: v0.16.4 labels: app.kubernetes.io/name: airlock-microgateway-operator - app.kubernetes.io/version: 4.4.0 + app.kubernetes.io/version: 4.4.3 name: envoyclusters.microgateway.airlock.com spec: group: microgateway.airlock.com diff --git a/charts/airlock/microgateway/4.4.0/crds/envoyconfigurations.microgateway.airlock.com.yaml b/charts/airlock/microgateway/4.4.3/crds/envoyconfigurations.microgateway.airlock.com.yaml similarity index 99% rename from charts/airlock/microgateway/4.4.0/crds/envoyconfigurations.microgateway.airlock.com.yaml rename to charts/airlock/microgateway/4.4.3/crds/envoyconfigurations.microgateway.airlock.com.yaml index 97c3a03845..2cd9294827 100644 --- a/charts/airlock/microgateway/4.4.0/crds/envoyconfigurations.microgateway.airlock.com.yaml +++ b/charts/airlock/microgateway/4.4.3/crds/envoyconfigurations.microgateway.airlock.com.yaml @@ -5,7 +5,7 @@ metadata: controller-gen.kubebuilder.io/version: v0.16.4 labels: app.kubernetes.io/name: airlock-microgateway-operator - app.kubernetes.io/version: 4.4.0 + app.kubernetes.io/version: 4.4.3 name: envoyconfigurations.microgateway.airlock.com spec: group: microgateway.airlock.com diff --git a/charts/airlock/microgateway/4.4.0/crds/envoyhttpfilters.microgateway.airlock.com.yaml b/charts/airlock/microgateway/4.4.3/crds/envoyhttpfilters.microgateway.airlock.com.yaml similarity index 98% rename from charts/airlock/microgateway/4.4.0/crds/envoyhttpfilters.microgateway.airlock.com.yaml rename to charts/airlock/microgateway/4.4.3/crds/envoyhttpfilters.microgateway.airlock.com.yaml index 7df49ba42b..f296895d8f 100644 --- a/charts/airlock/microgateway/4.4.0/crds/envoyhttpfilters.microgateway.airlock.com.yaml +++ b/charts/airlock/microgateway/4.4.3/crds/envoyhttpfilters.microgateway.airlock.com.yaml @@ -5,7 +5,7 @@ metadata: controller-gen.kubebuilder.io/version: v0.16.4 labels: app.kubernetes.io/name: airlock-microgateway-operator - app.kubernetes.io/version: 4.4.0 + app.kubernetes.io/version: 4.4.3 name: envoyhttpfilters.microgateway.airlock.com spec: group: microgateway.airlock.com diff --git a/charts/airlock/microgateway/4.4.0/crds/graphqls.microgateway.airlock.com.yaml b/charts/airlock/microgateway/4.4.3/crds/graphqls.microgateway.airlock.com.yaml similarity index 98% rename from charts/airlock/microgateway/4.4.0/crds/graphqls.microgateway.airlock.com.yaml rename to charts/airlock/microgateway/4.4.3/crds/graphqls.microgateway.airlock.com.yaml index 215cbb6427..60c1efc8a4 100644 --- a/charts/airlock/microgateway/4.4.0/crds/graphqls.microgateway.airlock.com.yaml +++ b/charts/airlock/microgateway/4.4.3/crds/graphqls.microgateway.airlock.com.yaml @@ -5,7 +5,7 @@ metadata: controller-gen.kubebuilder.io/version: v0.16.4 labels: app.kubernetes.io/name: airlock-microgateway-operator - app.kubernetes.io/version: 4.4.0 + app.kubernetes.io/version: 4.4.3 name: graphqls.microgateway.airlock.com spec: group: microgateway.airlock.com diff --git a/charts/airlock/microgateway/4.4.0/crds/headerrewrites.microgateway.airlock.com.yaml b/charts/airlock/microgateway/4.4.3/crds/headerrewrites.microgateway.airlock.com.yaml similarity index 99% rename from charts/airlock/microgateway/4.4.0/crds/headerrewrites.microgateway.airlock.com.yaml rename to charts/airlock/microgateway/4.4.3/crds/headerrewrites.microgateway.airlock.com.yaml index c7c4f7f63d..1473bba581 100644 --- a/charts/airlock/microgateway/4.4.0/crds/headerrewrites.microgateway.airlock.com.yaml +++ b/charts/airlock/microgateway/4.4.3/crds/headerrewrites.microgateway.airlock.com.yaml @@ -5,7 +5,7 @@ metadata: controller-gen.kubebuilder.io/version: v0.16.4 labels: app.kubernetes.io/name: airlock-microgateway-operator - app.kubernetes.io/version: 4.4.0 + app.kubernetes.io/version: 4.4.3 name: headerrewrites.microgateway.airlock.com spec: group: microgateway.airlock.com diff --git a/charts/airlock/microgateway/4.4.0/crds/identitypropagations.microgateway.airlock.com.yaml b/charts/airlock/microgateway/4.4.3/crds/identitypropagations.microgateway.airlock.com.yaml similarity index 99% rename from charts/airlock/microgateway/4.4.0/crds/identitypropagations.microgateway.airlock.com.yaml rename to charts/airlock/microgateway/4.4.3/crds/identitypropagations.microgateway.airlock.com.yaml index 4a5df8ed02..6c97ece439 100644 --- a/charts/airlock/microgateway/4.4.0/crds/identitypropagations.microgateway.airlock.com.yaml +++ b/charts/airlock/microgateway/4.4.3/crds/identitypropagations.microgateway.airlock.com.yaml @@ -5,7 +5,7 @@ metadata: controller-gen.kubebuilder.io/version: v0.16.4 labels: app.kubernetes.io/name: airlock-microgateway-operator - app.kubernetes.io/version: 4.4.0 + app.kubernetes.io/version: 4.4.3 name: identitypropagations.microgateway.airlock.com spec: group: microgateway.airlock.com diff --git a/charts/airlock/microgateway/4.4.0/crds/jwks.microgateway.airlock.com.yaml b/charts/airlock/microgateway/4.4.3/crds/jwks.microgateway.airlock.com.yaml similarity index 99% rename from charts/airlock/microgateway/4.4.0/crds/jwks.microgateway.airlock.com.yaml rename to charts/airlock/microgateway/4.4.3/crds/jwks.microgateway.airlock.com.yaml index 4ea6213819..995f2daadd 100644 --- a/charts/airlock/microgateway/4.4.0/crds/jwks.microgateway.airlock.com.yaml +++ b/charts/airlock/microgateway/4.4.3/crds/jwks.microgateway.airlock.com.yaml @@ -5,7 +5,7 @@ metadata: controller-gen.kubebuilder.io/version: v0.16.4 labels: app.kubernetes.io/name: airlock-microgateway-operator - app.kubernetes.io/version: 4.4.0 + app.kubernetes.io/version: 4.4.3 name: jwks.microgateway.airlock.com spec: group: microgateway.airlock.com diff --git a/charts/airlock/microgateway/4.4.0/crds/limits.microgateway.airlock.com.yaml b/charts/airlock/microgateway/4.4.3/crds/limits.microgateway.airlock.com.yaml similarity index 99% rename from charts/airlock/microgateway/4.4.0/crds/limits.microgateway.airlock.com.yaml rename to charts/airlock/microgateway/4.4.3/crds/limits.microgateway.airlock.com.yaml index 3f3cae2eef..4d9aea33ae 100644 --- a/charts/airlock/microgateway/4.4.0/crds/limits.microgateway.airlock.com.yaml +++ b/charts/airlock/microgateway/4.4.3/crds/limits.microgateway.airlock.com.yaml @@ -5,7 +5,7 @@ metadata: controller-gen.kubebuilder.io/version: v0.16.4 labels: app.kubernetes.io/name: airlock-microgateway-operator - app.kubernetes.io/version: 4.4.0 + app.kubernetes.io/version: 4.4.3 name: limits.microgateway.airlock.com spec: group: microgateway.airlock.com diff --git a/charts/airlock/microgateway/4.4.0/crds/oidcproviders.microgateway.airlock.com.yaml b/charts/airlock/microgateway/4.4.3/crds/oidcproviders.microgateway.airlock.com.yaml similarity index 99% rename from charts/airlock/microgateway/4.4.0/crds/oidcproviders.microgateway.airlock.com.yaml rename to charts/airlock/microgateway/4.4.3/crds/oidcproviders.microgateway.airlock.com.yaml index a4b737db88..125db6c25f 100644 --- a/charts/airlock/microgateway/4.4.0/crds/oidcproviders.microgateway.airlock.com.yaml +++ b/charts/airlock/microgateway/4.4.3/crds/oidcproviders.microgateway.airlock.com.yaml @@ -5,7 +5,7 @@ metadata: controller-gen.kubebuilder.io/version: v0.16.4 labels: app.kubernetes.io/name: airlock-microgateway-operator - app.kubernetes.io/version: 4.4.0 + app.kubernetes.io/version: 4.4.3 name: oidcproviders.microgateway.airlock.com spec: group: microgateway.airlock.com diff --git a/charts/airlock/microgateway/4.4.0/crds/oidcrelyingparties.microgateway.airlock.com.yaml b/charts/airlock/microgateway/4.4.3/crds/oidcrelyingparties.microgateway.airlock.com.yaml similarity index 99% rename from charts/airlock/microgateway/4.4.0/crds/oidcrelyingparties.microgateway.airlock.com.yaml rename to charts/airlock/microgateway/4.4.3/crds/oidcrelyingparties.microgateway.airlock.com.yaml index 3c071bc668..57169f5366 100644 --- a/charts/airlock/microgateway/4.4.0/crds/oidcrelyingparties.microgateway.airlock.com.yaml +++ b/charts/airlock/microgateway/4.4.3/crds/oidcrelyingparties.microgateway.airlock.com.yaml @@ -5,7 +5,7 @@ metadata: controller-gen.kubebuilder.io/version: v0.16.4 labels: app.kubernetes.io/name: airlock-microgateway-operator - app.kubernetes.io/version: 4.4.0 + app.kubernetes.io/version: 4.4.3 name: oidcrelyingparties.microgateway.airlock.com spec: group: microgateway.airlock.com diff --git a/charts/airlock/microgateway/4.4.0/crds/openapis.microgateway.airlock.com.yaml b/charts/airlock/microgateway/4.4.3/crds/openapis.microgateway.airlock.com.yaml similarity index 99% rename from charts/airlock/microgateway/4.4.0/crds/openapis.microgateway.airlock.com.yaml rename to charts/airlock/microgateway/4.4.3/crds/openapis.microgateway.airlock.com.yaml index f4b9bc2857..6b15b1351f 100644 --- a/charts/airlock/microgateway/4.4.0/crds/openapis.microgateway.airlock.com.yaml +++ b/charts/airlock/microgateway/4.4.3/crds/openapis.microgateway.airlock.com.yaml @@ -5,7 +5,7 @@ metadata: controller-gen.kubebuilder.io/version: v0.16.4 labels: app.kubernetes.io/name: airlock-microgateway-operator - app.kubernetes.io/version: 4.4.0 + app.kubernetes.io/version: 4.4.3 name: openapis.microgateway.airlock.com spec: group: microgateway.airlock.com diff --git a/charts/airlock/microgateway/4.4.0/crds/parsers.microgateway.airlock.com.yaml b/charts/airlock/microgateway/4.4.3/crds/parsers.microgateway.airlock.com.yaml similarity index 99% rename from charts/airlock/microgateway/4.4.0/crds/parsers.microgateway.airlock.com.yaml rename to charts/airlock/microgateway/4.4.3/crds/parsers.microgateway.airlock.com.yaml index ab0a5f4d26..28c5532cb6 100644 --- a/charts/airlock/microgateway/4.4.0/crds/parsers.microgateway.airlock.com.yaml +++ b/charts/airlock/microgateway/4.4.3/crds/parsers.microgateway.airlock.com.yaml @@ -5,7 +5,7 @@ metadata: controller-gen.kubebuilder.io/version: v0.16.4 labels: app.kubernetes.io/name: airlock-microgateway-operator - app.kubernetes.io/version: 4.4.0 + app.kubernetes.io/version: 4.4.3 name: parsers.microgateway.airlock.com spec: group: microgateway.airlock.com diff --git a/charts/airlock/microgateway/4.4.0/crds/redisproviders.microgateway.airlock.com.yaml b/charts/airlock/microgateway/4.4.3/crds/redisproviders.microgateway.airlock.com.yaml similarity index 99% rename from charts/airlock/microgateway/4.4.0/crds/redisproviders.microgateway.airlock.com.yaml rename to charts/airlock/microgateway/4.4.3/crds/redisproviders.microgateway.airlock.com.yaml index 44b70be9ba..f67737f3c8 100644 --- a/charts/airlock/microgateway/4.4.0/crds/redisproviders.microgateway.airlock.com.yaml +++ b/charts/airlock/microgateway/4.4.3/crds/redisproviders.microgateway.airlock.com.yaml @@ -5,7 +5,7 @@ metadata: controller-gen.kubebuilder.io/version: v0.16.4 labels: app.kubernetes.io/name: airlock-microgateway-operator - app.kubernetes.io/version: 4.4.0 + app.kubernetes.io/version: 4.4.3 name: redisproviders.microgateway.airlock.com spec: group: microgateway.airlock.com diff --git a/charts/airlock/microgateway/4.4.0/crds/sessionhandlings.microgateway.airlock.com.yaml b/charts/airlock/microgateway/4.4.3/crds/sessionhandlings.microgateway.airlock.com.yaml similarity index 99% rename from charts/airlock/microgateway/4.4.0/crds/sessionhandlings.microgateway.airlock.com.yaml rename to charts/airlock/microgateway/4.4.3/crds/sessionhandlings.microgateway.airlock.com.yaml index 4014275146..39fe5902b8 100644 --- a/charts/airlock/microgateway/4.4.0/crds/sessionhandlings.microgateway.airlock.com.yaml +++ b/charts/airlock/microgateway/4.4.3/crds/sessionhandlings.microgateway.airlock.com.yaml @@ -5,7 +5,7 @@ metadata: controller-gen.kubebuilder.io/version: v0.16.4 labels: app.kubernetes.io/name: airlock-microgateway-operator - app.kubernetes.io/version: 4.4.0 + app.kubernetes.io/version: 4.4.3 name: sessionhandlings.microgateway.airlock.com spec: group: microgateway.airlock.com diff --git a/charts/airlock/microgateway/4.4.0/crds/sidecargateways.microgateway.airlock.com.yaml b/charts/airlock/microgateway/4.4.3/crds/sidecargateways.microgateway.airlock.com.yaml similarity index 99% rename from charts/airlock/microgateway/4.4.0/crds/sidecargateways.microgateway.airlock.com.yaml rename to charts/airlock/microgateway/4.4.3/crds/sidecargateways.microgateway.airlock.com.yaml index d4de013f64..6ae99f15ae 100644 --- a/charts/airlock/microgateway/4.4.0/crds/sidecargateways.microgateway.airlock.com.yaml +++ b/charts/airlock/microgateway/4.4.3/crds/sidecargateways.microgateway.airlock.com.yaml @@ -5,7 +5,7 @@ metadata: controller-gen.kubebuilder.io/version: v0.16.4 labels: app.kubernetes.io/name: airlock-microgateway-operator - app.kubernetes.io/version: 4.4.0 + app.kubernetes.io/version: 4.4.3 name: sidecargateways.microgateway.airlock.com spec: group: microgateway.airlock.com diff --git a/charts/airlock/microgateway/4.4.0/crds/telemetries.microgateway.airlock.com.yaml b/charts/airlock/microgateway/4.4.3/crds/telemetries.microgateway.airlock.com.yaml similarity index 99% rename from charts/airlock/microgateway/4.4.0/crds/telemetries.microgateway.airlock.com.yaml rename to charts/airlock/microgateway/4.4.3/crds/telemetries.microgateway.airlock.com.yaml index 7dcfbc1be4..c9de32789b 100644 --- a/charts/airlock/microgateway/4.4.0/crds/telemetries.microgateway.airlock.com.yaml +++ b/charts/airlock/microgateway/4.4.3/crds/telemetries.microgateway.airlock.com.yaml @@ -5,7 +5,7 @@ metadata: controller-gen.kubebuilder.io/version: v0.16.4 labels: app.kubernetes.io/name: airlock-microgateway-operator - app.kubernetes.io/version: 4.4.0 + app.kubernetes.io/version: 4.4.3 name: telemetries.microgateway.airlock.com spec: group: microgateway.airlock.com diff --git a/charts/airlock/microgateway/4.4.0/dashboards/blockLogs.json b/charts/airlock/microgateway/4.4.3/dashboards/blockLogs.json similarity index 100% rename from charts/airlock/microgateway/4.4.0/dashboards/blockLogs.json rename to charts/airlock/microgateway/4.4.3/dashboards/blockLogs.json diff --git a/charts/airlock/microgateway/4.4.0/dashboards/blockMetrics.json b/charts/airlock/microgateway/4.4.3/dashboards/blockMetrics.json similarity index 100% rename from charts/airlock/microgateway/4.4.0/dashboards/blockMetrics.json rename to charts/airlock/microgateway/4.4.3/dashboards/blockMetrics.json diff --git a/charts/airlock/microgateway/4.4.0/dashboards/headerLogs.json b/charts/airlock/microgateway/4.4.3/dashboards/headerLogs.json similarity index 100% rename from charts/airlock/microgateway/4.4.0/dashboards/headerLogs.json rename to charts/airlock/microgateway/4.4.3/dashboards/headerLogs.json diff --git a/charts/airlock/microgateway/4.4.0/dashboards/license.json b/charts/airlock/microgateway/4.4.3/dashboards/license.json similarity index 100% rename from charts/airlock/microgateway/4.4.0/dashboards/license.json rename to charts/airlock/microgateway/4.4.3/dashboards/license.json diff --git a/charts/airlock/microgateway/4.4.0/dashboards/logOnlyLogs.json b/charts/airlock/microgateway/4.4.3/dashboards/logOnlyLogs.json similarity index 100% rename from charts/airlock/microgateway/4.4.0/dashboards/logOnlyLogs.json rename to charts/airlock/microgateway/4.4.3/dashboards/logOnlyLogs.json diff --git a/charts/airlock/microgateway/4.4.0/dashboards/logOnlyMetrics.json b/charts/airlock/microgateway/4.4.3/dashboards/logOnlyMetrics.json similarity index 100% rename from charts/airlock/microgateway/4.4.0/dashboards/logOnlyMetrics.json rename to charts/airlock/microgateway/4.4.3/dashboards/logOnlyMetrics.json diff --git a/charts/airlock/microgateway/4.4.0/dashboards/overview.json b/charts/airlock/microgateway/4.4.3/dashboards/overview.json similarity index 100% rename from charts/airlock/microgateway/4.4.0/dashboards/overview.json rename to charts/airlock/microgateway/4.4.3/dashboards/overview.json diff --git a/charts/airlock/microgateway/4.4.3/templates/NOTES.txt b/charts/airlock/microgateway/4.4.3/templates/NOTES.txt new file mode 100644 index 0000000000..a607483f9c --- /dev/null +++ b/charts/airlock/microgateway/4.4.3/templates/NOTES.txt @@ -0,0 +1,61 @@ +Thank you for installing Airlock Microgateway. +{{- if .Values.operator.gatewayAPI.enabled }} + +K8s Gateway API support enabled. +Note that the K8s Gateway API support is an incubating Airlock Microgateway feature. We encourage you to try the installation and configuration for testing and evaluation. Your feedback is welcome. + + {{- if or .Values.operator.watchNamespaces .Values.operator.watchNamespaceSelector -}} + {{- fail ` + +K8s Gateway API is only supported using the 'AllNamespaces' installation mode type, ensure that 'operator.watchNamespaces' and 'operator.watchNamespaceSelector' are not configured. +` + -}} + {{- end -}} +{{- end }} + +Please ensure the following prerequisites are fulfilled: +* cert-manager is installed. + https://cert-manager.io/docs/installation/helm/ +* A valid Airlock Microgateway license is deployed in the Kubernetes secret '{{ .Release.Namespace }}/{{ .Values.license.secretName }}' + * Get a free Community license: https://airlock.com/en/microgateway-community + * Order a Premium license: https://airlock.com/en/microgateway-premium +* Airlock Microgateway CNI is installed on the cluster, when running data plane mode sidecar + https://artifacthub.io/packages/helm/airlock-microgateway-cni/microgateway-cni. + For more information about data plane modes, see https://docs.airlock.com/microgateway/{{ include "airlock-microgateway.docsVersion" . }}/#data/1660804709650.html + +Further information: +* Documentation: https://docs.airlock.com/microgateway/{{ include "airlock-microgateway.docsVersion" . }} +* CRD API reference documentation: https://docs.airlock.com/microgateway/{{ include "airlock-microgateway.docsVersion" . }}/api/crds +* Airlock Microgateway Labs: https://play.instruqt.com/airlock/invite/hyi9fy4b4jzc?icp_referrer=helm +{{- if .Values.crds.skipVersionCheck }} + +Warning: CRD version check skipped +{{- else -}} +{{- $outdatedCRDs := (include "airlock-microgateway.outdatedCRDs" .) -}} +{{- if $outdatedCRDs -}} + {{- fail (printf ` + +Helm does not automatically upgrade CRDs from the chart's 'crds/' directory during 'helm install/upgrade'. +Therefore, the CRDs must be manually upgraded with the following command before deploying this chart: + +kubectl apply -k https://github.com/airlock/microgateway/deploy/charts/airlock-microgateway/crds/?ref=%s --server-side --force-conflicts + +If you are not using the helm install/upgrade command and instead rely on some other mechanism which is able to upgrade CRDs for deploying this chart, you can suppress this error by setting the helm value 'crds.skipVersionCheck=true'.` + .Chart.AppVersion) + -}} +{{- end -}} +{{- end -}} +{{- if .Values.tests.enabled -}} + {{- if .Values.operator.watchNamespaces -}} + {{- if not (has .Release.Namespace .Values.operator.watchNamespaces) -}} + {{- fail (printf ` + +To execute 'helm test', it is necessary that the release namespace '%s' is part of the operator's watch scope. Either disable the tests or ensure that the release namespace is added to watch namspace list ('operator.watchNamespaces') in the helm values. +` + .Release.Namespace) + -}} + {{- end -}} + {{- end -}} +{{- end }} + +Your release version is {{ .Chart.Version }}. \ No newline at end of file diff --git a/charts/airlock/microgateway/4.4.3/templates/_helpers.tpl b/charts/airlock/microgateway/4.4.3/templates/_helpers.tpl new file mode 100644 index 0000000000..733ba96486 --- /dev/null +++ b/charts/airlock/microgateway/4.4.3/templates/_helpers.tpl @@ -0,0 +1,153 @@ +{{/* +Expand the name of the chart. +We truncate at 49 chars because some Kubernetes name fields are limited to 63 chars (by the DNS naming spec) +and the longest explicit suffix is 14 characters. +*/}} +{{- define "airlock-microgateway.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 49 | trimSuffix "-" }} +{{- end }} + +{{/* +Convert an image configuration object into an image ref string. +*/}} +{{- define "airlock-microgateway.image" -}} + {{- if .digest -}} + {{- printf "%s@%s" .repository .digest -}} + {{- else if .tag -}} + {{- printf "%s:%s" .repository .tag -}} + {{- else -}} + {{- printf "%s" .repository -}} + {{- end -}} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 36 chars because some Kubernetes name fields are limited to 63 chars (by the DNS naming spec) +and the longest implicit suffix is 27 characters. +If release name contains chart name it will be used as a full name. +*/}} +{{- define "airlock-microgateway.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 36 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 36 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 36 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "airlock-microgateway.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "airlock-microgateway.sharedLabels" -}} +helm.sh/chart: {{ include "airlock-microgateway.chart" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/part-of: {{ .Chart.Name }} +{{- with .Values.commonLabels }} +{{ toYaml .}} +{{- end }} +{{- end }} + +{{/* +Common Selector labels +*/}} +{{- define "airlock-microgateway.sharedSelectorLabels" -}} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Restricted Container Security Context +*/}} +{{- define "airlock-microgateway.restrictedSecurityContext" -}} +allowPrivilegeEscalation: false +privileged: false +runAsNonRoot: true +capabilities: + drop: ["ALL"] +readOnlyRootFilesystem: true +seccompProfile: + type: RuntimeDefault +{{- end }} + +{{/* Precondition: May only be used if AppVersion is isSemver */}} +{{- define "airlock-microgateway.supportedCRDVersionPattern" -}} +{{- $version := (semver .Chart.AppVersion) -}} +{{- if $version.Prerelease -}} +>= {{ $version.Major }}.{{ $version.Minor }}.{{ $version.Patch }}-{{ $version.Prerelease }} +{{- else -}} +>= {{ $version.Major }}.{{ $version.Minor }}.0 || >= {{ $version.Major }}.{{ $version.Minor }}.{{ add1 $version.Patch }}-0 +{{- end -}} +{{- end -}} + +{{- define "airlock-microgateway.outdatedCRDs" -}} +{{- if (eq "true" (include "airlock-microgateway.isSemver" .Chart.AppVersion)) -}} + {{- $supportedVersion := (include "airlock-microgateway.supportedCRDVersionPattern" .) -}} + {{- range $path, $_ := .Files.Glob "crds/*.yaml" -}} + {{- $api := ($.Files.Get $path | fromYaml).metadata.name -}} + {{- $crd := (lookup "apiextensions.k8s.io/v1" "CustomResourceDefinition" "" $api) -}} + {{- $isOutdated := false -}} + {{- if $crd -}} + {{/* If CRD is already present in the cluster, it must have the minimum supported version */}} + {{- $isOutdated = true -}} + {{- if hasKey $crd.metadata "labels" -}} + {{- $crdVersion := get $crd.metadata.labels "app.kubernetes.io/version" -}} + {{- if (eq "true" (include "airlock-microgateway.isSemver" $crdVersion)) -}} + {{- if (semverCompare $supportedVersion $crdVersion) }} + {{- $isOutdated = false -}} + {{- end }} + {{- end -}} + {{- end -}} + {{- end -}} + {{- if $isOutdated }} +{{ base $path }} + {{- end }} + {{- end -}} +{{- end -}} +{{- end -}} + +{{- define "airlock-microgateway.isSemver" -}} +{{- regexMatch `^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$` . -}} +{{- end -}} + +{{- define "airlock-microgateway.docsVersion" -}} +{{- if and (eq "true" (include "airlock-microgateway.isSemver" .Chart.AppVersion)) (not (contains "-" .Chart.AppVersion)) -}} + {{- $version := (semver .Chart.AppVersion) -}} + {{- $version.Major }}.{{ $version.Minor -}} +{{- else -}} + {{- print "latest" -}} +{{- end -}} +{{- end -}} + +{{- define "airlock-microgateway.watchNamespaceSelector.labelQuery" -}} +{{- $list := list -}} +{{- with .matchLabels -}} + {{- range $key, $value := . -}} + {{- $list = append $list (printf "%s=%s" $key $value) -}} + {{- end -}} +{{- end -}} +{{- with .matchExpressions -}} + {{- range . -}} + {{- if has .operator (list "In" "NotIn") -}} + {{- $list = append $list (printf "%s %s (%s)" .key (lower .operator) (join "," .values)) -}} + {{- else if eq .operator "Exists" -}} + {{- $list = append $list .key -}} + {{- else if eq .operator "DoesNotExist" -}} + {{- $list = append $list (printf "!%s" .key) -}} + {{- end -}} + {{- end -}} +{{- end -}} +{{- join "," $list -}} +{{- end -}} \ No newline at end of file diff --git a/charts/airlock/microgateway/4.4.3/templates/operator/_operator_helpers.tpl b/charts/airlock/microgateway/4.4.3/templates/operator/_operator_helpers.tpl new file mode 100644 index 0000000000..a540ff9f4f --- /dev/null +++ b/charts/airlock/microgateway/4.4.3/templates/operator/_operator_helpers.tpl @@ -0,0 +1,42 @@ +{{/* +Create a default fully qualified name for operator components. +*/}} +{{- define "airlock-microgateway.operator.fullname" -}} +{{ include "airlock-microgateway.fullname" . }}-operator +{{- end }} + + +{{/* +Common operator labels +*/}} +{{- define "airlock-microgateway.operator.labels" -}} +{{ include "airlock-microgateway.sharedLabels" . }} +{{ include "airlock-microgateway.operator.selectorLabels" . }} +{{- end }} + +{{/* +Operator Selector labels +*/}} +{{- define "airlock-microgateway.operator.selectorLabels" -}} +{{ include "airlock-microgateway.sharedSelectorLabels" . }} +app.kubernetes.io/name: {{ include "airlock-microgateway.name" . }}-operator +app.kubernetes.io/component: controller +{{- end }} + +{{/* +Create the name of the service account to use for the operator +*/}} +{{- define "airlock-microgateway.operator.serviceAccountName" -}} +{{- if .Values.operator.serviceAccount.create }} +{{- default (include "airlock-microgateway.operator.fullname" .) .Values.operator.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.operator.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +ServiceMonitor metrics regex pattern for leader only metrics +*/}} +{{- define "airlock-microgateway.operator.metricsLeaderOnlyRegexPattern" -}} +^(microgateway_license|microgateway_sidecars).*$ +{{- end }} diff --git a/charts/airlock/microgateway/4.4.0/templates/operator/_rbac.gen.tpl b/charts/airlock/microgateway/4.4.3/templates/operator/_rbac.gen.tpl similarity index 100% rename from charts/airlock/microgateway/4.4.0/templates/operator/_rbac.gen.tpl rename to charts/airlock/microgateway/4.4.3/templates/operator/_rbac.gen.tpl diff --git a/charts/airlock/microgateway/4.4.0/templates/operator/_webhooks.gen.tpl b/charts/airlock/microgateway/4.4.3/templates/operator/_webhooks.gen.tpl similarity index 100% rename from charts/airlock/microgateway/4.4.0/templates/operator/_webhooks.gen.tpl rename to charts/airlock/microgateway/4.4.3/templates/operator/_webhooks.gen.tpl diff --git a/charts/airlock/microgateway/4.4.0/templates/operator/configmap.yaml b/charts/airlock/microgateway/4.4.3/templates/operator/configmap.yaml similarity index 99% rename from charts/airlock/microgateway/4.4.0/templates/operator/configmap.yaml rename to charts/airlock/microgateway/4.4.3/templates/operator/configmap.yaml index 276a632e86..a9c07b5f16 100644 --- a/charts/airlock/microgateway/4.4.0/templates/operator/configmap.yaml +++ b/charts/airlock/microgateway/4.4.3/templates/operator/configmap.yaml @@ -125,6 +125,7 @@ data: - name: xds_cluster connect_timeout: 1s type: STRICT_DNS + respect_dns_ttl: true load_assignment: cluster_name: xds_cluster endpoints: diff --git a/charts/airlock/microgateway/4.4.3/templates/operator/dashboard-configmap.yaml b/charts/airlock/microgateway/4.4.3/templates/operator/dashboard-configmap.yaml new file mode 100644 index 0000000000..b71ac89b65 --- /dev/null +++ b/charts/airlock/microgateway/4.4.3/templates/operator/dashboard-configmap.yaml @@ -0,0 +1,28 @@ +{{- if .Values.dashboards.create -}} +{{- range $instance := (keys .Values.dashboards.instances | sortAlpha) -}} +{{- $dashboard := get $.Values.dashboards.instances $instance -}} +{{- if $dashboard.create }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "airlock-microgateway.fullname" $ }}-dashboard-{{ $instance | lower }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "airlock-microgateway.operator.labels" $ | nindent 4 }} + {{- with $.Values.dashboards.config.grafana.dashboardLabel -}} + {{- .name | nindent 4 -}}: {{ .value | quote }} + {{- end }} + annotations: + {{- with $.Values.dashboards.config.grafana.folderAnnotation -}} + {{- .name | nindent 4 -}}: {{ .value | quote }} + {{- end }} + {{- with $.Values.commonAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +data: + {{- printf "%s.json" $instance | nindent 2 }}: |- + {{- ($.Files.Get (printf "dashboards/%s.json" $instance)) | nindent 4 -}} +{{- end -}} +{{- end -}} +{{- end -}} \ No newline at end of file diff --git a/charts/airlock/microgateway/4.4.3/templates/operator/deployment.yaml b/charts/airlock/microgateway/4.4.3/templates/operator/deployment.yaml new file mode 100644 index 0000000000..db340cdecc --- /dev/null +++ b/charts/airlock/microgateway/4.4.3/templates/operator/deployment.yaml @@ -0,0 +1,143 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "airlock-microgateway.operator.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "airlock-microgateway.operator.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.operator.replicaCount }} + {{- with .Values.operator.updateStrategy }} + strategy: + {{- toYaml . | trim | nindent 4 }} + {{- end }} + selector: + matchLabels: + {{- include "airlock-microgateway.operator.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + checksum/config: {{ include (print $.Template.BasePath "/operator/configmap.yaml") . | sha256sum }} + kubectl.kubernetes.io/default-container: manager + {{- with mustMerge .Values.operator.podAnnotations .Values.commonAnnotations}} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "airlock-microgateway.operator.labels" . | nindent 8 }} + {{- with .Values.operator.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + containers: + - args: + - --config=operator_config.yaml + env: + - name: ENGINE_IMAGE + value: {{ include "airlock-microgateway.image" .Values.engine.image }} + - name: NETWORK_VALIDATOR_IMAGE + value: {{ include "airlock-microgateway.image" .Values.networkValidator.image }} + - name: SESSION_AGENT_IMAGE + value: {{ include "airlock-microgateway.image" .Values.sessionAgent.image }} + - name: OPERATOR_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: OPERATOR_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: {{ include "airlock-microgateway.image" .Values.operator.image }} + imagePullPolicy: {{ .Values.operator.image.pullPolicy }} + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + timeoutSeconds: 5 + name: manager + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP + - containerPort: 13377 + name: xds-server + protocol: TCP + - containerPort: 8080 + protocol: TCP + - containerPort: 8081 + protocol: TCP + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + {{- with .Values.operator.resources }} + resources: + {{- toYaml . | nindent 10 }} + {{- end }} + securityContext: + {{- include "airlock-microgateway.restrictedSecurityContext" . | nindent 10 }} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + - mountPath: /opt/airlock/license/ + name: airlock-microgateway-license + readOnly: true + - mountPath: /operator_config.yaml + name: operator-config + subPath: operator_config.yaml + - mountPath: /sidecar/engine_container_template.yaml + name: operator-config + subPath: engine_container_template.yaml + - mountPath: /sidecar/network_validator_container_template.yaml + name: operator-config + subPath: network_validator_container_template.yaml + - mountPath: /sidecar/session_agent_container_template.yaml + name: operator-config + subPath: session_agent_container_template.yaml + - mountPath: /engine_bootstrap_config_template.yaml + name: operator-config + subPath: engine_bootstrap_config_template.yaml + securityContext: + runAsNonRoot: true + serviceAccountName: {{ include "airlock-microgateway.operator.serviceAccountName" . }} + terminationGracePeriodSeconds: 10 + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.operator.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.operator.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.operator.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: {{ include "airlock-microgateway.operator.fullname" . }}-webhook-server-cert + - name: airlock-microgateway-license + secret: + defaultMode: 292 + optional: true + secretName: {{ .Values.license.secretName }} + - configMap: + name: {{ include "airlock-microgateway.operator.fullname" . }}-config + name: operator-config diff --git a/charts/airlock/microgateway/4.4.3/templates/operator/manager-role.yaml b/charts/airlock/microgateway/4.4.3/templates/operator/manager-role.yaml new file mode 100644 index 0000000000..90335bcfe1 --- /dev/null +++ b/charts/airlock/microgateway/4.4.3/templates/operator/manager-role.yaml @@ -0,0 +1,33 @@ +{{- if .Values.operator.rbac.create }} +{{- if empty .Values.operator.watchNamespaces }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "airlock-microgateway.operator.fullname" . }}-manager-{{ .Release.Namespace }} + labels: + {{- include "airlock-microgateway.operator.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +rules: +{{ include "airlock-microgateway-operator.rbacRules" . -}} +{{- else }} +{{- range $namespace := (append .Values.operator.watchNamespaces .Release.Namespace | uniq) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "airlock-microgateway.operator.fullname" $ }}-manager + namespace: {{ $namespace }} + labels: + {{- include "airlock-microgateway.operator.labels" $ | nindent 4 }} + {{- with $.Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +rules: +{{ include "airlock-microgateway-operator.rbacRules" $ }} +--- +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/charts/airlock/microgateway/4.4.3/templates/operator/manager-rolebinding.yaml b/charts/airlock/microgateway/4.4.3/templates/operator/manager-rolebinding.yaml new file mode 100644 index 0000000000..ae99cfb7b6 --- /dev/null +++ b/charts/airlock/microgateway/4.4.3/templates/operator/manager-rolebinding.yaml @@ -0,0 +1,45 @@ +{{- if .Values.operator.rbac.create }} +{{- if empty .Values.operator.watchNamespaces }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "airlock-microgateway.operator.fullname" . }}-manager-{{ .Release.Namespace }} + labels: + {{- include "airlock-microgateway.operator.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "airlock-microgateway.operator.fullname" . }}-manager-{{ .Release.Namespace }} +subjects: + - kind: ServiceAccount + name: {{ include "airlock-microgateway.operator.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- else }} +{{- range $namespace := (append .Values.operator.watchNamespaces .Release.Namespace | uniq) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "airlock-microgateway.operator.fullname" $ }}-manager + namespace: {{ $namespace }} + labels: + {{- include "airlock-microgateway.operator.labels" $ | nindent 4 }} + {{- with $.Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "airlock-microgateway.operator.fullname" $ }}-manager +subjects: + - kind: ServiceAccount + name: {{ include "airlock-microgateway.operator.serviceAccountName" $ }} + namespace: {{ $.Release.Namespace }} +--- +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/charts/airlock/microgateway/4.4.3/templates/operator/metrics-service.yaml b/charts/airlock/microgateway/4.4.3/templates/operator/metrics-service.yaml new file mode 100644 index 0000000000..34d23f6d67 --- /dev/null +++ b/charts/airlock/microgateway/4.4.3/templates/operator/metrics-service.yaml @@ -0,0 +1,47 @@ +apiVersion: v1 +kind: Service +metadata: + name: airlock-microgateway-operator-metrics + namespace: {{ .Release.Namespace }} + labels: + {{- include "airlock-microgateway.operator.labels" . | nindent 4 }} + {{- with .Values.operator.serviceLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with mustMerge .Values.operator.serviceAnnotations .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + ports: + - appProtocol: http + name: metrics + port: 8080 + protocol: TCP + selector: + {{- include "airlock-microgateway.operator.selectorLabels" . | nindent 4 }} +--- +apiVersion: v1 +kind: Service +metadata: + name: airlock-microgateway-operator-leader-metrics + namespace: {{ .Release.Namespace }} + labels: + {{- include "airlock-microgateway.operator.labels" . | nindent 4 }} + {{- with .Values.operator.serviceLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + operator.microgateway.airlock.com/isLeader: "true" + {{- with mustMerge .Values.operator.serviceAnnotations .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + ports: + - appProtocol: http + name: metrics + port: 8080 + protocol: TCP + selector: + {{- include "airlock-microgateway.operator.selectorLabels" . | nindent 4 }} + operator.microgateway.airlock.com/isLeader: "true" \ No newline at end of file diff --git a/charts/airlock/microgateway/4.4.3/templates/operator/mutating-webhook.yaml b/charts/airlock/microgateway/4.4.3/templates/operator/mutating-webhook.yaml new file mode 100644 index 0000000000..311f9726ad --- /dev/null +++ b/charts/airlock/microgateway/4.4.3/templates/operator/mutating-webhook.yaml @@ -0,0 +1,28 @@ +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: {{ include "airlock-microgateway.operator.fullname" . }}-webhook-{{ .Release.Namespace }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "airlock-microgateway.operator.labels" . | nindent 4 }} + annotations: + cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ include "airlock-microgateway.operator.fullname" . }}-serving-cert + {{- with .Values.commonAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +webhooks: +{{- range $webhook := (include "airlock-microgateway-operator.mutatingWebhooks" .) | fromYamlArray }} +- {{ toYaml $webhook | indent 2 | trim }} + {{- with $.Values.operator.watchNamespaceSelector }} + namespaceSelector: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with $.Values.operator.watchNamespaces }} + namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: In + values: + {{- toYaml . | nindent 10 }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/airlock/microgateway/4.4.3/templates/operator/podmonitor.yaml b/charts/airlock/microgateway/4.4.3/templates/operator/podmonitor.yaml new file mode 100644 index 0000000000..1fe34fcb35 --- /dev/null +++ b/charts/airlock/microgateway/4.4.3/templates/operator/podmonitor.yaml @@ -0,0 +1,27 @@ +{{- if .Values.engine.sidecar.podMonitor.create }} +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: {{ include "airlock-microgateway.fullname" . }}-engine + namespace: {{ .Release.Namespace }} + labels: + {{- include "airlock-microgateway.operator.labels" . | nindent 4 }} + {{- with .Values.engine.sidecar.podMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + namespaceSelector: + any: true + selector: + matchLabels: + sidecar.microgateway.airlock.com/inject: "true" + microgateway.airlock.com/managedBy: {{ .Release.Namespace }} + podMetricsEndpoints: + - targetPort: 19002 + path: /metrics + scheme: http +{{- end -}} \ No newline at end of file diff --git a/charts/airlock/microgateway/4.4.3/templates/operator/role.yaml b/charts/airlock/microgateway/4.4.3/templates/operator/role.yaml new file mode 100644 index 0000000000..5378be8ef9 --- /dev/null +++ b/charts/airlock/microgateway/4.4.3/templates/operator/role.yaml @@ -0,0 +1,45 @@ +{{- if .Values.operator.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "airlock-microgateway.operator.fullname" . }}-leader-election + namespace: {{ .Release.Namespace }} + labels: + {{- include "airlock-microgateway.operator.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +{{- end -}} \ No newline at end of file diff --git a/charts/airlock/microgateway/4.4.3/templates/operator/rolebinding.yaml b/charts/airlock/microgateway/4.4.3/templates/operator/rolebinding.yaml new file mode 100644 index 0000000000..bafec10156 --- /dev/null +++ b/charts/airlock/microgateway/4.4.3/templates/operator/rolebinding.yaml @@ -0,0 +1,20 @@ +{{- if .Values.operator.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "airlock-microgateway.operator.fullname" . }}-leader-election + namespace: {{ .Release.Namespace }} + labels: + {{- include "airlock-microgateway.operator.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "airlock-microgateway.operator.fullname" . }}-leader-election +subjects: + - kind: ServiceAccount + name: {{ include "airlock-microgateway.operator.serviceAccountName" . }} +{{- end -}} diff --git a/charts/airlock/microgateway/4.4.3/templates/operator/selfsigned-issuer.yaml b/charts/airlock/microgateway/4.4.3/templates/operator/selfsigned-issuer.yaml new file mode 100644 index 0000000000..466c56338e --- /dev/null +++ b/charts/airlock/microgateway/4.4.3/templates/operator/selfsigned-issuer.yaml @@ -0,0 +1,13 @@ +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ include "airlock-microgateway.operator.fullname" . }}-selfsigned-issuer + namespace: {{ .Release.Namespace }} + labels: + {{- include "airlock-microgateway.operator.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selfSigned: {} diff --git a/charts/airlock/microgateway/4.4.3/templates/operator/serviceaccount.yaml b/charts/airlock/microgateway/4.4.3/templates/operator/serviceaccount.yaml new file mode 100644 index 0000000000..434d7e9d30 --- /dev/null +++ b/charts/airlock/microgateway/4.4.3/templates/operator/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.operator.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "airlock-microgateway.operator.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "airlock-microgateway.operator.labels" . | nindent 4 }} + {{- with mustMerge .Values.operator.serviceAccount.annotations .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end -}} diff --git a/charts/airlock/microgateway/4.4.3/templates/operator/servicemonitor.yaml b/charts/airlock/microgateway/4.4.3/templates/operator/servicemonitor.yaml new file mode 100644 index 0000000000..ff85a9a310 --- /dev/null +++ b/charts/airlock/microgateway/4.4.3/templates/operator/servicemonitor.yaml @@ -0,0 +1,60 @@ +{{- if .Values.operator.serviceMonitor.create }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "airlock-microgateway.operator.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "airlock-microgateway.operator.labels" . | nindent 4 }} + {{- with .Values.operator.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "airlock-microgateway.operator.selectorLabels" . | nindent 6 }} + matchExpressions: + - { key: "operator.microgateway.airlock.com/isLeader", operator: DoesNotExist } + endpoints: + - path: /metrics + port: metrics + scheme: http + metricRelabelings: + - sourceLabels: + - __name__ + regex: {{ include "airlock-microgateway.operator.metricsLeaderOnlyRegexPattern" . }} + action: drop +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "airlock-microgateway.operator.fullname" . }}-leader + namespace: {{ .Release.Namespace }} + labels: + {{- include "airlock-microgateway.operator.labels" . | nindent 4 }} + {{- with .Values.operator.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "airlock-microgateway.operator.selectorLabels" . | nindent 6 }} + operator.microgateway.airlock.com/isLeader: "true" + endpoints: + - path: /metrics + port: metrics + scheme: http + metricRelabelings: + - sourceLabels: + - __name__ + regex: {{ include "airlock-microgateway.operator.metricsLeaderOnlyRegexPattern" . }} + action: keep +{{- end -}} \ No newline at end of file diff --git a/charts/airlock/microgateway/4.4.3/templates/operator/serving-certificate.yaml b/charts/airlock/microgateway/4.4.3/templates/operator/serving-certificate.yaml new file mode 100644 index 0000000000..60b92e1e2c --- /dev/null +++ b/charts/airlock/microgateway/4.4.3/templates/operator/serving-certificate.yaml @@ -0,0 +1,19 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ include "airlock-microgateway.operator.fullname" . }}-serving-cert + namespace: {{ .Release.Namespace }} + labels: + {{- include "airlock-microgateway.operator.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + dnsNames: + - airlock-microgateway-operator-webhook.{{ .Release.Namespace }}.svc + - airlock-microgateway-operator-webhook.{{ .Release.Namespace }}.svc.cluster.local + issuerRef: + kind: Issuer + name: {{ include "airlock-microgateway.operator.fullname" . }}-selfsigned-issuer + secretName: {{ include "airlock-microgateway.operator.fullname" . }}-webhook-server-cert diff --git a/charts/airlock/microgateway/4.4.3/templates/operator/validating-webhook.yaml b/charts/airlock/microgateway/4.4.3/templates/operator/validating-webhook.yaml new file mode 100644 index 0000000000..5d6b4396ba --- /dev/null +++ b/charts/airlock/microgateway/4.4.3/templates/operator/validating-webhook.yaml @@ -0,0 +1,28 @@ +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: {{ include "airlock-microgateway.operator.fullname" . }}-webhook-{{ .Release.Namespace }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "airlock-microgateway.operator.labels" . | nindent 4 }} + annotations: + cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ include "airlock-microgateway.operator.fullname" . }}-serving-cert + {{- with .Values.commonAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +webhooks: +{{- range $webhook := (include "airlock-microgateway-operator.validatingWebhooks" .) | fromYamlArray }} +- {{ toYaml $webhook | indent 2 | trim }} + {{- with $.Values.operator.watchNamespaceSelector }} + namespaceSelector: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with $.Values.operator.watchNamespaces }} + namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: In + values: + {{- toYaml . | nindent 10 }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/airlock/microgateway/4.4.3/templates/operator/webhook-service.yaml b/charts/airlock/microgateway/4.4.3/templates/operator/webhook-service.yaml new file mode 100644 index 0000000000..477ea839f3 --- /dev/null +++ b/charts/airlock/microgateway/4.4.3/templates/operator/webhook-service.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: Service +metadata: + name: airlock-microgateway-operator-webhook + namespace: {{ .Release.Namespace }} + labels: + {{- include "airlock-microgateway.operator.labels" . | nindent 4 }} + {{- with .Values.operator.serviceLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with mustMerge .Values.operator.serviceAnnotations .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + ports: + - appProtocol: https + name: webhook + port: 443 + protocol: TCP + targetPort: 9443 + selector: + {{- include "airlock-microgateway.operator.selectorLabels" . | nindent 4 }} \ No newline at end of file diff --git a/charts/airlock/microgateway/4.4.3/templates/operator/xds-service.yaml b/charts/airlock/microgateway/4.4.3/templates/operator/xds-service.yaml new file mode 100644 index 0000000000..81b41acf5b --- /dev/null +++ b/charts/airlock/microgateway/4.4.3/templates/operator/xds-service.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: Service +metadata: + name: airlock-microgateway-operator-xds + namespace: {{ .Release.Namespace }} + labels: + {{- include "airlock-microgateway.operator.labels" . | nindent 4 }} + {{- with .Values.operator.serviceLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with mustMerge .Values.operator.serviceAnnotations .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + ports: + - appProtocol: grpc + name: xds + port: 13377 + protocol: TCP + targetPort: 13377 + selector: + {{- include "airlock-microgateway.operator.selectorLabels" . | nindent 4 }} + operator.microgateway.airlock.com/isLeader: "true" diff --git a/charts/airlock/microgateway/4.4.3/templates/tests/rbac.yaml b/charts/airlock/microgateway/4.4.3/templates/tests/rbac.yaml new file mode 100644 index 0000000000..93bd4cd1bd --- /dev/null +++ b/charts/airlock/microgateway/4.4.3/templates/tests/rbac.yaml @@ -0,0 +1,143 @@ +{{- if .Values.tests.enabled -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/component: tests + app.kubernetes.io/name: {{ include "airlock-microgateway.name" . }}-tests + {{- include "airlock-microgateway.sharedLabels" . | nindent 4 }} + name: "{{ include "airlock-microgateway.fullname" . }}-tests" + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/component: tests + app.kubernetes.io/name: {{ include "airlock-microgateway.name" . }}-tests + {{- include "airlock-microgateway.sharedLabels" . | nindent 4 }} + name: "{{ include "airlock-microgateway.fullname" . }}-tests" + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: "{{ include "airlock-microgateway.fullname" . }}-tests" +subjects: +- kind: ServiceAccount + name: "{{ include "airlock-microgateway.fullname" . }}-tests" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app.kubernetes.io/component: tests + app.kubernetes.io/name: {{ include "airlock-microgateway.name" . }}-tests + {{- include "airlock-microgateway.sharedLabels" . | nindent 4 }} + name: "{{ include "airlock-microgateway.fullname" . }}-tests" + namespace: {{ .Release.Namespace }} +rules: +- apiGroups: + - microgateway.airlock.com + resources: + - sidecargateways + resourceNames: + - "{{ include "airlock-microgateway.fullname" . }}-test-sidecargateway" + verbs: + - get + - list + - watch + - delete +- apiGroups: + - microgateway.airlock.com + resources: + - sidecargateways + verbs: + - create +- apiGroups: + - "" + resources: + - events + verbs: + - list +- apiGroups: + - "apps" + resources: + - deployments + resourceNames: + - "{{ include "airlock-microgateway.operator.fullname" . }}" + verbs: + - get + - list + - watch +- apiGroups: + - "apps" + resources: + - statefulsets + - statefulsets/scale + resourceNames: + - "{{ include "airlock-microgateway.fullname" . }}-test-backend" + verbs: + - get + - list + - watch + - patch +- apiGroups: + - "" + resources: + - pods + - pods/log + - pods/status + - pods/attach + resourceNames: + - "{{ include "airlock-microgateway.fullname" . }}-test-backend-0" + - "{{ include "airlock-microgateway.fullname" . }}-test-valid-request" + - "{{ include "airlock-microgateway.fullname" . }}-test-injection-request" + verbs: + - get + - list + - create + - watch + - delete +- apiGroups: + - "" + resources: + - pods + verbs: + - create +{{- if .Values.operator.watchNamespaceSelector }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/component: tests + app.kubernetes.io/name: {{ include "airlock-microgateway.name" . }}-tests + {{- include "airlock-microgateway.sharedLabels" . | nindent 4 }} + name: "{{ include "airlock-microgateway.fullname" . }}-tests-{{ .Release.Namespace }}" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: "{{ include "airlock-microgateway.fullname" . }}-tests-{{ .Release.Namespace }}" +subjects: + - kind: ServiceAccount + name: "{{ include "airlock-microgateway.fullname" . }}-tests" + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: tests + app.kubernetes.io/name: {{ include "airlock-microgateway.name" . }}-tests + {{- include "airlock-microgateway.sharedLabels" . | nindent 4 }} + name: "{{ include "airlock-microgateway.fullname" . }}-tests-{{ .Release.Namespace }}" +rules: +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - list +{{- end }} +{{- end -}} diff --git a/charts/airlock/microgateway/4.4.3/templates/tests/service.yaml b/charts/airlock/microgateway/4.4.3/templates/tests/service.yaml new file mode 100644 index 0000000000..30ddc278d6 --- /dev/null +++ b/charts/airlock/microgateway/4.4.3/templates/tests/service.yaml @@ -0,0 +1,23 @@ +{{- if .Values.tests.enabled -}} +apiVersion: v1 +kind: Service +metadata: + name: "{{ include "airlock-microgateway.fullname" . }}-test-service" + namespace: {{ .Release.Namespace }} + labels: + app: test-service + app.kubernetes.io/component: test-install + app.kubernetes.io/name: {{ include "airlock-microgateway.name" . }}-tests + {{- include "airlock-microgateway.sharedLabels" . | nindent 4 }} + {{- include "airlock-microgateway.sharedSelectorLabels" . | nindent 4 }} +spec: + selector: + app.kubernetes.io/component: test-install + app.kubernetes.io/name: {{ include "airlock-microgateway.name" . }}-tests + app: "{{ include "airlock-microgateway.fullname" . }}-test-backend" + {{- include "airlock-microgateway.sharedSelectorLabels" . | nindent 4 }} + ports: + - name: http + port: 8080 + targetPort: 8080 +{{- end -}} \ No newline at end of file diff --git a/charts/airlock/microgateway/4.4.3/templates/tests/statefulset.yaml b/charts/airlock/microgateway/4.4.3/templates/tests/statefulset.yaml new file mode 100644 index 0000000000..710a7b9f67 --- /dev/null +++ b/charts/airlock/microgateway/4.4.3/templates/tests/statefulset.yaml @@ -0,0 +1,56 @@ +{{- if .Values.tests.enabled -}} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: "{{ include "airlock-microgateway.fullname" . }}-test-backend" + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/component: test-install + app.kubernetes.io/name: {{ include "airlock-microgateway.name" . }}-tests + app: "{{ include "airlock-microgateway.fullname" . }}-test-backend" + {{- include "airlock-microgateway.sharedLabels" . | nindent 4 }} + {{- include "airlock-microgateway.sharedSelectorLabels" . | nindent 4 }} +spec: + serviceName: nginx + replicas: 0 + selector: + matchLabels: + app.kubernetes.io/component: test-install + app.kubernetes.io/name: {{ include "airlock-microgateway.name" . }}-tests + app: "{{ include "airlock-microgateway.fullname" . }}-test-backend" + {{- include "airlock-microgateway.sharedSelectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + k8s.v1.cni.cncf.io/networks: default/airlock-microgateway-cni + labels: + sidecar.microgateway.airlock.com/inject: "true" + sidecar.istio.io/inject: "false" + app.kubernetes.io/component: test-install + app.kubernetes.io/name: {{ include "airlock-microgateway.name" . }}-tests + app: "{{ include "airlock-microgateway.fullname" . }}-test-backend" + {{- include "airlock-microgateway.sharedLabels" . | nindent 8 }} + {{- include "airlock-microgateway.sharedSelectorLabels" . | nindent 8 }} + spec: + containers: + - image: cgr.dev/chainguard/nginx + name: nginx + ports: + - containerPort: 8080 + volumeMounts: + - mountPath: /var/lib/nginx/tmp/ + name: nginx-tmp + - mountPath: /var/run + name: nginx-run + securityContext: + {{- include "airlock-microgateway.restrictedSecurityContext" . | nindent 12 }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - emptyDir: {} + name: nginx-tmp + - emptyDir: {} + name: nginx-run +{{- end -}} \ No newline at end of file diff --git a/charts/airlock/microgateway/4.4.3/templates/tests/test-install.yaml b/charts/airlock/microgateway/4.4.3/templates/tests/test-install.yaml new file mode 100644 index 0000000000..721ae2b82e --- /dev/null +++ b/charts/airlock/microgateway/4.4.3/templates/tests/test-install.yaml @@ -0,0 +1,227 @@ +{{- if .Values.tests.enabled -}} +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "airlock-microgateway.fullname" . }}-test-install" + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/component: test-install + app.kubernetes.io/name: {{ include "airlock-microgateway.name" . }}-tests + sidecar.istio.io/inject: "false" + {{- include "airlock-microgateway.sharedLabels" . | nindent 4 }} + {{- include "airlock-microgateway.sharedSelectorLabels" . | nindent 4 }} + annotations: + helm.sh/hook: test + helm.sh/hook-delete-policy: before-hook-creation +spec: + restartPolicy: Never + containers: + - name: test + image: "bitnami/kubectl:{{ .Capabilities.KubeVersion.Major }}.{{ .Capabilities.KubeVersion.Minor }}" + securityContext: + {{- include "airlock-microgateway.restrictedSecurityContext" . | nindent 6 }} + command: + - sh + - -c + - | + set -eu + + clean_up() { + echo "" + echo "### Clean up test resources" + kubectl delete --ignore-not-found=true -n {{ .Release.Namespace }} sidecargateways.microgateway.airlock.com {{ include "airlock-microgateway.fullname" . }}-test-sidecargateway || true + echo "" + echo "### Scale down '{{ include "airlock-microgateway.fullname" . }}-test-backend'" + kubectl scale -n {{ .Release.Namespace }} statefulset/{{ include "airlock-microgateway.fullname" . }}-test-backend --replicas=0 --timeout=60s + sleep 3s + echo "" + } + + fail() { + echo "" + echo "### Error: ${1}" + echo "" + + if kubectl get -n {{ .Release.Namespace }} sidecargateway.microgateway.airlock.com/{{ include "airlock-microgateway.fullname" . }}-test-sidecargateway >/dev/null 2>&1; then + echo "" + echo 'Microgateway Sidecargateway status:' + kubectl get -n {{ .Release.Namespace }} sidecargateway.microgateway.airlock.com/{{ include "airlock-microgateway.fullname" . }}-test-sidecargateway -o jsonpath-as-json='{.status}' || true + echo "" + echo "" + fi + + if kubectl get -n {{ .Release.Namespace }} pods/{{ include "airlock-microgateway.fullname" . }}-test-backend-0 >/dev/null 2>&1; then + echo "Pod '{{ include "airlock-microgateway.fullname" . }}-test-backend-0':" + kubectl describe -n {{ .Release.Namespace }} pods/{{ include "airlock-microgateway.fullname" . }}-test-backend-0 || true + echo "" + echo "" + echo 'Logs of Nginx container:' + kubectl logs -n {{ .Release.Namespace }} pods/{{ include "airlock-microgateway.fullname" . }}-test-backend-0 -c nginx --tail 5 || true + echo "" + echo "" + # Wait for engine logs + sleep 10s + echo 'Logs of Microgateway Engine container:' + kubectl logs -n {{ .Release.Namespace }} pods/{{ include "airlock-microgateway.fullname" . }}-test-backend-0 -c airlock-microgateway-engine --tail 5 || true + fi + + exit 1 + } + + create_sidecargateway() { + # create SidecarGateway resource for testing purposes + kubectl delete --ignore-not-found=true -n {{ .Release.Namespace }} sidecargateways.microgateway.airlock.com {{ include "airlock-microgateway.fullname" . }}-test-sidecargateway || true + kubectl apply -f - </dev/null 2>&1; do sleep 1s; i=$((i+1)); done + kubectl logs -f -n {{ .Release.Namespace }} {{ include "airlock-microgateway.fullname" . }}-test-valid-request + kubectl delete pod --ignore-not-found=true -n {{ .Release.Namespace }} {{ include "airlock-microgateway.fullname" . }}-test-valid-request + } + + {{- if .Values.operator.watchNamespaceSelector }} + echo "### Verify that Namespace Selector matches Namespace '{{ .Release.Namespace }}'" + if ! kubectl get namespace -l '{{ include "airlock-microgateway.watchNamespaceSelector.labelQuery" .Values.operator.watchNamespaceSelector }}' | grep -q {{ .Release.Namespace }}; then + labels=$(kubectl get namespace {{ .Release.Namespace }} -o jsonpath={.metadata.labels} | jq | awk '{print " " $0}') + fail {{printf `"Operator namespace '%s' is not part of the operator's watch scope. To execute 'helm test', the selector configured in the helm value 'operator.watchNamespaceSelector' must match the namespace's labels:\n* Current selector:\n%s\n\n* Current labels:\n$labels\n###"` + .Release.Namespace + (replace "\"" "\\\"" (replace "\n" "\\n" (.Values.operator.watchNamespaceSelector | toPrettyJson | indent 2))) + }} + fi + echo "" + {{- end }} + + trap clean_up EXIT + echo "" + + echo "### Waiting for Microgateway Operator Deployments to be ready" + if ! kubectl rollout status -n {{ .Release.Namespace }} --timeout=90s \ + deployments/{{ include "airlock-microgateway.operator.fullname" . }}; then + fail 'Timeout occurred' + fi + echo "" + + echo "### Scale '{{ include "airlock-microgateway.fullname" . }}-test-backend' to '1' replica" + # scale to zero replicas to ensure no pods are present from previous runs + kubectl scale -n {{ .Release.Namespace }} statefulset/{{ include "airlock-microgateway.fullname" . }}-test-backend --replicas=0 --timeout=10s + kubectl scale -n {{ .Release.Namespace }} statefulset/{{ include "airlock-microgateway.fullname" . }}-test-backend --replicas=1 --timeout=10s + echo "" + + echo "### Waiting for backend pod" + i=0 + while true; do + if kubectl get -n {{ .Release.Namespace }} pods/{{ include "airlock-microgateway.fullname" . }}-test-backend-0; then + break + elif [ $i -gt 3 ]; then + fail 'Pod not ready' + fi + sleep 2s + i=$((i+1)) + done + + echo "### Checking Microgateway Engine sidecar container was injected" + if ! kubectl get -n {{ .Release.Namespace }} pods/{{ include "airlock-microgateway.fullname" . }}-test-backend-0 -o jsonpath='{.spec.containers[?(@.name=="airlock-microgateway-engine")]}' | grep -q "airlock-microgateway-engine"; then + fail 'Microgateway Engine sidecar container not injected' + fi + echo "True" + echo "" + + echo "### Checking for valid license" + i=0 + while true; do + if [ "$(kubectl get -n {{ .Release.Namespace }} pods/{{ include "airlock-microgateway.fullname" . }}-test-backend-0 -o jsonpath='{.metadata.labels.sidecar\.microgateway\.airlock\.com/licensed}')" = 'true' ]; then + break + elif [ $i -gt 30 ]; then + fail 'Microgateway license is missing or invalid' + fi + sleep 2s + i=$((i+1)) + done + echo "True" + echo "" + + echo "### Create SidecarGateway resource for testing" + if ! create_sidecargateway ; then + fail 'Creation of SidecarGateway resource failed' + fi + echo "" + + echo "### Waiting for '{{ include "airlock-microgateway.fullname" . }}-test-backend' to be ready" + if ! kubectl rollout status -n {{ .Release.Namespace }} statefulset/{{ include "airlock-microgateway.fullname" . }}-test-backend --timeout=90s; then + fail 'Timeout occurred' + fi + echo "" + + echo "### Waiting for 'engine-config-valid' condition" + if ! kubectl wait -n {{ .Release.Namespace }} pods --field-selector=metadata.name={{ include "airlock-microgateway.fullname" . }}-test-backend-0 --timeout=90s --for=condition=microgateway.airlock.com/engine-config-valid=True; then + fail 'Configuration was never accepted by the Microgateway Engine' + fi + sleep 5s + echo "" + echo "" + + echo "### Checking whether a valid request is successful and returns HTTP status code '200'" + out=$(curl -vsS --retry 3 --retry-connrefused --connect-timeout 10 "http://{{ include "airlock-microgateway.fullname" . }}-test-service:8080/" || true) + echo "Response:" + echo "${out}" + if ! echo "${out}" | grep -q "200 OK"; then + fail 'A valid request was not successful' + fi + echo "" + echo "" + + echo "### Checking whether a request with an injection attack is blocked and returns HTTP status code '400'" + out=$(curl -vsS --retry 3 --retry-connrefused --connect-timeout 10 "http://{{ include "airlock-microgateway.fullname" . }}-test-service:8080/?token='%20UnION%20all%20select%20A" || true) + echo "Response:" + echo "${out}" + if ! echo "${out}" | grep -q "400 Bad Request"; then + fail 'A malicious request was not blocked' + fi + echo "" + echo "" + + echo "### Installation of '{{ include "airlock-microgateway.fullname" . }}' succeeded" + exit 0 + serviceAccountName: "{{ include "airlock-microgateway.fullname" . }}-tests" +{{- end -}} diff --git a/charts/airlock/microgateway/4.4.3/values.schema.json b/charts/airlock/microgateway/4.4.3/values.schema.json new file mode 100644 index 0000000000..05c7d77175 --- /dev/null +++ b/charts/airlock/microgateway/4.4.3/values.schema.json @@ -0,0 +1,572 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nameOverride": { + "type": "string" + }, + "fullnameOverride": { + "type": "string" + }, + "commonLabels": { + "$ref": "#/definitions/StringMap" + }, + "commonAnnotations": { + "$ref": "#/definitions/StringMap" + }, + "crds": { + "type": "object", + "properties": { + "skipVersionCheck": { + "type": "boolean" + } + }, + "additionalProperties": false + }, + "imagePullSecrets": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "name" + ], + "additionalProperties": true + } + }, + "operator": { + "type": "object", + "properties": { + "replicaCount": { + "type": "integer", + "minimum": 0 + }, + "updateStrategy": { + "$ref": "#/definitions/UpdateStrategy" + }, + "image": { + "$ref": "#/definitions/Image" + }, + "podAnnotations": { + "$ref": "#/definitions/StringMap" + }, + "podLabels": { + "$ref": "#/definitions/StringMap" + }, + "serviceAnnotations": { + "$ref": "#/definitions/StringMap" + }, + "serviceLabels": { + "$ref": "#/definitions/StringMap" + }, + "resources": { + "type": "object" + }, + "nodeSelector": { + "$ref": "#/definitions/StringMap" + }, + "tolerations": { + "type": "array", + "items": { + "type": "object" + } + }, + "affinity": { + "type": "object" + }, + "config": { + "type": "object", + "properties": { + "logLevel": { + "type": "string", + "enum": [ + "debug", + "info", + "warn", + "error" + ] + } + }, + "required": [ + "logLevel" + ], + "additionalProperties": false + }, + "serviceAccount": { + "type": "object", + "properties": { + "create": { + "type": "boolean" + }, + "annotations": { + "$ref": "#/definitions/StringMap" + }, + "name": { + "type": "string" + } + }, + "required": [ + "annotations", + "create", + "name" + ], + "additionalProperties": false + }, + "watchNamespaces": { + "type": "array", + "items": { + "type": "string" + } + }, + "watchNamespaceSelector": { + "$ref": "#/definitions/LabelSelector" + }, + "rbac": { + "type": "object", + "properties": { + "create": { + "type": "boolean" + } + }, + "required": [ + "create" + ], + "additionalProperties": false + }, + "serviceMonitor": { + "type": "object", + "properties": { + "create": { + "type": "boolean" + }, + "labels": { + "$ref": "#/definitions/StringMap" + } + }, + "required": [ + "create" + ], + "additionalProperties": false + }, + "gatewayAPI": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "controllerName" : { + "type": "string", + "pattern": "^microgateway\\.airlock\\.com\/[A-Za-z0-9\/\\-._~%!$&'()*+,;=:]+$" + } + }, + "required": [ + "enabled" + ], + "additionalProperties": false + } + }, + "oneOf": [ + { + "properties": { + "watchNamespaces": { + "minItems": 1 + }, + "watchNamespaceSelector": { + "additionalProperties": false + } + } + }, + { + "properties": { + "watchNamespaces": { + "maxItems": 0 + }, + "watchNamespaceSelector": { + "$ref": "#/definitions/LabelSelector" + } + } + } + ], + "required": [ + "affinity", + "config", + "image", + "updateStrategy", + "nodeSelector", + "podAnnotations", + "podLabels", + "rbac", + "replicaCount", + "resources", + "serviceAccount", + "serviceAnnotations", + "serviceLabels", + "serviceMonitor", + "tolerations" + ], + "additionalProperties": false + }, + "engine": { + "type": "object", + "properties": { + "image": { + "$ref": "#/definitions/Image" + }, + "resources": { + "type": "object" + }, + "sidecar": { + "type": "object", + "properties":{ + "podMonitor": { + "type": "object", + "properties": { + "create": { + "type": "boolean" + }, + "labels": { + "$ref": "#/definitions/StringMap" + } + }, + "required": [ + "create" + ], + "additionalProperties": false + } + }, + "required": [ + "podMonitor" + ], + "additionalProperties": false + } + }, + "required": [ + "image", + "resources", + "sidecar" + ], + "additionalProperties": false + }, + "networkValidator": { + "type": "object", + "properties": { + "image": { + "$ref": "#/definitions/Image" + }, + "resources": { + "type": "object" + } + }, + "required": [ + "image", + "resources" + ], + "additionalProperties": false + }, + "sessionAgent": { + "type": "object", + "properties": { + "image": { + "$ref": "#/definitions/Image" + }, + "resources": { + "type": "object" + } + }, + "required": [ + "image", + "resources" + ], + "additionalProperties": false + }, + "license": { + "type": "object", + "properties": { + "secretName": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "secretName" + ], + "additionalProperties": false + }, + "dashboards": { + "type": "object", + "properties" : { + "create": { + "type": "boolean" + }, + "config": { + "type": "object", + "properties": { + "grafana": { + "type": "object", + "properties": { + "folderAnnotation": { + "$ref": "#/definitions/NameValuePair" + }, + "dashboardLabel": { + "$ref": "#/definitions/NameValuePair" + } + }, + "required": [ + "folderAnnotation", + "dashboardLabel" + ], + "additionalProperties": false + } + }, + "required": [ + "grafana" + ], + "additionalProperties": false + }, + "instances": { + "type": "object", + "properties": { + "overview": { + "$ref": "#/definitions/DashboardInstance" + }, + "license" : { + "$ref": "#/definitions/DashboardInstance" + }, + "blockMetrics" : { + "$ref": "#/definitions/DashboardInstance" + }, + "blockLogs" : { + "$ref": "#/definitions/DashboardInstance" + }, + "headerLogs" : { + "$ref": "#/definitions/DashboardInstance" + }, + "logOnlyMetrics" : { + "$ref": "#/definitions/DashboardInstance" + }, + "logOnlyLogs" : { + "$ref": "#/definitions/DashboardInstance" + } + }, + "required": [ + "overview", + "license", + "blockMetrics", + "blockLogs", + "headerLogs", + "logOnlyMetrics", + "logOnlyLogs" + ], + "additionalProperties": false + } + }, + "required": [ + "create", + "config", + "instances" + ], + "additionalProperties": false + }, + "tests": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + } + }, + "required": [ + "enabled" + ], + "additionalProperties": false + }, + "global": { + "type": "object" + } + }, + "required": [ + "commonAnnotations", + "commonLabels", + "crds", + "engine", + "fullnameOverride", + "imagePullSecrets", + "license", + "nameOverride", + "operator", + "networkValidator", + "sessionAgent", + "dashboards", + "tests" + ], + "additionalProperties": false, + "definitions": { + "StringMap": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "Image": { + "type": "object", + "properties": { + "repository": { + "type": "string", + "minLength": 1 + }, + "tag": { + "type": "string" + }, + "digest": { + "type": "string", + "pattern": "^$|^sha256:[a-f0-9]{64}$" + }, + "pullPolicy": { + "type": "string", + "enum": [ + "Always", + "IfNotPresent", + "Never" + ] + } + }, + "required": [ + "digest", + "pullPolicy", + "repository", + "tag" + ], + "additionalProperties": false + }, + "LabelSelector": { + "type": "object", + "properties": { + "matchExpressions": { + "type": "array", + "items": { + "type": "object", + "required": [ + "key", + "operator" + ], + "properties": { + "key": { + "type": "string" + }, + "operator": { + "type": "string" + }, + "values": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "additionalProperties": false + } + }, + "matchLabels": { + "$ref": "#/definitions/StringMap" + } + }, + "additionalProperties": false + }, + "UpdateStrategy": { + "type": "object", + "oneOf" : [ + { + "properties": { + "type": { + "$ref": "#/definitions/RecreateType" + } + }, + "required": [ + "type" + ], + "additionalProperties": false + }, + { + "properties": { + "type": { + "$ref": "#/definitions/RollingUpdateType" + }, + "rollingUpdate": { + "$ref": "#/definitions/RollingUpdate" + } + }, + "required": [ + "type" + ], + "additionalProperties": false + } + ] + }, + "RecreateType": { + "type": "string", + "enum": [ + "Recreate" + ] + }, + "RollingUpdateType": { + "type": "string", + "enum": [ + "RollingUpdate" + ] + }, + "RollingUpdate": { + "type": "object", + "properties": { + "maxSurge": { + "type": ["integer", "string"], + "minimum": 0, + "pattern": "^\\d+%?$" + }, + "maxUnavailable": { + "type": ["integer", "string"], + "minimum": 0, + "pattern": "^\\d+%?$" + } + }, + "anyOf": [ + {"required": ["maxSurge"]}, + {"required": ["maxUnavailable"]} + ], + "additionalProperties": false + }, + "DashboardInstance" : { + "type" : "object", + "properties" : { + "create" : { + "type" : "boolean" + } + }, + "required" : [ + "create" + ], + "additionalProperties": false + }, + "NameValuePair" : { + "type" : "object", + "properties" : { + "name" : { + "type": "string", + "minLength": 1 + }, + "value" : { + "type" : "string", + "minLength": 1 + } + }, + "required" : [ + "name", + "value" + ], + "additionalProperties": false + } + } +} diff --git a/charts/airlock/microgateway/4.4.3/values.yaml b/charts/airlock/microgateway/4.4.3/values.yaml new file mode 100644 index 0000000000..b59a41f9c0 --- /dev/null +++ b/charts/airlock/microgateway/4.4.3/values.yaml @@ -0,0 +1,237 @@ +# -- Allows overriding the name to use instead of "microgateway". +nameOverride: "" +# -- Allows overriding the name to use as full name of resources. +fullnameOverride: "" +# -- Labels to add to all resources. +commonLabels: {} +# -- Annotations to add to all resources. +commonAnnotations: {} +# -- ImagePullSecrets to use when pulling images. +imagePullSecrets: [] +# - name: myRegistryKeySecretName + +crds: + # -- Whether to skip the sanity check which prevents installing/upgrading the helm chart in a cluster with outdated Airlock Microgateway CRDs. + # The check aims to prevent unexpected behavior and issues due to Helm v3 not automatically upgrading CRDs which are already present in the cluster + # when performing a "helm install/upgrade". + skipVersionCheck: false +operator: + # -- Number of replicas for the operator Deployment. + replicaCount: 2 + # -- Specifies the operator update strategy. + updateStrategy: + type: RollingUpdate + # Specifies the Airlock Microgateway Operator image. + image: + # -- Image repository from which to pull the Airlock Microgateway Operator image. + repository: "quay.io/airlock/microgateway-operator" + # -- Image tag to pull. + tag: "4.4.3" + # -- SHA256 image digest to pull (in the format "sha256:c79ee3f85862fb386e9dd62b901b607161d27807f512d7fbdece05e9ee3d7c63"). + # Overrides tag when specified. + digest: "sha256:0cf190def105f5721bcf234c216861b5150c74bbac480497721012ea9574cc7c" + # -- Pull policy for this image. + pullPolicy: IfNotPresent + # -- Annotations to add to all Pods. + podAnnotations: {} + # -- Labels to add to all Pods. + podLabels: {} + # -- Annotations to add to the Service. + serviceAnnotations: {} + # prometheus.io/scrape: "true" + # prometheus.io/port: "8080" + + # -- Labels to add to the Service. + serviceLabels: {} + # -- Resource restrictions to apply to the operator container. + resources: {} + # We recommend at least the following resource specification. + # limits: + # cpu: 1000m + # memory: 512Mi + # requests: + # cpu: 100m + # memory: 512Mi + + # -- Custom nodeSelector to apply to the operator Deployment in order to constrain its Pods to certain nodes. + nodeSelector: {} + # -- Custom tolerations to apply to the operator Deployment in order to allow its Pods to run on tainted nodes. + tolerations: [] + # -- Custom affinity to apply to the operator Deployment. Used to influence the scheduling. + affinity: {} + # Parameters for the operator configuration. + config: + # -- Operator application log level. + logLevel: "info" + # Configures the generation of the ServiceAccount. + serviceAccount: + # -- Whether a ServiceAccount should be created. + create: true + # -- Annotations to add to the ServiceAccount. + annotations: {} + # -- Name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template. + name: "" + # -- Allows to restrict the operator to specific namespaces, depending on your needs. + # For a `OwnNamespace` or `SingleNamespace` installation the list may only contain one namespace (e.g., `watchNamespaces: ["airlock-microgateway-system"]`). + # In case of the `OwnNamespace` installation mode the specified namespace should be equal to the installation namespace. + # For a static `MultiNamespace` installation, the complete list of namespaces must be provided in the `watchNamespaces`. + # An `AllNamespaces` installation or the usage of the `watchNamespaceSelector` requires the `watchNamespaces` to be empty. + # Regardless of the installation modes supported by `watchNamespaces`, RBAC is created only namespace-scoped (using Roles and RoleBindings) in the respective namespaces. + # Please note that this feature requires a Premium license. + watchNamespaces: [] + # -- Allows to dynamically select watch namespaces of the operator and the scope of the webhooks based on a Namespace label selector. + # It is able to detect and reconcile resources in all namespaces that match the label selector automatically, even for new namespaces, without restarting the operator. + # This facilitates a dynamic `MultiNamespace` installation mode, but still requires cluster-scoped permissions (i.e., ClusterRoles and ClusterRoleBindings). + # An `AllNamespaces` installation or the usage of the `watchNamespaces` requires the `watchNamespaceSelector` to be empty. + # Please note that this feature requires a Premium license. + watchNamespaceSelector: {} + # For further examples, see: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#resources-that-support-set-based-requirements. + # matchLabels: + # microgateway.airlock.com/enable: "true" + # matchExpressions: + # - { key: environment, operator: NotIn, values: [dev] } + + # Configures the generation of Role and RoleBinding as well as ClusterRoles and ClusterRoleBinding pairs for the ServiceAccount specified above. + rbac: + # -- Whether to create RBAC resources which are required for the Airlock Microgateway Operator to function. + create: true + # Configures the generation of a Prometheus Operator ServiceMonitor. + serviceMonitor: + # -- Whether to create a ServiceMonitor resource for monitoring. + create: false + # -- Labels to add to the ServiceMonitor. + labels: {} + # release: "" + # Configures the Kubernetes Gateway API integration. + gatewayAPI: + # -- Whether to enable the Kubernetes Gateway API related controllers. + # Requires that the gateway.networking.k8s.io/v1 resources are installed on the cluster. + enabled: false + # -- Controller name referred in the GatewayClasses managed by this operator. The value must be a path prefixed by the domain `microgateway.airlock.com`. + controllerName: microgateway.airlock.com/gatewayclass-controller +engine: + # Specifies the Airlock Microgateway Engine image. + image: + # -- Image repository from which to pull the Airlock Microgateway Engine image. + repository: "quay.io/airlock/microgateway-engine" + # -- Image tag to pull. + tag: "4.4.3" + # -- SHA256 image digest to pull (in the format "sha256:a3051f42d3013813b05f7513bb86ed6a3209cb3003f1bb2f7b72df249aa544d3"). + # Overrides tag when specified. + digest: "sha256:bf2d56de0f0acc1563db6c80f17e71ec5aee489e4287f328fa7529f553a07cc1" + # -- Pull policy for this image. + pullPolicy: IfNotPresent + # -- Resource restrictions to apply to the Airlock Microgateway Engine container. + resources: {} + # We recommend at least the following resource specification. + # limits: + # cpu: 500m + # memory: 128Mi + # requests: + # cpu: 10m + # memory: 40Mi + + # Additional configuration when deployed as a sidecar. + sidecar: + # Configures the generation of a Prometheus Operator PodMonitor. + podMonitor: + # -- Whether to create a PodMonitor resource for monitoring. + create: false + # -- Labels to add to the PodMonitor. + labels: {} + # release: "" +networkValidator: + # Specifies the Airlock Microgateway Network Validator image to be injected as an init-container. + image: + # -- Image repository from which to pull the netcat image for the Airlock Microgateway Network Validator init-container. + repository: "cgr.dev/chainguard/netcat" + # -- Image tag to pull. + tag: "" + # -- SHA256 image digest to pull (in the format "sha256:7ef657ce316ce9d86f90c1dc99702d1190877c6ac2e923e696dc82c30050a14c"). + # Overrides tag when specified. + digest: "sha256:7ef657ce316ce9d86f90c1dc99702d1190877c6ac2e923e696dc82c30050a14c" + # -- Pull policy for this image. + pullPolicy: IfNotPresent + # -- Resource restrictions to apply to the Airlock Microgateway Network Validator init-container. + resources: + limits: + cpu: 25m + memory: 12Mi + requests: + cpu: 5m + memory: 1Mi +sessionAgent: + # Specifies the Airlock Microgateway Session Agent image. + image: + # -- Image repository from which to pull the Airlock Microgateway Session Agent image. + repository: "quay.io/airlock/microgateway-session-agent" + # -- Image tag to pull. + tag: "4.4.3" + # -- SHA256 image digest to pull (in the format "sha256:a3051f42d3013813b05f7513bb86ed6a3209cb3003f1bb2f7b72df249aa544d3"). + # Overrides tag when specified. + digest: "sha256:d1be15a6b516b1bd1c105d01f915e5e5ba5dc6c8c5870226233f542d3e6ac5f8" + # -- Pull policy for this image. + pullPolicy: IfNotPresent + # -- Resource restrictions to apply to the Airlock Microgateway Session Agent container. + resources: {} + # We recommend at least the following resource specification. + # limits: + # cpu: 150m + # memory: 32Mi + # requests: + # cpu: 10m + # memory: 8Mi +license: + # -- Name of the secret containing the "microgateway-license.txt" key. + secretName: "airlock-microgateway-license" +# Creates dashboards in the form of ConfigMaps that can be imported +# by Grafana using its sidecar setup. +dashboards: + # -- Whether to create any ConfigMaps containing Grafana dashboards to import. + create: false + config: + # Configures the necessary label and annotations along with their values + # to enable Grafana to correctly identify the ConfigMaps containing + # dashboards and file them within a dedicated folder in the dashboard overview. + # These settings need to match the Grafana sidecar configuration. + grafana: + folderAnnotation: + # -- Name of the annotation containing the folder name to file dashboards into. + name: "grafana_folder" + # -- Name of the folder dashboards are filed into within the Grafana UI. + value: "Airlock Microgateway" + dashboardLabel: + # -- Name of the label that lets Grafana identify ConfigMaps that represent dashboards. + name: "grafana_dashboard" + # -- Value of the label that lets Grafana identify ConfigMaps that represent dashboards. + value: "1" + instances: + # Available dashboard instances that can be individually created/deployed. + overview: + # -- Whether to create the overview dashboard. + create: true + license: + # -- Whether to create the license dashboard. + create: true + blockMetrics: + # -- Whether to create the block metrics dashboard. + create: true + blockLogs: + # -- Whether to create the block logs dashboard. + create: true + headerLogs: + # -- Whether to create the header rewrite logs dashboard. + create: true + logOnlyMetrics: + # -- Whether to create the log only metrics dashboard + create: true + logOnlyLogs: + # -- Whether to create the log only logs dashboard. + create: true +# Check whether the installation of the Airlock Microgateway Helm Chart was successful. +# Requires a secret with a valid Airlock Microgateway license key already to be present. +tests: + # -- Whether additional resources required for running `helm test` should be created (e.g. Roles and ServiceAccounts). + # If set to false, `helm test` will not run any tests. + enabled: false diff --git a/charts/kubecost/cost-analyzer/2.5.3/Chart.yaml b/charts/kubecost/cost-analyzer/2.5.3/Chart.yaml new file mode 100644 index 0000000000..2b3a357875 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/Chart.yaml @@ -0,0 +1,13 @@ +annotations: + artifacthub.io/links: | + - name: Homepage + url: https://www.kubecost.com + catalog.cattle.io/certified: partner + catalog.cattle.io/display-name: Kubecost + catalog.cattle.io/release-name: cost-analyzer +apiVersion: v2 +appVersion: 2.5.3 +description: Kubecost Helm chart - monitor your cloud costs! +icon: file://assets/icons/cost-analyzer.png +name: cost-analyzer +version: 2.5.3 diff --git a/charts/kubecost/cost-analyzer/2.5.3/README.md b/charts/kubecost/cost-analyzer/2.5.3/README.md new file mode 100644 index 0000000000..66ae92c921 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/README.md @@ -0,0 +1,110 @@ +# Kubecost Helm chart + +This is the official Helm chart for [Kubecost](https://www.kubecost.com/), an enterprise-grade application to monitor and manage Kubernetes spend. Please see the [website](https://www.kubecost.com/) for more details on what Kubecost can do for you and the official documentation [here](https://docs.kubecost.com/), or contact [team@kubecost.com](mailto:team@kubecost.com) for assistance. + +To install via Helm, run the following command. + +```sh +helm upgrade --install kubecost -n kubecost --create-namespace \ + --repo https://kubecost.github.io/cost-analyzer/ cost-analyzer \ + --set kubecostToken="aGVsbUBrdWJlY29zdC5jb20=xm343yadf98" +``` + +Alternatively, add the Helm repository first and scan for updates. + +```sh +helm repo add kubecost https://kubecost.github.io/cost-analyzer/ +helm repo update +``` + +Next, install the chart. + +```sh +helm install kubecost kubecost/cost-analyzer -n kubecost --create-namespace \ + --set kubecostToken="aGVsbUBrdWJlY29zdC5jb20=xm343yadf98" +``` + +While Helm is the [recommended install path](http://kubecost.com/install) for Kubecost especially in production, Kubecost can alternatively be deployed with a single-file manifest using the following command. Keep in mind when choosing this method, Kubecost will be installed from a development branch and may include unreleased changes. + +```sh +kubectl apply -f https://raw.githubusercontent.com/kubecost/cost-analyzer-helm-chart/develop/kubecost.yaml +``` + +The following table lists commonly used configuration parameters for the Kubecost Helm chart and their default values. Please see the [values file](values.yaml) for the complete set of definable values. + +| Parameter | Description | Default | +|------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------| +| `global.prometheus.enabled` | If false, use an existing Prometheus install. [More info](http://docs.kubecost.com/custom-prom). | `true` | +| `prometheus.server.persistentVolume.enabled` | If true, Prometheus server will create a Persistent Volume Claim. | `true` | +| `prometheus.server.persistentVolume.size` | Prometheus server data Persistent Volume size. Default set to retain ~6000 samples per second for 15 days. | `32Gi` | +| `prometheus.server.retention` | Determines when to remove old data. | `15d` | +| `prometheus.server.resources` | Prometheus server resource requests and limits. | `{}` | +| `prometheus.nodeExporter.resources` | Node exporter resource requests and limits. | `{}` | +| `prometheus.nodeExporter.enabled` `prometheus.serviceAccounts.nodeExporter.create` | If false, do not crate NodeExporter daemonset. | `true` | +| `prometheus.alertmanager.persistentVolume.enabled` | If true, Alertmanager will create a Persistent Volume Claim. | `true` | +| `prometheus.pushgateway.persistentVolume.enabled` | If true, Prometheus Pushgateway will create a Persistent Volume Claim. | `true` | +| `persistentVolume.enabled` | If true, Kubecost will create a Persistent Volume Claim for product config data. | `true` | +| `persistentVolume.size` | Define PVC size for cost-analyzer | `32.0Gi` | +| `persistentVolume.dbSize` | Define PVC size for cost-analyzer's flat file database | `32.0Gi` | +| `ingress.enabled` | If true, Ingress will be created | `false` | +| `ingress.annotations` | Ingress annotations | `{}` | +| `ingress.className` | Ingress class name | `{}` | +| `ingress.paths` | Ingress paths | `["/"]` | +| `ingress.hosts` | Ingress hostnames | `[cost-analyzer.local]` | +| `ingress.tls` | Ingress TLS configuration (YAML) | `[]` | +| `networkCosts.enabled` | If true, collect network allocation metrics [More info](http://docs.kubecost.com/network-allocation) | `false` | +| `networkCosts.podMonitor.enabled` | If true, a [PodMonitor](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#podmonitor) for the network-cost daemonset is created | `false` | +| `serviceMonitor.enabled` | Set this to `true` to create ServiceMonitor for Prometheus operator | `false` | +| `serviceMonitor.additionalLabels` | Additional labels that can be used so ServiceMonitor will be discovered by Prometheus | `{}` | +| `serviceMonitor.relabelings` | Sets Prometheus metric_relabel_configs on the scrape job | `[]` | +| `serviceMonitor.metricRelabelings` | Sets Prometheus relabel_configs on the scrape job | `[]` | +| `prometheusRule.enabled` | Set this to `true` to create PrometheusRule for Prometheus operator | `false` | +| `prometheusRule.additionalLabels` | Additional labels that can be used so PrometheusRule will be discovered by Prometheus | `{}` | +| `grafana.resources` | Grafana resource requests and limits. | `{}` | +| `grafana.serviceAccount.create` | If true, create a Service Account for Grafana. | `true` | +| `grafana.serviceAccount.name` | Grafana Service Account name. | `{}` | +| `grafana.sidecar.datasources.defaultDatasourceEnabled` | Set this to `false` to disable creation of Prometheus datasource in Grafana | `true` | +| `serviceAccount.create` | Set this to `false` if you want to create the service account `kubecost-cost-analyzer` on your own | `true` | +| `tolerations` | node taints to tolerate | `[]` | +| `affinity` | pod affinity | `{}` | +| `kubecostProductConfigs.productKey.mountPath` | Use instead of `kubecostProductConfigs.productKey.secretname` to declare the path at which the product key file is mounted (eg. by a secrets provisioner) | `N/A` | +| `kubecostFrontend.api.fqdn` | Customize the upstream api FQDN | `computed in terms of the service name and namespace` | +| `kubecostFrontend.model.fqdn` | Customize the upstream model FQDN | `computed in terms of the service name and namespace` | +| `clusterController.fqdn` | Customize the upstream cluster controller FQDN | `computed in terms of the service name and namespace` | +| `global.grafana.fqdn` | Customize the upstream grafana FQDN | `computed in terms of the release name and namespace` | + +## Adjusting Log Output + +The log output can be customized during deployment by using the `LOG_LEVEL` and/or `LOG_FORMAT` environment variables. + +### Adjusting Log Level + +Adjusting the log level increases or decreases the level of verbosity written to the logs. To set the log level to `trace`, the following flag can be added to the `helm` command. + +```sh +--set 'kubecostModel.extraEnv[0].name=LOG_LEVEL,kubecostModel.extraEnv[0].value=trace' +``` + +### Adjusting Log Format + +Adjusting the log format changes the format in which the logs are output making it easier for log aggregators to parse and display logged messages. The `LOG_FORMAT` environment variable accepts the values `JSON`, for a structured output, and `pretty` for a nice, human-readable output. + +| Value | Output | +|--------|----------------------------------------------------------------------------------------------------------------------------| +| `JSON` | `{"level":"info","time":"2006-01-02T15:04:05.999999999Z07:00","message":"Starting cost-model (git commit \"1.91.0-rc.0\")"}` | +| `pretty` | `2006-01-02T15:04:05.999999999Z07:00 INF Starting cost-model (git commit "1.91.0-rc.0")` | + +## Testing +To perform local testing do next: +- install locally [kind](https://github.com/kubernetes-sigs/kind) according to documentation. +- install locally [ct](https://github.com/helm/chart-testing) according to documentation. +- create local cluster using `kind` \ +use image version from https://github.com/kubernetes-sigs/kind/releases e.g. `kindest/node:v1.25.11@sha256:227fa11ce74ea76a0474eeefb84cb75d8dad1b08638371ecf0e86259b35be0c8` +```shell +kind create cluster --image kindest/node:v1.25.11@sha256:227fa11ce74ea76a0474eeefb84cb75d8dad1b08638371ecf0e86259b35be0c8 +``` +- perform ct execution +```shell +ct install --chart-dirs="." --charts="." +``` + diff --git a/charts/kubecost/cost-analyzer/2.5.3/app-readme.md b/charts/kubecost/cost-analyzer/2.5.3/app-readme.md new file mode 100644 index 0000000000..90fd50607a --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/app-readme.md @@ -0,0 +1,25 @@ +# Kubecost + +[Kubecost](https://kubecost.com/) is an open-source Kubernetes cost monitoring solution. + +Kubecost gives teams visibility into current and historical Kubernetes spend and resource allocation. These models provide cost transparency in Kubernetes environments that support multiple applications, teams, departments, etc. + +To see more on the functionality of the full Kubecost product, please visit the [features page](https://kubecost.com/#features) on our website. + +Here is a summary of features enabled by this cost model: + +- Real-time cost allocation by Kubernetes service, deployment, namespace, label, statefulset, daemonset, pod, and container +- Dynamic asset pricing enabled by integrations with AWS, Azure, and GCP billing APIs +- Supports on-prem k8s clusters with custom pricing sheets +- Allocation for in-cluster resources like CPU, GPU, memory, and persistent volumes. +- Allocation for AWS & GCP out-of-cluster resources like RDS instances and S3 buckets with key (optional) +- Easily export pricing data to Prometheus with /metrics endpoint ([learn more](https://github.com/kubecost/cost-model/blob/develop/PROMETHEUS.md)) +- Free and open source distribution (Apache2 license) + +## Requirements + +- Kubernetes 1.8+ +- kube-state-metrics +- Grafana +- Prometheus +- Node Exporter diff --git a/charts/kubecost/cost-analyzer/2.5.3/ci/aggregator-values.yaml b/charts/kubecost/cost-analyzer/2.5.3/ci/aggregator-values.yaml new file mode 100644 index 0000000000..523b9e81b5 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/ci/aggregator-values.yaml @@ -0,0 +1,17 @@ +kubecostAggregator: + enabled: true + cloudCost: + enabled: true + aggregatorDbStorage: + storageRequest: 10Gi +kubecostModel: + federatedStorageConfigSecret: federated-store +kubecostProductConfigs: + cloudIntegrationSecret: cloud-integration + clusterName: CLUSTER_NAME +prometheus: + server: + global: + external_labels: + # cluster_id should be unique for all clusters and the same value as .kubecostProductConfigs.clusterName + cluster_id: CLUSTER_NAME diff --git a/charts/kubecost/cost-analyzer/2.5.3/ci/federatedetl-primary-netcosts-values.yaml b/charts/kubecost/cost-analyzer/2.5.3/ci/federatedetl-primary-netcosts-values.yaml new file mode 100644 index 0000000000..78ad057258 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/ci/federatedetl-primary-netcosts-values.yaml @@ -0,0 +1,35 @@ +kubecostProductConfigs: + clusterName: CLUSTER_NAME + # cloudIntegrationSecret: cloud-integration +federatedETL: + useExistingS3Config: false + federatedCluster: true +kubecostModel: + containerStatsEnabled: true + federatedStorageConfigSecret: federated-store +serviceAccount: # this example uses AWS IRSA, which creates a service account with rights to the s3 bucket. If using keys+secrets in the federated-store, set create: true + create: true +global: + prometheus: + enabled: true + # fqdn: http://prometheus-operated.monitoring:9090 + grafana: # prometheus metrics will be local cluster only, disable grafana to save resources + enabled: false + proxy: false +prometheus: + nodeExporter: + enabled: false + server: + global: + external_labels: + # cluster_id should be unique for all clusters and the same value as .kubecostProductConfigs.clusterName + cluster_id: CLUSTER_NAME +networkCosts: + # optional, see: https://docs.kubecost.com/install-and-configure/advanced-configuration/network-costs-configuration + enabled: true + config: + services: + # set the appropriate cloud provider to true + amazon-web-services: true + # google-cloud-services: true + # azure-cloud-services: true diff --git a/charts/kubecost/cost-analyzer/2.5.3/ci/statefulsets-cc.yaml b/charts/kubecost/cost-analyzer/2.5.3/ci/statefulsets-cc.yaml new file mode 100644 index 0000000000..626a0c2e5e --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/ci/statefulsets-cc.yaml @@ -0,0 +1,46 @@ +### This test is to verify that Kubecost aggregator is deployed as a StatefulSet, +### cluster controller is installed, and the various Prometheus components are installed. +global: + podAnnotations: + kubecost.io/test1: value1 + kubecost.io/test2: value2 + additionalLabels: + kubecosttest1: value1 + kubecosttest2: value2 + prometheus: + enabled: true + # fqdn: http://prometheus-operated.monitoring:9090 + grafana: # prometheus metrics will be local cluster only, disable grafana to save resources + enabled: false + proxy: false +kubecostProductConfigs: + clusterName: CLUSTER_NAME +kubecostAggregator: + deployMethod: statefulset +kubecostModel: + federatedStorageConfigSecret: federated-store +clusterController: + enabled: true + actionConfigs: + clusterTurndown: + - name: my-schedule2 + start: "2034-02-09T00:00:00Z" + end: "2034-02-09T01:00:00Z" + repeat: none +prometheus: + nodeExporter: + enabled: true + alertmanager: + enabled: true + configmapReload: + prometheus: + enabled: true + pushgateway: + enabled: true + server: + statefulSet: + enabled: true + global: + external_labels: + # cluster_id should be unique for all clusters and the same value as .kubecostProductConfigs.clusterName + cluster_id: CLUSTER_NAME \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/crds/cluster-turndown-crd.yaml b/charts/kubecost/cost-analyzer/2.5.3/crds/cluster-turndown-crd.yaml new file mode 100644 index 0000000000..8c87644cc9 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/crds/cluster-turndown-crd.yaml @@ -0,0 +1,78 @@ +# TurndownSchedule Custom Resource Definition for persistence +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: turndownschedules.kubecost.com +spec: + group: kubecost.com + names: + kind: TurndownSchedule + singular: turndownschedule + plural: turndownschedules + shortNames: + - td + - tds + scope: Cluster + versions: + - name: v1alpha1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + start: + type: string + format: date-time + end: + type: string + format: date-time + repeat: + type: string + enum: [none, daily, weekly] + status: + type: object + properties: + state: + type: string + lastUpdated: + format: date-time + type: string + current: + type: string + scaleDownId: + type: string + nextScaleDownTime: + format: date-time + type: string + scaleDownMetadata: + additionalProperties: + type: string + type: object + scaleUpID: + type: string + nextScaleUpTime: + format: date-time + type: string + scaleUpMetadata: + additionalProperties: + type: string + type: object + additionalPrinterColumns: + - name: State + type: string + description: The state of the turndownschedule + jsonPath: .status.state + - name: Next Turndown + type: string + description: The next turndown date-time + jsonPath: .status.nextScaleDownTime + - name: Next Turn Up + type: string + description: The next turn up date-time + jsonPath: .status.nextScaleUpTime diff --git a/charts/kubecost/cost-analyzer/2.5.3/custom-pricing.csv b/charts/kubecost/cost-analyzer/2.5.3/custom-pricing.csv new file mode 100644 index 0000000000..c3e6d23674 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/custom-pricing.csv @@ -0,0 +1,7 @@ +EndTimestamp,InstanceID,Region,AssetClass,InstanceIDField,InstanceType,MarketPriceHourly,Version +2028-01-06 23:34:45 UTC,,us-east-2,node,metadata.name,g4dn.xlarge,5.55, +2028-01-06 23:34:45 UTC,,,node,metadata.name,R730-type1,1.35, +2028-01-06 23:34:45 UTC,,,pv,metadata.name,standard,0.44, +2028-01-06 23:34:45 UTC,a100,,gpu,gpu.nvidia.com/class,,0.75, +2028-01-06 23:34:45 UTC,RTX3090,,gpu,nvidia.com/gpu_type,,0.65, +2028-01-06 23:34:45 UTC,i-01045ab6d13179700,,,spec.providerID,,1.2, diff --git a/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/README.md b/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/README.md new file mode 100644 index 0000000000..160316ab6c --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/README.md @@ -0,0 +1,45 @@ +# Kubecost Grafana Dashboards + +## Overview + +Kubecost, by default, is bundled with a Grafana instance that already contains the dashboards in this folder. + +The dashboards in this repo are imported into Kubecost, unless disabled with + + +The same dashboards have template versions in [grafana-templates/](grafana-templates/) for those wanting to load the dashboards into an existing Grafana instance. + +## Caveats + +The primary purpose of the dashboards provided is to allow visibility into the metrics used by Kubecost to create the cost-model. + +The networkCosts-metrics dashboard requires the optional networkCosts daemonset to be [enabled](https://docs.kubecost.com/install-and-configure/advanced-configuration/network-costs-configuration). + +## Metrics Required + +`kubecost-container-stats` metrics: + +``` +container_cpu_usage_seconds_total +kube_pod_container_resource_requests +container_memory_working_set_bytes +container_cpu_cfs_throttled_periods_total +container_cpu_cfs_periods_total +``` + +`network-transfer-data` metrics: + +``` +kubecost_pod_network_ingress_bytes_total +kubecost_pod_network_egress_bytes_total +``` + +`disk-usage` metrics: +``` +container_fs_limit_bytes +container_fs_usage_bytes +``` + +## Additional Information + +Kubecost Grafana [Configuration Guide](https://docs.kubecost.com/install-and-configure/advanced-configuration/custom-grafana) \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/attached-disks.json b/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/attached-disks.json new file mode 100644 index 0000000000..49c8d6c1a0 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/attached-disks.json @@ -0,0 +1,549 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 16, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(container_fs_limit_bytes{instance=~'$disk', device!=\"tmpfs\", id=\"/\", cluster_id=~'$cluster'}) by (cluster_id, instance)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{cluster_id}}/{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Disk Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(container_fs_usage_bytes{instance=~'$disk',id=\"/\", cluster_id=~'$cluster'}) by (cluster_id, instance) / sum(container_fs_limit_bytes{instance=~'$disk',device!=\"tmpfs\", id=\"/\", cluster_id=~'$cluster'}) by (cluster_id,instance)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{cluster_id}}-{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Disk Utilization", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "1 - sum(container_fs_inodes_free{instance=~'$disk',id=\"/\", cluster_id=~'$cluster'}) by (cluster_id, instance) / sum(container_fs_inodes_total{instance=~'$disk',id=\"/\", cluster_id=~'$cluster'}) by (cluster_id, instance)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{cluster_id}}/{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "iNode Utilization", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(container_fs_usage_bytes{instance=~'$disk',id=\"/\", cluster_id=~'$cluster'}) by (cluster_id, instance)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{cluster_id}}/{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Disk Usage", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "tags": [ + "kubecost", + "cost", + "utilization", + "metrics" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "PBFA97CFB590B2093" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(cluster_id)", + "hide": 0, + "includeAll": true, + "multi": false, + "name": "cluster", + "options": [], + "query": { + "query": "label_values(cluster_id)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "type": "query" + }, + { + "current": { + "selected": true, + "text": "ip-192-168-147-146.us-east-2.compute.internal", + "value": "ip-192-168-147-146.us-east-2.compute.internal" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(container_fs_limit_bytes{cluster_id=~\"$cluster\"}, instance)", + "hide": 0, + "includeAll": true, + "multi": false, + "name": "disk", + "options": [], + "query": { + "query": "label_values(container_fs_limit_bytes{cluster_id=~\"$cluster\"}, instance)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-7d", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Attached disk metrics", + "uid": "nBH7qBgMk", + "version": 7, + "weekStart": "" +} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/cluster-metrics.json b/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/cluster-metrics.json new file mode 100644 index 0000000000..2535560006 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/cluster-metrics.json @@ -0,0 +1,1683 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Cost metrics from the Kubecost product", + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 7, + "iteration": 1558062099204, + "links": [], + "panels": [ + { + "content": "Deprecated - It is not expected to match Kubecost UI/API.", + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 27, + "links": [], + "mode": "markdown", + "title": "", + "transparent": true, + "type": "text" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "${datasource}", + "decimals": 2, + "description": "Monthly run rate of CPU + GPU costs based on currently provisioned resources.", + "format": "currencyUSD", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 2 + }, + "hideTimeOverride": true, + "id": 2, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(\n avg(kube_node_status_capacity{resource=\"cpu\", unit=\"core\"}) by (node) * avg(node_cpu_hourly_cost) by (node) * 730 * (1-$useDiscount/100) +\n avg(node_gpu_hourly_cost) by (node) * 730 * (1-$useDiscount/100)\n)", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": " {{ node }}", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": "15m", + "timeShift": null, + "title": "CPU Cost", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "${datasource}", + "decimals": 2, + "description": "Monthly run rate of memory costs based on currently provisioned expenses.", + "format": "currencyUSD", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 2 + }, + "hideTimeOverride": true, + "id": 3, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(\n avg(kube_node_status_capacity{resource=\"memory\", unit=\"byte\"}) by (node) / 1024 / 1024 / 1024 * avg(node_ram_hourly_cost) by (node) * 730 * (1-$useDiscount/100)\n)", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": " {{ node }}", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": "15m", + "timeShift": null, + "title": "Memory Cost", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "${datasource}", + "decimals": 2, + "description": "Monthly run rate of attached storage and PV costs based on currently provisioned resources.", + "format": "currencyUSD", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 12, + "y": 2 + }, + "hideTimeOverride": true, + "id": 4, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(avg(pv_hourly_cost) by (persistentvolume) * 730 * avg(kube_persistentvolume_capacity_bytes) by (persistentvolume) / 1024 / 1024 / 1024) \n+\nsum(sum(container_fs_limit_bytes{device!=\"tmpfs\", id=\"/\"}) by (instance) / 1024 / 1024 / 1024) * $localStorageGBCost", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": " {{ node }}", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": "15m", + "timeShift": null, + "title": "Storage Cost", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "${datasource}", + "decimals": 2, + "description": "Sum of compute, memory, storage and network costs.", + "format": "currencyUSD", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 2 + }, + "hideTimeOverride": true, + "id": 11, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "# Compute\nsum(\n avg(kube_node_status_capacity{resource=\"cpu\", unit=\"core\"}) by (node) * avg(node_cpu_hourly_cost) by (node) * 730 * (1-$useDiscount/100) +\n avg(node_gpu_hourly_cost) by (node) * 730 * (1-$useDiscount/100)\n) +\n\n\n# Memory\nsum(\n avg(kube_node_status_capacity{resource=\"memory\", unit=\"byte\"}) by (node) / 1024 / 1024 / 1024 * avg(node_ram_hourly_cost) by (node) * 730 * (1-$useDiscount/100)\n) +\n\n# Storage \n\nsum(avg(pv_hourly_cost) by (persistentvolume) * 730 * avg(kube_persistentvolume_capacity_bytes) by (persistentvolume) / 1024 / 1024 / 1024) \n+\nsum(sum(container_fs_limit_bytes{device!=\"tmpfs\", id=\"/\"}) by (instance) / 1024 / 1024 / 1024) * $localStorageGBCost", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": " {{ node }}", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": "15m", + "timeShift": null, + "title": "Total Cost", + "type": "singlestat", + "valueFontSize": "120%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(50, 172, 45, 0.97)", + "#c15c17" + ], + "datasource": "${datasource}", + "decimals": 2, + "description": "Current CPU use from applications divided by allocatable CPUs", + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 5 + }, + "height": "180px", + "hideTimeOverride": true, + "id": 13, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(\n sum(\n count(irate(container_cpu_usage_seconds_total{id=\"/\"}[10m])) by (instance)\n * on (instance) \n sum(irate(container_cpu_usage_seconds_total{id=\"/\"}[10m])) by (instance)\n ) \n / \n (sum (kube_node_status_allocatable{resource=\"cpu\", unit=\"core\"}))\n) * 100", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "30, 80", + "timeFrom": "", + "title": "CPU Utilization", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(50, 172, 45, 0.97)", + "#c15c17" + ], + "datasource": "${datasource}", + "decimals": 2, + "description": "Current CPU reservation requests from applications vs allocatable CPU", + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 5 + }, + "height": "180px", + "id": 15, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "SUM(kube_pod_container_resource_requests{resource=\"cpu\", unit=\"core\"}) / SUM(kube_node_status_allocatable{resource=\"cpu\", unit=\"core\"}) * 100", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "30, 80", + "title": "CPU Requests", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(50, 172, 45, 0.97)", + "#c15c17" + ], + "datasource": "${datasource}", + "description": "Current RAM use vs RAM available", + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 6, + "y": 5 + }, + "height": "180px", + "hideTimeOverride": true, + "id": 17, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "SUM(container_memory_usage_bytes{namespace!=\"\"}) / SUM(kube_node_status_allocatable{resource=\"memory\", unit=\"byte\"}) * 100", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "refId": "A", + "step": 10 + }, + { + "expr": "", + "format": "time_series", + "intervalFactor": 1, + "refId": "B" + } + ], + "thresholds": "30,80", + "timeFrom": "", + "title": "RAM Utilization", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(50, 172, 45, 0.97)", + "#c15c17" + ], + "datasource": "${datasource}", + "description": "Current RAM requests vs RAM available", + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 9, + "y": 5 + }, + "height": "180px", + "id": 19, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(\n sum(kube_pod_container_resource_requests{resource=\"memory\", unit=\"byte\", namespace!=\"\"})\n /\n sum(kube_node_status_allocatable{resource=\"memory\", unit=\"byte\"})\n) * 100", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "30,80", + "title": "RAM Requests", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(50, 172, 45, 0.97)", + "#c15c17" + ], + "datasource": "${datasource}", + "decimals": 2, + "description": "This gauge shows the current standard storage use, including cluster storage, vs storage available", + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 5 + }, + "height": "180px", + "hideTimeOverride": true, + "id": 21, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum (\n sum(kube_persistentvolumeclaim_info) by (persistentvolumeclaim, namespace)\n + on (persistentvolumeclaim, namespace)\n sum(pod_pvc_allocation) by (persistentvolumeclaim, namespace) or up * 0\n + sum(container_fs_usage_bytes{device=~\"^/dev/[sv]d[a-z][1-9]$\",id=\"/\"})\n) /\nsum (\n sum(kube_persistentvolumeclaim_info) by (persistentvolumeclaim, namespace)\n + on (persistentvolumeclaim, namespace)\n sum(kube_persistentvolumeclaim_resource_requests_storage_bytes) by (persistentvolumeclaim, namespace) or up * 0\n + sum(container_fs_limit_bytes{device=~\"^/dev/[sv]d[a-z][1-9]$\",id=\"/\"})\n) * 100", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "30, 80", + "timeFrom": "", + "title": "Storage Utilization", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "description": "Monthly run rate of CPU + GPU costs", + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 9 + }, + "id": 6, + "interval": "1m", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n avg(kube_node_status_capacity{resource=\"cpu\", unit=\"core\"}) by (node) * avg(node_cpu_hourly_cost) by (node) * 730 +\n avg(node_gpu_hourly_cost) by (node) * 730\n)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "compute cost", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Compute Cost", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "currencyUSD", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "description": "Monthly run rate of memory costs", + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 9 + }, + "id": 9, + "interval": "1m", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n avg(kube_node_status_capacity{resource=\"memory\", unit=\"byte\"}) by (node) / 1024 / 1024 / 1024 * avg(node_ram_hourly_cost) by (node) * 730\n)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "memory cost", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Cost", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "currencyUSD", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "description": "Monthly run rate of attached disk + PV storage costs", + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 9 + }, + "id": 10, + "interval": "1m", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n avg(avg_over_time(pv_hourly_cost[$timeRange] offset 1m)) by (persistentvolume) * 730 \n * avg(avg_over_time(kube_persistentvolume_capacity_bytes[$timeRange] offset 1m)) by (persistentvolume) / 1024 / 1024 / 1024\n) +\nsum(avg(container_fs_limit_bytes{device!=\"tmpfs\", id=\"/\"}) by (instance) / 1024 / 1024 / 1024) * $localStorageGBCost", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "storage cost", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Storage Cost", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "currencyUSD", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "description": "Sum of compute, memory, and storage costs", + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 9 + }, + "id": 22, + "interval": "1m", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "# Compute\nsum(\n avg(kube_node_status_capacity{resource=\"cpu\", unit=\"core\"}) by (node) * avg(node_cpu_hourly_cost) by (node) * 730 * (1-$useDiscount/100) +\n avg(node_gpu_hourly_cost) by (node) * 730 * (1-$useDiscount/100)\n) +\n\n\n# Memory\nsum(\n avg(kube_node_status_capacity{resource=\"memory\", unit=\"byte\"}) by (node) / 1024 / 1024 / 1024 * avg(node_ram_hourly_cost) by (node) * 730 * (1-$useDiscount/100)\n) +\n\n# Storage \n\nsum(avg(pv_hourly_cost) by (persistentvolume) * 730 * avg(kube_persistentvolume_capacity_bytes) by (persistentvolume) / 1024 / 1024 / 1024) \n+\nsum(sum(container_fs_limit_bytes{device!=\"tmpfs\", id=\"/\"}) by (instance) / 1024 / 1024 / 1024) * $localStorageGBCost", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "total cost", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Total Cost", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "currencyUSD", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "columns": [], + "datasource": "${datasource}", + "description": "Cost of by resource class of currently provisioned nodes", + "fontSize": "100%", + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 8, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 4, + "desc": false + }, + "styles": [ + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Time", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Compute Cost", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Cost", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value #A", + "thresholds": [], + "type": "number", + "unit": "currencyUSD" + }, + { + "alias": "Mem Cost", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value #B", + "thresholds": [], + "type": "number", + "unit": "currencyUSD" + }, + { + "alias": "Total", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value #C", + "thresholds": [], + "type": "number", + "unit": "currencyUSD" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "instance", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "GPU", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value #D", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "avg(kube_node_status_capacity{resource=\"cpu\", unit=\"core\"}) by (node) * avg(node_cpu_hourly_cost or up * 0) by (node) * 730 * (1-$useDiscount/100)", + "format": "table", + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + }, + { + "expr": "avg(kube_node_status_capacity{resource=\"memory\", unit=\"byte\"}) by (node) / 1024 / 1024 / 1024 * avg(node_ram_hourly_cost) by (node) * 730 * (1-$useDiscount/100)", + "format": "table", + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "refId": "B" + }, + { + "expr": "avg(node_gpu_hourly_cost) by (node) * 730 * (1-$useDiscount/100)", + "format": "table", + "instant": true, + "intervalFactor": 1, + "refId": "D" + }, + { + "expr": "# CPU \navg(kube_node_status_capacity{resource=\"cpu\", unit=\"core\"}) by (node) * avg(node_cpu_hourly_cost or up * 0) by (node) * 730 * (1-$useDiscount/100) +\n# GPU\navg(node_gpu_hourly_cost) by (node) * 730 * (1-$useDiscount/100) +\n# Memory\navg(kube_node_status_capacity{resource=\"memory\", unit=\"byte\"}) by (node) / 1024 / 1024 / 1024 * avg(node_ram_hourly_cost) by (node) * 730 * (1-$useDiscount/100)\n", + "format": "table", + "instant": true, + "intervalFactor": 1, + "refId": "C" + } + ], + "title": "Cost by node", + "transform": "table", + "type": "table" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "description": "Monthly run rate of attached disk + PV storage costs based on currently provisioned resources.", + "fill": 1, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 25, + "interval": "1m", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n avg(kube_node_status_capacity{resource=\"cpu\", unit=\"core\"}) by (node) * avg(node_cpu_hourly_cost) by (node) * 730 +\n avg(node_gpu_hourly_cost) by (node) * 730\n)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "cpu", + "refId": "B" + }, + { + "expr": "sum(\n avg(kube_node_status_capacity{resource=\"memory\", unit=\"byte\"}) by (node) / 1024 / 1024 / 1024 * avg(node_ram_hourly_cost) by (node) * 730\n)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "memory", + "refId": "A" + }, + { + "expr": "sum(\n avg(avg_over_time(pv_hourly_cost[$timeRange] offset 1m)) by (persistentvolume) * 730 \n * avg(avg_over_time(kube_persistentvolume_capacity_bytes[$timeRange] offset 1m)) by (persistentvolume) / 1024 / 1024 / 1024\n) +\nsum(avg(container_fs_limit_bytes{device!=\"tmpfs\", id=\"/\"}) by (instance) / 1024 / 1024 / 1024) * $localStorageGBCost", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "storage", + "refId": "C" + }, + { + "expr": "SUM(rate(node_network_transmit_bytes_total{device=\"eth0\"}[60m]) / 1024 / 1024 / 1024 ) * (60 * 60 * 24 * 30) * $percentEgress * $egressCost ", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "network", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Cost by Resource", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "currencyUSD", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 16, + "style": "dark", + "tags": [ + "kubecost", + "cost", + "utilization", + "metrics" + ], + "templating": { + "list": [ + { + "auto": true, + "auto_count": 1, + "auto_min": "1m", + "current": { + "text": "auto", + "value": "$__auto_interval_timeRange" + }, + "hide": 2, + "label": null, + "name": "timeRange", + "options": [ + { + "selected": true, + "text": "auto", + "value": "$__auto_interval_timeRange" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + }, + { + "selected": false, + "text": "90d", + "value": "90d" + } + ], + "query": "1h,6h,12h,1d,7d,14d,30d,90d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + }, + { + "current": { + "text": ".04", + "value": ".04" + }, + "hide": 2, + "label": "Cost per Gb hour for attached disks", + "name": "localStorageGBCost", + "options": [ + { + "selected": true, + "text": ".04", + "value": ".04" + } + ], + "query": ".04", + "skipUrlSync": false, + "type": "constant" + }, + { + "current": { + "tags": [], + "text": "0", + "value": "0" + }, + "hide": 0, + "label": "Sustained Use Discount %", + "name": "useDiscount", + "options": [ + { + "selected": true, + "text": "0", + "value": "0" + } + ], + "query": "0", + "skipUrlSync": false, + "type": "constant" + }, + { + "current": { + "text": ".1", + "value": ".1" + }, + "hide": 2, + "label": null, + "name": "percentEgress", + "options": [ + { + "selected": true, + "text": ".1", + "value": ".1" + } + ], + "query": ".1", + "skipUrlSync": false, + "type": "constant" + }, + { + "current": { + "text": ".12", + "value": ".12" + }, + "hide": 2, + "label": null, + "name": "egressCost", + "options": [ + { + "selected": true, + "text": ".12", + "value": ".12" + } + ], + "query": ".12", + "skipUrlSync": false, + "type": "constant" + }, + { + "current": { + "selected": true, + "text": "default-kubecost", + "value": "default-kubecost" + }, + "error": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-7d", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Deprecated - Kubecost cluster metrics", + "uid": "JOUdHGZZz", + "version": 20 +} diff --git a/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/cluster-utilization.json b/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/cluster-utilization.json new file mode 100644 index 0000000000..8a17f26c04 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/cluster-utilization.json @@ -0,0 +1,3196 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "A dashboard to help manage Kubernetes cluster costs and resources", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 6873, + "graphTooltip": 0, + "id": 10, + "iteration": 1645112913364, + "links": [], + "liveNow": false, + "panels": [ + { + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 86, + "links": [], + "options": { + "content": "Deprecated - It is not expected to match Kubecost UI/API. This dashboard shows monthly cost estimates for the cluster, based on **current** CPU, RAM and storage provisioned.", + "mode": "markdown" + }, + "pluginVersion": "8.3.2", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "currencyUSD" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 2 + }, + "hideTimeOverride": true, + "id": 75, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P0C970EB638C812D0" + }, + "exemplar": false, + "expr": "sum(\n (\n (\n sum(kube_node_status_capacity_cpu_cores) by (node)\n * on (node) group_left (label_cloud_google_com_gke_preemptible)\n avg(kube_node_labels{label_cloud_google_com_gke_preemptible=\"true\"}) by (node)\n ) * $costpcpu\n )\n or\n (\n (\n sum(kube_node_status_capacity{resource=\"cpu\", unit=\"core\"}) by (node)\n * on (node) group_left (label_cloud_google_com_gke_preemptible)\n avg(kube_node_labels{label_cloud_google_com_gke_preemptible!=\"true\"}) by (node)\n ) * ($costcpu - ($costcpu / 100 * $costDiscount))\n )\n) ", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": " {{ node }}", + "refId": "A" + } + ], + "timeFrom": "15m", + "title": "CPU Cost", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "currencyUSD" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 2 + }, + "hideTimeOverride": true, + "id": 77, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P0C970EB638C812D0" + }, + "exemplar": false, + "expr": "sum(\n (\n (\n sum(kube_node_status_capacity_memory_bytes) by (node)\n * on (node) group_left (label_cloud_google_com_gke_preemptible)\n avg(kube_node_labels{label_cloud_google_com_gke_preemptible=\"true\"}) by (node)\n ) /1024/1024/1024 * $costpram\n )\n or\n (\n (\n sum(kube_node_status_capacity{resource=\"memory\", unit=\"byte\"}) by (node)\n * on (node) group_left (label_cloud_google_com_gke_preemptible)\n avg(kube_node_labels{label_cloud_google_com_gke_preemptible!=\"true\"}) by (node)\n ) /1024/1024/1024 * ($costram - ($costram / 100 * $costDiscount))\n)\n) ", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": " {{ node }}", + "refId": "A" + } + ], + "timeFrom": "15m", + "title": "RAM Cost", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "currencyUSD" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 2 + }, + "hideTimeOverride": true, + "id": 78, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P0C970EB638C812D0" + }, + "exemplar": false, + "expr": "sum (\n sum(kube_persistentvolumeclaim_info{storageclass=~\".*ssd.*\"}) by (persistentvolumeclaim, namespace, storageclass)\n + on (persistentvolumeclaim, namespace) group_right(storageclass)\n sum(kube_persistentvolumeclaim_resource_requests_storage_bytes) by (persistentvolumeclaim, namespace) or up * 0\n) / 1024 / 1024 /1024 * $costStorageSSD\n\n+\n\nsum (\n sum(kube_persistentvolumeclaim_info{storageclass!~\".*ssd.*\"}) by (persistentvolumeclaim, namespace, storageclass)\n + on (persistentvolumeclaim, namespace) group_right(storageclass)\n sum(kube_persistentvolumeclaim_resource_requests_storage_bytes) by (persistentvolumeclaim, namespace) or up * 0\n) / 1024 / 1024 /1024 * $costStorageStandard\n\n+ \n\nsum(container_fs_limit_bytes{id=\"/\"}) / 1024 / 1024 / 1024 * 1.03 * $costStorageStandard", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": " {{ node }}", + "refId": "A" + } + ], + "timeFrom": "15m", + "title": "Storage Cost (Cluster and PVC)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Represents a near worst-case approximation of network costs.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "currencyUSD" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 2 + }, + "hideTimeOverride": true, + "id": 129, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P0C970EB638C812D0" + }, + "exemplar": false, + "expr": "SUM(rate(node_network_transmit_bytes_total{device=\"eth0\"}[60m]) / 1024 / 1024 / 1024 ) * (60 * 60 * 24 * 30) * $costEgress", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": " {{ node }}", + "refId": "A" + } + ], + "timeFrom": "15m", + "title": "Network Egress Cost", + "type": "stat" + }, + { + "datasource": { + "uid": "${datasource}" + }, + "description": "Current CPU use from applications divided by allocatable CPUs", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(245, 54, 54, 0.9)", + "value": null + }, + { + "color": "rgba(50, 172, 45, 0.97)", + "value": 30 + }, + { + "color": "#c15c17", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 6 + }, + "hideTimeOverride": true, + "id": 82, + "links": [], + "maxDataPoints": 100, + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "8.3.2", + "targets": [ + { + "expr": "(\n sum(\n count(irate(container_cpu_usage_seconds_total{id=\"/\"}[10m])) by (instance)\n * on (instance) \n sum(irate(container_cpu_usage_seconds_total{id=\"/\"}[10m])) by (instance)\n ) \n / \n (sum (kube_node_status_allocatable{resource=\"cpu\", unit=\"core\"}))\n) * 100", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "timeFrom": "", + "title": "CPU Utilization", + "type": "gauge" + }, + { + "datasource": { + "uid": "${datasource}" + }, + "description": "Current CPU reservation requests from applications vs allocatable CPU", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(245, 54, 54, 0.9)", + "value": null + }, + { + "color": "rgba(50, 172, 45, 0.97)", + "value": 30 + }, + { + "color": "#c15c17", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 6 + }, + "id": 91, + "links": [], + "maxDataPoints": 100, + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "8.3.2", + "targets": [ + { + "expr": "SUM(kube_pod_container_resource_requests{resource=\"cpu\", unit=\"core\"}) / SUM(kube_node_status_allocatable{resource=\"cpu\", unit=\"core\"}) * 100", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "title": "CPU Requests", + "type": "gauge" + }, + { + "datasource": { + "uid": "${datasource}" + }, + "description": "Current RAM use vs RAM available", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(245, 54, 54, 0.9)", + "value": null + }, + { + "color": "rgba(50, 172, 45, 0.97)", + "value": 30 + }, + { + "color": "#c15c17", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 6, + "y": 6 + }, + "hideTimeOverride": true, + "id": 80, + "links": [], + "maxDataPoints": 100, + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "8.3.2", + "targets": [ + { + "expr": "SUM(container_memory_working_set_bytes{name!=\"POD\", container!=\"\", namespace!=\"\"}) / SUM(kube_node_status_allocatable{resource=\"memory\", unit=\"byte\"}) * 100", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "refId": "A", + "step": 10 + }, + { + "expr": "", + "format": "time_series", + "intervalFactor": 1, + "refId": "B" + } + ], + "timeFrom": "", + "title": "RAM Utilization", + "type": "gauge" + }, + { + "datasource": { + "uid": "${datasource}" + }, + "description": "Current RAM requests vs RAM available", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(245, 54, 54, 0.9)", + "value": null + }, + { + "color": "rgba(50, 172, 45, 0.97)", + "value": 30 + }, + { + "color": "#c15c17", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 9, + "y": 6 + }, + "id": 92, + "links": [], + "maxDataPoints": 100, + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "8.3.2", + "targets": [ + { + "expr": "(\n sum(kube_pod_container_resource_requests{resource=\"memory\", unit=\"byte\", namespace!=\"\"})\n /\n sum(kube_node_status_allocatable{resource=\"memory\", unit=\"byte\"})\n) * 100", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "title": "RAM Requests", + "type": "gauge" + }, + { + "datasource": { + "uid": "${datasource}" + }, + "description": "This gauge shows the current standard storage use, including cluster storage, vs storage available", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(245, 54, 54, 0.9)", + "value": null + }, + { + "color": "rgba(50, 172, 45, 0.97)", + "value": 30 + }, + { + "color": "#c15c17", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 6 + }, + "hideTimeOverride": true, + "id": 95, + "links": [], + "maxDataPoints": 100, + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "8.3.2", + "targets": [ + { + "expr": "sum (\n sum(kube_persistentvolumeclaim_info{storageclass!~\".*ssd.*\"}) by (persistentvolumeclaim, namespace, storageclass)\n + on (persistentvolumeclaim, namespace) group_right(storageclass)\n sum(kubelet_volume_stats_used_bytes) by (persistentvolumeclaim, namespace) or up * 0\n + sum(container_fs_usage_bytes{device=~\"^/dev/[sv]d[a-z][1-9]$\",id=\"/\"})\n) /\nsum (\n sum(kube_persistentvolumeclaim_info{storageclass!~\".*ssd.*\"}) by (persistentvolumeclaim, namespace, storageclass)\n + on (persistentvolumeclaim, namespace) group_right(storageclass)\n sum(kube_persistentvolumeclaim_resource_requests_storage_bytes) by (persistentvolumeclaim, namespace) or up * 0\n + sum(container_fs_limit_bytes{device=~\"^/dev/[sv]d[a-z][1-9]$\",id=\"/\"})\n) * 100", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "timeFrom": "", + "title": "Storage Utilization", + "type": "gauge" + }, + { + "datasource": { + "uid": "${datasource}" + }, + "description": "This gauge shows the current SSD use vs SSD available", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(245, 54, 54, 0.9)", + "value": null + }, + { + "color": "rgba(50, 172, 45, 0.97)", + "value": 30 + }, + { + "color": "#c15c17", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 6 + }, + "hideTimeOverride": true, + "id": 96, + "links": [], + "maxDataPoints": 100, + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "8.3.2", + "targets": [ + { + "expr": "sum (\n sum(kube_persistentvolumeclaim_info{storageclass=~\".*ssd.*\"}) by (persistentvolumeclaim, namespace, storageclass)\n + on (persistentvolumeclaim, namespace) group_right(storageclass)\n sum(kubelet_volume_stats_used_bytes) by (persistentvolumeclaim, namespace)\n) /\nsum (\n sum(kube_persistentvolumeclaim_info{storageclass=~\".*ssd.*\"}) by (persistentvolumeclaim, namespace, storageclass)\n + on (persistentvolumeclaim, namespace) group_right(storageclass)\n sum(kube_persistentvolumeclaim_resource_requests_storage_bytes) by (persistentvolumeclaim, namespace)\n) * 100", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "timeFrom": "", + "title": "SSD Utilization", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Expected monthly cost given current CPU, memory storage, and network resource consumption", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "currencyUSD" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 6 + }, + "hideTimeOverride": true, + "id": 93, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P0C970EB638C812D0" + }, + "exemplar": false, + "expr": "# CPU\nsum(\n (\n (\n sum(kube_node_status_capacity_cpu_cores) by (node)\n * on (node) group_left (label_cloud_google_com_gke_preemptible)\n avg(kube_node_labels{label_cloud_google_com_gke_preemptible=\"true\"}) by (node)\n ) * $costpcpu\n )\n or\n (\n (\n sum(kube_node_status_capacity{resource=\"cpu\", unit=\"core\"}) by (node)\n * on (node) group_left (label_cloud_google_com_gke_preemptible)\n avg(kube_node_labels{label_cloud_google_com_gke_preemptible!=\"true\"}) by (node)\n ) * ($costcpu - ($costcpu / 100 * $costDiscount))\n )\n) \n\n+ \n\n# Storage\nsum (\n sum(kube_persistentvolumeclaim_info{storageclass=~\".*ssd.*\"}) by (persistentvolumeclaim, namespace, storageclass)\n + on (persistentvolumeclaim, namespace) group_right(storageclass)\n sum(kube_persistentvolumeclaim_resource_requests_storage_bytes) by (persistentvolumeclaim, namespace) or up * 0\n) / 1024 / 1024 /1024 * $costStorageSSD\n\n+\n\nsum (\n sum(kube_persistentvolumeclaim_info{storageclass!~\".*ssd.*\"}) by (persistentvolumeclaim, namespace, storageclass)\n + on (persistentvolumeclaim, namespace) group_right(storageclass)\n sum(kube_persistentvolumeclaim_resource_requests_storage_bytes) by (persistentvolumeclaim, namespace) or up * 0\n) / 1024 / 1024 /1024 * $costStorageStandard\n\n+ \n\nsum(container_fs_limit_bytes{id=\"/\"}) / 1024 / 1024 / 1024 * 1.03 * $costStorageStandard \n\n+\n\n# END STORAGE\n# RAM \nsum(\n (\n (\n sum(kube_node_status_capacity_memory_bytes) by (node)\n * on (node) group_left (label_cloud_google_com_gke_preemptible)\n avg(kube_node_labels{label_cloud_google_com_gke_preemptible=\"true\"}) by (node)\n ) /1024/1024/1024 * $costpram\n )\n or\n (\n (\n sum(kube_node_status_capacity{resource=\"memory\", unit=\"byte\"}) by (node)\n * on (node) group_left (label_cloud_google_com_gke_preemptible)\n avg(kube_node_labels{label_cloud_google_com_gke_preemptible!=\"true\"}) by (node)\n ) /1024/1024/1024 * ($costram - ($costram / 100 * $costDiscount))\n)\n)\n\n+\n\n#Network \nSUM(rate(node_network_transmit_bytes_total{device=\"eth0\"}[60m]) / 1024 / 1024 / 1024 ) * (60 * 60 * 24 * 30) * $costEgress", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": " {{ node }}", + "refId": "A" + } + ], + "timeFrom": "15m", + "title": "Total Monthly Cost", + "type": "stat" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${datasource}" + }, + "description": "Expected monthly CPU, memory and storage costs given provisioned resources", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 10 + }, + "hiddenSeries": false, + "id": 120, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.3.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "# CPU\nsum(\n (\n (\n sum(kube_node_status_capacity_cpu_cores) by (node)\n * on (node) group_left (label_cloud_google_com_gke_preemptible)\n avg(kube_node_labels{label_cloud_google_com_gke_preemptible=\"true\"}) by (node)\n ) * $costpcpu\n )\n or\n (\n (\n sum(kube_node_status_capacity{resource=\"cpu\", unit=\"core\"}) by (node)\n * on (node) group_left (label_cloud_google_com_gke_preemptible)\n avg(kube_node_labels{label_cloud_google_com_gke_preemptible!=\"true\"}) by (node)\n ) * ($costcpu - ($costcpu / 100 * $costDiscount))\n )\n) \n\n+ \n\n# Storage\nsum (\n sum(kube_persistentvolumeclaim_info{storageclass=~\".*ssd.*\"}) by (persistentvolumeclaim, namespace, storageclass)\n + on (persistentvolumeclaim, namespace) group_right(storageclass)\n sum(kube_persistentvolumeclaim_resource_requests_storage_bytes) by (persistentvolumeclaim, namespace) or up * 0\n) / 1024 / 1024 /1024 * $costStorageSSD\n\n+\n\nsum (\n sum(kube_persistentvolumeclaim_info{storageclass!~\".*ssd.*\"}) by (persistentvolumeclaim, namespace, storageclass)\n + on (persistentvolumeclaim, namespace) group_right(storageclass)\n sum(kube_persistentvolumeclaim_resource_requests_storage_bytes) by (persistentvolumeclaim, namespace) or up * 0\n) / 1024 / 1024 /1024 * $costStorageStandard\n\n+ \n\nsum(container_fs_limit_bytes{id=\"/\"}) / 1024 / 1024 / 1024 * 1.03 * $costStorageStandard \n\n+\n\n# END STORAGE\n# RAM \nsum(\n (\n (\n sum(kube_node_status_capacity_memory_bytes) by (node)\n * on (node) group_left (label_cloud_google_com_gke_preemptible)\n avg(kube_node_labels{label_cloud_google_com_gke_preemptible=\"true\"}) by (node)\n ) /1024/1024/1024 * $costpram\n )\n or\n (\n (\n sum(kube_node_status_capacity{resource=\"memory\", unit=\"byte\"}) by (node)\n * on (node) group_left (label_cloud_google_com_gke_preemptible)\n avg(kube_node_labels{label_cloud_google_com_gke_preemptible!=\"true\"}) by (node)\n ) /1024/1024/1024 * ($costram - ($costram / 100 * $costDiscount))\n)\n) \n\n+\n\n#Network \nSUM(rate(node_network_transmit_bytes_total{device=\"eth0\"}[60m]) / 1024 / 1024 / 1024 ) * (60 * 60 * 24 * 30) * $costEgress", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "cluster cost", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Total monthly cost", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "currencyUSD", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "columns": [ + { + "text": "Avg", + "value": "avg" + } + ], + "datasource": { + "uid": "${datasource}" + }, + "description": "Resources allocated to namespace based on container requests", + "fontSize": "100%", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 10 + }, + "hideTimeOverride": false, + "id": 73, + "links": [], + "pageSize": 10, + "repeatDirection": "v", + "scroll": true, + "showHeader": true, + "sort": { + "col": 7, + "desc": true + }, + "styles": [ + { + "alias": "Namespace", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(50, 172, 45, 0.97)", + "#c15c17" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "View namespace cost metrics", + "linkUrl": "d/at-cost-analysis-namespace2/namespace-cost-metrics?&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + "30", + "80" + ], + "type": "string", + "unit": "currencyUSD" + }, + { + "alias": "RAM", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #B", + "thresholds": [], + "type": "number", + "unit": "currencyUSD" + }, + { + "alias": "CPU", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value #A", + "thresholds": [], + "type": "number", + "unit": "currencyUSD" + }, + { + "alias": "", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Time", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "PV Storage", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value #C", + "thresholds": [], + "type": "number", + "unit": "currencyUSD" + }, + { + "alias": "Total", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value #D", + "thresholds": [], + "type": "number", + "unit": "currencyUSD" + }, + { + "alias": "CPU Utilization", + "align": "auto", + "colorMode": "value", + "colors": [ + "#bf1b00", + "rgba(50, 172, 45, 0.97)", + "#ef843c" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value #E", + "thresholds": [ + "30", + "80" + ], + "type": "number", + "unit": "percent" + }, + { + "alias": "RAM Utilization", + "align": "auto", + "colorMode": "value", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(50, 172, 45, 0.97)", + "#ef843c" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value #F", + "thresholds": [ + "30", + "80" + ], + "type": "number", + "unit": "percent" + } + ], + "targets": [ + { + "expr": "(\n sum(kube_pod_container_resource_requests{resource=\"cpu\", unit=\"core\", namespace!=\"\",namespace!=\"kube-system\",cloud_google_com_gke_preemptible!=\"true\"}*($costcpu - ($costcpu / 100 * $costDiscount))) by(namespace)\n or\n count(\n count(container_spec_cpu_shares{namespace!=\"\",namespace!=\"kube-system\"}) by(namespace)\n ) by(namespace) -1\n)\n\n+\n\n(\n sum(kube_pod_container_resource_requests{resource=\"cpu\", unit=\"core\", namespace!=\"\",namespace!=\"kube-system\",cloud_google_com_gke_preemptible=\"true\"}*$costpcpu) by(namespace)\n or\n count(\n count(container_spec_cpu_shares{namespace!=\"\",namespace!=\"kube-system\"}) by(namespace)\n ) by(namespace) -1\n)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ namespace }}", + "refId": "A" + }, + { + "expr": "(\n sum(kube_pod_container_resource_requests{resource=\"memory\", unit=\"byte\", namespace!=\"\",namespace!=\"kube-system\",cloud_google_com_gke_preemptible!=\"true\"} / 1024 / 1024 / 1024*($costram- ($costram / 100 * $costDiscount))) by (namespace) \n or\n count(\n count(container_spec_memory_limit_bytes{namespace!=\"\",namespace!=\"kube-system\"}) by(namespace)\n ) by(namespace) -1\n)\n\n+\n\n(\n sum(kube_pod_container_resource_requests{resource=\"memory\", unit=\"byte\", namespace!=\"\",namespace!=\"kube-system\",cloud_google_com_gke_preemptible=\"true\"} / 1024 / 1024 / 1024 * $costpram ) by (namespace) \n or\n count(\n count(container_spec_memory_limit_bytes{namespace!=\"\",namespace!=\"kube-system\"}) by(namespace)\n ) by(namespace) -1\n)", + "format": "table", + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ namespace }}", + "refId": "B" + }, + { + "expr": "sum (\n sum(kube_persistentvolumeclaim_info{storageclass=~\".*ssd.*\"}) by (persistentvolumeclaim, namespace, storageclass)\n + on (persistentvolumeclaim, namespace) group_right(storageclass)\n sum(kube_persistentvolumeclaim_resource_requests_storage_bytes) by (persistentvolumeclaim, namespace) \n) by (namespace) / 1024 / 1024 /1024 * $costStorageSSD \n\nor\n\nsum (\n sum(kube_persistentvolumeclaim_info{storageclass!~\".*ssd.*\"}) by (persistentvolumeclaim, namespace, storageclass)\n + on (persistentvolumeclaim, namespace) group_right(storageclass)\n sum(kube_persistentvolumeclaim_resource_requests_storage_bytes) by (persistentvolumeclaim, namespace) \n) by (namespace) / 1024 / 1024 /1024 * $costStorageStandard", + "format": "table", + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ namespace }}", + "refId": "C" + }, + { + "expr": "# CPU \n(\n sum(kube_pod_container_resource_requests{resource=\"cpu\", unit=\"core\", namespace!=\"\",namespace!=\"kube-system\",cloud_google_com_gke_preemptible!=\"true\"}*($costcpu - ($costcpu / 100 * $costDiscount))) by(namespace)\n or\n count(\n count(container_spec_cpu_shares{namespace!=\"\",namespace!=\"kube-system\"}) by(namespace)\n ) by(namespace) -1\n)\n\n+\n\n(\n sum(kube_pod_container_resource_requests{resource=\"cpu\", unit=\"core\", namespace!=\"\",namespace!=\"kube-system\",cloud_google_com_gke_preemptible=\"true\"}*$costpcpu) by(namespace)\n or\n count(\n count(container_spec_cpu_shares{namespace!=\"\",namespace!=\"kube-system\"}) by(namespace)\n ) by(namespace) -1\n)\n\n+\n\n#END CPU \n# Memory \n\n(\n sum(kube_pod_container_resource_requests{resource=\"memory\", unit=\"byte\", namespace!=\"\",namespace!=\"kube-system\",cloud_google_com_gke_preemptible!=\"true\"} / 1024 / 1024 / 1024*($costram- ($costram / 100 * $costDiscount))) by (namespace) \n or\n count(\n count(container_spec_memory_limit_bytes{namespace!=\"\",namespace!=\"kube-system\"}) by(namespace)\n ) by(namespace) -1\n)\n\n+\n\n(\n sum(kube_pod_container_resource_requests{resource=\"memory\", unit=\"byte\", namespace!=\"\",namespace!=\"kube-system\",cloud_google_com_gke_preemptible=\"true\"} / 1024 / 1024 / 1024 * $costpram ) by (namespace) \n or\n count(\n count(container_spec_memory_limit_bytes{namespace!=\"\",namespace!=\"kube-system\"}) by(namespace)\n ) by(namespace) -1\n)\n\n+\n\n# PV storage\n\n(\nsum (\n sum(kube_persistentvolumeclaim_info{storageclass=~\".*ssd.*\"}) by (persistentvolumeclaim, namespace, storageclass)\n + on (persistentvolumeclaim, namespace) group_right(storageclass)\n sum(kube_persistentvolumeclaim_resource_requests_storage_bytes) by (persistentvolumeclaim, namespace) \n) by (namespace) / 1024 / 1024 /1024 * $costStorageSSD \n\nor\n\nsum (\n sum(kube_persistentvolumeclaim_info{storageclass!~\".*ssd.*\"}) by (persistentvolumeclaim, namespace, storageclass)\n + on (persistentvolumeclaim, namespace) group_right(storageclass)\n sum(kube_persistentvolumeclaim_resource_requests_storage_bytes) by (persistentvolumeclaim, namespace) \n) by (namespace) / 1024 / 1024 /1024 * $costStorageStandard \n)", + "format": "table", + "instant": true, + "intervalFactor": 1, + "legendFormat": "Total", + "refId": "D" + } + ], + "timeFrom": "", + "title": "Namespace cost allocation", + "transform": "table", + "type": "table-old" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 108, + "panels": [], + "title": "CPU Metrics", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${datasource}" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 19 + }, + "hiddenSeries": false, + "id": 116, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.3.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "SUM(kube_node_status_capacity{resource=\"cpu\", unit=\"core\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "capacity", + "refId": "A" + }, + { + "expr": "SUM(kube_pod_container_resource_requests{resource=\"cpu\", unit=\"core\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "requests", + "refId": "C" + }, + { + "expr": "SUM(irate(container_cpu_usage_seconds_total{id=\"/\"}[5m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "usage", + "refId": "B" + }, + { + "expr": "SUM(kube_pod_container_resource_limits{resource=\"cpu\", unit=\"core\"}) ", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "limits", + "refId": "D" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Cluster CPUs", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${datasource}" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 27 + }, + "hiddenSeries": false, + "id": 130, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.3.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(irate(node_cpu_seconds_total{mode!=\"idle\"}[5m])) by (mode) * 100", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{mode}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "CPU Mode", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "logBase": 1, + "show": true + }, + { + "format": "percent", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "columns": [ + { + "text": "Avg", + "value": "avg" + } + ], + "datasource": { + "uid": "${datasource}" + }, + "description": "This table shows the comparison of CPU requests and usage by namespace", + "fontSize": "100%", + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 35 + }, + "hideTimeOverride": true, + "id": 104, + "links": [], + "pageSize": 8, + "repeatDirection": "v", + "scroll": true, + "showHeader": true, + "sort": { + "col": 1, + "desc": true + }, + "styles": [ + { + "alias": "CPU Requests", + "align": "auto", + "colors": [ + "#fceaca", + "#fce2de", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value #A", + "thresholds": [ + "" + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Node", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "node", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Time", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "CPU Requests", + "align": "auto", + "colorMode": "value", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(50, 172, 45, 0.97)", + "#cffaff" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value #B", + "thresholds": [ + "" + ], + "type": "number", + "unit": "short" + }, + { + "alias": "24h CPU Usage", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value #C", + "thresholds": [ + "30" + ], + "type": "number", + "unit": "none" + }, + { + "alias": "", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "View namespace cost metrics", + "linkUrl": "d/at-cost-analysis-namespace2/namespace-cost-metrics?&var-namespace=$__cell", + "mappingType": 1, + "pattern": "namespace", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(kube_pod_container_resource_requests{resource=\"cpu\", unit=\"core\", namespace!=\"\"}) by (namespace) ", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + }, + { + "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",namespace!=\"\"}[24h])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ namespace }}", + "refId": "C" + } + ], + "title": "CPU request utilization by namespace", + "transform": "table", + "type": "table-old" + }, + { + "columns": [ + { + "text": "Avg", + "value": "avg" + } + ], + "datasource": { + "uid": "${datasource}" + }, + "description": "This table shows the comparison of application CPU usage vs the capacity of the node (measured over last 60 minutes)", + "fontSize": "100%", + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 35 + }, + "hideTimeOverride": true, + "id": 90, + "links": [], + "pageSize": 8, + "repeatDirection": "v", + "scroll": true, + "showHeader": true, + "sort": { + "col": 2, + "desc": true + }, + "styles": [ + { + "alias": "CPU Request Utilization", + "align": "auto", + "colorMode": "value", + "colors": [ + "#ef843c", + "rgba(50, 172, 45, 0.97)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value #A", + "thresholds": [ + ".30", + " .80" + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Node", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "node", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Time", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "CPU Utilization", + "align": "auto", + "colorMode": "value", + "colors": [ + "#ef843c", + "rgba(50, 172, 45, 0.97)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value #B", + "thresholds": [ + ".20", + " .80" + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "24h Utilization ", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value", + "thresholds": [], + "type": "number", + "unit": "percentunit" + } + ], + "targets": [ + { + "expr": "SUM(\nSUM(rate(container_cpu_usage_seconds_total[24h])) by (pod_name)\n* on (pod_name) group_left (node) \nlabel_replace(\n avg(kube_pod_info{}),\n \"pod_name\", \n \"$1\", \n \"pod\", \n \"(.+)\"\n)\n) by (node) \n/ \nsum(kube_node_status_capacity{resource=\"cpu\", unit=\"core\"}) by (node)", + "format": "table", + "instant": true, + "intervalFactor": 1, + "refId": "B" + }, + { + "expr": "sum(kube_pod_container_resource_requests{resource=\"cpu\", unit=\"core\"}) by (node) / sum(kube_node_status_capacity{resource=\"cpu\", unit=\"core\"}) by (node)", + "format": "table", + "instant": true, + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Cluster cost & utilization by node", + "transform": "table", + "type": "table-old" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 45 + }, + "id": 113, + "panels": [], + "title": "Memory Metrics", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${datasource}" + }, + "fill": 1, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 46 + }, + "id": 117, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "8.3.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "SUM(kube_node_status_capacity{resource=\"memory\", unit=\"byte\"} / 1024 / 1024 / 1024)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "capacity", + "refId": "A" + }, + { + "expr": "SUM(kube_pod_container_resource_requests{resource=\"memory\", unit=\"byte\", namespace!=\"\"} / 1024 / 1024 / 1024)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "requests", + "refId": "C" + }, + { + "expr": "SUM(container_memory_usage_bytes{image!=\"\"} / 1024 / 1024 / 1024)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "usage", + "refId": "B" + }, + { + "expr": "SUM(kube_pod_container_resource_limits{resource=\"memory\", unit=\"byte\", namespace!=\"\"} / 1024 / 1024 / 1024)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "limits", + "refId": "D" + } + ], + "thresholds": [], + "title": "Cluster memory (GB)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decgbytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${datasource}" + }, + "fill": 1, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 54 + }, + "id": 131, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "8.3.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - sum(node_memory_MemAvailable_bytes) by (node) / sum(node_memory_MemTotal_bytes) by (node)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "usage", + "refId": "A" + } + ], + "thresholds": [], + "title": "Cluster Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "columns": [ + { + "text": "Avg", + "value": "avg" + } + ], + "datasource": { + "uid": "${datasource}" + }, + "description": "Comparison of memory requests and current usage by namespace", + "fontSize": "100%", + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 62 + }, + "hideTimeOverride": true, + "id": 109, + "links": [], + "pageSize": 7, + "repeatDirection": "v", + "scroll": true, + "showHeader": true, + "sort": { + "col": 1, + "desc": true + }, + "styles": [ + { + "alias": "Mem Requests (GB)", + "align": "auto", + "colors": [ + "#fceaca", + "#fce2de", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value #A", + "thresholds": [ + "" + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Node", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "node", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Time", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "CPU Requests", + "align": "auto", + "colorMode": "value", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(50, 172, 45, 0.97)", + "#cffaff" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value #B", + "thresholds": [ + "" + ], + "type": "number", + "unit": "short" + }, + { + "alias": "24h Mem Usage", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "#508642", + "#e5ac0e" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value #C", + "thresholds": [ + ".30", + ".75" + ], + "type": "number", + "unit": "none" + }, + { + "alias": "Namespace", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "View namespace cost metrics", + "linkUrl": "d/at-cost-analysis-namespace2/namespace-cost-metrics?&var-namespace=$__cell", + "mappingType": 1, + "pattern": "namespace", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(kube_pod_container_resource_requests{resource=\"memory\", unit=\"byte\", namespace!=\"\"} / 1024 / 1024 / 1024) by (namespace) ", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + }, + { + "expr": "SUM(container_memory_usage_bytes{image!=\"\",namespace!=\"\"} / 1024 / 1024 / 1024) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "refId": "C" + } + ], + "title": "Memory requests & utilization by namespace", + "transform": "table", + "type": "table-old" + }, + { + "columns": [ + { + "text": "Avg", + "value": "avg" + } + ], + "datasource": { + "uid": "${datasource}" + }, + "description": "Container RAM usage vs node capacity", + "fontSize": "100%", + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 62 + }, + "hideTimeOverride": true, + "id": 114, + "links": [], + "pageSize": 8, + "repeatDirection": "v", + "scroll": true, + "showHeader": true, + "sort": { + "col": 1, + "desc": true + }, + "styles": [ + { + "alias": "RAM Requests", + "align": "auto", + "colorMode": "value", + "colors": [ + "#ef843c", + "rgba(50, 172, 45, 0.97)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value #A", + "thresholds": [ + "30", + " 80" + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Node", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "node", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Time", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "RAM Usage", + "align": "auto", + "colorMode": "value", + "colors": [ + "#ef843c", + "rgba(50, 172, 45, 0.97)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value #B", + "thresholds": [ + "25", + " 80" + ], + "type": "number", + "unit": "percent" + }, + { + "alias": "", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "SUM(label_replace(container_memory_usage_bytes{namespace!=\"\"}, \"node\", \"$1\", \"instance\",\"(.+)\")) by (node) * 100\n/\nSUM(kube_node_status_capacity{resource=\"memory\", unit=\"byte\"}) by (node)", + "format": "table", + "instant": true, + "intervalFactor": 1, + "refId": "B" + }, + { + "expr": "sum(kube_pod_container_resource_requests{resource=\"memory\", unit=\"byte\", namespace!=\"\"}) by (node) / SUM(kube_node_status_capacity{resource=\"memory\", unit=\"byte\"}) by (node)", + "format": "table", + "instant": true, + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Node utilization of allocatable RAM", + "transform": "table", + "type": "table-old" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 72 + }, + "id": 101, + "panels": [], + "title": "Storage Metrics", + "type": "row" + }, + { + "columns": [ + { + "text": "Avg", + "value": "avg" + } + ], + "datasource": { + "uid": "${datasource}" + }, + "fontSize": "100%", + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 73 + }, + "hideTimeOverride": true, + "id": 97, + "links": [], + "pageSize": 8, + "repeatDirection": "v", + "scroll": true, + "showHeader": true, + "sort": { + "col": 4, + "desc": true + }, + "styles": [ + { + "alias": "Node", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "instance", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "PVC Name", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "persistentvolumeclaim", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "Storage Class", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "storageclass", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "Cost", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value", + "thresholds": [], + "type": "number", + "unit": "currencyUSD" + }, + { + "alias": "", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Time", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Cost", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value #A", + "thresholds": [], + "type": "number", + "unit": "currencyUSD" + }, + { + "alias": "Size (GB)", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value #B", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "Usage", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value #C", + "thresholds": [], + "type": "number", + "unit": "percentunit" + } + ], + "targets": [ + { + "expr": "SUM(container_fs_limit_bytes{id=\"/\"}) by (instance) / 1024 / 1024 / 1024 * 1.03", + "format": "table", + "instant": true, + "intervalFactor": 1, + "refId": "B" + }, + { + "expr": "SUM(container_fs_limit_bytes{id=\"/\"}) by (instance) / 1024 / 1024 / 1024 * 1.03 * $costStorageStandard\n", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ persistentvolumeclaim }}", + "refId": "A" + }, + { + "expr": "sum(container_fs_usage_bytes{device=~\"^/dev/[sv]d[a-z][1-9]$\",id=\"/\"} / container_fs_limit_bytes{device=~\"^/dev/[sv]d[a-z][1-9]$\",id=\"/\"}) by (instance) \n", + "format": "table", + "instant": true, + "intervalFactor": 1, + "refId": "C" + } + ], + "title": "Local Storage", + "transform": "table", + "type": "table-old" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${datasource}" + }, + "fill": 1, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 73 + }, + "id": 128, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "8.3.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "SUM(container_fs_usage_bytes{id=\"/\"}) / SUM(container_fs_limit_bytes{id=\"/\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "reads", + "refId": "D" + } + ], + "thresholds": [], + "title": "Local storage utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": "IOPS", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "columns": [ + { + "text": "Avg", + "value": "avg" + } + ], + "datasource": { + "uid": "${datasource}" + }, + "fontSize": "100%", + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 82 + }, + "hideTimeOverride": true, + "id": 94, + "links": [], + "pageSize": 10, + "repeatDirection": "v", + "scroll": true, + "showHeader": true, + "sort": { + "col": 2, + "desc": true + }, + "styles": [ + { + "alias": "Namespace", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "View namespace cost metrics", + "linkUrl": "d/at-cost-analysis-namespace2/namespace-cost-metrics?&var-namespace=$__cell", + "mappingType": 1, + "pattern": "namespace", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "PVC Name", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "persistentvolumeclaim", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "Storage Class", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "storageclass", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "Cost", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value", + "thresholds": [], + "type": "number", + "unit": "currencyUSD" + }, + { + "alias": "", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Time", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Cost", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value #A", + "thresholds": [], + "type": "number", + "unit": "currencyUSD" + }, + { + "alias": "Usage", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value #B", + "thresholds": [], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Size (GB)", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value #C", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum (\n sum(kube_persistentvolumeclaim_info{storageclass=~\".*ssd.*\"}) by (persistentvolumeclaim, namespace, storageclass)\n * on (persistentvolumeclaim, namespace) group_right(storageclass)\n sum(kube_persistentvolumeclaim_resource_requests_storage_bytes{storageclass=~\".*ssd.*\"}) by (persistentvolumeclaim, namespace)\n) by (namespace,persistentvolumeclaim,storageclass) / 1024 / 1024 /1024\n\nor\n\nsum (\n sum(kube_persistentvolumeclaim_info{storageclass!~\".*ssd.*\"}) by (persistentvolumeclaim, namespace, storageclass)\n * on (persistentvolumeclaim, namespace) group_right(storageclass)\n sum(kube_persistentvolumeclaim_resource_requests_storage_bytes{storageclass!~\".*ssd.*\"}) by (persistentvolumeclaim, namespace)\n) by (namespace,persistentvolumeclaim,storageclass) / 1024 / 1024 /1024\n\n\n", + "format": "table", + "instant": true, + "intervalFactor": 1, + "refId": "C" + }, + { + "expr": "sum (\n sum(kube_persistentvolumeclaim_info{storageclass=~\".*ssd.*\"}) by (persistentvolumeclaim, namespace, storageclass)\n * on (persistentvolumeclaim, namespace) group_right(storageclass)\n sum(kube_persistentvolumeclaim_resource_requests_storage_bytes{storageclass=~\".*ssd.*\"}) by (persistentvolumeclaim, namespace)\n) by (namespace,persistentvolumeclaim,storageclass) / 1024 / 1024 /1024 * $costStorageSSD\n\nor\n\nsum (\n sum(kube_persistentvolumeclaim_info{storageclass!~\".*ssd.*\"}) by (persistentvolumeclaim, namespace, storageclass)\n * on (persistentvolumeclaim, namespace) group_right(storageclass)\n sum(kube_persistentvolumeclaim_resource_requests_storage_bytes{storageclass!~\".*ssd.*\"}) by (persistentvolumeclaim, namespace)\n) by (namespace,persistentvolumeclaim,storageclass) / 1024 / 1024 /1024 * $costStorageStandard\n", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ persistentvolumeclaim }}", + "refId": "A" + }, + { + "expr": "sum(kubelet_volume_stats_used_bytes) by (persistentvolumeclaim, namespace) \n/\nsum (\n sum(kube_persistentvolumeclaim_info{storageclass!~\".*ssd.*\"}) by (persistentvolumeclaim, namespace, storageclass)\n * on (persistentvolumeclaim, namespace) group_right(storageclass)\n sum(kube_persistentvolumeclaim_resource_requests_storage_bytes{storageclass!~\".*ssd.*\"}) by (persistentvolumeclaim, namespace)\n) by (namespace,persistentvolumeclaim)", + "format": "table", + "instant": true, + "intervalFactor": 1, + "refId": "B" + } + ], + "title": "Persistent Volume Claims", + "transform": "table", + "type": "table-old" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${datasource}" + }, + "fill": 1, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 82 + }, + "id": 132, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "8.3.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "SUM(rate(node_disk_reads_completed_total[10m])) or SUM(rate(node_disk_reads_completed[10m]))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "reads", + "refId": "D" + }, + { + "expr": "SUM(rate(node_disk_writes_completed_total[10m])) or SUM(rate(node_disk_writes_completed[10m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "writes", + "refId": "A" + } + ], + "thresholds": [], + "title": "Disk IOPS", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": "IOPS", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${datasource}" + }, + "fill": 1, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 92 + }, + "id": 122, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "8.3.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "SUM( kubelet_volume_stats_inodes_used / kubelet_volume_stats_inodes) by (persistentvolumeclaim) * 100", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "", + "refId": "D" + } + ], + "thresholds": [], + "title": "Inode usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 101 + }, + "id": 127, + "panels": [], + "title": "Network", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${datasource}" + }, + "fill": 1, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 102 + }, + "id": 123, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "8.3.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (rate (node_network_transmit_bytes_total{}[60m]))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "node_out", + "refId": "B" + }, + { + "expr": "SUM ( rate(node_network_transmit_bytes_total{device=\"eth0\"}[60m]))", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "eth0 out", + "refId": "C" + } + ], + "thresholds": [], + "title": "Node network transmit", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "refresh": "15m", + "schemaVersion": 33, + "style": "dark", + "tags": [ + "cost", + "utilization", + "metrics" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "23.076", + "value": "23.076" + }, + "hide": 0, + "label": "CPU", + "name": "costcpu", + "options": [ + { + "selected": true, + "text": "23.076", + "value": "23.076" + } + ], + "query": "23.076", + "skipUrlSync": false, + "type": "textbox" + }, + { + "current": { + "selected": true, + "text": "5.10", + "value": "5.10" + }, + "hide": 0, + "label": "PE CPU", + "name": "costpcpu", + "options": [ + { + "selected": true, + "text": "5.10", + "value": "5.10" + } + ], + "query": "5.10", + "skipUrlSync": false, + "type": "textbox" + }, + { + "current": { + "selected": true, + "text": "3.25", + "value": "3.25" + }, + "hide": 0, + "label": "RAM", + "name": "costram", + "options": [ + { + "selected": true, + "text": "3.25", + "value": "3.25" + } + ], + "query": "3.25", + "skipUrlSync": false, + "type": "textbox" + }, + { + "current": { + "selected": true, + "text": "0.6862", + "value": "0.6862" + }, + "hide": 0, + "label": "PE RAM", + "name": "costpram", + "options": [ + { + "selected": true, + "text": "0.6862", + "value": "0.6862" + } + ], + "query": "0.6862", + "skipUrlSync": false, + "type": "textbox" + }, + { + "current": { + "selected": true, + "text": "0.040", + "value": "0.040" + }, + "hide": 0, + "label": "Storage", + "name": "costStorageStandard", + "options": [ + { + "selected": true, + "text": "0.040", + "value": "0.040" + } + ], + "query": "0.040", + "skipUrlSync": false, + "type": "textbox" + }, + { + "current": { + "selected": true, + "text": ".17", + "value": ".17" + }, + "hide": 0, + "label": "SSD", + "name": "costStorageSSD", + "options": [ + { + "selected": true, + "text": ".17", + "value": ".17" + } + ], + "query": ".17", + "skipUrlSync": false, + "type": "textbox" + }, + { + "current": { + "selected": true, + "text": ".12", + "value": ".12" + }, + "hide": 0, + "label": "Egress", + "name": "costEgress", + "options": [ + { + "selected": true, + "text": ".12", + "value": ".12" + } + ], + "query": ".12", + "skipUrlSync": false, + "type": "textbox" + }, + { + "current": { + "selected": true, + "text": "30", + "value": "30" + }, + "hide": 0, + "label": "Discount", + "name": "costDiscount", + "options": [ + { + "selected": true, + "text": "30", + "value": "30" + } + ], + "query": "30", + "skipUrlSync": false, + "type": "textbox" + }, + { + "current": { + "selected": false, + "text": "default-kubecost", + "value": "default-kubecost" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": { + "hidden": false, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Deprecated - Cluster cost & utilization metrics", + "uid": "cluster-costs", + "version": 1, + "weekStart": "" + } \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/deployment-utilization.json b/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/deployment-utilization.json new file mode 100644 index 0000000000..1fd2b1d9ee --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/deployment-utilization.json @@ -0,0 +1,1386 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Monitors Kubernetes deployments in cluster using Prometheus and kube-state-metrics. Shows resource utilization of deployments, daemonsets, and statefulsets.", + "editable": true, + "gnetId": 8588, + "graphTooltip": 0, + "id": 2, + "iteration": 1586200623748, + "links": [], + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "${datasource}", + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 0 + }, + "height": "180px", + "id": 1, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum (container_memory_working_set_bytes{container!=\"\",pod_name=~\"^$Deployment$Statefulset$Daemonset.*$\", kubernetes_io_hostname=~\"^$Node$\", pod_name!=\"\"}) / sum (kube_node_status_allocatable{resource=\"memory\", unit=\"byte\", node=~\"^$Node.*$\"}) * 100", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 900 + } + ], + "thresholds": "65, 90", + "title": "Deployment memory usage", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "${datasource}", + "decimals": 2, + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 0 + }, + "height": "180px", + "id": 2, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum (rate (container_cpu_usage_seconds_total{pod_name=~\"^$Deployment$Statefulset$Daemonset.*$\", kubernetes_io_hostname=~\"^$Node$\"}[2m])) / sum (machine_cpu_cores{kubernetes_io_hostname=~\"^$Node$\"}) * 100", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 900 + } + ], + "thresholds": "65, 90", + "title": "Deployment CPU usage", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "${datasource}", + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 0 + }, + "height": "180px", + "id": 3, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(((sum(kube_deployment_status_replicas{deployment=~\".*$Deployment$Statefulset$Daemonset\"}) or vector(0)) + (sum(kube_statefulset_replicas{statefulset=~\".*$Deployment$Statefulset$Daemonset\"}) or vector(0)) + (sum(kube_daemonset_status_desired_number_scheduled{daemonset=~\".*$Deployment$Statefulset$Daemonset\"}) or vector(0))) - ((sum(kube_deployment_status_replicas_available{deployment=~\".*$Deployment$Statefulset$Daemonset\"}) or vector(0)) + (sum(kube_statefulset_status_replicas{statefulset=~\".*$Deployment$Statefulset$Daemonset\"}) or vector(0)) + (sum(kube_daemonset_status_number_ready{daemonset=~\".*$Deployment$Statefulset$Daemonset\"}) or vector(0)))) / ((sum(kube_deployment_status_replicas{deployment=~\".*$Deployment$Statefulset$Daemonset\"}) or vector(0)) + (sum(kube_statefulset_replicas{statefulset=~\".*$Deployment$Statefulset$Daemonset\"}) or vector(0)) + (sum(kube_daemonset_status_desired_number_scheduled{daemonset=~\".*$Deployment$Statefulset$Daemonset\"}) or vector(0))) * 100", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 1800 + } + ], + "thresholds": "1,30", + "title": "Unavailable Replicas", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${datasource}", + "editable": true, + "error": false, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 5 + }, + "height": "100px", + "id": 4, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{container!=\"\",pod_name=~\"^$Deployment$Statefulset$Daemonset.*$\", kubernetes_io_hostname=~\"^$Node$\", pod_name!=\"\"})", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 1800 + } + ], + "thresholds": "", + "title": "Used", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${datasource}", + "editable": true, + "error": false, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 5 + }, + "height": "100px", + "id": 5, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum (kube_node_status_allocatable{resource=\"memory\", unit=\"byte\", node=~\"^$Node.*$\"})", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 1800 + } + ], + "thresholds": "", + "title": "Total", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${datasource}", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 5 + }, + "height": "100px", + "id": 6, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " cores", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum (rate (container_cpu_usage_seconds_total{pod_name=~\"^$Deployment$Statefulset$Daemonset.*$\", kubernetes_io_hostname=~\"^$Node$\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 1800 + } + ], + "thresholds": "", + "title": "Used", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${datasource}", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 5 + }, + "height": "100px", + "id": 7, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " cores", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum (machine_cpu_cores{kubernetes_io_hostname=~\"^$Node$\"})", + "intervalFactor": 2, + "refId": "A", + "step": 1800 + } + ], + "thresholds": "", + "title": "Total", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${datasource}", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 5 + }, + "height": "100px", + "id": 8, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(sum(kube_deployment_status_replicas_available{deployment=~\".*$Deployment$Statefulset$Daemonset\"}) or vector(0)) + (sum(kube_statefulset_status_replicas{statefulset=~\".*$Deployment$Statefulset$Daemonset\"}) or vector(0)) + (sum(kube_daemonset_status_number_ready{daemonset=~\".*$Deployment$Statefulset$Daemonset\"}) or vector(0))", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 1800 + } + ], + "thresholds": "", + "title": "Available (cluster)", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${datasource}", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 5 + }, + "height": "100px", + "id": 9, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(sum(kube_deployment_status_replicas{deployment=~\".*$Deployment$Statefulset$Daemonset\"}) or vector(0)) + (sum(kube_statefulset_replicas{statefulset=~\".*$Deployment$Statefulset$Daemonset\"}) or vector(0)) + (sum(kube_daemonset_status_desired_number_scheduled{daemonset=~\".*$Deployment$Statefulset$Daemonset\"}) or vector(0))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ $Daemonset }}", + "refId": "A", + "step": 1800 + } + ], + "thresholds": "", + "title": "Total (cluster)", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "decimals": 3, + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 8 + }, + "height": "", + "id": 10, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/avlbl.*/", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (irate (container_cpu_usage_seconds_total{container!=\"\",image!=\"\",name=~\"^k8s_.*\",io_kubernetes_container_name!=\"POD\",pod_name=~\"^$Deployment$Statefulset$Daemonset.*$\",kubernetes_io_hostname=~\"^$Node$\"}[5m])) by (pod_name,kubernetes_io_hostname)", + "format": "time_series", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "usage: {{ kubernetes_io_hostname }} | {{ pod_name }} ", + "metric": "container_cpu", + "refId": "A", + "step": 60 + }, + { + "expr": "sum (kube_pod_container_resource_requests{resource=\"cpu\", unit=\"core\", pod=~\"^$Deployment$Statefulset$Daemonset.*$\",node=~\"^$Node$\"}) by (pod,node)", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "rqst: {{ node }} | {{ pod }}", + "refId": "B", + "step": 120 + }, + { + "expr": "sum ((kube_node_status_allocatable{resource=\"cpu\", unit=\"core\", node=~\"^$Node$\"})) by (node)", + "format": "time_series", + "hide": true, + "intervalFactor": 2, + "legendFormat": "avlbl: {{ node }}", + "refId": "C", + "step": 30 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage & requests", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": "cores", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "decimals": 2, + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 13, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 11, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^avlbl.*$/", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max (container_memory_working_set_bytes{container!=\"POD\",container!=\"\",id!=\"/\",pod_name=~\"^$Deployment$Statefulset$Daemonset.*$\",kubernetes_io_hostname=~\"^$Node$\"}) by (pod_name,kubernetes_io_hostname)", + "format": "time_series", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "usage: {{kubernetes_io_hostname }} | {{ pod_name }}", + "metric": "container_memory_usage:sort_desc", + "refId": "A", + "step": 60 + }, + { + "expr": "sum ((kube_pod_container_resource_requests{resource=\"memory\", unit=\"byte\", pod=~\"^$Deployment$Statefulset$Daemonset.*$\",node=~\"^$Node$\"})) by (pod,node)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "rqst: {{ node }} | {{ pod }}", + "refId": "B", + "step": 120 + }, + { + "expr": "sum ((kube_node_status_allocatable{resource=\"memory\", unit=\"byte\", node=~\"^$Node$\"})) by (node)", + "format": "time_series", + "hide": true, + "intervalFactor": 2, + "legendFormat": "avlbl: {{ node }}", + "refId": "C", + "step": 30 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory usage & requests", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "fill": 1, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 12, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "100 * (kubelet_volume_stats_used_bytes{kubernetes_io_hostname=~\"^$Node$\", persistentvolumeclaim=~\".*$Deployment$Statefulset$Daemonset.*$\"} / kubelet_volume_stats_capacity_bytes{kubernetes_io_hostname=~\"^$Node$\", persistentvolumeclaim=~\".*$Deployment$Statefulset$Daemonset.*$\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ persistentvolumeclaim }} | {{ kubernetes_io_hostname }}", + "refId": "A", + "step": 120 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk Usage", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 13, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 13, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (rate (container_network_receive_bytes_total{id!=\"/\",pod_name=~\"^$Deployment$Statefulset$Daemonset.*$\",kubernetes_io_hostname=~\"^$Node$\"}[2m])) by (pod_name, kubernetes_io_hostname)", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "-> {{ kubernetes_io_hostname }} | {{ pod_name }}", + "metric": "network", + "refId": "A", + "step": 60 + }, + { + "expr": "- sum( rate (container_network_transmit_bytes_total{id!=\"/\",pod_name=~\"^$Deployment$Statefulset$Daemonset.*$\",kubernetes_io_hostname=~\"^$Node$\"}[2m])) by (pod_name, kubernetes_io_hostname)", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "<- {{ kubernetes_io_hostname }} | {{ pod_name }}", + "metric": "network", + "refId": "B", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "All processes network I/O", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 16, + "style": "dark", + "tags": [ + "kubecost", + "utilization", + "metrics" + ], + "templating": { + "list": [ + { + "allValue": "()", + "current": { + "selected": false, + "tags": [], + "text": "All", + "value": "$__all" + }, + "datasource": "${datasource}", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "Deployment", + "options": [], + "query": "label_values(deployment)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": "()", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "${datasource}", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "Statefulset", + "options": [], + "query": "label_values(statefulset)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": "()", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "${datasource}", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "Daemonset", + "options": [], + "query": "label_values(daemonset)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "${datasource}", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "Node", + "options": [], + "query": "label_values(kubernetes_io_hostname)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": true, + "text": "default-kubecost", + "value": "default-kubecost" + }, + "error": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Deprecated - Deployment/Statefulset/Daemonset utilization metrics", + "uid": "deployment-metrics", + "version": 2 +} diff --git a/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/grafana-templates/aggregator-dashboard.json b/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/grafana-templates/aggregator-dashboard.json new file mode 100644 index 0000000000..e7c5b3691b --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/grafana-templates/aggregator-dashboard.json @@ -0,0 +1,668 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(container_fs_writes_bytes_total{pod=~\".+-aggregator-0\",namespace=~\"$namespace\"}[2m])) by (namespace)", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Storage Write", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(container_fs_reads_bytes_total{pod=~\".+-aggregator-0\",namespace=~\"$namespace\"}[2m])) by (namespace)", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Storage Read", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(container_memory_working_set_bytes{container=\"aggregator\",pod!=\"\",namespace=~\"$namespace\"} ) by (namespace)", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Memory Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate\r\n{container=\"aggregator\",pod!=\"\",namespace=~\"$namespace\"}[2m])) by (namespace)", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(kubelet_volume_stats_available_bytes{persistentvolumeclaim=~\"aggregator.+\",namespace=~\"$namespace\"}) by (namespace)", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "C" + } + ], + "title": "Storage Available", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(container_network_receive_bytes_total{pod=~\".+aggregator-0\",namespace=~\"$namespace\"}[2m])) by (namespace)", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Network Receive Bytes", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "utilization", + "metrics", + "kubecost" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "Prometheus", + "value": "prometheus" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(container_memory_working_set_bytes{container=\"aggregator\"},namespace)", + "hide": 0, + "includeAll": true, + "multi": false, + "name": "namespace", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(container_memory_working_set_bytes{container=\"aggregator\"},namespace)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "type": "query" + } + ] + }, + "time": { + "from": "now-1d", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Kubecost Aggregator Metrics", + "uid": "kubecost_aggregator_metrics", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/grafana-templates/multi-cluster-container-stats.json b/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/grafana-templates/multi-cluster-container-stats.json new file mode 100644 index 0000000000..5c592b3399 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/grafana-templates/multi-cluster-container-stats.json @@ -0,0 +1,787 @@ +{ + "__inputs": [ + { + "name": "DS_THANOS", + "label": "Thanos", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "10.3.1" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 9063, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_THANOS}" + }, + "description": "Maximum CPU Core Usage vs avg Requested", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 3600000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 94, + "links": [], + "options": { + "legend": { + "calcs": [ + "max" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.4.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_THANOS}" + }, + "editorMode": "code", + "expr": "max(irate(container_cpu_usage_seconds_total\r\n {cluster_id=\"$cluster\",namespace=~\"$namespace\",pod=~\"$pod\", container=~\"$container\", container!=\"POD\",container!=\"\"}\r\n [$__rate_interval])) \r\n by (cluster_id, namespace, pod, container)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{cluster_id}} {{namespace}}/{{pod}}/{{container}} (usage max)", + "metric": "container_cpu", + "refId": "A", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_THANOS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "avg(kube_pod_container_resource_requests\r\n {cluster_id=\"$cluster\",resource=\"cpu\",unit=\"core\",namespace=~\"$namespace\",pod=~\"$pod\",container=~\"$container\",container!=\"POD\"}\r\n ) \r\nby (cluster_id,namespace,pod,container)", + "legendFormat": "{{cluster_id}} {{namespace}}/{{pod}}/{{container}} (requested)", + "range": true, + "refId": "B" + } + ], + "timeFrom": "", + "title": "CPU Core Usage vs Requested", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_THANOS}" + }, + "description": "Max memory used vs avg requested", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 3600000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 96, + "links": [], + "options": { + "legend": { + "calcs": [ + "max" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "pluginVersion": "9.4.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_THANOS}" + }, + "editorMode": "code", + "expr": "max(max_over_time(container_memory_working_set_bytes\r\n {namespace=~\"$namespace\",pod=~\"$pod\",cluster_id=\"$cluster\",container=~\"$container\",container!=\"POD\",container!=\"\"}\r\n [$__rate_interval])) \r\nby (cluster_id,namespace,pod,container)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{cluster_id}} {{namespace}}/{{pod}}/{{container}} (usage max)", + "metric": "container_cpu", + "refId": "MEMORY_USAGE", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_THANOS}" + }, + "editorMode": "code", + "expr": "avg(kube_pod_container_resource_requests\n {resource=\"memory\",unit=\"byte\",cluster_id=\"$cluster\",namespace=~\"$namespace\",pod=~\"$pod\", container=~\"$container\",container!=\"POD\"}\n )\nby (cluster_id,namespace,pod,container)", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{cluster_id}} {{namespace}}/{{pod}}/{{container}} (requested)", + "refId": "MEMORY_REQUESTED" + } + ], + "timeFrom": "", + "title": "Memory Usage vs Requested", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_THANOS}" + }, + "description": "Network traffic by pod", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 3600000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 95, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.4.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_THANOS}" + }, + "editorMode": "code", + "expr": "sum(irate(container_network_receive_bytes_total\n {cluster_id=~\"$cluster\",namespace=~\"$namespace\",pod=~\"$pod\"}\n [$__rate_interval])) \nby (cluster_id, namespace, pod)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{cluster_id}} {{namespace}}/{{pod}}<- in", + "metric": "container_cpu", + "refId": "A", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_THANOS}" + }, + "editorMode": "code", + "expr": "- sum(irate(container_network_transmit_bytes_total\n {cluster_id=\"$cluster\",namespace=~\"$namespace\",pod=~\"$pod\"}\n [$__rate_interval])) \nby (cluster_id, namespace, pod)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{cluster_id}} {{namespace}}/{{pod}}-> out", + "refId": "B" + } + ], + "timeFrom": "", + "title": "Network IO", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_THANOS}" + }, + "description": "Disk read writes", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 3600000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 97, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.4.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_THANOS}" + }, + "editorMode": "code", + "expr": "sum(irate(container_fs_writes_bytes_total\r\n {cluster_id=\"$cluster\",namespace=~\"$namespace\",container!=\"POD\",pod!=\"\",pod=~\"$pod\",container=~\"$container\"}\r\n [$__rate_interval])) \r\nby (cluster_id,namespace,pod,container)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{cluster_id}} {{pod}}/{{container}}<- write", + "metric": "container_cpu", + "refId": "A", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_THANOS}" + }, + "editorMode": "code", + "expr": "- sum(irate(container_fs_reads_bytes_total\r\n {cluster_id=\"$cluster\",namespace=~\"$namespace\",container!=\"POD\",pod!=\"\",pod=~\"$pod\",container=~\"$container\"}\r\n [$__rate_interval])) \r\nby (cluster_id,namespace,pod,container)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{cluster_id}} {{pod}}/{{container}}-> read", + "refId": "B" + } + ], + "timeFrom": "", + "title": "Disk IO", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_THANOS}" + }, + "description": "This graph shows the % of periods where a pod is being throttled. Values range from 0-100", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 14 + }, + "id": 99, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.4.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_THANOS}" + }, + "editorMode": "code", + "expr": "100\n * sum by(cluster_id, namespace, pod, container) (increase(container_cpu_cfs_throttled_periods_total{container!=\"\",cluster_id=\"$cluster\", namespace=~\"$namespace\", pod=~\"$pod\", container=~\"$container\", container!=\"POD\"}[$__rate_interval]))\n / sum by(cluster_id,namespace,pod,container) (increase(container_cpu_cfs_periods_total{container!=\"\",cluster_id=\"$cluster\",namespace=~\"$namespace\",pod=~\"$pod\",container=~\"$container\",container!=\"POD\"}[$__rate_interval]))", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "B" + } + ], + "timeFrom": "", + "title": "CPU throttle percent", + "type": "timeseries" + } + ], + "refresh": "", + "revision": 1, + "schemaVersion": 39, + "tags": [ + "utilization", + "metrics", + "kubecost" +], + "templating": { + "list": [ + { + "current": {}, + "definition": "label_values(cluster_id)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "cluster", + "options": [], + "query": { + "query": "label_values(cluster_id)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "type": "query" + }, + { + "current": {}, + "definition": "label_values(kube_namespace_labels{cluster_id=\"$cluster\"}, namespace) ", + "hide": 0, + "includeAll": true, + "label": "", + "multi": false, + "name": "namespace", + "options": [], + "query": { + "query": "label_values(kube_namespace_labels{cluster_id=\"$cluster\"}, namespace) ", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": {}, + "definition": "label_values(kube_pod_labels{cluster_id=\"$cluster\",namespace=~\"$namespace\"}, pod) ", + "hide": 0, + "includeAll": true, + "label": "pod", + "multi": false, + "name": "pod", + "options": [], + "query": { + "query": "label_values(kube_pod_labels{cluster_id=\"$cluster\",namespace=~\"$namespace\"}, pod) ", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": {}, + "definition": "label_values(container_memory_working_set_bytes{cluster_id=\"$cluster\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"POD\"}, container) ", + "hide": 0, + "includeAll": true, + "multi": false, + "name": "container", + "options": [], + "query": { + "query": "label_values(container_memory_working_set_bytes{cluster_id=\"$cluster\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"POD\"}, container) ", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "type": "query" + } + ] + }, + "time": { + "from": "now-2d", + "to": "now" + }, + "timepicker": { + "hidden": false, + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Pod utilization metrics (multi-cluster)", + "uid": "at-cost-analysis-pod2", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/grafana-templates/multi-cluster-disk-usage.json b/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/grafana-templates/multi-cluster-disk-usage.json new file mode 100644 index 0000000000..6dc0b153c8 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/grafana-templates/multi-cluster-disk-usage.json @@ -0,0 +1,571 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "10.4.2" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(container_fs_limit_bytes{instance=~'$disk', device!=\"tmpfs\", id=\"/\", cluster_id=~'$cluster'}) by (cluster_id, instance)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{cluster_id}}/{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Disk Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(container_fs_usage_bytes{instance=~'$disk',id=\"/\", cluster_id=~'$cluster'}) by (cluster_id, instance) / sum(container_fs_limit_bytes{instance=~'$disk',device!=\"tmpfs\", id=\"/\", cluster_id=~'$cluster'}) by (cluster_id,instance)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{cluster_id}}-{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Disk Utilization", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "1 - sum(container_fs_inodes_free{instance=~'$disk',id=\"/\", cluster_id=~'$cluster'}) by (cluster_id, instance) / sum(container_fs_inodes_total{instance=~'$disk',id=\"/\", cluster_id=~'$cluster'}) by (cluster_id, instance)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{cluster_id}}/{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "iNode Utilization", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(container_fs_usage_bytes{instance=~'$disk',id=\"/\", cluster_id=~'$cluster'}) by (cluster_id, instance)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{cluster_id}}/{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Disk Usage", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "tags": [ + "kubecost", + "utilization", + "metrics" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(cluster_id)", + "hide": 0, + "includeAll": true, + "multi": false, + "name": "cluster", + "options": [], + "query": { + "query": "label_values(cluster_id)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(container_fs_limit_bytes{cluster_id=~\"$cluster\"}, instance)", + "hide": 0, + "includeAll": true, + "multi": false, + "name": "disk", + "options": [], + "query": { + "query": "label_values(container_fs_limit_bytes{cluster_id=~\"$cluster\"}, instance)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-7d", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Attached disk metrics (multi-cluster)", + "uid": "nBH7qBgMk", + "version": 2, + "weekStart": "" +} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/grafana-templates/multi-cluster-network-transfer-data.json b/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/grafana-templates/multi-cluster-network-transfer-data.json new file mode 100644 index 0000000000..40bf4e787b --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/grafana-templates/multi-cluster-network-transfer-data.json @@ -0,0 +1,685 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "10.4.2" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "https://docs.kubecost.com/install-and-configure/advanced-configuration/network-costs-configuration", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 12, + "panels": [], + "title": "Network Data Transfers (negative is egress data)", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 11, + "x": 0, + "y": 1 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(increase(kubecost_pod_network_ingress_bytes_total\n {namespace=~\"$namespace\", cluster_id=~\"$cluster\", pod_name=~\"$pod_name\", service=~\"$service\"}\n [1h]\n ))\nby($aggregation) ", + "interval": "1h", + "legendFormat": "__auto", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "-sum(increase(kubecost_pod_network_egress_bytes_total\n {namespace=~\"$namespace\", cluster_id=~\"$cluster\", pod_name=~\"$pod_name\", service=~\"$service\"}\n [1h]\n ))\nby($aggregation) ", + "hide": false, + "interval": "1h", + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "All Data", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 13, + "x": 11, + "y": 1 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(increase(kubecost_pod_network_ingress_bytes_total\n {internet=\"true\", namespace=~\"$namespace\", cluster_id=~\"$cluster\", pod_name=~\"$pod_name\", service=~\"$service\"}\n [1h]\n))\nby($aggregation) ", + "hide": false, + "interval": "1h", + "legendFormat": "__auto", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "- sum(increase(kubecost_pod_network_egress_bytes_total\n {internet=\"true\", namespace=~\"$namespace\", cluster_id=~\"$cluster\", pod_name=~\"$pod_name\", service=~\"$service\"}\n [1h]))\nby($aggregation) ", + "hide": false, + "interval": "1h", + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "Internet Data", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Cross region and cross zone subnets must be defined via the configMap. \nSee: \n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 11, + "x": 0, + "y": 15 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(increase(kubecost_pod_network_ingress_bytes_total\n {internet=\"false\",namespace=~\"$namespace\",cluster_id=~\"$cluster\",pod_name=~\"$pod_name\", sameRegion=\"false\",sameZone=\"false\", service=~\"$service\"}\n [1h]))\nby($aggregation)", + "interval": "1h", + "legendFormat": "__auto", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "- sum(increase(kubecost_pod_network_egress_bytes_total\n {internet=\"false\", namespace=~\"$namespace\",cluster_id=~\"$cluster\",pod_name=~\"$pod_name\",sameRegion=\"false\", sameZone=\"false\", service=~\"$service\"}\n [1h]))\nby($aggregation) ", + "hide": false, + "interval": "1h", + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "Cross Region Data", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Cross region and cross zone subnets must be defined via the configMap. \nSee: \n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 13, + "x": 11, + "y": 15 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(increase(kubecost_pod_network_ingress_bytes_total\n {internet=\"false\", namespace=~\"$namespace\", cluster_id=~\"$cluster\", pod_name=~\"$pod_name\", sameRegion=\"true\", sameZone=\"false\", service=~\"$service\"}\n [1h]))\nby($aggregation)", + "interval": "1h", + "legendFormat": "__auto", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "- sum(increase(kubecost_pod_network_egress_bytes_total\n {internet=\"false\", namespace=~\"$namespace\", cluster_id=~\"$cluster\", pod_name=~\"$pod_name\", sameRegion=\"true\", sameZone=\"false\", service=~\"$service\"}\n [1h]))\nby($aggregation)", + "hide": false, + "interval": "1h", + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "Cross Zone Data", + "type": "timeseries" + } + ], + "refresh": "", + "revision": 1, + "schemaVersion": 39, + "tags": [ + "kubecost" + ], + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "namespace", + "value": "namespace" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "aggregation", + "options": [ + { + "selected": false, + "text": "cluster_id", + "value": "cluster_id" + }, + { + "selected": true, + "text": "namespace", + "value": "namespace" + }, + { + "selected": false, + "text": "pod_name", + "value": "pod_name" + } + ], + "query": "cluster_id, namespace, pod_name", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(cluster_id)", + "hide": 0, + "includeAll": true, + "multi": false, + "name": "cluster", + "options": [], + "query": { + "query": "label_values(cluster_id)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "type": "query" + }, + { + "current": { + "selected": true, + "text": "kubecost", + "value": "kubecost" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(kube_namespace_labels{cluster_id=~\"$cluster\"}, namespace) ", + "hide": 0, + "includeAll": true, + "multi": false, + "name": "namespace", + "options": [], + "query": { + "query": "label_values(kube_namespace_labels{cluster_id=~\"$cluster\"}, namespace) ", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(kubecost_pod_network_egress_bytes_total{cluster_id=~\"$cluster\", namespace=~\"$namespace\"},pod_name)", + "hide": 0, + "includeAll": true, + "multi": false, + "name": "pod_name", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(kubecost_pod_network_egress_bytes_total{cluster_id=~\"$cluster\", namespace=~\"$namespace\"},pod_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "filters": [], + "hide": 0, + "name": "filter", + "skipUrlSync": false, + "type": "adhoc" + }, + { + "current": {}, + "definition": "label_values(kubecost_pod_network_egress_bytes_total{namespace=~\"$namespace\"},service)", + "hide": 0, + "includeAll": true, + "multi": false, + "name": "service", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(kubecost_pod_network_egress_bytes_total{namespace=~\"$namespace\"},service)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "type": "query" + } + ] + }, + "time": { + "from": "now-7d", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Kubecost Network Costs Metrics", + "uid": "kubecost-networkCosts-metrics", + "version": 8, + "weekStart": "" +} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/kubernetes-resource-efficiency.json b/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/kubernetes-resource-efficiency.json new file mode 100644 index 0000000000..156b3c292f --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/kubernetes-resource-efficiency.json @@ -0,0 +1,408 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 29, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [], + "title": "Requests - Usage (negative values are unused reservations)", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 3600000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 16, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by ($aggregation) (\n (sum by (cluster_id,namespace,pod,container) (container_memory_usage_bytes{cluster_id=~\"$cluster\",namespace=~\"$namespace\",container=~\"$container\",container!=\"POD\",container!=\"\"}))\n -(sum by (cluster_id,namespace,pod,container) (kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",cluster_id=~\"$cluster\",namespace=~\"$namespace\",container=~\"$container\",container!=\"POD\",container!=\"\"}))\n)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by ($aggregation) (\n -(sum by (cluster_id,namespace,pod,container) (kube_pod_container_resource_requests{resource=\"memory\",unit=\"byte\",cluster_id=~\"$cluster\",namespace=~\"$namespace\",container=~\"$container\",container!=\"POD\",container!=\"\"}))\n)", + "hide": true, + "legendFormat": "{{$aggregation}} Request", + "range": true, + "refId": "B" + } + ], + "title": "Memory Request-Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 3600000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by ($aggregation)(\n (sum by (cluster_id,namespace,pod,container) (rate(container_cpu_usage_seconds_total{cluster_id=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\", container!=\"POD\",container!=\"\"}[1h])))\n - \n (sum by (cluster_id,namespace,pod,container) (kube_pod_container_resource_requests{resource=\"cpu\",cluster_id=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\", container!=\"POD\",container!=\"\"}))\n)\n \n", + "legendFormat": "__auto", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "-sum by ($aggregation)(\n (sum by (cluster_id,namespace,pod,container) (kube_pod_container_resource_requests{resource=\"cpu\",cluster_id=~\"$cluster\", namespace=~\"$namespace\", container=~\"$container\", container!=\"POD\",container!=\"\"}))\n)", + "hide": true, + "legendFormat": "{{$aggregation}} Request", + "range": true, + "refId": "B" + } + ], + "title": "CPU Request-Usage", + "type": "timeseries" + } + ], + "schemaVersion": 37, + "style": "dark", + "tags": [ + "utilization", + "metrics", + "kubecost" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "default", + "value": "default" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": true, + "text": "namespace", + "value": "namespace" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "aggregation", + "options": [ + { + "selected": false, + "text": "cluster_id", + "value": "cluster_id" + }, + { + "selected": true, + "text": "namespace", + "value": "namespace" + }, + { + "selected": false, + "text": "container", + "value": "container" + } + ], + "query": "cluster_id,namespace,container", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + }, + { + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(kube_namespace_labels, cluster_id)", + "hide": 0, + "includeAll": true, + "multi": false, + "name": "cluster", + "options": [], + "query": { + "query": "label_values(kube_namespace_labels, cluster_id)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "type": "query" + }, + { + "current": { + "selected": true, + "text": "kubecost", + "value": "kubecost" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(kube_namespace_labels{cluster_id=~\"$cluster\"}, namespace) ", + "hide": 0, + "includeAll": true, + "multi": false, + "name": "namespace", + "options": [], + "query": { + "query": "label_values(kube_namespace_labels{cluster_id=~\"$cluster\"}, namespace) ", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(container_memory_working_set_bytes{cluster_id=~\"$cluster\",namespace=~\"$namespace\", container!=\"POD\"}, container) ", + "hide": 0, + "includeAll": true, + "multi": false, + "name": "container", + "options": [], + "query": { + "query": "label_values(container_memory_working_set_bytes{cluster_id=~\"$cluster\",namespace=~\"$namespace\", container!=\"POD\"}, container) ", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Kubernetes Resource Efficiency", + "uid": "kubernetes-resource-efficiency", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/label-cost-utilization.json b/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/label-cost-utilization.json new file mode 100644 index 0000000000..dc1963edb3 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/label-cost-utilization.json @@ -0,0 +1,1146 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "iteration": 1645115160709, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Monthly projected CPU cost given last 10m", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "currencyUSD" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 0 + }, + "hideTimeOverride": true, + "id": 15, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P0C970EB638C812D0" + }, + "exemplar": false, + "expr": "sum(\n avg(container_cpu_allocation) by (pod,node)\n\n * on (node) group_left()\n avg(avg_over_time(node_cpu_hourly_cost[10m])) by (node)\n\n * on (pod) group_left()\n label_replace(\n max(kube_pod_labels{label_$label=~\"$label_value\"}) by (pod),\n \"pod_name\",\n \"$1\", \n \"pod\", \n \"(.+)\"\n )\n) * 730", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": " {{ node }}", + "refId": "A" + } + ], + "timeFrom": "15m", + "title": "CPU Cost", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Based on CPU usage over last 24 hours", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "currencyUSD" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 0 + }, + "hideTimeOverride": true, + "id": 16, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P0C970EB638C812D0" + }, + "exemplar": false, + "expr": "sum(\n avg(container_memory_allocation_bytes) by (pod,node) / 1024 / 1024 / 1024\n\n * on (node) group_left()\n avg(avg_over_time(node_ram_hourly_cost[10m])) by (node)\n\n * on (pod) group_left()\n label_replace(\n max(kube_pod_labels{label_$label=~\"$label_value\"}) by (pod),\n \"pod_name\",\n \"$1\", \n \"pod\", \n \"(.+)\"\n )\n) * 730", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": " {{ node }}", + "refId": "A" + } + ], + "timeFrom": "15m", + "title": "Memory Cost", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "currencyUSD" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 0 + }, + "hideTimeOverride": true, + "id": 21, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P0C970EB638C812D0" + }, + "exemplar": false, + "expr": "sum(\n sum(kube_persistentvolumeclaim_info{storageclass!=\".*ssd.*\"}) by (persistentvolumeclaim, storageclass)\n * on (persistentvolumeclaim) group_right(storageclass)\n sum(kube_persistentvolumeclaim_resource_requests_storage_bytes) by (persistentvolumeclaim)\n * on (persistentvolumeclaim) group_left(label_app)\n max(kube_persistentvolumeclaim_labels{label_$label=~\"$label_value\"}) by (persistentvolumeclaim) or up * 0\n) / 1024 / 1024 /1024 * .04 \n\n+\n\nsum(\n sum(kube_persistentvolumeclaim_info{storageclass=~\".*ssd.*\"}) by (persistentvolumeclaim, storageclass)\n * on (persistentvolumeclaim) group_right(storageclass)\n sum(kube_persistentvolumeclaim_resource_requests_storage_bytes) by (persistentvolumeclaim)\n * on (persistentvolumeclaim) group_left(label_app)\n max(kube_persistentvolumeclaim_labels{label_$label=~\"$label_value\"}) by (persistentvolumeclaim) or up * 0\n) / 1024 / 1024 /1024 * .17 \n", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": " {{ node }}", + "refId": "A" + } + ], + "timeFrom": "15m", + "title": "Storage Cost", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Cost of memory + CPU usage", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "currencyUSD" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 0 + }, + "hideTimeOverride": true, + "id": 20, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P0C970EB638C812D0" + }, + "exemplar": false, + "expr": "# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ CPU ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\nsum(\n avg(container_cpu_allocation) by (pod,node)\n\n * on (node) group_left()\n avg(avg_over_time(node_cpu_hourly_cost[10m])) by (node)\n\n * on (pod) group_left()\n label_replace(\n max(kube_pod_labels{label_$label=~\"$label_value\"}) by (pod),\n \"pod_name\",\n \"$1\", \n \"pod\", \n \"(.+)\"\n )\n) * 730\n\n#END CPU\n+\n\n# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Memory ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\nsum(\n avg(container_memory_allocation_bytes) by (pod,node) / 1024 / 1024 / 1024\n\n * on (node) group_left()\n avg(avg_over_time(node_ram_hourly_cost[10m])) by (node)\n\n * on (pod) group_left()\n label_replace(\n max(kube_pod_labels{label_$label=~\"$label_value\"}) by (pod),\n \"pod_name\",\n \"$1\", \n \"pod\", \n \"(.+)\"\n )\n) * 730\n\n# END MEMORY\n\n+\n\n# ~~~~~~~~~~~~~~~~~~~~~~~~~~~ STORAGE ~~~~~~~~~~~~~~~~~~~~~~~~~\n\nsum(\n sum(kube_persistentvolumeclaim_info{storageclass!=\".*ssd.*\"}) by (persistentvolumeclaim, storageclass)\n * on (persistentvolumeclaim) group_right(storageclass)\n sum(kube_persistentvolumeclaim_resource_requests_storage_bytes) by (persistentvolumeclaim)\n * on (persistentvolumeclaim) group_left(label_app)\n max(kube_persistentvolumeclaim_labels{label_$label=~\"$label_value\"}) by (persistentvolumeclaim) or up * 0\n) / 1024 / 1024 /1024 * .04 \n\n+\n\nsum(\n sum(kube_persistentvolumeclaim_info{storageclass=~\".*ssd.*\"}) by (persistentvolumeclaim, storageclass)\n * on (persistentvolumeclaim) group_right(storageclass)\n sum(kube_persistentvolumeclaim_resource_requests_storage_bytes) by (persistentvolumeclaim)\n * on (persistentvolumeclaim) group_left(label_app)\n max(kube_persistentvolumeclaim_labels{label_$label=~\"$label_value\"}) by (persistentvolumeclaim) or up * 0\n) / 1024 / 1024 /1024 * .17 \n\n\n# END STORAGE\n", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": " {{ node }}", + "refId": "A" + } + ], + "timeFrom": "15m", + "title": "Total Cost", + "type": "stat" + }, + { + "datasource": { + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 5 + }, + "id": 10, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(\n sum (kube_pod_container_resource_requests{resource=\"cpu\", unit=\"core\"}) by (pod)\n * on (pod) group_left()\n max(kube_pod_labels{label_$label=~\"$label_value\"}) by (pod)\n or up * 0\n) ", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "CPU Request", + "type": "stat" + }, + { + "datasource": { + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 5 + }, + "id": 17, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(\n label_replace(\n sum(rate(container_cpu_usage_seconds_total{image!=\"\",container_name!=\"POD\"}[1h])) by (kubernetes_io_hostname,pod_name),\n \"node\",\n \"$1\", \n \"kubernetes_io_hostname\", \n \"(.+)\"\n ) \n * on (pod_name) group_left()\n label_replace(\n max(kube_pod_labels{label_$label=~\"$label_value\"}) by (pod),\n \"pod_name\",\n \"$1\", \n \"pod\", \n \"(.+)\"\n ) or up * 0\n) ", + "format": "time_series", + "intervalFactor": 2, + "refId": "A" + } + ], + "title": "CPU Used", + "type": "stat" + }, + { + "datasource": { + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 6, + "y": 5 + }, + "id": 11, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(\n sum (kube_pod_container_resource_requests{resource=\"memory\", unit=\"byte\"}) by (pod)\n * on (pod) group_left()\n max(kube_pod_labels{label_$label=~\"$label_value\"}) by (pod)\n or up * 0\n) ", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Memory Request", + "type": "stat" + }, + { + "datasource": { + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 9, + "y": 5 + }, + "id": 18, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(\n label_replace(\n sum (container_memory_working_set_bytes{pod_name!=\"\",container!=\"POD\",container!=\"\"}) by (pod_name),\n \"pod\",\n \"$1\", \n \"pod_name\", \n \"(.+)\")\n * on (pod) group_left()\n max(kube_pod_labels{label_$label=~\"$label_value\"}) by (pod)\n or up * 0\n)", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Memory Usage", + "type": "stat" + }, + { + "datasource": { + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 5 + }, + "id": 22, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(\n max(kube_persistentvolumeclaim_info) by (persistentvolumeclaim, storageclass)\n * on (persistentvolumeclaim) group_right(storageclass)\n sum(kube_persistentvolumeclaim_resource_requests_storage_bytes) by (persistentvolumeclaim)\n * on (persistentvolumeclaim) group_left(label_app)\n max(kube_persistentvolumeclaim_labels{label_$label=~\"$label_value\"}) by (persistentvolumeclaim) or up * 0\n) \n", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Storage Request", + "type": "stat" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${datasource}" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n label_replace(\n sum (kube_pod_container_resource_limits{resource=\"cpu\", unit=\"core\"}) by (pod, container)\n * on (pod) group_left()\n max(kube_pod_labels{label_$label=~\"$label_value\"}) by (pod,container),\n \"container_name\",\n \"$1\", \n \"container\", \n \"(.+)\"\n )\n) \n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "limit", + "refId": "C" + }, + { + "expr": "sum(\n label_replace(\n sum (kube_pod_container_resource_requests{resource=\"cpu\", unit=\"core\"}) by (pod, container)\n * on (pod) group_left()\n max(kube_pod_labels{label_$label=~\"$label_value\"}) by (pod,container),\n \"container_name\",\n \"$1\", \n \"container\", \n \"(.+)\"\n )\n) \n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "request", + "refId": "B" + }, + { + "expr": "sum(\n label_replace(\n sum (rate (container_cpu_usage_seconds_total{image!=\"\",container!=\"POD\",container!=\"\"}[10m])) by (container,pod),\n \"pod\", \n \"$1\", \n \"pod_name\", \n \"(.+)\"\n )\n * on (pod) group_left (label_$label)\n max(kube_pod_labels{label_$label=~\"$label_value\"}) by (pod,container)\n)\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "usage", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "CPU Usage vs Requests vs Limits", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "${datasource}" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "hiddenSeries": false, + "id": 23, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n label_replace(\n sum (kube_pod_container_resource_limits{resource=\"memory\", unit=\"byte\"}) by (pod, container)\n * on (pod) group_left()\n max(kube_pod_labels{label_$label=~\"$label_value\"}) by (pod,container),\n \"container_name\",\n \"$1\", \n \"container\", \n \"(.+)\"\n )\n) \n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "limit", + "refId": "C" + }, + { + "expr": "sum(\n label_replace(\n sum (kube_pod_container_resource_requests{resource=\"memory\", unit=\"byte\"}) by (pod, container)\n * on (pod) group_left()\n max(kube_pod_labels{label_$label=~\"$label_value\"}) by (pod,container),\n \"container_name\",\n \"$1\", \n \"container\", \n \"(.+)\"\n )\n) \n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "request", + "refId": "B" + }, + { + "expr": "sum(\n label_replace(\n sum (container_memory_working_set_bytes{container!=\"\",container!=\"POD\"}) by (container,pod),\n \"pod\", \n \"$1\", \n \"pod_name\", \n \"(.+)\"\n )\n * on (pod) group_left (label_$label)\n max(kube_pod_labels{label_$label=~\"$label_value\"}) by (pod,container)\n)\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "usage", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory Usage vs Requests vs Limits", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "refresh": false, + "schemaVersion": 34, + "style": "dark", + "tags": [ + "kubecost", + "cost", + "utilization", + "metrics" + ], + "templating": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "filters": [], + "hide": 0, + "label": "", + "name": "Filters", + "skipUrlSync": false, + "type": "adhoc" + }, + { + "current": { + "tags": [], + "text": "app", + "value": "app" + }, + "hide": 0, + "includeAll": false, + "label": "Label", + "multi": false, + "name": "label", + "options": [ + { + "selected": true, + "text": "app", + "value": "app" + }, + { + "selected": false, + "text": "tier", + "value": "tier" + }, + { + "selected": false, + "text": "component", + "value": "component" + }, + { + "selected": false, + "text": "release", + "value": "release" + }, + { + "selected": false, + "text": "name", + "value": "name" + }, + { + "selected": false, + "text": "team", + "value": "team" + }, + { + "selected": false, + "text": "department", + "value": "department" + }, + { + "selected": false, + "text": "owner", + "value": "owner" + }, + { + "selected": false, + "text": "contact", + "value": "contact" + } + ], + "query": "app, tier, component, release, name, team, department, owner, contact", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "label": "Value", + "multi": false, + "name": "label_value", + "options": [], + "query": { + "query": "query_result(SUM(kube_pod_labels{label_$label!=\"\",namespace!=\"kube-system\"}) by (label_$label))", + "refId": "default-kubecost-label_value-Variable-Query" + }, + "refresh": 1, + "regex": "/label_$label=\\\"(.*?)(\\\")/", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": "()", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "label": "", + "multi": false, + "name": "Deployments", + "options": [], + "query": { + "query": "label_values(deployment)", + "refId": "default-kubecost-Deployments-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "multi": false, + "name": "Secondary", + "options": [], + "query": { + "query": "query_result(kube_pod_labels)", + "refId": "default-kubecost-Secondary-Variable-Query" + }, + "refresh": 1, + "regex": "/.+?label_([^=]*).*/", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": "default-kubecost", + "value": "default-kubecost" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Label costs & utilization", + "uid": "lWMhIA-ik", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/namespace-utilization.json b/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/namespace-utilization.json new file mode 100644 index 0000000000..a2e60c1f20 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/namespace-utilization.json @@ -0,0 +1,1175 @@ +{ + "annotations":{ + "list":[ + { + "builtIn":1, + "datasource":"-- Grafana --", + "enable":true, + "hide":true, + "iconColor":"rgba(0, 211, 255, 1)", + "name":"Annotations & Alerts", + "type":"dashboard" + } + ] + }, + "description":"A dashboard to help with utilization and resource allocation", + "editable":true, + "gnetId":8673, + "graphTooltip":0, + "id":9, + "iteration":1553150922105, + "links":[ + + ], + "panels":[ + { + "columns":[ + { + "text":"Avg", + "value":"avg" + } + ], + "datasource":"${datasource}", + "fontSize":"100%", + "gridPos":{ + "h":9, + "w":16, + "x":0, + "y":0 + }, + "hideTimeOverride":true, + "id":73, + "links":[ + + ], + "pageSize":8, + "repeat":null, + "repeatDirection":"v", + "scroll":true, + "showHeader":true, + "sort":{ + "col":2, + "desc":false + }, + "styles":[ + { + "alias":"Pod", + "colorMode":null, + "colors":[ + "rgba(245, 54, 54, 0.9)", + "rgba(50, 172, 45, 0.97)", + "#c15c17" + ], + "dateFormat":"YYYY-MM-DD HH:mm:ss", + "decimals":2, + "link":false, + "linkTooltip":"", + "linkUrl":"", + "pattern":"pod_name", + "thresholds":[ + "30", + "80" + ], + "type":"string", + "unit":"currencyUSD" + }, + { + "alias":"RAM", + "colorMode":null, + "colors":[ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat":"YYYY-MM-DD HH:mm:ss", + "decimals":2, + "pattern":"Value #B", + "thresholds":[ + + ], + "type":"number", + "unit":"decbytes" + }, + { + "alias":"CPU %", + "colorMode":null, + "colors":[ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat":"YYYY-MM-DD HH:mm:ss", + "decimals":2, + "mappingType":1, + "pattern":"Value #A", + "thresholds":[ + + ], + "type":"number", + "unit":"percent" + }, + { + "alias":"", + "colorMode":null, + "colors":[ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat":"YYYY-MM-DD HH:mm:ss", + "decimals":2, + "mappingType":1, + "pattern":"Time", + "thresholds":[ + + ], + "type":"hidden", + "unit":"short" + }, + { + "alias":"Storage", + "colorMode":null, + "colors":[ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat":"YYYY-MM-DD HH:mm:ss", + "decimals":2, + "mappingType":1, + "pattern":"Value #C", + "thresholds":[ + + ], + "type":"number", + "unit":"currencyUSD" + }, + { + "alias":"Total", + "colorMode":null, + "colors":[ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat":"YYYY-MM-DD HH:mm:ss", + "decimals":2, + "mappingType":1, + "pattern":"Value #D", + "thresholds":[ + + ], + "type":"number", + "unit":"currencyUSD" + }, + { + "alias":"CPU Utilization", + "colorMode":"value", + "colors":[ + "#bf1b00", + "rgba(50, 172, 45, 0.97)", + "#ef843c" + ], + "dateFormat":"YYYY-MM-DD HH:mm:ss", + "decimals":2, + "mappingType":1, + "pattern":"Value #E", + "thresholds":[ + "30", + "80" + ], + "type":"number", + "unit":"percent" + }, + { + "alias":"RAM Utilization", + "colorMode":"value", + "colors":[ + "rgba(245, 54, 54, 0.9)", + "rgba(50, 172, 45, 0.97)", + "#ef843c" + ], + "dateFormat":"YYYY-MM-DD HH:mm:ss", + "decimals":2, + "mappingType":1, + "pattern":"Value #F", + "thresholds":[ + "30", + "80" + ], + "type":"number", + "unit":"percent" + } + ], + "targets":[ + { + "expr":"sum (rate (container_cpu_usage_seconds_total{namespace=\"$namespace\"}[10m])) by (pod_name) * 100", + "format":"table", + "hide":false, + "instant":true, + "interval":"", + "intervalFactor":1, + "legendFormat":"{{ pod_name }}", + "refId":"A" + }, + { + "expr":"sum (avg_over_time (container_memory_working_set_bytes{namespace=\"$namespace\", container_name!=\"POD\"}[10m])) by (pod_name)", + "format":"table", + "hide":false, + "instant":true, + "intervalFactor":1, + "legendFormat":"{{ pod_name }}", + "refId":"B" + } + ], + "timeFrom":"1M", + "timeShift":null, + "title":"Pod utilization analysis", + "transform":"table", + "transparent":false, + "type":"table" + }, + { + "columns":[ + { + "text":"Avg", + "value":"avg" + } + ], + "datasource":"${datasource}", + "fontSize":"100%", + "gridPos":{ + "h":9, + "w":8, + "x":16, + "y":0 + }, + "hideTimeOverride":true, + "id":90, + "links":[ + + ], + "pageSize":8, + "repeatDirection":"v", + "scroll":true, + "showHeader":true, + "sort":{ + "col":4, + "desc":true + }, + "styles":[ + { + "alias":"Namespace", + "colorMode":null, + "colors":[ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat":"YYYY-MM-DD HH:mm:ss", + "decimals":2, + "mappingType":1, + "pattern":"namespace", + "thresholds":[ + + ], + "type":"hidden", + "unit":"short" + }, + { + "alias":"PVC Name", + "colorMode":null, + "colors":[ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat":"YYYY-MM-DD HH:mm:ss", + "decimals":2, + "mappingType":1, + "pattern":"persistentvolumeclaim", + "thresholds":[ + + ], + "type":"number", + "unit":"short" + }, + { + "alias":"Storage Class", + "colorMode":null, + "colors":[ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat":"YYYY-MM-DD HH:mm:ss", + "decimals":2, + "mappingType":1, + "pattern":"storageclass", + "thresholds":[ + + ], + "type":"number", + "unit":"short" + }, + { + "alias":"Size", + "colorMode":null, + "colors":[ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat":"YYYY-MM-DD HH:mm:ss", + "decimals":1, + "mappingType":1, + "pattern":"Value", + "thresholds":[ + + ], + "type":"number", + "unit":"gbytes" + }, + { + "alias":"", + "colorMode":null, + "colors":[ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat":"YYYY-MM-DD HH:mm:ss", + "decimals":2, + "mappingType":1, + "pattern":"Time", + "thresholds":[ + + ], + "type":"hidden", + "unit":"short" + } + ], + "targets":[ + { + "expr":"sum (\n sum(kube_persistentvolumeclaim_info) by (persistentvolumeclaim, namespace, storageclass)\n + on (persistentvolumeclaim, namespace) group_right (storageclass)\n sum(kube_persistentvolumeclaim_resource_requests_storage_bytes{namespace=~\"$namespace\"}) by (persistentvolumeclaim, namespace)\n) by (namespace,persistentvolumeclaim,storageclass) / 1024 / 1024 /1024 ", + "format":"table", + "hide":false, + "instant":true, + "interval":"", + "intervalFactor":1, + "legendFormat":"{{ persistentvolumeclaim }}", + "refId":"A" + } + ], + "timeFrom":null, + "timeShift":null, + "title":"Persistent Volume Claims", + "transform":"table", + "transparent":false, + "type":"table" + }, + { + "aliasColors":{ + + }, + "bars":false, + "dashLength":10, + "dashes":false, + "datasource":"${datasource}", + "description":"CPU requests by pod divided by the rate of CPU usage over the last hour", + "fill":1, + "gridPos":{ + "h":9, + "w":24, + "x":0, + "y":9 + }, + "id":100, + "legend":{ + "avg":false, + "current":false, + "max":false, + "min":false, + "show":true, + "total":false, + "values":false + }, + "lines":true, + "linewidth":1, + "links":[ + + ], + "nullPointMode":"null", + "percentage":false, + "pointradius":5, + "points":false, + "renderer":"flot", + "seriesOverrides":[ + + ], + "spaceLength":10, + "stack":false, + "steppedLine":false, + "targets":[ + { + "expr":"topk(10,\n label_replace(\n sum(kube_pod_container_resource_requests{resource=\"cpu\", unit=\"core\", namespace=\"$namespace\"}) by (pod),\n \"pod_name\", \n \"$1\", \n \"pod\", \n \"(.+)\"\n ) \n/ on (pod_name) sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\",pod_name=~\".+\"}[1h])) by (pod_name))", + "format":"time_series", + "intervalFactor":1, + "refId":"A" + } + ], + "thresholds":[ + + ], + "timeFrom":null, + "timeShift":null, + "title":"Ratio of CPU requests to usage (Top 10 pods)", + "tooltip":{ + "shared":true, + "sort":0, + "value_type":"individual" + }, + "type":"graph", + "xaxis":{ + "buckets":null, + "mode":"time", + "name":null, + "show":true, + "values":[ + + ] + }, + "yaxes":[ + { + "format":"short", + "label":null, + "logBase":1, + "max":null, + "min":null, + "show":true + }, + { + "format":"short", + "label":null, + "logBase":1, + "max":null, + "min":null, + "show":true + } + ], + "yaxis":{ + "align":false, + "alignLevel":null + } + }, + { + "aliasColors":{ + + }, + "bars":false, + "dashLength":10, + "dashes":false, + "datasource":"${datasource}", + "decimals":3, + "description":"This panel shows historical utilization as an average across all pods in this namespace. It only accounts for currently deployed pods", + "editable":true, + "error":false, + "fill":0, + "grid":{ + + }, + "gridPos":{ + "h":6, + "w":12, + "x":0, + "y":18 + }, + "height":"", + "id":94, + "isNew":true, + "legend":{ + "alignAsTable":false, + "avg":false, + "current":false, + "hideEmpty":false, + "hideZero":false, + "max":false, + "min":false, + "rightSide":false, + "show":false, + "sideWidth":null, + "sort":"current", + "sortDesc":true, + "total":false, + "values":true + }, + "lines":true, + "linewidth":2, + "links":[ + + ], + "nullPointMode":"connected", + "percentage":false, + "pointradius":5, + "points":false, + "renderer":"flot", + "seriesOverrides":[ + + ], + "spaceLength":10, + "stack":false, + "steppedLine":true, + "targets":[ + { + "expr":"sum (rate (container_cpu_usage_seconds_total{namespace=\"$namespace\"}[10m])) by (namespace)\n", + "format":"time_series", + "hide":false, + "instant":false, + "interval":"10s", + "intervalFactor":1, + "legendFormat":"cpu utilization", + "metric":"container_cpu", + "refId":"A", + "step":10 + } + ], + "thresholds":[ + + ], + "timeFrom":"", + "timeShift":null, + "title":"Overall CPU Utilization", + "tooltip":{ + "msResolution":true, + "shared":true, + "sort":2, + "value_type":"cumulative" + }, + "type":"graph", + "xaxis":{ + "buckets":null, + "mode":"time", + "name":null, + "show":true, + "values":[ + + ] + }, + "yaxes":[ + { + "decimals":null, + "format":"percent", + "label":"", + "logBase":1, + "max":"110", + "min":"0", + "show":true + }, + { + "format":"short", + "label":null, + "logBase":1, + "max":null, + "min":null, + "show":false + } + ], + "yaxis":{ + "align":false, + "alignLevel":null + } + }, + { + "aliasColors":{ + + }, + "bars":false, + "dashLength":10, + "dashes":false, + "datasource":"${datasource}", + "decimals":2, + "description":"This panel shows historical utilization as an average across all pods in this namespace. It only accounts for currently deployed pods", + "editable":true, + "error":false, + "fill":0, + "grid":{ + + }, + "gridPos":{ + "h":6, + "w":12, + "x":12, + "y":18 + }, + "id":92, + "isNew":true, + "legend":{ + "alignAsTable":false, + "avg":false, + "current":false, + "max":false, + "min":false, + "rightSide":false, + "show":false, + "sideWidth":200, + "sort":"current", + "sortDesc":true, + "total":false, + "values":true + }, + "lines":true, + "linewidth":2, + "links":[ + + ], + "nullPointMode":"connected", + "percentage":false, + "pointradius":5, + "points":false, + "renderer":"flot", + "seriesOverrides":[ + + ], + "spaceLength":10, + "stack":false, + "steppedLine":true, + "targets":[ + { + "expr":"sum (container_memory_working_set_bytes{namespace=\"$namespace\"})\n/\nsum(node_memory_MemTotal_bytes)", + "format":"time_series", + "instant":false, + "intervalFactor":1, + "legendFormat":"mem utilization", + "refId":"B" + } + ], + "thresholds":[ + + ], + "timeFrom":"", + "timeShift":null, + "title":"Overall RAM Utilization", + "tooltip":{ + "msResolution":false, + "shared":true, + "sort":2, + "value_type":"cumulative" + }, + "type":"graph", + "xaxis":{ + "buckets":null, + "mode":"time", + "name":null, + "show":true, + "values":[ + + ] + }, + "yaxes":[ + { + "decimals":null, + "format":"percent", + "label":null, + "logBase":1, + "max":"110", + "min":"0", + "show":true + }, + { + "format":"short", + "label":null, + "logBase":1, + "max":null, + "min":null, + "show":false + } + ], + "yaxis":{ + "align":false, + "alignLevel":null + } + }, + { + "aliasColors":{ + + }, + "bars":false, + "dashLength":10, + "dashes":false, + "datasource":"${datasource}", + "decimals":2, + "description":"Traffic in and out of this namespace, as a sum of the pods within it", + "editable":true, + "error":false, + "fill":1, + "grid":{ + + }, + "gridPos":{ + "h":6, + "w":12, + "x":0, + "y":24 + }, + "height":"", + "id":96, + "isNew":true, + "legend":{ + "alignAsTable":false, + "avg":true, + "current":true, + "hideEmpty":false, + "hideZero":false, + "max":false, + "min":false, + "rightSide":false, + "show":true, + "sideWidth":null, + "sort":"current", + "sortDesc":true, + "total":false, + "values":true + }, + "lines":true, + "linewidth":2, + "links":[ + + ], + "nullPointMode":"connected", + "percentage":false, + "pointradius":5, + "points":false, + "renderer":"flot", + "seriesOverrides":[ + + ], + "spaceLength":10, + "stack":false, + "steppedLine":false, + "targets":[ + { + "expr":"sum (rate (container_network_receive_bytes_total{namespace=\"$namespace\"}[10m])) by (namespace)", + "format":"time_series", + "hide":false, + "instant":false, + "interval":"", + "intervalFactor":1, + "legendFormat":"<- in", + "metric":"container_cpu", + "refId":"A", + "step":10 + }, + { + "expr":"- sum (rate (container_network_transmit_bytes_total{namespace=\"$namespace\"}[10m])) by (namespace)", + "format":"time_series", + "hide":false, + "instant":false, + "interval":"", + "intervalFactor":1, + "legendFormat":"-> out", + "refId":"B" + } + ], + "thresholds":[ + + ], + "timeFrom":"", + "timeShift":null, + "title":"Network IO", + "tooltip":{ + "msResolution":true, + "shared":true, + "sort":2, + "value_type":"cumulative" + }, + "type":"graph", + "xaxis":{ + "buckets":null, + "mode":"time", + "name":null, + "show":true, + "values":[ + + ] + }, + "yaxes":[ + { + "format":"Bps", + "label":"", + "logBase":1, + "max":null, + "min":null, + "show":true + }, + { + "format":"short", + "label":null, + "logBase":1, + "max":null, + "min":null, + "show":false + } + ], + "yaxis":{ + "align":false, + "alignLevel":null + } + }, + { + "aliasColors":{ + + }, + "bars":false, + "dashLength":10, + "dashes":false, + "datasource":"${datasource}", + "decimals":2, + "description":"Disk reads and writes for the namespace, as a sum of the pods within it", + "editable":true, + "error":false, + "fill":1, + "grid":{ + + }, + "gridPos":{ + "h":6, + "w":12, + "x":12, + "y":24 + }, + "height":"", + "id":98, + "isNew":true, + "legend":{ + "alignAsTable":false, + "avg":true, + "current":true, + "hideEmpty":false, + "hideZero":false, + "max":false, + "min":false, + "rightSide":false, + "show":true, + "sideWidth":null, + "sort":"current", + "sortDesc":true, + "total":false, + "values":true + }, + "lines":true, + "linewidth":2, + "links":[ + + ], + "nullPointMode":"connected", + "percentage":false, + "pointradius":5, + "points":false, + "renderer":"flot", + "seriesOverrides":[ + + ], + "spaceLength":10, + "stack":false, + "steppedLine":false, + "targets":[ + { + "expr":"sum (rate (container_fs_writes_bytes_total{namespace=\"$namespace\"}[10m])) by (namespace)", + "format":"time_series", + "hide":false, + "instant":false, + "interval":"", + "intervalFactor":1, + "legendFormat":"<- write", + "metric":"container_cpu", + "refId":"A", + "step":10 + }, + { + "expr":"- sum (rate (container_fs_reads_bytes_total{namespace=\"$namespace\"}[10m])) by (namespace)", + "format":"time_series", + "hide":false, + "instant":false, + "interval":"", + "intervalFactor":1, + "legendFormat":"-> read", + "refId":"B" + } + ], + "thresholds":[ + + ], + "timeFrom":"", + "timeShift":null, + "title":"Disk IO", + "tooltip":{ + "msResolution":true, + "shared":true, + "sort":2, + "value_type":"cumulative" + }, + "type":"graph", + "xaxis":{ + "buckets":null, + "mode":"time", + "name":null, + "show":true, + "values":[ + + ] + }, + "yaxes":[ + { + "format":"Bps", + "label":"", + "logBase":1, + "max":null, + "min":null, + "show":true + }, + { + "format":"short", + "label":null, + "logBase":1, + "max":null, + "min":null, + "show":false + } + ], + "yaxis":{ + "align":false, + "alignLevel":null + } + } + ], + "refresh":"10s", + "schemaVersion":16, + "style":"dark", + "tags":[ + "cost", + "utilization", + "metrics" + ], + "templating":{ + "list":[ + { + "current":{ + "text":"23.06", + "value":"23.06" + }, + "hide":0, + "label":"CPU", + "name":"costcpu", + "options":[ + { + "text":"23.06", + "value":"23.06" + } + ], + "query":"23.06", + "skipUrlSync":false, + "type":"constant" + }, + { + "current":{ + "text":"7.28", + "value":"7.28" + }, + "hide":0, + "label":"PE CPU", + "name":"costpcpu", + "options":[ + { + "text":"7.28", + "value":"7.28" + } + ], + "query":"7.28", + "skipUrlSync":false, + "type":"constant" + }, + { + "current":{ + "text":"3.25", + "value":"3.25" + }, + "hide":0, + "label":"RAM", + "name":"costram", + "options":[ + { + "text":"3.25", + "value":"3.25" + } + ], + "query":"3.25", + "skipUrlSync":false, + "type":"constant" + }, + { + "current":{ + "text":"0.6862", + "value":"0.6862" + }, + "hide":0, + "label":"PE RAM", + "name":"costpram", + "options":[ + { + "text":"0.6862", + "value":"0.6862" + } + ], + "query":"0.6862", + "skipUrlSync":false, + "type":"constant" + }, + { + "current":{ + "text":"0.04", + "value":"0.04" + }, + "hide":0, + "label":"Storage", + "name":"costStorageStandard", + "options":[ + { + "text":"0.04", + "value":"0.04" + } + ], + "query":"0.04", + "skipUrlSync":false, + "type":"constant" + }, + { + "current":{ + "text":".17", + "value":".17" + }, + "hide":0, + "label":"SSD", + "name":"costStorageSSD", + "options":[ + { + "text":".17", + "value":".17" + } + ], + "query":".17", + "skipUrlSync":false, + "type":"constant" + }, + { + "current":{ + "text":"30", + "value":"30" + }, + "hide":0, + "label":"Disc.", + "name":"costDiscount", + "options":[ + { + "text":"30", + "value":"30" + } + ], + "query":"30", + "skipUrlSync":false, + "type":"constant" + }, + { + "allValue":null, + "current":{ + "text":"kube-system", + "value":"kube-system" + }, + "datasource":"${datasource}", + "hide":0, + "includeAll":false, + "label":"NS", + "multi":false, + "name":"namespace", + "options":[ + + ], + "query":"query_result(sum(kube_namespace_created{namespace!=\"\"}) by (namespace))", + "refresh":1, + "regex":"/namespace=\\\"(.*?)(\\\")/", + "skipUrlSync":false, + "sort":0, + "tagValuesQuery":"", + "tags":[ + + ], + "tagsQuery":"", + "type":"query", + "useTags":false + }, + { + "datasource":"${datasource}", + "filters":[ + + ], + "hide":0, + "label":"", + "name":"Filters", + "skipUrlSync":false, + "type":"adhoc" + }, + { + "current": { + "selected": true, + "text": "default-kubecost", + "value": "default-kubecost" + }, + "error": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time":{ + "from":"now-15m", + "to":"now" + }, + "timepicker":{ + "hidden":false, + "refresh_intervals":[ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options":[ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone":"browser", + "title":"Namespace utilization metrics", + "uid":"at-cost-analysis-namespace2", + "version":1 +} diff --git a/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/network-cloud-services.json b/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/network-cloud-services.json new file mode 100644 index 0000000000..2729b6ca7d --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/network-cloud-services.json @@ -0,0 +1,408 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Most used metrics when troubleshooting applications", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 14, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum(\n rate(kubecost_pod_network_ingress_bytes_total\n {namespace=~\"$namespace\", pod_name=~\"$pod\",service!=\"\",service=~\"$service\"}\n [1h]\n )\n) \nby (namespace,service) ", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "1h", + "legendFormat": "{{namespace}}/{{service}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "- sum(\n rate(kubecost_pod_network_egress_bytes_total\n {namespace=~\"$namespace\", pod_name=~\"$pod\",service!=\"\",service=~\"$service\"}\n [1h]\n )\n) \nby(namespace, service) ", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "interval": "1h", + "legendFormat": "{{namespace}}/{{service}}", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Kubecost Network Cloud Service by Namespace (egress is negative)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum(topk(5,\n rate(kubecost_pod_network_ingress_bytes_total\n {namespace=~\"$namespace\", pod_name=~\"$pod\",service!=\"\",service=~\"$service\"}\n [1h]\n )\n) )\nby(namespace, pod_name,service) ", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "1h", + "legendFormat": "{{namespace}}/{{pod_name}}/{{service}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "- sum(topk(5,\n rate(kubecost_pod_network_egress_bytes_total\n {namespace=~\"$namespace\", pod_name=~\"$pod\",service!=\"\",service=~\"$service\"}\n [1h]\n )\n) )\nby(namespace, pod_name,service) ", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "interval": "1h", + "legendFormat": "{{namespace}}/{{pod_name}}/{{service}}", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Kubecost Network Cloud Service by Pod (egress is negative)", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 39, + "tags": [ + "utilization", + "metrics", + "kubecost" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "PBFA97CFB590B2093" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(kube_namespace_labels,namespace)", + "hide": 0, + "includeAll": true, + "multi": false, + "name": "namespace", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(kube_namespace_labels,namespace)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(kube_pod_owner{namespace=~\"$namespace\"},pod)", + "hide": 0, + "includeAll": true, + "multi": false, + "name": "pod", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(kube_pod_owner{namespace=~\"$namespace\"},pod)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(kube_pod_container_status_running{namespace=\"$namespace\"},container)", + "hide": 0, + "includeAll": true, + "multi": false, + "name": "container", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(kube_pod_container_status_running{namespace=\"$namespace\"},container)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "type": "query" + }, + { + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "definition": "label_values(kubecost_pod_network_egress_bytes_total{namespace=~\"$namespace\"},service)", + "hide": 0, + "includeAll": true, + "multi": false, + "name": "service", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(kubecost_pod_network_egress_bytes_total{namespace=~\"$namespace\"},service)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "type": "query" + } + ] + }, + "time": { + "from": "now-2d", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Kubecost Network Cloud Service Metrics", + "uid": "kubecost-network-cloud-services", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/networkCosts-metrics.json b/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/networkCosts-metrics.json new file mode 100644 index 0000000000..79e568ccb1 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/networkCosts-metrics.json @@ -0,0 +1,672 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "https://docs.kubecost.com/install-and-configure/advanced-configuration/network-costs-configuration", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 2, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 12, + "panels": [], + "title": "Network Data Transfers (negative is egress data)", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 11, + "x": 0, + "y": 1 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(increase(kubecost_pod_network_ingress_bytes_total\n {namespace=~\"$namespace\", cluster_id=~\"$cluster\", pod_name=~\"$pod_name\", service=~\"$service\"}\n [1h]\n ))\nby($aggregation) ", + "interval": "1h", + "legendFormat": "__auto", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "-sum(increase(kubecost_pod_network_egress_bytes_total\n {namespace=~\"$namespace\", cluster_id=~\"$cluster\", pod_name=~\"$pod_name\", service=~\"$service\"}\n [1h]\n ))\nby($aggregation) ", + "hide": false, + "interval": "1h", + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "All Data", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 13, + "x": 11, + "y": 1 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(increase(kubecost_pod_network_ingress_bytes_total\n {internet=\"true\", namespace=~\"$namespace\", cluster_id=~\"$cluster\", pod_name=~\"$pod_name\", service=~\"$service\"}\n [1h]\n))\nby($aggregation) ", + "hide": false, + "interval": "1h", + "legendFormat": "__auto", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "- sum(increase(kubecost_pod_network_egress_bytes_total\n {internet=\"true\", namespace=~\"$namespace\", cluster_id=~\"$cluster\", pod_name=~\"$pod_name\", service=~\"$service\"}\n [1h]))\nby($aggregation) ", + "hide": false, + "interval": "1h", + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "Internet Data", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Cross region and cross zone subnets must be defined via the configMap. \nSee: \n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 11, + "x": 0, + "y": 15 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(increase(kubecost_pod_network_ingress_bytes_total\n {internet=\"false\",namespace=~\"$namespace\",cluster_id=~\"$cluster\",pod_name=~\"$pod_name\", sameRegion=\"false\",sameZone=\"false\", service=~\"$service\"}\n [1h]))\nby($aggregation)", + "interval": "1h", + "legendFormat": "__auto", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "- sum(increase(kubecost_pod_network_egress_bytes_total\n {internet=\"false\", namespace=~\"$namespace\",cluster_id=~\"$cluster\",pod_name=~\"$pod_name\",sameRegion=\"false\", sameZone=\"false\", service=~\"$service\"}\n [1h]))\nby($aggregation) ", + "hide": false, + "interval": "1h", + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "Cross Region Data", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Cross region and cross zone subnets must be defined via the configMap. \nSee: \n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 13, + "x": 11, + "y": 15 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(increase(kubecost_pod_network_ingress_bytes_total\n {internet=\"false\", namespace=~\"$namespace\", cluster_id=~\"$cluster\", pod_name=~\"$pod_name\", sameRegion=\"true\", sameZone=\"false\", service=~\"$service\"}\n [1h]))\nby($aggregation)", + "interval": "1h", + "legendFormat": "__auto", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "- sum(increase(kubecost_pod_network_egress_bytes_total\n {internet=\"false\", namespace=~\"$namespace\", cluster_id=~\"$cluster\", pod_name=~\"$pod_name\", sameRegion=\"true\", sameZone=\"false\", service=~\"$service\"}\n [1h]))\nby($aggregation)", + "hide": false, + "interval": "1h", + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "Cross Zone Data", + "type": "timeseries" + } + ], + "refresh": "", + "revision": 1, + "schemaVersion": 39, + "tags": [ + "utilization", + "metrics", + "kubecost" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "PBFA97CFB590B2093" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "namespace", + "value": "namespace" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "aggregation", + "options": [ + { + "selected": false, + "text": "cluster_id", + "value": "cluster_id" + }, + { + "selected": true, + "text": "namespace", + "value": "namespace" + }, + { + "selected": false, + "text": "pod_name", + "value": "pod_name" + } + ], + "query": "cluster_id, namespace, pod_name", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(cluster_id)", + "hide": 0, + "includeAll": true, + "multi": false, + "name": "cluster", + "options": [], + "query": { + "query": "label_values(cluster_id)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "type": "query" + }, + { + "current": { + "selected": true, + "text": "kubecost", + "value": "kubecost" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(kube_namespace_labels{cluster_id=~\"$cluster\"}, namespace) ", + "hide": 0, + "includeAll": true, + "multi": false, + "name": "namespace", + "options": [], + "query": { + "query": "label_values(kube_namespace_labels{cluster_id=~\"$cluster\"}, namespace) ", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(kubecost_pod_network_egress_bytes_total{cluster_id=~\"$cluster\", namespace=~\"$namespace\"},pod_name)", + "hide": 0, + "includeAll": true, + "multi": false, + "name": "pod_name", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(kubecost_pod_network_egress_bytes_total{cluster_id=~\"$cluster\", namespace=~\"$namespace\"},pod_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "filters": [], + "hide": 0, + "name": "filter", + "skipUrlSync": false, + "type": "adhoc" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "definition": "label_values(kubecost_pod_network_egress_bytes_total{namespace=~\"$namespace\"},service)", + "hide": 0, + "includeAll": true, + "multi": false, + "name": "service", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(kubecost_pod_network_egress_bytes_total{namespace=~\"$namespace\"},service)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "type": "query" + } + ] + }, + "time": { + "from": "now-7d", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Kubecost Network Costs Metrics", + "uid": "kubecost-networkCosts-metrics", + "version": 2, + "weekStart": "" +} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/node-utilization.json b/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/node-utilization.json new file mode 100644 index 0000000000..dc03cc0743 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/node-utilization.json @@ -0,0 +1,1389 @@ +{ + "annotations":{ + "list":[ + { + "builtIn":1, + "datasource":"-- Grafana --", + "enable":true, + "hide":true, + "iconColor":"rgba(0, 211, 255, 1)", + "name":"Annotations & Alerts", + "type":"dashboard" + } + ] + }, + "editable":true, + "gnetId":null, + "graphTooltip":0, + "id":6, + "iteration":1557245882378, + "links":[ + + ], + "panels":[ + { + "cacheTimeout":null, + "colorBackground":false, + "colorValue":false, + "colors":[ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource":"${datasource}", + "format":"percentunit", + "gauge":{ + "maxValue":100, + "minValue":0, + "show":true, + "thresholdLabels":false, + "thresholdMarkers":true + }, + "gridPos":{ + "h":7, + "w":8, + "x":0, + "y":0 + }, + "id":2, + "interval":null, + "links":[ + + ], + "mappingType":1, + "mappingTypes":[ + { + "name":"value to text", + "value":1 + }, + { + "name":"range to text", + "value":2 + } + ], + "maxDataPoints":100, + "nullPointMode":"connected", + "nullText":null, + "postfix":"", + "postfixFontSize":"50%", + "prefix":"", + "prefixFontSize":"50%", + "rangeMaps":[ + { + "from":"null", + "text":"N/A", + "to":"null" + } + ], + "sparkline":{ + "fillColor":"rgba(31, 118, 189, 0.18)", + "full":false, + "lineColor":"rgb(31, 120, 193)", + "show":false + }, + "tableColumn":"", + "targets":[ + { + "expr":"sum(irate(container_cpu_usage_seconds_total{id=\"/\",instance=\"$node\"}[10m]))", + "format":"time_series", + "intervalFactor":1, + "refId":"A" + } + ], + "thresholds":"", + "title":"CPU Usage", + "type":"singlestat", + "valueFontSize":"80%", + "valueMaps":[ + { + "op":"=", + "text":"N/A", + "value":"null" + } + ], + "valueName":"avg" + }, + { + "cacheTimeout":null, + "colorBackground":false, + "colorValue":false, + "colors":[ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource":"${datasource}", + "format":"percentunit", + "gauge":{ + "maxValue":100, + "minValue":0, + "show":true, + "thresholdLabels":false, + "thresholdMarkers":true + }, + "gridPos":{ + "h":7, + "w":8, + "x":8, + "y":0 + }, + "id":3, + "interval":null, + "links":[ + + ], + "mappingType":1, + "mappingTypes":[ + { + "name":"value to text", + "value":1 + }, + { + "name":"range to text", + "value":2 + } + ], + "maxDataPoints":100, + "nullPointMode":"connected", + "nullText":null, + "postfix":"", + "postfixFontSize":"50%", + "prefix":"", + "prefixFontSize":"50%", + "rangeMaps":[ + { + "from":"null", + "text":"N/A", + "to":"null" + } + ], + "sparkline":{ + "fillColor":"rgba(31, 118, 189, 0.18)", + "full":false, + "lineColor":"rgb(31, 120, 193)", + "show":false + }, + "tableColumn":"", + "targets":[ + { + "expr":"SUM(container_memory_usage_bytes{namespace!=\"\",instance=\"$node\"}) / SUM(kube_node_status_capacity{resource=\"memory\", unit=\"byte\", node=\"$node\"})", + "format":"time_series", + "intervalFactor":1, + "refId":"A" + } + ], + "thresholds":"", + "title":"Memory Usage", + "type":"singlestat", + "valueFontSize":"80%", + "valueMaps":[ + { + "op":"=", + "text":"N/A", + "value":"null" + } + ], + "valueName":"avg" + }, + { + "cacheTimeout":null, + "colorBackground":false, + "colorValue":false, + "colors":[ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource":"${datasource}", + "format":"percentunit", + "gauge":{ + "maxValue":100, + "minValue":0, + "show":true, + "thresholdLabels":false, + "thresholdMarkers":true + }, + "gridPos":{ + "h":7, + "w":8, + "x":16, + "y":0 + }, + "id":4, + "interval":null, + "links":[ + + ], + "mappingType":1, + "mappingTypes":[ + { + "name":"value to text", + "value":1 + }, + { + "name":"range to text", + "value":2 + } + ], + "maxDataPoints":100, + "nullPointMode":"connected", + "nullText":null, + "postfix":"", + "postfixFontSize":"50%", + "prefix":"", + "prefixFontSize":"50%", + "rangeMaps":[ + { + "from":"null", + "text":"N/A", + "to":"null" + } + ], + "sparkline":{ + "fillColor":"rgba(31, 118, 189, 0.18)", + "full":false, + "lineColor":"rgb(31, 120, 193)", + "show":false + }, + "tableColumn":"", + "targets":[ + { + "expr":"sum(container_fs_usage_bytes{device=~\"^/dev/[sv]d[a-z][1-9]$\",id=\"/\",instance=\"$node\"}) /\nsum(container_fs_limit_bytes{device=~\"^/dev/[sv]d[a-z][1-9]$\",id=\"/\",instance=\"$node\"})", + "format":"time_series", + "intervalFactor":1, + "refId":"A" + } + ], + "thresholds":"", + "title":"Storage Usage", + "type":"singlestat", + "valueFontSize":"80%", + "valueMaps":[ + { + "op":"=", + "text":"N/A", + "value":"null" + } + ], + "valueName":"avg" + }, + { + "columns":[ + { + "text":"Avg", + "value":"avg" + } + ], + "datasource":"${datasource}", + "fontSize":"100%", + "gridPos":{ + "h":8, + "w":16, + "x":0, + "y":7 + }, + "hideTimeOverride":true, + "id":21, + "links":[ + + ], + "pageSize":8, + "repeat":null, + "repeatDirection":"v", + "scroll":true, + "showHeader":true, + "sort":{ + "col":4, + "desc":true + }, + "styles":[ + { + "alias":"Pod", + "colorMode":null, + "colors":[ + "rgba(245, 54, 54, 0.9)", + "rgba(50, 172, 45, 0.97)", + "#c15c17" + ], + "dateFormat":"YYYY-MM-DD HH:mm:ss", + "decimals":2, + "link":false, + "linkTooltip":"", + "linkUrl":"", + "pattern":"pod_name", + "thresholds":[ + "30", + "80" + ], + "type":"string", + "unit":"currencyUSD" + }, + { + "alias":"", + "colorMode":null, + "colors":[ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat":"YYYY-MM-DD HH:mm:ss", + "decimals":2, + "mappingType":1, + "pattern":"Time", + "thresholds":[ + + ], + "type":"hidden", + "unit":"short" + }, + { + "alias":"CPU Usage", + "colorMode":null, + "colors":[ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat":"YYYY-MM-DD HH:mm:ss", + "decimals":2, + "mappingType":1, + "pattern":"Value #C", + "thresholds":[ + + ], + "type":"number", + "unit":"short" + }, + { + "alias":"CPU Request", + "colorMode":null, + "colors":[ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat":"YYYY-MM-DD HH:mm:ss", + "decimals":2, + "mappingType":1, + "pattern":"Value #A", + "thresholds":[ + + ], + "type":"number", + "unit":"short" + }, + { + "alias":"CPU Limit", + "colorMode":null, + "colors":[ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat":"YYYY-MM-DD HH:mm:ss", + "decimals":2, + "mappingType":1, + "pattern":"Value #B", + "thresholds":[ + + ], + "type":"number", + "unit":"short" + }, + { + "alias":"Mem Usage", + "colorMode":null, + "colors":[ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat":"YYYY-MM-DD HH:mm:ss", + "decimals":2, + "mappingType":1, + "pattern":"Value #D", + "thresholds":[ + + ], + "type":"number", + "unit":"bytes" + }, + { + "alias":"Mem Request", + "colorMode":null, + "colors":[ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat":"YYYY-MM-DD HH:mm:ss", + "decimals":2, + "mappingType":1, + "pattern":"Value #E", + "thresholds":[ + + ], + "type":"number", + "unit":"bytes" + }, + { + "alias":"Mem Limit", + "colorMode":null, + "colors":[ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat":"YYYY-MM-DD HH:mm:ss", + "decimals":2, + "mappingType":1, + "pattern":"Value #F", + "thresholds":[ + + ], + "type":"number", + "unit":"bytes" + } + ], + "targets":[ + { + "expr":"sum(rate(container_cpu_usage_seconds_total{container_name!=\"\",container_name!=\"POD\",pod_name!=\"\",instance=\"$node\"}[24h])) by (pod_name)", + "format":"table", + "instant":true, + "intervalFactor":1, + "refId":"C" + }, + { + "expr":"sum(label_replace(\nsum(avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\", unit=\"core\", container!=\"\",container!=\"POD\",node=\"$node\"}[24h])) by (pod), \n\"pod_name\",\"$1\",\"pod\",\"(.+)\")\nor up * 0\n) by (pod_name)", + "format":"table", + "instant":true, + "intervalFactor":1, + "refId":"A" + }, + { + "expr":"sum(avg_over_time(container_memory_usage_bytes{container_name!=\"\",container_name!=\"POD\",pod_name!=\"\",instance=\"$node\"}[24h])) by (pod_name)\n", + "format":"table", + "instant":true, + "intervalFactor":1, + "refId":"D" + }, + { + "expr":"sum(label_replace(label_replace(\nsum(avg_over_time(kube_pod_container_resource_requests{resource=\"memory\", unit=\"byte\", container!=\"\",container!=\"POD\",node=\"$node\"}[24h])) by (pod),\n\"container_name\",\"$1\",\"container\",\"(.+)\"), \"pod_name\",\"$1\",\"pod\",\"(.+)\")\nor up * 0\n) by (pod_name)\n", + "format":"table", + "instant":true, + "intervalFactor":1, + "refId":"E" + } + ], + "timeFrom":"1M", + "timeShift":null, + "title":"Current pods", + "transform":"table", + "transparent":false, + "type":"table" + }, + { + "cacheTimeout":null, + "colorBackground":false, + "colorValue":false, + "colors":[ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource":"${datasource}", + "decimals":0, + "format":"none", + "gauge":{ + "maxValue":100, + "minValue":0, + "show":false, + "thresholdLabels":false, + "thresholdMarkers":true + }, + "gridPos":{ + "h":4, + "w":4, + "x":16, + "y":7 + }, + "id":8, + "interval":null, + "links":[ + + ], + "mappingType":1, + "mappingTypes":[ + { + "name":"value to text", + "value":1 + }, + { + "name":"range to text", + "value":2 + } + ], + "maxDataPoints":100, + "nullPointMode":"connected", + "nullText":null, + "postfix":"", + "postfixFontSize":"50%", + "prefix":"", + "prefixFontSize":"50%", + "rangeMaps":[ + { + "from":"null", + "text":"N/A", + "to":"null" + } + ], + "sparkline":{ + "fillColor":"rgba(31, 118, 189, 0.18)", + "full":false, + "lineColor":"rgb(31, 120, 193)", + "show":false + }, + "tableColumn":"", + "targets":[ + { + "expr":"sum(\n count(avg_over_time(kube_pod_container_resource_requests{resource=\"cpu\", unit=\"core\", container!=\"\",container!=\"POD\",node=\"$node\"}[24h])) by (pod)\n * on (pod) group_right()\n sum(kube_pod_container_status_running) by (pod)\n)", + "format":"time_series", + "instant":true, + "intervalFactor":1, + "refId":"A" + } + ], + "thresholds":"", + "title":"Pods Running", + "type":"singlestat", + "valueFontSize":"80%", + "valueMaps":[ + { + "op":"=", + "text":"N/A", + "value":"null" + } + ], + "valueName":"current" + }, + { + "cacheTimeout":null, + "colorBackground":false, + "colorValue":false, + "colors":[ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource":"${datasource}", + "format":"bytes", + "gauge":{ + "maxValue":100, + "minValue":0, + "show":false, + "thresholdLabels":false, + "thresholdMarkers":true + }, + "gridPos":{ + "h":4, + "w":4, + "x":20, + "y":7 + }, + "id":18, + "interval":null, + "links":[ + + ], + "mappingType":1, + "mappingTypes":[ + { + "name":"value to text", + "value":1 + }, + { + "name":"range to text", + "value":2 + } + ], + "maxDataPoints":100, + "nullPointMode":"connected", + "nullText":null, + "postfix":"", + "postfixFontSize":"50%", + "prefix":"", + "prefixFontSize":"50%", + "rangeMaps":[ + { + "from":"null", + "text":"N/A", + "to":"null" + } + ], + "sparkline":{ + "fillColor":"rgba(31, 118, 189, 0.18)", + "full":false, + "lineColor":"rgb(31, 120, 193)", + "show":false + }, + "tableColumn":"", + "targets":[ + { + "expr":"sum(container_fs_limit_bytes{device=~\"^/dev/[sv]d[a-z][1-9]$\",id=\"/\",instance=\"$node\"})", + "format":"time_series", + "intervalFactor":1, + "refId":"A" + } + ], + "thresholds":"", + "title":"Storage Capacity", + "type":"singlestat", + "valueFontSize":"80%", + "valueMaps":[ + { + "op":"=", + "text":"N/A", + "value":"null" + } + ], + "valueName":"current" + }, + { + "cacheTimeout":null, + "colorBackground":false, + "colorValue":false, + "colors":[ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource":"${datasource}", + "format":"none", + "gauge":{ + "maxValue":100, + "minValue":0, + "show":false, + "thresholdLabels":false, + "thresholdMarkers":true + }, + "gridPos":{ + "h":4, + "w":4, + "x":16, + "y":11 + }, + "id":9, + "interval":null, + "links":[ + + ], + "mappingType":1, + "mappingTypes":[ + { + "name":"value to text", + "value":1 + }, + { + "name":"range to text", + "value":2 + } + ], + "maxDataPoints":100, + "nullPointMode":"connected", + "nullText":null, + "postfix":"", + "postfixFontSize":"50%", + "prefix":"", + "prefixFontSize":"50%", + "rangeMaps":[ + { + "from":"null", + "text":"N/A", + "to":"null" + } + ], + "sparkline":{ + "fillColor":"rgba(31, 118, 189, 0.18)", + "full":false, + "lineColor":"rgb(31, 120, 193)", + "show":false + }, + "tableColumn":"", + "targets":[ + { + "expr":"kube_node_status_capacity{resource=\"cpu\", unit=\"core\", node=\"$node\"}", + "format":"time_series", + "intervalFactor":1, + "refId":"A" + } + ], + "thresholds":"", + "title":"CPU Capacity", + "type":"singlestat", + "valueFontSize":"80%", + "valueMaps":[ + { + "op":"=", + "text":"N/A", + "value":"null" + } + ], + "valueName":"avg" + }, + { + "cacheTimeout":null, + "colorBackground":false, + "colorValue":false, + "colors":[ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource":"${datasource}", + "format":"bytes", + "gauge":{ + "maxValue":100, + "minValue":0, + "show":false, + "thresholdLabels":false, + "thresholdMarkers":true + }, + "gridPos":{ + "h":4, + "w":4, + "x":20, + "y":11 + }, + "id":19, + "interval":null, + "links":[ + + ], + "mappingType":1, + "mappingTypes":[ + { + "name":"value to text", + "value":1 + }, + { + "name":"range to text", + "value":2 + } + ], + "maxDataPoints":100, + "nullPointMode":"connected", + "nullText":null, + "postfix":"", + "postfixFontSize":"50%", + "prefix":"", + "prefixFontSize":"50%", + "rangeMaps":[ + { + "from":"null", + "text":"N/A", + "to":"null" + } + ], + "sparkline":{ + "fillColor":"rgba(31, 118, 189, 0.18)", + "full":false, + "lineColor":"rgb(31, 120, 193)", + "show":false + }, + "tableColumn":"", + "targets":[ + { + "expr":"kube_node_status_capacity{resource=\"memory\", unit=\"byte\", node=\"$node\"}", + "format":"time_series", + "intervalFactor":1, + "refId":"A" + } + ], + "thresholds":"", + "title":"RAM Capacity", + "type":"singlestat", + "valueFontSize":"80%", + "valueMaps":[ + { + "op":"=", + "text":"N/A", + "value":"null" + } + ], + "valueName":"current" + }, + { + "aliasColors":{ + + }, + "bars":false, + "dashLength":10, + "dashes":false, + "datasource":"${datasource}", + "decimals":3, + "description":"This panel shows historical utilization for the node.", + "editable":true, + "error":false, + "fill":0, + "grid":{ + + }, + "gridPos":{ + "h":6, + "w":12, + "x":0, + "y":15 + }, + "height":"", + "id":11, + "isNew":true, + "legend":{ + "alignAsTable":false, + "avg":false, + "current":false, + "hideEmpty":false, + "hideZero":false, + "max":false, + "min":false, + "rightSide":false, + "show":false, + "sideWidth":null, + "sort":"current", + "sortDesc":true, + "total":false, + "values":true + }, + "lines":true, + "linewidth":2, + "links":[ + + ], + "nullPointMode":"connected", + "percentage":false, + "pointradius":5, + "points":false, + "renderer":"flot", + "seriesOverrides":[ + + ], + "spaceLength":10, + "stack":false, + "steppedLine":true, + "targets":[ + { + "expr":"sum(irate(container_cpu_usage_seconds_total{id=\"/\",instance=\"$node\"}[10m]))", + "format":"time_series", + "hide":false, + "instant":false, + "interval":"10s", + "intervalFactor":1, + "legendFormat":"cpu utilization", + "metric":"container_cpu", + "refId":"A", + "step":10 + } + ], + "thresholds":[ + + ], + "timeFrom":"", + "timeShift":null, + "title":"CPU Utilization", + "tooltip":{ + "msResolution":true, + "shared":true, + "sort":2, + "value_type":"cumulative" + }, + "type":"graph", + "xaxis":{ + "buckets":null, + "mode":"time", + "name":null, + "show":true, + "values":[ + + ] + }, + "yaxes":[ + { + "decimals":null, + "format":"percentunit", + "label":"", + "logBase":1, + "max":"1.1", + "min":"0", + "show":true + }, + { + "format":"short", + "label":null, + "logBase":1, + "max":null, + "min":null, + "show":false + } + ], + "yaxis":{ + "align":false, + "alignLevel":null + } + }, + { + "aliasColors":{ + + }, + "bars":false, + "dashLength":10, + "dashes":false, + "datasource":"${datasource}", + "decimals":2, + "description":"This panel shows historical utilization for the node.", + "editable":true, + "error":false, + "fill":0, + "grid":{ + + }, + "gridPos":{ + "h":6, + "w":12, + "x":12, + "y":15 + }, + "id":13, + "isNew":true, + "legend":{ + "alignAsTable":false, + "avg":false, + "current":false, + "max":false, + "min":false, + "rightSide":false, + "show":false, + "sideWidth":200, + "sort":"current", + "sortDesc":true, + "total":false, + "values":true + }, + "lines":true, + "linewidth":2, + "links":[ + + ], + "nullPointMode":"connected", + "percentage":false, + "pointradius":5, + "points":false, + "renderer":"flot", + "seriesOverrides":[ + + ], + "spaceLength":10, + "stack":false, + "steppedLine":true, + "targets":[ + { + "expr":"SUM(container_memory_usage_bytes{namespace!=\"\",instance=\"$node\"}) / SUM(kube_node_status_capacity{resource=\"memory\", unit=\"byte\", node=\"$node\"})", + "format":"time_series", + "instant":false, + "interval":"10s", + "intervalFactor":1, + "legendFormat":"ram utilization", + "metric":"container_memory_usage:sort_desc", + "refId":"A", + "step":10 + } + ], + "thresholds":[ + + ], + "timeFrom":"", + "timeShift":null, + "title":"RAM Utilization", + "tooltip":{ + "msResolution":false, + "shared":true, + "sort":2, + "value_type":"cumulative" + }, + "type":"graph", + "xaxis":{ + "buckets":null, + "mode":"time", + "name":null, + "show":true, + "values":[ + + ] + }, + "yaxes":[ + { + "decimals":null, + "format":"percentunit", + "label":null, + "logBase":1, + "max":"1.1", + "min":"0", + "show":true + }, + { + "format":"short", + "label":null, + "logBase":1, + "max":null, + "min":null, + "show":false + } + ], + "yaxis":{ + "align":false, + "alignLevel":null + } + }, + { + "aliasColors":{ + + }, + "bars":false, + "dashLength":10, + "dashes":false, + "datasource":"${datasource}", + "decimals":2, + "description":"Traffic in and out of this namespace, as a sum of the pods within it", + "editable":true, + "error":false, + "fill":1, + "grid":{ + + }, + "gridPos":{ + "h":6, + "w":12, + "x":0, + "y":21 + }, + "height":"", + "id":15, + "isNew":true, + "legend":{ + "alignAsTable":false, + "avg":true, + "current":true, + "hideEmpty":false, + "hideZero":false, + "max":false, + "min":false, + "rightSide":false, + "show":true, + "sideWidth":null, + "sort":"current", + "sortDesc":true, + "total":false, + "values":true + }, + "lines":true, + "linewidth":2, + "links":[ + + ], + "nullPointMode":"connected", + "percentage":false, + "pointradius":5, + "points":false, + "renderer":"flot", + "seriesOverrides":[ + + ], + "spaceLength":10, + "stack":false, + "steppedLine":false, + "targets":[ + { + "expr":"sum (rate (container_network_receive_bytes_total{instance=\"$node\"}[10m]))", + "format":"time_series", + "hide":false, + "instant":false, + "interval":"", + "intervalFactor":1, + "legendFormat":"<- in", + "metric":"container_cpu", + "refId":"A", + "step":10 + }, + { + "expr":"- sum (rate (container_network_transmit_bytes_total{instance=\"$node\"}[10m]))", + "format":"time_series", + "hide":false, + "instant":false, + "interval":"", + "intervalFactor":1, + "legendFormat":"-> out", + "refId":"B" + } + ], + "thresholds":[ + + ], + "timeFrom":"", + "timeShift":null, + "title":"Network IO", + "tooltip":{ + "msResolution":true, + "shared":true, + "sort":2, + "value_type":"cumulative" + }, + "type":"graph", + "xaxis":{ + "buckets":null, + "mode":"time", + "name":null, + "show":true, + "values":[ + + ] + }, + "yaxes":[ + { + "format":"Bps", + "label":"", + "logBase":1, + "max":null, + "min":null, + "show":true + }, + { + "format":"short", + "label":null, + "logBase":1, + "max":null, + "min":null, + "show":false + } + ], + "yaxis":{ + "align":false, + "alignLevel":null + } + }, + { + "aliasColors":{ + + }, + "bars":false, + "dashLength":10, + "dashes":false, + "datasource":"${datasource}", + "decimals":2, + "description":"Disk reads and writes for the namespace, as a sum of the pods within it", + "editable":true, + "error":false, + "fill":1, + "grid":{ + + }, + "gridPos":{ + "h":6, + "w":12, + "x":12, + "y":21 + }, + "height":"", + "id":17, + "isNew":true, + "legend":{ + "alignAsTable":false, + "avg":true, + "current":true, + "hideEmpty":false, + "hideZero":false, + "max":false, + "min":false, + "rightSide":false, + "show":true, + "sideWidth":null, + "sort":"current", + "sortDesc":true, + "total":false, + "values":true + }, + "lines":true, + "linewidth":2, + "links":[ + + ], + "nullPointMode":"connected", + "percentage":false, + "pointradius":5, + "points":false, + "renderer":"flot", + "seriesOverrides":[ + + ], + "spaceLength":10, + "stack":false, + "steppedLine":false, + "targets":[ + { + "expr":"sum (rate (container_fs_writes_bytes_total{instance=\"$node\"}[10m]))", + "format":"time_series", + "hide":false, + "instant":false, + "interval":"", + "intervalFactor":1, + "legendFormat":"<- write", + "metric":"container_cpu", + "refId":"A", + "step":10 + }, + { + "expr":"- sum (rate (container_fs_reads_bytes_total{instance=\"$node\"}[10m]))", + "format":"time_series", + "hide":false, + "instant":false, + "interval":"", + "intervalFactor":1, + "legendFormat":"-> read", + "refId":"B" + } + ], + "thresholds":[ + + ], + "timeFrom":"", + "timeShift":null, + "title":"Disk IO", + "tooltip":{ + "msResolution":true, + "shared":true, + "sort":2, + "value_type":"cumulative" + }, + "type":"graph", + "xaxis":{ + "buckets":null, + "mode":"time", + "name":null, + "show":true, + "values":[ + + ] + }, + "yaxes":[ + { + "format":"Bps", + "label":"", + "logBase":1, + "max":null, + "min":null, + "show":true + }, + { + "format":"short", + "label":null, + "logBase":1, + "max":null, + "min":null, + "show":false + } + ], + "yaxis":{ + "align":false, + "alignLevel":null + } + } + ], + "schemaVersion":16, + "style":"dark", + "tags":[ + "cost", + "utilization", + "metrics" + ], + "templating":{ + "list":[ + { + "allValue":null, + "current":{ + "text":"ip-172-20-44-170.us-east-2.compute.internal", + "value":"ip-172-20-44-170.us-east-2.compute.internal" + }, + "datasource":"${datasource}", + "hide":0, + "includeAll":false, + "label":null, + "multi":false, + "name":"node", + "options":[ + + ], + "query":"query_result(kube_node_labels)", + "refresh":1, + "regex":"/node=\\\"(.*?)(\\\")/", + "skipUrlSync":false, + "sort":0, + "tagValuesQuery":"", + "tags":[ + + ], + "tagsQuery":"", + "type":"query", + "useTags":false + }, + { + "current": { + "selected": true, + "text": "default-kubecost", + "value": "default-kubecost" + }, + "error": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time":{ + "from":"now-6h", + "to":"now" + }, + "timepicker":{ + "refresh_intervals":[ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options":[ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone":"", + "title":"Node utilization metrics", + "uid":"NUQW37Lmk", + "version":1 +} diff --git a/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/pod-utilization-multi-cluster.json b/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/pod-utilization-multi-cluster.json new file mode 100644 index 0000000000..3054b9fdde --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/pod-utilization-multi-cluster.json @@ -0,0 +1,788 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 9063, + "graphTooltip": 0, + "id": 4, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "Thanos" + }, + "description": "Maximum CPU Core Usage vs avg Requested", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 3600000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 94, + "links": [], + "options": { + "legend": { + "calcs": [ + "max" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.4.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Thanos" + }, + "editorMode": "code", + "expr": "max(irate(container_cpu_usage_seconds_total\r\n {cluster_id=\"$cluster\",namespace=~\"$namespace\",pod=~\"$pod\", container=~\"$container\", container!=\"POD\",container!=\"\"}\r\n [$__rate_interval])) \r\n by (cluster_id, namespace, pod, container)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{cluster_id}} {{namespace}}/{{pod}}/{{container}} (usage max)", + "metric": "container_cpu", + "refId": "A", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "Thanos" + }, + "editorMode": "code", + "exemplar": true, + "expr": "avg(kube_pod_container_resource_requests\r\n {cluster_id=\"$cluster\",resource=\"cpu\",unit=\"core\",namespace=~\"$namespace\",pod=~\"$pod\",container=~\"$container\",container!=\"POD\"}\r\n ) \r\nby (cluster_id,namespace,pod,container)", + "legendFormat": "{{cluster_id}} {{namespace}}/{{pod}}/{{container}} (requested)", + "range": true, + "refId": "B" + } + ], + "timeFrom": "", + "title": "CPU Core Usage vs Requested", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Thanos" + }, + "description": "Max memory used vs avg requested", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 3600000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 96, + "links": [], + "options": { + "legend": { + "calcs": [ + "max" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "pluginVersion": "9.4.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Thanos" + }, + "editorMode": "code", + "expr": "max(max_over_time(container_memory_working_set_bytes\r\n {namespace=~\"$namespace\",pod=~\"$pod\",cluster_id=\"$cluster\",container=~\"$container\",container!=\"POD\",container!=\"\"}\r\n [$__rate_interval])) \r\nby (cluster_id,namespace,pod,container)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{cluster_id}} {{namespace}}/{{pod}}/{{container}} (usage max)", + "metric": "container_cpu", + "refId": "MEMORY_USAGE", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "Thanos" + }, + "editorMode": "code", + "expr": "avg(kube_pod_container_resource_requests\n {resource=\"memory\",unit=\"byte\",cluster_id=\"$cluster\",namespace=~\"$namespace\",pod=~\"$pod\", container=~\"$container\",container!=\"POD\"}\n )\nby (cluster_id,namespace,pod,container)", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{cluster_id}} {{namespace}}/{{pod}}/{{container}} (requested)", + "refId": "MEMORY_REQUESTED" + } + ], + "timeFrom": "", + "title": "Memory Usage vs Requested", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Thanos" + }, + "description": "Network traffic by pod", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 3600000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 95, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.4.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Thanos" + }, + "editorMode": "code", + "expr": "sum(irate(container_network_receive_bytes_total\n {cluster_id=~\"$cluster\",namespace=~\"$namespace\",pod=~\"$pod\"}\n [$__rate_interval])) \nby (cluster_id, namespace, pod)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{cluster_id}} {{namespace}}/{{pod}}<- in", + "metric": "container_cpu", + "refId": "A", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "Thanos" + }, + "editorMode": "code", + "expr": "- sum(irate(container_network_transmit_bytes_total\n {cluster_id=\"$cluster\",namespace=~\"$namespace\",pod=~\"$pod\"}\n [$__rate_interval])) \nby (cluster_id, namespace, pod)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{cluster_id}} {{namespace}}/{{pod}}-> out", + "refId": "B" + } + ], + "timeFrom": "", + "title": "Network IO", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Thanos" + }, + "description": "Disk read writes", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 3600000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 97, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.4.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Thanos" + }, + "editorMode": "code", + "expr": "sum(irate(container_fs_writes_bytes_total\r\n {cluster_id=\"$cluster\",namespace=~\"$namespace\",container!=\"POD\",pod!=\"\",pod=~\"$pod\",container=~\"$container\"}\r\n [$__rate_interval])) \r\nby (cluster_id,namespace,pod,container)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{cluster_id}} {{pod}}/{{container}}<- write", + "metric": "container_cpu", + "refId": "A", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "Thanos" + }, + "editorMode": "code", + "expr": "- sum(irate(container_fs_reads_bytes_total\r\n {cluster_id=\"$cluster\",namespace=~\"$namespace\",container!=\"POD\",pod!=\"\",pod=~\"$pod\",container=~\"$container\"}\r\n [$__rate_interval])) \r\nby (cluster_id,namespace,pod,container)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{cluster_id}} {{pod}}/{{container}}-> read", + "refId": "B" + } + ], + "timeFrom": "", + "title": "Disk IO", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Thanos" + }, + "description": "This graph shows the % of periods where a pod is being throttled. Values range from 0-100", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 14 + }, + "id": 99, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.4.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Thanos" + }, + "editorMode": "code", + "expr": "100\n * sum by(cluster_id, namespace, pod, container) (increase(container_cpu_cfs_throttled_periods_total{container!=\"\",cluster_id=\"$cluster\", namespace=~\"$namespace\", pod=~\"$pod\", container=~\"$container\", container!=\"POD\"}[$__rate_interval]))\n / sum by(cluster_id,namespace,pod,container) (increase(container_cpu_cfs_periods_total{container!=\"\",cluster_id=\"$cluster\",namespace=~\"$namespace\",pod=~\"$pod\",container=~\"$container\",container!=\"POD\"}[$__rate_interval]))", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "B" + } + ], + "timeFrom": "", + "title": "CPU throttle percent", + "type": "timeseries" + } + ], + "refresh": "", + "revision": 1, + "schemaVersion": 39, + "tags": [ + "utilization", + "metrics", + "kubecost" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "CostManagement", + "value": "CostManagement" + }, + "datasource": { + "type": "prometheus", + "uid": "Thanos" + }, + "definition": "label_values(cluster_id)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "cluster", + "options": [], + "query": { + "query": "label_values(cluster_id)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "kubecost", + "value": "kubecost" + }, + "datasource": { + "type": "prometheus", + "uid": "Thanos" + }, + "definition": "label_values(kube_namespace_labels{cluster_id=\"$cluster\"}, namespace) ", + "hide": 0, + "includeAll": true, + "label": "", + "multi": false, + "name": "namespace", + "options": [], + "query": { + "query": "label_values(kube_namespace_labels{cluster_id=\"$cluster\"}, namespace) ", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "Thanos" + }, + "definition": "label_values(kube_pod_labels{cluster_id=\"$cluster\",namespace=~\"$namespace\"}, pod) ", + "hide": 0, + "includeAll": true, + "label": "pod", + "multi": false, + "name": "pod", + "options": [], + "query": { + "query": "label_values(kube_pod_labels{cluster_id=\"$cluster\",namespace=~\"$namespace\"}, pod) ", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "Thanos" + }, + "definition": "label_values(container_memory_working_set_bytes{cluster_id=\"$cluster\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"POD\"}, container) ", + "hide": 0, + "includeAll": true, + "multi": false, + "name": "container", + "options": [], + "query": { + "query": "label_values(container_memory_working_set_bytes{cluster_id=\"$cluster\",namespace=~\"$namespace\",pod=~\"$pod\", container!=\"POD\"}, container) ", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "type": "query" + } + ] + }, + "time": { + "from": "now-2d", + "to": "now" + }, + "timepicker": { + "hidden": false, + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Pod utilization metrics (multi-cluster)", + "uid": "at-cost-analysis-pod-multicluster", + "version": 2, + "weekStart": "" +} diff --git a/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/pod-utilization.json b/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/pod-utilization.json new file mode 100644 index 0000000000..6596cef761 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/pod-utilization.json @@ -0,0 +1,860 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 9063, + "graphTooltip": 0, + "id": 11, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Maximum CPU Core Usage vs Requested", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 3600000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 94, + "links": [], + "options": { + "legend": { + "calcs": [ + "max" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.4.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "max(irate(\r\n container_cpu_usage_seconds_total\r\n {namespace=~\"$namespace\",pod=~\"$pod\",container=~\"$container\",container!=\"POD\",container!=\"\"}\r\n [$__rate_interval])) \r\n by (cluster_id, namespace, pod, container)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{cluster_id}} {{pod}}/{{container}} (usage max)", + "metric": "container_cpu", + "refId": "A", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "avg(kube_pod_container_resource_requests\r\n {resource=\"cpu\",unit=\"core\",namespace=~\"$namespace\",pod=~\"$pod\",container=~\"$container\",container!=\"POD\"}\r\n ) \r\nby (cluster_id, namespace, pod, container)", + "legendFormat": "{{cluster_id}} {{pod}}/{{container}} (avg requested)", + "range": true, + "refId": "B" + } + ], + "timeFrom": "", + "title": "CPU Core Usage vs Requested", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Max Memory usage vs avg requested", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 3600000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 96, + "links": [], + "options": { + "legend": { + "calcs": [ + "max" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "pluginVersion": "9.4.7", + "targets": [ + { + "datasource": { + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "max(max_over_time(\r\n container_memory_working_set_bytes\r\n {namespace=~\"$namespace\",pod=~\"$pod\",container=~\"$container\",container!=\"POD\",container!=\"\"}\r\n [$__rate_interval])) \r\nby (cluster_id,namespace,pod,container)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{cluster_id}} {{pod}}/{{container}} (usage max)", + "metric": "container_cpu", + "refId": "A", + "step": 10 + }, + { + "datasource": { + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg(\n kube_pod_container_resource_requests\n {resource=\"memory\",unit=\"byte\",namespace=~\"$namespace\",pod=~\"$pod\",container=~\"$container\", container!=\"POD\"}\n )\nby (cluster_id,namespace,pod,container)", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{cluster_id}} {{pod}}/{{container}} (avg requested)", + "refId": "B" + } + ], + "timeFrom": "", + "title": "Memory Usage vs Requested", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Network traffic by pod", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 3600000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 95, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.4.7", + "targets": [ + { + "datasource": { + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(irate(container_network_receive_bytes_total\n {namespace=~\"$namespace\",pod=~\"$pod\"}\n [$__rate_interval])) \nby (cluster_id, namespace, pod)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{cluster_id}} {{namespace}}/{{pod}}<- in", + "metric": "container_cpu", + "refId": "A", + "step": 10 + }, + { + "datasource": { + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "- sum(irate(container_network_transmit_bytes_total\n {namespace=~\"$namespace\",pod=~\"$pod\"}\n [$__rate_interval])) \nby (cluster_id, namespace, pod)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{cluster_id}} {{namespace}}/{{pod}}-> out", + "refId": "B" + } + ], + "timeFrom": "", + "title": "Network IO", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Disk read writes", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 3600000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 97, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.4.7", + "targets": [ + { + "datasource": { + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(irate(container_fs_writes_bytes_total\r\n {container!=\"POD\",pod!=\"\",pod=~\"$pod\",container=~\"$container\"}\r\n [$__rate_interval])) \r\nby (cluster_id,namespace,pod,container)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{cluster_id}} {{pod}}/{{container}}<- write", + "metric": "container_cpu", + "refId": "A", + "step": 10 + }, + { + "datasource": { + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "- sum(irate(container_fs_reads_bytes_total\r\n {container!=\"POD\",pod!=\"\",pod=~\"$pod\",container=~\"$container\"}\r\n [$__rate_interval])) \r\nby (cluster_id,namespace,pod,container)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{cluster_id}} {{pod}}/{{container}}-> read", + "refId": "B" + } + ], + "timeFrom": "", + "title": "Disk IO", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "This graph shows the % of periods where a pod is being throttled. Values range from 0-100", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 1800000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 14 + }, + "id": 99, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.4.7", + "targets": [ + { + "datasource": { + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "100\n * sum by(cluster_id, namespace, pod, container) (increase(container_cpu_cfs_throttled_periods_total{container!=\"\", namespace=~\"$namespace\", pod=~\"$pod\", container=~\"$container\", container!=\"POD\"}[$__rate_interval]))\n / sum by(cluster_id, namespace, pod, container) (increase(container_cpu_cfs_periods_total{container!=\"\", namespace=~\"$namespace\", pod=~\"$pod\", container=~\"$container\", container!=\"POD\"}[$__rate_interval]))", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "__auto", + "refId": "B" + } + ], + "timeFrom": "", + "title": "CPU throttle percent", + "type": "timeseries" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "NVIDIA GPU usage for this container.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 3600000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 14 + }, + "id": 100, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.4.7", + "targets": [ + { + "datasource": { + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "DCGM_FI_PROF_GR_ENGINE_ACTIVE{namespace=~\"$namespace\",container=~\"$container\",pod=~\"$pod\"}", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "__auto", + "metric": "container_cpu", + "refId": "A", + "step": 10 + } + ], + "timeFrom": "", + "title": "GPU Usage", + "type": "timeseries" + } + ], + "refresh": "", + "revision": 1, + "schemaVersion": 39, + "tags": [ + "kubecost", + "utilization", + "metrics" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "uid": "${datasource}" + }, + "definition": "label_values(kube_namespace_labels, namespace) ", + "hide": 0, + "includeAll": true, + "label": "", + "multi": false, + "name": "namespace", + "options": [], + "query": { + "query": "label_values(kube_namespace_labels, namespace) ", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "uid": "${datasource}" + }, + "definition": "label_values(kube_pod_labels{namespace=~\"$namespace\"}, pod) ", + "hide": 0, + "includeAll": true, + "label": "pod", + "multi": false, + "name": "pod", + "options": [], + "query": { + "query": "label_values(kube_pod_labels{namespace=~\"$namespace\"}, pod) ", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(container_memory_working_set_bytes{namespace=~\"$namespace\",pod=~\"$pod\", container!=\"POD\"}, container) ", + "hide": 0, + "includeAll": true, + "multi": false, + "name": "container", + "options": [], + "query": { + "query": "label_values(container_memory_working_set_bytes{namespace=~\"$namespace\",pod=~\"$pod\", container!=\"POD\"}, container) ", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "type": "query" + } + ] + }, + "time": { + "from": "now-2d", + "to": "now" + }, + "timepicker": { + "hidden": false, + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Pod utilization metrics", + "uid": "at-cost-analysis-pod", + "version": 2, + "weekStart": "" +} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/prom-benchmark.json b/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/prom-benchmark.json new file mode 100644 index 0000000000..ff054acc2f --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/prom-benchmark.json @@ -0,0 +1,5691 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Metrics useful for benchmarking and loadtesting Prometheus itself. Designed primarily for Prometheus 2.17.x.", + "editable": true, + "gnetId": 12054, + "graphTooltip": 1, + "id": 9, + "iteration": 1603144824023, + "links": [], + "panels": [ + { + "collapsed": false, + "datasource": "${datasource}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 49, + "panels": [], + "title": "Basics", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 1 + }, + "hiddenSeries": false, + "id": 40, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_build_info{job=\"prometheus\",instance=\"$Prometheus:9090\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{version}} - {{revision}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Prometheus Version", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 1 + }, + "hiddenSeries": false, + "id": 72, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "time() - process_start_time_seconds{job=\"prometheus\",instance=\"$Prometheus:9090\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Age", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Uptime", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "dtdurations", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 1 + }, + "hiddenSeries": false, + "id": 107, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "time() - prometheus_config_last_reload_success_timestamp_seconds{job=\"prometheus\",instance=\"$Prometheus:9090\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Age", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Last Successful Config Reload", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "dtdurations", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": "${datasource}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 46, + "panels": [], + "title": "Ingestion", + "type": "row" + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833", + "Time series": "#70dbed" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 9 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_head_series{job=\"prometheus\",instance=\"$Prometheus:9090\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Time series", + "metric": "prometheus_local_storage_memory_series", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Head Time series", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 9 + }, + "hiddenSeries": false, + "id": 26, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_head_active_appenders{job=\"prometheus\",instance=\"$Prometheus:9090\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Head Appenders", + "metric": "prometheus_local_storage_memory_series", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Head Active Appenders", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "samples/s": "#e5a8e2" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 9 + }, + "hiddenSeries": false, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_head_samples_appended_total{job=\"prometheus\",instance=\"$Prometheus:9090\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "samples/s", + "metric": "prometheus_local_storage_ingested_samples_total", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Samples Appended/s", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833", + "To persist": "#9AC48A" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 16 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/Max.*/", + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_head_chunks{job=\"prometheus\",instance=\"$Prometheus:9090\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Chunks", + "metric": "prometheus_local_storage_memory_chunks", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Head Chunks", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 16 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_head_chunks_created_total{job=\"prometheus\",instance=\"$Prometheus:9090\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Created", + "metric": "prometheus_local_storage_chunk_ops_total", + "refId": "A", + "step": 10 + }, + { + "expr": "rate(prometheus_tsdb_head_chunks_removed_total{job=\"prometheus\",instance=\"$Prometheus:9090\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Removed", + "metric": "prometheus_local_storage_chunk_ops_total", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Head Chunks/s", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833", + "Removed": "#e5ac0e" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 16 + }, + "hiddenSeries": false, + "id": 25, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_isolation_high_watermark{job=\"prometheus\",instance=\"$Prometheus:9090\"} - prometheus_tsdb_isolation_low_watermark{job=\"prometheus\",instance=\"$Prometheus:9090\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Difference", + "metric": "prometheus_local_storage_chunk_ops_total", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Isolation Watermarks", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": "${datasource}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 52, + "panels": [], + "title": "Compaction", + "type": "row" + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max": "#447ebc", + "Max chunks": "#052B51", + "Max to persist": "#3F6833", + "Min": "#447ebc", + "Now": "#7eb26d" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 24 + }, + "hiddenSeries": false, + "id": 28, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Max", + "fillBelowTo": "Min", + "lines": false + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_head_min_time{job=\"prometheus\",instance=\"$Prometheus:9090\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Min", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "A", + "step": 10 + }, + { + "expr": "time() * 1000", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Now", + "refId": "C" + }, + { + "expr": "prometheus_tsdb_head_max_time{job=\"prometheus\",instance=\"$Prometheus:9090\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Max", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Head Time Range", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "dateTimeAsIso", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 24 + }, + "hiddenSeries": false, + "id": 29, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_head_gc_duration_seconds_sum{job=\"prometheus\",instance=\"$Prometheus:9090\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "GC Time/s", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Head GC Time/s", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 24 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Queue length", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_blocks_loaded{job=\"prometheus\",instance=\"$Prometheus:9090\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Blocks Loaded", + "metric": "prometheus_local_storage_indexing_batch_sizes_sum", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Blocks Loaded", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Failed Compactions": "#bf1b00", + "Failed Reloads": "#bf1b00", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 31 + }, + "hiddenSeries": false, + "id": 30, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_reloads_total{job=\"prometheus\",instance=\"$Prometheus:9090\"}[10m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Reloads", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "TSDB Reloads/s", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Failed Compactions": "#bf1b00", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 31 + }, + "hiddenSeries": false, + "id": 31, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_wal_fsync_duration_seconds_sum{job=\"prometheus\",instance=\"$Prometheus:9090\"}[2m]) / rate(prometheus_tsdb_wal_fsync_duration_seconds_count{job=\"prometheus\",instance=\"$Prometheus:9090\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Fsync Latency", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "A", + "step": 10 + }, + { + "expr": "rate(prometheus_tsdb_wal_truncate_duration_seconds_sum{job=\"prometheus\",instance=\"$Prometheus:9090\"}[1m]) / rate(prometheus_tsdb_wal_trunacte_duration_seconds_count{job=\"prometheus\",instance=\"$Prometheus:9090\"}[1m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Truncate Latency", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "WAL Fsync&Truncate Latency", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Failed Compactions": "#bf1b00", + "Max chunks": "#052B51", + "Max to persist": "#3F6833", + "{instance=\"demo.robustperception.io:9090\",job=\"prometheus\"}": "#bf1b00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 31 + }, + "hiddenSeries": false, + "id": 32, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_wal_corruptions_total{job=\"prometheus\",instance=\"$Prometheus:9090\"}[10m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "WAL Corruptions", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "A", + "step": 10 + }, + { + "expr": "rate(prometheus_tsdb_reloads_failures_total{job=\"prometheus\",instance=\"$Prometheus:9090\"}[10m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Reload Failures", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "B", + "step": 10 + }, + { + "expr": "rate(prometheus_tsdb_head_series_not_found{job=\"prometheus\",instance=\"$Prometheus:9090\"}[10m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Head Series Not Found", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "C", + "step": 10 + }, + { + "expr": "rate(prometheus_tsdb_compactions_failed_total{job=\"prometheus\",instance=\"$Prometheus:9090\"}[10m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Compaction Failures", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "D", + "step": 10 + }, + { + "expr": "rate(prometheus_tsdb_retention_cutoffs_failures_total{job=\"prometheus\",instance=\"$Prometheus:9090\"}[10m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Retention Cutoff Failures", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "E", + "step": 10 + }, + { + "expr": "rate(prometheus_tsdb_checkpoint_creations_failed_total{job=\"prometheus\",instance=\"$Prometheus:9090\"}[10m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "WAL Checkpoint Creation Failures", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "F", + "step": 10 + }, + { + "expr": "rate(prometheus_tsdb_checkpoint_deletions_failed_total{job=\"prometheus\",instance=\"$Prometheus:9090\"}[10m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "WAL Checkpoint Deletion Failures", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "G", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "TSDB Problems/s", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Failed Compactions": "#bf1b00", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 38 + }, + "hiddenSeries": false, + "id": 19, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_compactions_total{job=\"prometheus\",instance=\"$Prometheus:9090\"}[10m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Compactions", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Compactions/s", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 38 + }, + "hiddenSeries": false, + "id": 33, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_compaction_duration_seconds_sum{job=\"prometheus\",instance=\"$Prometheus:9090\"}[10m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Compaction Time/s", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Allocated bytes": "#F9BA8F", + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833", + "RSS": "#890F02" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 38 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_time_retentions_total{job=\"prometheus\",instance=\"$Prometheus:9090\"}[10m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Time Cutoffs", + "metric": "last", + "refId": "A", + "step": 10 + }, + { + "expr": "rate(prometheus_tsdb_size_retentions_total{job=\"prometheus\",instance=\"$Prometheus:9090\"}[10m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Size Cutoffs", + "metric": "last", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Retention Cutoffs/s", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 45 + }, + "hiddenSeries": false, + "id": 27, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_compaction_chunk_range_seconds_sum{job=\"prometheus\",instance=\"$Prometheus:9090\"}[10m]) / rate(prometheus_tsdb_compaction_chunk_range_seconds_count{job=\"prometheus\",instance=\"$Prometheus:9090\"}[10m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Chunk Time Range", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "First Compaction, Avg Chunk Time Range", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "dtdurationms", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 45 + }, + "hiddenSeries": false, + "id": 35, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_compaction_chunk_size_bytes_sum{job=\"prometheus\",instance=\"$Prometheus:9090\"}[10m]) / rate(prometheus_tsdb_compaction_chunk_samples_sum{job=\"prometheus\",instance=\"$Prometheus:9090\"}[10m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Bytes/Sample", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "First Compaction, Avg Bytes/Sample", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 45 + }, + "hiddenSeries": false, + "id": 34, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_compaction_chunk_samples_sum{job=\"prometheus\",instance=\"$Prometheus:9090\"}[10m]) / rate(prometheus_tsdb_compaction_chunk_samples_count{job=\"prometheus\",instance=\"$Prometheus:9090\"}[10m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Chunk Samples", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "First Compaction, Avg Chunk Samples", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": "${datasource}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 52 + }, + "id": 55, + "panels": [], + "title": "Resource Usage", + "type": "row" + }, + { + "aliasColors": { + "Allocated bytes": "#7EB26D", + "Allocated bytes - 1m max": "#BF1B00", + "Allocated bytes - 1m min": "#BF1B00", + "Allocated bytes - 5m max": "#BF1B00", + "Allocated bytes - 5m min": "#BF1B00", + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833", + "RSS": "#447EBC" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "decimals": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 53 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/-/", + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{job=\"prometheus\",instance=\"$Prometheus:9090\"}", + "intervalFactor": 2, + "legendFormat": "RSS", + "metric": "process_resident_memory_bytes", + "refId": "B", + "step": 10 + }, + { + "expr": "prometheus_local_storage_target_heap_size_bytes{job=\"prometheus\",instance=\"$Prometheus:9090\"}", + "intervalFactor": 2, + "legendFormat": "Target heap size", + "metric": "go_memstats_alloc_bytes", + "refId": "D", + "step": 10 + }, + { + "expr": "go_memstats_next_gc_bytes{job=\"prometheus\",instance=\"$Prometheus:9090\"}", + "intervalFactor": 2, + "legendFormat": "Next GC", + "metric": "go_memstats_next_gc_bytes", + "refId": "C", + "step": 10 + }, + { + "expr": "go_memstats_alloc_bytes{job=\"prometheus\",instance=\"$Prometheus:9090\"}", + "intervalFactor": 2, + "legendFormat": "Allocated", + "metric": "go_memstats_alloc_bytes", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Allocated bytes": "#F9BA8F", + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833", + "RSS": "#890F02" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 53 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(go_memstats_alloc_bytes_total{job=\"prometheus\",instance=\"$Prometheus:9090\"}[2m])", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Allocated Bytes/s", + "metric": "go_memstats_alloc_bytes", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Allocations", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 53 + }, + "hiddenSeries": false, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(process_cpu_seconds_total{job=\"prometheus\",instance=\"$Prometheus:9090\"}[1m])", + "intervalFactor": 2, + "legendFormat": "Irate", + "metric": "prometheus_local_storage_ingested_samples_total", + "refId": "A", + "step": 10 + }, + { + "expr": "rate(process_cpu_seconds_total{job=\"prometheus\",instance=\"$Prometheus:9090\"}[5m])", + "intervalFactor": 2, + "legendFormat": "5m rate", + "metric": "prometheus_local_storage_ingested_samples_total", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + "avg" + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 60 + }, + "hiddenSeries": false, + "id": 70, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_symbol_table_size_bytes{job=\"prometheus\",instance=\"$Prometheus:9090\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "RAM Used", + "metric": "prometheus_local_storage_ingested_samples_total", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Symbol Tables Size", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + "avg" + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 60 + }, + "hiddenSeries": false, + "id": 71, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_storage_blocks_bytes_total{job=\"prometheus\",instance=\"$Prometheus:9090\"} or prometheus_tsdb_storage_blocks_bytes{job=\"prometheus\",instance=\"$Prometheus:9090\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Disk Used", + "metric": "prometheus_local_storage_ingested_samples_total", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Block Size", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + "avg" + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Max": "#e24d42", + "Open": "#508642" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 60 + }, + "hiddenSeries": false, + "id": 41, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_max_fds{job=\"prometheus\",instance=\"$Prometheus:9090\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Max", + "refId": "A" + }, + { + "expr": "process_open_fds{job=\"prometheus\",instance=\"$Prometheus:9090\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Open", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "File Descriptors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": "${datasource}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 67 + }, + "id": 91, + "panels": [], + "title": "Service Discovery", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 68 + }, + "hiddenSeries": false, + "id": 42, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_sd_discovered_targets{job=\"prometheus\",instance=\"$Prometheus:9090\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{name}}-{{config}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Discovered Targets", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 68 + }, + "hiddenSeries": false, + "id": 96, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_sd_updates_total{job=\"prometheus\",instance=\"$Prometheus:9090\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Sent Updates/s", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 68 + }, + "hiddenSeries": false, + "id": 97, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_sd_received_updates_total{job=\"prometheus\",instance=\"$Prometheus:9090\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Received Updates/s", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": "${datasource}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 75 + }, + "id": 99, + "panels": [], + "title": "Scraping", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 76 + }, + "hiddenSeries": false, + "id": 105, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_target_interval_length_seconds_sum{job=\"prometheus\",instance=\"$Prometheus:9090\"}[2m]) / rate(prometheus_target_interval_length_seconds_count{job=\"prometheus\",instance=\"$Prometheus:9090\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{interval}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Scrape Interval", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 76 + }, + "hiddenSeries": false, + "id": 104, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_target_scrapes_exceeded_sample_limit_total{job=\"prometheus\",instance=\"$Prometheus:9090\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Exceeded Sample Limit", + "refId": "A" + }, + { + "expr": "rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{job=\"prometheus\",instance=\"$Prometheus:9090\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Duplicate Timestamp", + "refId": "C" + }, + { + "expr": "rate(prometheus_target_scrapes_sample_out_of_bounds_total{job=\"prometheus\",instance=\"$Prometheus:9090\"}[1m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Out Of Bounds ", + "refId": "D" + }, + { + "expr": "rate(prometheus_target_scrapes_sample_out_of_order_total{job=\"prometheus\",instance=\"$Prometheus:9090\"}[1m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Out of Order", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Scrape Problems/s", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 76 + }, + "hiddenSeries": false, + "id": 95, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_target_metadata_cache_bytes{job=\"prometheus\",instance=\"$Prometheus:9090\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{scrape_job}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Metadata Cache Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": "${datasource}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 83 + }, + "id": 63, + "panels": [], + "title": "Query Engine", + "type": "row" + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "description": "Time spent in each mode, per second", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 84 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_engine_query_duration_seconds_sum{job=\"prometheus\",}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{slice}}", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Query engine timings/s", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 84 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_rule_group_iterations_missed_total{job=\"prometheus\",instance=\"$Prometheus:9090\"}[2m]) ", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Rule group missed", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "B", + "step": 10 + }, + { + "expr": "rate(prometheus_rule_evaluation_failures_total{job=\"prometheus\",instance=\"$Prometheus:9090\"}[1m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Rule evals failed", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "C", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Rule group evaulation problems/s", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 84 + }, + "hiddenSeries": false, + "id": 23, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_rule_group_duration_seconds_sum{job=\"prometheus\",instance=\"$Prometheus:9090\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Rule evaluation duration", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Evaluation time of rule groups/s", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": true, + "datasource": "${datasource}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 91 + }, + "id": 77, + "panels": [ + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 92 + }, + "hiddenSeries": false, + "id": 86, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_notifications_sent_total{job=\"prometheus\",instance=\"$Prometheus:9090\"}[1m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{alertmanager}}", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Notification Sent/s", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 92 + }, + "hiddenSeries": false, + "id": 87, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_notifications_errors_total{job=\"prometheus\",instance=\"$Prometheus:9090\"}[2m]) / rate(prometheus_notifications_sent_total{job=\"prometheus\",instance=\"$Prometheus:9090\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{alertmanager}}", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Notification Error Ratio", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 92 + }, + "hiddenSeries": false, + "id": 81, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_notifications_latency_seconds_sum{job=\"prometheus\",instance=\"$Prometheus:9090\"}[1m]) / rate(prometheus_notifications_latency_seconds_count{job=\"prometheus\",instance=\"$Prometheus:9090\"}[1m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{alertmanager}}", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Notification Latency", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 99 + }, + "hiddenSeries": false, + "id": 85, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_notifications_alertmanagers_discovered{job=\"prometheus\",instance=\"$Prometheus:9090\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Alertmanagers", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Alertmanagers Discovered", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 99 + }, + "hiddenSeries": false, + "id": 89, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_notifications_dropped_total{job=\"prometheus\",instance=\"$Prometheus:9090\"}[1m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Dropped", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Notifications Dropped/s", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 99 + }, + "hiddenSeries": false, + "id": 88, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_notifications_queue_length{job=\"prometheus\",instance=\"$Prometheus:9090\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Pending", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + }, + { + "expr": "prometheus_notifications_queue_capacity{job=\"prometheus\",instance=\"$Prometheus:9090\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Max", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Notification Queue", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "title": "Notification", + "type": "row" + }, + { + "collapsed": false, + "datasource": "${datasource}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 92 + }, + "id": 58, + "panels": [], + "title": "HTTP Server", + "type": "row" + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 93 + }, + "hiddenSeries": false, + "id": 38, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_http_request_duration_seconds_count{job=\"prometheus\",instance=\"$Prometheus:9090\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{handler}}", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "HTTP requests/s", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 93 + }, + "hiddenSeries": false, + "id": 37, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_http_request_duration_seconds_sum{job=\"prometheus\",instance=\"$Prometheus:9090\"}[2m]) / rate(prometheus_http_request_duration_seconds_count{job=\"prometheus\",instance=\"$Prometheus:9090\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{handler}}", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "HTTP request latency", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 93 + }, + "hiddenSeries": false, + "id": 36, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_http_request_duration_seconds_sum{job=\"prometheus\",instance=\"$Prometheus:9090\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{handler}}", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Time spent in HTTP requests/s", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": "${datasource}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 100 + }, + "id": 61, + "panels": [], + "repeat": "RuleGroup", + "scopedVars": { + "RuleGroup": { + "selected": false, + "text": "/etc/config/rules;CPU", + "value": "/etc/config/rules;CPU" + } + }, + "title": "Rule Group: $RuleGroup", + "type": "row" + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Interval": "#890f02", + "Last Duration": "#f9934e", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 101 + }, + "hiddenSeries": false, + "id": 43, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "scopedVars": { + "RuleGroup": { + "selected": false, + "text": "/etc/config/rules;CPU", + "value": "/etc/config/rules;CPU" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_rule_group_interval_seconds{job=\"prometheus\",instance=\"$Prometheus:9090\",rule_group=~\"$RuleGroup\"}\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Interval", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + }, + { + "expr": "prometheus_rule_group_last_duration_seconds{job=\"prometheus\",instance=\"$Prometheus:9090\",rule_group=~\"$RuleGroup\"}\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Last Duration", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "$RuleGroup: Duration", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Interval": "#890f02", + "Last Duration": "#f9934e", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 101 + }, + "hiddenSeries": false, + "id": 66, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeatDirection": "h", + "scopedVars": { + "RuleGroup": { + "selected": false, + "text": "/etc/config/rules;CPU", + "value": "/etc/config/rules;CPU" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_rule_group_rules{job=\"prometheus\",instance=\"$Prometheus:9090\",rule_group=~\"$RuleGroup\"}\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Rules", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "$RuleGroup: Rules", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": "${datasource}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 108 + }, + "id": 108, + "panels": [], + "repeat": null, + "repeatIteration": 1603144824023, + "repeatPanelId": 61, + "scopedVars": { + "RuleGroup": { + "selected": false, + "text": "/etc/config/rules;Savings", + "value": "/etc/config/rules;Savings" + } + }, + "title": "Rule Group: $RuleGroup", + "type": "row" + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Interval": "#890f02", + "Last Duration": "#f9934e", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 109 + }, + "hiddenSeries": false, + "id": 109, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "repeatIteration": 1603144824023, + "repeatPanelId": 43, + "repeatedByRow": true, + "scopedVars": { + "RuleGroup": { + "selected": false, + "text": "/etc/config/rules;Savings", + "value": "/etc/config/rules;Savings" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_rule_group_interval_seconds{job=\"prometheus\",instance=\"$Prometheus:9090\",rule_group=~\"$RuleGroup\"}\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Interval", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + }, + { + "expr": "prometheus_rule_group_last_duration_seconds{job=\"prometheus\",instance=\"$Prometheus:9090\",rule_group=~\"$RuleGroup\"}\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Last Duration", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "$RuleGroup: Duration", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Interval": "#890f02", + "Last Duration": "#f9934e", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${datasource}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 109 + }, + "hiddenSeries": false, + "id": 110, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeatDirection": "h", + "repeatIteration": 1603144824023, + "repeatPanelId": 66, + "repeatedByRow": true, + "scopedVars": { + "RuleGroup": { + "selected": false, + "text": "/etc/config/rules;Savings", + "value": "/etc/config/rules;Savings" + } + }, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_rule_group_rules{job=\"prometheus\",instance=\"$Prometheus:9090\",rule_group=~\"$RuleGroup\"}\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Rules", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "$RuleGroup: Rules", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": [ + "kubecost" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": false, + "text": "localhost", + "value": "localhost" + }, + "datasource": "${datasource}", + "definition": "", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "Prometheus", + "options": [], + "query": "query_result(up{job=\"prometheus\"} == 1)", + "refresh": 2, + "regex": ".*instance=\"([^\"]+):9090\".*", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": null, + "tags": [], + "tagsQuery": null, + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": "${datasource}", + "definition": "", + "hide": 2, + "includeAll": true, + "label": null, + "multi": false, + "name": "RuleGroup", + "options": [], + "query": "prometheus_rule_group_last_duration_seconds{job=\"prometheus\",instance=\"$Prometheus:9090\"}", + "refresh": 2, + "regex": ".*rule_group=\"(.*?)\".*", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": true, + "text": "default-kubecost", + "value": "default-kubecost" + }, + "error": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Prometheus Benchmark - 2.17.x", + "uid": "L0HBvojWz", + "version": 4 +} diff --git a/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/workload-metrics-aggregator.json b/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/workload-metrics-aggregator.json new file mode 100644 index 0000000000..660358905e --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/workload-metrics-aggregator.json @@ -0,0 +1,988 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Most used metrics when troubleshooting applications", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 7, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "topk(5,sum(container_memory_working_set_bytes{container=\"aggregator\",namespace=~\"$namespace\"} ) by (namespace,pod,container))", + "instant": false, + "legendFormat": "{{namespace}}/{{container}}", + "range": true, + "refId": "A" + } + ], + "title": "Top Memory Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "topk(5, (\r\n sum(rate(container_cpu_usage_seconds_total{image!=\"\",namespace=~\"$namespace\",container=\"aggregator\"}[$__rate_interval])) by (namespace,pod,container)\r\n )\r\n)", + "instant": false, + "legendFormat": "{{namespace}}/{{container}}", + "range": true, + "refId": "A" + } + ], + "title": "Top CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum(topk(5,\n rate(kubecost_pod_network_ingress_bytes_total\n {namespace=~\"$namespace\", pod_name=~\"$pod\"}\n [$__rate_interval]\n )\n) )\nby(namespace, pod_name) ", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{namespace}}/{{pod_name}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "- sum(topk(5,\n rate(kubecost_pod_network_egress_bytes_total\n {namespace=~\"$namespace\", pod_name=~\"$pod\"}\n [$__rate_interval]\n )\n) )\nby(namespace, pod_name) ", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{namespace}}/{{pod_name}}", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Kubecost Top Network (egress is negative)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": 3600000, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(topk(5,rate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$__rate_interval]))) by (namespace,pod) ", + "hide": false, + "instant": false, + "legendFormat": "{{namespace}}/{{pod}}", + "range": true, + "refId": "receive" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "-sum(topk(5,rate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$__rate_interval]))) by (namespace,pod) ", + "hide": false, + "instant": false, + "legendFormat": "{{namespace}}/{{pod}}", + "range": true, + "refId": "transmit" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "container_network_transmit_bytes_total{}", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Top Network (transmit is negative)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": 3600000, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "\n sum(rate(container_fs_writes_bytes_total\n {container=\"aggregator\",namespace=~\"$namespace\",image!=\"\"}\n [$__rate_interval]))\n by (namespace,pod,container)\n>0 ", + "hide": false, + "instant": false, + "legendFormat": "{{namespace}}/{{container}}", + "range": true, + "refId": "storage_write" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "-(sum\r\n (rate(container_fs_reads_bytes_total\r\n {container=\"aggregator\",namespace=~\"$namespace\",image!=\"\"}\r\n [$__rate_interval])) \r\nby (namespace,pod,container) \r\n) <0", + "hide": false, + "instant": false, + "legendFormat": "{{namespace}}/{{container}}", + "range": true, + "refId": "storage_read" + } + ], + "title": "Storage (read is negative)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "This may work depending on the CRI", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(kubelet_volume_stats_available_bytes{namespace=~\"$namespace\"}) by (namespace,persistentvolumeclaim)", + "hide": false, + "instant": false, + "legendFormat": "{{namespace}}/{{persistentvolumeclaim}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(kube_persistentvolume_capacity_bytes) by (persistentvolume)", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Storage ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 82, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(kube_pod_container_status_restarts_total{namespace=~\"$namespace\",pod=~\".+-aggregator-0\"}[1h])) by (namespace,container)>0", + "instant": false, + "interval": "1h", + "legendFormat": "{{namespace}}/{{container}}", + "range": true, + "refId": "A" + } + ], + "title": "Pod restarts per hour", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "kubecost_read_db_size", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "kubecost_read_db_size" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(kubecost_write_db_size)", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "kubecost_write_db_size" + } + ], + "title": "Aggregator DB Size", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 39, + "tags": [ + "kubecost", + "utilization", + "metrics" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "PBFA97CFB590B2093" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": true, + "text": "kubecost", + "value": "kubecost" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(kube_namespace_labels,namespace)", + "hide": 0, + "includeAll": true, + "multi": false, + "name": "namespace", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(kube_namespace_labels,namespace)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(kube_pod_owner{namespace=~\"$namespace\"},pod)", + "hide": 0, + "includeAll": true, + "multi": false, + "name": "pod", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(kube_pod_owner{namespace=~\"$namespace\"},pod)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "type": "query" + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Workload Metrics - Aggregator", + "uid": "kubecost-aggregator-metrics", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/workload-metrics.json b/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/workload-metrics.json new file mode 100644 index 0000000000..248afc1348 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/grafana-dashboards/workload-metrics.json @@ -0,0 +1,893 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Most used metrics when troubleshooting applications", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 2, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "topk(5,sum(container_memory_working_set_bytes{container=~\"$container\",pod=~\"$pod\",container!=\"\",namespace=~\"$namespace\"} ) by (namespace,pod,container))", + "instant": false, + "legendFormat": "{{namespace}}/{{container}}", + "range": true, + "refId": "A" + } + ], + "title": "Top Memory Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "topk(5, (\r\n sum(rate(container_cpu_usage_seconds_total{image!=\"\",namespace=~\"$namespace\",pod=~\"$pod\",container=~\"$container\"}[10m])) by (namespace,pod,container)\r\n )\r\n)", + "instant": false, + "legendFormat": "{{namespace}}/{{container}}", + "range": true, + "refId": "A" + } + ], + "title": "Top CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum(topk(5,\n rate(kubecost_pod_network_ingress_bytes_total\n {namespace=~\"$namespace\", pod_name=~\"$pod\"}\n [$__rate_interval]\n )\n) )\nby(namespace, pod_name) ", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{namespace}}/{{pod_name}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "- sum(topk(5,\n rate(kubecost_pod_network_egress_bytes_total\n {namespace=~\"$namespace\", pod_name=~\"$pod\"}\n [$__rate_interval]\n )\n) )\nby(namespace, pod_name) ", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{namespace}}/{{pod_name}}", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Kubecost Top Network (egress is negative)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": 3600000, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(topk(5,rate(container_network_receive_bytes_total{pod=~\"$pod\",namespace=~\"$namespace\"}[$__rate_interval]))) by (namespace,pod) ", + "hide": false, + "instant": false, + "legendFormat": "{{namespace}}/{{pod}}", + "range": true, + "refId": "receive" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "-sum(topk(5,rate(container_network_transmit_bytes_total{pod=~\"$pod\",namespace=~\"$namespace\"}[$__rate_interval]))) by (namespace,pod) ", + "hide": false, + "instant": false, + "legendFormat": "{{namespace}}/{{pod}}", + "range": true, + "refId": "transmit" + } + ], + "title": "Top Network (transmit is negative)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": 3600000, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "\n sum(rate(container_fs_writes_bytes_total\n {pod=~\"$pod\",namespace=~\"$namespace\",image!=\"\"}\n [$__rate_interval]))\n by (namespace,pod,container)\n>0 ", + "hide": false, + "instant": false, + "legendFormat": "{{namespace}}/{{container}}", + "range": true, + "refId": "storage_write" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "-(sum\r\n (rate(container_fs_reads_bytes_total\r\n {pod=~\"$pod\",namespace=~\"$namespace\",image!=\"\"}\r\n [$__rate_interval])) \r\nby (namespace,pod,container) \r\n) <0", + "hide": false, + "instant": false, + "legendFormat": "{{namespace}}/{{container}}", + "range": true, + "refId": "storage_read" + } + ], + "title": "Storage (read is negative)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "This may work depending on the CRI", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(kubelet_volume_stats_available_bytes{namespace=~\"$namespace\"}) by (namespace,persistentvolumeclaim)", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(kube_persistentvolume_capacity_bytes) by (persistentvolume)", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Storage ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 82, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(kube_pod_container_status_restarts_total{namespace=~\"$namespace\",pod=~\"$pod\"}[1h])) by (namespace,container)>0", + "instant": false, + "interval": "1h", + "legendFormat": "{{namespace}}/{{container}}", + "range": true, + "refId": "A" + } + ], + "title": "Pod restarts per hour", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 39, + "tags": [ + "kubecost", + "utilization", + "metrics" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "PBFA97CFB590B2093" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": true, + "text": "kubecost", + "value": "kubecost" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(kube_namespace_labels,namespace)", + "hide": 0, + "includeAll": true, + "multi": false, + "name": "namespace", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(kube_namespace_labels,namespace)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(kube_pod_owner{namespace=~\"$namespace\"},pod)", + "hide": 0, + "includeAll": true, + "multi": false, + "name": "pod", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(kube_pod_owner{namespace=~\"$namespace\"},pod)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values({namespace=~\"$namespace\", pod=~\"$pod\"},container)", + "hide": 0, + "includeAll": true, + "multi": false, + "name": "container", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({namespace=~\"$namespace\", pod=~\"$pod\"},container)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Workload Metrics", + "uid": "kubecost-workload-metrics", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/questions.yaml b/charts/kubecost/cost-analyzer/2.5.3/questions.yaml new file mode 100644 index 0000000000..7717d04dc4 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/questions.yaml @@ -0,0 +1,187 @@ +questions: + # General Settings + - variable: kubecostProductConfigs.clusterName + label: Cluster Name + description: "Used for display in the cost-analyzer UI (Can be renamed in the UI)" + type: string + required: true + default: "" + group: General Settings + - variable: persistentVolume.enabled + label: Enable Persistent Volume for CostAnalyzer + description: "If true, Kubecost will create a Persistent Volume Claim for product config data" + type: boolean + default: false + show_subquestion_if: true + group: "General Settings" + subquestions: + - variable: persistentVolume.size + label: CostAnalyzer Persistent Volume Size + type: string + default: "0.2Gi" + # Amazon EKS + - variable: AmazonEKS.enabled + label: Amazon EKS cluster + description: "If true, Kubecost will be installed with the images and helm chart from https://gallery.ecr.aws/kubecost/" + type: boolean + default: false + show_subquestion_if: true + group: General Settings + subquestions: + - variable: kubecostFrontend.image + label: Kubecost frontend image for Amazon EKS + description: "Use this image for the Amazon EKS cluster: public.ecr.aws/kubecost/frontend" + type: string + default: "" + - variable: kubecostModel.image + label: Kubecost cost-model image for Amazon EKS + description: "Use this image for the Amazon EKS cluster: public.ecr.aws/kubecost/cost-model" + type: string + default: "" + - variable: prometheus.server.image.repository + label: Kubecost Prometheus image for Amazon EKS + description: "Use this image for the Amazon EKS cluster: public.ecr.aws/kubecost/prometheus" + type: string + default: "" + - variable: prometheus.server.image.tag + label: Kubecost Prometheus image tag for Amazon EKS + type: string + default: "v2.35.0" + # Prometheus Server + - variable: global.prometheus.enabled + label: Enable Prometheus + description: If false, use an existing Prometheus install + type: boolean + default: true + group: "Prometheus" + - variable: prometheus.kubeStateMetrics.enabled + label: Enable KubeStateMetrics + description: "If true, deploy kube-state-metrics for Kubernetes metrics" + type: boolean + default: true + show_if: "global.prometheus.enabled=true" + group: "Prometheus" + - variable: prometheus.server.retention + label: Prometheus Server Retention + description: "Determines when to remove old data" + type: string + default: "15d" + show_if: "global.prometheus.enabled=true" + group: "Prometheus" + - variable: prometheus.server.persistentVolume.enabled + label: Create Persistent Volume for Prometheus + description: "If true, prometheus will create a persistent volume claim" + type: boolean + required: true + default: false + group: "Prometheus" + show_if: "global.prometheus.enabled=true" + show_subquestion_if: true + subquestions: + - variable: prometheus.server.persistentVolume.size + label: Prometheus Persistent Volume Size + type: string + default: "8Gi" + - variable: prometheus.server.persistentVolume.storageClass + label: Prometheus Persistent Volume StorageClass + description: "Prometheus data persistent volume storageClass, if not set use default StorageClass" + default: "" + type: storageclass + - variable: prometheus.server.persistentVolume.existingClaim + label: Existing Persistent Volume Claim for Prometheus + description: "If not empty, uses the specified existing PVC instead of creating new one" + type: pvc + default: "" + + # Prometheus Node Exporter + - variable: prometheus.nodeExporter.enabled + label: Enable NodeExporter + description: "If false, do not create NodeExporter daemonset" + type: boolean + default: true + group: "NodeExporter" + - variable: prometheus.serviceAccounts.nodeExporter.create + label: Enable Service Accounts NodeExporter + description: "If false, do not create NodeExporter daemonset" + type: boolean + default: true + group: "NodeExporter" + + # Prometheus AlertManager + - variable: prometheus.alertmanager.enabled + label: Enable AlertManager + type: boolean + default: false + group: "AlertManager" + - variable: prometheus.alertmanager.persistentVolume.enabled + label: Create Persistent Volume for AlertManager + description: "If true, alertmanager will create a persistent volume claim" + type: boolean + required: true + default: false + group: "AlertManager" + show_if: "prometheus.alertmanager.enabled=true" + show_subquestion_if: true + subquestions: + - variable: prometheus.alertmanager.persistentVolume.size + default: "2Gi" + description: "AlertManager data persistent volume size" + type: string + label: AlertManager Persistent Volume Size + - variable: prometheus.alertmanager.persistentVolume.storageClass + default: "" + description: "Alertmanager data persistent volume storageClass, if not set use default StorageClass" + type: storageclass + label: AlertManager Persistent Volume StorageClass + - variable: prometheus.alertmanager.persistentVolume.existingClaim + default: "" + description: "If not empty, uses the specified existing PVC instead of creating new one" + type: pvc + label: Existing Persistent Volume Claim for AlertManager + + # PushGateway + - variable: prometheus.pushgateway.enabled + label: Enable PushGateway + type: boolean + default: false + group: "PushGateway" + - variable: prometheus.pushgateway.persistentVolume.enabled + label: Create Persistent Volume for PushGateway + description: "If true, PushGateway will create a persistent volume claim" + required: true + type: boolean + default: false + group: "PushGateway" + show_if: "prometheus.pushgateway.enabled=true" + show_subquestion_if: true + subquestions: + - variable: prometheus.prometheus.pushgateway.persistentVolume.size + label: PushGateway Persistent Volume Size + type: string + default: "2Gi" + - variable: prometheus.pushgateway.persistentVolume.storageClass + label: PushGateway Persistent Volume StorageClass + description: "PushGateway data persistent volume storageClass, if not set use default StorageClass" + type: storageclass + default: "" + - variable: prometheus.pushgateway.persistentVolume.existingClaim + label: Existing Persistent Volume Claim for PushGateway + description: "If not empty, uses the specified existing PVC instead of creating new one" + type: pvc + default: "" + + # Services and Load Balancing + - variable: ingress.enabled + label: Enable Ingress + description: "Expose app using Ingress (Layer 7 Load Balancer)" + default: false + type: boolean + show_subquestion_if: true + group: "Services and Load Balancing" + subquestions: + - variable: ingress.hosts[0] + default: "xip.io" + description: "Hostname to your CostAnalyzer installation" + type: hostname + required: true + label: Hostname diff --git a/charts/kubecost/cost-analyzer/2.5.3/scripts/create-admission-controller-tls.sh b/charts/kubecost/cost-analyzer/2.5.3/scripts/create-admission-controller-tls.sh new file mode 100644 index 0000000000..2290cadd1b --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/scripts/create-admission-controller-tls.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +set -eo pipefail + +if [ -z "$1" ]; then + namespace=kubecost +else + namespace="$1" +fi + +echo -e "\nCreating certificates ..." +mkdir certs +openssl genrsa -out certs/tls.key 2048 +openssl req -new -key certs/tls.key -out certs/tls.csr -subj "/CN=webhook-server.${namespace}.svc" +openssl x509 -req -days 500 -extfile <(printf "subjectAltName=DNS:webhook-server.%s.svc" "${namespace}") -in certs/tls.csr -signkey certs/tls.key -out certs/tls.crt + +echo -e "\nCreating Webhook Server TLS Secret ..." +kubectl create secret tls webhook-server-tls \ + --cert "certs/tls.crt" \ + --key "certs/tls.key" -n "${namespace}" + +ENCODED_CA=$(base64 < certs/tls.crt | tr -d '\n') + +if [ -f "../values.yaml" ]; then + echo -e "\nUpdating values.yaml ..." + sed -i '' 's@${CA_BUNDLE}@'"${ENCODED_CA}"'@g' ../values.yaml +else + echo -e "\nThe CA bundle to use in your values file is: \n${ENCODED_CA}" +fi \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/NOTES.txt b/charts/kubecost/cost-analyzer/2.5.3/templates/NOTES.txt new file mode 100644 index 0000000000..e653b3e717 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/NOTES.txt @@ -0,0 +1,31 @@ +-------------------------------------------------- +{{- include "kubecostV2-preconditions" . -}} +{{- include "cloudIntegrationSourceCheck" . -}} +{{- include "eksCheck" . -}} +{{- include "cloudIntegrationSecretCheck" . -}} +{{- include "gcpCloudIntegrationCheck" . -}} +{{- include "azureCloudIntegrationCheck" . -}} +{{- include "federatedStorageConfigSecretCheck" . -}} +{{- include "federatedStorageSourceCheck" . -}} +{{- include "prometheusRetentionCheck" . -}} +{{- include "clusterIDCheck" . -}} +{{- include "kubeRBACProxyBearerTokenCheck" . -}} +{{- include "caCertsSecretConfigCheck" . -}} + +{{- $servicePort := .Values.service.port | default 9090 }} +Kubecost {{ .Chart.Version }} has been successfully installed. + +Kubecost 2.x is a major upgrade from previous versions and includes major new features including a brand new API Backend. Please review the following documentation to ensure a smooth transition: https://docs.kubecost.com/install-and-configure/install/kubecostv2 + +When pods are Ready, you can enable port-forwarding with the following command: + + kubectl port-forward --namespace {{ .Release.Namespace }} deployment/{{ template "cost-analyzer.fullname" . }} {{ $servicePort }} + +Then, navigate to http://localhost:{{ $servicePort }} in a web browser. + +Please allow 25 minutes for Kubecost to gather metrics. A progress indicator will appear at the top of the UI. + +Having installation issues? View our Troubleshooting Guide at http://docs.kubecost.com/troubleshoot-install + +{{- include "kubecostV2-3-notices" . -}} +{{- include "federatedStorageCheck" . -}} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/_helpers.tpl b/charts/kubecost/cost-analyzer/2.5.3/templates/_helpers.tpl new file mode 100644 index 0000000000..67ff512f5d --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/_helpers.tpl @@ -0,0 +1,1558 @@ +{{/* vim: set filetype=mustache: */}} + +{{/* +Set important variables before starting main templates +*/}} +{{- define "aggregator.deployMethod" -}} + {{- if (.Values.federatedETL).primaryCluster }} + {{- printf "statefulset" }} + {{- else if or ((.Values.federatedETL).agentOnly) (.Values.agent) (.Values.cloudAgent) }} + {{- printf "disabled" }} + {{- else if (not .Values.kubecostAggregator) }} + {{- printf "singlepod" }} + {{- else if .Values.kubecostAggregator.enabled }} + {{- printf "statefulset" }} + {{- else if eq .Values.kubecostAggregator.deployMethod "singlepod" }} + {{- printf "singlepod" }} + {{- else if eq .Values.kubecostAggregator.deployMethod "statefulset" }} + {{- printf "statefulset" }} + {{- else if eq .Values.kubecostAggregator.deployMethod "disabled" }} + {{- printf "disabled" }} + {{- else }} + {{- fail "Unknown kubecostAggregator.deployMethod value" }} + {{- end }} +{{- end }} + +{{- define "frontend.deployMethod" -}} + {{- if eq .Values.kubecostFrontend.deployMethod "haMode" -}} + {{- printf "haMode" -}} + {{- else -}} + {{- printf "singlepod" -}} + {{- end -}} +{{- end -}} + +{{/* +Kubecost 2.3 notices +*/}} +{{- define "kubecostV2-3-notices" -}} + {{- if (.Values.kubecostAggregator).env -}} + {{- printf "\n\n\nNotice: Issue in values detected.\nKubecost 2.3 has updated the aggregator's environment variables. Please update your Helm values to use the new key pairs.\nFor more information, see: https://docs.kubecost.com/install-and-configure/install/multi-cluster/federated-etl/aggregator#aggregator-optimizations\nIn Kubecost 2.3, kubecostAggregator.env is no longer used in favor of the new key pairs. This was done to prevent unexpected behavior and to simplify the aggregator's configuration." -}} + {{- end -}} +{{- end -}} + +{{/* +Kubecost 2.0 preconditions +*/}} +{{- define "kubecostV2-preconditions" -}} + {{/* Iterate through all StatefulSets in the namespace and check if any of them have a label indicating they are from + a pre-2.0 Helm Chart (e.g. "helm.sh/chart: cost-analyzer-1.108.1"). If so, return an error message with details and + documentation for how to properly upgrade to Kubecost 2.0 */}} + {{- $sts := (lookup "apps/v1" "StatefulSet" .Release.Namespace "") -}} + {{- if not (empty $sts.items) -}} + {{- range $index, $sts := $sts.items -}} + {{- if contains "aggregator" $sts.metadata.name -}} + {{- if $sts.metadata.labels -}} + {{- $stsLabels := $sts.metadata.labels -}} {{/* helm.sh/chart: cost-analyzer-1.108.1 */}} + {{- if hasKey $stsLabels "helm.sh/chart" -}} + {{- $chartLabel := index $stsLabels "helm.sh/chart" -}} {{/* cost-analyzer-1.108.1 */}} + {{- $chartNameAndVersion := split "-" $chartLabel -}} {{/* _0:cost _1:analyzer _2:1.108.1 */}} + {{- if gt (len $chartNameAndVersion) 2 -}} + {{- $chartVersion := $chartNameAndVersion._2 -}} {{/* 1.108.1 */}} + {{- if semverCompare ">=1.0.0-0 <2.0.0-0" $chartVersion -}} + {{- fail "\n\nAn existing Aggregator StatefulSet was found in your namespace.\nBefore upgrading to Kubecost 2.x, please `kubectl delete` this Statefulset.\nRefer to the following documentation for more information: https://docs.kubecost.com/install-and-configure/install/kubecostv2" -}} + {{- end -}} + {{- end -}} + {{- end -}} + {{- end -}} + {{- end -}} + {{- end -}} + {{- end -}} + + {{/*https://github.com/helm/helm/issues/8026#issuecomment-881216078*/}} + {{- if ((.Values.thanos).store).enabled -}} + {{- fail "\n\nYou are attempting to upgrade to Kubecost 2.x.\nKubecost no longer includes Thanos by default. \nPlease see https://docs.kubecost.com/install-and-configure/install/kubecostv2 for more information.\nIf you have any questions or concerns, please reach out to us at product@kubecost.com" -}} + {{- end -}} + + {{- if or ((.Values.saml).rbac).enabled ((.Values.oidc).rbac).enabled -}} + {{- if (not (.Values.upgrade).toV2) -}} + {{- fail "\n\nSSO with RBAC is enabled.\nNote that Kubecost 2.x has significant architectural changes that may impact RBAC.\nThis should be tested before giving end-users access to the UI.\nKubecost has tested various configurations and believe that 2.x will be 100% compatible with existing configurations.\nRefer to the following documentation for more information: https://docs.kubecost.com/install-and-configure/install/kubecostv2\n\nWhen ready to upgrade, add `--set upgrade.toV2=true`." -}} + {{- end -}} + {{- end -}} + + {{/* Aggregator config reconciliation and common config */}} + {{- if eq (include "aggregator.deployMethod" .) "statefulset" -}} + {{- if .Values.kubecostAggregator -}} + {{- if (not .Values.kubecostAggregator.aggregatorDbStorage) -}} + {{- fail "In Enterprise configuration, Aggregator DB storage is required" -}} + {{- end -}} + {{- end -}} + {{- end -}} + + {{- if (.Values.podSecurityPolicy).enabled }} + {{- fail "Kubecost no longer includes PodSecurityPolicy by default. Please take steps to preserve your existing PSPs before attempting the installation/upgrade again with the podSecurityPolicy values removed." }} + {{- end }} + + {{- if ((.Values.kubecostDeployment).leaderFollower).enabled -}} + {{- fail "\nIn Kubecost 2.0, kubecostDeployment does not support running as leaderFollower. Please reach out to support to discuss upgrade paths." -}} + {{- end -}} + + {{- if ((.Values.kubecostDeployment).statefulSet).enabled -}} + {{- fail "\nIn Kubecost 2.0, kubecostDeployment does not support running as a statefulSet. Please reach out to support to discuss upgrade paths." -}} + {{- end -}} + {{- if and (eq (include "aggregator.deployMethod" .) "statefulset") (.Values.federatedETL).agentOnly }} + {{- fail "\nKubecost does not support running federatedETL.agentOnly with the aggregator statefulset" }} + {{- end }} +{{- end -}} + +{{- define "federatedStorageCheck" -}} + {{- if or (.Values.federatedETL).federatedStore (.Values.kubecostModel).federatedStorageConfig }} + {{- if and (not (eq (include "aggregator.deployMethod" .) "statefulset")) (not (.Values.federatedETL).agentOnly) }} + {{- printf "\n\n***Configuration issue detected:***\nWhen a federated store is provided, Kubecost should either be running as agentOnly or as a statefulset.\n.Values.federatedETL.agentOnly=true\nOr\n.Values.kubecostAggregator.deployMethod=statefulset\n***" }} + {{- end }} + {{- end }} +{{- end }} + +{{- define "cloudIntegrationFromProductConfigs" }} + { + {{- if ((.Values.kubecostProductConfigs).athenaBucketName) }} + "aws": [ + { + "athenaBucketName": "{{ .Values.kubecostProductConfigs.athenaBucketName }}", + "athenaRegion": "{{ .Values.kubecostProductConfigs.athenaRegion }}", + "athenaDatabase": "{{ .Values.kubecostProductConfigs.athenaDatabase }}", + "athenaTable": "{{ .Values.kubecostProductConfigs.athenaTable }}", + "projectID": "{{ .Values.kubecostProductConfigs.athenaProjectID }}" + {{ if (.Values.kubecostProductConfigs).athenaWorkgroup }} + , "athenaWorkgroup": "{{ .Values.kubecostProductConfigs.athenaWorkgroup }}" + {{ else }} + , "athenaWorkgroup": "primary" + {{ end }} + {{ if (.Values.kubecostProductConfigs).masterPayerARN }} + , "masterPayerARN": "{{ .Values.kubecostProductConfigs.masterPayerARN }}" + {{ end }} + {{- if and ((.Values.kubecostProductConfigs).awsServiceKeyName) ((.Values.kubecostProductConfigs).awsServiceKeyPassword) }}, + "serviceKeyName": "{{ .Values.kubecostProductConfigs.awsServiceKeyName }}", + "serviceKeySecret": "{{ .Values.kubecostProductConfigs.awsServiceKeyPassword }}" + {{- end }} + } + ] + {{- end }} + } +{{- end }} + +{{/* +Cloud integration source contents check. Either the Secret must be specified or the JSON, not both. +Additionally, for upgrade protection, certain individual values populated under the kubecostProductConfigs map, if found, +will result in failure. Users are asked to select one of the two presently-available sources for cloud integration information. +*/}} +{{- define "cloudIntegrationSourceCheck" -}} + {{- if and (.Values.kubecostProductConfigs).cloudIntegrationSecret (.Values.kubecostProductConfigs).cloudIntegrationJSON -}} + {{- fail "\nkubecostProductConfigs.cloudIntegrationSecret and kubecostProductConfigs.cloudIntegrationJSON are mutually exclusive. Please specify only one." -}} + {{- end -}} + {{- if and (.Values.kubecostProductConfigs).cloudIntegrationSecret ((.Values.kubecostProductConfigs).athenaBucketName) }} + {{- fail "\nkubecostProductConfigs.cloudIntegrationSecret and kubecostProductConfigs.athena* values are mutually exclusive. Please specifiy only one." -}} + {{- end -}} +{{- if and (.Values.kubecostProductConfigs).cloudIntegrationJSON ((.Values.kubecostProductConfigs).athenaBucketName) }} + {{- fail "\nkubecostProductConfigs.cloudIntegrationJSON and kubecostProductConfigs.athena* values are mutually exclusive. Please specifiy only one." -}} + {{- end -}} +{{- end -}} + +{{/* +Federated Storage source contents check. Either the Secret must be specified or the JSON, not both. +*/}} +{{- define "federatedStorageSourceCheck" -}} + {{- if and (.Values.kubecostModel).federatedStorageConfigSecret (.Values.kubecostModel).federatedStorageConfig -}} + {{- fail "\nkubecostkubecostModel.federatedStorageConfigSecret and kubecostModel.federatedStorageConfig are mutually exclusive. Please specify only one." -}} + {{- end -}} +{{- end -}} + +{{/* +Print a warning if PV is enabled AND EKS is detected AND the EBS-CSI driver is not installed +*/}} +{{- define "eksCheck" }} +{{- $isEKS := (regexMatch ".*eks.*" (.Capabilities.KubeVersion | quote) )}} +{{- $isGT22 := (semverCompare ">=1.23-0" .Capabilities.KubeVersion.GitVersion) }} +{{- $PVNotExists := (empty (lookup "v1" "PersistentVolume" "" "")) }} +{{- $EBSCSINotExists := (empty (lookup "apps/v1" "Deployment" "kube-system" "ebs-csi-controller")) }} +{{- if (and $isEKS $isGT22 .Values.persistentVolume.enabled $EBSCSINotExists) -}} + +ERROR: MISSING EBS-CSI DRIVER WHICH IS REQUIRED ON EKS v1.23+ TO MANAGE PERSISTENT VOLUMES. LEARN MORE HERE: https://docs.kubecost.com/install-and-configure/install/provider-installations/aws-eks-cost-monitoring#prerequisites + +{{- end -}} +{{- end -}} + +{{/* +Verify a cluster_id is set in the Prometheus global config +*/}} +{{- define "clusterIDCheck" -}} + {{- if or (.Values.kubecostModel).federatedStorageConfigSecret (.Values.kubecostModel).federatedStorageConfig }} + {{- if not .Values.prometheus.server.clusterIDConfigmap }} + {{- if eq .Values.prometheus.server.global.external_labels.cluster_id "cluster-one" }} + {{- fail "\n\nWhen using multi-cluster Kubecost, you must specify a unique `.Values.prometheus.server.global.external_labels.cluster_id` for each cluster.\nNote this must be set even if you are using your own Prometheus or another identifier.\n" -}} + {{- end -}} + {{- end -}} + {{- end -}} +{{- end -}} + +{{/* + Verify if both kube-rbac-proxy and bearer token are set +*/}} +{{- define "kubeRBACProxyBearerTokenCheck" -}} +{{- if and (.Values.global.prometheus.kubeRBACProxy) (.Values.global.prometheus.queryServiceBearerTokenSecretName) }} + {{- fail "\n\nBoth kubeRBACProxy and queryServiceBearerTokenSecretName are set. Please specify only one." -}} +{{- end -}} +{{- end -}} + + +{{/* +Verify the cloud integration secret exists with the expected key when cloud integration is enabled. +Skip the check if CI/CD is enabled and skipSanityChecks is set. Argo CD, for example, does not +support templating a chart which uses the lookup function. +*/}} +{{- define "cloudIntegrationSecretCheck" -}} +{{- if (.Values.kubecostProductConfigs).cloudIntegrationSecret }} +{{- if not (and .Values.global.platforms.cicd.enabled .Values.global.platforms.cicd.skipSanityChecks) }} +{{- if .Capabilities.APIVersions.Has "v1/Secret" }} + {{- $secret := lookup "v1" "Secret" .Release.Namespace .Values.kubecostProductConfigs.cloudIntegrationSecret }} + {{- if or (not $secret) (not (index $secret.data "cloud-integration.json")) }} + {{- fail (printf "The cloud integration secret '%s' does not exist or does not contain the expected key 'cloud-integration.json'\nIf you are using `--dry-run`, please add `--dry-run=server`. This requires Helm 3.13+." .Values.kubecostProductConfigs.cloudIntegrationSecret) }} + {{- end }} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Verify the federated storage config secret exists with the expected key. +Skip the check if CI/CD is enabled and skipSanityChecks is set. Argo CD, for +example, does not support templating a chart which uses the lookup function. +*/}} +{{- define "federatedStorageConfigSecretCheck" -}} +{{- if (.Values.kubecostModel).federatedStorageConfigSecret }} +{{- if not (and .Values.global.platforms.cicd.enabled .Values.global.platforms.cicd.skipSanityChecks) }} +{{- if .Capabilities.APIVersions.Has "v1/Secret" }} + {{- $secret := lookup "v1" "Secret" .Release.Namespace .Values.kubecostModel.federatedStorageConfigSecret }} + {{- if or (not $secret) (not (index $secret.data "federated-store.yaml")) }} + {{- fail (printf "The federated storage config secret '%s' does not exist or does not contain the expected key 'federated-store.yaml'" .Values.kubecostModel.federatedStorageConfigSecret) }} + {{- end }} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* + Ensure that the Prometheus retention is not set too low +*/}} +{{- define "prometheusRetentionCheck" }} +{{- if ((.Values.prometheus).server).enabled }} + + {{- $retention := .Values.prometheus.server.retention }} + {{- $etlHourlyDurationHours := (int .Values.kubecostModel.etlHourlyStoreDurationHours) }} + + {{- if (hasSuffix "d" $retention) }} + {{- $retentionDays := (int (trimSuffix "d" $retention)) }} + {{- if lt $retentionDays 3 }} + {{- fail (printf "With a daily resolution, Prometheus retention must be set >= 3 days. Provided retention is %s" $retention) }} + {{- else if le (mul $retentionDays 24) $etlHourlyDurationHours }} + {{- fail (printf "Prometheus retention (%s) must be greater than .Values.kubecostModel.etlHourlyStoreDurationHours (%d)" $retention $etlHourlyDurationHours) }} + {{- end }} + + {{- else if (hasSuffix "h" $retention) }} + {{- $retentionHours := (int (trimSuffix "h" $retention)) }} + {{- if lt $retentionHours 50 }} + {{- fail (printf "With an hourly resolution, Prometheus retention must be set >= 50 hours. Provided retention is %s" $retention) }} + {{- else if le $retentionHours $etlHourlyDurationHours }} + {{- fail (printf "Prometheus retention (%s) must be greater than .Values.kubecostModel.etlHourlyStoreDurationHours (%d)" $retention $etlHourlyDurationHours) }} + {{- end }} + + {{- else }} + {{- fail "prometheus.server.retention must be set in days (e.g. 5d) or hours (e.g. 97h)"}} + + {{- end }} +{{- end }} +{{- end }} + +{{/* +Expand the name of the chart. +*/}} +{{- define "cost-analyzer.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- define "aggregator.name" -}} +{{- default "aggregator" | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- define "cloudCost.name" -}} +{{- default "cloud-cost" | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- define "etlUtils.name" -}} +{{- default "etl-utils" | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- define "forecasting.name" -}} +{{- default "forecasting" | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- define "frontend.name" -}} +{{- default "frontend" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "cost-analyzer.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{- define "diagnostics.fullname" -}} +{{- if .Values.diagnosticsFullnameOverride -}} +{{- .Values.diagnosticsFullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name "diagnostics" | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} + +{{- define "aggregator.fullname" -}} +{{- printf "%s-%s" .Release.Name "aggregator" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "cloudCost.fullname" -}} +{{- printf "%s-%s" .Release.Name (include "cloudCost.name" .) | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "etlUtils.fullname" -}} +{{- printf "%s-%s" .Release.Name (include "etlUtils.name" .) | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- define "forecasting.fullname" -}} +{{- printf "%s-%s" .Release.Name (include "forecasting.name" .) | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- define "frontend.fullname" -}} +{{- printf "%s-%s" .Release.Name (include "frontend.name" .) | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create the fully qualified name for Prometheus server service. +*/}} +{{- define "cost-analyzer.prometheus.server.name" -}} +{{- if .Values.prometheus -}} +{{- if .Values.prometheus.server -}} +{{- if .Values.prometheus.server.fullnameOverride -}} +{{- .Values.prometheus.server.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-prometheus-server" .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- else -}} +{{- printf "%s-prometheus-server" .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- else -}} +{{- printf "%s-prometheus-server" .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} + +{{/* +Create the fully qualified name for Prometheus alertmanager service. +*/}} +{{- define "cost-analyzer.prometheus.alertmanager.name" -}} +{{- if .Values.prometheus -}} +{{- if .Values.prometheus.alertmanager -}} +{{- if .Values.prometheus.alertmanager.fullnameOverride -}} +{{- .Values.prometheus.alertmanager.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-prometheus-alertmanager" .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- else -}} +{{- printf "%s-prometheus-alertmanager" .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- else -}} +{{- printf "%s-prometheus-alertmanager" .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} + +{{- define "cost-analyzer.serviceName" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{- define "frontend.serviceName" -}} +{{ include "frontend.fullname" . }} +{{- end -}} + +{{- define "diagnostics.serviceName" -}} +{{- printf "%s-%s" .Release.Name "diagnostics" | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- define "aggregator.serviceName" -}} +{{- printf "%s-%s" .Release.Name "aggregator" | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- define "cloudCost.serviceName" -}} +{{ include "cloudCost.fullname" . }} +{{- end -}} +{{- define "etlUtils.serviceName" -}} +{{ include "etlUtils.fullname" . }} +{{- end -}} +{{- define "forecasting.serviceName" -}} +{{ include "forecasting.fullname" . }} +{{- end -}} + +{{/* +Create the name of the service account +*/}} +{{- define "cost-analyzer.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "cost-analyzer.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} +{{- define "aggregator.serviceAccountName" -}} +{{- if .Values.kubecostAggregator.serviceAccountName -}} + {{ .Values.kubecostAggregator.serviceAccountName }} +{{- else -}} + {{ template "cost-analyzer.serviceAccountName" . }} +{{- end -}} +{{- end -}} +{{- define "cloudCost.serviceAccountName" -}} +{{- if .Values.kubecostAggregator.cloudCost.serviceAccountName -}} + {{ .Values.kubecostAggregator.cloudCost.serviceAccountName }} +{{- else -}} + {{ template "cost-analyzer.serviceAccountName" . }} +{{- end -}} +{{- end -}} +{{/* +Network Costs name used to tie autodiscovery of metrics to daemon set pods +*/}} +{{- define "cost-analyzer.networkCostsName" -}} +{{- printf "%s-%s" .Release.Name "network-costs" -}} +{{- end -}} + +{{- define "kubecost.clusterControllerName" -}} +{{- printf "%s-%s" .Release.Name "cluster-controller" -}} +{{- end -}} + +{{- define "kubecost.kubeMetricsName" -}} +{{- if .Values.agent }} +{{- printf "%s-%s" .Release.Name "agent" -}} +{{- else if .Values.cloudAgent }} +{{- printf "%s-%s" .Release.Name "cloud-agent" -}} +{{- else }} +{{- printf "%s-%s" .Release.Name "metrics" -}} +{{- end }} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "cost-analyzer.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create the chart labels. +*/}} +{{- define "cost-analyzer.chartLabels" -}} +helm.sh/chart: {{ include "cost-analyzer.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- if .Values.chartLabels }} +{{ toYaml .Values.chartLabels }} +{{- end }} +{{- end -}} + + +{{/* +Create the common labels. +*/}} +{{- define "cost-analyzer.commonLabels" -}} +app.kubernetes.io/name: {{ include "cost-analyzer.name" . }} +helm.sh/chart: {{ include "cost-analyzer.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app: cost-analyzer +{{- end -}} + +{{- define "aggregator.commonLabels" -}} +{{ include "cost-analyzer.chartLabels" . }} +app: aggregator +{{- end -}} + +{{- define "diagnostics.commonLabels" -}} +{{ include "cost-analyzer.chartLabels" . }} +app: diagnostics +{{- end -}} + +{{- define "cloudCost.commonLabels" -}} +{{ include "cost-analyzer.chartLabels" . }} +{{ include "cloudCost.selectorLabels" . }} +{{- end -}} + +{{- define "etlUtils.commonLabels" -}} +{{ include "cost-analyzer.chartLabels" . }} +{{ include "etlUtils.selectorLabels" . }} +{{- end -}} +{{- define "forecasting.commonLabels" -}} +{{ include "cost-analyzer.chartLabels" . }} +{{ include "forecasting.selectorLabels" . }} +{{- end -}} + +{{/* +Create the networkcosts common labels. Note that because this is a daemonset, we don't want app.kubernetes.io/instance: to take the release name, which allows the scrape config to be static. +*/}} +{{- define "networkcosts.commonLabels" -}} +app.kubernetes.io/instance: kubecost +app.kubernetes.io/name: network-costs +helm.sh/chart: {{ include "cost-analyzer.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app: {{ template "cost-analyzer.networkCostsName" . }} +{{- end -}} +{{- define "networkcosts.selectorLabels" -}} +app: {{ template "cost-analyzer.networkCostsName" . }} +{{- end }} +{{- define "diagnostics.selectorLabels" -}} +app.kubernetes.io/name: diagnostics +app.kubernetes.io/instance: {{ .Release.Name }} +app: diagnostics +{{- end }} + +{{/* +Create the selector labels. +*/}} +{{- define "cost-analyzer.selectorLabels" -}} +app.kubernetes.io/name: {{ include "cost-analyzer.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app: cost-analyzer +{{- end -}} + +{{/* +Create the selector labels for haMode frontend. +*/}} +{{- define "frontend.selectorLabels" -}} +app.kubernetes.io/name: {{ include "frontend.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app: cost-analyzer +{{- end -}} + +{{- define "aggregator.selectorLabels" -}} +{{- if eq (include "aggregator.deployMethod" .) "statefulset" }} +app.kubernetes.io/name: {{ include "aggregator.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app: aggregator +{{- else if eq (include "aggregator.deployMethod" .) "singlepod" }} +{{- include "cost-analyzer.selectorLabels" . }} +{{- else }} +{{ fail "Failed to set aggregator.selectorLabels" }} +{{- end }} +{{- end }} + +{{- define "cloudCost.selectorLabels" -}} +{{- if eq (include "aggregator.deployMethod" .) "statefulset" }} +app.kubernetes.io/name: {{ include "cloudCost.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app: {{ include "cloudCost.name" . }} +{{- else }} +{{- include "cost-analyzer.selectorLabels" . }} +{{- end }} +{{- end }} + +{{- define "forecasting.selectorLabels" -}} +app.kubernetes.io/name: {{ include "forecasting.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app: {{ include "forecasting.name" . }} +{{- end -}} +{{- define "etlUtils.selectorLabels" -}} +app.kubernetes.io/name: {{ include "etlUtils.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app: {{ include "etlUtils.name" . }} +{{- end -}} + +{{/* +Recursive filter which accepts a map containing an input map (.v) and an output map (.r). The template +will traverse all values inside .v recursively writing non-map values to the output .r. If a nested map +is discovered, we look for an 'enabled' key. If it doesn't exist, we continue traversing the +map. If it does exist, we omit the inner map traversal iff enabled is false. This filter writes the +enabled only version to the output .r +*/}} +{{- define "cost-analyzer.filter" -}} +{{- $v := .v }} +{{- $r := .r }} +{{- range $key, $value := .v }} + {{- $tp := kindOf $value -}} + {{- if eq $tp "map" -}} + {{- $isEnabled := true -}} + {{- if (hasKey $value "enabled") -}} + {{- $isEnabled = $value.enabled -}} + {{- end -}} + {{- if $isEnabled -}} + {{- $rr := "{}" | fromYaml }} + {{- template "cost-analyzer.filter" (dict "v" $value "r" $rr) }} + {{- $_ := set $r $key $rr -}} + {{- end -}} + {{- else -}} + {{- $_ := set $r $key $value -}} + {{- end -}} +{{- end -}} +{{- end -}} + +{{/* +This template accepts a map and returns a base64 encoded json version of the map where all disabled +leaf nodes are omitted. + +The implied use case is {{ template "cost-analyzer.filterEnabled" .Values }} +*/}} +{{- define "cost-analyzer.filterEnabled" -}} +{{- $result := "{}" | fromYaml }} +{{- template "cost-analyzer.filter" (dict "v" . "r" $result) }} +{{- $result | toJson | b64enc }} +{{- end -}} + +{{/* +============================================================== +Begin Prometheus templates +============================================================== +*/}} +{{/* +Expand the name of the chart. +*/}} +{{- define "prometheus.name" -}} +{{- "prometheus" -}} +{{- end -}} + +{{/* +Define common selector labels for all Prometheus components +*/}} +{{- define "prometheus.common.matchLabels" -}} +app: {{ template "prometheus.name" . }} +release: {{ .Release.Name }} +{{- end -}} + +{{/* +Define common top-level labels for all Prometheus components +*/}} +{{- define "prometheus.common.metaLabels" -}} +heritage: {{ .Release.Service }} +{{- end -}} + +{{/* +Define top-level labels for Alert Manager +*/}} +{{- define "prometheus.alertmanager.labels" -}} +{{ include "prometheus.alertmanager.matchLabels" . }} +{{ include "prometheus.common.metaLabels" . }} +{{- end -}} + +{{/* +Define selector labels for Alert Manager +*/}} +{{- define "prometheus.alertmanager.matchLabels" -}} +component: {{ .Values.prometheus.alertmanager.name | quote }} +{{ include "prometheus.common.matchLabels" . }} +{{- end -}} + +{{/* +Define top-level labels for Node Exporter +*/}} +{{- define "prometheus.nodeExporter.labels" -}} +{{ include "prometheus.nodeExporter.matchLabels" . }} +{{ include "prometheus.common.metaLabels" . }} +{{- end -}} + +{{/* +Define selector labels for Node Exporter +*/}} +{{- define "prometheus.nodeExporter.matchLabels" -}} +component: {{ .Values.prometheus.nodeExporter.name | quote }} +{{ include "prometheus.common.matchLabels" . }} +{{- end -}} + +{{/* +Define top-level labels for Push Gateway +*/}} +{{- define "prometheus.pushgateway.labels" -}} +{{ include "prometheus.pushgateway.matchLabels" . }} +{{ include "prometheus.common.metaLabels" . }} +{{- end -}} + +{{/* +Define selector labels for Push Gateway +*/}} +{{- define "prometheus.pushgateway.matchLabels" -}} +component: {{ .Values.prometheus.pushgateway.name | quote }} +{{ include "prometheus.common.matchLabels" . }} +{{- end -}} + +{{/* +Define top-level labels for Server +*/}} +{{- define "prometheus.server.labels" -}} +{{ include "prometheus.server.matchLabels" . }} +{{ include "prometheus.common.metaLabels" . }} +{{- end -}} + +{{/* +Define selector labels for Server +*/}} +{{- define "prometheus.server.matchLabels" -}} +component: {{ .Values.prometheus.server.name | quote }} +{{ include "prometheus.common.matchLabels" . }} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "prometheus.fullname" -}} +{{- if .Values.prometheus.fullnameOverride -}} +{{- .Values.prometheus.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default "prometheus" .Values.prometheus.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create a fully qualified alertmanager name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} + +{{- define "prometheus.alertmanager.fullname" -}} +{{- if .Values.prometheus.alertmanager.fullnameOverride -}} +{{- .Values.prometheus.alertmanager.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default "prometheus" .Values.prometheus.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- printf "%s-%s" .Release.Name .Values.prometheus.alertmanager.name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s-%s" .Release.Name $name .Values.prometheus.alertmanager.name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + + +{{/* +Create a fully qualified node-exporter name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "prometheus.nodeExporter.fullname" -}} +{{- if .Values.prometheus.nodeExporter.fullnameOverride -}} +{{- .Values.prometheus.nodeExporter.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default "prometheus" .Values.prometheus.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- printf "%s-%s" .Release.Name .Values.prometheus.nodeExporter.name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s-%s" .Release.Name $name .Values.prometheus.nodeExporter.name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create a fully qualified Prometheus server name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "prometheus.server.fullname" -}} +{{- if .Values.prometheus.server.fullnameOverride -}} +{{- .Values.prometheus.server.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default "prometheus" .Values.prometheus.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- printf "%s-%s" .Release.Name .Values.prometheus.server.name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s-%s" .Release.Name $name .Values.prometheus.server.name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create a fully qualified pushgateway name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "prometheus.pushgateway.fullname" -}} +{{- if .Values.prometheus.pushgateway.fullnameOverride -}} +{{- .Values.prometheus.pushgateway.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default "prometheus" .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- printf "%s-%s" .Release.Name .Values.prometheus.pushgateway.name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s-%s" .Release.Name $name .Values.prometheus.pushgateway.name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create the name of the service account to use for the alertmanager component +*/}} +{{- define "prometheus.serviceAccountName.alertmanager" -}} +{{- if .Values.prometheus.serviceAccounts.alertmanager.create -}} + {{ default (include "prometheus.alertmanager.fullname" .) .Values.prometheus.serviceAccounts.alertmanager.name }} +{{- else -}} + {{ default "default" .Values.prometheus.serviceAccounts.alertmanager.name }} +{{- end -}} +{{- end -}} + +{{/* +Create the name of the service account to use for the nodeExporter component +*/}} +{{- define "prometheus.serviceAccountName.nodeExporter" -}} +{{- if .Values.prometheus.serviceAccounts.nodeExporter.create -}} + {{ default (include "prometheus.nodeExporter.fullname" .) .Values.prometheus.serviceAccounts.nodeExporter.name }} +{{- else -}} + {{ default "default" .Values.prometheus.serviceAccounts.nodeExporter.name }} +{{- end -}} +{{- end -}} + +{{/* +Create the name of the service account to use for the pushgateway component +*/}} +{{- define "prometheus.serviceAccountName.pushgateway" -}} +{{- if .Values.prometheus.serviceAccounts.pushgateway.create -}} + {{ default (include "prometheus.pushgateway.fullname" .) .Values.prometheus.serviceAccounts.pushgateway.name }} +{{- else -}} + {{ default "default" .Values.prometheus.serviceAccounts.pushgateway.name }} +{{- end -}} +{{- end -}} + +{{/* +Create the name of the service account to use for the server component +*/}} +{{- define "prometheus.serviceAccountName.server" -}} +{{- if .Values.prometheus.serviceAccounts.server.create -}} + {{ default (include "prometheus.server.fullname" .) .Values.prometheus.serviceAccounts.server.name }} +{{- else -}} + {{ default "default" .Values.prometheus.serviceAccounts.server.name }} +{{- end -}} +{{- end -}} + +{{/* +============================================================== +Begin Grafana templates +============================================================== +*/}} +{{/* +Expand the name of the chart. +*/}} +{{- define "grafana.name" -}} +{{- "grafana" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "grafana.fullname" -}} +{{- if .Values.grafana.fullnameOverride -}} +{{- .Values.grafana.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default "grafana" .Values.grafana.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create the name of the service account +*/}} +{{- define "grafana.serviceAccountName" -}} +{{- if .Values.grafana.serviceAccount.create -}} + {{ default (include "grafana.fullname" .) .Values.grafana.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.grafana.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +============================================================== +Begin Kubecost 2.0 templates +============================================================== +*/}} + +{{- define "aggregator.containerTemplate" }} +- name: aggregator +{{- if .Values.kubecostAggregator.containerSecurityContext }} + securityContext: + {{- toYaml .Values.kubecostAggregator.containerSecurityContext | nindent 4 }} +{{- else if .Values.global.containerSecurityContext }} + securityContext: + {{- toYaml .Values.global.containerSecurityContext | nindent 4 }} +{{- end }} + {{- if .Values.kubecostModel }} + {{- if .Values.kubecostAggregator.fullImageName }} + image: {{ .Values.kubecostAggregator.fullImageName }} + {{- else if .Values.imageVersion }} + image: {{ .Values.kubecostModel.image }}:{{ .Values.imageVersion }} + {{- else if eq "development" .Chart.AppVersion }} + image: gcr.io/kubecost1/cost-model-nightly:latest + {{- else }} + image: {{ .Values.kubecostModel.image }}:prod-{{ $.Chart.AppVersion }} + {{- end }} + {{- else }} + image: gcr.io/kubecost1/cost-model:prod-{{ $.Chart.AppVersion }} + {{- end }} + {{- if .Values.kubecostAggregator.readinessProbe.enabled }} + readinessProbe: + httpGet: + path: /healthz + port: 9004 + initialDelaySeconds: {{ .Values.kubecostAggregator.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.kubecostAggregator.readinessProbe.periodSeconds }} + failureThreshold: {{ .Values.kubecostAggregator.readinessProbe.failureThreshold }} + {{- end }} + {{- if .Values.kubecostAggregator.imagePullPolicy }} + imagePullPolicy: {{ .Values.kubecostAggregator.imagePullPolicy }} + {{- else }} + imagePullPolicy: Always + {{- end }} + args: ["waterfowl"] + ports: + - name: tcp-api + containerPort: 9004 + protocol: TCP + {{- with.Values.kubecostAggregator.extraPorts }} + {{- toYaml . | nindent 4 }} + {{- end }} + resources: + {{- toYaml .Values.kubecostAggregator.resources | nindent 4 }} + volumeMounts: + - name: persistent-configs + mountPath: /var/configs + {{- if or (.Values.kubecostModel).federatedStorageConfigSecret (.Values.kubecostModel).federatedStorageConfig }} + - name: federated-storage-config + mountPath: /var/configs/etl + readOnly: true + {{- end }} + {{- if and .Values.persistentVolume.dbPVEnabled (eq (include "aggregator.deployMethod" .) "singlepod") }} + - name: persistent-db + mountPath: /var/db + # aggregator should only need read access to ETL data + readOnly: true + {{- end }} + {{- if eq (include "aggregator.deployMethod" .) "statefulset" }} + - name: aggregator-db-storage + mountPath: /var/configs/waterfowl/duckdb + - name: aggregator-staging + # Aggregator uses /var/configs/waterfowl as a "staging" directory for + # things like intermediate-state files pre-ingestion. In order to avoid a + # permission problem similar to + # https://github.com/kubernetes/kubernetes/issues/81676, we create an + # emptyDir at this path. + # + # This hasn't been observed as a problem in cost-analyzer, likely because + # of the init container that gives everything under /var/configs 777. + mountPath: /var/configs/waterfowl + {{- end }} + {{- if and ((.Values.kubecostProductConfigs).productKey).enabled ((.Values.kubecostProductConfigs).productKey).secretname (eq (include "aggregator.deployMethod" .) "statefulset") }} + - name: productkey-secret + mountPath: /var/configs/productkey + {{- end }} + {{- if and ((.Values.kubecostProductConfigs).smtp).secretname (eq (include "aggregator.deployMethod" .) "statefulset") }} + - name: smtp-secret + mountPath: /var/configs/smtp + {{- end }} + {{- if .Values.saml }} + {{- if .Values.saml.enabled }} + {{- if .Values.saml.secretName }} + - name: secret-volume + mountPath: /var/configs/secret-volume + {{- end }} + {{- if .Values.saml.encryptionCertSecret }} + - name: saml-encryption-cert + mountPath: /var/configs/saml-encryption-cert + {{- end }} + {{- if .Values.saml.decryptionKeySecret }} + - name: saml-decryption-key + mountPath: /var/configs/saml-decryption-key + {{- end }} + {{- if .Values.saml.metadataSecretName }} + - name: metadata-secret-volume + mountPath: /var/configs/metadata-secret-volume + {{- end }} + - name: saml-auth-secret + mountPath: /var/configs/saml-auth-secret + {{- if .Values.saml.rbac.enabled }} + - name: saml-roles + mountPath: /var/configs/saml + {{- end }} + {{- end }} + {{- end }} + {{- if .Values.oidc }} + {{- if .Values.oidc.enabled }} + - name: oidc-config + mountPath: /var/configs/oidc + {{- if or .Values.oidc.existingCustomSecret.name .Values.oidc.secretName }} + - name: oidc-client-secret + mountPath: /var/configs/oidc-client-secret + {{- end }} + {{- end }} + {{- end }} + {{- if .Values.global.integrations.postgres.enabled }} + - name: postgres-creds + mountPath: /var/configs/integrations/postgres-creds + - name: postgres-queries + mountPath: /var/configs/integrations/postgres-queries + {{- end }} + {{- /* Only adds extraVolumeMounts if aggregator is running as its own pod */}} + {{- if and .Values.kubecostAggregator.extraVolumeMounts (eq (include "aggregator.deployMethod" .) "statefulset") }} + {{- toYaml .Values.kubecostAggregator.extraVolumeMounts | nindent 4 }} + {{- end }} + {{- if .Values.global.integrations.turbonomic.enabled }} + - name: turbonomic-credentials + mountPath: /var/configs/turbonomic + {{- end }} + env: + {{- if and (.Values.prometheus.server.global.external_labels.cluster_id) (not .Values.prometheus.server.clusterIDConfigmap) }} + - name: CLUSTER_ID + value: {{ .Values.prometheus.server.global.external_labels.cluster_id }} + {{- end }} + {{- if .Values.prometheus.server.clusterIDConfigmap }} + - name: CLUSTER_ID + valueFrom: + configMapKeyRef: + name: {{ .Values.prometheus.server.clusterIDConfigmap }} + key: CLUSTER_ID + {{- end }} + {{- if and ((.Values.kubecostProductConfigs).productKey).mountPath (eq (include "aggregator.deployMethod" .) "statefulset") }} + - name: PRODUCT_KEY_MOUNT_PATH + value: {{ .Values.kubecostProductConfigs.productKey.mountPath }} + {{- end }} + {{- if and ((.Values.kubecostProductConfigs).smtp).mountPath (eq (include "aggregator.deployMethod" .) "statefulset") }} + - name: SMTP_CONFIG_MOUNT_PATH + value: {{ .Values.kubecostProductConfigs.smtp.mountPath }} + {{- end }} + {{- if .Values.smtpConfigmapName }} + - name: SMTP_CONFIGMAP_NAME + value: {{ .Values.smtpConfigmapName }} + {{- end }} + {{- if (gt (int .Values.kubecostAggregator.numDBCopyPartitions) 0) }} + - name: NUM_DB_COPY_CHUNKS + value: {{ .Values.kubecostAggregator.numDBCopyPartitions | quote }} + {{- end }} + {{- if .Values.kubecostAggregator.jaeger.enabled }} + - name: TRACING_URL + value: "http://localhost:14268/api/traces" + {{- end }} + - name: CONFIG_PATH + value: /var/configs/ + {{- if and .Values.persistentVolume.dbPVEnabled (eq (include "aggregator.deployMethod" .) "singlepod") }} + - name: ETL_PATH_PREFIX + value: "/var/db" + {{- end }} + - name: CLOUD_PROVIDER_API_KEY + value: "AIzaSyDXQPG_MHUEy9neR7stolq6l0ujXmjJlvk" # The GCP Pricing API key.This GCP api key is expected to be here and is limited to accessing google's billing API.' + {{- if .Values.global.integrations.postgres.enabled }} + - name: AGGREGATOR_ADDRESS + {{- if or .Values.saml.enabled .Values.oidc.enabled }} + value: localhost:9008 + {{- else }} + value: localhost:9004 + {{- end }} + - name: INT_PG_ENABLED + value: "true" + - name: INT_PG_RUN_INTERVAL + value: {{ quote .Values.global.integrations.postgres.runInterval }} + {{- end }} + - name: READ_ONLY + value: {{ (quote .Values.readonly) | default (quote false) }} + {{- if .Values.systemProxy.enabled }} + - name: HTTP_PROXY + value: {{ .Values.systemProxy.httpProxyUrl }} + - name: http_proxy + value: {{ .Values.systemProxy.httpProxyUrl }} + - name: HTTPS_PROXY + value: {{ .Values.systemProxy.httpsProxyUrl }} + - name: https_proxy + value: {{ .Values.systemProxy.httpsProxyUrl }} + - name: NO_PROXY + value: {{ .Values.systemProxy.noProxy }} + - name: no_proxy + value: {{ .Values.systemProxy.noProxy }} + {{- end }} + {{- if ((.Values.kubecostProductConfigs).carbonEstimates) }} + - name: CARBON_ESTIMATES_ENABLED + value: "true" + {{- end }} + - name: CUSTOM_COST_ENABLED + value: {{ .Values.kubecostModel.plugins.enabled | quote }} + {{- if .Values.kubecostAggregator.extraEnv -}} + {{- toYaml .Values.kubecostAggregator.extraEnv | nindent 4 }} + {{- end }} + {{- if eq (include "aggregator.deployMethod" .) "statefulset" }} + # If this isn't set, we pretty much have to be in a read only state, + # initialization will probably fail otherwise. + - name: ETL_BUCKET_CONFIG + {{- if and (not .Values.kubecostModel.federatedStorageConfigSecret) (not .Values.kubecostModel.federatedStorageConfig) }} + value: /var/configs/etl/object-store.yaml + {{- else }} + value: /var/configs/etl/federated-store.yaml + - name: FEDERATED_STORE_CONFIG + value: /var/configs/etl/federated-store.yaml + - name: FEDERATED_PRIMARY_CLUSTER # this ensures the ingester runs assuming federated primary paths in the bucket + value: "true" + - name: FEDERATED_CLUSTER # this ensures the ingester runs assuming federated primary paths in the bucket + value: "true" + {{- if (.Values.kubecostProductConfigs).standardDiscount }} + {{- if .Values.ingestionConfigmapName }} + - name: INGESTION_CONFIGMAP_NAME + value: {{ .Values.ingestionConfigmapName }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + - name: LOG_LEVEL + value: {{ .Values.kubecostAggregator.logLevel }} + - name: DB_READ_THREADS + value: {{ .Values.kubecostAggregator.dbReadThreads | quote }} + - name: DB_WRITE_THREADS + value: {{ .Values.kubecostAggregator.dbWriteThreads | quote }} + - name: DB_CONCURRENT_INGESTION_COUNT + value: {{ .Values.kubecostAggregator.dbConcurrentIngestionCount | quote }} + {{- if ne .Values.kubecostAggregator.dbMemoryLimit "0GB" }} + - name: DB_MEMORY_LIMIT + value: {{ .Values.kubecostAggregator.dbMemoryLimit | quote }} + {{- end }} + {{- if ne .Values.kubecostAggregator.dbWriteMemoryLimit "0GB" }} + - name: DB_WRITE_MEMORY_LIMIT + value: {{ .Values.kubecostAggregator.dbWriteMemoryLimit | quote }} + {{- end }} + - name: ETL_DAILY_STORE_DURATION_DAYS + value: {{ .Values.kubecostAggregator.etlDailyStoreDurationDays | quote }} + - name: ETL_HOURLY_STORE_DURATION_HOURS + value: {{ .Values.kubecostAggregator.etlHourlyStoreDurationHours | quote }} + - name: CONTAINER_RESOURCE_USAGE_RETENTION_DAYS + value: {{ .Values.kubecostAggregator.containerResourceUsageRetentionDays | quote }} + - name: DB_TRIM_MEMORY_ON_CLOSE + value: {{ .Values.kubecostAggregator.dbTrimMemoryOnClose | quote }} + - name: KUBECOST_NAMESPACE + value: {{ .Release.Namespace }} + {{- if .Values.global.grafana }} + - name: GRAFANA_ENABLED + value: "{{ template "cost-analyzer.grafanaEnabled" . }}" + {{- end}} + {{- if .Values.oidc.enabled }} + - name: OIDC_ENABLED + value: "true" + - name: OIDC_SKIP_ONLINE_VALIDATION + value: {{ (quote .Values.oidc.skipOnlineTokenValidation) | default (quote false) }} + {{- end}} + {{- if .Values.kubecostAggregator }} + {{- if .Values.kubecostAggregator.collections }} + {{- if (((.Values.kubecostAggregator).collections).cache) }} + - name: COLLECTIONS_MEMORY_CACHE_ENABLED + value: {{ (quote .Values.kubecostAggregator.collections.cache.enabled) | default (quote true) }} + {{- end }} + {{- end }} + {{- end }} + {{- if .Values.global.integrations.turbonomic.enabled }} + - name: TURBONOMIC_ENABLED + value: "true" + {{- end }} + {{- if .Values.saml }} + {{- if .Values.saml.enabled }} + - name: SAML_ENABLED + value: "true" + - name: IDP_URL + value: {{ .Values.saml.idpMetadataURL }} + - name: SP_HOST + value: {{ .Values.saml.appRootURL }} + {{- if .Values.saml.audienceURI }} + - name: AUDIENCE_URI + value: {{ .Values.saml.audienceURI }} + {{- end }} + {{- if .Values.saml.isGLUUProvider }} + - name: GLUU_SAML_PROVIDER + value: {{ (quote .Values.saml.isGLUUProvider) }} + {{- end }} + {{- if .Values.saml.nameIDFormat }} + - name: NAME_ID_FORMAT + value: {{ .Values.saml.nameIDFormat }} + {{- end}} + {{- if .Values.saml.authTimeout }} + - name: AUTH_TOKEN_TIMEOUT + value: {{ (quote .Values.saml.authTimeout) }} + {{- end}} + {{- if .Values.saml.redirectURL }} + - name: LOGOUT_REDIRECT_URL + value: {{ .Values.saml.redirectURL }} + {{- end}} + {{- if .Values.saml.rbac.enabled }} + - name: SAML_RBAC_ENABLED + value: "true" + {{- end }} + {{- if and .Values.saml.encryptionCertSecret .Values.saml.decryptionKeySecret }} + - name: SAML_RESPONSE_ENCRYPTED + value: "true" + {{- end}} + {{- end }} + {{- end }} +{{- end }} + + +{{- define "aggregator.jaeger.sidecarContainerTemplate" }} +- name: embedded-jaeger + env: + - name: SPAN_STORAGE_TYPE + value: badger + - name: BADGER_EPHEMERAL + value: "true" + - name: BADGER_DIRECTORY_VALUE + value: /tmp/badger/data + - name: BADGER_DIRECTORY_KEY + value: /tmp/badger/key + securityContext: + {{- toYaml .Values.kubecostAggregator.jaeger.containerSecurityContext | nindent 4 }} + image: {{ .Values.kubecostAggregator.jaeger.image }}:{{ .Values.kubecostAggregator.jaeger.imageVersion }} +{{- end }} + + +{{- define "aggregator.cloudCost.containerTemplate" }} +- name: cloud-cost + {{- if .Values.kubecostModel }} + {{- if .Values.kubecostAggregator.fullImageName }} + image: {{ .Values.kubecostAggregator.fullImageName }} + {{- else if .Values.kubecostModel.fullImageName }} + image: {{ .Values.kubecostModel.fullImageName }} + {{- else if .Values.imageVersion }} + image: {{ .Values.kubecostModel.image }}:{{ .Values.imageVersion }} + {{- else if eq "development" .Chart.AppVersion }} + image: gcr.io/kubecost1/cost-model-nightly:latest + {{- else }} + image: {{ .Values.kubecostModel.image }}:prod-{{ $.Chart.AppVersion }} + {{ end }} + {{- else }} + image: gcr.io/kubecost1/cost-model:prod-{{ $.Chart.AppVersion }} + {{ end }} + {{- if .Values.kubecostAggregator.cloudCost.readinessProbe.enabled }} + readinessProbe: + httpGet: + path: /healthz + port: 9005 + initialDelaySeconds: {{ .Values.kubecostAggregator.cloudCost.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.kubecostAggregator.cloudCost.readinessProbe.periodSeconds }} + failureThreshold: {{ .Values.kubecostAggregator.cloudCost.readinessProbe.failureThreshold }} + {{- end }} + {{- if .Values.kubecostAggregator.imagePullPolicy }} + imagePullPolicy: {{ .Values.kubecostAggregator.imagePullPolicy }} + {{- else }} + imagePullPolicy: Always + {{- end }} + args: ["cloud-cost"] + ports: + - name: tcp-api + containerPort: 9005 + protocol: TCP + resources: + {{- toYaml .Values.kubecostAggregator.cloudCost.resources | nindent 4 }} + securityContext: + {{- if .Values.global.containerSecurityContext }} + {{- toYaml .Values.global.containerSecurityContext | nindent 4 }} + {{- end }} + volumeMounts: + - name: persistent-configs + mountPath: /var/configs + {{- if or (.Values.kubecostModel).federatedStorageConfigSecret (.Values.kubecostModel).federatedStorageConfig }} + - name: federated-storage-config + mountPath: /var/configs/etl/federated + readOnly: true + {{- end }} + {{- if or (.Values.kubecostProductConfigs).cloudIntegrationSecret (.Values.kubecostProductConfigs).cloudIntegrationJSON ((.Values.kubecostProductConfigs).athenaBucketName) }} + - name: cloud-integration + mountPath: /var/configs/cloud-integration + {{- end }} + {{- if .Values.kubecostModel.plugins.enabled }} + - mountPath: {{ .Values.kubecostModel.plugins.folder }} + name: plugins-dir + readOnly: false + - name: tmp + mountPath: /tmp + - mountPath: {{ $.Values.kubecostModel.plugins.folder }}/config + name: plugins-config + readOnly: true + {{- end }} + {{- /* Only adds extraVolumeMounts when cloudcosts is running as its own pod */}} + {{- if and .Values.kubecostAggregator.cloudCost.extraVolumeMounts (eq (include "aggregator.deployMethod" .) "statefulset") }} + {{- toYaml .Values.kubecostAggregator.cloudCost.extraVolumeMounts | nindent 4 }} + {{- end }} + env: + - name: CONFIG_PATH + value: /var/configs/ + {{- if or .Values.kubecostModel.federatedStorageConfigSecret .Values.kubecostModel.federatedStorageConfig }} + - name: FEDERATED_STORE_CONFIG + value: /var/configs/etl/federated/federated-store.yaml + - name: FEDERATED_CLUSTER + value: "true" + {{- end}} + - name: ETL_DAILY_STORE_DURATION_DAYS + value: {{ (quote .Values.kubecostModel.etlDailyStoreDurationDays) }} + - name: CLOUD_COST_REFRESH_RATE_HOURS + value: {{ .Values.kubecostAggregator.cloudCost.refreshRateHours | default 6 | quote }} + - name: CLOUD_COST_QUERY_WINDOW_DAYS + value: {{ .Values.kubecostAggregator.cloudCost.queryWindowDays | default 7 | quote }} + - name: CLOUD_COST_RUN_WINDOW_DAYS + value: {{ .Values.kubecostAggregator.cloudCost.runWindowDays | default 3 | quote }} + - name: CUSTOM_COST_ENABLED + value: {{ .Values.kubecostModel.plugins.enabled | quote }} + {{- range $key, $value := .Values.kubecostAggregator.cloudCost.env }} + - name: {{ $key | quote }} + value: {{ $value | quote }} + {{- end }} + {{- if .Values.systemProxy.enabled }} + - name: HTTP_PROXY + value: {{ .Values.systemProxy.httpProxyUrl }} + - name: http_proxy + value: {{ .Values.systemProxy.httpProxyUrl }} + - name: HTTPS_PROXY + value: {{ .Values.systemProxy.httpsProxyUrl }} + - name: https_proxy + value: {{ .Values.systemProxy.httpsProxyUrl }} + - name: NO_PROXY + value: {{ .Values.systemProxy.noProxy }} + - name: no_proxy + value: {{ .Values.systemProxy.noProxy }} + {{- end }} +{{- end }} + +{{/* +SSO enabled flag for nginx configmap +*/}} +{{- define "ssoEnabled" -}} + {{- if or (.Values.saml).enabled (.Values.oidc).enabled -}} + {{- printf "true" -}} + {{- else -}} + {{- printf "false" -}} + {{- end -}} +{{- end -}} + +{{/* +To use the Kubecost built-in Teams UI RBAC< you must enable SSO and RBAC and not specify any groups. +Groups is only used when using external RBAC. +*/}} +{{- define "rbacTeamsEnabled" -}} + {{- if or (.Values.saml).enabled (.Values.oidc).enabled -}} + {{- if or ((.Values.saml).rbac).enabled ((.Values.oidc).rbac).enabled -}} + {{- if not (or (.Values.saml).groups (.Values.oidc).groups) -}} + {{- printf "true" -}} + {{- else -}} + {{- printf "false" -}} + {{- end -}} + {{- else -}} + {{- printf "false" -}} + {{- end -}} + {{- else -}} + {{- printf "false" -}} + {{- end -}} +{{- end -}} + +{{/* +Backups configured flag for nginx configmap +*/}} +{{- define "dataBackupConfigured" -}} + {{- if or (.Values.kubecostModel).federatedStorageConfigSecret (.Values.kubecostModel).federatedStorageConfig -}} + {{- printf "true" -}} + {{- else -}} + {{- printf "false" -}} + {{- end -}} +{{- end -}} + +{{/* +costEventsAuditEnabled flag for nginx configmap +*/}} +{{- define "costEventsAuditEnabled" -}} + {{- if or (.Values.costEventsAudit).enabled -}} + {{- printf "true" -}} + {{- else -}} + {{- printf "false" -}} + {{- end -}} +{{- end -}} + +{{- define "cost-analyzer.grafanaEnabled" -}} + {{- if and (.Values.global.grafana.enabled) (not .Values.federatedETL.agentOnly) -}} + {{- printf "true" -}} + {{- else -}} + {{- printf "false" -}} + {{- end -}} +{{- end -}} + +{{- define "gcpCloudIntegrationJSON" }} +Kubecost 2.x requires a change to the method that cloud-provider billing integrations are configured. +Please use this output to create a cloud-integration.json config. See: + +for more information + + { + "gcp": + { + [ + { + "bigQueryBillingDataDataset": "{{ .Values.kubecostProductConfigs.bigQueryBillingDataDataset }}", + "bigQueryBillingDataProject": "{{ .Values.kubecostProductConfigs.bigQueryBillingDataProject }}", + "bigQueryBillingDataTable": "{{ .Values.kubecostProductConfigs.bigQueryBillingDataTable }}", + "projectID": "{{ .Values.kubecostProductConfigs.projectID }}" + } + ] + } + } +{{- end }} + +{{- define "gcpCloudIntegrationCheck" }} +{{- if ((.Values.kubecostProductConfigs).bigQueryBillingDataDataset) }} +{{- fail (include "gcpCloudIntegrationJSON" .) }} +{{- end }} +{{- end }} + + +{{- define "azureCloudIntegrationJSON" }} + +Kubecost 2.x requires a change to the method that cloud-provider billing integrations are configured. +Please use this output to create a cloud-integration.json config. See: + +for more information + { + "azure": + [ + { + "azureStorageContainer": "{{ .Values.kubecostProductConfigs.azureStorageContainer }}", + "azureSubscriptionID": "{{ .Values.kubecostProductConfigs.azureSubscriptionID }}", + "azureStorageAccount": "{{ .Values.kubecostProductConfigs.azureStorageAccount }}", + "azureStorageAccessKey": "{{ .Values.kubecostProductConfigs.azureStorageKey }}", + "azureContainerPath": "{{ .Values.kubecostProductConfigs.azureContainerPath }}", + "azureCloud": "{{ .Values.kubecostProductConfigs.azureCloud }}" + } + ] + } +{{- end }} + +{{- define "azureCloudIntegrationCheck" }} +{{- if ((.Values.kubecostProductConfigs).azureStorageContainer) }} +{{- fail (include "azureCloudIntegrationJSON" .) }} +{{- end }} +{{- end }} + +{{- define "caCertsSecretConfigCheck" }} + {{- if .Values.global.updateCaTrust.enabled }} + {{- if and .Values.global.updateCaTrust.caCertsSecret .Values.global.updateCaTrust.caCertsConfig }} + {{- fail "Both caCertsSecret and caCertsConfig are defined. Please specify only one." }} + {{- else if and (not .Values.global.updateCaTrust.caCertsSecret) (not .Values.global.updateCaTrust.caCertsConfig) }} + {{- fail "Neither caCertsSecret nor caCertsConfig is defined, but updateCaTrust is enabled. Please specify one." }} + {{- end }} + {{- end }} +{{- end }} + +{{- define "clusterControllerEnabled" }} +{{- if (.Values.clusterController).enabled }} +{{- printf "true" -}} +{{- else -}} +{{- printf "false" -}} +{{- end -}} +{{- end -}} + +{{- define "forecastingEnabled" }} +{{- if (.Values.forecasting).enabled }} +{{- printf "true" -}} +{{- else -}} +{{- printf "false" -}} +{{- end -}} +{{- end -}} + +{{- define "pluginsEnabled" }} +{{- if (.Values.kubecostModel.plugins).enabled }} +{{- printf "true" -}} +{{- else -}} +{{- printf "false" -}} +{{- end -}} +{{- end -}} + +{{- define "carbonEstimatesEnabled" }} +{{- if ((.Values.kubecostProductConfigs).carbonEstimates) }} +{{- printf "true" -}} +{{- else -}} +{{- printf "false" -}} +{{- end -}} +{{- end -}} + +{{- /* + Compute a checksum based on the rendered content of specific ConfigMaps and Secrets. +*/ -}} +{{- define "configsChecksum" -}} +{{- $files := list + "alibaba-service-key-secret.yaml" + "aws-service-key-secret.yaml" + "azure-service-key-secret.yaml" + "cloud-integration-secret.yaml" + "cost-analyzer-account-mapping-configmap.yaml" + "cost-analyzer-alerts-configmap.yaml" + "cost-analyzer-asset-reports-configmap.yaml" + "cost-analyzer-cloud-cost-reports-configmap.yaml" + "cost-analyzer-config-map-template.yaml" + "cost-analyzer-frontend-config-map-template.yaml" + "cost-analyzer-metrics-config-map-template.yaml" + "cost-analyzer-network-costs-config-map-template.yaml" + "cost-analyzer-oidc-config-map-template.yaml" + "cost-analyzer-pkey-configmap.yaml" + "cost-analyzer-pricing-configmap.yaml" + "cost-analyzer-saml-config-map-template.yaml" + "cost-analyzer-saved-reports-configmap.yaml" + "cost-analyzer-server-configmap.yaml" + "cost-analyzer-smtp-configmap.yaml" + "external-grafana-config-map-template.yaml" + "gcpstore-config-map-template.yaml" + "grafana/grafana-secret.yaml" + "install-plugins.yaml" + "integrations-postgres-queries-configmap.yaml" + "integrations-postgres-secret.yaml" + "kubecost-cluster-context-switcher.yaml" + "kubecost-cluster-controller-actions-config.yaml" + "kubecost-oidc-secret-template.yaml" + "kubecost-saml-secret-template.yaml" + "mimir-proxy-configmap-template.yaml" + "savings-recommendations-allowlists-config-map-template.yaml" +-}} +{{- $checksum := "" -}} +{{- range $files -}} + {{- $content := include (print $.Template.BasePath (printf "/%s" .)) $ -}} + {{- $checksum = printf "%s%s" $checksum $content | sha256sum -}} +{{- end -}} +{{- $checksum | sha256sum -}} +{{- end -}} + +{{- define "cost-model.image" }} +{{- if .Values.kubecostModel }} + {{- if .Values.kubecostModel.fullImageName }} + {{ .Values.kubecostModel.fullImageName }} + {{- else if .Values.imageVersion }} + {{ .Values.kubecostModel.image }}:{{ .Values.imageVersion }} + {{- else if eq "development" .Chart.AppVersion }} + gcr.io/kubecost1/cost-model-nightly:latest + {{- else }} + {{ .Values.kubecostModel.image }}:prod-{{ $.Chart.AppVersion }} + {{- end }} +{{- else }} + gcr.io/kubecost1/cost-model:prod-{{ $.Chart.AppVersion }} +{{- end }} +{{- end }} + +{{- define "cost-model.imagetag" }} +{{- $image := include "cost-model.image" . }} +{{- $parts := splitList ":" $image }} +{{- $tag := last $parts }} +{{- $tag }} +{{- end }} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/aggregator-cloud-cost-deployment.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/aggregator-cloud-cost-deployment.yaml new file mode 100644 index 0000000000..c862f9c34a --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/aggregator-cloud-cost-deployment.yaml @@ -0,0 +1,173 @@ +{{- if eq (include "aggregator.deployMethod" .) "statefulset" }} + +{{/* + A cloud integration secret is required for cloud cost to function as a dedicated pod. + UI based configuration is not supported for cloud cost with aggregator. +*/}} +{{- if ((.Values.kubecostAggregator).cloudCost).enabled }} +{{- if not ( or (.Values.kubecostProductConfigs).cloudIntegrationSecret (.Values.kubecostProductConfigs).cloudIntegrationJSON ((.Values.kubecostProductConfigs).athenaBucketName)) }} +{{- fail "\n\nA cloud-integration secret is required when using the aggregator statefulset and cloudCost is enabled." }} +{{- end }} +{{- end }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "cloudCost.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "cloudCost.commonLabels" . | nindent 4 }} + {{- with .Values.global.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- with .Values.global.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.kubecostAggregator.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: 1 + selector: + matchLabels: + {{- include "cloudCost.selectorLabels" . | nindent 6 }} + strategy: + type: Recreate + template: + metadata: + labels: + app.kubernetes.io/name: cloud-cost + app.kubernetes.io/instance: {{ .Release.Name }} + app: cloud-cost + {{- with .Values.global.additionalLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + {{- with .Values.global.podAnnotations}} + {{- toYaml . | nindent 8 }} + {{- end }} + checksum/configs: {{ include "configsChecksum" . }} + spec: + {{- if .Values.global.platforms.openshift.enabled }} + securityContext: + {{- toYaml .Values.global.platforms.openshift.securityContext | nindent 8 }} + {{- else if .Values.global.securityContext }} + securityContext: + {{- toYaml .Values.global.securityContext | nindent 8 }} + {{- else }} + securityContext: + runAsUser: 1001 + runAsGroup: 1001 + fsGroup: 1001 + {{- end }} + restartPolicy: Always + serviceAccountName: {{ template "cloudCost.serviceAccountName" . }} + volumes: + {{- if or .Values.kubecostModel.federatedStorageConfigSecret .Values.kubecostModel.federatedStorageConfig }} + - name: federated-storage-config + secret: + defaultMode: 420 + {{- if .Values.kubecostModel.federatedStorageConfigSecret }} + secretName: {{ .Values.kubecostModel.federatedStorageConfigSecret }} + {{- else }} + secretName: federated-store + {{- end }} + {{- end }} + {{- if (.Values.kubecostProductConfigs).cloudIntegrationSecret }} + - name: cloud-integration + secret: + secretName: {{ .Values.kubecostProductConfigs.cloudIntegrationSecret }} + items: + - key: cloud-integration.json + path: cloud-integration.json + {{- else if or (.Values.kubecostProductConfigs).cloudIntegrationJSON ((.Values.kubecostProductConfigs).athenaProjectID) }} + - name: cloud-integration + secret: + secretName: cloud-integration + items: + - key: cloud-integration.json + path: cloud-integration.json + {{- end }} + {{/* Despite the name, this is not persistent-configs. + The name is for compatibility with single-pod install. + All data stored here is ephemeral, and does not require persistence. */}} + - name: persistent-configs + emptyDir: {} + {{- if .Values.kubecostModel.plugins.enabled }} + {{- if .Values.kubecostModel.plugins.install.enabled}} + - name: install-script + configMap: + name: {{ template "cost-analyzer.fullname" . }}-install-plugins + {{- end }} + - name: plugins-dir + emptyDir: {} + {{- if and (not .Values.kubecostModel.plugins.existingCustomSecret.enabled) .Values.kubecostModel.plugins.secretName }} + - name: plugins-config + secret: + secretName: {{ .Values.kubecostModel.plugins.secretName }} + items: + {{- range $key, $config := .Values.kubecostModel.plugins.enabledPlugins }} + - key: {{ $config }}_config.json + path: {{ $config }}_config.json + {{- end }} + {{- end }} + {{- if .Values.kubecostModel.plugins.existingCustomSecret.enabled }} + - name: plugins-config + secret: + secretName: {{ .Values.kubecostModel.plugins.existingCustomSecret.name }} + items: + {{- range $key, $config := .Values.kubecostModel.plugins.enabledPlugins }} + - key: {{ $config }}_config.json + path: {{ $config }}_config.json + {{- end }} + {{- end }} + - name: tmp + emptyDir: {} + {{- end }} + {{- if .Values.kubecostAggregator.cloudCost.extraVolumes }} + {{- toYaml .Values.kubecostAggregator.cloudCost.extraVolumes | nindent 8 }} + {{- end }} + initContainers: + {{- if (and .Values.kubecostModel.plugins.enabled .Values.kubecostModel.plugins.install.enabled )}} + - name: plugin-installer + image: {{ .Values.kubecostModel.plugins.install.fullImageName }} + command: ["sh", "/install/install_plugins.sh"] + {{- with .Values.kubecostModel.plugins.install.securityContext }} + securityContext: {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: install-script + mountPath: /install + - name: plugins-dir + mountPath: {{ .Values.kubecostModel.plugins.folder }} + {{- end }} + containers: + {{- include "aggregator.cloudCost.containerTemplate" . | nindent 8 }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{- range $.Values.imagePullSecrets }} + - name: {{ .name }} + {{- end }} + {{- end }} + {{- if .Values.kubecostAggregator.priority }} + {{- if .Values.kubecostAggregator.priority.enabled }} + {{- if .Values.kubecostAggregator.priority.name }} + priorityClassName: {{ .Values.kubecostAggregator.priority.name }} + {{- else }} + priorityClassName: {{ template "cost-analyzer.fullname" . }}-aggregator-priority + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.kubecostAggregator.cloudCost.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.kubecostAggregator.cloudCost.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.kubecostAggregator.cloudCost.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/aggregator-cloud-cost-service-account.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/aggregator-cloud-cost-service-account.yaml new file mode 100644 index 0000000000..0f72059f54 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/aggregator-cloud-cost-service-account.yaml @@ -0,0 +1,23 @@ +{{- if eq (include "aggregator.deployMethod" .) "statefulset" }} + +{{/* + A cloud integration secret is required for cloud cost to function as a dedicated pod. + UI based configuration is not supported for cloud cost with aggregator. +*/}} + +{{- if or (.Values.kubecostProductConfigs).cloudIntegrationSecret (.Values.kubecostProductConfigs).cloudIntegrationJSON ((.Values.kubecostProductConfigs).athenaBucketName) }} +{{- if and .Values.serviceAccount.create .Values.kubecostAggregator.cloudCost.serviceAccountName }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "cloudCost.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "cloudCost.commonLabels" . | nindent 4 }} +{{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/aggregator-cloud-cost-service.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/aggregator-cloud-cost-service.yaml new file mode 100644 index 0000000000..bb38a2164a --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/aggregator-cloud-cost-service.yaml @@ -0,0 +1,17 @@ +{{- if not (eq (include "aggregator.deployMethod" .) "disabled") -}} +kind: Service +apiVersion: v1 +metadata: + name: {{ template "cloudCost.serviceName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "cloudCost.commonLabels" . | nindent 4 }} +spec: + selector: + {{- include "cloudCost.selectorLabels" . | nindent 4 }} + type: "ClusterIP" + ports: + - name: tcp-api + port: 9005 + targetPort: 9005 +{{- end }} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/aggregator-service.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/aggregator-service.yaml new file mode 100644 index 0000000000..956ebef06a --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/aggregator-service.yaml @@ -0,0 +1,28 @@ +{{- if not (eq (include "aggregator.deployMethod" .) "disabled") -}} +kind: Service +apiVersion: v1 +metadata: + name: {{ template "aggregator.serviceName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "aggregator.commonLabels" . | nindent 4 }} + {{- if .Values.kubecostAggregator.service.labels }} + {{- toYaml .Values.kubecostAggregator.service.labels | nindent 4 }} + {{- end }} +spec: + selector: + {{- include "aggregator.selectorLabels" . | nindent 4 }} + type: "ClusterIP" + ports: + - name: tcp-api + port: 9004 + targetPort: 9004 + {{- if or .Values.saml.enabled .Values.oidc.enabled}} + - name: apiserver + port: 9008 + targetPort: 9008 + {{- end }} + {{- with .Values.kubecostAggregator.extraPorts }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/aggregator-servicemonitor.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/aggregator-servicemonitor.yaml new file mode 100644 index 0000000000..7f9c939375 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/aggregator-servicemonitor.yaml @@ -0,0 +1,31 @@ +{{- if .Values.serviceMonitor.aggregatorMetrics.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "aggregator.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "aggregator.commonLabels" . | nindent 4 }} + {{- if .Values.serviceMonitor.aggregatorMetrics.additionalLabels }} + {{ toYaml .Values.serviceMonitor.aggregatorMetrics.additionalLabels | nindent 4 }} + {{- end }} +spec: + endpoints: + - port: tcp-api + interval: {{ .Values.serviceMonitor.aggregatorMetrics.interval }} + scrapeTimeout: {{ .Values.serviceMonitor.aggregatorMetrics.scrapeTimeout }} + path: /metrics + scheme: http + {{- with .Values.serviceMonitor.aggregatorMetrics.metricRelabelings }} + metricRelabelings: {{ toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.serviceMonitor.aggregatorMetrics.relabelings }} + relabelings: {{ toYaml . | nindent 8 }} + {{- end }} + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} + selector: + matchLabels: + {{- include "aggregator.commonLabels" . | nindent 6 }} +{{- end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/aggregator-statefulset.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/aggregator-statefulset.yaml new file mode 100644 index 0000000000..3432d13e61 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/aggregator-statefulset.yaml @@ -0,0 +1,222 @@ +{{- if and (not .Values.agent) (not .Values.cloudAgent) }} +{{- if eq (include "aggregator.deployMethod" .) "statefulset" }} + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ template "aggregator.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "aggregator.commonLabels" . | nindent 4 }} + {{- with .Values.global.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- with .Values.global.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.kubecostAggregator.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.kubecostAggregator.replicas }} + serviceName: {{ template "aggregator.serviceName" . }} + selector: + matchLabels: + {{- include "aggregator.selectorLabels" . | nindent 6 }} + volumeClaimTemplates: + - metadata: + name: aggregator-db-storage + spec: + accessModes: [ "ReadWriteOnce" ] + storageClassName: {{ .Values.kubecostAggregator.aggregatorDbStorage.storageClass }} + resources: + requests: + storage: {{ .Values.kubecostAggregator.aggregatorDbStorage.storageRequest }} + - metadata: + # In the StatefulSet config, Aggregator should not share any filesystem + # state with the cost-model to maintain independence and improve + # stability (in the event of bad file-locking state). Still, there is + # a need to "mount" ConfigMap files (using the watcher) to a file system; + # that's what this per-replica Volume is used for. + name: persistent-configs + spec: + accessModes: [ "ReadWriteOnce" ] + storageClassName: {{ .Values.kubecostAggregator.persistentConfigsStorage.storageClass }} + resources: + requests: + storage: {{ .Values.kubecostAggregator.persistentConfigsStorage.storageRequest }} + template: + metadata: + labels: + app.kubernetes.io/name: aggregator + app.kubernetes.io/instance: {{ .Release.Name }} + app: aggregator + {{- with .Values.global.additionalLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + {{- with .Values.global.podAnnotations}} + {{- toYaml . | nindent 8 }} + {{- end }} + checksum/configs: {{ include "configsChecksum" . }} + spec: + restartPolicy: Always + {{- if .Values.kubecostAggregator.securityContext }} + securityContext: + {{- toYaml .Values.kubecostAggregator.securityContext | nindent 8 }} + {{- else if and (.Values.global.platforms.openshift.enabled) (.Values.global.platforms.openshift.securityContext) }} + securityContext: + {{- toYaml .Values.global.platforms.openshift.securityContext | nindent 8 }} + {{- else if .Values.global.securityContext }} + securityContext: + {{- toYaml .Values.global.securityContext | nindent 8 }} + {{- end }} + serviceAccountName: {{ template "aggregator.serviceAccountName" . }} + volumes: + - name: aggregator-staging + emptyDir: + sizeLimit: {{ .Values.kubecostAggregator.stagingEmptyDirSizeLimit }} + {{- $etlBackupBucketSecret := "" }} + {{- if .Values.kubecostModel.federatedStorageConfigSecret }} + {{- $etlBackupBucketSecret = .Values.kubecostModel.federatedStorageConfigSecret }} + {{- end }} + {{- if or $etlBackupBucketSecret .Values.kubecostModel.federatedStorageConfig }} + {{- if or .Values.kubecostModel.federatedStorageConfigSecret .Values.kubecostModel.federatedStorageConfig }} + - name: federated-storage-config + secret: + defaultMode: 420 + {{- if .Values.kubecostModel.federatedStorageConfigSecret }} + secretName: {{ .Values.kubecostModel.federatedStorageConfigSecret }} + {{- else }} + secretName: federated-store + {{- end }} + {{- end }} + - name: etl-bucket-config + secret: + defaultMode: 420 + secretName: {{ $etlBackupBucketSecret | default "federated-store" }} + {{- else }} + {{- fail "\n\nKubecost Aggregator Enterprise Config requires either .Values.kubecostModel.federatedStorageConfigSecret or .Values.kubecostModel.federatedStorageConfig" }} + {{- end }} + {{- if and ((.Values.kubecostProductConfigs).productKey).enabled ((.Values.kubecostProductConfigs).productKey).secretname }} + - name: productkey-secret + secret: + secretName: {{ .Values.kubecostProductConfigs.productKey.secretname }} + items: + - key: productkey.json + path: productkey.json + {{- end }} + {{- if ((.Values.kubecostProductConfigs).smtp).secretname }} + - name: smtp-secret + secret: + secretName: {{ .Values.kubecostProductConfigs.smtp.secretname }} + items: + - key: smtp.json + path: smtp.json + {{- end }} + {{- if .Values.saml }} + {{- if .Values.saml.enabled }} + {{- if .Values.saml.secretName }} + - name: secret-volume + secret: + secretName: {{ .Values.saml.secretName }} + {{- end }} + {{- if .Values.saml.encryptionCertSecret }} + - name: saml-encryption-cert + secret: + secretName: {{ .Values.saml.encryptionCertSecret }} + {{- end }} + {{- if .Values.saml.decryptionKeySecret }} + - name: saml-decryption-key + secret: + secretName: {{ .Values.saml.decryptionKeySecret }} + {{- end }} + {{- if .Values.saml.metadataSecretName }} + - name: metadata-secret-volume + secret: + secretName: {{ .Values.saml.metadataSecretName }} + {{- end }} + - name: saml-auth-secret + secret: + secretName: {{ .Values.saml.authSecretName | default "kubecost-saml-secret" }} + {{- if .Values.saml.rbac.enabled }} + - name: saml-roles + configMap: + name: {{ template "cost-analyzer.fullname" . }}-saml + {{- end }} + {{- end }} + {{- end }} + {{- if .Values.oidc }} + {{- if .Values.oidc.enabled }} + - name: oidc-config + configMap: + name: {{ template "cost-analyzer.fullname" . }}-oidc + {{- if and (not .Values.oidc.existingCustomSecret.enabled) .Values.oidc.secretName }} + - name: oidc-client-secret + secret: + secretName: {{ .Values.oidc.secretName }} + {{- end }} + {{- if .Values.oidc.existingCustomSecret.enabled }} + - name: oidc-client-secret + secret: + secretName: {{ .Values.oidc.existingCustomSecret.name }} + {{- end }} + {{- end }} + {{- end }} + {{- if .Values.global.integrations.postgres.enabled }} + - name: postgres-creds + secret: + {{- if not (eq .Values.global.integrations.postgres.databaseSecretName "") }} + secretName: {{ .Values.global.integrations.postgres.databaseSecretName }} + {{- else }} + secretName: kubecost-integrations-postgres + {{- end }} + - name: postgres-queries + configMap: + name: kubecost-integrations-postgres-queries + {{- end }} + {{- if .Values.global.integrations.turbonomic.enabled }} + - name: turbonomic-credentials + secret: + secretName: kubecost-integrations-turbonomic + {{- end }} + {{- if .Values.kubecostAggregator.extraVolumes }} + {{- toYaml .Values.kubecostAggregator.extraVolumes | nindent 8 }} + {{- end }} + containers: + {{- include "aggregator.containerTemplate" . | nindent 8 }} + + {{- if .Values.kubecostAggregator.jaeger.enabled }} + {{ include "aggregator.jaeger.sidecarContainerTemplate" . | nindent 8 }} + {{- end }} + + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{- range $.Values.imagePullSecrets }} + - name: {{ .name }} + {{- end }} + {{- end }} + {{- if .Values.kubecostAggregator.priority }} + {{- if .Values.kubecostAggregator.priority.enabled }} + {{- if .Values.kubecostAggregator.priority.name }} + priorityClassName: {{ .Values.kubecostAggregator.priority.name }} + {{- else }} + priorityClassName: {{ template "cost-analyzer.fullname" . }}-aggregator-priority + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.kubecostAggregator.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.kubecostAggregator.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.kubecostAggregator.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/alibaba-service-key-secret.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/alibaba-service-key-secret.yaml new file mode 100644 index 0000000000..4bdded0b68 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/alibaba-service-key-secret.yaml @@ -0,0 +1,20 @@ +{{- if .Values.kubecostProductConfigs }} +{{- if .Values.kubecostProductConfigs.createServiceKeySecret }} +{{- if .Values.kubecostProductConfigs.alibabaServiceKeyName }} +apiVersion: v1 +kind: Secret +metadata: + name: cloud-service-key + namespace: {{ .Release.Namespace }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} +type: Opaque +stringData: + service-key.json: |- + { + "alibaba_access_key_id": "{{ .Values.kubecostProductConfigs.alibabaServiceKeyName }}", + "alibaba_secret_access_key": "{{ .Values.kubecostProductConfigs.alibabaServiceKeyPassword }}" + } +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/aws-service-key-secret.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/aws-service-key-secret.yaml new file mode 100644 index 0000000000..51966f27e4 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/aws-service-key-secret.yaml @@ -0,0 +1,20 @@ +{{- if .Values.kubecostProductConfigs }} +{{- if .Values.kubecostProductConfigs.createServiceKeySecret }} +{{- if .Values.kubecostProductConfigs.awsServiceKeyName }} +apiVersion: v1 +kind: Secret +metadata: + name: cloud-service-key + namespace: {{ .Release.Namespace }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} +type: Opaque +stringData: + service-key.json: |- + { + "aws_access_key_id": "{{ .Values.kubecostProductConfigs.awsServiceKeyName }}", + "aws_secret_access_key": "{{ .Values.kubecostProductConfigs.awsServiceKeyPassword }}" + } +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/awsstore-deployment-template.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/awsstore-deployment-template.yaml new file mode 100644 index 0000000000..6f51ffa200 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/awsstore-deployment-template.yaml @@ -0,0 +1,56 @@ +{{- if .Values.awsstore }} +{{- if .Values.awsstore.useAwsStore }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "cost-analyzer.fullname" . }}-awsstore + namespace: {{ .Release.Namespace }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} + {{- with .Values.global.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- with .Values.global.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.awsstore.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + app: awsstore + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + labels: + app: awsstore + {{- with .Values.global.additionalLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.global.podAnnotations}} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: awsstore-serviceaccount + {{- if .Values.awsstore.priorityClassName }} + priorityClassName: "{{ .Values.awsstore.priorityClassName }}" + {{- end }} + {{- with .Values.awsstore.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - image: {{ .Values.awsstore.imageNameAndVersion }} + name: awsstore + # Just sleep forever + command: [ "sleep" ] + args: [ "infinity" ] +{{- end }} +{{- end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/awsstore-service-account-template.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/awsstore-service-account-template.yaml new file mode 100644 index 0000000000..086a8dc1f2 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/awsstore-service-account-template.yaml @@ -0,0 +1,15 @@ +{{- if .Values.awsstore }} +{{- if .Values.awsstore.createServiceAccount }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: awsstore-serviceaccount + namespace: {{ .Release.Namespace }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} +{{- with .Values.awsstore.annotations }} + annotations: + {{- toYaml . | nindent 4 }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/azure-service-key-secret.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/azure-service-key-secret.yaml new file mode 100644 index 0000000000..64b9231a6e --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/azure-service-key-secret.yaml @@ -0,0 +1,24 @@ +{{- if .Values.kubecostProductConfigs }} +{{- if .Values.kubecostProductConfigs.createServiceKeySecret }} +{{- if .Values.kubecostProductConfigs.azureSubscriptionID }} +apiVersion: v1 +kind: Secret +metadata: + name: cloud-service-key + namespace: {{ .Release.Namespace }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} +type: Opaque +stringData: + service-key.json: |- + { + "subscriptionId": "{{ .Values.kubecostProductConfigs.azureSubscriptionID }}", + "serviceKey": { + "appId": "{{ .Values.kubecostProductConfigs.azureClientID }}", + "password": "{{ .Values.kubecostProductConfigs.azureClientPassword }}", + "tenant": "{{ .Values.kubecostProductConfigs.azureTenantID }}" + } + } +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/cloud-integration-secret.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/cloud-integration-secret.yaml new file mode 100644 index 0000000000..e6023e59b3 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/cloud-integration-secret.yaml @@ -0,0 +1,16 @@ +{{- if or ((.Values.kubecostProductConfigs).cloudIntegrationJSON) ((.Values.kubecostProductConfigs).athenaBucketName) }} +apiVersion: v1 +kind: Secret +type: Opaque +metadata: + name: cloud-integration + namespace: {{ .Release.Namespace }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} +data: + {{- if (.Values.kubecostProductConfigs).cloudIntegrationJSON }} + cloud-integration.json: {{ .Values.kubecostProductConfigs.cloudIntegrationJSON | replace "\n" "" | b64enc }} + {{- else }} + cloud-integration.json: {{ include "cloudIntegrationFromProductConfigs" . |nindent 4| replace "\n" "" | b64enc }} + {{- end }} +{{- end -}} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-account-mapping-configmap.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-account-mapping-configmap.yaml new file mode 100644 index 0000000000..1668526b32 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-account-mapping-configmap.yaml @@ -0,0 +1,13 @@ +{{- if .Values.kubecostProductConfigs }} +{{- if .Values.kubecostProductConfigs.cloudAccountMapping }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: "account-mapping" + namespace: {{ .Release.Namespace }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} +data: + account-map.json: '{{ toJson .Values.kubecostProductConfigs.cloudAccountMapping }}' +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-alerts-configmap.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-alerts-configmap.yaml new file mode 100644 index 0000000000..ba30ca541a --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-alerts-configmap.yaml @@ -0,0 +1,11 @@ +{{- if .Values.global.notifications.alertConfigs }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ default "alert-configs" .Values.alertConfigmapName }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} +data: + alerts.json: '{{ toJson .Values.global.notifications.alertConfigs }}' +{{- end -}} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-asset-reports-configmap.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-asset-reports-configmap.yaml new file mode 100644 index 0000000000..076e3adbd7 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-asset-reports-configmap.yaml @@ -0,0 +1,14 @@ +{{- if .Values.global.assetReports }} +{{- if .Values.global.assetReports.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ default "asset-report-configs" .Values.assetReportConfigmapName }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} +data: + asset-reports.json: '{{ toJson .Values.global.assetReports.reports }}' +{{- end -}} +{{- end -}} + diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-cloud-cost-reports-configmap.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-cloud-cost-reports-configmap.yaml new file mode 100644 index 0000000000..783f47dab4 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-cloud-cost-reports-configmap.yaml @@ -0,0 +1,13 @@ +{{- if .Values.global.cloudCostReports }} +{{- if .Values.global.cloudCostReports.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{default "cloud-cost-report-configs" .Values.cloudCostReportConfigmapName }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} +data: + cloud-cost-reports.json: '{{ toJson .Values.global.cloudCostReports.reports }}' +{{- end -}} +{{- end -}} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-cluster-role-binding-template.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-cluster-role-binding-template.yaml new file mode 100644 index 0000000000..c5e3ec0ad0 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-cluster-role-binding-template.yaml @@ -0,0 +1,56 @@ +{{- if .Values.reporting }} +{{- if .Values.reporting.logCollection }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "cost-analyzer.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "cost-analyzer.serviceAccountName" . }} +subjects: + - kind: ServiceAccount + name: {{ template "cost-analyzer.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +--- +{{- end }} +{{- end }} +{{- if (not .Values.kubecostModel.etlReadOnlyMode) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "cost-analyzer.serviceAccountName" . }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "cost-analyzer.serviceAccountName" . }} +subjects: + - kind: ServiceAccount + name: {{ template "cost-analyzer.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +--- +{{- end }} +{{- if and .Values.global.platforms.openshift.enabled .Values.global.platforms.openshift.createMonitoringClusterRoleBinding }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "cost-analyzer.serviceAccountName" . }}-openshift-monitoring + labels: + {{ include "cost-analyzer.commonLabels" . | nindent 4 }} +roleRef: + # Grant the kubecost service account the cluster-monitoring-view role to enable it to query OpenShift Prometheus. + # This is necessary for Kubecost to get access and query the in-cluster Prometheus instance using its service account token. + # https://docs.redhat.com/en/documentation/openshift_container_platform/4.2/html/monitoring/cluster-monitoring#monitoring-accessing-prometheus-alerting-ui-grafana-using-the-web-console_accessing-prometheus + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-monitoring-view +subjects: + - kind: ServiceAccount + name: {{ template "cost-analyzer.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-cluster-role-template-readonly.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-cluster-role-template-readonly.yaml new file mode 100644 index 0000000000..c84f105e9c --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-cluster-role-template-readonly.yaml @@ -0,0 +1,26 @@ +{{- if (.Values.kubecostModel.etlReadOnlyMode) }} +{{- if and .Values.reporting .Values.reporting.logCollection -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + namespace: {{ .Release.Namespace }} + name: {{ template "cost-analyzer.serviceAccountName" . }} +rules: + - apiGroups: + - '' + resources: + - "pods/log" + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - configmaps + verbs: + - get + - list + - watch +{{- end -}} +{{- end -}} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-cluster-role-template.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-cluster-role-template.yaml new file mode 100644 index 0000000000..ff64e820cc --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-cluster-role-template.yaml @@ -0,0 +1,108 @@ +{{- if not .Values.kubecostModel.etlReadOnlyMode -}} +{{- if and .Values.reporting .Values.reporting.logCollection -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + namespace: {{ .Release.Namespace }} + name: {{ template "cost-analyzer.serviceAccountName" . }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} +rules: +- apiGroups: + - '' + resources: + - "pods/log" + verbs: + - get + - list + - watch +- apiGroups: + - '' + resources: + - configmaps + verbs: + - get + - list + - watch + - update +--- +{{- end }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "cost-analyzer.serviceAccountName" . }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} +rules: + - apiGroups: + - '' + resources: + - configmaps + - nodes + - pods + - events + - services + - resourcequotas + - replicationcontrollers + - limitranges + - persistentvolumeclaims + - persistentvolumes + - namespaces + - endpoints + verbs: + - get + - list + - watch + - apiGroups: + - apps + resources: + - statefulsets + - deployments + - daemonsets + - replicasets + verbs: + - get + - list + - watch + - apiGroups: + - batch + resources: + - cronjobs + - jobs + verbs: + - get + - list + - watch + - apiGroups: + - autoscaling + resources: + - horizontalpodautoscalers + verbs: + - get + - list + - watch + - apiGroups: + - policy + resources: + - poddisruptionbudgets + verbs: + - get + - list + - watch + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - events.k8s.io + resources: + - events + verbs: + - get + - list + - watch +{{- end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-config-map-template.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-config-map-template.yaml new file mode 100644 index 0000000000..7bd8892e4b --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-config-map-template.yaml @@ -0,0 +1,37 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "cost-analyzer.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} +data: + {{- if .Values.global.prometheus.enabled }} + {{- if .Values.global.zone }} + prometheus-alertmanager-endpoint: http://{{ template "cost-analyzer.prometheus.alertmanager.name" . }}.{{ .Release.Namespace }}.svc.{{ .Values.global.zone }} + {{ else }} + prometheus-alertmanager-endpoint: http://{{ template "cost-analyzer.prometheus.alertmanager.name" . }}.{{ .Release.Namespace }} + {{- end -}} + {{ else }} + prometheus-alertmanager-endpoint: {{ .Values.global.notifications.alertmanager.fqdn }} + {{- end -}} + {{ if .Values.global.gmp.enabled }} + prometheus-server-endpoint: {{ .Values.global.gmp.prometheusServerEndpoint }} + {{- else if .Values.global.amp.enabled }} + prometheus-server-endpoint: {{ .Values.global.amp.prometheusServerEndpoint }} + {{- else if .Values.global.ammsp.enabled }} + prometheus-server-endpoint: {{ .Values.global.ammsp.prometheusServerEndpoint }} + {{- else if .Values.global.prometheus.enabled }} + {{- if .Values.global.zone }} + prometheus-server-endpoint: http://{{ template "cost-analyzer.prometheus.server.name" . }}.{{ .Release.Namespace }}.svc.{{ .Values.global.zone }} + {{ else }} + prometheus-server-endpoint: http://{{ template "cost-analyzer.prometheus.server.name" . }}.{{ .Release.Namespace }} + {{- end -}} + {{ else }} + prometheus-server-endpoint: {{ .Values.global.prometheus.fqdn }} + {{- end -}} + {{- if .Values.kubecostToken }} + kubecost-token: {{ .Values.kubecostToken }} + {{ else }} + kubecost-token: not-applied + {{- end -}} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-deployment-template.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-deployment-template.yaml new file mode 100644 index 0000000000..ae4abf4eb9 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-deployment-template.yaml @@ -0,0 +1,1241 @@ +{{- if and (not .Values.agent) (not .Values.cloudAgent) }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "cost-analyzer.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} + {{- with .Values.global.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if and .Values.kubecostDeployment .Values.kubecostDeployment.labels }} + {{- toYaml .Values.kubecostDeployment.labels | nindent 4 }} + {{- end }} + annotations: + {{- with .Values.global.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if and .Values.kubecostDeployment .Values.kubecostDeployment.annotations }} + {{- toYaml .Values.kubecostDeployment.annotations | nindent 4 }} + {{- end }} +spec: +{{- if .Values.kubecostDeployment }} + replicas: {{ .Values.kubecostDeployment.replicas | default 1 }} +{{- end }} + selector: + matchLabels: + {{- include "cost-analyzer.selectorLabels" . | nindent 8}} +{{- if .Values.kubecostDeployment }} +{{- if .Values.kubecostDeployment.deploymentStrategy }} +{{- with .Values.kubecostDeployment.deploymentStrategy }} + strategy: {{ toYaml . | nindent 4 }} +{{- end }} +{{- else }} + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: RollingUpdate +{{- end }} +{{- end }} + template: + metadata: + labels: + {{- include "cost-analyzer.selectorLabels" . | nindent 8 }} + {{- if .Values.global.additionalLabels }} + {{ toYaml .Values.global.additionalLabels | nindent 8 }} + {{- end }} + {{- if and .Values.kubecostDeployment .Values.kubecostDeployment.labels }} + {{- toYaml .Values.kubecostDeployment.labels | nindent 8 }} + {{- end }} + annotations: + {{- with .Values.global.podAnnotations}} + {{- toYaml . | nindent 8 }} + {{- end }} + checksum/configs: {{ include "configsChecksum" . }} + spec: + {{- if .Values.global.platforms.openshift.enabled }} + securityContext: + {{- toYaml .Values.global.platforms.openshift.securityContext | nindent 8 }} + {{- else if .Values.global.securityContext }} + securityContext: + {{- toYaml .Values.global.securityContext | nindent 8 }} + {{- else }} + securityContext: + runAsUser: 1001 + runAsGroup: 1001 + fsGroup: 1001 + {{- end }} + restartPolicy: Always + serviceAccountName: {{ template "cost-analyzer.serviceAccountName" . }} + volumes: + {{- if .Values.kubecostModel.plugins.enabled }} + - name: plugins-dir + emptyDir: {} + {{- if and (not .Values.kubecostModel.plugins.existingCustomSecret.enabled) .Values.kubecostModel.plugins.secretName }} + - name: plugins-config + secret: + secretName: {{ .Values.kubecostModel.plugins.secretName }} + items: + {{- range $key, $config := .Values.kubecostModel.plugins.enabledPlugins}} + - key: {{ $config }}_config.json + path: {{ $config }}_config.json + {{- end }} + {{- end }} + {{- if .Values.kubecostModel.plugins.existingCustomSecret.enabled }} + - name: plugins-config + secret: + secretName: {{ .Values.kubecostModel.plugins.existingCustomSecret.name }} + items: + {{- range $key, $config := .Values.kubecostModel.plugins.enabledPlugins }} + - key: {{ $config }}_config.json + path: {{ $config }}_config.json + {{- end }} + {{- end }} + {{- if .Values.kubecostModel.plugins.install.enabled}} + - name: install-script + configMap: + name: {{ template "cost-analyzer.fullname" . }}-install-plugins + {{- end }} + {{- end }} + {{- if .Values.global.gcpstore.enabled }} + - name: ubbagent-config + configMap: + name: ubbagent-config + {{- end }} + - name: tmp + emptyDir: {} + {{- if and .Values.kubecostFrontend.enabled (not .Values.federatedETL.agentOnly) (not (eq (include "frontend.deployMethod" .) "haMode")) }} + - name: nginx-conf + configMap: + name: nginx-conf + items: + - key: nginx.conf + path: default.conf + {{- end }} + {{- if .Values.global.containerSecuritycontext }} + - name: var-run + emptyDir: { } + - name: cache + emptyDir: { } + {{- end }} + {{- if or .Values.kubecostModel.federatedStorageConfigSecret .Values.kubecostModel.federatedStorageConfig }} + - name: federated-storage-config + secret: + defaultMode: 420 + secretName: {{ .Values.kubecostModel.federatedStorageConfigSecret | default "federated-store" }} + {{- end }} + {{- if .Values.global.updateCaTrust.enabled }} + - name: ca-certs-secret + {{- if .Values.global.updateCaTrust.caCertsSecret }} + secret: + defaultMode: 420 + secretName: {{ .Values.global.updateCaTrust.caCertsSecret }} + {{- else }} + configMap: + name: {{ .Values.global.updateCaTrust.caCertsConfig }} + {{- end }} + - name: ssl-path + emptyDir: {} + {{- end }} + {{- if .Values.kubecostProductConfigs }} + {{- if and ((.Values.kubecostProductConfigs).productKey).enabled ((.Values.kubecostProductConfigs).productKey).secretname }} + - name: productkey-secret + secret: + secretName: {{ .Values.kubecostProductConfigs.productKey.secretname }} + items: + - key: productkey.json + path: productkey.json + {{- end }} + {{- if ((.Values.kubecostProductConfigs).smtp).secretname }} + - name: smtp-secret + secret: + secretName: {{ .Values.kubecostProductConfigs.smtp.secretname }} + items: + - key: smtp.json + path: smtp.json + {{- end }} + {{- if .Values.kubecostProductConfigs }} + {{- if .Values.kubecostProductConfigs.gcpSecretName }} + - name: gcp-key-secret + secret: + secretName: {{ .Values.kubecostProductConfigs.gcpSecretName }} + items: + - key: {{ .Values.kubecostProductConfigs.gcpSecretKeyName | default "compute-viewer-kubecost-key.json" }} + path: service-key.json + {{- end }} + {{- end -}} + {{- if .Values.kubecostProductConfigs.serviceKeySecretName }} + - name: service-key-secret + secret: + secretName: {{ .Values.kubecostProductConfigs.serviceKeySecretName }} + {{- else if .Values.kubecostProductConfigs.createServiceKeySecret }} + - name: service-key-secret + secret: + secretName: cloud-service-key + {{- end }} + {{- if .Values.kubecostProductConfigs.cloudIntegrationSecret }} + - name: cloud-integration + secret: + secretName: {{ .Values.kubecostProductConfigs.cloudIntegrationSecret }} + items: + - key: cloud-integration.json + path: cloud-integration.json + {{- else if or .Values.kubecostProductConfigs.cloudIntegrationJSON ((.Values.kubecostProductConfigs).athenaBucketName) }} + - name: cloud-integration + secret: + secretName: cloud-integration + items: + - key: cloud-integration.json + path: cloud-integration.json + {{- end }} + {{- if .Values.kubecostProductConfigs.clusters }} + - name: kubecost-clusters + configMap: + name: kubecost-clusters + {{- range .Values.kubecostProductConfigs.clusters }} + {{- if .auth }} + {{- if .auth.secretName }} + - name: {{ .auth.secretName }} + secret: + secretName: {{ .auth.secretName }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- if .Values.kubecostFrontend.tls }} + {{- if .Values.kubecostFrontend.tls.enabled }} + - name: tls + secret: + secretName : {{ .Values.kubecostFrontend.tls.secretName }} + items: + - key: tls.crt + path: kc.crt + - key: tls.key + path: kc.key + {{- end }} + {{- end }} + {{- if .Values.kubecostAdmissionController }} + {{- if .Values.kubecostAdmissionController.enabled }} + {{- if .Values.kubecostAdmissionController.secretName }} + - name: webhook-server-tls + secret: + secretName: {{ .Values.kubecostAdmissionController.secretName }} + items: + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key + {{- end }} + {{- end }} + {{- end }} + {{- if .Values.saml }} + {{- if .Values.saml.enabled }} + {{- if .Values.saml.secretName }} + - name: secret-volume + secret: + secretName: {{ .Values.saml.secretName }} + {{- end }} + {{- if .Values.saml.encryptionCertSecret }} + - name: saml-encryption-cert + secret: + secretName: {{ .Values.saml.encryptionCertSecret }} + {{- end }} + {{- if .Values.saml.decryptionKeySecret }} + - name: saml-decryption-key + secret: + secretName: {{ .Values.saml.decryptionKeySecret }} + {{- end }} + {{- if .Values.saml.metadataSecretName }} + - name: metadata-secret-volume + secret: + secretName: {{ .Values.saml.metadataSecretName }} + {{- end }} + - name: saml-auth-secret + secret: + secretName: {{ .Values.saml.authSecretName | default "kubecost-saml-secret" }} + {{- if .Values.saml.rbac.enabled }} + - name: saml-roles + configMap: + name: {{ template "cost-analyzer.fullname" . }}-saml + {{- end }} + {{- end }} + {{- end }} + {{- if .Values.oidc }} + {{- if .Values.oidc.enabled }} + - name: oidc-config + configMap: + name: {{ template "cost-analyzer.fullname" . }}-oidc + {{- if and (not .Values.oidc.existingCustomSecret.enabled) .Values.oidc.secretName }} + - name: oidc-client-secret + secret: + secretName: {{ .Values.oidc.secretName }} + {{- end }} + {{- if .Values.oidc.existingCustomSecret.enabled }} + - name: oidc-client-secret + secret: + secretName: {{ .Values.oidc.existingCustomSecret.name }} + {{- end }} + {{- end }} + {{- end }} + {{- if .Values.extraVolumes }} + # Extra volume(s) + {{- toYaml .Values.extraVolumes | nindent 8 }} + {{- end }} + - name: persistent-configs +{{- if .Values.persistentVolume }} +{{- if .Values.persistentVolume.enabled }} + persistentVolumeClaim: +{{- if .Values.persistentVolume.existingClaim }} + claimName: {{ .Values.persistentVolume.existingClaim }} +{{- else }} + claimName: {{ template "cost-analyzer.fullname" . }} +{{- end -}} +{{- else }} + emptyDir: {} +{{- end -}} +{{- else }} + persistentVolumeClaim: + claimName: {{ template "cost-analyzer.fullname" . }} +{{- end }} +{{- if .Values.persistentVolume.dbPVEnabled }} + - name: persistent-db +{{- if .Values.persistentVolume }} +{{- if .Values.persistentVolume.enabled }} + persistentVolumeClaim: +{{- if .Values.persistentVolume.dbExistingClaim }} + claimName: {{ .Values.persistentVolume.dbExistingClaim }} +{{- else }} + claimName: {{ template "cost-analyzer.fullname" . }}-db +{{- end -}} +{{- else }} + emptyDir: {} +{{- end -}} +{{- else }} + persistentVolumeClaim: + claimName: {{ template "cost-analyzer.fullname" . }}-db +{{- end }} +{{- end }} + initContainers: + {{- if and .Values.kubecostModel.plugins.enabled (not (eq (include "aggregator.deployMethod" .) "statefulset")) }} + - name: plugin-installer + image: {{ .Values.kubecostModel.plugins.install.fullImageName }} + command: ["sh", "/install/install_plugins.sh"] + {{- with .Values.kubecostModel.plugins.install.securityContext }} + securityContext: {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: install-script + mountPath: /install + - name: plugins-dir + mountPath: {{ .Values.kubecostModel.plugins.folder }} + {{- end }} + {{- if .Values.supportNFS }} + - name: config-db-perms-fix + {{- if .Values.initChownDataImage }} + image: {{ .Values.initChownDataImage }} + {{- else }} + image: busybox + {{- end }} + {{- with .Values.initChownData.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.persistentVolume.dbPVEnabled }} + command: ["sh", "-c", "/bin/chmod -R 777 /var/configs && /bin/chmod -R 777 /var/db"] + {{- else }} + command: ["sh", "-c", "/bin/chmod -R 777 /var/configs"] + {{- end }} + volumeMounts: + - name: persistent-configs + mountPath: /var/configs + {{- if .Values.persistentVolume.dbPVEnabled }} + - name: persistent-db + mountPath: /var/db + {{- end }} + securityContext: + runAsUser: 0 +{{ end }} + {{- if .Values.global.updateCaTrust.enabled }} + - name: update-ca-trust + image: {{ include "cost-model.image" . | trim | quote}} + {{- if .Values.kubecostModel.imagePullPolicy }} + imagePullPolicy: {{ .Values.kubecostModel.imagePullPolicy }} + {{- else }} + imagePullPolicy: Always + {{- end }} + {{- with .Values.global.updateCaTrust.securityContext }} + securityContext: {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.global.updateCaTrust.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + command: + - 'sh' + - '-c' + - > + mkdir -p /etc/pki/ca-trust/extracted/{edk2,java,openssl,pem}; + /usr/bin/update-ca-trust extract; + volumeMounts: + - name: ca-certs-secret + mountPath: {{ .Values.global.updateCaTrust.caCertsMountPath | quote }} + - name: ssl-path + mountPath: "/etc/pki/ca-trust/extracted" + readOnly: false + {{- end}} + containers: + {{- if .Values.global.gmp.enabled }} + - name: {{ .Values.global.gmp.gmpProxy.name }} + image: {{ .Values.global.gmp.gmpProxy.image }} + {{- if .Values.global.gmp.gmpProxy.imagePullPolicy }} + imagePullPolicy: {{ .Values.global.gmp.gmpProxy.imagePullPolicy }} + {{- else }} + imagePullPolicy: Always + {{- end }} + args: + - "--web.listen-address=:{{ .Values.global.gmp.gmpProxy.port }}" + - "--query.project-id={{ .Values.global.gmp.gmpProxy.projectId }}" + {{- if .Values.systemProxy.enabled }} + env: + - name: HTTP_PROXY + value: "{{ .Values.systemProxy.httpProxyUrl }}" + - name: http_proxy + value: "{{ .Values.systemProxy.httpProxyUrl }}" + - name: HTTPS_PROXY + value: "{{ .Values.systemProxy.httpsProxyUrl }}" + - name: https_proxy + value: "{{ .Values.systemProxy.httpsProxyUrl }}" + - name: NO_PROXY + value: "{{ .Values.systemProxy.noProxy }}" + - name: no_proxy + value: "{{ .Values.systemProxy.noProxy }}" + {{- end }} + ports: + - name: web + containerPort: {{ .Values.global.gmp.gmpProxy.port | int }} + readinessProbe: + httpGet: + path: /-/ready + port: web + livenessProbe: + httpGet: + path: /-/healthy + port: web + {{- end }} + {{- if .Values.global.amp.enabled }} + - name: sigv4proxy + image: {{ .Values.sigV4Proxy.image }} + {{- if .Values.sigV4Proxy.imagePullPolicy }} + imagePullPolicy: {{ .Values.sigV4Proxy.imagePullPolicy }} + {{- else }} + imagePullPolicy: Always + {{- end }} + {{- if .Values.global.containerSecurityContext }} + securityContext: + {{- toYaml .Values.global.containerSecurityContext | nindent 12 -}} + {{- end }} + {{- with .Values.sigV4Proxy.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + args: + - --name + - {{ .Values.sigV4Proxy.name }} + - --region + - {{ .Values.sigV4Proxy.region }} + - --host + - {{ .Values.sigV4Proxy.host }} + {{- if .Values.sigV4Proxy.role_arn }} + - --role-arn + - {{ .Values.sigV4Proxy.role_arn }} + {{- end }} + - --port + - :{{ .Values.sigV4Proxy.port }} + ports: + - name: aws-sigv4-proxy + containerPort: {{ .Values.sigV4Proxy.port | int }} + env: + - name: AGENT_LOCAL_PORT + value: "{{ .Values.sigV4Proxy.port | int }}" + {{- if .Values.systemProxy.enabled }} + - name: HTTP_PROXY + value: "{{ .Values.systemProxy.httpProxyUrl }}" + - name: http_proxy + value: "{{ .Values.systemProxy.httpProxyUrl }}" + - name: HTTPS_PROXY + value: "{{ .Values.systemProxy.httpsProxyUrl }}" + - name: https_proxy + value: "{{ .Values.systemProxy.httpsProxyUrl }}" + - name: NO_PROXY + value: "{{ .Values.systemProxy.noProxy }}" + - name: no_proxy + value: "{{ .Values.systemProxy.noProxy }}" + {{- end }} + {{- if .Values.sigV4Proxy.extraEnv }} + {{- toYaml .Values.sigV4Proxy.extraEnv | nindent 10 }} + {{- end }} + {{- end }} + {{- if .Values.global.gcpstore.enabled }} + - name: ubbagent + image: gcr.io/kubecost1/gcp-mp/ent/cost-model/ubbagent:1.0 + env: + {{- if .Values.systemProxy.enabled }} + - name: HTTP_PROXY + value: {{ .Values.systemProxy.httpProxyUrl }} + - name: http_proxy + value: {{ .Values.systemProxy.httpProxyUrl }} + - name: HTTPS_PROXY + value: {{ .Values.systemProxy.httpsProxyUrl }} + - name: https_proxy + value: {{ .Values.systemProxy.httpsProxyUrl }} + - name: NO_PROXY + value: {{ .Values.systemProxy.noProxy }} + - name: no_proxy + value: {{ .Values.systemProxy.noProxy }} + {{- end }} + - name: AGENT_CONFIG_FILE + value: "/etc/ubbagent/config.yaml" + - name: AGENT_LOCAL_PORT + value: "6080" + - name: AGENT_ENCODED_KEY + valueFrom: + secretKeyRef: + name: {{ default "kubecost-reporting-secret" .Values.reportingSecret }} + key: reporting-key + - name: AGENT_CONSUMER_ID + valueFrom: + secretKeyRef: + name: {{ default "kubecost-reporting-secret" .Values.reportingSecret }} + key: consumer-id + volumeMounts: + - name: ubbagent-config + mountPath: /etc/ubbagent + {{- end }} + {{- if .Values.global.ammsp.enabled }} + # This section of the chart borrows liberally from + # https://github.com/Azure/aad-auth-proxy/blob/main/deploy/chart/aad-auth-proxy/templates/deployment.yaml + - name: {{ .Values.global.ammsp.aadAuthProxy.name }} + image: {{ .Values.global.ammsp.aadAuthProxy.image }} + {{- if .Values.global.ammsp.aadAuthProxy.imagePullPolicy }} + imagePullPolicy: {{ .Values.global.ammsp.aadAuthProxy.imagePullPolicy }} + {{- else }} + imagePullPolicy: Always + {{- end }} + env: + - name: AUDIENCE + value: {{ .Values.global.ammsp.aadAuthProxy.audience }} + - name: TARGET_HOST + value: {{ .Values.global.ammsp.queryEndpoint }} + - name: LISTENING_PORT + value: {{ .Values.global.ammsp.aadAuthProxy.port | quote }} + - name: IDENTITY_TYPE + value: {{ .Values.global.ammsp.aadAuthProxy.identityType }} + {{- if eq .Values.global.ammsp.aadAuthProxy.identityType "userAssigned" }} + - name: AAD_CLIENT_ID + value: {{ required "aadClientId is required for userAssigned identity types" .Values.global.ammsp.aadAuthProxy.aadClientId | toString | trim | quote }} + {{- else if eq .Values.global.ammsp.aadAuthProxy.identityType "aadApplication" }} + - name: AAD_CLIENT_ID + value: {{ required "aadClientId is required for aadApplication identity types" .Values.global.ammsp.aadAuthProxy.aadClientId | toString | trim | quote }} + - name: AAD_TENANT_ID + value: {{ required "aadTenantId is required for aadApplication identity type" .Values.global.ammsp.aadAuthProxy.aadTenantId | toString | trim | quote }} + - name: AAD_CLIENT_CERTIFICATE_PATH + value: {{ required "aadClientCertificatePath is required for aadApplication identity type" .Values.global.ammsp.aadAuthProxy.aadClientCertificatePath | toString | trim | quote }} + {{- end }} + - name: AAD_TOKEN_REFRESH_INTERVAL_IN_PERCENTAGE + value: "10" + - name: OTEL_SERVICE_NAME + value: {{ .Values.global.ammsp.aadAuthProxy.name | replace "-" "_" }} + {{- if .Values.systemProxy.enabled }} + - name: HTTP_PROXY + value: "{{ .Values.systemProxy.httpProxyUrl }}" + - name: http_proxy + value: "{{ .Values.systemProxy.httpProxyUrl }}" + - name: HTTPS_PROXY + value: "{{ .Values.systemProxy.httpsProxyUrl }}" + - name: https_proxy + value: "{{ .Values.systemProxy.httpsProxyUrl }}" + - name: NO_PROXY + value: "{{ .Values.systemProxy.noProxy }}" + - name: no_proxy + value: "{{ .Values.systemProxy.noProxy }}" + {{- end }} + ports: + - name: http + containerPort: {{ .Values.global.ammsp.aadAuthProxy.port | int }} + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /ready + port: http + scheme: HTTP + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + livenessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: http + scheme: HTTP + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + {{- end }} + - image: {{ include "cost-model.image" . | trim | quote}} + name: cost-model + {{- if .Values.kubecostModel.extraArgs }} + args: + {{- toYaml .Values.kubecostModel.extraArgs | nindent 12 }} + {{- end }} + securityContext: + {{- if .Values.kubecostModel.securityContext }} + {{- toYaml .Values.kubecostModel.securityContext | nindent 12 -}} + {{- else if .Values.global.containerSecurityContext }} + {{- toYaml .Values.global.containerSecurityContext | nindent 12 -}} + {{- end }} + {{- if .Values.kubecostModel.imagePullPolicy }} + imagePullPolicy: {{ .Values.kubecostModel.imagePullPolicy }} + {{- else }} + imagePullPolicy: Always + {{- end }} + ports: + - name: tcp-model + containerPort: 9003 + protocol: TCP + {{- if and .Values.kubecostFrontend.enabled (not .Values.federatedETL.agentOnly) (not (eq (include "frontend.deployMethod" .) "haMode")) }} + - name: tcp-frontend + containerPort: 9090 + protocol: TCP + {{- end }} + {{- with .Values.kubecostModel.extraPorts }} + {{- toYaml . | nindent 10 }} + {{- end }} + resources: +{{ toYaml .Values.kubecostModel.resources | indent 12 }} + {{- if .Values.kubecostModel.readinessProbe.enabled }} + readinessProbe: + httpGet: + path: /healthz + port: 9003 + initialDelaySeconds: {{ .Values.kubecostModel.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.kubecostModel.readinessProbe.periodSeconds}} + failureThreshold: {{ .Values.kubecostModel.readinessProbe.failureThreshold}} + {{- end }} + {{- if .Values.kubecostModel.livenessProbe.enabled }} + livenessProbe: + httpGet: + path: /healthz + port: 9003 + initialDelaySeconds: {{ .Values.kubecostModel.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.kubecostModel.livenessProbe.periodSeconds }} + failureThreshold: {{ .Values.kubecostModel.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.global.containerSecuritycontext }} + securityContext: + {{- toYaml .Values.global.containerSecuritycontext | nindent 12 }} + {{- end }} + volumeMounts: + - name: persistent-configs + mountPath: /var/configs + {{- if .Values.extraVolumeMounts }} + # Extra volume mount(s) + {{- toYaml .Values.extraVolumeMounts | nindent 12 }} + {{- end }} + {{- if .Values.persistentVolume.dbPVEnabled }} + - name: persistent-db + mountPath: /var/db + {{- end }} + {{- if or .Values.kubecostModel.federatedStorageConfigSecret .Values.kubecostModel.federatedStorageConfig }} + - name: federated-storage-config + mountPath: /var/configs/etl/federated + readOnly: true + {{- end }} + {{- if .Values.global.updateCaTrust.enabled }} + - name: ca-certs-secret + mountPath: {{ .Values.global.updateCaTrust.caCertsMountPath | quote }} + - name: ssl-path + mountPath: "/etc/pki/ca-trust/extracted" + readOnly: false + {{- end }} + {{- if .Values.kubecostAdmissionController }} + {{- if .Values.kubecostAdmissionController.enabled }} + {{- if .Values.kubecostAdmissionController.secretName }} + - name: {{ .Values.kubecostAdmissionController.secretName }} + mountPath: /certs + {{- end }} + {{- end }} + {{- end }} + {{- if .Values.kubecostProductConfigs }} + {{- if and ((.Values.kubecostProductConfigs).productKey).enabled ((.Values.kubecostProductConfigs).productKey).secretname }} + - name: productkey-secret + mountPath: /var/configs/productkey + {{- end }} + {{- if ((.Values.kubecostProductConfigs).smtp).secretname }} + - name: smtp-secret + mountPath: /var/configs/smtp + {{- end }} + {{- if .Values.kubecostProductConfigs.gcpSecretName }} + - name: gcp-key-secret + mountPath: /var/secrets + {{- end }} + {{- if or .Values.kubecostProductConfigs.serviceKeySecretName .Values.kubecostProductConfigs.createServiceKeySecret }} + - name: service-key-secret + mountPath: /var/secrets + {{- end }} + {{- if .Values.kubecostProductConfigs.clusters }} + - name: kubecost-clusters + mountPath: /var/configs/clusters + {{- range .Values.kubecostProductConfigs.clusters }} + {{- if .auth }} + {{- if .auth.secretName }} + - name: {{ .auth.secretName }} + mountPath: /var/secrets/{{ .auth.secretName }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- if .Values.saml }} + {{- if .Values.saml.enabled }} + {{- if .Values.saml.secretName }} + - name: secret-volume + mountPath: /var/configs/secret-volume + {{- end }} + {{- if .Values.saml.encryptionCertSecret }} + - name: saml-encryption-cert + mountPath: /var/configs/saml-encryption-cert + {{- end }} + {{- if .Values.saml.decryptionKeySecret }} + - name: saml-decryption-key + mountPath: /var/configs/saml-decryption-key + {{- end }} + {{- if .Values.saml.metadataSecretName }} + - name: metadata-secret-volume + mountPath: /var/configs/metadata-secret-volume + {{- end }} + - name: saml-auth-secret + mountPath: /var/configs/saml-auth-secret + {{- if .Values.saml.rbac.enabled }} + - name: saml-roles + mountPath: /var/configs/saml + {{- end }} + {{- end }} + {{- end }} + {{- if .Values.oidc }} + {{- if .Values.oidc.enabled }} + - name: oidc-config + mountPath: /var/configs/oidc + {{- if or .Values.oidc.existingCustomSecret.name .Values.oidc.secretName }} + - name: oidc-client-secret + mountPath: /var/configs/oidc-client-secret + {{- end }} + {{- end }} + {{- end }} + env: + - name: CONTAINER_IMAGE_TAG + value: {{ include "cost-model.imagetag" . }} + {{- if .Values.global.grafana }} + - name: GRAFANA_ENABLED + value: "{{ template "cost-analyzer.grafanaEnabled" . }}" + {{- end}} + - name: LOG_LEVEL + value: {{ .Values.kubecostModel.logLevel }} + {{- if .Values.kubecostModel.extraEnv -}} + {{ toYaml .Values.kubecostModel.extraEnv | nindent 12 }} + {{- end }} + {{- if .Values.reporting }} + {{- if .Values.reporting.valuesReporting }} + - name: HELM_VALUES + value: {{ template "cost-analyzer.filterEnabled" .Values }} + {{- end }} + {{- end }} + {{- if .Values.alertConfigmapName }} + - name: ALERT_CONFIGMAP_NAME + value: {{ .Values.alertConfigmapName }} + {{- end }} + {{- if .Values.productConfigmapName }} + - name: PRODUCT_CONFIGMAP_NAME + value: {{ .Values.productConfigmapName }} + {{- end }} + {{- if .Values.smtpConfigmapName }} + - name: SMTP_CONFIGMAP_NAME + value: {{ .Values.smtpConfigmapName }} + {{- end }} + {{- if .Values.appConfigmapName }} + - name: APP_CONFIGMAP_NAME + value: {{ .Values.appConfigmapName }} + {{- end }} + {{- if .Values.kubecostModel.softMemoryLimit }} + - name: GOMEMLIMIT + value: {{ .Values.kubecostModel.softMemoryLimit }} + {{- end }} + {{- if .Values.assetReportConfigmapName }} + - name: ASSET_REPORT_CONFIGMAP_NAME + value: {{ .Values.assetReportConfigmapName }} + {{- end }} + {{- if .Values.cloudCostReportConfigmapName }} + - name: CLOUD_COST_REPORT_CONFIGMAP_NAME + value: {{ .Values.cloudCostReportConfigmapName }} + {{- end }} + {{- if .Values.savedReportConfigmapName }} + - name: SAVED_REPORT_CONFIGMAP_NAME + value: {{ .Values.savedReportConfigmapName }} + {{- end }} + {{- if .Values.groupFiltersConfigmapName }} + - name: GROUP_FILTERS_CONFIGMAP_NAME + value: {{ .Values.groupFiltersConfigmapName }} + {{- end }} + {{- if .Values.pricingConfigmapName }} + - name: PRICING_CONFIGMAP_NAME + value: {{ .Values.pricingConfigmapName }} + {{- end }} + {{- if .Values.metricsConfigmapName }} + - name: METRICS_CONFIGMAP_NAME + value: {{ .Values.metricsConfigmapName }} + {{- end }} + - name: READ_ONLY + value: {{ (quote .Values.readonly) | default (quote false) }} + - name: PROMETHEUS_SERVER_ENDPOINT + valueFrom: + configMapKeyRef: + name: {{ template "cost-analyzer.fullname" . }} + key: prometheus-server-endpoint + - name: CLOUD_PROVIDER_API_KEY + value: "AIzaSyDXQPG_MHUEy9neR7stolq6l0ujXmjJlvk" # The GCP Pricing API key.This GCP api key is expected to be here and is limited to accessing google's billing API. + {{- if .Values.kubecostProductConfigs }} + {{- if .Values.kubecostProductConfigs.gcpSecretName }} + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /var/configs/key.json + {{- end }} + {{- end }} + - name: CONFIG_PATH + value: /var/configs/ + - name: DB_PATH + value: /var/db/ + - name: CLUSTER_PROFILE + {{- if .Values.kubecostProductConfigs }} + value: {{ .Values.kubecostProductConfigs.clusterProfile | default "production" }} + {{- else }} + value: production + {{- end }} + {{- if .Values.kubecostProductConfigs }} + {{- if ((.Values.kubecostProductConfigs).productKey).mountPath }} + - name: PRODUCT_KEY_MOUNT_PATH + value: {{ .Values.kubecostProductConfigs.productKey.mountPath }} + {{- end }} + {{- if ((.Values.kubecostProductConfigs).smtp).mountPath }} + - name: SMTP_CONFIG_MOUNT_PATH + value: {{ .Values.kubecostProductConfigs.smtp.mountPath }} + {{- end }} + {{- if .Values.kubecostProductConfigs.ingestPodUID }} + - name: INGEST_POD_UID + value: {{ (quote .Values.kubecostProductConfigs.ingestPodUID) }} + {{- end }} + {{- if .Values.kubecostProductConfigs.regionOverrides }} + - name: REGION_OVERRIDE_LIST + value: {{ (quote .Values.kubecostProductConfigs.regionOverrides) }} + {{- end }} + {{- end }} + {{- if .Values.global.prometheus.queryServiceBasicAuthSecretName}} + - name: DB_BASIC_AUTH_USERNAME + valueFrom: + secretKeyRef: + name: {{ .Values.global.prometheus.queryServiceBasicAuthSecretName }} + key: USERNAME + - name: DB_BASIC_AUTH_PW + valueFrom: + secretKeyRef: + name: {{ .Values.global.prometheus.queryServiceBasicAuthSecretName }} + key: PASSWORD + {{- end }} + {{- if .Values.global.prometheus.queryServiceBearerTokenSecretName }} + - name: DB_BEARER_TOKEN + valueFrom: + secretKeyRef: + name: {{ .Values.global.prometheus.queryServiceBearerTokenSecretName }} + key: TOKEN + {{- end }} + {{- if .Values.global.prometheus.insecureSkipVerify }} + - name: INSECURE_SKIP_VERIFY + value: {{ (quote .Values.global.prometheus.insecureSkipVerify) }} + {{- end }} + {{- if .Values.global.prometheus.kubeRBACProxy }} + - name: KUBE_RBAC_PROXY_ENABLED + value: {{ (quote .Values.global.prometheus.kubeRBACProxy) }} + {{- end }} + {{- if .Values.pricingCsv }} + {{- if .Values.pricingCsv.enabled }} + - name: USE_CSV_PROVIDER + value: "true" + - name: CSV_PATH + value: {{ .Values.pricingCsv.location.URI }} + - name: CSV_REGION + value: {{ .Values.pricingCsv.location.region }} + {{- if eq .Values.pricingCsv.location.provider "AWS"}} + {{- if .Values.pricingCsv.location.csvAccessCredentials }} + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: {{ .Values.pricingCsv.location.csvAccessCredentials }} + key: AWS_ACCESS_KEY_ID + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.pricingCsv.location.csvAccessCredentials }} + key: AWS_SECRET_ACCESS_KEY + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- if .Values.kubecostMetrics }} + - name: EMIT_POD_ANNOTATIONS_METRIC + value: {{ (quote .Values.kubecostMetrics.emitPodAnnotations) | default (quote false) }} + - name: EMIT_NAMESPACE_ANNOTATIONS_METRIC + value: {{ (quote .Values.kubecostMetrics.emitNamespaceAnnotations) | default (quote false) }} + {{- end }} + {{- if .Values.kubecostMetrics }} + - name: EMIT_KSM_V1_METRICS + value: {{ (quote .Values.kubecostMetrics.emitKsmV1Metrics) | default (quote true) }} + {{- end }} + {{- if .Values.kubecostMetrics }} + - name: EMIT_KSM_V1_METRICS_ONLY # ONLY emit KSM v1 metrics that do not exist in KSM 2 by default + value: {{ (quote .Values.kubecostMetrics.emitKsmV1MetricsOnly) | default (quote false) }} + {{- end }} + {{- if .Values.reporting }} + - name: LOG_COLLECTION_ENABLED + value: {{ (quote .Values.reporting.logCollection) | default (quote true) }} + - name: PRODUCT_ANALYTICS_ENABLED + value: {{ (quote .Values.reporting.productAnalytics) | default (quote true) }} + - name: ERROR_REPORTING_ENABLED + value: {{ (quote .Values.reporting.errorReporting ) | default (quote true) }} + - name: VALUES_REPORTING_ENABLED + value: {{ (quote .Values.reporting.valuesReporting) | default (quote true) }} + {{- if .Values.reporting.errorReporting }} + - name: SENTRY_DSN + value: "https://71964476292e4087af8d5072afe43abd@o394722.ingest.sentry.io/5245431" + {{- end }} + {{- end }} + {{- if or .Values.kubecostModel.federatedStorageConfigSecret .Values.kubecostModel.federatedStorageConfig }} + - name: FEDERATED_STORE_CONFIG + value: "/var/configs/etl/federated/federated-store.yaml" + {{- end }} + {{- if or .Values.federatedETL.federatedCluster .Values.kubecostModel.federatedStorageConfigSecret .Values.kubecostModel.federatedStorageConfig }} + - name: FEDERATED_CLUSTER + {{- if eq .Values.federatedETL.readOnlyPrimary true }} + value: "false" + {{- else }} + value: "true" + {{- end }} + {{- end }} + {{- if .Values.federatedETL.redirectS3Backup }} + - name: FEDERATED_REDIRECT_BACKUP + value: "true" + {{- end }} + {{- if .Values.federatedETL.useMultiClusterDB }} + - name: CURRENT_CLUSTER_ID_FILTER_ENABLED + value: "true" + {{- end }} + {{- if .Values.persistentVolume.dbPVEnabled }} + - name: ETL_PATH_PREFIX + value: "/var/db" + {{- end }} + - name: ETL_RESOLUTION_SECONDS + value: {{ (quote .Values.kubecostModel.etlResolutionSeconds) | default (quote 300) }} + - name: ETL_MAX_PROMETHEUS_QUERY_DURATION_MINUTES + value: {{ (quote .Values.kubecostModel.maxPrometheusQueryDurationMinutes) | default (quote 1440) }} + - name: ETL_DAILY_STORE_DURATION_DAYS + value: {{ (quote .Values.kubecostModel.etlDailyStoreDurationDays) }} + - name: ETL_HOURLY_STORE_DURATION_HOURS + value: {{ (quote .Values.kubecostModel.etlHourlyStoreDurationHours) | default (quote 49) }} + {{- if .Values.kubecostModel }} + {{- if .Values.kubecostModel.allocation }} + {{- if .Values.kubecostModel.allocation.nodeLabels }} + {{- with .Values.kubecostModel.allocation.nodeLabels }} + - name: ALLOCATION_NODE_LABELS_ENABLED + value: {{ (quote .enabled) | default (quote true) }} + - name: ALLOCATION_NODE_LABELS_INCLUDE_LIST + value: {{ (quote .includeList) }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + - name: CONTAINER_STATS_ENABLED + value: {{ (quote .Values.kubecostModel.containerStatsEnabled) | default (quote false) }} + {{- if .Values.systemProxy.enabled }} + - name: HTTP_PROXY + value: {{ .Values.systemProxy.httpProxyUrl }} + - name: http_proxy + value: {{ .Values.systemProxy.httpProxyUrl }} + - name: HTTPS_PROXY + value: {{ .Values.systemProxy.httpsProxyUrl }} + - name: https_proxy + value: {{ .Values.systemProxy.httpsProxyUrl }} + - name: NO_PROXY + value: {{ .Values.systemProxy.noProxy }} + - name: no_proxy + value: {{ .Values.systemProxy.noProxy }} + {{- end }} + - name: PV_ENABLED + value: {{ (quote .Values.persistentVolume.enabled) | default (quote true) }} + - name: MAX_QUERY_CONCURRENCY + value: {{ (quote .Values.kubecostModel.maxQueryConcurrency) | default (quote 5) }} + - name: UTC_OFFSET + value: {{ (quote .Values.kubecostModel.utcOffset) | default (quote ) }} + {{- if .Values.networkCosts }} + {{- if .Values.networkCosts.enabled }} + - name: NETWORK_COSTS_PORT + value: {{ quote .Values.networkCosts.port | default (quote 3001) }} + # ADVANCED_NETWORK_STATS is a feature offered by Kubecost that gives you network + # insights of your Kubernetes resources with cloud services. The feature is + # enabled when network cost is enabled and one of the service tagging is enabled + {{- if .Values.networkCosts.config.services }} + {{- $services := .Values.networkCosts.config.services -}} + {{- if or (index $services "google-cloud-services") (index $services "amazon-web-services") (index $services "azure-cloud-services")}} + - name: ADVANCED_NETWORK_STATS + value: "true" + {{- else}} + - name: ADVANCED_NETWORK_STATS + value: "false" + {{- end}} + {{- end }} + {{- end }} + {{- end }} + {{- if .Values.oidc.enabled }} + - name: OIDC_ENABLED + value: "true" + - name: OIDC_SKIP_ONLINE_VALIDATION + value: {{ (quote .Values.oidc.skipOnlineTokenValidation) | default (quote false) }} + {{- end}} + {{- if .Values.saml }} + {{- if .Values.saml.enabled }} + - name: SAML_ENABLED + value: "true" + - name: IDP_URL + value: {{ .Values.saml.idpMetadataURL }} + - name: SP_HOST + value: {{ .Values.saml.appRootURL }} + {{- if .Values.saml.audienceURI }} + - name: AUDIENCE_URI + value: {{ .Values.saml.audienceURI }} + {{- end }} + {{- if .Values.saml.isGLUUProvider }} + - name: GLUU_SAML_PROVIDER + value: {{ (quote .Values.saml.isGLUUProvider) }} + {{- end }} + {{- if .Values.saml.nameIDFormat }} + - name: NAME_ID_FORMAT + value: {{ .Values.saml.nameIDFormat }} + {{- end}} + {{- if .Values.saml.authTimeout }} + - name: AUTH_TOKEN_TIMEOUT + value: {{ (quote .Values.saml.authTimeout) }} + {{- end}} + {{- if .Values.saml.redirectURL }} + - name: LOGOUT_REDIRECT_URL + value: {{ .Values.saml.redirectURL }} + {{- end}} + {{- if .Values.saml.rbac.enabled }} + - name: SAML_RBAC_ENABLED + value: "true" + {{- end }} + {{- if and .Values.saml.encryptionCertSecret .Values.saml.decryptionKeySecret }} + - name: SAML_RESPONSE_ENCRYPTED + value: "true" + {{- end}} + {{- end }} + {{- end }} + {{- if and (.Values.prometheus.server.global.external_labels.cluster_id) (not .Values.prometheus.server.clusterIDConfigmap) }} + - name: CLUSTER_ID + value: {{ .Values.prometheus.server.global.external_labels.cluster_id }} + {{- end }} + {{- if .Values.prometheus.server.clusterIDConfigmap }} + - name: CLUSTER_ID + valueFrom: + configMapKeyRef: + name: {{ .Values.prometheus.server.clusterIDConfigmap }} + key: CLUSTER_ID + {{- end }} + {{- if .Values.kubecostModel.promClusterIDLabel }} + - name: PROM_CLUSTER_ID_LABEL + value: {{ .Values.kubecostModel.promClusterIDLabel }} + {{- end }} + {{- if .Values.reporting.googleAnalyticsTag }} + - name: GOOGLE_ANALYTICS_TAG + value: {{ .Values.reporting.googleAnalyticsTag }} + {{- end }} + {{- if .Values.costEventsAudit }} + - name: COST_EVENTS_AUDIT_ENABLED + value: {{ (quote .Values.costEventsAudit.enabled) | default (quote false) }} + {{- end }} + - name: RELEASE_NAME + value: {{ .Release.Name }} + - name: KUBECOST_NAMESPACE + value: {{ .Release.Namespace }} + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: KUBECOST_TOKEN + valueFrom: + configMapKeyRef: + name: {{ template "cost-analyzer.fullname" . }} + key: kubecost-token + - name: WATERFOWL_ENABLED + value: "true" + {{- if not (.Values.diagnostics.enabled) }} + - name: DIAGNOSTICS_RUN_IN_COST_MODEL + value: "false" + {{- /* Cannot run MultiClusterDiagnostics in the cost-model container without federated-store config */}} + {{- else if and (empty .Values.kubecostModel.federatedStorageConfigSecret) (not .Values.kubecostModel.federatedStorageConfig) }} + - name: DIAGNOSTICS_RUN_IN_COST_MODEL + value: "false" + {{- else if .Values.diagnostics.deployment.enabled }} + - name: DIAGNOSTICS_RUN_IN_COST_MODEL + value: "false" + {{- else }} + - name: DIAGNOSTICS_RUN_IN_COST_MODEL + value: "true" + - name: DIAGNOSTICS_KUBECOST_FQDN + value: "localhost" + - name: DIAGNOSTICS_KUBECOST_NAMESPACE + value: {{ .Release.Namespace }} + - name: DIAGNOSTICS_PRIMARY + value: {{ quote .Values.diagnostics.primary.enabled }} + - name: DIAGNOSTICS_RETENTION + value: {{ .Values.diagnostics.primary.retention }} + - name: DIAGNOSTICS_PRIMARY_READONLY + value: {{ quote .Values.diagnostics.primary.readonly }} + - name: DIAGNOSTICS_POLLING_INTERVAL + value: {{ .Values.diagnostics.pollingInterval }} + - name: DIAGNOSTICS_KEEP_HISTORY + value: {{ quote .Values.diagnostics.keepDiagnosticHistory }} + - name: DIAGNOSTICS_COLLECT_HELM_VALUES + value: {{ quote .Values.diagnostics.collectHelmValues }} + {{- end }} + {{- if and .Values.kubecostFrontend.enabled (not .Values.federatedETL.agentOnly) (not (eq (include "frontend.deployMethod" .) "haMode")) }} + {{- if .Values.kubecostFrontend }} + {{- if .Values.kubecostFrontend.fullImageName }} + - image: {{ .Values.kubecostFrontend.fullImageName }} + {{- else if .Values.imageVersion }} + - image: {{ .Values.kubecostFrontend.image }}:{{ .Values.imageVersion }} + {{- else if eq "development" .Chart.AppVersion }} + - image: gcr.io/kubecost1/frontend-nightly:latest + {{- else }} + - image: {{ .Values.kubecostFrontend.image }}:prod-{{ $.Chart.AppVersion }} + {{- end }} + {{- else }} + - image: gcr.io/kubecost1/frontend:prod-{{ $.Chart.AppVersion }} + {{- end }} + env: + - name: GET_HOSTS_FROM + value: dns + {{- if .Values.kubecostFrontend.extraEnv -}} + {{ toYaml .Values.kubecostFrontend.extraEnv | nindent 12 }} + {{- end }} + name: cost-analyzer-frontend + {{- if .Values.kubecostFrontend.securityContext }} + securityContext: + {{- toYaml .Values.kubecostFrontend.securityContext | nindent 12 }} + {{- else }} + securityContext: + {{- toYaml .Values.global.containerSecurityContext | nindent 12 }} + {{- end }} + volumeMounts: + - name: tmp + mountPath: /tmp + - name: tmp + mountPath: /var/lib/nginx/tmp + - name: tmp + mountPath: /var/run + - name: nginx-conf + mountPath: /etc/nginx/conf.d/ + {{- if .Values.global.containerSecuritycontext }} + - mountPath: /var/cache/nginx + name: cache + - mountPath: /var/run + name: var-run + {{- end }} + {{- if .Values.kubecostFrontend.tls }} + {{- if .Values.kubecostFrontend.tls.enabled }} + - name: tls + mountPath: /etc/ssl/certs + {{- end }} + {{- end }} + resources: +{{ toYaml .Values.kubecostFrontend.resources | indent 12 }} + {{- if .Values.kubecostFrontend.imagePullPolicy }} + imagePullPolicy: {{ .Values.kubecostFrontend.imagePullPolicy }} + {{- else }} + imagePullPolicy: Always + {{- end }} + {{- if .Values.kubecostFrontend.readinessProbe.enabled }} + readinessProbe: + httpGet: + path: /healthz + port: 9003 + initialDelaySeconds: {{ .Values.kubecostFrontend.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.kubecostFrontend.readinessProbe.periodSeconds }} + failureThreshold: {{ .Values.kubecostFrontend.readinessProbe.failureThreshold }} + {{- end }} + {{- if .Values.kubecostFrontend.livenessProbe.enabled }} + livenessProbe: + httpGet: + path: /healthz + port: 9003 + initialDelaySeconds: {{ .Values.kubecostFrontend.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.kubecostFrontend.livenessProbe.periodSeconds }} + failureThreshold: {{ .Values.kubecostFrontend.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.global.containerSecuritycontext }} + securityContext: + {{- toYaml .Values.global.containerSecuritycontext | nindent 12 }} + {{- end }} + {{ end }} + + {{- if and (eq (include "aggregator.deployMethod" .) "singlepod") (not .Values.federatedETL.agentOnly) }} + {{- include "aggregator.containerTemplate" . | nindent 8 }} + {{- if .Values.kubecostAggregator.jaeger.enabled }} + {{- include "aggregator.jaeger.sidecarContainerTemplate" . | nindent 8 }} + {{- end }} + {{- include "aggregator.cloudCost.containerTemplate" . | nindent 8 }} + {{- end }} + + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{- range $.Values.imagePullSecrets }} + - name: {{ .name }} + {{- end }} + {{- end }} + {{- if .Values.priority }} + {{- if .Values.priority.enabled }} + {{- if gt (len .Values.priority.name) 0 }} + priorityClassName: {{ .Values.priority.name }} + {{- else }} + priorityClassName: {{ template "cost-analyzer.fullname" . }}-priority + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-frontend-config-map-template.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-frontend-config-map-template.yaml new file mode 100644 index 0000000000..3b289bb4ee --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-frontend-config-map-template.yaml @@ -0,0 +1,1512 @@ +{{- if .Values.kubecostFrontend.enabled }} +{{- if and (not .Values.agent) (not .Values.cloudAgent) (not .Values.federatedETL.agentOnly) }} +{{- $serviceName := include "cost-analyzer.serviceName" . -}} +{{- if .Values.saml.enabled }} +{{- if .Values.oidc.enabled }} +{{- fail "SAML and OIDC cannot both be enabled" }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: nginx-conf + namespace: {{ .Release.Namespace }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} +data: + nginx.conf: | + gzip_static on; + + # Enable gzip encoding for content of the provided types of 50kb and higher. + gzip on; + gzip_min_length 50000; + gzip_proxied expired no-cache no-store private auth; + gzip_types + application/atom+xml + application/geo+json + application/javascript + application/x-javascript + application/json + application/ld+json + application/manifest+json + application/rdf+xml + application/rss+xml + application/vnd.ms-fontobject + application/wasm + application/x-web-app-manifest+json + application/xhtml+xml + application/xml + font/eot + font/otf + font/ttf + image/bmp + image/svg+xml + text/cache-manifest + text/calendar + text/css + text/javascript + text/markdown + text/plain + text/xml + text/x-component + text/x-cross-domain-policy; + + upstream api { +{{- if .Values.kubecostFrontend.useDefaultFqdn }} + server {{ $serviceName }}.{{ .Release.Namespace }}.svc.cluster.local:9001; +{{- else if (.Values.kubecostFrontend.api).fqdn }} + server {{ .Values.kubecostFrontend.api.fqdn }}; +{{- else }} + server {{ $serviceName }}.{{ .Release.Namespace }}:9001; +{{- end }} + } + + upstream model { +{{- if .Values.kubecostFrontend.useDefaultFqdn }} + server {{ $serviceName }}.{{ .Release.Namespace }}.svc.cluster.local:9003; +{{- else if (.Values.kubecostFrontend.model).fqdn }} + server {{ .Values.kubecostFrontend.model.fqdn }}; +{{- else }} + server {{ $serviceName }}.{{ .Release.Namespace }}:9003; +{{- end }} + } + +{{- if and .Values.clusterController .Values.clusterController.enabled }} + upstream clustercontroller { +{{- if .Values.kubecostFrontend.useDefaultFqdn }} + server {{ template "kubecost.clusterControllerName" . }}-service.{{ .Release.Namespace }}.svc.cluster.local:9731; +{{- else }} +{{- if (.Values.kubecostFrontend.clusterController).fqdn }} + server {{ .Values.kubecostFrontend.clusterController.fqdn }}; +{{- else }} + server {{ template "kubecost.clusterControllerName" . }}-service.{{ .Release.Namespace }}:9731; +{{- end }} +{{- end }} + } +{{- end }} + +{{- if .Values.global.grafana.proxy }} + upstream grafana { +{{- if .Values.global.grafana.enabled }} +{{- if .Values.kubecostFrontend.useDefaultFqdn }} + server {{ .Release.Name }}-grafana.{{ .Release.Namespace }}.svc.cluster.local; +{{- else }} +{{- if .Values.global.grafana.fqdn }} + server {{ .Values.global.grafana.fqdn }}; +{{- else }} + server {{ .Release.Name }}-grafana.{{ .Release.Namespace }}; +{{- end }} +{{- end }} +{{- else }} + server {{.Values.global.grafana.domainName}}; +{{- end }} + } +{{- end }} + + {{- if .Values.forecasting.enabled }} + upstream forecasting { + {{- if .Values.kubecostFrontend.useDefaultFqdn }} + server {{ .Release.Name }}-forecasting.{{ .Release.Namespace }}.svc.cluster.local:5000; + {{- else }} + {{- if (.Values.kubecostFrontend.forcasting).fqdn }} + server {{ .Values.kubecostFrontend.forcasting.fqdn }}; + {{- else }} + server {{ .Release.Name }}-forecasting.{{ .Release.Namespace }}:5000; + {{- end }} + {{- end }} + } + {{- end }} + + {{- if and (not .Values.agent) (not .Values.cloudAgent) (not (eq (include "aggregator.deployMethod" .) "disabled")) }} + upstream aggregator { + {{- if .Values.kubecostFrontend.useDefaultFqdn }} + server {{ .Release.Name }}-aggregator.{{ .Release.Namespace }}.svc.cluster.local:9004; + {{- else }} + {{- if (.Values.kubecostFrontend.aggregator).fqdn }} + server {{ .Values.kubecostFrontend.aggregator.fqdn }}; + {{- else }} + server {{ .Release.Name }}-aggregator.{{ .Release.Namespace }}:9004; + {{- end }} + {{- end }} + } + upstream cloudCost { + {{- if .Values.kubecostFrontend.useDefaultFqdn }} + server {{ template "cloudCost.serviceName" . }}.{{ .Release.Namespace }}.svc.cluster.local:9005; + {{- else }} + {{- if (.Values.kubecostFrontend.cloudCost).fqdn }} + server {{ .Values.kubecostFrontend.cloudCost.fqdn }}; + {{- else }} + server {{ template "cloudCost.serviceName" . }}.{{ .Release.Namespace }}:9005; + {{- end }} + {{- end }} + } + {{- end }} + + {{- if and .Values.diagnostics.enabled .Values.diagnostics.primary.enabled .Values.diagnostics.deployment.enabled }} + {{- if or (not (empty .Values.kubecostModel.federatedStorageConfigSecret )) .Values.kubecostModel.federatedStorageConfig }} + upstream multi-cluster-diagnostics { + {{- if .Values.kubecostFrontend.useDefaultFqdn }} + server {{ template "diagnostics.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local:9007; + {{- else}} + {{- if (.Values.kubecostFrontend.multiClusterDiagnostics).fqdn }} + server {{ .Values.kubecostFrontend.multiClusterDiagnostics.fqdn }}; + {{- else }} + server {{ template "diagnostics.fullname" . }}.{{ .Release.Namespace }}:9007; + {{- end }} + {{- end }} + } + {{- end }} + {{- end }} + + server { + server_name _; + root /var/www; + index index.html; + + add_header Cache-Control "must-revalidate"; + + {{- if .Values.kubecostFrontend.extraServerConfig }} + {{- .Values.kubecostFrontend.extraServerConfig | toString | nindent 8 -}} + {{- else }} + large_client_header_buffers 4 32k; + {{- end }} + + error_page 504 /custom_504.html; + location = /custom_504.html { + internal; + } + +{{- if or .Values.saml.enabled .Values.oidc.enabled }} + add_header Cache-Control "max-age=0"; + location / { + auth_request /auth; + proxy_redirect off; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + error_page 401 = /login; + try_files $uri $uri/ /index.html; + } + location /healthz { + add_header 'Content-Type' 'text/plain'; + return 200 "healthy\n"; + } +{{- else }} + add_header Cache-Control "max-age=300"; + location / { + try_files $uri $uri/ /index.html; + } +{{- end }} +{{- if .Values.imageVersion }} + add_header ETag "{{ $.Values.imageVersion }}"; +{{- else }} + add_header ETag "{{ $.Chart.Version }}"; +{{- end }} +{{- if .Values.kubecostFrontend.tls }} +{{- if .Values.kubecostFrontend.tls.enabled }} +{{- if .Values.kubecostFrontend.tls.specifyProtocols }} + ssl_protocols {{ $.Values.kubecostFrontend.tls.protocols }}; +{{- end }} + ssl_certificate /etc/ssl/certs/kc.crt; + ssl_certificate_key /etc/ssl/certs/kc.key; + listen {{ .Values.service.targetPort }} ssl; +{{- if .Values.kubecostFrontend.ipv6.enabled }} + listen [::]:{{ .Values.service.targetPort }} ssl; +{{- end }} +{{- else }} + listen {{ .Values.service.targetPort }}; +{{- if .Values.kubecostFrontend.ipv6.enabled }} + listen [::]:{{ .Values.service.targetPort }}; +{{- end }} +{{- end }} +{{- else }} + listen {{ .Values.service.targetPort }}; +{{- if .Values.kubecostFrontend.ipv6.enabled }} + listen [::]:{{ .Values.service.targetPort }}; +{{- end }} +{{- end }} + location /api/ { + {{- if or .Values.saml.enabled .Values.oidc.enabled }} + auth_request /auth; + {{- end }} + proxy_pass http://api/; + proxy_redirect off; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location /model/ { + proxy_connect_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_send_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://model/; + proxy_redirect off; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + {{- if .Values.kubecostFrontend.extraModelConfigs }} + {{- .Values.kubecostFrontend.extraModelConfigs | toString | nindent 12 -}} + {{- end }} + } + + location ~ ^/(turndown|cluster)/ { + + add_header 'Access-Control-Allow-Origin' '*' always; + add_header 'Access-Control-Allow-Methods' 'GET, PUT, POST, DELETE, OPTIONS' always; +{{- if .Values.clusterController }} +{{- if .Values.clusterController.enabled }} + {{- if or .Values.saml .Values.oidc }} + {{- if or .Values.saml.enabled .Values.oidc.enabled }} + auth_request /auth; + {{- else if .Values.saml.rbac.enabled}} + auth_request /authrbac; + {{- end }} + {{- end }} + + rewrite ^/(?:turndown|cluster)/(.*)$ /$1 break; + proxy_pass http://clustercontroller; + proxy_connect_timeout 180; + proxy_send_timeout 180; + proxy_read_timeout 180; + proxy_redirect off; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + +{{- else }} + return 404; +{{- end }} +{{- else }} + return 404; +{{- end }} + } +{{- if and (or .Values.saml.enabled .Values.oidc.enabled) (not (eq (include "aggregator.deployMethod" .) "disabled")) }} + {{- if .Values.oidc.enabled }} + location /oidc/ { + proxy_connect_timeout 180; + proxy_send_timeout 180; + proxy_read_timeout 180; + proxy_pass http://aggregator/oidc/; + proxy_redirect off; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + {{- end }} + {{- if .Values.saml.enabled }} + location /saml/ { + proxy_connect_timeout 180; + proxy_send_timeout 180; + proxy_read_timeout 180; + proxy_pass http://aggregator/saml/; + proxy_redirect off; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + {{- end }} + {{- if or .Values.saml.enabled .Values.oidc.enabled}} + location /login { + proxy_connect_timeout 180; + proxy_send_timeout 180; + proxy_read_timeout 180; + proxy_pass http://aggregator/login; + proxy_redirect off; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Original-URI $request_uri; + } + + location /logout { + proxy_connect_timeout 180; + proxy_send_timeout 180; + proxy_read_timeout 180; + proxy_pass http://aggregator/logout; + proxy_redirect off; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + {{- end }} +{{- end }} + {{- if .Values.global.grafana.proxy }} + location /grafana/ { + {{- if .Values.saml.enabled }} + auth_request /auth; + {{- end }} + proxy_pass http://grafana/; + proxy_redirect off; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $http_host; + } + {{ end }} + {{- if .Values.oidc.enabled }} + location /auth { + proxy_pass http://aggregator/isAuthenticated; + } + {{- end }} + {{- if .Values.saml.enabled }} + location /auth { + proxy_pass http://aggregator/isAuthenticated; + } + {{- if .Values.saml.rbac.enabled }} + location /authrbac { + proxy_pass http://aggregator/isAdminAuthenticated; + } + {{- end }} + {{- end }} + + +{{- if and (not .Values.agent) (not .Values.cloudAgent) (not (eq (include "aggregator.deployMethod" .) "disabled")) }} + # TODO make aggregator route the default, start special-casing + # cost-model APIs + + # Aggregator proxy + {{- if and (.Values.kubecostDeployment) (.Values.kubecostDeployment.queryServiceReplicas) (gt (.Values.kubecostDeployment.queryServiceReplicas | toString | atoi) 0) }} + {{- fail "The Kubecost Aggregator should not be used at the same time as Query Service Replicas" }} + {{- end }} + + location = /model/allocation { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/allocation; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + {{- if not .Values.kubecostFrontend.trendsDisabled }} + location = /model/allocation/trends { + proxy_read_timeout 300; + proxy_pass http://aggregator/allocation/trends; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + {{ end }} + location = /model/allocation/view { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/allocation/view; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/allocation/summary { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/allocation/summary; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/allocation/summary/topline { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/allocation/summary/topline; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/allocation/autocomplete { + proxy_read_timeout 300; + proxy_pass http://aggregator/allocation/autocomplete; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/allocation/carbon { + proxy_read_timeout 300; + proxy_pass http://aggregator/allocation/carbon; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/assets { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/assets; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/assets/topline { + proxy_read_timeout 300; + proxy_pass http://aggregator/assets/topline; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/assets/graph { + proxy_read_timeout 300; + proxy_pass http://aggregator/assets/graph; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/assets/totals { + return 501 "Aggregator does not support this endpoint."; + } + location = /model/assets/diff { + return 501 "Aggregator does not support this endpoint."; + } + location = /model/assets/breakdown { + return 501 "Aggregator does not support this endpoint."; + } + location = /model/assets/autocomplete { + proxy_read_timeout 300; + proxy_pass http://aggregator/assets/autocomplete; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/assets/carbon { + proxy_read_timeout 300; + proxy_pass http://aggregator/assets/carbon; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/savings/requestSizingV2 { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/savings/requestSizingV2; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/savings/requestSizingV2/topline { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/savings/requestSizingV2/topline; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/savings/clusterSizingETL { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/savings/clusterSizingETL; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/savings/nodeGroupSizingETL { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/savings/nodeGroupSizingETL; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/savings/recommendations/allowLists { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/savings/recommendations/allowLists; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location ~* /model/savings/gpuContainersDetails(.*) { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/savings/gpuContainersDetails$1$is_args$args; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location ~* /model/savings/gpuWorkloadUtilization(.*) { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/savings/gpuWorkloadUtilization$1$is_args$args; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + {{- if .Values.global.integrations.turbonomic.enabled }} + location = /model/savings/turbonomic/resizeWorkloadControllers { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/savings/turbonomic/resizeWorkloadControllers; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/savings/turbonomic/suspendContainerPods { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/savings/turbonomic/suspendContainerPods; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/savings/turbonomic/moveContainerPods { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/savings/turbonomic/moveContainerPods; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/savings/turbonomic/suspendVirtualMachines { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/savings/turbonomic/suspendVirtualMachines; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + {{- end }} + location ~* /model/savings/gpuRecommendation(.*) { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/savings/gpuRecommendation$1$is_args$args; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/cloudCost { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/cloudCost; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/cloudCost/view/graph { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/cloudCost/view/graph; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/cloudCost/view/totals { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/cloudCost/view/totals; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/cloudCost/view/table { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/cloudCost/view/table; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/cloudCost/view/trends { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/cloudCost/view/trends; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/cloudCost/autocomplete { + proxy_read_timeout 300; + proxy_pass http://aggregator/cloudCost/autocomplete; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/clusters/status { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/clusters/status; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/savings { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/savings; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/savings/abandonedWorkloads { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/savings/abandonedWorkloads; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/savings/abandonedWorkloads/topline { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/savings/abandonedWorkloads/topline; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/savings/unclaimedVolumes { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/savings/unclaimedVolumes; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/savings/localLowDisks { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/savings/localLowDisks; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/savings/persistentVolumeSizing { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/savings/persistentVolumeSizing; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/savings/persistentVolumeSizing/topline { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/savings/persistentVolumeSizing/topline; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/reports/allocation { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/reports/allocation; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/reports/asset { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/reports/asset; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/reports/cloudCost { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/reports/cloudCost; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/reports/group { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/reports/group; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + # this is a special case to handle /reports/group/:group in the Kubecost Aggregator. prior to aggregator, this endpoint + # was handled by /model/, so no special case proxies were required. without this, /model/reports/groups/?foo=bar + # will be directed to /reports/groups?foo=bar (note the missing /model prefix) + location ~ ^/model/reports/group/ { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/reports/group/$is_args$args; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/budget { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/budget; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/budgets { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/budgets; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/collection { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/collection; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/collections { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/collections; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/collection/query/total { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/collection/query/total; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/collection/query/timeseries { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/collection/query/timeseries; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/collection/query/complement { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/collection/query/complement; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/collection/query/complement/cloud { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/collection/query/complement/cloud; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/collection/query/complement/kubernetes { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/collection/query/complement/kubernetes; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + location = /model/collections/query/total { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/collections/query/total; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/collections/query/timeseries { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/collections/query/timeseries; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/collections/query/complement { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/collections/query/complement; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/collections/query/complement/cloud { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/collections/query/complement/cloud; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/collections/query/complement/kubernetes { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/collections/query/complement/kubernetes; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/collection/cache/status { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/collection/cache/status; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/collection/configs { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/collection/configs; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/kubernetes/containers/resources { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/kubernetes/containers/resources; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/kubernetes/containers/resources/timeseries { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/kubernetes/containers/resources/timeseries; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/kubernetes/containers/costs { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/kubernetes/containers/costs; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/kubernetes/containers/costs/timeseries { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/kubernetes/containers/costs/timeseries; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/networkinsights { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/networkinsights; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/networkinsights/graph { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/networkinsights/graph; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/rbacGroups { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/rbacGroups; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/smtp { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/smtp; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/smtp/test { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/smtp/test; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/teams { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/teams; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/team { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/team; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/users { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/users; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/user { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/user; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/serviceAccounts { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/serviceAccounts; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/serviceAccount { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/serviceAccount; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/debug/orchestrator { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/debug/orchestrator; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/prediction/speccost { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/prediction/speccost; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/diagnostic/coreCount { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/diagnostic/coreCount; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/diagnostic/nodeCount { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/diagnostic/nodeCount; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/diagnostic/tableWindowCount { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/diagnostic/tableWindowCount; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/diagnostic/containersPerDay { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/diagnostic/containersPerDay; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/diagnostic/nodesPerDay { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/diagnostic/nodesPerDay; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/diagnostic/containerLabelStats { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/diagnostic/containerLabelStats; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/diagnostic/containerAnnotationStats { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/diagnostic/containerAnnotationStats; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/diagnostic/cloudCostsPerDay { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/diagnostic/cloudCostsPerDay; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/diagnostic/containerWithoutMatchingNode { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/diagnostic/containerWithoutMatchingNode; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/diagnostic/containerDuplicateNoId { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/diagnostic/containerDuplicateNoId; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/diagnostic/containerDuplicateWithId { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/diagnostic/containerDuplicateWithId; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/diagnostic/nodeDuplicateNoId { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/diagnostic/nodeDuplicateNoId; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/debug/ingestionRecords { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/debug/ingestionRecords; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/debug/ingestionSummary { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/debug/ingestionSummary; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/debug/derivationRecords { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/debug/derivationRecords; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/diagnostic/databaseDirectory { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/diagnostic/databaseDirectory; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/getApiConfig { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/getApiConfig; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/setApiConfig { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/setApiConfig; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/getIngestionConfig { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/getIngestionConfig; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/setIngestionConfig { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/setIngestionConfig; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/enablements { + proxy_read_timeout 300; + proxy_pass http://aggregator/enablements; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/customCost/total { + proxy_read_timeout 300; + proxy_pass http://aggregator/customCost/total; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/customCost/timeseries { + proxy_read_timeout 300; + proxy_pass http://aggregator/customCost/timeseries; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/providerOptimization { + proxy_read_timeout 300; + proxy_pass http://aggregator/providerOptimization; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + location = /model/getProductKey { + proxy_read_timeout 300; + proxy_pass http://aggregator/getProductKey; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/setProductKey { + proxy_read_timeout 300; + proxy_pass http://aggregator/setProductKey; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/trialStatus { + proxy_read_timeout 300; + proxy_pass http://aggregator/trialStatus; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/startProductTrial { + proxy_read_timeout 300; + proxy_pass http://aggregator/startProductTrial; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/resetProductTrial { + proxy_read_timeout 300; + proxy_pass http://aggregator/resetProductTrial; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/extendProductTrial { + proxy_read_timeout 300; + proxy_pass http://aggregator/extendProductTrial; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/expireProductTrial { + proxy_read_timeout 300; + proxy_pass http://aggregator/expireProductTrial; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + #Cloud Cost Endpoints + location = /model/cloudCost/status { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://cloudCost/cloudCost/status; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/cloudCost/rebuild { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://cloudCost/cloudCost/rebuild; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/cloudCost/repair { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://cloudCost/cloudCost/repair; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/cloud/config { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://cloudCost/cloud/config; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/cloud/config/export { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://cloudCost/cloud/config/export; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/cloud/config/enable { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://cloudCost/cloud/config/enable; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/cloud/config/disable { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://cloudCost/cloud/config/disable; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/cloudCost/integration/validate { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://cloudCost/cloudCost/integration/validate; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/customCost/status { + proxy_read_timeout 300; + proxy_pass http://cloudCost/customCost/status; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location = /model/customCost/rebuild { + proxy_read_timeout 300; + proxy_pass http://cloudCost/customCost/rebuild; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + # alert end points with v2 will be routed to aggregator server + location = /model/v2/alerts { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/alerts; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location ~* ^/model/v2/alerts/(.*) { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/alerts/$1; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + # alert end points without any version will be routed to model server + location = /model/alerts { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://model/alerts; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location ~* ^/model/alerts/(.*) { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://model/alerts/$1; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location ~* ^/model/reports/(.*)/schedule/test { + proxy_read_timeout {{ .Values.kubecostFrontend.timeoutSeconds | default 300 }}; + proxy_pass http://aggregator/reports/$1/schedule/test; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } +{{- end }} + location = /model/hideOrphanedResources { + default_type 'application/json'; + add_header 'Access-Control-Allow-Origin' '*' always; + add_header 'Access-Control-Allow-Methods' 'GET, PUT, POST, DELETE, OPTIONS' always; + {{- if .Values.kubecostFrontend.hideOrphanedResources }} + return 200 '{"hideOrphanedResources": "true"}'; + {{- else }} + return 200 '{"hideOrphanedResources": "false"}'; + {{- end }} + } + location = /model/hideDiagnostics { + default_type 'application/json'; + add_header 'Access-Control-Allow-Origin' '*' always; + add_header 'Access-Control-Allow-Methods' 'GET, PUT, POST, DELETE, OPTIONS' always; + {{- if .Values.kubecostFrontend.hideDiagnostics }} + return 200 '{"hideDiagnostics": "true"}'; + {{- else }} + return 200 '{"hideDiagnostics": "false"}'; + {{- end }} + } + + {{- if .Values.kubecostFrontend.trendsDisabled }} + location /model/allocation/trends { + return 204 'endpoint disabled'; + } + {{ end }} + + location /model/multi-cluster-diagnostics-enabled { + default_type 'application/json'; + add_header 'Access-Control-Allow-Origin' '*' always; + add_header 'Access-Control-Allow-Methods' 'GET, PUT, POST, DELETE, OPTIONS' always; + {{- if and .Values.diagnostics.enabled .Values.diagnostics.primary.enabled }} + {{- if or (not (empty .Values.kubecostModel.federatedStorageConfigSecret )) .Values.kubecostModel.federatedStorageConfig }} + return 200 '{"multiClusterDiagnosticsEnabled": true}'; + {{- end }} + {{- else }} + return 200 '{"multiClusterDiagnosticsEnabled": false}'; + {{- end }} + } + + {{- if and .Values.diagnostics.enabled .Values.diagnostics.primary.enabled .Values.diagnostics.deployment.enabled }} + {{- if or (not (empty .Values.kubecostModel.federatedStorageConfigSecret )) .Values.kubecostModel.federatedStorageConfig }} + + # When the Multi-cluster Diagnostics Service is run within the + # cost-model container, its endpoint is available at the path + # `/model/diagnostics/multicluster`. No additional Nginx path forwarding + # needed. When the Multi-cluster Diagnostics Service is run as a K8s + # Deployment, we should forward that path to the K8s Service. + location /model/diagnostics/multicluster { + default_type 'application/json'; + add_header 'Access-Control-Allow-Origin' '*' always; + add_header 'Access-Control-Allow-Methods' 'GET, PUT, POST, DELETE, OPTIONS' always; + proxy_read_timeout 300; + proxy_pass http://multi-cluster-diagnostics/status; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + # simple alias for support + location /mcd { + default_type 'application/json'; + add_header 'Access-Control-Allow-Origin' '*' always; + add_header 'Access-Control-Allow-Methods' 'GET, PUT, POST, DELETE, OPTIONS' always; + proxy_read_timeout 300; + proxy_pass http://multi-cluster-diagnostics/status?window=7d; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + {{- end }} + {{- end }} + + location /model/aggregatorEnabled { + default_type 'application/json'; + return 200 '{"aggregatorEnabled": "true"}'; + } + + {{- if .Values.forecasting.enabled }} + location /forecasting { + default_type 'application/json'; + add_header 'Access-Control-Allow-Origin' '*' always; + add_header 'Access-Control-Allow-Methods' 'GET, PUT, POST, DELETE, OPTIONS' always; + proxy_read_timeout 300; + proxy_pass http://forecasting/; + proxy_redirect off; + proxy_set_header Connection ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + {{- else }} + location /forecasting { + default_type 'application/json'; + return 405 '{"forecastingEnabled": "false"}'; + } + {{- end }} + + location /model/productConfigs { + default_type 'application/json'; + add_header 'Access-Control-Allow-Origin' '*' always; + add_header 'Access-Control-Allow-Methods' 'GET, PUT, POST, DELETE, OPTIONS' always; + return 200 '\n + { + "ssoConfigured": "{{ template "ssoEnabled" . }}", + "rbacTeamsEnabled": "{{ template "rbacTeamsEnabled" . }}", + "dataBackupConfigured": "{{ template "dataBackupConfigured" . }}", + "costEventsAuditEnabled": "{{ template "costEventsAuditEnabled" . }}", + "frontendDeployMethod": "{{ template "frontend.deployMethod" . }}", + "pluginsEnabled": "{{ template "pluginsEnabled" . }}", + "carbonEstimatesEnabled": "{{ template "carbonEstimatesEnabled" . }}", + "clusterControllerEnabled": "{{ template "clusterControllerEnabled" . }}", + "forecastingEnabled": "{{ template "forecastingEnabled" . }}", + "chartVersion": "2.5.3", + "hourlyDataRetention": "{{ (.Values.kubecostAggregator.etlHourlyStoreDurationHours) }}", + "dailyDataRetention": "{{ (.Values.kubecostAggregator.etlDailyStoreDurationDays) }}", + "hideDiagnostics": "{{ default false ((.Values.kubecostProductConfigs).hideDiagnostics) }}", + "hideOrphanedResources": "{{ default false ((.Values.kubecostProductConfigs).hideOrphanedResources) }}", + "hideKubecostActions": "{{ default false ((.Values.kubecostProductConfigs).hideKubecostActions) }}", + "hideReservedInstances": "{{ default false ((.Values.kubecostProductConfigs).hideReservedInstances) }}", + "hideSpotCommander": "{{ default false ((.Values.kubecostProductConfigs).hideSpotCommander) }}", + "hideUnclaimedVolumes": "{{ default false ((.Values.kubecostProductConfigs).hideUnclaimedVolumes) }}", + "hideCloudIntegrationsUI": "{{ default false ((.Values.kubecostProductConfigs).hideCloudIntegrationsUI) }}", + "hideBellIcon": "{{ default false ((.Values.kubecostProductConfigs).hideBellIcon) }}", + "hideTeams": "{{ default false ((.Values.kubecostProductConfigs).hideTeams) }}" + } + '; + } + } + +{{- end }} +{{- end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-ingestion-configmap.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-ingestion-configmap.yaml new file mode 100644 index 0000000000..5283c2f0d1 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-ingestion-configmap.yaml @@ -0,0 +1,14 @@ +{{- if eq (include "aggregator.deployMethod" .) "statefulset" }} +{{- if (.Values.kubecostProductConfigs).standardDiscount }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ default "ingestion-configs" .Values.ingestionConfigmapName }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} +data: + standardDiscount: "{{ .Values.kubecostProductConfigs.standardDiscount }}" + helmConfig: "true" +{{- end -}} +{{- end -}} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-ingress-template.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-ingress-template.yaml new file mode 100644 index 0000000000..4ac0693ddf --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-ingress-template.yaml @@ -0,0 +1,56 @@ +{{- if .Values.ingress -}} +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "cost-analyzer.fullname" . -}} +{{- $serviceName := "" -}} +{{- if eq (include "frontend.deployMethod" .) "haMode" }} +{{- $serviceName = include "frontend.serviceName" . }} +{{- else }} +{{- $serviceName = include "cost-analyzer.serviceName" . -}} +{{- end }} +{{- $ingressPaths := .Values.ingress.paths -}} +{{- $ingressPathType := .Values.ingress.pathType -}} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ $fullName }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} + {{- with .Values.ingress.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: +{{- if .Values.ingress.className }} + ingressClassName: {{ .Values.ingress.className }} +{{- end }} +{{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} +{{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ . | quote }} + http: + paths: + {{- range $ingressPaths }} + - path: {{ . }} + pathType: {{ $ingressPathType }} + backend: + service: + name: {{ $serviceName }} + port: + name: tcp-frontend + {{- end }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-metrics-config-map-template.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-metrics-config-map-template.yaml new file mode 100644 index 0000000000..3fe7ff5eea --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-metrics-config-map-template.yaml @@ -0,0 +1,13 @@ +{{- if .Values.kubecostProductConfigs -}} +{{- if .Values.kubecostProductConfigs.metricsConfigs -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ default "metrics-config" .Values.metricsConfigmapName }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} +data: + metrics.json: '{{ toJson .Values.kubecostProductConfigs.metricsConfigs }}' +{{- end -}} +{{- end -}} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-network-costs-config-map-template.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-network-costs-config-map-template.yaml new file mode 100644 index 0000000000..1ea73aa546 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-network-costs-config-map-template.yaml @@ -0,0 +1,16 @@ +{{- if .Values.networkCosts -}} +{{- if .Values.networkCosts.enabled -}} +{{- if .Values.networkCosts.config -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: network-costs-config + namespace: {{ .Release.Namespace }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} +data: + config.yaml: | +{{- toYaml .Values.networkCosts.config | nindent 4 }} +{{- end -}} +{{- end -}} +{{- end -}} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-network-costs-podmonitor-template.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-network-costs-podmonitor-template.yaml new file mode 100644 index 0000000000..d0b5b5dd8f --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-network-costs-podmonitor-template.yaml @@ -0,0 +1,32 @@ +{{- if .Values.networkCosts }} +{{- if .Values.networkCosts.enabled }} +{{- if .Values.networkCosts.podMonitor }} +{{- if .Values.networkCosts.podMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: {{ include "cost-analyzer.networkCostsName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{ include "cost-analyzer.commonLabels" . | nindent 4 }} + {{- if .Values.networkCosts.podMonitor.additionalLabels }} + {{ toYaml .Values.networkCosts.podMonitor.additionalLabels | nindent 4 }} + {{- end }} +spec: + podMetricsEndpoints: + - port: http-server + honorLabels: true + interval: 1m + scrapeTimeout: 10s + path: /metrics + scheme: http + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} + selector: + matchLabels: + app: {{ template "cost-analyzer.networkCostsName" . }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-network-costs-service-template.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-network-costs-service-template.yaml new file mode 100644 index 0000000000..0ac70718d4 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-network-costs-service-template.yaml @@ -0,0 +1,34 @@ +{{- if .Values.networkCosts }} +{{- if .Values.networkCosts.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "cost-analyzer.networkCostsName" . }} + namespace: {{ .Release.Namespace }} +{{- if (or .Values.networkCosts.service.annotations .Values.networkCosts.prometheusScrape) }} + annotations: +{{- if .Values.networkCosts.service.annotations }} +{{ toYaml .Values.networkCosts.service.annotations | indent 4 }} +{{- end }} +{{- if .Values.networkCosts.prometheusScrape }} + prometheus.io/scrape: "true" + prometheus.io/port: {{ (quote .Values.networkCosts.port) | default (quote 3001) }} +{{- end }} +{{- end }} + labels: + {{- include "networkcosts.commonLabels" . | nindent 4 }} + {{- if .Values.networkCosts.service.labels }} + {{ toYaml .Values.networkCosts.service.labels | nindent 4 }} + {{- end }} +spec: + clusterIP: None + ports: + - name: metrics + port: {{ .Values.networkCosts.port | default 3001 }} + protocol: TCP + targetPort: {{ .Values.networkCosts.port | default 3001 }} + selector: + {{- include "networkcosts.selectorLabels" . | nindent 4 }} + type: ClusterIP +{{- end }} +{{- end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-network-costs-template.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-network-costs-template.yaml new file mode 100644 index 0000000000..ec7e0595dc --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-network-costs-template.yaml @@ -0,0 +1,161 @@ +{{- if .Values.networkCosts -}} +{{- if .Values.networkCosts.enabled -}} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ template "cost-analyzer.networkCostsName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "networkcosts.commonLabels" . | nindent 4 }} + {{- if .Values.networkCosts.additionalLabels }} + {{- toYaml .Values.networkCosts.additionalLabels | nindent 4 }} + {{- end }} + annotations: + {{- with .Values.global.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.networkCosts.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if .Values.networkCosts.updateStrategy }} + updateStrategy: + {{- toYaml .Values.networkCosts.updateStrategy | nindent 4 }} + {{- end }} + selector: + matchLabels: + {{- include "networkcosts.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + {{- with .Values.global.podAnnotations}} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.networkCosts.annotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "networkcosts.commonLabels" . | nindent 8 }} + {{- if .Values.networkCosts.additionalLabels }} + {{- toYaml .Values.networkCosts.additionalLabels | nindent 8 }} + {{- end }} + spec: + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{- range $.Values.imagePullSecrets }} + - name: {{ .name }} + {{- end }} + {{- end }} + hostNetwork: true + serviceAccountName: {{ template "cost-analyzer.serviceAccountName" . }} + containers: + - name: {{ template "cost-analyzer.networkCostsName" . }} + {{- if eq (typeOf .Values.networkCosts.image) "string" }} + image: {{ .Values.networkCosts.image }} + {{- else }} + image: {{ .Values.networkCosts.image.repository }}:{{ .Values.networkCosts.image.tag }} + {{- end}} + {{- if .Values.networkCosts.extraArgs }} + args: + {{- toYaml .Values.networkCosts.extraArgs | nindent 8 }} + {{- end }} +{{- if .Values.networkCosts.imagePullPolicy }} + imagePullPolicy: {{ .Values.networkCosts.imagePullPolicy }} +{{- else }} + imagePullPolicy: Always +{{- end }} +{{- if .Values.networkCosts.resources }} + resources: {{- toYaml .Values.networkCosts.resources | nindent 10 }} +{{- end }} + env: + {{- if .Values.networkCosts.hostProc }} + - name: HOST_PROC + value: {{ .Values.networkCosts.hostProc.mountPath }} + {{- end }} + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: HOST_PORT + value: {{ (quote .Values.networkCosts.port) | default (quote 3001) }} + - name: TRAFFIC_LOGGING_ENABLED + value: {{ (quote .Values.networkCosts.trafficLogging) | default (quote true) }} + - name: LOG_LEVEL + value: {{ .Values.networkCosts.logLevel }} + {{- if .Values.networkCosts.softMemoryLimit }} + - name: GOMEMLIMIT + value: {{ .Values.networkCosts.softMemoryLimit }} + {{- end }} + {{- if .Values.networkCosts.heapMonitor }} + {{- if .Values.networkCosts.heapMonitor.enabled }} + - name: HEAP_MONITOR_ENABLED + value: "true" + - name: HEAP_MONITOR_THRESHOLD + value: {{ .Values.networkCosts.heapMonitor.threshold }} + {{- if .Values.networkCosts.heapMonitor.outFile }} + - name: HEAP_MONITOR_OUTPUT + value: {{ .Values.networkCosts.heapMonitor.outFile }} + {{- end }} + {{- end }} + {{- end }} + {{- if .Values.networkCosts.healthCheckProbes }} + {{- toYaml .Values.networkCosts.healthCheckProbes | nindent 8 }} + {{- end }} + volumeMounts: + {{- if .Values.networkCosts.hostProc }} + - mountPath: {{ .Values.networkCosts.hostProc.mountPath }} + name: host-proc + {{- else }} + - mountPath: /net + name: nf-conntrack + - mountPath: /netfilter + name: netfilter + {{- end }} + {{- if .Values.networkCosts.config }} + - mountPath: /network-costs/config + name: network-costs-config + {{- end }} + securityContext: + privileged: true + {{- if .Values.networkCosts.additionalSecurityContext }} + {{- toYaml .Values.networkCosts.additionalSecurityContext | nindent 10 }} + {{- end }} + ports: + - name: http-server + containerPort: {{ .Values.networkCosts.port | default 3001 }} + hostPort: {{ .Values.networkCosts.port | default 3001 }} +{{- if .Values.networkCosts.priorityClassName }} + priorityClassName: "{{ .Values.networkCosts.priorityClassName }}" +{{- end }} + {{- with .Values.networkCosts.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 10 }} + {{- end }} +{{- if .Values.networkCosts.tolerations }} + tolerations: +{{ toYaml .Values.networkCosts.tolerations | indent 8 }} + {{- end }} + {{- with .Values.networkCosts.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + {{- if .Values.networkCosts.config }} + - name: network-costs-config + configMap: + name: network-costs-config + {{- end }} + {{- if .Values.networkCosts.hostProc }} + - name: host-proc + hostPath: + path: {{ default "/proc" .Values.networkCosts.hostProc.hostPath }} + {{- else }} + - name: nf-conntrack + hostPath: + path: /proc/net + - name: netfilter + hostPath: + path: /proc/sys/net/netfilter + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-networks-costs-ocp-scc.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-networks-costs-ocp-scc.yaml new file mode 100644 index 0000000000..8602cb0c61 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-networks-costs-ocp-scc.yaml @@ -0,0 +1,30 @@ +{{- if and (.Capabilities.APIVersions.Has "security.openshift.io/v1/SecurityContextConstraints") (.Values.global.platforms.openshift.scc.networkCosts) (.Values.networkCosts.enabled) }} +apiVersion: security.openshift.io/v1 +kind: SecurityContextConstraints +metadata: + name: {{ template "cost-analyzer.networkCostsName" . }} +priority: 10 +allowPrivilegedContainer: true +allowHostDirVolumePlugin: true +allowHostNetwork: true +allowHostPorts: true +allowHostPID: false +allowHostIPC: false +readOnlyRootFilesystem: false +runAsUser: + type: RunAsAny +fsGroup: + type: RunAsAny +seLinuxContext: + type: RunAsAny +supplementalGroups: + type: RunAsAny +seccompProfiles: +- runtime/default +volumes: + - hostPath + - projected + - configMap +users: + - system:serviceaccount:{{ .Release.Namespace }}:{{ template "cost-analyzer.serviceAccountName" . }} +{{- end }} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-ocp-route.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-ocp-route.yaml new file mode 100644 index 0000000000..3438dcd546 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-ocp-route.yaml @@ -0,0 +1,25 @@ +{{- if and (.Capabilities.APIVersions.Has "route.openshift.io/v1/Route") (.Values.global.platforms.openshift.enabled) (.Values.global.platforms.openshift.route.enabled) }} +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: {{ template "cost-analyzer.fullname" . }}-route + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} + {{- with .Values.global.platforms.openshift.route.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if .Values.global.platforms.openshift.route.host }} + host: "{{ .Values.global.platforms.openshift.route.host }}" + {{- end }} + port: + targetPort: tcp-frontend + tls: + termination: edge + to: + kind: Service + name: {{ template "cost-analyzer.serviceName" . }} + weight: 100 + wildcardPolicy: None +{{- end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-oidc-config-map-template.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-oidc-config-map-template.yaml new file mode 100644 index 0000000000..9f6461c56f --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-oidc-config-map-template.yaml @@ -0,0 +1,49 @@ +{{- if .Values.oidc }} +{{- if .Values.oidc.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "cost-analyzer.fullname" . }}-oidc + namespace: {{ .Release.Namespace }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} +data: +{{- $root := . }} + oidc.json: |- + { + "enabled" : {{ .Values.oidc.enabled }}, + "useIDToken" : {{ .Values.oidc.useIDToken | default "false" }}, + "clientID" : "{{ .Values.oidc.clientID }}", + {{- if .Values.oidc.existingCustomSecret.enabled }} + "secretName" : "{{ .Values.oidc.existingCustomSecret.name }}", + {{- else }} + "secretName" : "{{ .Values.oidc.secretName }}", + {{- end }} + "authURL" : "{{ .Values.oidc.authURL }}", + "loginRedirectURL" : "{{ .Values.oidc.loginRedirectURL }}", + "discoveryURL" : "{{ .Values.oidc.discoveryURL }}", + "hostedDomain" : "{{ .Values.oidc.hostedDomain }}", + "skipOnlineTokenValidation" : "{{ .Values.oidc.skipOnlineTokenValidation | default "false" }}", + "useClientSecretPost": {{ .Values.oidc.useClientSecretPost }}, + "rbac" : { + "enabled" : {{ .Values.oidc.rbac.enabled }}, + "groups" : [ + {{- range $i, $g := .Values.oidc.rbac.groups }} + {{- if ne $i 0 }},{{- end }} + { + "roleName": "{{ $g.name }}", + "enabled": {{ $g.enabled }}, + "claimName": "{{ $g.claimName }}", + "claimValues": [ + {{- range $j, $v := $g.claimValues }} + {{- if ne $j 0 }},{{- end }} + "{{ $v }}" + {{- end }} + ] + } + {{- end }} + ] + } + } +{{- end -}} +{{- end -}} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-pkey-configmap.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-pkey-configmap.yaml new file mode 100644 index 0000000000..9afdb7f73d --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-pkey-configmap.yaml @@ -0,0 +1,23 @@ +{{- if .Values.kubecostProductConfigs }} +{{- if .Values.kubecostProductConfigs.productKey }} +{{- if .Values.kubecostProductConfigs.productKey.enabled }} +# If the productKey.key is not specified, the configmap will not be created +{{- if .Values.kubecostProductConfigs.productKey.key }} +# If the secretname is specified, the configmap will not be created +{{- if not .Values.kubecostProductConfigs.productKey.secretname }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ default "product-configs" .Values.productConfigmapName }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} +data: + {{- if .Values.kubecostProductConfigs.productKey.key }} + key: {{ .Values.kubecostProductConfigs.productKey.key | quote }} + {{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-pricing-configmap.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-pricing-configmap.yaml new file mode 100644 index 0000000000..9ca6ba33c9 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-pricing-configmap.yaml @@ -0,0 +1,141 @@ +{{- if .Values.kubecostProductConfigs }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ default "pricing-configs" .Values.pricingConfigmapName }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} +data: + {{- if .Values.kubecostProductConfigs.defaultModelPricing }} + {{- if .Values.kubecostProductConfigs.defaultModelPricing.enabled }} + {{- if .Values.kubecostProductConfigs.customPricesEnabled }} + customPricesEnabled: "{{ .Values.kubecostProductConfigs.customPricesEnabled }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.defaultModelPricing.CPU }} + CPU: "{{ .Values.kubecostProductConfigs.defaultModelPricing.CPU | toString }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.defaultModelPricing.spotCPU }} + spotCPU: "{{ .Values.kubecostProductConfigs.defaultModelPricing.spotCPU | toString }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.defaultModelPricing.RAM }} + RAM: "{{ .Values.kubecostProductConfigs.defaultModelPricing.RAM | toString }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.defaultModelPricing.spotRAM }} + spotRAM: "{{ .Values.kubecostProductConfigs.defaultModelPricing.spotRAM | toString }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.defaultModelPricing.GPU }} + GPU: "{{ .Values.kubecostProductConfigs.defaultModelPricing.GPU | toString }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.defaultModelPricing.spotGPU }} + spotGPU: "{{ .Values.kubecostProductConfigs.defaultModelPricing.spotGPU | toString }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.defaultModelPricing.storage }} + storage: "{{ .Values.kubecostProductConfigs.defaultModelPricing.storage | toString }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.defaultModelPricing.zoneNetworkEgress }} + zoneNetworkEgress: "{{ .Values.kubecostProductConfigs.defaultModelPricing.zoneNetworkEgress | toString }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.defaultModelPricing.regionNetworkEgress }} + regionNetworkEgress: "{{ .Values.kubecostProductConfigs.defaultModelPricing.regionNetworkEgress | toString }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.defaultModelPricing.internetNetworkEgress }} + internetNetworkEgress: "{{ .Values.kubecostProductConfigs.defaultModelPricing.internetNetworkEgress | toString }}" + {{- end -}} + {{- end -}} + {{- end -}} + {{- if .Values.kubecostProductConfigs.clusterName }} + clusterName: "{{ .Values.kubecostProductConfigs.clusterName }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.clusterAccountID }} + clusterAccountID: "{{ .Values.kubecostProductConfigs.clusterAccountID }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.currencyCode }} + currencyCode: "{{ .Values.kubecostProductConfigs.currencyCode }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.azureBillingRegion }} + azureBillingRegion: "{{ .Values.kubecostProductConfigs.azureBillingRegion }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.azureSubscriptionID }} + azureSubscriptionID: "{{ .Values.kubecostProductConfigs.azureSubscriptionID }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.azureClientID }} + azureClientID: "{{ .Values.kubecostProductConfigs.azureClientID }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.azureTenantID }} + azureTenantID: "{{ .Values.kubecostProductConfigs.azureTenantID }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.azureOfferDurableID }} + azureOfferDurableID: "{{ .Values.kubecostProductConfigs.azureOfferDurableID }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.discount }} + discount: "{{ .Values.kubecostProductConfigs.discount }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.negotiatedDiscount }} + negotiatedDiscount: "{{ .Values.kubecostProductConfigs.negotiatedDiscount }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.defaultIdle }} + defaultIdle: "{{ .Values.kubecostProductConfigs.defaultIdle }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.sharedNamespaces }} + sharedNamespaces: "{{ .Values.kubecostProductConfigs.sharedNamespaces }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.sharedOverhead }} + sharedOverhead: "{{ .Values.kubecostProductConfigs.sharedOverhead }}" + {{- end -}} + {{- if gt (len (toString .Values.kubecostProductConfigs.shareTenancyCosts)) 0 }} + {{- if eq (toString .Values.kubecostProductConfigs.shareTenancyCosts) "false" }} + shareTenancyCosts: "false" + {{- else if eq (toString .Values.kubecostProductConfigs.shareTenancyCosts) "true" }} + shareTenancyCosts: "true" + {{- end -}} + {{- end -}} + {{- if .Values.kubecostProductConfigs.spotLabel }} + spotLabel: "{{ .Values.kubecostProductConfigs.spotLabel }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.spotLabelValue }} + spotLabelValue: "{{ .Values.kubecostProductConfigs.spotLabelValue }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.awsSpotDataRegion }} + spotDataRegion: "{{ .Values.kubecostProductConfigs.awsSpotDataRegion }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.awsSpotDataBucket }} + spotDataBucket: "{{ .Values.kubecostProductConfigs.awsSpotDataBucket }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.awsSpotDataPrefix }} + spotDataPrefix: "{{ .Values.kubecostProductConfigs.awsSpotDataPrefix }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.projectID }} + projectID: "{{ .Values.kubecostProductConfigs.projectID }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.bigQueryBillingDataDataset }} + billingDataDataset: "{{ .Values.kubecostProductConfigs.bigQueryBillingDataDataset }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.athenaProjectID }} + athenaProjectID: "{{ .Values.kubecostProductConfigs.athenaProjectID }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.athenaBucketName }} + athenaBucketName: "{{ .Values.kubecostProductConfigs.athenaBucketName }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.athenaRegion }} + athenaRegion: "{{ .Values.kubecostProductConfigs.athenaRegion }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.athenaDatabase }} + athenaDatabase: "{{ .Values.kubecostProductConfigs.athenaDatabase }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.athenaTable }} + athenaTable: "{{ .Values.kubecostProductConfigs.athenaTable }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.athenaWorkgroup }} + athenaWorkgroup: "{{ .Values.kubecostProductConfigs.athenaWorkgroup }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.masterPayerARN}} + masterPayerARN: "{{ .Values.kubecostProductConfigs.masterPayerARN }}" + {{- end }} + {{- if .Values.kubecostProductConfigs.gpuLabel }} + gpuLabel: "{{ .Values.kubecostProductConfigs.gpuLabel }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.gpuLabelValue }} + gpuLabelValue: "{{ .Values.kubecostProductConfigs.gpuLabelValue }}" + {{- end -}} +{{- end -}} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-prometheusrule-template.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-prometheusrule-template.yaml new file mode 100644 index 0000000000..28afac4d0b --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-prometheusrule-template.yaml @@ -0,0 +1,22 @@ +{{- if .Values.prometheus }} +{{- if .Values.prometheus.serverFiles }} +{{- if .Values.prometheus.serverFiles.rules }} +{{- if .Values.prometheusRule }} +{{- if .Values.prometheusRule.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ include "cost-analyzer.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} + {{- if .Values.prometheusRule.additionalLabels }} + {{ toYaml .Values.prometheusRule.additionalLabels | nindent 4 }} + {{- end }} +spec: + {{ toYaml .Values.prometheus.serverFiles.rules | nindent 2 }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-pvc-template.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-pvc-template.yaml new file mode 100644 index 0000000000..002ea3c51e --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-pvc-template.yaml @@ -0,0 +1,33 @@ +{{- if .Values.persistentVolume -}} +{{- if not .Values.persistentVolume.existingClaim -}} +{{- if .Values.persistentVolume.enabled -}} +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: {{ template "cost-analyzer.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} + {{- with .Values.persistentVolume.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.persistentVolume.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + accessModes: + - ReadWriteOnce + {{- if .Values.persistentVolume.storageClass }} + storageClassName: {{ .Values.persistentVolume.storageClass }} + {{ end }} + resources: + requests: + {{- if .Values.persistentVolume }} + storage: {{ .Values.persistentVolume.size }} + {{- else }} + storage: 32.0Gi + {{ end }} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-saml-config-map-template.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-saml-config-map-template.yaml new file mode 100644 index 0000000000..df5dbb32e2 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-saml-config-map-template.yaml @@ -0,0 +1,14 @@ +{{- if .Values.saml }} +{{- if .Values.saml.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "cost-analyzer.fullname" . }}-saml + namespace: {{ .Release.Namespace }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} +data: +{{- $root := . }} + saml.json: '{{ toJson .Values.saml }}' +{{- end -}} +{{- end -}} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-saved-reports-configmap.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-saved-reports-configmap.yaml new file mode 100644 index 0000000000..a04c6ea6d3 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-saved-reports-configmap.yaml @@ -0,0 +1,13 @@ +{{- if .Values.global.savedReports }} +{{- if .Values.global.savedReports.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{default "saved-report-configs" .Values.savedReportConfigmapName }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} +data: + saved-reports.json: '{{ toJson .Values.global.savedReports.reports }}' +{{- end -}} +{{- end -}} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-server-configmap.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-server-configmap.yaml new file mode 100644 index 0000000000..c310eb6299 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-server-configmap.yaml @@ -0,0 +1,73 @@ +{{- if .Values.kubecostProductConfigs }} +{{- if or .Values.kubecostProductConfigs.grafanaURL .Values.kubecostProductConfigs.labelMappingConfigs .Values.kubecostProductConfigs.cloudAccountMapping}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ default "app-configs" .Values.appConfigmapName }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} +data: +{{- if .Values.kubecostProductConfigs.labelMappingConfigs }} +{{- if .Values.kubecostProductConfigs.labelMappingConfigs.enabled }} + {{- if .Values.kubecostProductConfigs.labelMappingConfigs.owner_label }} + owner_label: "{{ .Values.kubecostProductConfigs.labelMappingConfigs.owner_label }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.labelMappingConfigs.team_label }} + team_label: "{{ .Values.kubecostProductConfigs.labelMappingConfigs.team_label }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.labelMappingConfigs.department_label }} + department_label: "{{ .Values.kubecostProductConfigs.labelMappingConfigs.department_label }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.labelMappingConfigs.product_label }} + product_label: "{{ .Values.kubecostProductConfigs.labelMappingConfigs.product_label }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.labelMappingConfigs.environment_label }} + environment_label: "{{ .Values.kubecostProductConfigs.labelMappingConfigs.environment_label }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.labelMappingConfigs.namespace_external_label }} + namespace_external_label: "{{ .Values.kubecostProductConfigs.labelMappingConfigs.namespace_external_label }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.labelMappingConfigs.cluster_external_label }} + cluster_external_label: "{{ .Values.kubecostProductConfigs.labelMappingConfigs.cluster_external_label }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.labelMappingConfigs.controller_external_label }} + controller_external_label: "{{ .Values.kubecostProductConfigs.labelMappingConfigs.controller_external_label }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.labelMappingConfigs.product_external_label }} + product_external_label: "{{ .Values.kubecostProductConfigs.labelMappingConfigs.product_external_label }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.labelMappingConfigs.service_external_label }} + service_external_label: "{{ .Values.kubecostProductConfigs.labelMappingConfigs.service_external_label }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.labelMappingConfigs.deployment_external_label }} + deployment_external_label: "{{ .Values.kubecostProductConfigs.labelMappingConfigs.deployment_external_label }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.labelMappingConfigs.team_external_label }} + team_external_label: "{{ .Values.kubecostProductConfigs.labelMappingConfigs.team_external_label }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.labelMappingConfigs.environment_external_label }} + environment_external_label: "{{ .Values.kubecostProductConfigs.labelMappingConfigs.environment_external_label }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.labelMappingConfigs.department_external_label }} + department_external_label: "{{ .Values.kubecostProductConfigs.labelMappingConfigs.department_external_label }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.labelMappingConfigs.statefulset_external_label }} + statefulset_external_label: "{{ .Values.kubecostProductConfigs.labelMappingConfigs.statefulset_external_label }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.labelMappingConfigs.daemonset_external_label }} + daemonset_external_label: "{{ .Values.kubecostProductConfigs.labelMappingConfigs.daemonset_external_label }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.labelMappingConfigs.pod_external_label }} + pod_external_label: "{{ .Values.kubecostProductConfigs.labelMappingConfigs.pod_external_label }}" + {{- end -}} + {{- if .Values.kubecostProductConfigs.labelMappingConfigs.owner_external_label }} + owner_external_label: "{{ .Values.kubecostProductConfigs.labelMappingConfigs.owner_external_label }}" + {{- end -}} +{{- end -}} +{{- end -}} + {{- if .Values.kubecostProductConfigs.grafanaURL }} + grafanaURL: "{{ .Values.kubecostProductConfigs.grafanaURL }}" + {{- end -}} +{{- end -}} +{{- end -}} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-service-account-template.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-service-account-template.yaml new file mode 100644 index 0000000000..1aaa628afd --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-service-account-template.yaml @@ -0,0 +1,13 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "cost-analyzer.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} +{{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} +{{- end }} +{{- end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-service-template.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-service-template.yaml new file mode 100644 index 0000000000..82d957fca7 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-service-template.yaml @@ -0,0 +1,66 @@ +{{- if and (not .Values.agent) (not .Values.cloudAgent) }} +kind: Service +apiVersion: v1 +metadata: + name: {{ template "cost-analyzer.serviceName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} +{{- if .Values.service.labels }} +{{ toYaml .Values.service.labels | indent 4 }} +{{- end }} +{{- if .Values.service.annotations }} + annotations: +{{ toYaml .Values.service.annotations | indent 4 }} +{{- end }} +spec: + selector: + {{- include "cost-analyzer.selectorLabels" . | nindent 4 }} +{{- if .Values.service -}} +{{- if .Values.service.type }} + type: "{{ .Values.service.type }}" +{{- else }} + type: ClusterIP +{{- end }} +{{- else }} + type: ClusterIP +{{- end }} +{{- if (eq .Values.service.type "LoadBalancer") }} + {{- if .Values.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: +{{ toYaml .Values.service.loadBalancerSourceRanges | indent 4 }} + {{- end -}} +{{- end }} + ports: + - name: tcp-model + port: 9003 + targetPort: 9003 + {{- with .Values.kubecostModel.extraPorts }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if and (.Values.kubecostFrontend.enabled) (not (eq (include "frontend.deployMethod" .) "haMode")) }} + - name: tcp-frontend + {{- if (eq .Values.service.type "NodePort") }} + {{- if .Values.service.nodePort }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + {{- end }} + port: {{ .Values.service.port }} + targetPort: {{ .Values.service.targetPort }} + {{- end }} + {{- if or .Values.saml.enabled .Values.oidc.enabled}} + - name: apiserver + port: 9007 + targetPort: 9007 + {{- end }} +{{- if .Values.service.sessionAffinity.enabled }} + sessionAffinity: ClientIP + {{- if .Values.service.sessionAffinity.timeoutSeconds }} + sessionAffinityConfig: + clientIP: + timeoutSeconds: {{ .Values.service.sessionAffinity.timeoutSeconds }} + {{- end }} +{{- else }} + sessionAffinity: None +{{- end }} +{{- end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-servicemonitor-template.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-servicemonitor-template.yaml new file mode 100644 index 0000000000..3fece145db --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-servicemonitor-template.yaml @@ -0,0 +1,34 @@ +{{- if .Values.serviceMonitor }} +{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "cost-analyzer.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} + {{- if .Values.serviceMonitor.additionalLabels }} + {{ toYaml .Values.serviceMonitor.additionalLabels | nindent 4 }} + {{- end }} +spec: + endpoints: + - port: tcp-model + honorLabels: true + interval: {{ .Values.serviceMonitor.interval }} + scrapeTimeout: {{ .Values.serviceMonitor.scrapeTimeout }} + path: /metrics + scheme: http + {{- with .Values.serviceMonitor.metricRelabelings }} + metricRelabelings: {{ toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.serviceMonitor.relabelings }} + relabelings: {{ toYaml . | nindent 8 }} + {{- end }} + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} + selector: + matchLabels: + {{- include "cost-analyzer.selectorLabels" . | nindent 6 }} +{{- end }} +{{- end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-smtp-configmap.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-smtp-configmap.yaml new file mode 100644 index 0000000000..fd00091ce3 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/cost-analyzer-smtp-configmap.yaml @@ -0,0 +1,12 @@ + +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ default "smtp-configs" .Values.smtpConfigmapName }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} +{{- if (((.Values.kubecostProductConfigs).smtp).config) }} +data: + config: {{ .Values.kubecostProductConfigs.smtp.config | quote }} +{{- end -}} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/diagnostics-deployment.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/diagnostics-deployment.yaml new file mode 100644 index 0000000000..12ff95a70f --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/diagnostics-deployment.yaml @@ -0,0 +1,183 @@ +{{- if and .Values.diagnostics.enabled .Values.diagnostics.deployment.enabled }} +{{- if or (not (empty .Values.kubecostModel.federatedStorageConfigSecret )) .Values.kubecostModel.federatedStorageConfig -}} + +{{- if eq .Values.prometheus.server.global.external_labels.cluster_id "cluster-one" }} +{{- fail "Error: The 'cluster_id' is set to default 'cluster-one'. Please update so that the diagnostics service can uniquely identify data coming from this cluster." }} +{{- end }} + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "diagnostics.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "diagnostics.selectorLabels" . | nindent 4 }} + {{- with .Values.global.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if .Values.diagnostics.deployment.labels }} + {{- toYaml .Values.diagnostics.deployment.labels | nindent 4 }} + {{- end }} + annotations: + {{- with .Values.global.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.diagnostics.deployment.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: 1 + selector: + matchLabels: + {{- include "diagnostics.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "diagnostics.selectorLabels" . | nindent 8 }} + {{- with .Values.global.additionalLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + {{- with .Values.global.podAnnotations}} + {{- toYaml . | nindent 8 }} + {{- end }} + checksum/configs: {{ include "configsChecksum" . }} + spec: + restartPolicy: Always + {{- if .Values.diagnostics.deployment.securityContext }} + securityContext: + {{- toYaml .Values.diagnostics.deployment.securityContext | nindent 8 }} + {{- else if .Values.global.securityContext }} + securityContext: + {{- toYaml .Values.global.securityContext | nindent 8 }} + {{- end }} + serviceAccountName: {{ template "cost-analyzer.serviceAccountName" . }} + volumes: + {{- if or .Values.kubecostModel.federatedStorageConfigSecret .Values.kubecostModel.federatedStorageConfig }} + - name: federated-storage-config + secret: + defaultMode: 420 + secretName: {{ .Values.kubecostModel.federatedStorageConfigSecret | default "federated-store" }} + {{- end }} + - name: config-db + {{- /* #TODO: make pv? */}} + emptyDir: {} + containers: + - name: diagnostics + args: ["diagnostics"] + {{- if .Values.kubecostModel }} + {{- if .Values.kubecostModel.fullImageName }} + image: {{ .Values.kubecostModel.fullImageName }} + {{- else if .Values.imageVersion }} + image: {{ .Values.kubecostModel.image }}:{{ .Values.imageVersion }} + {{- else if eq "development" .Chart.AppVersion }} + image: gcr.io/kubecost1/cost-model-nightly:latest + {{- else }} + image: {{ .Values.kubecostModel.image }}:prod-{{ $.Chart.AppVersion }} + {{- end }} + {{- else }} + image: gcr.io/kubecost1/cost-model:prod-{{ $.Chart.AppVersion }} + {{- end }} + {{- if .Values.kubecostModel.imagePullPolicy }} + imagePullPolicy: {{ .Values.kubecostModel.imagePullPolicy }} + {{- else }} + imagePullPolicy: Always + {{- end }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{- range $.Values.imagePullSecrets }} + - name: {{ .name }} + {{- end }} + {{- end }} + {{- if .Values.diagnostics.deployment.containerSecurityContext }} + securityContext: + {{- toYaml .Values.diagnostics.deployment.containerSecurityContext | nindent 12 }} + {{- else if .Values.global.containerSecurityContext }} + securityContext: + {{- toYaml .Values.global.containerSecurityContext | nindent 12 }} + {{- end }} + volumeMounts: + - name: config-db + mountPath: /var/configs/db + readOnly: false + - name: federated-storage-config + mountPath: /var/configs/etl + readOnly: true + env: + {{- if and (.Values.prometheus.server.global.external_labels.cluster_id) (not .Values.prometheus.server.clusterIDConfigmap) }} + - name: CLUSTER_ID + value: {{ .Values.prometheus.server.global.external_labels.cluster_id }} + {{- end }} + {{- if .Values.prometheus.server.clusterIDConfigmap }} + - name: CLUSTER_ID + valueFrom: + configMapKeyRef: + name: {{ .Values.prometheus.server.clusterIDConfigmap }} + key: CLUSTER_ID + {{- end }} + - name: FEDERATED_STORE_CONFIG + value: /var/configs/etl/federated-store.yaml + - name: DIAGNOSTICS_RUN_IN_COST_MODEL + value: "false" + - name: DIAGNOSTICS_KUBECOST_FQDN + value: {{ template "cost-analyzer.serviceName" . }} + - name: DIAGNOSTICS_KUBECOST_NAMESPACE + value: {{ .Release.Namespace }} + - name: DIAGNOSTICS_PRIMARY + value: {{ quote .Values.diagnostics.primary.enabled }} + - name: DIAGNOSTICS_RETENTION + value: {{ .Values.diagnostics.primary.retention }} + - name: DIAGNOSTICS_PRIMARY_READONLY + value: {{ quote .Values.diagnostics.primary.readonly }} + - name: DIAGNOSTICS_POLLING_INTERVAL + value: {{ .Values.diagnostics.pollingInterval }} + - name: DIAGNOSTICS_KEEP_HISTORY + value: {{ quote .Values.diagnostics.keepDiagnosticHistory }} + - name: DIAGNOSTICS_COLLECT_HELM_VALUES + value: {{ quote .Values.diagnostics.collectHelmValues }} + {{- if .Values.systemProxy.enabled }} + - name: HTTP_PROXY + value: {{ .Values.systemProxy.httpProxyUrl }} + - name: http_proxy + value: {{ .Values.systemProxy.httpProxyUrl }} + - name: HTTPS_PROXY + value: {{ .Values.systemProxy.httpsProxyUrl }} + - name: https_proxy + value: {{ .Values.systemProxy.httpsProxyUrl }} + - name: NO_PROXY + value: {{ .Values.systemProxy.noProxy }} + - name: no_proxy + value: {{ .Values.systemProxy.noProxy }} + {{- end }} + {{- range $key, $value := .Values.diagnostics.deployment.env }} + - name: {{ $key | quote }} + value: {{ $value | quote }} + {{- end }} + {{- /* TODO: heatlhcheck that validates the diagnotics pod is healthy */}} + {{- if .Values.diagnostics.primary.enabled}} + readinessProbe: + httpGet: + path: /healthz + port: 9007 + ports: + - name: diagnostics-api + containerPort: 9007 + protocol: TCP + {{- end }} + resources: + {{- toYaml .Values.diagnostics.deployment.resources | nindent 12 }} + {{- with .Values.diagnostics.deployment.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.diagnostics.deployment.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.diagnostics.deployment.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + +{{- end }} +{{- end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/diagnostics-service.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/diagnostics-service.yaml new file mode 100644 index 0000000000..deb67bce65 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/diagnostics-service.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.diagnostics.enabled .Values.diagnostics.deployment.enabled .Values.diagnostics.primary.enabled }} +{{- if or (not (empty .Values.kubecostModel.federatedStorageConfigSecret )) .Values.kubecostModel.federatedStorageConfig -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "diagnostics.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "diagnostics.selectorLabels" . | nindent 4 }} +spec: + ports: + - name: diagnostics-api + protocol: TCP + port: 9007 + targetPort: diagnostics-api + selector: + {{- include "diagnostics.selectorLabels" . | nindent 4 }} + type: ClusterIP +{{- end }} +{{- end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/etl-utils-deployment.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/etl-utils-deployment.yaml new file mode 100644 index 0000000000..0bd71c4a51 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/etl-utils-deployment.yaml @@ -0,0 +1,129 @@ +{{- if .Values.etlUtils.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "etlUtils.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "etlUtils.commonLabels" . | nindent 4 }} + {{- with .Values.global.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- with .Values.global.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.etlUtils.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: 1 + selector: + matchLabels: + {{- include "etlUtils.selectorLabels" . | nindent 6 }} + strategy: + type: Recreate + template: + metadata: + labels: + app.kubernetes.io/name: {{ template "etlUtils.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app: {{ template "etlUtils.name" . }} + {{- with .Values.global.additionalLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + {{- with .Values.global.podAnnotations}} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + restartPolicy: Always + volumes: + {{- if .Values.etlUtils.thanosSourceBucketSecret }} + - name: etl-bucket-config + secret: + defaultMode: 420 + secretName: {{ .Values.etlUtils.thanosSourceBucketSecret }} + {{- end }} + {{- if or .Values.kubecostModel.federatedStorageConfigSecret .Values.kubecostModel.federatedStorageConfig}} + - name: federated-storage-config + secret: + defaultMode: 420 + secretName: {{ .Values.kubecostModel.federatedStorageConfigSecret | default "federated-store" }} + {{- end }} + serviceAccountName: {{ template "cost-analyzer.serviceAccountName" . }} + containers: + - name: {{ template "etlUtils.name" . }} + {{- if .Values.kubecostModel }} + {{- if .Values.etlUtils.fullImageName }} + image: {{ .Values.etlUtils.fullImageName }} + {{- else if .Values.kubecostModel.fullImageName }} + image: {{ .Values.kubecostModel.fullImageName }} + {{- else if .Values.imageVersion }} + image: {{ .Values.kubecostModel.image }}:{{ .Values.imageVersion }} + {{- else if eq "development" .Chart.AppVersion }} + image: gcr.io/kubecost1/cost-model-nightly:latest + {{- else }} + image: {{ .Values.kubecostModel.image }}:prod-{{ $.Chart.AppVersion }} + {{ end }} + {{- else }} + image: gcr.io/kubecost1/cost-model:prod-{{ $.Chart.AppVersion }} + {{ end }} + readinessProbe: + httpGet: + path: /healthz + port: 9006 + initialDelaySeconds: 10 + periodSeconds: 5 + failureThreshold: 200 + livenessProbe: + httpGet: + path: /healthz + port: 9006 + initialDelaySeconds: 10 + periodSeconds: 5 + imagePullPolicy: Always + args: ["etl-utils"] + ports: + - name: api + containerPort: 9006 + protocol: TCP + resources: + {{- toYaml .Values.etlUtils.resources | nindent 12 }} + volumeMounts: + {{- if .Values.etlUtils.thanosSourceBucketSecret }} + - name: etl-bucket-config + mountPath: /var/configs/etl + readOnly: true + {{- end }} + env: + - name: CONFIG_PATH + value: /var/configs/ + {{- if .Values.etlUtils.thanosSourceBucketSecret }} + - name: ETL_BUCKET_CONFIG + value: "/var/configs/etl/object-store.yaml" + {{- end}} + {{- range $key, $value := .Values.etlUtils.env }} + - name: {{ $key | quote }} + value: {{ $value | quote }} + {{- end }} + + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{- range $.Values.imagePullSecrets }} + - name: {{ .name }} + {{- end }} + {{- end }} + {{- with .Values.etlUtils.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.etlUtils.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.etlUtils.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/etl-utils-service.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/etl-utils-service.yaml new file mode 100644 index 0000000000..70148d77d3 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/etl-utils-service.yaml @@ -0,0 +1,17 @@ +{{- if .Values.etlUtils.enabled }} +kind: Service +apiVersion: v1 +metadata: + name: {{ template "etlUtils.serviceName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "etlUtils.commonLabels" . | nindent 4 }} +spec: + selector: + {{- include "etlUtils.selectorLabels" . | nindent 4 }} + type: "ClusterIP" + ports: + - name: api + port: 9006 + targetPort: 9006 +{{- end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/external-grafana-config-map-template.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/external-grafana-config-map-template.yaml new file mode 100644 index 0000000000..b185a9dcf8 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/external-grafana-config-map-template.yaml @@ -0,0 +1,11 @@ +{{- if eq .Values.global.grafana.proxy false -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: external-grafana-config-map + namespace: {{ .Release.Namespace }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} +data: + grafanaURL: {{ .Values.global.grafana.scheme | default "http" }}://{{- .Values.global.grafana.domainName }} +{{- end -}} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/extra-manifests.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/extra-manifests.yaml new file mode 100644 index 0000000000..edad397d93 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/extra-manifests.yaml @@ -0,0 +1,8 @@ +{{ range .Values.extraObjects }} +--- +{{- if typeIs "string" . }} + {{- tpl . $ }} +{{- else }} + {{- tpl (toYaml .) $ }} +{{- end }} +{{ end }} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/federated-store-secret.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/federated-store-secret.yaml new file mode 100644 index 0000000000..8119f0b51f --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/federated-store-secret.yaml @@ -0,0 +1,12 @@ +{{- if .Values.kubecostModel.federatedStorageConfig -}} +apiVersion: v1 +kind: Secret +type: Opaque +metadata: + name: federated-store + namespace: {{ .Release.Namespace }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} +data: + federated-store.yaml: {{ .Values.kubecostModel.federatedStorageConfig | b64enc }} +{{- end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/forecasting-deployment.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/forecasting-deployment.yaml new file mode 100644 index 0000000000..04e7fb85a3 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/forecasting-deployment.yaml @@ -0,0 +1,154 @@ +{{- if and .Values.forecasting.enabled (not .Values.federatedETL.agentOnly) }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "forecasting.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "forecasting.commonLabels" . | nindent 4 }} + {{- with .Values.global.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- with .Values.global.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.forecasting.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: 1 + selector: + matchLabels: + {{- include "forecasting.selectorLabels" . | nindent 6 }} + strategy: + type: RollingUpdate + template: + metadata: + labels: + app.kubernetes.io/name: forecasting + app.kubernetes.io/instance: {{ .Release.Name }} + app: forecasting + {{- with .Values.global.additionalLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.global.podAnnotations}} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + automountServiceAccountToken: false + {{- if .Values.global.platforms.openshift.enabled }} + securityContext: + {{- toYaml .Values.global.platforms.openshift.securityContext | nindent 8 }} + {{- else if .Values.global.securityContext }} + securityContext: + {{- toYaml .Values.global.securityContext | nindent 8 }} + {{- else }} + securityContext: + runAsUser: 1001 + runAsGroup: 1001 + fsGroup: 1001 + {{- end }} + restartPolicy: Always + containers: + - name: forecasting + {{- if .Values.forecasting.fullImageName }} + image: {{ .Values.forecasting.fullImageName }} + {{- else }} + image: gcr.io/kubecost1/kubecost-modeling:prod-{{ $.Chart.AppVersion }} + {{ end }} + {{- if .Values.forecasting.readinessProbe.enabled }} + volumeMounts: + - name: tmp + {{- /* In the future, this path should be configurable and not under tmp */}} + mountPath: /tmp + securityContext: + {{- toYaml .Values.global.containerSecurityContext | nindent 12 }} + {{- if .Values.forecasting.imagePullPolicy }} + imagePullPolicy: {{ .Values.forecasting.imagePullPolicy }} + {{- else }} + imagePullPolicy: Always + {{- end }} + ports: + - name: tcp-api + containerPort: 5000 + protocol: TCP + {{- with .Values.forecasting.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + env: + - name: CONFIG_PATH + value: /var/configs/ + {{- if or .Values.saml.enabled .Values.oidc.enabled }} + - name: KCM_BASE_URL + value: http://{{ template "aggregator.serviceName" . }}:9008 + {{- else }} + - name: KCM_BASE_URL + value: http://{{ template "aggregator.serviceName" . }}:9004 + {{- end }} + - name: MODEL_STORAGE_PATH + value: "/tmp/localrun/models" + - name: PAGE_ITEM_LIMIT + value: "1000" + {{- range $key, $value := .Values.forecasting.env }} + - name: {{ $key | quote }} + value: {{ $value | quote }} + {{- end }} + readinessProbe: + httpGet: + path: /healthz + port: 5000 + initialDelaySeconds: {{ .Values.forecasting.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.forecasting.readinessProbe.periodSeconds }} + failureThreshold: {{ .Values.forecasting.readinessProbe.failureThreshold }} + {{- end }} + {{- if .Values.forecasting.livenessProbe.enabled }} + livenessProbe: + httpGet: + path: /healthz + port: 5000 + initialDelaySeconds: {{ .Values.forecasting.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.forecasting.livenessProbe.periodSeconds }} + failureThreshold: {{ .Values.forecasting.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{- range $.Values.imagePullSecrets }} + - name: {{ .name }} + {{- end }} + {{- end }} + {{- if .Values.forecasting.priority }} + {{- if .Values.forecasting.priority.enabled }} + {{- if .Values.forecasting.priority.name }} + priorityClassName: {{ .Values.forecasting.priority.name }} + {{- else }} + priorityClassName: {{ template "forecasting.fullname" . }}-priority + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.forecasting.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.forecasting.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.forecasting.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: tmp + {{- /* + An emptyDir for models is necessary because of the + readOnlyRootFilesystem default In the future, this may optionally be a + PV. To allow Python to auto-detect a temp directory, which the code + currently relies on, we mount it at /tmp. In the future this will be a + configurable path. + */}} + emptyDir: + sizeLimit: 500Mi +{{- end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/forecasting-service.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/forecasting-service.yaml new file mode 100644 index 0000000000..41e69961e3 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/forecasting-service.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.forecasting.enabled (not .Values.federatedETL.agentOnly) }} +kind: Service +apiVersion: v1 +metadata: + name: {{ template "forecasting.serviceName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "forecasting.commonLabels" . | nindent 4 }} +spec: + selector: + {{- include "forecasting.selectorLabels" . | nindent 4 }} + type: ClusterIP + ports: + - name: tcp-api + port: 5000 + targetPort: 5000 +{{- end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/frontend-deployment-template.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/frontend-deployment-template.yaml new file mode 100644 index 0000000000..e5d26ff786 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/frontend-deployment-template.yaml @@ -0,0 +1,222 @@ +{{- if eq (include "frontend.deployMethod" .) "haMode" }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "frontend.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} + {{- with .Values.global.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if and .Values.kubecostDeployment .Values.kubecostDeployment.labels }} + {{- toYaml .Values.kubecostDeployment.labels | nindent 4 }} + {{- end }} + annotations: + {{- with .Values.global.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if and .Values.kubecostDeployment .Values.kubecostDeployment.annotations }} + {{- toYaml .Values.kubecostDeployment.annotations | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.kubecostFrontend.haReplicas | default 2 }} + selector: + matchLabels: + {{- include "frontend.selectorLabels" . | nindent 6 }} + {{- if .Values.kubecostFrontend.deploymentStrategy }} + {{- with .Values.kubecostFrontend.deploymentStrategy }} + strategy: {{ toYaml . | nindent 4 }} + {{- end }} + {{- else }} + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: RollingUpdate + {{- end }} + template: + metadata: + labels: + {{- include "frontend.selectorLabels" . | nindent 8 }} + {{- if .Values.global.additionalLabels }} + {{- toYaml .Values.global.additionalLabels | nindent 8 }} + {{- end }} + {{- if and .Values.kubecostDeployment .Values.kubecostDeployment.labels }} + {{- toYaml .Values.kubecostDeployment.labels | nindent 8 }} + {{- end }} + annotations: + {{- with .Values.global.podAnnotations}} + {{- toYaml . | nindent 8 }} + {{- end }} + checksum/configs: {{ include "configsChecksum" . }} + spec: + {{- if .Values.global.platforms.openshift.enabled }} + securityContext: + {{- toYaml .Values.global.platforms.openshift.securityContext | nindent 8 }} + {{- else if .Values.global.securityContext }} + securityContext: + {{- toYaml .Values.global.securityContext | nindent 8 }} + {{- else }} + securityContext: + runAsUser: 1001 + runAsGroup: 1001 + fsGroup: 1001 + {{- end }} + restartPolicy: Always + serviceAccountName: {{ template "cost-analyzer.serviceAccountName" . }} + volumes: + - name: tmp + emptyDir: {} + - name: nginx-conf + configMap: + name: nginx-conf + items: + - key: nginx.conf + path: default.conf + {{- if .Values.global.containerSecuritycontext }} + - name: var-run + emptyDir: {} + - name: cache + emptyDir: {} + {{- end }} + {{- if .Values.kubecostFrontend.tls }} + {{- if .Values.kubecostFrontend.tls.enabled }} + - name: tls + secret: + secretName : {{ .Values.kubecostFrontend.tls.secretName }} + items: + - key: tls.crt + path: kc.crt + - key: tls.key + path: kc.key + {{- end }} + {{- end }} + {{- if .Values.kubecostAdmissionController }} + {{- if .Values.kubecostAdmissionController.enabled }} + {{- if .Values.kubecostAdmissionController.secretName }} + - name: webhook-server-tls + secret: + secretName: {{ .Values.kubecostAdmissionController.secretName }} + items: + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key + {{- end }} + {{- end }} + {{- end }} + containers: + {{- if .Values.kubecostFrontend }} + {{- if .Values.kubecostFrontend.fullImageName }} + - image: {{ .Values.kubecostFrontend.fullImageName }} + {{- else if .Values.imageVersion }} + - image: {{ .Values.kubecostFrontend.image }}:{{ .Values.imageVersion }} + {{- else if eq "development" .Chart.AppVersion }} + - image: gcr.io/kubecost1/frontend-nightly:latest + {{- else }} + - image: {{ .Values.kubecostFrontend.image }}:prod-{{ $.Chart.AppVersion }} + {{- end }} + {{- else }} + - image: gcr.io/kubecost1/frontend:prod-{{ $.Chart.AppVersion }} + {{- end }} + name: cost-analyzer-frontend + ports: + - name: tcp-frontend + containerPort: 9090 + protocol: TCP + env: + - name: GET_HOSTS_FROM + value: dns + {{- if .Values.kubecostFrontend.extraEnv -}} + {{ toYaml .Values.kubecostFrontend.extraEnv | nindent 12 }} + {{- end }} + {{- if .Values.kubecostFrontend.securityContext }} + securityContext: + {{- toYaml .Values.kubecostFrontend.securityContext | nindent 12 }} + {{- else }} + securityContext: + {{- toYaml .Values.global.containerSecurityContext | nindent 12 }} + {{- end }} + volumeMounts: + - name: tmp + mountPath: /tmp + - name: tmp + mountPath: /var/lib/nginx/tmp + - name: tmp + mountPath: /var/run + - name: nginx-conf + mountPath: /etc/nginx/conf.d/ + {{- if .Values.global.containerSecuritycontext }} + - mountPath: /var/cache/nginx + name: cache + - mountPath: /var/run + name: var-run + {{- end }} + {{- if .Values.kubecostFrontend.tls }} + {{- if .Values.kubecostFrontend.tls.enabled }} + - name: tls + mountPath: /etc/ssl/certs + {{- end }} + {{- end }} + resources: + {{- toYaml .Values.kubecostFrontend.resources | nindent 12 }} + {{- if .Values.kubecostFrontend.imagePullPolicy }} + imagePullPolicy: {{ .Values.kubecostFrontend.imagePullPolicy }} + {{- else }} + imagePullPolicy: Always + {{- end }} + {{- if .Values.kubecostFrontend.readinessProbe.enabled }} + readinessProbe: + httpGet: + path: /healthz + port: 9090 + initialDelaySeconds: {{ .Values.kubecostFrontend.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.kubecostFrontend.readinessProbe.periodSeconds }} + failureThreshold: {{ .Values.kubecostFrontend.readinessProbe.failureThreshold }} + {{- end }} + {{- if .Values.kubecostFrontend.livenessProbe.enabled }} + livenessProbe: + httpGet: + path: /healthz + port: 9090 + initialDelaySeconds: {{ .Values.kubecostFrontend.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.kubecostFrontend.livenessProbe.periodSeconds }} + failureThreshold: {{ .Values.kubecostFrontend.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.global.containerSecuritycontext }} + securityContext: + {{- toYaml .Values.global.containerSecuritycontext | nindent 12 }} + {{- end }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{- range $.Values.imagePullSecrets }} + - name: {{ .name }} + {{- end }} + {{- end }} + {{- if .Values.priority }} + {{- if .Values.priority.enabled }} + {{- if gt (len .Values.priority.name) 0 }} + priorityClassName: {{ .Values.priority.name }} + {{- else }} + priorityClassName: {{ template "cost-analyzer.fullname" . }}-priority + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/frontend-service-template.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/frontend-service-template.yaml new file mode 100644 index 0000000000..22c2d4fde8 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/frontend-service-template.yaml @@ -0,0 +1,53 @@ +{{- if eq (include "frontend.deployMethod" .) "haMode" }} +kind: Service +apiVersion: v1 +metadata: + name: {{ template "frontend.serviceName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} +{{- if .Values.service.labels }} +{{ toYaml .Values.service.labels | indent 4 }} +{{- end }} +{{- if .Values.service.annotations }} + annotations: +{{ toYaml .Values.service.annotations | indent 4 }} +{{- end }} +spec: + selector: + {{- include "frontend.selectorLabels" . | nindent 4 }} +{{- if .Values.service -}} +{{- if .Values.service.type }} + type: "{{ .Values.service.type }}" +{{- else }} + type: ClusterIP +{{- end }} +{{- else }} + type: ClusterIP +{{- end }} +{{- if (eq .Values.service.type "LoadBalancer") }} + {{- if .Values.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: +{{ toYaml .Values.service.loadBalancerSourceRanges | indent 4 }} + {{- end -}} +{{- end }} + ports: + - name: tcp-frontend + {{- if (eq .Values.service.type "NodePort") }} + {{- if .Values.service.nodePort }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + {{- end }} + port: {{ .Values.service.port }} + targetPort: {{ .Values.service.targetPort }} +{{- if .Values.service.sessionAffinity.enabled }} + sessionAffinity: ClientIP + {{- if .Values.service.sessionAffinity.timeoutSeconds }} + sessionAffinityConfig: + clientIP: + timeoutSeconds: {{ .Values.service.sessionAffinity.timeoutSeconds }} + {{- end }} +{{- else }} + sessionAffinity: None +{{- end }} +{{- end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/gcpstore-config-map-template.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/gcpstore-config-map-template.yaml new file mode 100644 index 0000000000..0c5da0df9a --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/gcpstore-config-map-template.yaml @@ -0,0 +1,61 @@ +{{- if .Values.global.gcpstore.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: ubbagent-config +data: + config.yaml: | + # The identity section contains authentication information used + # by the agent. + identities: + - name: gcp + gcp: + # This parameter accepts a base64-encoded JSON service + # account key. The value comes from the reporting secret. + encodedServiceAccountKey: $AGENT_ENCODED_KEY + + # The metrics section defines the metric that will be reported. + # Metric names should match verbatim the identifiers created + # during pricing setup. + metrics: + + - name: commercial_ent_node_hr + type: int + endpoints: + - name: servicecontrol + + # The passthrough marker indicates that no aggregation should + # occur for this metric. Reports received are immediately sent + # to the reporting endpoint. We use passthrough for the + # instance_time metric since reports are generated + # automatically by a heartbeat source defined in a later + # section. + passthrough: {} + + # The endpoints section defines where metering data is ultimately + # sent. Currently supported endpoints include: + # * disk - some directory on the local filesystem + # * servicecontrol - Google Service Control + endpoints: + - name: servicecontrol + servicecontrol: + identity: gcp + # The service name is unique to your application and will be + # provided during onboarding. + serviceName: kubecost-ent.endpoints.kubecost-public.cloud.goog + consumerId: $AGENT_CONSUMER_ID # From the reporting secret. + + + # The sources section lists metric data sources run by the agent + # itself. The currently-supported source is 'heartbeat', which + # sends a defined value to a metric at a defined interval. In + # this example, the heartbeat sends a 60-second value through the + # "instance_time" metric every minute. + sources: + - name: commercial_ent_node_hr_heartbeat + heartbeat: + metric: commercial_ent_node_hr + intervalSeconds: 3600 + value: + int64Value: 1 +{{- end }} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-clusterrole.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-clusterrole.yaml new file mode 100644 index 0000000000..ca1666823d --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-clusterrole.yaml @@ -0,0 +1,24 @@ +{{- if (eq (include "cost-analyzer.grafanaEnabled" .) "true") }} +{{- if .Values.grafana.rbac.create }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + labels: + app: {{ template "grafana.name" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +{{- with .Values.grafana.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} + name: {{ template "grafana.fullname" . }}-clusterrole +{{- if or .Values.grafana.sidecar.dashboards.enabled .Values.grafana.sidecar.datasources.enabled }} +rules: +- apiGroups: [""] # "" indicates the core API group + resources: ["configmaps"] + verbs: ["get", "watch", "list"] +{{- else }} +rules: [] +{{- end}} +{{- end}} +{{ end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-clusterrolebinding.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-clusterrolebinding.yaml new file mode 100644 index 0000000000..4fc7267f32 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-clusterrolebinding.yaml @@ -0,0 +1,24 @@ +{{- if (eq (include "cost-analyzer.grafanaEnabled" .) "true") }} +{{- if .Values.grafana.rbac.create }} +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "grafana.fullname" . }}-clusterrolebinding + labels: + app: {{ template "grafana.name" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +{{- with .Values.grafana.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +subjects: + - kind: ServiceAccount + name: {{ template "grafana.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ template "grafana.fullname" . }}-clusterrole + apiGroup: rbac.authorization.k8s.io +{{- end}} +{{ end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-configmap-dashboard-provider.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-configmap-dashboard-provider.yaml new file mode 100644 index 0000000000..78c7717be6 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-configmap-dashboard-provider.yaml @@ -0,0 +1,28 @@ +{{- if (eq (include "cost-analyzer.grafanaEnabled" .) "true") }} +{{- if .Values.grafana.sidecar.dashboards.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + app: {{ template "grafana.name" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +{{- with .Values.grafana.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} + name: {{ template "grafana.fullname" . }}-config-dashboards + namespace: {{ .Release.Namespace }} +data: + provider.yaml: |- + apiVersion: 1 + providers: + - name: 'default' + orgId: 1 + folder: '' + type: file + disableDeletion: false + options: + path: {{ .Values.grafana.sidecar.dashboards.folder }} +{{- end}} +{{ end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-configmap.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-configmap.yaml new file mode 100644 index 0000000000..04d6146674 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-configmap.yaml @@ -0,0 +1,90 @@ +{{- if (eq (include "cost-analyzer.grafanaEnabled" .) "true") }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "grafana.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "grafana.name" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +data: +{{- if .Values.grafana.plugins }} + plugins: {{ join "," .Values.grafana.plugins }} +{{- end }} + grafana.ini: | +{{- range $key, $value := index .Values.grafana "grafana.ini" }} + [{{ $key }}] + {{- range $elem, $elemVal := $value }} + {{ $elem }} = {{ $elemVal }} + {{- end }} +{{- end }} + +{{- if .Values.grafana.datasources }} + {{- range $key, $value := .Values.grafana.datasources }} + {{ $key }}: | +{{ toYaml $value | trim | indent 4 }} + {{- end -}} +{{- end }} +{{- if not .Values.grafana.datasources }} + datasources.yaml: | + apiVersion: 1 + datasources: +{{- if .Values.global.prometheus.enabled }} + - access: proxy + isDefault: true + name: Prometheus + type: prometheus + url: http://{{ template "cost-analyzer.prometheus.server.name" . }}.{{ .Release.Namespace }}.svc + jsonData: + httpMethod: POST + prometheusType: Prometheus + prometheusVersion: 2.35.0 + timeInterval: 1m +{{- else }} + - access: proxy + isDefault: true + name: Prometheus + type: prometheus + url: {{ .Values.global.prometheus.fqdn }} + jsonData: + httpMethod: POST + prometheusType: Prometheus + prometheusVersion: 2.35.0 + timeInterval: 1m +{{- end -}} +{{- end }} +{{- if .Values.grafana.dashboardProviders }} + {{- range $key, $value := .Values.grafana.dashboardProviders }} + {{ $key }}: | +{{ toYaml $value | indent 4 }} + {{- end -}} +{{- end -}} + +{{- if .Values.grafana.dashboards }} + download_dashboards.sh: | + #!/usr/bin/env sh + set -euf + {{- if .Values.grafana.dashboardProviders }} + {{- range $key, $value := .Values.grafana.dashboardProviders }} + {{- range $value.providers }} + mkdir -p {{ .options.path }} + {{- end }} + {{- end }} + {{- end }} + + {{- range $provider, $dashboards := .Values.grafana.dashboards }} + {{- range $key, $value := $dashboards }} + {{- if (or (hasKey $value "gnetId") (hasKey $value "url")) }} + curl -sk \ + --connect-timeout 60 \ + --max-time 60 \ + -H "Accept: application/json" \ + -H "Content-Type: application/json;charset=UTF-8" \ + {{- if $value.url -}}{{ $value.url }}{{- else -}} https://grafana.com/api/dashboards/{{ $value.gnetId }}/revisions/{{- if $value.revision -}}{{ $value.revision }}{{- else -}}1{{- end -}}/download{{- end -}}{{ if $value.datasource }}| sed 's|\"datasource\":[^,]*|\"datasource\": \"{{ $value.datasource }}\"|g'{{ end }} \ + > /var/lib/grafana/dashboards/{{ $provider }}/{{ $key }}.json + {{- end }} + {{- end }} + {{- end }} +{{- end }} +{{ end }} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-attached-disks.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-attached-disks.yaml new file mode 100644 index 0000000000..d00ece4be1 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-attached-disks.yaml @@ -0,0 +1,21 @@ +{{- if (((.Values.grafana).sidecar).dashboards).enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-attached-disk-metrics + {{- if $.Values.grafana.namespace_dashboards }} + namespace: {{ $.Values.grafana.namespace_dashboards }} + {{- end }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "{{ $.Values.grafana.sidecar.dashboards.labelValue }}" + {{- else }} + grafana_dashboard: "1" + {{- end }} + annotations: +{{- toYaml .Values.grafana.sidecar.dashboards.annotations | nindent 4 }} +data: + attached-disks.json: |- +{{- .Files.Get "grafana-dashboards/attached-disks.json" | nindent 4 }} +{{- end -}} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-cluster-metrics-template.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-cluster-metrics-template.yaml new file mode 100644 index 0000000000..56dbbb704a --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-cluster-metrics-template.yaml @@ -0,0 +1,21 @@ +{{- if (((.Values.grafana).sidecar).dashboards).enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-cluster-metrics + {{- if $.Values.grafana.namespace_dashboards }} + namespace: {{ $.Values.grafana.namespace_dashboards }} + {{- end }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "{{ $.Values.grafana.sidecar.dashboards.labelValue }}" + {{- else }} + grafana_dashboard: "1" + {{- end }} + annotations: +{{- toYaml .Values.grafana.sidecar.dashboards.annotations | nindent 4 }} +data: + cluster-metrics.json: |- +{{- .Files.Get "grafana-dashboards/cluster-metrics.json" | nindent 4 }} +{{- end -}} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-cluster-utilization-template.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-cluster-utilization-template.yaml new file mode 100644 index 0000000000..3f29d0e24d --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-cluster-utilization-template.yaml @@ -0,0 +1,21 @@ +{{- if (((.Values.grafana).sidecar).dashboards).enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-cluster-utilization + {{- if $.Values.grafana.namespace_dashboards }} + namespace: {{ $.Values.grafana.namespace_dashboards }} + {{- end }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "{{ $.Values.grafana.sidecar.dashboards.labelValue }}" + {{- else }} + grafana_dashboard: "1" + {{- end }} + annotations: +{{- toYaml .Values.grafana.sidecar.dashboards.annotations | nindent 4 }} +data: + cluster-utilization.json: |- +{{- .Files.Get "grafana-dashboards/cluster-utilization.json" | nindent 4 }} +{{- end -}} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-deployment-utilization-template.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-deployment-utilization-template.yaml new file mode 100644 index 0000000000..a349f8ac44 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-deployment-utilization-template.yaml @@ -0,0 +1,21 @@ +{{- if (((.Values.grafana).sidecar).dashboards).enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-deployment-utilization + {{- if $.Values.grafana.namespace_dashboards }} + namespace: {{ $.Values.grafana.namespace_dashboards }} + {{- end }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "{{ $.Values.grafana.sidecar.dashboards.labelValue }}" + {{- else }} + grafana_dashboard: "1" + {{- end }} + annotations: +{{- toYaml .Values.grafana.sidecar.dashboards.annotations | nindent 4 }} +data: + deployment-utilization.json: |- +{{- .Files.Get "grafana-dashboards/deployment-utilization.json" | nindent 4 }} +{{- end -}} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-kubernetes-resource-efficiency-template.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-kubernetes-resource-efficiency-template.yaml new file mode 100644 index 0000000000..b272ee7e2d --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-kubernetes-resource-efficiency-template.yaml @@ -0,0 +1,21 @@ +{{- if (((.Values.grafana).sidecar).dashboards).enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-kubernetes-resource-efficiency + {{- if $.Values.grafana.namespace_dashboards }} + namespace: {{ $.Values.grafana.namespace_dashboards }} + {{- end }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "{{ $.Values.grafana.sidecar.dashboards.labelValue }}" + {{- else }} + grafana_dashboard: "1" + {{- end }} + annotations: +{{- toYaml .Values.grafana.sidecar.dashboards.annotations | nindent 4 }} +data: + kubernetes-resource-efficiency.json: |- +{{- .Files.Get "grafana-dashboards/kubernetes-resource-efficiency.json" | nindent 4 }} +{{- end -}} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-label-cost-utilization-template.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-label-cost-utilization-template.yaml new file mode 100644 index 0000000000..3fd558703e --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-label-cost-utilization-template.yaml @@ -0,0 +1,21 @@ +{{- if (((.Values.grafana).sidecar).dashboards).enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-label-cost + {{- if $.Values.grafana.namespace_dashboards }} + namespace: {{ $.Values.grafana.namespace_dashboards }} + {{- end }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "{{ $.Values.grafana.sidecar.dashboards.labelValue }}" + {{- else }} + grafana_dashboard: "1" + {{- end }} + annotations: +{{- toYaml .Values.grafana.sidecar.dashboards.annotations | nindent 4 }} +data: + label-cost-utilization.json: |- +{{- .Files.Get "grafana-dashboards/label-cost-utilization.json" | nindent 4 }} +{{- end -}} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-namespace-utilization-template.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-namespace-utilization-template.yaml new file mode 100644 index 0000000000..7c54d8fd7a --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-namespace-utilization-template.yaml @@ -0,0 +1,21 @@ +{{- if (((.Values.grafana).sidecar).dashboards).enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-namespace-utilization + {{- if $.Values.grafana.namespace_dashboards }} + namespace: {{ $.Values.grafana.namespace_dashboards }} + {{- end }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "{{ $.Values.grafana.sidecar.dashboards.labelValue }}" + {{- else }} + grafana_dashboard: "1" + {{- end }} + annotations: +{{- toYaml .Values.grafana.sidecar.dashboards.annotations | nindent 4 }} +data: + namespace-utilization.json: |- +{{- .Files.Get "grafana-dashboards/namespace-utilization.json" | nindent 4 }} +{{- end -}} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-network-cloud-sevices.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-network-cloud-sevices.yaml new file mode 100644 index 0000000000..b4dd541d95 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-network-cloud-sevices.yaml @@ -0,0 +1,21 @@ +{{- if (((.Values.grafana).sidecar).dashboards).enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-network-cloud-services + {{- if $.Values.grafana.namespace_dashboards }} + namespace: {{ $.Values.grafana.namespace_dashboards }} + {{- end }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "{{ $.Values.grafana.sidecar.dashboards.labelValue }}" + {{- else }} + grafana_dashboard: "1" + {{- end }} + annotations: +{{- toYaml .Values.grafana.sidecar.dashboards.annotations | nindent 4 }} +data: + grafana-network-cloud-services.json: |- +{{- .Files.Get "grafana-dashboards/network-cloud-services.json" | nindent 4 }} +{{- end -}} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-network-costs.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-network-costs.yaml new file mode 100644 index 0000000000..f51cb11abc --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-network-costs.yaml @@ -0,0 +1,21 @@ +{{- if (((.Values.grafana).sidecar).dashboards).enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-network-costs-metrics + {{- if $.Values.grafana.namespace_dashboards }} + namespace: {{ $.Values.grafana.namespace_dashboards }} + {{- end }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "{{ $.Values.grafana.sidecar.dashboards.labelValue }}" + {{- else }} + grafana_dashboard: "1" + {{- end }} + annotations: +{{- toYaml .Values.grafana.sidecar.dashboards.annotations | nindent 4 }} +data: + networkCosts-metrics.json: |- +{{- .Files.Get "grafana-dashboards/networkCosts-metrics.json" | nindent 4 }} +{{- end -}} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-node-utilization-template.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-node-utilization-template.yaml new file mode 100644 index 0000000000..f364358d58 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-node-utilization-template.yaml @@ -0,0 +1,21 @@ +{{- if (((.Values.grafana).sidecar).dashboards).enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-node-utilization + {{- if $.Values.grafana.namespace_dashboards }} + namespace: {{ $.Values.grafana.namespace_dashboards }} + {{- end }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "{{ $.Values.grafana.sidecar.dashboards.labelValue }}" + {{- else }} + grafana_dashboard: "1" + {{- end }} + annotations: +{{- toYaml .Values.grafana.sidecar.dashboards.annotations | nindent 4 }} +data: + node-utilization.json: |- +{{- .Files.Get "grafana-dashboards/node-utilization.json" | nindent 4 }} +{{- end -}} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-pod-utilization-multi-cluster.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-pod-utilization-multi-cluster.yaml new file mode 100644 index 0000000000..5766c9a4be --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-pod-utilization-multi-cluster.yaml @@ -0,0 +1,21 @@ +{{- if (((.Values.grafana).sidecar).dashboards).enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-pod-utilization-multi-cluster + {{- if $.Values.grafana.namespace_dashboards }} + namespace: {{ $.Values.grafana.namespace_dashboards }} + {{- end }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "{{ $.Values.grafana.sidecar.dashboards.labelValue }}" + {{- else }} + grafana_dashboard: "1" + {{- end }} + annotations: +{{- toYaml .Values.grafana.sidecar.dashboards.annotations | nindent 4 }} +data: + pod-utilization-multi-cluster.json: |- +{{- .Files.Get "grafana-dashboards/pod-utilization-multi-cluster.json" | nindent 4 }} +{{- end -}} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-pod-utilization-template.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-pod-utilization-template.yaml new file mode 100644 index 0000000000..0a21343da4 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-pod-utilization-template.yaml @@ -0,0 +1,21 @@ +{{- if (((.Values.grafana).sidecar).dashboards).enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-pod-utilization + {{- if $.Values.grafana.namespace_dashboards }} + namespace: {{ $.Values.grafana.namespace_dashboards }} + {{- end }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "{{ $.Values.grafana.sidecar.dashboards.labelValue }}" + {{- else }} + grafana_dashboard: "1" + {{- end }} + annotations: +{{- toYaml .Values.grafana.sidecar.dashboards.annotations | nindent 4 }} +data: + pod-utilization.json: |- +{{- .Files.Get "grafana-dashboards/pod-utilization.json" | nindent 4 }} +{{- end -}} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-prometheus-metrics-template.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-prometheus-metrics-template.yaml new file mode 100644 index 0000000000..fa86630476 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-prometheus-metrics-template.yaml @@ -0,0 +1,21 @@ +{{- if (((.Values.grafana).sidecar).dashboards).enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-prom-benchmark + {{- if $.Values.grafana.namespace_dashboards }} + namespace: {{ $.Values.grafana.namespace_dashboards }} + {{- end }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "{{ $.Values.grafana.sidecar.dashboards.labelValue }}" + {{- else }} + grafana_dashboard: "1" + {{- end }} + annotations: +{{- toYaml .Values.grafana.sidecar.dashboards.annotations | nindent 4 }} +data: + prom-benchmark.json: |- +{{- .Files.Get "grafana-dashboards/prom-benchmark.json" | nindent 4 }} +{{- end -}} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-workload-aggregator.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-workload-aggregator.yaml new file mode 100644 index 0000000000..bd384373a3 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-workload-aggregator.yaml @@ -0,0 +1,21 @@ +{{- if (((.Values.grafana).sidecar).dashboards).enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-workload-aggregator + {{- if $.Values.grafana.namespace_dashboards }} + namespace: {{ $.Values.grafana.namespace_dashboards }} + {{- end }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "{{ $.Values.grafana.sidecar.dashboards.labelValue }}" + {{- else }} + grafana_dashboard: "1" + {{- end }} + annotations: +{{- toYaml .Values.grafana.sidecar.dashboards.annotations | nindent 4 }} +data: + workload-metrics-aggregator.json: |- +{{- .Files.Get "grafana-dashboards/workload-metrics-aggregator.json" | nindent 4 }} +{{- end -}} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-workload-metrics.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-workload-metrics.yaml new file mode 100644 index 0000000000..429909c5aa --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboard-workload-metrics.yaml @@ -0,0 +1,21 @@ +{{- if (((.Values.grafana).sidecar).dashboards).enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-workload-metrics + {{- if $.Values.grafana.namespace_dashboards }} + namespace: {{ $.Values.grafana.namespace_dashboards }} + {{- end }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "{{ $.Values.grafana.sidecar.dashboards.labelValue }}" + {{- else }} + grafana_dashboard: "1" + {{- end }} + annotations: +{{- toYaml .Values.grafana.sidecar.dashboards.annotations | nindent 4 }} +data: + grafana-workload-metrics.json: |- +{{- .Files.Get "grafana-dashboards/workload-metrics.json" | nindent 4 }} +{{- end -}} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboards-json-configmap.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboards-json-configmap.yaml new file mode 100644 index 0000000000..b7ccb3cb54 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-dashboards-json-configmap.yaml @@ -0,0 +1,24 @@ +{{- if (eq (include "cost-analyzer.grafanaEnabled" .) "true") }} +{{- if .Values.grafana.dashboards }} + {{- range $provider, $dashboards := .Values.grafana.dashboards }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "grafana.fullname" $ }}-dashboards-{{ $provider }} + namespace: {{ $.Release.Namespace }} + labels: + app: {{ template "grafana.name" $ }} + release: {{ $.Release.Name }} + heritage: {{ $.Release.Service }} + dashboard-provider: {{ $provider }} +data: + {{- range $key, $value := $dashboards }} + {{- if hasKey $value "json" }} + {{ $key }}.json: | +{{ $value.json | indent 4 }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-datasource-template.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-datasource-template.yaml new file mode 100644 index 0000000000..ba4ecea8c9 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-datasource-template.yaml @@ -0,0 +1,38 @@ +{{- if .Values.grafana -}} +{{- if .Values.grafana.sidecar -}} +{{- if .Values.grafana.sidecar.datasources -}} +{{- if .Values.grafana.sidecar.datasources.enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-datasource + {{- if $.Values.grafana.namespace_datasources }} + namespace: {{ $.Values.grafana.namespace_datasources }} + {{- end }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} + {{- if $.Values.grafana.sidecar.datasources.label }} + {{ $.Values.grafana.sidecar.datasources.label }}: "1" + {{- else }} + {{- if .Values.global.grafana.enabled }} + kubecost_grafana_datasource: "1" + {{- else }} + grafana_datasource: "1" + {{- end }} + {{- end }} +data: + {{ default "datasource.yaml" .Values.grafana.sidecar.datasources.dataSourceFilename }}: |- + apiVersion: 1 + datasources: + - access: proxy + name: default-kubecost + type: prometheus +{{- if .Values.grafana.sidecar.datasources.defaultDatasourceEnabled }} + isDefault: true +{{- else }} + isDefault: false +{{- end }} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-deployment.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-deployment.yaml new file mode 100644 index 0000000000..041039e94c --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-deployment.yaml @@ -0,0 +1,291 @@ +{{- if (eq (include "cost-analyzer.grafanaEnabled" .) "true") }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "grafana.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "grafana.name" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + {{- with .Values.global.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- with .Values.global.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.grafana.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.grafana.replicas }} + selector: + matchLabels: + app: {{ template "grafana.name" . }} + release: {{ .Release.Name }} + strategy: + type: {{ .Values.grafana.deploymentStrategy }} + {{- if ne .Values.grafana.deploymentStrategy "RollingUpdate" }} + rollingUpdate: null + {{- end }} + template: + metadata: + labels: + app: {{ template "grafana.name" . }} + release: {{ .Release.Name }} + {{- if .Values.global.additionalLabels }} + {{ toYaml .Values.global.additionalLabels | nindent 8 }} + {{- end }} + {{- with .Values.grafana.podAnnotations }} + annotations: + {{ toYaml . | indent 8 }} + {{- end }} + spec: + serviceAccountName: {{ template "grafana.serviceAccountName" . }} + {{- if .Values.grafana.schedulerName }} + schedulerName: "{{ .Values.grafana.schedulerName }}" + {{- end }} + {{- if .Values.grafana.securityContext }} + securityContext: + {{- toYaml .Values.grafana.securityContext | nindent 8 }} + {{- else if and (.Values.global.platforms.openshift.enabled) (.Values.global.platforms.openshift.securityContext) }} + securityContext: + {{- toYaml .Values.global.platforms.openshift.securityContext | nindent 8 }} + {{- else if .Values.global.securityContext }} + securityContext: + {{- toYaml .Values.global.securityContext | nindent 8 }} + {{- end }} + {{- if .Values.grafana.priorityClassName }} + priorityClassName: "{{ .Values.grafana.priorityClassName }}" + {{- end }} + {{- if .Values.grafana.dashboards }} + initContainers: + - name: download-dashboards + image: "{{ .Values.grafana.downloadDashboardsImage.repository }}:{{ .Values.grafana.downloadDashboardsImage.tag }}" + imagePullPolicy: {{ .Values.grafana.downloadDashboardsImage.pullPolicy }} + command: ["sh", "/etc/grafana/download_dashboards.sh"] + {{- with .Values.global.containerSecurityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: config + mountPath: "/etc/grafana/download_dashboards.sh" + subPath: download_dashboards.sh + - name: storage + mountPath: "/var/lib/grafana" + {{- if .Values.grafana.persistence.subPath }} + subPath: {{ .Values.grafana.persistence.subPath }} + {{- end }} + {{- range .Values.grafana.extraSecretMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + readOnly: {{ .readOnly }} + {{- end }} + {{- end }} + {{- if .Values.grafana.image.pullSecrets }} + imagePullSecrets: + {{- range .Values.grafana.image.pullSecrets }} + - name: {{ . }} + {{- end}} + {{- end }} + containers: + {{- if .Values.grafana.sidecar.dashboards.enabled }} + - name: {{ template "grafana.name" . }}-sc-dashboard + image: "{{ .Values.grafana.sidecar.image.repository }}:{{ .Values.grafana.sidecar.image.tag }}" + imagePullPolicy: {{ .Values.grafana.sidecar.image.pullPolicy }} + {{- if .Values.global.containerSecurityContext }} + securityContext: + {{- toYaml .Values.global.containerSecurityContext | nindent 12 -}} + {{- end }} + env: + - name: LABEL + value: "{{ .Values.grafana.sidecar.dashboards.label }}" + - name: FOLDER + value: "{{ .Values.grafana.sidecar.dashboards.folder }}" + - name: ERROR_THROTTLE_SLEEP + value: "{{ .Values.grafana.sidecar.dashboards.error_throttle_sleep }}" + {{- with .Values.grafana.sidecar.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: sc-dashboard-volume + mountPath: {{ .Values.grafana.sidecar.dashboards.folder | quote }} + {{- end}} + {{- if .Values.grafana.sidecar.datasources.enabled }} + - name: {{ template "grafana.name" . }}-sc-datasources + image: "{{ .Values.grafana.sidecar.image.repository }}:{{ .Values.grafana.sidecar.image.tag }}" + imagePullPolicy: {{ .Values.grafana.sidecar.image.pullPolicy }} + {{- with .Values.global.containerSecurityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + env: + - name: LABEL + value: "{{ .Values.grafana.sidecar.datasources.label }}" + - name: FOLDER + value: "/etc/grafana/provisioning/datasources" + - name: ERROR_THROTTLE_SLEEP + value: "{{ .Values.grafana.sidecar.datasources.error_throttle_sleep }}" + resources: + {{ toYaml .Values.grafana.sidecar.resources | indent 12 }} + volumeMounts: + - name: sc-datasources-volume + mountPath: "/etc/grafana/provisioning/datasources" + {{- end}} + - name: grafana + image: "{{ .Values.grafana.image.repository }}:{{ .Values.grafana.image.tag }}" + imagePullPolicy: {{ .Values.grafana.image.pullPolicy }} + {{- with .Values.global.containerSecurityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: config + mountPath: "/etc/grafana/grafana.ini" + subPath: grafana.ini +{{- if .Values.grafana.dashboards }} + {{- range $provider, $dashboards := .Values.grafana.dashboards }} + {{- range $key, $value := $dashboards }} + {{- if hasKey $value "json" }} + - name: dashboards-{{ $provider }} + mountPath: "/var/lib/grafana/dashboards/{{ $provider }}/{{ $key }}.json" + subPath: "{{ $key }}.json" + {{- end }} + {{- end }} + {{- end }} +{{- end -}} +{{- if .Values.grafana.dashboardsConfigMaps }} + {{- range keys .Values.grafana.dashboardsConfigMaps }} + - name: dashboards-{{ . }} + mountPath: "/var/lib/grafana/dashboards/{{ . }}" + {{- end }} +{{- end }} +{{- if or (.Values.grafana.datasources) (include "cost-analyzer.grafanaEnabled" .) }} + - name: config + mountPath: "/etc/grafana/provisioning/datasources/datasources.yaml" + subPath: datasources.yaml +{{- end }} +{{- if .Values.grafana.dashboardProviders }} + - name: config + mountPath: "/etc/grafana/provisioning/dashboards/dashboardproviders.yaml" + subPath: dashboardproviders.yaml +{{- end }} +{{- if .Values.grafana.sidecar.dashboards.enabled }} + - name: sc-dashboard-volume + mountPath: {{ .Values.grafana.sidecar.dashboards.folder | quote }} + - name: sc-dashboard-provider + mountPath: "/etc/grafana/provisioning/dashboards/sc-dashboardproviders.yaml" + subPath: provider.yaml +{{- end}} +{{- if .Values.grafana.sidecar.datasources.enabled }} + - name: sc-datasources-volume + mountPath: "/etc/grafana/provisioning/datasources" +{{- end}} + - name: storage + mountPath: "/var/lib/grafana" + {{- if .Values.grafana.persistence.subPath }} + subPath: {{ .Values.grafana.persistence.subPath }} + {{- end }} + {{- range .Values.grafana.extraSecretMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + readOnly: {{ .readOnly }} + {{- end }} + ports: + - name: service + containerPort: {{ .Values.grafana.service.port }} + protocol: TCP + - name: grafana + containerPort: 3000 + protocol: TCP + env: + - name: GF_SECURITY_ADMIN_USER + valueFrom: + secretKeyRef: + name: {{ template "grafana.fullname" . }} + key: admin-user + - name: GF_SECURITY_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: {{ template "grafana.fullname" . }} + key: admin-password + {{- if .Values.grafana.plugins }} + - name: GF_INSTALL_PLUGINS + valueFrom: + configMapKeyRef: + name: {{ template "grafana.fullname" . }} + key: plugins + {{- end }} +{{- range $key, $value := .Values.grafana.env }} + - name: "{{ $key }}" + value: "{{ $value }}" +{{- end }} + {{- if .Values.grafana.envFromSecret }} + envFrom: + - secretRef: + name: {{ .Values.grafana.envFromSecret }} + {{- end }} + livenessProbe: +{{ toYaml .Values.grafana.livenessProbe | indent 12 }} + readinessProbe: +{{ toYaml .Values.grafana.readinessProbe | indent 12 }} + resources: +{{ toYaml .Values.grafana.resources | indent 12 }} + {{- with .Values.grafana.nodeSelector }} + nodeSelector: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.grafana.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.grafana.tolerations }} + tolerations: +{{ toYaml . | indent 8 }} + {{- end }} + volumes: + - name: config + configMap: + name: {{ template "grafana.fullname" . }} + {{- if .Values.grafana.dashboards }} + {{- range keys .Values.grafana.dashboards }} + - name: dashboards-{{ . }} + configMap: + name: {{ template "grafana.fullname" $ }}-dashboards-{{ . }} + {{- end }} + {{- end }} + {{- if .Values.grafana.dashboardsConfigMaps }} + {{- range $provider, $name := .Values.grafana.dashboardsConfigMaps }} + - name: dashboards-{{ $provider }} + configMap: + name: {{ $name }} + {{- end }} + {{- end }} + - name: storage + {{- if .Values.grafana.persistence.enabled }} + persistentVolumeClaim: + claimName: {{ .Values.grafana.persistence.existingClaim | default (include "grafana.fullname" .) }} + {{- else }} + emptyDir: {} + {{- end -}} + {{- if .Values.grafana.sidecar.dashboards.enabled }} + - name: sc-dashboard-volume + emptyDir: {} + - name: sc-dashboard-provider + configMap: + name: {{ template "grafana.fullname" . }}-config-dashboards + {{- end }} + {{- if .Values.grafana.sidecar.datasources.enabled }} + - name: sc-datasources-volume + emptyDir: {} + {{- end -}} + {{- range .Values.grafana.extraSecretMounts }} + - name: {{ .name }} + secret: + secretName: {{ .secretName }} + defaultMode: {{ .defaultMode }} + {{- end }} +{{ end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-pvc.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-pvc.yaml new file mode 100644 index 0000000000..d90e7f7477 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-pvc.yaml @@ -0,0 +1,26 @@ +{{- if (eq (include "cost-analyzer.grafanaEnabled" .) "true") }} +{{- if and .Values.grafana.persistence.enabled (not .Values.grafana.persistence.existingClaim) }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ template "grafana.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "grafana.name" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + {{- with .Values.grafana.persistence.annotations }} + annotations: +{{ toYaml . | indent 4 }} + {{- end }} +spec: + accessModes: + {{- range .Values.grafana.persistence.accessModes }} + - {{ . | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.grafana.persistence.size | quote }} + storageClassName: {{ .Values.grafana.persistence.storageClassName }} +{{- end -}} +{{ end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-secret.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-secret.yaml new file mode 100644 index 0000000000..ce2ca7c90c --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-secret.yaml @@ -0,0 +1,19 @@ +{{- if (eq (include "cost-analyzer.grafanaEnabled" .) "true") }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "grafana.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "grafana.name" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +type: Opaque +data: + admin-user: {{ .Values.grafana.adminUser | b64enc | quote }} + {{- if .Values.grafana.adminPassword }} + admin-password: {{ .Values.grafana.adminPassword | b64enc | quote }} + {{- else }} + admin-password: {{ randAlphaNum 40 | b64enc | quote }} + {{- end }} +{{ end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-service.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-service.yaml new file mode 100644 index 0000000000..3bf668ed8a --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-service.yaml @@ -0,0 +1,51 @@ +{{- if (eq (include "cost-analyzer.grafanaEnabled" .) "true") }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "grafana.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "grafana.name" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +{{- if .Values.grafana.service.labels }} +{{ toYaml .Values.grafana.service.labels | indent 4 }} +{{- end }} +{{- with .Values.grafana.service.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +spec: +{{- if (or (eq .Values.grafana.service.type "ClusterIP") (empty .Values.grafana.service.type)) }} + type: ClusterIP + {{- if .Values.grafana.service.clusterIP }} + clusterIP: {{ .Values.grafana.service.clusterIP }} + {{end}} +{{- else if eq .Values.grafana.service.type "LoadBalancer" }} + type: {{ .Values.grafana.service.type }} + {{- if .Values.grafana.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.grafana.service.loadBalancerIP }} + {{- end }} + {{- if .Values.grafana.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: +{{ toYaml .Values.grafana.service.loadBalancerSourceRanges | indent 4 }} + {{- end -}} +{{- else }} + type: {{ .Values.grafana.service.type }} +{{- end }} +{{- if .Values.grafana.service.externalIPs }} + externalIPs: +{{ toYaml .Values.grafana.service.externalIPs | indent 4 }} +{{- end }} + ports: + - name: tcp-service + port: {{ .Values.grafana.service.port }} + protocol: TCP + targetPort: 3000 +{{ if (and (eq .Values.grafana.service.type "NodePort") (not (empty .Values.grafana.service.nodePort))) }} + nodePort: {{.Values.grafana.service.nodePort}} +{{ end }} + selector: + app: {{ template "grafana.name" . }} + release: {{ .Release.Name }} +{{ end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-serviceaccount.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-serviceaccount.yaml new file mode 100644 index 0000000000..bf2f21db6e --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/grafana/grafana-serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if (eq (include "cost-analyzer.grafanaEnabled" .) "true") }} +{{- if .Values.grafana.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app: {{ template "grafana.name" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + name: {{ template "grafana.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end }} +{{- end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/install-plugins.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/install-plugins.yaml new file mode 100644 index 0000000000..372af04786 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/install-plugins.yaml @@ -0,0 +1,43 @@ +{{- if .Values.kubecostModel.plugins.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "cost-analyzer.fullname" . }}-install-plugins + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} +data: + install_plugins.sh: |- + {{- if .Values.kubecostModel.plugins.install.enabled }} + set -ex + rm -f {{ .Values.kubecostModel.plugins.folder }}/bin/* + mkdir -p {{ .Values.kubecostModel.plugins.folder }}/bin + cd {{ .Values.kubecostModel.plugins.folder }}/bin + OSTYPE=$(cat /etc/os-release) + OS='' + case "$OSTYPE" in + *Linux*) OS='linux';; + *) echo "$OSTYPE is unsupported" && exit 1 ;; + esac + + UNAME_OUTPUT=$(uname -m) + ARCH='' + case "$UNAME_OUTPUT" in + *x86_64*) ARCH='amd64';; + *amd64*) ARCH='amd64';; + *aarch64*) ARCH='arm64';; + *arm64*) ARCH='arm64';; + *) echo "$UNAME_OUTPUT is unsupported" && exit 1 ;; + esac + + {{- if .Values.kubecostModel.plugins.version }} + VER={{ .Values.kubecostModel.plugins.version | quote}} + {{- else }} + VER=$(curl --silent https://api.github.com/repos/opencost/opencost-plugins/releases/latest | grep ".tag_name" | awk -F\" '{print $4}') + {{- end }} + + {{- range $pluginName := .Values.kubecostModel.plugins.enabledPlugins }} + curl -fsSLO "https://github.com/opencost/opencost-plugins/releases/download/$VER/{{ $pluginName }}.ocplugin.$OS.$ARCH" + chmod a+rx "{{ $pluginName }}.ocplugin.$OS.$ARCH" + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/integrations-postgres-queries-configmap.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/integrations-postgres-queries-configmap.yaml new file mode 100644 index 0000000000..59f08cdd3f --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/integrations-postgres-queries-configmap.yaml @@ -0,0 +1,14 @@ +{{- if .Values.global.integrations.postgres.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: kubecost-integrations-postgres-queries + namespace: {{ .Release.Namespace }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} +data: + kubecost-queries.json: |- + {{- with .Values.global.integrations.postgres.queryConfigs }} + {{- . | toJson | nindent 6 }} + {{- end }} +{{- end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/integrations-postgres-secret.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/integrations-postgres-secret.yaml new file mode 100644 index 0000000000..1f5e5a7049 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/integrations-postgres-secret.yaml @@ -0,0 +1,19 @@ +{{- if and (.Values.global.integrations.postgres.enabled) (eq .Values.global.integrations.postgres.databaseSecretName "") }} +apiVersion: v1 +kind: Secret +metadata: + name: kubecost-integrations-postgres + namespace: {{ .Release.Namespace }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} +type: Opaque +stringData: + creds.json: |- + { + "host": "{{ .Values.global.integrations.postgres.databaseHost }}", + "port": "{{ .Values.global.integrations.postgres.databasePort }}", + "databaseName": "{{ .Values.global.integrations.postgres.databaseName }}", + "user": "{{ .Values.global.integrations.postgres.databaseUser }}", + "password": "{{ .Values.global.integrations.postgres.databasePassword }}" + } +{{- end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/integrations-turbonomic-secret.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/integrations-turbonomic-secret.yaml new file mode 100644 index 0000000000..9d63240648 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/integrations-turbonomic-secret.yaml @@ -0,0 +1,19 @@ +{{- if .Values.global.integrations.turbonomic.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: kubecost-integrations-turbonomic + namespace: {{ .Release.Namespace }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} +type: Opaque +stringData: + turbonomic-credentials.json: |- + { + "host": "{{ .Values.global.integrations.turbonomic.host }}", + "role": "{{ .Values.global.integrations.turbonomic.role }}", + "clientId": "{{ .Values.global.integrations.turbonomic.clientId }}", + "clientSecret": "{{ .Values.global.integrations.turbonomic.clientSecret }}", + "insecureClient": {{ .Values.global.integrations.turbonomic.insecureClient }} + } +{{- end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/kubecost-admission-controller-service-template.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/kubecost-admission-controller-service-template.yaml new file mode 100644 index 0000000000..b2b7c60a4d --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/kubecost-admission-controller-service-template.yaml @@ -0,0 +1,15 @@ +{{- if .Values.kubecostAdmissionController -}} +{{- if .Values.kubecostAdmissionController.enabled -}} +apiVersion: v1 +kind: Service +metadata: + name: webhook-server + namespace: {{.Release.Namespace}} +spec: + selector: + {{- include "cost-analyzer.selectorLabels" . | nindent 4 }} + ports: + - port: 443 + targetPort: 8443 +{{- end -}} +{{- end -}} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/kubecost-admission-controller-template.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/kubecost-admission-controller-template.yaml new file mode 100644 index 0000000000..be68bcea14 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/kubecost-admission-controller-template.yaml @@ -0,0 +1,30 @@ +{{- if .Values.kubecostAdmissionController -}} +{{- if .Values.kubecostAdmissionController.enabled -}} +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: kubecost-deployment-validation +webhooks: + - name: "kubecost-deployment-validation.kubecost.svc" + failurePolicy: Ignore + rules: + - operations: [ "CREATE", "UPDATE" ] + apiGroups: [ "apps" ] + apiVersions: [ "v1" ] + resources: [ "deployments" ] + scope: "*" + clientConfig: + service: + namespace: {{.Release.Namespace}} + name: webhook-server + path: "/validate" + {{- if .Values.kubecostAdmissionController.caBundle }} + caBundle: {{ .Values.kubecostAdmissionController.caBundle }} + {{- else }} + caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURCRENDQWV5Z0F3SUJBZ0lVR3E2YkdOaEowVjRsb0NiWHhUa0pocWkwUnB3d0RRWUpLb1pJaHZjTkFRRUwKQlFBd0pqRWtNQ0lHQTFVRUF3d2JkMlZpYUc5dmF5MXpaWEoyWlhJdWEzVmlaV052YzNRdWMzWmpNQjRYRFRJegpNREl3T1RFNU1UVTFNbG9YRFRJME1EWXlNekU1TVRVMU1sb3dKakVrTUNJR0ExVUVBd3diZDJWaWFHOXZheTF6ClpYSjJaWEl1YTNWaVpXTnZjM1F1YzNaak1JSUJJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBUThBTUlJQkNnS0MKQVFFQXpvU2JBejBhZFJTdEN3eVRPSGd2S2VuQ29GbWE2OC9nYTFHZjVST2dXeGJhamhQRTZKbEtBcENwK1pzKwo2bHJzL2J3bkx5SDdoMUFJa1NmZ25EYlNadDJjdHRFSmhSd25vKy90WElMYk84WndRQTErYXpUQzVtSkluZVF3CktRMkErYy9CUnk3N3B0SnZIRStkTEllcWhRelV2M25nWUwvSDZaMUZPa20xUCtlR0FwSWxyVHVPV1ozUVhRYkMKemhOQXppRWNjL3o3RERBdlFBMlpIQ1I2OGl1V0ptd0RYZEdjWmEwenNVb1hDbGIvWXdiWFgvMlp2dklIbkdtawp5VTlZdEhxNVpscFZjT0V5MTVBWFVEOFZVUU1jVXQ5NkJvVThMMXJKbTZJK0E0YmFySEs5QjlxcjdzRmFaY2wvCnBncHZGd0NBaHZHYUM2VzA5UnM3T0NrdXh3SURBUUFCb3lvd0tEQW1CZ05WSFJFRUh6QWRnaHQzWldKb2IyOXIKTFhObGNuWmxjaTVyZFdKbFkyOXpkQzV6ZG1Nd0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFDdVhNcUgzYmhsVApGKzlRUFplS2xiUTZlWSs0NlhMVGtEdlZzenAyZysweWhlMVNRRHZRUTVad1l6MnMwODNqb2loTXVzeFZ1TmFGCk1LdE9vbGY2bitsaUZFcEw4OU9XZ1VjdzJRdFdqVWUraU1zby91dWN0eGVPTzZLam9JcUVrUlg5YXh1cGxxVm0KakZRaGZtNlRYZ2pxWmttUVNsbHdLVkcxSFJZTkRveFpFa0JHK1l6RWF5QmdQdXl4bW5iTDdlck5IOVJQSVZtbAoxaWFnS1NVVG5vN0hJY3IwdHYzT3JEWDZRN3VJUGdWanBRSHMzNXBZSWlBYjVNR0RjWFZvY050SEZ0YnluREhzCi80WGhYMjFhOXdnSVF6dUF3ck0zQ0VDRnVocHJzWlZmQjBKQ1dBOG1aVEZneTVBL0tLUjJmTXRMRWRQS1ZsSXUKZjc1MjB3T3JzME09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K + {{- end }} + admissionReviewVersions: ["v1"] + sideEffects: None + timeoutSeconds: 5 +{{- end -}} +{{- end -}} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/kubecost-cluster-context-switcher.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/kubecost-cluster-context-switcher.yaml new file mode 100644 index 0000000000..2179d221db --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/kubecost-cluster-context-switcher.yaml @@ -0,0 +1,14 @@ +{{- if .Values.kubecostProductConfigs }} +{{- if .Values.kubecostProductConfigs.clusters }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: kubecost-clusters + namespace: {{ .Release.Namespace }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} +data: + default-clusters.yaml: | +{{- toYaml .Values.kubecostProductConfigs.clusters | nindent 4 }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/kubecost-cluster-controller-actions-config.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/kubecost-cluster-controller-actions-config.yaml new file mode 100644 index 0000000000..114f381b02 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/kubecost-cluster-controller-actions-config.yaml @@ -0,0 +1,56 @@ +{{- if .Values.clusterController }} +{{- if .Values.clusterController.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: cluster-controller-continuous-cluster-sizing + namespace: {{ .Release.Namespace }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} +{{- if .Values.clusterController.actionConfigs.clusterRightsize }} +binaryData: + config: | +{{- toJson .Values.clusterController.actionConfigs.clusterRightsize | b64enc | nindent 4 }} +{{- end }} +--- + +apiVersion: v1 +kind: ConfigMap +metadata: + name: cluster-controller-nsturndown-config + namespace: {{ .Release.Namespace }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} +{{- if .Values.clusterController.actionConfigs.namespaceTurndown }} +binaryData: +{{- range .Values.clusterController.actionConfigs.namespaceTurndown }} + {{ .name }}: | + {{- toJson . | b64enc | nindent 4 }} +{{- end }} +{{- end }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: cluster-controller-container-rightsizing-config + namespace: {{ .Release.Namespace }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} +{{- if .Values.clusterController.actionConfigs.containerRightsize }} +binaryData: + config: | +{{- toJson .Values.clusterController.actionConfigs.containerRightsize | b64enc | nindent 4 }} +{{- end }} +{{- range .Values.clusterController.actionConfigs.clusterTurndown }} +--- +apiVersion: kubecost.com/v1alpha1 +kind: TurndownSchedule +metadata: + name: {{ .name }} +spec: + start: {{ .start }} + end: {{ .end }} + repeat: {{ .repeat }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/kubecost-cluster-controller-template.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/kubecost-cluster-controller-template.yaml new file mode 100644 index 0000000000..f15c70674b --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/kubecost-cluster-controller-template.yaml @@ -0,0 +1,310 @@ +{{- if .Values.clusterController }} +{{- if .Values.clusterController.enabled }} +{{- $serviceName := include "cost-analyzer.serviceName" . -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kubecost.clusterControllerName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} +--- +# +# NOTE: +# The following ClusterRole permissions are only created and assigned for the +# cluster controller feature. They will not be added to any clusters by default. +# +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kubecost.clusterControllerName" . }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} +rules: + - apiGroups: + - kubecost.com + resources: + - turndownschedules + - turndownschedules/status + verbs: + - get + - list + - watch + - create + - patch + - update + - delete + - apiGroups: + - '' + - events.k8s.io + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - '' + resources: + - deployments + - nodes + - pods + - resourcequotas + - replicationcontrollers + - limitranges + - pods/eviction + verbs: + - get + - list + - watch + - create + - patch + - update + - delete + - apiGroups: + - '' + resources: + - configmaps + verbs: + - get + - list + - watch + - update + - patch + - delete + - apiGroups: + - '' + resources: + - configmaps + - namespaces + - persistentvolumeclaims + - persistentvolumes + - endpoints + - events + - services + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - configmaps + resourceNames: + - 'cluster-controller-nsturndown-config' + verbs: + - get + - create + - update + - apiGroups: + - apps + resources: + - statefulsets + - deployments + - daemonsets + - replicasets + verbs: + - get + - list + - watch + - create + - patch + - update + - delete + - apiGroups: + - batch + resources: + - cronjobs + - jobs + verbs: + - get + - list + - watch + - create + - patch + - update + - delete + - apiGroups: + - autoscaling + resources: + - horizontalpodautoscalers + verbs: + - get + - list + - watch + - apiGroups: + - policy + resources: + - poddisruptionbudgets + verbs: + - get + - list + - watch + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - events.k8s.io + resources: + - events + verbs: + - get + - list + - watch + # Used for namespace turndown + # When cleaning a namespace, we need the ability to remove + # arbitrary resources (since we helm uninstall all releases in that NS first) + {{- if .Values.clusterController.namespaceTurndown.rbac.enabled }} + - apiGroups: ["*"] + resources: ["*"] + verbs: + - list + - get + - delete + {{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kubecost.clusterControllerName" . }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kubecost.clusterControllerName" . }} +subjects: + - kind: ServiceAccount + name: {{ template "kubecost.clusterControllerName" . }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "kubecost.clusterControllerName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} + annotations: + {{- with .Values.global.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.clusterController.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: RollingUpdate + selector: + matchLabels: + app: {{ template "kubecost.clusterControllerName" . }} + template: + metadata: + labels: + app: {{ template "kubecost.clusterControllerName" . }} + {{- with .Values.global.podAnnotations}} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- if .Values.clusterController.priorityClassName }} + priorityClassName: "{{ .Values.clusterController.priorityClassName }}" + {{- end }} + containers: + - name: {{ template "kubecost.clusterControllerName" . }} + {{- if eq (typeOf .Values.clusterController.image) "string" }} + image: {{ .Values.clusterController.image }} + {{- else }} + image: {{ .Values.clusterController.image.repository }}:{{ .Values.clusterController.image.tag }} + {{- end}} + imagePullPolicy: {{ .Values.clusterController.imagePullPolicy }} + volumeMounts: + - name: cluster-controller-keys + mountPath: /var/keys + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CLUSTER_ID + value: {{ .Values.prometheus.server.global.external_labels.cluster_id }} + - name: TURNDOWN_NAMESPACE + value: {{ .Release.Namespace }} + - name: TURNDOWN_DEPLOYMENT + value: {{ template "kubecost.clusterControllerName" . }} + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /var/keys/service-key.json + - name: CC_LOG_LEVEL + value: {{ .Values.clusterController.logLevel | default "info" }} + - name: CC_KUBESCALER_COST_MODEL_PATH + value: http://{{ $serviceName }}.{{ .Release.Namespace }}:{{ .Values.service.targetPort | default 9090 }}/model + - name: CC_CCL_COST_MODEL_PATH + value: http://{{ $serviceName }}.{{ .Release.Namespace }}:{{ .Values.service.targetPort | default 9090 }}/model + {{- if .Values.clusterController.kubescaler }} + - name: CC_KUBESCALER_DEFAULT_RESIZE_ALL + value: {{ .Values.clusterController.kubescaler.defaultResizeAll | default "false" | quote }} + {{- end }} + ports: + - name: http-server + containerPort: 9731 + hostPort: 9731 + resources: + {{- toYaml .Values.clusterController.resources | nindent 12 }} + serviceAccount: {{ template "kubecost.clusterControllerName" . }} + serviceAccountName: {{ template "kubecost.clusterControllerName" . }} + {{- with .Values.clusterController.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.clusterController.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.clusterController.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: cluster-controller-keys + secret: + secretName: {{ .Values.clusterController.secretName | default "cluster-controller-service-key" }} + # The secret is optional because not all of cluster controller's + # functionality requires this secret. Cluster controller will + # partially or fully initialize based on the presence of these keys + # and their validity. + optional: true +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kubecost.clusterControllerName" . }}-service + namespace: {{ .Release.Namespace }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} +spec: + type: ClusterIP + ports: + - name: http + protocol: TCP + port: 9731 + targetPort: 9731 + selector: + app: {{ template "kubecost.clusterControllerName" . }} +{{- end }} +{{- end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/kubecost-oidc-secret-template.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/kubecost-oidc-secret-template.yaml new file mode 100644 index 0000000000..4735d951c9 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/kubecost-oidc-secret-template.yaml @@ -0,0 +1,16 @@ +{{- if .Values.oidc }} +{{- if and (not .Values.oidc.existingCustomSecret.enabled) .Values.oidc.secretName }} +{{- if .Values.oidc.clientSecret }} +apiVersion: v1 +kind: Secret +type: Opaque +metadata: + name: {{ .Values.oidc.secretName }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} +stringData: + clientSecret: {{ .Values.oidc.clientSecret }} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/kubecost-priority-class-template.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/kubecost-priority-class-template.yaml new file mode 100644 index 0000000000..9710b5a225 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/kubecost-priority-class-template.yaml @@ -0,0 +1,15 @@ +{{- if .Values.priority }} +{{- if .Values.priority.enabled }} +{{- if eq (len .Values.priority.name) 0 }} +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: {{ template "cost-analyzer.fullname" . }}-priority + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} +value: {{ .Values.priority.value | default "1000000" }} +globalDefault: false +description: "Priority class for scheduling the cost-analyzer pod" +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/kubecost-saml-secret-template.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/kubecost-saml-secret-template.yaml new file mode 100644 index 0000000000..cd6c9d3d7d --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/kubecost-saml-secret-template.yaml @@ -0,0 +1,12 @@ +{{- if .Values.saml.enabled }} +apiVersion: v1 +kind: Secret +type: Opaque +metadata: + name: {{ .Values.saml.authSecretName | default "kubecost-saml-secret" }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} +stringData: + clientSecret: {{ .Values.saml.authSecret | default (randAlphaNum 32 | quote) }} +{{- end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/mimir-proxy-configmap-template.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/mimir-proxy-configmap-template.yaml new file mode 100644 index 0000000000..08b93ee84e --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/mimir-proxy-configmap-template.yaml @@ -0,0 +1,21 @@ +{{- if .Values.global.mimirProxy }} +{{- if .Values.global.mimirProxy.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "cost-analyzer.fullname" . }}-mimir-proxy + namespace: {{ .Release.Namespace }} +data: + default.conf: | + server { + listen {{ .Values.global.mimirProxy.port }}; + location / { + proxy_pass {{ .Values.global.mimirProxy.mimirEndpoint }}; + proxy_set_header X-Scope-OrgID "{{ .Values.global.mimirProxy.orgIdentifier }}"; + {{- if .Values.global.mimirProxy.basicAuth }} + proxy_set_header Authorization "Basic {{ (printf "%s:%s" .Values.global.mimirProxy.basicAuth.username .Values.global.mimirProxy.basicAuth.password) | b64enc }}"; + {{- end }} + } + } +{{- end }} +{{- end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/mimir-proxy-deployment-template.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/mimir-proxy-deployment-template.yaml new file mode 100644 index 0000000000..b2ae81c26e --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/mimir-proxy-deployment-template.yaml @@ -0,0 +1,57 @@ +{{- if .Values.global.mimirProxy }} +{{- if .Values.global.mimirProxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "cost-analyzer.fullname" . }}-mimir-proxy + namespace: {{ .Release.Namespace }} + labels: + app: mimir-proxy + {{- with .Values.global.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- with .Values.global.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.global.mimirProxy.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: 1 + selector: + matchLabels: + app: mimir-proxy + template: + metadata: + labels: + app: mimir-proxy + {{- with .Values.global.additionalLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + {{- with .Values.global.podAnnotations}} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + containers: + - name: {{ .Values.global.mimirProxy.name }} + image: {{ .Values.global.mimirProxy.image }} + ports: + - containerPort: {{ .Values.global.mimirProxy.port }} + protocol: TCP + resources: {} + imagePullPolicy: IfNotPresent + volumeMounts: + - mountPath: /etc/nginx/conf.d + name: default-conf + readOnly: true + volumes: + - name: default-conf + configMap: + name: {{ template "cost-analyzer.fullname" . }}-mimir-proxy + items: + - key: default.conf + path: default.conf +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/mimir-proxy-service-template.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/mimir-proxy-service-template.yaml new file mode 100644 index 0000000000..5e46b62f99 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/mimir-proxy-service-template.yaml @@ -0,0 +1,18 @@ +{{- if .Values.global.mimirProxy }} +{{- if .Values.global.mimirProxy.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "cost-analyzer.fullname" . }}-mimir-proxy + namespace: {{ .Release.Namespace }} +spec: + ports: + - name: mimir-proxy + protocol: TCP + port: {{ .Values.global.mimirProxy.port }} + targetPort: {{ .Values.global.mimirProxy.port }} + selector: + app: mimir-proxy + type: ClusterIP +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/monitoring-role-binding-template.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/monitoring-role-binding-template.yaml new file mode 100644 index 0000000000..72c22f354d --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/monitoring-role-binding-template.yaml @@ -0,0 +1,17 @@ +{{- if (.Values.global.platforms.openshift.enabled) }} +{{- if (.Values.global.platforms.openshift.createMonitoringResourceReaderRoleBinding) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + namespace: {{ .Release.Namespace }} + name: {{ template "cost-analyzer.fullname" . }}-reader +subjects: +- kind: ServiceAccount + name: {{ .Values.global.platforms.openshift.monitoringServiceAccountName | quote }} + namespace: {{ .Values.global.platforms.openshift.monitoringServiceAccountNamespace | quote }} +roleRef: + kind: Role + name: {{ template "cost-analyzer.fullname" . }}-reader + apiGroup: rbac.authorization.k8s.io +{{- end -}} +{{- end -}} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/monitoring-role-template.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/monitoring-role-template.yaml new file mode 100644 index 0000000000..5890eb27a1 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/monitoring-role-template.yaml @@ -0,0 +1,19 @@ +{{- if (.Values.global.platforms.openshift.enabled) }} +{{- if (.Values.global.platforms.openshift.createMonitoringResourceReaderRoleBinding) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + namespace: {{ .Release.Namespace }} + name: {{ template "cost-analyzer.fullname" . }}-reader +rules: + - apiGroups: + - '' + resources: + - "pods" + - "services" + - "endpoints" + verbs: + - list + - watch +{{- end -}} +{{- end -}} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/network-costs-servicemonitor-template.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/network-costs-servicemonitor-template.yaml new file mode 100644 index 0000000000..53c9a89b85 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/network-costs-servicemonitor-template.yaml @@ -0,0 +1,32 @@ +{{- if .Values.serviceMonitor.networkCosts.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "cost-analyzer.networkCostsName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} + {{- if .Values.serviceMonitor.networkCosts.additionalLabels }} + {{ toYaml .Values.serviceMonitor.networkCosts.additionalLabels | nindent 4 }} + {{- end }} +spec: + endpoints: + - port: metrics + honorLabels: true + interval: {{ .Values.serviceMonitor.networkCosts.interval }} + scrapeTimeout: {{ .Values.serviceMonitor.networkCosts.scrapeTimeout }} + path: /metrics + scheme: http + {{- with .Values.serviceMonitor.networkCosts.metricRelabelings }} + metricRelabelings: {{ toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.serviceMonitor.networkCosts.relabelings }} + relabelings: {{ toYaml . | nindent 8 }} + {{- end }} + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} + selector: + matchLabels: + app: {{ include "cost-analyzer.networkCostsName" . }} +{{- end }} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/plugins-config.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/plugins-config.yaml new file mode 100644 index 0000000000..1a37f8b78d --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/plugins-config.yaml @@ -0,0 +1,13 @@ +{{- if and (not .Values.kubecostModel.plugins.existingCustomSecret.enabled) .Values.kubecostModel.plugins.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.kubecostModel.plugins.secretName }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} +data: + {{- range $key, $config := .Values.kubecostModel.plugins.configs }} + {{ $key }}_config.json: + {{ $config | b64enc | indent 4}} + {{- end }} +{{- end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-alertmanager-configmap.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-alertmanager-configmap.yaml new file mode 100644 index 0000000000..8f5b8315f3 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-alertmanager-configmap.yaml @@ -0,0 +1,21 @@ +{{ if .Values.global.prometheus.enabled }} +{{- if and .Values.prometheus.alertmanager.enabled (and (empty .Values.prometheus.alertmanager.configMapOverrideName) (empty .Values.prometheus.alertmanager.configFromSecret)) -}} +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + {{- include "prometheus.alertmanager.labels" . | nindent 4 }} + name: {{ template "prometheus.alertmanager.fullname" . }} + namespace: {{ .Release.Namespace }} +data: +{{- $root := . -}} +{{- range $key, $value := .Values.prometheus.alertmanagerFiles }} + {{- if $key | regexMatch ".*\\.ya?ml$" }} + {{ $key }}: | +{{ toYaml $value | default "{}" | indent 4 }} + {{- else }} + {{ $key }}: {{ toYaml $value | indent 4 }} + {{- end }} +{{- end -}} +{{- end -}} +{{ end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-alertmanager-deployment.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-alertmanager-deployment.yaml new file mode 100644 index 0000000000..fd70f11091 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-alertmanager-deployment.yaml @@ -0,0 +1,160 @@ +{{ if .Values.global.prometheus.enabled }} +{{- if and .Values.prometheus.alertmanager.enabled (not .Values.prometheus.alertmanager.statefulSet.enabled) -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + {{- include "prometheus.alertmanager.labels" . | nindent 4 }} + {{- with .Values.global.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- with .Values.global.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.prometheus.alertmanager.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ template "prometheus.alertmanager.fullname" . }} + namespace: {{ .Release.Namespace }} +spec: + selector: + matchLabels: + {{- include "prometheus.alertmanager.matchLabels" . | nindent 6 }} + replicas: {{ .Values.prometheus.alertmanager.replicaCount }} + {{- if .Values.prometheus.alertmanager.strategy }} + strategy: +{{ toYaml .Values.prometheus.alertmanager.strategy | indent 4 }} + {{- end }} + template: + metadata: + annotations: + {{- with .Values.global.podAnnotations}} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.prometheus.alertmanager.podAnnotations }} + {{ toYaml .Values.prometheus.alertmanager.podAnnotations | indent 8 }} + {{- end }} + labels: + {{- include "prometheus.alertmanager.labels" . | nindent 8 }} + {{- with .Values.global.additionalLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.prometheus.alertmanager.podLabels}} + {{ toYaml .Values.prometheus.alertmanager.podLabels | nindent 8 }} + {{- end}} + spec: +{{- if .Values.prometheus.alertmanager.schedulerName }} + schedulerName: "{{ .Values.prometheus.alertmanager.schedulerName }}" +{{- end }} + serviceAccountName: {{ template "prometheus.serviceAccountName.alertmanager" . }} +{{- if .Values.prometheus.alertmanager.priorityClassName }} + priorityClassName: "{{ .Values.prometheus.alertmanager.priorityClassName }}" +{{- end }} + containers: + - name: {{ template "prometheus.name" . }}-{{ .Values.prometheus.alertmanager.name }} + image: "{{ .Values.prometheus.alertmanager.image.repository }}:{{ .Values.prometheus.alertmanager.image.tag }}" + imagePullPolicy: "{{ .Values.prometheus.alertmanager.image.pullPolicy }}" + env: + {{- range $key, $value := .Values.prometheus.alertmanager.extraEnv }} + - name: {{ $key }} + value: {{ $value }} + {{- end }} + - name: POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + args: + - --config.file=/etc/config/{{ .Values.prometheus.alertmanager.configFileName }} + - --storage.path={{ .Values.prometheus.alertmanager.persistentVolume.mountPath }} + - --cluster.advertise-address=$(POD_IP):6783 + {{- range $key, $value := .Values.prometheus.alertmanager.extraArgs }} + - --{{ $key }}={{ $value }} + {{- end }} + {{- if .Values.prometheus.alertmanager.baseURL }} + - --web.external-url={{ .Values.prometheus.alertmanager.baseURL }} + {{- end }} + + ports: + - containerPort: 9093 + readinessProbe: + httpGet: + path: {{ .Values.prometheus.alertmanager.prefixURL }}/-/ready + port: 9093 + initialDelaySeconds: 30 + timeoutSeconds: 30 + resources: +{{ toYaml .Values.prometheus.alertmanager.resources | indent 12 }} + volumeMounts: + - name: config-volume + mountPath: /etc/config + - name: storage-volume + mountPath: "{{ .Values.prometheus.alertmanager.persistentVolume.mountPath }}" + subPath: "{{ .Values.prometheus.alertmanager.persistentVolume.subPath }}" + {{- range .Values.prometheus.alertmanager.extraSecretMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + subPath: {{ .subPath }} + readOnly: {{ .readOnly }} + {{- end }} + + {{- if .Values.prometheus.configmapReload.alertmanager.enabled }} + - name: {{ template "prometheus.name" . }}-{{ .Values.prometheus.alertmanager.name }}-{{ .Values.prometheus.configmapReload.alertmanager.name }} + image: "{{ .Values.prometheus.configmapReload.alertmanager.image.repository }}:{{ .Values.prometheus.configmapReload.alertmanager.image.tag }}" + imagePullPolicy: "{{ .Values.prometheus.configmapReload.alertmanager.image.pullPolicy }}" + args: + - --watched-dir=/etc/config + - --reload-url=http://127.0.0.1:9093{{ .Values.prometheus.alertmanager.prefixURL }}/-/reload + resources: +{{ toYaml .Values.prometheus.configmapReload.alertmanager.resources | indent 12 }} + volumeMounts: + - name: config-volume + mountPath: /etc/config + readOnly: true + {{- end }} + {{- if .Values.prometheus.imagePullSecrets }} + imagePullSecrets: + {{- range $.Values.prometheus.imagePullSecrets}} + - name: {{ .name }} + {{- end }} + {{- end }} + {{- if .Values.prometheus.alertmanager.nodeSelector }} + nodeSelector: +{{ toYaml .Values.prometheus.alertmanager.nodeSelector | indent 8 }} + {{- end }} + {{- if .Values.prometheus.alertmanager.securityContext }} + securityContext: +{{ toYaml .Values.prometheus.alertmanager.securityContext | indent 8 }} + {{- end }} + {{- if .Values.prometheus.alertmanager.tolerations }} + tolerations: +{{ toYaml .Values.prometheus.alertmanager.tolerations | indent 8 }} + {{- end }} + {{- if .Values.prometheus.alertmanager.affinity }} + affinity: +{{ toYaml .Values.prometheus.alertmanager.affinity | indent 8 }} + {{- end }} + volumes: + - name: config-volume + {{- if empty .Values.prometheus.alertmanager.configFromSecret }} + configMap: + name: {{ if .Values.prometheus.alertmanager.configMapOverrideName }}{{ .Release.Name }}-{{ .Values.prometheus.alertmanager.configMapOverrideName }}{{- else }}{{ template "prometheus.alertmanager.fullname" . }}{{- end }} + {{- else }} + secret: + secretName: {{ .Values.prometheus.alertmanager.configFromSecret }} + {{- end }} + {{- range .Values.prometheus.alertmanager.extraSecretMounts }} + - name: {{ .name }} + secret: + secretName: {{ .secretName }} + {{- end }} + - name: storage-volume + {{- if .Values.prometheus.alertmanager.persistentVolume.enabled }} + persistentVolumeClaim: + claimName: {{ if .Values.prometheus.alertmanager.persistentVolume.existingClaim }}{{ .Values.prometheus.alertmanager.persistentVolume.existingClaim }}{{- else }}{{ template "prometheus.alertmanager.fullname" . }}{{- end }} + {{- else }} + emptyDir: {} + {{- end -}} +{{- end }} +{{ end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-alertmanager-ingress.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-alertmanager-ingress.yaml new file mode 100644 index 0000000000..41757e0e13 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-alertmanager-ingress.yaml @@ -0,0 +1,41 @@ +{{ if .Values.global.prometheus.enabled }} +{{- if and .Values.prometheus.alertmanager.enabled .Values.prometheus.alertmanager.ingress.enabled -}} +{{- $releaseName := .Release.Name -}} +{{- $serviceName := include "prometheus.alertmanager.fullname" . }} +{{- $servicePort := .Values.prometheus.alertmanager.service.servicePort -}} +{{- $extraPaths := .Values.prometheus.alertmanager.ingress.extraPaths -}} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: +{{- if .Values.prometheus.alertmanager.ingress.annotations }} + annotations: +{{ toYaml .Values.prometheus.alertmanager.ingress.annotations | indent 4 }} +{{- end }} + labels: + {{- include "prometheus.alertmanager.labels" . | nindent 4 }} +{{- range $key, $value := .Values.prometheus.alertmanager.ingress.extraLabels }} + {{ $key }}: {{ $value }} +{{- end }} + name: {{ template "prometheus.alertmanager.fullname" . }} + namespace: {{ .Release.Namespace }} +spec: + rules: + {{- range .Values.prometheus.alertmanager.ingress.hosts }} + {{- $url := splitList "/" . }} + - host: {{ first $url }} + http: + paths: +{{ if $extraPaths }} +{{ toYaml $extraPaths | indent 10 }} +{{- end }} + - path: /{{ rest $url | join "/" }} + backend: + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end -}} +{{- if .Values.prometheus.alertmanager.ingress.tls }} + tls: +{{ toYaml .Values.prometheus.alertmanager.ingress.tls | indent 4 }} + {{- end -}} +{{- end -}} +{{ end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-alertmanager-networkpolicy.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-alertmanager-networkpolicy.yaml new file mode 100644 index 0000000000..c24a76ae70 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-alertmanager-networkpolicy.yaml @@ -0,0 +1,22 @@ +{{ if .Values.global.prometheus.enabled }} +{{- if and .Values.prometheus.alertmanager.enabled .Values.prometheus.networkPolicy.enabled -}} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ template "prometheus.alertmanager.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "prometheus.alertmanager.labels" . | nindent 4 }} +spec: + podSelector: + matchLabels: + {{- include "prometheus.alertmanager.matchLabels" . | nindent 6 }} + ingress: + - from: + - podSelector: + matchLabels: + {{- include "prometheus.server.matchLabels" . | nindent 12 }} + - ports: + - port: 9093 +{{- end -}} +{{ end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-alertmanager-pdb.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-alertmanager-pdb.yaml new file mode 100644 index 0000000000..123d24ee01 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-alertmanager-pdb.yaml @@ -0,0 +1,16 @@ +{{ if .Values.global.prometheus.enabled }} +{{- if .Values.prometheus.alertmanager.podDisruptionBudget.enabled }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ template "prometheus.alertmanager.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "prometheus.alertmanager.labels" . | nindent 4 }} +spec: + maxUnavailable: {{ .Values.prometheus.alertmanager.podDisruptionBudget.maxUnavailable }} + selector: + matchLabels: + {{- include "prometheus.alertmanager.labels" . | nindent 6 }} +{{- end }} +{{ end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-alertmanager-pvc.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-alertmanager-pvc.yaml new file mode 100644 index 0000000000..dea65e5e5e --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-alertmanager-pvc.yaml @@ -0,0 +1,35 @@ +{{ if .Values.global.prometheus.enabled }} +{{- if not .Values.prometheus.alertmanager.statefulSet.enabled -}} +{{- if and .Values.prometheus.alertmanager.enabled .Values.prometheus.alertmanager.persistentVolume.enabled -}} +{{- if not .Values.prometheus.alertmanager.persistentVolume.existingClaim -}} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + {{- if .Values.prometheus.alertmanager.persistentVolume.annotations }} + annotations: +{{ toYaml .Values.prometheus.alertmanager.persistentVolume.annotations | indent 4 }} + {{- end }} + labels: + {{- include "prometheus.alertmanager.labels" . | nindent 4 }} + name: {{ template "prometheus.alertmanager.fullname" . }} + namespace: {{ .Release.Namespace }} +spec: + accessModes: +{{ toYaml .Values.prometheus.alertmanager.persistentVolume.accessModes | indent 4 }} +{{- if .Values.prometheus.alertmanager.persistentVolume.storageClass }} +{{- if (eq "-" .Values.prometheus.alertmanager.persistentVolume.storageClass) }} + storageClassName: "" +{{- else }} + storageClassName: "{{ .Values.prometheus.alertmanager.persistentVolume.storageClass }}" +{{- end }} +{{- end }} +{{- if .Values.prometheus.alertmanager.persistentVolume.volumeBindingMode }} + volumeBindingModeName: "{{ .Values.prometheus.alertmanager.persistentVolume.volumeBindingMode }}" +{{- end }} + resources: + requests: + storage: "{{ .Values.prometheus.alertmanager.persistentVolume.size }}" +{{- end -}} +{{- end -}} +{{- end -}} +{{ end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-alertmanager-service-headless.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-alertmanager-service-headless.yaml new file mode 100644 index 0000000000..2f68f41260 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-alertmanager-service-headless.yaml @@ -0,0 +1,33 @@ +{{ if .Values.global.prometheus.enabled }} +{{- if and .Values.prometheus.alertmanager.enabled .Values.prometheus.alertmanager.statefulSet.enabled -}} +apiVersion: v1 +kind: Service +metadata: +{{- if .Values.prometheus.alertmanager.statefulSet.headless.annotations }} + annotations: +{{ toYaml .Values.prometheus.alertmanager.statefulSet.headless.annotations | indent 4 }} +{{- end }} + labels: + {{- include "prometheus.alertmanager.labels" . | nindent 4 }} +{{- if .Values.prometheus.alertmanager.statefulSet.headless.labels }} +{{ toYaml .Values.prometheus.alertmanager.statefulSet.headless.labels | indent 4 }} +{{- end }} + name: {{ template "prometheus.alertmanager.fullname" . }}-headless + namespace: {{ .Release.Namespace }} +spec: + clusterIP: None + ports: + - name: http + port: {{ .Values.prometheus.alertmanager.statefulSet.headless.servicePort }} + protocol: TCP + targetPort: 9093 +{{- if .Values.prometheus.alertmanager.statefulSet.headless.enableMeshPeer }} + - name: meshpeer + port: 6783 + protocol: TCP + targetPort: 6783 +{{- end }} + selector: + {{- include "prometheus.alertmanager.matchLabels" . | nindent 4 }} +{{- end }} +{{ end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-alertmanager-service.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-alertmanager-service.yaml new file mode 100644 index 0000000000..838d39ba44 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-alertmanager-service.yaml @@ -0,0 +1,55 @@ +{{ if .Values.global.prometheus.enabled }} +{{- if .Values.prometheus.alertmanager.enabled -}} +apiVersion: v1 +kind: Service +metadata: +{{- if .Values.prometheus.alertmanager.service.annotations }} + annotations: +{{ toYaml .Values.prometheus.alertmanager.service.annotations | indent 4 }} +{{- end }} + labels: + {{- include "prometheus.alertmanager.labels" . | nindent 4 }} +{{- if .Values.prometheus.alertmanager.service.labels }} +{{ toYaml .Values.prometheus.alertmanager.service.labels | indent 4 }} +{{- end }} + name: {{ template "prometheus.alertmanager.fullname" . }} + namespace: {{ .Release.Namespace }} +spec: +{{- if .Values.prometheus.alertmanager.service.clusterIP }} + clusterIP: {{ .Values.prometheus.alertmanager.service.clusterIP }} +{{- end }} +{{- if .Values.prometheus.alertmanager.service.externalIPs }} + externalIPs: +{{ toYaml .Values.prometheus.alertmanager.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.prometheus.alertmanager.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.prometheus.alertmanager.service.loadBalancerIP }} +{{- end }} +{{- if .Values.prometheus.alertmanager.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.prometheus.alertmanager.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} + ports: + - name: http + port: {{ .Values.prometheus.alertmanager.service.servicePort }} + protocol: TCP + targetPort: 9093 + {{- if .Values.prometheus.alertmanager.service.nodePort }} + nodePort: {{ .Values.prometheus.alertmanager.service.nodePort }} + {{- end }} +{{- if .Values.prometheus.alertmanager.service.enableMeshPeer }} + - name: meshpeer + port: 6783 + protocol: TCP + targetPort: 6783 +{{- end }} + selector: + {{- include "prometheus.alertmanager.matchLabels" . | nindent 4 }} +{{- if .Values.prometheus.alertmanager.service.sessionAffinity }} + sessionAffinity: {{ .Values.prometheus.alertmanager.service.sessionAffinity }} +{{- end }} + type: "{{ .Values.prometheus.alertmanager.service.type }}" +{{- end }} +{{ end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-alertmanager-serviceaccount.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-alertmanager-serviceaccount.yaml new file mode 100644 index 0000000000..99257bbf88 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-alertmanager-serviceaccount.yaml @@ -0,0 +1,11 @@ +{{ if .Values.global.prometheus.enabled }} +{{- if and .Values.prometheus.alertmanager.enabled .Values.prometheus.serviceAccounts.alertmanager.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + {{- include "prometheus.alertmanager.labels" . | nindent 4 }} + name: {{ template "prometheus.serviceAccountName.alertmanager" . }} + namespace: {{ .Release.Namespace }} +{{- end -}} +{{ end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-alertmanager-statefulset.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-alertmanager-statefulset.yaml new file mode 100644 index 0000000000..9070fef1e7 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-alertmanager-statefulset.yaml @@ -0,0 +1,167 @@ +{{ if .Values.global.prometheus.enabled }} +{{- if and .Values.prometheus.alertmanager.enabled .Values.prometheus.alertmanager.statefulSet.enabled -}} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + {{- include "prometheus.alertmanager.labels" . | nindent 4 }} + annotations: + {{- with .Values.global.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.prometheus.alertmanager.statefulSet.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ template "prometheus.alertmanager.fullname" . }} + namespace: {{ .Release.Namespace }} +spec: + serviceName: {{ template "prometheus.alertmanager.fullname" . }}-headless + selector: + matchLabels: + {{- include "prometheus.alertmanager.matchLabels" . | nindent 6 }} + replicas: {{ .Values.prometheus.alertmanager.replicaCount }} + podManagementPolicy: {{ .Values.prometheus.alertmanager.statefulSet.podManagementPolicy }} + template: + metadata: + annotations: + {{- with .Values.global.podAnnotations}} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.prometheus.alertmanager.podAnnotations }} + {{ toYaml .Values.prometheus.alertmanager.podAnnotations | nindent 8 }} + {{- end }} + labels: + {{- include "prometheus.alertmanager.labels" . | nindent 8 }} + spec: +{{- if .Values.prometheus.alertmanager.affinity }} + affinity: +{{ toYaml .Values.prometheus.alertmanager.affinity | indent 8 }} +{{- end }} +{{- if .Values.prometheus.alertmanager.schedulerName }} + schedulerName: "{{ .Values.prometheus.alertmanager.schedulerName }}" +{{- end }} + serviceAccountName: {{ template "prometheus.serviceAccountName.alertmanager" . }} +{{- if .Values.prometheus.alertmanager.priorityClassName }} + priorityClassName: "{{ .Values.prometheus.alertmanager.priorityClassName }}" +{{- end }} + containers: + - name: {{ template "prometheus.name" . }}-{{ .Values.prometheus.alertmanager.name }} + image: "{{ .Values.prometheus.alertmanager.image.repository }}:{{ .Values.prometheus.alertmanager.image.tag }}" + imagePullPolicy: "{{ .Values.prometheus.alertmanager.image.pullPolicy }}" + env: + {{- range $key, $value := .Values.prometheus.alertmanager.extraEnv }} + - name: {{ $key }} + value: {{ $value }} + {{- end }} + - name: POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + args: + - --config.file=/etc/config/alertmanager.yml + - --storage.path={{ .Values.prometheus.alertmanager.persistentVolume.mountPath }} + - --cluster.advertise-address=$(POD_IP):6783 + {{- if .Values.prometheus.alertmanager.statefulSet.headless.enableMeshPeer }} + - --cluster.listen-address=0.0.0.0:6783 + {{- range $n := until (.Values.prometheus.alertmanager.replicaCount | int) }} + - --cluster.peer={{ template "prometheus.alertmanager.fullname" $ }}-{{ $n }}.{{ template "prometheus.alertmanager.fullname" $ }}-headless:6783 + {{- end }} + {{- end }} + {{- range $key, $value := .Values.prometheus.alertmanager.extraArgs }} + - --{{ $key }}={{ $value }} + {{- end }} + {{- if .Values.prometheus.alertmanager.baseURL }} + - --web.external-url={{ .Values.prometheus.alertmanager.baseURL }} + {{- end }} + + ports: + - containerPort: 9093 + readinessProbe: + httpGet: + path: {{ .Values.prometheus.alertmanager.prefixURL }}/#/status + port: 9093 + initialDelaySeconds: 30 + timeoutSeconds: 30 + resources: +{{ toYaml .Values.prometheus.alertmanager.resources | indent 12 }} + volumeMounts: + - name: config-volume + mountPath: /etc/config + - name: storage-volume + mountPath: "{{ .Values.prometheus.alertmanager.persistentVolume.mountPath }}" + subPath: "{{ .Values.prometheus.alertmanager.persistentVolume.subPath }}" + {{- range .Values.prometheus.alertmanager.extraSecretMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + subPath: {{ .subPath }} + readOnly: {{ .readOnly }} + {{- end }} + {{- if .Values.prometheus.configmapReload.alertmanager.enabled }} + - name: {{ template "prometheus.name" . }}-{{ .Values.prometheus.alertmanager.name }}-{{ .Values.prometheus.configmapReload.alertmanager.name }} + image: "{{ .Values.prometheus.configmapReload.alertmanager.image.repository }}:{{ .Values.prometheus.configmapReload.alertmanager.image.tag }}" + imagePullPolicy: "{{ .Values.prometheus.configmapReload.alertmanager.image.pullPolicy }}" + args: + - --watched-dir=/etc/config + - --reload-url=http://localhost:9093{{ .Values.prometheus.alertmanager.prefixURL }}/-/reload + resources: +{{ toYaml .Values.prometheus.configmapReload.alertmanager.resources | indent 12 }} + volumeMounts: + - name: config-volume + mountPath: /etc/config + readOnly: true + {{- end }} + {{- if .Values.prometheus.imagePullSecrets }} + imagePullSecrets: + {{- range $.Values.prometheus.imagePullSecrets}} + - name: {{ .name }} + {{- end }} + {{- end }} + {{- if .Values.prometheus.alertmanager.nodeSelector }} + nodeSelector: +{{ toYaml .Values.prometheus.alertmanager.nodeSelector | indent 8 }} + {{- end }} + {{- if .Values.prometheus.alertmanager.securityContext }} + securityContext: +{{ toYaml .Values.prometheus.alertmanager.securityContext | indent 8 }} + {{- end }} + {{- if .Values.prometheus.alertmanager.tolerations }} + tolerations: +{{ toYaml .Values.prometheus.alertmanager.tolerations | indent 8 }} + {{- end }} + volumes: + - name: config-volume + configMap: + name: {{ if .Values.prometheus.alertmanager.configMapOverrideName }}{{ .Release.Name }}-{{ .Values.prometheus.alertmanager.configMapOverrideName }}{{- else }}{{ template "prometheus.alertmanager.fullname" . }}{{- end }} + {{- range .Values.prometheus.alertmanager.extraSecretMounts }} + - name: {{ .name }} + secret: + secretName: {{ .secretName }} + {{- end }} +{{- if .Values.prometheus.alertmanager.persistentVolume.enabled }} + volumeClaimTemplates: + - metadata: + name: storage-volume + {{- if .Values.prometheus.alertmanager.persistentVolume.annotations }} + annotations: +{{ toYaml .Values.prometheus.alertmanager.persistentVolume.annotations | indent 10 }} + {{- end }} + spec: + accessModes: +{{ toYaml .Values.prometheus.alertmanager.persistentVolume.accessModes | indent 10 }} + resources: + requests: + storage: "{{ .Values.prometheus.alertmanager.persistentVolume.size }}" + {{- if .Values.prometheus.server.persistentVolume.storageClass }} + {{- if (eq "-" .Values.prometheus.server.persistentVolume.storageClass) }} + storageClassName: "" + {{- else }} + storageClassName: "{{ .Values.prometheus.alertmanager.persistentVolume.storageClass }}" + {{- end }} + {{- end }} +{{- else }} + - name: storage-volume + emptyDir: {} +{{- end }} +{{- end }} +{{ end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-node-exporter-daemonset.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-node-exporter-daemonset.yaml new file mode 100644 index 0000000000..eb541514cc --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-node-exporter-daemonset.yaml @@ -0,0 +1,147 @@ +{{ if .Values.global.prometheus.enabled }} +{{- if .Values.prometheus.nodeExporter.enabled -}} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + annotations: + {{- with .Values.global.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if .Values.prometheus.nodeExporter.annotations }} + {{- toYaml .Values.prometheus.nodeExporter.annotations | nindent 4 }} + {{- end }} + labels: + {{- include "prometheus.nodeExporter.labels" . | nindent 4 }} + {{- with .Values.global.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ template "prometheus.nodeExporter.fullname" . }} + namespace: {{ .Release.Namespace }} +spec: + selector: + matchLabels: + {{- include "prometheus.nodeExporter.matchLabels" . | nindent 6 }} + {{- if .Values.prometheus.nodeExporter.updateStrategy }} + updateStrategy: +{{ toYaml .Values.prometheus.nodeExporter.updateStrategy | indent 4 }} + {{- end }} + template: + metadata: + annotations: + {{- with .Values.global.podAnnotations}} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.prometheus.nodeExporter.podAnnotations }} + {{ toYaml .Values.prometheus.nodeExporter.podAnnotations | indent 8 }} + {{- end }} + labels: + {{- include "prometheus.nodeExporter.labels" . | nindent 8 }} + {{- with .Values.global.additionalLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} +{{- if .Values.prometheus.nodeExporter.pod.labels }} +{{ toYaml .Values.prometheus.nodeExporter.pod.labels | indent 8 }} +{{- end }} + spec: +{{- if .Values.prometheus.nodeExporter.affinity }} + affinity: +{{ toYaml .Values.prometheus.nodeExporter.affinity | indent 8 }} +{{- end }} + serviceAccountName: {{ template "prometheus.serviceAccountName.nodeExporter" . }} +{{- if .Values.prometheus.nodeExporter.dnsPolicy }} + dnsPolicy: "{{ .Values.prometheus.nodeExporter.dnsPolicy }}" +{{- end }} +{{- if .Values.prometheus.nodeExporter.priorityClassName }} + priorityClassName: "{{ .Values.prometheus.nodeExporter.priorityClassName }}" +{{- end }} + containers: + - name: {{ template "prometheus.name" . }}-{{ .Values.prometheus.nodeExporter.name }} + image: "{{ .Values.prometheus.nodeExporter.image.repository }}:{{ .Values.prometheus.nodeExporter.image.tag }}" + imagePullPolicy: "{{ .Values.prometheus.nodeExporter.image.pullPolicy }}" + args: + - --path.procfs=/host/proc + - --path.sysfs=/host/sys + {{- if .Values.prometheus.nodeExporter.hostNetwork }} + - --web.listen-address=:{{ .Values.prometheus.nodeExporter.service.hostPort }} + {{- end }} + {{- range $key, $value := .Values.prometheus.nodeExporter.extraArgs }} + {{- if $value }} + - --{{ $key }}={{ $value }} + {{- else }} + - --{{ $key }} + {{- end }} + {{- end }} + ports: + - name: metrics + {{- if .Values.prometheus.nodeExporter.hostNetwork }} + containerPort: {{ .Values.prometheus.nodeExporter.service.hostPort }} + {{- else }} + containerPort: 9100 + {{- end }} + hostPort: {{ .Values.prometheus.nodeExporter.service.hostPort }} + resources: +{{ toYaml .Values.prometheus.nodeExporter.resources | indent 12 }} + volumeMounts: + - name: proc + mountPath: /host/proc + readOnly: true + - name: sys + mountPath: /host/sys + readOnly: true + {{- range .Values.prometheus.nodeExporter.extraHostPathMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + readOnly: {{ .readOnly }} + {{- if .mountPropagation }} + mountPropagation: {{ .mountPropagation }} + {{- end }} + {{- end }} + {{- range .Values.prometheus.nodeExporter.extraConfigmapMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + readOnly: {{ .readOnly }} + {{- end }} + {{- if .Values.prometheus.imagePullSecrets }} + imagePullSecrets: + {{- range $.Values.prometheus.imagePullSecrets}} + - name: {{ .name }} + {{- end }} + {{- end }} + {{- if .Values.prometheus.nodeExporter.hostNetwork }} + hostNetwork: true + {{- end }} + {{- if .Values.prometheus.nodeExporter.hostPID }} + hostPID: true + {{- end }} + {{- if .Values.prometheus.nodeExporter.tolerations }} + tolerations: +{{ toYaml .Values.prometheus.nodeExporter.tolerations | indent 8 }} + {{- end }} + {{- if .Values.prometheus.nodeExporter.nodeSelector }} + nodeSelector: +{{ toYaml .Values.prometheus.nodeExporter.nodeSelector | indent 8 }} + {{- end }} + {{- if .Values.prometheus.nodeExporter.securityContext }} + securityContext: +{{ toYaml .Values.prometheus.nodeExporter.securityContext | indent 8 }} + {{- end }} + volumes: + - name: proc + hostPath: + path: /proc + - name: sys + hostPath: + path: /sys + {{- range .Values.prometheus.nodeExporter.extraHostPathMounts }} + - name: {{ .name }} + hostPath: + path: {{ .hostPath }} + {{- end }} + {{- range .Values.prometheus.nodeExporter.extraConfigmapMounts }} + - name: {{ .name }} + configMap: + name: {{ .configMap }} + {{- end }} + +{{- end -}} +{{ end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-node-exporter-ocp-scc.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-node-exporter-ocp-scc.yaml new file mode 100644 index 0000000000..e226f9beac --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-node-exporter-ocp-scc.yaml @@ -0,0 +1,29 @@ +{{- if and (.Capabilities.APIVersions.Has "security.openshift.io/v1/SecurityContextConstraints") (.Values.global.platforms.openshift.scc.nodeExporter) (.Values.prometheus.nodeExporter.enabled) }} +apiVersion: security.openshift.io/v1 +kind: SecurityContextConstraints +metadata: + name: {{ template "prometheus.nodeExporter.fullname" . }} +priority: 10 +allowPrivilegedContainer: true +allowHostDirVolumePlugin: true +allowHostNetwork: true +allowHostPorts: true +allowHostPID: true +allowHostIPC: false +readOnlyRootFilesystem: false +runAsUser: + type: RunAsAny +fsGroup: + type: RunAsAny +seLinuxContext: + type: RunAsAny +supplementalGroups: + type: RunAsAny +seccompProfiles: +- runtime/default +volumes: + - hostPath + - projected +users: + - system:serviceaccount:{{ .Release.Namespace }}:{{ template "prometheus.serviceAccountName.nodeExporter" . }} +{{- end }} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-node-exporter-service.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-node-exporter-service.yaml new file mode 100644 index 0000000000..9b8167e8d7 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-node-exporter-service.yaml @@ -0,0 +1,47 @@ +{{ if .Values.global.prometheus.enabled }} +{{- if .Values.prometheus.nodeExporter.enabled -}} +apiVersion: v1 +kind: Service +metadata: +{{- if .Values.prometheus.nodeExporter.service.annotations }} + annotations: +{{ toYaml .Values.prometheus.nodeExporter.service.annotations | indent 4 }} +{{- end }} + labels: + {{- include "prometheus.nodeExporter.labels" . | nindent 4 }} +{{- if .Values.prometheus.nodeExporter.service.labels }} +{{ toYaml .Values.prometheus.nodeExporter.service.labels | indent 4 }} +{{- end }} + name: {{ template "prometheus.nodeExporter.fullname" . }} + namespace: {{ .Release.Namespace }} +spec: +{{- if .Values.prometheus.nodeExporter.service.clusterIP }} + clusterIP: {{ .Values.prometheus.nodeExporter.service.clusterIP }} +{{- end }} +{{- if .Values.prometheus.nodeExporter.service.externalIPs }} + externalIPs: +{{ toYaml .Values.prometheus.nodeExporter.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.prometheus.nodeExporter.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.prometheus.nodeExporter.service.loadBalancerIP }} +{{- end }} +{{- if .Values.prometheus.nodeExporter.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.prometheus.nodeExporter.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} + ports: + - name: tcp-metrics + port: {{ .Values.prometheus.nodeExporter.service.servicePort }} + protocol: TCP + {{- if .Values.prometheus.nodeExporter.hostNetwork }} + targetPort: {{ .Values.prometheus.nodeExporter.service.hostPort }} + {{- else }} + targetPort: 9100 + {{- end }} + selector: + {{- include "prometheus.nodeExporter.matchLabels" . | nindent 4 }} + type: "{{ .Values.prometheus.nodeExporter.service.type }}" +{{- end -}} +{{ end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-node-exporter-serviceaccount.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-node-exporter-serviceaccount.yaml new file mode 100644 index 0000000000..3cb68d8e46 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-node-exporter-serviceaccount.yaml @@ -0,0 +1,11 @@ +{{ if .Values.global.prometheus.enabled }} +{{- if and .Values.prometheus.nodeExporter.enabled .Values.prometheus.serviceAccounts.nodeExporter.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + {{- include "prometheus.nodeExporter.labels" . | nindent 4 }} + name: {{ template "prometheus.serviceAccountName.nodeExporter" . }} + namespace: {{ .Release.Namespace }} +{{- end -}} +{{ end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-server-clusterrole.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-server-clusterrole.yaml new file mode 100644 index 0000000000..3672195556 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-server-clusterrole.yaml @@ -0,0 +1,39 @@ +{{ if .Values.global.prometheus.enabled }} +{{- if and .Values.prometheus.server.enabled .Values.prometheus.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + {{- include "prometheus.server.labels" . | nindent 4 }} + name: {{ template "prometheus.server.fullname" . }} +rules: + - apiGroups: + - "" + resources: + - nodes + - nodes/proxy + - nodes/metrics + - services + - endpoints + - pods + - ingresses + - configmaps + verbs: + - get + - list + - watch + - apiGroups: + - networking.k8s.io + resources: + - ingresses/status + - ingresses + verbs: + - get + - list + - watch + - nonResourceURLs: + - "/metrics" + verbs: + - get +{{- end }} +{{ end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-server-clusterrolebinding.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-server-clusterrolebinding.yaml new file mode 100644 index 0000000000..e03d8e443f --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-server-clusterrolebinding.yaml @@ -0,0 +1,18 @@ +{{ if .Values.global.prometheus.enabled }} +{{- if and .Values.prometheus.server.enabled .Values.prometheus.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + {{- include "prometheus.server.labels" . | nindent 4 }} + name: {{ template "prometheus.server.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ template "prometheus.serviceAccountName.server" . }} + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "prometheus.server.fullname" . }} +{{- end }} +{{ end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-server-configmap.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-server-configmap.yaml new file mode 100644 index 0000000000..36403c64ca --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-server-configmap.yaml @@ -0,0 +1,100 @@ +{{ if .Values.global.prometheus.enabled }} +{{- if .Values.prometheus.server.enabled -}} +{{- if (empty .Values.prometheus.server.configMapOverrideName) -}} +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + {{- include "prometheus.server.labels" . | nindent 4 }} + name: {{ template "prometheus.server.fullname" . }} + namespace: {{ .Release.Namespace }} +data: +{{- $root := . -}} +{{- range $key, $value := .Values.prometheus.serverFiles }} + {{ $key }}: | +{{- if eq $key "prometheus.yml" }} + global: +{{ $root.Values.prometheus.server.global | toYaml | trimSuffix "\n" | indent 6 }} +{{- if $root.Values.global.amp.enabled }} + remote_write: + - url: {{ $root.Values.global.amp.remoteWriteService }} + sigv4: +{{ $root.Values.global.amp.sigv4 | toYaml | indent 8 }} +{{- end }} +{{- if $root.Values.global.ammsp.enabled }} + # See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write + # for guidance on this configuration option + remote_write: + - url: {{ $root.Values.global.ammsp.remoteWriteService }} + azuread: + cloud: 'AzurePublic' + managed_identity: + client_id: {{ $root.Values.global.ammsp.aadAuthProxy.aadClientId }} +{{- end }} +{{- if $root.Values.prometheus.server.remoteWrite }} + remote_write: +{{ $root.Values.prometheus.server.remoteWrite | toYaml | indent 4 }} +{{- end }} +{{- if $root.Values.prometheus.server.remoteRead }} + remote_read: +{{ $root.Values.prometheus.server.remoteRead | toYaml | indent 4 }} +{{- end }} +{{- end }} +{{- if eq $key "alerts" }} +{{- if and (not (empty $value)) (empty $value.groups) }} + groups: +{{- range $ruleKey, $ruleValue := $value }} + - name: {{ $ruleKey -}}.rules + rules: +{{ $ruleValue | toYaml | trimSuffix "\n" | indent 6 }} +{{- end }} +{{- else }} +{{ toYaml $value | indent 4 }} +{{- end }} +{{- else }} +{{ toYaml $value | default "{}" | indent 4 }} +{{- end }} +{{- if eq $key "prometheus.yml" -}} +{{- if $root.Values.prometheus.extraScrapeConfigs }} +{{ tpl $root.Values.prometheus.extraScrapeConfigs $root | indent 4 }} +{{- end -}} +{{- if or ($root.Values.prometheus.alertmanager.enabled) ($root.Values.prometheus.server.alertmanagers) }} + alerting: +{{- if $root.Values.prometheus.alertRelabelConfigs }} +{{ $root.Values.prometheus.alertRelabelConfigs | toYaml | trimSuffix "\n" | indent 6 }} +{{- end }} + alertmanagers: +{{- if $root.Values.prometheus.server.alertmanagers }} +{{ toYaml $root.Values.prometheus.server.alertmanagers | indent 8 }} +{{- else }} + - kubernetes_sd_configs: + - role: pod + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if $root.Values.prometheus.alertmanager.prefixURL }} + path_prefix: {{ $root.Values.prometheus.alertmanager.prefixURL }} + {{- end }} + relabel_configs: + - source_labels: [__meta_kubernetes_namespace] + regex: {{ $root.Release.Namespace }} + action: keep + - source_labels: [__meta_kubernetes_pod_label_app] + regex: {{ template "prometheus.name" $root }} + action: keep + - source_labels: [__meta_kubernetes_pod_label_component] + regex: alertmanager + action: keep + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_probe] + regex: {{ index $root.Values.prometheus.alertmanager.podAnnotations "prometheus.io/probe" | default ".*" }} + action: keep + - source_labels: [__meta_kubernetes_pod_container_port_number] + regex: + action: drop +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{ end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-server-deployment.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-server-deployment.yaml new file mode 100644 index 0000000000..e7b4565197 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-server-deployment.yaml @@ -0,0 +1,268 @@ +{{ if .Values.global.prometheus.enabled }} +{{- if .Values.prometheus.server.enabled -}} +{{- if not .Values.prometheus.server.statefulSet.enabled -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + {{- with .Values.global.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if .Values.prometheus.server.annotations }} + {{- toYaml .Values.prometheus.server.annotations | nindent 4 }} + {{- end }} + labels: + {{- include "prometheus.server.labels" . | nindent 4 }} + {{- with .Values.global.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ template "prometheus.server.fullname" . }} + namespace: {{ .Release.Namespace }} +spec: + selector: + matchLabels: + {{- include "prometheus.server.matchLabels" . | nindent 6 }} + replicas: {{ .Values.prometheus.server.replicaCount }} + {{- if .Values.prometheus.server.strategy }} + strategy: +{{ toYaml .Values.prometheus.server.strategy | indent 4 }} + {{- end }} + template: + metadata: + annotations: + {{- with .Values.global.podAnnotations}} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.prometheus.server.podAnnotations }} + {{ toYaml .Values.prometheus.server.podAnnotations | nindent 8 }} + {{- end }} + checksum/configs: {{ include "configsChecksum" . }} + labels: + {{- include "prometheus.server.labels" . | nindent 8 }} + {{- with .Values.global.additionalLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.prometheus.server.podLabels}} + {{ toYaml .Values.prometheus.server.podLabels | nindent 8 }} + {{- end}} + spec: +{{- if .Values.prometheus.server.priorityClassName }} + priorityClassName: "{{ .Values.prometheus.server.priorityClassName }}" +{{- end }} +{{- if .Values.prometheus.server.schedulerName }} + schedulerName: "{{ .Values.prometheus.server.schedulerName }}" +{{- end }} + serviceAccountName: {{ template "prometheus.serviceAccountName.server" . }} + {{- if .Values.prometheus.server.extraInitContainers }} + initContainers: +{{ toYaml .Values.prometheus.server.extraInitContainers | indent 8 }} + {{- end }} + containers: + {{- if .Values.prometheus.configmapReload.prometheus.enabled }} + - name: {{ template "prometheus.name" . }}-{{ .Values.prometheus.server.name }}-{{ .Values.prometheus.configmapReload.prometheus.name }} + image: "{{ .Values.prometheus.configmapReload.prometheus.image.repository }}:{{ .Values.prometheus.configmapReload.prometheus.image.tag }}" + imagePullPolicy: "{{ .Values.prometheus.configmapReload.prometheus.image.pullPolicy }}" + args: + - --watched-dir=/etc/config + - --reload-url=http://127.0.0.1:9090{{ .Values.prometheus.server.prefixURL }}/-/reload + {{- range $key, $value := .Values.prometheus.configmapReload.prometheus.extraArgs }} + - --{{ $key }}={{ $value }} + {{- end }} + {{- range .Values.prometheus.configmapReload.prometheus.extraVolumeDirs }} + - --watched-dir={{ . }} + {{- end }} + resources: + {{- toYaml .Values.prometheus.configmapReload.prometheus.resources | nindent 12 }} + securityContext: + {{- if .Values.global.containerSecurityContext }} + {{- toYaml .Values.global.containerSecurityContext | nindent 12 }} + {{- else }} + {{- toYaml .Values.prometheus.configmapReload.prometheus.containerSecurityContext | nindent 12 }} + {{- end }} + volumeMounts: + {{- if .Values.prometheus.selfsignedCertConfigMapName }} + - name: {{ .Values.prometheus.selfsignedCertConfigMapName }} + mountPath: /etc/ssl/certs/my-cert.pem + subPath: my-cert.pem + readOnly: false + {{- end }} + - name: config-volume + mountPath: /etc/config + readOnly: true + {{- range .Values.prometheus.configmapReload.prometheus.extraConfigmapMounts }} + - name: {{ $.Values.prometheus.configmapReload.prometheus.name }}-{{ .name }} + mountPath: {{ .mountPath }} + subPath: {{ .subPath }} + readOnly: {{ .readOnly }} + {{- end }} + {{- end }} + + - name: {{ template "prometheus.name" . }}-{{ .Values.prometheus.server.name }} + image: "{{ .Values.prometheus.server.image.repository }}:{{ .Values.prometheus.server.image.tag }}" + imagePullPolicy: "{{ .Values.prometheus.server.image.pullPolicy }}" + {{- if .Values.prometheus.server.env }} + env: +{{ toYaml .Values.prometheus.server.env | indent 12}} + {{- end }} + args: + {{- if .Values.prometheus.server.retention }} + - --storage.tsdb.retention.time={{ .Values.prometheus.server.retention }} + {{- end }} + {{- if .Values.prometheus.server.retentionSize }} + - --storage.tsdb.retention.size={{ .Values.prometheus.server.retentionSize }} + {{- end }} + - --config.file={{ .Values.prometheus.server.configPath }} + - --storage.tsdb.path={{ .Values.prometheus.server.persistentVolume.mountPath }} + - --web.console.libraries=/etc/prometheus/console_libraries + - --web.console.templates=/etc/prometheus/consoles + {{- range .Values.prometheus.server.extraFlags }} + - --{{ . }} + {{- end }} + {{- if .Values.prometheus.server.baseURL }} + - --web.external-url={{ .Values.prometheus.server.baseURL }} + {{- end }} + + {{- range $key, $value := .Values.prometheus.server.extraArgs }} + - --{{ $key }}={{ $value }} + {{- end }} + ports: + - containerPort: 9090 + readinessProbe: + httpGet: + path: {{ .Values.prometheus.server.prefixURL }}/-/ready + port: 9090 + initialDelaySeconds: {{ .Values.prometheus.server.readinessProbeInitialDelay }} + timeoutSeconds: {{ .Values.prometheus.server.readinessProbeTimeout }} + failureThreshold: {{ .Values.prometheus.server.readinessProbeFailureThreshold }} + successThreshold: {{ .Values.prometheus.server.readinessProbeSuccessThreshold }} + livenessProbe: + httpGet: + path: {{ .Values.prometheus.server.prefixURL }}/-/healthy + port: 9090 + initialDelaySeconds: {{ .Values.prometheus.server.livenessProbeInitialDelay }} + timeoutSeconds: {{ .Values.prometheus.server.livenessProbeTimeout }} + failureThreshold: {{ .Values.prometheus.server.livenessProbeFailureThreshold }} + successThreshold: {{ .Values.prometheus.server.livenessProbeSuccessThreshold }} + resources: + {{- toYaml .Values.prometheus.server.resources | nindent 12 }} + securityContext: + {{- if .Values.global.containerSecurityContext }} + {{- toYaml .Values.global.containerSecurityContext | nindent 12 }} + {{- else }} + {{- toYaml .Values.prometheus.server.containerSecurityContext | nindent 12 }} + {{- end }} + volumeMounts: + - name: config-volume + mountPath: /etc/config + - name: storage-volume + mountPath: {{ .Values.prometheus.server.persistentVolume.mountPath }} + subPath: "{{ .Values.prometheus.server.persistentVolume.subPath }}" + {{- range .Values.prometheus.server.extraHostPathMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + subPath: {{ .subPath }} + readOnly: {{ .readOnly }} + {{- end }} + {{- range .Values.prometheus.server.extraConfigmapMounts }} + - name: {{ $.Values.prometheus.server.name }}-{{ .name }} + mountPath: {{ .mountPath }} + subPath: {{ .subPath }} + readOnly: {{ .readOnly }} + {{- end }} + {{- range .Values.prometheus.server.extraSecretMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + subPath: {{ .subPath }} + readOnly: {{ .readOnly }} + {{- end }} + {{- if .Values.prometheus.server.extraVolumeMounts }} + {{ toYaml .Values.prometheus.server.extraVolumeMounts | nindent 12 }} + {{- end }} + {{- if .Values.prometheus.server.sidecarContainers }} + {{- toYaml .Values.prometheus.server.sidecarContainers | nindent 8 }} + {{- end }} + {{- if .Values.prometheus.imagePullSecrets }} + imagePullSecrets: + {{- range $.Values.prometheus.imagePullSecrets}} + - name: {{ .name }} + {{- end }} + {{- end }} + {{- if .Values.prometheus.server.nodeSelector }} + nodeSelector: + {{- toYaml .Values.prometheus.server.nodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.prometheus.server.securityContext }} + securityContext: + {{- if not .Values.prometheus.server.securityContext.fsGroup }} + fsGroupChangePolicy: OnRootMismatch + fsGroup: 1001 + {{- end }} + {{- toYaml .Values.prometheus.server.securityContext | nindent 8 }} + {{- else if and (.Values.global.platforms.openshift.enabled) (.Values.global.platforms.openshift.securityContext) }} + securityContext: + {{- toYaml .Values.global.platforms.openshift.securityContext | nindent 8 }} + {{- else if .Values.global.securityContext }} + securityContext: + {{- toYaml .Values.global.securityContext | nindent 8 }} + {{- end }} + {{- if .Values.prometheus.server.tolerations }} + tolerations: +{{ toYaml .Values.prometheus.server.tolerations | indent 8 }} + {{- end }} + {{- if .Values.prometheus.server.affinity }} + affinity: +{{ toYaml .Values.prometheus.server.affinity | indent 8 }} + {{- end }} + terminationGracePeriodSeconds: {{ .Values.prometheus.server.terminationGracePeriodSeconds }} + volumes: + {{- if .Values.prometheus.selfsignedCertConfigMapName }} + - name: {{ .Values.prometheus.selfsignedCertConfigMapName }} + configMap: + name: {{ .Values.prometheus.selfsignedCertConfigMapName }} + {{- end }} + - name: config-volume + configMap: + name: {{ if .Values.prometheus.server.configMapOverrideName }}{{ .Release.Name }}-{{ .Values.prometheus.server.configMapOverrideName }}{{- else }}{{ template "prometheus.server.fullname" . }}{{- end }} + - name: storage-volume + {{- if .Values.prometheus.server.persistentVolume.enabled }} + persistentVolumeClaim: + claimName: {{ if .Values.prometheus.server.persistentVolume.existingClaim }}{{ .Values.prometheus.server.persistentVolume.existingClaim }}{{- else }}{{ template "prometheus.server.fullname" . }}{{- end }} + {{- else }} + emptyDir: + {{- if .Values.prometheus.server.emptyDir.sizeLimit }} + sizeLimit: {{ .Values.prometheus.server.emptyDir.sizeLimit }} + {{- else }} + {} + {{- end -}} + {{- end -}} +{{- if .Values.prometheus.server.extraVolumes }} +{{ toYaml .Values.prometheus.server.extraVolumes | indent 8}} +{{- end }} + {{- range .Values.prometheus.server.extraHostPathMounts }} + - name: {{ .name }} + hostPath: + path: {{ .hostPath }} + {{- end }} + {{- range .Values.prometheus.configmapReload.prometheus.extraConfigmapMounts }} + - name: {{ $.Values.prometheus.configmapReload.prometheus.name }}-{{ .name }} + configMap: + name: {{ .configMap }} + {{- end }} + {{- range .Values.prometheus.server.extraConfigmapMounts }} + - name: {{ $.Values.prometheus.server.name }}-{{ .name }} + configMap: + name: {{ .configMap }} + {{- end }} + {{- range .Values.prometheus.server.extraSecretMounts }} + - name: {{ .name }} + secret: + secretName: {{ tpl .secretName $ }} + {{- end }} + {{- range .Values.prometheus.configmapReload.prometheus.extraConfigmapMounts }} + - name: {{ .name }} + configMap: + name: {{ .configMap }} + {{- end }} +{{- end -}} +{{- end -}} +{{ end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-server-ingress.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-server-ingress.yaml new file mode 100644 index 0000000000..18a7835fce --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-server-ingress.yaml @@ -0,0 +1,45 @@ +{{- if and (.Values.global.prometheus.enabled) (.Values.prometheus.server.enabled) (.Values.prometheus.server.ingress.enabled) }} +{{- $serviceName := include "prometheus.server.fullname" . }} +{{- $servicePort := .Values.prometheus.server.service.servicePort -}} +{{- $extraPaths := .Values.prometheus.server.ingress.extraPaths -}} +{{- $pathType := .Values.prometheus.server.ingress.pathType -}} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: +{{- if .Values.prometheus.server.ingress.annotations }} + annotations: +{{ toYaml .Values.prometheus.server.ingress.annotations | indent 4 }} +{{- end }} + labels: + {{- include "prometheus.server.labels" . | nindent 4 }} +{{- range $key, $value := .Values.prometheus.server.ingress.extraLabels }} + {{ $key }}: {{ $value }} +{{- end }} + name: {{ template "prometheus.server.fullname" . }} + namespace: {{ .Release.Namespace }} +spec: +{{- if .Values.prometheus.server.ingress.className }} + ingressClassName: {{ .Values.prometheus.server.ingress.className }} +{{- end }} + rules: + {{- range .Values.prometheus.server.ingress.hosts }} + {{- $url := splitList "/" . }} + - host: {{ first $url }} + http: + paths: +{{ if $extraPaths }} +{{ toYaml $extraPaths | indent 10 }} +{{- end }} + - path: /{{ rest $url | join "/" }} + pathType: {{ $pathType }} + backend: + service: + name: {{ $serviceName }} + port: + number: {{ $servicePort }} + {{- end -}} +{{- if .Values.prometheus.server.ingress.tls }} + tls: +{{ toYaml .Values.prometheus.server.ingress.tls | indent 4 }} + {{- end -}} +{{- end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-server-pdb.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-server-pdb.yaml new file mode 100644 index 0000000000..52ceeb2484 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-server-pdb.yaml @@ -0,0 +1,15 @@ +{{ if .Values.global.prometheus.enabled }} +{{- if .Values.prometheus.server.podDisruptionBudget.enabled }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ template "prometheus.server.fullname" . }} + labels: + {{- include "prometheus.server.labels" . | nindent 4 }} +spec: + maxUnavailable: {{ .Values.prometheus.server.podDisruptionBudget.maxUnavailable }} + selector: + matchLabels: + {{- include "prometheus.server.labels" . | nindent 6 }} +{{- end }} +{{ end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-server-pvc.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-server-pvc.yaml new file mode 100644 index 0000000000..301a33e1a1 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-server-pvc.yaml @@ -0,0 +1,37 @@ +{{ if .Values.global.prometheus.enabled }} +{{- if .Values.prometheus.server.enabled -}} +{{- if not .Values.prometheus.server.statefulSet.enabled -}} +{{- if .Values.prometheus.server.persistentVolume.enabled -}} +{{- if not .Values.prometheus.server.persistentVolume.existingClaim -}} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + {{- if .Values.prometheus.server.persistentVolume.annotations }} + annotations: +{{ toYaml .Values.prometheus.server.persistentVolume.annotations | indent 4 }} + {{- end }} + labels: + {{- include "prometheus.server.labels" . | nindent 4 }} + name: {{ template "prometheus.server.fullname" . }} + namespace: {{ .Release.Namespace }} +spec: + accessModes: +{{ toYaml .Values.prometheus.server.persistentVolume.accessModes | indent 4 }} +{{- if .Values.prometheus.server.persistentVolume.storageClass }} +{{- if (eq "-" .Values.prometheus.server.persistentVolume.storageClass) }} + storageClassName: "" +{{- else }} + storageClassName: "{{ .Values.prometheus.server.persistentVolume.storageClass }}" +{{- end }} +{{- end }} +{{- if .Values.prometheus.server.persistentVolume.volumeBindingMode }} + volumeBindingModeName: "{{ .Values.prometheus.server.persistentVolume.volumeBindingMode }}" +{{- end }} + resources: + requests: + storage: "{{ .Values.prometheus.server.persistentVolume.size }}" +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{ end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-server-service-headless.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-server-service-headless.yaml new file mode 100644 index 0000000000..019803d30c --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-server-service-headless.yaml @@ -0,0 +1,29 @@ +{{ if .Values.global.prometheus.enabled }} +{{- if .Values.prometheus.server.enabled -}} +{{- if .Values.prometheus.server.statefulSet.enabled -}} +apiVersion: v1 +kind: Service +metadata: +{{- if .Values.prometheus.server.statefulSet.headless.annotations }} + annotations: +{{ toYaml .Values.prometheus.server.statefulSet.headless.annotations | indent 4 }} +{{- end }} + labels: + {{- include "prometheus.server.labels" . | nindent 4 }} +{{- if .Values.prometheus.server.statefulSet.headless.labels }} +{{ toYaml .Values.prometheus.server.statefulSet.headless.labels | indent 4 }} +{{- end }} + name: {{ template "prometheus.server.fullname" . }}-headless + namespace: {{ .Release.Namespace }} +spec: + clusterIP: None + ports: + - name: http + port: {{ .Values.prometheus.server.statefulSet.headless.servicePort }} + protocol: TCP + targetPort: 9090 + selector: + {{- include "prometheus.server.matchLabels" . | nindent 4 }} +{{- end -}} +{{- end -}} +{{ end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-server-service.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-server-service.yaml new file mode 100644 index 0000000000..69f093c38e --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-server-service.yaml @@ -0,0 +1,62 @@ +{{ if .Values.global.prometheus.enabled }} +{{- if .Values.prometheus.server.enabled -}} +apiVersion: v1 +kind: Service +metadata: +{{- if .Values.prometheus.server.service.annotations }} + annotations: +{{ toYaml .Values.prometheus.server.service.annotations | indent 4 }} +{{- end }} + labels: + {{- include "prometheus.server.labels" . | nindent 4 }} +{{- if .Values.prometheus.server.service.labels }} +{{ toYaml .Values.prometheus.server.service.labels | indent 4 }} +{{- end }} + name: {{ template "prometheus.server.fullname" . }} + namespace: {{ .Release.Namespace }} +spec: +{{- if .Values.prometheus.server.service.clusterIP }} + clusterIP: {{ .Values.prometheus.server.service.clusterIP }} +{{- end }} +{{- if .Values.prometheus.server.service.externalIPs }} + externalIPs: +{{ toYaml .Values.prometheus.server.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.prometheus.server.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.prometheus.server.service.loadBalancerIP }} +{{- end }} +{{- if .Values.prometheus.server.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.prometheus.server.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} + ports: + - name: http + port: {{ .Values.prometheus.server.service.servicePort }} + protocol: TCP + targetPort: 9090 + {{- if .Values.prometheus.server.service.nodePort }} + nodePort: {{ .Values.prometheus.server.service.nodePort }} + {{- end }} + {{- if .Values.prometheus.server.service.gRPC.enabled }} + - name: grpc + port: {{ .Values.prometheus.server.service.gRPC.servicePort }} + protocol: TCP + targetPort: 10901 + {{- if .Values.prometheus.server.service.gRPC.nodePort }} + nodePort: {{ .Values.prometheus.server.service.gRPC.nodePort }} + {{- end }} + {{- end }} + selector: + {{- if and .Values.prometheus.server.statefulSet.enabled .Values.prometheus.server.service.statefulsetReplica.enabled }} + statefulset.kubernetes.io/pod-name: {{ .Release.Name }}-{{ .Values.prometheus.server.name }}-{{ .Values.prometheus.server.service.statefulsetReplica.replica }} + {{- else -}} + {{- include "prometheus.server.matchLabels" . | nindent 4 }} +{{- if .Values.prometheus.server.service.sessionAffinity }} + sessionAffinity: {{ .Values.prometheus.server.service.sessionAffinity }} +{{- end }} + {{- end }} + type: "{{ .Values.prometheus.server.service.type }}" +{{- end -}} +{{ end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-server-serviceaccount.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-server-serviceaccount.yaml new file mode 100644 index 0000000000..17ee234bb0 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-server-serviceaccount.yaml @@ -0,0 +1,17 @@ +{{ if .Values.global.prometheus.enabled }} +{{- if .Values.prometheus.server.enabled -}} +{{- if .Values.prometheus.serviceAccounts.server.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + {{- include "prometheus.server.labels" . | nindent 4 }} + name: {{ template "prometheus.serviceAccountName.server" . }} + namespace: {{ .Release.Namespace }} + {{- with .Values.prometheus.serviceAccounts.server.annotations }} + annotations: + {{- . | toYaml | nindent 4 }} + {{- end }} +{{- end }} +{{- end }} +{{ end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-server-statefulset.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-server-statefulset.yaml new file mode 100644 index 0000000000..c87ad6b97e --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-server-statefulset.yaml @@ -0,0 +1,241 @@ +{{ if .Values.global.prometheus.enabled }} +{{- if .Values.prometheus.server.enabled -}} +{{- if .Values.prometheus.server.statefulSet.enabled -}} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + annotations: + {{- with .Values.global.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if .Values.prometheus.server.statefulSet.annotations }} + {{- toYaml .Values.prometheus.server.statefulSet.annotations | nindent 4 }} + {{- end }} + labels: + {{- include "prometheus.server.labels" . | nindent 4 }} + {{- with .Values.global.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if .Values.prometheus.server.statefulSet.labels}} + {{ toYaml .Values.prometheus.server.statefulSet.labels | nindent 4 }} + {{- end}} + name: {{ template "prometheus.server.fullname" . }} + namespace: {{ .Release.Namespace }} +spec: + serviceName: {{ template "prometheus.server.fullname" . }}-headless + selector: + matchLabels: + {{- include "prometheus.server.matchLabels" . | nindent 6 }} + replicas: {{ .Values.prometheus.server.replicaCount }} + podManagementPolicy: {{ .Values.prometheus.server.statefulSet.podManagementPolicy }} + template: + metadata: + annotations: + {{- with .Values.global.podAnnotations}} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.prometheus.server.podAnnotations }} + {{ toYaml .Values.prometheus.server.podAnnotations | indent 8 }} + {{- end }} + labels: + {{- include "prometheus.server.labels" . | nindent 8 }} + {{- with .Values.global.additionalLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.prometheus.server.statefulSet.labels}} + {{ toYaml .Values.prometheus.server.statefulSet.labels | nindent 8 }} + {{- end}} + spec: +{{- if .Values.prometheus.server.priorityClassName }} + priorityClassName: "{{ .Values.prometheus.server.priorityClassName }}" +{{- end }} +{{- if .Values.prometheus.server.schedulerName }} + schedulerName: "{{ .Values.prometheus.server.schedulerName }}" +{{- end }} + serviceAccountName: {{ template "prometheus.serviceAccountName.server" . }} + containers: + {{- if .Values.prometheus.configmapReload.prometheus.enabled }} + - name: {{ template "prometheus.name" . }}-{{ .Values.prometheus.server.name }}-{{ .Values.prometheus.configmapReload.prometheus.name }} + image: "{{ .Values.prometheus.configmapReload.prometheus.image.repository }}:{{ .Values.prometheus.configmapReload.prometheus.image.tag }}" + imagePullPolicy: "{{ .Values.prometheus.configmapReload.prometheus.image.pullPolicy }}" + args: + - --watched-dir=/etc/config + - --reload-url=http://127.0.0.1:9090{{ .Values.prometheus.server.prefixURL }}/-/reload + {{- range $key, $value := .Values.prometheus.configmapReload.prometheus.extraArgs }} + - --{{ $key }}={{ $value }} + {{- end }} + {{- range .Values.prometheus.configmapReload.prometheus.extraVolumeDirs }} + - --watched-dir={{ . }} + {{- end }} + resources: +{{ toYaml .Values.prometheus.configmapReload.prometheus.resources | indent 12 }} + volumeMounts: + - name: config-volume + mountPath: /etc/config + readOnly: true + {{- range .Values.prometheus.configmapReload.prometheus.extraConfigmapMounts }} + - name: {{ $.Values.prometheus.configmapReload.prometheus.name }}-{{ .name }} + mountPath: {{ .mountPath }} + subPath: {{ .subPath }} + readOnly: {{ .readOnly }} + {{- end }} + {{- end }} + - name: {{ template "prometheus.name" . }}-{{ .Values.prometheus.server.name }} + image: "{{ .Values.prometheus.server.image.repository }}:{{ .Values.prometheus.server.image.tag }}" + imagePullPolicy: "{{ .Values.prometheus.server.image.pullPolicy }}" + {{- if .Values.prometheus.server.env }} + env: +{{ toYaml .Values.prometheus.server.env | indent 12}} + {{- end }} + args: + {{- if .Values.prometheus.server.retention }} + - --storage.tsdb.retention.time={{ .Values.prometheus.server.retention }} + {{- end }} + - --config.file={{ .Values.prometheus.server.configPath }} + - --storage.tsdb.path={{ .Values.prometheus.server.persistentVolume.mountPath }} + - --web.console.libraries=/etc/prometheus/console_libraries + - --web.console.templates=/etc/prometheus/consoles + {{- range .Values.prometheus.server.extraFlags }} + - --{{ . }} + {{- end }} + {{- range $key, $value := .Values.prometheus.server.extraArgs }} + - --{{ $key }}={{ $value }} + {{- end }} + {{- if .Values.prometheus.server.baseURL }} + - --web.external-url={{ .Values.prometheus.server.baseURL }} + {{- end }} + ports: + - containerPort: 9090 + readinessProbe: + httpGet: + path: {{ .Values.prometheus.server.prefixURL }}/-/ready + port: 9090 + initialDelaySeconds: {{ .Values.prometheus.server.readinessProbeInitialDelay }} + timeoutSeconds: {{ .Values.prometheus.server.readinessProbeTimeout }} + livenessProbe: + httpGet: + path: {{ .Values.prometheus.server.prefixURL }}/-/healthy + port: 9090 + initialDelaySeconds: {{ .Values.prometheus.server.livenessProbeInitialDelay }} + timeoutSeconds: {{ .Values.prometheus.server.livenessProbeTimeout }} + securityContext: + {{- if .Values.prometheus.server.containerSecurityContext }} + {{- toYaml .Values.prometheus.server.containerSecurityContext | nindent 12 }} + {{- else }} + {{- toYaml .Values.global.containerSecurityContext | nindent 12 }} + {{- end }} + resources: +{{ toYaml .Values.prometheus.server.resources | indent 12 }} + volumeMounts: + - name: config-volume + mountPath: /etc/config + - name: storage-volume + mountPath: {{ .Values.prometheus.server.persistentVolume.mountPath }} + subPath: "{{ .Values.prometheus.server.persistentVolume.subPath }}" + {{- range .Values.prometheus.server.extraHostPathMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + subPath: {{ .subPath }} + readOnly: {{ .readOnly }} + {{- end }} + {{- range .Values.prometheus.server.extraConfigmapMounts }} + - name: {{ $.Values.prometheus.server.name }}-{{ .name }} + mountPath: {{ .mountPath }} + subPath: {{ .subPath }} + readOnly: {{ .readOnly }} + {{- end }} + {{- range .Values.prometheus.server.extraSecretMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + subPath: {{ .subPath }} + readOnly: {{ .readOnly }} + {{- end }} + {{- if .Values.prometheus.server.extraVolumeMounts }} + {{ toYaml .Values.prometheus.server.extraVolumeMounts | nindent 12 }} + {{- end }} + {{- if .Values.prometheus.server.sidecarContainers }} + {{- toYaml .Values.prometheus.server.sidecarContainers | nindent 8 }} + {{- end }} + {{- if .Values.prometheus.imagePullSecrets }} + imagePullSecrets: + {{- range $.Values.prometheus.imagePullSecrets}} + - name: {{ .name }} + {{- end }} + {{- end }} + {{- if .Values.prometheus.server.nodeSelector }} + nodeSelector: +{{ toYaml .Values.prometheus.server.nodeSelector | indent 8 }} + {{- end }} + {{- if .Values.prometheus.server.securityContext }} + securityContext: +{{ toYaml .Values.prometheus.server.securityContext | indent 8 }} + {{- end }} + {{- if .Values.prometheus.server.tolerations }} + tolerations: +{{ toYaml .Values.prometheus.server.tolerations | indent 8 }} + {{- end }} + {{- if .Values.prometheus.server.affinity }} + affinity: +{{ toYaml .Values.prometheus.server.affinity | indent 8 }} + {{- end }} + terminationGracePeriodSeconds: {{ .Values.prometheus.server.terminationGracePeriodSeconds }} + volumes: + - name: config-volume + configMap: + name: {{ if .Values.prometheus.server.configMapOverrideName }}{{ .Release.Name }}-{{ .Values.prometheus.server.configMapOverrideName }}{{- else }}{{ template "prometheus.server.fullname" . }}{{- end }} + {{- range .Values.prometheus.server.extraHostPathMounts }} + - name: {{ .name }} + hostPath: + path: {{ .hostPath }} + {{- end }} + {{- range .Values.prometheus.configmapReload.prometheus.extraConfigmapMounts }} + - name: {{ $.Values.prometheus.configmapReload.prometheus.name }}-{{ .name }} + configMap: + name: {{ .configMap }} + {{- end }} + {{- range .Values.prometheus.server.extraConfigmapMounts }} + - name: {{ $.Values.prometheus.server.name }}-{{ .name }} + configMap: + name: {{ .configMap }} + {{- end }} + {{- range .Values.prometheus.server.extraSecretMounts }} + - name: {{ .name }} + secret: + secretName: {{ .secretName }} + {{- end }} + {{- range .Values.prometheus.configmapReload.prometheus.extraConfigmapMounts }} + - name: {{ .name }} + configMap: + name: {{ .configMap }} + {{- end }} +{{- if .Values.prometheus.server.extraVolumes }} +{{ toYaml .Values.prometheus.server.extraVolumes | indent 8}} +{{- end }} +{{- if .Values.prometheus.server.persistentVolume.enabled }} + volumeClaimTemplates: + - metadata: + name: storage-volume + {{- if .Values.prometheus.server.persistentVolume.annotations }} + annotations: +{{ toYaml .Values.prometheus.server.persistentVolume.annotations | indent 10 }} + {{- end }} + spec: + accessModes: +{{ toYaml .Values.prometheus.server.persistentVolume.accessModes | indent 10 }} + resources: + requests: + storage: "{{ .Values.prometheus.server.persistentVolume.size }}" + {{- if .Values.prometheus.server.persistentVolume.storageClass }} + {{- if (eq "-" .Values.prometheus.server.persistentVolume.storageClass) }} + storageClassName: "" + {{- else }} + storageClassName: "{{ .Values.prometheus.server.persistentVolume.storageClass }}" + {{- end }} + {{- end }} +{{- else }} + - name: storage-volume + emptyDir: {} +{{- end }} +{{- end }} +{{- end }} +{{ end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-server-vpa.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-server-vpa.yaml new file mode 100644 index 0000000000..25a61f2532 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/prometheus/prometheus-server-vpa.yaml @@ -0,0 +1,22 @@ +{{- if and (.Values.global.prometheus.enabled) (.Values.prometheus.server.enabled) (.Values.prometheus.server.verticalAutoscaler.enabled) }} +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + labels: + {{- include "prometheus.server.labels" . | nindent 4 }} + name: {{ template "prometheus.server.fullname" . }}-vpa + namespace: {{ .Release.Namespace }} +spec: + targetRef: + apiVersion: apps/v1 +{{- if .Values.prometheus.server.statefulSet.enabled }} + kind: StatefulSet +{{- else }} + kind: Deployment +{{- end }} + name: {{ template "prometheus.server.fullname" . }} + updatePolicy: + updateMode: {{ .Values.prometheus.server.verticalAutoscaler.updateMode | default "Off" | quote }} + resourcePolicy: + containerPolicies: {{ .Values.prometheus.server.verticalAutoscaler.containerPolicies | default list | toYaml | trim | nindent 4 }} +{{- end }} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/savings-recommendations-allowlists-config-map-template.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/savings-recommendations-allowlists-config-map-template.yaml new file mode 100644 index 0000000000..b6e20a1fed --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/savings-recommendations-allowlists-config-map-template.yaml @@ -0,0 +1,13 @@ +{{- if .Values.kubecostProductConfigs }} +{{- if .Values.kubecostProductConfigs.savingsRecommendationsAllowLists }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: "savings-recommendations-instance-allow-lists" + namespace: {{ .Release.Namespace }} + labels: + {{- include "cost-analyzer.commonLabels" . | nindent 4 }} +data: + allow-lists.json: '{{ toJson .Values.kubecostProductConfigs.savingsRecommendationsAllowLists }}' +{{- end -}} +{{- end -}} \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/tests/_helpers.tpl b/charts/kubecost/cost-analyzer/2.5.3/templates/tests/_helpers.tpl new file mode 100644 index 0000000000..8c1e53a560 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/tests/_helpers.tpl @@ -0,0 +1,5 @@ +{{/* vim: set filetype=mustache: */}} + +{{- define "kubecost.test.annotations" -}} +helm.sh/hook: test +{{- end -}} diff --git a/charts/kubecost/cost-analyzer/2.5.3/templates/tests/basic-health.yaml b/charts/kubecost/cost-analyzer/2.5.3/templates/tests/basic-health.yaml new file mode 100644 index 0000000000..d75705491a --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/templates/tests/basic-health.yaml @@ -0,0 +1,48 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: basic-health + namespace: {{ .Release.Namespace }} + annotations: + {{- include "kubecost.test.annotations" . | nindent 4 }} + labels: + app: basic-health + app.kubernetes.io/name: basic-health + app.kubernetes.io/instance: {{ .Release.Name }} +spec: + automountServiceAccountToken: false + restartPolicy: Never + securityContext: + seccompProfile: + type: RuntimeDefault + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 3000 + fsGroup: 2000 + containers: + - name: test-kubecost + image: {{ (.Values.basicHealth).fullImageName | default "alpine/k8s:1.26.9" }} + securityContext: + privileged: false + capabilities: + drop: + - ALL + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + command: + - /bin/sh + args: + - -c + - >- + svc="{{ .Release.Name }}-cost-analyzer"; + echo Getting current Kubecost state.; + response=$(curl -sL http://${svc}:9090/model/getConfigs); + code=$(echo ${response} | jq .code); + if [ "$code" -eq 200 ]; then + echo "Got Kubecost working configuration. Successful." + exit 0 + else + echo "Failed to fetch Kubecost configuration. Response was $response" + exit 1 + fi diff --git a/charts/kubecost/cost-analyzer/2.5.3/values-amp.yaml b/charts/kubecost/cost-analyzer/2.5.3/values-amp.yaml new file mode 100644 index 0000000000..4242ba3002 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/values-amp.yaml @@ -0,0 +1,20 @@ +global: + amp: + enabled: true + prometheusServerEndpoint: http://localhost:8005/workspaces/$ + remoteWriteService: https://aps-workspaces.us-west-2.amazonaws.com/workspaces/$/api/v1/remote_write + sigv4: + region: us-west-2 + +sigV4Proxy: + region: us-west-2 + host: aps-workspaces.us-west-2.amazonaws.com + +kubecostProductConfigs: + clusterName: AWS-cluster-one + +prometheus: + server: + global: + external_labels: + cluster_id: AWS-cluster-one \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/values-custom-pricing.yaml b/charts/kubecost/cost-analyzer/2.5.3/values-custom-pricing.yaml new file mode 100644 index 0000000000..82a0c55408 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/values-custom-pricing.yaml @@ -0,0 +1,17 @@ +pricingCsv: + enabled: true + location: + URI: /var/kubecost-csv/custom-pricing.csv # local configMap or s3://bucket/path/custom-pricing.csv + # provider: "AWS" + # region: "us-east-1" + # URI: s3://kc-csv-test/pricing_schema.csv # a valid file URI + # csvAccessCredentials: pricing-schema-access-secret + +# when using configmap: kubectl create configmap -n kubecost csv-pricing --from-file custom-pricing.csv +extraVolumes: +- name: kubecost-csv + configMap: + name: csv-pricing +extraVolumeMounts: +- name: kubecost-csv + mountPath: /var/kubecost-csv diff --git a/charts/kubecost/cost-analyzer/2.5.3/values-eks-cost-monitoring.yaml b/charts/kubecost/cost-analyzer/2.5.3/values-eks-cost-monitoring.yaml new file mode 100644 index 0000000000..1eece00856 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/values-eks-cost-monitoring.yaml @@ -0,0 +1,44 @@ +# grafana is disabled by default, but can be enabled by setting the following values. +# or proxy to an existing grafana: https://docs.kubecost.com/install-and-configure/advanced-configuration/custom-grafana +global: + grafana: + enabled: false + proxy: false +# grafana: +# image: +# repository: YOUR_REGISTRY/grafana +# sidecar: +# image: +# repository: YOUR_REGISTRY/k8s-sidecar + +kubecostFrontend: + image: public.ecr.aws/kubecost/frontend + +kubecostModel: + image: public.ecr.aws/kubecost/cost-model + +forecasting: + fullImageName: public.ecr.aws/kubecost/kubecost-modeling:v0.1.19 + +networkCosts: + image: + repository: public.ecr.aws/kubecost/kubecost-network-costs + tag: v0.17.6 + +clusterController: + image: + repository: public.ecr.aws/kubecost/cluster-controller + +prometheus: + server: + image: + repository: public.ecr.aws/kubecost/prometheus + + configmapReload: + prometheus: + image: + repository: public.ecr.aws/kubecost/prometheus-config-reloader + +reporting: + productAnalytics: false + diff --git a/charts/kubecost/cost-analyzer/2.5.3/values-openshift-cluster-prometheus.yaml b/charts/kubecost/cost-analyzer/2.5.3/values-openshift-cluster-prometheus.yaml new file mode 100644 index 0000000000..1284e7dd4d --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/values-openshift-cluster-prometheus.yaml @@ -0,0 +1,26 @@ +# This Helm values file is a modified version of `values-openshift.yaml`. +# The primary difference is that this file is configured to disable the Kubecost-bundled Prometheus, and instead leverage the Prometheus instance that is typically pre-installed in OpenShift clusters. +global: + prometheus: + enabled: false # Kubecost depends on Prometheus data, it is not optional. When enabled: false, Prometheus will not be installed and you must configure your in-cluster Prometheus to scrape kubecost as well as provide the fqdn below. -- Warning: Before changing using this setting, please read to understand the risks https://docs.kubecost.com/install-and-configure/install/custom-prom + fqdn: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 # example address of a Prometheus to connect to. Include protocol (http:// or https://) Ignored if enabled: true + kubeRBACProxy: true # If true, kubecost will use kube-rbac-proxy to authenticate with in cluster Prometheus for openshift + grafana: + enabled: false # If false, Grafana will not be installed + domainName: grafana.grafana + proxy: false + + platforms: + # Deploying to OpenShift (OCP) requires enabling this option. + openshift: + enabled: true # Deploy Kubecost to OpenShift. + createMonitoringClusterRoleBinding: true # Create a ClusterRoleBinding to grant the Kubecost serviceaccount access to query Prometheus. + createMonitoringResourceReaderRoleBinding: true # Create a Role and Role Binding to allow Prometheus to list and watch Kubecost resources. + monitoringServiceAccountName: prometheus-k8s # Name of the Prometheus serviceaccount to bind to the Resource Reader Role Binding. + monitoringServiceAccountNamespace: openshift-monitoring # Namespace of the Prometheus serviceaccount to bind to the Resource Reader Role Binding. + +serviceMonitor: + enabled: true + +prometheusRule: + enabled: true diff --git a/charts/kubecost/cost-analyzer/2.5.3/values-openshift.yaml b/charts/kubecost/cost-analyzer/2.5.3/values-openshift.yaml new file mode 100644 index 0000000000..ffd38545e1 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/values-openshift.yaml @@ -0,0 +1,8 @@ +# This Helm values file is a modified version of `values.yaml`. +# This file is meant to be used by users deploying Kubecost to OpenShift (OCP) clusters. For more configuration options, see `values.yaml`. +global: + # Platforms is a higher-level abstraction for platform-specific values and settings. + platforms: + # Deploying to OpenShift (OCP) requires enabling this option. + openshift: + enabled: true # Deploy Kubecost to OpenShift. \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/values-savings-rec-allowlist-aws.yaml b/charts/kubecost/cost-analyzer/2.5.3/values-savings-rec-allowlist-aws.yaml new file mode 100644 index 0000000000..e86af6dc47 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/values-savings-rec-allowlist-aws.yaml @@ -0,0 +1,790 @@ +kubecostProductConfigs: + savingsRecommendationsAllowLists: + AWS: + - a1.2xlarge + - a1.4xlarge + - a1.large + - a1.medium + - a1.metal + - a1.xlarge + - c1.medium + - c1.xlarge + - c3.2xlarge + - c3.4xlarge + - c3.8xlarge + - c3.large + - c3.xlarge + - c4.2xlarge + - c4.4xlarge + - c4.8xlarge + - c4.large + - c4.xlarge + - c5.12xlarge + - c5.18xlarge + - c5.24xlarge + - c5.2xlarge + - c5.4xlarge + - c5.9xlarge + - c5.large + - c5.metal + - c5.xlarge + - c5a.12xlarge + - c5a.16xlarge + - c5a.24xlarge + - c5a.2xlarge + - c5a.4xlarge + - c5a.8xlarge + - c5a.large + - c5a.xlarge + - c5ad.12xlarge + - c5ad.16xlarge + - c5ad.24xlarge + - c5ad.2xlarge + - c5ad.4xlarge + - c5ad.8xlarge + - c5ad.large + - c5ad.xlarge + - c5d.12xlarge + - c5d.18xlarge + - c5d.24xlarge + - c5d.2xlarge + - c5d.4xlarge + - c5d.9xlarge + - c5d.large + - c5d.metal + - c5d.xlarge + - c5n.18xlarge + - c5n.2xlarge + - c5n.4xlarge + - c5n.9xlarge + - c5n.large + - c5n.metal + - c5n.xlarge + - c6a.12xlarge + - c6a.16xlarge + - c6a.24xlarge + - c6a.2xlarge + - c6a.32xlarge + - c6a.48xlarge + - c6a.4xlarge + - c6a.8xlarge + - c6a.large + - c6a.metal + - c6a.xlarge + - c6g.12xlarge + - c6g.16xlarge + - c6g.2xlarge + - c6g.4xlarge + - c6g.8xlarge + - c6g.large + - c6g.medium + - c6g.metal + - c6g.xlarge + - c6gd.12xlarge + - c6gd.16xlarge + - c6gd.2xlarge + - c6gd.4xlarge + - c6gd.8xlarge + - c6gd.large + - c6gd.medium + - c6gd.metal + - c6gd.xlarge + - c6gn.12xlarge + - c6gn.16xlarge + - c6gn.2xlarge + - c6gn.4xlarge + - c6gn.8xlarge + - c6gn.large + - c6gn.medium + - c6gn.xlarge + - c6i.12xlarge + - c6i.16xlarge + - c6i.24xlarge + - c6i.2xlarge + - c6i.32xlarge + - c6i.4xlarge + - c6i.8xlarge + - c6i.large + - c6i.metal + - c6i.xlarge + - c6id.12xlarge + - c6id.16xlarge + - c6id.24xlarge + - c6id.2xlarge + - c6id.32xlarge + - c6id.4xlarge + - c6id.8xlarge + - c6id.large + - c6id.metal + - c6id.xlarge + - c6in.12xlarge + - c6in.16xlarge + - c6in.24xlarge + - c6in.2xlarge + - c6in.32xlarge + - c6in.4xlarge + - c6in.8xlarge + - c6in.large + - c6in.metal + - c6in.xlarge + - c7a.12xlarge + - c7a.16xlarge + - c7a.24xlarge + - c7a.2xlarge + - c7a.32xlarge + - c7a.48xlarge + - c7a.4xlarge + - c7a.8xlarge + - c7a.large + - c7a.medium + - c7a.metal-48xl + - c7a.xlarge + - c7g.12xlarge + - c7g.16xlarge + - c7g.2xlarge + - c7g.4xlarge + - c7g.8xlarge + - c7g.large + - c7g.medium + - c7g.metal + - c7g.xlarge + - c7gd.12xlarge + - c7gd.16xlarge + - c7gd.2xlarge + - c7gd.4xlarge + - c7gd.8xlarge + - c7gd.large + - c7gd.medium + - c7gd.metal + - c7gd.xlarge + - c7gn.12xlarge + - c7gn.16xlarge + - c7gn.2xlarge + - c7gn.4xlarge + - c7gn.8xlarge + - c7gn.large + - c7gn.medium + - c7gn.metal + - c7gn.xlarge + - c7i-flex.2xlarge + - c7i-flex.4xlarge + - c7i-flex.8xlarge + - c7i-flex.large + - c7i-flex.xlarge + - c7i.12xlarge + - c7i.16xlarge + - c7i.24xlarge + - c7i.2xlarge + - c7i.48xlarge + - c7i.4xlarge + - c7i.8xlarge + - c7i.large + - c7i.metal-24xl + - c7i.metal-48xl + - c7i.xlarge + - d2.2xlarge + - d2.4xlarge + - d2.8xlarge + - d2.xlarge + - d3.2xlarge + - d3.4xlarge + - d3.8xlarge + - d3.xlarge + - d3en.12xlarge + - d3en.2xlarge + - d3en.4xlarge + - d3en.6xlarge + - d3en.8xlarge + - d3en.xlarge + - dl1.24xlarge + - dl2q.24xlarge + - f1.16xlarge + - f1.2xlarge + - f1.4xlarge + - g3.16xlarge + - g3.4xlarge + - g3.8xlarge + - g3s.xlarge + - g4ad.16xlarge + - g4ad.2xlarge + - g4ad.4xlarge + - g4ad.8xlarge + - g4ad.xlarge + - g4dn.12xlarge + - g4dn.16xlarge + - g4dn.2xlarge + - g4dn.4xlarge + - g4dn.8xlarge + - g4dn.metal + - g4dn.xlarge + - g5.12xlarge + - g5.16xlarge + - g5.24xlarge + - g5.2xlarge + - g5.48xlarge + - g5.4xlarge + - g5.8xlarge + - g5.xlarge + - g5g.16xlarge + - g5g.2xlarge + - g5g.4xlarge + - g5g.8xlarge + - g5g.metal + - g5g.xlarge + - g6.12xlarge + - g6.16xlarge + - g6.24xlarge + - g6.2xlarge + - g6.48xlarge + - g6.4xlarge + - g6.8xlarge + - g6.xlarge + - g6e.12xlarge + - g6e.16xlarge + - g6e.24xlarge + - g6e.2xlarge + - g6e.48xlarge + - g6e.4xlarge + - g6e.8xlarge + - g6e.xlarge + - gr6.4xlarge + - gr6.8xlarge + - h1.16xlarge + - h1.2xlarge + - h1.4xlarge + - h1.8xlarge + - i2.2xlarge + - i2.4xlarge + - i2.8xlarge + - i2.xlarge + - i3.16xlarge + - i3.2xlarge + - i3.4xlarge + - i3.8xlarge + - i3.large + - i3.metal + - i3.xlarge + - i3en.12xlarge + - i3en.24xlarge + - i3en.2xlarge + - i3en.3xlarge + - i3en.6xlarge + - i3en.large + - i3en.metal + - i3en.xlarge + - i4g.16xlarge + - i4g.2xlarge + - i4g.4xlarge + - i4g.8xlarge + - i4g.large + - i4g.xlarge + - i4i.12xlarge + - i4i.16xlarge + - i4i.24xlarge + - i4i.2xlarge + - i4i.32xlarge + - i4i.4xlarge + - i4i.8xlarge + - i4i.large + - i4i.metal + - i4i.xlarge + - im4gn.16xlarge + - im4gn.2xlarge + - im4gn.4xlarge + - im4gn.8xlarge + - im4gn.large + - im4gn.xlarge + - inf1.24xlarge + - inf1.2xlarge + - inf1.6xlarge + - inf1.xlarge + - inf2.24xlarge + - inf2.48xlarge + - inf2.8xlarge + - inf2.xlarge + - is4gen.2xlarge + - is4gen.4xlarge + - is4gen.8xlarge + - is4gen.large + - is4gen.medium + - is4gen.xlarge + - m1.large + - m1.medium + - m1.small + - m1.xlarge + - m2.2xlarge + - m2.4xlarge + - m2.xlarge + - m3.2xlarge + - m3.large + - m3.medium + - m3.xlarge + - m4.10xlarge + - m4.16xlarge + - m4.2xlarge + - m4.4xlarge + - m4.large + - m4.xlarge + - m5.12xlarge + - m5.16xlarge + - m5.24xlarge + - m5.2xlarge + - m5.4xlarge + - m5.8xlarge + - m5.large + - m5.metal + - m5.xlarge + - m5a.12xlarge + - m5a.16xlarge + - m5a.24xlarge + - m5a.2xlarge + - m5a.4xlarge + - m5a.8xlarge + - m5a.large + - m5a.xlarge + - m5ad.12xlarge + - m5ad.16xlarge + - m5ad.24xlarge + - m5ad.2xlarge + - m5ad.4xlarge + - m5ad.8xlarge + - m5ad.large + - m5ad.xlarge + - m5d.12xlarge + - m5d.16xlarge + - m5d.24xlarge + - m5d.2xlarge + - m5d.4xlarge + - m5d.8xlarge + - m5d.large + - m5d.metal + - m5d.xlarge + - m5dn.12xlarge + - m5dn.16xlarge + - m5dn.24xlarge + - m5dn.2xlarge + - m5dn.4xlarge + - m5dn.8xlarge + - m5dn.large + - m5dn.metal + - m5dn.xlarge + - m5n.12xlarge + - m5n.16xlarge + - m5n.24xlarge + - m5n.2xlarge + - m5n.4xlarge + - m5n.8xlarge + - m5n.large + - m5n.metal + - m5n.xlarge + - m5zn.12xlarge + - m5zn.2xlarge + - m5zn.3xlarge + - m5zn.6xlarge + - m5zn.large + - m5zn.metal + - m5zn.xlarge + - m6a.12xlarge + - m6a.16xlarge + - m6a.24xlarge + - m6a.2xlarge + - m6a.32xlarge + - m6a.48xlarge + - m6a.4xlarge + - m6a.8xlarge + - m6a.large + - m6a.metal + - m6a.xlarge + - m6g.12xlarge + - m6g.16xlarge + - m6g.2xlarge + - m6g.4xlarge + - m6g.8xlarge + - m6g.large + - m6g.medium + - m6g.metal + - m6g.xlarge + - m6gd.12xlarge + - m6gd.16xlarge + - m6gd.2xlarge + - m6gd.4xlarge + - m6gd.8xlarge + - m6gd.large + - m6gd.medium + - m6gd.metal + - m6gd.xlarge + - m6i.12xlarge + - m6i.16xlarge + - m6i.24xlarge + - m6i.2xlarge + - m6i.32xlarge + - m6i.4xlarge + - m6i.8xlarge + - m6i.large + - m6i.metal + - m6i.xlarge + - m6id.12xlarge + - m6id.16xlarge + - m6id.24xlarge + - m6id.2xlarge + - m6id.32xlarge + - m6id.4xlarge + - m6id.8xlarge + - m6id.large + - m6id.metal + - m6id.xlarge + - m6idn.12xlarge + - m6idn.16xlarge + - m6idn.24xlarge + - m6idn.2xlarge + - m6idn.32xlarge + - m6idn.4xlarge + - m6idn.8xlarge + - m6idn.large + - m6idn.metal + - m6idn.xlarge + - m6in.12xlarge + - m6in.16xlarge + - m6in.24xlarge + - m6in.2xlarge + - m6in.32xlarge + - m6in.4xlarge + - m6in.8xlarge + - m6in.large + - m6in.metal + - m6in.xlarge + - m7a.12xlarge + - m7a.16xlarge + - m7a.24xlarge + - m7a.2xlarge + - m7a.32xlarge + - m7a.48xlarge + - m7a.4xlarge + - m7a.8xlarge + - m7a.large + - m7a.medium + - m7a.metal-48xl + - m7a.xlarge + - m7g.12xlarge + - m7g.16xlarge + - m7g.2xlarge + - m7g.4xlarge + - m7g.8xlarge + - m7g.large + - m7g.medium + - m7g.metal + - m7g.xlarge + - m7gd.12xlarge + - m7gd.16xlarge + - m7gd.2xlarge + - m7gd.4xlarge + - m7gd.8xlarge + - m7gd.large + - m7gd.medium + - m7gd.metal + - m7gd.xlarge + - m7i-flex.2xlarge + - m7i-flex.4xlarge + - m7i-flex.8xlarge + - m7i-flex.large + - m7i-flex.xlarge + - m7i.12xlarge + - m7i.16xlarge + - m7i.24xlarge + - m7i.2xlarge + - m7i.48xlarge + - m7i.4xlarge + - m7i.8xlarge + - m7i.large + - m7i.metal-24xl + - m7i.metal-48xl + - m7i.xlarge + - p2.16xlarge + - p2.8xlarge + - p2.xlarge + - p3.16xlarge + - p3.2xlarge + - p3.8xlarge + - p3dn.24xlarge + - p4d.24xlarge + - p5.48xlarge + - r3.2xlarge + - r3.4xlarge + - r3.8xlarge + - r3.large + - r3.xlarge + - r4.16xlarge + - r4.2xlarge + - r4.4xlarge + - r4.8xlarge + - r4.large + - r4.xlarge + - r5.12xlarge + - r5.16xlarge + - r5.24xlarge + - r5.2xlarge + - r5.4xlarge + - r5.8xlarge + - r5.large + - r5.metal + - r5.xlarge + - r5a.12xlarge + - r5a.16xlarge + - r5a.24xlarge + - r5a.2xlarge + - r5a.4xlarge + - r5a.8xlarge + - r5a.large + - r5a.xlarge + - r5ad.12xlarge + - r5ad.16xlarge + - r5ad.24xlarge + - r5ad.2xlarge + - r5ad.4xlarge + - r5ad.8xlarge + - r5ad.large + - r5ad.xlarge + - r5b.12xlarge + - r5b.16xlarge + - r5b.24xlarge + - r5b.2xlarge + - r5b.4xlarge + - r5b.8xlarge + - r5b.large + - r5b.metal + - r5b.xlarge + - r5d.12xlarge + - r5d.16xlarge + - r5d.24xlarge + - r5d.2xlarge + - r5d.4xlarge + - r5d.8xlarge + - r5d.large + - r5d.metal + - r5d.xlarge + - r5dn.12xlarge + - r5dn.16xlarge + - r5dn.24xlarge + - r5dn.2xlarge + - r5dn.4xlarge + - r5dn.8xlarge + - r5dn.large + - r5dn.metal + - r5dn.xlarge + - r5n.12xlarge + - r5n.16xlarge + - r5n.24xlarge + - r5n.2xlarge + - r5n.4xlarge + - r5n.8xlarge + - r5n.large + - r5n.metal + - r5n.xlarge + - r6a.12xlarge + - r6a.16xlarge + - r6a.24xlarge + - r6a.2xlarge + - r6a.32xlarge + - r6a.48xlarge + - r6a.4xlarge + - r6a.8xlarge + - r6a.large + - r6a.metal + - r6a.xlarge + - r6g.12xlarge + - r6g.16xlarge + - r6g.2xlarge + - r6g.4xlarge + - r6g.8xlarge + - r6g.large + - r6g.medium + - r6g.metal + - r6g.xlarge + - r6gd.12xlarge + - r6gd.16xlarge + - r6gd.2xlarge + - r6gd.4xlarge + - r6gd.8xlarge + - r6gd.large + - r6gd.medium + - r6gd.metal + - r6gd.xlarge + - r6i.12xlarge + - r6i.16xlarge + - r6i.24xlarge + - r6i.2xlarge + - r6i.32xlarge + - r6i.4xlarge + - r6i.8xlarge + - r6i.large + - r6i.metal + - r6i.xlarge + - r6id.12xlarge + - r6id.16xlarge + - r6id.24xlarge + - r6id.2xlarge + - r6id.32xlarge + - r6id.4xlarge + - r6id.8xlarge + - r6id.large + - r6id.metal + - r6id.xlarge + - r6idn.12xlarge + - r6idn.16xlarge + - r6idn.24xlarge + - r6idn.2xlarge + - r6idn.32xlarge + - r6idn.4xlarge + - r6idn.8xlarge + - r6idn.large + - r6idn.metal + - r6idn.xlarge + - r6in.12xlarge + - r6in.16xlarge + - r6in.24xlarge + - r6in.2xlarge + - r6in.32xlarge + - r6in.4xlarge + - r6in.8xlarge + - r6in.large + - r6in.metal + - r6in.xlarge + - r7a.12xlarge + - r7a.16xlarge + - r7a.24xlarge + - r7a.2xlarge + - r7a.32xlarge + - r7a.48xlarge + - r7a.4xlarge + - r7a.8xlarge + - r7a.large + - r7a.medium + - r7a.metal-48xl + - r7a.xlarge + - r7g.12xlarge + - r7g.16xlarge + - r7g.2xlarge + - r7g.4xlarge + - r7g.8xlarge + - r7g.large + - r7g.medium + - r7g.metal + - r7g.xlarge + - r7gd.12xlarge + - r7gd.16xlarge + - r7gd.2xlarge + - r7gd.4xlarge + - r7gd.8xlarge + - r7gd.large + - r7gd.medium + - r7gd.metal + - r7gd.xlarge + - r7i.12xlarge + - r7i.16xlarge + - r7i.24xlarge + - r7i.2xlarge + - r7i.48xlarge + - r7i.4xlarge + - r7i.8xlarge + - r7i.large + - r7i.metal-24xl + - r7i.metal-48xl + - r7i.xlarge + - r7iz.12xlarge + - r7iz.16xlarge + - r7iz.2xlarge + - r7iz.32xlarge + - r7iz.4xlarge + - r7iz.8xlarge + - r7iz.large + - r7iz.metal-16xl + - r7iz.metal-32xl + - r7iz.xlarge + - r8g.12xlarge + - r8g.16xlarge + - r8g.24xlarge + - r8g.2xlarge + - r8g.48xlarge + - r8g.4xlarge + - r8g.8xlarge + - r8g.large + - r8g.medium + - r8g.metal-24xl + - r8g.metal-48xl + - r8g.xlarge + - t1.micro + - t2.2xlarge + - t2.large + - t2.medium + - t2.micro + - t2.small + - t2.xlarge + - t3.2xlarge + - t3.large + - t3.medium + - t3.micro + - t3.nano + - t3.small + - t3.xlarge + - t3a.2xlarge + - t3a.large + - t3a.medium + - t3a.micro + - t3a.nano + - t3a.small + - t3a.xlarge + - t4g.2xlarge + - t4g.large + - t4g.medium + - t4g.micro + - t4g.nano + - t4g.small + - t4g.xlarge + - trn1.2xlarge + - trn1.32xlarge + - trn1n.32xlarge + - vt1.24xlarge + - vt1.3xlarge + - vt1.6xlarge + - x1.16xlarge + - x1.32xlarge + - x1e.16xlarge + - x1e.2xlarge + - x1e.32xlarge + - x1e.4xlarge + - x1e.8xlarge + - x1e.xlarge + - x2gd.12xlarge + - x2gd.16xlarge + - x2gd.2xlarge + - x2gd.4xlarge + - x2gd.8xlarge + - x2gd.large + - x2gd.medium + - x2gd.metal + - x2gd.xlarge + - x2idn.16xlarge + - x2idn.24xlarge + - x2idn.32xlarge + - x2idn.metal + - x2iedn.16xlarge + - x2iedn.24xlarge + - x2iedn.2xlarge + - x2iedn.32xlarge + - x2iedn.4xlarge + - x2iedn.8xlarge + - x2iedn.metal + - x2iedn.xlarge + - x2iezn.12xlarge + - x2iezn.2xlarge + - x2iezn.4xlarge + - x2iezn.6xlarge + - x2iezn.8xlarge + - x2iezn.metal + - z1d.12xlarge + - z1d.2xlarge + - z1d.3xlarge + - z1d.6xlarge + - z1d.large + - z1d.metal + - z1d.xlarge \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/values-savings-rec-allowlist-azure.yaml b/charts/kubecost/cost-analyzer/2.5.3/values-savings-rec-allowlist-azure.yaml new file mode 100644 index 0000000000..e324c53a00 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/values-savings-rec-allowlist-azure.yaml @@ -0,0 +1,283 @@ +kubecostProductConfigs: + savingsRecommendationsAllowLists: + Azure: + - A1 v2 + - A2 v2 + - A2m v2 + - A4 v2 + - A4m v2 + - A8 v2 + - A8m v2 + - B12ms + - B16ms + - B1ls + - B1ms + - B1s + - B20ms + - B2ms + - B2s + - B4ms + - B8ms + - D1 v2 + - D11 v2 + - D12 v2 + - D13 v2 + - D14 v2 + - D15 v2 + - D15i v2 + - D16 v3 + - D16 v4 + - D16a v4 + - D16as v4 + - D16d v4 + - D16ds v4 + - D16ds v5 + - D16s v3 + - D16s v4 + - D16s v5 + - D2 v2 + - D2 v3 + - D2 v4 + - D2a v4 + - D2as v4 + - D2d v4 + - D2ds v4 + - D2ds v5 + - D2s v3 + - D2s v4 + - D2s v5 + - D3 v2 + - D32 v3 + - D32 v4 + - D32a v4 + - D32as v4 + - D32d v4 + - D32ds v4 + - D32ds v5 + - D32s v3 + - D32s v4 + - D32s v5 + - D4 v2 + - D4 v3 + - D4 v4 + - D48 v3 + - D48 v4 + - D48a v4 + - D48as v4 + - D48d v4 + - D48ds v4 + - D48ds v5 + - D48s v3 + - D48s v4 + - D48s v5 + - D4a v4 + - D4as v4 + - D4d v4 + - D4ds v4 + - D4ds v5 + - D4s v3 + - D4s v4 + - D4s v5 + - D5 v2 + - D64 v3 + - D64 v4 + - D64a v4 + - D64as v4 + - D64d v4 + - D64ds v4 + - D64ds v5 + - D64s v3 + - D64s v4 + - D64s v5 + - D8 v3 + - D8 v4 + - D8a v4 + - D8as v4 + - D8d v4 + - D8ds v4 + - D8ds v5 + - D8s v3 + - D8s v4 + - D8s v5 + - D96a v4 + - D96as v4 + - D96ds v5 + - D96s v5 + - DC1s v2 + - DC2s v2 + - DC4s v2 + - DC8 v2 + - DS1 v2 + - DS11 v2 + - DS12 v2 + - DS13 v2 + - DS14 v2 + - DS15 v2 + - DS15i v2 + - DS2 v2 + - DS3 v2 + - DS4 v2 + - DS5 v2 + - E16 v3 + - E16 v4 + - E16a v4 + - E16as v4 + - E16d v4 + - E16ds v4 + - E16s v3 + - E16s v4 + - E2 v3 + - E2 v4 + - E20 v3 + - E20a v4 + - E20as v4 + - E20d v4 + - E20ds v4 + - E20s v3 + - E20s v4 + - E2a v4 + - E2as v4 + - E2d v4 + - E2ds v4 + - E2s v3 + - E2s v4 + - E32 v3 + - E32 v4 + - E32a v4 + - E32as v4 + - E32d v4 + - E32ds v4 + - E32s v3 + - E32s v4 + - E4 v3 + - E4 v4 + - E48 v3 + - E48 v4 + - E48a v4 + - E48as v4 + - E48d v4 + - E48ds v4 + - E48s v3 + - E48s v4 + - E4a v4 + - E4as v4 + - E4d v4 + - E4ds v4 + - E4s v3 + - E4s v4 + - E64 v3 + - E64 v4 + - E64a v4 + - E64as v4 + - E64d v4 + - E64ds v4 + - E64i v3 + - E64is v3 + - E64s v3 + - E64s v4 + - E8 v3 + - E8 v4 + - E80ids v4 + - E80is v4 + - E8a v4 + - E8as v4 + - E8d v4 + - E8ds v4 + - E8s v3 + - E8s v4 + - E96a v4 + - E96as v4 + - F1 + - F16 + - F16s + - F16s v2 + - F1s + - F2 + - F2s + - F2s v2 + - F32s v2 + - F4 + - F48s v2 + - F4s + - F4s v2 + - F64s v2 + - F72s v2 + - F8 + - F8s + - F8s v2 + - G1 + - G2 + - G3 + - G4 + - G5 + - GS1 + - GS2 + - GS3 + - GS4 + - GS5 + - H16 + - H16m + - H16mr + - H16r + - H8 + - H8m + - HB120rs v2 + - HC44rs + - L16s + - L16s v2 + - L32s + - L32s v2 + - L48s v2 + - L4s + - L64s v2 + - L80s v2 + - L8s + - L8s v2 + - M128 + - M128m + - M128ms + - M128s + - M16ms + - M208ms v2 + - M208s v2 + - M32ls + - M32ms + - M32ts + - M416ms v2 + - M416s v2 + - M64 + - M64ls + - M64m + - M64ms + - M64s + - M8ms + - NC12 + - NC12s v2 + - NC12s v3 + - NC16as T4 v3 + - NC24 + - NC24r + - NC24rs v2 + - NC24rs v3 + - NC24s v2 + - NC24s v3 + - NC4as T4 v3 + - NC6 + - NC64as T4 v3 + - NC6s v2 + - NC6s v3 + - NC8as T4 v3 + - ND12s + - ND24rs + - ND24s + - ND40rs v2 + - ND6s + - NP10s + - NP20s + - NP40s + - NV12 + - NV12s v3 + - NV24 + - NV24s v3 + - NV48s v3 + - NV6 \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/values-savings-rec-allowlist-gcp.yaml b/charts/kubecost/cost-analyzer/2.5.3/values-savings-rec-allowlist-gcp.yaml new file mode 100644 index 0000000000..2de05f0b43 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/values-savings-rec-allowlist-gcp.yaml @@ -0,0 +1,76 @@ +kubecostProductConfigs: + savingsRecommendationsAllowLists: + GCP: + - e2-highcpu-2 + - e2-highcpu-4 + - e2-highcpu-8 + - e2-highcpu-16 + - e2-highcpu-32 + - e2-highmem-2 + - e2-highmem-4 + - e2-highmem-8 + - e2-highmem-16 + - e2-micro + - e2-small + - e2-medium + - e2-standard-2 + - e2-standard-4 + - e2-standard-8 + - e2-standard-16 + - e2-standard-32 + - f1-micro + - g1-small + - m1-megamem-96 + - m1-ultramem-40 + - m1-ultramem-80 + - m1-ultramem-160 + - n1-highcpu-2 + - n1-highcpu-4 + - n1-highcpu-8 + - n1-highcpu-16 + - n1-highcpu-32 + - n1-highcpu-64 + - n1-highcpu-96 + - n1-highmem-2 + - n1-highmem-4 + - n1-highmem-8 + - n1-highmem-16 + - n1-highmem-32 + - n1-highmem-64 + - n1-highmem-96 + - n1-megamem-96 + - n1-standard-1 + - n1-standard-2 + - n1-standard-4 + - n1-standard-8 + - n1-standard-16 + - n1-standard-32 + - n1-standard-64 + - n1-standard-96 + - n1-ultramem-40 + - n1-ultramem-80 + - n1-ultramem-160 + - n2-highcpu-2 + - n2-highcpu-4 + - n2-highcpu-8 + - n2-highcpu-16 + - n2-highcpu-32 + - n2-highcpu-48 + - n2-highcpu-64 + - n2-highcpu-80 + - n2-highmem-2 + - n2-highmem-4 + - n2-highmem-8 + - n2-highmem-16 + - n2-highmem-32 + - n2-highmem-48 + - n2-highmem-64 + - n2-highmem-80 + - n2-standard-2 + - n2-standard-4 + - n2-standard-8 + - n2-standard-16 + - n2-standard-32 + - n2-standard-48 + - n2-standard-64 + - n2-standard-80 \ No newline at end of file diff --git a/charts/kubecost/cost-analyzer/2.5.3/values-windows-node-affinity.yaml b/charts/kubecost/cost-analyzer/2.5.3/values-windows-node-affinity.yaml new file mode 100644 index 0000000000..f67d595672 --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/values-windows-node-affinity.yaml @@ -0,0 +1,25 @@ +nodeSelector: + kubernetes.io/os: linux + +networkCosts: + nodeSelector: + kubernetes.io/os: linux + +prometheus: + server: + nodeSelector: + kubernetes.io/os: linux + nodeExporter: + enabled: true + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: In + values: + - linux +grafana: + nodeSelector: + kubernetes.io/os: linux diff --git a/charts/kubecost/cost-analyzer/2.5.3/values.yaml b/charts/kubecost/cost-analyzer/2.5.3/values.yaml new file mode 100644 index 0000000000..7d79edf9ac --- /dev/null +++ b/charts/kubecost/cost-analyzer/2.5.3/values.yaml @@ -0,0 +1,2419 @@ +global: + # zone: cluster.local (use only if your DNS server doesn't live in the same zone as kubecost) + prometheus: + enabled: true # Kubecost depends on Prometheus data, it is not optional. When enabled: false, Prometheus will not be installed and you must configure your own Prometheus to scrape kubecost as well as provide the fqdn below. -- Warning: Before changing this setting, please read to understand the risks https://docs.kubecost.com/install-and-configure/install/custom-prom + fqdn: http://cost-analyzer-prometheus-server.default.svc # example address of a prometheus to connect to. Include protocol (http:// or https://) Ignored if enabled: true + insecureSkipVerify: false # If true, kubecost will not check the TLS cert of prometheus + # queryServiceBasicAuthSecretName: dbsecret # kubectl create secret generic dbsecret -n kubecost --from-file=USERNAME --from-file=PASSWORD + # queryServiceBearerTokenSecretName: mcdbsecret # kubectl create secret generic mcdbsecret -n kubecost --from-file=TOKEN + kubeRBACProxy: false # If true, kubecost will use kube-rbac-proxy to authenticate with in cluster Prometheus for openshift + + grafana: + enabled: true # If false, Grafana will not be installed + domainName: cost-analyzer-grafana.default.svc # example grafana domain Ignored if enabled: true + scheme: "http" # http or https, for the domain name above. + proxy: true # If true, the kubecost frontend will route to your grafana through its service endpoint + # fqdn: cost-analyzer-grafana.default.svc + + # Enable only when you are using GCP Marketplace ENT listing. Learn more at https://console.cloud.google.com/marketplace/product/kubecost-public/kubecost-ent + gcpstore: + enabled: false + + # Google Cloud Managed Service for Prometheus + gmp: + # Remember to set up these parameters when install the Kubecost Helm chart with `global.gmp.enabled=true` if you want to use GMP self-deployed collection (Recommended) to utilize Kubecost scrape configs. + # If enabling GMP, it is highly recommended to utilize Google's distribution of Prometheus. + # Learn more at https://cloud.google.com/stackdriver/docs/managed-prometheus/setup-unmanaged + # --set prometheus.server.image.repository="gke.gcr.io/prometheus-engine/prometheus" \ + # --set prometheus.server.image.tag="v2.35.0-gmp.2-gke.0" + enabled: false # If true, kubecost will be configured to use GMP Prometheus image and query from Google Cloud Managed Service for Prometheus. + prometheusServerEndpoint: http://localhost:8085/ # The prometheus service endpoint used by kubecost. The calls are forwarded through the GMP Prom proxy side car to the GMP database. + gmpProxy: + enabled: false + image: gke.gcr.io/prometheus-engine/frontend:v0.4.1-gke.0 # GMP Prometheus proxy image that serve as an endpoint to query metrics from GMP + imagePullPolicy: IfNotPresent + name: gmp-proxy + port: 8085 + projectId: YOUR_PROJECT_ID # example GCP project ID + + # Amazon Managed Service for Prometheus + amp: + enabled: false # If true, kubecost will be configured to remote_write and query from Amazon Managed Service for Prometheus. + prometheusServerEndpoint: http://localhost:8005/workspaces// # The prometheus service endpoint used by kubecost. The calls are forwarded through the SigV4Proxy side car to the AMP workspace. + remoteWriteService: https://aps-workspaces.us-west-2.amazonaws.com/workspaces//api/v1/remote_write # The remote_write endpoint for the AMP workspace. + sigv4: + region: us-west-2 + # access_key: ACCESS_KEY # AWS Access key + # secret_key: SECRET_KEY # AWS Secret key + # role_arn: ROLE_ARN # AWS role arn + # profile: PROFILE # AWS profile + + # Mimir Proxy to help Kubecost to query metrics from multi-tenant Grafana Mimir. + # Set `global.mimirProxy.enabled=true` and `global.prometheus.enabled=false` to enable Mimir Proxy. + # You also need to set `global.prometheus.fqdn=http://kubecost-cost-analyzer-mimir-proxy.kubecost.svc:8085/prometheus` + # or `global.prometheus.fqdn=http://{{ template "cost-analyzer.fullname" . }}-mimir-proxy.{{ .Release.Namespace }}.svc:8085/prometheus' + # Learn more at https://grafana.com/docs/mimir/latest/operators-guide/secure/authentication-and-authorization/#without-an-authenticating-reverse-proxy + mimirProxy: + enabled: false + ## Annotations to be added to the Mimir Proxy deployment template + annotations: {} + name: mimir-proxy + image: nginxinc/nginx-unprivileged + port: 8085 + mimirEndpoint: $mimir_endpoint # Your Mimir query endpoint. If your Mimir query endpoint is http://example.com/prometheus, replace $mimir_endpoint with http://example.com/ + orgIdentifier: $your_tenant_ID # Your Grafana Mimir tenant ID + # basicAuth: + # username: user + # password: pwd + + ## Azure Monitor Managed Service for Prometheus + ## Ref: https://learn.microsoft.com/en-us/azure/azure-monitor/essentials/prometheus-remote-write-virtual-machines + ammsp: + enabled: false + prometheusServerEndpoint: http://localhost:8081/ + remoteWriteService: $ + queryEndpoint: $ + + aadAuthProxy: + enabled: false + # per https://github.com/Azure/aad-auth-proxy/releases/tag/0.1.0-main-04-10-2024-7067ac84 + image: $ # Example: mcr.microsoft.com/azuremonitor/auth-proxy/prod/aad-auth-proxy/images/aad-auth-proxy:0.1.0-main-04-10-2024-7067ac84 + imagePullPolicy: IfNotPresent + name: aad-auth-proxy + port: 8081 + audience: https://prometheus.monitor.azure.com/.default + identityType: userAssigned + aadClientId: $ + aadTenantId: $ + + ## Kubecost Alerting + ## Ref: http://docs.kubecost.com/alerts + notifications: + # alertConfigs: + # frontendUrl: http://localhost:9090 # Optional + # globalSlackWebhookUrl: https://hooks.slack.com/services/T00000000/B00000000/XXXXXXXXXXXXXXXXXXXXXXXX # Optional + # globalMsTeamsWebhookUrl: https://xxxxx.webhook.office.com/webhookb2/XXXXXXXXXXXXXXXXXXXXXXXX/IncomingWebhook/XXXXXXXXXXXXXXXXXXXXXXXX # Optional + # globalAlertEmails: + # - recipient@example.com + # - additionalRecipient@example.com + # globalEmailSubject: Custom Subject + # alerts: + # # Daily namespace budget alert on namespace `kubecost` + # - type: budget # supported: budget, recurringUpdate + # threshold: 50 # optional, required for budget alerts + # window: daily # or 1d + # aggregation: namespace + # filter: kubecost + # ownerContact: # optional, overrides globalAlertEmails default + # - owner@example.com + # - owner2@example.com + # slackWebhookUrl: https://hooks.slack.com/services/T00000000/B00000000/XXXXXXXXXXXXXXXXXXXXXXXX # Optional + # msTeamsWebhookUrl: https://xxxxx.webhook.office.com/webhookb2/XXXXXXXXXXXXXXXXXXXXXXXX/IncomingWebhook/XXXXXXXXXXXXXXXXXXXXXXXX # Optional + # # Daily cluster budget alert on cluster `cluster-one` + # - type: budget + # threshold: 200.8 # optional, required for budget alerts + # window: daily # or 1d + # aggregation: cluster + # filter: cluster-one # does not accept csv + # # Recurring weekly update (weeklyUpdate alert) + # - type: recurringUpdate + # window: weekly # or 7d + # aggregation: namespace + # filter: '*' + # # Recurring weekly namespace update on kubecost namespace + # - type: recurringUpdate + # window: weekly # or 7d + # aggregation: namespace + # filter: kubecost + # # Spend Change Alert + # - type: spendChange # change relative to moving avg + # relativeThreshold: 0.20 # Proportional change relative to baseline. Must be greater than -1 (can be negative) + # window: 1d # accepts ‘d’, ‘h’ + # baselineWindow: 30d # previous window, offset by window + # aggregation: namespace + # filter: kubecost, default # accepts csv + # # Health Score Alert + # - type: health # Alerts when health score changes by a threshold + # window: 10m + # threshold: 5 # Send Alert if health scores changes by 5 or more + # # Kubecost Health Diagnostic + # - type: diagnostic # Alerts when kubecost is unable to compute costs - ie: Prometheus unreachable + # window: 10m + + alertmanager: # Supply an alertmanager FQDN to receive notifications from the app. + enabled: false # If true, allow kubecost to write to your alertmanager + fqdn: http://cost-analyzer-prometheus-server.default.svc # example fqdn. Ignored if prometheus.enabled: true + + ## Kubecost Saved Reports + ## Ref: http://docs.kubecost.com/saved-reports + savedReports: + enabled: false # If true, overwrites report parameters set through UI + reports: + - title: "Example Saved Report 0" + window: "today" + aggregateBy: "namespace" + chartDisplay: "category" + idle: "separate" + rate: "cumulative" + accumulate: false # daily resolution + filters: # Ref: https://docs.kubecost.com/apis/filters-api + - key: "cluster" # Ref: https://docs.kubecost.com/apis/filters-api#allocation-apis-request-sizing-v2-api + operator: ":" # Ref: https://docs.kubecost.com/apis/filters-api#filter-operators + value: "dev" + - title: "Example Saved Report 1" + window: "month" + aggregateBy: "controllerKind" + chartDisplay: "category" + idle: "share" + rate: "monthly" + accumulate: false + filters: # Ref: https://docs.kubecost.com/apis/filters-api + - key: "namespace" # Ref: https://docs.kubecost.com/apis/filters-api#allocation-apis-request-sizing-v2-api + operator: "!:" # Ref: https://docs.kubecost.com/apis/filters-api#filter-operators + value: "kubecost" + - title: "Example Saved Report 2" + window: "2020-11-11T00:00:00Z,2020-12-09T23:59:59Z" + aggregateBy: "service" + chartDisplay: "category" + idle: "hide" + rate: "daily" + accumulate: true # entire window resolution + filters: [] # if no filters, specify empty array + assetReports: + enabled: false # If true, overwrites report parameters set through UI + reports: + - title: "Example Asset Report 0" + window: "today" + aggregateBy: "type" + accumulate: false # daily resolution + filters: + - property: "cluster" + value: "cluster-one" + cloudCostReports: + enabled: false # If true, overwrites report parameters set through UI + reports: + - title: "Cloud Cost Report 0" + window: "today" + aggregateBy: "service" + accumulate: false # daily resolution + # filters: + # - property: "service" + # value: "service1" # corresponds to a value to filter cloud cost aggregate by service data on. + + podAnnotations: {} + # iam.amazonaws.com/role: role-arn + + # Annotations to be added for all controllers (StatefulSets, Deployments, DaemonSets) + annotations: {} + # iam.amazonaws.com/role: role-arn + + # Applies these labels to all Deployments, StatefulSets, DaemonSets, and their pod templates. + additionalLabels: {} + + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + fsGroup: 1001 + runAsGroup: 1001 + runAsUser: 1001 + fsGroupChangePolicy: OnRootMismatch + containerSecurityContext: + allowPrivilegeEscalation: false + privileged: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + + # Installs custom CA certificates onto Kubecost pods + updateCaTrust: + enabled: false # Set to true to enable the init container for updating CA trust + # Security context settings for the init container. + securityContext: + runAsUser: 0 + runAsGroup: 0 + runAsNonRoot: false + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + seccompProfile: + type: RuntimeDefault + caCertsSecret: ca-certs-secret # The name of the Secret containing custom CA certificates to mount to the cost-model container. + # caCertsConfig: ca-certs-config # The name of the ConfigMap containing the CA trust configuration. + resources: {} # Resource requests and limits for the init container. + caCertsMountPath: /etc/pki/ca-trust/source/anchors # The path where the custom CA certificates will be mounted in the init container + + # Platforms is a higher-level abstraction for platform-specific values and settings. + platforms: + # Deploying to OpenShift (OCP) requires enabling this option. + openshift: + enabled: false # Deploy Kubecost to OpenShift. + route: + enabled: false # Create an OpenShift Route. + annotations: {} # Add annotations to the Route. + # host: kubecost.apps.okd4.example.com # Add a custom host for your Route. + + # OPTIONAL. The following configs only to be enabled when using a Prometheus instance already installed in the cluster. + createMonitoringClusterRoleBinding: false # Create a ClusterRoleBinding to grant the Kubecost serviceaccount access to query Prometheus. + createMonitoringResourceReaderRoleBinding: false # Create a Role and Role Binding to allow Prometheus to list and watch Kubecost resources. + monitoringServiceAccountName: prometheus-k8s # Name of the Prometheus serviceaccount to bind to the Resource Reader Role Binding. + monitoringServiceAccountNamespace: openshift-monitoring # Namespace of the Prometheus serviceaccount to bind to the Resource Reader Role Binding. + + # Create Security Context Constraint resources for the DaemonSets requiring additional privileges. + scc: + nodeExporter: false # Creates an SCC for Prometheus Node Exporter. This requires Node Exporter be enabled. + networkCosts: false # Creates an SCC for Kubecost network-costs. This requires network-costs be enabled. + # When OpenShift is enabled, the following securityContext will be applied to all resources unless they define their own. + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + # Set options for deploying with CI/CD tools like Argo CD. + cicd: + enabled: false # Set to true when using affected CI/CD tools for access to the below configuration options. + skipSanityChecks: false # If true, skip all sanity/existence checks for resources like Secrets. + + ## Kubecost Integrations + ## Ref: https://docs.kubecost.com/integrations + integrations: + turbonomic: + enabled: false # Set to true to enable the Turbonomic integration + clientId: "" # Client ID generated from the OAuth Client created + clientSecret: "" # Client Secret generated from the OAuth Client created + role: "" # Role that the OAuth Client was created with (e.g. ADMINISTRATOR, SITE_ADMIN, etc.) + host: "" # URL to your turbonomic API. EG: https://turbonomic.example.com/ + insecureClient: false # Do not verify certificate + postgres: + enabled: false + runInterval: "12h" # How frequently to run the integration. + databaseHost: "" # REQUIRED. ex: my.postgres.database.azure.com + databasePort: "" # REQUIRED. ex: 5432 + databaseName: "" # REQUIRED. ex: postgres + databaseUser: "" # REQUIRED. ex: myusername + databasePassword: "" # REQUIRED. ex: mypassword + databaseSecretName: "" # OPTIONAL. Specify your own k8s secret containing the above credentials. Must have key "creds.json". + + ## Configure what Postgres table to write to, and what parameters to pass + ## when querying Kubecost's APIs. Ensure all parameters are enclosed in + ## quotes. Ref: https://docs.kubecost.com/apis/apis-overview + queryConfigs: + allocations: [] + # - databaseTable: "kubecost_allocation_data" + # window: "7d" + # aggregate: "namespace" + # idle: "true" + # shareIdle: "true" + # shareNamespaces: "kubecost,kube-system" + # shareLabels: "" + # - databaseTable: "kubecost_allocation_data_by_cluster" + # window: "10d" + # aggregate: "cluster" + # idle: "true" + # shareIdle: "false" + # shareNamespaces: "" + # shareLabels: "" + assets: [] + # - databaseTable: "kubecost_assets_data" + # window: "7d" + # aggregate: "cluster" + cloudCosts: [] + # - databaseTable: "kubecost_cloudcosts_data" + # window: "7d" + # aggregate: "service" + +## Provide a name override for the chart. +# nameOverride: "" +## Provide a full name override option for the chart. +# fullnameOverride: "" + +## Provide additional labels for the chart. +# chartLabels: +# app.kubernetes.io/name: kubecost-cost-analyzer + +## This flag is only required for users upgrading to a new version of Kubecost. +## The flag is used to ensure users are aware of important +## (potentially breaking) changes included in the new version. +## +upgrade: + toV2: false + +# generated at http://kubecost.com/install, used for alerts tracking and free trials +kubecostToken: # "" + +# Advanced pipeline for custom prices, enterprise key required +pricingCsv: + enabled: false + location: + provider: "AWS" + region: "us-east-1" + URI: s3://kc-csv-test/pricing_schema.csv # a valid file URI + csvAccessCredentials: pricing-schema-access-secret + +## Kubecost SAML (enterprise key required) +## Ref: https://docs.kubecost.com/install-and-configure/advanced-configuration/user-management-saml +saml: + enabled: false + # secretName: "" + # metadataSecretName: "" # One of metadataSecretName or idpMetadataURL must be set. Defaults to idpMetadataURL if set. + # idpMetadataURL: "" + # appRootURL: "" + # authTimeout: 1440 # Number of minutes the JWT will be valid + # redirectURL: "" # Callback URL redirected to after logout + # audienceURI: "" # Usually the same as the appRootURL. Optionally any string uniquely identifying kubecost to your SAML IDP. + # nameIDFormat: "" # If your SAML provider requires a specific nameid format + # isGLUUProvider: false # An additional URL parameter must be appended for GLUU providers + # encryptionCertSecret: "" # K8s secret storing the x509 certificate used to encrypt an Okta SAML response + # decryptionKeySecret: "" # K8s secret storing the private key associated with the encryptionCertSecret + # authSecret: "" # Value of SAML secret used to issue tokens, will be autogenerated as random string if not provided + # authSecretName: "" # Name of K8s secret where the authSecret will be stored. Defaults to "kubecost-saml-secret" if not provided. + rbac: + enabled: false + # groups: + # - name: admin + # enabled: false # If admin is disabled, all SAML users will be able to make configuration changes to the Kubecost frontend + # assertionName: "" + # assertionValues: + # - "admin" + # - "superusers" + # - name: readonly + # enabled: false # If readonly is disabled, all users authorized on SAML will default to readonly + # assertionName: "" + # assertionValues: + # - "readonly" + # - name: editor + # enabled: true # If editor is enabled, editors will be allowed to edit reports/alerts scoped to them, and act as readers otherwise. Users will never default to editor. + # assertionName: "" + # assertionValues: + # - "editor" + +## Kubecost OIDC (enterprise key required) +## Ref: https://docs.kubecost.com/install-and-configure/advanced-configuration/user-management-oidc +oidc: + enabled: false + clientID: "" # Application client_id parameter obtained from provider. Used to make requests to server. + clientSecret: "" # Application/client client_secret parameter obtained from provider. Used to make requests to server. + secretName: "kubecost-oidc-secret" # K8s secret where clientsecret will be stored + existingCustomSecret: + enabled: false + name: "" # Name of an existing clientSecret. Overrides the usage of oidc.clientSecret and oidc.secretName. + authURL: "" # Authorization endpoint for your identity provider + loginRedirectURL: "" # Kubecost URL endpoint which handles auth flow + discoveryURL: "" # Your identity provider's endpoint sharing OIDC configuration + skipOnlineTokenValidation: false # If true, validate JWT claims locally + useClientSecretPost: false # If true, only use client_secret_post method. Otherwise attempt to send the secret in both the header and the body. + hostedDomain: "" # Optional, blocks access to the auth domain specified in the hd claim of the provider ID token + rbac: + enabled: false + # groups: + # - name: admin # Admins have permissions to edit Kubecost settings and save reports + # enabled: false + # claimName: "roles" # Kubecost matches this string against the JWT's payload key containing RBAC info (this value is unique across identity providers) + # claimValues: # Kubecost matches these strings with the roles created in your identity provider + # - "admin" + # - "superusers" + # - name: readonly # Readonly users do not have permissions to edit Kubecost settings or save reports. + # enabled: false + # claimName: "roles" + # claimValues: + # - "readonly" + # - name: editor # Editors have permissions to edit reports/alerts and act as readers otherwise + # enabled: false + # claimName: "roles" + # claimValues: + # - "editor" + +## Adds the HTTP_PROXY, HTTPS_PROXY, and NO_PROXY environment variables to all +## containers. Typically used in environments that have firewall rules which +## prevent kubecost from accessing cloud provider resources. +## Ref: https://www.oreilly.com/library/view/security-with-go/9781788627917/5ea6a02b-3d96-44b1-ad3c-6ab60fcbbe4f.xhtml +## +systemProxy: + enabled: false + httpProxyUrl: "" + httpsProxyUrl: "" + noProxy: "" + +# imagePullSecrets: +# - name: "image-pull-secret" + +# imageVersion uses the base image name (image:) but overrides the version +# pulled. It should be avoided. If non-default behavior is needed, use +# fullImageName for the relevant component. +# imageVersion: + +kubecostFrontend: + enabled: true + deployMethod: singlepod # haMode or singlepod - haMode is currently only supported with Enterprise tier + haReplicas: 2 # only used with haMode + image: "gcr.io/kubecost1/frontend" + imagePullPolicy: IfNotPresent + # fullImageName overrides the default image construction logic. The exact + # image provided (registry, image, tag) will be used for the frontend. + # fullImageName: + + # extraEnv: + # - name: NGINX_ENTRYPOINT_WORKER_PROCESSES_AUTOTUNE + # value: "1" + # securityContext: + # readOnlyRootFilesystem: true + resources: + requests: + cpu: "10m" + memory: "55Mi" + deploymentStrategy: {} + readinessProbe: + enabled: true + initialDelaySeconds: 1 + periodSeconds: 5 + failureThreshold: 6 + livenessProbe: + enabled: true + initialDelaySeconds: 1 + periodSeconds: 5 + failureThreshold: 6 + ipv6: + enabled: true # disable if the cluster does not support ipv6 + # timeoutSeconds: 600 # should be rarely used, but can be increased if needed + # allow customizing nginx-conf server block + # extraServerConfig: |- + # proxy_busy_buffers_size 512k; + # proxy_buffers 4 512k; + # proxy_buffer_size 256k; + # large_client_header_buffers 4 64k; + # hideDiagnostics: false # useful if the primary is not monitored. Supported in limited environments. + # hideOrphanedResources: false # OrphanedResources works on the primary-cluster's cloud-provider only. + + # set to true to set all upstreams to use ..svc.cluster.local instead of just . + useDefaultFqdn: false +# api: +# fqdn: kubecost-api.kubecost.svc.cluster.local:9001 +# model: +# fqdn: kubecost-model.kubecost.svc.cluster.local:9003 +# forecasting: +# fqdn: kubecost-forcasting.kubecost.svc.cluster.local:5000 +# aggregator: +# fqdn: kubecost-aggregator.kubecost.svc.cluster.local:9004 +# cloudCost: +# fqdn: kubecost-cloud-cost.kubecost.svc.cluster.local:9005 +# multiClusterDiagnostics: +# fqdn: kubecost-multi-diag.kubecost.svc.cluster.local:9007 +# clusterController: +# fqdn: cluster-controller.kubecost.svc.cluster.local:9731 + +# Kubecost Metrics deploys a separate pod which will emit kubernetes specific metrics required +# by the cost-model. This pod is designed to remain active and decoupled from the cost-model itself. +# However, disabling this service/pod deployment will flag the cost-model to emit the metrics instead. +kubecostMetrics: + # emitPodAnnotations: false + # emitNamespaceAnnotations: false + # emitKsmV1Metrics: true # emit all KSM metrics in KSM v1. + # emitKsmV1MetricsOnly: false # emit only the KSM metrics missing from KSM v2. Advanced users only. + +sigV4Proxy: + image: public.ecr.aws/aws-observability/aws-sigv4-proxy:latest + imagePullPolicy: IfNotPresent + name: aps + port: 8005 + region: us-west-2 # The AWS region + host: aps-workspaces.us-west-2.amazonaws.com # The hostname for AMP service. + # role_arn: arn:aws:iam:::role/role-name # The AWS IAM role to assume. + extraEnv: # Pass extra env variables to sigV4Proxy + # - name: AWS_ACCESS_KEY_ID + # value: + # - name: AWS_SECRET_ACCESS_KEY + # value: + resources: {} + +kubecostModel: + image: "gcr.io/kubecost1/cost-model" + imagePullPolicy: IfNotPresent + # fullImageName overrides the default image construction logic. The exact + # image provided (registry, image, tag) will be used for cost-model. + # fullImageName: + + # Log level for the cost model container. Options are "trace", "debug", "info", "warn", "error", "fatal", "panic" + logLevel: info + + # securityContext: + # readOnlyRootFilesystem: true + + # The total number of days the ETL pipelines will build + # Set to 0 to disable daily ETL (not recommended) + etlDailyStoreDurationDays: 91 + # The total number of hours the ETL pipelines will build + # Set to 0 to disable hourly ETL (recommended for large environments) + # Must be < prometheus server retention, otherwise empty data may overwrite + # known-good data + etlHourlyStoreDurationHours: 49 + # For deploying kubecost in a cluster that does not self-monitor + etlReadOnlyMode: false + + ## The name of the Secret containing a bucket config for Federated storage. + ## The contents should be stored under a key named federated-store.yaml. + ## Ref: https://docs.kubecost.com/install-and-configure/install/multi-cluster/long-term-storage-configuration + # federatedStorageConfigSecret: federated-store + + ## Federated storage config can be supplied via a secret or the yaml block + ## below when using the block below, only a single provider is supported, + ## others are for example purposes. + ## Ref: https://docs.kubecost.com/install-and-configure/install/multi-cluster/long-term-storage-configuration + # federatedStorageConfig: |- + # # AWS EXAMPLE + # type: S3 + # config: + # bucket: kubecost-federated-storage-bucket + # endpoint: s3.amazonaws.com + # region: us-east-1 + # # best practice is to use pod identities to access AWS resources. Otherwise it is possible to use an access_key and secret_key + # access_key: "" + # secret_key: "" + # # AZURE EXAMPLE + # type: AZURE + # config: + # storage_account: "" + # storage_account_key: "" + # container: "" + # max_retries: 0 + # # GCP EXAMPLE + # type: GCS + # config: + # bucket: kubecost-federated-storage-bucket + # service_account: |- + # { + # "type": "service_account", + # "project_id": "...", + # "private_key_id": "...", + # "private_key": "...", + # "client_email": "...", + # "client_id": "...", + # "auth_uri": "https://accounts.google.com/o/oauth2/auth", + # "token_uri": "https://oauth2.googleapis.com/token", + # "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + # "client_x509_cert_url": "" + # } + + # Installs Kubecost/OpenCost plugins + plugins: + enabled: false + install: + enabled: false + fullImageName: curlimages/curl:latest + securityContext: + allowPrivilegeEscalation: false + seccompProfile: + type: RuntimeDefault + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1001 + folder: /opt/opencost/plugin + + # leave this commented to always download most recent version of plugins + # version: + + # the list of enabled plugins + enabledPlugins: [] + # - datadog + + # pre-existing secret for plugin configuration + existingCustomSecret: + enabled: false + name: "" # name of the secret containing plugin config + + secretName: kubecost-plugin-secret + + # uncomment this to define plugin configuration via the values file + # configs: + # datadog: | + # { + # "datadog_site": "", + # "datadog_api_key": "", + # "datadog_app_key": "" + # } + + allocation: + # Enables or disables adding node labels to allocation data (i.e. workloads). + # Defaults to "true" and starts with a sensible includeList for basics like + # topology (e.g. zone, region) and instance type labels. + # nodeLabels: + # enabled: true + # includeList: "node.kubernetes.io/instance-type,topology.kubernetes.io/region,topology.kubernetes.io/zone" + + # Enables or disables the ContainerStats pipeline, used for quantile-based + # queries like for request sizing recommendations. + # ContainerStats provides support for quantile-based request right-sizing + # recommendations. + # + # It is disabled by default to avoid problems in extremely high-scale Thanos + # environments. If you would like to try quantile-based request-sizing + # recommendations, enable this! If you are in a high-scale environment, + # please monitor Kubecost logs, Thanos query logs, and Thanos load closely. + # We hope to make major improvements at scale here soon! + # + containerStatsEnabled: true # enabled by default as of v2.2.0 + + # max number of concurrent Prometheus queries + maxQueryConcurrency: 5 + resources: + requests: + cpu: "200m" + memory: "55Mi" + # limits: + # cpu: "800m" + # memory: "256Mi" + + readinessProbe: + enabled: true + initialDelaySeconds: 10 + periodSeconds: 10 + failureThreshold: 200 + livenessProbe: + enabled: true + initialDelaySeconds: 10 + periodSeconds: 10 + failureThreshold: 200 + extraArgs: [] + + # Optional. A list of extra environment variables to be added to the cost-model container. + # extraEnv: + # - name: LOG_FORMAT + # value: json + # # When false, Kubecost will not show Asset costs for local disks physically + # # attached to nodes (e.g. ephemeral storage). This needs to be applied to + # # each cluster monitored. + # - name: ASSET_INCLUDE_LOCAL_DISK_COST + # value: "true" + + utcOffset: "+00:00" + extraPorts: [] + +## etlUtils is a utility typically used by Enterprise customers transitioning +## from v1 to v2 of Kubecost. It translates the data from the "/etl" dir of the +## bucket, to the "/federated" dir of the bucket. +## Ref: https://docs.kubecost.com/install-and-configure/install/multi-cluster/federated-etl/thanos-migration-guide +## +etlUtils: + enabled: false + fullImageName: null + resources: {} + env: {} + nodeSelector: {} + tolerations: [] + ## Annotations to be added to etlutils deployment + annotations: {} + affinity: {} + +# Basic Kubecost ingress, more examples available at https://docs.kubecost.com/install-and-configure/install/ingress-examples +ingress: + enabled: false + # className: nginx + labels: + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + annotations: + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + paths: ["/"] # There's no need to route specifically to the pods-- we have an nginx deployed that handles routing + pathType: ImplementationSpecific + hosts: + - cost-analyzer.local + tls: [] + # - secretName: cost-analyzer-tls + # hosts: + # - cost-analyzer.local + +nodeSelector: {} +tolerations: [] +affinity: {} +topologySpreadConstraints: [] +priority: + enabled: false + name: "" +extraVolumes: [] +extraVolumeMounts: [] + +# Define persistence volume for cost-analyzer, more information at https://docs.kubecost.com/install-and-configure/install/storage +persistentVolume: + size: 32Gi + enabled: true # Note that setting this to false means configurations will be wiped out on pod restart. + # storageClass: "-" # + # existingClaim: kubecost-cost-analyzer # a claim in the same namespace as kubecost + labels: {} + annotations: {} + +service: + type: ClusterIP + port: 9090 + targetPort: 9090 + nodePort: {} + labels: {} + annotations: {} + # loadBalancerSourceRanges: [] + sessionAffinity: + enabled: false # Makes sure that connections from a client are passed to the same Pod each time, when set to `true`. You should set it when you enabled authentication through OIDC or SAML integration. + timeoutSeconds: 10800 + +prometheus: + ## Provide a full name override for Prometheus. + # fullnameOverride: "" + ## Provide a name override for Prometheus. + # nameOverride: "" + + rbac: + create: true # Create the RBAC resources for Prometheus. + + serviceAccounts: + alertmanager: + create: true + name: + nodeExporter: + create: true + name: + server: + create: true + name: + ## Prometheus server ServiceAccount annotations. + ## Can be used for AWS IRSA annotations when using Remote Write mode with Amazon Managed Prometheus. + annotations: {} + + ## Specify an existing ConfigMap to be used by Prometheus when using self-signed certificates. + ## + # selfsignedCertConfigMapName: "" + + imagePullSecrets: + + extraScrapeConfigs: | + - job_name: kubecost + honor_labels: true + scrape_interval: 1m + scrape_timeout: 60s + metrics_path: /metrics + scheme: http + dns_sd_configs: + - names: + - {{ template "cost-analyzer.serviceName" . }} + type: 'A' + port: 9003 + - job_name: kubecost-networking + kubernetes_sd_configs: + - role: pod + relabel_configs: + # Scrape only the the targets matching the following metadata + - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance] + action: keep + regex: kubecost + - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name] + action: keep + regex: network-costs + - job_name: kubecost-aggregator + scrape_interval: 1m + scrape_timeout: 60s + metrics_path: /metrics + scheme: http + dns_sd_configs: + - names: + - {{ template "aggregator.serviceName" . }} + type: 'A' + {{- if or .Values.saml.enabled .Values.oidc.enabled }} + port: 9008 + {{- else }} + port: 9004 + {{- end }} + ## Enables scraping of NVIDIA GPU metrics via dcgm-exporter. Scrapes all + ## endpoints which contain "dcgm-exporter" in labels "app", + ## "app.kubernetes.io/component", or "app.kubernetes.io/name" with a case + ## insensitive match. The label must be present on the K8s service endpoints and not just pods. + ## Refs: + ## https://github.com/NVIDIA/gpu-operator/blob/d4316a415bbd684ce8416a88042305fc1a093aa4/assets/state-dcgm-exporter/0600_service.yaml#L7 + ## https://github.com/NVIDIA/dcgm-exporter/blob/54fd1ca137c66511a87a720390613680b9bdabdd/deployment/templates/service.yaml#L23 + - job_name: kubecost-dcgm-exporter + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - source_labels: [__meta_kubernetes_pod_label_app, __meta_kubernetes_pod_label_app_kubernetes_io_component, __meta_kubernetes_pod_label_app_kubernetes_io_name] + action: keep + regex: (?i)(.*dcgm-exporter.*|.*dcgm-exporter.*|.*dcgm-exporter.*) + + server: + # If clusterIDConfigmap is defined, instead use user-generated configmap with key CLUSTER_ID + # to use as unique cluster ID in kubecost cost-analyzer deployment. + # This overrides the cluster_id set in prometheus.server.global.external_labels. + # NOTE: This does not affect the external_labels set in prometheus config. + # clusterIDConfigmap: cluster-id-configmap + + ## Provide a full name override for the Prometheus server. + # fullnameOverride: "" + + enabled: true + name: server + sidecarContainers: + strategy: + type: Recreate + rollingUpdate: null + image: + repository: quay.io/prometheus/prometheus + tag: v2.55.1 + pullPolicy: IfNotPresent + priorityClassName: "" + prefixURL: "" + baseURL: "" + env: [] + extraFlags: + - web.enable-lifecycle + configPath: /etc/config/prometheus.yml + global: + scrape_interval: 1m + scrape_timeout: 60s + evaluation_interval: 1m + external_labels: + cluster_id: cluster-one # Each cluster should have a unique ID + remoteWrite: {} + remoteRead: {} + extraArgs: + query.max-concurrency: 1 + query.max-samples: 100000000 + extraInitContainers: [] + extraVolumeMounts: [] + extraVolumes: [] + extraHostPathMounts: [] + extraConfigmapMounts: [] + extraSecretMounts: [] + configMapOverrideName: "" + ingress: + enabled: false + # className: nginx + annotations: {} + extraLabels: {} + hosts: [] + pathType: "Prefix" + extraPaths: [] + tls: [] + # strategy: + # type: Recreate + tolerations: [] + nodeSelector: {} + affinity: {} + podDisruptionBudget: + enabled: false + maxUnavailable: 1 + # schedulerName: + persistentVolume: + enabled: true + accessModes: + - ReadWriteOnce + annotations: {} + existingClaim: "" + mountPath: /data + size: 32Gi + # storageClass: "-" + # volumeBindingMode: "" + subPath: "" + emptyDir: + sizeLimit: "" + podAnnotations: {} + annotations: {} + podLabels: {} + alertmanagers: [] + replicaCount: 1 + statefulSet: + enabled: false + annotations: {} + labels: {} + podManagementPolicy: OrderedReady + headless: + annotations: {} + labels: {} + servicePort: 80 + readinessProbeInitialDelay: 5 + readinessProbeTimeout: 3 + readinessProbeFailureThreshold: 3 + readinessProbeSuccessThreshold: 1 + livenessProbeInitialDelay: 5 + livenessProbeTimeout: 3 + livenessProbeFailureThreshold: 3 + livenessProbeSuccessThreshold: 1 + resources: {} + verticalAutoscaler: + enabled: false + ## Optional. Defaults to "Auto" if not specified. + # updateMode: "Auto" + ## Mandatory. Without, VPA will not be created. + # containerPolicies: + # - containerName: 'prometheus-server' + securityContext: {} + containerSecurityContext: {} + service: + annotations: {} + labels: {} + clusterIP: "" + externalIPs: [] + loadBalancerIP: "" + loadBalancerSourceRanges: [] + servicePort: 80 + sessionAffinity: None + type: ClusterIP + gRPC: + enabled: false + servicePort: 10901 + statefulsetReplica: + enabled: false + replica: 0 + terminationGracePeriodSeconds: 300 + + ## Prometheus data retention period (default if not specified is 97 hours) + ## + ## Kubecost builds up its own persistent store of metric data on the + ## filesystem (usually a PV) and, when using ETL Backup and/or Federated + ## ETL, in more durable object storage like S3 or GCS. Kubecost's data + ## retention is _not_ tied to the configured Prometheus retention. + ## + ## For data durability, we recommend using ETL Backup instead of relying on + ## Prometheus retention. + ## + ## Lower retention values will affect Prometheus by reducing resource + ## consumption and increasing stability. It _must not_ be set below or equal + ## to kubecostModel.etlHourlyStoreDurationHours, otherwise empty data sets + ## may overwrite good data sets. For now, it must also be >= 49h for Daily + ## ETL stability. + ## + ## "ETL Rebuild" and "ETL Repair" is only possible on data available within + ## this retention window. This is an extremely rare operation. + ## + ## If you want maximum security in the event of a Kubecost agent + ## (cost-model) outage, increase this value. The current default of 97h is + ## intended to balance Prometheus stability and resource consumption + ## against the event of an outage in Kubecost which would necessitate a + ## version change. 4 days should provide enough time for most users to + ## notice a problem and initiate corrective action. + retention: 97h + # retentionSize: should be significantly greater than the storage used in the number of hours set in etlHourlyStoreDurationHours + + # Install Prometheus Alert Manager + alertmanager: + enabled: false + ## Provide a full name override for Prometheus alertmanager. + # fullnameOverride: "" + strategy: + type: Recreate + rollingUpdate: null + name: alertmanager + image: + repository: quay.io/prometheus/alertmanager + tag: v0.27.0 + pullPolicy: IfNotPresent + priorityClassName: "" + extraArgs: {} + prefixURL: "" + baseURL: "http://localhost:9093" + extraEnv: {} + extraSecretMounts: [] + configMapOverrideName: "" + configFromSecret: "" + configFileName: alertmanager.yml + ingress: + enabled: false + annotations: {} + extraLabels: {} + hosts: [] + extraPaths: [] + tls: [] + # strategy: + # type: Recreate + tolerations: [] + nodeSelector: {} + affinity: {} + podDisruptionBudget: + enabled: false + maxUnavailable: 1 + # schedulerName: + persistentVolume: + enabled: true + accessModes: + - ReadWriteOnce + annotations: {} + existingClaim: "" + mountPath: /data + size: 2Gi + # storageClass: "-" + # volumeBindingMode: "" + subPath: "" + podAnnotations: {} + annotations: {} + podLabels: {} + replicaCount: 1 + statefulSet: + enabled: false + annotations: {} + podManagementPolicy: OrderedReady + headless: + annotations: {} + labels: {} + # enableMeshPeer : true + servicePort: 80 + resources: {} + securityContext: + runAsUser: 1001 + runAsNonRoot: true + runAsGroup: 1001 + fsGroup: 1001 + service: + annotations: {} + labels: {} + clusterIP: "" + # enableMeshPeer : true + externalIPs: [] + loadBalancerIP: "" + loadBalancerSourceRanges: [] + servicePort: 80 + # nodePort: 30000 + sessionAffinity: None + type: ClusterIP + + alertmanagerFiles: + alertmanager.yml: + global: {} + receivers: + - name: default-receiver + route: + group_wait: 10s + group_interval: 5m + receiver: default-receiver + repeat_interval: 3h + + ## Monitors ConfigMap changes and POSTs to a URL + configmapReload: + prometheus: + enabled: false + name: configmap-reload + image: + repository: quay.io/prometheus-operator/prometheus-config-reloader + tag: v0.78.2 + pullPolicy: IfNotPresent + extraArgs: {} + extraVolumeDirs: [] + extraConfigmapMounts: [] + resources: {} + containerSecurityContext: {} + + alertmanager: + enabled: false + name: configmap-reload + image: + repository: quay.io/prometheus-operator/prometheus-config-reloader + tag: v0.78.2 + pullPolicy: IfNotPresent + extraArgs: {} + extraVolumeDirs: [] + extraConfigmapMounts: [] + resources: {} + + nodeExporter: + ## If false, node-exporter will not be installed. + ## This is disabled by default in Kubecost 2.0, though it can be enabled as needed. + ## + enabled: false + + ## Provide a full name override for node exporter. + # fullnameOverride: "" + + hostNetwork: true + hostPID: true + dnsPolicy: ClusterFirstWithHostNet + name: node-exporter + image: + repository: prom/node-exporter + tag: v1.8.2 + pullPolicy: IfNotPresent + priorityClassName: "" + updateStrategy: + type: RollingUpdate + extraArgs: {} + extraHostPathMounts: [] + extraConfigmapMounts: [] + # affinity: + tolerations: [] + nodeSelector: {} + podAnnotations: {} + annotations: {} + pod: + labels: {} + podDisruptionBudget: + enabled: false + maxUnavailable: 1 + resources: {} + securityContext: {} + service: + annotations: + prometheus.io/scrape: "true" + labels: {} + clusterIP: None + externalIPs: [] + hostPort: 9100 + loadBalancerIP: "" + loadBalancerSourceRanges: [] + servicePort: 9100 + type: ClusterIP + + serverFiles: + ## Alerts configuration + ## Ref: https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/ + alerting_rules.yml: {} + + ## Records configuration + ## Ref: https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/ + recording_rules.yml: {} + + prometheus.yml: + rule_files: + - /etc/config/recording_rules.yml + - /etc/config/alerting_rules.yml + + scrape_configs: + - job_name: prometheus + static_configs: + - targets: + - localhost:9090 + + # A scrape configuration for running Prometheus on a Kubernetes cluster. + # This uses separate scrape configs for cluster components (i.e. API server, node) + # and services to allow each to use different authentication configs. + # + # Kubernetes labels will be added as Prometheus labels on metrics via the + # `labelmap` relabeling action. + - job_name: 'kubernetes-nodes-cadvisor' + + # Default to scraping over https. If required, just disable this or change to + # `http`. + scheme: https + + # This TLS & bearer token file config is used to connect to the actual scrape + # endpoints for cluster components. This is separate to discovery auth + # configuration because discovery & scraping are two separate concerns in + # Prometheus. The discovery auth config is automatic if Prometheus runs inside + # the cluster. Otherwise, more config options have to be provided within the + # . + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + # If your node certificates are self-signed or use a different CA to the + # master CA, then disable certificate verification below. Note that + # certificate verification is an integral part of a secure infrastructure + # so this should only be disabled in a controlled environment. You can + # disable certificate verification by uncommenting the line below. + # + insecure_skip_verify: true + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + + kubernetes_sd_configs: + - role: node + + # This configuration will work only on kubelet 1.7.3+ + # As the scrape endpoints for cAdvisor have changed + # if you are using older version you need to change the replacement to + # replacement: /api/v1/nodes/$1:4194/proxy/metrics + # more info here https://github.com/coreos/prometheus-operator/issues/633 + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/$1/proxy/metrics/cadvisor + + metric_relabel_configs: + - source_labels: [__name__] + regex: (container_cpu_usage_seconds_total|container_memory_working_set_bytes|container_network_receive_errors_total|container_network_transmit_errors_total|container_network_receive_packets_dropped_total|container_network_transmit_packets_dropped_total|container_memory_usage_bytes|container_cpu_cfs_throttled_periods_total|container_cpu_cfs_periods_total|container_fs_usage_bytes|container_fs_limit_bytes|container_cpu_cfs_periods_total|container_fs_inodes_free|container_fs_inodes_total|container_fs_usage_bytes|container_fs_limit_bytes|container_cpu_cfs_throttled_periods_total|container_cpu_cfs_periods_total|container_network_receive_bytes_total|container_network_transmit_bytes_total|container_fs_inodes_free|container_fs_inodes_total|container_fs_usage_bytes|container_fs_limit_bytes|container_spec_cpu_shares|container_spec_memory_limit_bytes|container_network_receive_bytes_total|container_network_transmit_bytes_total|container_fs_reads_bytes_total|container_network_receive_bytes_total|container_fs_writes_bytes_total|container_fs_reads_bytes_total|cadvisor_version_info|kubecost_pv_info) + action: keep + - source_labels: [container] + target_label: container_name + regex: (.+) + action: replace + - source_labels: [pod] + target_label: pod_name + regex: (.+) + action: replace + + # A scrape configuration for running Prometheus on a Kubernetes cluster. + # This uses separate scrape configs for cluster components (i.e. API server, node) + # and services to allow each to use different authentication configs. + # + # Kubernetes labels will be added as Prometheus labels on metrics via the + # `labelmap` relabeling action. + - job_name: 'kubernetes-nodes' + + # Default to scraping over https. If required, just disable this or change to + # `http`. + scheme: https + + # This TLS & bearer token file config is used to connect to the actual scrape + # endpoints for cluster components. This is separate to discovery auth + # configuration because discovery & scraping are two separate concerns in + # Prometheus. The discovery auth config is automatic if Prometheus runs inside + # the cluster. Otherwise, more config options have to be provided within the + # . + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + # If your node certificates are self-signed or use a different CA to the + # master CA, then disable certificate verification below. Note that + # certificate verification is an integral part of a secure infrastructure + # so this should only be disabled in a controlled environment. You can + # disable certificate verification by uncommenting the line below. + # + insecure_skip_verify: true + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + + kubernetes_sd_configs: + - role: node + + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/$1/proxy/metrics + + metric_relabel_configs: + - source_labels: [__name__] + regex: (kubelet_volume_stats_used_bytes) # this metric is in alpha + action: keep + + # Scrape config for service endpoints. + # + # The relabeling allows the actual service scrape endpoint to be configured + # via the following annotations: + # + # * `prometheus.io/scrape`: Only scrape services that have a value of `true` + # * `prometheus.io/scheme`: If the metrics endpoint is secured then you will need + # to set this to `https` & most likely set the `tls_config` of the scrape config. + # * `prometheus.io/path`: If the metrics path is not `/metrics` override this. + # * `prometheus.io/port`: If the metrics are exposed on a different port to the + # service then set this appropriately. + - job_name: 'kubernetes-service-endpoints' + + kubernetes_sd_configs: + - role: endpoints + + relabel_configs: + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__meta_kubernetes_endpoints_name] + action: keep + regex: (.*node-exporter|kubecost-network-costs) + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme] + action: replace + target_label: __scheme__ + regex: (https?) + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port] + action: replace + target_label: __address__ + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_service_name] + action: replace + target_label: kubernetes_name + - source_labels: [__meta_kubernetes_pod_node_name] + action: replace + target_label: kubernetes_node + metric_relabel_configs: + - source_labels: [__name__] + regex: (container_cpu_allocation|container_cpu_usage_seconds_total|container_fs_limit_bytes|container_fs_writes_bytes_total|container_gpu_allocation|container_memory_allocation_bytes|container_memory_usage_bytes|container_memory_working_set_bytes|container_network_receive_bytes_total|container_network_transmit_bytes_total|DCGM_FI_DEV_GPU_UTIL|deployment_match_labels|kube_daemonset_status_desired_number_scheduled|kube_daemonset_status_number_ready|kube_deployment_spec_replicas|kube_deployment_status_replicas|kube_deployment_status_replicas_available|kube_job_status_failed|kube_namespace_annotations|kube_namespace_labels|kube_node_info|kube_node_labels|kube_node_status_allocatable|kube_node_status_allocatable_cpu_cores|kube_node_status_allocatable_memory_bytes|kube_node_status_capacity|kube_node_status_capacity_cpu_cores|kube_node_status_capacity_memory_bytes|kube_node_status_condition|kube_persistentvolume_capacity_bytes|kube_persistentvolume_status_phase|kube_persistentvolumeclaim_info|kube_persistentvolumeclaim_resource_requests_storage_bytes|kube_pod_container_info|kube_pod_container_resource_limits|kube_pod_container_resource_limits_cpu_cores|kube_pod_container_resource_limits_memory_bytes|kube_pod_container_resource_requests|kube_pod_container_resource_requests_cpu_cores|kube_pod_container_resource_requests_memory_bytes|kube_pod_container_status_restarts_total|kube_pod_container_status_running|kube_pod_container_status_terminated_reason|kube_pod_labels|kube_pod_owner|kube_pod_status_phase|kube_replicaset_owner|kube_statefulset_replicas|kube_statefulset_status_replicas|kubecost_cluster_info|kubecost_cluster_management_cost|kubecost_cluster_memory_working_set_bytes|kubecost_load_balancer_cost|kubecost_network_internet_egress_cost|kubecost_network_region_egress_cost|kubecost_network_zone_egress_cost|kubecost_node_is_spot|kubecost_pod_network_egress_bytes_total|node_cpu_hourly_cost|node_cpu_seconds_total|node_disk_reads_completed|node_disk_reads_completed_total|node_disk_writes_completed|node_disk_writes_completed_total|node_filesystem_device_error|node_gpu_count|node_gpu_hourly_cost|node_memory_Buffers_bytes|node_memory_Cached_bytes|node_memory_MemAvailable_bytes|node_memory_MemFree_bytes|node_memory_MemTotal_bytes|node_network_transmit_bytes_total|node_ram_hourly_cost|node_total_hourly_cost|pod_pvc_allocation|pv_hourly_cost|service_selector_labels|statefulSet_match_labels|kubecost_pv_info|up) + action: keep + rules: + groups: + - name: CPU + rules: + - expr: sum(rate(container_cpu_usage_seconds_total{container!=""}[5m])) + record: cluster:cpu_usage:rate5m + - expr: rate(container_cpu_usage_seconds_total{container!=""}[5m]) + record: cluster:cpu_usage_nosum:rate5m + - expr: avg(irate(container_cpu_usage_seconds_total{container!="POD", container!=""}[5m])) by (container,pod,namespace) + record: kubecost_container_cpu_usage_irate + - expr: sum(container_memory_working_set_bytes{container!="POD",container!=""}) by (container,pod,namespace) + record: kubecost_container_memory_working_set_bytes + - expr: sum(container_memory_working_set_bytes{container!="POD",container!=""}) + record: kubecost_cluster_memory_working_set_bytes + - name: Savings + rules: + - expr: sum(avg(kube_pod_owner{owner_kind!="DaemonSet"}) by (pod) * sum(container_cpu_allocation) by (pod)) + record: kubecost_savings_cpu_allocation + labels: + daemonset: "false" + - expr: sum(avg(kube_pod_owner{owner_kind="DaemonSet"}) by (pod) * sum(container_cpu_allocation) by (pod)) / sum(kube_node_info) + record: kubecost_savings_cpu_allocation + labels: + daemonset: "true" + - expr: sum(avg(kube_pod_owner{owner_kind!="DaemonSet"}) by (pod) * sum(container_memory_allocation_bytes) by (pod)) + record: kubecost_savings_memory_allocation_bytes + labels: + daemonset: "false" + - expr: sum(avg(kube_pod_owner{owner_kind="DaemonSet"}) by (pod) * sum(container_memory_allocation_bytes) by (pod)) / sum(kube_node_info) + record: kubecost_savings_memory_allocation_bytes + labels: + daemonset: "true" + + # Adds option to add alert_relabel_configs to avoid duplicate alerts in alertmanager + # useful in H/A prometheus with different external labels but the same alerts + alertRelabelConfigs: + # alert_relabel_configs: + # - source_labels: [dc] + # regex: (.+)\d+ + # target_label: dc + + networkPolicy: + enabled: false + +## Optional daemonset to more accurately attribute network costs to the correct workload +## https://docs.kubecost.com/install-and-configure/advanced-configuration/network-costs-configuration +networkCosts: + enabled: false + image: + repository: gcr.io/kubecost1/kubecost-network-costs + tag: v0.17.6 + imagePullPolicy: IfNotPresent + updateStrategy: + type: RollingUpdate + # For existing Prometheus Installs, use the serviceMonitor: or prometheusScrape below. + # the below setting annotates the networkCost service endpoints for each of the network-costs pods. + # The Service is annotated with prometheus.io/scrape: "true" to automatically get picked up by the prometheus config. + # NOTE: Setting this option to true and leaving the above extraScrapeConfig "job_name: kubecost-networking" configured will cause the + # NOTE: pods to be scraped twice. + prometheusScrape: false + # Traffic Logging will enable logging the top 5 destinations for each source + # every 30 minutes. + trafficLogging: true + + # Log level for the network cost containers. Options are "trace", "debug", "info", "warn", "error", "fatal", "panic" + logLevel: info + + # Port will set both the containerPort and hostPort to this value. + # These must be identical due to network-costs being run on hostNetwork + port: 3001 + # this daemonset can use significant resources on large clusters: https://docs.kubecost.com/using-kubecost/navigating-the-kubecost-ui/cost-allocation/network-allocation + resources: + limits: # remove the limits by setting cpu: null + cpu: 500m # can be less, will depend on cluster size + # memory: it is not recommended to set a memory limit + requests: + cpu: 50m + memory: 20Mi + extraArgs: [] + config: + # Configuration for traffic destinations, including specific classification + # for IPs and CIDR blocks. This configuration will act as an override to the + # automatic classification provided by network-costs. + destinations: + # In Zone contains a list of address/range that will be + # classified as in zone. + in-zone: + # Loopback Addresses in "IANA IPv4 Special-Purpose Address Registry" + - "127.0.0.0/8" + # IPv4 Link Local Address Space + - "169.254.0.0/16" + # Private Address Ranges in RFC-1918 + - "10.0.0.0/8" # Remove this entry if using Multi-AZ Kubernetes + - "172.16.0.0/12" + - "192.168.0.0/16" + + # In Region contains a list of address/range that will be + # classified as in region. This is synonymous with cross + # zone traffic, where the regions between source and destinations + # are the same, but the zone is different. + in-region: [] + + # Cross Region contains a list of address/range that will be + # classified as non-internet egress from one region to another. + cross-region: [] + + # Internet contains a list of address/range that will be + # classified as internet traffic. This is synonymous with traffic + # that cannot be classified within the cluster. + # NOTE: Internet classification filters are executed _after_ + # NOTE: direct-classification, but before in-zone, in-region, + # NOTE: and cross-region. + internet: [] + + # Direct Classification specifically maps an ip address or range + # to a region (required) and/or zone (optional). This classification + # takes priority over in-zone, in-region, and cross-region configurations. + direct-classification: [] + # - region: "us-east1" + # zone: "us-east1-c" + # ips: + # - "10.0.0.0/24" + services: + # google-cloud-services: when set to true, enables labeling traffic metrics with google cloud + # service endpoints + google-cloud-services: true + # amazon-web-services: when set to true, enables labeling traffic metrics with amazon web service + # endpoints. + amazon-web-services: true + # azure-cloud-services: when set to true, enables labeling traffic metrics with azure cloud service + # endpoints + azure-cloud-services: true + # user defined services provide a way to define custom service endpoints which will label traffic metrics + # falling within the defined address range. + # services: + # - service: "test-service-1" + # ips: + # - "19.1.1.2" + # - service: "test-service-2" + # ips: + # - "15.128.15.2" + # - "20.0.0.0/8" + + tolerations: [] + affinity: {} + service: + annotations: {} + labels: {} + priorityClassName: "" + podMonitor: + enabled: false + additionalLabels: {} + additionalLabels: {} + nodeSelector: {} + # Annotations to be added to network cost daemonset template and pod template annotations + annotations: {} + healthCheckProbes: {} + additionalSecurityContext: {} + +## Kubecost Deployment Configuration +## Used for HA mode in Business & Enterprise tier +## +kubecostDeployment: + replicas: 1 + labels: {} + annotations: {} + +## Kubecost Forecasting forecasts future cost patterns based on historical +## patterns observed by Kubecost. +forecasting: + enabled: true + + # fullImageName overrides the default image construction logic. The exact + # image provided (registry, image, tag) will be used for the forecasting + # container. + fullImageName: gcr.io/kubecost1/kubecost-modeling:v0.1.19 + imagePullPolicy: IfNotPresent + + # Resource specification block for the forecasting container. + resources: + requests: + cpu: 200m + memory: 300Mi + limits: + cpu: 1500m + memory: 1Gi + + # Set environment variables for the forecasting container as key/value pairs. + env: + # -t is the worker timeout which primarily affects model training time; + # if it is not high enough, training workers may die mid training + "GUNICORN_CMD_ARGS": "--log-level info -t 1200" + + priority: + enabled: false + name: "" + nodeSelector: {} + tolerations: [] + annotations: {} + affinity: {} + readinessProbe: + enabled: true + initialDelaySeconds: 10 + periodSeconds: 10 + failureThreshold: 200 + livenessProbe: + enabled: true + initialDelaySeconds: 10 + periodSeconds: 10 + failureThreshold: 200 + +## The Kubecost Aggregator is the primary query backend for Kubecost +## Ref: https://docs.kubecost.com/install-and-configure/install/multi-cluster/federated-etl/aggregator +## +kubecostAggregator: + # deployMethod determines how Aggregator is deployed. Current options are + # "singlepod" (within cost-analyzer Pod) "statefulset" (separate + # StatefulSet), and "disabled". Only use "disabled" if this is a secondary + # Federated ETL cluster which does not need to answer queries. + deployMethod: singlepod + + # fullImageName overrides the default image construction logic. The exact + # image provided (registry, image, tag) will be used for aggregator. + # fullImageName: + imagePullPolicy: IfNotPresent + + # For legacy configuration support, `enabled: true` overrides deployMethod + # and causes `deployMethod: "statefulset"` + enabled: false + + # Replicas sets the number of Aggregator replicas. It only has an effect if + # `deployMethod: "statefulset"` + replicas: 1 + + # Log level for the aggregator container. Options are "trace", "debug", "info", "warn", "error", "fatal", "panic" + logLevel: info + + # stagingEmptyDirSizeLimit changes how large the "staging" + # /var/configs/waterfowl emptyDir is. It only takes effect in StatefulSet + # configurations of Aggregator, other configurations are unaffected. + # + # It should be set to approximately 8x the size of the largest bingen file in + # object storage. For example, if your largest bingen file is a daily + # Allocation file with size 300MiB, this value should be set to approximately + # 2400Mi. In most environments, the default should suffice. + stagingEmptyDirSizeLimit: 2Gi + + # this is the number of partitions the datastore is split into for copying + # the higher this number, the lower the ram usage but the longer it takes for + # new data to show in the kubecost UI + # set to 0 for max partitioning (minimum possible ram usage, but the slowest) + # the default of 25 is sufficient for 95%+ of users. This should only be modified + # after consulting with Kubecost's support team + numDBCopyPartitions: 25 + + # How many threads the read database is configured with (i.e. Kubecost API / + # UI queries). If increasing this value, it is recommended to increase the + # aggregator's memory requests & limits. + # default: 1 + dbReadThreads: 1 + + # How many threads the write database is configured with (i.e. ingestion of + # new data from S3). If increasing this value, it is recommended to increase + # the aggregator's memory requests & limits. + # default: 1 + dbWriteThreads: 1 + + # How many threads to use when ingesting Asset/Allocation/CloudCost data + # from the federated store bucket. In most cases the default is sufficient, + # but can be increased if trying to backfill historical data. + # default: 1 + dbConcurrentIngestionCount: 1 + + # Memory limit applied to read database and write database connections. The + # default of "no limit" is appropriate when first establishing a baseline of + # resource usage required. It is eventually recommended to set these values + # such that dbMemoryLimit + dbWriteMemoryLimit < the total memory available + # to the aggregator pod. + # default: 0GB is no limit + dbMemoryLimit: 0GB + dbWriteMemoryLimit: 0GB + + # How much data to ingest from the federated store bucket, and how much data + # to keep in the DB before rolling the data off. + # + # Note: If increasing this value to backfill historical data, it will take + # time to gradually ingest and process those historical ETL files. Consider + # also increasing the resources available to the aggregator as well as the + # refresh and concurrency env vars. + # + # default: 91 + etlDailyStoreDurationDays: 91 + + # How much hourly data to ingest from the federated store bucket, and how much + # to keep in the DB before rolling the data off. + # + # In high scale environments setting this to `0` can improve performance if hourly + # resolution is not a requirement. + # + # default: 49 + etlHourlyStoreDurationHours: 49 + + # How much container resource usage data to retain in the DB, in terms of days. + # + # In high scale environments setting this to `0` can improve performance if hourly + # resolution is not a requirement. + # + # default: 1 + containerResourceUsageRetentionDays: 1 + + # Trim memory on close, only change if advised by Kubecost support. + dbTrimMemoryOnClose: true + + persistentConfigsStorage: + storageClass: "" # default storage class + storageRequest: 1Gi + aggregatorDbStorage: + storageClass: "" # default storage class + storageRequest: 128Gi + + resources: {} + # requests: + # cpu: 1000m + # memory: 1Gi + + readinessProbe: + enabled: true + initialDelaySeconds: 10 + periodSeconds: 10 + failureThreshold: 200 + + ## Set additional environment variables for the aggregator pod + # extraEnv: + # - name: SOME_VARIABLE + # value: "some_value" + + ## Add a priority class to the aggregator pod + # priority: + # enabled: false + # name: "" + + ## Optional - add extra ports to the aggregator container. For kubecost development purposes only - not recommended for users. + # extraPorts: [] + # - name: debug + # port: 40000 + # targetPort: 40000 + # containerPort: 40000 + + ## Define a securityContext for the aggregator pod. This will take highest precedence. + # securityContext: {} + + ## Define the container-level security context for the aggregator pod. This will take highest precedence. + # containerSecurityContext: {} + + ## Provide a Service Account name for aggregator. + # serviceAccountName: "" + + ## Define a nodeSelector for the aggregator pod + # nodeSelector: {} + + ## Define tolerations for the aggregator pod + # tolerations: [] + + ## Annotations to be added for aggregator deployment or statefulset + # annotations: {} + + ## Define Pod affinity for the aggregator pod + # affinity: {} + + ## Define extra volumes for the aggregator pod + # extraVolumes: [] + + ## Define extra volumemounts for the aggregator pod + # extraVolumeMounts: [] + + ## Creates a new container/pod to retrieve CloudCost data. By default it uses + ## the same serviceaccount as the cost-analyzer pod. A custom serviceaccount + ## can be specified. + cloudCost: + # The cloudCost component of Aggregator depends on + # kubecostAggregator.deployMethod: + # kA.dM = "singlepod" -> cloudCost is run as container inside cost-analyzer + # kA.dM = "statefulset" -> cloudCost is run as single-replica Deployment + resources: {} + # requests: + # cpu: 1000m + # memory: 1Gi + # refreshRateHours: + # queryWindowDays: + # runWindowDays: + # serviceAccountName: + readinessProbe: + enabled: true + initialDelaySeconds: 10 + periodSeconds: 10 + failureThreshold: 200 + + ## Add a nodeSelector for aggregator cloud costs + # nodeSelector: {} + + ## Tolerations for the aggregator cloud costs + # tolerations: [] + + ## Affinity for the aggregator cloud costs + # affinity: {} + + ## ServiceAccount for the aggregator cloud costs + # serviceAccountName: "" + + ## Define environment variables for cloud cost + # env: {} + + ## Define extra volumes for the cloud cost pod + # extraVolumes: [] + + ## Define extra volumemounts for the cloud cost pod + # extraVolumeMounts: [] + + ## Configure the Collections service for aggregator. + # collections: + # cache: + # enabled: false + + # Jaeger is an optional container attached to wherever the Aggregator + # container is running. It is used for performance investigation. Enable if + # Kubecost Support asks. + jaeger: + enabled: false + image: jaegertracing/all-in-one + imageVersion: latest + + service: + labels: {} + +## Kubecost Multi-cluster Diagnostics (beta) +## A single view into the health of all agent clusters. Each agent cluster sends +## its diagnostic data to a storage bucket. Future versions may include +## repairing & alerting from the primary. +## Ref: https://docs.kubecost.com/install-and-configure/install/multi-cluster/multi-cluster-diagnostics +## +diagnostics: + enabled: true + + ## The primary aggregates all diagnostic data and handles API requests. It's + ## also responsible for deleting diagnostic data (on disk & bucket) beyond + ## retention. When in readonly mode it does not push its own diagnostic data + ## to the bucket. + primary: + enabled: false + retention: "7d" + readonly: false + + ## How frequently to run & push diagnostics. Defaults to 5 minutes. + pollingInterval: "300s" + + ## Creates a new Diagnostic file in the bucket for every run. + keepDiagnosticHistory: false + + ## Pushes the cluster's Kubecost Helm Values to the bucket once upon startup. + ## This may contain sensitive information and is roughly 30kb per cluster. + collectHelmValues: false + + ## By default, the Multi-cluster Diagnostics service runs within the + ## cost-model container in the cost-analyzer pod. For higher availability, it + ## can be run as a separate deployment. + deployment: + enabled: false + resources: + requests: + cpu: "10m" + memory: "20Mi" + env: {} + labels: {} + securityContext: {} + containerSecurityContext: {} + nodeSelector: {} + tolerations: [] + ## Annotations to be added for diagnostics Deployment. + annotations: {} + affinity: {} + +## Provide a full name override for the diagnostics Deployment. +# diagnosticsFullnameOverride: "" + +# Kubecost Cluster Controller for Right Sizing and Cluster Turndown +clusterController: + enabled: false + image: + repository: gcr.io/kubecost1/cluster-controller + tag: v0.16.11 + imagePullPolicy: IfNotPresent + priorityClassName: "" + tolerations: [] + + ## Annotations to be added for cluster controller template + annotations: {} + resources: {} + affinity: {} + nodeSelector: {} + actionConfigs: + # this configures the Kubecost Cluster Turndown action + # for more details, see documentation at https://github.com/kubecost/cluster-turndown/tree/develop?tab=readme-ov-file#setting-a-turndown-schedule + clusterTurndown: [] + # - name: my-schedule + # start: "2024-02-09T00:00:00Z" + # end: "2024-02-09T12:00:00Z" + # repeat: daily + # - name: my-schedule2 + # start: "2024-02-09T00:00:00Z" + # end: "2024-02-09T01:00:00Z" + # repeat: weekly + # this configures the Kubecost Namespace Turndown action + # for more details, see documentation at https://docs.kubecost.com/using-kubecost/navigating-the-kubecost-ui/savings/savings-actions#namespace-turndown + namespaceTurndown: + # - name: my-ns-turndown-action + # dryRun: false + # schedule: "0 0 * * *" + # type: Scheduled + # targetObjs: + # - namespace + # keepPatterns: + # - ignorednamespace + # keepLabels: + # turndown: ignore + # params: + # minNamespaceAge: 4h + # this configures the Kubecost Cluster Sizing action + # for more details, see documentation at https://docs.kubecost.com/using-kubecost/navigating-the-kubecost-ui/savings/savings-actions#cluster-sizing + clusterRightsize: + # startTime: '2024-01-02T15:04:05Z' + # frequencyMinutes: 1440 + # lastCompleted: '' + # recommendationParams: + # window: 48h + # architecture: '' + # targetUtilization: 0.8 + # minNodeCount: 1 + # allowSharedCore: false + # allowCostIncrease: false + # recommendationType: '' + # This configures the Kubecost Continuous Request Sizing Action + # + # Using this configuration overrides annotation-based configuration of + # Continuous Request Sizing. Annotation configuration will be ignored while + # this configuration method is present in the cluster. + # + # For more details, see documentation at https://docs.kubecost.com/using-kubecost/navigating-the-kubecost-ui/savings/savings-actions#automated-request-sizing + containerRightsize: + # Workloads can be selected by an _exact_ key (namespace, controllerKind, + # controllerName). This will only match a single controller. The cluster + # ID is current irrelevant because Cluster Controller can only modify + # workloads within the cluster it is running in. + # workloads: + # - clusterID: cluster-one + # namespace: my-namespace + # controllerKind: deployment + # controllerName: my-controller + # An alternative to exact key selection is filter selection. The filters + # are syntactically identical to Kubecost's "v2" filters [1] but only + # support a small set of filter fields, those being: + # - namespace + # - controllerKind + # - controllerName + # - label + # - annotation + # + # If multiple filters are listed, they will be ORed together at the top + # level. + # + # See the examples below. + # + # [1] https://docs.kubecost.com/apis/filters-api + # filterConfig: + # - filter: | + # namespace:"abc"+controllerKind:"deployment" + # - filter: | + # controllerName:"abc123"+controllerKind:"daemonset" + # - filter: | + # namespace:"foo"+controllerKind!:"statefulset" + # - filter: | + # namespace:"bar","baz" + # schedule: + # start: "2024-01-30T15:04:05Z" + # frequencyMinutes: 5 + # recommendationQueryWindow: "48h" + # lastModified: '' + # targetUtilizationCPU: 0.8 # results in a cpu request setting that is 20% higher than the max seen over last 48h + # targetUtilizationMemory: 0.8 # results in a RAM request setting that is 20% higher than the max seen over last 48h + + kubescaler: + # If true, will cause all (supported) workloads to be have their requests + # automatically right-sized on a regular basis. + defaultResizeAll: false +# fqdn: kubecost-cluster-controller.kubecost.svc.cluster.local:9731 + namespaceTurndown: + rbac: + enabled: true + +reporting: + # Kubecost bug report feature: Logs access/collection limited to .Release.Namespace + # Ref: http://docs.kubecost.com/bug-report + logCollection: true + # Basic frontend analytics + productAnalytics: true + + # Report Javascript errors + errorReporting: true + valuesReporting: true + # googleAnalyticsTag allows you to embed your Google Global Site Tag to track usage of Kubecost. + # googleAnalyticsTag is only included in our Enterprise offering. + # googleAnalyticsTag: G-XXXXXXXXX + +serviceMonitor: # the kubecost included prometheus uses scrapeConfigs and does not support service monitors. The following options assume an existing prometheus that supports serviceMonitors. + enabled: false + interval: 1m + scrapeTimeout: 10s + additionalLabels: {} + metricRelabelings: [] + relabelings: [] + networkCosts: + enabled: false + interval: 1m + scrapeTimeout: 10s + additionalLabels: {} + metricRelabelings: [] + relabelings: [] + aggregatorMetrics: + enabled: false + interval: 1m + scrapeTimeout: 10s + additionalLabels: {} + metricRelabelings: [] + relabelings: + - action: replace + sourceLabels: + - __meta_kubernetes_namespace + targetLabel: namespace +prometheusRule: + enabled: false + additionalLabels: {} + +supportNFS: false +# initChownDataImage ensures all Kubecost filepath permissions on PV or local storage are set up correctly. +initChownDataImage: "busybox" # Supports a fully qualified Docker image, e.g. registry.hub.docker.com/library/busybox:latest +initChownData: + resources: {} + +## Kubecost's Bundled Grafana +## You can access it by visiting http://kubecost.me.com/grafana/ +## Ref: https://docs.kubecost.com/install-and-configure/advanced-configuration/custom-grafana +grafana: + # namespace_datasources: kubecost # override the default namespace here + # namespace_dashboards: kubecost # override the default namespace here + rbac: + create: true + + serviceAccount: + create: true + name: "" + + ## Provide a full name override for the Grafana Deployment. + # fullnameOverride: "" + ## Provide a name override for the Grafana Deployment. + # nameOverride: "" + + ## Configure grafana datasources + ## ref: http://docs.grafana.org/administration/provisioning/#datasources + ## + # datasources: + # datasources.yaml: + # apiVersion: 1 + # datasources: + # - name: prometheus-kubecost + # type: prometheus + # url: http://kubecost-prometheus-server.kubecost.svc.cluster.local + # access: proxy + # isDefault: false + # jsonData: + # httpMethod: POST + # prometheusType: Prometheus + # prometheusVersion: 2.35.0 + # timeInterval: 1m + + replicas: 1 + deploymentStrategy: RollingUpdate + readinessProbe: + httpGet: + path: /api/health + port: 3000 + livenessProbe: + httpGet: + path: /api/health + port: 3000 + initialDelaySeconds: 60 + timeoutSeconds: 30 + failureThreshold: 10 + image: + repository: grafana/grafana + tag: 11.3.1 + pullPolicy: IfNotPresent + # pullSecrets: + securityContext: {} + priorityClassName: "" + + ## Container image settings for Grafana initContainer used to download dashboards. Will only be used when dashboards are present. + downloadDashboardsImage: + repository: curlimages/curl + tag: latest + pullPolicy: IfNotPresent + podAnnotations: {} + annotations: {} + service: + type: ClusterIP + port: 80 + annotations: {} + labels: {} + resources: {} + nodeSelector: {} + tolerations: [] + affinity: {} + persistence: + enabled: false + # storageClassName: default + # accessModes: + # - ReadWriteOnce + # size: 10Gi + # annotations: {} + # subPath: "" + # existingClaim: + adminUser: admin + adminPassword: strongpassword + # schedulerName: + env: {} + envFromSecret: "" + extraSecretMounts: [] + plugins: [] + dashboardProviders: {} + dashboards: {} + dashboardsConfigMaps: {} + ## Grafana sidecars that collect the configmaps with specified label and stores the included files them into the respective folders + ## Requires at least Grafana 5 to work and can't be used together with parameters dashboardProviders, datasources and dashboards + sidecar: + image: + repository: ghcr.io/kiwigrid/k8s-sidecar + tag: 1.28.1 + pullPolicy: IfNotPresent + resources: {} + dashboards: + enabled: true + # label that the configmaps with dashboards are marked with + label: grafana_dashboard + labelValue: "1" + # set sidecar ERROR_THROTTLE_SLEEP env var from default 5s to 0s -> fixes https://github.com/kubecost/cost-analyzer-helm-chart/issues/877 + annotations: {} + error_throttle_sleep: 0 + folder: /tmp/dashboards + datasources: + # dataSourceFilename: foo.yml # If you need to change the name of the datasource file + enabled: false + error_throttle_sleep: 0 + # label that the configmaps with datasources are marked with + label: grafana_datasource + ## Grafana's primary configuration + ## NOTE: values in map will be converted to ini format + ## ref: http://docs.grafana.org/installation/configuration/ + ## + ## For grafana to be accessible, add the path to root_url. For example, if you run kubecost at www.foo.com:9090/kubecost + ## set root_url to "%(protocol)s://%(domain)s:%(http_port)s/kubecost/grafana". No change is necessary here if kubecost runs at a root URL + grafana.ini: + server: + serve_from_sub_path: false # Set to false on Grafana v10+ + root_url: "%(protocol)s://%(domain)s:%(http_port)s/grafana" + paths: + data: /var/lib/grafana/data + logs: /var/log/grafana + plugins: /var/lib/grafana/plugins + provisioning: /etc/grafana/provisioning + analytics: + check_for_updates: true + log: + mode: console + grafana_net: + url: https://grafana.net + auth.anonymous: + enabled: true + org_role: Editor + org_name: Main Org. + +serviceAccount: + create: true # Set this to false if you're bringing your own service account. + annotations: {} + +awsstore: + useAwsStore: false + imageNameAndVersion: gcr.io/kubecost1/awsstore:latest + createServiceAccount: false + priorityClassName: "" + nodeSelector: {} + annotations: {} + +## Federated ETL Architecture +## Ref: https://docs.kubecost.com/install-and-configure/install/multi-cluster/federated-etl +## +federatedETL: + + ## If true, installs the minimal set of components required for a Federated ETL cluster. + agentOnly: false + + ## If true, push ETL data to the federated storage bucket + federatedCluster: false + + ## If true, this cluster will be able to read from the federated-store but will + ## not write to it. This is useful in situations when you want to deploy a + ## primary cluster, but don't want the primary cluster's ETL data to be + ## pushed to the bucket + readOnlyPrimary: false + + ## If true, changes the dir of S3 backup to the Federated combined store. + ## Commonly used when transitioning from Thanos to Federated ETL architecture. + redirectS3Backup: false + + ## If true, will query metrics from a central PromQL DB (e.g. Amazon Managed + ## Prometheus) + useMultiClusterDB: false + +## Kubecost Admission Controller (beta feature) +## To use this feature, ensure you have run the `create-admission-controller.sh` +## script. This generates a k8s secret with TLS keys/certificats and a +## corresponding CA bundle. +## +kubecostAdmissionController: + enabled: false + secretName: webhook-server-tls + caBundle: ${CA_BUNDLE} + +# Enables or disables the Cost Event Audit pipeline, which tracks recent changes at cluster level +# and provides an estimated cost impact via the Kubecost Predict API. +# +# It is disabled by default to avoid problems in high-scale environments. +costEventsAudit: + enabled: false + +## Disable updates to kubecost from the frontend UI and via POST request +## This feature is considered beta, entrprise users should use teams: +## https://docs.kubecost.com/using-kubecost/navigating-the-kubecost-ui/teams +# readonly: false + +# # These configs can also be set from the Settings page in the Kubecost product +# # UI. Values in this block override config changes in the Settings UI on pod +# # restart +# kubecostProductConfigs: +# # An optional list of cluster definitions that can be added for frontend +# # access. The local cluster is *always* included by default, so this list is +# # for non-local clusters. +# clusters: +# - name: "Cluster A" +# address: http://cluster-a.kubecost.com:9090 +# # Optional authentication credentials - only basic auth is currently supported. +# auth: +# type: basic +# # Secret name should be a secret formatted based on: https://github.com/kubecost/poc-common-configurations/tree/main/ingress-examples +# secretName: cluster-a-auth +# # Or pass auth directly as base64 encoded user:pass +# data: YWRtaW46YWRtaW4= +# # Or user and pass directly +# user: admin +# pass: admin +# - name: "Cluster B" +# address: http://cluster-b.kubecost.com:9090 +# # Enabling customPricesEnabled and defaultModelPricing instructs Kubecost to +# # use these custom monthly resource prices when reporting node costs. Note, +# # that the below configuration is for the monthly cost of the resource. +# # Kubecost considers there to be 730 hours in a month. Also note, that these +# # configurations will have no effect on metrics emitted such as +# # `node_ram_hourly_cost` or `node_cpu_hourly_cost`. +# # Ref: https://docs.kubecost.com/install-and-configure/install/provider-installations/air-gapped +# customPricesEnabled: false +# defaultModelPricing: +# enabled: true +# CPU: "28.0" +# spotCPU: "4.86" +# RAM: "3.09" +# spotRAM: "0.65" +# GPU: "693.50" +# spotGPU: "225.0" +# storage: "0.04" +# zoneNetworkEgress: "0.01" +# regionNetworkEgress: "0.01" +# internetNetworkEgress: "0.12" +# # The cluster profile represents a predefined set of parameters to use when calculating savings. +# # Possible values are: [ development, production, high-availability ] +# clusterProfile: production +# spotLabel: lifecycle +# spotLabelValue: Ec2Spot +# gpuLabel: gpu +# gpuLabelValue: true +# alibabaServiceKeyName: "" +# alibabaServiceKeyPassword: "" +# awsServiceKeyName: "" +# awsServiceKeyPassword: "" +# awsSpotDataRegion: us-east-1 +# awsSpotDataBucket: spot-data-feed-s3-bucket +# awsSpotDataPrefix: dev +# athenaProjectID: "530337586277" # The AWS AccountID where the Athena CUR is. Generally your masterpayer account +# athenaBucketName: "s3://aws-athena-query-results-530337586277-us-east-1" +# athenaRegion: us-east-1 +# athenaDatabase: athenacurcfn_athena_test1 +# athenaTable: "athena_test1" +# athenaWorkgroup: "primary" # The default workgroup in AWS is 'primary' +# masterPayerARN: "" +# projectID: "123456789" # Also known as AccountID on AWS -- the current account/project that this instance of Kubecost is deployed on. +# gcpSecretName: gcp-secret # Name of a secret representing the gcp service key +# gcpSecretKeyName: compute-viewer-kubecost-key.json # Name of the secret's key containing the gcp service key +# bigQueryBillingDataDataset: billing_data.gcp_billing_export_v1_01AC9F_74CF1D_5565A2 +# labelMappingConfigs: # names of k8s labels or annotations used to designate different allocation concepts +# enabled: true +# owner_label: "owner" +# team_label: "team" +# department_label: "dept" +# product_label: "product" +# environment_label: "env" +# namespace_external_label: "kubernetes_namespace" # external labels/tags are used to map external cloud costs to kubernetes concepts +# cluster_external_label: "kubernetes_cluster" +# controller_external_label: "kubernetes_controller" +# product_external_label: "kubernetes_label_app" +# service_external_label: "kubernetes_service" +# deployment_external_label: "kubernetes_deployment" +# owner_external_label: "kubernetes_label_owner" +# team_external_label: "kubernetes_label_team" +# environment_external_label: "kubernetes_label_env" +# department_external_label: "kubernetes_label_department" +# statefulset_external_label: "kubernetes_statefulset" +# daemonset_external_label: "kubernetes_daemonset" +# pod_external_label: "kubernetes_pod" +# grafanaURL: "" +# # Provide a mapping from Account ID to a readable Account Name in a key/value object. Provide Account IDs as they are displayed in CloudCost +# # as the 'key' and the Account Name associated with it as the 'value' +# cloudAccountMapping: +# EXAMPLE_ACCOUNT_ID: EXAMPLE_ACCOUNT_NAME +# clusterName: "" # clusterName is the default context name in settings. +# clusterAccountID: "" # Manually set Account property for assets +# currencyCode: "USD" # official support for USD, AUD, BRL, CAD, CHF, CNY, DKK, EUR, GBP, IDR, INR, JPY, NOK, PLN, SEK +# azureBillingRegion: US # Represents 2-letter region code, e.g. West Europe = NL, Canada = CA. ref: https://en.wikipedia.org/wiki/List_of_ISO_3166_country_codes +# azureSubscriptionID: 0bd50fdf-c923-4e1e-850c-196dd3dcc5d3 +# azureClientID: f2ef6f7d-71fb-47c8-b766-8d63a19db017 +# azureTenantID: 72faf3ff-7a3f-4597-b0d9-7b0b201bb23a +# azureClientPassword: fake key # Only use if your values.yaml are stored encrypted. Otherwise provide an existing secret via serviceKeySecretName +# azureOfferDurableID: "MS-AZR-0003p" +# discount: "" # percentage discount applied to compute +# negotiatedDiscount: "" # custom negotiated cloud provider discount +# standardDiscount: "" # custom negotiated cloud provider discount, applied to all incoming asset compute costs in a federated environment. Overrides negotiatedDiscount on any cluster in the federated environment. +# defaultIdle: false +# serviceKeySecretName: "" # Use an existing AWS or Azure secret with format as in aws-service-key-secret.yaml or azure-service-key-secret.yaml. Leave blank if using createServiceKeySecret +# createServiceKeySecret: true # Creates a secret representing your cloud service key based on data in values.yaml. If you are storing unencrypted values, add a secret manually +# sharedNamespaces: "" # namespaces with shared workloads, example value: "kube-system\,ingress-nginx\,kubecost\,monitoring" +# sharedOverhead: "" # value representing a fixed external cost per month to be distributed among aggregations. +# shareTenancyCosts: true # enable or disable sharing costs such as cluster management fees (defaults to "true" on Settings page) +# metricsConfigs: # configuration for metrics emitted by Kubecost +# disabledMetrics: [] # list of metrics that Kubecost will not emit. Note that disabling metrics can lead to unexpected behavior in the cost-model. +# productKey: # Apply enterprise product license +# enabled: false +# key: "" +# secretname: productkeysecret # Reference an existing k8s secret created from a file named productkey.json of format { "key": "enterprise-key-here" }. If the secretname is specified, a configmap with the key will not be created. +# mountPath: "/some/custom/path/productkey.json" # (use instead of secretname) Declare the path at which the product key file is mounted (eg. by a secrets provisioner). The file must be of format { "key": "enterprise-key-here" }. +# # The following block enables the use of a custom SMTP server which overrides Kubecost's built-in, external SMTP server for alerts and reports +# smtp: +# config: | +# { +# "sender_email": "", +# "host": "", +# "port": 587, +# "authentication": true, +# "username": "", +# "password": "", +# "secure": true +# } +# secretname: smtpconfigsecret # Reference an existing k8s secret created from a file named smtp.json of format specified by config above. If the secretname is specified, a configmap with the key will not be created. +# mountPath: "/some/custom/path/smtp.json" # (use instead of secretname) Declare the path at which the SMTP config file is mounted (eg. by a secrets provisioner). The file must be of format specified by config above. +# carbonEstimates: false # Enables Kubecost beta carbon estimation endpoints /assets/carbon and /allocations/carbon +# The below options to hide UI elements are only supported in Enterprise +# hideDiagnostics: false # useful if the primary is not monitored. Supported in limited environments. +# hideOrphanedResources: false # OrphanedResources works on the primary-cluster's cloud-provider only. +# hideKubecostActions: false +# hideReservedInstances: false +# hideSpotCommander: false +# hideUnclaimedVolumes: false +# hideCloudIntegrationsUI: false +# hideBellIcon: false +# hideTeams: false +# savingsRecommendationsAllowLists: # Define select list of instance types to be evaluated in computing Savings Recommendations +# AWS: [] +# GCP: [] +# Azure: [] + + ## Specify an existing Kubernetes Secret holding the cloud integration information. This Secret must contain + ## a key with name `cloud-integration.json` and the contents must be in a specific format. It is expected + ## to exist in the release Namespace. This is mutually exclusive with cloudIntegrationJSON where only one must be defined. + # cloudIntegrationSecret: "cloud-integration" + + ## Specify the cloud integration information in JSON form if pointing to an existing Secret is not desired or you'd rather + ## define the cloud integration information directly in the values file. This will result in a new Secret being created + ## named `cloud-integration` in the release Namespace. It is mutually exclusive with the cloudIntegrationSecret where only one must be defined. + # cloudIntegrationJSON: |- + # { + # "aws": [ + # { + # "athenaBucketName": "s3://AWS_cloud_integration_athenaBucketName", + # "athenaRegion": "AWS_cloud_integration_athenaRegion", + # "athenaDatabase": "AWS_cloud_integration_athenaDatabase", + # "athenaTable": "AWS_cloud_integration_athenaBucketName", + # "projectID": "AWS_cloud_integration_athena_projectID", + # "serviceKeyName": "AWS_cloud_integration_athena_serviceKeyName", + # "serviceKeySecret": "AWS_cloud_integration_athena_serviceKeySecret" + # } + # ], + # "azure": [ + # { + # "azureSubscriptionID": "my-subscription-id", + # "azureStorageAccount": "my-storage-account", + # "azureStorageAccessKey": "my-storage-access-key", + # "azureStorageContainer": "my-storage-container" + # } + # ], + # "gcp": [ + # { + # "projectID": "my-project-id", + # "billingDataDataset": "detailedbilling.my-billing-dataset", + # "key": { + # "type": "service_account", + # "project_id": "my-project-id", + # "private_key_id": "my-private-key-id", + # "private_key": "my-pem-encoded-private-key", + # "client_email": "my-service-account-name@my-project-id.iam.gserviceaccount.com", + # "client_id": "my-client-id", + # "auth_uri": "auth-uri", + # "token_uri": "token-uri", + # "auth_provider_x509_cert_url": "my-x509-provider-cert", + # "client_x509_cert_url": "my-x509-cert-url" + # } + # } + # ] + # } + + # ingestPodUID: false # Enables using UIDs to uniquely ID pods. This requires either Kubecost's replicated KSM metrics, or KSM v2.1.0+. This may impact performance, and changes the default cost-model allocation behavior. + # regionOverrides: "region1,region2,region3" # list of regions which will override default costmodel provider regions + +# Explicit names of various ConfigMaps to use. If not set, a default will apply. +# pricingConfigmapName: "" +# productConfigmapName: "" +# smtpConfigmapName: "" + +# -- Array of extra K8s manifests to deploy +## Note: Supports use of custom Helm templates +extraObjects: [] +# Cloud Billing Integration: +# - apiVersion: v1 +# kind: Secret +# metadata: +# name: cloud-integration +# namespace: kubecost +# type: Opaque +# data: +# cloud-integration.json: BASE64_SECRET +# Istio: +# - apiVersion: networking.istio.io/v1alpha3 +# kind: VirtualService +# metadata: +# name: my-virtualservice +# spec: +# hosts: +# - kubecost.myorg.com +# gateways: +# - my-gateway +# http: +# - route: +# - destination: +# host: kubecost.kubecost.svc.cluster.local +# port: +# number: 80 + +# -- Optional override for the image used for the basic health test container +# basicHealth: +# fullImageName: alpine/k8s:1.26.9 diff --git a/index.yaml b/index.yaml index 85c4e3fe35..9f97e1d943 100644 --- a/index.yaml +++ b/index.yaml @@ -8626,6 +8626,23 @@ entries: - assets/hashicorp/consul-1.1.2.tgz version: 1.1.2 cost-analyzer: + - annotations: + artifacthub.io/links: | + - name: Homepage + url: https://www.kubecost.com + catalog.cattle.io/certified: partner + catalog.cattle.io/display-name: Kubecost + catalog.cattle.io/release-name: cost-analyzer + apiVersion: v2 + appVersion: 2.5.3 + created: "2025-01-18T00:02:04.383176742Z" + description: Kubecost Helm chart - monitor your cloud costs! + digest: 3e986863e66a2505857b4018dddad0cb3ce85303f42f49de00401f2917a3321e + icon: file://assets/icons/cost-analyzer.png + name: cost-analyzer + urls: + - assets/kubecost/cost-analyzer-2.5.3.tgz + version: 2.5.3 - annotations: artifacthub.io/links: | - name: Homepage @@ -26640,6 +26657,54 @@ entries: - assets/loft/loft-3.2.0.tgz version: 3.2.0 microgateway: + - annotations: + artifacthub.io/category: security + artifacthub.io/license: MIT + artifacthub.io/links: | + - name: Airlock Microgateway Documentation + url: https://docs.airlock.com/microgateway/4.4/ + - name: Airlock Microgateway Labs + url: https://play.instruqt.com/airlock/invite/hyi9fy4b4jzc?icp_referrer=artifacthub.io + - name: Airlock Microgateway Forum + url: https://forum.airlock.com/ + catalog.cattle.io/certified: partner + catalog.cattle.io/display-name: Airlock Microgateway + catalog.cattle.io/kube-version: '>=1.25.0-0' + catalog.cattle.io/release-name: "" + charts.openshift.io/name: Airlock Microgateway + apiVersion: v2 + appVersion: 4.4.3 + created: "2025-01-18T00:02:00.768386813Z" + description: A Helm chart for deploying the Airlock Microgateway + digest: 6114cc00a10fb416b19e5cb8f81cf7dacf208a7210e383d43b29512133a79e6a + home: https://www.airlock.com/en/microgateway + icon: file://assets/icons/microgateway.svg + keywords: + - WAF + - Web Application Firewall + - WAAP + - Web Application and API protection + - OWASP + - Airlock + - Microgateway + - Security + - Filtering + - DevSecOps + - shift left + - control plane + - Operator + kubeVersion: '>=1.25.0-0' + maintainers: + - email: support@airlock.com + name: Airlock + url: https://www.airlock.com/ + name: microgateway + sources: + - https://github.com/airlock/microgateway + type: application + urls: + - assets/airlock/microgateway-4.4.3.tgz + version: 4.4.3 - annotations: artifacthub.io/category: security artifacthub.io/license: MIT @@ -27073,6 +27138,53 @@ entries: - assets/airlock/microgateway-4.2.3.tgz version: 4.2.3 microgateway-cni: + - annotations: + artifacthub.io/category: security + artifacthub.io/license: MIT + artifacthub.io/links: | + - name: Airlock Microgateway Documentation + url: https://docs.airlock.com/microgateway/4.4/ + - name: Airlock Microgateway Labs + url: https://play.instruqt.com/airlock/invite/hyi9fy4b4jzc?icp_referrer=artifacthub.io + - name: Airlock Microgateway Forum + url: https://forum.airlock.com/ + catalog.cattle.io/certified: partner + catalog.cattle.io/display-name: Airlock Microgateway CNI + catalog.cattle.io/kube-version: '>=1.25.0-0' + catalog.cattle.io/release-name: "" + charts.openshift.io/name: Airlock Microgateway CNI + apiVersion: v2 + appVersion: 4.4.3 + created: "2025-01-18T00:02:00.774999612Z" + description: A Helm chart for deploying the Airlock Microgateway CNI plugin + digest: 53b8ec35836a9a2546cc085392f65b75ce821ea451f5241f612fca407ceb8103 + home: https://www.airlock.com/en/microgateway + icon: file://assets/icons/microgateway-cni.svg + keywords: + - WAF + - Web Application Firewall + - WAAP + - Web Application and API protection + - OWASP + - Airlock + - Microgateway + - Security + - Filtering + - DevSecOps + - shift left + - CNI + kubeVersion: '>=1.25.0-0' + maintainers: + - email: support@airlock.com + name: Airlock + url: https://www.airlock.com/ + name: microgateway-cni + sources: + - https://github.com/airlock/microgateway + type: application + urls: + - assets/airlock/microgateway-cni-4.4.3.tgz + version: 4.4.3 - annotations: artifacthub.io/category: security artifacthub.io/license: MIT @@ -47637,4 +47749,4 @@ entries: urls: - assets/netfoundry/ziti-host-1.5.1.tgz version: 1.5.1 -generated: "2025-01-17T00:01:58.398652304Z" +generated: "2025-01-18T00:02:00.745382082Z"