From c625260575c0d968309c2e0c11de7e39797fad05 Mon Sep 17 00:00:00 2001 From: Richard Gibson Date: Sun, 3 Nov 2024 17:59:52 -0500 Subject: [PATCH] fixup! test(a3p-integration): Try to get more insight into the failing test --- .../proposals/z:acceptance/genesis-test.sh | 2 +- .../proposals/z:acceptance/psm.test.js | 2 +- .../test-lib/enable-prometheus.sh | 110 +++++++++++++----- a3p-integration/proposals/z:acceptance/use.sh | 5 +- 4 files changed, 87 insertions(+), 32 deletions(-) mode change 100644 => 100755 a3p-integration/proposals/z:acceptance/test-lib/enable-prometheus.sh diff --git a/a3p-integration/proposals/z:acceptance/genesis-test.sh b/a3p-integration/proposals/z:acceptance/genesis-test.sh index dd9aa62d7a85..458e195518c2 100755 --- a/a3p-integration/proposals/z:acceptance/genesis-test.sh +++ b/a3p-integration/proposals/z:acceptance/genesis-test.sh @@ -13,7 +13,7 @@ export_genesis() { fi agd export --export-dir "$GENESIS_EXPORT_DIR" $GENESIS_HEIGHT_ARG "$@" - CONFIG_FILE="$GENESIS_EXPORT_DIR/config.toml" + CONFIG_FILE="$GENESIS_EXPORT_DIR/app.toml" echo "Enabling Prometheus in $CONFIG_FILE ..." BACKUP_FILE="$(./test-lib/enable-prometheus.sh -b "$CONFIG_FILE")" diff -u "$BACKUP_FILE" "$CONFIG_FILE" || true diff --git a/a3p-integration/proposals/z:acceptance/psm.test.js b/a3p-integration/proposals/z:acceptance/psm.test.js index cfdc2ab287d1..743b415aeee8 100644 --- a/a3p-integration/proposals/z:acceptance/psm.test.js +++ b/a3p-integration/proposals/z:acceptance/psm.test.js @@ -139,7 +139,7 @@ test.serial('swap into IST', async t => { const rejectionPatt = /admission_refused|inbound_not_allowed/; const getTxRejectionMetrics = async () => { - const resp = await fetch('http://localhost:26660/metrics'); + const resp = await fetch('http://localhost:1317/metrics?format=prometheus'); const metrics = await (resp.ok ? resp.text() : Promise.reject(resp)); return metrics.split('\n').filter(line => line.match(rejectionPatt)); }; diff --git a/a3p-integration/proposals/z:acceptance/test-lib/enable-prometheus.sh b/a3p-integration/proposals/z:acceptance/test-lib/enable-prometheus.sh old mode 100644 new mode 100755 index b3cb433ea6f1..28332e80e311 --- a/a3p-integration/proposals/z:acceptance/test-lib/enable-prometheus.sh +++ b/a3p-integration/proposals/z:acceptance/test-lib/enable-prometheus.sh @@ -1,8 +1,9 @@ #!/bin/bash -# Usage: $0 [-b|--backup] /path/to/config.toml -# Perform in-place mutation of a config.toml file to enable Prometheus metrics, +# Usage: $0 [-b|--backup] /path/to/app.toml +# Perform in-place mutation of a app.toml file to enable Prometheus metrics, # optionally emitting the path to a backup of the original. -# https://github.com/cometbft/cometbft/blob/main/docs/explanation/core/metrics.md +# https://github.com/agoric-labs/cosmos-sdk/blob/Agoric/docs/core/telemetry.md +# https://github.com/cosmos/cosmos-sdk/blob/main/docs/learn/advanced/09-telemetry.md set -ueo pipefail @@ -17,41 +18,96 @@ tmp="$(mktemp "$(basename "$src").XXXXXXXXXX.bak")" cp "$src" "$tmp" # redirection preserves file permissions cat "$tmp" | awk > "$src" ' - BEGIN { FS = "[[:space:]]*=[[:space:]]*"; } - !done { + BEGIN { + FS = "[[:space:]]*[=#][[:space:]]*"; # split on `=`/`#` with optional whitespace + enableApi = "enable = true"; + enableTelemetry = "enabled = true"; + retainTelemetry = "prometheus-retention-time = 3600 # 60 minutes"; + } + enableApi || enableTelemetry || retainTelemetry { if (match($0, /^[[][^]]*[]]/)) { + # New section; append to the previous one if warranted. section = substr($0, RSTART + 1, RLENGTH - 2); - if (current_section == "instrumentation") { - done = 1; - print "prometheus = true"; - if (blanks != "") print substr(blanks, 2, length(blanks) - 1); + if (current_section == "api") { + finishApi(); + } else if (current_section == "telemetry") { + finishTelemetry(); } current_section = section; - } else if (current_section == "instrumentation") { - if ($1 == "prometheus" && NF > 1) { - done = 1; - if (!match($2, /^true([[:space:]]|$)/)) { - if (blanks != "") print substr(blanks, 2, length(blanks) - 1); - print "prometheus = true"; + } else if (current_section == "api") { + if ($1 == "enable") { + tmp = enableApi; + enableApi = ""; + if ($2 != "true") { + flushBuffer(tmp); + next; + } + } else if (maybeBuffer($0)) { + next; + } + } else if (current_section == "telemetry") { + if ($1 == "enabled") { + tmp = enableTelemetry; + enableTelemetry = ""; + if ($2 != "true") { + flushBuffer(tmp); next; } - } else if (match($0, /^[[:space:]]*(#.*)?$/)) { - blanks = blanks "\n" $0; + } else if ($1 == "prometheus-retention-time") { + tmp = retainTelemetry; + retainTelemetry = ""; + # Preserve a simple decimal integer >= 3600, but no other variants. + # https://toml.io/en/v1.0.0#integer + if (!(match($2, /^[1-9][0-9]*$/) && $2 >= 3600)) { + flushBuffer(tmp); + next; + } + } else if (maybeBuffer($0)) { next; } } } - 1 + { + flushBuffer(); + print; + } END { - if (done) { - # noop - } else if (current_section == "instrumentation") { - print "prometheus = true"; - } else { - print ""; - print "[instrumentation]"; - print "prometheus = true"; - } + if (current_section == "api") { + finishApi(); + } else if (current_section == "telemetry") { + finishTelemetry(); + } + + if (enableApi) finishApi("[api]"); + if (enableTelemetry || retainTelemetry) finishTelemetry("[telemetry]"); + } + + function finishApi(heading) { + if (heading) print "\n" heading; + if (enableApi) print enableApi; + enableApi = ""; + flushBuffer(); + } + + function finishTelemetry(heading) { + if (heading) print "\n" heading; + if (enableTelemetry) print enableTelemetry; + if (retainTelemetry) print retainTelemetry; + enableTelemetry = retainTelemetry = ""; + flushBuffer(); + } + + function maybeBuffer(line) { + if (match(line, /^[[:space:]]*(#.*)?$/)) { + buf = buf "\n" $0; + return 1; + } + } + + function flushBuffer(extra) { + if (buf != "") print substr(buf, 2, length(buf) - 1); + buf = ""; + if (extra) print extra; } ' diff --git a/a3p-integration/proposals/z:acceptance/use.sh b/a3p-integration/proposals/z:acceptance/use.sh index fccc474e9342..6064a253ae35 100644 --- a/a3p-integration/proposals/z:acceptance/use.sh +++ b/a3p-integration/proposals/z:acceptance/use.sh @@ -9,8 +9,7 @@ source /usr/src/upgrade-test-scripts/env_setup.sh # later steps, such as the "test" step, or further proposal layers. # Enable Prometheus metrics. -# https://github.com/cometbft/cometbft/blob/main/docs/explanation/core/metrics.md -CONFIG_FILE="$HOME/.agoric/config/config.toml" +CONFIG_FILE="$HOME/.agoric/config/app.toml" echo "Enabling Prometheus in $CONFIG_FILE ..." echo "# Before" && cat "$CONFIG_FILE" BACKUP_FILE="$(./test-lib/enable-prometheus.sh -b "$CONFIG_FILE")" @@ -22,5 +21,5 @@ startAgd waitForBlock 3 set -v # https://prometheus.io/docs/instrumenting/exposition_formats/#text-based-format -curl -sSL http://localhost:26660/metrics \ +curl -sSL 'http://localhost:1317/metrics?format=prometheus' \ | grep -E '^[[:space:]]*#[[:space:]]*(HELP|TYPE)\b' || true