Handler metrics

Author: AlCutter

cmd/gcp/main.go

@@ -28,7 +28,6 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 	sctfe "github.com/transparency-dev/static-ct"
 	"github.com/transparency-dev/static-ct/storage"
 	gcpSCTFE "github.com/transparency-dev/static-ct/storage/gcp"
@@ -49,7 +48,6 @@ var (
 	notAfterLimit timestampFlag
 
 	httpEndpoint               = flag.String("http_endpoint", "localhost:6962", "Endpoint for HTTP (host:port).")
-	metricsEndpoint            = flag.String("metrics_endpoint", "", "Endpoint for serving metrics; if left empty, metrics will be visible on --http_endpoint.")
 	httpDeadline               = flag.Duration("http_deadline", time.Second*10, "Deadline for HTTP requests.")
 	maskInternalErrors         = flag.Bool("mask_internal_errors", false, "Don't return error strings with Internal Server Error HTTP responses.")
 	origin                     = flag.String("origin", "", "Origin of the log, for checkpoints and the monitoring prefix.")
@@ -99,25 +97,6 @@ func main() {
 	klog.Info("**** CT HTTP Server Starting ****")
 	http.Handle("/", logHandler)
 
-	metricsAt := *metricsEndpoint
-	if metricsAt == "" {
-		metricsAt = *httpEndpoint
-	}
-
-	if metricsAt != *httpEndpoint {
-		// Run a separate handler for metrics.
-		go func() {
-			mux := http.NewServeMux()
-			mux.Handle("/metrics", promhttp.Handler())
-			metricsServer := http.Server{Addr: metricsAt, Handler: mux}
-			err := metricsServer.ListenAndServe()
-			klog.Warningf("Metrics server exited: %v", err)
-		}()
-	} else {
-		// Handle metrics on the DefaultServeMux.
-		http.Handle("/metrics", promhttp.Handler())
-	}
-
 	// Bring up the HTTP server and serve until we get a signal not to.
 	srv := http.Server{Addr: *httpEndpoint}
 	shutdownWG := new(sync.WaitGroup)

ctlog.go

@@ -138,7 +138,7 @@ func NewLogHandler(ctx context.Context, origin string, signer crypto.Signer, cfg
 		TimeSource:         sysTimeSource,
 	}
 
-	handlers := scti.NewPathHandlers(opts, log)
+	handlers := scti.NewPathHandlers(ctx, opts, log)
 	mux := http.NewServeMux()
 	// Register handlers for all the configured logs.
 	for path, handler := range handlers {

internal/otel/cast.go

@@ -0,0 +1,33 @@
+// Copyright 2025 The Tessera authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package otel
+
+import "math"
+
+var (
+	// LatencyHistogramBuckets is a range of millisecond scale bucket boundaries which remain useful at around 1-2 seconds timescale in addition to smaller latencies.
+	LatencyHistogramBuckets = []float64{0, 10, 50, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000, 1200, 1400, 1600, 1800, 2000, 2500, 3000, 4000, 5000, 6000, 8000, 10000}
+)
+
+// Clamp64 casts a uint64 to an int64, clamping it at MaxInt64 if the value is above.
+//
+// Intended only for converting Tessera uint64 internal values to int64 for use with
+// open telemetry metrics.
+func Clamp64(u uint64) int64 {
+	if u > math.MaxInt64 {
+		return math.MaxInt64
+	}
+	return int64(u)
+}

internal/scti/handlers.go

@@ -24,19 +24,18 @@ import (
 	"fmt"
 	"io"
 	"net/http"
-	"strconv"
 	"strings"
 	"sync"
 	"time"
 
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
+	"github.com/transparency-dev/static-ct/internal/otel"
 	"github.com/transparency-dev/static-ct/internal/types/rfc6962"
 	"github.com/transparency-dev/static-ct/internal/types/tls"
 	"github.com/transparency-dev/static-ct/internal/x509util"
 	"github.com/transparency-dev/static-ct/modules/dedup"
 	tessera "github.com/transparency-dev/trillian-tessera"
 	"github.com/transparency-dev/trillian-tessera/ctonly"
+	"go.opentelemetry.io/otel/metric"
 	"k8s.io/klog/v2"
 )
 
@@ -50,7 +49,7 @@ const (
 )
 
 // entrypointName identifies a CT entrypoint as defined in section 4 of RFC 6962.
-type entrypointName string
+type entrypointName = string
 
 // Constants for entrypoint names, as exposed in statistics/logging.
 const (
@@ -63,53 +62,38 @@ var (
 	// Metrics are all per-log (label "origin"), but may also be
 	// per-entrypoint (label "ep") or per-return-code (label "rc").
 	once             sync.Once
-	knownLogs        *prometheus.GaugeVec     // origin => value (always 1.0)
-	lastSCTIndex     *prometheus.GaugeVec     // origin => value
-	lastSCTTimestamp *prometheus.GaugeVec     // origin => value
-	reqsCounter      *prometheus.CounterVec   // origin, op => value
-	rspsCounter      *prometheus.CounterVec   // origin, op, code => value
-	rspLatency       *prometheus.HistogramVec // origin, op, code => value
+	knownLogs        metric.Int64Gauge       // origin => value (always 1.0)
+	lastSCTIndex     metric.Int64Gauge       // origin => value
+	lastSCTTimestamp metric.Int64Gauge       // origin => value
+	reqsCounter      metric.Int64Counter     // origin, op => value
+	rspsCounter      metric.Int64Counter     // origin, op, code => value
+	rspLatency       metric.Float64Histogram // origin, op, code => value
 )
 
 // setupMetrics initializes all the exported metrics.
 func setupMetrics() {
 	// TODO(phboneff): add metrics for deduplication and chain storage.
-	knownLogs = promauto.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Name: "known_logs",
-			Help: "Set to 1 for known logs",
-		},
-		[]string{"origin"})
-	lastSCTTimestamp = promauto.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Name: "last_sct_timestamp",
-			Help: "Time of last SCT in ms since epoch",
-		},
-		[]string{"origin"})
-	lastSCTIndex = promauto.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Name: "last_sct_index",
-			Help: "Index of last SCT",
-		},
-		[]string{"origin"})
-	reqsCounter = promauto.NewCounterVec(
-		prometheus.CounterOpts{
-			Name: "http_reqs",
-			Help: "Number of requests",
-		},
-		[]string{"origin", "ep"})
-	rspsCounter = promauto.NewCounterVec(
-		prometheus.CounterOpts{
-			Name: "http_rsps",
-			Help: "Number of responses",
-		},
-		[]string{"origin", "op", "code"})
-	rspLatency = promauto.NewHistogramVec(
-		prometheus.HistogramOpts{
-			Name: "http_latency",
-			Help: "Latency of responses in seconds",
-		},
-		[]string{"origin", "op", "code"})
+	knownLogs = mustCreate(meter.Int64Gauge("tesseract.known_logs",
+		metric.WithDescription("Set to 1 for known logs")))
+
+	lastSCTTimestamp = mustCreate(meter.Int64Gauge("tesseract.last_sct.timestamp",
+		metric.WithDescription("Time of last SCT since epoch"),
+		metric.WithUnit("ms")))
+
+	lastSCTIndex = mustCreate(meter.Int64Gauge("tesseract.last_sct.index",
+		metric.WithDescription("Index of last SCT"),
+		metric.WithUnit("{entry}")))
+
+	reqsCounter = mustCreate(meter.Int64Counter("tesseract.http_request.count",
+		metric.WithDescription("CT HTTP requests")))
+
+	rspsCounter = mustCreate(meter.Int64Counter("tesseract.http_response.count",
+		metric.WithDescription("CT HTTP responses")))
+
+	rspLatency = mustCreate(meter.Float64Histogram("tesseract.http_response.duration",
+		metric.WithDescription("CT HTTP response duration"),
+		metric.WithExplicitBucketBoundaries(otel.LatencyHistogramBuckets...),
+		metric.WithUnit("ms")))
 }
 
 // entrypoints is a list of entrypoint names as exposed in statistics/logging.
@@ -132,16 +116,18 @@ type appHandler struct {
 // does additional common error and stats processing.
 func (a appHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	var statusCode int
-	label0 := a.log.origin
-	label1 := string(a.name)
-	reqsCounter.WithLabelValues(label0, label1).Inc()
+
+	originAttr := originKey.String(a.log.origin)
+	operationAttr := operationKey.String(a.name)
+	reqsCounter.Add(r.Context(), 1, metric.WithAttributes(originAttr, operationAttr))
 	startTime := a.opts.TimeSource.Now()
 	logCtx := a.opts.RequestLog.start(r.Context())
 	a.opts.RequestLog.origin(logCtx, a.log.origin)
 	defer func() {
 		latency := a.opts.TimeSource.Now().Sub(startTime).Seconds()
-		rspLatency.WithLabelValues(label0, label1, strconv.Itoa(statusCode)).Observe(latency)
+		rspLatency.Record(r.Context(), latency, metric.WithAttributes(originAttr, operationAttr, codeKey.Int(statusCode)))
 	}()
+
 	klog.V(2).Infof("%s: request %v %q => %s", a.log.origin, r.Method, r.URL, a.name)
 	// TODO(phboneff): add a.Method directly on the handler path and remove this test.
 	if r.Method != a.method {
@@ -169,7 +155,7 @@ func (a appHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	statusCode, err = a.handler(ctx, a.opts, a.log, w, r)
 	a.opts.RequestLog.status(ctx, statusCode)
 	klog.V(2).Infof("%s: %s <= st=%d", a.log.origin, a.name, statusCode)
-	rspsCounter.WithLabelValues(label0, label1, strconv.Itoa(statusCode)).Inc()
+	rspsCounter.Add(r.Context(), 1, metric.WithAttributes(originAttr, operationAttr, codeKey.Int(statusCode)))
 	if err != nil {
 		klog.Warningf("%s: %s handler error: %v", a.log.origin, a.name, err)
 		a.opts.sendHTTPError(w, statusCode, err)
@@ -197,9 +183,9 @@ type HandlerOptions struct {
 	TimeSource TimeSource
 }
 
-func NewPathHandlers(opts *HandlerOptions, log *log) pathHandlers {
+func NewPathHandlers(ctx context.Context, opts *HandlerOptions, log *log) pathHandlers {
 	once.Do(func() { setupMetrics() })
-	knownLogs.WithLabelValues(log.origin).Set(1.0)
+	knownLogs.Record(ctx, 1, metric.WithAttributes(originKey.String(log.origin)))
 
 	prefix := strings.TrimRight(log.origin, "/")
 	if !strings.HasPrefix(prefix, "/") {
@@ -351,8 +337,8 @@ func addChainInternal(ctx context.Context, opts *HandlerOptions, log *log, w htt
 	}
 	klog.V(3).Infof("%s: %s <= SCT", log.origin, method)
 	if sct.Timestamp == timeMillis {
-		lastSCTTimestamp.WithLabelValues(log.origin).Set(float64(sct.Timestamp))
-		lastSCTIndex.WithLabelValues(log.origin).Set(float64(idx))
+		lastSCTTimestamp.Record(ctx, otel.Clamp64(sct.Timestamp), metric.WithAttributes(originKey.String(log.origin)))
+		lastSCTIndex.Record(ctx, otel.Clamp64(idx), metric.WithAttributes(originKey.String(log.origin)))
 	}
 
 	return http.StatusOK, nil

internal/scti/handlers_test.go

@@ -123,7 +123,7 @@ func setupTestLog(t *testing.T) (*log, string) {
 func setupTestServer(t *testing.T, log *log, path string) *httptest.Server {
 	t.Helper()
 
-	handlers := NewPathHandlers(&hOpts, log)
+	handlers := NewPathHandlers(t.Context(), &hOpts, log)
 	handler, ok := handlers[path]
 	if !ok {
 		t.Fatalf("Handler not found: %s", path)
@@ -208,7 +208,7 @@ func postHandlers(t *testing.T, handlers pathHandlers) pathHandlers {
 
 func TestPostHandlersRejectGet(t *testing.T) {
 	log, _ := setupTestLog(t)
-	handlers := NewPathHandlers(&hOpts, log)
+	handlers := NewPathHandlers(t.Context(), &hOpts, log)
 
 	// Anything in the post handler list should reject GET
 	for path, handler := range postHandlers(t, handlers) {
@@ -229,7 +229,7 @@ func TestPostHandlersRejectGet(t *testing.T) {
 
 func TestGetHandlersRejectPost(t *testing.T) {
 	log, _ := setupTestLog(t)
-	handlers := NewPathHandlers(&hOpts, log)
+	handlers := NewPathHandlers(t.Context(), &hOpts, log)
 
 	// Anything in the get handler list should reject POST.
 	for path, handler := range getHandlers(t, handlers) {
@@ -262,7 +262,7 @@ func TestPostHandlersFailure(t *testing.T) {
 	}
 
 	log, _ := setupTestLog(t)
-	handlers := NewPathHandlers(&hOpts, log)
+	handlers := NewPathHandlers(t.Context(), &hOpts, log)
 
 	for path, handler := range postHandlers(t, handlers) {
 		t.Run(path, func(t *testing.T) {
@@ -285,7 +285,7 @@ func TestPostHandlersFailure(t *testing.T) {
 func TestNewPathHandlers(t *testing.T) {
 	log, _ := setupTestLog(t)
 	t.Run("Handlers", func(t *testing.T) {
-		handlers := NewPathHandlers(&HandlerOptions{}, log)
+		handlers := NewPathHandlers(t.Context(), &HandlerOptions{}, log)
 		// Check each entrypoint has a handler
 		if got, want := len(handlers), len(entrypoints); got != want {
 			t.Fatalf("len(info.handler)=%d; want %d", got, want)

internal/scti/otel.go

@@ -0,0 +1,41 @@
+// Copyright 2025 The Tessera authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package scti
+
+import (
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"k8s.io/klog/v2"
+)
+
+const name = "github.com/transparency-dev/static-ct/internal/scti"
+
+var (
+	meter  = otel.Meter(name)
+	tracer = otel.Tracer(name)
+)
+
+var (
+	codeKey      = attribute.Key("tesseract.code")
+	operationKey = attribute.Key("tesseract.operation")
+	originKey    = attribute.Key("tesseract.origin")
+)
+
+func mustCreate[T any](t T, err error) T {
+	if err != nil {
+		klog.Exit(err.Error())
+	}
+	return t
+}