From 216d9da8fcdafacd8f2fc9ab239147ec32cfba98 Mon Sep 17 00:00:00 2001 From: Al Cutter Date: Tue, 8 Apr 2025 16:47:26 +0100 Subject: [PATCH 1/7] Bump Tessera to HEAD --- go.mod | 8 ++++---- go.sum | 20 ++++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index 03491740..93daff0e 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.24.0 require ( cloud.google.com/go/secretmanager v1.14.6 - cloud.google.com/go/spanner v1.78.0 + cloud.google.com/go/spanner v1.79.0 cloud.google.com/go/storage v1.51.0 github.com/RobinUS2/golang-moving-average v1.0.0 github.com/aws/aws-sdk-go-v2 v1.36.3 @@ -13,18 +13,18 @@ require ( github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.35.3 github.com/aws/smithy-go v1.22.3 github.com/gdamore/tcell/v2 v2.8.1 - github.com/go-sql-driver/mysql v1.9.1 + github.com/go-sql-driver/mysql v1.9.2 github.com/google/go-cmp v0.7.0 github.com/kylelemons/godebug v1.1.0 github.com/prometheus/client_golang v1.21.1 github.com/rivo/tview v0.0.0-20240625185742-b0a7293b8130 github.com/transparency-dev/formats v0.0.0-20250127084410-134797944be6 github.com/transparency-dev/merkle v0.0.2 - github.com/transparency-dev/trillian-tessera v0.1.2-0.20250320160837-ae724376e1ac + github.com/transparency-dev/trillian-tessera v0.1.2-0.20250408153912-a650aa01f2a4 go.etcd.io/bbolt v1.4.0 golang.org/x/crypto v0.37.0 golang.org/x/mod v0.24.0 - golang.org/x/net v0.38.0 + golang.org/x/net v0.39.0 google.golang.org/api v0.228.0 google.golang.org/grpc v1.71.1 k8s.io/klog/v2 v2.130.1 diff --git a/go.sum b/go.sum index ef84e3bf..b56766fa 100644 --- a/go.sum +++ b/go.sum @@ -530,8 +530,8 @@ cloud.google.com/go/shell v1.6.0/go.mod h1:oHO8QACS90luWgxP3N9iZVuEiSF84zNyLytb+ cloud.google.com/go/spanner v1.41.0/go.mod h1:MLYDBJR/dY4Wt7ZaMIQ7rXOTLjYrmxLE/5ve9vFfWos= cloud.google.com/go/spanner v1.44.0/go.mod h1:G8XIgYdOK+Fbcpbs7p2fiprDw4CaZX63whnSMLVBxjk= cloud.google.com/go/spanner v1.45.0/go.mod h1:FIws5LowYz8YAE1J8fOS7DJup8ff7xJeetWEo5REA2M= -cloud.google.com/go/spanner v1.78.0 h1:lO0W6rnGRH1ILpFgoVr0tO+ffnE0Xbw+vNtU6VCDXQo= -cloud.google.com/go/spanner v1.78.0/go.mod h1:224ub0ngSaiy7SJI7QZ1pu9zoVPt6CgfwDGBNhUUuzU= +cloud.google.com/go/spanner v1.79.0 h1:HSg+P01K6I1ZFxvLGYToLdZkp+noM7P777Hd/ZHFvdk= +cloud.google.com/go/spanner v1.79.0/go.mod h1:224ub0ngSaiy7SJI7QZ1pu9zoVPt6CgfwDGBNhUUuzU= cloud.google.com/go/speech v1.6.0/go.mod h1:79tcr4FHCimOp56lwC01xnt/WPJZc4v3gzyT7FoBkCM= cloud.google.com/go/speech v1.7.0/go.mod h1:KptqL+BAQIhMsj1kOP2la5DSEEerPDuOP/2mmkhHhZQ= cloud.google.com/go/speech v1.8.0/go.mod h1:9bYIl1/tjsAnMgKGHKmBZzXKEkGgtU+MpdDPTE9f7y0= @@ -781,8 +781,8 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-pdf/fpdf v0.5.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= github.com/go-pdf/fpdf v0.6.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= -github.com/go-sql-driver/mysql v1.9.1 h1:FrjNGn/BsJQjVRuSa8CBrM5BWA9BWoXXat3KrtSb/iI= -github.com/go-sql-driver/mysql v1.9.1/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU= +github.com/go-sql-driver/mysql v1.9.2 h1:4cNKDYQ1I84SXslGddlsrMhc8k4LeDVj6Ad6WRjiHuU= +github.com/go-sql-driver/mysql v1.9.2/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU= github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -1016,8 +1016,8 @@ github.com/transparency-dev/formats v0.0.0-20250127084410-134797944be6 h1:TVUG0R github.com/transparency-dev/formats v0.0.0-20250127084410-134797944be6/go.mod h1:tSjZBSQ1ZMxgaOMppnyw48SbTDL947PD/8KYbvrx+lE= github.com/transparency-dev/merkle v0.0.2 h1:Q9nBoQcZcgPamMkGn7ghV8XiTZ/kRxn1yCG81+twTK4= github.com/transparency-dev/merkle v0.0.2/go.mod h1:pqSy+OXefQ1EDUVmAJ8MUhHB9TXGuzVAT58PqBoHz1A= -github.com/transparency-dev/trillian-tessera v0.1.2-0.20250320160837-ae724376e1ac h1:LwnfCnox2ZHW3TgMCdL60C4xp8oR3lDew13MZeHDhPA= -github.com/transparency-dev/trillian-tessera v0.1.2-0.20250320160837-ae724376e1ac/go.mod h1:fARQE1UN2Z0wv/02J0uAsTIli/+flEJGCMWlBs9Ps9E= +github.com/transparency-dev/trillian-tessera v0.1.2-0.20250408153912-a650aa01f2a4 h1:11tQLp5M54Dpzwr6rwGDMtb4226KYxCVHP2PLYFxjmE= +github.com/transparency-dev/trillian-tessera v0.1.2-0.20250408153912-a650aa01f2a4/go.mod h1:Eyunn8E4gvIY4iun15FXzH9A2ovhfn9zc8zRtkxfbfI= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -1048,8 +1048,8 @@ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0 h1:CV7UdSG go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0/go.mod h1:FRmFuRJfag1IZ2dPkHnEoSFVgTVPUd2qf5Vi69hLb8I= go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= -go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.29.0 h1:WDdP9acbMYjbKIyJUhTvtzj601sVJOqgWdUxSdR/Ysc= -go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.29.0/go.mod h1:BLbf7zbNIONBLPwvFnwNHGj4zge8uTCM/UPIVW1Mq2I= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.35.0 h1:PB3Zrjs1sG1GBX51SXyTSoOTqcDglmsk7nT6tkKPb/k= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.35.0/go.mod h1:U2R3XyVPzn0WX7wOIypPuptulsMcPDPs/oiSVOMVnHY= go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY= @@ -1202,8 +1202,8 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= -golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= +golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= From e0f8fb784d25ce6d213ab69df2537f3410ce35a2 Mon Sep 17 00:00:00 2001 From: Al Cutter Date: Tue, 8 Apr 2025 16:50:10 +0100 Subject: [PATCH 2/7] Add OTel support to GCP binary --- cmd/gcp/main.go | 4 +++ cmd/gcp/otel.go | 75 +++++++++++++++++++++++++++++++++++++++++++++++++ go.mod | 10 ++++--- go.sum | 4 +++ 4 files changed, 89 insertions(+), 4 deletions(-) create mode 100644 cmd/gcp/otel.go diff --git a/cmd/gcp/main.go b/cmd/gcp/main.go index f88191f4..e83a4794 100644 --- a/cmd/gcp/main.go +++ b/cmd/gcp/main.go @@ -63,6 +63,7 @@ var ( rejectExtensions = flag.String("reject_extension", "", "A list of X.509 extension OIDs, in dotted string form (e.g. '2.3.4.5') which, if present, should cause submissions to be rejected.") signerPublicKeySecretName = flag.String("signer_public_key_secret_name", "", "Public key secret name for checkpoints and SCTs signer. Format: projects/{projectId}/secrets/{secretName}/versions/{secretVersion}.") signerPrivateKeySecretName = flag.String("signer_private_key_secret_name", "", "Private key secret name for checkpoints and SCTs signer. Format: projects/{projectId}/secrets/{secretName}/versions/{secretVersion}.") + traceFraction = flag.Float64("trace_fraction", 0, "Fraction of open-telemetry span traces to sample") ) // nolint:staticcheck @@ -71,6 +72,9 @@ func main() { flag.Parse() ctx := context.Background() + shutdownOTel := initOTel(ctx, *traceFraction) + defer shutdownOTel(ctx) + signer, err := NewSecretManagerSigner(ctx, *signerPublicKeySecretName, *signerPrivateKeySecretName) if err != nil { klog.Exitf("Can't create secret manager signer: %v", err) diff --git a/cmd/gcp/otel.go b/cmd/gcp/otel.go new file mode 100644 index 00000000..bdfa0447 --- /dev/null +++ b/cmd/gcp/otel.go @@ -0,0 +1,75 @@ +// Copyright 2025 The Tessera authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "errors" + + "go.opentelemetry.io/otel" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + + mexporter "github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric" + texporter "github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace" + "k8s.io/klog/v2" +) + +// initOTel initialises the open telemetry support for metrics and tracing. +// +// Tracing is enabled with statistical sampling, with the probability passed in. +// Returns a shutdown function which should be called just before exiting the process. +func initOTel(ctx context.Context, traceFraction float64) func(context.Context) { + var shutdownFuncs []func(context.Context) error + // shutdown combines shutdown functions from multiple OpenTelemetry + // components into a single function. + shutdown := func(ctx context.Context) { + var err error + for _, fn := range shutdownFuncs { + err = errors.Join(err, fn(ctx)) + } + shutdownFuncs = nil + if err != nil { + klog.Errorf("OTel shutdown: %v", err) + } + } + + me, err := mexporter.New() + if err != nil { + klog.Exitf("Failed to create metric exporter: %v", err) + return nil + } + // initialize a MeterProvider that periodically exports to the GCP exporter. + mp := sdkmetric.NewMeterProvider( + sdkmetric.WithReader(sdkmetric.NewPeriodicReader(me)), + ) + shutdownFuncs = append(shutdownFuncs, mp.Shutdown) + otel.SetMeterProvider(mp) + + te, err := texporter.New() + if err != nil { + klog.Exitf("Failed to create trace exporter: %v", err) + return nil + } + // initialize a TracerProvier that periodically exports to the GCP exporter. + tp := sdktrace.NewTracerProvider( + sdktrace.WithSampler(sdktrace.TraceIDRatioBased(traceFraction)), + sdktrace.WithBatcher(te), + ) + shutdownFuncs = append(shutdownFuncs, mp.Shutdown) + otel.SetTracerProvider(tp) + + return shutdown +} diff --git a/go.mod b/go.mod index 93daff0e..9af5c7df 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,8 @@ require ( cloud.google.com/go/secretmanager v1.14.6 cloud.google.com/go/spanner v1.79.0 cloud.google.com/go/storage v1.51.0 + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.27.0 github.com/RobinUS2/golang-moving-average v1.0.0 github.com/aws/aws-sdk-go-v2 v1.36.3 github.com/aws/aws-sdk-go-v2/config v1.29.13 @@ -22,6 +24,9 @@ require ( github.com/transparency-dev/merkle v0.0.2 github.com/transparency-dev/trillian-tessera v0.1.2-0.20250408153912-a650aa01f2a4 go.etcd.io/bbolt v1.4.0 + go.opentelemetry.io/otel v1.35.0 + go.opentelemetry.io/otel/sdk v1.35.0 + go.opentelemetry.io/otel/sdk/metric v1.35.0 golang.org/x/crypto v0.37.0 golang.org/x/mod v0.24.0 golang.org/x/net v0.39.0 @@ -39,10 +44,10 @@ require ( cloud.google.com/go/iam v1.4.2 // indirect cloud.google.com/go/longrunning v0.6.6 // indirect cloud.google.com/go/monitoring v1.24.1 // indirect + cloud.google.com/go/trace v1.11.3 // indirect filippo.io/edwards25519 v1.1.0 // indirect github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.5.2 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 // indirect github.com/avast/retry-go/v4 v4.6.1 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 // indirect @@ -89,10 +94,7 @@ require ( go.opentelemetry.io/contrib/detectors/gcp v1.35.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0 // indirect - go.opentelemetry.io/otel v1.35.0 // indirect go.opentelemetry.io/otel/metric v1.35.0 // indirect - go.opentelemetry.io/otel/sdk v1.35.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.35.0 // indirect go.opentelemetry.io/otel/trace v1.35.0 // indirect golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 // indirect golang.org/x/oauth2 v0.28.0 // indirect diff --git a/go.sum b/go.sum index b56766fa..6f79d6ae 100644 --- a/go.sum +++ b/go.sum @@ -631,6 +631,8 @@ github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0/go.mod h1:yAZHSGnqScoU556rBOVkwLze6WP5N+U11RHuWaGVxwY= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 h1:fYE9p3esPxA/C0rQ0AHhP0drtPXDRhaWiwg1DPqO7IU= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0/go.mod h1:BnBReJLvVYx2CS/UHOgVz2BXKXD9wsQPxZug20nZhd0= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.27.0 h1:Jtr816GUk6+I2ox9L/v+VcOwN6IyGOEDTSNHfD6m9sY= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.27.0/go.mod h1:E05RN++yLx9W4fXPtX978OLo9P0+fBacauUdET1BckA= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.51.0 h1:OqVGm6Ei3x5+yZmSJG1Mh2NwHvpVmZ08CB5qJhT9Nuk= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.51.0/go.mod h1:SZiPHWGOOk3bl8tkevxkoiwPgsIl6CwrWcbwjfHZpdM= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 h1:6/0iUd0xrnX7qt+mLNRwg5c0PGv8wpE8K90ryANQwMI= @@ -1061,6 +1063,8 @@ go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= From edec4dd532b9d8caecd5ab2a9e46be77117d830d Mon Sep 17 00:00:00 2001 From: Al Cutter Date: Tue, 8 Apr 2025 17:34:16 +0100 Subject: [PATCH 3/7] Handler metrics --- cmd/gcp/main.go | 21 -------- ctlog.go | 2 +- internal/otel/cast.go | 33 ++++++++++++ internal/scti/handlers.go | 96 +++++++++++++++------------------- internal/scti/handlers_test.go | 10 ++-- internal/scti/otel.go | 41 +++++++++++++++ 6 files changed, 121 insertions(+), 82 deletions(-) create mode 100644 internal/otel/cast.go create mode 100644 internal/scti/otel.go diff --git a/cmd/gcp/main.go b/cmd/gcp/main.go index e83a4794..2478338d 100644 --- a/cmd/gcp/main.go +++ b/cmd/gcp/main.go @@ -28,7 +28,6 @@ import ( "syscall" "time" - "github.com/prometheus/client_golang/prometheus/promhttp" sctfe "github.com/transparency-dev/static-ct" "github.com/transparency-dev/static-ct/storage" gcpSCTFE "github.com/transparency-dev/static-ct/storage/gcp" @@ -49,7 +48,6 @@ var ( notAfterLimit timestampFlag httpEndpoint = flag.String("http_endpoint", "localhost:6962", "Endpoint for HTTP (host:port).") - metricsEndpoint = flag.String("metrics_endpoint", "", "Endpoint for serving metrics; if left empty, metrics will be visible on --http_endpoint.") httpDeadline = flag.Duration("http_deadline", time.Second*10, "Deadline for HTTP requests.") maskInternalErrors = flag.Bool("mask_internal_errors", false, "Don't return error strings with Internal Server Error HTTP responses.") origin = flag.String("origin", "", "Origin of the log, for checkpoints and the monitoring prefix.") @@ -99,25 +97,6 @@ func main() { klog.Info("**** CT HTTP Server Starting ****") http.Handle("/", logHandler) - metricsAt := *metricsEndpoint - if metricsAt == "" { - metricsAt = *httpEndpoint - } - - if metricsAt != *httpEndpoint { - // Run a separate handler for metrics. - go func() { - mux := http.NewServeMux() - mux.Handle("/metrics", promhttp.Handler()) - metricsServer := http.Server{Addr: metricsAt, Handler: mux} - err := metricsServer.ListenAndServe() - klog.Warningf("Metrics server exited: %v", err) - }() - } else { - // Handle metrics on the DefaultServeMux. - http.Handle("/metrics", promhttp.Handler()) - } - // Bring up the HTTP server and serve until we get a signal not to. srv := http.Server{Addr: *httpEndpoint} shutdownWG := new(sync.WaitGroup) diff --git a/ctlog.go b/ctlog.go index 7c570212..fb8af2d6 100644 --- a/ctlog.go +++ b/ctlog.go @@ -138,7 +138,7 @@ func NewLogHandler(ctx context.Context, origin string, signer crypto.Signer, cfg TimeSource: sysTimeSource, } - handlers := scti.NewPathHandlers(opts, log) + handlers := scti.NewPathHandlers(ctx, opts, log) mux := http.NewServeMux() // Register handlers for all the configured logs. for path, handler := range handlers { diff --git a/internal/otel/cast.go b/internal/otel/cast.go new file mode 100644 index 00000000..9e601e4f --- /dev/null +++ b/internal/otel/cast.go @@ -0,0 +1,33 @@ +// Copyright 2025 The Tessera authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package otel + +import "math" + +var ( + // LatencyHistogramBuckets is a range of millisecond scale bucket boundaries which remain useful at around 1-2 seconds timescale in addition to smaller latencies. + LatencyHistogramBuckets = []float64{0, 10, 50, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000, 1200, 1400, 1600, 1800, 2000, 2500, 3000, 4000, 5000, 6000, 8000, 10000} +) + +// Clamp64 casts a uint64 to an int64, clamping it at MaxInt64 if the value is above. +// +// Intended only for converting Tessera uint64 internal values to int64 for use with +// open telemetry metrics. +func Clamp64(u uint64) int64 { + if u > math.MaxInt64 { + return math.MaxInt64 + } + return int64(u) +} diff --git a/internal/scti/handlers.go b/internal/scti/handlers.go index dab80528..5378c6df 100644 --- a/internal/scti/handlers.go +++ b/internal/scti/handlers.go @@ -24,19 +24,18 @@ import ( "fmt" "io" "net/http" - "strconv" "strings" "sync" "time" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/transparency-dev/static-ct/internal/otel" "github.com/transparency-dev/static-ct/internal/types/rfc6962" "github.com/transparency-dev/static-ct/internal/types/tls" "github.com/transparency-dev/static-ct/internal/x509util" "github.com/transparency-dev/static-ct/modules/dedup" tessera "github.com/transparency-dev/trillian-tessera" "github.com/transparency-dev/trillian-tessera/ctonly" + "go.opentelemetry.io/otel/metric" "k8s.io/klog/v2" ) @@ -50,7 +49,7 @@ const ( ) // entrypointName identifies a CT entrypoint as defined in section 4 of RFC 6962. -type entrypointName string +type entrypointName = string // Constants for entrypoint names, as exposed in statistics/logging. const ( @@ -63,53 +62,38 @@ var ( // Metrics are all per-log (label "origin"), but may also be // per-entrypoint (label "ep") or per-return-code (label "rc"). once sync.Once - knownLogs *prometheus.GaugeVec // origin => value (always 1.0) - lastSCTIndex *prometheus.GaugeVec // origin => value - lastSCTTimestamp *prometheus.GaugeVec // origin => value - reqsCounter *prometheus.CounterVec // origin, op => value - rspsCounter *prometheus.CounterVec // origin, op, code => value - rspLatency *prometheus.HistogramVec // origin, op, code => value + knownLogs metric.Int64Gauge // origin => value (always 1.0) + lastSCTIndex metric.Int64Gauge // origin => value + lastSCTTimestamp metric.Int64Gauge // origin => value + reqsCounter metric.Int64Counter // origin, op => value + rspsCounter metric.Int64Counter // origin, op, code => value + rspLatency metric.Float64Histogram // origin, op, code => value ) // setupMetrics initializes all the exported metrics. func setupMetrics() { // TODO(phboneff): add metrics for deduplication and chain storage. - knownLogs = promauto.NewGaugeVec( - prometheus.GaugeOpts{ - Name: "known_logs", - Help: "Set to 1 for known logs", - }, - []string{"origin"}) - lastSCTTimestamp = promauto.NewGaugeVec( - prometheus.GaugeOpts{ - Name: "last_sct_timestamp", - Help: "Time of last SCT in ms since epoch", - }, - []string{"origin"}) - lastSCTIndex = promauto.NewGaugeVec( - prometheus.GaugeOpts{ - Name: "last_sct_index", - Help: "Index of last SCT", - }, - []string{"origin"}) - reqsCounter = promauto.NewCounterVec( - prometheus.CounterOpts{ - Name: "http_reqs", - Help: "Number of requests", - }, - []string{"origin", "ep"}) - rspsCounter = promauto.NewCounterVec( - prometheus.CounterOpts{ - Name: "http_rsps", - Help: "Number of responses", - }, - []string{"origin", "op", "code"}) - rspLatency = promauto.NewHistogramVec( - prometheus.HistogramOpts{ - Name: "http_latency", - Help: "Latency of responses in seconds", - }, - []string{"origin", "op", "code"}) + knownLogs = mustCreate(meter.Int64Gauge("tesseract.known_logs", + metric.WithDescription("Set to 1 for known logs"))) + + lastSCTTimestamp = mustCreate(meter.Int64Gauge("tesseract.last_sct.timestamp", + metric.WithDescription("Time of last SCT since epoch"), + metric.WithUnit("ms"))) + + lastSCTIndex = mustCreate(meter.Int64Gauge("tesseract.last_sct.index", + metric.WithDescription("Index of last SCT"), + metric.WithUnit("{entry}"))) + + reqsCounter = mustCreate(meter.Int64Counter("tesseract.http_request.count", + metric.WithDescription("CT HTTP requests"))) + + rspsCounter = mustCreate(meter.Int64Counter("tesseract.http_response.count", + metric.WithDescription("CT HTTP responses"))) + + rspLatency = mustCreate(meter.Float64Histogram("tesseract.http_response.duration", + metric.WithDescription("CT HTTP response duration"), + metric.WithExplicitBucketBoundaries(otel.LatencyHistogramBuckets...), + metric.WithUnit("ms"))) } // entrypoints is a list of entrypoint names as exposed in statistics/logging. @@ -132,16 +116,18 @@ type appHandler struct { // does additional common error and stats processing. func (a appHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { var statusCode int - label0 := a.log.origin - label1 := string(a.name) - reqsCounter.WithLabelValues(label0, label1).Inc() + + originAttr := originKey.String(a.log.origin) + operationAttr := operationKey.String(a.name) + reqsCounter.Add(r.Context(), 1, metric.WithAttributes(originAttr, operationAttr)) startTime := a.opts.TimeSource.Now() logCtx := a.opts.RequestLog.start(r.Context()) a.opts.RequestLog.origin(logCtx, a.log.origin) defer func() { latency := a.opts.TimeSource.Now().Sub(startTime).Seconds() - rspLatency.WithLabelValues(label0, label1, strconv.Itoa(statusCode)).Observe(latency) + rspLatency.Record(r.Context(), latency, metric.WithAttributes(originAttr, operationAttr, codeKey.Int(statusCode))) }() + klog.V(2).Infof("%s: request %v %q => %s", a.log.origin, r.Method, r.URL, a.name) // TODO(phboneff): add a.Method directly on the handler path and remove this test. if r.Method != a.method { @@ -169,7 +155,7 @@ func (a appHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { statusCode, err = a.handler(ctx, a.opts, a.log, w, r) a.opts.RequestLog.status(ctx, statusCode) klog.V(2).Infof("%s: %s <= st=%d", a.log.origin, a.name, statusCode) - rspsCounter.WithLabelValues(label0, label1, strconv.Itoa(statusCode)).Inc() + rspsCounter.Add(r.Context(), 1, metric.WithAttributes(originAttr, operationAttr, codeKey.Int(statusCode))) if err != nil { klog.Warningf("%s: %s handler error: %v", a.log.origin, a.name, err) a.opts.sendHTTPError(w, statusCode, err) @@ -197,9 +183,9 @@ type HandlerOptions struct { TimeSource TimeSource } -func NewPathHandlers(opts *HandlerOptions, log *log) pathHandlers { +func NewPathHandlers(ctx context.Context, opts *HandlerOptions, log *log) pathHandlers { once.Do(func() { setupMetrics() }) - knownLogs.WithLabelValues(log.origin).Set(1.0) + knownLogs.Record(ctx, 1, metric.WithAttributes(originKey.String(log.origin))) prefix := strings.TrimRight(log.origin, "/") if !strings.HasPrefix(prefix, "/") { @@ -351,8 +337,8 @@ func addChainInternal(ctx context.Context, opts *HandlerOptions, log *log, w htt } klog.V(3).Infof("%s: %s <= SCT", log.origin, method) if sct.Timestamp == timeMillis { - lastSCTTimestamp.WithLabelValues(log.origin).Set(float64(sct.Timestamp)) - lastSCTIndex.WithLabelValues(log.origin).Set(float64(idx)) + lastSCTTimestamp.Record(ctx, otel.Clamp64(sct.Timestamp), metric.WithAttributes(originKey.String(log.origin))) + lastSCTIndex.Record(ctx, otel.Clamp64(idx), metric.WithAttributes(originKey.String(log.origin))) } return http.StatusOK, nil diff --git a/internal/scti/handlers_test.go b/internal/scti/handlers_test.go index b064d268..0b70479c 100644 --- a/internal/scti/handlers_test.go +++ b/internal/scti/handlers_test.go @@ -124,7 +124,7 @@ func setupTestLog(t *testing.T) (*log, string) { func setupTestServer(t *testing.T, log *log, path string) *httptest.Server { t.Helper() - handlers := NewPathHandlers(&hOpts, log) + handlers := NewPathHandlers(t.Context(), &hOpts, log) handler, ok := handlers[path] if !ok { t.Fatalf("Handler not found: %s", path) @@ -209,7 +209,7 @@ func postHandlers(t *testing.T, handlers pathHandlers) pathHandlers { func TestPostHandlersRejectGet(t *testing.T) { log, _ := setupTestLog(t) - handlers := NewPathHandlers(&hOpts, log) + handlers := NewPathHandlers(t.Context(), &hOpts, log) // Anything in the post handler list should reject GET for path, handler := range postHandlers(t, handlers) { @@ -230,7 +230,7 @@ func TestPostHandlersRejectGet(t *testing.T) { func TestGetHandlersRejectPost(t *testing.T) { log, _ := setupTestLog(t) - handlers := NewPathHandlers(&hOpts, log) + handlers := NewPathHandlers(t.Context(), &hOpts, log) // Anything in the get handler list should reject POST. for path, handler := range getHandlers(t, handlers) { @@ -263,7 +263,7 @@ func TestPostHandlersFailure(t *testing.T) { } log, _ := setupTestLog(t) - handlers := NewPathHandlers(&hOpts, log) + handlers := NewPathHandlers(t.Context(), &hOpts, log) for path, handler := range postHandlers(t, handlers) { t.Run(path, func(t *testing.T) { @@ -286,7 +286,7 @@ func TestPostHandlersFailure(t *testing.T) { func TestNewPathHandlers(t *testing.T) { log, _ := setupTestLog(t) t.Run("Handlers", func(t *testing.T) { - handlers := NewPathHandlers(&HandlerOptions{}, log) + handlers := NewPathHandlers(t.Context(), &HandlerOptions{}, log) // Check each entrypoint has a handler if got, want := len(handlers), len(entrypoints); got != want { t.Fatalf("len(info.handler)=%d; want %d", got, want) diff --git a/internal/scti/otel.go b/internal/scti/otel.go new file mode 100644 index 00000000..d0e62fa4 --- /dev/null +++ b/internal/scti/otel.go @@ -0,0 +1,41 @@ +// Copyright 2025 The Tessera authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scti + +import ( + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "k8s.io/klog/v2" +) + +const name = "github.com/transparency-dev/static-ct/internal/scti" + +var ( + meter = otel.Meter(name) + tracer = otel.Tracer(name) +) + +var ( + codeKey = attribute.Key("tesseract.code") + operationKey = attribute.Key("tesseract.operation") + originKey = attribute.Key("tesseract.origin") +) + +func mustCreate[T any](t T, err error) T { + if err != nil { + klog.Exit(err.Error()) + } + return t +} From 2585c5c57dc0a3758cfd1a75a32815150d531bb4 Mon Sep 17 00:00:00 2001 From: Al Cutter Date: Tue, 8 Apr 2025 17:36:25 +0100 Subject: [PATCH 4/7] Remove prom from AWS --- cmd/aws/main.go | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/cmd/aws/main.go b/cmd/aws/main.go index a5d1c2b5..482b1e46 100644 --- a/cmd/aws/main.go +++ b/cmd/aws/main.go @@ -28,7 +28,6 @@ import ( "time" "github.com/go-sql-driver/mysql" - "github.com/prometheus/client_golang/prometheus/promhttp" sctfe "github.com/transparency-dev/static-ct" "github.com/transparency-dev/static-ct/storage" awsSCTFE "github.com/transparency-dev/static-ct/storage/aws" @@ -50,7 +49,6 @@ var ( notAfterLimit timestampFlag httpEndpoint = flag.String("http_endpoint", "localhost:6962", "Endpoint for HTTP (host:port).") - metricsEndpoint = flag.String("metrics_endpoint", "", "Endpoint for serving metrics; if left empty, metrics will be visible on --http_endpoint.") httpDeadline = flag.Duration("http_deadline", time.Second*10, "Deadline for HTTP requests.") maskInternalErrors = flag.Bool("mask_internal_errors", false, "Don't return error strings with Internal Server Error HTTP responses.") origin = flag.String("origin", "", "Origin of the log, for checkpoints and the monitoring prefix.") @@ -102,25 +100,6 @@ func main() { klog.Info("**** CT HTTP Server Starting ****") http.Handle("/", logHandler) - metricsAt := *metricsEndpoint - if metricsAt == "" { - metricsAt = *httpEndpoint - } - - if metricsAt != *httpEndpoint { - // Run a separate handler for metrics. - go func() { - mux := http.NewServeMux() - mux.Handle("/metrics", promhttp.Handler()) - metricsServer := http.Server{Addr: metricsAt, Handler: mux} - err := metricsServer.ListenAndServe() - klog.Warningf("Metrics server exited: %v", err) - }() - } else { - // Handle metrics on the DefaultServeMux. - http.Handle("/metrics", promhttp.Handler()) - } - // Bring up the HTTP server and serve until we get a signal not to. srv := http.Server{Addr: *httpEndpoint} shutdownWG := new(sync.WaitGroup) From 5145bafba85bda13d7ee668ad41dd39f314c9738 Mon Sep 17 00:00:00 2001 From: Al Cutter Date: Tue, 8 Apr 2025 17:37:01 +0100 Subject: [PATCH 5/7] Mod tidy --- go.mod | 9 +-------- go.sum | 14 -------------- 2 files changed, 1 insertion(+), 22 deletions(-) diff --git a/go.mod b/go.mod index 9af5c7df..f8453595 100644 --- a/go.mod +++ b/go.mod @@ -18,13 +18,13 @@ require ( github.com/go-sql-driver/mysql v1.9.2 github.com/google/go-cmp v0.7.0 github.com/kylelemons/godebug v1.1.0 - github.com/prometheus/client_golang v1.21.1 github.com/rivo/tview v0.0.0-20240625185742-b0a7293b8130 github.com/transparency-dev/formats v0.0.0-20250127084410-134797944be6 github.com/transparency-dev/merkle v0.0.2 github.com/transparency-dev/trillian-tessera v0.1.2-0.20250408153912-a650aa01f2a4 go.etcd.io/bbolt v1.4.0 go.opentelemetry.io/otel v1.35.0 + go.opentelemetry.io/otel/metric v1.35.0 go.opentelemetry.io/otel/sdk v1.35.0 go.opentelemetry.io/otel/sdk/metric v1.35.0 golang.org/x/crypto v0.37.0 @@ -64,7 +64,6 @@ require ( github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.33.18 // indirect - github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42 // indirect github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect @@ -80,21 +79,15 @@ require ( github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect github.com/googleapis/gax-go/v2 v2.14.1 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect - github.com/klauspost/compress v1.17.11 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect - github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect - github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.62.0 // indirect - github.com/prometheus/procfs v0.15.1 // indirect github.com/rivo/uniseg v0.4.7 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/contrib/detectors/gcp v1.35.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0 // indirect - go.opentelemetry.io/otel/metric v1.35.0 // indirect go.opentelemetry.io/otel/trace v1.35.0 // indirect golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 // indirect golang.org/x/oauth2 v0.28.0 // indirect diff --git a/go.sum b/go.sum index 6f79d6ae..ae831ca9 100644 --- a/go.sum +++ b/go.sum @@ -690,8 +690,6 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.33.18 h1:xz7WvTMfSStb9Y8NpCT82FXLNC3 github.com/aws/aws-sdk-go-v2/service/sts v1.33.18/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4= github.com/aws/smithy-go v1.22.3 h1:Z//5NuZCSW6R4PhQ93hShNbyBbn8BWCmCVCt+Q8Io5k= github.com/aws/smithy-go v1.22.3/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= -github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -920,8 +918,6 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:C github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= -github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= -github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -944,8 +940,6 @@ github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -971,17 +965,9 @@ github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.21.1 h1:DOvXXTqVzvkIewV/CDPFdejpMCGeMcbGCQ8YOmu+Ibk= -github.com/prometheus/client_golang v1.21.1/go.mod h1:U9NM32ykUErtVBxdvD3zfi+EuFkkaBvMb09mIfe0Zgg= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= -github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= -github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= -github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= -github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= -github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/tview v0.0.0-20240625185742-b0a7293b8130 h1:o1CYtoFOm6xJK3DvDAEG5wDJPLj+SoxUtUDFaQgt1iY= github.com/rivo/tview v0.0.0-20240625185742-b0a7293b8130/go.mod h1:02iFIz7K/A9jGCvrizLPvoqr4cEIx7q54RH5Qudkrss= From 538a4d6bbd9b68d1f9ddaa05506752e7a8456a26 Mon Sep 17 00:00:00 2001 From: Al Cutter Date: Tue, 8 Apr 2025 17:51:35 +0100 Subject: [PATCH 6/7] Add top-level spans --- internal/scti/handlers.go | 11 ++++++++++- storage/otel.go | 25 +++++++++++++++++++++++++ storage/storage.go | 12 ++++++++++++ 3 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 storage/otel.go diff --git a/internal/scti/handlers.go b/internal/scti/handlers.go index 5378c6df..fdd89b5b 100644 --- a/internal/scti/handlers.go +++ b/internal/scti/handlers.go @@ -345,14 +345,23 @@ func addChainInternal(ctx context.Context, opts *HandlerOptions, log *log, w htt } func addChain(ctx context.Context, opts *HandlerOptions, log *log, w http.ResponseWriter, r *http.Request) (int, error) { + ctx, span := tracer.Start(ctx, "tesseract.addChain") + defer span.End() + return addChainInternal(ctx, opts, log, w, r, false) } func addPreChain(ctx context.Context, opts *HandlerOptions, log *log, w http.ResponseWriter, r *http.Request) (int, error) { + ctx, span := tracer.Start(ctx, "tesseract.addPreChain") + defer span.End() + return addChainInternal(ctx, opts, log, w, r, true) } -func getRoots(_ context.Context, opts *HandlerOptions, log *log, w http.ResponseWriter, _ *http.Request) (int, error) { +func getRoots(ctx context.Context, opts *HandlerOptions, log *log, w http.ResponseWriter, _ *http.Request) (int, error) { + _, span := tracer.Start(ctx, "tesseract.getRoots") + defer span.End() + // Pull out the raw certificates from the parsed versions rawCerts := make([][]byte, 0, len(log.chainValidationOpts.trustedRoots.RawCertificates())) for _, cert := range log.chainValidationOpts.trustedRoots.RawCertificates() { diff --git a/storage/otel.go b/storage/otel.go new file mode 100644 index 00000000..10f63848 --- /dev/null +++ b/storage/otel.go @@ -0,0 +1,25 @@ +// Copyright 2025 The Tessera authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "go.opentelemetry.io/otel" +) + +const name = "github.com/transparency-dev/static-ct/storage" + +var ( + tracer = otel.Tracer(name) +) diff --git a/storage/storage.go b/storage/storage.go index 076b4e8d..be4798b9 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -68,6 +68,9 @@ func NewCTStorage(logStorage *tessera.Appender, issuerStorage IssuerStorage, ded // Add stores CT entries. func (cts *CTStorage) Add(ctx context.Context, entry *ctonly.Entry) tessera.IndexFuture { + ctx, span := tracer.Start(ctx, "tesseract.storage.Add") + defer span.End() + // TODO(phboneff): add deduplication and chain storage return cts.storeData(ctx, entry) } @@ -76,6 +79,9 @@ func (cts *CTStorage) Add(ctx context.Context, entry *ctonly.Entry) tessera.Inde // // If an object is already stored under this hash, continues. func (cts *CTStorage) AddIssuerChain(ctx context.Context, chain []*x509.Certificate) error { + ctx, span := tracer.Start(ctx, "tesseract.storage.AddIssuerChain") + defer span.End() + kvs := []KV{} for _, c := range chain { id := sha256.Sum256(c.Raw) @@ -125,6 +131,9 @@ func cachedStoreIssuers(s IssuerStorage) func(context.Context, []KV) error { // AddCertDedupInfo stores in the deduplication storage. func (cts CTStorage) AddCertDedupInfo(ctx context.Context, c *x509.Certificate, sctDedupInfo dedup.SCTDedupInfo) error { + ctx, span := tracer.Start(ctx, "tesseract.storage.AddCertDedupInfo") + defer span.End() + key := sha256.Sum256(c.Raw) if err := cts.dedupStorage.Add(ctx, []dedup.LeafDedupInfo{{LeafID: key[:], SCTDedupInfo: sctDedupInfo}}); err != nil { return fmt.Errorf("error storing SCTDedupInfo %+v of \"%x\": %v", sctDedupInfo, key, err) @@ -134,6 +143,9 @@ func (cts CTStorage) AddCertDedupInfo(ctx context.Context, c *x509.Certificate, // GetCertDedupInfo fetches the SCTDedupInfo of a given certificate from the deduplication storage. func (cts CTStorage) GetCertDedupInfo(ctx context.Context, c *x509.Certificate) (dedup.SCTDedupInfo, bool, error) { + ctx, span := tracer.Start(ctx, "tesseract.storageGetCertDedupInfo") + defer span.End() + key := sha256.Sum256(c.Raw) sctC, ok, err := cts.dedupStorage.Get(ctx, key[:]) if err != nil { From 54dc4b466c0b1c42ce02b1812e4c06a532651d1e Mon Sep 17 00:00:00 2001 From: Al Cutter Date: Wed, 9 Apr 2025 10:39:11 +0100 Subject: [PATCH 7/7] Fixes --- internal/scti/handlers.go | 27 ++++++++++++++------------- internal/scti/otel.go | 2 +- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/internal/scti/handlers.go b/internal/scti/handlers.go index fdd89b5b..3e4fe2fd 100644 --- a/internal/scti/handlers.go +++ b/internal/scti/handlers.go @@ -65,9 +65,9 @@ var ( knownLogs metric.Int64Gauge // origin => value (always 1.0) lastSCTIndex metric.Int64Gauge // origin => value lastSCTTimestamp metric.Int64Gauge // origin => value - reqsCounter metric.Int64Counter // origin, op => value - rspsCounter metric.Int64Counter // origin, op, code => value - rspLatency metric.Float64Histogram // origin, op, code => value + reqCounter metric.Int64Counter // origin, op => value + rspCounter metric.Int64Counter // origin, op, code => value + reqDuration metric.Float64Histogram // origin, op, code => value ) // setupMetrics initializes all the exported metrics. @@ -84,16 +84,17 @@ func setupMetrics() { metric.WithDescription("Index of last SCT"), metric.WithUnit("{entry}"))) - reqsCounter = mustCreate(meter.Int64Counter("tesseract.http_request.count", - metric.WithDescription("CT HTTP requests"))) + reqCounter = mustCreate(meter.Int64Counter("tesseract.http.request.count", + metric.WithDescription("CT HTTP requests"), + metric.WithUnit("{request}"))) - rspsCounter = mustCreate(meter.Int64Counter("tesseract.http_response.count", - metric.WithDescription("CT HTTP responses"))) + rspCounter = mustCreate(meter.Int64Counter("tesseract.http.response.count", + metric.WithDescription("CT HTTP responses"), + metric.WithUnit("{response}"))) - rspLatency = mustCreate(meter.Float64Histogram("tesseract.http_response.duration", + reqDuration = mustCreate(meter.Float64Histogram("tesseract.http.request.duration", metric.WithDescription("CT HTTP response duration"), - metric.WithExplicitBucketBoundaries(otel.LatencyHistogramBuckets...), - metric.WithUnit("ms"))) + metric.WithUnit("s"))) } // entrypoints is a list of entrypoint names as exposed in statistics/logging. @@ -119,13 +120,13 @@ func (a appHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { originAttr := originKey.String(a.log.origin) operationAttr := operationKey.String(a.name) - reqsCounter.Add(r.Context(), 1, metric.WithAttributes(originAttr, operationAttr)) + reqCounter.Add(r.Context(), 1, metric.WithAttributes(originAttr, operationAttr)) startTime := a.opts.TimeSource.Now() logCtx := a.opts.RequestLog.start(r.Context()) a.opts.RequestLog.origin(logCtx, a.log.origin) defer func() { latency := a.opts.TimeSource.Now().Sub(startTime).Seconds() - rspLatency.Record(r.Context(), latency, metric.WithAttributes(originAttr, operationAttr, codeKey.Int(statusCode))) + reqDuration.Record(r.Context(), latency, metric.WithAttributes(originAttr, operationAttr, codeKey.Int(statusCode))) }() klog.V(2).Infof("%s: request %v %q => %s", a.log.origin, r.Method, r.URL, a.name) @@ -155,7 +156,7 @@ func (a appHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { statusCode, err = a.handler(ctx, a.opts, a.log, w, r) a.opts.RequestLog.status(ctx, statusCode) klog.V(2).Infof("%s: %s <= st=%d", a.log.origin, a.name, statusCode) - rspsCounter.Add(r.Context(), 1, metric.WithAttributes(originAttr, operationAttr, codeKey.Int(statusCode))) + rspCounter.Add(r.Context(), 1, metric.WithAttributes(originAttr, operationAttr, codeKey.Int(statusCode))) if err != nil { klog.Warningf("%s: %s handler error: %v", a.log.origin, a.name, err) a.opts.sendHTTPError(w, statusCode, err) diff --git a/internal/scti/otel.go b/internal/scti/otel.go index d0e62fa4..3a604bb6 100644 --- a/internal/scti/otel.go +++ b/internal/scti/otel.go @@ -28,7 +28,7 @@ var ( ) var ( - codeKey = attribute.Key("tesseract.code") + codeKey = attribute.Key("http.response.status_code") operationKey = attribute.Key("tesseract.operation") originKey = attribute.Key("tesseract.origin") )