From 8488859b9fbbd5f67fbeea440b361506d8f023a6 Mon Sep 17 00:00:00 2001 From: Al Cutter Date: Thu, 20 Mar 2025 16:10:46 +0000 Subject: [PATCH 1/6] Bump Tessera to ae724376e1ace4046767511c72c6006bde3ec87e --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 069a11aa..2406882b 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/rivo/tview v0.0.0-20240625185742-b0a7293b8130 github.com/transparency-dev/formats v0.0.0-20250127084410-134797944be6 github.com/transparency-dev/merkle v0.0.2 - github.com/transparency-dev/trillian-tessera v0.1.1-0.20250317151915-e61e2d86f685 + github.com/transparency-dev/trillian-tessera v0.1.2-0.20250320160837-ae724376e1ac go.etcd.io/bbolt v1.4.0 golang.org/x/crypto v0.36.0 golang.org/x/mod v0.24.0 diff --git a/go.sum b/go.sum index 73c44334..e49f5cb9 100644 --- a/go.sum +++ b/go.sum @@ -974,8 +974,8 @@ github.com/transparency-dev/formats v0.0.0-20250127084410-134797944be6 h1:TVUG0R github.com/transparency-dev/formats v0.0.0-20250127084410-134797944be6/go.mod h1:tSjZBSQ1ZMxgaOMppnyw48SbTDL947PD/8KYbvrx+lE= github.com/transparency-dev/merkle v0.0.2 h1:Q9nBoQcZcgPamMkGn7ghV8XiTZ/kRxn1yCG81+twTK4= github.com/transparency-dev/merkle v0.0.2/go.mod h1:pqSy+OXefQ1EDUVmAJ8MUhHB9TXGuzVAT58PqBoHz1A= -github.com/transparency-dev/trillian-tessera v0.1.1-0.20250317151915-e61e2d86f685 h1:Cu1sKlj37BnhBybTQ5V1/zgqPQHlBEXIyLiAK+rVlvA= -github.com/transparency-dev/trillian-tessera v0.1.1-0.20250317151915-e61e2d86f685/go.mod h1:uvyZ7WGpaRDPY+4Lme+s1vEUOluYevTYzrDg9j05cYU= +github.com/transparency-dev/trillian-tessera v0.1.2-0.20250320160837-ae724376e1ac h1:LwnfCnox2ZHW3TgMCdL60C4xp8oR3lDew13MZeHDhPA= +github.com/transparency-dev/trillian-tessera v0.1.2-0.20250320160837-ae724376e1ac/go.mod h1:fARQE1UN2Z0wv/02J0uAsTIli/+flEJGCMWlBs9Ps9E= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= From 831b436d37f97689645a2848090665402fc1ff96 Mon Sep 17 00:00:00 2001 From: Al Cutter Date: Thu, 20 Mar 2025 16:18:55 +0000 Subject: [PATCH 2/6] Update migrate cmd to match --- cmd/experimental/migrate/gcp/main.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/cmd/experimental/migrate/gcp/main.go b/cmd/experimental/migrate/gcp/main.go index 2a98812e..fababbf4 100644 --- a/cmd/experimental/migrate/gcp/main.go +++ b/cmd/experimental/migrate/gcp/main.go @@ -40,7 +40,7 @@ var ( spanner = flag.String("spanner", "", "Spanner resource URI ('projects/.../...')") sourceURL = flag.String("source_url", "", "Base URL for the source log.") - numWorkers = flag.Int("num_workers", 30, "Number of migration worker goroutines.") + numWorkers = flag.Uint("num_workers", 30, "Number of migration worker goroutines.") persistentAntispam = flag.Bool("antispam", false, "EXPERIMENTAL: Set to true to enable GCP-based persistent antispam storage") ) @@ -88,7 +88,13 @@ func main() { var antispam tessera.Antispam // Persistent antispam is currently experimental, so there's no terraform or documentation yet! if *persistentAntispam { - antispam, err = gcp_as.NewAntispam(ctx, fmt.Sprintf("%s-antispam", *spanner)) + as_opts := gcp_as.AntispamOpts{ + // 1500 appears to be give good performance for migrating logs, but you may need to lower it if you have + // less than 300 Spanner PU available. (Consider temporarily raising your Spanner CPU quota to be at least + // this amount for the duration of the migration.) + MaxBatchSize: 1500, + } + antispam, err = gcp_as.NewAntispam(ctx, fmt.Sprintf("%s-antispam", *spanner), as_opts) if err != nil { klog.Exitf("Failed to create new GCP antispam storage: %v", err) } @@ -101,7 +107,7 @@ func main() { } readEntryBundle := readCTEntryBundle(*sourceURL) - if err := tessera.Migrate(context.Background(), *numWorkers, sourceSize, sourceRoot, readEntryBundle, m); err != nil { + if err := m.Migrate(context.Background(), *numWorkers, sourceSize, sourceRoot, readEntryBundle); err != nil { klog.Exitf("Migrate failed: %v", err) } From 4d218bb41461300ea99150c2bb7bb94c7f46c4e8 Mon Sep 17 00:00:00 2001 From: Al Cutter Date: Thu, 20 Mar 2025 16:29:59 +0000 Subject: [PATCH 3/6] Update handlers --- internal/scti/handlers.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/scti/handlers.go b/internal/scti/handlers.go index 3cdea775..5b6e46fe 100644 --- a/internal/scti/handlers.go +++ b/internal/scti/handlers.go @@ -301,7 +301,7 @@ func addChainInternal(ctx context.Context, opts *HandlerOptions, log *log, w htt } klog.V(2).Infof("%s: %s => storage.Add", log.origin, method) - idx, err = log.storage.Add(ctx, entry)() + index, err := log.storage.Add(ctx, entry)() if err != nil { if errors.Is(err, tessera.ErrPushback) { w.Header().Add("Retry-After", "1") @@ -309,6 +309,9 @@ func addChainInternal(ctx context.Context, opts *HandlerOptions, log *log, w htt } return http.StatusInternalServerError, fmt.Errorf("couldn't store the leaf: %v", err) } + // TODO(phbnf): figure out whether to use Tessera's index.IsDup() or a separate "external" antispam impl. + idx = index.Index + // We store the index for this certificate in the deduplication storage immediately. // It might be stored again later, if a local deduplication storage is synced, potentially // with a smaller value. From 9431629a4c0f55b1835728b9a98c30a645b89393 Mon Sep 17 00:00:00 2001 From: Al Cutter Date: Thu, 20 Mar 2025 16:47:06 +0000 Subject: [PATCH 4/6] Fix mock generation --- mockstorage/gen.go | 2 +- mockstorage/mock_ct_storage.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mockstorage/gen.go b/mockstorage/gen.go index e63aca33..26382d4d 100644 --- a/mockstorage/gen.go +++ b/mockstorage/gen.go @@ -15,4 +15,4 @@ // Package mockclient provides a mockable version of the Trillian log client API. package mockstorage -//go:generate mockgen -package mockstorage -destination mock_ct_storage.go github.com/transparency-dev/static-ct Storage +//go:generate mockgen -package mockstorage -destination mock_ct_storage.go github.com/transparency-dev/static-ct/internal/scti Storage diff --git a/mockstorage/mock_ct_storage.go b/mockstorage/mock_ct_storage.go index b336b4a5..8582b4cb 100644 --- a/mockstorage/mock_ct_storage.go +++ b/mockstorage/mock_ct_storage.go @@ -1,13 +1,13 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/transparency-dev/static-ct (interfaces: Storage) +// Source: github.com/transparency-dev/static-ct/internal/scti (interfaces: Storage) // Package mockstorage is a generated GoMock package. package mockstorage import ( context "context" - reflect "reflect" x509 "crypto/x509" + reflect "reflect" gomock "github.com/golang/mock/gomock" dedup "github.com/transparency-dev/static-ct/modules/dedup" From bab4f11e468cbfba4b1144e7766ea7208c8ae624 Mon Sep 17 00:00:00 2001 From: Al Cutter Date: Thu, 20 Mar 2025 16:50:19 +0000 Subject: [PATCH 5/6] Fix up handlers_test.go --- internal/scti/handlers_test.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/internal/scti/handlers_test.go b/internal/scti/handlers_test.go index e7ca822b..2989799c 100644 --- a/internal/scti/handlers_test.go +++ b/internal/scti/handlers_test.go @@ -38,6 +38,7 @@ import ( "github.com/transparency-dev/static-ct/internal/x509util" "github.com/transparency-dev/static-ct/mockstorage" "github.com/transparency-dev/static-ct/modules/dedup" + tessera "github.com/transparency-dev/trillian-tessera" "github.com/transparency-dev/trillian-tessera/ctonly" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -293,7 +294,7 @@ func TestAddChainWhitespace(t *testing.T) { epilog := "]}\n" req, leafChain := parseChain(t, false, pemChain, info.roots.RawCertificates()[0]) - rsp := uint64(0) + rsp := tessera.Index{Index: 0} var tests = []struct { descr string @@ -332,7 +333,7 @@ func TestAddChainWhitespace(t *testing.T) { if test.want == http.StatusOK { info.storage.EXPECT().GetCertDedupInfo(deadlineMatcher(), cmpMatcher{leafChain[0]}).Return(dedup.SCTDedupInfo{Idx: uint64(0), Timestamp: fakeTimeMillis}, false, nil) info.storage.EXPECT().AddIssuerChain(deadlineMatcher(), cmpMatcher{leafChain[1:]}).Return(nil) - info.storage.EXPECT().Add(deadlineMatcher(), cmpMatcher{req}).Return(func() (uint64, error) { return rsp, nil }) + info.storage.EXPECT().Add(deadlineMatcher(), cmpMatcher{req}).Return(func() (tessera.Index, error) { return rsp, nil }) info.storage.EXPECT().AddCertDedupInfo(deadlineMatcher(), cmpMatcher{leafChain[0]}, dedup.SCTDedupInfo{Idx: uint64(0), Timestamp: fakeTimeMillis}).Return(nil) } @@ -408,10 +409,10 @@ func TestAddChain(t *testing.T) { chain := createJSONChain(t, *pool) if len(test.toSign) > 0 { req, leafChain := parseChain(t, false, test.chain, info.roots.RawCertificates()[0]) - rsp := uint64(0) + rsp := tessera.Index{Index: 0} info.storage.EXPECT().GetCertDedupInfo(deadlineMatcher(), cmpMatcher{leafChain[0]}).Return(dedup.SCTDedupInfo{Idx: uint64(0), Timestamp: fakeTimeMillis}, false, nil) info.storage.EXPECT().AddIssuerChain(deadlineMatcher(), cmpMatcher{leafChain[1:]}).Return(nil) - info.storage.EXPECT().Add(deadlineMatcher(), cmpMatcher{req}).Return(func() (uint64, error) { return rsp, test.err }) + info.storage.EXPECT().Add(deadlineMatcher(), cmpMatcher{req}).Return(func() (tessera.Index, error) { return rsp, test.err }) if test.want == http.StatusOK { info.storage.EXPECT().AddCertDedupInfo(deadlineMatcher(), cmpMatcher{leafChain[0]}, dedup.SCTDedupInfo{Idx: uint64(0), Timestamp: fakeTimeMillis}).Return(nil) } @@ -505,10 +506,10 @@ func TestAddPrechain(t *testing.T) { chain := createJSONChain(t, *pool) if len(test.toSign) > 0 { req, leafChain := parseChain(t, true, test.chain, info.roots.RawCertificates()[0]) - rsp := uint64(0) + rsp := tessera.Index{Index: 0} info.storage.EXPECT().GetCertDedupInfo(deadlineMatcher(), cmpMatcher{leafChain[0]}).Return(dedup.SCTDedupInfo{Idx: uint64(0), Timestamp: fakeTimeMillis}, false, nil) info.storage.EXPECT().AddIssuerChain(deadlineMatcher(), cmpMatcher{leafChain[1:]}).Return(nil) - info.storage.EXPECT().Add(deadlineMatcher(), cmpMatcher{req}).Return(func() (uint64, error) { return rsp, test.err }) + info.storage.EXPECT().Add(deadlineMatcher(), cmpMatcher{req}).Return(func() (tessera.Index, error) { return rsp, test.err }) if test.want == http.StatusOK { info.storage.EXPECT().AddCertDedupInfo(deadlineMatcher(), cmpMatcher{leafChain[0]}, dedup.SCTDedupInfo{Idx: uint64(0), Timestamp: fakeTimeMillis}).Return(nil) } From b2816851939829f988fa7f96aa308a889117a6d0 Mon Sep 17 00:00:00 2001 From: Al Cutter Date: Thu, 20 Mar 2025 17:31:41 +0000 Subject: [PATCH 6/6] add flag --- cmd/experimental/migrate/gcp/main.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/experimental/migrate/gcp/main.go b/cmd/experimental/migrate/gcp/main.go index fababbf4..2d88ce1d 100644 --- a/cmd/experimental/migrate/gcp/main.go +++ b/cmd/experimental/migrate/gcp/main.go @@ -41,7 +41,8 @@ var ( sourceURL = flag.String("source_url", "", "Base URL for the source log.") numWorkers = flag.Uint("num_workers", 30, "Number of migration worker goroutines.") - persistentAntispam = flag.Bool("antispam", false, "EXPERIMENTAL: Set to true to enable GCP-based persistent antispam storage") + persistentAntispam = flag.Bool("antispam", false, "EXPERIMENTAL: Set to true to enable GCP-based persistent antispam storage.") + antispamBatchSize = flag.Uint("antispam_batch_size", 1500, "EXPERIMENTAL: maximum number of antispam rows to insert in a batch (1500 gives good performance with 300 Spanner PU and above, smaller values may be required for smaller allocs).") ) func main() { @@ -92,7 +93,7 @@ func main() { // 1500 appears to be give good performance for migrating logs, but you may need to lower it if you have // less than 300 Spanner PU available. (Consider temporarily raising your Spanner CPU quota to be at least // this amount for the duration of the migration.) - MaxBatchSize: 1500, + MaxBatchSize: *antispamBatchSize, } antispam, err = gcp_as.NewAntispam(ctx, fmt.Sprintf("%s-antispam", *spanner), as_opts) if err != nil {