diff --git a/cmd/experimental/migrate/gcp/main.go b/cmd/experimental/migrate/gcp/main.go index 2a98812e..2d88ce1d 100644 --- a/cmd/experimental/migrate/gcp/main.go +++ b/cmd/experimental/migrate/gcp/main.go @@ -40,8 +40,9 @@ var ( spanner = flag.String("spanner", "", "Spanner resource URI ('projects/.../...')") sourceURL = flag.String("source_url", "", "Base URL for the source log.") - numWorkers = flag.Int("num_workers", 30, "Number of migration worker goroutines.") - persistentAntispam = flag.Bool("antispam", false, "EXPERIMENTAL: Set to true to enable GCP-based persistent antispam storage") + numWorkers = flag.Uint("num_workers", 30, "Number of migration worker goroutines.") + persistentAntispam = flag.Bool("antispam", false, "EXPERIMENTAL: Set to true to enable GCP-based persistent antispam storage.") + antispamBatchSize = flag.Uint("antispam_batch_size", 1500, "EXPERIMENTAL: maximum number of antispam rows to insert in a batch (1500 gives good performance with 300 Spanner PU and above, smaller values may be required for smaller allocs).") ) func main() { @@ -88,7 +89,13 @@ func main() { var antispam tessera.Antispam // Persistent antispam is currently experimental, so there's no terraform or documentation yet! if *persistentAntispam { - antispam, err = gcp_as.NewAntispam(ctx, fmt.Sprintf("%s-antispam", *spanner)) + as_opts := gcp_as.AntispamOpts{ + // 1500 appears to be give good performance for migrating logs, but you may need to lower it if you have + // less than 300 Spanner PU available. (Consider temporarily raising your Spanner CPU quota to be at least + // this amount for the duration of the migration.) + MaxBatchSize: *antispamBatchSize, + } + antispam, err = gcp_as.NewAntispam(ctx, fmt.Sprintf("%s-antispam", *spanner), as_opts) if err != nil { klog.Exitf("Failed to create new GCP antispam storage: %v", err) } @@ -101,7 +108,7 @@ func main() { } readEntryBundle := readCTEntryBundle(*sourceURL) - if err := tessera.Migrate(context.Background(), *numWorkers, sourceSize, sourceRoot, readEntryBundle, m); err != nil { + if err := m.Migrate(context.Background(), *numWorkers, sourceSize, sourceRoot, readEntryBundle); err != nil { klog.Exitf("Migrate failed: %v", err) } diff --git a/go.mod b/go.mod index 069a11aa..2406882b 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/rivo/tview v0.0.0-20240625185742-b0a7293b8130 github.com/transparency-dev/formats v0.0.0-20250127084410-134797944be6 github.com/transparency-dev/merkle v0.0.2 - github.com/transparency-dev/trillian-tessera v0.1.1-0.20250317151915-e61e2d86f685 + github.com/transparency-dev/trillian-tessera v0.1.2-0.20250320160837-ae724376e1ac go.etcd.io/bbolt v1.4.0 golang.org/x/crypto v0.36.0 golang.org/x/mod v0.24.0 diff --git a/go.sum b/go.sum index 73c44334..e49f5cb9 100644 --- a/go.sum +++ b/go.sum @@ -974,8 +974,8 @@ github.com/transparency-dev/formats v0.0.0-20250127084410-134797944be6 h1:TVUG0R github.com/transparency-dev/formats v0.0.0-20250127084410-134797944be6/go.mod h1:tSjZBSQ1ZMxgaOMppnyw48SbTDL947PD/8KYbvrx+lE= github.com/transparency-dev/merkle v0.0.2 h1:Q9nBoQcZcgPamMkGn7ghV8XiTZ/kRxn1yCG81+twTK4= github.com/transparency-dev/merkle v0.0.2/go.mod h1:pqSy+OXefQ1EDUVmAJ8MUhHB9TXGuzVAT58PqBoHz1A= -github.com/transparency-dev/trillian-tessera v0.1.1-0.20250317151915-e61e2d86f685 h1:Cu1sKlj37BnhBybTQ5V1/zgqPQHlBEXIyLiAK+rVlvA= -github.com/transparency-dev/trillian-tessera v0.1.1-0.20250317151915-e61e2d86f685/go.mod h1:uvyZ7WGpaRDPY+4Lme+s1vEUOluYevTYzrDg9j05cYU= +github.com/transparency-dev/trillian-tessera v0.1.2-0.20250320160837-ae724376e1ac h1:LwnfCnox2ZHW3TgMCdL60C4xp8oR3lDew13MZeHDhPA= +github.com/transparency-dev/trillian-tessera v0.1.2-0.20250320160837-ae724376e1ac/go.mod h1:fARQE1UN2Z0wv/02J0uAsTIli/+flEJGCMWlBs9Ps9E= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/internal/scti/handlers.go b/internal/scti/handlers.go index 3cdea775..5b6e46fe 100644 --- a/internal/scti/handlers.go +++ b/internal/scti/handlers.go @@ -301,7 +301,7 @@ func addChainInternal(ctx context.Context, opts *HandlerOptions, log *log, w htt } klog.V(2).Infof("%s: %s => storage.Add", log.origin, method) - idx, err = log.storage.Add(ctx, entry)() + index, err := log.storage.Add(ctx, entry)() if err != nil { if errors.Is(err, tessera.ErrPushback) { w.Header().Add("Retry-After", "1") @@ -309,6 +309,9 @@ func addChainInternal(ctx context.Context, opts *HandlerOptions, log *log, w htt } return http.StatusInternalServerError, fmt.Errorf("couldn't store the leaf: %v", err) } + // TODO(phbnf): figure out whether to use Tessera's index.IsDup() or a separate "external" antispam impl. + idx = index.Index + // We store the index for this certificate in the deduplication storage immediately. // It might be stored again later, if a local deduplication storage is synced, potentially // with a smaller value. diff --git a/internal/scti/handlers_test.go b/internal/scti/handlers_test.go index e7ca822b..2989799c 100644 --- a/internal/scti/handlers_test.go +++ b/internal/scti/handlers_test.go @@ -38,6 +38,7 @@ import ( "github.com/transparency-dev/static-ct/internal/x509util" "github.com/transparency-dev/static-ct/mockstorage" "github.com/transparency-dev/static-ct/modules/dedup" + tessera "github.com/transparency-dev/trillian-tessera" "github.com/transparency-dev/trillian-tessera/ctonly" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -293,7 +294,7 @@ func TestAddChainWhitespace(t *testing.T) { epilog := "]}\n" req, leafChain := parseChain(t, false, pemChain, info.roots.RawCertificates()[0]) - rsp := uint64(0) + rsp := tessera.Index{Index: 0} var tests = []struct { descr string @@ -332,7 +333,7 @@ func TestAddChainWhitespace(t *testing.T) { if test.want == http.StatusOK { info.storage.EXPECT().GetCertDedupInfo(deadlineMatcher(), cmpMatcher{leafChain[0]}).Return(dedup.SCTDedupInfo{Idx: uint64(0), Timestamp: fakeTimeMillis}, false, nil) info.storage.EXPECT().AddIssuerChain(deadlineMatcher(), cmpMatcher{leafChain[1:]}).Return(nil) - info.storage.EXPECT().Add(deadlineMatcher(), cmpMatcher{req}).Return(func() (uint64, error) { return rsp, nil }) + info.storage.EXPECT().Add(deadlineMatcher(), cmpMatcher{req}).Return(func() (tessera.Index, error) { return rsp, nil }) info.storage.EXPECT().AddCertDedupInfo(deadlineMatcher(), cmpMatcher{leafChain[0]}, dedup.SCTDedupInfo{Idx: uint64(0), Timestamp: fakeTimeMillis}).Return(nil) } @@ -408,10 +409,10 @@ func TestAddChain(t *testing.T) { chain := createJSONChain(t, *pool) if len(test.toSign) > 0 { req, leafChain := parseChain(t, false, test.chain, info.roots.RawCertificates()[0]) - rsp := uint64(0) + rsp := tessera.Index{Index: 0} info.storage.EXPECT().GetCertDedupInfo(deadlineMatcher(), cmpMatcher{leafChain[0]}).Return(dedup.SCTDedupInfo{Idx: uint64(0), Timestamp: fakeTimeMillis}, false, nil) info.storage.EXPECT().AddIssuerChain(deadlineMatcher(), cmpMatcher{leafChain[1:]}).Return(nil) - info.storage.EXPECT().Add(deadlineMatcher(), cmpMatcher{req}).Return(func() (uint64, error) { return rsp, test.err }) + info.storage.EXPECT().Add(deadlineMatcher(), cmpMatcher{req}).Return(func() (tessera.Index, error) { return rsp, test.err }) if test.want == http.StatusOK { info.storage.EXPECT().AddCertDedupInfo(deadlineMatcher(), cmpMatcher{leafChain[0]}, dedup.SCTDedupInfo{Idx: uint64(0), Timestamp: fakeTimeMillis}).Return(nil) } @@ -505,10 +506,10 @@ func TestAddPrechain(t *testing.T) { chain := createJSONChain(t, *pool) if len(test.toSign) > 0 { req, leafChain := parseChain(t, true, test.chain, info.roots.RawCertificates()[0]) - rsp := uint64(0) + rsp := tessera.Index{Index: 0} info.storage.EXPECT().GetCertDedupInfo(deadlineMatcher(), cmpMatcher{leafChain[0]}).Return(dedup.SCTDedupInfo{Idx: uint64(0), Timestamp: fakeTimeMillis}, false, nil) info.storage.EXPECT().AddIssuerChain(deadlineMatcher(), cmpMatcher{leafChain[1:]}).Return(nil) - info.storage.EXPECT().Add(deadlineMatcher(), cmpMatcher{req}).Return(func() (uint64, error) { return rsp, test.err }) + info.storage.EXPECT().Add(deadlineMatcher(), cmpMatcher{req}).Return(func() (tessera.Index, error) { return rsp, test.err }) if test.want == http.StatusOK { info.storage.EXPECT().AddCertDedupInfo(deadlineMatcher(), cmpMatcher{leafChain[0]}, dedup.SCTDedupInfo{Idx: uint64(0), Timestamp: fakeTimeMillis}).Return(nil) } diff --git a/mockstorage/gen.go b/mockstorage/gen.go index e63aca33..26382d4d 100644 --- a/mockstorage/gen.go +++ b/mockstorage/gen.go @@ -15,4 +15,4 @@ // Package mockclient provides a mockable version of the Trillian log client API. package mockstorage -//go:generate mockgen -package mockstorage -destination mock_ct_storage.go github.com/transparency-dev/static-ct Storage +//go:generate mockgen -package mockstorage -destination mock_ct_storage.go github.com/transparency-dev/static-ct/internal/scti Storage diff --git a/mockstorage/mock_ct_storage.go b/mockstorage/mock_ct_storage.go index b336b4a5..8582b4cb 100644 --- a/mockstorage/mock_ct_storage.go +++ b/mockstorage/mock_ct_storage.go @@ -1,13 +1,13 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/transparency-dev/static-ct (interfaces: Storage) +// Source: github.com/transparency-dev/static-ct/internal/scti (interfaces: Storage) // Package mockstorage is a generated GoMock package. package mockstorage import ( context "context" - reflect "reflect" x509 "crypto/x509" + reflect "reflect" gomock "github.com/golang/mock/gomock" dedup "github.com/transparency-dev/static-ct/modules/dedup"