Bump tessera to ae724376e1ace4046767511c72c6006bde3ec87e (#203)

Author: AlCutter

cmd/experimental/migrate/gcp/main.go

@@ -40,8 +40,9 @@ var (
 	spanner = flag.String("spanner", "", "Spanner resource URI ('projects/.../...')")
 
 	sourceURL          = flag.String("source_url", "", "Base URL for the source log.")
-	numWorkers         = flag.Int("num_workers", 30, "Number of migration worker goroutines.")
-	persistentAntispam = flag.Bool("antispam", false, "EXPERIMENTAL: Set to true to enable GCP-based persistent antispam storage")
+	numWorkers         = flag.Uint("num_workers", 30, "Number of migration worker goroutines.")
+	persistentAntispam = flag.Bool("antispam", false, "EXPERIMENTAL: Set to true to enable GCP-based persistent antispam storage.")
+	antispamBatchSize  = flag.Uint("antispam_batch_size", 1500, "EXPERIMENTAL: maximum number of antispam rows to insert in a batch (1500 gives good performance with 300 Spanner PU and above, smaller values may be required for smaller allocs).")
 )
 
 func main() {
@@ -88,7 +89,13 @@ func main() {
 	var antispam tessera.Antispam
 	// Persistent antispam is currently experimental, so there's no terraform or documentation yet!
 	if *persistentAntispam {
-		antispam, err = gcp_as.NewAntispam(ctx, fmt.Sprintf("%s-antispam", *spanner))
+		as_opts := gcp_as.AntispamOpts{
+			// 1500 appears to be give good performance for migrating logs, but you may need to lower it if you have
+			// less than 300 Spanner PU available. (Consider temporarily raising your Spanner CPU quota to be at least
+			// this amount for the duration of the migration.)
+			MaxBatchSize: *antispamBatchSize,
+		}
+		antispam, err = gcp_as.NewAntispam(ctx, fmt.Sprintf("%s-antispam", *spanner), as_opts)
 		if err != nil {
 			klog.Exitf("Failed to create new GCP antispam storage: %v", err)
 		}
@@ -101,7 +108,7 @@ func main() {
 	}
 
 	readEntryBundle := readCTEntryBundle(*sourceURL)
-	if err := tessera.Migrate(context.Background(), *numWorkers, sourceSize, sourceRoot, readEntryBundle, m); err != nil {
+	if err := m.Migrate(context.Background(), *numWorkers, sourceSize, sourceRoot, readEntryBundle); err != nil {
 		klog.Exitf("Migrate failed: %v", err)
 	}
 

go.mod

@@ -15,7 +15,7 @@ require (
 	github.com/rivo/tview v0.0.0-20240625185742-b0a7293b8130
 	github.com/transparency-dev/formats v0.0.0-20250127084410-134797944be6
 	github.com/transparency-dev/merkle v0.0.2
-	github.com/transparency-dev/trillian-tessera v0.1.1-0.20250317151915-e61e2d86f685
+	github.com/transparency-dev/trillian-tessera v0.1.2-0.20250320160837-ae724376e1ac
 	go.etcd.io/bbolt v1.4.0
 	golang.org/x/crypto v0.36.0
 	golang.org/x/mod v0.24.0

go.sum

@@ -974,8 +974,8 @@ github.com/transparency-dev/formats v0.0.0-20250127084410-134797944be6 h1:TVUG0R
 github.com/transparency-dev/formats v0.0.0-20250127084410-134797944be6/go.mod h1:tSjZBSQ1ZMxgaOMppnyw48SbTDL947PD/8KYbvrx+lE=
 github.com/transparency-dev/merkle v0.0.2 h1:Q9nBoQcZcgPamMkGn7ghV8XiTZ/kRxn1yCG81+twTK4=
 github.com/transparency-dev/merkle v0.0.2/go.mod h1:pqSy+OXefQ1EDUVmAJ8MUhHB9TXGuzVAT58PqBoHz1A=
-github.com/transparency-dev/trillian-tessera v0.1.1-0.20250317151915-e61e2d86f685 h1:Cu1sKlj37BnhBybTQ5V1/zgqPQHlBEXIyLiAK+rVlvA=
-github.com/transparency-dev/trillian-tessera v0.1.1-0.20250317151915-e61e2d86f685/go.mod h1:uvyZ7WGpaRDPY+4Lme+s1vEUOluYevTYzrDg9j05cYU=
+github.com/transparency-dev/trillian-tessera v0.1.2-0.20250320160837-ae724376e1ac h1:LwnfCnox2ZHW3TgMCdL60C4xp8oR3lDew13MZeHDhPA=
+github.com/transparency-dev/trillian-tessera v0.1.2-0.20250320160837-ae724376e1ac/go.mod h1:fARQE1UN2Z0wv/02J0uAsTIli/+flEJGCMWlBs9Ps9E=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

internal/scti/handlers.go

@@ -301,14 +301,17 @@ func addChainInternal(ctx context.Context, opts *HandlerOptions, log *log, w htt
 		}
 
 		klog.V(2).Infof("%s: %s => storage.Add", log.origin, method)
-		idx, err = log.storage.Add(ctx, entry)()
+		index, err := log.storage.Add(ctx, entry)()
 		if err != nil {
 			if errors.Is(err, tessera.ErrPushback) {
 				w.Header().Add("Retry-After", "1")
 				return http.StatusServiceUnavailable, fmt.Errorf("Tessera sequencer pushed back: %v", err)
 			}
 			return http.StatusInternalServerError, fmt.Errorf("couldn't store the leaf: %v", err)
 		}
+		// TODO(phbnf): figure out whether to use Tessera's index.IsDup() or a separate "external" antispam impl.
+		idx = index.Index
+
 		// We store the index for this certificate in the deduplication storage immediately.
 		// It might be stored again later, if a local deduplication storage is synced, potentially
 		// with a smaller value.

internal/scti/handlers_test.go

@@ -38,6 +38,7 @@ import (
 	"github.com/transparency-dev/static-ct/internal/x509util"
 	"github.com/transparency-dev/static-ct/mockstorage"
 	"github.com/transparency-dev/static-ct/modules/dedup"
+	tessera "github.com/transparency-dev/trillian-tessera"
 	"github.com/transparency-dev/trillian-tessera/ctonly"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
@@ -293,7 +294,7 @@ func TestAddChainWhitespace(t *testing.T) {
 	epilog := "]}\n"
 
 	req, leafChain := parseChain(t, false, pemChain, info.roots.RawCertificates()[0])
-	rsp := uint64(0)
+	rsp := tessera.Index{Index: 0}
 
 	var tests = []struct {
 		descr string
@@ -332,7 +333,7 @@ func TestAddChainWhitespace(t *testing.T) {
 			if test.want == http.StatusOK {
 				info.storage.EXPECT().GetCertDedupInfo(deadlineMatcher(), cmpMatcher{leafChain[0]}).Return(dedup.SCTDedupInfo{Idx: uint64(0), Timestamp: fakeTimeMillis}, false, nil)
 				info.storage.EXPECT().AddIssuerChain(deadlineMatcher(), cmpMatcher{leafChain[1:]}).Return(nil)
-				info.storage.EXPECT().Add(deadlineMatcher(), cmpMatcher{req}).Return(func() (uint64, error) { return rsp, nil })
+				info.storage.EXPECT().Add(deadlineMatcher(), cmpMatcher{req}).Return(func() (tessera.Index, error) { return rsp, nil })
 				info.storage.EXPECT().AddCertDedupInfo(deadlineMatcher(), cmpMatcher{leafChain[0]}, dedup.SCTDedupInfo{Idx: uint64(0), Timestamp: fakeTimeMillis}).Return(nil)
 			}
 
@@ -408,10 +409,10 @@ func TestAddChain(t *testing.T) {
 			chain := createJSONChain(t, *pool)
 			if len(test.toSign) > 0 {
 				req, leafChain := parseChain(t, false, test.chain, info.roots.RawCertificates()[0])
-				rsp := uint64(0)
+				rsp := tessera.Index{Index: 0}
 				info.storage.EXPECT().GetCertDedupInfo(deadlineMatcher(), cmpMatcher{leafChain[0]}).Return(dedup.SCTDedupInfo{Idx: uint64(0), Timestamp: fakeTimeMillis}, false, nil)
 				info.storage.EXPECT().AddIssuerChain(deadlineMatcher(), cmpMatcher{leafChain[1:]}).Return(nil)
-				info.storage.EXPECT().Add(deadlineMatcher(), cmpMatcher{req}).Return(func() (uint64, error) { return rsp, test.err })
+				info.storage.EXPECT().Add(deadlineMatcher(), cmpMatcher{req}).Return(func() (tessera.Index, error) { return rsp, test.err })
 				if test.want == http.StatusOK {
 					info.storage.EXPECT().AddCertDedupInfo(deadlineMatcher(), cmpMatcher{leafChain[0]}, dedup.SCTDedupInfo{Idx: uint64(0), Timestamp: fakeTimeMillis}).Return(nil)
 				}
@@ -505,10 +506,10 @@ func TestAddPrechain(t *testing.T) {
 			chain := createJSONChain(t, *pool)
 			if len(test.toSign) > 0 {
 				req, leafChain := parseChain(t, true, test.chain, info.roots.RawCertificates()[0])
-				rsp := uint64(0)
+				rsp := tessera.Index{Index: 0}
 				info.storage.EXPECT().GetCertDedupInfo(deadlineMatcher(), cmpMatcher{leafChain[0]}).Return(dedup.SCTDedupInfo{Idx: uint64(0), Timestamp: fakeTimeMillis}, false, nil)
 				info.storage.EXPECT().AddIssuerChain(deadlineMatcher(), cmpMatcher{leafChain[1:]}).Return(nil)
-				info.storage.EXPECT().Add(deadlineMatcher(), cmpMatcher{req}).Return(func() (uint64, error) { return rsp, test.err })
+				info.storage.EXPECT().Add(deadlineMatcher(), cmpMatcher{req}).Return(func() (tessera.Index, error) { return rsp, test.err })
 				if test.want == http.StatusOK {
 					info.storage.EXPECT().AddCertDedupInfo(deadlineMatcher(), cmpMatcher{leafChain[0]}, dedup.SCTDedupInfo{Idx: uint64(0), Timestamp: fakeTimeMillis}).Return(nil)
 				}

mockstorage/gen.go

@@ -15,4 +15,4 @@
 // Package mockclient provides a mockable version of the Trillian log client API.
 package mockstorage
 
-//go:generate mockgen -package mockstorage -destination mock_ct_storage.go github.com/transparency-dev/static-ct Storage
+//go:generate mockgen -package mockstorage -destination mock_ct_storage.go github.com/transparency-dev/static-ct/internal/scti Storage