Deduplicate SCT timestamps

modules/dedup/dedup.go

@@ -28,7 +28,7 @@ import (
 	"k8s.io/klog/v2"
 )
 
-// LeafDedupInfo, enables building idempotent deduplicated add-chain responses.
+// LeafDedupInfo, enables building deduplicated add-chain responses.
 type LeafDedupInfo struct {
 	LeafID []byte
 	SCTDedupInfo

serialize.go

@@ -184,7 +184,7 @@ func DedupFromBundle(bundle []byte, bundleIdx uint64) ([]dedup.LeafDedupInfo, er
 	kvs := []dedup.LeafDedupInfo{}
 	s := cryptobyte.String(bundle)
 
-	for len(s) > 0 {
+	for i := bundleIdx * 256; len(s) > 0; i++ {
 		var timestamp uint64
 		var entryType uint16
 		var extensions, fingerprints cryptobyte.String
@@ -213,7 +213,7 @@ func DedupFromBundle(bundle []byte, bundleIdx uint64) ([]dedup.LeafDedupInfo, er
 			return nil, fmt.Errorf("invalid data tile: unknown type %d", entryType)
 		}
 		k := sha256.Sum256(crt)
-		sctDedupInfo := dedup.SCTDedupInfo{Idx: bundleIdx*256 + uint64(len(kvs)), Timestamp: timestamp}
+		sctDedupInfo := dedup.SCTDedupInfo{Idx: uint64(i), Timestamp: timestamp}
 		kvs = append(kvs, dedup.LeafDedupInfo{LeafID: k[:], SCTDedupInfo: sctDedupInfo})
 	}
 	return kvs, nil

storage.go

@@ -131,11 +131,11 @@ func cachedStoreIssuers(s IssuerStorage) func(context.Context, []KV) error {
 	}
 }
 
-// AddCertDedupInfo stores <cert_hash, SCTDEdupInfo> in the deduplication storage.
+// AddCertDedupInfo stores <cert_hash, SCTDedupInfo> in the deduplication storage.
 func (cts CTStorage) AddCertDedupInfo(ctx context.Context, c *x509.Certificate, sctDedupInfo dedup.SCTDedupInfo) error {
 	key := sha256.Sum256(c.Raw)
 	if err := cts.dedupStorage.Add(ctx, []dedup.LeafDedupInfo{{LeafID: key[:], SCTDedupInfo: sctDedupInfo}}); err != nil {
-		return fmt.Errorf("error storing SCTDedupInfo %+v of %q: %v", sctDedupInfo, hex.EncodeToString(key[:]), err)
+		return fmt.Errorf("error storing SCTDedupInfo %+v of \"%x\": %v", sctDedupInfo, key, err)
 	}
 	return nil
 }
@@ -145,7 +145,7 @@ func (cts CTStorage) GetCertDedupInfo(ctx context.Context, c *x509.Certificate)
 	key := sha256.Sum256(c.Raw)
 	sctC, ok, err := cts.dedupStorage.Get(ctx, key[:])
 	if err != nil {
-		return dedup.SCTDedupInfo{}, false, fmt.Errorf("error fetching index of %q: %v", hex.EncodeToString(key[:]), err)
+		return dedup.SCTDedupInfo{}, false, fmt.Errorf("error fetching index of \"%x\": %v", key, err)
 	}
 	return sctC, ok, nil
 }

storage/bbolt/dedup.go

@@ -15,7 +15,7 @@
 // Package bbolt implements modules/dedup using BBolt.
 //
 // It contains two buckets:
-//   - The dedup bucket stores <leafID, idx::timestamp> pairs. Entries can either be added after
+//   - The dedup bucket stores <leafID, {idx, timestamp}> pairs. Entries can either be added after
 //     sequencing, by the server that received the request, or later when synchronising the dedup
 //     storage with the log state.
 //   - The size bucket has a single entry: <"size", X>, where X is the largest contiguous index
@@ -48,7 +48,7 @@ type Storage struct {
 
 // NewStorage returns a new BBolt storage instance with a dedup and size bucket.
 //
-// The dedup bucket stores <leafID, idx::timestamp> pairs, where idx::timestamp is the
+// The dedup bucket stores <leafID, {idx, timestamp}> pairs, where idx::timestamp is the
 // concatenation of two uint64 8 bytes BigEndian representation.
 // The size bucket has a single entry: <"size", X>, where X is the largest contiguous index from 0
 // that has been inserted in the dedup bucket.
@@ -113,7 +113,7 @@ func (s *Storage) Add(_ context.Context, ldis []dedup.LeafDedupInfo) error {
 			size := btoi(sizeB)
 			vB, err := vtob(ldi.Idx, ldi.Timestamp)
 			if err != nil {
-				return fmt.Errorf("btov(): %v", err)
+				return fmt.Errorf("vtob(): %v", err)
 			}
 
 			if old := db.Get(ldi.LeafID); len(old) == 16 && btoi(old[:8]) <= ldi.Idx {
@@ -158,7 +158,7 @@ func (s *Storage) Get(_ context.Context, leafID []byte) (dedup.SCTDedupInfo, boo
 	}
 	idx, t, err := btov(v)
 	if err != nil {
-		return dedup.SCTDedupInfo{}, false, fmt.Errorf("vtob(): %v", err)
+		return dedup.SCTDedupInfo{}, false, fmt.Errorf("btov(): %v", err)
 	}
 	return dedup.SCTDedupInfo{Idx: idx, Timestamp: t}, true, nil
 }
@@ -196,21 +196,31 @@ func btoi(b []byte) uint64 {
 
 // vtob concatenates an index and timestamp values into a byte array.
 func vtob(idx uint64, timestamp uint64) ([]byte, error) {
-	i := itob(idx)
-	if len(i) != 8 {
-		return nil, fmt.Errorf("input error, idx should be %d bytes long, got %d", 8, len(i))
+	b := make([]byte, 16)
+	var err error
+
+	b, err = binary.Append(b, binary.BigEndian, idx)
+	if err != nil {
+		return nil, fmt.Errorf("binary.Append() could not encode idx: %v", err)
 	}
-	t := itob(timestamp)
-	if len(t) != 8 {
-		return nil, fmt.Errorf("input error, idx should be %d bytes long, got %d", 8, len(t))
+	b, err = binary.Append(b, binary.BigEndian, timestamp)
+	if err != nil {
+		return nil, fmt.Errorf("binary.Append() could not encode timestamp: %v", err)
 	}
-	return append(itob(idx), itob(timestamp)...), nil
+
+	return b, nil
 }
 
 // btov parses a byte array into an index and timestamp values.
-func btov(b []byte) (idx uint64, timestamp uint64, err error) {
-	if len(b) != 16 {
-		return 0, 0, fmt.Errorf("input error, value should be %d bytes long, got %d", 16, len(b))
+func btov(b []byte) (uint64, uint64, error) {
+	var idx, timestamp uint64
+	n, err := binary.Decode(b, binary.BigEndian, &idx)
+	if err != nil {
+		return 0, 0, fmt.Errorf("binary.Decode() could not decode idx: %v", err)
+	}
+	_, err = binary.Decode(b[n:], binary.BigEndian, &timestamp)
+	if err != nil {
+		return 0, 0, fmt.Errorf("binary.Decode() could not decode timestamp: %v", err)
 	}
-	return btoi(b[0:8]), btoi(b[8:16]), nil
+	return idx, timestamp, nil
 }