Skip to content

Deduplicate SCT timestamps #57

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Nov 22, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion modules/dedup/dedup.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ import (
"k8s.io/klog/v2"
)

// LeafDedupInfo, enables building idempotent deduplicated add-chain responses.
// LeafDedupInfo, enables building deduplicated add-chain responses.
type LeafDedupInfo struct {
LeafID []byte
SCTDedupInfo
Expand Down
4 changes: 2 additions & 2 deletions serialize.go
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,7 @@ func DedupFromBundle(bundle []byte, bundleIdx uint64) ([]dedup.LeafDedupInfo, er
kvs := []dedup.LeafDedupInfo{}
s := cryptobyte.String(bundle)

for len(s) > 0 {
for i := bundleIdx * 256; len(s) > 0; i++ {
var timestamp uint64
var entryType uint16
var extensions, fingerprints cryptobyte.String
Expand Down Expand Up @@ -213,7 +213,7 @@ func DedupFromBundle(bundle []byte, bundleIdx uint64) ([]dedup.LeafDedupInfo, er
return nil, fmt.Errorf("invalid data tile: unknown type %d", entryType)
}
k := sha256.Sum256(crt)
sctDedupInfo := dedup.SCTDedupInfo{Idx: bundleIdx*256 + uint64(len(kvs)), Timestamp: timestamp}
sctDedupInfo := dedup.SCTDedupInfo{Idx: uint64(i), Timestamp: timestamp}
kvs = append(kvs, dedup.LeafDedupInfo{LeafID: k[:], SCTDedupInfo: sctDedupInfo})
}
return kvs, nil
Expand Down
6 changes: 3 additions & 3 deletions storage.go
Original file line number Diff line number Diff line change
Expand Up @@ -131,11 +131,11 @@ func cachedStoreIssuers(s IssuerStorage) func(context.Context, []KV) error {
}
}

// AddCertDedupInfo stores <cert_hash, SCTDEdupInfo> in the deduplication storage.
// AddCertDedupInfo stores <cert_hash, SCTDedupInfo> in the deduplication storage.
func (cts CTStorage) AddCertDedupInfo(ctx context.Context, c *x509.Certificate, sctDedupInfo dedup.SCTDedupInfo) error {
key := sha256.Sum256(c.Raw)
if err := cts.dedupStorage.Add(ctx, []dedup.LeafDedupInfo{{LeafID: key[:], SCTDedupInfo: sctDedupInfo}}); err != nil {
return fmt.Errorf("error storing SCTDedupInfo %+v of %q: %v", sctDedupInfo, hex.EncodeToString(key[:]), err)
return fmt.Errorf("error storing SCTDedupInfo %+v of \"%x\": %v", sctDedupInfo, key, err)
}
return nil
}
Expand All @@ -145,7 +145,7 @@ func (cts CTStorage) GetCertDedupInfo(ctx context.Context, c *x509.Certificate)
key := sha256.Sum256(c.Raw)
sctC, ok, err := cts.dedupStorage.Get(ctx, key[:])
if err != nil {
return dedup.SCTDedupInfo{}, false, fmt.Errorf("error fetching index of %q: %v", hex.EncodeToString(key[:]), err)
return dedup.SCTDedupInfo{}, false, fmt.Errorf("error fetching index of \"%x\": %v", key, err)
}
return sctC, ok, nil
}
40 changes: 25 additions & 15 deletions storage/bbolt/dedup.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
// Package bbolt implements modules/dedup using BBolt.
//
// It contains two buckets:
// - The dedup bucket stores <leafID, idx::timestamp> pairs. Entries can either be added after
// - The dedup bucket stores <leafID, {idx, timestamp}> pairs. Entries can either be added after
// sequencing, by the server that received the request, or later when synchronising the dedup
// storage with the log state.
// - The size bucket has a single entry: <"size", X>, where X is the largest contiguous index
Expand Down Expand Up @@ -48,7 +48,7 @@ type Storage struct {

// NewStorage returns a new BBolt storage instance with a dedup and size bucket.
//
// The dedup bucket stores <leafID, idx::timestamp> pairs, where idx::timestamp is the
// The dedup bucket stores <leafID, {idx, timestamp}> pairs, where idx::timestamp is the
// concatenation of two uint64 8 bytes BigEndian representation.
// The size bucket has a single entry: <"size", X>, where X is the largest contiguous index from 0
// that has been inserted in the dedup bucket.
Expand Down Expand Up @@ -113,7 +113,7 @@ func (s *Storage) Add(_ context.Context, ldis []dedup.LeafDedupInfo) error {
size := btoi(sizeB)
vB, err := vtob(ldi.Idx, ldi.Timestamp)
if err != nil {
return fmt.Errorf("btov(): %v", err)
return fmt.Errorf("vtob(): %v", err)
}

if old := db.Get(ldi.LeafID); len(old) == 16 && btoi(old[:8]) <= ldi.Idx {
Expand Down Expand Up @@ -158,7 +158,7 @@ func (s *Storage) Get(_ context.Context, leafID []byte) (dedup.SCTDedupInfo, boo
}
idx, t, err := btov(v)
if err != nil {
return dedup.SCTDedupInfo{}, false, fmt.Errorf("vtob(): %v", err)
return dedup.SCTDedupInfo{}, false, fmt.Errorf("btov(): %v", err)
}
return dedup.SCTDedupInfo{Idx: idx, Timestamp: t}, true, nil
}
Expand Down Expand Up @@ -196,21 +196,31 @@ func btoi(b []byte) uint64 {

// vtob concatenates an index and timestamp values into a byte array.
func vtob(idx uint64, timestamp uint64) ([]byte, error) {
i := itob(idx)
if len(i) != 8 {
return nil, fmt.Errorf("input error, idx should be %d bytes long, got %d", 8, len(i))
b := make([]byte, 16)
var err error

b, err = binary.Append(b, binary.BigEndian, idx)
if err != nil {
return nil, fmt.Errorf("binary.Append() could not encode idx: %v", err)
}
t := itob(timestamp)
if len(t) != 8 {
return nil, fmt.Errorf("input error, idx should be %d bytes long, got %d", 8, len(t))
b, err = binary.Append(b, binary.BigEndian, timestamp)
if err != nil {
return nil, fmt.Errorf("binary.Append() could not encode timestamp: %v", err)
}
return append(itob(idx), itob(timestamp)...), nil

return b, nil
}

// btov parses a byte array into an index and timestamp values.
func btov(b []byte) (idx uint64, timestamp uint64, err error) {
if len(b) != 16 {
return 0, 0, fmt.Errorf("input error, value should be %d bytes long, got %d", 16, len(b))
func btov(b []byte) (uint64, uint64, error) {
var idx, timestamp uint64
n, err := binary.Decode(b, binary.BigEndian, &idx)
if err != nil {
return 0, 0, fmt.Errorf("binary.Decode() could not decode idx: %v", err)
}
_, err = binary.Decode(b[n:], binary.BigEndian, &timestamp)
if err != nil {
return 0, 0, fmt.Errorf("binary.Decode() could not decode timestamp: %v", err)
}
return btoi(b[0:8]), btoi(b[8:16]), nil
return idx, timestamp, nil
}
Loading