Skip to content

Commit b98391d

Browse files
phbnfphbnf
committed
dedupeverywhere
# Conflicts: # go.mod # go.sum # Conflicts: # go.sum # Conflicts: # go.sum
1 parent 5de64b4 commit b98391d

File tree

19 files changed

+149
-602
lines changed

19 files changed

+149
-602
lines changed

‎cmd/aws/main.go

Lines changed: 43 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,9 @@ import (
3131
tesseract "github.com/transparency-dev/static-ct"
3232
"github.com/transparency-dev/static-ct/storage"
3333
"github.com/transparency-dev/static-ct/storage/aws"
34-
"github.com/transparency-dev/static-ct/storage/bbolt"
3534
tessera "github.com/transparency-dev/trillian-tessera"
3635
taws "github.com/transparency-dev/trillian-tessera/storage/aws"
36+
aws_as "github.com/transparency-dev/trillian-tessera/storage/aws/antispam"
3737
"golang.org/x/mod/sumdb/note"
3838
"k8s.io/klog/v2"
3939
)
@@ -54,6 +54,7 @@ var (
5454
origin = flag.String("origin", "", "Origin of the log, for checkpoints and the monitoring prefix.")
5555
bucket = flag.String("bucket", "", "Name of the bucket to store the log in.")
5656
dbName = flag.String("db_name", "", "AuroraDB name")
57+
antispamDBName = flag.String("antispam_db_name", "", "AuroraDB antispam name")
5758
dbHost = flag.String("db_host", "", "AuroraDB host")
5859
dbPort = flag.Int("db_port", 3306, "AuroraDB port")
5960
dbUser = flag.String("db_user", "", "AuroraDB user")
@@ -147,9 +148,19 @@ func newAWSStorage(ctx context.Context, signer note.Signer) (*storage.CTStorage,
147148
if err != nil {
148149
return nil, fmt.Errorf("failed to initialize AWS Tessera storage driver: %v", err)
149150
}
151+
152+
var antispam tessera.Antispam
153+
if *antispamDBName != "" {
154+
antispam, err = aws_as.NewAntispam(ctx, antispamMysqlConfig().FormatDSN(), aws_as.AntispamOpts{})
155+
if err != nil {
156+
klog.Exitf("Failed to create new GCP antispam storage: %v", err)
157+
}
158+
}
159+
150160
appender, _, _, err := tessera.NewAppender(ctx, driver, tessera.NewAppendOptions().
151161
WithCheckpointSigner(signer).
152-
WithCTLayout())
162+
WithCTLayout().
163+
WithAntispam(2<<18, antispam)) // TODO(phbnf): do the math to see what fits in memory
153164
if err != nil {
154165
return nil, fmt.Errorf("failed to initialize AWS Tessera storage: %v", err)
155166
}
@@ -159,12 +170,7 @@ func newAWSStorage(ctx context.Context, signer note.Signer) (*storage.CTStorage,
159170
return nil, fmt.Errorf("failed to initialize AWS issuer storage: %v", err)
160171
}
161172

162-
beDedupStorage, err := bbolt.NewStorage(*dedupPath)
163-
if err != nil {
164-
return nil, fmt.Errorf("failed to initialize BBolt deduplication database: %v", err)
165-
}
166-
167-
return storage.NewCTStorage(appender, issuerStorage, beDedupStorage)
173+
return storage.NewCTStorage(appender, issuerStorage)
168174
}
169175

170176
type timestampFlag struct {
@@ -230,3 +236,32 @@ func storageConfigFromFlags() taws.Config {
230236
MaxIdleConns: *dbMaxIdle,
231237
}
232238
}
239+
240+
func antispamMysqlConfig() *mysql.Config {
241+
if *antispamDBName == "" {
242+
klog.Exit("--antispam_db_name must be set")
243+
}
244+
if *dbHost == "" {
245+
klog.Exit("--db_host must be set")
246+
}
247+
if *dbPort == 0 {
248+
klog.Exit("--db_port must be set")
249+
}
250+
if *dbUser == "" {
251+
klog.Exit("--db_user must be set")
252+
}
253+
// Empty passord isn't an option with AuroraDB MySQL.
254+
if *dbPassword == "" {
255+
klog.Exit("--db_password must be set")
256+
}
257+
258+
return &mysql.Config{
259+
User: *dbUser,
260+
Passwd: *dbPassword,
261+
Net: "tcp",
262+
Addr: fmt.Sprintf("%s:%d", *dbHost, *dbPort),
263+
DBName: *antispamDBName,
264+
AllowCleartextPasswords: true,
265+
AllowNativePasswords: true,
266+
}
267+
}

‎cmd/gcp/main.go

Lines changed: 3 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,6 @@ var (
5454
origin = flag.String("origin", "", "Origin of the log, for checkpoints and the monitoring prefix.")
5555
bucket = flag.String("bucket", "", "Name of the bucket to store the log in.")
5656
spannerDB = flag.String("spanner_db_path", "", "Spanner database path: projects/{projectId}/instances/{instanceId}/databases/{databaseId}.")
57-
spannerDedupDB = flag.String("spanner_dedup_db_path", "", "Spanner deduplication database path: projects/{projectId}/instances/{instanceId}/databases/{databaseId}.")
5857
spannerAntispamDB = flag.String("spanner_antispam_db_path", "", "EXPERIMENTAL: Spanner antispam deduplication database path projects/{projectId}/instances/{instanceId}/databases/{databaseId}.")
5958
rootsPemFile = flag.String("roots_pem_file", "", "Path to the file containing root certificates that are acceptable to the log. The certs are served through get-roots endpoint.")
6059
rejectExpired = flag.Bool("reject_expired", false, "If true then the certificate validity period will be checked against the current time during the validation of submissions. This will cause expired certificates to be rejected.")
@@ -162,11 +161,7 @@ func newGCPStorage(ctx context.Context, signer note.Signer) (*storage.CTStorage,
162161
var antispam tessera.Antispam
163162
// Persistent antispam is currently experimental, so there's no terraform or documentation yet!
164163
if *spannerAntispamDB != "" {
165-
asOpts := gcp_as.AntispamOpts{
166-
MaxBatchSize: 5000,
167-
PushbackThreshold: 1024,
168-
}
169-
antispam, err = gcp_as.NewAntispam(ctx, *spannerAntispamDB, asOpts)
164+
antispam, err = gcp_as.NewAntispam(ctx, *spannerAntispamDB, gcp_as.AntispamOpts{})
170165
if err != nil {
171166
klog.Exitf("Failed to create new GCP antispam storage: %v", err)
172167
}
@@ -175,7 +170,7 @@ func newGCPStorage(ctx context.Context, signer note.Signer) (*storage.CTStorage,
175170
opts := tessera.NewAppendOptions().
176171
WithCheckpointSigner(signer).
177172
WithCTLayout().
178-
WithAntispam(256, antispam)
173+
WithAntispam(2<<18, antispam) // TODO(phbnf): do the math to see what fits in memory
179174

180175
// TODO(phbnf): figure out the best way to thread the `shutdown` func NewAppends returns back out to main so we can cleanly close Tessera down
181176
// when it's time to exit.
@@ -189,12 +184,7 @@ func newGCPStorage(ctx context.Context, signer note.Signer) (*storage.CTStorage,
189184
return nil, fmt.Errorf("failed to initialize GCP issuer storage: %v", err)
190185
}
191186

192-
beDedupStorage, err := gcp.NewDedupeStorage(ctx, *spannerDedupDB)
193-
if err != nil {
194-
return nil, fmt.Errorf("failed to initialize GCP Spanner deduplication database: %v", err)
195-
}
196-
197-
return storage.NewCTStorage(appender, issuerStorage, beDedupStorage)
187+
return storage.NewCTStorage(appender, issuerStorage)
198188
}
199189

200190
type timestampFlag struct {

‎deployment/live/aws/test/.terraform.lock.hcl

Lines changed: 23 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎deployment/live/aws/test/README.md

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -66,8 +66,10 @@ export AWS_PROFILE=AdministratorAccess-<REDACTED>
6666

6767
Terraforming the account can be done by:
6868
1. `cd` to [/deployment/live/aws/test/](/deployment/live/aws/test/) to deploy/change.
69-
2. Run `terragrunt apply`.
70-
69+
2. Run `terragrunt apply`. If this fails to create the antispam database,
70+
connect the RDS instance to your VM using the instrunctions bellow, and run
71+
`terragrunt apply` again.
72+
7173
Store the Aurora RDS database and S3 bucket information into the environment variables:
7274

7375
```sh
@@ -95,7 +97,7 @@ go run ./cmd/aws \
9597
--db_port=3306 \
9698
--db_user=tesseract \
9799
--db_password=${TESSERACT_DB_PASSWORD} \
98-
--dedup_path=test-static-ct
100+
--antispam_db_name=antispam_db
99101
```
100102

101103
In a different terminal you can either mint and submit certificates manually, or
@@ -187,7 +189,7 @@ go run ./cmd/aws \
187189
--db_port=3306 \
188190
--db_user=tesseract \
189191
--db_password=${TESSERACT_DB_PASSWORD} \
190-
--dedup_path=test-static-ct
192+
--antispam_db_name=antispam_db
191193
-v=3
192194
```
193195

‎deployment/modules/aws/storage/main.tf

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,10 @@ terraform {
44
source = "hashicorp/aws"
55
version = "5.92.0"
66
}
7+
mysql = {
8+
source = "petoju/mysql"
9+
version = "3.0.71"
10+
}
711
}
812
}
913

@@ -50,3 +54,20 @@ data "aws_secretsmanager_secret_version" "db_credentials" {
5054
aws_rds_cluster_instance.cluster_instances
5155
]
5256
}
57+
58+
# Configure the MySQL provider based on the outcome of
59+
# creating the aws_db_instance.
60+
# This requires that the machine running terraform has access
61+
# to the DB instance created above. This is _NOT_ the case when
62+
# github actions are applying the terraform.
63+
provider "mysql" {
64+
endpoint = aws_rds_cluster_instance.cluster_instances[0].endpoint
65+
username = aws_rds_cluster.log_rds_cluster.master_username
66+
password = jsondecode(data.aws_secretsmanager_secret_version.db_credentials.secret_string)["password"]
67+
}
68+
69+
# Create a second database for antispam.
70+
resource "mysql_database" "antispam_db" {
71+
name = "antispam_db"
72+
count = var.create_antispam_db ? 1 : 0
73+
}

‎deployment/modules/aws/storage/outputs.tf

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,3 +33,7 @@ output "rds_aurora_cluster_master_user_secret_unsafe" {
3333
value = jsondecode(data.aws_secretsmanager_secret_version.db_credentials.secret_string)["password"]
3434
sensitive = true # Mark as sensitive, but it can still be exposed
3535
}
36+
37+
output "antispam_database_name" {
38+
value = mysql_database.antispam_db[0].name
39+
}

‎deployment/modules/aws/storage/variables.tf

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,12 @@ variable "region" {
1313
type = string
1414
}
1515

16+
variable "create_antispam_db" {
17+
description = "Set to true to create another database to be used by the antispam implementation."
18+
type = bool
19+
default = false
20+
}
21+
1622
variable "ephemeral" {
1723
description = "Set to true if this is a throwaway/temporary log instance. Will set attributes on created resources to allow them to be disabled/deleted more easily."
1824
type = bool

‎deployment/modules/aws/tesseract/conformance/main.tf

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,11 @@ locals {
1616
module "storage" {
1717
source = "../../storage"
1818

19-
prefix_name = var.prefix_name
20-
base_name = var.base_name
21-
region = var.region
22-
ephemeral = var.ephemeral
19+
prefix_name = var.prefix_name
20+
base_name = var.base_name
21+
region = var.region
22+
ephemeral = var.ephemeral
23+
create_antispam_db = true
2324
}
2425

2526
module "secretsmanager" {
@@ -171,6 +172,7 @@ resource "aws_ecs_task_definition" "conformance" {
171172
"--dedup_path=ci-static-ct",
172173
"--signer_public_key_secret_name=${module.secretsmanager.ecdsa_p256_public_key_id}",
173174
"--signer_private_key_secret_name=${module.secretsmanager.ecdsa_p256_private_key_id}",
175+
"--antispam_db_name=${module.storage.antispam_database_name}",
174176
"-v=2"
175177
],
176178
"logConfiguration" : {

‎deployment/modules/aws/tesseract/test/main.tf

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,11 @@ terraform {
55
module "storage" {
66
source = "../../storage"
77

8-
prefix_name = var.prefix_name
9-
base_name = var.base_name
10-
region = var.region
11-
ephemeral = var.ephemeral
8+
prefix_name = var.prefix_name
9+
base_name = var.base_name
10+
region = var.region
11+
ephemeral = var.ephemeral
12+
create_antispam_db = true
1213
}
1314

1415
module "secretsmanager" {

‎go.mod

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,9 @@ require (
6767
github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 // indirect
6868
github.com/cespare/xxhash/v2 v2.3.0 // indirect
6969
github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42 // indirect
70+
github.com/dgraph-io/badger/v4 v4.7.0 // indirect
71+
github.com/dgraph-io/ristretto/v2 v2.2.0 // indirect
72+
github.com/dustin/go-humanize v1.0.1 // indirect
7073
github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect
7174
github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
7275
github.com/felixge/httpsnoop v1.0.4 // indirect
@@ -76,11 +79,13 @@ require (
7679
github.com/go-logr/logr v1.4.2 // indirect
7780
github.com/go-logr/stdr v1.2.2 // indirect
7881
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
82+
github.com/google/flatbuffers v25.2.10+incompatible // indirect
7983
github.com/google/s2a-go v0.1.9 // indirect
8084
github.com/google/uuid v1.6.0 // indirect
8185
github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect
8286
github.com/googleapis/gax-go/v2 v2.14.1 // indirect
8387
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
88+
github.com/klauspost/compress v1.18.0 // indirect
8489
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
8590
github.com/mattn/go-runewidth v0.0.16 // indirect
8691
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect

‎go.sum

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -724,8 +724,14 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
724724
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
725725
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
726726
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
727+
github.com/dgraph-io/badger/v4 v4.7.0 h1:Q+J8HApYAY7UMpL8d9owqiB+odzEc0zn/aqOD9jhc6Y=
728+
github.com/dgraph-io/badger/v4 v4.7.0/go.mod h1:He7TzG3YBy3j4f5baj5B7Zl2XyfNe5bl4Udl0aPemVA=
729+
github.com/dgraph-io/ristretto/v2 v2.2.0 h1:bkY3XzJcXoMuELV8F+vS8kzNgicwQFAaGINAEJdWGOM=
730+
github.com/dgraph-io/ristretto/v2 v2.2.0/go.mod h1:RZrm63UmcBAaYWC1DotLYBmTvgkrs0+XhBd7Npn7/zI=
727731
github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
728732
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
733+
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
734+
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
729735
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
730736
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
731737
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
@@ -831,6 +837,8 @@ github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW
831837
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
832838
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
833839
github.com/google/flatbuffers v2.0.8+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
840+
github.com/google/flatbuffers v25.2.10+incompatible h1:F3vclr7C3HpB1k9mxCGRMXq6FdUalZ6H/pNX4FP1v0Q=
841+
github.com/google/flatbuffers v25.2.10+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
834842
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
835843
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
836844
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -920,6 +928,8 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:C
920928
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
921929
github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
922930
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
931+
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
932+
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
923933
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
924934
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
925935
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=

‎internal/ct/ctlog.go

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ import (
99
"fmt"
1010

1111
"github.com/transparency-dev/static-ct/internal/types/rfc6962"
12-
"github.com/transparency-dev/static-ct/modules/dedup"
1312
"github.com/transparency-dev/static-ct/storage"
1413
tessera "github.com/transparency-dev/trillian-tessera"
1514
"github.com/transparency-dev/trillian-tessera/ctonly"
@@ -39,10 +38,6 @@ type Storage interface {
3938
Add(context.Context, *ctonly.Entry) tessera.IndexFuture
4039
// AddIssuerChain stores every the chain certificate in a content-addressable store under their sha256 hash.
4140
AddIssuerChain(context.Context, []*x509.Certificate) error
42-
// AddCertDedupInfo stores the SCTDedupInfo of certificate in a log under its hash.
43-
AddCertDedupInfo(context.Context, *x509.Certificate, dedup.SCTDedupInfo) error
44-
// GetCertDedupInfo gets the SCTDedupInfo of certificate in a log from its hash.
45-
GetCertDedupInfo(context.Context, *x509.Certificate) (dedup.SCTDedupInfo, bool, error)
4641
}
4742

4843
// ChainValidator provides functions to validate incoming chains.

0 commit comments

Comments
 (0)