Skip to content

Commit a07a779

Browse files
committed
aws copy storage and addapt to s3
1 parent 9d5daa4 commit a07a779

File tree

5 files changed

+227
-1
lines changed

5 files changed

+227
-1
lines changed

go.mod

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@ require (
2424
k8s.io/klog/v2 v2.130.1
2525
)
2626

27+
replace github.com/transparency-dev/trillian-tessera => /usr/local/google/home/phboneff/git/phbnf/trillian-tessera
28+
2729
require (
2830
cel.dev/expr v0.16.1 // indirect
2931
cloud.google.com/go v0.116.0 // indirect
@@ -38,7 +40,25 @@ require (
3840
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.24.1 // indirect
3941
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1 // indirect
4042
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1 // indirect
41-
github.com/aws/aws-sdk-go v1.51.8 // indirect
43+
github.com/aws/aws-sdk-go v1.55.5 // indirect
44+
github.com/aws/aws-sdk-go-v2 v1.32.4 // indirect
45+
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 // indirect
46+
github.com/aws/aws-sdk-go-v2/config v1.28.3 // indirect
47+
github.com/aws/aws-sdk-go-v2/credentials v1.17.44 // indirect
48+
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.19 // indirect
49+
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.23 // indirect
50+
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.23 // indirect
51+
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
52+
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.23 // indirect
53+
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 // indirect
54+
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.4 // indirect
55+
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.4 // indirect
56+
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.4 // indirect
57+
github.com/aws/aws-sdk-go-v2/service/s3 v1.66.3 // indirect
58+
github.com/aws/aws-sdk-go-v2/service/sso v1.24.5 // indirect
59+
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.4 // indirect
60+
github.com/aws/aws-sdk-go-v2/service/sts v1.32.4 // indirect
61+
github.com/aws/smithy-go v1.22.0 // indirect
4262
github.com/beorn7/perks v1.0.1 // indirect
4363
github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect
4464
github.com/cespare/xxhash/v2 v2.3.0 // indirect

go.sum

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -648,6 +648,44 @@ github.com/apache/arrow/go/v11 v11.0.0/go.mod h1:Eg5OsL5H+e299f7u5ssuXsuHQVEGC4x
648648
github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU=
649649
github.com/aws/aws-sdk-go v1.51.8 h1:tD7gQq5XKuKdhA6UMEH26ZNQH0s+HbL95rzv/ACz5TQ=
650650
github.com/aws/aws-sdk-go v1.51.8/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
651+
github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU=
652+
github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
653+
github.com/aws/aws-sdk-go-v2 v1.32.4 h1:S13INUiTxgrPueTmrm5DZ+MiAo99zYzHEFh1UNkOxNE=
654+
github.com/aws/aws-sdk-go-v2 v1.32.4/go.mod h1:2SK5n0a2karNTv5tbP1SjsX0uhttou00v/HpXKM1ZUo=
655+
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 h1:pT3hpW0cOHRJx8Y0DfJUEQuqPild8jRGmSFmBgvydr0=
656+
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6/go.mod h1:j/I2++U0xX+cr44QjHay4Cvxj6FUbnxrgmqN3H1jTZA=
657+
github.com/aws/aws-sdk-go-v2/config v1.28.3 h1:kL5uAptPcPKaJ4q0sDUjUIdueO18Q7JDzl64GpVwdOM=
658+
github.com/aws/aws-sdk-go-v2/config v1.28.3/go.mod h1:SPEn1KA8YbgQnwiJ/OISU4fz7+F6Fe309Jf0QTsRCl4=
659+
github.com/aws/aws-sdk-go-v2/credentials v1.17.44 h1:qqfs5kulLUHUEXlHEZXLJkgGoF3kkUeFUTVA585cFpU=
660+
github.com/aws/aws-sdk-go-v2/credentials v1.17.44/go.mod h1:0Lm2YJ8etJdEdw23s+q/9wTpOeo2HhNE97XcRa7T8MA=
661+
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.19 h1:woXadbf0c7enQ2UGCi8gW/WuKmE0xIzxBF/eD94jMKQ=
662+
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.19/go.mod h1:zminj5ucw7w0r65bP6nhyOd3xL6veAUMc3ElGMoLVb4=
663+
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.23 h1:A2w6m6Tmr+BNXjDsr7M90zkWjsu4JXHwrzPg235STs4=
664+
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.23/go.mod h1:35EVp9wyeANdujZruvHiQUAo9E3vbhnIO1mTCAxMlY0=
665+
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.23 h1:pgYW9FCabt2M25MoHYCfMrVY2ghiiBKYWUVXfwZs+sU=
666+
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.23/go.mod h1:c48kLgzO19wAu3CPkDWC28JbaJ+hfQlsdl7I2+oqIbk=
667+
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ=
668+
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc=
669+
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.23 h1:1SZBDiRzzs3sNhOMVApyWPduWYGAX0imGy06XiBnCAM=
670+
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.23/go.mod h1:i9TkxgbZmHVh2S0La6CAXtnyFhlCX/pJ0JsOvBAS6Mk=
671+
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 h1:TToQNkvGguu209puTojY/ozlqy2d/SFNcoLIqTFi42g=
672+
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0/go.mod h1:0jp+ltwkf+SwG2fm/PKo8t4y8pJSgOCO4D8Lz3k0aHQ=
673+
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.4 h1:aaPpoG15S2qHkWm4KlEyF01zovK1nW4BBbyXuHNSE90=
674+
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.4/go.mod h1:eD9gS2EARTKgGr/W5xwgY/ik9z/zqpW+m/xOQbVxrMk=
675+
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.4 h1:tHxQi/XHPK0ctd/wdOw0t7Xrc2OxcRCnVzv8lwWPu0c=
676+
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.4/go.mod h1:4GQbF1vJzG60poZqWatZlhP31y8PGCCVTvIGPdaaYJ0=
677+
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.4 h1:E5ZAVOmI2apR8ADb72Q63KqwwwdW1XcMeXIlrZ1Psjg=
678+
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.4/go.mod h1:wezzqVUOVVdk+2Z/JzQT4NxAU0NbhRe5W8pIE72jsWI=
679+
github.com/aws/aws-sdk-go-v2/service/s3 v1.66.3 h1:neNOYJl72bHrz9ikAEED4VqWyND/Po0DnEx64RW6YM4=
680+
github.com/aws/aws-sdk-go-v2/service/s3 v1.66.3/go.mod h1:TMhLIyRIyoGVlaEMAt+ITMbwskSTpcGsCPDq91/ihY0=
681+
github.com/aws/aws-sdk-go-v2/service/sso v1.24.5 h1:HJwZwRt2Z2Tdec+m+fPjvdmkq2s9Ra+VR0hjF7V2o40=
682+
github.com/aws/aws-sdk-go-v2/service/sso v1.24.5/go.mod h1:wrMCEwjFPms+V86TCQQeOxQF/If4vT44FGIOFiMC2ck=
683+
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.4 h1:zcx9LiGWZ6i6pjdcoE9oXAB6mUdeyC36Ia/QEiIvYdg=
684+
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.4/go.mod h1:Tp/ly1cTjRLGBBmNccFumbZ8oqpZlpdhFf80SrRh4is=
685+
github.com/aws/aws-sdk-go-v2/service/sts v1.32.4 h1:yDxvkz3/uOKfxnv8YhzOi9m+2OGIxF+on3KOISbK5IU=
686+
github.com/aws/aws-sdk-go-v2/service/sts v1.32.4/go.mod h1:9XEUty5v5UAsMiFOBJrNibZgwCeOma73jgGwwhgffa8=
687+
github.com/aws/smithy-go v1.22.0 h1:uunKnWlcoL3zO7q+gG2Pk53joueEOsnNB28QdMsmiMM=
688+
github.com/aws/smithy-go v1.22.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
651689
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
652690
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
653691
github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=

storage/aws/client.go

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
// Copyright 2024 The Tessera authors. All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package aws
16+
17+
import (
18+
"context"
19+
"fmt"
20+
"io"
21+
22+
"github.com/aws/aws-sdk-go-v2/config"
23+
"github.com/aws/aws-sdk-go-v2/service/s3"
24+
"github.com/aws/aws-sdk-go/aws"
25+
)
26+
27+
// GetFetcher returns an S3 read function for objects in a given bucket.
28+
func GetFetcher(ctx context.Context, projectID string, bucket string) (func(ctx context.Context, path string) ([]byte, error), error) {
29+
// TODO(phboneff): this should probably move somewhere else
30+
sdkConfig, err := config.LoadDefaultConfig(ctx)
31+
if err != nil {
32+
return nil, fmt.Errorf("failed to load default AWS configuration: %v", err)
33+
}
34+
c := s3.NewFromConfig(sdkConfig)
35+
36+
return func(ctx context.Context, path string) ([]byte, error) {
37+
r, err := c.GetObject(ctx, &s3.GetObjectInput{
38+
Bucket: aws.String(bucket),
39+
Key: aws.String(path),
40+
})
41+
42+
if err != nil {
43+
return nil, fmt.Errorf("getObject: failed to create reader for object %q in bucket %q: %w", path, bucket, err)
44+
}
45+
46+
d, err := io.ReadAll(r.Body)
47+
if err != nil {
48+
return nil, fmt.Errorf("failed to read %q: %v", path, err)
49+
}
50+
return d, r.Body.Close()
51+
}, nil
52+
}

storage/aws/doc.go

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
// Copyright 2024 The Tessera authors. All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
/*
16+
Package aws allows the SCTFE to interact with AWS to:
17+
- store issuers
18+
- read log entries
19+
*/
20+
package aws

storage/aws/issuers.go

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
// Copyright 2024 The Tessera authors. All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package aws
16+
17+
import (
18+
"bytes"
19+
"context"
20+
"errors"
21+
"fmt"
22+
"path"
23+
24+
"github.com/aws/aws-sdk-go-v2/config"
25+
"github.com/aws/aws-sdk-go-v2/service/s3"
26+
"github.com/aws/aws-sdk-go/aws"
27+
"github.com/aws/smithy-go"
28+
sctfe "github.com/transparency-dev/static-ct"
29+
"k8s.io/klog/v2"
30+
)
31+
32+
// IssuersStorage is a key value store backed by S3 on AWS to store issuer chains.
33+
type IssuersStorage struct {
34+
s3Client *s3.Client
35+
bucket string
36+
prefix string
37+
contentType string
38+
}
39+
40+
// NewIssuerStorage creates a new IssuerStorage.
41+
//
42+
// The specified bucket must exist or an error will be returned.
43+
func NewIssuerStorage(ctx context.Context, projectID string, bucket string, prefix string, contentType string) (*IssuersStorage, error) {
44+
// TODO(phboneff): this should probably move somewhere else
45+
// TODO(phboneff): withjsonreads?
46+
sdkConfig, err := config.LoadDefaultConfig(ctx)
47+
if err != nil {
48+
return nil, fmt.Errorf("failed to load default AWS configuration: %v", err)
49+
}
50+
c := s3.NewFromConfig(sdkConfig)
51+
if err != nil {
52+
return nil, fmt.Errorf("failed to create GCS client: %v", err)
53+
}
54+
55+
r := &IssuersStorage{
56+
s3Client: c,
57+
bucket: bucket,
58+
prefix: prefix,
59+
contentType: contentType,
60+
}
61+
62+
return r, nil
63+
}
64+
65+
// keyToObjName converts bytes to a GCS object name.
66+
func (s *IssuersStorage) keyToObjName(key []byte) string {
67+
return path.Join(s.prefix, string(key))
68+
}
69+
70+
// AddIssuers stores Issuers values under their Key if there isn't an object under Key already.
71+
func (s *IssuersStorage) AddIssuersIfNotExist(ctx context.Context, kv []sctfe.KV) error {
72+
// We first try and see if this issuer cert has already been stored since reads
73+
// are cheaper than writes.
74+
// TODO(phboneff): add parallel operations
75+
for _, kv := range kv {
76+
objName := s.keyToObjName(kv.K)
77+
put := &s3.PutObjectInput{
78+
Bucket: aws.String(s.bucket),
79+
Key: aws.String(objName),
80+
Body: bytes.NewReader(kv.V),
81+
ContentType: aws.String(s.contentType),
82+
IfNoneMatch: aws.String("*"),
83+
}
84+
85+
if _, err := s.s3Client.PutObject(ctx, put); err != nil {
86+
var apiErr smithy.APIError
87+
if errors.As(err, &apiErr); apiErr.ErrorCode() == "PreconditionFailed" {
88+
klog.V(2).Infof("AddIssuersIfNotExist: object %q already exists in bucket %q, continuing", objName, s.bucket)
89+
return nil
90+
}
91+
return fmt.Errorf("failed to write object %q to bucket %q: %w", objName, s.bucket, err)
92+
}
93+
klog.V(2).Infof("AddIssuersIfNotExist: added %q in bucket %q", objName, s.bucket)
94+
}
95+
return nil
96+
}

0 commit comments

Comments
 (0)