Skip to content

Commit

Permalink
Merge pull request #102 from covalenthq/extractor_one_file
Browse files Browse the repository at this point in the history 
script to convert avro encoded replicas into specimen and result jsons
  • Loading branch information
@noslav
noslav authored Aug 22, 2022
2 parents 1eacba2 + 45a4aa3 commit 905f2d7
Show file tree
Hide file tree
Showing 15 changed files with 1,034 additions and 502 deletions.
3 changes: 3 additions & 0 deletions .golangci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@ run:
modules-download-mode: readonly
skip-dirs:
- internal/metrics
- bin/
- docs/
- data/

linters:
enable:
Expand Down
5 changes: 5 additions & 0 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,3 +38,8 @@ run-agent-elrond:
--consumer-timeout=8000 \
--ipfs-service=web3.storage

test:
@echo "---- Testing Agent from cmd/bspagent ----"
@go test ./... -coverprofile=coverage.out
@echo "---- Done Testing for cmd/bspagent ----"

8 changes: 8 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -309,3 +309,11 @@ go run extractor.go \
```

Please make sure that the --binary-file-path and --avro-codec-path matches the ones given while running the agent above. --indent-json (0,1,2) can be used to pretty print and inspect the AVRO json objects.

#### extractor2.go

similar to extractor.go, but outputs the specimen, result and block_replica in separate json files.

```bash
go run extractor2.go --binary-file-path="/Users/sudeep/repos/bsp-agent/data/block-ethereum/" --codec-path "../../codec/block-ethereum.avsc" --indent-json 0 --end-block-number 15185258 --start-block-number 15185258 --chain-id "1"
```
117 changes: 97 additions & 20 deletions codec/block-ethereum.avsc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,11 @@
},
{
"name":"TotalDifficulty",
"type":"double"
"type":{
"type":"bytes",
"logicalType":"decimal",
"precision":1000
}
},
{
"name":"Header",
Expand Down Expand Up @@ -76,11 +80,19 @@
},
{
"name":"difficulty",
"type":"double"
"type":{
"type":"bytes",
"logicalType":"decimal",
"precision":1000
}
},
{
"name":"number",
"type":"int"
"type":{
"type":"bytes",
"logicalType":"decimal",
"precision":1000
}
},
{
"name":"gasLimit",
Expand Down Expand Up @@ -111,7 +123,11 @@
},
{
"name":"baseFeePerGas",
"type":"double"
"type":{
"type":"bytes",
"logicalType":"decimal",
"precision":1000
}
}
]
}
Expand Down Expand Up @@ -153,31 +169,51 @@
},
{
"name":"chainId",
"type":"long"
"type":{
"type":"bytes",
"logicalType":"decimal",
"precision":1000
}
},
{
"name":"nonce",
"type":"long"
},
{
"name":"gasPrice",
"type":"long"
"type":{
"type":"bytes",
"logicalType":"decimal",
"precision":1000
}
},
{
"name":"gasTipCap",
"type":"long"
"type":{
"type":"bytes",
"logicalType":"decimal",
"precision":1000
}
},
{
"name":"gasFeeCap",
"type":"long"
"type":{
"type":"bytes",
"logicalType":"decimal",
"precision":1000
}
},
{
"name":"gas",
"type":"long"
},
{
"name":"from",
"type":"string"
"type":[
"null",
"string"
],
"default":"null"
},
{
"name":"to",
Expand All @@ -187,9 +223,49 @@
],
"default":"null"
},
{
"name":"v",
"type":[
"null",
{
"type":"bytes",
"logicalType":"decimal",
"precision":1000
}
],
"default":"null"
},
{
"name":"r",
"type":[
"null",
{
"type":"bytes",
"logicalType":"decimal",
"precision":1000
}
],
"default":"null"
},
{
"name":"s",
"type":[
"null",
{
"type":"bytes",
"logicalType":"decimal",
"precision":1000
}
],
"default":"null"
},
{
"name":"value",
"type":"double"
"type":{
"type":"bytes",
"logicalType":"decimal",
"precision":1000
}
},
{
"name":"input",
Expand All @@ -201,14 +277,11 @@
},
{
"name":"uncles",
"type":[
"null",
{
"type":"array",
"items":"Header"
}
],
"default":null
"type":{
"type":"array",
"items":"Header"
},
"default": []
},
{
"name":"Receipts",
Expand Down Expand Up @@ -324,7 +397,11 @@
},
{
"name":"Balance",
"type":"double"
"type":{
"type":"bytes",
"logicalType":"decimal",
"precision":1000
}
},
{
"name":"CodeHash",
Expand Down Expand Up @@ -423,7 +500,7 @@
{
"name":"codecVersion",
"type":"double",
"default":0.2
"default":0.3
}
]
}
30 changes: 19 additions & 11 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,29 +20,37 @@ require (
github.com/satori/go.uuid v1.2.0
github.com/sirupsen/logrus v1.8.1
github.com/ubiq/go-ubiq v3.0.1+incompatible
golang.org/x/sys v0.0.0-20220624220833-87e55d714810
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8
google.golang.org/api v0.91.0
gopkg.in/avro.v0 v0.0.0-20171217001914-a730b5802183
)

require github.com/covalenthq/ipfs-pinner v0.1.7

require (
github.com/VictoriaMetrics/fastcache v1.10.0 // indirect
github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 // indirect
github.com/apache/arrow/go/v7 v7.0.0 // indirect
github.com/benbjohnson/immutable v0.3.0 // indirect
github.com/deepmap/oapi-codegen v1.8.2 // indirect
github.com/goccy/go-json v0.7.10 // indirect
github.com/edsrzf/mmap-go v1.1.0 // indirect
github.com/goccy/go-json v0.9.7 // indirect
github.com/gofrs/uuid v3.3.0+incompatible // indirect
github.com/golang-jwt/jwt/v4 v4.4.1 // indirect
github.com/google/flatbuffers v2.0.5+incompatible // indirect
github.com/googleapis/enterprise-certificate-proxy v0.1.0 // indirect
github.com/influxdata/flux v0.161.0 // indirect
github.com/influxdata/influxql v1.1.1-0.20211004132434-7e7d61973256 // indirect
github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 // indirect
github.com/rogpeppe/go-internal v1.8.1 // indirect
github.com/rs/cors v1.8.2 // indirect
github.com/sergi/go-diff v1.1.0 // indirect
github.com/stretchr/testify v1.8.0 // indirect
github.com/uber/jaeger-client-go v2.28.0+incompatible // indirect
github.com/uber/jaeger-lib v2.4.1+incompatible // indirect
github.com/xlab/treeprint v1.0.0 // indirect
golang.org/x/exp v0.0.0-20220713135740-79cabaa25d75 // indirect
golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
)

Expand All @@ -63,7 +71,7 @@ require (
github.com/cenkalti/backoff/v4 v4.1.1 // indirect
github.com/ceramicnetwork/go-dag-jose v0.1.0 // indirect
github.com/cespare/xxhash v1.1.0 // indirect
github.com/cespare/xxhash/v2 v2.1.1 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/cheekybits/genny v1.0.0 // indirect
github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327 // indirect
github.com/coreos/go-systemd/v22 v22.3.2 // indirect
Expand All @@ -79,12 +87,12 @@ require (
github.com/dustin/go-humanize v1.0.0 // indirect
github.com/elastic/gosigar v0.12.0 // indirect
github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 // indirect
github.com/fatih/color v1.13.0 // indirect
github.com/fatih/color v1.13.0
github.com/flynn/noise v1.0.0 // indirect
github.com/francoispqt/gojay v1.2.13 // indirect
github.com/fsnotify/fsnotify v1.5.4 // indirect
github.com/go-ole/go-ole v1.2.1 // indirect
github.com/go-stack/stack v1.8.0 // indirect
github.com/go-stack/stack v1.8.1 // indirect
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
github.com/godbus/dbus/v5 v5.0.4 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
Expand Down Expand Up @@ -227,7 +235,7 @@ require (
github.com/multiformats/go-multihash v0.1.0 // indirect
github.com/multiformats/go-multistream v0.2.2 // indirect
github.com/multiformats/go-varint v0.0.6 // indirect
github.com/nxadm/tail v1.4.8 // indirect
github.com/nxadm/tail v1.4.9-0.20211216163028-4472660a31a6 // indirect
github.com/onsi/ginkgo v1.16.5 // indirect
github.com/opencontainers/runtime-spec v1.0.2 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
Expand Down Expand Up @@ -275,17 +283,17 @@ require (
go.uber.org/multierr v1.7.0 // indirect
go.uber.org/zap v1.21.0 // indirect
go4.org v0.0.0-20200411211856-f5505b9728dd // indirect
golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29 // indirect
golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57 // indirect
golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e // indirect
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect
golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect
golang.org/x/net v0.0.0-20220630215102-69896b714898 // indirect
golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2 // indirect
golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/tools v0.1.9 // indirect
golang.org/x/tools v0.1.10 // indirect
golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f // indirect
google.golang.org/grpc v1.47.0 // indirect
google.golang.org/grpc v1.48.0 // indirect
google.golang.org/protobuf v1.28.0 // indirect
gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
gopkg.in/square/go-jose.v2 v2.5.1 // indirect
Expand Down
Loading

0 comments on commit 905f2d7

Please sign in to comment.