diff --git a/package.json b/package.json index 1e0f8cb..1c7549f 100644 --- a/package.json +++ b/package.json @@ -19,8 +19,10 @@ "@koa/cors": "^3.1.0", "@noble/ed25519": "^1.3.0", "@noble/hashes": "^0.4.1", + "@smithy/node-http-handler": "^2.2.2", "@subspace/reed-solomon-erasure.wasm": "^0.2.5", "borsh": "^0.6.0", + "cacheable-lookup": "6", "csv-stringify": "^6.2.1", "debug": "^4.3.2", "koa": "^2.13.4", diff --git a/scripts/load-raw-near-lake.js b/scripts/load-raw-near-lake.js index 9ebdec6..519f579 100644 --- a/scripts/load-raw-near-lake.js +++ b/scripts/load-raw-near-lake.js @@ -5,7 +5,19 @@ const { } = require('@aws-sdk/client-s3'); const fs = require('fs/promises'); -const { parse } = require('path'); + +// Setup keep-alive agents for AWS +const { NodeHttpHandler } = require('@smithy/node-http-handler'); +const { Agent: HttpAgent } = require('http'); +const { Agent: HttpsAgent } = require('https'); +const httpAgent = new HttpAgent({ keepAlive: true }); +const httpsAgent = new HttpsAgent({ keepAlive: true }); + +// Avoid DNS lookups for every request +const CacheableLookup = require('cacheable-lookup'); +const cacheable = new CacheableLookup(); +cacheable.install(httpAgent); +cacheable.install(httpsAgent); function normalizeBlockHeight(number) { return number.toString().padStart(12, '0'); @@ -50,6 +62,10 @@ async function* chunkStream(stream, chunkSize) { async function sync(bucketName, startAfter, limit = 1000) { const client = new S3Client({ region: 'eu-central-1', + requestHandler: new NodeHttpHandler({ + httpAgent, + httpsAgent, + }), }); const dstDir = `./lake-data/${bucketName}`; diff --git a/yarn.lock b/yarn.lock index 4d04df6..b89e99c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1156,6 +1156,56 @@ dependencies: "@sinonjs/commons" "^1.7.0" +"@smithy/abort-controller@^2.0.16": + version "2.0.16" + resolved "https://registry.yarnpkg.com/@smithy/abort-controller/-/abort-controller-2.0.16.tgz#31a86748e0c55a97ead1d179040160c6fc55ba1b" + integrity sha512-4foO7738k8kM9flMHu3VLabqu7nPgvIj8TB909S0CnKx0YZz/dcDH3pZ/4JHdatfxlZdKF1JWOYCw9+v3HVVsw== + dependencies: + "@smithy/types" "^2.8.0" + tslib "^2.5.0" + +"@smithy/node-http-handler@^2.2.2": + version "2.2.2" + resolved "https://registry.yarnpkg.com/@smithy/node-http-handler/-/node-http-handler-2.2.2.tgz#f9f8cd49f270bc50a0de8a4587bbdaae1c7c4e80" + integrity sha512-XO58TO/Eul/IBQKFKaaBtXJi0ItEQQCT+NI4IiKHCY/4KtqaUT6y/wC1EvDqlA9cP7Dyjdj7FdPs4DyynH3u7g== + dependencies: + "@smithy/abort-controller" "^2.0.16" + "@smithy/protocol-http" "^3.0.12" + "@smithy/querystring-builder" "^2.0.16" + "@smithy/types" "^2.8.0" + tslib "^2.5.0" + +"@smithy/protocol-http@^3.0.12": + version "3.0.12" + resolved "https://registry.yarnpkg.com/@smithy/protocol-http/-/protocol-http-3.0.12.tgz#9f606efd191593f6dbde58fa822465b92b8afbca" + integrity sha512-Xz4iaqLiaBfbQpB9Hgi3VcZYbP7xRDXYhd8XWChh4v94uw7qwmvlxdU5yxzfm6ACJM66phHrTbS5TVvj5uQ72w== + dependencies: + "@smithy/types" "^2.8.0" + tslib "^2.5.0" + +"@smithy/querystring-builder@^2.0.16": + version "2.0.16" + resolved "https://registry.yarnpkg.com/@smithy/querystring-builder/-/querystring-builder-2.0.16.tgz#1a9a02b1fb938688cdab5e585cb7c62c8054bc41" + integrity sha512-Q/GsJT0C0mijXMRs7YhZLLCP5FcuC4797lYjKQkME5CZohnLC4bEhylAd2QcD3gbMKNjCw8+T2I27WKiV/wToA== + dependencies: + "@smithy/types" "^2.8.0" + "@smithy/util-uri-escape" "^2.0.0" + tslib "^2.5.0" + +"@smithy/types@^2.8.0": + version "2.8.0" + resolved "https://registry.yarnpkg.com/@smithy/types/-/types-2.8.0.tgz#bdbaa0a54c9c3538d6c763c6f32d3e4f76fe0df9" + integrity sha512-h9sz24cFgt/W1Re22OlhQKmUZkNh244ApgRsUDYinqF8R+QgcsBIX344u2j61TPshsTz3CvL6HYU1DnQdsSrHA== + dependencies: + tslib "^2.5.0" + +"@smithy/util-uri-escape@^2.0.0": + version "2.0.0" + resolved "https://registry.yarnpkg.com/@smithy/util-uri-escape/-/util-uri-escape-2.0.0.tgz#19955b1a0f517a87ae77ac729e0e411963dfda95" + integrity sha512-ebkxsqinSdEooQduuk9CbKcI+wheijxEb3utGXkCoYQkJnwTnLbH1JXGimJtUkQwNQbsbuYwG2+aFVyZf5TLaw== + dependencies: + tslib "^2.5.0" + "@subspace/reed-solomon-erasure.wasm@^0.2.5": version "0.2.5" resolved "https://registry.npmjs.org/@subspace/reed-solomon-erasure.wasm/-/reed-solomon-erasure.wasm-0.2.5.tgz" @@ -1506,6 +1556,11 @@ cache-content-type@^1.0.0: mime-types "^2.1.18" ylru "^1.2.0" +cacheable-lookup@6: + version "6.1.0" + resolved "https://registry.yarnpkg.com/cacheable-lookup/-/cacheable-lookup-6.1.0.tgz#0330a543471c61faa4e9035db583aad753b36385" + integrity sha512-KJ/Dmo1lDDhmW2XDPMo+9oiy/CeqosPguPCrgcVzKyZrL6pM1gU2GmPY/xo6OQPTUaA/c0kwHuywB4E6nmT9ww== + cacheable-request@^6.0.0: version "6.1.0" resolved "https://registry.yarnpkg.com/cacheable-request/-/cacheable-request-6.1.0.tgz#20ffb8bd162ba4be11e9567d823db651052ca912" @@ -4038,6 +4093,11 @@ tslib@^2.3.1: resolved "https://registry.npmjs.org/tslib/-/tslib-2.4.0.tgz" integrity sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ== +tslib@^2.5.0: + version "2.6.2" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.2.tgz#703ac29425e7b37cd6fd456e92404d46d1f3e4ae" + integrity sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q== + tsscmp@1.0.6: version "1.0.6" resolved "https://registry.npmjs.org/tsscmp/-/tsscmp-1.0.6.tgz"