merged with main

Author: NikolasHaimerl

.github/workflows/ci.yml

@@ -1,16 +1,19 @@
 name: CI
-on: [push]
+on:
+  push:
+    branches: [main]
+  pull_request:
 
 env:
   ZINNIA_VERSION: v0.20.2
 jobs:
-  build:
+  test-linux:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - run: curl -L https://github.com/filecoin-station/zinnia/releases/download/${{ env.ZINNIA_VERSION }}/zinnia-linux-x64.tar.gz | tar -xz
-      - uses: actions/setup-node@v4
-      - run: npx standard
+      - run:
+          curl -L https://github.com/filecoin-station/zinnia/releases/download/${{
+          env.ZINNIA_VERSION }}/zinnia-linux-x64.tar.gz | tar -xz
       - run: ./zinnia run test.js
 
   test-windows:
@@ -25,3 +28,11 @@ jobs:
           extract: true
           token: ${{ secrets.GITHUB_TOKEN }}
       - run: ./zinnia.exe run test.js
+
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+      - run: npm ci
+      - run: npm run lint

.github/workflows/publish.yml

@@ -8,7 +8,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - run: curl -L ${{ github.event.release.tarball_url }} > source.tar.gz
-      - uses: filecoin-station/publish-zinnia-module-action@2e0211ca6021228bca6c33b0aa55a73260518f68
+      - uses: filecoin-station/publish-zinnia-module-action@v0.2.0
         id: publish
         with:
           source: source.tar.gz
@@ -37,7 +37,7 @@ jobs:
             }
       - uses: slackapi/slack-github-action@v1.27.0
         with:
-          channel-id: spark-public
+          channel-id: filecoin-slack-spark
           payload: |
             {
               "text": "SPARK checker version ${{ github.event.release.tag_name }} released",

.prettierignore

@@ -0,0 +1 @@
+/vendor

.prettierrc.yaml

@@ -0,0 +1,5 @@
+semi: false
+singleQuote: true
+printWidth: 100
+trailingComma: all
+proseWrap: always

README.md

@@ -1,4 +1,5 @@
 # spark
+
 SP Retrieval Checker Module
 
 - [Roadmap](https://pl-strflt.notion.site/SPARK-Roadmap-ac729c11c49b409fbec54751d1bc6c8a)
@@ -9,8 +10,12 @@ SP Retrieval Checker Module
 Install [Zinnia CLI](https://github.com/filecoin-station/zinnia).
 
 ```bash
+$ # Install dev tooling
+$ npm ci
 $ # Lint
-$ npx standard
+$ npm run lint
+$ # Fix linting issues
+$ npm run lint:fix
 $ # Run module
 $ zinnia run main.js
 $ # Test module
@@ -29,5 +34,4 @@ $ ./release.sh 1.0.0
 
 Use GitHub's changelog feature to fill out the release notes.
 
-Publish the new release and let the CI/CD workflow upload the sources
-to IPFS & IPNS.
+Publish the new release and let the CI/CD workflow upload the sources to IPFS & IPNS.

deps.ts

@@ -3,15 +3,15 @@
 // Run the following script after making change in this file:
 //   deno bundle deps.ts vendor/deno-deps.js
 //
+// You must use a 1.x version of Deno, e.g. v1.43.1
 
 export { encodeHex } from 'https://deno.land/std@0.203.0/encoding/hex.ts'
 export { decodeBase64 } from 'https://deno.land/std@0.203.0/encoding/base64.ts'
 export { decode as decodeVarint } from 'https://deno.land/x/varint@v2.0.0/varint.ts'
-export { retry } from 'https://deno.land/std@0.203.0/async/retry.ts';
-
+export { retry } from 'https://deno.land/std@0.203.0/async/retry.ts'
 
 // Deno Bundle does not support npm dependencies, we have to load them via CDN
-export { ethers } from "https://cdn.jsdelivr.net/npm/ethers@6.13.5/dist/ethers.min.js";
+export { ethers } from 'https://cdn.jsdelivr.net/npm/ethers@6.13.5/dist/ethers.min.js'
 export {
   getIndexProviderPeerId,
   MINER_TO_PEERID_CONTRACT_ADDRESS,
@@ -21,12 +21,16 @@ export { CarBlockIterator } from 'https://cdn.skypack.dev/@ipld/car@5.3.2/?dts'
 export {
   UnsupportedHashError,
   HashMismatchError,
-  validateBlock
+  validateBlock,
 } from 'https://cdn.skypack.dev/@web3-storage/car-block-validator@1.2.0/?dts'
 // cdn.skypack.dev cannot resolve import from @noble/hashes
 // jsdelivr.net seems to work better, it's also recommended by drand-client
 export {
   fetchBeaconByTime,
   HttpChainClient,
-  HttpCachingChain
+  HttpCachingChain,
 } from 'https://cdn.jsdelivr.net/npm/drand-client@1.2.6/index.js/+esm'
+
+export { assertOkResponse } from 'https://cdn.skypack.dev/assert-ok-response@1.0.0/?dts'
+import pRetry from 'https://cdn.skypack.dev/p-retry@6.2.1/?dts'
+export { pRetry }

eslint.config.js

@@ -0,0 +1,7 @@
+import neostandard from 'neostandard'
+
+export default neostandard({
+  noStyle: true, // Disable style-related rules, we use Prettier
+  ts: true,
+  ignores: ['vendor/**', 'deps.ts'],
+})

lib/activity-state.js

@@ -4,18 +4,18 @@
 export class ActivityState {
   #healthy = null
 
-  onOutdatedClient () {
+  onOutdatedClient() {
     this.onError('SPARK is outdated. Please upgrade Filecoin Station to the latest version.')
   }
 
-  onError (msg) {
+  onError(msg) {
     if (this.#healthy === null || this.#healthy) {
       this.#healthy = false
       Zinnia.activity.error(msg ?? 'SPARK failed reporting retrieval')
     }
   }
 
-  onHealthy () {
+  onHealthy() {
     if (this.#healthy === null) {
       this.#healthy = true
       Zinnia.activity.info('SPARK started reporting retrievals')

lib/constants.js

@@ -1,6 +1,8 @@
-export const SPARK_VERSION = '1.17.2'
+export const SPARK_VERSION = '1.19.1'
 export const MAX_CAR_SIZE = 200 * 1024 * 1024 // 200 MB
 export const APPROX_ROUND_LENGTH_IN_MS = 20 * 60_000 // 20 minutes
+export const MAX_JITTER_BETWEEN_TASKS_IN_MS = 10_000 // 10 seconds
 export const RPC_URL = 'https://api.node.glif.io/'
 export const RPC_AUTH = 'KZLIUb9ejreYOm-mZFM3UNADE0ux6CrHjxnS2D2Qgb8='
 export const MINER_TO_PEERID_CONTRACT_ADDRESS = '0x14183aD016Ddc83D638425D6328009aa390339Ce' // Contract address on the Filecoin EVM
+export const MAX_REQUEST_DURATION_MS = 90_000

lib/drand-client.js

@@ -1,8 +1,4 @@
-import {
-  fetchBeaconByTime,
-  HttpChainClient,
-  HttpCachingChain
-} from '../vendor/deno-deps.js'
+import { fetchBeaconByTime, HttpChainClient, HttpCachingChain } from '../vendor/deno-deps.js'
 
 // See https://docs.filecoin.io/networks/mainnet#genesis
 const FIL_MAINNET_GENESIS_TS = new Date('2020-08-24T22:00:00Z').getTime()
@@ -18,8 +14,9 @@ const DRAND_OPTIONS = {
   chainVerificationParams: {
     // quicknet
     chainHash: '52db9ba70e0cc0f6eaf7803dd07447a1f5477735fd3f661792ba94600c84e971',
-    publicKey: '83cf0f2896adee7eb8b5f01fcad3912212c437e0073e911fb90022d3e760183c8c4b450b6a0a6c3ac6a5776a2d1064510d1fec758c921cc22b0e17e63aaf4bcb5ed66304de9cf809bd274ca73bab4af5a6e9c76a4bc09e76eae8991ef5ece45a'
-  }
+    publicKey:
+      '83cf0f2896adee7eb8b5f01fcad3912212c437e0073e911fb90022d3e760183c8c4b450b6a0a6c3ac6a5776a2d1064510d1fec758c921cc22b0e17e63aaf4bcb5ed66304de9cf809bd274ca73bab4af5a6e9c76a4bc09e76eae8991ef5ece45a',
+  },
 }
 
 const DRAND_URL = `https://api2.drand.sh/${DRAND_OPTIONS.chainVerificationParams.chainHash}`
@@ -29,7 +26,7 @@ const client = new HttpChainClient(chain, DRAND_OPTIONS)
 /**
  * @param {number} roundStartEpoch
  */
-export async function getRandomnessForSparkRound (roundStartEpoch) {
+export async function getRandomnessForSparkRound(roundStartEpoch) {
   const roundStartedAt = roundStartEpoch * FIL_MAINNET_BLOCK_TIME + FIL_MAINNET_GENESIS_TS
   const beacon = await fetchBeaconByTime(client, roundStartedAt)
   return beacon.randomness

lib/http-assertions.js

@@ -1,33 +1,19 @@
 import { AssertionError } from 'zinnia:assert'
 
-/**
- * @param {Response} res
- * @param {string} [errorMsg]
- */
-export async function assertOkResponse (res, errorMsg) {
-  if (res.ok) return
-
-  let body
-  try {
-    body = await res.text()
-  } catch {}
-  const err = new Error(`${errorMsg ?? 'Fetch failed'} (${res.status}): ${body?.trimEnd()}`)
-  err.statusCode = res.status
-  err.serverMessage = body
-  throw err
-}
+export { assertOkResponse } from '../vendor/deno-deps.js'
 
 /**
  * @param {Response} res
  * @param {string} [errorMsg]
  */
-export async function assertRedirectResponse (res, errorMsg) {
+export async function assertRedirectResponse(res, errorMsg) {
   if ([301, 302, 303, 304, 307, 308].includes(res.status)) {
     const location = res.headers.get('location')
     if (!location) {
-      const msg = (errorMsg ? errorMsg + ' ' : '') +
+      const msg =
+        (errorMsg ? errorMsg + ' ' : '') +
         'The server response is missing the Location header. Headers found:\n' +
-         Array.from(res.headers.keys()).join('\n')
+        Array.from(res.headers.keys()).join('\n')
       throw new AssertionError(msg)
     }
     return
@@ -37,7 +23,9 @@ export async function assertRedirectResponse (res, errorMsg) {
   try {
     body = await res.text()
   } catch {}
-  const err = new Error(`${errorMsg ?? 'Server did not respond with redirect'} (${res.status}): ${body?.trimEnd()}`)
+  const err = new Error(
+    `${errorMsg ?? 'Server did not respond with redirect'} (${res.status}): ${body?.trimEnd()}`,
+  )
   err.statusCode = res.status
   err.serverMessage = body
   throw err

lib/ipni-client.js

@@ -1,4 +1,4 @@
-import { decodeBase64, decodeVarint } from '../vendor/deno-deps.js'
+import { decodeBase64, decodeVarint, pRetry, assertOkResponse } from '../vendor/deno-deps.js'
 
 /**
  *
@@ -9,23 +9,25 @@ import { decodeBase64, decodeVarint } from '../vendor/deno-deps.js'
  *  provider?: { address: string; protocol: string };
  * }>}
  */
-export async function queryTheIndex (cid, providerId) {
-  const url = `https://cid.contact/cid/${encodeURIComponent(cid)}`
-
+export async function queryTheIndex(cid, providerId) {
   let providerResults
   try {
-    const res = await fetch(url)
-    if (!res.ok) {
-      console.error('IPNI query failed, HTTP response: %s %s', res.status, (await res.text()).trimEnd())
-      return { indexerResult: `ERROR_${res.status}` }
-    }
-
-    const result = await res.json()
-    providerResults = result.MultihashResults.flatMap(r => r.ProviderResults)
-    console.log('IPNI returned %s provider results: %o', providerResults.length, providerResults)
+    providerResults = await pRetry(() => getRetrievalProviders(cid), {
+      retries: 5,
+      shouldRetry: (error) => {
+        return error.statusCode && error.statusCode >= 500
+      },
+      onFailedAttempt: (error) => {
+        console.error(error)
+        console.error('IPNI query failed, retrying...')
+      },
+    })
+    console.log('IPNI returned %s provider results', providerResults.length)
   } catch (err) {
     console.error('IPNI query failed.', err)
-    return { indexerResult: 'ERROR_FETCH' }
+    return {
+      indexerResult: typeof err.statusCode === 'number' ? `ERROR_${err.statusCode}` : 'ERROR_FETCH',
+    }
   }
 
   let graphsyncProvider
@@ -37,7 +39,7 @@ export async function queryTheIndex (cid, providerId) {
       0x900: 'bitswap',
       0x910: 'graphsync',
       0x0920: 'http',
-      4128768: 'graphsync'
+      4128768: 'graphsync',
     }[protocolCode]
 
     const address = p.Provider.Addrs[0]
@@ -47,14 +49,14 @@ export async function queryTheIndex (cid, providerId) {
       case 'http':
         return {
           indexerResult: 'OK',
-          provider: { address, protocol }
+          provider: { address, protocol },
         }
 
       case 'graphsync':
         if (!graphsyncProvider) {
           graphsyncProvider = {
             address: `${address}/p2p/${p.Provider.ID}`,
-            protocol
+            protocol,
           }
         }
     }
@@ -63,10 +65,19 @@ export async function queryTheIndex (cid, providerId) {
     console.log('HTTP protocol is not advertised, falling back to Graphsync.')
     return {
       indexerResult: 'HTTP_NOT_ADVERTISED',
-      provider: graphsyncProvider
+      provider: graphsyncProvider,
     }
   }
 
   console.log('All advertisements are from other miners or for unsupported protocols.')
   return { indexerResult: 'NO_VALID_ADVERTISEMENT' }
 }
+
+async function getRetrievalProviders(cid) {
+  const url = `https://cid.contact/cid/${encodeURIComponent(cid)}`
+  const res = await fetch(url)
+  await assertOkResponse(res)
+
+  const result = await res.json()
+  return result.MultihashResults.flatMap((r) => r.ProviderResults)
+}