Evaluate alternative provider measurement

lib/preprocess.js

@@ -17,6 +17,7 @@ export class Measurement {
   constructor (m, pointerize = (v) => v) {
     this.participantAddress = pointerize(parseParticipantAddress(m.participant_address))
     this.retrievalResult = pointerize(getRetrievalResult(m))
+    this.networkRetrievalResult = pointerize(getNetworkRetrievalResult(m))
     this.cid = pointerize(m.cid)
     this.minerId = pointerize(m.miner_id)
     // Note: providerId is recorded by spark-publish but we don't use it for evaluations yet
@@ -41,6 +42,13 @@ export class Measurement {
     this.stationId = pointerize(m.station_id)
     this.carChecksum = pointerize(m.car_checksum)
     this.carTooLarge = m.car_too_large
+    this.networkRetrieval = {
+      statusCode: m.network_retrieval_status_code,
+      timeout: m.network_retrieval_timeout,
+      carTooLarge: m.network_retrieval_car_too_large,
+      endAt: m.network_retrieval_end_at,
+      protocol: m.network_retrieval_protocol
+    }
   }
 }
 
@@ -319,3 +327,31 @@ export const getRetrievalResult = (measurement) => {
 
   return ok ? 'OK' : 'UNKNOWN_ERROR'
 }
+
+/**
+ * Evaluates the network retrieval result.
+ *
+ * Network retrieval results are evaluated only if the indexer result is `NO_VALID_ADVERTISEMENT`
+ * and the network retrieval status code is set.
+ *
+ * @param {Partial<import('./typings.js').RawMeasurement>} measurement
+ * @return {import('./typings.js').RetrievalResult}
+ */
+export const getNetworkRetrievalResult = (measurement) => {
+  if (measurement.indexer_result !== 'NO_VALID_ADVERTISEMENT' || !measurement.network_retrieval_status_code) {
+    return getRetrievalResult(measurement)
+  }
+
+  /** @type {Partial<import('./typings.js').RawMeasurement>} */
+  const networkMeasurement = {
+    indexer_result: 'OK',
+    status_code: measurement.network_retrieval_status_code,
+    timeout: measurement.network_retrieval_timeout,
+    network_retrieval_status_code: measurement.network_retrieval_status_code,
+    car_too_large: measurement.network_retrieval_car_too_large,
+    end_at: measurement.network_retrieval_end_at,
+    protocol: measurement.network_retrieval_protocol
+  }
+
+  return getRetrievalResult(networkMeasurement)
+}

lib/public-stats.js

@@ -30,6 +30,7 @@ export const updatePublicStats = async ({ createPgClient, committees, allMeasure
     await updateRetrievalTimings(pgClient, committees)
     await updateDailyClientRetrievalStats(pgClient, committees, findDealClients)
     await updateDailyAllocatorRetrievalStats(pgClient, committees, findDealAllocators)
+    await updateDailyNetworkRetrievalStats(pgClient, committees)
   } finally {
     await pgClient.end()
   }
@@ -407,3 +408,34 @@ function buildPerPartyStats (committees, perDealParty, partyName) {
   )
   return flatStats
 }
+
+/**
+ * @param {pg.Client} pgClient
+ * @param {object} stats
+ * @param {number} stats.total
+ * @param {number} stats.successful
+ */
+export const updateDailyNetworkRetrievalStats = async (pgClient, committees) => {
+  let total = 0
+  let successful = 0
+  for (const c of committees) {
+    for (const m of c.measurements) {
+      if (m.networkRetrievalResult === 'OK') successful++
+      total++
+    }
+  }
+
+  debug('Updating public daily network wide retrieval stats: total += %s successful += %s', total, successful)
+  await pgClient.query(`
+    INSERT INTO daily_network_retrieval_stats
+      (day, total, successful)
+    VALUES
+      (now(), $1, $2)
+    ON CONFLICT(day) DO UPDATE SET
+      total = daily_network_retrieval_stats.total + $1,
+      successful = daily_network_retrieval_stats.successful + $2
+  `, [
+    total,
+    successful
+  ])
+}

lib/retrieval-stats.js

@@ -60,6 +60,7 @@ export const buildRetrievalStats = (measurements, telemetryPoint) => {
   const sizeValues = []
   let httpSuccesses = 0
   let indexerServerErrorCount = 0
+  let networkRetrievalSuccesses = 0
 
   for (const m of measurements) {
     // `retrievalResult` should be always set by lib/preprocess.js, so we should never encounter
@@ -106,6 +107,8 @@ export const buildRetrievalStats = (measurements, telemetryPoint) => {
     // A successful HTTP response is a response with result breakdown set to OK and the protocol being used is set to HTTP.
     if (m.retrievalResult === 'OK' && m.protocol === 'http') { httpSuccesses++ }
 
+    if (m.networkRetrievalResult === 'OK') { networkRetrievalSuccesses++ }
+
     if (m.retrievalResult.match(/^IPNI_ERROR_5\d\d$/)) {
       indexerServerErrorCount++
     }
@@ -114,6 +117,7 @@ export const buildRetrievalStats = (measurements, telemetryPoint) => {
   const totalForRSR = totalCount - indexerServerErrorCount
   const successRate = totalForRSR ? resultBreakdown.OK / totalForRSR : 0
   const successRateHttp = totalForRSR ? httpSuccesses / totalForRSR : 0
+  const networkRSR = totalForRSR ? networkRetrievalSuccesses / totalForRSR : 0
   telemetryPoint.intField('total_for_success_rates', totalForRSR)
   telemetryPoint.intField('unique_tasks', uniqueTasksCount)
   telemetryPoint.floatField('success_rate', successRate)
@@ -122,6 +126,7 @@ export const buildRetrievalStats = (measurements, telemetryPoint) => {
   telemetryPoint.intField('inet_groups', inetGroups.size)
   telemetryPoint.intField('measurements', totalCount)
   telemetryPoint.intField('download_bandwidth', downloadBandwidth)
+  telemetryPoint.floatField('network_success_rate', networkRSR)
 
   addHistogramToPoint(telemetryPoint, ttfbValues, 'ttfb_')
   addHistogramToPoint(telemetryPoint, durationValues, 'duration_')

lib/typings.d.ts

@@ -100,6 +100,11 @@ export interface RawMeasurement {
     | 'NO_VALID_ADVERTISEMENT'
     | 'ERROR_FETCH'
     | `ERROR_${number}`;
+  network_retrieval_status_code: number | undefined | null;
+  network_retrieval_timeout: boolean;
+  network_retrieval_car_too_large: boolean;
+  network_retrieval_end_at: string;
+  network_retrieval_protocol: string;
 }
 
 export type CreatePgClient = () => Promise<import('pg').Client>;

migrations/026.do.add-daily-network-retrieval-stats-table.sql

@@ -0,0 +1,5 @@
+CREATE TABLE daily_network_retrieval_stats (
+  day DATE NOT NULL PRIMARY KEY,
+  total INT NOT NULL,
+  successful INT NOT NULL
+);

test/helpers/test-data.js

@@ -43,7 +43,15 @@ export const VALID_MEASUREMENT = {
   retrievalResult: 'OK',
   indexerResult: 'OK',
   taskingEvaluation: null,
-  consensusEvaluation: null
+  consensusEvaluation: null,
+  networkRetrievalResult: 'OK',
+  networkRetrieval: {
+    statusCode: null,
+    timeout: false,
+    carTooLarge: false,
+    endAt: null,
+    protocol: 'http'
+  }
 }
 
 // Fraud detection is mutating the measurements parsed from JSON

test/preprocess.js

@@ -4,7 +4,8 @@ import {
   preprocess,
   Measurement,
   parseMeasurements,
-  assertValidMeasurement
+  assertValidMeasurement,
+  getNetworkRetrievalResult
 } from '../lib/preprocess.js'
 import { Point } from '../lib/telemetry.js'
 import assert from 'node:assert'
@@ -416,3 +417,88 @@ describe('assertValidMeasurement', () => {
     assert.throws(() => assertValidMeasurement(measurement), /first_byte_at must be greater than or equal to start_at/)
   })
 })
+
+describe('getNetworkRetrievalResult', () => {
+  /** @type {Partial<import('../lib/typings.js').RawMeasurement>} */
+  const SUCCESSFUL_RETRIEVAL = {
+    spark_version: '1.5.2',
+    participant_address: 'f410fgkhpcrbmdvic52o3nivftrjxr7nzw47updmuzra',
+    station_id: VALID_STATION_ID,
+    finished_at: '2023-11-01T09:42:03.246Z',
+    timeout: false,
+    start_at: '2023-11-01T09:40:03.393Z',
+    status_code: 200,
+    first_byte_at: '1970-01-01T00:00:00.000Z',
+    end_at: '1970-01-01T00:00:00.000Z',
+    byte_length: 1234,
+    inet_group: 'ue49TX_JdYjI',
+    cid: 'bafkreihstuf2qcu3hs64ersidh46cjtilxcoipmzgu3pifwzmkqdjpraqq',
+    miner_id: 'f1abc',
+    provider_address: '/ip4/108.89.91.150/tcp/46717/p2p/12D3KooWSsaFCtzDJUEhLQYDdwoFtdCMqqfk562UMvccFz12kYxU',
+    provider_id: 'PROVIDERID',
+    protocol: 'http',
+    indexer_result: 'OK'
+  }
+
+  it('successful retrieval', () => {
+    const result = getNetworkRetrievalResult({
+      ...SUCCESSFUL_RETRIEVAL
+    })
+    assert.strictEqual(result, 'OK')
+  })
+
+  it('TIMEOUT - no network retrieval measurements', () => {
+    const result = getNetworkRetrievalResult({
+      ...SUCCESSFUL_RETRIEVAL,
+      timeout: true
+    })
+    assert.strictEqual(result, 'TIMEOUT')
+  })
+
+  it('TIMEOUT - successful network retrieval measurements', () => {
+    const result = getNetworkRetrievalResult({
+      ...SUCCESSFUL_RETRIEVAL,
+      timeout: true,
+      network_retrieval_car_too_large: false,
+      network_retrieval_timeout: false,
+      network_retrieval_status_code: 200,
+      network_retrieval_end_at: '2023-11-01T09:42:03.246Z',
+      network_retrieval_protocol: 'http'
+    })
+    assert.strictEqual(result, 'TIMEOUT')
+  })
+
+  it('NO_VALID_ADVERTISEMENT - no network retrieval measurements', () => {
+    const result = getNetworkRetrievalResult({
+      ...SUCCESSFUL_RETRIEVAL,
+      indexer_result: 'NO_VALID_ADVERTISEMENT'
+    })
+    assert.strictEqual(result, 'IPNI_NO_VALID_ADVERTISEMENT')
+  })
+
+  it('NO_VALID_ADVERTISEMENT - successful network retrieval measurements', () => {
+    const result = getNetworkRetrievalResult({
+      ...SUCCESSFUL_RETRIEVAL,
+      indexer_result: 'NO_VALID_ADVERTISEMENT',
+      network_retrieval_car_too_large: false,
+      network_retrieval_timeout: false,
+      network_retrieval_status_code: 200,
+      network_retrieval_end_at: '2023-11-01T09:42:03.246Z',
+      network_retrieval_protocol: 'http'
+    })
+    assert.strictEqual(result, 'OK')
+  })
+
+  it('NO_VALID_ADVERTISEMENT - TIMEOUT network retrieval measurements', () => {
+    const result = getNetworkRetrievalResult({
+      ...SUCCESSFUL_RETRIEVAL,
+      indexer_result: 'NO_VALID_ADVERTISEMENT',
+      network_retrieval_car_too_large: false,
+      network_retrieval_timeout: true,
+      network_retrieval_status_code: 500,
+      network_retrieval_end_at: '2023-11-01T09:42:03.246Z',
+      network_retrieval_protocol: 'http'
+    })
+    assert.strictEqual(result, 'TIMEOUT')
+  })
+})

test/public-stats.test.js

@@ -4,7 +4,7 @@ import pg from 'pg'
 import { DATABASE_URL } from '../lib/config.js'
 import { migrateWithPgClient } from '../lib/migrate.js'
 import { buildEvaluatedCommitteesFromMeasurements, VALID_MEASUREMENT } from './helpers/test-data.js'
-import { updateDailyAllocatorRetrievalStats, updateDailyClientRetrievalStats, updatePublicStats } from '../lib/public-stats.js'
+import { updateDailyAllocatorRetrievalStats, updateDailyClientRetrievalStats, updateDailyNetworkRetrievalStats, updatePublicStats } from '../lib/public-stats.js'
 import { beforeEach } from 'mocha'
 import { groupMeasurementsToCommittees } from '../lib/committee.js'
 
@@ -31,6 +31,7 @@ describe('public-stats', () => {
     await pgClient.query('DELETE FROM retrieval_timings')
     await pgClient.query('DELETE FROM daily_client_retrieval_stats')
     await pgClient.query('DELETE FROM daily_allocator_retrieval_stats')
+    await pgClient.query('DELETE FROM daily_network_retrieval_stats')
 
     // Run all tests inside a transaction to ensure `now()` always returns the same value
     // See https://dba.stackexchange.com/a/63549/125312
@@ -724,7 +725,7 @@ describe('public-stats', () => {
         findDealClients
       )
       const { rows } = await pgClient.query(
-          `SELECT
+        `SELECT
                day::TEXT,
                client_id,
                total,
@@ -770,7 +771,7 @@ describe('public-stats', () => {
         findDealClients
       )
       const { rows } = await pgClient.query(
-          `SELECT
+        `SELECT
                day::TEXT,
                client_id,
                total,
@@ -965,7 +966,7 @@ describe('public-stats', () => {
         findDealAllocators
       )
       const { rows } = await pgClient.query(
-            `SELECT
+        `SELECT
                  day::TEXT,
                  allocator_id,
                  total,
@@ -1011,7 +1012,7 @@ describe('public-stats', () => {
         findDealAllocators
       )
       const { rows } = await pgClient.query(
-            `SELECT
+        `SELECT
                  day::TEXT,
                  allocator_id,
                  total,
@@ -1168,6 +1169,40 @@ describe('public-stats', () => {
         { day: today, allocator_id: 'f0allocator', total: 4, successful: 3 }
       ])
     })
+
+    describe('updateDailyNetworkRetrievalStats ', () => {
+      it('creates or updates the row for today', async () => {
+        /** @type {Measurement[]} */
+        const allMeasurements = [
+          { ...VALID_MEASUREMENT, networkRetrievalResult: 'OK' },
+          { ...VALID_MEASUREMENT, networkRetrievalResult: 'OK' },
+
+          { ...VALID_MEASUREMENT, networkRetrievalResult: 'TIMEOUT' },
+          { ...VALID_MEASUREMENT, networkRetrievalResult: 'CAR_TOO_LARGE' },
+          { ...VALID_MEASUREMENT, networkRetrievalResult: 'IPNI_ERROR_FETCH' }
+        ]
+
+        const committees = buildEvaluatedCommitteesFromMeasurements(allMeasurements)
+        const { rows: created } = await pgClient.query(
+          'SELECT * FROM daily_network_retrieval_stats'
+        )
+        assert.deepStrictEqual(created, [])
+        await updateDailyNetworkRetrievalStats(
+          pgClient,
+          committees
+        )
+        const { rows } = await pgClient.query(
+          `SELECT
+               day::TEXT,
+               total,
+               successful
+            FROM daily_network_retrieval_stats`)
+
+        assert.deepStrictEqual(rows, [
+          { day: today, total: 5, successful: 2 }
+        ])
+      })
+    })
   })
 
   const getCurrentDate = async () => {

test/retrieval-stats.test.js

@@ -324,6 +324,26 @@ describe('retrieval statistics', () => {
     // Only one of the successful measurements used http
     assertPointFieldValue(point, 'success_rate_http', '0.25')
   })
+
+  it('records network retrieval success rate', async () => {
+    /** @type {Measurement[]} */
+    const measurements = [
+      {
+        ...VALID_MEASUREMENT,
+        networkRetrievalResult: 'OK'
+      },
+      {
+        ...VALID_MEASUREMENT,
+        networkRetrievalResult: 'HTTP_500'
+      }
+    ]
+
+    const point = new Point('stats')
+    buildRetrievalStats(measurements, point)
+    debug('stats', point.fields)
+
+    assertPointFieldValue(point, 'network_success_rate', '0.5')
+  })
 })
 
 describe('getValueAtPercentile', () => {