Skip to content

Add alternative provider retrieval measurement #571

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 20 commits into
base: main
Choose a base branch
from
Open

Add alternative provider retrieval measurement #571

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
7eecd81
Add support for network wide check
pyropy Apr 9, 2025
dd80cd0
Simplify name for network wide measurements
pyropy Apr 9, 2025
11af4fc
Extend network measurement with more columns
pyropy Apr 9, 2025
eae83a1
Rename network retrieval table
pyropy Apr 9, 2025
87d330e
Edit measurement test data
pyropy Apr 9, 2025
2b52cd4
Fix failing test
pyropy Apr 9, 2025
91f1a19
Convert network measurement to object inside the measurement
pyropy Apr 10, 2025
4357648
Fix failing test
pyropy Apr 10, 2025
b2bcb2b
Rename network check to alternative provider check
pyropy Apr 15, 2025
0ede95e
Fix failing tests
pyropy Apr 15, 2025
60ec48b
Merge branch 'main' into add/network-wide-retrieval-measurement
pyropy Apr 15, 2025
94201d1
Merge branch 'main' into add/network-wide-retrieval-measurement
pyropy Apr 28, 2025
bba43e4
Add alt provider id
pyropy Apr 28, 2025
3ea8dae
Fix insert query
pyropy Apr 28, 2025
428a4f6
Merge branch 'main' into add/network-wide-retrieval-measurement
pyropy Apr 28, 2025
6f74b20
Merge branch 'main' into add/network-wide-retrieval-measurement
pyropy Apr 28, 2025
85f497e
Publish new measurement fields
pyropy Apr 28, 2025
43bcbc8
Merge branch 'add/network-wide-retrieval-measurement' of github.com:f…
pyropy Apr 28, 2025
2bba71c
Group alternative provider check fields into single object
pyropy Apr 29, 2025
083bf60
Change alternativeProviderCheck validation message
pyropy Apr 29, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 29 additions & 3 deletions api/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,15 @@ const createMeasurement = async (req, res, client) => {
validate(measurement, 'stationId', { type: 'string', required: true })
assert(measurement.stationId.match(/^[0-9a-fA-F]{88}$/), 400, 'Invalid Station ID')

if (measurement.networkRetrieval) {
validate(measurement, 'networkRetrieval', { type: 'object', required: false })
validate(measurement.networkRetrieval, 'statusCode', { type: 'number', required: false })
validate(measurement.networkRetrieval, 'timeout', { type: 'boolean', required: false })
validate(measurement.networkRetrieval, 'carTooLarge', { type: 'boolean', required: false })
validate(measurement.networkRetrieval, 'endAt', { type: 'date', required: false })
validate(measurement.networkRetrieval, 'protocol', { type: 'string', required: false })
}

const inetGroup = await mapRequestToInetGroup(client, req)
logNetworkInfo(req.headers, measurement.stationId, recordNetworkInfoTelemetry)

Expand All @@ -124,10 +133,15 @@ const createMeasurement = async (req, res, client) => {
indexer_result,
miner_id,
provider_id,
network_retrieval_status_code,
network_retrieval_timeout,
network_retrieval_car_too_large,
network_retrieval_end_at,
network_retrieval_protocol,
completed_at_round
)
SELECT
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21,
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26,
id as completed_at_round
FROM spark_rounds
ORDER BY id DESC
Expand All @@ -154,7 +168,12 @@ const createMeasurement = async (req, res, client) => {
measurement.carChecksum,
measurement.indexerResult,
measurement.minerId,
measurement.providerId
measurement.providerId,
measurement.networkRetrieval?.statusCode,
measurement.networkRetrieval?.timeout,
measurement.networkRetrieval?.carTooLarge ?? false,
measurement.networkRetrieval?.endAt,
measurement.networkRetrieval?.protocol
])
json(res, { id: rows[0].id })
}
Expand Down Expand Up @@ -190,7 +209,14 @@ const getMeasurement = async (req, res, client, measurementId) => {
endAt: resultRow.end_at,
byteLength: resultRow.byte_length,
carTooLarge: resultRow.car_too_large,
attestation: resultRow.attestation
attestation: resultRow.attestation,
networkRetrieval: {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Kudos for re-creating this structure in the JSON data uploaded to Storacha! 👏🏻

Let's use the same property name as we use in spark-checker, see CheckerNetwork/spark-checker#132 (comment)

statusCode: resultRow.network_retrieval_status_code,
timeout: resultRow.network_retrieval_timeout,
carTooLarge: resultRow.network_retrieval_car_too_large,
endAt: resultRow.network_retrieval_end_at,
protocol: resultRow.network_retrieval_protocol
}
})
}

Expand Down
14 changes: 13 additions & 1 deletion api/test/test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,14 @@ const VALID_MEASUREMENT = {
carChecksum: 'somehash',
minerId: 'f02abc',
providerId: 'provider-pubkey',
indexerResult: 'OK'
indexerResult: 'OK',
networkRetrieval: {
statusCode: 200,
timeout: false,
carTooLarge: false,
endAt: new Date(),
protocol: 'graphsync'
}
}

const assertResponseStatus = async (res, status) => {
Expand Down Expand Up @@ -194,6 +201,11 @@ describe('Routes', () => {
assert.strictEqual(measurementRow.miner_id, measurement.minerId)
assert.strictEqual(measurementRow.provider_id, measurement.providerId)
assert.strictEqual(measurementRow.station_id, measurement.stationId)
assert.strictEqual(measurementRow.network_retrieval_status_code, measurement.networkRetrieval.statusCode)
assert.strictEqual(measurementRow.network_retrieval_timeout, measurement.networkRetrieval.timeout)
assert.strictEqual(measurementRow.network_retrieval_car_too_large, measurement.networkRetrieval.carTooLarge)
assert.strictEqual(measurementRow.network_retrieval_end_at, measurement.networkRetrieval.endAt)
assert.strictEqual(measurementRow.network_retrieval_protocol, measurement.networkRetrieval.protocol)
})

it('allows older format with walletAddress', async () => {
Expand Down
7 changes: 7 additions & 0 deletions migrations/067.do.measurement-network-retrieval.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
ALTER TABLE measurements
ADD COLUMN network_retrieval_status_code INTEGER,
ADD COLUMN network_retrieval_timeout BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN network_retrieval_car_too_large BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN network_retrieval_end_at TIMESTAMPTZ,
ADD COLUMN network_retrieval_protocol protocol;

Loading