Skip to content

Commit

Permalink
graph init: add subgraph composition (#1920)
Browse files Browse the repository at this point in the history 
* Fix subgraphs without abi field failing to build

* Fix graph init for composed subgraphs

* Add changeset

* Fix validation not working

* Support declared calls in manifest

* Lint fix

* Address review comments

* Dont allow adding new contracts when subgraph is a composed subgraph

* Allow init of subgraph datasource subgraphs without the interactive mode

* Reduce code duplication between subgraph datasource and normal data source

* prevent using --from-contract and --from-source-subgraph flags together

* cli: validate protocol and source subgraph relationship

* chore(dependencies): updated changesets for modified dependencies

* change flag name for source subgraph

* Refactor manifest validation util functions

* get start block from source manifest

* set fromSubgraph to be default value for graph init in interactive mode

* fix protocol flag validation

* Add init test for subgraphs

* Fix error message

* chore(dependencies): updated changesets for modified dependencies

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: YaroShkvorets <shkvorets@gmail.com>
  • Loading branch information
@incrypto32 @github-actions @YaroShkvorets
3 people authored Feb 13, 2025
1 parent 7f22631 commit b6d7f1c
Show file tree
Hide file tree
Showing 10 changed files with 330 additions and 68 deletions.
5 changes: 5 additions & 0 deletions .changeset/curly-buses-hang.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
'@graphprotocol/graph-cli': minor
---

Add support for subgraph datasource in `graph init`
1 change: 0 additions & 1 deletion packages/cli/src/commands/codegen.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@ export default class CodegenCommand extends Command {
summary: 'IPFS node to use for fetching subgraph data.',
char: 'i',
default: DEFAULT_IPFS_URL,
hidden: true,
}),
'uncrashable-config': Flags.file({
summary: 'Directory for uncrashable config.',
Expand Down
145 changes: 110 additions & 35 deletions packages/cli/src/commands/init.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,13 @@ import EthereumABI from '../protocols/ethereum/abi.js';
import Protocol, { ProtocolName } from '../protocols/index.js';
import { abiEvents } from '../scaffold/schema.js';
import Schema from '../schema.js';
import { createIpfsClient, loadSubgraphSchemaFromIPFS } from '../utils.js';
import {
createIpfsClient,
getMinStartBlock,
loadManifestYaml,
loadSubgraphSchemaFromIPFS,
validateSubgraphNetworkMatch,
} from '../utils.js';
import { validateContract } from '../validation/index.js';
import AddCommand from './add.js';

Expand Down Expand Up @@ -54,6 +60,10 @@ export default class InitCommand extends Command {
summary: 'Graph node for which to initialize.',
char: 'g',
}),
'from-subgraph': Flags.string({
description: 'Creates a scaffold based on an existing subgraph.',
exclusive: ['from-example', 'from-contract'],
}),
'from-contract': Flags.string({
description: 'Creates a scaffold based on an existing contract.',
exclusive: ['from-example'],
Expand Down Expand Up @@ -88,7 +98,6 @@ export default class InitCommand extends Command {
description: 'Block number to start indexing from.',
// TODO: using a default sets the value and therefore requires --from-contract
// default: '0',
dependsOn: ['from-contract'],
}),

abi: Flags.string({
Expand All @@ -110,7 +119,6 @@ export default class InitCommand extends Command {
summary: 'IPFS node to use for fetching subgraph data.',
char: 'i',
default: DEFAULT_IPFS_URL,
hidden: true,
}),
};

Expand All @@ -127,6 +135,7 @@ export default class InitCommand extends Command {
protocol,
node: nodeFlag,
'from-contract': fromContract,
'from-subgraph': fromSubgraph,
'contract-name': contractName,
'from-example': fromExample,
'index-events': indexEvents,
Expand All @@ -141,11 +150,20 @@ export default class InitCommand extends Command {

initDebugger('Flags: %O', flags);

if (startBlock && !(fromContract || fromSubgraph)) {
this.error('--start-block can only be used with --from-contract or --from-subgraph');
}

if (fromContract && fromSubgraph) {
this.error('Cannot use both --from-contract and --from-subgraph at the same time');
}

if (skipGit) {
this.warn(
'The --skip-git flag will be removed in the next major version. By default we will stop initializing a Git repository.',
);
}

if ((fromContract || spkgPath) && !network && !fromExample) {
this.error('--network is required when using --from-contract or --spkg');
}
Expand Down Expand Up @@ -199,16 +217,15 @@ export default class InitCommand extends Command {
let abi!: EthereumABI;

// If all parameters are provided from the command-line,
// go straight to creating the subgraph from an existing contract
if ((fromContract || spkgPath) && protocol && subgraphName && directory && network && node) {
const registry = await loadRegistry();
const contractService = new ContractService(registry);
const sourcifyContractInfo = await contractService.getFromSourcify(
EthereumABI,
network,
fromContract!,
);

// go straight to creating the subgraph from an existing contract or source subgraph
if (
(fromContract || spkgPath || fromSubgraph) &&
protocol &&
subgraphName &&
directory &&
network &&
node
) {
if (!protocolChoices.includes(protocol as ProtocolName)) {
this.error(
`Protocol '${protocol}' is not supported, choose from these options: ${protocolChoices.join(
Expand All @@ -220,7 +237,31 @@ export default class InitCommand extends Command {

const protocolInstance = new Protocol(protocol as ProtocolName);

if (protocolInstance.hasABIs()) {
if (fromSubgraph && !protocolInstance.isComposedSubgraph()) {
this.error('--protocol can only be subgraph when using --from-subgraph');
}

if (
fromContract &&
(protocolInstance.isComposedSubgraph() || protocolInstance.isSubstreams())
) {
this.error('--protocol cannot be subgraph or substreams when using --from-contract');
}

if (spkgPath && !protocolInstance.isSubstreams()) {
this.error('--protocol can only be substreams when using --spkg');
}

// Only fetch contract info and ABI for non-source-subgraph cases
if (!fromSubgraph && protocolInstance.hasABIs()) {
const registry = await loadRegistry();
const contractService = new ContractService(registry);
const sourcifyContractInfo = await contractService.getFromSourcify(
EthereumABI,
network,
fromContract!,
);

const ABI = protocolInstance.getABI();
if (abiPath) {
try {
Expand All @@ -244,7 +285,7 @@ export default class InitCommand extends Command {
protocolInstance,
abi,
directory,
source: fromContract!,
source: fromSubgraph || fromContract!,
indexEvents,
network,
subgraphName,
Expand Down Expand Up @@ -288,7 +329,7 @@ export default class InitCommand extends Command {
abi,
abiPath,
directory,
source: fromContract,
source: fromContract || fromSubgraph,
indexEvents,
fromExample,
subgraphName,
Expand Down Expand Up @@ -534,7 +575,7 @@ async function processInitForm(
value: 'contract',
},
{ message: 'Substreams', name: 'substreams', value: 'substreams' },
// { message: 'Subgraph', name: 'subgraph', value: 'subgraph' },
{ message: 'Subgraph', name: 'subgraph', value: 'subgraph' },
].filter(({ name }) => name),
});

Expand Down Expand Up @@ -604,6 +645,30 @@ async function processInitForm(
},
});

promptManager.addStep({
type: 'input',
name: 'ipfs',
message: `IPFS node to use for fetching subgraph manifest`,
initial: ipfsUrl,
skip: () => !isComposedSubgraph,
validate: value => {
if (!value) {
return 'IPFS node URL cannot be empty';
}
try {
new URL(value);
return true;
} catch {
return 'Please enter a valid URL';
}
},
result: value => {
ipfsNode = value;
initDebugger.extend('processInitForm')('ipfs: %O', value);
return value;
},
});

promptManager.addStep({
type: 'input',
name: 'source',
Expand All @@ -616,9 +681,16 @@ async function processInitForm(
isSubstreams ||
(!protocolInstance.hasContract() && !isComposedSubgraph),
initial: initContract,
validate: async (value: string) => {
validate: async (value: string): Promise<string | boolean> => {
if (isComposedSubgraph) {
return value.startsWith('Qm') ? true : 'Subgraph deployment ID must start with Qm';
const ipfs = createIpfsClient(ipfsNode);
const manifestYaml = await loadManifestYaml(ipfs, value);
const { valid, error } = validateSubgraphNetworkMatch(manifestYaml, network.id);
if (!valid) {
return error || 'Invalid subgraph network match';
}
startBlock ||= getMinStartBlock(manifestYaml)?.toString();
return true;
}
if (initFromExample !== undefined || !protocolInstance.hasContract()) {
return true;
Expand Down Expand Up @@ -668,6 +740,7 @@ async function processInitForm(
} else {
abiFromApi = initAbi;
}

// If startBlock is not provided, try to fetch it from Etherscan API
if (!initStartBlock) {
startBlock = await retryWithPrompt(() =>
Expand Down Expand Up @@ -699,19 +772,6 @@ async function processInitForm(
},
});

promptManager.addStep({
type: 'input',
name: 'ipfs',
message: `IPFS node to use for fetching subgraph manifest`,
initial: ipfsUrl,
skip: () => !isComposedSubgraph,
result: value => {
ipfsNode = value;
initDebugger.extend('processInitForm')('ipfs: %O', value);
return value;
},
});

promptManager.addStep({
type: 'input',
name: 'spkg',
Expand Down Expand Up @@ -751,7 +811,7 @@ async function processInitForm(
isSubstreams ||
!!initAbiPath ||
isComposedSubgraph,
validate: async (value: string) => {
validate: async (value: string): Promise<string | boolean> => {
if (
initFromExample ||
abiFromApi ||
Expand Down Expand Up @@ -1199,6 +1259,14 @@ async function initSubgraphFromContract(
},
});

// Validate network match first
const manifestYaml = await loadManifestYaml(ipfsClient, source);
const { valid, error } = validateSubgraphNetworkMatch(manifestYaml, network);
if (!valid) {
throw new Error(error || 'Invalid subgraph network match');
}

startBlock ||= getMinStartBlock(manifestYaml)?.toString();
const schemaString = await loadSubgraphSchemaFromIPFS(ipfsClient, source);
const schema = await Schema.loadFromString(schemaString);
entities = schema.getEntityNames();
Expand All @@ -1208,8 +1276,9 @@ async function initSubgraphFromContract(
}

if (
!protocolInstance.isComposedSubgraph() &&
!isComposedSubgraph &&
protocolInstance.hasABIs() &&
abi && // Add check for abi existence
(abiEvents(abi).size === 0 ||
// @ts-expect-error TODO: the abiEvents result is expected to be a List, how's it an array?
abiEvents(abi).length === 0)
Expand All @@ -1224,6 +1293,12 @@ async function initSubgraphFromContract(
`Failed to create subgraph scaffold`,
`Warnings while creating subgraph scaffold`,
async spinner => {
initDebugger('Generating scaffold with ABI:', abi);
initDebugger('ABI data:', abi?.data);
if (abi) {
initDebugger('ABI events:', abiEvents(abi));
}

const scaffold = await generateScaffold(
{
protocolInstance,
Expand Down Expand Up @@ -1280,7 +1355,7 @@ async function initSubgraphFromContract(
this.exit(1);
}

while (addContract) {
while (addContract && !isComposedSubgraph) {
addContract = await addAnotherContract
.bind(this)({
protocolInstance,
Expand Down
58 changes: 39 additions & 19 deletions packages/cli/src/compiler/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -508,6 +508,9 @@ export default class Compiler {
`Failed to write compiled subgraph to ${displayDir}`,
`Warnings while writing compiled subgraph to ${displayDir}`,
async spinner => {
// Add debug log for initial subgraph state
compilerDebug('Initial subgraph state:', subgraph.toJS());

// Copy schema and update its path
subgraph = subgraph.updateIn(['schema', 'file'], schemaFile => {
const schemaFilePath = path.resolve(this.sourceDir, schemaFile as string);
Expand All @@ -518,32 +521,49 @@ export default class Compiler {
return path.relative(this.options.outputDir, targetFile);
});

// Add debug log before processing data sources
compilerDebug('Processing dataSources:', subgraph.get('dataSources').toJS());

// Copy data source files and update their paths
subgraph = subgraph.update('dataSources', (dataSources: any[]) =>
dataSources.map(dataSource => {
// Add debug log for each data source
compilerDebug('Processing dataSource:', dataSource.toJS());

let updatedDataSource = dataSource;

if (this.protocol.hasABIs()) {
updatedDataSource = updatedDataSource
// Write data source ABIs to the output directory
.updateIn(['mapping', 'abis'], (abis: any[]) =>
abis.map((abi: any) =>
abi.update('file', (abiFile: string) => {
abiFile = path.resolve(this.sourceDir, abiFile);
const abiData = this.ABI.load(abi.get('name'), abiFile);
return path.relative(
this.options.outputDir,
this._writeSubgraphFile(
abiFile,
JSON.stringify(abiData.data.toJS(), null, 2),
this.sourceDir,
this.subgraphDir(this.options.outputDir, dataSource),
spinner,
),
);
}),
),
// Add debug log for ABIs
compilerDebug(
'Processing ABIs for dataSource:',
dataSource.getIn(['mapping', 'abis'])?.toJS() || 'undefined',
);

updatedDataSource = updatedDataSource.updateIn(['mapping', 'abis'], (abis: any[]) => {
compilerDebug('ABIs value:', Array.isArray(abis) ? abis : 'undefined');

if (!abis) {
compilerDebug('No ABIs found for dataSource');
return immutable.List();
}

return abis.map((abi: any) =>
abi.update('file', (abiFile: string) => {
abiFile = path.resolve(this.sourceDir, abiFile);
const abiData = this.ABI.load(abi.get('name'), abiFile);
return path.relative(
this.options.outputDir,
this._writeSubgraphFile(
abiFile,
JSON.stringify(abiData.data.toJS(), null, 2),
this.sourceDir,
this.subgraphDir(this.options.outputDir, dataSource),
spinner,
),
);
}),
);
});
}

if (protocol.name == 'substreams' || protocol.name == 'substreams/triggers') {
Expand Down
1 change: 1 addition & 0 deletions packages/cli/src/protocols/subgraph/manifest.graphql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ type ContractABI {
type EntityHandler {
handler: String!
entity: String!
calls: JSON
}

type Graft {
Expand Down
Loading

0 comments on commit b6d7f1c

Please sign in to comment.