From 8960f67f1b72a09da35879aa49a5ddfda77c336d Mon Sep 17 00:00:00 2001 From: prathamesh0 <42446521+prathamesh0@users.noreply.github.com> Date: Mon, 12 Sep 2022 17:44:53 +0530 Subject: [PATCH] Update compare CLI to verify IPLD block meta data (#179) * Update compare CLI to compare IPLD block meta data * Include init and checkpoints in IPLD blocks verification * Accomodate changes in codegen --- .../templates/resolvers-template.handlebars | 4 +- packages/eden-watcher/src/resolvers.ts | 4 +- packages/erc721-watcher/src/resolvers.ts | 4 +- .../src/cli/compare/compare-blocks.ts | 21 ++- packages/graph-node/src/cli/compare/utils.ts | 152 ++++++++++++++---- packages/graph-node/src/loader.ts | 2 +- packages/graph-node/tsconfig.json | 2 +- packages/graph-test-watcher/src/resolvers.ts | 4 +- packages/mobymask-watcher/src/resolvers.ts | 4 +- 9 files changed, 156 insertions(+), 41 deletions(-) diff --git a/packages/codegen/src/templates/resolvers-template.handlebars b/packages/codegen/src/templates/resolvers-template.handlebars index 1e9251e08..26494dc33 100644 --- a/packages/codegen/src/templates/resolvers-template.handlebars +++ b/packages/codegen/src/templates/resolvers-template.handlebars @@ -8,7 +8,7 @@ import debug from 'debug'; import Decimal from 'decimal.js'; import { GraphQLScalarType } from 'graphql'; -import { ValueResult, BlockHeight, StateKind, gqlTotalQueryCount, gqlQueryCount, jsonBigIntStringReplacer } from '@cerc-io/util'; +import { ValueResult, BlockHeight, gqlTotalQueryCount, gqlQueryCount, jsonBigIntStringReplacer } from '@cerc-io/util'; import { Indexer } from './indexer'; import { EventWatcher } from './events'; @@ -125,7 +125,7 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch return ipldBlock && ipldBlock.block.isComplete ? indexer.getResultIPLDBlock(ipldBlock) : undefined; }, - getState: async (_: any, { blockHash, contractAddress, kind = StateKind.Checkpoint }: { blockHash: string, contractAddress: string, kind: string }) => { + getState: async (_: any, { blockHash, contractAddress, kind }: { blockHash: string, contractAddress: string, kind: string }) => { log('getState', blockHash, contractAddress, kind); gqlTotalQueryCount.inc(1); gqlQueryCount.labels('getState').inc(1); diff --git a/packages/eden-watcher/src/resolvers.ts b/packages/eden-watcher/src/resolvers.ts index e4dc8516e..ba1ed63f2 100644 --- a/packages/eden-watcher/src/resolvers.ts +++ b/packages/eden-watcher/src/resolvers.ts @@ -8,7 +8,7 @@ import debug from 'debug'; import Decimal from 'decimal.js'; import { GraphQLScalarType } from 'graphql'; -import { BlockHeight, OrderDirection, StateKind, gqlTotalQueryCount, gqlQueryCount, jsonBigIntStringReplacer } from '@cerc-io/util'; +import { BlockHeight, OrderDirection, gqlTotalQueryCount, gqlQueryCount, jsonBigIntStringReplacer } from '@cerc-io/util'; import { Indexer } from './indexer'; import { EventWatcher } from './events'; @@ -311,7 +311,7 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch return ipldBlock && ipldBlock.block.isComplete ? indexer.getResultIPLDBlock(ipldBlock) : undefined; }, - getState: async (_: any, { blockHash, contractAddress, kind = StateKind.Checkpoint }: { blockHash: string, contractAddress: string, kind: string }) => { + getState: async (_: any, { blockHash, contractAddress, kind }: { blockHash: string, contractAddress: string, kind: string }) => { log('getState', blockHash, contractAddress, kind); gqlTotalQueryCount.inc(1); gqlQueryCount.labels('getState').inc(1); diff --git a/packages/erc721-watcher/src/resolvers.ts b/packages/erc721-watcher/src/resolvers.ts index 4581445e8..3f4306410 100644 --- a/packages/erc721-watcher/src/resolvers.ts +++ b/packages/erc721-watcher/src/resolvers.ts @@ -8,7 +8,7 @@ import debug from 'debug'; import Decimal from 'decimal.js'; import { GraphQLScalarType } from 'graphql'; -import { ValueResult, BlockHeight, StateKind } from '@cerc-io/util'; +import { ValueResult, BlockHeight } from '@cerc-io/util'; import { Indexer } from './indexer'; import { EventWatcher } from './events'; @@ -166,7 +166,7 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch return ipldBlock && ipldBlock.block.isComplete ? indexer.getResultIPLDBlock(ipldBlock) : undefined; }, - getState: async (_: any, { blockHash, contractAddress, kind = StateKind.Checkpoint }: { blockHash: string, contractAddress: string, kind: string }) => { + getState: async (_: any, { blockHash, contractAddress, kind }: { blockHash: string, contractAddress: string, kind: string }) => { log('getState', blockHash, contractAddress, kind); const ipldBlock = await indexer.getPrevIPLDBlock(blockHash, contractAddress, kind); diff --git a/packages/graph-node/src/cli/compare/compare-blocks.ts b/packages/graph-node/src/cli/compare/compare-blocks.ts index 826c7006a..60b2e196d 100644 --- a/packages/graph-node/src/cli/compare/compare-blocks.ts +++ b/packages/graph-node/src/cli/compare/compare-blocks.ts @@ -12,7 +12,7 @@ import _ from 'lodash'; import { getConfig as getWatcherConfig, wait } from '@cerc-io/util'; import { GraphQLClient } from '@cerc-io/ipld-eth-client'; -import { checkEntityInIPLDState, compareQuery, Config, getBlockIPLDState as getIPLDStateByBlock, getClients, getConfig } from './utils'; +import { checkEntityInIPLDState, compareQuery, Config, getIPLDsByBlock, checkIPLDMetaData, combineIPLDState, getClients, getConfig } from './utils'; import { Database } from '../../database'; import { getSubgraphConfig } from '../../utils'; @@ -64,6 +64,7 @@ export const main = async (): Promise => { let diffFound = false; let blockDelay = wait(0); let subgraphContracts: string[] = []; + const contractLatestStateCIDMap: Map = new Map(); let db: Database | undefined, subgraphGQLClient: GraphQLClient | undefined; if (config.watcher) { @@ -79,6 +80,10 @@ export const main = async (): Promise => { const watcherEndpoint = config.endpoints[config.watcher.endpoint] as string; subgraphGQLClient = new GraphQLClient({ gqlEndpoint: watcherEndpoint }); } + + subgraphContracts.forEach(subgraphContract => { + contractLatestStateCIDMap.set(subgraphContract, ''); + }); } for (let blockNumber = startBlock; blockNumber <= endBlock; blockNumber++) { @@ -110,7 +115,18 @@ export const main = async (): Promise => { assert(db); const [block] = await db?.getBlocksAtHeight(blockNumber, false); assert(subgraphGQLClient); - ipldStateByBlock = await getIPLDStateByBlock(subgraphGQLClient, subgraphContracts, block.blockHash); + const contractIPLDsByBlock = await getIPLDsByBlock(subgraphGQLClient, subgraphContracts, block.blockHash); + + // Check meta data for each IPLD block found + contractIPLDsByBlock.flat().forEach(contractIPLD => { + const ipldMetaDataDiff = checkIPLDMetaData(contractIPLD, contractLatestStateCIDMap, rawJson); + if (ipldMetaDataDiff) { + log('Results mismatch for IPLD meta data:', ipldMetaDataDiff); + diffFound = true; + } + }); + + ipldStateByBlock = combineIPLDState(contractIPLDsByBlock.flat()); } await blockDelay; @@ -166,6 +182,7 @@ export const main = async (): Promise => { } } catch (err: any) { log('Error:', err.message); + log('Error:', err); } } diff --git a/packages/graph-node/src/cli/compare/utils.ts b/packages/graph-node/src/cli/compare/utils.ts index 69e9b8bbf..c56faa65e 100644 --- a/packages/graph-node/src/cli/compare/utils.ts +++ b/packages/graph-node/src/cli/compare/utils.ts @@ -21,6 +21,14 @@ import { DEFAULT_LIMIT } from '../../database'; const IPLD_STATE_QUERY = ` query getState($blockHash: String!, $contractAddress: String!, $kind: String){ getState(blockHash: $blockHash, contractAddress: $contractAddress, kind: $kind){ + block { + cid + number + hash + } + contractAddress + cid + kind data } } @@ -150,43 +158,131 @@ export const getClients = async (config: Config, queryDir?: string):Promise<{ }; }; -export const getBlockIPLDState = async (client: GraphQLClient, contracts: string[], blockHash: string): Promise<{[key: string]: any}> => { - const contractIPLDStates: {[key: string]: any}[] = await Promise.all(contracts.map(async contract => { +export const getIPLDsByBlock = async (client: GraphQLClient, contracts: string[], blockHash: string): Promise<{[key: string]: any}[][]> => { + // Fetch IPLD states for all contracts + return Promise.all(contracts.map(async contract => { const { getState } = await client.query( gql(IPLD_STATE_QUERY), { blockHash, - contractAddress: contract, - kind: 'diff' + contractAddress: contract } ); - if (getState) { - const data = JSON.parse(getState.data); - - // Apply default limit on array type relation fields. - Object.values(data.state) - .forEach((idEntityMap: any) => { - Object.values(idEntityMap) - .forEach((entity: any) => { - Object.values(entity) - .forEach(fieldValue => { - if ( - Array.isArray(fieldValue) && - fieldValue.length && - fieldValue[0].id - ) { - fieldValue.splice(DEFAULT_LIMIT); - } - }); - }); - }); - - return data.state; + const stateIPLDs = []; + + // If 'checkpoint' is found at the same block, fetch 'diff' as well + if (getState && getState.kind === 'checkpoint' && getState.block.hash === blockHash) { + // Check if 'init' present at the same block + const { getState: getInitState } = await client.query( + gql(IPLD_STATE_QUERY), + { + blockHash, + contractAddress: contract, + kind: 'init' + } + ); + + if (getInitState && getInitState.block.hash === blockHash) { + // Append the 'init' IPLD block to the result + stateIPLDs.push(getInitState); + } + + // Check if 'diff' present at the same block + const { getState: getDiffState } = await client.query( + gql(IPLD_STATE_QUERY), + { + blockHash, + contractAddress: contract, + kind: 'diff' + } + ); + + if (getDiffState && getDiffState.block.hash === blockHash) { + // Append the 'diff' IPLD block to the result + stateIPLDs.push(getDiffState); + } } - return {}; + // Append the IPLD block to the result + stateIPLDs.push(getState); + + return stateIPLDs; })); +}; + +export const checkIPLDMetaData = (contractIPLD: {[key: string]: any}, contractLatestStateCIDMap: Map, rawJson: boolean) => { + // Return if IPLD for a contract not found + if (!contractIPLD) { + return; + } + + const { contractAddress, cid, kind, block } = contractIPLD; + + let parentCID = contractLatestStateCIDMap.get(contractAddress); + // If CID is same as the parent CID, skip the check + if (cid === parentCID) { + return; + } + + // Update the parent CID in the map + contractLatestStateCIDMap.set(contractAddress, cid); + + // Actual meta data from the GQL result + const data = JSON.parse(contractIPLD.data); + + // If parentCID not initialized (is empty at start) + // Take the expected parentCID from the actual data itself + if (parentCID === '') { + parentCID = data.meta.parent['/']; + } + + // Expected meta data + const expectedMetaData = { + id: contractAddress, + kind, + parent: { + '/': parentCID + }, + ethBlock: { + cid: { + '/': block.cid + }, + num: block.number + } + }; + + return compareObjects(expectedMetaData, data.meta, rawJson); +}; + +export const combineIPLDState = (contractIPLDs: {[key: string]: any}[]): {[key: string]: any} => { + const contractIPLDStates: {[key: string]: any}[] = contractIPLDs.map(contractIPLD => { + if (!contractIPLD) { + return {}; + } + + const data = JSON.parse(contractIPLD.data); + + // Apply default limit on array type relation fields. + Object.values(data.state) + .forEach((idEntityMap: any) => { + Object.values(idEntityMap) + .forEach((entity: any) => { + Object.values(entity) + .forEach(fieldValue => { + if ( + Array.isArray(fieldValue) && + fieldValue.length && + fieldValue[0].id + ) { + fieldValue.splice(DEFAULT_LIMIT); + } + }); + }); + }); + + return data.state; + }); return contractIPLDStates.reduce((acc, state) => _.merge(acc, state)); }; @@ -219,6 +315,8 @@ export const checkEntityInIPLDState = async ( return diff; }; +// obj1: expected +// obj2: actual const compareObjects = (obj1: any, obj2: any, rawJson: boolean): string => { if (rawJson) { const diffObj = diff(obj1, obj2); diff --git a/packages/graph-node/src/loader.ts b/packages/graph-node/src/loader.ts index a3885f723..d9a89b709 100644 --- a/packages/graph-node/src/loader.ts +++ b/packages/graph-node/src/loader.ts @@ -15,7 +15,7 @@ import debug from 'debug'; import { BaseProvider } from '@ethersproject/providers'; import loader from '@vulcanize/assemblyscript/lib/loader'; -import { IndexerInterface, GraphDecimal, getGraphDigitsAndExp, jsonBigIntStringReplacer } from '@cerc-io/util'; +import { IndexerInterface, GraphDecimal, getGraphDigitsAndExp } from '@cerc-io/util'; import { TypeId, Level } from './types'; import { diff --git a/packages/graph-node/tsconfig.json b/packages/graph-node/tsconfig.json index d68248572..fdb0c6b83 100644 --- a/packages/graph-node/tsconfig.json +++ b/packages/graph-node/tsconfig.json @@ -11,7 +11,7 @@ // "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */ /* Language and Environment */ - "target": "es5", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ + "target": "ES2019", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ // "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */ // "jsx": "preserve", /* Specify what JSX code is generated. */ // "experimentalDecorators": true, /* Enable experimental support for TC39 stage 2 draft decorators. */ diff --git a/packages/graph-test-watcher/src/resolvers.ts b/packages/graph-test-watcher/src/resolvers.ts index b09e3d3ab..56ab93242 100644 --- a/packages/graph-test-watcher/src/resolvers.ts +++ b/packages/graph-test-watcher/src/resolvers.ts @@ -8,7 +8,7 @@ import debug from 'debug'; import Decimal from 'decimal.js'; import { GraphQLScalarType } from 'graphql'; -import { ValueResult, BlockHeight, StateKind, jsonBigIntStringReplacer } from '@cerc-io/util'; +import { ValueResult, BlockHeight, jsonBigIntStringReplacer } from '@cerc-io/util'; import { Indexer } from './indexer'; import { EventWatcher } from './events'; @@ -122,7 +122,7 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch return ipldBlock && ipldBlock.block.isComplete ? indexer.getResultIPLDBlock(ipldBlock) : undefined; }, - getState: async (_: any, { blockHash, contractAddress, kind = StateKind.Checkpoint }: { blockHash: string, contractAddress: string, kind: string }) => { + getState: async (_: any, { blockHash, contractAddress, kind }: { blockHash: string, contractAddress: string, kind: string }) => { log('getState', blockHash, contractAddress, kind); const ipldBlock = await indexer.getPrevIPLDBlock(blockHash, contractAddress, kind); diff --git a/packages/mobymask-watcher/src/resolvers.ts b/packages/mobymask-watcher/src/resolvers.ts index bbc7d686b..90e77549d 100644 --- a/packages/mobymask-watcher/src/resolvers.ts +++ b/packages/mobymask-watcher/src/resolvers.ts @@ -8,7 +8,7 @@ import debug from 'debug'; import Decimal from 'decimal.js'; import { GraphQLScalarType } from 'graphql'; -import { ValueResult, StateKind, gqlTotalQueryCount, gqlQueryCount } from '@cerc-io/util'; +import { ValueResult, gqlTotalQueryCount, gqlQueryCount } from '@cerc-io/util'; import { Indexer } from './indexer'; import { EventWatcher } from './events'; @@ -131,7 +131,7 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch return ipldBlock && ipldBlock.block.isComplete ? indexer.getResultIPLDBlock(ipldBlock) : undefined; }, - getState: async (_: any, { blockHash, contractAddress, kind = StateKind.Checkpoint }: { blockHash: string, contractAddress: string, kind: string }) => { + getState: async (_: any, { blockHash, contractAddress, kind }: { blockHash: string, contractAddress: string, kind: string }) => { log('getState', blockHash, contractAddress, kind); gqlTotalQueryCount.inc(1); gqlQueryCount.labels('getState').inc(1);