Skip to content

Commit

Permalink
Update compare CLI to verify IPLD block meta data (#179)
Browse files Browse the repository at this point in the history 
* Update compare CLI to compare IPLD block meta data

* Include init and checkpoints in IPLD blocks verification

* Accomodate changes in codegen
  • Loading branch information
@prathamesh0
prathamesh0 authored Sep 12, 2022
1 parent 996f68a commit 8960f67
Show file tree
Hide file tree
Showing 9 changed files with 156 additions and 41 deletions.
4 changes: 2 additions & 2 deletions packages/codegen/src/templates/resolvers-template.handlebars
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import debug from 'debug';
import Decimal from 'decimal.js';
import { GraphQLScalarType } from 'graphql';

import { ValueResult, BlockHeight, StateKind, gqlTotalQueryCount, gqlQueryCount, jsonBigIntStringReplacer } from '@cerc-io/util';
import { ValueResult, BlockHeight, gqlTotalQueryCount, gqlQueryCount, jsonBigIntStringReplacer } from '@cerc-io/util';

import { Indexer } from './indexer';
import { EventWatcher } from './events';
Expand Down Expand Up @@ -125,7 +125,7 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
return ipldBlock && ipldBlock.block.isComplete ? indexer.getResultIPLDBlock(ipldBlock) : undefined;
},

getState: async (_: any, { blockHash, contractAddress, kind = StateKind.Checkpoint }: { blockHash: string, contractAddress: string, kind: string }) => {
getState: async (_: any, { blockHash, contractAddress, kind }: { blockHash: string, contractAddress: string, kind: string }) => {
log('getState', blockHash, contractAddress, kind);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('getState').inc(1);
Expand Down
4 changes: 2 additions & 2 deletions packages/eden-watcher/src/resolvers.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import debug from 'debug';
import Decimal from 'decimal.js';
import { GraphQLScalarType } from 'graphql';

import { BlockHeight, OrderDirection, StateKind, gqlTotalQueryCount, gqlQueryCount, jsonBigIntStringReplacer } from '@cerc-io/util';
import { BlockHeight, OrderDirection, gqlTotalQueryCount, gqlQueryCount, jsonBigIntStringReplacer } from '@cerc-io/util';

import { Indexer } from './indexer';
import { EventWatcher } from './events';
Expand Down Expand Up @@ -311,7 +311,7 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
return ipldBlock && ipldBlock.block.isComplete ? indexer.getResultIPLDBlock(ipldBlock) : undefined;
},

getState: async (_: any, { blockHash, contractAddress, kind = StateKind.Checkpoint }: { blockHash: string, contractAddress: string, kind: string }) => {
getState: async (_: any, { blockHash, contractAddress, kind }: { blockHash: string, contractAddress: string, kind: string }) => {
log('getState', blockHash, contractAddress, kind);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('getState').inc(1);
Expand Down
4 changes: 2 additions & 2 deletions packages/erc721-watcher/src/resolvers.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import debug from 'debug';
import Decimal from 'decimal.js';
import { GraphQLScalarType } from 'graphql';

import { ValueResult, BlockHeight, StateKind } from '@cerc-io/util';
import { ValueResult, BlockHeight } from '@cerc-io/util';

import { Indexer } from './indexer';
import { EventWatcher } from './events';
Expand Down Expand Up @@ -166,7 +166,7 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
return ipldBlock && ipldBlock.block.isComplete ? indexer.getResultIPLDBlock(ipldBlock) : undefined;
},

getState: async (_: any, { blockHash, contractAddress, kind = StateKind.Checkpoint }: { blockHash: string, contractAddress: string, kind: string }) => {
getState: async (_: any, { blockHash, contractAddress, kind }: { blockHash: string, contractAddress: string, kind: string }) => {
log('getState', blockHash, contractAddress, kind);

const ipldBlock = await indexer.getPrevIPLDBlock(blockHash, contractAddress, kind);
Expand Down
21 changes: 19 additions & 2 deletions packages/graph-node/src/cli/compare/compare-blocks.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import _ from 'lodash';
import { getConfig as getWatcherConfig, wait } from '@cerc-io/util';
import { GraphQLClient } from '@cerc-io/ipld-eth-client';

import { checkEntityInIPLDState, compareQuery, Config, getBlockIPLDState as getIPLDStateByBlock, getClients, getConfig } from './utils';
import { checkEntityInIPLDState, compareQuery, Config, getIPLDsByBlock, checkIPLDMetaData, combineIPLDState, getClients, getConfig } from './utils';
import { Database } from '../../database';
import { getSubgraphConfig } from '../../utils';

Expand Down Expand Up @@ -64,6 +64,7 @@ export const main = async (): Promise<void> => {
let diffFound = false;
let blockDelay = wait(0);
let subgraphContracts: string[] = [];
const contractLatestStateCIDMap: Map<string, string> = new Map();
let db: Database | undefined, subgraphGQLClient: GraphQLClient | undefined;

if (config.watcher) {
Expand All @@ -79,6 +80,10 @@ export const main = async (): Promise<void> => {
const watcherEndpoint = config.endpoints[config.watcher.endpoint] as string;
subgraphGQLClient = new GraphQLClient({ gqlEndpoint: watcherEndpoint });
}

subgraphContracts.forEach(subgraphContract => {
contractLatestStateCIDMap.set(subgraphContract, '');
});
}

for (let blockNumber = startBlock; blockNumber <= endBlock; blockNumber++) {
Expand Down Expand Up @@ -110,7 +115,18 @@ export const main = async (): Promise<void> => {
assert(db);
const [block] = await db?.getBlocksAtHeight(blockNumber, false);
assert(subgraphGQLClient);
ipldStateByBlock = await getIPLDStateByBlock(subgraphGQLClient, subgraphContracts, block.blockHash);
const contractIPLDsByBlock = await getIPLDsByBlock(subgraphGQLClient, subgraphContracts, block.blockHash);

// Check meta data for each IPLD block found
contractIPLDsByBlock.flat().forEach(contractIPLD => {
const ipldMetaDataDiff = checkIPLDMetaData(contractIPLD, contractLatestStateCIDMap, rawJson);
if (ipldMetaDataDiff) {
log('Results mismatch for IPLD meta data:', ipldMetaDataDiff);
diffFound = true;
}
});

ipldStateByBlock = combineIPLDState(contractIPLDsByBlock.flat());
}

await blockDelay;
Expand Down Expand Up @@ -166,6 +182,7 @@ export const main = async (): Promise<void> => {
}
} catch (err: any) {
log('Error:', err.message);
log('Error:', err);
}
}

Expand Down
152 changes: 125 additions & 27 deletions packages/graph-node/src/cli/compare/utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,14 @@ import { DEFAULT_LIMIT } from '../../database';
const IPLD_STATE_QUERY = `
query getState($blockHash: String!, $contractAddress: String!, $kind: String){
getState(blockHash: $blockHash, contractAddress: $contractAddress, kind: $kind){
block {
cid
number
hash
}
contractAddress
cid
kind
data
}
}
Expand Down Expand Up @@ -150,43 +158,131 @@ export const getClients = async (config: Config, queryDir?: string):Promise<{
};
};

export const getBlockIPLDState = async (client: GraphQLClient, contracts: string[], blockHash: string): Promise<{[key: string]: any}> => {
const contractIPLDStates: {[key: string]: any}[] = await Promise.all(contracts.map(async contract => {
export const getIPLDsByBlock = async (client: GraphQLClient, contracts: string[], blockHash: string): Promise<{[key: string]: any}[][]> => {
// Fetch IPLD states for all contracts
return Promise.all(contracts.map(async contract => {
const { getState } = await client.query(
gql(IPLD_STATE_QUERY),
{
blockHash,
contractAddress: contract,
kind: 'diff'
contractAddress: contract
}
);

if (getState) {
const data = JSON.parse(getState.data);

// Apply default limit on array type relation fields.
Object.values(data.state)
.forEach((idEntityMap: any) => {
Object.values(idEntityMap)
.forEach((entity: any) => {
Object.values(entity)
.forEach(fieldValue => {
if (
Array.isArray(fieldValue) &&
fieldValue.length &&
fieldValue[0].id
) {
fieldValue.splice(DEFAULT_LIMIT);
}
});
});
});

return data.state;
const stateIPLDs = [];

// If 'checkpoint' is found at the same block, fetch 'diff' as well
if (getState && getState.kind === 'checkpoint' && getState.block.hash === blockHash) {
// Check if 'init' present at the same block
const { getState: getInitState } = await client.query(
gql(IPLD_STATE_QUERY),
{
blockHash,
contractAddress: contract,
kind: 'init'
}
);

if (getInitState && getInitState.block.hash === blockHash) {
// Append the 'init' IPLD block to the result
stateIPLDs.push(getInitState);
}

// Check if 'diff' present at the same block
const { getState: getDiffState } = await client.query(
gql(IPLD_STATE_QUERY),
{
blockHash,
contractAddress: contract,
kind: 'diff'
}
);

if (getDiffState && getDiffState.block.hash === blockHash) {
// Append the 'diff' IPLD block to the result
stateIPLDs.push(getDiffState);
}
}

return {};
// Append the IPLD block to the result
stateIPLDs.push(getState);

return stateIPLDs;
}));
};

export const checkIPLDMetaData = (contractIPLD: {[key: string]: any}, contractLatestStateCIDMap: Map<string, string>, rawJson: boolean) => {
// Return if IPLD for a contract not found
if (!contractIPLD) {
return;
}

const { contractAddress, cid, kind, block } = contractIPLD;

let parentCID = contractLatestStateCIDMap.get(contractAddress);
// If CID is same as the parent CID, skip the check
if (cid === parentCID) {
return;
}

// Update the parent CID in the map
contractLatestStateCIDMap.set(contractAddress, cid);

// Actual meta data from the GQL result
const data = JSON.parse(contractIPLD.data);

// If parentCID not initialized (is empty at start)
// Take the expected parentCID from the actual data itself
if (parentCID === '') {
parentCID = data.meta.parent['/'];
}

// Expected meta data
const expectedMetaData = {
id: contractAddress,
kind,
parent: {
'/': parentCID
},
ethBlock: {
cid: {
'/': block.cid
},
num: block.number
}
};

return compareObjects(expectedMetaData, data.meta, rawJson);
};

export const combineIPLDState = (contractIPLDs: {[key: string]: any}[]): {[key: string]: any} => {
const contractIPLDStates: {[key: string]: any}[] = contractIPLDs.map(contractIPLD => {
if (!contractIPLD) {
return {};
}

const data = JSON.parse(contractIPLD.data);

// Apply default limit on array type relation fields.
Object.values(data.state)
.forEach((idEntityMap: any) => {
Object.values(idEntityMap)
.forEach((entity: any) => {
Object.values(entity)
.forEach(fieldValue => {
if (
Array.isArray(fieldValue) &&
fieldValue.length &&
fieldValue[0].id
) {
fieldValue.splice(DEFAULT_LIMIT);
}
});
});
});

return data.state;
});

return contractIPLDStates.reduce((acc, state) => _.merge(acc, state));
};
Expand Down Expand Up @@ -219,6 +315,8 @@ export const checkEntityInIPLDState = async (
return diff;
};

// obj1: expected
// obj2: actual
const compareObjects = (obj1: any, obj2: any, rawJson: boolean): string => {
if (rawJson) {
const diffObj = diff(obj1, obj2);
Expand Down
2 changes: 1 addition & 1 deletion packages/graph-node/src/loader.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import debug from 'debug';

import { BaseProvider } from '@ethersproject/providers';
import loader from '@vulcanize/assemblyscript/lib/loader';
import { IndexerInterface, GraphDecimal, getGraphDigitsAndExp, jsonBigIntStringReplacer } from '@cerc-io/util';
import { IndexerInterface, GraphDecimal, getGraphDigitsAndExp } from '@cerc-io/util';

import { TypeId, Level } from './types';
import {
Expand Down
2 changes: 1 addition & 1 deletion packages/graph-node/tsconfig.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
// "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */

/* Language and Environment */
"target": "es5", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */
"target": "ES2019", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */
// "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */
// "jsx": "preserve", /* Specify what JSX code is generated. */
// "experimentalDecorators": true, /* Enable experimental support for TC39 stage 2 draft decorators. */
Expand Down
4 changes: 2 additions & 2 deletions packages/graph-test-watcher/src/resolvers.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import debug from 'debug';
import Decimal from 'decimal.js';
import { GraphQLScalarType } from 'graphql';

import { ValueResult, BlockHeight, StateKind, jsonBigIntStringReplacer } from '@cerc-io/util';
import { ValueResult, BlockHeight, jsonBigIntStringReplacer } from '@cerc-io/util';

import { Indexer } from './indexer';
import { EventWatcher } from './events';
Expand Down Expand Up @@ -122,7 +122,7 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
return ipldBlock && ipldBlock.block.isComplete ? indexer.getResultIPLDBlock(ipldBlock) : undefined;
},

getState: async (_: any, { blockHash, contractAddress, kind = StateKind.Checkpoint }: { blockHash: string, contractAddress: string, kind: string }) => {
getState: async (_: any, { blockHash, contractAddress, kind }: { blockHash: string, contractAddress: string, kind: string }) => {
log('getState', blockHash, contractAddress, kind);

const ipldBlock = await indexer.getPrevIPLDBlock(blockHash, contractAddress, kind);
Expand Down
4 changes: 2 additions & 2 deletions packages/mobymask-watcher/src/resolvers.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import debug from 'debug';
import Decimal from 'decimal.js';
import { GraphQLScalarType } from 'graphql';

import { ValueResult, StateKind, gqlTotalQueryCount, gqlQueryCount } from '@cerc-io/util';
import { ValueResult, gqlTotalQueryCount, gqlQueryCount } from '@cerc-io/util';

import { Indexer } from './indexer';
import { EventWatcher } from './events';
Expand Down Expand Up @@ -131,7 +131,7 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
return ipldBlock && ipldBlock.block.isComplete ? indexer.getResultIPLDBlock(ipldBlock) : undefined;
},

getState: async (_: any, { blockHash, contractAddress, kind = StateKind.Checkpoint }: { blockHash: string, contractAddress: string, kind: string }) => {
getState: async (_: any, { blockHash, contractAddress, kind }: { blockHash: string, contractAddress: string, kind: string }) => {
log('getState', blockHash, contractAddress, kind);
gqlTotalQueryCount.inc(1);
gqlQueryCount.labels('getState').inc(1);
Expand Down

0 comments on commit 8960f67

Please sign in to comment.