Implement query for multiple entities and nested relation fields in e…

…den-watcher (#166)

* Implement query for multiple entities in eden-watcher

* Implement nested relation queries

* Implement GQL query params first, skip, orderBy, orderDirection

* Add blockNumber index to subgraph entities

* Add logs for timing eth-calls and storage calls

* Add prometheus metrics to monitor GQL queries

* Fix default limit and order of 1-N related field in GQL entitiy query

* Add timer logs for block processing

* Run transpiled js in all watchers

Co-authored-by: prathamesh0 <prathamesh.musale0@gmail.com>

packages/codegen/src/entity.ts

@@ -209,11 +209,15 @@ export class Entity {
 
       entityObject.imports.push(
         {
-          toImport: new Set(['Entity', 'PrimaryColumn', 'Column']),
+          toImport: new Set(['Entity', 'PrimaryColumn', 'Column', 'Index']),
           from: 'typeorm'
         }
       );
 
+      entityObject.indexOn.push({
+        columns: ['blockNumber']
+      });
+
       // Add common columns.
       entityObject.columns.push({
         name: 'id',

packages/codegen/src/templates/config-template.handlebars

@@ -27,6 +27,8 @@
 [metrics]
   host = "127.0.0.1"
   port = 9000
+  [metrics.gql]
+    port = 9001
 
 [database]
   type = "postgres"

packages/codegen/src/templates/indexer-template.handlebars

@@ -430,7 +430,7 @@ export class Indexer implements IPLDIndexerInterface {
   async getSubgraphEntity<Entity> (entity: new () => Entity, id: string, block?: BlockHeight): Promise<any> {
     const relations = this._relationsMap.get(entity) || {};
 
-    const data = await this._graphWatcher.getEntity(entity, id, relations, block);
+    const data = await this._graphWatcher.getEntity(entity, id, this._relationsMap, block);
 
     return data;
   }
@@ -704,6 +704,7 @@ export class Indexer implements IPLDIndexerInterface {
     const blockPromise = this._ethClient.getBlockByHash(blockHash);
     let logs: any[];
 
+    console.time('time:indexer#_fetchAndSaveEvents-fetch-logs');
     if (this._serverConfig.filterLogs) {
       const watchedContracts = this._baseIndexer.getWatchedContracts();
 
@@ -725,6 +726,7 @@ export class Indexer implements IPLDIndexerInterface {
     } else {
       ({ logs } = await this._ethClient.getLogs({ blockHash }));
     }
+    console.timeEnd('time:indexer#_fetchAndSaveEvents-fetch-logs');
 
     let [
       { block },
@@ -816,8 +818,10 @@ export class Indexer implements IPLDIndexerInterface {
         parentHash: block.parent.hash
       };
 
+      console.time('time:indexer#_fetchAndSaveEvents-save-block-events');
       const blockProgress = await this._db.saveEvents(dbTx, block, dbEvents);
       await dbTx.commitTransaction();
+      console.timeEnd('time:indexer#_fetchAndSaveEvents-save-block-events');
 
       return blockProgress;
     } catch (error) {

packages/codegen/src/templates/package-template.handlebars

@@ -6,9 +6,13 @@
   "main": "dist/index.js",
   "scripts": {
     "lint": "eslint .",
-    "build": "tsc",
-    "server": "DEBUG=vulcanize:* ts-node src/server.ts",
-    "job-runner": "DEBUG=vulcanize:* ts-node src/job-runner.ts",
+    "build": "yarn clean && tsc && yarn copy-assets",
+    "clean": "rm -rf ./dist",
+    "copy-assets": "copyfiles -u 1 src/**/*.gql dist/",
+    "server": "DEBUG=vulcanize:* node --enable-source-maps dist/server.js",
+    "server:dev": "DEBUG=vulcanize:* ts-node src/server.ts",
+    "job-runner": "DEBUG=vulcanize:* node --enable-source-maps dist/job-runner.js",
+    "job-runner:dev": "DEBUG=vulcanize:* ts-node src/job-runner.ts",
     "watch:contract": "DEBUG=vulcanize:* ts-node src/cli/watch-contract.ts",
     "fill": "DEBUG=vulcanize:* ts-node src/fill.ts",
     "reset": "DEBUG=vulcanize:* ts-node src/cli/reset.ts",
@@ -65,6 +69,7 @@
     "eslint-plugin-promise": "^5.1.0",
     "eslint-plugin-standard": "^5.0.0",
     "ts-node": "^10.0.0",
-    "typescript": "^4.3.2"
+    "typescript": "^4.3.2",
+    "copyfiles": "^2.4.1"
   }
 }

packages/codegen/src/templates/resolvers-template.handlebars

@@ -8,7 +8,7 @@ import debug from 'debug';
 import Decimal from 'decimal.js';
 import { GraphQLScalarType } from 'graphql';
 
-import { ValueResult, BlockHeight, StateKind } from '@vulcanize/util';
+import { ValueResult, BlockHeight, StateKind, gqlTotalQueryCount, gqlQueryCount } from '@vulcanize/util';
 
 import { Indexer } from './indexer';
 import { EventWatcher } from './events';
@@ -68,6 +68,9 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
       {{~#each this.params}}, {{this.name}}: {{this.type~}} {{/each}} }): Promise<ValueResult> => {
         log('{{this.name}}', blockHash, contractAddress
         {{~#each this.params}}, {{this.name~}} {{/each}});
+        gqlTotalQueryCount.inc(1);
+        gqlQueryCount.labels('{{this.name}}').inc(1);
+
         return indexer.{{this.name}}(blockHash, contractAddress
         {{~#each this.params}}, {{this.name~}} {{/each}});
       },
@@ -77,13 +80,17 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
       {{~#each subgraphQueries}}
       {{this.queryName}}: async (_: any, { id, block = {} }: { id: string, block: BlockHeight }) => {
         log('{{this.queryName}}', id, block);
+        gqlTotalQueryCount.inc(1);
+        gqlQueryCount.labels('{{this.queryName}}').inc(1);
 
         return indexer.getSubgraphEntity({{this.entityName}}, id, block);
       },
 
       {{/each}}
       events: async (_: any, { blockHash, contractAddress, name }: { blockHash: string, contractAddress: string, name?: string }) => {
         log('events', blockHash, contractAddress, name);
+        gqlTotalQueryCount.inc(1);
+        gqlQueryCount.labels('events').inc(1);
 
         const block = await indexer.getBlockProgress(blockHash);
         if (!block || !block.isComplete) {
@@ -96,6 +103,8 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
 
       eventsInRange: async (_: any, { fromBlockNumber, toBlockNumber }: { fromBlockNumber: number, toBlockNumber: number }) => {
         log('eventsInRange', fromBlockNumber, toBlockNumber);
+        gqlTotalQueryCount.inc(1);
+        gqlQueryCount.labels('eventsInRange').inc(1);
 
         const { expected, actual } = await indexer.getProcessedBlockCountForRange(fromBlockNumber, toBlockNumber);
         if (expected !== actual) {
@@ -108,6 +117,8 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
 
       getStateByCID: async (_: any, { cid }: { cid: string }) => {
         log('getStateByCID', cid);
+        gqlTotalQueryCount.inc(1);
+        gqlQueryCount.labels('getStateByCID').inc(1);
 
         const ipldBlock = await indexer.getIPLDBlockByCid(cid);
 
@@ -116,6 +127,8 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
 
       getState: async (_: any, { blockHash, contractAddress, kind = StateKind.Checkpoint }: { blockHash: string, contractAddress: string, kind: string }) => {
         log('getState', blockHash, contractAddress, kind);
+        gqlTotalQueryCount.inc(1);
+        gqlQueryCount.labels('getState').inc(1);
 
         const ipldBlock = await indexer.getPrevIPLDBlock(blockHash, contractAddress, kind);
 
@@ -124,6 +137,8 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
 
       getSyncStatus: async () => {
         log('getSyncStatus');
+        gqlTotalQueryCount.inc(1);
+        gqlQueryCount.labels('getSyncStatus').inc(1);
 
         return indexer.getSyncStatus();
       }

packages/codegen/src/templates/server-template.handlebars

@@ -14,7 +14,7 @@ import debug from 'debug';
 import 'graphql-import-node';
 import { createServer } from 'http';
 
-import { DEFAULT_CONFIG_PATH, getConfig, Config, JobQueue, KIND_ACTIVE, initClients } from '@vulcanize/util';
+import { DEFAULT_CONFIG_PATH, getConfig, Config, JobQueue, KIND_ACTIVE, initClients, startGQLMetricsServer } from '@vulcanize/util';
 {{#if (subgraphPath)}}
 import { GraphWatcher, Database as GraphDatabase } from '@vulcanize/graph-node';
 {{/if}}
@@ -45,10 +45,10 @@ export const main = async (): Promise<any> => {
   const db = new Database(config.database);
   await db.init();
   {{#if (subgraphPath)}}
-  
+
   const graphDb = new GraphDatabase(config.database, path.resolve(__dirname, 'entity/*'));
   await graphDb.init();
-  
+
   const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
   {{/if}}
 
@@ -98,6 +98,8 @@ export const main = async (): Promise<any> => {
     log(`Server is listening on host ${host} port ${port}`);
   });
 
+  startGQLMetricsServer(config);
+
   return { app, server };
 };
 

packages/codegen/src/templates/tsconfig-template.handlebars

@@ -12,9 +12,9 @@
     // "jsx": "preserve",                           /* Specify JSX code generation: 'preserve', 'react-native', 'react', 'react-jsx' or 'react-jsxdev'. */
     // "declaration": true,                         /* Generates corresponding '.d.ts' file. */
     // "declarationMap": true,                      /* Generates a sourcemap for each corresponding '.d.ts' file. */
-    // "sourceMap": true,                           /* Generates corresponding '.map' file. */
+    "sourceMap": true,                              /* Generates corresponding '.map' file. */
     // "outFile": "./",                             /* Concatenate and emit output to single file. */
-    "outDir": "dist",                            /* Redirect output structure to the directory. */
+    "outDir": "dist",                               /* Redirect output structure to the directory. */
     // "rootDir": "./",                             /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */
     // "composite": true,                           /* Enable project compilation */
     // "tsBuildInfoFile": "./",                     /* Specify file to store incremental compilation information */

packages/eden-watcher/environments/local.toml

@@ -25,6 +25,8 @@
 [metrics]
   host = "127.0.0.1"
   port = 9000
+  [metrics.gql]
+    port = 9001
 
 [database]
   type = "postgres"

packages/eden-watcher/package.json

@@ -6,9 +6,13 @@
   "main": "dist/index.js",
   "scripts": {
     "lint": "eslint .",
-    "build": "tsc",
-    "server": "DEBUG=vulcanize:* ts-node src/server.ts",
-    "job-runner": "DEBUG=vulcanize:* ts-node src/job-runner.ts",
+    "build": "yarn clean && tsc && yarn copy-assets",
+    "clean": "rm -rf ./dist",
+    "copy-assets": "copyfiles -u 1 src/**/*.gql dist/",
+    "server": "DEBUG=vulcanize:* node --enable-source-maps dist/server.js",
+    "server:dev": "DEBUG=vulcanize:* ts-node src/server.ts",
+    "job-runner": "DEBUG=vulcanize:* node --enable-source-maps dist/job-runner.js",
+    "job-runner:dev": "DEBUG=vulcanize:* ts-node src/job-runner.ts",
     "watch:contract": "DEBUG=vulcanize:* ts-node src/cli/watch-contract.ts",
     "fill": "DEBUG=vulcanize:* ts-node src/fill.ts",
     "reset": "DEBUG=vulcanize:* ts-node src/cli/reset.ts",
@@ -32,29 +36,30 @@
     "@apollo/client": "^3.3.19",
     "@ethersproject/providers": "^5.4.4",
     "@ipld/dag-cbor": "^6.0.12",
+    "@vulcanize/graph-node": "^0.1.0",
     "@vulcanize/ipld-eth-client": "^0.1.0",
     "@vulcanize/solidity-mapper": "^0.1.0",
     "@vulcanize/util": "^0.1.0",
-    "@vulcanize/graph-node": "^0.1.0",
     "apollo-server-express": "^2.25.0",
     "apollo-type-bigint": "^0.1.3",
     "debug": "^4.3.1",
+    "decimal.js": "^10.3.1",
     "ethers": "^5.4.4",
     "express": "^4.17.1",
     "graphql": "^15.5.0",
     "graphql-import-node": "^0.0.4",
     "json-bigint": "^1.0.0",
     "reflect-metadata": "^0.1.13",
     "typeorm": "^0.2.32",
-    "yargs": "^17.0.1",
-    "decimal.js": "^10.3.1"
+    "yargs": "^17.0.1"
   },
   "devDependencies": {
     "@ethersproject/abi": "^5.3.0",
     "@types/express": "^4.17.11",
     "@types/yargs": "^17.0.0",
     "@typescript-eslint/eslint-plugin": "^4.25.0",
     "@typescript-eslint/parser": "^4.25.0",
+    "copyfiles": "^2.4.1",
     "eslint": "^7.27.0",
     "eslint-config-semistandard": "^15.0.1",
     "eslint-config-standard": "^16.0.3",

packages/eden-watcher/src/entity/Account.ts

@@ -2,11 +2,12 @@
 // Copyright 2021 Vulcanize, Inc.
 //
 
-import { Entity, PrimaryColumn, Column } from 'typeorm';
+import { Entity, PrimaryColumn, Column, Index } from 'typeorm';
 
 import { bigintTransformer } from '@vulcanize/util';
 
 @Entity()
+@Index(['blockNumber'])
 export class Account {
   @PrimaryColumn('varchar')
   id!: string;

packages/eden-watcher/src/entity/Block.ts

@@ -2,10 +2,11 @@
 // Copyright 2021 Vulcanize, Inc.
 //
 
-import { Entity, PrimaryColumn, Column } from 'typeorm';
+import { Entity, PrimaryColumn, Column, Index } from 'typeorm';
 import { bigintTransformer } from '@vulcanize/util';
 
 @Entity()
+@Index(['blockNumber'])
 export class Block {
   @PrimaryColumn('varchar')
   id!: string;

packages/eden-watcher/src/entity/Claim.ts

@@ -2,11 +2,12 @@
 // Copyright 2021 Vulcanize, Inc.
 //
 
-import { Entity, PrimaryColumn, Column } from 'typeorm';
+import { Entity, PrimaryColumn, Column, Index } from 'typeorm';
 
 import { bigintTransformer } from '@vulcanize/util';
 
 @Entity()
+@Index(['blockNumber'])
 export class Claim {
   @PrimaryColumn('varchar')
   id!: string;

packages/eden-watcher/src/entity/Distribution.ts

@@ -2,11 +2,12 @@
 // Copyright 2021 Vulcanize, Inc.
 //
 
-import { Entity, PrimaryColumn, Column } from 'typeorm';
+import { Entity, PrimaryColumn, Column, Index } from 'typeorm';
 
 import { bigintTransformer } from '@vulcanize/util';
 
 @Entity()
+@Index(['blockNumber'])
 export class Distribution {
   @PrimaryColumn('varchar')
   id!: string;

packages/eden-watcher/src/entity/Distributor.ts

@@ -2,9 +2,10 @@
 // Copyright 2021 Vulcanize, Inc.
 //
 
-import { Entity, PrimaryColumn, Column } from 'typeorm';
+import { Entity, PrimaryColumn, Column, Index } from 'typeorm';
 
 @Entity()
+@Index(['blockNumber'])
 export class Distributor {
   @PrimaryColumn('varchar')
   id!: string;

packages/eden-watcher/src/entity/Epoch.ts

@@ -2,12 +2,13 @@
 // Copyright 2021 Vulcanize, Inc.
 //
 
-import { Entity, PrimaryColumn, Column } from 'typeorm';
+import { Entity, PrimaryColumn, Column, Index } from 'typeorm';
 import Decimal from 'decimal.js';
 
 import { bigintTransformer, decimalTransformer } from '@vulcanize/util';
 
 @Entity()
+@Index(['blockNumber'])
 export class Epoch {
   @PrimaryColumn('varchar')
   id!: string;

packages/eden-watcher/src/entity/Network.ts

@@ -2,11 +2,12 @@
 // Copyright 2021 Vulcanize, Inc.
 //
 
-import { Entity, PrimaryColumn, Column } from 'typeorm';
+import { Entity, PrimaryColumn, Column, Index } from 'typeorm';
 
 import { bigintArrayTransformer, bigintTransformer } from '@vulcanize/util';
 
 @Entity()
+@Index(['blockNumber'])
 export class Network {
   @PrimaryColumn('varchar')
   id!: string;

packages/eden-watcher/src/entity/Producer.ts

@@ -2,10 +2,11 @@
 // Copyright 2021 Vulcanize, Inc.
 //
 
-import { Entity, PrimaryColumn, Column } from 'typeorm';
+import { Entity, PrimaryColumn, Column, Index } from 'typeorm';
 import { bigintTransformer } from '@vulcanize/util';
 
 @Entity()
+@Index(['blockNumber'])
 export class Producer {
   @PrimaryColumn('varchar')
   id!: string;

packages/eden-watcher/src/entity/ProducerEpoch.ts

@@ -2,12 +2,13 @@
 // Copyright 2021 Vulcanize, Inc.
 //
 
-import { Entity, PrimaryColumn, Column } from 'typeorm';
+import { Entity, PrimaryColumn, Column, Index } from 'typeorm';
 import Decimal from 'decimal.js';
 
 import { bigintTransformer, decimalTransformer } from '@vulcanize/util';
 
 @Entity()
+@Index(['blockNumber'])
 export class ProducerEpoch {
   @PrimaryColumn('varchar')
   id!: string;