Implement cache for latest updated entities to be used in mapping code (

#194)

* Load relations according to GQL query

* Implement cache for latest entities to used in mapping code

* Add metrics for cache hit and fix caching pruned entities

* Changes in codegen and graph-test-watcher

* Remove entity load counter reset to zero

packages/codegen/src/templates/indexer-template.handlebars

@@ -8,6 +8,7 @@ import { DeepPartial, FindConditions, FindManyOptions } from 'typeorm';
 import JSONbig from 'json-bigint';
 import { ethers } from 'ethers';
 import _ from 'lodash';
+import { SelectionNode } from 'graphql';
 
 import { JsonFragment } from '@ethersproject/abi';
 import { BaseProvider } from '@ethersproject/providers';
@@ -350,12 +351,16 @@ export class Indexer implements IPLDIndexerInterface {
     return createStateCheckpoint(this, contractAddress, blockHash);
   }
 
-  async processCanonicalBlock (blockHash: string): Promise<void> {
+  async processCanonicalBlock (blockHash: string, blockNumber: number): Promise<void> {
     // Finalize staged diff blocks if any.
     await this._baseIndexer.finalizeDiffStaged(blockHash);
 
     // Call custom stateDiff hook.
     await createStateDiff(this, blockHash);
+    {{#if (subgraphPath)}}
+
+    this._graphWatcher.pruneEntityCacheFrothyBlocks(blockHash, blockNumber);
+    {{/if}}
   }
 
   async processCheckpoint (blockHash: string): Promise<void> {
@@ -442,8 +447,13 @@ export class Indexer implements IPLDIndexerInterface {
   }
 
   {{#if (subgraphPath)}}
-  async getSubgraphEntity<Entity> (entity: new () => Entity, id: string, block?: BlockHeight): Promise<any> {
-    const data = await this._graphWatcher.getEntity(entity, id, this._relationsMap, block);
+  async getSubgraphEntity<Entity> (
+    entity: new () => Entity,
+    id: string,
+    block: BlockHeight,
+    selections: ReadonlyArray<SelectionNode> = []
+  ): Promise<any> {
+    const data = await this._graphWatcher.getEntity(entity, id, this._relationsMap, block, selections);
 
     return data;
   }
@@ -466,9 +476,13 @@ export class Indexer implements IPLDIndexerInterface {
     await this.triggerIndexingOnEvent(event);
   }
 
-  async processBlock (blockHash: string, blockNumber: number): Promise<void> {
+  async processBlock (blockProgress: BlockProgress): Promise<void> {
     // Call a function to create initial state for contracts.
-    await this._baseIndexer.createInit(this, blockHash, blockNumber);
+    await this._baseIndexer.createInit(this, blockProgress.blockHash, blockProgress.blockNumber);
+
+    {{#if (subgraphPath)}}
+    this._graphWatcher.updateEntityCacheFrothyBlocks(blockProgress);
+    {{/if}}
   }
 
   {{#if (subgraphPath)}}

packages/codegen/src/templates/job-runner-template.handlebars

@@ -105,7 +105,7 @@ export class JobRunner {
       }
 
       // Process the hooks for the given block number.
-      await this._indexer.processCanonicalBlock(blockHash);
+      await this._indexer.processCanonicalBlock(blockHash, blockNumber);
 
       // Update the IPLD status.
       await this._indexer.updateIPLDStatusHooksBlock(blockNumber);

packages/codegen/src/templates/resolvers-template.handlebars

@@ -6,7 +6,7 @@ import assert from 'assert';
 import BigInt from 'apollo-type-bigint';
 import debug from 'debug';
 import Decimal from 'decimal.js';
-import { GraphQLScalarType } from 'graphql';
+import { GraphQLResolveInfo, GraphQLScalarType } from 'graphql';
 
 import { ValueResult, BlockHeight, gqlTotalQueryCount, gqlQueryCount, jsonBigIntStringReplacer } from '@cerc-io/util';
 
@@ -78,12 +78,18 @@ export const createResolvers = async (indexer: Indexer, eventWatcher: EventWatch
       {{/each}}
 
       {{~#each subgraphQueries}}
-      {{this.queryName}}: async (_: any, { id, block = {} }: { id: string, block: BlockHeight }) => {
+      {{this.queryName}}: async (
+         _: any,
+        { id, block = {} }: { id: string, block: BlockHeight },
+        __: any,
+        info: GraphQLResolveInfo
+      ) => {
         log('{{this.queryName}}', id, JSON.stringify(block, jsonBigIntStringReplacer));
         gqlTotalQueryCount.inc(1);
         gqlQueryCount.labels('{{this.queryName}}').inc(1);
+        assert(info.fieldNodes[0].selectionSet);
 
-        return indexer.getSubgraphEntity({{this.entityName}}, id, block);
+        return indexer.getSubgraphEntity({{this.entityName}}, id, block, info.fieldNodes[0].selectionSet.selections);
       },
 
       {{/each}}

packages/eden-watcher/src/indexer.ts

@@ -8,6 +8,7 @@ import { DeepPartial, FindConditions, FindManyOptions } from 'typeorm';
 import JSONbig from 'json-bigint';
 import { ethers } from 'ethers';
 import _ from 'lodash';
+import { SelectionNode } from 'graphql';
 
 import { JsonFragment } from '@ethersproject/abi';
 import { BaseProvider } from '@ethersproject/providers';
@@ -264,16 +265,16 @@ export class Indexer implements IPLDIndexerInterface {
     return createStateCheckpoint(this, contractAddress, blockHash);
   }
 
-  async processCanonicalBlock (blockHash: string): Promise<void> {
+  async processCanonicalBlock (blockHash: string, blockNumber: number): Promise<void> {
     console.time('time:indexer#processCanonicalBlock-finalize_auto_diffs');
-
     // Finalize staged diff blocks if any.
     await this._baseIndexer.finalizeDiffStaged(blockHash);
-
     console.timeEnd('time:indexer#processCanonicalBlock-finalize_auto_diffs');
 
     // Call custom stateDiff hook.
     await createStateDiff(this, blockHash);
+
+    this._graphWatcher.pruneEntityCacheFrothyBlocks(blockHash, blockNumber);
   }
 
   async processCheckpoint (blockHash: string): Promise<void> {
@@ -371,14 +372,20 @@ export class Indexer implements IPLDIndexerInterface {
     await this._baseIndexer.removeIPLDBlocks(blockNumber, kind);
   }
 
-  async getSubgraphEntity<Entity> (entity: new () => Entity, id: string, block?: BlockHeight): Promise<any> {
-    const data = await this._graphWatcher.getEntity(entity, id, this._relationsMap, block);
+  async getSubgraphEntity<Entity> (entity: new () => Entity, id: string, block: BlockHeight, selections: ReadonlyArray<SelectionNode> = []): Promise<any> {
+    const data = await this._graphWatcher.getEntity(entity, id, this._relationsMap, block, selections);
 
     return data;
   }
 
-  async getSubgraphEntities<Entity> (entity: new () => Entity, block: BlockHeight, where: { [key: string]: any } = {}, queryOptions: QueryOptions = {}): Promise<any[]> {
-    return this._graphWatcher.getEntities(entity, this._relationsMap, block, where, queryOptions);
+  async getSubgraphEntities<Entity> (
+    entity: new () => Entity,
+    block: BlockHeight,
+    where: { [key: string]: any } = {},
+    queryOptions: QueryOptions = {},
+    selections: ReadonlyArray<SelectionNode> = []
+  ): Promise<any[]> {
+    return this._graphWatcher.getEntities(entity, this._relationsMap, block, where, queryOptions, selections);
   }
 
   async triggerIndexingOnEvent (event: Event): Promise<void> {
@@ -400,13 +407,13 @@ export class Indexer implements IPLDIndexerInterface {
     await this.triggerIndexingOnEvent(event);
   }
 
-  async processBlock (blockHash: string, blockNumber: number): Promise<void> {
+  async processBlock (blockProgress: BlockProgress): Promise<void> {
     console.time('time:indexer#processBlock-init_state');
-
     // Call a function to create initial state for contracts.
-    await this._baseIndexer.createInit(this, blockHash, blockNumber);
-
+    await this._baseIndexer.createInit(this, blockProgress.blockHash, blockProgress.blockNumber);
     console.timeEnd('time:indexer#processBlock-init_state');
+
+    this._graphWatcher.updateEntityCacheFrothyBlocks(blockProgress);
   }
 
   async processBlockAfterEvents (blockHash: string): Promise<void> {

packages/eden-watcher/src/job-runner.ts

@@ -102,7 +102,7 @@ export class JobRunner {
       }
 
       // Process the hooks for the given block number.
-      await this._indexer.processCanonicalBlock(blockHash);
+      await this._indexer.processCanonicalBlock(blockHash, blockNumber);
 
       // Update the IPLD status.
       await this._indexer.updateIPLDStatusHooksBlock(blockNumber);