Skip to content

Commit

Permalink
Update codegen with changes implemented in mobymask watcher (#148)
Browse files Browse the repository at this point in the history
* Update codegen with index-block CLI and remove graph-node

* Add filter logs by contract flag

* Skip generating GQL API for immutable variables

* Add config for maxEventsBlockRange

* Add new flags in existing watchers
  • Loading branch information
nikugogoi authored Jul 22, 2022
1 parent b577db2 commit 1bcabd6
Show file tree
Hide file tree
Showing 69 changed files with 825 additions and 575 deletions.
8 changes: 7 additions & 1 deletion packages/codegen/src/generate-code.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ import { exportState } from './export-state';
import { importState } from './import-state';
import { exportInspectCID } from './inspect-cid';
import { getSubgraphConfig } from './utils/subgraph';
import { exportIndexBlock } from './index-block';

const main = async (): Promise<void> => {
const argv = await yargs(hideBin(process.argv))
Expand Down Expand Up @@ -166,7 +167,7 @@ function generateWatcher (visitor: Visitor, contracts: any[], config: any) {
});

// Register the handlebar helpers to be used in the templates.
registerHandlebarHelpers();
registerHandlebarHelpers(config);

visitor.visitSubgraph(config.subgraphPath);

Expand Down Expand Up @@ -300,6 +301,11 @@ function generateWatcher (visitor: Visitor, contracts: any[], config: any) {
? fs.createWriteStream(path.join(outputDir, 'src/cli/inspect-cid.ts'))
: process.stdout;
exportInspectCID(outStream, config.subgraphPath);

outStream = outputDir
? fs.createWriteStream(path.join(outputDir, 'src/cli/index-block.ts'))
: process.stdout;
exportIndexBlock(outStream);
}

function getConfig (configFile: string): any {
Expand Down
21 changes: 21 additions & 0 deletions packages/codegen/src/index-block.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
//
// Copyright 2022 Vulcanize, Inc.
//

import fs from 'fs';
import path from 'path';
import Handlebars from 'handlebars';
import { Writable } from 'stream';

const TEMPLATE_FILE = './templates/index-block-template.handlebars';

/**
* Writes the index-block file generated from a template to a stream.
* @param outStream A writable output stream to write the index-block file to.
*/
export function exportIndexBlock (outStream: Writable): void {
const templateString = fs.readFileSync(path.resolve(__dirname, TEMPLATE_FILE)).toString();
const template = Handlebars.compile(templateString);
const indexBlock = template({});
outStream.write(indexBlock);
}
14 changes: 10 additions & 4 deletions packages/codegen/src/templates/checkpoint-template.handlebars
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,18 @@
// Copyright 2021 Vulcanize, Inc.
//

{{#if (subgraphPath)}}
import path from 'path';
{{/if}}
import yargs from 'yargs';
import 'reflect-metadata';
import debug from 'debug';
import assert from 'assert';

import { Config, DEFAULT_CONFIG_PATH, getConfig, initClients, JobQueue } from '@vulcanize/util';
{{#if (subgraphPath)}}
import { GraphWatcher, Database as GraphDatabase } from '@vulcanize/graph-node';
{{/if}}

import { Database } from '../database';
import { Indexer } from '../indexer';
Expand Down Expand Up @@ -45,11 +49,13 @@ const main = async (): Promise<void> => {

const db = new Database(config.database);
await db.init();

{{#if (subgraphPath)}}

const graphDb = new GraphDatabase(config.database, path.resolve(__dirname, 'entity/*'));
await graphDb.init();

const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
{{/if}}

const jobQueueConfig = config.jobQueue;
assert(jobQueueConfig, 'Missing job queue config');
Expand All @@ -60,11 +66,11 @@ const main = async (): Promise<void> => {
const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs });
await jobQueue.start();

const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue, graphWatcher);
const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue{{#if (subgraphPath)}}, graphWatcher{{/if}});
await indexer.init();
{{#if (subgraphPath)}}

graphWatcher.setIndexer(indexer);
{{#if subgraphPath}}
await graphWatcher.init();
{{/if}}

Expand Down
9 changes: 8 additions & 1 deletion packages/codegen/src/templates/config-template.handlebars
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,17 @@
# IPFS API address (can be taken from the output on running the IPFS daemon).
ipfsApiAddr = "/ip4/127.0.0.1/tcp/5001"

{{#if subgraphPath}}
{{#if (subgraphPath)}}
subgraphPath = "{{subgraphPath}}"
wasmRestartBlocksInterval = 20

{{/if}}
# Boolean to filter logs by contract.
filterLogs = false

# Max block range for which to return events in eventsInRange GQL query.
# Use -1 for skipping check on block range.
maxEventsBlockRange = 1000

[database]
type = "postgres"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,9 @@ import fs from 'fs';
import path from 'path';

import { Config, DEFAULT_CONFIG_PATH, getConfig, initClients, JobQueue, StateKind } from '@vulcanize/util';
{{#if (subgraphPath)}}
import { GraphWatcher, Database as GraphDatabase } from '@vulcanize/graph-node';
{{/if}}
import * as codec from '@ipld/dag-cbor';

import { Database } from '../database';
Expand Down Expand Up @@ -42,11 +44,13 @@ const main = async (): Promise<void> => {

const db = new Database(config.database);
await db.init();
{{#if (subgraphPath)}}

const graphDb = new GraphDatabase(config.database, path.resolve(__dirname, 'entity/*'));
await graphDb.init();

const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
{{/if}}

const jobQueueConfig = config.jobQueue;
assert(jobQueueConfig, 'Missing job queue config');
Expand All @@ -57,11 +61,11 @@ const main = async (): Promise<void> => {
const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs });
await jobQueue.start();

const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue, graphWatcher);
const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue{{#if (subgraphPath)}}, graphWatcher{{/if}});
await indexer.init();
{{#if (subgraphPath)}}

graphWatcher.setIndexer(indexer);
{{#if subgraphPath}}
await graphWatcher.init();
{{/if}}

Expand Down
14 changes: 10 additions & 4 deletions packages/codegen/src/templates/fill-template.handlebars
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,9 @@
// Copyright 2021 Vulcanize, Inc.
//

{{#if (subgraphPath)}}
import path from 'path';
{{/if}}
import assert from 'assert';
import 'reflect-metadata';
import yargs from 'yargs';
Expand All @@ -11,7 +13,9 @@ import debug from 'debug';
import { PubSub } from 'apollo-server-express';

import { Config, getConfig, fillBlocks, JobQueue, DEFAULT_CONFIG_PATH, initClients } from '@vulcanize/util';
{{#if (subgraphPath)}}
import { GraphWatcher, Database as GraphDatabase } from '@vulcanize/graph-node';
{{/if}}

import { Database } from './database';
import { Indexer } from './indexer';
Expand Down Expand Up @@ -57,11 +61,13 @@ export const main = async (): Promise<any> => {

const db = new Database(config.database);
await db.init();

{{#if (subgraphPath)}}

const graphDb = new GraphDatabase(config.database, path.resolve(__dirname, 'entity/*'));
await graphDb.init();

const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
{{/if}}

const jobQueueConfig = config.jobQueue;
assert(jobQueueConfig, 'Missing job queue config');
Expand All @@ -72,11 +78,11 @@ export const main = async (): Promise<any> => {
const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs });
await jobQueue.start();

const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue, graphWatcher);
const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue{{#if (subgraphPath)}}, graphWatcher{{/if}});
await indexer.init();
{{#if (subgraphPath)}}

graphWatcher.setIndexer(indexer);
{{#if subgraphPath}}
await graphWatcher.init();
{{/if}}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,9 @@ import fs from 'fs';
import path from 'path';

import { getConfig, fillBlocks, JobQueue, DEFAULT_CONFIG_PATH, Config, initClients, StateKind } from '@vulcanize/util';
{{#if (subgraphPath)}}
import { GraphWatcher, Database as GraphDatabase } from '@vulcanize/graph-node';
{{/if}}
import * as codec from '@ipld/dag-cbor';

import { Database } from '../database';
Expand Down Expand Up @@ -46,11 +48,13 @@ export const main = async (): Promise<any> => {

const db = new Database(config.database);
await db.init();
{{#if (subgraphPath)}}

const graphDb = new GraphDatabase(config.database, path.resolve(__dirname, 'entity/*'));
await graphDb.init();

const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
{{/if}}

// Note: In-memory pubsub works fine for now, as each watcher is a single process anyway.
// Later: https://www.apollographql.com/docs/apollo-server/data/subscriptions/#production-pubsub-libraries
Expand All @@ -65,11 +69,11 @@ export const main = async (): Promise<any> => {
const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs });
await jobQueue.start();

const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue, graphWatcher);
const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue{{#if (subgraphPath)}}, graphWatcher{{/if}});
await indexer.init();
{{#if (subgraphPath)}}

graphWatcher.setIndexer(indexer);
{{#if subgraphPath}}
await graphWatcher.init();
{{/if}}

Expand Down
81 changes: 81 additions & 0 deletions packages/codegen/src/templates/index-block-template.handlebars
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
//
// Copyright 2022 Vulcanize, Inc.
//

{{#if (subgraphPath)}}
import path from 'path';
{{/if}}
import yargs from 'yargs';
import 'reflect-metadata';
import debug from 'debug';
import assert from 'assert';

import { Config, DEFAULT_CONFIG_PATH, getConfig, initClients, JobQueue, indexBlock } from '@vulcanize/util';
{{#if (subgraphPath)}}
import { GraphWatcher, Database as GraphDatabase } from '@vulcanize/graph-node';
{{/if}}

import { Database } from '../database';
import { Indexer } from '../indexer';

const log = debug('vulcanize:index-block');

const main = async (): Promise<void> => {
const argv = await yargs.parserConfiguration({
'parse-numbers': false
}).options({
configFile: {
alias: 'f',
type: 'string',
require: true,
demandOption: true,
describe: 'Configuration file path (toml)',
default: DEFAULT_CONFIG_PATH
},
block: {
type: 'number',
require: true,
demandOption: true,
describe: 'Block number to index'
}
}).argv;

const config: Config = await getConfig(argv.configFile);
const { ethClient, ethProvider } = await initClients(config);

const db = new Database(config.database);
await db.init();
{{#if (subgraphPath)}}

const graphDb = new GraphDatabase(config.database, path.resolve(__dirname, 'entity/*'));
await graphDb.init();

const graphWatcher = new GraphWatcher(graphDb, ethClient, ethProvider, config.server);
{{/if}}

const jobQueueConfig = config.jobQueue;
assert(jobQueueConfig, 'Missing job queue config');

const { dbConnectionString, maxCompletionLagInSecs } = jobQueueConfig;
assert(dbConnectionString, 'Missing job queue db connection string');

const jobQueue = new JobQueue({ dbConnectionString, maxCompletionLag: maxCompletionLagInSecs });

const indexer = new Indexer(config.server, db, ethClient, ethProvider, jobQueue{{#if (subgraphPath)}}, graphWatcher{{/if}});
await indexer.init();
{{#if (subgraphPath)}}

graphWatcher.setIndexer(indexer);
await graphWatcher.init();
{{/if}}

await indexBlock(indexer, jobQueueConfig.eventsInBatch, argv);

await db.close();
};

main().catch(err => {
log(err);
}).finally(() => {
process.exit(0);
});
Loading

0 comments on commit 1bcabd6

Please sign in to comment.