From 9da2fb4f32de9e337091a0c8899fe2f07a25139f Mon Sep 17 00:00:00 2001 From: Ramil Amerzyanov Date: Wed, 7 Apr 2021 15:32:48 +0300 Subject: [PATCH] Contracttest (#47) * fix saving storage for mapping * increase pg connection limit * fix merge issue * add fk to address table from contract_address_slots table to be able to derive mapping storage by address * fill address table and all related from events. Match addresses with storage * fix tests * fix linter warnings * fix issue with duplicated FK * check data format correct * fix db connection leak --- docker-compose.yml | 1 + ormconfig.js | 5 + .../data/addressIdSlotIdRepository.ts | 34 +- src/repositories/data/addressRepository.ts | 12 +- src/repositories/data/eventRepository.ts | 3 +- src/repositories/data/slotRepository.ts | 10 + src/run.ts | 8 +- src/server.ts | 3 +- src/services/dataService.ts | 173 ++++-- src/services/dataTypeParser.test.ts | 524 +++++++++++------- src/services/dataTypeParser.ts | 54 +- src/services/decodeService.ts | 4 +- src/store.ts | 4 +- src/types/index.ts | 1 + 14 files changed, 560 insertions(+), 276 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 40e9b6f..10a090c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -60,6 +60,7 @@ services: POSTGRES_USER: "vdbm" POSTGRES_DB: "vulcanize_public" POSTGRES_PASSWORD: "password" + command: ["postgres", "-c", "log_statement=all", "-N", "500"] volumes: - contract_watcher_js_db_data:/var/lib/postgresql/data ports: diff --git a/ormconfig.js b/ormconfig.js index ad40215..eb93bc9 100644 --- a/ormconfig.js +++ b/ormconfig.js @@ -9,6 +9,11 @@ module.exports = { 'password': env.DATABASE_PASSWORD, 'database': env.DATABASE_NAME, 'logging': env.DATABASE_LOGGING, + extra: { + max: 100, + idleTimeoutMillis: 5000, + //log: console.log + }, "entities": process.env.NODE_ENV === 'production' ? [__dirname + "/dist/models/*.js", __dirname + "/dist/models/**/*.js",] : ["src/models/*.ts", "src/models/**/*.ts"], diff --git a/src/repositories/data/addressIdSlotIdRepository.ts b/src/repositories/data/addressIdSlotIdRepository.ts index 86847a4..1cae3af 100644 --- a/src/repositories/data/addressIdSlotIdRepository.ts +++ b/src/repositories/data/addressIdSlotIdRepository.ts @@ -1,5 +1,6 @@ import {EntityRepository, QueryRunner, Table} from 'typeorm'; import { TableOptions } from 'typeorm/schema-builder/options/TableOptions'; +import {Structure} from "../../services/dataTypeParser"; @EntityRepository() export default class AddressIdSlotIdRepository { @@ -9,12 +10,16 @@ export default class AddressIdSlotIdRepository { this.queryRunner = queryRunner; } - public async createTable(addressId, slotId): Promise { - const tableName = `data.address_id_${addressId}_slot_id_${slotId}`; + private getTableName(contractId: number, slotId: number): string { + return `data.contract_id_${contractId}_address_slot_id_${slotId}`; + } + + public async createTable(contractId: number, slotId: number): Promise { + const tableName = this.getTableName(contractId, slotId); const table = await this.queryRunner.getTable(tableName); if (table) { - console.log(`Table ${tableName} already exists`); + // console.log(`Table ${tableName} already exists`); return; } @@ -41,16 +46,16 @@ export default class AddressIdSlotIdRepository { console.log('create new table', tableName); } - public async add(cotractAddressId: number, addressId, slotId: number, hash: string): Promise { - const tableName = `data.address_id_${cotractAddressId}_slot_id_${slotId}`; + public async add(contractId: number, addressId, slotId: number, hash: string): Promise { + const tableName = this.getTableName(contractId, slotId); const sql = `INSERT INTO ${tableName} (address_id, hash) VALUES (${addressId}, '${hash}');`; console.log(sql); return this.queryRunner.query(sql); } - public async isExist(cotractAddressId: number, slotId: number, addressId: number): Promise { - const tableName = `data.address_id_${cotractAddressId}_slot_id_${slotId}`; + public async isExist(contractId: number, slotId: number, addressId: number): Promise { + const tableName = this.getTableName(contractId, slotId); const sql = `SELECT * FROM ${tableName} WHERE address_id=${addressId};`; const data = await this.queryRunner.query(sql); @@ -61,8 +66,8 @@ export default class AddressIdSlotIdRepository { return data[0]?.address_id ? true : false; } - public async getAddressIdByHash(cotractAddressId: number, slotId: number, hash: string): Promise { - const tableName = `data.address_id_${cotractAddressId}_slot_id_${slotId}`; + public async getAddressIdByHash(contractId: number, slotId: number, hash: string): Promise { + const tableName = this.getTableName(contractId, slotId); const sql = `SELECT * FROM ${tableName} WHERE hash='${hash}';`; const data = await this.queryRunner.query(sql); @@ -72,4 +77,15 @@ export default class AddressIdSlotIdRepository { return data[0]?.address_id; } + + public async syncAddressSlotHashes(contractId: number, slotId: number, stateStructure: Structure): Promise { + const sql = `UPDATE data.contract_id_${contractId}_state_id_${slotId} a + SET address_id=b.address_id + FROM data.contract_id_${contractId}_address_slot_id_${slotId} b + WHERE + a.address_id IS NULL + AND a.${stateStructure.name}=b.hash`; + + return this.queryRunner.query(sql); + } } diff --git a/src/repositories/data/addressRepository.ts b/src/repositories/data/addressRepository.ts index c55acce..c5dafa1 100644 --- a/src/repositories/data/addressRepository.ts +++ b/src/repositories/data/addressRepository.ts @@ -10,8 +10,16 @@ export default class AddressRepository extends Repository
{ public async add(address: string, hash: string): Promise
{ return this.save({ - address, - hash, + address: address.toLowerCase(), + hash: hash.toLowerCase(), + }); + } + + public async get(address: string): Promise
{ + return await this.findOne({ + where: { + address: address.toLowerCase() + } }); } } diff --git a/src/repositories/data/eventRepository.ts b/src/repositories/data/eventRepository.ts index 6894688..7d3d7e7 100644 --- a/src/repositories/data/eventRepository.ts +++ b/src/repositories/data/eventRepository.ts @@ -1,4 +1,5 @@ import {EntityRepository, QueryRunner} from 'typeorm'; +import {ABIInputData} from "../../types"; @EntityRepository() export default class EventRepository { @@ -8,7 +9,7 @@ export default class EventRepository { this.queryRunner = queryRunner; } - public async add(tableName: string, data): Promise { + public async add(tableName: string, data: Array): Promise { const sql = `INSERT INTO ${tableName} (${data.map((line) => line.isStrict ? line.name : 'data_' + line.name.toLowerCase().trim()).join(',')}) VALUES diff --git a/src/repositories/data/slotRepository.ts b/src/repositories/data/slotRepository.ts index 8e62470..e8a5a82 100644 --- a/src/repositories/data/slotRepository.ts +++ b/src/repositories/data/slotRepository.ts @@ -8,6 +8,16 @@ export default class SlotRepository { this.queryRunner = queryRunner; } + public async getByValue(table: string, name: string, value: string): Promise { + const sql = `SELECT id FROM ${table} WHERE ${name}=$1`; + const res = await this.queryRunner.query(sql, [value]); + if (!res || !res.length) { + return null; + } + + return res[0].id; + } + public async add(tableName: string, name: string[], value): Promise { const sql = `INSERT INTO ${tableName} (${name.join(',')}) VALUES ('${value.map((v) => v.toString().replace(/\0/g, '')).join('\',\'')}') RETURNING id;`; console.log(sql); diff --git a/src/run.ts b/src/run.ts index 6eb922b..3428588 100644 --- a/src/run.ts +++ b/src/run.ts @@ -21,6 +21,12 @@ process.on('unhandledRejection', (reason, p) => { dataService.processState(data.relatedNode) }, processHeader: (data) => dataService.processHeader(data), - processEvent: (data) => dataService.processEvent(null, data.relatedNode, data.decoded, data.event), + processEvent: (data) => { + if (!data) { + return; + } + + dataService.processEvent(null, data.relatedNode, data.decoded, data.event); + }, }) })(); diff --git a/src/server.ts b/src/server.ts index 793bddd..715a283 100644 --- a/src/server.ts +++ b/src/server.ts @@ -26,7 +26,8 @@ const server = async function({ createConnection(connectionOptions).then(async () => { const app = new App(); - Store.init(); + // @TODO autoUpdate=true breaks + Store.init(false); const graphqlClient = new GraphqlClient(env.GRAPHQL_URI, ws); const graphqlService = new GraphqlService(graphqlClient); diff --git a/src/services/dataService.ts b/src/services/dataService.ts index e721ec2..37109eb 100644 --- a/src/services/dataService.ts +++ b/src/services/dataService.ts @@ -27,7 +27,7 @@ import DecodeService from './decodeService'; import { ABI, ABIElem, - ABIInput, + ABIInput, ABIInputData, EthHeaderCid, EthReceiptCid, EthStateCid, @@ -56,11 +56,6 @@ const INDEX = [ '000000000000000000000000000000000000000000000000000000000000000c', // 12 ]; -type ABIInputData = { - name: string; - value?: any; // eslint-disable-line -} - export default class DataService { public async createTables(contracts: Contract[] = []): Promise { @@ -195,6 +190,17 @@ VALUES relatedNode.mhKey, ); + for (const item of decoded) { + if (item.type === 'address') { + // create address if it doesn't exist + const addressModel = await this.saveAddress(`0x${item.value}`); + // fill address slot tables + await this.fillAddressSlotTables(target.contractId, addressModel); + + await this.matchAddressAndHash(target.contractId); + } + } + console.log(`Event ${event.name} saved`); } @@ -316,7 +322,7 @@ VALUES return; } - console.log(JSON.stringify(relatedNode, null, 2)); + //console.log(JSON.stringify(relatedNode, null, 2)); await this.processHeader(relatedNode?.ethHeaderCidByHeaderId); @@ -328,7 +334,7 @@ VALUES for (const state of states) { const structure = toStructure(state.type, state.variable); - console.log('structure', structure); + //console.log('structure', structure); const tableName = DataService._getTableName({ contractId: contract.contractId, @@ -336,11 +342,12 @@ VALUES id: state.stateId, }); const tableOptions = toTableOptions(tableName, structure) - console.log('tableOptions', JSON.stringify(tableOptions, null, 2)); + //console.log('tableOptions', JSON.stringify(tableOptions, null, 2)); if (structure.type === 'mapping') { - const addressIdSlotIdRepository: AddressIdSlotIdRepository = new AddressIdSlotIdRepository(getConnection().createQueryRunner()); - const slotRepository: SlotRepository = new SlotRepository(getConnection().createQueryRunner()); + const queryRunner = getConnection().createQueryRunner(); + const addressIdSlotIdRepository: AddressIdSlotIdRepository = new AddressIdSlotIdRepository(queryRunner); + const slotRepository: SlotRepository = new SlotRepository(queryRunner); if (structure.value.type === 'simple') { for (const storage of relatedNode?.storageCidsByStateId?.nodes) { @@ -348,6 +355,7 @@ VALUES // const addressId = await addressIdSlotIdRepository.getAddressIdByHash(address.addressId, state.stateId, storage.storageLeafKey); if (!storage.storageLeafKey) { + console.log('storage mapping. skip storage without leaf key'); continue; } @@ -359,34 +367,49 @@ VALUES let value; if (structure.value.kind === 'string') { - value = toAscii(storageDataDecoded.toString('hex')); + const result = this.deriveStringFromStorage(state.slot, relatedNode?.storageCidsByStateId?.nodes); + if (!result.success) { + continue; + } + + value = result.result; } else { value = abi.rawDecode([ structure.value.kind ], storageDataDecoded)[0]; } - console.log(decoded); - console.log(rlp.decode(Buffer.from(decoded[1], 'hex'))); + // console.log(decoded); + // console.log(rlp.decode(Buffer.from(decoded[1], 'hex'))); + // + // console.log(decoded[0].toString('hex')); + // console.log(value); - console.log(decoded[0].toString('hex')); - console.log(value); + console.log('saving state', structure.value.name, value); - const id = await slotRepository.add(tableOptions[0].name, [structure.name], [decoded[0].toString('hex')]); + // try to derive address by storageLeafKey (keccack256(slot+address)) + const addressId = await addressIdSlotIdRepository.getAddressIdByHash(contract.contractId, state.stateId, storage.storageLeafKey); + const id: number = await this.saveStorageKeyForMapping(tableOptions[0].name, structure.name, storage.storageLeafKey, addressId); await slotRepository.add(tableOptions[1].name, [ + 'state_id', + 'contract_id', + 'mh_key', `${structure.name}_id`, structure.value.name, ], [ + state.stateId, + contract.contractId, + storage.blockByMhKey.key, id, value, ]); } } else if (structure.value.type === 'struct') { - // asd mmaping -> struct + // mapping -> struct console.log('structure.value', structure.value.fields); let storageLeafKey; let addressId; for (const storage of relatedNode?.storageCidsByStateId?.nodes) { - addressId = await addressIdSlotIdRepository.getAddressIdByHash(contractAddress.addressId, state.stateId, storage.storageLeafKey); + addressId = await addressIdSlotIdRepository.getAddressIdByHash(contract.contractId, state.stateId, storage.storageLeafKey); if (!addressId) { continue; @@ -412,7 +435,7 @@ VALUES for (const field of structure.value.fields) { if (field.type === 'simple') { const storage = relatedNode?.storageCidsByStateId?.nodes.find((s) => s.storageLeafKey === hashes[index]); - console.log('storageLeafKey', hashes[index]); + // console.log('storageLeafKey', hashes[index]); index++; if (!storage) { @@ -442,15 +465,18 @@ VALUES } else { // TODO } + + await queryRunner.release(); } else if (structure.type === 'struct') { - const slotRepository: SlotRepository = new SlotRepository(getConnection().createQueryRunner()); + const queryRunner = getConnection().createQueryRunner(); + const slotRepository: SlotRepository = new SlotRepository(queryRunner); let index = state.slot; const data: { name: string; value: any }[] = []; // eslint-disable-line for (const field of structure.fields) { if (field.type === 'simple') { const storageLeafKey = DataService._getKeyForFixedType(index); - console.log('storageLeafKey', storageLeafKey); + // console.log('storageLeafKey', storageLeafKey); index++; const storage = relatedNode?.storageCidsByStateId?.nodes.find((s) => s.storageLeafKey === storageLeafKey); @@ -474,12 +500,12 @@ VALUES } await slotRepository.add(tableOptions[0].name, data.map((d) => d.name), data.map((d) => d.value)); + await queryRunner.release(); } else if (structure.type === 'simple') { const storageLeafKey = DataService._getKeyForFixedType(state.slot); - console.log('storageLeafKey', storageLeafKey); + // console.log('storageLeafKey', storageLeafKey); const storage = relatedNode?.storageCidsByStateId?.nodes.find((s) => s.storageLeafKey === storageLeafKey); - console.log('storage', storage); if (!storage) { continue; } @@ -709,34 +735,105 @@ VALUES return notSyncedIds; } - public async prepareAddresses(contracts: Contract[] = []): Promise { + private async saveAddress(address: string): Promise
{ const addressRepository: AddressRepository = getConnection().getCustomRepository(AddressRepository); - const addressIdSlotIdRepository: AddressIdSlotIdRepository = new AddressIdSlotIdRepository(getConnection().createQueryRunner()); + + let addressModel: Address = await addressRepository.get(address); + if (!addressModel) { + const hash = '0x' + keccakFromHexString(address).toString('hex'); + addressModel = await addressRepository.add(address, hash); + } + + return addressModel; + } + + // saveStorageKey used to save hierarchy structure for mapping + private async saveStorageKeyForMapping(tableName: string, name: string, value: string, addressId: number): Promise { + const queryRunner = getConnection().createQueryRunner(); + const slotRepository: SlotRepository = new SlotRepository(queryRunner); + + let id = await slotRepository.getByValue(tableName, name, value) + if (!id) { + const names = [name]; + const values = [value]; + if (addressId) { + names.push('address_id'); + values.push(addressId.toString()); + } + id = await slotRepository.add(tableName, names, values); + } + + await queryRunner.release(); + return id; + } + + private async fillAddressSlotTables(contractId: number, address: Address): Promise { + const queryRunner = getConnection().createQueryRunner(); + const addressIdSlotIdRepository: AddressIdSlotIdRepository = new AddressIdSlotIdRepository(queryRunner); + + const states = Store.getStore().getStatesByContractId(contractId); + for (const state of states) { + const structure = toStructure(state.type, state.variable); + if (structure.type === 'mapping' || structure.type === 'struct') { + const isExist = await addressIdSlotIdRepository.isExist(contractId, state.stateId, address.addressId); + if (!isExist) { + const hash = DataService._getKeyForMapping(address.address, state.slot); + await addressIdSlotIdRepository.add(contractId, address.addressId, state.stateId, hash); + } + } + } + + await queryRunner.release(); + } + + /* + we have tables: + - contract_id_${contractId}_address_slot_id_${slotId} which contains Address ID and its hash for specific slot + - contract_id_${contractId}_state_id_${slotId} - which stores mapping key hash for specific slot. + we need to match their hashes and update address_id in table contract_id_${contractId}_state_id_${slotId} + */ + private async matchAddressAndHash(contractId: number): Promise { + const queryRunner = getConnection().createQueryRunner(); + const addressIdSlotIdRepository: AddressIdSlotIdRepository = new AddressIdSlotIdRepository(queryRunner); + + const states = Store.getStore().getStatesByContractId(contractId); + for (const state of states) { + const structure = toStructure(state.type, state.variable); + if (structure.type === 'mapping') { + await addressIdSlotIdRepository.syncAddressSlotHashes(contractId, state.stateId, structure); + } + } + await queryRunner.release(); + } + + public async prepareAddresses(contracts: Contract[] = []): Promise { + const queryRunner = getConnection().createQueryRunner(); + const addressIdSlotIdRepository: AddressIdSlotIdRepository = new AddressIdSlotIdRepository(queryRunner); for (const contract of contracts) { - let address: Address = Store.getStore().getAddress(contract.address); - if (!address) { - const hash = '0x' + keccakFromHexString(contract.address).toString('hex'); - address = await addressRepository.add(contract.address, hash); - Store.getStore().addAddress(address); + let contractAddress: Address = Store.getStore().getAddress(contract.address); + if (!contractAddress) { + contractAddress = await this.saveAddress(contract.address); } + Store.getStore().addAddress(contractAddress); const states = Store.getStore().getStatesByContractId(contract.contractId); for (const state of states) { const structure = toStructure(state.type, state.variable); if (structure.type === 'mapping' || structure.type === 'struct') { - await addressIdSlotIdRepository.createTable(address.addressId, state.stateId); + await addressIdSlotIdRepository.createTable(contract.contractId, state.stateId); const addresses: Address[] = Store.getStore().getAddresses(); for (const adr of addresses) { - const isExist = await addressIdSlotIdRepository.isExist(address.addressId, state.stateId, adr.addressId); + const isExist = await addressIdSlotIdRepository.isExist(contract.contractId, state.stateId, adr.addressId); if (!isExist) { const hash = DataService._getKeyForMapping(adr.address, state.slot); - await addressIdSlotIdRepository.add(address.addressId, adr.addressId, state.stateId, hash); + await addressIdSlotIdRepository.add(contract.contractId, adr.addressId, state.stateId, hash); } } } } } + await queryRunner.release(); } private static _getTableName({ contractId, type = 'event', id}, withSchema = true): string { @@ -850,10 +947,10 @@ VALUES return; } - const tableOptions = toTableOptions(tableName, toStructure(state.type, state.variable)) - await Promise.all( - tableOptions.map((t) => entityManager.queryRunner.createTable(new Table(t), true)) - ); + const tableOptions = toTableOptions(tableName, toStructure(state.type, state.variable)); + for (const tableOption of tableOptions) { + await entityManager.queryRunner.createTable(new Table(tableOption), true) + } console.log('create new table', tableName); }); } diff --git a/src/services/dataTypeParser.test.ts b/src/services/dataTypeParser.test.ts index f2f7011..404e343 100644 --- a/src/services/dataTypeParser.test.ts +++ b/src/services/dataTypeParser.test.ts @@ -250,29 +250,30 @@ describe('toTableOptions', function () { "kind": "string", } as Structure; const tableOptions1 = toTableOptions('test', st1); - expect({ + expect(tableOptions1[0]).toStrictEqual({ "name": "test", + "foreignKeys": [], "columns": [{ - "generationStrategy": "increment", - "isGenerated": true, - "isPrimary": true, - "name": "id", - "type": "integer", - }, { - name: 'state_id', - type: 'integer', - }, { - name: 'contract_id', - type: 'integer', - }, { - name: 'mh_key', - type: 'text', - }, { - "isNullable": true, - "name": "name", - "type": "text", - }], - }).toStrictEqual(tableOptions1[0]); + "generationStrategy": "increment", + "isGenerated": true, + "isPrimary": true, + "name": "id", + "type": "integer", + }, { + name: 'state_id', + type: 'integer', + }, { + name: 'contract_id', + type: 'integer', + }, { + name: 'mh_key', + type: 'text', + }, { + "isNullable": true, + "name": "name", + "type": "text", + }], + }); const st2 = { "type": "simple", @@ -280,29 +281,30 @@ describe('toTableOptions', function () { "kind": "uint8", } as Structure; const tableOptions2 = toTableOptions('test', st2); - expect({ + expect(tableOptions2[0]).toStrictEqual({ "name": "test", + "foreignKeys": [], "columns": [{ - "generationStrategy": "increment", - "isGenerated": true, - "isPrimary": true, - "name": "id", - "type": "integer", - }, { - name: 'state_id', - type: 'integer', - }, { - name: 'contract_id', - type: 'integer', - }, { - name: 'mh_key', - type: 'text', - }, { - "isNullable": true, - "name": "decimals", - "type": "numeric", - }], - }).toStrictEqual(tableOptions2[0]); + "generationStrategy": "increment", + "isGenerated": true, + "isPrimary": true, + "name": "id", + "type": "integer", + }, { + name: 'state_id', + type: 'integer', + }, { + name: 'contract_id', + type: 'integer', + }, { + name: 'mh_key', + type: 'text', + }, { + "isNullable": true, + "name": "decimals", + "type": "numeric", + }], + }); }); test('array types', function () { @@ -317,21 +319,22 @@ describe('toTableOptions', function () { } as Structure; const tableOptions1 = toTableOptions('test', st1); - expect({ + expect(tableOptions1[0]).toStrictEqual({ "name": "test", + "foreignKeys": [], "columns": [{ - "generationStrategy": "increment", - "isGenerated": true, - "isPrimary": true, - "name": "id", - "type": "integer", - }, { - "isArray": true, - "isNullable": true, - "name": "value0", - "type": "text", - }], - }).toStrictEqual(tableOptions1[0]); + "generationStrategy": "increment", + "isGenerated": true, + "isPrimary": true, + "name": "id", + "type": "integer", + }, { + "isArray": true, + "isNullable": true, + "name": "value0", + "type": "text", + }], + }); }); test('mapping types', function () { @@ -349,48 +352,64 @@ describe('toTableOptions', function () { const tableOptions1 = toTableOptions('test', st1); - expect({ + expect(tableOptions1[0]).toStrictEqual({ "name": "test", + "foreignKeys": [{ + name: 'address_id_data.address', + columnNames: ['address_id'], + referencedTableName: 'data.addresses', + referencedColumnNames: ['address_id'], + }], "columns": [{ - "generationStrategy": "increment", - "isGenerated": true, - "isPrimary": true, - "name": "id", - "type": "integer", - }, { - "isNullable": true, - "name": "balances", - "type": "character varying(66)", - }], - }).toStrictEqual(tableOptions1[0]); + "generationStrategy": "increment", + "isGenerated": true, + "isPrimary": true, + "name": "id", + "type": "integer", + }, { + "isNullable": true, + "name": "balances", + "type": "character varying(66)", + }, { + "isNullable": true, + "name": "address_id", + "type": "integer", + }, ], + }); - expect({ + expect(tableOptions1[1]).toStrictEqual({ "name": "test_balances_id", + "foreignKeys": [{ + name: `test_balances_id`, + columnNames: [`balances_id`], + referencedTableName: 'test', + referencedColumnNames: ['id'], + }], "columns": [{ - "generationStrategy": "increment", - "isGenerated": true, - "isPrimary": true, - "name": "id", - "type": "integer", - }, { - "isNullable": false, - "name": "balances_id", - "type": "integer", - }, { - name: 'state_id', - type: 'integer', - }, { - name: 'contract_id', - type: 'integer', - }, { - name: 'mh_key', - type: 'text', - }, { - "isNullable": true, - "name": "value0", - "type": "numeric", - }], - }).toStrictEqual(tableOptions1[1]); + "generationStrategy": "increment", + "isGenerated": true, + "isPrimary": true, + "name": "id", + "type": "integer", + }, { + "isNullable": false, + "name": "balances_id", + "type": "integer", + }, { + name: 'state_id', + type: 'integer', + }, { + name: 'contract_id', + type: 'integer', + }, { + name: 'mh_key', + type: 'text', + }, { + "isNullable": true, + "name": "value0", + "type": "numeric", + }], + }); // mapping (address => mapping (address => uint96)) internal allowances; @@ -427,42 +446,73 @@ describe('toTableOptions', function () { const tableOptions2 = toTableOptions('test', st2); - expect({ + expect(tableOptions2[0]).toStrictEqual({ "name": "test", + "foreignKeys": [{ + name: `address_id_data.address`, + columnNames: [`address_id`], + referencedTableName: 'data.addresses', + referencedColumnNames: ['address_id'], + }], "columns": [{ - "generationStrategy": "increment", - "isGenerated": true, - "isPrimary": true, - "name": "id", - "type": "integer", - }, { - "isNullable": true, - "name": "allowances", - "type": "character varying(66)", - }], - }).toStrictEqual(tableOptions2[0]); - - expect({ - "name": "test_allowances_id", - "columns": [{ "generationStrategy": "increment", "isGenerated": true, "isPrimary": true, "name": "id", "type": "integer", }, { - "name": "allowances_id", - "type": "integer", - "isNullable": false - },{ "isNullable": true, - "name": "value0", + "name": "allowances", "type": "character varying(66)", + }, { + name: 'address_id', + type: 'integer', + isNullable: true, }], - }).toStrictEqual(tableOptions2[1]); + }); + + expect(tableOptions2[1]).toStrictEqual({ + "name": "test_allowances_id", + "foreignKeys": [{ + name: `test_allowances_id`, + columnNames: [`allowances_id`], + referencedTableName: 'test', + referencedColumnNames: ['id'], + }, { + name: `address_id_data.address`, + columnNames: [`address_id`], + referencedTableName: 'data.addresses', + referencedColumnNames: ['address_id'], + },], + "columns": [{ + "generationStrategy": "increment", + "isGenerated": true, + "isPrimary": true, + "name": "id", + "type": "integer", + }, { + "name": "allowances_id", + "type": "integer", + "isNullable": false + },{ + "isNullable": true, + "name": "value0", + "type": "character varying(66)", + }, { + "isNullable": true, + "name": "address_id", + "type": "integer", + },], + }); expect({ "name": "test_allowances_id_value0_id", + "foreignKeys": [{ + name: `test_allowances_id_value0_id`, + columnNames: [`value0_id`], + referencedTableName: 'test_allowances_id', + referencedColumnNames: ['id'], + }], "columns": [{ "generationStrategy": "increment", "isGenerated": true, @@ -511,6 +561,7 @@ describe('toTableOptions', function () { expect({ "name": "test", + "foreignKeys": [], "columns": [{ "generationStrategy": "increment", "isGenerated": true, @@ -552,6 +603,7 @@ describe('toTableOptions', function () { expect({ "name": "test", + "foreignKeys": [], "columns": [{ "generationStrategy": "increment", "isGenerated": true, @@ -561,28 +613,34 @@ describe('toTableOptions', function () { }], }).toStrictEqual(tableOptions2[0]); - expect({ + expect(tableOptions2[1]).toStrictEqual({ "name": "test_checkpoint_id", + "foreignKeys": [{ + name: `test_checkpoint_id`, + columnNames: [`checkpoint_id`], + referencedTableName: 'test', + referencedColumnNames: ['checkpoint'], + }], "columns": [{ - "generationStrategy": "increment", - "isGenerated": true, - "isPrimary": true, - "name": "id", - "type": "integer", - }, { - "name": "checkpoint_id", - "type": "integer", - "isNullable": false - }, { - "name": "fromBlock", - "type": "numeric", - "isNullable": true, - },{ - "name": "votes", - "type": "numeric", - "isNullable": true, - }], - }).toStrictEqual(tableOptions2[1]); + "generationStrategy": "increment", + "isGenerated": true, + "isPrimary": true, + "name": "id", + "type": "integer", + }, { + "name": "checkpoint_id", + "type": "integer", + "isNullable": false + }, { + "name": "fromBlock", + "type": "numeric", + "isNullable": true, + },{ + "name": "votes", + "type": "numeric", + "isNullable": true, + }], + }); /* mapping(address => Checkpoint) public checkpoint; @@ -607,43 +665,59 @@ describe('toTableOptions', function () { const tableOptions3 = toTableOptions('test', st3); - expect({ + expect(tableOptions3[0]).toStrictEqual({ "name": "test", + "foreignKeys": [{ + name: 'address_id_data.address', + columnNames: ['address_id'], + referencedTableName: 'data.addresses', + referencedColumnNames: ['address_id'], + }], "columns": [{ - "generationStrategy": "increment", - "isGenerated": true, - "isPrimary": true, - "name": "id", - "type": "integer", - }, { - "isNullable": true, - "name": "checkpoint", - "type": "character varying(66)", - }], - }).toStrictEqual(tableOptions3[0]); + "generationStrategy": "increment", + "isGenerated": true, + "isPrimary": true, + "name": "id", + "type": "integer", + }, { + "isNullable": true, + "name": "checkpoint", + "type": "character varying(66)", + }, { + name: 'address_id', + type: 'integer', + isNullable: true, + }], + }); - expect({ + expect(tableOptions3[1]).toStrictEqual({ "name": "test_checkpoint_id", + "foreignKeys": [{ + name: 'test_checkpoint_id', + columnNames: ['checkpoint_id'], + referencedTableName: 'test', + referencedColumnNames: ['id'], + }], "columns": [{ - "generationStrategy": "increment", - "isGenerated": true, - "isPrimary": true, - "name": "id", - "type": "integer", - }, { - "name": "checkpoint_id", - "type": "integer", - "isNullable": false - }, { - "name": "fromBlock", - "type": "numeric", - "isNullable": true, - },{ - "name": "votes", - "type": "numeric", - "isNullable": true, - }], - }).toStrictEqual(tableOptions3[1]); + "generationStrategy": "increment", + "isGenerated": true, + "isPrimary": true, + "name": "id", + "type": "integer", + }, { + "name": "checkpoint_id", + "type": "integer", + "isNullable": false + }, { + "name": "fromBlock", + "type": "numeric", + "isNullable": true, + },{ + "name": "votes", + "type": "numeric", + "isNullable": true, + }], + }); /* mapping(address => mapping(uint => Checkpoint)) public checkpoint; @@ -673,61 +747,83 @@ describe('toTableOptions', function () { const tableOptions4 = toTableOptions('test', st4); - expect({ + expect(tableOptions4[0]).toStrictEqual({ "name": "test", + "foreignKeys": [{ + name: 'address_id_data.address', + columnNames: ['address_id'], + referencedTableName: 'data.addresses', + referencedColumnNames: ['address_id'], + }], "columns": [{ - "generationStrategy": "increment", - "isGenerated": true, - "isPrimary": true, - "name": "id", - "type": "integer", - }, { - "isNullable": true, - "name": "checkpoint", - "type": "character varying(66)", - }], - }).toStrictEqual(tableOptions4[0]); + "generationStrategy": "increment", + "isGenerated": true, + "isPrimary": true, + "name": "id", + "type": "integer", + }, { + "isNullable": true, + "name": "checkpoint", + "type": "character varying(66)", + }, { + "isNullable": true, + "name": "address_id", + "type": "integer", + }, ], + }); - expect({ + expect(tableOptions4[1]).toStrictEqual({ "name": "test_checkpoint_id", + "foreignKeys": [{ + name: 'test_checkpoint_id', + columnNames: ['checkpoint_id'], + referencedTableName: 'test', + referencedColumnNames: ['id'], + }], "columns": [{ - "generationStrategy": "increment", - "isGenerated": true, - "isPrimary": true, - "name": "id", - "type": "integer", - }, { - "name": "checkpoint_id", - "type": "integer", - "isNullable": false - }, { - "name": "value0", - "type": "numeric", - "isNullable": true, - }], - }).toStrictEqual(tableOptions4[1]); + "generationStrategy": "increment", + "isGenerated": true, + "isPrimary": true, + "name": "id", + "type": "integer", + }, { + "name": "checkpoint_id", + "type": "integer", + "isNullable": false + }, { + "name": "value0", + "type": "numeric", + "isNullable": true, + }], + }); - expect({ + expect(tableOptions4[2]).toStrictEqual({ "name": "test_checkpoint_id_value0_id", + "foreignKeys": [{ + name: 'test_checkpoint_id_value0_id', + columnNames: ['value0_id'], + referencedTableName: 'test_checkpoint_id', + referencedColumnNames: ['id'], + }], "columns": [{ - "generationStrategy": "increment", - "isGenerated": true, - "isPrimary": true, - "name": "id", - "type": "integer", - }, { - "name": "value0_id", - "type": "integer", - "isNullable": false - }, { - "name": "fromBlock", - "type": "numeric", - "isNullable": true, - }, { - "name": "votes", - "type": "numeric", - "isNullable": true, - }], - }).toStrictEqual(tableOptions4[2]); + "generationStrategy": "increment", + "isGenerated": true, + "isPrimary": true, + "name": "id", + "type": "integer", + }, { + "name": "value0_id", + "type": "integer", + "isNullable": false + }, { + "name": "fromBlock", + "type": "numeric", + "isNullable": true, + }, { + "name": "votes", + "type": "numeric", + "isNullable": true, + }], + }); }); }); diff --git a/src/services/dataTypeParser.ts b/src/services/dataTypeParser.ts index 62b4fd6..9c2f7f8 100644 --- a/src/services/dataTypeParser.ts +++ b/src/services/dataTypeParser.ts @@ -1,5 +1,6 @@ import { parse, SourceUnit, ContractDefinition, StateVariableDeclaration, StructDefinition, TypeName } from 'solidity-parser-diligence'; import { TableOptions } from 'typeorm/schema-builder/options/TableOptions'; +import {TableForeignKeyOptions} from "typeorm/schema-builder/options/TableForeignKeyOptions"; export const errUnknownVariable = new Error('unknown variable'); @@ -39,7 +40,7 @@ export type Field = { } function getPgType(abiType: string): string { - let pgType = 'text'; + let pgType; // Fill in pg type based on abi type switch (abiType.replace(/\d+/g, '')) { @@ -218,7 +219,7 @@ export function toFields(obj: Structure): Field[] { return fields; } -export function toTableOptions(tableName: string, obj: Structure, fk?: string): TableOptions[] { +export function toTableOptions(tableName: string, obj: Structure, fk?: TableForeignKeyOptions): TableOptions[] { const tableOptions: TableOptions = { name: tableName, columns: [ @@ -229,15 +230,18 @@ export function toTableOptions(tableName: string, obj: Structure, fk?: string): isGenerated: true, generationStrategy: 'increment' } - ] + ], + foreignKeys: [], }; if (fk) { tableOptions.columns.push({ - name: fk, + name: fk.columnNames[0], type: 'integer', isNullable: false, }); + + tableOptions.foreignKeys.push(fk); } if (obj.type === 'simple') { @@ -268,7 +272,29 @@ export function toTableOptions(tableName: string, obj: Structure, fk?: string): isNullable: true, }); - return [tableOptions, ...toTableOptions(`${tableName}_${obj.name}_id`, obj.value, `${obj.name}_id`)]; + // add relation to address table for mapping(address => ..) + if (obj.key === 'address') { + tableOptions.columns.push({ + name: 'address_id', + type: 'integer', + isNullable: true, + }); + tableOptions.foreignKeys.push({ + name: `address_id_data.address`, + columnNames: ['address_id'], + referencedTableName: 'data.addresses', + referencedColumnNames: ['address_id'], + }); + } + + const fkChildTable: TableForeignKeyOptions = { + name: `${tableName}_${obj.name}_id`, + columnNames: [`${obj.name}_id`], + referencedTableName: tableName, + referencedColumnNames: ['id'], + }; + + return [tableOptions, ...toTableOptions(`${tableName}_${obj.name}_id`, obj.value, fkChildTable)]; } if (obj.type === 'array') { @@ -282,9 +308,23 @@ export function toTableOptions(tableName: string, obj: Structure, fk?: string): return [tableOptions]; } else if (obj.kind.type === 'mapping') { - return [tableOptions, ...toTableOptions(`${tableName}_${obj.kind.name}_id`, obj.kind.value, `${obj.kind.name}_id`)]; + const fkChildTable: TableForeignKeyOptions = { + name: `${tableName}_${obj.kind.name}_id`, + columnNames: [`${obj.kind.name}_id`], + referencedTableName: tableName, + referencedColumnNames: [obj.kind.name], + }; + + return [tableOptions, ...toTableOptions(`${tableName}_${obj.kind.name}_id`, obj.kind.value, fkChildTable)]; } else if (obj.kind.type === 'struct') { - return [tableOptions, ...toTableOptions(`${tableName}_${obj.name}_id`, obj.kind, `${obj.name}_id`)]; + const fkChildTable: TableForeignKeyOptions = { + name: `${tableName}_${obj.name}_id`, + columnNames: [`${obj.name}_id`], + referencedTableName: tableName, + referencedColumnNames: [obj.name], + }; + + return [tableOptions, ...toTableOptions(`${tableName}_${obj.name}_id`, obj.kind, fkChildTable)]; } } diff --git a/src/services/decodeService.ts b/src/services/decodeService.ts index 9e0302f..700daa2 100644 --- a/src/services/decodeService.ts +++ b/src/services/decodeService.ts @@ -110,6 +110,7 @@ export default class DecodeService { array.push({ name: input.name, value: abi.rawDecode([input.internalType], Buffer.from(topic, 'hex'))[0], + type: input.internalType, }); } catch (e) { console.log('Error abi decode', input.name, input.internalType, e.message); @@ -120,6 +121,7 @@ export default class DecodeService { array.push({ name: input.name, value: messages[index], + type: input.internalType, }); }); @@ -279,7 +281,7 @@ export default class DecodeService { } } else if (structure.type === 'simple') { const storageLeafKey = '0x' + keccak256(Buffer.from(INDEX[state.slot], 'hex')).toString('hex'); - console.log('storageLeafKey', storageLeafKey); + // console.log('storageLeafKey', storageLeafKey); const storage = relatedNode?.storageCidsByStateId?.nodes.find((s) => s.storageLeafKey === storageLeafKey); //console.log('storage', storage); diff --git a/src/store.ts b/src/store.ts index a4608fa..e45ed42 100644 --- a/src/store.ts +++ b/src/store.ts @@ -58,7 +58,7 @@ export default class Store { return null; } - return (this.contracts || []).find((contract) => contract.address === address.address); + return (this.contracts || []).find((contract) => contract.address.toLowerCase() === address.address.toLowerCase()); } public getEvents(): Event[] { @@ -100,7 +100,7 @@ export default class Store { } public getAddress(addressString: string): Address { - return (this.addresses || []).find((a) => a.address === addressString); + return (this.addresses || []).find((a) => a.address.toLowerCase() === addressString.toLowerCase()); } public getAddressByHash(hash: string): Address { diff --git a/src/types/index.ts b/src/types/index.ts index 1d740a4..b0d4636 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -10,6 +10,7 @@ export * from './graphql'; export type ABIInputData = { name: string; value?: any; // eslint-disable-line + type?: string; } export type DecodeReceiptResult = {