diff --git a/scripts/update-rtdb.ts b/scripts/update-rtdb.ts index 3c2dd244..49cd24ee 100644 --- a/scripts/update-rtdb.ts +++ b/scripts/update-rtdb.ts @@ -2,7 +2,7 @@ import { diffString } from 'json-diff' import { loadUpdateRTDBConfig } from '../src/config' import { FirebaseClient } from '../src/clients/firebase-client' -import { deleteMissingKeysUpdateRequest } from '../src/utils/utils' +import { keepInternalKeys } from '../src/utils/utils' import { getCeloRTDBMetadata } from '../src' async function main() { @@ -28,17 +28,20 @@ async function main() { await firebaseClient.writeToPath(rtdbLocation, data) console.log(`Wrote data to ${rtdbLocation}.`) } else { - let updateRequest = data - if (overrideType.deleteMissingKeys) { - const deleteMissingKeys = deleteMissingKeysUpdateRequest( - data, - rtdbData, + const updateRequest = keepInternalKeys( + data, + rtdbData, + overrideType.keptInternalKeys, + ) + const updateDiff = diffString(rtdbData, updateRequest) + if (!updateDiff) { + console.log( + `Diff is empty for data at ${rtdbLocation}, skipping...`, ) - updateRequest = { ...updateRequest, ...deleteMissingKeys } + } else { + await firebaseClient.writeToPath(rtdbLocation, updateRequest) + console.log(`Updated data at ${rtdbLocation}.`) } - // TODO: Avoid updating the node if we already know there aren't changes - await firebaseClient.updateToPath(rtdbLocation, updateRequest) - console.log(`Updated data at ${rtdbLocation}.`) } } } diff --git a/src/clients/firebase-client.ts b/src/clients/firebase-client.ts index f49cbd02..4d409c74 100644 --- a/src/clients/firebase-client.ts +++ b/src/clients/firebase-client.ts @@ -1,5 +1,4 @@ import * as admin from 'firebase-admin' -import { mapNestedJsonIntoPlain } from '../utils/utils' import { UpdateRTDBConfig } from '../types' export class FirebaseClient { @@ -21,11 +20,6 @@ export class FirebaseClient { return data.val() } - async updateToPath(path: string, data: any): Promise { - const ref = await this.firebaseDb.ref(path) - await ref.update(mapNestedJsonIntoPlain(data)) - } - async writeToPath(path: string, data: any): Promise { const ref = await this.firebaseDb.ref(path) await ref.set(data) diff --git a/src/index.ts b/src/index.ts index cfcf652c..2b465788 100644 --- a/src/index.ts +++ b/src/index.ts @@ -27,7 +27,11 @@ export function getCeloRTDBMetadata(environment: Environment): RTDBMetadata[] { data: transformCeloTokensForRTDB(tokensInfo), schema: RTDBAddressToTokenInfoSchema, rtdbLocation: 'tokensInfo', - overrideType: OverrideType.DeleteMissingKeysAndUpdate, + overrideType: OverrideType.KeepInternalKeys([ + 'historicalUsdPrices', + 'usdPrice', + 'priceFetchedAt', + ]), }, ] } diff --git a/src/types.ts b/src/types.ts index c7300007..b59ce69f 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,14 +1,14 @@ import Joi from 'joi' export class OverrideType { - static readonly OverrideAll = new OverrideType(true, true) - static readonly OnlyUpdates = new OverrideType(false, false) - static readonly DeleteMissingKeysAndUpdate = new OverrideType(false, true) + static readonly OverrideAll = new OverrideType(true, []) + static readonly KeepInternalKeys = (keepInternalKeys?: string[]) => + new OverrideType(false, keepInternalKeys) // private to disallow creating other instances of this type private constructor( public readonly shouldOverride: boolean, - public readonly deleteMissingKeys: boolean, + public readonly keptInternalKeys: string[] = [], ) {} } diff --git a/src/utils/utils.test.ts b/src/utils/utils.test.ts index 68d62b0c..22a30547 100644 --- a/src/utils/utils.test.ts +++ b/src/utils/utils.test.ts @@ -1,128 +1,91 @@ -import { deleteMissingKeysUpdateRequest, mapNestedJsonIntoPlain } from './utils' +import { keepInternalKeys } from './utils' + +describe('Keep Internal Keys', () => { + const valuesToUpdate = { + '0x1234': { + address: '0x1234', + decimal: 18, + imageUrl: 'https://example.com', + }, + '0x1235': { + address: '0x1235', + decimal: 8, + imageUrl: 'https://example2.com', + }, + '0x1236': { + address: '0x1236', + decimal: 18, + imageUrl: 'https://example3.com', + }, + } + + const currentValues = { + '0x1234': { + address: '0x1234old', + decimal: 18, + imageUrl: 'https://example.com', + usdPrice: '1', + historicalUsdPrices: { + lastDay: { + price: '1.2', + }, + }, + }, + '0x1235': { + address: '0x1235old', + decimal: 8, + imageUrl: 'https://example2.com', + usdPrice: '2', + historicalUsdPrices: { + lastDay: { + price: '2.2', + }, + }, + }, + } -describe('Map Nested Json Into', () => { - it('should map correctly when the json contains one level', () => { - const inputJson = { - key1: 'stringValue', - key2: 3, - } - const expectedJson = inputJson + it("returns the expected object when there aren't keys to keep", () => { + const updateObject = keepInternalKeys(valuesToUpdate, currentValues, []) - const result = mapNestedJsonIntoPlain(inputJson) - expect(result).toMatchObject(expectedJson) + expect(updateObject).toMatchObject(valuesToUpdate) }) - it('should map correctly when the json contains a nested level', () => { - const inputJson = { - key1: { - key3: 4, - key4: 'stringValue', + it('returns the expected object when there are internal keys to keep', () => { + const updateObject = keepInternalKeys(valuesToUpdate, currentValues, [ + 'usdPrice', + 'historicalUsdPrices', + ]) + + const expectedObject = { + '0x1234': { + address: '0x1234', + decimal: 18, + imageUrl: 'https://example.com', + usdPrice: '1', + historicalUsdPrices: { + lastDay: { + price: '1.2', + }, + }, }, - key2: 3, - } - const expectedJson = { - 'key1/key3': 4, - 'key1/key4': 'stringValue', - key2: 3, - } - - const result = mapNestedJsonIntoPlain(inputJson) - expect(result).toMatchObject(expectedJson) - }) - - it('should map correctly when the json contains multiple nested level', () => { - const inputJson = { - key1: { - key3: 4, - key4: 5, - key5: { - key6: 'stringValue', - key7: { - key8: 9, + '0x1235': { + address: '0x1235', + decimal: 8, + imageUrl: 'https://example2.com', + usdPrice: '2', + historicalUsdPrices: { + lastDay: { + price: '2.2', }, }, }, - key2: 3, - } - const expectedJson = { - 'key1/key3': 4, - 'key1/key4': 5, - 'key1/key5/key6': 'stringValue', - 'key1/key5/key7/key8': 9, - key2: 3, - } - - const result = mapNestedJsonIntoPlain(inputJson) - expect(result).toMatchObject(expectedJson) - }) - - it('should map array to object with indexes as key', () => { - const inputJson = { - key1: ['this', 'is', 'a', 'test'], - key2: 3, - } - - const expectedJson = { - 'key1/0': 'this', - 'key1/1': 'is', - 'key1/2': 'a', - 'key1/3': 'test', - key2: 3, - } - - const result = mapNestedJsonIntoPlain(inputJson) - expect(result).toMatchObject(expectedJson) - }) -}) - -describe('Delete Missing Keys UpdateRequest', () => { - it('return expected update request when there are missing keys', () => { - const expected = { - key1: 1, - key2: 'value', - } - - const current = { - key1: 2, - key3: 'shouldBeDeleted', - } - - const updateObject = deleteMissingKeysUpdateRequest(expected, current) - - const expectedResult = { - key3: null, - } - - expect(updateObject).toMatchObject(expectedResult) - }) - - it('return empty update request when there are not missing keys', () => { - const expected = { - key1: 1, - key2: 'value', - key3: 'extra', - } - - const current = { - key1: 2, - key2: 'shouldNotBeDeleted', - } - - const updateObject = deleteMissingKeysUpdateRequest(expected, current) - - expect(updateObject).toMatchObject({}) - }) - - it('creates all keys if the RTDB instance is clean', () => { - const expected = { - key1: 1, - key2: 'value', + '0x1236': { + address: '0x1236', + decimal: 18, + imageUrl: 'https://example3.com', + }, } - const current = null - - const updateObject = deleteMissingKeysUpdateRequest(expected, current) - - expect(updateObject).toMatchObject({}) + expect(expectedObject).toMatchObject(updateObject) }) }) diff --git a/src/utils/utils.ts b/src/utils/utils.ts index a4528f4c..ec7ee65e 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -1,41 +1,19 @@ -export function mapNestedJsonIntoPlain(json: any): any { - if (!isNonEmptyObject(json)) { - return json - } +export function keepInternalKeys( + expected: any, + current: any, + keptInternalKeys: string[], +) { + const result: any = {} - return Object.keys(json).reduce( - (result, current) => ({ - ...result, - ...prependKey(current, mapNestedJsonIntoPlain(json[current])), - }), - {}, - ) -} + for (const key of Object.keys(expected)) { + result[key] = expected[key] -function prependKey(key: string, json: any) { - if (!isNonEmptyObject(json)) { - return { [key]: json } + for (const internalKey of keptInternalKeys) { + if (current[key] && current[key][internalKey]) { + result[key][internalKey] = current[key][internalKey] + } + } } - return Object.keys(json).reduce( - (result, current) => ({ - ...result, - [`${key}/${current}`]: json[current], - }), - {}, - ) -} - -function isNonEmptyObject(json: any): boolean { - return typeof json === 'object' && json !== null -} - -export function deleteMissingKeysUpdateRequest(expected: any, current: any) { - if (!isNonEmptyObject(current)) return {} - const expectedKeys = new Set(Object.keys(expected)) - const currentKeys = Object.keys(current) - - return currentKeys - .filter((key) => !expectedKeys.has(key)) - .reduce((result, key) => ({ ...result, [key]: null }), {}) + return result }