From a18bf8df282c45e0f042af90913dedc1b12d5c56 Mon Sep 17 00:00:00 2001 From: m0ar Date: Fri, 27 Sep 2024 16:13:55 +0200 Subject: [PATCH 1/6] if resolving a legacy history entry from the dev contract, filter out the duplicate entries --- src/api/v2/resolvers/dpid.ts | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/src/api/v2/resolvers/dpid.ts b/src/api/v2/resolvers/dpid.ts index 0b0d6db..0d49e3c 100644 --- a/src/api/v2/resolvers/dpid.ts +++ b/src/api/v2/resolvers/dpid.ts @@ -3,7 +3,6 @@ import parentLogger from "../../../logger.js"; import { CACHE_TTL_ANCHORED, CACHE_TTL_PENDING, DPID_ENV, getDpidAliasRegistry } from "../../../util/config.js"; import { ResolverError } from "../../../errors.js"; import { getCodexHistory, type HistoryQueryResult, type HistoryVersion } from "../queries/history.js"; -import { BigNumber } from "ethers"; import { getFromCache, setToCache } from "../../../redis.js"; import type { DpidAliasRegistry } from "@desci-labs/desci-contracts/dist/typechain-types/DpidAliasRegistry.js"; @@ -152,7 +151,8 @@ export const resolveDpid = async (dpid: number, versionIx?: number): Promise ({ // No CommitID available version: "", - time: BigNumber.from(time).toNumber(), + time: time.toNumber(), manifest, })), }; @@ -179,5 +179,23 @@ export const resolveDpid = async (dpid: number, versionIx?: number): Promise {} + +type LegacyVersion = DpidAliasRegistry.LegacyVersionStructOutput; + +const undupeIfLegacyDevHistory = (versions: LegacyVersion[]) => { + if (DPID_ENV !== "dev") { + return versions; + } + + return versions.reduce((unduped, current) => { + if (unduped.length === 0 || !isLegacyDupe(current, unduped[unduped.length - 1])) { + unduped.push(current); + } + return unduped; + }, [] as LegacyVersion[]); +}; + +const isLegacyDupe = (a: LegacyVersion, b: LegacyVersion) => { + return a[0] === b[0] && a[1].toNumber() === b[1].toNumber(); +}; From 9b11bfb5593c7baceebf81095b1834e3af9a0777 Mon Sep 17 00:00:00 2001 From: m0ar Date: Mon, 30 Sep 2024 16:45:56 +0200 Subject: [PATCH 2/6] disable nvmrc versioning of base image, can't be used with docker hub integration --- Dockerfile | 3 +-- package.json | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1f51138..421de88 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,4 @@ -ARG NODE_VERSION -FROM node:${NODE_VERSION}-alpine3.20 AS base +FROM node:20.13.1-alpine3.20 AS base RUN apk update && apk add --no-cache bash dumb-init WORKDIR /usr/src/app diff --git a/package.json b/package.json index 8bb5d49..0ef11f2 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,6 @@ "type": "module", "scripts": { "build": "tsc", - "docker:build": "docker build --build-arg NODE_VERSION=$(< .nvmrc) .", "watch": "tsc --watch", "start": "node dist/index.js", "test": "PINO_LOG_LEVEL=silent PORT=5600 vitest --config vitest.config.ts", From dfe0f407b8e8d3fb4a0df7bf335a30f4ac23a497 Mon Sep 17 00:00:00 2001 From: m0ar Date: Tue, 1 Oct 2024 10:23:18 +0200 Subject: [PATCH 3/6] Fix test case using a now-filtered dupe from dev registry --- test/v2/resolvers.spec.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/v2/resolvers.spec.ts b/test/v2/resolvers.spec.ts index bdc4f19..5095323 100644 --- a/test/v2/resolvers.spec.ts +++ b/test/v2/resolvers.spec.ts @@ -279,7 +279,8 @@ describe("dPID", { timeout: 10_000 }, function () { .expect(200) .expect((res) => expect(res.body.manifest).toEqual( - "bafkreih5koqw5nvxucidlihwfslknj674oeuroclit74rkaqpe4mq6xuka", + // fourth published CID + "bafkreibn3jhdlsdsonv25t7i2bwtrbkl3jzwjbnnwylpeih3jmmzdhsfmi", ), ); }); From e2f3df3eb33669cc540e1971cefc5aa4633d83e9 Mon Sep 17 00:00:00 2001 From: m0ar Date: Tue, 1 Oct 2024 10:45:37 +0200 Subject: [PATCH 4/6] fix bug in deserialising BigNumbers from redis --- src/api/v2/resolvers/dpid.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/api/v2/resolvers/dpid.ts b/src/api/v2/resolvers/dpid.ts index 0d49e3c..0eccdef 100644 --- a/src/api/v2/resolvers/dpid.ts +++ b/src/api/v2/resolvers/dpid.ts @@ -5,6 +5,7 @@ import { ResolverError } from "../../../errors.js"; import { getCodexHistory, type HistoryQueryResult, type HistoryVersion } from "../queries/history.js"; import { getFromCache, setToCache } from "../../../redis.js"; import type { DpidAliasRegistry } from "@desci-labs/desci-contracts/dist/typechain-types/DpidAliasRegistry.js"; +import { BigNumber } from "ethers"; const MODULE_PATH = "/api/v2/resolvers/codex" as const; const logger = parentLogger.child({ @@ -163,7 +164,9 @@ export const resolveDpid = async (dpid: number, versionIx?: number): Promise ({ // No CommitID available version: "", - time: time.toNumber(), + // When restored from redis, the BigNumber is deserialised as a regular object + // Ethers can instantiate the class from that format + time: BigNumber.from(time).toNumber(), manifest, })), }; From e9090b6aa634014d8c2e9bded8527113ce6c7ab4 Mon Sep 17 00:00:00 2001 From: m0ar Date: Tue, 1 Oct 2024 10:53:39 +0200 Subject: [PATCH 5/6] demystify legacy version equality fn after review --- src/api/v2/resolvers/dpid.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/api/v2/resolvers/dpid.ts b/src/api/v2/resolvers/dpid.ts index 0eccdef..81efa0c 100644 --- a/src/api/v2/resolvers/dpid.ts +++ b/src/api/v2/resolvers/dpid.ts @@ -199,6 +199,11 @@ const undupeIfLegacyDevHistory = (versions: LegacyVersion[]) => { }, [] as LegacyVersion[]); }; -const isLegacyDupe = (a: LegacyVersion, b: LegacyVersion) => { - return a[0] === b[0] && a[1].toNumber() === b[1].toNumber(); +const isLegacyDupe = ( + [aCid, aTimeBn]: LegacyVersion, + [bCid, bTimeBn]: LegacyVersion +): Boolean => { + const cidIsEqual = aCid === bCid; + const timeIsEqual = aTimeBn.toNumber() === bTimeBn.toNumber(); + return cidIsEqual && timeIsEqual; }; From f4374ba45b5dc322d55a1e8df7d5cde1992f01e9 Mon Sep 17 00:00:00 2001 From: m0ar Date: Tue, 1 Oct 2024 13:58:19 +0200 Subject: [PATCH 6/6] fix cache deserialisation of bignumber, improve error serialisation in logging --- src/api/v2/queries/history.ts | 61 ++++++++++++++++++++++++++------- src/api/v2/resolvers/codex.ts | 6 ++-- src/api/v2/resolvers/dpid.ts | 22 ++++++------ src/api/v2/resolvers/generic.ts | 8 ++--- src/logger.ts | 2 ++ 5 files changed, 68 insertions(+), 31 deletions(-) diff --git a/src/api/v2/queries/history.ts b/src/api/v2/queries/history.ts index 0710d87..973a56c 100644 --- a/src/api/v2/queries/history.ts +++ b/src/api/v2/queries/history.ts @@ -1,15 +1,16 @@ import type { Request, Response } from "express"; import { CACHE_TTL_ANCHORED, CACHE_TTL_PENDING, DPID_ENV, getCeramicClient } from "../../../util/config.js"; import { type CeramicClient } from "@desci-labs/desci-codex-lib"; -import parentLogger from "../../../logger.js"; -import { resolveDpid } from "../resolvers/dpid.js"; +import parentLogger, { serializeError } from "../../../logger.js"; +import { DpidResolverError, resolveDpid } from "../resolvers/dpid.js"; import { isDpid } from "../../../util/validation.js"; import { CommitID, StreamID } from "@desci-labs/desci-codex-lib/dist/streams.js"; import { getFromCache, keyBump, setToCache } from "../../../redis.js"; import { cleanupEip155Address } from "../../../util/conversions.js"; +const MODULE_PATH = "api/v2/queries/history" as const; const logger = parentLogger.child({ - module: "api/v2/queries/history", + module: MODULE_PATH, }); export type HistoryQueryRequest = { @@ -22,7 +23,15 @@ export type HistoryQueryParams = { id?: string; }; -export type HistoryQueryResponse = HistoryQueryResult[] | HistoryQueryError; +export type ErrorResponse = { + error: string; + details: unknown; + body: unknown; + params: unknown; + path: typeof MODULE_PATH; +}; + +export type HistoryQueryResponse = HistoryQueryResult[] | ErrorResponse; export type HistoryVersion = { /** Manifest CID at this version */ @@ -44,8 +53,6 @@ export type HistoryQueryResult = { versions: HistoryVersion[]; }; -export type HistoryQueryError = string; - /** * For one or more IDs, fetch metadata and version history. * An ID can be both a streamID and a dPID, but a dPID lookup is a bit slower. @@ -57,10 +64,20 @@ export const historyQueryHandler = async ( const { id } = req.params; const { ids = [] } = req.body; + const baseError = { + params: req.params, + body: req.body, + path: MODULE_PATH, + }; + if (!Array.isArray(ids)) { // Received ids in body, but not as array - logger.error({ body: req.body, params: req.params }, "received malformed IDs"); - return res.status(400).send("body.ids expects string[]"); + logger.error(baseError, "received malformed IDs"); + return res.status(400).send({ + error: "invalid request", + details: "body.ids expects string[]", + ...baseError, + }); } if (id) { @@ -70,8 +87,12 @@ export const historyQueryHandler = async ( if (ids.length === 0) { // Neither ID format was supplied - logger.error({ body: req.body, params: req.params }, "request missing IDs"); - return res.status(400).send("missing /:id or ids array in body"); + logger.error(baseError, "request missing IDs"); + return res.status(400).send({ + error: "invalid request", + details: "missing /:id or ids array in body", + ...baseError, + }); } logger.info({ ids }, "handling history query"); @@ -87,9 +108,23 @@ export const historyQueryHandler = async ( ]); const result = [...codexHistories, ...dpidHistories]; return res.send(result); - } catch (error) { - logger.error({ ids, error }, "failed to compile histories"); - return res.status(500).send("failed to compile histories"); + } catch (e) { + if (e instanceof DpidResolverError) { + const errPayload = { + error: "failed to resolve dpid", + details: serializeError(e), + ...baseError, + }; + logger.error(errPayload, "failed to resolve dpid"); + return res.status(500).send(errPayload); + } + const errPayload = { + error: "failed to compile histories", + details: serializeError(e as Error), + ...baseError, + }; + logger.error(errPayload, "failed to compile histories"); + return res.status(500).send(errPayload); } }; diff --git a/src/api/v2/resolvers/codex.ts b/src/api/v2/resolvers/codex.ts index e35f5c9..41166c1 100644 --- a/src/api/v2/resolvers/codex.ts +++ b/src/api/v2/resolvers/codex.ts @@ -1,5 +1,5 @@ import type { Request, Response } from "express"; -import parentLogger from "../../../logger.js"; +import parentLogger, { serializeError } from "../../../logger.js"; import { pidFromStringID, type PID } from "@desci-labs/desci-codex-lib"; import { getCodexHistory, type HistoryQueryResult } from "../queries/history.js"; @@ -46,7 +46,7 @@ export const resolveCodexHandler = async ( } catch (e) { const errPayload = { error: "Invalid stream or commit ID", - details: "Could not coerce ID into neither stream nor commitID", + details: serializeError(e as Error), params: req.params, path: MODULE_PATH, }; @@ -67,7 +67,7 @@ export const resolveCodexHandler = async ( logger.error({ streamId, versionIx, err }, "failed to resolve stream"); return res.status(404).send({ error: "Could not resolve; does stream/version exist?", - details: err, + details: serializeError(err), params: req.params, path: MODULE_PATH, }); diff --git a/src/api/v2/resolvers/dpid.ts b/src/api/v2/resolvers/dpid.ts index 81efa0c..6d2941d 100644 --- a/src/api/v2/resolvers/dpid.ts +++ b/src/api/v2/resolvers/dpid.ts @@ -1,5 +1,5 @@ import type { Request, Response } from "express"; -import parentLogger from "../../../logger.js"; +import parentLogger, { serializeError } from "../../../logger.js"; import { CACHE_TTL_ANCHORED, CACHE_TTL_PENDING, DPID_ENV, getDpidAliasRegistry } from "../../../util/config.js"; import { ResolverError } from "../../../errors.js"; import { getCodexHistory, type HistoryQueryResult, type HistoryVersion } from "../queries/history.js"; @@ -7,7 +7,7 @@ import { getFromCache, setToCache } from "../../../redis.js"; import type { DpidAliasRegistry } from "@desci-labs/desci-contracts/dist/typechain-types/DpidAliasRegistry.js"; import { BigNumber } from "ethers"; -const MODULE_PATH = "/api/v2/resolvers/codex" as const; +const MODULE_PATH = "/api/v2/resolvers/dpid" as const; const logger = parentLogger.child({ module: MODULE_PATH, }); @@ -53,7 +53,7 @@ export const resolveDpidHandler = async ( if (e instanceof DpidResolverError) { const errPayload = { error: e.message, - details: e.cause, + details: serializeError(e.cause), params: req.params, path: MODULE_PATH, }; @@ -63,7 +63,7 @@ export const resolveDpidHandler = async ( const err = e as Error; const errPayload = { error: err.message, - details: err, + details: serializeError(err), params: req.params, path: MODULE_PATH, }; @@ -148,8 +148,11 @@ export const resolveDpid = async (dpid: number, versionIx?: number): Promise { + v[1] = BigNumber.from(v[1]); + }); } const owner = resolvedEntry[0]; @@ -166,11 +169,11 @@ export const resolveDpid = async (dpid: number, versionIx?: number): Promise { }, [] as LegacyVersion[]); }; -const isLegacyDupe = ( - [aCid, aTimeBn]: LegacyVersion, - [bCid, bTimeBn]: LegacyVersion -): Boolean => { +const isLegacyDupe = ([aCid, aTimeBn]: LegacyVersion, [bCid, bTimeBn]: LegacyVersion): boolean => { const cidIsEqual = aCid === bCid; const timeIsEqual = aTimeBn.toNumber() === bTimeBn.toNumber(); return cidIsEqual && timeIsEqual; diff --git a/src/api/v2/resolvers/generic.ts b/src/api/v2/resolvers/generic.ts index 94eaa11..6057f35 100644 --- a/src/api/v2/resolvers/generic.ts +++ b/src/api/v2/resolvers/generic.ts @@ -2,7 +2,7 @@ import type { Request, Response } from "express"; import axios from "axios"; import type { ResearchObjectV1 } from "@desci-labs/desci-models"; -import parentLogger from "../../../logger.js"; +import parentLogger, { serializeError } from "../../../logger.js"; import analytics, { LogEventType } from "../../../analytics.js"; import { IPFS_GATEWAY, getNodesUrl } from "../../../util/config.js"; import { DpidResolverError, resolveDpid } from "./dpid.js"; @@ -152,7 +152,7 @@ export const resolveGenericHandler = async ( if (e instanceof DpidResolverError) { const errPayload = { error: e.message, - details: e.cause, + details: serializeError(e.cause), ...baseError, }; logger.error(errPayload, "failed to resolve dpid"); @@ -161,7 +161,7 @@ export const resolveGenericHandler = async ( const err = e as Error; const errPayload = { error: err.message, - details: err, + details: serializeError(err), ...baseError, }; logger.error(errPayload, "unexpected error occurred"); @@ -216,7 +216,7 @@ export const resolveGenericHandler = async ( // Doesn't seem it was a validDagUrl const errPayload = { error: "Failed to resolve DAG URL; check path and versioning", - details: e, + details: serializeError(e as Error), ...baseError, }; logger.error(errPayload, "got invalid DAG URL"); diff --git a/src/logger.ts b/src/logger.ts index 06e13d2..eb1dba2 100644 --- a/src/logger.ts +++ b/src/logger.ts @@ -39,6 +39,8 @@ function omitBuffer(array: any) { }); } +export const serializeError = (e: Error) => pino.stdSerializers.err(e); + process.on("uncaughtException", (err) => { logger.fatal(err, "uncaught exception"); });