diff --git a/.gitignore b/.gitignore index 8230f87d6..b16add01c 100644 --- a/.gitignore +++ b/.gitignore @@ -13,8 +13,7 @@ database/ ganache-data/ ipfs-data/ local-data/ -node-modules/ -node_modules/* +**/node_modules .idea .composedbRuntimeDefinition.json -openalex-importer \ No newline at end of file +openalex-importer diff --git a/toolshed/README.md b/toolshed/README.md new file mode 100644 index 000000000..91323087a --- /dev/null +++ b/toolshed/README.md @@ -0,0 +1,4 @@ +# The Toolshed ๐Ÿ”ฉ +Zero warranty scripts and misc that could prove useful in the future. + +These tools are not necessarily maintained or kept up-to-date. diff --git a/toolshed/history-debugger/.gitignore b/toolshed/history-debugger/.gitignore new file mode 100644 index 000000000..03567fc46 --- /dev/null +++ b/toolshed/history-debugger/.gitignore @@ -0,0 +1 @@ +outputs diff --git a/toolshed/history-debugger/README.md b/toolshed/history-debugger/README.md new file mode 100644 index 000000000..31f7fc137 --- /dev/null +++ b/toolshed/history-debugger/README.md @@ -0,0 +1,23 @@ +# History Debugger +Utility to compare token and stream histories, useful for investigating migrations and version ordering. + +## Setup +Install dependencies: + +```bash +npm ci +``` + +If you want the rendered SVG graph, you need to make sure `dot` is available in your `PATH`, which probably means installing `graphviz` with your package manager. + +## Use +To debug a node, you need its (legacy) dPID, and it's streamID. With these at hand, run the script like so: + +```bash +DPID=149 STREAM=kjzl6kcym7w8y6wtboiio6jmbpml97m04bji4zphyboqhsckz75qg48efglcjox npx tsx src/index.ts +``` + +This will generate three files in the `outputs` directory: +- `149.md`: markdown table with metadata and version info +- `149.dot`: raw dot graph for both histories +- `149.svg`: rendered SVG from the dot graph diff --git a/toolshed/history-debugger/package-lock.json b/toolshed/history-debugger/package-lock.json new file mode 100644 index 000000000..77e1234b4 --- /dev/null +++ b/toolshed/history-debugger/package-lock.json @@ -0,0 +1,613 @@ +{ + "name": "history-debugger", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "history-debugger", + "version": "1.0.0", + "license": "ISC", + "dependencies": { + "@thi.ng/dot": "^2.1.85", + "which": "^5.0.0" + }, + "devDependencies": { + "@types/node": "^20.8.1", + "@types/which": "^3.0.4", + "tsx": "^4.19.1", + "typescript": "^5.6.3" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.23.1.tgz", + "integrity": "sha512-6VhYk1diRqrhBAqpJEdjASR/+WVRtfjpqKuNw11cLiaWpAT/Uu+nokB+UJnevzy/P9C/ty6AOe0dwueMrGh/iQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.23.1.tgz", + "integrity": "sha512-uz6/tEy2IFm9RYOyvKl88zdzZfwEfKZmnX9Cj1BHjeSGNuGLuMD1kR8y5bteYmwqKm1tj8m4cb/aKEorr6fHWQ==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.23.1.tgz", + "integrity": "sha512-xw50ipykXcLstLeWH7WRdQuysJqejuAGPd30vd1i5zSyKK3WE+ijzHmLKxdiCMtH1pHz78rOg0BKSYOSB/2Khw==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.23.1.tgz", + "integrity": "sha512-nlN9B69St9BwUoB+jkyU090bru8L0NA3yFvAd7k8dNsVH8bi9a8cUAUSEcEEgTp2z3dbEDGJGfP6VUnkQnlReg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.23.1.tgz", + "integrity": "sha512-YsS2e3Wtgnw7Wq53XXBLcV6JhRsEq8hkfg91ESVadIrzr9wO6jJDMZnCQbHm1Guc5t/CdDiFSSfWP58FNuvT3Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.23.1.tgz", + "integrity": "sha512-aClqdgTDVPSEGgoCS8QDG37Gu8yc9lTHNAQlsztQ6ENetKEO//b8y31MMu2ZaPbn4kVsIABzVLXYLhCGekGDqw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.23.1.tgz", + "integrity": "sha512-h1k6yS8/pN/NHlMl5+v4XPfikhJulk4G+tKGFIOwURBSFzE8bixw1ebjluLOjfwtLqY0kewfjLSrO6tN2MgIhA==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.23.1.tgz", + "integrity": "sha512-lK1eJeyk1ZX8UklqFd/3A60UuZ/6UVfGT2LuGo3Wp4/z7eRTRYY+0xOu2kpClP+vMTi9wKOfXi2vjUpO1Ro76g==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.23.1.tgz", + "integrity": "sha512-CXXkzgn+dXAPs3WBwE+Kvnrf4WECwBdfjfeYHpMeVxWE0EceB6vhWGShs6wi0IYEqMSIzdOF1XjQ/Mkm5d7ZdQ==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.23.1.tgz", + "integrity": "sha512-/93bf2yxencYDnItMYV/v116zff6UyTjo4EtEQjUBeGiVpMmffDNUyD9UN2zV+V3LRV3/on4xdZ26NKzn6754g==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.23.1.tgz", + "integrity": "sha512-VTN4EuOHwXEkXzX5nTvVY4s7E/Krz7COC8xkftbbKRYAl96vPiUssGkeMELQMOnLOJ8k3BY1+ZY52tttZnHcXQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.23.1.tgz", + "integrity": "sha512-Vx09LzEoBa5zDnieH8LSMRToj7ir/Jeq0Gu6qJ/1GcBq9GkfoEAoXvLiW1U9J1qE/Y/Oyaq33w5p2ZWrNNHNEw==", + "cpu": [ + "loong64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.23.1.tgz", + "integrity": "sha512-nrFzzMQ7W4WRLNUOU5dlWAqa6yVeI0P78WKGUo7lg2HShq/yx+UYkeNSE0SSfSure0SqgnsxPvmAUu/vu0E+3Q==", + "cpu": [ + "mips64el" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.23.1.tgz", + "integrity": "sha512-dKN8fgVqd0vUIjxuJI6P/9SSSe/mB9rvA98CSH2sJnlZ/OCZWO1DJvxj8jvKTfYUdGfcq2dDxoKaC6bHuTlgcw==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.23.1.tgz", + "integrity": "sha512-5AV4Pzp80fhHL83JM6LoA6pTQVWgB1HovMBsLQ9OZWLDqVY8MVobBXNSmAJi//Csh6tcY7e7Lny2Hg1tElMjIA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.23.1.tgz", + "integrity": "sha512-9ygs73tuFCe6f6m/Tb+9LtYxWR4c9yg7zjt2cYkjDbDpV/xVn+68cQxMXCjUpYwEkze2RcU/rMnfIXNRFmSoDw==", + "cpu": [ + "s390x" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.23.1.tgz", + "integrity": "sha512-EV6+ovTsEXCPAp58g2dD68LxoP/wK5pRvgy0J/HxPGB009omFPv3Yet0HiaqvrIrgPTBuC6wCH1LTOY91EO5hQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.23.1.tgz", + "integrity": "sha512-aevEkCNu7KlPRpYLjwmdcuNz6bDFiE7Z8XC4CPqExjTvrHugh28QzUXVOZtiYghciKUacNktqxdpymplil1beA==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.23.1.tgz", + "integrity": "sha512-3x37szhLexNA4bXhLrCC/LImN/YtWis6WXr1VESlfVtVeoFJBRINPJ3f0a/6LV8zpikqoUg4hyXw0sFBt5Cr+Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.23.1.tgz", + "integrity": "sha512-aY2gMmKmPhxfU+0EdnN+XNtGbjfQgwZj43k8G3fyrDM/UdZww6xrWxmDkuz2eCZchqVeABjV5BpildOrUbBTqA==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.23.1.tgz", + "integrity": "sha512-RBRT2gqEl0IKQABT4XTj78tpk9v7ehp+mazn2HbUeZl1YMdaGAQqhapjGTCe7uw7y0frDi4gS0uHzhvpFuI1sA==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.23.1.tgz", + "integrity": "sha512-4O+gPR5rEBe2FpKOVyiJ7wNDPA8nGzDuJ6gN4okSA1gEOYZ67N8JPk58tkWtdtPeLz7lBnY6I5L3jdsr3S+A6A==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.23.1.tgz", + "integrity": "sha512-BcaL0Vn6QwCwre3Y717nVHZbAa4UBEigzFm6VdsVdT/MbZ38xoj1X9HPkZhbmaBGUD1W8vxAfffbDe8bA6AKnQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.23.1.tgz", + "integrity": "sha512-BHpFFeslkWrXWyUPnbKm+xYYVYruCinGcftSBaa8zoF9hZO4BcSCFUvHVTtzpIY6YzUnYtuEhZ+C9iEXjxnasg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@thi.ng/api": { + "version": "8.11.11", + "resolved": "https://registry.npmjs.org/@thi.ng/api/-/api-8.11.11.tgz", + "integrity": "sha512-/GKDIgyQXA7JKxwgJ6W5xgQVHIWXmoPnWHsNqfg7aIyYNpfMYtjLjao2+nXmZhwjizullMahJPH7Xrrm6o/vyg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/postspectacular" + }, + { + "type": "patreon", + "url": "https://patreon.com/thing_umbrella" + } + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@thi.ng/checks": { + "version": "3.6.13", + "resolved": "https://registry.npmjs.org/@thi.ng/checks/-/checks-3.6.13.tgz", + "integrity": "sha512-LUK/S+QOkDMx1SEG3rubLfCu/KVSDTAEqAlwRP32WDPQXpWt95d7OSVI+oKDhAfpEPwdn13VufDSVcQqVNjEgw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/postspectacular" + }, + { + "type": "patreon", + "url": "https://patreon.com/thing_umbrella" + } + ], + "dependencies": { + "tslib": "^2.7.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@thi.ng/dot": { + "version": "2.1.85", + "resolved": "https://registry.npmjs.org/@thi.ng/dot/-/dot-2.1.85.tgz", + "integrity": "sha512-6PuXABcx1R3c+AOb+SIr0+MgsZSskZIuQu7JS3qavVGAleuTvS+Ckrtsj79kAaxM2CzMekCP9wjTcsSe3vYTIA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/postspectacular" + }, + { + "type": "patreon", + "url": "https://patreon.com/thing_umbrella" + } + ], + "dependencies": { + "@thi.ng/api": "^8.11.11", + "@thi.ng/checks": "^3.6.13" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@types/node": { + "version": "20.8.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.8.1.tgz", + "integrity": "sha512-iN6stS2QGMl50pjH0h/dIfmcEUogljAreQZ+cubPw3ISWp5fJrZw9NOh/sDHJfw92A41hCU+Ls5zTIsNYzcnuA==", + "dev": true + }, + "node_modules/@types/which": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/which/-/which-3.0.4.tgz", + "integrity": "sha512-liyfuo/106JdlgSchJzXEQCVArk0CvevqPote8F8HgWgJ3dRCcTHgJIsLDuee0kxk/mhbInzIZk3QWSZJ8R+2w==", + "dev": true + }, + "node_modules/esbuild": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.23.1.tgz", + "integrity": "sha512-VVNz/9Sa0bs5SELtn3f7qhJCDPCF5oMEl5cO9/SSinpE9hbPVvxbd572HH5AKiP7WD8INO53GgfDDhRjkylHEg==", + "dev": true, + "hasInstallScript": true, + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.23.1", + "@esbuild/android-arm": "0.23.1", + "@esbuild/android-arm64": "0.23.1", + "@esbuild/android-x64": "0.23.1", + "@esbuild/darwin-arm64": "0.23.1", + "@esbuild/darwin-x64": "0.23.1", + "@esbuild/freebsd-arm64": "0.23.1", + "@esbuild/freebsd-x64": "0.23.1", + "@esbuild/linux-arm": "0.23.1", + "@esbuild/linux-arm64": "0.23.1", + "@esbuild/linux-ia32": "0.23.1", + "@esbuild/linux-loong64": "0.23.1", + "@esbuild/linux-mips64el": "0.23.1", + "@esbuild/linux-ppc64": "0.23.1", + "@esbuild/linux-riscv64": "0.23.1", + "@esbuild/linux-s390x": "0.23.1", + "@esbuild/linux-x64": "0.23.1", + "@esbuild/netbsd-x64": "0.23.1", + "@esbuild/openbsd-arm64": "0.23.1", + "@esbuild/openbsd-x64": "0.23.1", + "@esbuild/sunos-x64": "0.23.1", + "@esbuild/win32-arm64": "0.23.1", + "@esbuild/win32-ia32": "0.23.1", + "@esbuild/win32-x64": "0.23.1" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/get-tsconfig": { + "version": "4.8.1", + "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.8.1.tgz", + "integrity": "sha512-k9PN+cFBmaLWtVz29SkUoqU5O0slLuHJXt/2P+tMVFT+phsSGXGkp9t3rQIqdz0e+06EHNGs3oM6ZX1s2zHxRg==", + "dev": true, + "dependencies": { + "resolve-pkg-maps": "^1.0.0" + }, + "funding": { + "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" + } + }, + "node_modules/isexe": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-3.1.1.tgz", + "integrity": "sha512-LpB/54B+/2J5hqQ7imZHfdU31OlgQqx7ZicVlkm9kzg9/w8GKLEcFfJl/t7DCEDueOyBAD6zCCwTO6Fzs0NoEQ==", + "engines": { + "node": ">=16" + } + }, + "node_modules/resolve-pkg-maps": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz", + "integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==", + "dev": true, + "funding": { + "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" + } + }, + "node_modules/tslib": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.0.tgz", + "integrity": "sha512-jWVzBLplnCmoaTr13V9dYbiQ99wvZRd0vNWaDRg+aVYRcjDF3nDksxFDE/+fkXnKhpnUUkmx5pK/v8mCtLVqZA==" + }, + "node_modules/tsx": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.19.1.tgz", + "integrity": "sha512-0flMz1lh74BR4wOvBjuh9olbnwqCPc35OOlfyzHba0Dc+QNUeWX/Gq2YTbnwcWPO3BMd8fkzRVrHcsR+a7z7rA==", + "dev": true, + "dependencies": { + "esbuild": "~0.23.0", + "get-tsconfig": "^4.7.5" + }, + "bin": { + "tsx": "dist/cli.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + } + }, + "node_modules/typescript": { + "version": "5.6.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.6.3.tgz", + "integrity": "sha512-hjcS1mhfuyi4WW8IWtjP7brDrG2cuDZukyrYrSauoXGNgx0S7zceP07adYkJycEr56BOUTNPzbInooiN3fn1qw==", + "dev": true, + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/which": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/which/-/which-5.0.0.tgz", + "integrity": "sha512-JEdGzHwwkrbWoGOlIHqQ5gtprKGOenpDHpxE9zVR1bWbOtYRyPPHMe9FaP6x61CmNaTThSkb0DAJte5jD+DmzQ==", + "dependencies": { + "isexe": "^3.1.1" + }, + "bin": { + "node-which": "bin/which.js" + }, + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + } + } +} diff --git a/toolshed/history-debugger/package.json b/toolshed/history-debugger/package.json new file mode 100644 index 000000000..bdbbe8377 --- /dev/null +++ b/toolshed/history-debugger/package.json @@ -0,0 +1,19 @@ +{ + "name": "history-debugger", + "version": "1.0.0", + "description": "", + "type": "module", + "scripts": {}, + "author": "Edvard Hรผbinette", + "license": "ISC", + "dependencies": { + "@thi.ng/dot": "^2.1.85", + "which": "^5.0.0" + }, + "devDependencies": { + "@types/node": "^20.8.1", + "@types/which": "^3.0.4", + "tsx": "^4.19.1", + "typescript": "^5.6.3" + } +} \ No newline at end of file diff --git a/toolshed/history-debugger/src/graph.ts b/toolshed/history-debugger/src/graph.ts new file mode 100644 index 000000000..056068c7d --- /dev/null +++ b/toolshed/history-debugger/src/graph.ts @@ -0,0 +1,142 @@ +import { Graph, Node, NodeAttribs, serializeEdge, serializeGraph } from "@thi.ng/dot"; +import { AllEvents, GenericEvent } from "./history"; + +const eventAttribs: Partial = { + fillcolor: "yellow", + shape: "Mrecord", + // outs: { out: "next" }, + fontcolor: "black", +}; + +const formatTimestamp = (timestamp: number) => + new Date(timestamp * 1000).toISOString().replace(".000", ""); + +type HistoryKind = "token" | "stream"; + +const makeHtmlEventNode = ( + kind: HistoryKind, + event: GenericEvent, +): Partial => { + const tableProps = 'BORDER="0" CELLBORDER="1" CELLSPACING="0" CELLPADDING="4"'; + const label = kind === "token" + ? `
${formatTimestamp(event.time)}v${event.v}
${event.cid}
` + : `
v${event.v}${formatTimestamp(event.time)}
${event.cid}
`; + + return { + ...eventAttribs, + shape: "none", // Required for HTML labels + group: kind, + label: `<${label}>`, + }; +}; + +const graphFromHistory = ( + kind: HistoryKind, label: string, history: GenericEvent[] +) => { + const nodes: Graph["nodes"] = {}; + const edges: Graph["edges"] = []; + history.forEach(event => { + nodes[`${kind}:${event.v}`] = makeHtmlEventNode(kind, event); + if (event.v > 0) { + edges.push({ + src: `${kind}:${event.v - 1}`, + dest: `${kind}:${event.v}` + }) + } + }); + + const attribs: Graph["attribs"] = { + label, + }; + + return { attribs, nodes, edges }; +} + +const cidMapEdges = (events: AllEvents) => { + const edges: Graph["edges"] = []; + const orphans: number[] = []; + + events.token.history.forEach(tokenEvent => { + const src = `token:${tokenEvent.v}`; + + const matchingStreamEvent = events.stream.history + .find(e => e.cid === tokenEvent.cid); + if (!matchingStreamEvent) { + orphans.push(tokenEvent.v); + } else { + const sameIndex = tokenEvent.v === matchingStreamEvent.v; + edges.push({ + src, + dest: `stream:${matchingStreamEvent.v}`, + constraint: false, + color: sameIndex ? "green" : "red", + style: "dashed", + tailport: "e", + headport: "w", + }); + } + }); + + return { edges, orphans }; +} + +export const generateDot = (allEvents: AllEvents) => { + const tokenHistorySubgraph = graphFromHistory( + "token", + `Token history\n\n${allEvents.token.owner}`, + allEvents.token.history, + ); + + const streamHistorySubgraph = graphFromHistory( + "stream", + `Stream history\n\n${allEvents.stream.owner}`, + allEvents.stream.history + ); + + const { edges: cidEdges, orphans } = cidMapEdges(allEvents); + orphans.forEach(o => { + tokenHistorySubgraph.nodes[`token:${o}`].fillcolor = "orange"; + }); + + const graph: Graph = { + directed: true, + // global graph attributes + attribs: { + rank: "same", + rankdir: "TB", + splines: "line", + fontname: "monospace", + fontsize: 18, + fontcolor: "gray", + labeljust: "c", + labelloc: "t", + nodesep: 2, + // node defaults + node: { + style: "filled", + fontname: "monospace", + fontsize: 12 + }, + // edge defaults + edge: { + arrowsize: 0.75, + } + }, + + sub: [ + tokenHistorySubgraph, + streamHistorySubgraph + ], + nodes: {}, + edges: cidEdges, + }; + + + const serialisedGraph = serializeGraph(graph); + + // Fix html serialisation + return serialisedGraph + .replace(/\\"/g, '"') + .replace(/label=">"\]/g, '>>]') +} diff --git a/toolshed/history-debugger/src/history.ts b/toolshed/history-debugger/src/history.ts new file mode 100644 index 000000000..208ea6a14 --- /dev/null +++ b/toolshed/history-debugger/src/history.ts @@ -0,0 +1,123 @@ +const ONE_WEEK = 7 * 24 * 60 * 60; + +export type TokenHistory = { + id: string; + owner: string; + history: { + v: number; + time: number; + cid: string; + }[]; +}; + +type RawToken = { + id: string; + owner: string; + versions: { + time: number; + cid: string; + }[]; +} + +const getTokenHistory = async (dpid: number): Promise => { + const legacyUrl = `https://beta.dpid.org/api/v1/dpid?page=${dpid + 1}&size=1&sort=asc`; + console.error(`๐Ÿ’ซ Fetching ${legacyUrl}`); + const legacyDpidRes = await fetch(legacyUrl); + const legacyDpid = await legacyDpidRes.json(); + + const researchObject = legacyDpid[0].researchObject as RawToken; + const history = researchObject.versions.map((event, i) => ({ + v: i, + time: event.time, + cid: event.cid, + })); + + return { + id: researchObject.id, + owner: researchObject.owner, + history, + }; +} + +export type StreamHistory = { + id: string; + owner: string; + history: { + v: number; + commit: string; + cid: string; + time: number; + anchor: number; + }[]; +} + +type RawStream = { + state: { + log: { + type: number; + expirationTime: number; + timestamp: number; + }[]; + }; +} + +type IndexedStream = { + owner: string; + versions: { + version: string; + time: number; + manifest: string; + }[]; +} + +const getStreamHistory = async (streamId: string): Promise => { + // We can compute actual signature time from the raw commits + const streamApiUrl = `https://ceramic-prod.desci.com/api/v0/streams/${streamId}`; + console.error(`๐Ÿ’ซ Fetching ${streamApiUrl}`); + const rawStreamReq = await fetch(streamApiUrl); + const rawStream = await rawStreamReq.json() as RawStream; + + const rawUserCommits = rawStream.state.log + .filter(event => event.type !== 2); + + // But we can get the manifest CID more easily from the resolver + const resolverUrl = `https://beta.dpid.org/api/v2/resolve/codex/${streamId}` + const streamQueryRes = await fetch(resolverUrl); + const streamQuery = await streamQueryRes.json() as IndexedStream; + + const owner = streamQuery.owner as string; + const history = streamQuery.versions.map((event, i) => ({ + v: i, + commit: event.version, + cid: event.manifest, + time: rawUserCommits.at(i)!.expirationTime - ONE_WEEK, + anchor: rawUserCommits.at(i)!.timestamp, + })); + return { id: streamId, owner, history }; +} + +export type GenericEvent = + | StreamHistory['history'][number] + | TokenHistory['history'][number]; + +export type AllEvents = { + token: TokenHistory; + stream: StreamHistory; + merged: GenericEvent[]; +}; + +export const getAllEvents = async (dpid: number, streamId: string): Promise => { + console.error('๐Ÿ”Ž Getting events for token and stream history', { dpid, streamId }); + + const token = await getTokenHistory(dpid); + const stream = await getStreamHistory(streamId); + + const merged = [...token.history, ...stream.history] + .sort((a, b) => a.time - b.time); + + return { + token, + stream, + merged, + }; +} diff --git a/toolshed/history-debugger/src/index.ts b/toolshed/history-debugger/src/index.ts new file mode 100644 index 000000000..983cb04f3 --- /dev/null +++ b/toolshed/history-debugger/src/index.ts @@ -0,0 +1,40 @@ +import { mkdirSync, writeFileSync } from "fs"; +import which from 'which'; + +import { generateDot } from "./graph"; +import { getAllEvents } from "./history"; +import { renderMarkdown } from "./markdown"; +import { exec, spawnSync } from "child_process"; + +if (!process.env.DPID || !process.env.STREAM) { + console.error('DPID and STREAM envvars required'); + process.exit(1); +} + +const dpid = parseInt(process.env.DPID); +const streamId = process.env.STREAM; + +const allEvents = await getAllEvents(dpid, streamId); + +// Create outdir if it doesn't exist +mkdirSync('outputs', { recursive: true }); + +// Generate and save markdown +const mdPath = `outputs/${dpid}.md`; +const markdown = renderMarkdown(allEvents); +console.error(`๐Ÿ“ƒ Saving markdown to ${mdPath}`); +writeFileSync(mdPath, markdown); + +// Generate and save dot graph +const dotPath = `outputs/${dpid}.dot`; +const dot = generateDot(allEvents); +console.error(`๐Ÿ“ˆ Saving dot graph to ${dotPath}`); +writeFileSync(dotPath, dot); + +if (await which('dot')) { + const svgPath = `outputs/${dpid}.svg`; + console.error(`๐ŸŽจ Rendering graph to ${svgPath}`); + exec(`dot -Tsvg -o ${svgPath} ${dotPath}`); +} else { + console.error(`๐Ÿคท 'dot' command not installed, skipping to render svg`) +} diff --git a/toolshed/history-debugger/src/markdown.ts b/toolshed/history-debugger/src/markdown.ts new file mode 100644 index 000000000..4be26e2fa --- /dev/null +++ b/toolshed/history-debugger/src/markdown.ts @@ -0,0 +1,33 @@ +import { AllEvents } from "./history"; + +export const renderMarkdown = (allEvents: AllEvents) => { + const { token, stream, merged } = allEvents; + const metadataMarkdown = [ + '# Metadata ', + '| Type | Owner | ID |', + '|--------|-----------------|--------------|', + `| token | ${token.owner} | ${token.id} |`, + `| stream | ${stream.owner} | ${stream.id} |` + ]; + + const historyMarkdown = [ + '# History', + '| Date | Token (by blocktime) | Stream (by JWT timestamp) |', + '|------|-------------------|------------------------|' + ]; + + merged.forEach(e => { + const dateStr = new Date(e.time * 1000).toISOString(); + const isStreamEvent = "anchor" in e; + const version = `(${e.v}) ${e.cid}`; + + if (isStreamEvent) { + historyMarkdown.push(`| ${dateStr} | | ${version} |`); + } else { + historyMarkdown.push(`| ${dateStr} | ${version} | |`); + } + }); + + return [...metadataMarkdown, ...historyMarkdown].join('\n'); +} +