From 8bfd8bdabc5f56a2f22eb5ecbe71940b4fee8576 Mon Sep 17 00:00:00 2001 From: Felix Nguyen Date: Thu, 17 Oct 2024 22:58:48 -0400 Subject: [PATCH] feat: 30-day retention for tracks --- .vscode/settings.json | 3 +- cleanup.Dockerfile | 21 ++++++ migration.Dockerfile | 2 +- package-lock.json | 108 ++++++++++++++++++------------- package.json | 1 + scripts/cron-monthly-cleanup.mjs | 102 +++++++++++++++++++++++++++++ 6 files changed, 190 insertions(+), 47 deletions(-) create mode 100644 cleanup.Dockerfile create mode 100644 scripts/cron-monthly-cleanup.mjs diff --git a/.vscode/settings.json b/.vscode/settings.json index 2e8b328..14d1c1d 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -2,5 +2,6 @@ "typescript.tsdk": "node_modules/typescript/lib", "files.associations": { "*.css": "tailwindcss" - } + }, + "makefile.configureOnOpen": false } diff --git a/cleanup.Dockerfile b/cleanup.Dockerfile new file mode 100644 index 0000000..a8d9eb4 --- /dev/null +++ b/cleanup.Dockerfile @@ -0,0 +1,21 @@ +FROM node:20-alpine AS base + +# Install dependencies only when needed +FROM base AS deps +# Check https://github.com/nodejs/docker-node/tree/b4117f9333da4138b03a546ec926ef50a31506c3#nodealpine to understand why libc6-compat might be needed. +RUN apk add --no-cache libc6-compat +WORKDIR /app +COPY package.json package-lock.json ./ +RUN ["npm", "pkg", "delete", "scripts.prepare"] +# RUN [ "npm", "ci", "--omit=dev" ] +RUN [ "npm", "i" ] + +FROM base AS cleanup +WORKDIR /app +COPY --from=deps /app/node_modules node_modules/ +COPY package.json . +COPY scripts scripts +ARG ENABLE_ALPINE_PRIVATE_NETWORKING +ENV NODE_ENV=production +USER node +CMD [ "node", "scripts/cron-monthly-cleanup.mjs" ] diff --git a/migration.Dockerfile b/migration.Dockerfile index f5ae49c..ed65a18 100644 --- a/migration.Dockerfile +++ b/migration.Dockerfile @@ -1,4 +1,4 @@ -FROM node:20-alpine as base +FROM node:20-alpine AS base # Install dependencies only when needed FROM base AS deps diff --git a/package-lock.json b/package-lock.json index cd6afb3..e6b6474 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2707,9 +2707,9 @@ } }, "node_modules/@next/env": { - "version": "14.1.1-canary.59", - "resolved": "https://registry.npmjs.org/@next/env/-/env-14.1.1-canary.59.tgz", - "integrity": "sha512-7tBm5NJmAkmiBZf/HRe5RUIYln24SsxVUKdJpDGW7/OHcrQXZ9iO1WsGpZLJFAWpfLQU9aUXyovKLuKvqhyOTQ==" + "version": "14.2.15", + "resolved": "https://registry.npmjs.org/@next/env/-/env-14.2.15.tgz", + "integrity": "sha512-S1qaj25Wru2dUpcIZMjxeMVSwkt8BK4dmWHHiBuRstcIyOsMapqT4A4jSB6onvqeygkSSmOkyny9VVx8JIGamQ==" }, "node_modules/@next/eslint-plugin-next": { "version": "14.1.0", @@ -2755,9 +2755,9 @@ } }, "node_modules/@next/swc-darwin-arm64": { - "version": "14.1.1-canary.59", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.1.1-canary.59.tgz", - "integrity": "sha512-WVSg1dkNvprYkakRIYYHhkIiSvaU7+S/bCffs0hGKRmKr45Lc7HfvwVAWHPxIO3d7cUtnz1d7fXRD42pWom4EQ==", + "version": "14.2.15", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.2.15.tgz", + "integrity": "sha512-Rvh7KU9hOUBnZ9TJ28n2Oa7dD9cvDBKua9IKx7cfQQ0GoYUwg9ig31O2oMwH3wm+pE3IkAQ67ZobPfEgurPZIA==", "cpu": [ "arm64" ], @@ -2770,9 +2770,9 @@ } }, "node_modules/@next/swc-darwin-x64": { - "version": "14.1.1-canary.59", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-14.1.1-canary.59.tgz", - "integrity": "sha512-yvVLVK9QPdU+JfviXNuUkGV2/dKo/Toy0g8VBN1n4oltTJWsZRnqguz5dZAfFX/fdQmc7QhK7TgJbpsk99+aUg==", + "version": "14.2.15", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-14.2.15.tgz", + "integrity": "sha512-5TGyjFcf8ampZP3e+FyCax5zFVHi+Oe7sZyaKOngsqyaNEpOgkKB3sqmymkZfowy3ufGA/tUgDPPxpQx931lHg==", "cpu": [ "x64" ], @@ -2785,9 +2785,9 @@ } }, "node_modules/@next/swc-linux-arm64-gnu": { - "version": "14.1.1-canary.59", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.1.1-canary.59.tgz", - "integrity": "sha512-aKAAEjTA0kQDuPcpSji26RiyIJVHMvSfa6V14oI270wkzGDzA+I7yDIdfAjghZg2AqpqP37sppTtk94hpg0Jpw==", + "version": "14.2.15", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.2.15.tgz", + "integrity": "sha512-3Bwv4oc08ONiQ3FiOLKT72Q+ndEMyLNsc/D3qnLMbtUYTQAmkx9E/JRu0DBpHxNddBmNT5hxz1mYBphJ3mfrrw==", "cpu": [ "arm64" ], @@ -2800,9 +2800,9 @@ } }, "node_modules/@next/swc-linux-arm64-musl": { - "version": "14.1.1-canary.59", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.1.1-canary.59.tgz", - "integrity": "sha512-63sUd3fspUDPP8pKfSDBpegxNV8t7QqM9uKCWiVVF253/fjLrm21ziTSP22Ij2HyxMzYDjUcXXw3oaV0/9+lHw==", + "version": "14.2.15", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.2.15.tgz", + "integrity": "sha512-k5xf/tg1FBv/M4CMd8S+JL3uV9BnnRmoe7F+GWC3DxkTCD9aewFRH1s5rJ1zkzDa+Do4zyN8qD0N8c84Hu96FQ==", "cpu": [ "arm64" ], @@ -2815,9 +2815,9 @@ } }, "node_modules/@next/swc-linux-x64-gnu": { - "version": "14.1.1-canary.59", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.1.1-canary.59.tgz", - "integrity": "sha512-XguNPS4Q7b3plMqfiYJvY4dQ+vyoEITt/rLxjRmnNmT0inaPA4Q9Lbl0rST6luVPYNaysmk1lzF1Rqodb5uudw==", + "version": "14.2.15", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.2.15.tgz", + "integrity": "sha512-kE6q38hbrRbKEkkVn62reLXhThLRh6/TvgSP56GkFNhU22TbIrQDEMrO7j0IcQHcew2wfykq8lZyHFabz0oBrA==", "cpu": [ "x64" ], @@ -2830,9 +2830,9 @@ } }, "node_modules/@next/swc-linux-x64-musl": { - "version": "14.1.1-canary.59", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.1.1-canary.59.tgz", - "integrity": "sha512-Hy+av18riWV3kYpO5vK0eQq6lN1NcZvnYG70mnsFwxiFaT2XVJimbIdSWiXNIB5QNF1Vv7AjcCVAw0nTojRKVg==", + "version": "14.2.15", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.2.15.tgz", + "integrity": "sha512-PZ5YE9ouy/IdO7QVJeIcyLn/Rc4ml9M2G4y3kCM9MNf1YKvFY4heg3pVa/jQbMro+tP6yc4G2o9LjAz1zxD7tQ==", "cpu": [ "x64" ], @@ -2845,9 +2845,9 @@ } }, "node_modules/@next/swc-win32-arm64-msvc": { - "version": "14.1.1-canary.59", - "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.1.1-canary.59.tgz", - "integrity": "sha512-Z7dj8ox/ih0HiKp5J2R5SV9sZLQ8mbuYy9g6MvwwmtGLkoCH9trLuo966ubYZ+nIFrzItWGf1HnMrnvNS00KlA==", + "version": "14.2.15", + "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.2.15.tgz", + "integrity": "sha512-2raR16703kBvYEQD9HNLyb0/394yfqzmIeyp2nDzcPV4yPjqNUG3ohX6jX00WryXz6s1FXpVhsCo3i+g4RUX+g==", "cpu": [ "arm64" ], @@ -2860,9 +2860,9 @@ } }, "node_modules/@next/swc-win32-ia32-msvc": { - "version": "14.1.1-canary.59", - "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.1.1-canary.59.tgz", - "integrity": "sha512-VQpCBFfbUiQax+/CWwjPwYHGZwUydpd7ZoUskkHHA0ZTbIDsSj64EzMFhK3tdo8JOOnNfVDAT/l95afmxYnzSA==", + "version": "14.2.15", + "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.2.15.tgz", + "integrity": "sha512-fyTE8cklgkyR1p03kJa5zXEaZ9El+kDNM5A+66+8evQS5e/6v0Gk28LqA0Jet8gKSOyP+OTm/tJHzMlGdQerdQ==", "cpu": [ "ia32" ], @@ -2875,9 +2875,9 @@ } }, "node_modules/@next/swc-win32-x64-msvc": { - "version": "14.1.1-canary.59", - "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.1.1-canary.59.tgz", - "integrity": "sha512-p7h/35PoqYxDNxce2EPkegIhe0149Wvdcq+HyVYkN8tp2fwAy2kKQM0RK8HlMU8NbiRwmlAx4lST/0pu+sd95g==", + "version": "14.2.15", + "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.2.15.tgz", + "integrity": "sha512-SzqGbsLsP9OwKNUG9nekShTwhj6JSB9ZLMWQ8g1gG6hdE5gQLncbnbymrwy2yVmH9nikSLYRYxYMFu78Ggp7/g==", "cpu": [ "x64" ], @@ -11621,6 +11621,19 @@ "version": "1.0.0", "license": "ISC" }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/function-bind": { "version": "1.1.2", "license": "MIT", @@ -11834,9 +11847,10 @@ } }, "node_modules/get-tsconfig": { - "version": "4.7.2", + "version": "4.8.1", + "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.8.1.tgz", + "integrity": "sha512-k9PN+cFBmaLWtVz29SkUoqU5O0slLuHJXt/2P+tMVFT+phsSGXGkp9t3rQIqdz0e+06EHNGs3oM6ZX1s2zHxRg==", "dev": true, - "license": "MIT", "dependencies": { "resolve-pkg-maps": "^1.0.0" }, @@ -14812,11 +14826,11 @@ "license": "MIT" }, "node_modules/next": { - "version": "14.1.1-canary.59", - "resolved": "https://registry.npmjs.org/next/-/next-14.1.1-canary.59.tgz", - "integrity": "sha512-MZOg68cUQQ3owGiPkBNr25X45LaEMqC0uX1uvTzfRWZISw9l6BR7aeI24Zv/g6YWszGG/YGfMDt/g/37FgstYA==", + "version": "14.2.15", + "resolved": "https://registry.npmjs.org/next/-/next-14.2.15.tgz", + "integrity": "sha512-h9ctmOokpoDphRvMGnwOJAedT6zKhwqyZML9mDtspgf4Rh3Pn7UTYKqePNoDvhsWBAO5GoPNYshnAUGIazVGmw==", "dependencies": { - "@next/env": "14.1.1-canary.59", + "@next/env": "14.2.15", "@swc/helpers": "0.5.5", "busboy": "1.6.0", "caniuse-lite": "^1.0.30001579", @@ -14831,18 +14845,19 @@ "node": ">=18.17.0" }, "optionalDependencies": { - "@next/swc-darwin-arm64": "14.1.1-canary.59", - "@next/swc-darwin-x64": "14.1.1-canary.59", - "@next/swc-linux-arm64-gnu": "14.1.1-canary.59", - "@next/swc-linux-arm64-musl": "14.1.1-canary.59", - "@next/swc-linux-x64-gnu": "14.1.1-canary.59", - "@next/swc-linux-x64-musl": "14.1.1-canary.59", - "@next/swc-win32-arm64-msvc": "14.1.1-canary.59", - "@next/swc-win32-ia32-msvc": "14.1.1-canary.59", - "@next/swc-win32-x64-msvc": "14.1.1-canary.59" + "@next/swc-darwin-arm64": "14.2.15", + "@next/swc-darwin-x64": "14.2.15", + "@next/swc-linux-arm64-gnu": "14.2.15", + "@next/swc-linux-arm64-musl": "14.2.15", + "@next/swc-linux-x64-gnu": "14.2.15", + "@next/swc-linux-x64-musl": "14.2.15", + "@next/swc-win32-arm64-msvc": "14.2.15", + "@next/swc-win32-ia32-msvc": "14.2.15", + "@next/swc-win32-x64-msvc": "14.2.15" }, "peerDependencies": { "@opentelemetry/api": "^1.1.0", + "@playwright/test": "^1.41.2", "react": "^18.2.0", "react-dom": "^18.2.0", "sass": "^1.3.0" @@ -14851,6 +14866,9 @@ "@opentelemetry/api": { "optional": true }, + "@playwright/test": { + "optional": true + }, "sass": { "optional": true } diff --git a/package.json b/package.json index b38e4d2..abf634f 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "db:introspect": "drizzle-kit introspect:pg", "db:generate": "drizzle-kit generate:pg", "db:migrate": "node -r dotenv/config scripts/migrate.mjs", + "cleanup": "node -r dotenv/config scripts/cron-monthly-cleanup.mjs", "email:dev": "email dev --port 3001 --dir src/components/emails" }, "dependencies": { diff --git a/scripts/cron-monthly-cleanup.mjs b/scripts/cron-monthly-cleanup.mjs new file mode 100644 index 0000000..8c1d2fc --- /dev/null +++ b/scripts/cron-monthly-cleanup.mjs @@ -0,0 +1,102 @@ +// import { PostgresJsDatabase, drizzle } from 'drizzle-orm/postgres-js'; +import { sql } from 'drizzle-orm'; +import { drizzle } from 'drizzle-orm/postgres-js'; +import * as Minio from 'minio'; +import postgres from 'postgres'; + +// import * as schema from '../src/infra/schema'; + +if ( + !process.env.DATABASE_URL || + !process.env.S3_ENDPOINT || + !process.env.S3_PORT || + !process.env.S3_USE_SSL || + !process.env.S3_ACCESS_KEY || + !process.env.S3_SECRET_KEY || + !process.env.S3_BUCKET_NAME +) { + throw new Error('S3 and database environment variables are not set'); +} + +const main = async () => { + const db = drizzle(postgres(process.env.DATABASE_URL)); + const fileStorage = new Minio.Client({ + endPoint: process.env.S3_ENDPOINT, + port: parseInt(process.env.S3_PORT, 10), + useSSL: 'true' === process.env.S3_USE_SSL, + accessKey: process.env.S3_ACCESS_KEY, + secretKey: process.env.S3_SECRET_KEY, + }); + + const exists = await fileStorage.bucketExists(process.env.S3_BUCKET_NAME); + if (exists) { + console.info(`Bucket ${process.env.S3_BUCKET_NAME} exists.`); + fileStorage.removeObjects; + } else { + throw new Error(`Bucket ${process.env.S3_BUCKET_NAME} does not exist`); + } + + // Remove all objects from the bucket + const objectsStream = fileStorage.listObjects(process.env.S3_BUCKET_NAME); + const incompleteUploadsStream = fileStorage.listIncompleteUploads( + process.env.S3_BUCKET_NAME, + ); + const objectNames = []; + const incompleteUploadNames = []; + objectsStream.on('data', (obj) => { + if (obj.name) { + objectNames.push(obj.name); + } + }); + objectsStream.on('error', (error) => { + console.error(error); + }); + objectsStream.on('end', async () => { + console.log( + `Removing ${objectNames.length} objects from bucket ${process.env.S3_BUCKET_NAME}`, + ); + await fileStorage.removeObjects( + process.env.S3_BUCKET_NAME, + objectNames, + ); + console.log( + `Removed ${objectNames.length} objects from bucket ${process.env.S3_BUCKET_NAME}`, + ); + }); + incompleteUploadsStream.on('data', (obj) => { + incompleteUploadNames.push(obj.key); + }); + incompleteUploadsStream.on('error', (error) => { + console.error(error); + }); + incompleteUploadsStream.on('end', async () => { + console.log( + `Removing ${incompleteUploadNames.length} incomplete uploads from bucket ${process.env.S3_BUCKET_NAME}`, + ); + for (const name of incompleteUploadNames) { + await fileStorage.removeIncompleteUpload( + process.env.S3_BUCKET_NAME, + name, + ); + } + console.log( + `Removed ${incompleteUploadNames.length} incomplete uploads from bucket ${process.env.S3_BUCKET_NAME}`, + ); + }); + + // Delete assets and tracks tables + await db.transaction(async (tx) => { + // have to write raw SQL, careful to sync with table names in schema + await tx.execute(sql`DELETE FROM asset`); + await tx.execute(sql`DELETE FROM track`); + }); + + process.exit(0); +}; + +try { + await main(); +} catch (error) { + console.error(error); + process.exit(1); +}