From 87650e9723c88f935a0cf88f23a53132cf849c31 Mon Sep 17 00:00:00 2001 From: "E. Cooper" Date: Thu, 9 Jan 2025 11:18:24 -0800 Subject: [PATCH 01/19] Add CSV colorization --- src/lib/formatting/codeToAnsi.mjs | 3 ++- src/lib/formatting/colorize.mjs | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/lib/formatting/codeToAnsi.mjs b/src/lib/formatting/codeToAnsi.mjs index 78848300..cdcb81b6 100644 --- a/src/lib/formatting/codeToAnsi.mjs +++ b/src/lib/formatting/codeToAnsi.mjs @@ -1,6 +1,7 @@ import chalk from "chalk"; import { createHighlighterCoreSync } from "shiki/core"; import { createJavaScriptRegexEngine } from "shiki/engine/javascript"; +import csv from "shiki/langs/csv.mjs"; import json from "shiki/langs/json.mjs"; import log from "shiki/langs/log.mjs"; import yaml from "shiki/langs/yaml.mjs"; @@ -14,7 +15,7 @@ const THEME = "github-dark-high-contrast"; export const createHighlighter = () => { const highlighter = createHighlighterCoreSync({ themes: [githubDarkHighContrast], - langs: [fql, log, json, yaml], + langs: [fql, log, json, yaml, csv], engine: createJavaScriptRegexEngine(), }); diff --git a/src/lib/formatting/colorize.mjs b/src/lib/formatting/colorize.mjs index 2df1f1cc..556e42ae 100644 --- a/src/lib/formatting/colorize.mjs +++ b/src/lib/formatting/colorize.mjs @@ -10,6 +10,7 @@ export const Format = { JSON: "json", TEXT: "text", YAML: "yaml", + CSV: "csv", }; const objToString = (obj) => JSON.stringify(obj, null, 2); @@ -66,6 +67,21 @@ const yamlToAnsi = (obj) => { return res.trim(); }; +const csvToAnsi = (obj) => { + if (typeof obj !== "string") { + throw new Error("Unable to format CSV unless it is already a string."); + } + + const codeToAnsi = container.resolve("codeToAnsi"); + const res = codeToAnsi(obj, "csv"); + + if (!res) { + return ""; + } + + return res.trim(); +}; + /** * Formats an object for display with ANSI color codes. * @param {any} obj - The object to format @@ -83,6 +99,8 @@ export const toAnsi = (obj, { format = Format.TEXT } = {}) => { return logToAnsi(obj); case Format.YAML: return yamlToAnsi(obj); + case Format.CSV: + return csvToAnsi(obj); default: return textToAnsi(obj); } From 1fdb99d3cb549ee151b844c2b6814652c1804257 Mon Sep 17 00:00:00 2001 From: "E. Cooper" Date: Thu, 9 Jan 2025 15:09:16 -0800 Subject: [PATCH 02/19] Add create, list, and get export commands --- src/cli.mjs | 6 +- src/commands/database/list.mjs | 4 +- src/commands/export/create.mjs | 112 +++++ src/commands/export/export.mjs | 45 ++ src/commands/export/get.mjs | 31 ++ src/commands/export/list.mjs | 113 +++++ src/config/setup-test-container.mjs | 3 + src/lib/account-api.mjs | 92 +++- test/commands/export/create.mjs | 138 ++++++ test/commands/export/export.mjs | 49 +++ test/commands/export/get.mjs | 76 ++++ test/commands/export/list.mjs | 82 ++++ test/lib/account-api.mjs | 409 ------------------ test/lib/account-api/account-api.mjs | 393 +++++++++++++++++ .../account-api/fetch-with-account-key.mjs | 104 +++++ test/lib/account-api/response-handler.mjs | 189 ++++++++ 16 files changed, 1433 insertions(+), 413 deletions(-) create mode 100644 src/commands/export/create.mjs create mode 100644 src/commands/export/export.mjs create mode 100644 src/commands/export/get.mjs create mode 100644 src/commands/export/list.mjs create mode 100644 test/commands/export/create.mjs create mode 100644 test/commands/export/export.mjs create mode 100644 test/commands/export/get.mjs create mode 100644 test/commands/export/list.mjs delete mode 100644 test/lib/account-api.mjs create mode 100644 test/lib/account-api/account-api.mjs create mode 100644 test/lib/account-api/fetch-with-account-key.mjs create mode 100644 test/lib/account-api/response-handler.mjs diff --git a/src/cli.mjs b/src/cli.mjs index f770a075..9ead3498 100644 --- a/src/cli.mjs +++ b/src/cli.mjs @@ -8,6 +8,7 @@ import chalk from "chalk"; import yargs from "yargs"; import databaseCommand from "./commands/database/database.mjs"; +import exportCommand from "./commands/export/export.mjs"; import localCommand from "./commands/local.mjs"; import loginCommand from "./commands/login.mjs"; import queryCommand from "./commands/query.mjs"; @@ -114,11 +115,12 @@ function buildYargs(argvInput) { [applyLocalArg, fixPaths, applyAccountUrl, buildCredentials], false, ) + .command(loginCommand) + .command(databaseCommand) .command(queryCommand) .command(shellCommand) - .command(loginCommand) .command(schemaCommand) - .command(databaseCommand) + .command(exportCommand) .command(localCommand) .demandCommand() .strictCommands(true) diff --git a/src/commands/database/list.mjs b/src/commands/database/list.mjs index e82a9113..60baec58 100644 --- a/src/commands/database/list.mjs +++ b/src/commands/database/list.mjs @@ -58,7 +58,9 @@ async function doListDatabases(argv) { logger.stdout(colorize(res, { format: Format.JSON, color: argv.color })); } else { res.forEach(({ path, name }) => { - logger.stdout(path ?? name); + logger.stdout( + colorize(path ?? name, { format: Format.CSV, color: argv.color }), + ); }); } } diff --git a/src/commands/export/create.mjs b/src/commands/export/create.mjs new file mode 100644 index 00000000..d655d273 --- /dev/null +++ b/src/commands/export/create.mjs @@ -0,0 +1,112 @@ +// @ts-check + +import { container } from "../../config/container.mjs"; +import { ValidationError } from "../../lib/errors.mjs"; +import { colorize, Format } from "../../lib/formatting/colorize.mjs"; +import { DATABASE_PATH_OPTIONS } from "../../lib/options.mjs"; + +async function createS3Export(argv) { + const logger = container.resolve("logger"); + const { + database, + path, + bucket, + format, + json, + color, + collection: collections, + } = argv; + const { createExport } = container.resolve("accountAPI"); + + const response = await createExport({ + database, + collections, + destination: { + s3: { + bucket, + path, + }, + }, + format, + }); + + if (json) { + logger.stdout(colorize(response, { color, format: Format.JSON })); + } else { + logger.stdout(response.id); + } +} + +function buildCreateS3ExportCommand(yargs) { + return yargs + .options({ + bucket: { + type: "string", + required: true, + description: "Name of the bucket to export to.", + }, + path: { + type: "string", + required: true, + description: "Key prefix to export to.", + }, + format: { + type: "string", + required: true, + description: "Format to export to.", + choices: ["simple", "tagged"], + default: "simple", + }, + }) + .check((argv) => { + if (!argv.database) { + throw new ValidationError( + "--database is required to create an export.", + ); + } + + return true; + }) + .example([ + [ + "$0 export create s3 -d us/my_db --bucket my-bucket --path my-prefix", + "Create an export of collections to the 'my-bucket' bucket with the 'my-prefix' key prefix in simple format.", + ], + [ + "$0 export create s3 -d us/my_db --bucket my-bucket --path my-prefix --collections my-collection", + "Create an export of the 'my-collection' collection to the 'my-bucket' bucket with the 'my-prefix' key prefix in simple format.", + ], + [ + "$0 export create s3 -d us/my_db --bucket my-bucket --path my-prefix --collection my-collection --format tagged", + "Create an export of the 'my-collection' collection to the 'my-bucket' bucket with the 'my-prefix' key prefix in tagged format.", + ], + ]); +} + +function buildCreateCommand(yargs) { + return yargs + .options(DATABASE_PATH_OPTIONS) + .options({ + collection: { + type: "array", + required: false, + description: + "The name of the collections to export. If empty, all collections will be exported.", + default: [], + }, + }) + .command({ + command: "s3", + description: "Create a database export to an S3 bucket.", + builder: buildCreateS3ExportCommand, + handler: createS3Export, + }); +} + +export default { + command: "create ", + description: "Create a database export to a given destination.", + builder: buildCreateCommand, + // eslint-disable-next-line no-empty-function + handler: () => {}, +}; diff --git a/src/commands/export/export.mjs b/src/commands/export/export.mjs new file mode 100644 index 00000000..a3e4c7b0 --- /dev/null +++ b/src/commands/export/export.mjs @@ -0,0 +1,45 @@ +import { ValidationError } from "../../lib/errors.mjs"; +import { ACCOUNT_OPTIONS } from "../../lib/options.mjs"; +import createCommand from "./create.mjs"; +import getCommand from "./get.mjs"; +import listCommand from "./list.mjs"; + +/** + * Validates the arguments do not include Core API authentication options. + * In the CLI, we don't validate unknown options, but because these commands are unique and + * only used the Account API, we aggressively validate the options here to avoid confusion. + * @param {import("yargs").Arguments} argv + * @returns {boolean} + */ +function validateAccountOnlyOptions(argv) { + const { secret, local } = argv; + + if (local) { + throw new ValidationError( + "Exports do not support --local and the Fauna docker container.", + ); + } + + if (secret) { + throw new ValidationError("Exports are not supported with --secret."); + } + + return true; +} + +function buildExportCommand(yargs) { + return yargs + .options(ACCOUNT_OPTIONS) + .check(validateAccountOnlyOptions) + .command(createCommand) + .command(listCommand) + .command(getCommand); +} + +export default { + command: "export ", + description: "Create and manage database exports.", + builder: buildExportCommand, + // eslint-disable-next-line no-empty-function + handler: () => {}, +}; diff --git a/src/commands/export/get.mjs b/src/commands/export/get.mjs new file mode 100644 index 00000000..602bf8e6 --- /dev/null +++ b/src/commands/export/get.mjs @@ -0,0 +1,31 @@ +import { container } from "../../config/container.mjs"; +import { colorize, Format } from "../../lib/formatting/colorize.mjs"; + +async function getExport(argv) { + const logger = container.resolve("logger"); + const { getExport } = container.resolve("accountAPI"); + const { exportId, json, color } = argv; + const response = await getExport({ exportId }); + + if (json) { + logger.stdout(colorize(response, { color, format: Format.JSON })); + } else { + logger.stdout(colorize(response, { color, format: Format.YAML })); + } +} + +function buildGetExportCommand(yargs) { + return yargs.positional("exportId", { + type: "string", + description: "The ID of the export to get.", + nargs: 1, + required: true, + }); +} + +export default { + command: "get ", + description: "Get an export by ID.", + builder: buildGetExportCommand, + handler: getExport, +}; diff --git a/src/commands/export/list.mjs b/src/commands/export/list.mjs new file mode 100644 index 00000000..5283d8c2 --- /dev/null +++ b/src/commands/export/list.mjs @@ -0,0 +1,113 @@ +import { container } from "../../config/container.mjs"; +import { colorize, Format } from "../../lib/formatting/colorize.mjs"; + +/* eslint-disable camelcase */ +/** + * Converts an export object to a CSV string. + * @param {{ id: string, database: string, created_at: string, updated_at: string, state: string }} export + * @returns {string} + */ +function exportToCSV({ id, database, created_at, updated_at, state }) { + return `${database},${id},${created_at},${updated_at},${state}`; +} +/* eslint-enable camelcase */ + +/** + * Outputs the exports to the console. + * @param {Object} params - The parameters for outputting the exports. + * @param {Function} params.stdout - The function to use for outputting the exports. + * @param {string[]} params.exports - The exports to output. + * @param {boolean} params.color - Whether to colorize the output. + */ +export function outputExports({ stdout, exports, color }) { + if (!exports || exports.length === 0) { + return; + } + + stdout( + colorize( + exportToCSV({ + id: "id", + database: "database", + state: "state", + /* eslint-disable camelcase */ + created_at: "created_at", + updated_at: "updated_at", + /* eslint-enable camelcase */ + }), + { color, format: Format.CSV }, + ), + ); + + exports.forEach((r) => { + stdout(colorize(exportToCSV(r), { color, format: Format.CSV })); + }); +} + +async function listExports(argv) { + const logger = container.resolve("logger"); + const { json, color, maxResults, state } = argv; + const { listExports } = container.resolve("accountAPI"); + + const { results } = await listExports({ + maxResults, + state: state, + }); + + if (json) { + logger.stdout(colorize(results, { color, format: Format.JSON })); + } else { + logger.stdout( + colorize( + ["database", "id", "created_at", "updated_at", "state"].join(","), + { color, format: Format.CSV }, + ), + ); + + results.forEach((r) => { + const { database, id, state, created_at, updated_at } = r; // eslint-disable-line camelcase + logger.stdout( + colorize( + [database, id, created_at, updated_at, state].join(","), // eslint-disable-line camelcase + { + color, + format: Format.CSV, + }, + ), + ); + }); + } +} + +function buildListExportsCommand(yargs) { + return yargs + .options({ + "max-results": { + alias: "max", + type: "number", + description: "Maximum number of exports to return. Defaults to 100.", + default: 100, + group: "API:", + }, + state: { + type: "array", + description: "Filter exports by state.", + default: [], + group: "API:", + choices: ["Pending", "InProgress", "Complete", "Failed"], + }, + }) + .example([ + ["$0 export list", "List exports in CSV format."], + ["$0 export list --max-results 100", "List a max of 100 exports."], + ["$0 export list --json", "List exports in JSON format."], + ["$0 export list --states Pending", "List exports in Pending state."], + ]); +} + +export default { + command: "list", + describe: "List database exports.", + builder: buildListExportsCommand, + handler: listExports, +}; diff --git a/src/config/setup-test-container.mjs b/src/config/setup-test-container.mjs index c5ce80f3..debc5b69 100644 --- a/src/config/setup-test-container.mjs +++ b/src/config/setup-test-container.mjs @@ -96,6 +96,9 @@ export function setupTestContainer() { createKey: stub(), refreshSession: stub(), getSession: stub(), + createExport: stub(), + getExport: stub(), + listExports: stub(), }), errorHandler: awilix.asValue((error, exitCode) => { error.code = exitCode; diff --git a/src/lib/account-api.mjs b/src/lib/account-api.mjs index 9fbeca0e..1fc8e1ca 100644 --- a/src/lib/account-api.mjs +++ b/src/lib/account-api.mjs @@ -67,7 +67,11 @@ export function toResource({ }) { const url = new URL(`${version}${endpoint}`, getAccountUrl()); for (const [key, value] of Object.entries(params)) { - url.searchParams.set(key, value); + if (Array.isArray(value)) { + value.forEach((v) => url.searchParams.append(key, v)); + } else { + url.searchParams.set(key, value); + } } return url; @@ -388,6 +392,89 @@ async function createKey({ path, role, ttl, name }) { return await responseHandler(response); } +/** + * Creates an export for a given database. + * + * @param {Object} params - The parameters for creating the export. + * @param {string} params.database - The path of the database, including region group. + * @param {string[] | undefined} [params.collections] - The collections to export. + * @param {Object} params.destination - The destination for the export. + * @param {Object} params.destination.s3 - The S3 destination for the export. + * @param {string} params.destination.s3.bucket - The name of the S3 bucket to export to. + * @param {string} params.destination.s3.path - The key prefix to export to. + * @param {string} params.format - The format for the export. + * @returns {Promise} - A promise that resolves when the export is created. + * @throws {AuthorizationError | AuthenticationError | CommandError | Error} If the response is not OK + */ +async function createExport({ + database, + destination, + format, + collections = undefined, +}) { + const url = toResource({ endpoint: "/exports", version: API_VERSIONS.v2 }); + const response = await fetchWithAccountKey(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + database: standardizeRegion(database), + destination, + format, + ...(collections && collections.length > 0 ? { collections } : {}), + }), + }); + return await responseHandler(response); +} + +/** + * Lists exports associated with the given account key. + * + * @param {Object} [params] - The parameters for listing the exports. + * @param {number} [params.maxResults] - The number of exports to return. Default 16. + * @param {string} [params.nextToken] - The next token for pagination. + * @param {string[]} [params.state] - The states to filter exports by. + * @returns {Promise} - A promise that resolves when the exports are listed. + * @throws {AuthorizationError | AuthenticationError | CommandError | Error} If the response is not OK + */ +async function listExports({ maxResults = 100, nextToken, state } = {}) { + const url = toResource({ + endpoint: "/exports", + version: API_VERSIONS.v2, + params: { + /* eslint-disable camelcase */ + max_results: maxResults, + ...(nextToken && { next_token: nextToken }), + ...(state && state.length > 0 ? { state } : {}), + /* eslint-enable camelcase */ + }, + }); + + const response = await fetchWithAccountKey(url, { + method: "GET", + }); + + return await responseHandler(response); +} + +/** + * Get an export by ID. + * + * @param {Object} params - The parameters for getting the export. + * @param {string} params.exportId - The ID of the export to get. + * @returns {Promise} - A promise that resolves when the export is retrieved. + * @throws {AuthorizationError | AuthenticationError | CommandError | Error} If the response is not OK + */ +async function getExport({ exportId }) { + const url = toResource({ + endpoint: `/exports/${exportId}`, + version: API_VERSIONS.v2, + }); + const response = await fetchWithAccountKey(url, { method: "GET" }); + return await responseHandler(response); +} + /** * The account API module with the currently supported endpoints. */ @@ -396,6 +483,9 @@ const accountAPI = { createKey, refreshSession, getSession, + createExport, + listExports, + getExport, }; export default accountAPI; diff --git a/test/commands/export/create.mjs b/test/commands/export/create.mjs new file mode 100644 index 00000000..13be14ac --- /dev/null +++ b/test/commands/export/create.mjs @@ -0,0 +1,138 @@ +// @ts-check + +import { expect } from "chai"; +import sinon from "sinon"; + +import { run } from "../../../src/cli.mjs"; +import { setupTestContainer as setupContainer } from "../../../src/config/setup-test-container.mjs"; +import { colorize, Format } from "../../../src/lib/formatting/colorize.mjs"; + +const createExportStub = (opts) => ({ + id: "test-export-id", + state: "Pending", + database: "us-std/example", + format: "simple", + destination: { + s3: { + path: "/test/key", + bucket: "test-bucket", + }, + }, + created_at: "2025-01-02T22:59:51", + ...opts, +}); + +describe("export create s3", () => { + let container, stderr, stdout, createExport; + + beforeEach(() => { + container = setupContainer(); + stderr = container.resolve("stderrStream"); + stdout = container.resolve("stdoutStream"); + ({ createExport } = container.resolve("accountAPI")); + }); + + it("creates an export", async () => { + const database = "us-std/example"; + const bucket = "test-bucket"; + const path = "/test/key"; + const stubbedResponse = createExportStub({ + database, + destination: { + s3: { + path, + bucket, + }, + }, + format: "simple", + }); + createExport.resolves(stubbedResponse); + + await run( + `export create s3 --database '${database}' --bucket '${bucket}' --path '${path}'`, + container, + ); + await stdout.waitForWritten(); + + expect(stdout.getWritten()).to.equal("test-export-id\n"); + expect(createExport).to.have.been.calledWith({ + database, + collections: [], + destination: { + s3: { + bucket, + path, + }, + }, + format: "simple", + }); + }); + + it("outputs the full response with --json", async () => { + const database = "us-std/example"; + const bucket = "test-bucket"; + const path = "/test/key"; + const stubbedResponse = createExportStub({ + database, + destination: { + s3: { + path, + bucket, + }, + }, + format: "simple", + }); + createExport.resolves(stubbedResponse); + + await run( + `export create s3 --database '${database}' --bucket '${bucket}' --path '${path}' --json`, + container, + ); + await stdout.waitForWritten(); + + expect(stdout.getWritten()).to.equal( + `${colorize(stubbedResponse, { format: Format.JSON })}\n`, + ); + }); + + it("passes the format to the account api", async () => { + createExport.resolves(createExportStub({ format: "tagged" })); + await run( + `export create s3 --database 'us-std/example' --bucket 'test-bucket' --path 'test/key' --format 'tagged'`, + container, + ); + expect(createExport).to.have.been.calledWith( + sinon.match({ + format: "tagged", + }), + ); + }); + + it("should allow providing multiple collections", async () => { + createExport.resolves(createExportStub({ collections: ["foo", "bar"] })); + await run( + `export create s3 --database 'us-std/example' --bucket 'test-bucket' --path 'test/key' --collection foo --collection bar`, + container, + ); + expect(createExport).to.have.been.calledWith( + sinon.match({ + database: "us-std/example", + collections: ["foo", "bar"], + }), + ); + }); + + it("should output an error if --database is not provided", async () => { + try { + await run( + "export create s3 --bucket test-bucket --path test/key", + container, + ); + } catch {} + + await stderr.waitForWritten(); + expect(stderr.getWritten()).to.contain( + "--database is required to create an export.", + ); + }); +}); diff --git a/test/commands/export/export.mjs b/test/commands/export/export.mjs new file mode 100644 index 00000000..430f4985 --- /dev/null +++ b/test/commands/export/export.mjs @@ -0,0 +1,49 @@ +// @ts-check + +import { expect } from "chai"; + +import { run } from "../../../src/cli.mjs"; +import { setupTestContainer as setupContainer } from "../../../src/config/setup-test-container.mjs"; + +describe("export", () => { + let container, stderr; + + beforeEach(() => { + container = setupContainer(); + stderr = container.resolve("stderrStream"); + }); + + [ + "export create s3 -d us/demo --bucket test-bucket --path test/key --local", + "export list --local", + "export get 1234567890 --local", + ].forEach((cmd) => { + it(`should output an error if --local is used: ${cmd}`, async () => { + try { + await run(cmd, container); + } catch {} + + await stderr.waitForWritten(); + expect(stderr.getWritten()).to.contain( + "Exports do not support --local and the Fauna docker container.", + ); + }); + }); + + [ + "export create s3 --bucket test-bucket --path test/key --secret=some-test-secret", + "export list --secret=some-test-secret", + "export get 1234567890 --secret=some-test-secret", + ].forEach((cmd) => { + it(`should output an error if --secret is provided: ${cmd}`, async () => { + try { + await run(cmd, container); + } catch {} + + await stderr.waitForWritten(); + expect(stderr.getWritten()).to.contain( + "Exports are not supported with --secret.", + ); + }); + }); +}); diff --git a/test/commands/export/get.mjs b/test/commands/export/get.mjs new file mode 100644 index 00000000..c77a7aa1 --- /dev/null +++ b/test/commands/export/get.mjs @@ -0,0 +1,76 @@ +import { expect } from "chai"; + +import { run } from "../../../src/cli.mjs"; +import { setupTestContainer as setupContainer } from "../../../src/config/setup-test-container.mjs"; + +const getExportStub = (opts) => ({ + id: "419630463817089613", + state: "Pending", + database: "us-std/demo", + format: "simple", + destination: { + s3: { + bucket: "test-bucket", + path: "some/key/prefix", + }, + }, + created_at: "2025-01-09T19:07:25.642703Z", + updated_at: "2025-01-09T19:07:25.642703Z", + ...opts, +}); + +describe("export get", () => { + let container, getExport, stdout; + + beforeEach(() => { + container = setupContainer(); + getExport = container.resolve("accountAPI").getExport; + stdout = container.resolve("stdoutStream"); + }); + + it("should get an export by ID", async () => { + const stubbedResponse = getExportStub({ + failure: { + code: "validation_error", + message: "failed to get bucket region: bucket not found", + }, + failed_at: "2025-01-09T19:07:26.600811Z", + state: "Failed", + }); + getExport.resolves(stubbedResponse); + + await run(`export get 419630463817089613`, container); + await stdout.waitForWritten(); + + expect(stdout.getWritten()).to.equal(`id: "419630463817089613" +state: Failed +database: us-std/demo +format: simple +destination: + s3: + bucket: test-bucket + path: some/key/prefix +created_at: 2025-01-09T19:07:25.642703Z +updated_at: 2025-01-09T19:07:25.642703Z +failure: + code: validation_error + message: "failed to get bucket region: bucket not found" +failed_at: 2025-01-09T19:07:26.600811Z +`); + expect(getExport).to.have.been.calledWith({ + exportId: "419630463817089613", + }); + }); + + it("should output JSON when --json is passed", async () => { + const stubbedResponse = getExportStub(); + getExport.resolves(stubbedResponse); + + await run(`export get 419630463817089613 --json`, container); + await stdout.waitForWritten(); + + expect(stdout.getWritten()).to.equal( + `${JSON.stringify(stubbedResponse, null, 2)}\n`, + ); + }); +}); diff --git a/test/commands/export/list.mjs b/test/commands/export/list.mjs new file mode 100644 index 00000000..d081fbed --- /dev/null +++ b/test/commands/export/list.mjs @@ -0,0 +1,82 @@ +// @ts-check + +import { expect } from "chai"; + +import { run } from "../../../src/cli.mjs"; +import { setupTestContainer as setupContainer } from "../../../src/config/setup-test-container.mjs"; +import { colorize, Format } from "../../../src/lib/formatting/colorize.mjs"; + +const listExportStub = (opts) => ({ + id: "test-export-id", + state: "Pending", + database: "us-std/example", + created_at: "2025-01-02T22:59:51", + updated_at: "2025-01-02T22:59:51", + ...opts, +}); + +describe("export list", () => { + let container, stdout, listExports; + + beforeEach(() => { + container = setupContainer(); + stdout = container.resolve("stdoutStream"); + ({ listExports } = container.resolve("accountAPI")); + }); + + it("lists exports", async () => { + const stubbedResponse = listExportStub({ + id: "tid", + database: "us-std/test", + }); + listExports.resolves({ results: [stubbedResponse] }); + + await run(`export list`, container); + await stdout.waitForWritten(); + + expect(stdout.getWritten()).to.equal( + `${[ + "database,id,created_at,updated_at,state", + "us-std/test,tid,2025-01-02T22:59:51,2025-01-02T22:59:51,Pending", + ].join("\n")}\n`, + ); + expect(listExports).to.have.been.calledWith({ + maxResults: 100, + state: [], + }); + }); + + it("supports --json", async () => { + const stubbedResponse = listExportStub(); + listExports.resolves({ results: [stubbedResponse] }); + + await run(`export list --json`, container); + await stdout.waitForWritten(); + + expect(stdout.getWritten()).to.equal( + `${colorize([stubbedResponse], { format: Format.JSON })}\n`, + ); + }); + + it("supports --max-results", async () => { + listExports.resolves({ results: [listExportStub()] }); + + await run(`export list --max-results 1`, container); + + expect(listExports).to.have.been.calledWith({ + maxResults: 1, + state: [], + }); + }); + + it("supports --state", async () => { + listExports.resolves({ results: [listExportStub()] }); + + await run(`export list --state Pending --state InProgress`, container); + + expect(listExports).to.have.been.calledWith({ + maxResults: 100, + state: ["Pending", "InProgress"], + }); + }); +}); diff --git a/test/lib/account-api.mjs b/test/lib/account-api.mjs deleted file mode 100644 index 98de28fe..00000000 --- a/test/lib/account-api.mjs +++ /dev/null @@ -1,409 +0,0 @@ -import * as awilix from "awilix"; -import { expect } from "chai"; -import sinon from "sinon"; - -import { setContainer } from "../../src/config/container.mjs"; -import { setupTestContainer as setupContainer } from "../../src/config/setup-test-container.mjs"; -import accountAPI, { - fetchWithAccountKey, - responseHandler, - toResource, -} from "../../src/lib/account-api.mjs"; -import { - AuthenticationError, - AuthorizationError, - CommandError, -} from "../../src/lib/errors.mjs"; -import { f } from "../helpers.mjs"; - -describe("toResource", () => { - it("should build a URL with the correct endpoint and parameters", () => { - const url = toResource({ endpoint: "/users", params: { limit: 10 } }); - expect(url.toString()).to.equal( - "https://account.fauna.com/api/v1/users?limit=10", - ); - }); - - it("should respect v2 endpoints when specified", () => { - const url = toResource({ - endpoint: "/users", - params: { limit: 10 }, - version: "/v2", - }); - expect(url.toString()).to.equal( - "https://account.fauna.com/v2/users?limit=10", - ); - }); -}); - -describe("responseHandler", () => { - const createMockResponse = ( - status, - body = {}, - contentType = "application/json", - ) => { - return { - ok: status >= 200 && status < 300, - status, - headers: { - get: () => contentType, - }, - json: async () => body, - }; - }; - - it("should standardize v1 and v2 endpoint errors to the same CommandError", async () => { - const v1Response = createMockResponse(400, { - code: "bad_request", - reason: "Database is not specified", - }); - const v2Response = createMockResponse(400, { - error: { - code: "bad_request", - message: "Database is not specified", - }, - }); - - let v1Error, v2Error; - try { - await responseHandler(v1Response); - } catch (error) { - v1Error = error; - } - - try { - await responseHandler(v2Response); - } catch (error) { - v2Error = error; - } - - // Check that the errors are equal instances of a CommandError - expect(v1Error).to.be.instanceOf(CommandError); - expect(v2Error).to.be.instanceOf(CommandError); - expect(v1Error.message).to.equal(v2Error.message); - expect(v1Error.cause).to.deep.equal(v2Error.cause); - - // Check that the errors have the correct code and message - expect(v1Error.message).to.equal("Database is not specified"); - }); - - it("should throw AuthenticationError for 401 status", async () => { - const response = createMockResponse(401, { - code: "unauthorized", - reason: "Invalid credentials", - }); - - try { - await responseHandler(response); - } catch (error) { - expect(error).to.be.instanceOf(AuthenticationError); - } - }); - - it("should throw AuthorizationError for 403 status", async () => { - const response = createMockResponse(403, { - code: "permission_denied", - reason: "Insufficient permissions", - }); - - try { - await responseHandler(response); - } catch (error) { - expect(error).to.be.instanceOf(AuthorizationError); - } - }); - - it("should throw CommandError for 400 status", async () => { - const response = createMockResponse(400, { - code: "bad_request", - reason: "Invalid parameters", - }); - - try { - await responseHandler(response); - } catch (error) { - expect(error).to.be.instanceOf(CommandError); - } - }); - - it("should throw CommandError for 404 status", async () => { - const response = createMockResponse(404, { - code: "not_found", - reason: "Resource not found", - }); - - try { - await responseHandler(response); - } catch (error) { - expect(error).to.be.instanceOf(CommandError); - } - }); - - it("should throw generic Error for other error status codes", async () => { - const response = createMockResponse(500, { - code: "internal_error", - reason: "This is a server error", - }); - - try { - await responseHandler(response); - } catch (error) { - expect(error).to.be.instanceOf(Error); - } - }); - - it("should handle non-JSON responses", async () => { - const response = { - status: 400, - headers: { - get: () => "text/plain", - }, - }; - - try { - await responseHandler(response); - } catch (error) { - expect(error).to.be.instanceOf(CommandError); - expect(error.message).to.equal( - "An unknown error occurred while making a request to the Account API.", - ); - } - }); - - it("should preserve error details in cause", async () => { - const responseBody = { - code: "bad_request", - reason: "Invalid parameters", - }; - const response = createMockResponse(400, responseBody); - - try { - await responseHandler(response); - } catch (error) { - expect(error.cause).to.exist; - expect(error.cause.status).to.equal(400); - expect(error.cause.body).to.deep.equal(responseBody); - expect(error.cause.code).to.equal("bad_request"); - expect(error.cause.message).to.equal("Invalid parameters"); - } - }); - - it("should return parsed JSON for successful responses", async () => { - const responseBody = { data: "success" }; - const response = createMockResponse(200, responseBody); - - const result = await responseHandler(response); - expect(result).to.deep.equal(responseBody); - }); -}); - -describe("accountAPI", () => { - let container, fetch; - - beforeEach(() => { - container = setupContainer(); - fetch = container.resolve("fetch"); - - container.register({ - credentials: awilix.asValue({ - accountKeys: { - key: "some-account-key", - onInvalidCreds: async () => { - container.resolve("credentials").accountKeys.key = - "new-account-key"; - return Promise.resolve(); - }, - promptLogin: sinon.stub(), - }, - }), - }); - - setContainer(container); - }); - - describe("fetchWithAccountKey", () => { - it("should call the endpoint with the correct headers", async () => { - await fetchWithAccountKey("https://account.fauna.com/api/v1/databases", { - method: "GET", - }); - - expect(fetch).to.have.been.calledWith( - "https://account.fauna.com/api/v1/databases", - { - method: "GET", - headers: { - Authorization: "Bearer some-account-key", - }, - }, - ); - }); - - it("should retry once when the response is a 401", async () => { - fetch - .withArgs("https://account.fauna.com/api/v1/databases") - .onCall(0) - .resolves(f(null, 401)); - - fetch - .withArgs("https://account.fauna.com/api/v1/databases") - .onCall(1) - .resolves(f({ results: [] }, 200)); - - const response = await fetchWithAccountKey( - "https://account.fauna.com/api/v1/databases", - { - method: "GET", - }, - ); - - expect(fetch).to.have.been.calledWith( - "https://account.fauna.com/api/v1/databases", - { - method: "GET", - headers: { - Authorization: "Bearer some-account-key", - }, - }, - ); - expect(fetch).to.have.been.calledWith( - "https://account.fauna.com/api/v1/databases", - { - method: "GET", - headers: { - Authorization: "Bearer new-account-key", - }, - }, - ); - expect(await response.json()).to.deep.equal({ results: [] }); - }); - - it("should only retry authorization errors once", async () => { - fetch - .withArgs("https://account.fauna.com/api/v1/databases") - .resolves(f(null, 401)); - - const response = await fetchWithAccountKey( - "https://account.fauna.com/api/v1/databases", - { - method: "GET", - }, - ); - - expect(response.status).to.equal(401); - expect(await response.json()).to.deep.equal(null); - }); - }); - - describe("listDatabases", () => { - it("should call the endpoint", async () => { - fetch - .withArgs( - sinon.match({ - href: "https://account.fauna.com/api/v1/databases?max_results=1000", - }), - sinon.match.any, - ) - .resolves( - f({ - results: [{ name: "test-db", path: "us-std/test-db" }], - }), - ); - - const data = await accountAPI.listDatabases({}); - - expect(fetch).to.have.been.calledWith( - sinon.match({ - href: "https://account.fauna.com/api/v1/databases?max_results=1000", - }), - sinon.match({ - method: "GET", - headers: { - Authorization: "Bearer some-account-key", - }, - }), - ); - - expect(data).to.deep.equal({ - results: [{ name: "test-db", path: "us-std/test-db" }], - }); - }); - - it("should call the endpoint with a path", async () => { - fetch - .withArgs( - sinon.match({ - href: "https://account.fauna.com/api/v1/databases?max_results=1000&path=us-std%2Ftest-db", - }), - ) - .resolves( - f({ - results: [{ name: "test-db", path: "us-std/test-db" }], - }), - ); - - const data = await accountAPI.listDatabases({ path: "us-std/test-db" }); - - expect(data).to.deep.equal({ - results: [{ name: "test-db", path: "us-std/test-db" }], - }); - }); - }); - - describe("createKey", () => { - it("should call the endpoint", async () => { - fetch - .withArgs( - sinon.match({ - href: "https://account.fauna.com/api/v1/databases/keys", - }), - sinon.match.any, - ) - .resolves( - f( - { - id: "key-id", - role: "admin", - path: "us-std/test-db", - ttl: "2025-01-01T00:00:00.000Z", - name: "test-key", - }, - 201, - ), - ); - - const data = await accountAPI.createKey({ - path: "us/test-db", - role: "admin", - ttl: "2025-01-01T00:00:00.000Z", - name: "test-key", - }); - - expect(fetch).to.have.been.calledWith( - sinon.match({ - href: "https://account.fauna.com/api/v1/databases/keys", - }), - sinon.match({ - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: "Bearer some-account-key", - }, - body: JSON.stringify({ - role: "admin", - path: "us-std/test-db", - ttl: "2025-01-01T00:00:00.000Z", - name: "test-key", - }), - }), - ); - - expect(data).to.deep.equal({ - id: "key-id", - role: "admin", - path: "us-std/test-db", - ttl: "2025-01-01T00:00:00.000Z", - name: "test-key", - }); - }); - }); -}); diff --git a/test/lib/account-api/account-api.mjs b/test/lib/account-api/account-api.mjs new file mode 100644 index 00000000..e60280ec --- /dev/null +++ b/test/lib/account-api/account-api.mjs @@ -0,0 +1,393 @@ +import * as awilix from "awilix"; +import { expect } from "chai"; +import sinon from "sinon"; + +import { setContainer } from "../../../src/config/container.mjs"; +import { setupTestContainer as setupContainer } from "../../../src/config/setup-test-container.mjs"; +import accountAPI from "../../../src/lib/account-api.mjs"; +import { f } from "../../helpers.mjs"; + +describe("accountAPI", () => { + let container, fetch; + + beforeEach(() => { + container = setupContainer(); + fetch = container.resolve("fetch"); + + container.register({ + credentials: awilix.asValue({ + accountKeys: { + key: "some-account-key", + onInvalidCreds: async () => { + container.resolve("credentials").accountKeys.key = + "new-account-key"; + return Promise.resolve(); + }, + promptLogin: sinon.stub(), + }, + }), + }); + + setContainer(container); + }); + + describe("listDatabases", () => { + const testResults = { + results: [{ name: "test-db", path: "us-std/test-db" }], + }; + it("should call the endpoint", async () => { + fetch + .withArgs( + sinon.match({ + href: "https://account.fauna.com/api/v1/databases?max_results=1000", + }), + sinon.match.any, + ) + .resolves(f(testResults, 200)); + + const data = await accountAPI.listDatabases(); + + expect(fetch).to.have.been.calledWith( + sinon.match({ + href: "https://account.fauna.com/api/v1/databases?max_results=1000", + }), + sinon.match({ + method: "GET", + headers: { + Authorization: "Bearer some-account-key", + }, + }), + ); + expect(data).to.deep.equal(testResults); + }); + + it("should call the endpoint with a path", async () => { + fetch + .withArgs( + sinon.match({ + href: "https://account.fauna.com/api/v1/databases?max_results=1000&path=us-std%2Ftest-db", + }), + ) + .resolves( + f({ + results: [{ name: "test-db", path: "us-std/test-db" }], + }), + ); + + await accountAPI.listDatabases({ path: "us-std/test-db" }); + + expect(fetch).to.have.been.calledWith( + sinon.match({ + href: "https://account.fauna.com/api/v1/databases?max_results=1000&path=us-std%2Ftest-db", + }), + sinon.match({ + method: "GET", + headers: { + Authorization: "Bearer some-account-key", + }, + }), + ); + }); + }); + + describe("createKey", () => { + const testKey = { + id: "key-id", + role: "admin", + path: "us-std/test-db", + ttl: "2025-01-01T00:00:00.000Z", + name: "test-key", + }; + + it("should call the endpoint", async () => { + fetch + .withArgs( + sinon.match({ + href: "https://account.fauna.com/api/v1/databases/keys", + }), + sinon.match.any, + ) + .resolves(f(testKey, 201)); + + const { role, path, ttl, name } = testKey; + const data = await accountAPI.createKey({ role, path, ttl, name }); + + expect(fetch).to.have.been.calledWith( + sinon.match({ + href: "https://account.fauna.com/api/v1/databases/keys", + }), + sinon.match({ + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: "Bearer some-account-key", + }, + body: JSON.stringify({ role, path, ttl, name }), + }), + ); + + expect(data).to.deep.equal(testKey); + }); + }); + + describe("createExport", () => { + const testExport = { + id: "419633606504219216", + state: "Pending", + database: "us-std/demo", + format: "simple", + destination: { + s3: { + bucket: "test-bucket", + path: "some/key", + }, + }, + created_at: "2025-01-09T19:57:22.735201Z", + }; + + it("should call the endpoint", async () => { + fetch + .withArgs( + sinon.match({ href: "https://account.fauna.com/v2/exports" }), + sinon.match({ method: "POST" }), + ) + .resolves(f(testExport, 201)); + + const data = await accountAPI.createExport({ + database: "us/demo", + format: "simple", + destination: { + s3: { + bucket: "test-bucket", + path: "some/key", + }, + }, + }); + + expect(fetch).to.have.been.calledWith( + sinon.match({ href: "https://account.fauna.com/v2/exports" }), + sinon.match({ + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: "Bearer some-account-key", + }, + body: JSON.stringify({ + database: "us-std/demo", + destination: { + s3: { bucket: "test-bucket", path: "some/key" }, + }, + format: "simple", + }), + }), + ); + expect(data).to.deep.equal(testExport); + }); + + it("should support collections", async () => { + fetch + .withArgs( + sinon.match({ href: "https://account.fauna.com/v2/exports" }), + sinon.match({ method: "POST" }), + ) + .resolves(f({ ...testExport, collections: ["test-collection"] }, 201)); + + const data = await accountAPI.createExport({ + database: "us/demo", + format: "simple", + destination: { + s3: { + bucket: "test-bucket", + path: "some/key", + }, + }, + collections: ["test-collection"], + }); + + expect(fetch).to.have.been.calledWith( + sinon.match({ href: "https://account.fauna.com/v2/exports" }), + sinon.match({ + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: "Bearer some-account-key", + }, + body: JSON.stringify({ + database: "us-std/demo", + destination: { + s3: { bucket: "test-bucket", path: "some/key" }, + }, + format: "simple", + collections: ["test-collection"], + }), + }), + ); + expect(data).to.deep.equal({ + ...testExport, + collections: ["test-collection"], + }); + }); + + it("should support tagged format", async () => { + fetch + .withArgs( + sinon.match({ href: "https://account.fauna.com/v2/exports" }), + sinon.match({ method: "POST" }), + ) + .resolves(f({ ...testExport, format: "tagged" }, 201)); + + const data = await accountAPI.createExport({ + database: "us/demo", + format: "tagged", + destination: { + s3: { + bucket: "test-bucket", + path: "some/key", + }, + }, + }); + + expect(fetch).to.have.been.calledWith( + sinon.match({ href: "https://account.fauna.com/v2/exports" }), + sinon.match({ + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: "Bearer some-account-key", + }, + body: JSON.stringify({ + database: "us-std/demo", + destination: { + s3: { bucket: "test-bucket", path: "some/key" }, + }, + format: "tagged", + }), + }), + ); + expect(data).to.deep.equal({ + ...testExport, + format: "tagged", + }); + }); + }); + + describe("listExports", () => { + const testExport = { + id: "419630463817089613", + state: "Failed", + database: "us-std/demo", + created_at: "2025-01-09T19:07:25.642703Z", + updated_at: "2025-01-09T19:07:25.642703Z", + }; + + beforeEach(() => { + fetch + .withArgs( + sinon.match({ + href: sinon.match(/exports/), + }), + ) + .resolves(f({ results: [testExport], next_token: "456" }, 200)); + }); + + it("should call the endpoint and return its data", async () => { + const data = await accountAPI.listExports(); + + expect(fetch).to.have.been.calledWith( + sinon.match({ + href: "https://account.fauna.com/v2/exports?max_results=100", + }), + sinon.match({ + method: "GET", + headers: { + Authorization: "Bearer some-account-key", + }, + }), + ); + + expect(data).to.deep.equal({ + results: [testExport], + next_token: "456", + }); + }); + + it("should support nextToken", async () => { + await accountAPI.listExports({ nextToken: "123" }); + + expect(fetch).to.have.been.calledWith( + sinon.match({ + href: "https://account.fauna.com/v2/exports?max_results=100&next_token=123", + }), + sinon.match({ + method: "GET", + headers: { + Authorization: "Bearer some-account-key", + }, + }), + ); + }); + + it("should support state", async () => { + await accountAPI.listExports({ + state: ["Pending", "Complete"], + }); + + expect(fetch).to.have.been.calledWith( + sinon.match({ + href: "https://account.fauna.com/v2/exports?max_results=100&state=Pending&state=Complete", + }), + sinon.match({ + method: "GET", + headers: { + Authorization: "Bearer some-account-key", + }, + }), + ); + }); + }); + + describe("getExport", () => { + const testExport = { + id: "419633606504219216", + state: "Pending", + database: "us-std/demo", + format: "simple", + destination: { + s3: { + bucket: "test-bucket", + path: "some/key", + }, + }, + created_at: "2025-01-09T19:57:22.735201Z", + updated_at: "2025-01-09T19:07:25.642703Z", + }; + + it("should call the endpoint", async () => { + fetch + .withArgs( + sinon.match({ + href: "https://account.fauna.com/v2/exports/419633606504219216", + }), + sinon.match({ method: "GET" }), + ) + .resolves(f(testExport, 200)); + + const data = await accountAPI.getExport({ + exportId: "419633606504219216", + }); + + expect(fetch).to.have.been.calledWith( + sinon.match({ + href: "https://account.fauna.com/v2/exports/419633606504219216", + }), + sinon.match({ + method: "GET", + headers: { + Authorization: "Bearer some-account-key", + }, + }), + ); + expect(data).to.deep.equal(testExport); + }); + }); +}); diff --git a/test/lib/account-api/fetch-with-account-key.mjs b/test/lib/account-api/fetch-with-account-key.mjs new file mode 100644 index 00000000..565f1d6b --- /dev/null +++ b/test/lib/account-api/fetch-with-account-key.mjs @@ -0,0 +1,104 @@ +import * as awilix from "awilix"; +import { expect } from "chai"; +import sinon from "sinon"; + +import { setContainer } from "../../../src/config/container.mjs"; +import { setupTestContainer as setupContainer } from "../../../src/config/setup-test-container.mjs"; +import { fetchWithAccountKey } from "../../../src/lib/account-api.mjs"; +import { f } from "../../helpers.mjs"; + +describe("fetchWithAccountKey", () => { + let container, fetch; + + beforeEach(() => { + container = setupContainer(); + fetch = container.resolve("fetch"); + + container.register({ + credentials: awilix.asValue({ + accountKeys: { + key: "some-account-key", + onInvalidCreds: async () => { + container.resolve("credentials").accountKeys.key = + "new-account-key"; + return Promise.resolve(); + }, + promptLogin: sinon.stub(), + }, + }), + }); + + setContainer(container); + }); + + it("should call the endpoint with the correct headers", async () => { + await fetchWithAccountKey("https://account.fauna.com/api/v1/databases", { + method: "GET", + }); + + expect(fetch).to.have.been.calledWith( + "https://account.fauna.com/api/v1/databases", + { + method: "GET", + headers: { + Authorization: "Bearer some-account-key", + }, + }, + ); + }); + + it("should retry once when the response is a 401", async () => { + fetch + .withArgs("https://account.fauna.com/api/v1/databases") + .onCall(0) + .resolves(f(null, 401)); + + fetch + .withArgs("https://account.fauna.com/api/v1/databases") + .onCall(1) + .resolves(f({ results: [] }, 200)); + + const response = await fetchWithAccountKey( + "https://account.fauna.com/api/v1/databases", + { + method: "GET", + }, + ); + + expect(fetch).to.have.been.calledWith( + "https://account.fauna.com/api/v1/databases", + { + method: "GET", + headers: { + Authorization: "Bearer some-account-key", + }, + }, + ); + expect(fetch).to.have.been.calledWith( + "https://account.fauna.com/api/v1/databases", + { + method: "GET", + headers: { + Authorization: "Bearer new-account-key", + }, + }, + ); + expect(await response.json()).to.deep.equal({ results: [] }); + }); + + it("should only retry authorization errors once", async () => { + fetch + .withArgs("https://account.fauna.com/api/v1/databases") + .resolves(f(null, 401)); + + const response = await fetchWithAccountKey( + "https://account.fauna.com/api/v1/databases", + { + method: "GET", + }, + ); + + expect(response.status).to.equal(401); + expect(await response.json()).to.deep.equal(null); + }); +}); diff --git a/test/lib/account-api/response-handler.mjs b/test/lib/account-api/response-handler.mjs new file mode 100644 index 00000000..e3763334 --- /dev/null +++ b/test/lib/account-api/response-handler.mjs @@ -0,0 +1,189 @@ +import { expect } from "chai"; + +import { responseHandler, toResource } from "../../../src/lib/account-api.mjs"; +import { + AuthenticationError, + AuthorizationError, + CommandError, +} from "../../../src/lib/errors.mjs"; + +describe("toResource", () => { + it("should build a URL with the correct endpoint and parameters", () => { + const url = toResource({ endpoint: "/users", params: { limit: 10 } }); + expect(url.toString()).to.equal( + "https://account.fauna.com/api/v1/users?limit=10", + ); + }); + + it("should respect v2 endpoints when specified", () => { + const url = toResource({ + endpoint: "/users", + params: { limit: 10 }, + version: "/v2", + }); + expect(url.toString()).to.equal( + "https://account.fauna.com/v2/users?limit=10", + ); + }); +}); + +describe("responseHandler", () => { + const createMockResponse = ( + status, + body = {}, + contentType = "application/json", + ) => { + return { + ok: status >= 200 && status < 300, + status, + headers: { + get: () => contentType, + }, + json: async () => body, + }; + }; + + it("should standardize v1 and v2 endpoint errors to the same CommandError", async () => { + const v1Response = createMockResponse(400, { + code: "bad_request", + reason: "Database is not specified", + }); + const v2Response = createMockResponse(400, { + error: { + code: "bad_request", + message: "Database is not specified", + }, + }); + + let v1Error, v2Error; + try { + await responseHandler(v1Response); + } catch (error) { + v1Error = error; + } + + try { + await responseHandler(v2Response); + } catch (error) { + v2Error = error; + } + + // Check that the errors are equal instances of a CommandError + expect(v1Error).to.be.instanceOf(CommandError); + expect(v2Error).to.be.instanceOf(CommandError); + expect(v1Error.message).to.equal(v2Error.message); + expect(v1Error.cause).to.deep.equal(v2Error.cause); + + // Check that the errors have the correct code and message + expect(v1Error.message).to.equal("Database is not specified"); + }); + + it("should throw AuthenticationError for 401 status", async () => { + const response = createMockResponse(401, { + code: "unauthorized", + reason: "Invalid credentials", + }); + + try { + await responseHandler(response); + } catch (error) { + expect(error).to.be.instanceOf(AuthenticationError); + } + }); + + it("should throw AuthorizationError for 403 status", async () => { + const response = createMockResponse(403, { + code: "permission_denied", + reason: "Insufficient permissions", + }); + + try { + await responseHandler(response); + } catch (error) { + expect(error).to.be.instanceOf(AuthorizationError); + } + }); + + it("should throw CommandError for 400 status", async () => { + const response = createMockResponse(400, { + code: "bad_request", + reason: "Invalid parameters", + }); + + try { + await responseHandler(response); + } catch (error) { + expect(error).to.be.instanceOf(CommandError); + } + }); + + it("should throw CommandError for 404 status", async () => { + const response = createMockResponse(404, { + code: "not_found", + reason: "Resource not found", + }); + + try { + await responseHandler(response); + } catch (error) { + expect(error).to.be.instanceOf(CommandError); + } + }); + + it("should throw generic Error for other error status codes", async () => { + const response = createMockResponse(500, { + code: "internal_error", + reason: "This is a server error", + }); + + try { + await responseHandler(response); + } catch (error) { + expect(error).to.be.instanceOf(Error); + } + }); + + it("should handle non-JSON responses", async () => { + const response = { + status: 400, + headers: { + get: () => "text/plain", + }, + }; + + try { + await responseHandler(response); + } catch (error) { + expect(error).to.be.instanceOf(CommandError); + expect(error.message).to.equal( + "An unknown error occurred while making a request to the Account API.", + ); + } + }); + + it("should preserve error details in cause", async () => { + const responseBody = { + code: "bad_request", + reason: "Invalid parameters", + }; + const response = createMockResponse(400, responseBody); + + try { + await responseHandler(response); + } catch (error) { + expect(error.cause).to.exist; + expect(error.cause.status).to.equal(400); + expect(error.cause.body).to.deep.equal(responseBody); + expect(error.cause.code).to.equal("bad_request"); + expect(error.cause.message).to.equal("Invalid parameters"); + } + }); + + it("should return parsed JSON for successful responses", async () => { + const responseBody = { data: "success" }; + const response = createMockResponse(200, responseBody); + + const result = await responseHandler(response); + expect(result).to.deep.equal(responseBody); + }); +}); From 695e56701a6bff4b454452c190cde5b82cb52f5a Mon Sep 17 00:00:00 2001 From: "E. Cooper" Date: Thu, 9 Jan 2025 15:20:22 -0800 Subject: [PATCH 03/19] Update examples for export create s3 --- src/commands/export/create.mjs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/commands/export/create.mjs b/src/commands/export/create.mjs index d655d273..e5a78afe 100644 --- a/src/commands/export/create.mjs +++ b/src/commands/export/create.mjs @@ -44,11 +44,13 @@ function buildCreateS3ExportCommand(yargs) { type: "string", required: true, description: "Name of the bucket to export to.", + group: "API:", }, path: { type: "string", required: true, description: "Key prefix to export to.", + group: "API:", }, format: { type: "string", @@ -56,6 +58,7 @@ function buildCreateS3ExportCommand(yargs) { description: "Format to export to.", choices: ["simple", "tagged"], default: "simple", + group: "API:", }, }) .check((argv) => { @@ -73,7 +76,7 @@ function buildCreateS3ExportCommand(yargs) { "Create an export of collections to the 'my-bucket' bucket with the 'my-prefix' key prefix in simple format.", ], [ - "$0 export create s3 -d us/my_db --bucket my-bucket --path my-prefix --collections my-collection", + "$0 export create s3 -d us/my_db --bucket my-bucket --path my-prefix --collection my-collection", "Create an export of the 'my-collection' collection to the 'my-bucket' bucket with the 'my-prefix' key prefix in simple format.", ], [ @@ -93,6 +96,7 @@ function buildCreateCommand(yargs) { description: "The name of the collections to export. If empty, all collections will be exported.", default: [], + group: "API:", }, }) .command({ From 9f30d94e4fe347b00fb99d4d55e477948e0bf042 Mon Sep 17 00:00:00 2001 From: "E. Cooper" Date: Tue, 14 Jan 2025 09:27:29 -0800 Subject: [PATCH 04/19] Update export for new API shapes --- src/lib/account-api.mjs | 7 ++++--- test/lib/account-api/account-api.mjs | 6 ++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/lib/account-api.mjs b/src/lib/account-api.mjs index 1fc8e1ca..9224d226 100644 --- a/src/lib/account-api.mjs +++ b/src/lib/account-api.mjs @@ -454,8 +454,8 @@ async function listExports({ maxResults = 100, nextToken, state } = {}) { const response = await fetchWithAccountKey(url, { method: "GET", }); - - return await responseHandler(response); + const data = await responseHandler(response); + return data.response; } /** @@ -472,7 +472,8 @@ async function getExport({ exportId }) { version: API_VERSIONS.v2, }); const response = await fetchWithAccountKey(url, { method: "GET" }); - return await responseHandler(response); + const data = await responseHandler(response); + return data.response; } /** diff --git a/test/lib/account-api/account-api.mjs b/test/lib/account-api/account-api.mjs index e60280ec..c1700b5f 100644 --- a/test/lib/account-api/account-api.mjs +++ b/test/lib/account-api/account-api.mjs @@ -287,7 +287,9 @@ describe("accountAPI", () => { href: sinon.match(/exports/), }), ) - .resolves(f({ results: [testExport], next_token: "456" }, 200)); + .resolves( + f({ response: { results: [testExport], next_token: "456" } }, 200), + ); }); it("should call the endpoint and return its data", async () => { @@ -370,7 +372,7 @@ describe("accountAPI", () => { }), sinon.match({ method: "GET" }), ) - .resolves(f(testExport, 200)); + .resolves(f({ response: testExport }, 200)); const data = await accountAPI.getExport({ exportId: "419633606504219216", From 423b3fb28fa92717b5b1c05db7b5326ce5d20957 Mon Sep 17 00:00:00 2001 From: "E. Cooper" Date: Tue, 14 Jan 2025 15:24:41 -0800 Subject: [PATCH 05/19] Update commands for API changes --- src/commands/export/get.mjs | 1 + src/commands/export/list.mjs | 82 ++++++++++------------------ src/lib/account-api.mjs | 29 ++++++++-- src/lib/formatting/codeToAnsi.mjs | 4 +- src/lib/formatting/colorize.mjs | 12 ++-- test/commands/export/create.mjs | 2 + test/commands/export/get.mjs | 2 + test/commands/export/list.mjs | 24 +++++--- test/lib/account-api/account-api.mjs | 32 +++++++++-- 9 files changed, 110 insertions(+), 78 deletions(-) diff --git a/src/commands/export/get.mjs b/src/commands/export/get.mjs index 602bf8e6..48e6c4fc 100644 --- a/src/commands/export/get.mjs +++ b/src/commands/export/get.mjs @@ -5,6 +5,7 @@ async function getExport(argv) { const logger = container.resolve("logger"); const { getExport } = container.resolve("accountAPI"); const { exportId, json, color } = argv; + const response = await getExport({ exportId }); if (json) { diff --git a/src/commands/export/list.mjs b/src/commands/export/list.mjs index 5283d8c2..dd65c138 100644 --- a/src/commands/export/list.mjs +++ b/src/commands/export/list.mjs @@ -1,48 +1,15 @@ import { container } from "../../config/container.mjs"; import { colorize, Format } from "../../lib/formatting/colorize.mjs"; -/* eslint-disable camelcase */ -/** - * Converts an export object to a CSV string. - * @param {{ id: string, database: string, created_at: string, updated_at: string, state: string }} export - * @returns {string} - */ -function exportToCSV({ id, database, created_at, updated_at, state }) { - return `${database},${id},${created_at},${updated_at},${state}`; -} -/* eslint-enable camelcase */ - -/** - * Outputs the exports to the console. - * @param {Object} params - The parameters for outputting the exports. - * @param {Function} params.stdout - The function to use for outputting the exports. - * @param {string[]} params.exports - The exports to output. - * @param {boolean} params.color - Whether to colorize the output. - */ -export function outputExports({ stdout, exports, color }) { - if (!exports || exports.length === 0) { - return; - } - - stdout( - colorize( - exportToCSV({ - id: "id", - database: "database", - state: "state", - /* eslint-disable camelcase */ - created_at: "created_at", - updated_at: "updated_at", - /* eslint-enable camelcase */ - }), - { color, format: Format.CSV }, - ), - ); - - exports.forEach((r) => { - stdout(colorize(exportToCSV(r), { color, format: Format.CSV })); - }); -} +const COLUMN_SEPARATOR = "\t"; +const COLLECTION_SEPARATOR = ", "; +const COLUMN_HEADERS = [ + "id", + "database", + "collections", + "destination_uri", + "state", +]; async function listExports(argv) { const logger = container.resolve("logger"); @@ -57,23 +24,30 @@ async function listExports(argv) { if (json) { logger.stdout(colorize(results, { color, format: Format.JSON })); } else { + if (!results.length) { + return; + } + logger.stdout( - colorize( - ["database", "id", "created_at", "updated_at", "state"].join(","), - { color, format: Format.CSV }, - ), + colorize(COLUMN_HEADERS.join(COLUMN_SEPARATOR), { + color, + format: Format.TSV, + }), ); results.forEach((r) => { - const { database, id, state, created_at, updated_at } = r; // eslint-disable-line camelcase + const row = [ + r.id, + r.database, + (r.collections ?? []).join(COLLECTION_SEPARATOR), + r.destination_uri, + r.state, + ]; logger.stdout( - colorize( - [database, id, created_at, updated_at, state].join(","), // eslint-disable-line camelcase - { - color, - format: Format.CSV, - }, - ), + colorize(row.join(COLUMN_SEPARATOR), { + color, + format: Format.TSV, + }), ); }); } diff --git a/src/lib/account-api.mjs b/src/lib/account-api.mjs index 9224d226..36dc0655 100644 --- a/src/lib/account-api.mjs +++ b/src/lib/account-api.mjs @@ -392,6 +392,15 @@ async function createKey({ path, role, ttl, name }) { return await responseHandler(response); } +const getExportUri = (data) => { + const { destination, state } = data; + if (!destination || !state || state.toUpperCase() !== "COMPLETE") { + return ""; + } + const path = destination.s3.path.replace(/^\/+/, ""); + return `s3://${destination.s3.bucket}/${path}`; +}; + /** * Creates an export for a given database. * @@ -425,7 +434,9 @@ async function createExport({ ...(collections && collections.length > 0 ? { collections } : {}), }), }); - return await responseHandler(response); + + const data = await responseHandler(response); + return { ...data.response, destination_uri: getExportUri(data.response) }; // eslint-disable-line camelcase } /** @@ -454,8 +465,15 @@ async function listExports({ maxResults = 100, nextToken, state } = {}) { const response = await fetchWithAccountKey(url, { method: "GET", }); - const data = await responseHandler(response); - return data.response; + const { response: data } = await responseHandler(response); + + if (data.results && Array.isArray(data.results)) { + data.results.forEach((r) => { + r.destination_uri = getExportUri(r); // eslint-disable-line camelcase + }); + } + + return data; } /** @@ -473,7 +491,10 @@ async function getExport({ exportId }) { }); const response = await fetchWithAccountKey(url, { method: "GET" }); const data = await responseHandler(response); - return data.response; + return { + ...data.response, + destination_uri: getExportUri(data.response), // eslint-disable-line camelcase + }; } /** diff --git a/src/lib/formatting/codeToAnsi.mjs b/src/lib/formatting/codeToAnsi.mjs index cdcb81b6..e42d18e3 100644 --- a/src/lib/formatting/codeToAnsi.mjs +++ b/src/lib/formatting/codeToAnsi.mjs @@ -1,9 +1,9 @@ import chalk from "chalk"; import { createHighlighterCoreSync } from "shiki/core"; import { createJavaScriptRegexEngine } from "shiki/engine/javascript"; -import csv from "shiki/langs/csv.mjs"; import json from "shiki/langs/json.mjs"; import log from "shiki/langs/log.mjs"; +import tsv from "shiki/langs/tsv.mjs"; import yaml from "shiki/langs/yaml.mjs"; import githubDarkHighContrast from "shiki/themes/github-dark-high-contrast.mjs"; @@ -15,7 +15,7 @@ const THEME = "github-dark-high-contrast"; export const createHighlighter = () => { const highlighter = createHighlighterCoreSync({ themes: [githubDarkHighContrast], - langs: [fql, log, json, yaml, csv], + langs: [fql, log, json, yaml, tsv], engine: createJavaScriptRegexEngine(), }); diff --git a/src/lib/formatting/colorize.mjs b/src/lib/formatting/colorize.mjs index 556e42ae..1324f589 100644 --- a/src/lib/formatting/colorize.mjs +++ b/src/lib/formatting/colorize.mjs @@ -10,7 +10,7 @@ export const Format = { JSON: "json", TEXT: "text", YAML: "yaml", - CSV: "csv", + TSV: "tsv", }; const objToString = (obj) => JSON.stringify(obj, null, 2); @@ -67,13 +67,13 @@ const yamlToAnsi = (obj) => { return res.trim(); }; -const csvToAnsi = (obj) => { +const tsvToAnsi = (obj) => { if (typeof obj !== "string") { - throw new Error("Unable to format CSV unless it is already a string."); + throw new Error("Unable to format TSV unless it is already a string."); } const codeToAnsi = container.resolve("codeToAnsi"); - const res = codeToAnsi(obj, "csv"); + const res = codeToAnsi(obj, "tsv"); if (!res) { return ""; @@ -99,8 +99,8 @@ export const toAnsi = (obj, { format = Format.TEXT } = {}) => { return logToAnsi(obj); case Format.YAML: return yamlToAnsi(obj); - case Format.CSV: - return csvToAnsi(obj); + case Format.TSV: + return tsvToAnsi(obj); default: return textToAnsi(obj); } diff --git a/test/commands/export/create.mjs b/test/commands/export/create.mjs index 13be14ac..c90cd38a 100644 --- a/test/commands/export/create.mjs +++ b/test/commands/export/create.mjs @@ -19,6 +19,8 @@ const createExportStub = (opts) => ({ }, }, created_at: "2025-01-02T22:59:51", + updated_at: "2025-01-02T22:59:51", + destination_uri: "", ...opts, }); diff --git a/test/commands/export/get.mjs b/test/commands/export/get.mjs index c77a7aa1..90f52689 100644 --- a/test/commands/export/get.mjs +++ b/test/commands/export/get.mjs @@ -16,6 +16,7 @@ const getExportStub = (opts) => ({ }, created_at: "2025-01-09T19:07:25.642703Z", updated_at: "2025-01-09T19:07:25.642703Z", + destination_uri: "", ...opts, }); @@ -52,6 +53,7 @@ destination: path: some/key/prefix created_at: 2025-01-09T19:07:25.642703Z updated_at: 2025-01-09T19:07:25.642703Z +destination_uri: "" failure: code: validation_error message: "failed to get bucket region: bucket not found" diff --git a/test/commands/export/list.mjs b/test/commands/export/list.mjs index d081fbed..0d0d771a 100644 --- a/test/commands/export/list.mjs +++ b/test/commands/export/list.mjs @@ -7,11 +7,19 @@ import { setupTestContainer as setupContainer } from "../../../src/config/setup- import { colorize, Format } from "../../../src/lib/formatting/colorize.mjs"; const listExportStub = (opts) => ({ - id: "test-export-id", + id: "419630463817089613", state: "Pending", - database: "us-std/example", - created_at: "2025-01-02T22:59:51", - updated_at: "2025-01-02T22:59:51", + database: "us-std/demo", + format: "simple", + destination: { + s3: { + bucket: "test-bucket", + path: "some/key/prefix", + }, + }, + created_at: "2025-01-09T19:07:25.642703Z", + updated_at: "2025-01-09T19:07:25.642703Z", + destination_uri: "", ...opts, }); @@ -26,7 +34,7 @@ describe("export list", () => { it("lists exports", async () => { const stubbedResponse = listExportStub({ - id: "tid", + id: "419630463817089613", database: "us-std/test", }); listExports.resolves({ results: [stubbedResponse] }); @@ -36,8 +44,8 @@ describe("export list", () => { expect(stdout.getWritten()).to.equal( `${[ - "database,id,created_at,updated_at,state", - "us-std/test,tid,2025-01-02T22:59:51,2025-01-02T22:59:51,Pending", + "id\tdatabase\tcollections\tdestination_uri\tstate", + "419630463817089613\tus-std/test\t\t\tPending", ].join("\n")}\n`, ); expect(listExports).to.have.been.calledWith({ @@ -54,7 +62,7 @@ describe("export list", () => { await stdout.waitForWritten(); expect(stdout.getWritten()).to.equal( - `${colorize([stubbedResponse], { format: Format.JSON })}\n`, + `${colorize([{ ...stubbedResponse }], { format: Format.JSON })}\n`, ); }); diff --git a/test/lib/account-api/account-api.mjs b/test/lib/account-api/account-api.mjs index c1700b5f..f28ba610 100644 --- a/test/lib/account-api/account-api.mjs +++ b/test/lib/account-api/account-api.mjs @@ -278,6 +278,12 @@ describe("accountAPI", () => { database: "us-std/demo", created_at: "2025-01-09T19:07:25.642703Z", updated_at: "2025-01-09T19:07:25.642703Z", + destination: { + s3: { + bucket: "test-bucket", + path: "some/key", + }, + }, }; beforeEach(() => { @@ -288,7 +294,15 @@ describe("accountAPI", () => { }), ) .resolves( - f({ response: { results: [testExport], next_token: "456" } }, 200), + f( + { + response: { + results: [testExport, { ...testExport, state: "Complete" }], + next_token: "456", + }, + }, + 200, + ), ); }); @@ -308,7 +322,14 @@ describe("accountAPI", () => { ); expect(data).to.deep.equal({ - results: [testExport], + results: [ + { ...testExport, destination_uri: "" }, + { + ...testExport, + state: "Complete", + destination_uri: "s3://test-bucket/some/key", + }, + ], next_token: "456", }); }); @@ -351,7 +372,7 @@ describe("accountAPI", () => { describe("getExport", () => { const testExport = { id: "419633606504219216", - state: "Pending", + state: "Complete", database: "us-std/demo", format: "simple", destination: { @@ -389,7 +410,10 @@ describe("accountAPI", () => { }, }), ); - expect(data).to.deep.equal(testExport); + expect(data).to.deep.equal({ + ...testExport, + destination_uri: "s3://test-bucket/some/key", + }); }); }); }); From e2df4d5ebe9ae7ba71b0d7c648e0e5d495bcd107 Mon Sep 17 00:00:00 2001 From: "E. Cooper" Date: Tue, 14 Jan 2025 15:36:16 -0800 Subject: [PATCH 06/19] Minor updates to export help text --- src/commands/export/create.mjs | 12 ++++++++---- src/commands/export/get.mjs | 23 +++++++++++++++++------ src/commands/export/list.mjs | 4 ++-- 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/src/commands/export/create.mjs b/src/commands/export/create.mjs index e5a78afe..73a53e14 100644 --- a/src/commands/export/create.mjs +++ b/src/commands/export/create.mjs @@ -73,15 +73,19 @@ function buildCreateS3ExportCommand(yargs) { .example([ [ "$0 export create s3 -d us/my_db --bucket my-bucket --path my-prefix", - "Create an export of collections to the 'my-bucket' bucket with the 'my-prefix' key prefix in simple format.", + "Output the ID of a new export for the database us-std/my_db to the 'my-bucket' bucket with the 'my-prefix' key prefix in simple format.", + ], + [ + "$0 export create s3 -d us/my_db --bucket my-bucket --path my-prefix --json", + "Output the full JSON of a new export for the database us-std/my_db to the 'my-bucket' bucket with the 'my-prefix' key prefix in simple format.", ], [ "$0 export create s3 -d us/my_db --bucket my-bucket --path my-prefix --collection my-collection", - "Create an export of the 'my-collection' collection to the 'my-bucket' bucket with the 'my-prefix' key prefix in simple format.", + "Output the ID of a new export for my-collection in us-std/my_db database to the 'my-bucket' bucket with the 'my-prefix' key prefix in simple format.", ], [ - "$0 export create s3 -d us/my_db --bucket my-bucket --path my-prefix --collection my-collection --format tagged", - "Create an export of the 'my-collection' collection to the 'my-bucket' bucket with the 'my-prefix' key prefix in tagged format.", + "$0 export create s3 -d us/my_db --bucket my-bucket --path my-prefix --format tagged", + "Output the ID of a new export in the tagged format for the database us-std/my_db to the 'my-bucket' bucket with the 'my-prefix' key prefix.", ], ]); } diff --git a/src/commands/export/get.mjs b/src/commands/export/get.mjs index 48e6c4fc..ba6a33a7 100644 --- a/src/commands/export/get.mjs +++ b/src/commands/export/get.mjs @@ -16,12 +16,23 @@ async function getExport(argv) { } function buildGetExportCommand(yargs) { - return yargs.positional("exportId", { - type: "string", - description: "The ID of the export to get.", - nargs: 1, - required: true, - }); + return yargs + .positional("exportId", { + type: "string", + description: "The ID of the export to get.", + nargs: 1, + required: true, + }) + .example([ + [ + "$0 export get 420099555438101069", + "Output the YAML for the export with an ID of 420099555438101069.", + ], + [ + "$0 export get 420099555438101069 --json", + "Output the JSON for the export with an ID of 420099555438101069.", + ], + ]); } export default { diff --git a/src/commands/export/list.mjs b/src/commands/export/list.mjs index dd65c138..3ed35e4d 100644 --- a/src/commands/export/list.mjs +++ b/src/commands/export/list.mjs @@ -72,10 +72,10 @@ function buildListExportsCommand(yargs) { }, }) .example([ - ["$0 export list", "List exports in CSV format."], + ["$0 export list", "List exports in TSV format."], ["$0 export list --max-results 100", "List a max of 100 exports."], ["$0 export list --json", "List exports in JSON format."], - ["$0 export list --states Pending", "List exports in Pending state."], + ["$0 export list --states Pending", "List exports in the Pending state."], ]); } From d6866ebab44cd81adb33a985c09b623acfa20098 Mon Sep 17 00:00:00 2001 From: James Rodewig Date: Tue, 14 Jan 2025 19:20:03 -0500 Subject: [PATCH 07/19] Edit help strings --- src/commands/export/create.mjs | 28 +++++++++++++++------------- src/commands/export/export.mjs | 6 +++--- src/commands/export/get.mjs | 11 ++++------- src/commands/export/list.mjs | 9 ++++++--- test/commands/export/export.mjs | 4 ++-- 5 files changed, 30 insertions(+), 28 deletions(-) diff --git a/src/commands/export/create.mjs b/src/commands/export/create.mjs index 73a53e14..f6ca5c82 100644 --- a/src/commands/export/create.mjs +++ b/src/commands/export/create.mjs @@ -43,19 +43,21 @@ function buildCreateS3ExportCommand(yargs) { bucket: { type: "string", required: true, - description: "Name of the bucket to export to.", + description: "Name of the S3 bucket where the export will be stored.", group: "API:", }, path: { type: "string", required: true, - description: "Key prefix to export to.", + description: + "Path prefix for the S3 bucket. Separate subfolders using a slash (`/`).", group: "API:", }, format: { type: "string", required: true, - description: "Format to export to.", + description: + "Data format used to encode the exported FQL document data as JSON.", choices: ["simple", "tagged"], default: "simple", group: "API:", @@ -72,20 +74,20 @@ function buildCreateS3ExportCommand(yargs) { }) .example([ [ - "$0 export create s3 -d us/my_db --bucket my-bucket --path my-prefix", - "Output the ID of a new export for the database us-std/my_db to the 'my-bucket' bucket with the 'my-prefix' key prefix in simple format.", + "$0 export create s3 --database us/my_db --bucket my-bucket --path exports/my_db", + "Export the 'us-std/my_db' database to the 'exports/my_db' path of the 'my-bucket' S3 bucket. Outputs the export ID.", ], [ - "$0 export create s3 -d us/my_db --bucket my-bucket --path my-prefix --json", - "Output the full JSON of a new export for the database us-std/my_db to the 'my-bucket' bucket with the 'my-prefix' key prefix in simple format.", + "$0 export create s3 --database us/my_db --bucket my-bucket --path my-prefix --json", + "Output the full JSON of the export request.", ], [ - "$0 export create s3 -d us/my_db --bucket my-bucket --path my-prefix --collection my-collection", - "Output the ID of a new export for my-collection in us-std/my_db database to the 'my-bucket' bucket with the 'my-prefix' key prefix in simple format.", + "$0 export create s3 --database us/my_db --bucket my-bucket --path my-prefix --collection my-collection", + "Export the 'my-collection' collection only.", ], [ - "$0 export create s3 -d us/my_db --bucket my-bucket --path my-prefix --format tagged", - "Output the ID of a new export in the tagged format for the database us-std/my_db to the 'my-bucket' bucket with the 'my-prefix' key prefix.", + "$0 export create s3 --database us/my_db --bucket my-bucket --path my-prefix --format tagged", + "Encode the export's document data using the 'tagged' format.", ], ]); } @@ -98,7 +100,7 @@ function buildCreateCommand(yargs) { type: "array", required: false, description: - "The name of the collections to export. If empty, all collections will be exported.", + "Used-defined collections to export. to export. Pass values as a space-separated list. If omitted, all user-defined collections are exported.", default: [], group: "API:", }, @@ -113,7 +115,7 @@ function buildCreateCommand(yargs) { export default { command: "create ", - description: "Create a database export to a given destination.", + description: "Start the export of a database or collections to an S3 bucket.", builder: buildCreateCommand, // eslint-disable-next-line no-empty-function handler: () => {}, diff --git a/src/commands/export/export.mjs b/src/commands/export/export.mjs index a3e4c7b0..da2de345 100644 --- a/src/commands/export/export.mjs +++ b/src/commands/export/export.mjs @@ -16,12 +16,12 @@ function validateAccountOnlyOptions(argv) { if (local) { throw new ValidationError( - "Exports do not support --local and the Fauna docker container.", + "Exports do not support --local or Fauna containers.", ); } if (secret) { - throw new ValidationError("Exports are not supported with --secret."); + throw new ValidationError("Exports do not support --secret."); } return true; @@ -38,7 +38,7 @@ function buildExportCommand(yargs) { export default { command: "export ", - description: "Create and manage database exports.", + description: "Create and manage exports.", builder: buildExportCommand, // eslint-disable-next-line no-empty-function handler: () => {}, diff --git a/src/commands/export/get.mjs b/src/commands/export/get.mjs index ba6a33a7..e84b25cb 100644 --- a/src/commands/export/get.mjs +++ b/src/commands/export/get.mjs @@ -19,19 +19,16 @@ function buildGetExportCommand(yargs) { return yargs .positional("exportId", { type: "string", - description: "The ID of the export to get.", + description: "ID of the export to retrieve.", nargs: 1, required: true, }) .example([ [ - "$0 export get 420099555438101069", - "Output the YAML for the export with an ID of 420099555438101069.", - ], - [ - "$0 export get 420099555438101069 --json", - "Output the JSON for the export with an ID of 420099555438101069.", + "$0 export get 123456789", + "Output the YAML for the export with an ID of '123456789'.", ], + ["$0 export get 123456789 --json", "Output the export as JSON."], ]); } diff --git a/src/commands/export/list.mjs b/src/commands/export/list.mjs index 3ed35e4d..3402fe46 100644 --- a/src/commands/export/list.mjs +++ b/src/commands/export/list.mjs @@ -73,15 +73,18 @@ function buildListExportsCommand(yargs) { }) .example([ ["$0 export list", "List exports in TSV format."], - ["$0 export list --max-results 100", "List a max of 100 exports."], ["$0 export list --json", "List exports in JSON format."], - ["$0 export list --states Pending", "List exports in the Pending state."], + ["$0 export list --max-results 50", "List up to 50 exports."], + [ + "$0 export list --states Pending Complete", + "List exports in the 'Pending' or 'Complete' state.", + ], ]); } export default { command: "list", - describe: "List database exports.", + describe: "List exports.", builder: buildListExportsCommand, handler: listExports, }; diff --git a/test/commands/export/export.mjs b/test/commands/export/export.mjs index 430f4985..9ab640d4 100644 --- a/test/commands/export/export.mjs +++ b/test/commands/export/export.mjs @@ -25,7 +25,7 @@ describe("export", () => { await stderr.waitForWritten(); expect(stderr.getWritten()).to.contain( - "Exports do not support --local and the Fauna docker container.", + "Exports do not support --local or Fauna containers.", ); }); }); @@ -42,7 +42,7 @@ describe("export", () => { await stderr.waitForWritten(); expect(stderr.getWritten()).to.contain( - "Exports are not supported with --secret.", + "Exports do not support --secret.", ); }); }); From a8474035a57e423205d20a410d957c4e29916e70 Mon Sep 17 00:00:00 2001 From: "E. Cooper" Date: Wed, 15 Jan 2025 11:09:49 -0800 Subject: [PATCH 08/19] Demand command for export and export create --- src/commands/export/create.mjs | 3 ++- src/commands/export/export.mjs | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/commands/export/create.mjs b/src/commands/export/create.mjs index f6ca5c82..1d4ac77d 100644 --- a/src/commands/export/create.mjs +++ b/src/commands/export/create.mjs @@ -110,7 +110,8 @@ function buildCreateCommand(yargs) { description: "Create a database export to an S3 bucket.", builder: buildCreateS3ExportCommand, handler: createS3Export, - }); + }) + .demandCommand(); } export default { diff --git a/src/commands/export/export.mjs b/src/commands/export/export.mjs index da2de345..efa4baf8 100644 --- a/src/commands/export/export.mjs +++ b/src/commands/export/export.mjs @@ -33,7 +33,8 @@ function buildExportCommand(yargs) { .check(validateAccountOnlyOptions) .command(createCommand) .command(listCommand) - .command(getCommand); + .command(getCommand) + .demandCommand(); } export default { From 784f0c699e4d7e08341a648e88343105d53adfe9 Mon Sep 17 00:00:00 2001 From: James Rodewig Date: Wed, 15 Jan 2025 14:40:15 -0500 Subject: [PATCH 09/19] Fix help text typo --- src/commands/export/create.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/export/create.mjs b/src/commands/export/create.mjs index 1d4ac77d..2a74d95a 100644 --- a/src/commands/export/create.mjs +++ b/src/commands/export/create.mjs @@ -100,7 +100,7 @@ function buildCreateCommand(yargs) { type: "array", required: false, description: - "Used-defined collections to export. to export. Pass values as a space-separated list. If omitted, all user-defined collections are exported.", + "Used-defined collections to export. Pass values as a space-separated list. If omitted, all user-defined collections are exported.", default: [], group: "API:", }, From 13e1e5f2024313e52f492d1b05e27799669bb84a Mon Sep 17 00:00:00 2001 From: James Rodewig Date: Wed, 15 Jan 2025 14:50:17 -0500 Subject: [PATCH 10/19] Show create examples. Rename dest positional arg. --- src/commands/export/create.mjs | 44 +++++++++++++++++----------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/src/commands/export/create.mjs b/src/commands/export/create.mjs index 2a74d95a..753b4ba0 100644 --- a/src/commands/export/create.mjs +++ b/src/commands/export/create.mjs @@ -71,25 +71,7 @@ function buildCreateS3ExportCommand(yargs) { } return true; - }) - .example([ - [ - "$0 export create s3 --database us/my_db --bucket my-bucket --path exports/my_db", - "Export the 'us-std/my_db' database to the 'exports/my_db' path of the 'my-bucket' S3 bucket. Outputs the export ID.", - ], - [ - "$0 export create s3 --database us/my_db --bucket my-bucket --path my-prefix --json", - "Output the full JSON of the export request.", - ], - [ - "$0 export create s3 --database us/my_db --bucket my-bucket --path my-prefix --collection my-collection", - "Export the 'my-collection' collection only.", - ], - [ - "$0 export create s3 --database us/my_db --bucket my-bucket --path my-prefix --format tagged", - "Encode the export's document data using the 'tagged' format.", - ], - ]); + }); } function buildCreateCommand(yargs) { @@ -107,16 +89,34 @@ function buildCreateCommand(yargs) { }) .command({ command: "s3", - description: "Create a database export to an S3 bucket.", + description: "Export to an S3 bucket.", builder: buildCreateS3ExportCommand, handler: createS3Export, }) + .example([ + [ + "$0 export create s3 --database us/my_db --bucket my-bucket --path exports/my_db", + "Export the 'us-std/my_db' database to the 'exports/my_db' path of the 'my-bucket' S3 bucket. Outputs the export ID.", + ], + [ + "$0 export create s3 --database us/my_db --bucket my-bucket --path my-prefix --json", + "Output the full JSON of the export request.", + ], + [ + "$0 export create s3 --database us/my_db --bucket my-bucket --path my-prefix --collection my-collection", + "Export the 'my-collection' collection only.", + ], + [ + "$0 export create s3 --database us/my_db --bucket my-bucket --path my-prefix --format tagged", + "Encode the export's document data using the 'tagged' format.", + ], + ]) .demandCommand(); } export default { - command: "create ", - description: "Start the export of a database or collections to an S3 bucket.", + command: "create ", + description: "Start the export of a database or collections. Outputs the export ID.", builder: buildCreateCommand, // eslint-disable-next-line no-empty-function handler: () => {}, From 5fb2732746d707380ba544600c90c720534268ed Mon Sep 17 00:00:00 2001 From: James Rodewig Date: Wed, 15 Jan 2025 14:50:33 -0500 Subject: [PATCH 11/19] Show create examples. Rename dest positional arg. --- src/commands/export/create.mjs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/commands/export/create.mjs b/src/commands/export/create.mjs index 753b4ba0..1a72f873 100644 --- a/src/commands/export/create.mjs +++ b/src/commands/export/create.mjs @@ -116,7 +116,8 @@ function buildCreateCommand(yargs) { export default { command: "create ", - description: "Start the export of a database or collections. Outputs the export ID.", + description: + "Start the export of a database or collections. Outputs the export ID.", builder: buildCreateCommand, // eslint-disable-next-line no-empty-function handler: () => {}, From df716ae47a4dbae687111f248b8b8a40424cf65c Mon Sep 17 00:00:00 2001 From: James Rodewig Date: Wed, 15 Jan 2025 15:00:26 -0500 Subject: [PATCH 12/19] Add examples to root export cmd --- src/commands/export/export.mjs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/commands/export/export.mjs b/src/commands/export/export.mjs index efa4baf8..f259d147 100644 --- a/src/commands/export/export.mjs +++ b/src/commands/export/export.mjs @@ -34,6 +34,17 @@ function buildExportCommand(yargs) { .command(createCommand) .command(listCommand) .command(getCommand) + .example([ + [ + "$0 export create s3 --database us/my_db --bucket my-bucket --path exports/my_db", + "Export the 'us-std/my_db' database to the 'exports/my_db' path of the 'my-bucket' S3 bucket. Outputs the export ID.", + ], + [ + "$0 export get 123456789", + "Output the YAML for the export with an ID of '123456789'.", + ], + ["$0 export list", "List exports in TSV format."], + ]) .demandCommand(); } From 38a1841e833063d858fe8b37cb65c4ca1cbc9c0c Mon Sep 17 00:00:00 2001 From: "E. Cooper" Date: Wed, 15 Jan 2025 15:27:28 -0800 Subject: [PATCH 13/19] Fix createExport tests for new API envelope shape --- src/lib/account-api.mjs | 1 + test/lib/account-api/account-api.mjs | 18 ++++++++++++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/lib/account-api.mjs b/src/lib/account-api.mjs index 36dc0655..636b94f0 100644 --- a/src/lib/account-api.mjs +++ b/src/lib/account-api.mjs @@ -1,4 +1,5 @@ //@ts-check +/* eslint-disable max-lines */ import { container } from "../config/container.mjs"; import { diff --git a/test/lib/account-api/account-api.mjs b/test/lib/account-api/account-api.mjs index f28ba610..9f63257e 100644 --- a/test/lib/account-api/account-api.mjs +++ b/test/lib/account-api/account-api.mjs @@ -151,7 +151,7 @@ describe("accountAPI", () => { sinon.match({ href: "https://account.fauna.com/v2/exports" }), sinon.match({ method: "POST" }), ) - .resolves(f(testExport, 201)); + .resolves(f({ response: testExport }, 201)); const data = await accountAPI.createExport({ database: "us/demo", @@ -181,7 +181,10 @@ describe("accountAPI", () => { }), }), ); - expect(data).to.deep.equal(testExport); + expect(data).to.deep.equal({ + ...testExport, + destination_uri: "", + }); }); it("should support collections", async () => { @@ -190,7 +193,12 @@ describe("accountAPI", () => { sinon.match({ href: "https://account.fauna.com/v2/exports" }), sinon.match({ method: "POST" }), ) - .resolves(f({ ...testExport, collections: ["test-collection"] }, 201)); + .resolves( + f( + { response: { ...testExport, collections: ["test-collection"] } }, + 201, + ), + ); const data = await accountAPI.createExport({ database: "us/demo", @@ -225,6 +233,7 @@ describe("accountAPI", () => { expect(data).to.deep.equal({ ...testExport, collections: ["test-collection"], + destination_uri: "", }); }); @@ -234,7 +243,7 @@ describe("accountAPI", () => { sinon.match({ href: "https://account.fauna.com/v2/exports" }), sinon.match({ method: "POST" }), ) - .resolves(f({ ...testExport, format: "tagged" }, 201)); + .resolves(f({ response: { ...testExport, format: "tagged" } }, 201)); const data = await accountAPI.createExport({ database: "us/demo", @@ -267,6 +276,7 @@ describe("accountAPI", () => { expect(data).to.deep.equal({ ...testExport, format: "tagged", + destination_uri: "", }); }); }); From fe1a6b471b3042a8de698fc9c52c94562a48026b Mon Sep 17 00:00:00 2001 From: "E. Cooper" Date: Wed, 15 Jan 2025 15:28:33 -0800 Subject: [PATCH 14/19] Add beta message for export commands --- src/commands/export/export.mjs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/commands/export/export.mjs b/src/commands/export/export.mjs index f259d147..1bb3bbba 100644 --- a/src/commands/export/export.mjs +++ b/src/commands/export/export.mjs @@ -1,3 +1,6 @@ +import chalk from "chalk"; + +import { container } from "../../config/container.mjs"; import { ValidationError } from "../../lib/errors.mjs"; import { ACCOUNT_OPTIONS } from "../../lib/options.mjs"; import createCommand from "./create.mjs"; @@ -30,6 +33,14 @@ function validateAccountOnlyOptions(argv) { function buildExportCommand(yargs) { return yargs .options(ACCOUNT_OPTIONS) + .middleware(() => { + const logger = container.resolve("logger"); + logger.stderr( + chalk.yellow( + `Warning: fauna export is currently in beta. To learn more, visit https://docs.fauna.com/fauna/current/build/cli/v4/commands/export/`, + ), + ); + }) .check(validateAccountOnlyOptions) .command(createCommand) .command(listCommand) @@ -52,6 +63,4 @@ export default { command: "export ", description: "Create and manage exports.", builder: buildExportCommand, - // eslint-disable-next-line no-empty-function - handler: () => {}, }; From 767ab99d30d4ac15985ec0bf56b8dd3292bb47ba Mon Sep 17 00:00:00 2001 From: James Rodewig Date: Wed, 15 Jan 2025 19:01:18 -0500 Subject: [PATCH 15/19] Share `fauna create` and `fauna create s3` examples --- src/commands/export/create.mjs | 39 ++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/src/commands/export/create.mjs b/src/commands/export/create.mjs index 1a72f873..695c0ea1 100644 --- a/src/commands/export/create.mjs +++ b/src/commands/export/create.mjs @@ -37,6 +37,25 @@ async function createS3Export(argv) { } } +const sharedExamples = [ + [ + "$0 export create s3 --database us/my_db --bucket my-bucket --path exports/my_db", + "Export the 'us-std/my_db' database to the 'exports/my_db' path of the 'my-bucket' S3 bucket. Outputs the export ID.", + ], + [ + "$0 export create s3 --database us/my_db --bucket my-bucket --path my-prefix --json", + "Output the full JSON of the export request.", + ], + [ + "$0 export create s3 --database us/my_db --bucket my-bucket --path my-prefix --collection my-collection", + "Export the 'my-collection' collection only.", + ], + [ + "$0 export create s3 --database us/my_db --bucket my-bucket --path my-prefix --format tagged", + "Encode the export's document data using the 'tagged' format.", + ], +]; + function buildCreateS3ExportCommand(yargs) { return yargs .options({ @@ -63,6 +82,7 @@ function buildCreateS3ExportCommand(yargs) { group: "API:", }, }) + .example(sharedExamples) .check((argv) => { if (!argv.database) { throw new ValidationError( @@ -93,24 +113,7 @@ function buildCreateCommand(yargs) { builder: buildCreateS3ExportCommand, handler: createS3Export, }) - .example([ - [ - "$0 export create s3 --database us/my_db --bucket my-bucket --path exports/my_db", - "Export the 'us-std/my_db' database to the 'exports/my_db' path of the 'my-bucket' S3 bucket. Outputs the export ID.", - ], - [ - "$0 export create s3 --database us/my_db --bucket my-bucket --path my-prefix --json", - "Output the full JSON of the export request.", - ], - [ - "$0 export create s3 --database us/my_db --bucket my-bucket --path my-prefix --collection my-collection", - "Export the 'my-collection' collection only.", - ], - [ - "$0 export create s3 --database us/my_db --bucket my-bucket --path my-prefix --format tagged", - "Encode the export's document data using the 'tagged' format.", - ], - ]) + .example(sharedExamples) .demandCommand(); } From 4bd0bbdddfff132e23a25867dad40b2cc6132b8e Mon Sep 17 00:00:00 2001 From: "E. Cooper" Date: Thu, 16 Jan 2025 11:50:48 -0800 Subject: [PATCH 16/19] Address feedback and add --wait flag for create and get export commands --- src/commands/export/create.mjs | 30 ++-- src/commands/export/export.mjs | 4 +- src/commands/export/get.mjs | 20 ++- src/commands/export/list.mjs | 29 ++-- src/commands/export/wait.mjs | 199 +++++++++++++++++++++++++++ src/config/setup-container.mjs | 2 + src/config/setup-test-container.mjs | 1 + src/lib/account-api.mjs | 15 +- src/lib/utils.mjs | 10 ++ test/commands/export/create.mjs | 13 +- test/commands/export/list.mjs | 9 +- test/commands/export/wait.mjs | 155 +++++++++++++++++++++ test/lib/account-api/account-api.mjs | 8 +- 13 files changed, 451 insertions(+), 44 deletions(-) create mode 100644 src/commands/export/wait.mjs create mode 100644 test/commands/export/wait.mjs diff --git a/src/commands/export/create.mjs b/src/commands/export/create.mjs index 695c0ea1..846b43d9 100644 --- a/src/commands/export/create.mjs +++ b/src/commands/export/create.mjs @@ -1,12 +1,13 @@ // @ts-check import { container } from "../../config/container.mjs"; +import { EXPORT_TERMINAL_STATES } from "../../lib/account-api.mjs"; import { ValidationError } from "../../lib/errors.mjs"; import { colorize, Format } from "../../lib/formatting/colorize.mjs"; import { DATABASE_PATH_OPTIONS } from "../../lib/options.mjs"; +import { WAIT_OPTIONS, waitUntilExportIsReady } from "./wait.mjs"; async function createS3Export(argv) { - const logger = container.resolve("logger"); const { database, path, @@ -15,10 +16,14 @@ async function createS3Export(argv) { json, color, collection: collections, + wait, + maxWait, + quiet, } = argv; + const logger = container.resolve("logger"); const { createExport } = container.resolve("accountAPI"); - const response = await createExport({ + let createdExport = await createExport({ database, collections, destination: { @@ -30,10 +35,20 @@ async function createS3Export(argv) { format, }); + if (wait && !EXPORT_TERMINAL_STATES.includes(createdExport.state)) { + createdExport = await waitUntilExportIsReady({ + id: createdExport.id, + opts: { + maxWait, + quiet, + }, + }); + } + if (json) { - logger.stdout(colorize(response, { color, format: Format.JSON })); + logger.stdout(colorize(createdExport, { color, format: Format.JSON })); } else { - logger.stdout(response.id); + logger.stdout(colorize(createdExport, { color, format: Format.YAML })); } } @@ -82,7 +97,7 @@ function buildCreateS3ExportCommand(yargs) { group: "API:", }, }) - .example(sharedExamples) + .options(WAIT_OPTIONS) .check((argv) => { if (!argv.database) { throw new ValidationError( @@ -91,7 +106,8 @@ function buildCreateS3ExportCommand(yargs) { } return true; - }); + }) + .example(sharedExamples); } function buildCreateCommand(yargs) { @@ -122,6 +138,4 @@ export default { description: "Start the export of a database or collections. Outputs the export ID.", builder: buildCreateCommand, - // eslint-disable-next-line no-empty-function - handler: () => {}, }; diff --git a/src/commands/export/export.mjs b/src/commands/export/export.mjs index 1bb3bbba..b9520a20 100644 --- a/src/commands/export/export.mjs +++ b/src/commands/export/export.mjs @@ -37,7 +37,7 @@ function buildExportCommand(yargs) { const logger = container.resolve("logger"); logger.stderr( chalk.yellow( - `Warning: fauna export is currently in beta. To learn more, visit https://docs.fauna.com/fauna/current/build/cli/v4/commands/export/`, + `Warning: fauna export is currently in beta. To learn more, visit https://docs.fauna.com/fauna/current/build/cli/v4/commands/export/\n`, ), ); }) @@ -61,6 +61,6 @@ function buildExportCommand(yargs) { export default { command: "export ", - description: "Create and manage exports.", + description: "Create and manage exports. Currently in beta.", builder: buildExportCommand, }; diff --git a/src/commands/export/get.mjs b/src/commands/export/get.mjs index e84b25cb..eb93e2a7 100644 --- a/src/commands/export/get.mjs +++ b/src/commands/export/get.mjs @@ -1,12 +1,23 @@ import { container } from "../../config/container.mjs"; +import { EXPORT_TERMINAL_STATES } from "../../lib/account-api.mjs"; import { colorize, Format } from "../../lib/formatting/colorize.mjs"; +import { WAIT_OPTIONS, waitUntilExportIsReady } from "./wait.mjs"; async function getExport(argv) { const logger = container.resolve("logger"); const { getExport } = container.resolve("accountAPI"); - const { exportId, json, color } = argv; + const { exportId, json, color, wait, maxWait, quiet } = argv; - const response = await getExport({ exportId }); + let response = await getExport({ exportId }); + if (wait && !EXPORT_TERMINAL_STATES.includes(response.state)) { + response = await waitUntilExportIsReady({ + id: exportId, + opts: { + maxWait, + quiet, + }, + }); + } if (json) { logger.stdout(colorize(response, { color, format: Format.JSON })); @@ -23,12 +34,17 @@ function buildGetExportCommand(yargs) { nargs: 1, required: true, }) + .options(WAIT_OPTIONS) .example([ [ "$0 export get 123456789", "Output the YAML for the export with an ID of '123456789'.", ], ["$0 export get 123456789 --json", "Output the export as JSON."], + [ + "$0 export get 123456789 --wait", + "Wait for the export to be in a terminal state before exiting.", + ], ]); } diff --git a/src/commands/export/list.mjs b/src/commands/export/list.mjs index 3402fe46..42a78009 100644 --- a/src/commands/export/list.mjs +++ b/src/commands/export/list.mjs @@ -1,15 +1,8 @@ import { container } from "../../config/container.mjs"; import { colorize, Format } from "../../lib/formatting/colorize.mjs"; - +import { EXPORT_STATES } from "../../lib/account-api.mjs"; const COLUMN_SEPARATOR = "\t"; -const COLLECTION_SEPARATOR = ", "; -const COLUMN_HEADERS = [ - "id", - "database", - "collections", - "destination_uri", - "state", -]; +const COLLECTION_SEPARATOR = ","; async function listExports(argv) { const logger = container.resolve("logger"); @@ -28,13 +21,6 @@ async function listExports(argv) { return; } - logger.stdout( - colorize(COLUMN_HEADERS.join(COLUMN_SEPARATOR), { - color, - format: Format.TSV, - }), - ); - results.forEach((r) => { const row = [ r.id, @@ -59,8 +45,8 @@ function buildListExportsCommand(yargs) { "max-results": { alias: "max", type: "number", - description: "Maximum number of exports to return. Defaults to 100.", - default: 100, + description: "Maximum number of exports to return. Defaults to 10.", + default: 10, group: "API:", }, state: { @@ -68,11 +54,14 @@ function buildListExportsCommand(yargs) { description: "Filter exports by state.", default: [], group: "API:", - choices: ["Pending", "InProgress", "Complete", "Failed"], + choices: EXPORT_STATES, }, }) .example([ - ["$0 export list", "List exports in TSV format."], + [ + "$0 export list", + "List exports in TSV format with export ID, database, collections, destination, and state as the columns.", + ], ["$0 export list --json", "List exports in JSON format."], ["$0 export list --max-results 50", "List up to 50 exports."], [ diff --git a/src/commands/export/wait.mjs b/src/commands/export/wait.mjs new file mode 100644 index 00000000..1966db9c --- /dev/null +++ b/src/commands/export/wait.mjs @@ -0,0 +1,199 @@ +// @ts-check + +import { container } from "../../config/container.mjs"; +import { EXPORT_TERMINAL_STATES } from "../../lib/account-api.mjs"; +import { CommandError } from "../../lib/errors.mjs"; +import { colorize, Format } from "../../lib/formatting/colorize.mjs"; +import { isTTY } from "../../lib/utils.mjs"; + +const INITIAL_INTERVAL_MS = 1000; // 1 second +const MAX_INTERVAL_MS = 1000 * 60 * 5; // 5 minutes +const MAX_WAIT_MINS = 60 * 2; // 2 hours + +const WAITING_FRAMES = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]; +export const WAIT_OPTIONS = { + wait: { + type: "boolean", + required: false, + description: "Wait for the export to complete before exiting.", + }, + maxWait: { + type: "number", + required: false, + description: "The maximum wait time in minutes. Maximum is 0 minutes.", + default: MAX_WAIT_MINS, + }, +}; + +let currentInterval = null; + +function defaultStatusHandler(message, isDone = false) { + const stream = container.resolve("stderrStream"); + + // If there's an interval, always just clear it. + if (currentInterval) { + clearInterval(currentInterval); + } + + if ( + !isTTY() || + typeof stream.clearLine !== "function" || + typeof stream.cursorTo !== "function" + ) { + const logger = container.resolve("logger"); + logger.stderr(`[Export] ${message}`); + + return; + } + + if (isDone) { + const stream = container.resolve("stderrStream"); + + // Clear the line and move the cursor to the beginning + stream.clearLine(0); + stream.cursorTo(0); + + // Write the message with the current frame + stream.write(`Done. ${message}\n\n`); + + return; + } + + const frames = [...WAITING_FRAMES]; + let currentFrame = frames[0]; + currentInterval = setInterval(() => { + const stream = container.resolve("stderrStream"); + + // Clear the line and move the cursor to the beginning + stream.clearLine(0); + stream.cursorTo(0); + + // Write the message with the current frame + stream.write(`${currentFrame} ${message}`); + + // Rotate the frames + frames.push(currentFrame); + currentFrame = frames.shift() ?? frames[0]; + }, 80); +} + +/** + * Waits for an export to complete and returns the export data. + * + * @param {object} params + * @param {string} params.id + * @param {object} [params.opts] + * @param {number} [params.opts.maxWait] + * @param {boolean} [params.opts.color] + * @param {string} [params.opts.quiet] + * @param {function} [params.opts.statusHandler] + * @returns {Promise} The export data + * @throws {CommandError} If the export did not complete within the allotted time + */ +export async function waitUntilExportIsReady({ id, opts = {} }) { + const { + maxWait = MAX_WAIT_MINS, + color = true, + statusHandler = defaultStatusHandler, + quiet = false, + } = opts; + + // eslint-disable-next-line no-empty-function + const sendStatus = quiet ? () => {} : statusHandler; + + sendStatus( + colorize(`${id} is Pending and not yet started.`, { + format: Format.LOG, + color, + }), + ); + + const waitTimeMs = Math.min(maxWait, MAX_WAIT_MINS) * 60 * 1000; + const exitAt = Date.now() + waitTimeMs; + const terminalExport = await waitAndCheckExportState({ + id, + exitAt, + color, + statusHandler: sendStatus, + }); + + return terminalExport; +} + +/** + * @param {object} params + * @param {string} params.id + * @param {number} params.exitAt + * @param {number} [params.interval] + * @param {boolean} [params.color] + * @param {function} [params.statusHandler] + * @returns {Promise} The export data + * @throws {CommandError} If the export did not complete within the allotted time + */ +export async function waitAndCheckExportState({ + id, + exitAt, + interval = INITIAL_INTERVAL_MS, + color, + statusHandler = defaultStatusHandler, +}) { + const sleep = container.resolve("sleep"); + const { getExport } = container.resolve("accountAPI"); + + // If the export did not complete within the allotted time, throw an error + if (Date.now() >= exitAt) { + statusHandler( + colorize(`${id} did not complete within the allotted time...exiting`, { + format: Format.LOG, + color, + }), + true, + ); + throw new CommandError( + colorize( + `${id} did not complete within the allotted time. To continue to check export status use: fauna export get ${id}`, + { + format: Format.LOG, + color, + }, + ), + ); + } + + // Sleep and then check the export state. Never wait longer than MAX_INTERVAL. + await sleep(Math.min(interval, MAX_INTERVAL_MS)); + + // Fetch the export data + const data = await getExport({ exportId: id }); + + // If the export is ready, return the data + if (EXPORT_TERMINAL_STATES.includes(data.state)) { + statusHandler( + colorize(`${id} has a terminal state of ${data.state}.`, { + format: Format.LOG, + color, + }), + true, + ); + return data; + } + + const nextInterval = Math.min(interval * 2, MAX_INTERVAL_MS); + statusHandler( + colorize( + `${id} is ${data.state} and not ready. Waiting for ${nextInterval}ms before checking again.`, + { + format: Format.LOG, + color, + }, + ), + ); + + // If the export is not ready, sleep and then check again + return waitAndCheckExportState({ + id, + exitAt, + interval: Math.min(interval * 2, MAX_INTERVAL_MS), + statusHandler, + }); +} diff --git a/src/config/setup-container.mjs b/src/config/setup-container.mjs index ac518145..1f067a69 100644 --- a/src/config/setup-container.mjs +++ b/src/config/setup-container.mjs @@ -37,6 +37,7 @@ import { getAllSchemaFileContents, writeSchemaFiles, } from "../lib/schema.mjs"; +import { sleep } from "../lib/utils.mjs"; export function setupCommonContainer() { const container = awilix.createContainer({ @@ -91,6 +92,7 @@ export const injectables = { accountAPI: awilix.asValue(accountAPI), makeFaunaRequest: awilix.asValue(makeRetryableFaunaRequest), errorHandler: awilix.asValue((_error, exitCode) => exit(exitCode)), + sleep: awilix.asValue(sleep), // While we inject the class instance before this in middleware, // we need to register it here to resolve types. diff --git a/src/config/setup-test-container.mjs b/src/config/setup-test-container.mjs index debc5b69..f0528594 100644 --- a/src/config/setup-test-container.mjs +++ b/src/config/setup-test-container.mjs @@ -126,6 +126,7 @@ export function setupTestContainer() { formatQueryResponse: faunaClientV4.formatQueryResponse, formatError: faunaClientV4.formatError, }), + sleep: awilix.asValue(stub().resolves()), }; confirmManualMocks(manualMocks, thingsToManuallyMock); diff --git a/src/lib/account-api.mjs b/src/lib/account-api.mjs index 636b94f0..be103489 100644 --- a/src/lib/account-api.mjs +++ b/src/lib/account-api.mjs @@ -14,6 +14,19 @@ const API_VERSIONS = { v2: "/v2", }; +export const ExportState = { + Pending: "Pending", + InProgress: "InProgress", + Complete: "Complete", + Failed: "Failed", +}; + +export const EXPORT_STATES = Object.values(ExportState); +export const EXPORT_TERMINAL_STATES = [ + ExportState.Complete, + ExportState.Failed, +]; + let accountUrl = process.env.FAUNA_ACCOUNT_URL ?? "https://account.fauna.com"; /** @@ -395,7 +408,7 @@ async function createKey({ path, role, ttl, name }) { const getExportUri = (data) => { const { destination, state } = data; - if (!destination || !state || state.toUpperCase() !== "COMPLETE") { + if (!destination || !state) { return ""; } const path = destination.s3.path.replace(/^\/+/, ""); diff --git a/src/lib/utils.mjs b/src/lib/utils.mjs index 95afeb4f..47cf04b5 100644 --- a/src/lib/utils.mjs +++ b/src/lib/utils.mjs @@ -46,3 +46,13 @@ export function standardizeRegion(databasePath) { const standardRegion = regionMap[region] || region; return rest ? `${standardRegion}/${rest}` : standardRegion; } + +/** + * @param {number} ms - The number of milliseconds to sleep. + * @returns {Promise} A promise that resolves after the specified number of milliseconds. + */ +export async function sleep(ms) { + return await new Promise((resolve) => { + setTimeout(resolve, ms); + }); +} diff --git a/test/commands/export/create.mjs b/test/commands/export/create.mjs index c90cd38a..4282b532 100644 --- a/test/commands/export/create.mjs +++ b/test/commands/export/create.mjs @@ -56,7 +56,18 @@ describe("export create s3", () => { ); await stdout.waitForWritten(); - expect(stdout.getWritten()).to.equal("test-export-id\n"); + expect(stdout.getWritten()).to.equal(`id: test-export-id +state: Pending +database: us-std/example +format: simple +destination: + s3: + path: /test/key + bucket: test-bucket +created_at: 2025-01-02T22:59:51 +updated_at: 2025-01-02T22:59:51 +destination_uri: "" +`); expect(createExport).to.have.been.calledWith({ database, collections: [], diff --git a/test/commands/export/list.mjs b/test/commands/export/list.mjs index 0d0d771a..25b9a884 100644 --- a/test/commands/export/list.mjs +++ b/test/commands/export/list.mjs @@ -43,13 +43,10 @@ describe("export list", () => { await stdout.waitForWritten(); expect(stdout.getWritten()).to.equal( - `${[ - "id\tdatabase\tcollections\tdestination_uri\tstate", - "419630463817089613\tus-std/test\t\t\tPending", - ].join("\n")}\n`, + `${["419630463817089613\tus-std/test\t\t\tPending"].join("\n")}\n`, ); expect(listExports).to.have.been.calledWith({ - maxResults: 100, + maxResults: 10, state: [], }); }); @@ -83,7 +80,7 @@ describe("export list", () => { await run(`export list --state Pending --state InProgress`, container); expect(listExports).to.have.been.calledWith({ - maxResults: 100, + maxResults: 10, state: ["Pending", "InProgress"], }); }); diff --git a/test/commands/export/wait.mjs b/test/commands/export/wait.mjs new file mode 100644 index 00000000..4dbd0663 --- /dev/null +++ b/test/commands/export/wait.mjs @@ -0,0 +1,155 @@ +import { expect } from "chai"; +import sinon from "sinon"; + +import { + waitAndCheckExportState, + waitUntilExportIsReady, +} from "../../../src/commands/export/wait.mjs"; +import { setContainer } from "../../../src/config/container.mjs"; +import { setupTestContainer as setupContainer } from "../../../src/config/setup-test-container.mjs"; +import { ExportState } from "../../../src/lib/account-api.mjs"; +import { CommandError } from "../../../src/lib/errors.mjs"; + +describe("export wait helpers", () => { + let container, getExport, sleep; + + beforeEach(() => { + container = setupContainer(); + sleep = container.resolve("sleep"); + ({ getExport } = container.resolve("accountAPI")); + + setContainer(container); + }); + + describe("waitUntilExportIsReady", () => { + it("should return export data when export completes successfully", async () => { + const exportId = "test-export-id"; + const exportData = { id: exportId, state: ExportState.Complete }; + const statusHandler = sinon.stub(); + + getExport.resolves(exportData); + + const result = await waitUntilExportIsReady({ + id: exportId, + opts: { quiet: false, color: false, statusHandler }, + }); + + expect(getExport).to.have.been.calledWith({ exportId }); + expect(result).to.deep.equal(exportData); + expect(sleep.calledOnce).to.be.true; + expect(statusHandler).to.have.been.calledWith( + `test-export-id is Pending and not yet started.`, + ); + expect(statusHandler).to.have.been.calledWith( + "test-export-id is Pending and not yet started.", + ); + expect(statusHandler).to.have.been.calledWith( + "test-export-id has a terminal state of Complete.", + ); + }); + + it("should not print status when quiet is true", async () => { + const exportId = "test-export-id"; + const exportData = { id: exportId, state: ExportState.Complete }; + const statusHandler = sinon.stub(); + + getExport.resolves(exportData); + + const result = await waitUntilExportIsReady({ + id: exportId, + opts: { quiet: true, color: false, statusHandler }, + }); + + expect(getExport).to.have.been.calledWith({ exportId }); + expect(result).to.deep.equal(exportData); + expect(sleep.calledOnce).to.be.true; + expect(statusHandler).to.have.not.been.called; + }); + + it("should respect maxWait parameter", async () => { + const exportId = "test-export-id"; + + // Force timeout by setting maxWait to 0 + try { + await waitUntilExportIsReady({ + id: exportId, + opts: { maxWait: 0, quiet: true }, + }); + expect.fail("Should have thrown an error"); + } catch (error) { + expect(error).to.be.instanceOf(CommandError); + expect(error.message).to.include( + "did not complete within the allotted time", + ); + } + }); + }); + + describe("waitAndCheckExportState", () => { + it("should retry until export reaches terminal state", async () => { + const exportId = "test-export-id"; + const exitAt = Date.now() + 5000; + + getExport + .onFirstCall() + .resolves({ id: exportId, state: ExportState.Pending }) + .onSecondCall() + .resolves({ id: exportId, state: ExportState.Complete }); + + const result = await waitAndCheckExportState({ + id: exportId, + exitAt, + color: false, + statusHandler: () => {}, + }); + + expect(result.state).to.equal(ExportState.Complete); + expect(getExport.calledTwice).to.be.true; + expect(sleep.calledTwice).to.be.true; + }); + + it("should throw error when timeout is reached", async () => { + const exportId = "test-export-id"; + const exitAt = Date.now() - 1000; // Already expired + + try { + await waitAndCheckExportState({ + id: exportId, + exitAt, + color: false, + statusHandler: () => {}, + }); + expect.fail("Should have thrown an error"); + } catch (error) { + expect(error).to.be.instanceOf(CommandError); + expect(error.message).to.include( + "did not complete within the allotted time", + ); + } + }); + + it("should respect interval backoff with maximum limit", async () => { + const exportId = "test-export-id"; + const exitAt = Date.now() + 10000; + + getExport + .onFirstCall() + .resolves({ id: exportId, state: ExportState.Pending }) + .onSecondCall() + .resolves({ id: exportId, state: ExportState.Pending }) + .onThirdCall() + .resolves({ id: exportId, state: ExportState.Complete }); + + await waitAndCheckExportState({ + id: exportId, + exitAt, + interval: 1000, + color: false, + statusHandler: () => {}, + }); + + expect(sleep.firstCall.args[0]).to.equal(1000); + expect(sleep.secondCall.args[0]).to.equal(2000); + }); + }); +}); diff --git a/test/lib/account-api/account-api.mjs b/test/lib/account-api/account-api.mjs index 9f63257e..228a4f4e 100644 --- a/test/lib/account-api/account-api.mjs +++ b/test/lib/account-api/account-api.mjs @@ -183,7 +183,7 @@ describe("accountAPI", () => { ); expect(data).to.deep.equal({ ...testExport, - destination_uri: "", + destination_uri: "s3://test-bucket/some/key", }); }); @@ -233,7 +233,7 @@ describe("accountAPI", () => { expect(data).to.deep.equal({ ...testExport, collections: ["test-collection"], - destination_uri: "", + destination_uri: "s3://test-bucket/some/key", }); }); @@ -276,7 +276,7 @@ describe("accountAPI", () => { expect(data).to.deep.equal({ ...testExport, format: "tagged", - destination_uri: "", + destination_uri: "s3://test-bucket/some/key", }); }); }); @@ -333,7 +333,7 @@ describe("accountAPI", () => { expect(data).to.deep.equal({ results: [ - { ...testExport, destination_uri: "" }, + { ...testExport, destination_uri: "s3://test-bucket/some/key" }, { ...testExport, state: "Complete", From d3176b114715e3f3b360a22df9d584b45ed9ad47 Mon Sep 17 00:00:00 2001 From: "E. Cooper" Date: Thu, 16 Jan 2025 11:50:48 -0800 Subject: [PATCH 17/19] Address feedback and add --wait flag for create and get export commands --- src/commands/export/list.mjs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/commands/export/list.mjs b/src/commands/export/list.mjs index 42a78009..276ffda5 100644 --- a/src/commands/export/list.mjs +++ b/src/commands/export/list.mjs @@ -1,6 +1,7 @@ import { container } from "../../config/container.mjs"; -import { colorize, Format } from "../../lib/formatting/colorize.mjs"; import { EXPORT_STATES } from "../../lib/account-api.mjs"; +import { colorize, Format } from "../../lib/formatting/colorize.mjs"; + const COLUMN_SEPARATOR = "\t"; const COLLECTION_SEPARATOR = ","; From d725810920d8508335ac56058a632bed074f802c Mon Sep 17 00:00:00 2001 From: "E. Cooper" Date: Thu, 16 Jan 2025 12:09:51 -0800 Subject: [PATCH 18/19] Output next interval as seconds --- src/commands/export/wait.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/export/wait.mjs b/src/commands/export/wait.mjs index 1966db9c..6a80b6e3 100644 --- a/src/commands/export/wait.mjs +++ b/src/commands/export/wait.mjs @@ -181,7 +181,7 @@ export async function waitAndCheckExportState({ const nextInterval = Math.min(interval * 2, MAX_INTERVAL_MS); statusHandler( colorize( - `${id} is ${data.state} and not ready. Waiting for ${nextInterval}ms before checking again.`, + `${id} is ${data.state} and not ready. Waiting for ${nextInterval / 1000}s before checking again.`, { format: Format.LOG, color, From 19a4f3f6d5e50e05c8a65f480566ec8136b6c34f Mon Sep 17 00:00:00 2001 From: James Rodewig Date: Thu, 16 Jan 2025 15:22:38 -0500 Subject: [PATCH 19/19] Edit help strings and examples --- src/commands/export/create.mjs | 4 ++++ src/commands/export/get.mjs | 2 +- src/commands/export/wait.mjs | 5 +++-- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/commands/export/create.mjs b/src/commands/export/create.mjs index 846b43d9..2fa8cb9f 100644 --- a/src/commands/export/create.mjs +++ b/src/commands/export/create.mjs @@ -69,6 +69,10 @@ const sharedExamples = [ "$0 export create s3 --database us/my_db --bucket my-bucket --path my-prefix --format tagged", "Encode the export's document data using the 'tagged' format.", ], + [ + "$0 export create s3 --database us/my_db --bucket my-bucket --path my-prefix --wait --max-wait 180", + "Wait for the export to complete or fail before exiting. Waits up to 180 minutes.", + ], ]; function buildCreateS3ExportCommand(yargs) { diff --git a/src/commands/export/get.mjs b/src/commands/export/get.mjs index eb93e2a7..43e00fbe 100644 --- a/src/commands/export/get.mjs +++ b/src/commands/export/get.mjs @@ -43,7 +43,7 @@ function buildGetExportCommand(yargs) { ["$0 export get 123456789 --json", "Output the export as JSON."], [ "$0 export get 123456789 --wait", - "Wait for the export to be in a terminal state before exiting.", + "Wait for the export to complete or fail before exiting.", ], ]); } diff --git a/src/commands/export/wait.mjs b/src/commands/export/wait.mjs index 6a80b6e3..28fe41a1 100644 --- a/src/commands/export/wait.mjs +++ b/src/commands/export/wait.mjs @@ -15,12 +15,13 @@ export const WAIT_OPTIONS = { wait: { type: "boolean", required: false, - description: "Wait for the export to complete before exiting.", + description: + "Wait for the export to complete or fail before exiting. Use '--max-wait' to set a timeout.", }, maxWait: { type: "number", required: false, - description: "The maximum wait time in minutes. Maximum is 0 minutes.", + description: "Maximum wait time in minutes. Defaults to 120 minutes.", default: MAX_WAIT_MINS, }, };