From 1ccf38ed9b604cc00a2beececf2ff11cc549ec20 Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Tue, 19 Nov 2024 13:26:20 -0500 Subject: [PATCH 01/33] temp --- .github/workflows/encryption-tests.yml | 74 ++++++++++++++++++++++++++ test/encryption/encryption.test.js | 11 ++++ 2 files changed, 85 insertions(+) create mode 100644 .github/workflows/encryption-tests.yml create mode 100644 test/encryption/encryption.test.js diff --git a/.github/workflows/encryption-tests.yml b/.github/workflows/encryption-tests.yml new file mode 100644 index 0000000000..87ffa37fcc --- /dev/null +++ b/.github/workflows/encryption-tests.yml @@ -0,0 +1,74 @@ +on: + push: + branches: ['main'] + workflow_dispatch: {} + +permissions: + contents: write + pull-requests: write + id-token: write + +name: encryption-tests + +jobs: + ssdlc: + permissions: + # required for all workflows + security-events: write + id-token: write + contents: write + environment: release + runs-on: ubuntu-latest + steps: + - name: Install mongodb-client-encryption + uses: mongodb-client-encryption + + - name: Set-up cluster + + + + - name: Run tests + uses: mongodb-labs/drivers-github-tools/node/get_version_info@v2 + with: + npm_package_name: mongodb + + - name: actions/compress_sign_and_upload + uses: mongodb-labs/drivers-github-tools/node/sign_node_package@v2 + with: + aws_role_arn: ${{ secrets.AWS_ROLE_ARN }} + aws_region_name: us-east-1 + aws_secret_id: ${{ secrets.AWS_SECRET_ID }} + npm_package_name: mongodb + dry_run: ${{ needs.release_please.outputs.release_created == '' }} + + - name: Copy sbom file to release assets + shell: bash + if: ${{ '' == '' }} + run: cp sbom.json ${{ env.S3_ASSETS }}/sbom.json + + # only used for mongodb-client-encryption + - name: Augment SBOM and copy to release assets + if: ${{ '' != '' }} + uses: mongodb-labs/drivers-github-tools/sbom@v2 + with: + silk_asset_group: '' + sbom_file_name: sbom.json + + - name: Generate authorized pub report + uses: mongodb-labs/drivers-github-tools/full-report@v2 + with: + release_version: ${{ env.package_version }} + product_name: mongodb + sarif_report_target_ref: 'main' + third_party_dependency_tool: n/a + dist_filenames: artifacts/* + token: ${{ github.token }} + sbom_file_name: sbom.json + evergreen_project: mongo-node-driver-next + evergreen_commit: ${{ env.commit }} + + - uses: mongodb-labs/drivers-github-tools/upload-s3-assets@v2 + with: + version: ${{ env.package_version }} + product_name: mongodb + dry_run: ${{ needs.release_please.outputs.release_created == '' }} \ No newline at end of file diff --git a/test/encryption/encryption.test.js b/test/encryption/encryption.test.js new file mode 100644 index 0000000000..cb9d3be2d6 --- /dev/null +++ b/test/encryption/encryption.test.js @@ -0,0 +1,11 @@ +'use strict'; + +const assert = require('assert'); +const EJSON = require('bson').EJSON; + +describe('setup check', () => { + it('environment variables are set', async function() { + const { local } = EJSON.parse(process.env.CSFLE_KMS_PROVIDERS || '{}'); + assert.ok(local); + }); +}); From aeda36be987adfaf29fa6ce9c5d3ca03aea13e11 Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Wed, 20 Nov 2024 02:14:22 -0500 Subject: [PATCH 02/33] temp --- .github/workflows/encryption-tests.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/encryption-tests.yml b/.github/workflows/encryption-tests.yml index 87ffa37fcc..82214877e6 100644 --- a/.github/workflows/encryption-tests.yml +++ b/.github/workflows/encryption-tests.yml @@ -17,14 +17,15 @@ jobs: security-events: write id-token: write contents: write - environment: release runs-on: ubuntu-latest steps: - name: Install mongodb-client-encryption uses: mongodb-client-encryption - name: Set-up cluster - + - shell: bash + + - name: Run tests From e5d8cadbf86e924f8db30317cddd659366dad390 Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Wed, 20 Nov 2024 16:24:58 -0500 Subject: [PATCH 03/33] temp 2 --- .github/scripts/run-kms-servers.sh | 11 +++ .github/scripts/run-orchestration.sh | 36 +++++++++ .github/workflows/encryption-tests.yml | 103 ++++++++++++++----------- 3 files changed, 103 insertions(+), 47 deletions(-) create mode 100644 .github/scripts/run-kms-servers.sh create mode 100644 .github/scripts/run-orchestration.sh diff --git a/.github/scripts/run-kms-servers.sh b/.github/scripts/run-kms-servers.sh new file mode 100644 index 0000000000..988b07bfef --- /dev/null +++ b/.github/scripts/run-kms-servers.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -o errexit # Exit the script with error if any of the commands fail + +cd ${DRIVERS_TOOLS}/.evergreen/csfle +. ./prepare-kmsvenv.sh + +echo "$PYTHON_EXEC" + +$PYTHON_EXEC -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/expired.pem --port 8000 & +$PYTHON_EXEC -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/wrong-host.pem --port 8001 & +$PYTHON_EXEC -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/server.pem --port 8002 --require_client_cert & \ No newline at end of file diff --git a/.github/scripts/run-orchestration.sh b/.github/scripts/run-orchestration.sh new file mode 100644 index 0000000000..3708c7bc33 --- /dev/null +++ b/.github/scripts/run-orchestration.sh @@ -0,0 +1,36 @@ +#! /usr/bin/env bash + +# This is a thin wrapper around drivers-tools run orchestration meant to print each of the configuration settings we make use of +# Additionally it ensures the downloaded binaries are in the PATH for the script to find (namely, the legacy shell for server set up) + +export MONGODB_VERSION=${VERSION} +echo "MONGODB_VERSION=${VERSION}" + +export TOPOLOGY=${TOPOLOGY} +echo "TOPOLOGY=${TOPOLOGY}" + +export AUTH=${AUTH} +echo "AUTH=${AUTH}" + +export SSL=${SSL} +echo "SSL=${SSL}" + +export ORCHESTRATION_FILE=${ORCHESTRATION_FILE} +echo "ORCHESTRATION_FILE=${ORCHESTRATION_FILE}" + +export REQUIRE_API_VERSION=${REQUIRE_API_VERSION} +echo "REQUIRE_API_VERSION=${REQUIRE_API_VERSION}" + +export LOAD_BALANCER=${LOAD_BALANCER} +echo "LOAD_BALANCER=${LOAD_BALANCER}" + +export COMPRESSOR=${COMPRESSOR} +echo "COMPRESSOR=${COMPRESSOR}" + +export PATH="$MONGODB_BINARIES:$PATH" +echo "MONGODB_BINARIES=${MONGODB_BINARIES}" + +export SKIP_LEGACY_SHELL="true" +echo "SKIP_LEGACY_SHELL=${SKIP_LEGACY_SHELL}" + +bash "${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh" \ No newline at end of file diff --git a/.github/workflows/encryption-tests.yml b/.github/workflows/encryption-tests.yml index 82214877e6..e8afec2d56 100644 --- a/.github/workflows/encryption-tests.yml +++ b/.github/workflows/encryption-tests.yml @@ -11,7 +11,7 @@ permissions: name: encryption-tests jobs: - ssdlc: + set-up: permissions: # required for all workflows security-events: write @@ -24,52 +24,61 @@ jobs: - name: Set-up cluster - shell: bash - - - - - name: Run tests - uses: mongodb-labs/drivers-github-tools/node/get_version_info@v2 - with: - npm_package_name: mongodb - - - name: actions/compress_sign_and_upload - uses: mongodb-labs/drivers-github-tools/node/sign_node_package@v2 - with: - aws_role_arn: ${{ secrets.AWS_ROLE_ARN }} - aws_region_name: us-east-1 - aws_secret_id: ${{ secrets.AWS_SECRET_ID }} - npm_package_name: mongodb - dry_run: ${{ needs.release_please.outputs.release_created == '' }} - - - name: Copy sbom file to release assets - shell: bash - if: ${{ '' == '' }} - run: cp sbom.json ${{ env.S3_ASSETS }}/sbom.json - - # only used for mongodb-client-encryption - - name: Augment SBOM and copy to release assets - if: ${{ '' != '' }} - uses: mongodb-labs/drivers-github-tools/sbom@v2 - with: - silk_asset_group: '' - sbom_file_name: sbom.json - - - name: Generate authorized pub report - uses: mongodb-labs/drivers-github-tools/full-report@v2 + run-tests: + needs: + - set-up + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + node: [20] + os: [ubuntu-latest] + mongodb: [8.0.0] + #clone drivers-evergreen-tools (will delete itself once the run is over) + git clone --depth=1 https://github.com/mongodb-labs/drivers-evergreen-tools.git /data/mci/ead9f9a67ef900991e09bd9589dfee0e/drivers-tools + #set path + #calling run orchestration sets up a server on your computer + # it outputs relevants uri in expansions.yml + # put uri into an environment variable + - name: set-up-orchestration + - command: subprocess.exec + params: + binary: bash + add_expansions_to_env: true + env: + # The following is the settings for how we want to launch mongodb + MONGODB_VERSION: ${VERSION} + TOPOLOGY: ${TOPOLOGY} + AUTH: ${AUTH} + SSL: ${SSL} + ORCHESTRATION_FILE: ${ORCHESTRATION_FILE} + REQUIRE_API_VERSION: ${REQUIRE_API_VERSION} + LOAD_BALANCER: ${LOAD_BALANCER} + COMPRESSOR: ${COMPRESSOR} + # These are paths necessary for launching mongodb + DRIVERS_TOOLS: ${DRIVERS_TOOLS} + MONGODB_BINARIES: ${MONGODB_BINARIES} + args: + - .github/scripts/run-orchestration.sh + name: Encryption tests + env: + FORCE_COLOR: true + steps: + - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + - name: Setup node + uses: actions/setup-node@0a44ba7841725637a19e28fa30b79a866c81b0a6 # v4.0.4 with: - release_version: ${{ env.package_version }} - product_name: mongodb - sarif_report_target_ref: 'main' - third_party_dependency_tool: n/a - dist_filenames: artifacts/* - token: ${{ github.token }} - sbom_file_name: sbom.json - evergreen_project: mongo-node-driver-next - evergreen_commit: ${{ env.commit }} - - - uses: mongodb-labs/drivers-github-tools/upload-s3-assets@v2 + node-version: latest + - name: Load MongoDB binary cache + id: cache-mongodb-binaries + uses: actions/cache@v4 with: - version: ${{ env.package_version }} - product_name: mongodb - dry_run: ${{ needs.release_please.outputs.release_created == '' }} \ No newline at end of file + path: ~/.cache/mongodb-binaries + key: ${{ matrix.os }}-${{ matrix.mongodb }} + - name: Install Dependencies + run: npm install + - name: Install drivers-evergreen-tools + + - name: Test + run: mocha --exit ./test/encryption/*.test.js \ No newline at end of file From a041206260d287d0746df5c4973f7735c9238418 Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Thu, 21 Nov 2024 16:59:50 -0500 Subject: [PATCH 04/33] test --- .github/workflows/encryption-tests.yml | 62 ++++++++++++++++---------- 1 file changed, 39 insertions(+), 23 deletions(-) diff --git a/.github/workflows/encryption-tests.yml b/.github/workflows/encryption-tests.yml index e8afec2d56..586cc450da 100644 --- a/.github/workflows/encryption-tests.yml +++ b/.github/workflows/encryption-tests.yml @@ -10,6 +10,28 @@ permissions: name: encryption-tests +functions: + "bootstrap mongo-orchestration": + - command: subprocess.exec + params: + binary: bash + add_expansions_to_env: true + env: + # The following is the settings for how we want to launch mongodb + MONGODB_VERSION: ${VERSION} + TOPOLOGY: ${TOPOLOGY} + AUTH: ${AUTH} + SSL: ${SSL} + ORCHESTRATION_FILE: ${ORCHESTRATION_FILE} + REQUIRE_API_VERSION: ${REQUIRE_API_VERSION} + LOAD_BALANCER: ${LOAD_BALANCER} + COMPRESSOR: ${COMPRESSOR} + # These are paths necessary for launching mongodb + DRIVERS_TOOLS: ${DRIVERS_TOOLS} + MONGODB_BINARIES: ${MONGODB_BINARIES} + args: + - ./workflows/run-orchestration.sh + jobs: set-up: permissions: @@ -21,7 +43,6 @@ jobs: steps: - name: Install mongodb-client-encryption uses: mongodb-client-encryption - - name: Set-up cluster - shell: bash @@ -36,31 +57,10 @@ jobs: os: [ubuntu-latest] mongodb: [8.0.0] #clone drivers-evergreen-tools (will delete itself once the run is over) - git clone --depth=1 https://github.com/mongodb-labs/drivers-evergreen-tools.git /data/mci/ead9f9a67ef900991e09bd9589dfee0e/drivers-tools #set path #calling run orchestration sets up a server on your computer # it outputs relevants uri in expansions.yml # put uri into an environment variable - - name: set-up-orchestration - - command: subprocess.exec - params: - binary: bash - add_expansions_to_env: true - env: - # The following is the settings for how we want to launch mongodb - MONGODB_VERSION: ${VERSION} - TOPOLOGY: ${TOPOLOGY} - AUTH: ${AUTH} - SSL: ${SSL} - ORCHESTRATION_FILE: ${ORCHESTRATION_FILE} - REQUIRE_API_VERSION: ${REQUIRE_API_VERSION} - LOAD_BALANCER: ${LOAD_BALANCER} - COMPRESSOR: ${COMPRESSOR} - # These are paths necessary for launching mongodb - DRIVERS_TOOLS: ${DRIVERS_TOOLS} - MONGODB_BINARIES: ${MONGODB_BINARIES} - args: - - .github/scripts/run-orchestration.sh name: Encryption tests env: FORCE_COLOR: true @@ -79,6 +79,22 @@ jobs: - name: Install Dependencies run: npm install - name: Install drivers-evergreen-tools - + run: git clone --depth=1 https://github.com/mongodb-labs/drivers-evergreen-tools.git ~/.cache/drivers-evergreen-tools + - name: Set up cluster + + run: bash .github/workflows/run-orchestration.sh + with: + # The following is the settings for how we want to launch mongodb + MONGODB_VERSION: 8.0.0 + TOPOLOGY: sharded + AUTH: auth + # ORCHESTRATION_FILE: ${ORCHESTRATION_FILE} + # REQUIRE_API_VERSION: ${REQUIRE_API_VERSION} + # LOAD_BALANCER: ${LOAD_BALANCER} + # COMPRESSOR: ${COMPRESSOR} + # These are paths necessary for launching mongodb + DRIVERS_TOOLS: ~/.cache/drivers-evergreen-tools + MONGODB_BINARIES: ~/.cache/mongodb-binaries + - - name: Test run: mocha --exit ./test/encryption/*.test.js \ No newline at end of file From 0c978a283aa6bce08293ff3e82489b98f0d16990 Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Thu, 21 Nov 2024 17:51:23 -0500 Subject: [PATCH 05/33] edit --- .github/workflows/encryption-tests.yml | 26 ++------------------------ 1 file changed, 2 insertions(+), 24 deletions(-) diff --git a/.github/workflows/encryption-tests.yml b/.github/workflows/encryption-tests.yml index 586cc450da..69580c06f4 100644 --- a/.github/workflows/encryption-tests.yml +++ b/.github/workflows/encryption-tests.yml @@ -1,3 +1,5 @@ +name: encryption-tests + on: push: branches: ['main'] @@ -8,30 +10,6 @@ permissions: pull-requests: write id-token: write -name: encryption-tests - -functions: - "bootstrap mongo-orchestration": - - command: subprocess.exec - params: - binary: bash - add_expansions_to_env: true - env: - # The following is the settings for how we want to launch mongodb - MONGODB_VERSION: ${VERSION} - TOPOLOGY: ${TOPOLOGY} - AUTH: ${AUTH} - SSL: ${SSL} - ORCHESTRATION_FILE: ${ORCHESTRATION_FILE} - REQUIRE_API_VERSION: ${REQUIRE_API_VERSION} - LOAD_BALANCER: ${LOAD_BALANCER} - COMPRESSOR: ${COMPRESSOR} - # These are paths necessary for launching mongodb - DRIVERS_TOOLS: ${DRIVERS_TOOLS} - MONGODB_BINARIES: ${MONGODB_BINARIES} - args: - - ./workflows/run-orchestration.sh - jobs: set-up: permissions: From f8cbb9aef63b9c8f261c81f5a86383ce828a027c Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Thu, 21 Nov 2024 18:13:16 -0500 Subject: [PATCH 06/33] removed errors --- .github/workflows/encryption-tests.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/encryption-tests.yml b/.github/workflows/encryption-tests.yml index 69580c06f4..a39ed25b08 100644 --- a/.github/workflows/encryption-tests.yml +++ b/.github/workflows/encryption-tests.yml @@ -1,4 +1,4 @@ -name: encryption-tests +name: Encryption Tests on: push: @@ -21,8 +21,6 @@ jobs: steps: - name: Install mongodb-client-encryption uses: mongodb-client-encryption - - name: Set-up cluster - - shell: bash run-tests: needs: @@ -59,9 +57,8 @@ jobs: - name: Install drivers-evergreen-tools run: git clone --depth=1 https://github.com/mongodb-labs/drivers-evergreen-tools.git ~/.cache/drivers-evergreen-tools - name: Set up cluster - - run: bash .github/workflows/run-orchestration.sh - with: + run: bash /.github/scripts/run-orchestration.sh + env: # The following is the settings for how we want to launch mongodb MONGODB_VERSION: 8.0.0 TOPOLOGY: sharded @@ -73,6 +70,5 @@ jobs: # These are paths necessary for launching mongodb DRIVERS_TOOLS: ~/.cache/drivers-evergreen-tools MONGODB_BINARIES: ~/.cache/mongodb-binaries - - - name: Test - run: mocha --exit ./test/encryption/*.test.js \ No newline at end of file + run: mocha --exit ./test/encryption/*.test.js From 2fa3417ce20c45f3398b7b91e2951d5240b1c005 Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Thu, 21 Nov 2024 18:16:21 -0500 Subject: [PATCH 07/33] on push --- .github/workflows/encryption-tests.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/encryption-tests.yml b/.github/workflows/encryption-tests.yml index a39ed25b08..24c0fd2362 100644 --- a/.github/workflows/encryption-tests.yml +++ b/.github/workflows/encryption-tests.yml @@ -1,9 +1,8 @@ name: Encryption Tests on: - push: - branches: ['main'] - workflow_dispatch: {} + push + #workflow_dispatch: {} permissions: contents: write From c71d3c287fd94a21de683005ae97920c9b585e42 Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Thu, 21 Nov 2024 18:21:34 -0500 Subject: [PATCH 08/33] test --- .github/workflows/encryption-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/encryption-tests.yml b/.github/workflows/encryption-tests.yml index 24c0fd2362..6bb982c03b 100644 --- a/.github/workflows/encryption-tests.yml +++ b/.github/workflows/encryption-tests.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Install mongodb-client-encryption - uses: mongodb-client-encryption + uses: mongodb-client-encryption@v6.1 run-tests: needs: From e6facfe19ce58bc2ebba3e6334406f7ee77cceff Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Thu, 21 Nov 2024 18:27:21 -0500 Subject: [PATCH 09/33] path --- .github/workflows/encryption-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/encryption-tests.yml b/.github/workflows/encryption-tests.yml index 6bb982c03b..a5d8717755 100644 --- a/.github/workflows/encryption-tests.yml +++ b/.github/workflows/encryption-tests.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Install mongodb-client-encryption - uses: mongodb-client-encryption@v6.1 + uses: mongodb-js/mongodb-client-encryption@v6.1.0 run-tests: needs: From 06f9758cc2be1a62c196c1d09a05de613c518318 Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Thu, 21 Nov 2024 18:33:59 -0500 Subject: [PATCH 10/33] path --- .github/workflows/encryption-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/encryption-tests.yml b/.github/workflows/encryption-tests.yml index a5d8717755..788a57d1aa 100644 --- a/.github/workflows/encryption-tests.yml +++ b/.github/workflows/encryption-tests.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Install mongodb-client-encryption - uses: mongodb-js/mongodb-client-encryption@v6.1.0 + run: npm install mongodb-client-encryption run-tests: needs: From 86f7874c5898ff298167666e25c24b4b047f3f3f Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Thu, 21 Nov 2024 18:35:59 -0500 Subject: [PATCH 11/33] path --- .github/workflows/encryption-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/encryption-tests.yml b/.github/workflows/encryption-tests.yml index 788a57d1aa..90a10f1aa2 100644 --- a/.github/workflows/encryption-tests.yml +++ b/.github/workflows/encryption-tests.yml @@ -56,7 +56,7 @@ jobs: - name: Install drivers-evergreen-tools run: git clone --depth=1 https://github.com/mongodb-labs/drivers-evergreen-tools.git ~/.cache/drivers-evergreen-tools - name: Set up cluster - run: bash /.github/scripts/run-orchestration.sh + run: bash ../scripts/run-orchestration.sh env: # The following is the settings for how we want to launch mongodb MONGODB_VERSION: 8.0.0 From 7a30734ed0959f673b2046a5e19ba503f167a01a Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Thu, 21 Nov 2024 18:37:54 -0500 Subject: [PATCH 12/33] path --- .github/workflows/encryption-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/encryption-tests.yml b/.github/workflows/encryption-tests.yml index 90a10f1aa2..00e3324072 100644 --- a/.github/workflows/encryption-tests.yml +++ b/.github/workflows/encryption-tests.yml @@ -56,7 +56,7 @@ jobs: - name: Install drivers-evergreen-tools run: git clone --depth=1 https://github.com/mongodb-labs/drivers-evergreen-tools.git ~/.cache/drivers-evergreen-tools - name: Set up cluster - run: bash ../scripts/run-orchestration.sh + run: bash .github/scripts/run-orchestration.sh env: # The following is the settings for how we want to launch mongodb MONGODB_VERSION: 8.0.0 From b39c75442305b51598efa78966c3c16e7150046e Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Fri, 22 Nov 2024 14:34:08 -0500 Subject: [PATCH 13/33] fixed --- .github/workflows/encryption-tests.yml | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/.github/workflows/encryption-tests.yml b/.github/workflows/encryption-tests.yml index 00e3324072..9234f2b508 100644 --- a/.github/workflows/encryption-tests.yml +++ b/.github/workflows/encryption-tests.yml @@ -53,21 +53,15 @@ jobs: key: ${{ matrix.os }}-${{ matrix.mongodb }} - name: Install Dependencies run: npm install - - name: Install drivers-evergreen-tools - run: git clone --depth=1 https://github.com/mongodb-labs/drivers-evergreen-tools.git ~/.cache/drivers-evergreen-tools - name: Set up cluster - run: bash .github/scripts/run-orchestration.sh - env: - # The following is the settings for how we want to launch mongodb - MONGODB_VERSION: 8.0.0 - TOPOLOGY: sharded - AUTH: auth - # ORCHESTRATION_FILE: ${ORCHESTRATION_FILE} - # REQUIRE_API_VERSION: ${REQUIRE_API_VERSION} - # LOAD_BALANCER: ${LOAD_BALANCER} - # COMPRESSOR: ${COMPRESSOR} - # These are paths necessary for launching mongodb - DRIVERS_TOOLS: ~/.cache/drivers-evergreen-tools - MONGODB_BINARIES: ~/.cache/mongodb-binaries - - name: Test + id: setup-cluster + uses: mongodb-labs/drivers-evergreen-tools@master + with: + version: ${{ matrix.mongodb }} + topology: sharded + auth: auth + - name: Run Tests run: mocha --exit ./test/encryption/*.test.js + env: + MONGODB_URI: ${{ steps.setup-cluster.outputs.cluster-uri }} + CRYPT_SHARED_LIB_PATH: ${{ steps.setup-cluster.outputs.crypt-shared-lib-path }} From ccb726edbb9fcd6a253b2759967a5f3285ed6813 Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Fri, 22 Nov 2024 14:47:20 -0500 Subject: [PATCH 14/33] typo --- .github/workflows/encryption-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/encryption-tests.yml b/.github/workflows/encryption-tests.yml index 9234f2b508..c83217958d 100644 --- a/.github/workflows/encryption-tests.yml +++ b/.github/workflows/encryption-tests.yml @@ -58,7 +58,7 @@ jobs: uses: mongodb-labs/drivers-evergreen-tools@master with: version: ${{ matrix.mongodb }} - topology: sharded + topology: sharded_cluster auth: auth - name: Run Tests run: mocha --exit ./test/encryption/*.test.js From b0a1c3c08e186925d179dc05a952bd073e768c92 Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Fri, 22 Nov 2024 14:51:24 -0500 Subject: [PATCH 15/33] install mocha --- .github/workflows/encryption-tests.yml | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/.github/workflows/encryption-tests.yml b/.github/workflows/encryption-tests.yml index c83217958d..c4ee9e3172 100644 --- a/.github/workflows/encryption-tests.yml +++ b/.github/workflows/encryption-tests.yml @@ -10,20 +10,12 @@ permissions: id-token: write jobs: - set-up: + run-tests: permissions: # required for all workflows security-events: write id-token: write contents: write - runs-on: ubuntu-latest - steps: - - name: Install mongodb-client-encryption - run: npm install mongodb-client-encryption - - run-tests: - needs: - - set-up runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -40,17 +32,19 @@ jobs: env: FORCE_COLOR: true steps: + # - name: Install mongodb-client-encryption + # run: npm install mongodb-client-encryption - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 - name: Setup node uses: actions/setup-node@0a44ba7841725637a19e28fa30b79a866c81b0a6 # v4.0.4 with: node-version: latest - - name: Load MongoDB binary cache - id: cache-mongodb-binaries - uses: actions/cache@v4 - with: - path: ~/.cache/mongodb-binaries - key: ${{ matrix.os }}-${{ matrix.mongodb }} + #- name: Load MongoDB binary cache + # id: cache-mongodb-binaries + # uses: actions/cache@v4 + # with: + # path: ~/.cache/mongodb-binaries + # key: ${{ matrix.os }}-${{ matrix.mongodb }} - name: Install Dependencies run: npm install - name: Set up cluster @@ -60,6 +54,8 @@ jobs: version: ${{ matrix.mongodb }} topology: sharded_cluster auth: auth + - name: Install mocha + run: brew install mocha - name: Run Tests run: mocha --exit ./test/encryption/*.test.js env: From 6198ade074d21b0d0d94df92d01930ca7ac9d995 Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Fri, 22 Nov 2024 14:56:31 -0500 Subject: [PATCH 16/33] fixed? --- .github/scripts/run-orchestration.sh | 36 -------------------------- .github/workflows/encryption-tests.yml | 17 +----------- 2 files changed, 1 insertion(+), 52 deletions(-) delete mode 100644 .github/scripts/run-orchestration.sh diff --git a/.github/scripts/run-orchestration.sh b/.github/scripts/run-orchestration.sh deleted file mode 100644 index 3708c7bc33..0000000000 --- a/.github/scripts/run-orchestration.sh +++ /dev/null @@ -1,36 +0,0 @@ -#! /usr/bin/env bash - -# This is a thin wrapper around drivers-tools run orchestration meant to print each of the configuration settings we make use of -# Additionally it ensures the downloaded binaries are in the PATH for the script to find (namely, the legacy shell for server set up) - -export MONGODB_VERSION=${VERSION} -echo "MONGODB_VERSION=${VERSION}" - -export TOPOLOGY=${TOPOLOGY} -echo "TOPOLOGY=${TOPOLOGY}" - -export AUTH=${AUTH} -echo "AUTH=${AUTH}" - -export SSL=${SSL} -echo "SSL=${SSL}" - -export ORCHESTRATION_FILE=${ORCHESTRATION_FILE} -echo "ORCHESTRATION_FILE=${ORCHESTRATION_FILE}" - -export REQUIRE_API_VERSION=${REQUIRE_API_VERSION} -echo "REQUIRE_API_VERSION=${REQUIRE_API_VERSION}" - -export LOAD_BALANCER=${LOAD_BALANCER} -echo "LOAD_BALANCER=${LOAD_BALANCER}" - -export COMPRESSOR=${COMPRESSOR} -echo "COMPRESSOR=${COMPRESSOR}" - -export PATH="$MONGODB_BINARIES:$PATH" -echo "MONGODB_BINARIES=${MONGODB_BINARIES}" - -export SKIP_LEGACY_SHELL="true" -echo "SKIP_LEGACY_SHELL=${SKIP_LEGACY_SHELL}" - -bash "${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh" \ No newline at end of file diff --git a/.github/workflows/encryption-tests.yml b/.github/workflows/encryption-tests.yml index c4ee9e3172..158cde2e10 100644 --- a/.github/workflows/encryption-tests.yml +++ b/.github/workflows/encryption-tests.yml @@ -23,28 +23,15 @@ jobs: node: [20] os: [ubuntu-latest] mongodb: [8.0.0] - #clone drivers-evergreen-tools (will delete itself once the run is over) - #set path - #calling run orchestration sets up a server on your computer - # it outputs relevants uri in expansions.yml - # put uri into an environment variable name: Encryption tests env: FORCE_COLOR: true steps: - # - name: Install mongodb-client-encryption - # run: npm install mongodb-client-encryption - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 - name: Setup node uses: actions/setup-node@0a44ba7841725637a19e28fa30b79a866c81b0a6 # v4.0.4 with: node-version: latest - #- name: Load MongoDB binary cache - # id: cache-mongodb-binaries - # uses: actions/cache@v4 - # with: - # path: ~/.cache/mongodb-binaries - # key: ${{ matrix.os }}-${{ matrix.mongodb }} - name: Install Dependencies run: npm install - name: Set up cluster @@ -54,10 +41,8 @@ jobs: version: ${{ matrix.mongodb }} topology: sharded_cluster auth: auth - - name: Install mocha - run: brew install mocha - name: Run Tests - run: mocha --exit ./test/encryption/*.test.js + run: npx mocha --exit ./test/encryption/*.test.js env: MONGODB_URI: ${{ steps.setup-cluster.outputs.cluster-uri }} CRYPT_SHARED_LIB_PATH: ${{ steps.setup-cluster.outputs.crypt-shared-lib-path }} From a135e79c93a1836acc0639c7dfd3304565c11eda Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Fri, 22 Nov 2024 15:02:53 -0500 Subject: [PATCH 17/33] fix tests --- test/encryption/encryption.test.js | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/test/encryption/encryption.test.js b/test/encryption/encryption.test.js index cb9d3be2d6..66088cf0c5 100644 --- a/test/encryption/encryption.test.js +++ b/test/encryption/encryption.test.js @@ -3,9 +3,14 @@ const assert = require('assert'); const EJSON = require('bson').EJSON; -describe('setup check', () => { - it('environment variables are set', async function() { - const { local } = EJSON.parse(process.env.CSFLE_KMS_PROVIDERS || '{}'); - assert.ok(local); +describe('environmental variables', () => { + it('MONGODB_URI is set', async function() { + const uri = process.env.MONGODB_URI; + assert.ok(uri); + }); + + it('CRYPT_SHARED_LIB_PATH is set', async function() { + const shared_library_path = process.env.CRYPT_SHARED_LIB_PATH; + assert.ok(shared_library_path); }); }); From ca258684b1db84b5cfb13a9b8f8eb9b75dc95279 Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Fri, 22 Nov 2024 15:59:31 -0500 Subject: [PATCH 18/33] run tests --- .github/workflows/encryption-tests.yml | 4 ++- mongocryptd.pid | 0 package.json | 1 + test/encryption/encryption.test.js | 35 +++++++++++++++++++++++--- 4 files changed, 36 insertions(+), 4 deletions(-) create mode 100644 mongocryptd.pid diff --git a/.github/workflows/encryption-tests.yml b/.github/workflows/encryption-tests.yml index 158cde2e10..09dc64df74 100644 --- a/.github/workflows/encryption-tests.yml +++ b/.github/workflows/encryption-tests.yml @@ -34,6 +34,8 @@ jobs: node-version: latest - name: Install Dependencies run: npm install + - name: Install mongodb-client-encryption + run: npm install mongodb-client-encryption - name: Set up cluster id: setup-cluster uses: mongodb-labs/drivers-evergreen-tools@master @@ -44,5 +46,5 @@ jobs: - name: Run Tests run: npx mocha --exit ./test/encryption/*.test.js env: - MONGODB_URI: ${{ steps.setup-cluster.outputs.cluster-uri }} + MONGOOSE_TEST_URI: ${{ steps.setup-cluster.outputs.cluster-uri }} CRYPT_SHARED_LIB_PATH: ${{ steps.setup-cluster.outputs.crypt-shared-lib-path }} diff --git a/mongocryptd.pid b/mongocryptd.pid new file mode 100644 index 0000000000..e69de29bb2 diff --git a/package.json b/package.json index 085d0655a7..c24f244497 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "bson": "^6.7.0", "kareem": "2.6.3", "mongodb": "~6.10.0", + "mongodb-client-encryption": "^6.1.0", "mpath": "0.9.0", "mquery": "5.0.0", "ms": "2.1.3", diff --git a/test/encryption/encryption.test.js b/test/encryption/encryption.test.js index 66088cf0c5..e6aa2f1d7b 100644 --- a/test/encryption/encryption.test.js +++ b/test/encryption/encryption.test.js @@ -1,11 +1,11 @@ 'use strict'; const assert = require('assert'); -const EJSON = require('bson').EJSON; +const mdb = require('mongodb'); describe('environmental variables', () => { - it('MONGODB_URI is set', async function() { - const uri = process.env.MONGODB_URI; + it('MONGODB_TEST_URI is set', async function() { + const uri = process.env.MONGOOSE_TEST_URI; assert.ok(uri); }); @@ -14,3 +14,32 @@ describe('environmental variables', () => { assert.ok(shared_library_path); }); }); + +describe('basic integration', () => { + it('supports mongodb csfle auto-encryption integration', async() => { + // 1. Create a MongoClient configured with auto encryption (referred to as `client_encrypted`) + const client = new mdb.MongoClient( + process.env.MONGOOSE_TEST_URI, + { + autoEncryption: { + keyVaultNamespace: 'keyvault.datakeys', + kmsProviders: { local: { key: Buffer.from( + 'Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk', + 'base64' + ) + } }, + extraOptions: { + cryptdSharedLibRequired: true, + cryptSharedLibPath: process.env.CRYPT_SHARED_LIB_PATH + } + } + } + ); + await client.connect(); + const insertResult = await client + .db('db') + .collection('coll') + .insertOne({ unencrypted: 'test' }); + assert.ok(insertResult.insertedId); + }); +}); From 270d151fcb14ef8bfff2be5b4420d9e69c368bb4 Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Mon, 25 Nov 2024 16:54:17 -0500 Subject: [PATCH 19/33] ready for rereview --- .github/scripts/run-kms-servers.sh | 11 ---- .github/workflows/encryption-tests.yml | 17 ++---- package.json | 1 + test/encryption/encryption.test.js | 80 +++++++++++++++++++++----- 4 files changed, 72 insertions(+), 37 deletions(-) delete mode 100644 .github/scripts/run-kms-servers.sh diff --git a/.github/scripts/run-kms-servers.sh b/.github/scripts/run-kms-servers.sh deleted file mode 100644 index 988b07bfef..0000000000 --- a/.github/scripts/run-kms-servers.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -set -o errexit # Exit the script with error if any of the commands fail - -cd ${DRIVERS_TOOLS}/.evergreen/csfle -. ./prepare-kmsvenv.sh - -echo "$PYTHON_EXEC" - -$PYTHON_EXEC -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/expired.pem --port 8000 & -$PYTHON_EXEC -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/wrong-host.pem --port 8001 & -$PYTHON_EXEC -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/server.pem --port 8002 --require_client_cert & \ No newline at end of file diff --git a/.github/workflows/encryption-tests.yml b/.github/workflows/encryption-tests.yml index 09dc64df74..2151b4eb3e 100644 --- a/.github/workflows/encryption-tests.yml +++ b/.github/workflows/encryption-tests.yml @@ -1,8 +1,9 @@ name: Encryption Tests on: - push - #workflow_dispatch: {} + push: + branches: ['master'] + workflow_dispatch: {} permissions: contents: write @@ -16,13 +17,7 @@ jobs: security-events: write id-token: write contents: write - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - node: [20] - os: [ubuntu-latest] - mongodb: [8.0.0] + runs-on: ubuntu-latest name: Encryption tests env: FORCE_COLOR: true @@ -40,11 +35,11 @@ jobs: id: setup-cluster uses: mongodb-labs/drivers-evergreen-tools@master with: - version: ${{ matrix.mongodb }} + version: 8.0.0 topology: sharded_cluster auth: auth - name: Run Tests - run: npx mocha --exit ./test/encryption/*.test.js + run: npm run encryption-test env: MONGOOSE_TEST_URI: ${{ steps.setup-cluster.outputs.cluster-uri }} CRYPT_SHARED_LIB_PATH: ${{ steps.setup-cluster.outputs.crypt-shared-lib-path }} diff --git a/package.json b/package.json index c24f244497..405af204ad 100644 --- a/package.json +++ b/package.json @@ -105,6 +105,7 @@ "test-deno": "deno run --allow-env --allow-read --allow-net --allow-run --allow-sys --allow-write ./test/deno.js", "test-rs": "START_REPLICA_SET=1 mocha --timeout 30000 --exit ./test/*.test.js", "test-tsd": "node ./test/types/check-types-filename && tsd", + "test-encryption": "mocha --exit ./test/encryption/*.test.js", "tdd": "mocha ./test/*.test.js --inspect --watch --recursive --watch-files ./**/*.{js,ts}", "test-coverage": "nyc --reporter=html --reporter=text npm test", "ts-benchmark": "cd ./benchmarks/typescript/simple && npm install && npm run benchmark | node ../../../scripts/tsc-diagnostics-check" diff --git a/test/encryption/encryption.test.js b/test/encryption/encryption.test.js index e6aa2f1d7b..a1b082433a 100644 --- a/test/encryption/encryption.test.js +++ b/test/encryption/encryption.test.js @@ -2,9 +2,12 @@ const assert = require('assert'); const mdb = require('mongodb'); +const isBsonType = require('../../lib/helpers/isBsonType'); + +const LOCAL_KEY = Buffer.from('Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk', 'base64'); describe('environmental variables', () => { - it('MONGODB_TEST_URI is set', async function() { + it('MONGOOSE_TEST_URI is set', async function() { const uri = process.env.MONGOOSE_TEST_URI; assert.ok(uri); }); @@ -16,18 +19,44 @@ describe('environmental variables', () => { }); describe('basic integration', () => { - it('supports mongodb csfle auto-encryption integration', async() => { - // 1. Create a MongoClient configured with auto encryption (referred to as `client_encrypted`) - const client = new mdb.MongoClient( + let keyVaultClient; + let dataKey; + let encryptedClient; + let dummyClient; + + beforeEach(async function() { + keyVaultClient = new mdb.MongoClient(process.env.MONGOOSE_TEST_URI); + await keyVaultClient.connect(); + await keyVaultClient.db('keyvault').collection('datakeys'); + const clientEncryption = new mdb.ClientEncryption(keyVaultClient, { + keyVaultNamespace: 'keyvault.datakeys', + kmsProviders: { local: { key: LOCAL_KEY } } + }); + dataKey = await clientEncryption.createDataKey('local'); + + encryptedClient = new mdb.MongoClient( process.env.MONGOOSE_TEST_URI, { autoEncryption: { keyVaultNamespace: 'keyvault.datakeys', - kmsProviders: { local: { key: Buffer.from( - 'Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk', - 'base64' - ) - } }, + kmsProviders: { local: { key: LOCAL_KEY } }, + schemaMap: { + 'db.coll': { + bsonType: 'object', + encryptMetadata: { + keyId: [new mdb.UUID(dataKey)] + }, + properties: { + a: { + encrypt: { + bsonType: 'int', + algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Random', + keyId: [new mdb.UUID(dataKey)] + } + } + } + } + }, extraOptions: { cryptdSharedLibRequired: true, cryptSharedLibPath: process.env.CRYPT_SHARED_LIB_PATH @@ -35,11 +64,32 @@ describe('basic integration', () => { } } ); - await client.connect(); - const insertResult = await client - .db('db') - .collection('coll') - .insertOne({ unencrypted: 'test' }); - assert.ok(insertResult.insertedId); + + dummyClient = new mdb.MongoClient(process.env.MONGOOSE_TEST_URI); + }); + + afterEach(async function() { + await keyVaultClient.close(); + await encryptedClient.close(); + await dummyClient.close(); + }); + + it('supports mongodb csfle auto-encryption integration', async() => { + await encryptedClient.connect(); + await encryptedClient.db('db').collection('coll').insertOne({ a: 1 }); + + // a dummyClient not configured with autoEncryption, returns a encrypted binary type, meaning that encryption succeeded + const encryptedCursor = await dummyClient.db('db').collection('coll').find(); + const encryptedResult = await encryptedCursor.next(); + assert.ok(encryptedResult); + assert.ok(encryptedResult.a); + assert.ok(isBsonType(encryptedResult.a, 'Binary')); + assert.ok(encryptedResult.a.sub_type === 6); + + // when the encryptedClient runs a find, the original unencrypted value is returned + const unencryptedCursor = await encryptedClient.db('db').collection('coll').find(); + const unencryptedResult = await unencryptedCursor.next(); + assert.ok(unencryptedResult); + assert.ok(unencryptedResult.a === 1); }); }); From 69dbda600aaaee77b07d497c43b9830f4a78f13b Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Mon, 25 Nov 2024 16:55:28 -0500 Subject: [PATCH 20/33] ready for rereview 2 --- .github/workflows/encryption-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/encryption-tests.yml b/.github/workflows/encryption-tests.yml index 2151b4eb3e..dc586d3111 100644 --- a/.github/workflows/encryption-tests.yml +++ b/.github/workflows/encryption-tests.yml @@ -2,7 +2,7 @@ name: Encryption Tests on: push: - branches: ['master'] + branches: ['master', 'NODE-6505/ci-setup'] workflow_dispatch: {} permissions: From 3ce14a41f8ba476bef63c0018d20e5fdd9258126 Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Mon, 25 Nov 2024 16:58:55 -0500 Subject: [PATCH 21/33] typo --- .github/workflows/encryption-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/encryption-tests.yml b/.github/workflows/encryption-tests.yml index dc586d3111..17ffbe3ca2 100644 --- a/.github/workflows/encryption-tests.yml +++ b/.github/workflows/encryption-tests.yml @@ -39,7 +39,7 @@ jobs: topology: sharded_cluster auth: auth - name: Run Tests - run: npm run encryption-test + run: npm run test-encryption env: MONGOOSE_TEST_URI: ${{ steps.setup-cluster.outputs.cluster-uri }} CRYPT_SHARED_LIB_PATH: ${{ steps.setup-cluster.outputs.crypt-shared-lib-path }} From eac708f94c7318adcd86e9214e0630fe7b5a4222 Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Tue, 10 Dec 2024 17:24:32 -0500 Subject: [PATCH 22/33] ready for review --- .github/workflows/encryption-tests.yml | 4 +- .gitignore | 2 + CONTRIBUTING.md | 1 + mongocryptd.pid | 0 package.json | 2 + scripts/encrypted-cluster.sh | 57 ++++++++++++++++++++++++++ test/encryption/encryption.test.js | 10 +++-- 7 files changed, 71 insertions(+), 5 deletions(-) delete mode 100644 mongocryptd.pid create mode 100755 scripts/encrypted-cluster.sh diff --git a/.github/workflows/encryption-tests.yml b/.github/workflows/encryption-tests.yml index 17ffbe3ca2..e605778920 100644 --- a/.github/workflows/encryption-tests.yml +++ b/.github/workflows/encryption-tests.yml @@ -2,7 +2,9 @@ name: Encryption Tests on: push: - branches: ['master', 'NODE-6505/ci-setup'] + branches: ['master'] + pull_request: + branches: [ 'master' ] workflow_dispatch: {} permissions: diff --git a/.gitignore b/.gitignore index 47c0742bb1..f88cd4f6ad 100644 --- a/.gitignore +++ b/.gitignore @@ -67,3 +67,5 @@ examples/ecommerce-netlify-functions/.netlify/state.json notes.md list.out + +encrypted-cluster \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6ba098d389..a6d1cfac10 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,6 +46,7 @@ If you have a question about Mongoose (not a bug report) please post it to eithe * execute `npm run test-tsd` to run the typescript tests * execute `npm run ts-benchmark` to run the typescript benchmark "performance test" for a single time. * execute `npm run ts-benchmark-watch` to run the typescript benchmark "performance test" while watching changes on types folder. Note: Make sure to commit all changes before executing this command. +* in order to run tests that require an encrypted cluster locally, run `npm run test-encryption-local`. Alternatively, you can start an encrypted cluster using the `scripts/encrypted-cluster.sh` file ## Documentation diff --git a/mongocryptd.pid b/mongocryptd.pid deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/package.json b/package.json index 405af204ad..660698caaa 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "license": "MIT", "dependencies": { "bson": "^6.7.0", + "hugo": "^0.0.3", "kareem": "2.6.3", "mongodb": "~6.10.0", "mongodb-client-encryption": "^6.1.0", @@ -106,6 +107,7 @@ "test-rs": "START_REPLICA_SET=1 mocha --timeout 30000 --exit ./test/*.test.js", "test-tsd": "node ./test/types/check-types-filename && tsd", "test-encryption": "mocha --exit ./test/encryption/*.test.js", + "test-encryption-local": "chmod +x scripts/encrypted-cluster.sh && scripts/encrypted-cluster.sh", "tdd": "mocha ./test/*.test.js --inspect --watch --recursive --watch-files ./**/*.{js,ts}", "test-coverage": "nyc --reporter=html --reporter=text npm test", "ts-benchmark": "cd ./benchmarks/typescript/simple && npm install && npm run benchmark | node ../../../scripts/tsc-diagnostics-check" diff --git a/scripts/encrypted-cluster.sh b/scripts/encrypted-cluster.sh new file mode 100755 index 0000000000..e919f1870c --- /dev/null +++ b/scripts/encrypted-cluster.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +# sets up an encrypted mongodb cluster + +if [ -d "encrypted-cluster" ]; then + cd encrypted-cluster +else + mkdir encrypted-cluster + cd encrypted-cluster + + if [-d drivers-evergreen-tools]; then + git clone --depth=1 "https://github.com/mongodb-labs/drivers-evergreen-tools.git" + fi + + export DRIVERS_TOOLS=$(pwd)/drivers-evergreen-tools + export MONGODB_VERSION=8.0 + export AUTH=true + export MONGODB_BINARIES=$DRIVERS_TOOLS/mongodb/bin + export NODE_DRIVER=~/dev/node-mongodb-native + export MONGO_ORCHESTRATION_HOME=$DRIVERS_TOOLS/mo + export PROJECT_ORCHESTRATION_HOME=$DRIVERS_TOOLS/.evergreen/orchestration + export TOPOLOGY=sharded_cluster + export SSL=nossl + + cd $DRIVERS_TOOLS + rm -rf mongosh mongodb mo + mkdir mo + cd - + + rm expansions.sh 2> /dev/null + + bash $DRIVERS_TOOLS/.evergreen/run-orchestration.sh +fi + +# IMPORTANT: extracts mongodb-uri, and starts the cluster of servers, store the uri for GitHub output + +read -r -d '' SOURCE_SCRIPT << EOM +const fs = require('fs'); +const file = fs.readFileSync('mo-expansion.yml', { encoding: 'utf-8' }) + .trim().split('\\n'); +const regex = /^(?.*): "(?.*)"$/; +const variables = file.map( + (line) => regex.exec(line.trim()).groups +).map( + ({key, value}) => \`export \${key}='\${value}'\` +).join('\n'); + +process.stdout.write(variables); +process.stdout.write('\n'); +EOM + +node --eval "$SOURCE_SCRIPT" | tee expansions.sh +source expansions.sh + +export MONGOOSE_TEST_URI=$MONGODB_URI + +npm run test-encryption diff --git a/test/encryption/encryption.test.js b/test/encryption/encryption.test.js index a1b082433a..ba29506965 100644 --- a/test/encryption/encryption.test.js +++ b/test/encryption/encryption.test.js @@ -44,14 +44,14 @@ describe('basic integration', () => { 'db.coll': { bsonType: 'object', encryptMetadata: { - keyId: [new mdb.UUID(dataKey)] + keyId: [dataKey] }, properties: { a: { encrypt: { bsonType: 'int', algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Random', - keyId: [new mdb.UUID(dataKey)] + keyId: [dataKey] } } } @@ -78,9 +78,11 @@ describe('basic integration', () => { await encryptedClient.connect(); await encryptedClient.db('db').collection('coll').insertOne({ a: 1 }); + const { insertedId } = await encryptedClient.db('db').collection('coll').insertOne({ a: 1 }); + // a dummyClient not configured with autoEncryption, returns a encrypted binary type, meaning that encryption succeeded - const encryptedCursor = await dummyClient.db('db').collection('coll').find(); - const encryptedResult = await encryptedCursor.next(); + const encryptedResult = await dummyClient.db('db').collection('coll').findOne({ _id: insertedId }); + assert.ok(encryptedResult); assert.ok(encryptedResult.a); assert.ok(isBsonType(encryptedResult.a, 'Binary')); From f38366bd1ea3fcb9d8e3f0e1c0e07cb51159e17c Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Tue, 10 Dec 2024 17:41:46 -0500 Subject: [PATCH 23/33] change to two files for local testing --- .gitignore | 2 +- CONTRIBUTING.md | 2 +- package.json | 2 +- ...uster.sh => run-encryption-tests-local.sh} | 28 ++----------------- scripts/start-encrypted-cluster.sh | 27 ++++++++++++++++++ 5 files changed, 33 insertions(+), 28 deletions(-) rename scripts/{encrypted-cluster.sh => run-encryption-tests-local.sh} (51%) create mode 100644 scripts/start-encrypted-cluster.sh diff --git a/.gitignore b/.gitignore index f88cd4f6ad..88f5a87efb 100644 --- a/.gitignore +++ b/.gitignore @@ -68,4 +68,4 @@ examples/ecommerce-netlify-functions/.netlify/state.json notes.md list.out -encrypted-cluster \ No newline at end of file +encrypted-cluster diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a6d1cfac10..086e2734a2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,7 +46,7 @@ If you have a question about Mongoose (not a bug report) please post it to eithe * execute `npm run test-tsd` to run the typescript tests * execute `npm run ts-benchmark` to run the typescript benchmark "performance test" for a single time. * execute `npm run ts-benchmark-watch` to run the typescript benchmark "performance test" while watching changes on types folder. Note: Make sure to commit all changes before executing this command. -* in order to run tests that require an encrypted cluster locally, run `npm run test-encryption-local`. Alternatively, you can start an encrypted cluster using the `scripts/encrypted-cluster.sh` file +* in order to run tests that require an encrypted cluster locally, run `npm run test-encryption-local`. Alternatively, you can start an encrypted cluster using the `scripts/start-encrypted-cluster.sh` file ## Documentation diff --git a/package.json b/package.json index 660698caaa..06adb0af08 100644 --- a/package.json +++ b/package.json @@ -107,7 +107,7 @@ "test-rs": "START_REPLICA_SET=1 mocha --timeout 30000 --exit ./test/*.test.js", "test-tsd": "node ./test/types/check-types-filename && tsd", "test-encryption": "mocha --exit ./test/encryption/*.test.js", - "test-encryption-local": "chmod +x scripts/encrypted-cluster.sh && scripts/encrypted-cluster.sh", + "test-encryption-local": "chmod +x scripts/run-encryption-tests-local.sh && scripts/run-encryption-tests-local.sh", "tdd": "mocha ./test/*.test.js --inspect --watch --recursive --watch-files ./**/*.{js,ts}", "test-coverage": "nyc --reporter=html --reporter=text npm test", "ts-benchmark": "cd ./benchmarks/typescript/simple && npm install && npm run benchmark | node ../../../scripts/tsc-diagnostics-check" diff --git a/scripts/encrypted-cluster.sh b/scripts/run-encryption-tests-local.sh similarity index 51% rename from scripts/encrypted-cluster.sh rename to scripts/run-encryption-tests-local.sh index e919f1870c..dd29eb9880 100755 --- a/scripts/encrypted-cluster.sh +++ b/scripts/run-encryption-tests-local.sh @@ -2,34 +2,12 @@ # sets up an encrypted mongodb cluster +export CWD=$(pwd); + if [ -d "encrypted-cluster" ]; then cd encrypted-cluster else - mkdir encrypted-cluster - cd encrypted-cluster - - if [-d drivers-evergreen-tools]; then - git clone --depth=1 "https://github.com/mongodb-labs/drivers-evergreen-tools.git" - fi - - export DRIVERS_TOOLS=$(pwd)/drivers-evergreen-tools - export MONGODB_VERSION=8.0 - export AUTH=true - export MONGODB_BINARIES=$DRIVERS_TOOLS/mongodb/bin - export NODE_DRIVER=~/dev/node-mongodb-native - export MONGO_ORCHESTRATION_HOME=$DRIVERS_TOOLS/mo - export PROJECT_ORCHESTRATION_HOME=$DRIVERS_TOOLS/.evergreen/orchestration - export TOPOLOGY=sharded_cluster - export SSL=nossl - - cd $DRIVERS_TOOLS - rm -rf mongosh mongodb mo - mkdir mo - cd - - - rm expansions.sh 2> /dev/null - - bash $DRIVERS_TOOLS/.evergreen/run-orchestration.sh + source $CWD/scripts/start-encrypted-cluster.sh fi # IMPORTANT: extracts mongodb-uri, and starts the cluster of servers, store the uri for GitHub output diff --git a/scripts/start-encrypted-cluster.sh b/scripts/start-encrypted-cluster.sh new file mode 100644 index 0000000000..b04c4b655c --- /dev/null +++ b/scripts/start-encrypted-cluster.sh @@ -0,0 +1,27 @@ + +export CWD=$(pwd); +mkdir encrypted-cluster +cd encrypted-cluster + +if [ ! -d "drivers-evergreen-tools/" ]; then + git clone --depth=1 "https://github.com/mongodb-labs/drivers-evergreen-tools.git" +fi + +export DRIVERS_TOOLS=$CWD/encrypted-cluster/drivers-evergreen-tools +export MONGODB_VERSION=8.0 +export AUTH=true +export MONGODB_BINARIES=$DRIVERS_TOOLS/mongodb/bin +export NODE_DRIVER=~/dev/node-mongodb-native +export MONGO_ORCHESTRATION_HOME=$DRIVERS_TOOLS/mo +export PROJECT_ORCHESTRATION_HOME=$DRIVERS_TOOLS/.evergreen/orchestration +export TOPOLOGY=sharded_cluster +export SSL=nossl + +cd $DRIVERS_TOOLS +rm -rf mongosh mongodb mo +mkdir mo +cd - + +rm expansions.sh 2> /dev/null + +bash $DRIVERS_TOOLS/.evergreen/run-orchestration.sh \ No newline at end of file From 62d18d830a74d24218b24cef26253317c38a2086 Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Tue, 10 Dec 2024 17:43:43 -0500 Subject: [PATCH 24/33] fixed deps --- CONTRIBUTING.md | 2 +- package.json | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 086e2734a2..97f084447d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,7 +46,7 @@ If you have a question about Mongoose (not a bug report) please post it to eithe * execute `npm run test-tsd` to run the typescript tests * execute `npm run ts-benchmark` to run the typescript benchmark "performance test" for a single time. * execute `npm run ts-benchmark-watch` to run the typescript benchmark "performance test" while watching changes on types folder. Note: Make sure to commit all changes before executing this command. -* in order to run tests that require an encrypted cluster locally, run `npm run test-encryption-local`. Alternatively, you can start an encrypted cluster using the `scripts/start-encrypted-cluster.sh` file +* in order to run tests that require an encrypted cluster locally, run `npm run test-encryption-local`. Alternatively, you can start an encrypted cluster using the `scripts/start-encrypted-cluster.sh` file. ## Documentation diff --git a/package.json b/package.json index 06adb0af08..0d0fe6f99c 100644 --- a/package.json +++ b/package.json @@ -20,10 +20,8 @@ "license": "MIT", "dependencies": { "bson": "^6.7.0", - "hugo": "^0.0.3", "kareem": "2.6.3", "mongodb": "~6.10.0", - "mongodb-client-encryption": "^6.1.0", "mpath": "0.9.0", "mquery": "5.0.0", "ms": "2.1.3", From 40858d490b529dac1eec8b3e4d69fcb98e9e6b49 Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Wed, 11 Dec 2024 14:25:03 -0500 Subject: [PATCH 25/33] requested changes --- .eslintrc.js | 3 +- mongocryptd.pid | 0 package.json | 2 +- scripts/run-encryption-tests-local.sh | 8 +- scripts/start-encrypted-cluster.sh | 13 ++- test/encryption/encryption.test.js | 139 +++++++++++++------------- 6 files changed, 88 insertions(+), 77 deletions(-) create mode 100644 mongocryptd.pid diff --git a/.eslintrc.js b/.eslintrc.js index b4d7d1652d..7670926d24 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -14,7 +14,8 @@ module.exports = { '**/docs/js/native.js', '!.*', 'node_modules', - '.git' + '.git', + 'encrypted-cluster' ], overrides: [ { diff --git a/mongocryptd.pid b/mongocryptd.pid new file mode 100644 index 0000000000..e69de29bb2 diff --git a/package.json b/package.json index 0d0fe6f99c..5c726e144c 100644 --- a/package.json +++ b/package.json @@ -105,7 +105,7 @@ "test-rs": "START_REPLICA_SET=1 mocha --timeout 30000 --exit ./test/*.test.js", "test-tsd": "node ./test/types/check-types-filename && tsd", "test-encryption": "mocha --exit ./test/encryption/*.test.js", - "test-encryption-local": "chmod +x scripts/run-encryption-tests-local.sh && scripts/run-encryption-tests-local.sh", + "test-encryption-local": "bash scripts/run-encryption-tests-local.sh", "tdd": "mocha ./test/*.test.js --inspect --watch --recursive --watch-files ./**/*.{js,ts}", "test-coverage": "nyc --reporter=html --reporter=text npm test", "ts-benchmark": "cd ./benchmarks/typescript/simple && npm install && npm run benchmark | node ../../../scripts/tsc-diagnostics-check" diff --git a/scripts/run-encryption-tests-local.sh b/scripts/run-encryption-tests-local.sh index dd29eb9880..ff4332370b 100755 --- a/scripts/run-encryption-tests-local.sh +++ b/scripts/run-encryption-tests-local.sh @@ -1,17 +1,18 @@ #!/usr/bin/env bash -# sets up an encrypted mongodb cluster +# sets up an encrypted mongodb cluster, adds relevant variables to the environment, and runs encryption tests export CWD=$(pwd); +# set up encrypted mongodb cluster if the encrypted-cluster folder does not exist +# note: for tooling, cluster set-up and configuration look into the 'scripts/start-encrypted-cluster.sh' script if [ -d "encrypted-cluster" ]; then cd encrypted-cluster else source $CWD/scripts/start-encrypted-cluster.sh fi -# IMPORTANT: extracts mongodb-uri, and starts the cluster of servers, store the uri for GitHub output - +# extracts MONGOOSE_TEST_URI and CRYPT_SHARED_LIB_PATH from .yml file into environment variables for this test run read -r -d '' SOURCE_SCRIPT << EOM const fs = require('fs'); const file = fs.readFileSync('mo-expansion.yml', { encoding: 'utf-8' }) @@ -32,4 +33,5 @@ source expansions.sh export MONGOOSE_TEST_URI=$MONGODB_URI +# run encryption tests npm run test-encryption diff --git a/scripts/start-encrypted-cluster.sh b/scripts/start-encrypted-cluster.sh index b04c4b655c..1fb8a4aa22 100644 --- a/scripts/start-encrypted-cluster.sh +++ b/scripts/start-encrypted-cluster.sh @@ -1,17 +1,25 @@ +# creates a encrypted cluster (sharded on 8.0 server) export CWD=$(pwd); mkdir encrypted-cluster cd encrypted-cluster +# note: + # we're using drivers-evergreen-tools which is a repo that handles cluster set-up for us. + # if you'd like to make changes to the cluster settings, edit the exported variables below. + # for configuration options for the exported variables, see here: https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/run-orchestration.sh + # after this script is run, the encrypted-cluster/ folder will notably contain the following: + # 'mo-expansion.yml' file which contains for your cluster URI and crypt shared library path + # 'drivers-evergreen-tools/mongodb/bin' which contain executables for other mongodb libraries such as mongocryptd, mongosh, and mongod if [ ! -d "drivers-evergreen-tools/" ]; then git clone --depth=1 "https://github.com/mongodb-labs/drivers-evergreen-tools.git" fi +# configure cluster settings export DRIVERS_TOOLS=$CWD/encrypted-cluster/drivers-evergreen-tools export MONGODB_VERSION=8.0 export AUTH=true export MONGODB_BINARIES=$DRIVERS_TOOLS/mongodb/bin -export NODE_DRIVER=~/dev/node-mongodb-native export MONGO_ORCHESTRATION_HOME=$DRIVERS_TOOLS/mo export PROJECT_ORCHESTRATION_HOME=$DRIVERS_TOOLS/.evergreen/orchestration export TOPOLOGY=sharded_cluster @@ -24,4 +32,5 @@ cd - rm expansions.sh 2> /dev/null -bash $DRIVERS_TOOLS/.evergreen/run-orchestration.sh \ No newline at end of file +# start cluster +bash $DRIVERS_TOOLS/.evergreen/run-orchestration.sh diff --git a/test/encryption/encryption.test.js b/test/encryption/encryption.test.js index ba29506965..14e18306d9 100644 --- a/test/encryption/encryption.test.js +++ b/test/encryption/encryption.test.js @@ -6,92 +6,91 @@ const isBsonType = require('../../lib/helpers/isBsonType'); const LOCAL_KEY = Buffer.from('Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk', 'base64'); -describe('environmental variables', () => { - it('MONGOOSE_TEST_URI is set', async function() { - const uri = process.env.MONGOOSE_TEST_URI; - assert.ok(uri); - }); +describe('ci', () => { + describe('environmental variables', () => { + it('MONGOOSE_TEST_URI is set', async function() { + const uri = process.env.MONGOOSE_TEST_URI; + assert.ok(uri); + }); - it('CRYPT_SHARED_LIB_PATH is set', async function() { - const shared_library_path = process.env.CRYPT_SHARED_LIB_PATH; - assert.ok(shared_library_path); + it('CRYPT_SHARED_LIB_PATH is set', async function() { + const shared_library_path = process.env.CRYPT_SHARED_LIB_PATH; + assert.ok(shared_library_path); + }); }); -}); -describe('basic integration', () => { - let keyVaultClient; - let dataKey; - let encryptedClient; - let dummyClient; + describe('basic integration', () => { + let keyVaultClient; + let dataKey; + let encryptedClient; + let unencryptedClient; - beforeEach(async function() { - keyVaultClient = new mdb.MongoClient(process.env.MONGOOSE_TEST_URI); - await keyVaultClient.connect(); - await keyVaultClient.db('keyvault').collection('datakeys'); - const clientEncryption = new mdb.ClientEncryption(keyVaultClient, { - keyVaultNamespace: 'keyvault.datakeys', - kmsProviders: { local: { key: LOCAL_KEY } } - }); - dataKey = await clientEncryption.createDataKey('local'); + beforeEach(async function() { + keyVaultClient = new mdb.MongoClient(process.env.MONGOOSE_TEST_URI); + await keyVaultClient.connect(); + await keyVaultClient.db('keyvault').collection('datakeys'); + const clientEncryption = new mdb.ClientEncryption(keyVaultClient, { + keyVaultNamespace: 'keyvault.datakeys', + kmsProviders: { local: { key: LOCAL_KEY } } + }); + dataKey = await clientEncryption.createDataKey('local'); - encryptedClient = new mdb.MongoClient( - process.env.MONGOOSE_TEST_URI, - { - autoEncryption: { - keyVaultNamespace: 'keyvault.datakeys', - kmsProviders: { local: { key: LOCAL_KEY } }, - schemaMap: { - 'db.coll': { - bsonType: 'object', - encryptMetadata: { - keyId: [dataKey] - }, - properties: { - a: { - encrypt: { - bsonType: 'int', - algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Random', - keyId: [dataKey] + encryptedClient = new mdb.MongoClient( + process.env.MONGOOSE_TEST_URI, + { + autoEncryption: { + keyVaultNamespace: 'keyvault.datakeys', + kmsProviders: { local: { key: LOCAL_KEY } }, + schemaMap: { + 'db.coll': { + bsonType: 'object', + encryptMetadata: { + keyId: [dataKey] + }, + properties: { + a: { + encrypt: { + bsonType: 'int', + algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Random', + keyId: [dataKey] + } } } } + }, + extraOptions: { + cryptdSharedLibRequired: true, + cryptSharedLibPath: process.env.CRYPT_SHARED_LIB_PATH } - }, - extraOptions: { - cryptdSharedLibRequired: true, - cryptSharedLibPath: process.env.CRYPT_SHARED_LIB_PATH } } - } - ); + ); - dummyClient = new mdb.MongoClient(process.env.MONGOOSE_TEST_URI); - }); - - afterEach(async function() { - await keyVaultClient.close(); - await encryptedClient.close(); - await dummyClient.close(); - }); + unencryptedClient = new mdb.MongoClient(process.env.MONGOOSE_TEST_URI); + }); - it('supports mongodb csfle auto-encryption integration', async() => { - await encryptedClient.connect(); - await encryptedClient.db('db').collection('coll').insertOne({ a: 1 }); + afterEach(async function() { + await keyVaultClient.close(); + await encryptedClient.close(); + await unencryptedClient.close(); + }); - const { insertedId } = await encryptedClient.db('db').collection('coll').insertOne({ a: 1 }); + it('ci set-up should support basic mongodb auto-encryption integration', async() => { + await encryptedClient.connect(); + const { insertedId } = await encryptedClient.db('db').collection('coll').insertOne({ a: 1 }); - // a dummyClient not configured with autoEncryption, returns a encrypted binary type, meaning that encryption succeeded - const encryptedResult = await dummyClient.db('db').collection('coll').findOne({ _id: insertedId }); + // client not configured with autoEncryption, returns a encrypted binary type, meaning that encryption succeeded + const encryptedResult = await unencryptedClient.db('db').collection('coll').findOne({ _id: insertedId }); - assert.ok(encryptedResult); - assert.ok(encryptedResult.a); - assert.ok(isBsonType(encryptedResult.a, 'Binary')); - assert.ok(encryptedResult.a.sub_type === 6); + assert.ok(encryptedResult); + assert.ok(encryptedResult.a); + assert.ok(isBsonType(encryptedResult.a, 'Binary')); + assert.ok(encryptedResult.a.sub_type === 6); - // when the encryptedClient runs a find, the original unencrypted value is returned - const unencryptedCursor = await encryptedClient.db('db').collection('coll').find(); - const unencryptedResult = await unencryptedCursor.next(); - assert.ok(unencryptedResult); - assert.ok(unencryptedResult.a === 1); + // when the encryptedClient runs a find, the original unencrypted value is returned + const unencryptedResult = await encryptedClient.db('db').collection('coll').findOne({ _id: insertedId }); + assert.ok(unencryptedResult); + assert.ok(unencryptedResult.a === 1); + }); }); }); From d6044e79358fd574a1b7190b593f7452d614069b Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Wed, 11 Dec 2024 15:28:33 -0500 Subject: [PATCH 26/33] fix --- mongocryptd.pid | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 mongocryptd.pid diff --git a/mongocryptd.pid b/mongocryptd.pid deleted file mode 100644 index e69de29bb2..0000000000 From 08f4c2345a89a9d70dbe089b05f56a38a809cd40 Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Wed, 18 Dec 2024 10:36:05 -0500 Subject: [PATCH 27/33] fix wording --- .gitignore | 1 + CONTRIBUTING.md | 2 +- ...ed-cluster.sh => configure-cluster-with-encryption.sh} | 8 ++++++-- scripts/run-encryption-tests-local.sh | 8 ++++---- 4 files changed, 12 insertions(+), 7 deletions(-) rename scripts/{start-encrypted-cluster.sh => configure-cluster-with-encryption.sh} (76%) diff --git a/.gitignore b/.gitignore index 88f5a87efb..e78311260a 100644 --- a/.gitignore +++ b/.gitignore @@ -69,3 +69,4 @@ notes.md list.out encrypted-cluster +*.pid \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 97f084447d..a94baf554b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,7 +46,7 @@ If you have a question about Mongoose (not a bug report) please post it to eithe * execute `npm run test-tsd` to run the typescript tests * execute `npm run ts-benchmark` to run the typescript benchmark "performance test" for a single time. * execute `npm run ts-benchmark-watch` to run the typescript benchmark "performance test" while watching changes on types folder. Note: Make sure to commit all changes before executing this command. -* in order to run tests that require an encrypted cluster locally, run `npm run test-encryption-local`. Alternatively, you can start an encrypted cluster using the `scripts/start-encrypted-cluster.sh` file. +* in order to run tests that require an cluster with encryption locally, run `npm run test-encryption-local`. Alternatively, you can start an encrypted cluster using the `scripts/start-cluster-with-encryption.sh` file. ## Documentation diff --git a/scripts/start-encrypted-cluster.sh b/scripts/configure-cluster-with-encryption.sh similarity index 76% rename from scripts/start-encrypted-cluster.sh rename to scripts/configure-cluster-with-encryption.sh index 1fb8a4aa22..1a77feee78 100644 --- a/scripts/start-encrypted-cluster.sh +++ b/scripts/configure-cluster-with-encryption.sh @@ -1,11 +1,15 @@ -# creates a encrypted cluster (sharded on 8.0 server) +# note: in order to use FLE with mongodb, we must +# have mongocryptd or the shared library downloaded +# have an enterprise server >= 4.2 + +# this script downloads all tools required to use FLE with mongodb, then starts a cluster of the provided configuration (sharded on 8.0 server) export CWD=$(pwd); mkdir encrypted-cluster cd encrypted-cluster # note: - # we're using drivers-evergreen-tools which is a repo that handles cluster set-up for us. + # we're using drivers-evergreen-tools which is a repo used by MongoDB drivers to start clusters for testing. # if you'd like to make changes to the cluster settings, edit the exported variables below. # for configuration options for the exported variables, see here: https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/run-orchestration.sh # after this script is run, the encrypted-cluster/ folder will notably contain the following: diff --git a/scripts/run-encryption-tests-local.sh b/scripts/run-encryption-tests-local.sh index ff4332370b..1b83c32403 100755 --- a/scripts/run-encryption-tests-local.sh +++ b/scripts/run-encryption-tests-local.sh @@ -1,15 +1,15 @@ #!/usr/bin/env bash -# sets up an encrypted mongodb cluster, adds relevant variables to the environment, and runs encryption tests +# sets up mongodb cluster and encryption configuration, adds relevant variables to the environment, and runs encryption tests export CWD=$(pwd); -# set up encrypted mongodb cluster if the encrypted-cluster folder does not exist -# note: for tooling, cluster set-up and configuration look into the 'scripts/start-encrypted-cluster.sh' script +# set up mongodb cluster and encryption configuration if the encrypted-cluster folder does not exist +# note: for tooling, cluster set-up and configuration look into the 'scripts/start-cluster-with-encryption.sh' script if [ -d "encrypted-cluster" ]; then cd encrypted-cluster else - source $CWD/scripts/start-encrypted-cluster.sh + source $CWD/scripts/start-cluster-with-encryption.sh fi # extracts MONGOOSE_TEST_URI and CRYPT_SHARED_LIB_PATH from .yml file into environment variables for this test run From 1098636943915c39c2f381aa69d071b565fd85e2 Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Thu, 26 Dec 2024 13:00:42 -0500 Subject: [PATCH 28/33] change all occurences of encrypted-cluster to data --- .eslintrc.js | 2 +- .gitignore | 2 +- scripts/configure-cluster-with-encryption.sh | 8 ++++---- scripts/run-encryption-tests-local.sh | 11 ++++++----- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/.eslintrc.js b/.eslintrc.js index 7670926d24..4b4f2f312c 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -15,7 +15,7 @@ module.exports = { '!.*', 'node_modules', '.git', - 'encrypted-cluster' + 'data' ], overrides: [ { diff --git a/.gitignore b/.gitignore index e78311260a..c66816fcd4 100644 --- a/.gitignore +++ b/.gitignore @@ -68,5 +68,5 @@ examples/ecommerce-netlify-functions/.netlify/state.json notes.md list.out -encrypted-cluster +data *.pid \ No newline at end of file diff --git a/scripts/configure-cluster-with-encryption.sh b/scripts/configure-cluster-with-encryption.sh index 1a77feee78..4584920ed4 100644 --- a/scripts/configure-cluster-with-encryption.sh +++ b/scripts/configure-cluster-with-encryption.sh @@ -5,14 +5,14 @@ # this script downloads all tools required to use FLE with mongodb, then starts a cluster of the provided configuration (sharded on 8.0 server) export CWD=$(pwd); -mkdir encrypted-cluster -cd encrypted-cluster +mkdir data +cd data # note: # we're using drivers-evergreen-tools which is a repo used by MongoDB drivers to start clusters for testing. # if you'd like to make changes to the cluster settings, edit the exported variables below. # for configuration options for the exported variables, see here: https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/run-orchestration.sh - # after this script is run, the encrypted-cluster/ folder will notably contain the following: + # after this script is run, the data/ folder will notably contain the following: # 'mo-expansion.yml' file which contains for your cluster URI and crypt shared library path # 'drivers-evergreen-tools/mongodb/bin' which contain executables for other mongodb libraries such as mongocryptd, mongosh, and mongod if [ ! -d "drivers-evergreen-tools/" ]; then @@ -20,7 +20,7 @@ if [ ! -d "drivers-evergreen-tools/" ]; then fi # configure cluster settings -export DRIVERS_TOOLS=$CWD/encrypted-cluster/drivers-evergreen-tools +export DRIVERS_TOOLS=$CWD/data/drivers-evergreen-tools export MONGODB_VERSION=8.0 export AUTH=true export MONGODB_BINARIES=$DRIVERS_TOOLS/mongodb/bin diff --git a/scripts/run-encryption-tests-local.sh b/scripts/run-encryption-tests-local.sh index 1b83c32403..244c125459 100755 --- a/scripts/run-encryption-tests-local.sh +++ b/scripts/run-encryption-tests-local.sh @@ -4,12 +4,13 @@ export CWD=$(pwd); -# set up mongodb cluster and encryption configuration if the encrypted-cluster folder does not exist -# note: for tooling, cluster set-up and configuration look into the 'scripts/start-cluster-with-encryption.sh' script -if [ -d "encrypted-cluster" ]; then - cd encrypted-cluster +# set up mongodb cluster and encryption configuration if the data/ folder does not exist +# note: for tooling, cluster set-up and configuration look into the 'scripts/configure-cluster-with-encryption.sh' script + +if [ -d "data" ]; then + cd data else - source $CWD/scripts/start-cluster-with-encryption.sh + source $CWD/scripts/configure-cluster-with-encryption.sh fi # extracts MONGOOSE_TEST_URI and CRYPT_SHARED_LIB_PATH from .yml file into environment variables for this test run From 955cedf05b0db8cff1f72165c540a4c353d4efa0 Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Fri, 27 Dec 2024 13:22:00 -0500 Subject: [PATCH 29/33] remove extra gha call - use local script instead lint typo lint --- .github/workflows/encryption-tests.yml | 10 ---------- .gitignore | 2 +- CONTRIBUTING.md | 2 +- package.json | 3 +-- ...cryption-tests-local.sh => run-encryption-tests.sh} | 3 ++- 5 files changed, 5 insertions(+), 15 deletions(-) rename scripts/{run-encryption-tests-local.sh => run-encryption-tests.sh} (95%) diff --git a/.github/workflows/encryption-tests.yml b/.github/workflows/encryption-tests.yml index e605778920..263ebaedc1 100644 --- a/.github/workflows/encryption-tests.yml +++ b/.github/workflows/encryption-tests.yml @@ -33,15 +33,5 @@ jobs: run: npm install - name: Install mongodb-client-encryption run: npm install mongodb-client-encryption - - name: Set up cluster - id: setup-cluster - uses: mongodb-labs/drivers-evergreen-tools@master - with: - version: 8.0.0 - topology: sharded_cluster - auth: auth - name: Run Tests run: npm run test-encryption - env: - MONGOOSE_TEST_URI: ${{ steps.setup-cluster.outputs.cluster-uri }} - CRYPT_SHARED_LIB_PATH: ${{ steps.setup-cluster.outputs.crypt-shared-lib-path }} diff --git a/.gitignore b/.gitignore index c66816fcd4..9a52110981 100644 --- a/.gitignore +++ b/.gitignore @@ -69,4 +69,4 @@ notes.md list.out data -*.pid \ No newline at end of file +*.pid diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a94baf554b..06073758d9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,7 +46,7 @@ If you have a question about Mongoose (not a bug report) please post it to eithe * execute `npm run test-tsd` to run the typescript tests * execute `npm run ts-benchmark` to run the typescript benchmark "performance test" for a single time. * execute `npm run ts-benchmark-watch` to run the typescript benchmark "performance test" while watching changes on types folder. Note: Make sure to commit all changes before executing this command. -* in order to run tests that require an cluster with encryption locally, run `npm run test-encryption-local`. Alternatively, you can start an encrypted cluster using the `scripts/start-cluster-with-encryption.sh` file. +* in order to run tests that require an cluster with encryption locally, run `npm run test-encryption`. Alternatively, you can start an encrypted cluster using the `scripts/configure-cluster-with-encryption.sh` file. ## Documentation diff --git a/package.json b/package.json index 5c726e144c..39d4451eeb 100644 --- a/package.json +++ b/package.json @@ -104,8 +104,7 @@ "test-deno": "deno run --allow-env --allow-read --allow-net --allow-run --allow-sys --allow-write ./test/deno.js", "test-rs": "START_REPLICA_SET=1 mocha --timeout 30000 --exit ./test/*.test.js", "test-tsd": "node ./test/types/check-types-filename && tsd", - "test-encryption": "mocha --exit ./test/encryption/*.test.js", - "test-encryption-local": "bash scripts/run-encryption-tests-local.sh", + "test-encryption": "bash scripts/run-encryption-tests.sh", "tdd": "mocha ./test/*.test.js --inspect --watch --recursive --watch-files ./**/*.{js,ts}", "test-coverage": "nyc --reporter=html --reporter=text npm test", "ts-benchmark": "cd ./benchmarks/typescript/simple && npm install && npm run benchmark | node ../../../scripts/tsc-diagnostics-check" diff --git a/scripts/run-encryption-tests-local.sh b/scripts/run-encryption-tests.sh similarity index 95% rename from scripts/run-encryption-tests-local.sh rename to scripts/run-encryption-tests.sh index 244c125459..0209292168 100755 --- a/scripts/run-encryption-tests-local.sh +++ b/scripts/run-encryption-tests.sh @@ -35,4 +35,5 @@ source expansions.sh export MONGOOSE_TEST_URI=$MONGODB_URI # run encryption tests -npm run test-encryption +cd .. +npx mocha --exit ./test/encryption/*.test.js From 464107c7ccf8d7dbf111ce1a795e011a42f27898 Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Thu, 9 Jan 2025 17:08:37 -0500 Subject: [PATCH 30/33] most of Val's comments addressed --- .github/workflows/encryption-tests.yml | 2 -- CONTRIBUTING.md | 3 +- scripts/configure-cluster-with-encryption.sh | 3 ++ scripts/run-encryption-tests.sh | 28 ++++--------------- test/encryption/encryption.test.js | 29 ++++++++++++++++---- 5 files changed, 35 insertions(+), 30 deletions(-) diff --git a/.github/workflows/encryption-tests.yml b/.github/workflows/encryption-tests.yml index 263ebaedc1..521aae66ab 100644 --- a/.github/workflows/encryption-tests.yml +++ b/.github/workflows/encryption-tests.yml @@ -31,7 +31,5 @@ jobs: node-version: latest - name: Install Dependencies run: npm install - - name: Install mongodb-client-encryption - run: npm install mongodb-client-encryption - name: Run Tests run: npm run test-encryption diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 06073758d9..103d03a6ef 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,7 +46,8 @@ If you have a question about Mongoose (not a bug report) please post it to eithe * execute `npm run test-tsd` to run the typescript tests * execute `npm run ts-benchmark` to run the typescript benchmark "performance test" for a single time. * execute `npm run ts-benchmark-watch` to run the typescript benchmark "performance test" while watching changes on types folder. Note: Make sure to commit all changes before executing this command. -* in order to run tests that require an cluster with encryption locally, run `npm run test-encryption`. Alternatively, you can start an encrypted cluster using the `scripts/configure-cluster-with-encryption.sh` file. +* in order to run tests that require an cluster with encryption locally, run `npm run test-encryption`. Alternatively, you can start an encrypted cluster using the `scripts/configure-cluster-with-encryption.sh` file. +* These scripts can take a few minutes to run. If a encryption script is exited prematurely, restart the shell and delete the `data/` directory to ensure clean-up. ## Documentation diff --git a/scripts/configure-cluster-with-encryption.sh b/scripts/configure-cluster-with-encryption.sh index 4584920ed4..9c4adf330b 100644 --- a/scripts/configure-cluster-with-encryption.sh +++ b/scripts/configure-cluster-with-encryption.sh @@ -8,6 +8,9 @@ export CWD=$(pwd); mkdir data cd data +# install encryption dependency +npm install mongodb-client-encryption > /dev/null + # note: # we're using drivers-evergreen-tools which is a repo used by MongoDB drivers to start clusters for testing. # if you'd like to make changes to the cluster settings, edit the exported variables below. diff --git a/scripts/run-encryption-tests.sh b/scripts/run-encryption-tests.sh index 0209292168..60b7dfae24 100755 --- a/scripts/run-encryption-tests.sh +++ b/scripts/run-encryption-tests.sh @@ -4,36 +4,20 @@ export CWD=$(pwd); +# install encryption dependency +npm install mongodb-client-encryption > /dev/null + # set up mongodb cluster and encryption configuration if the data/ folder does not exist # note: for tooling, cluster set-up and configuration look into the 'scripts/configure-cluster-with-encryption.sh' script - if [ -d "data" ]; then cd data else source $CWD/scripts/configure-cluster-with-encryption.sh fi -# extracts MONGOOSE_TEST_URI and CRYPT_SHARED_LIB_PATH from .yml file into environment variables for this test run -read -r -d '' SOURCE_SCRIPT << EOM -const fs = require('fs'); -const file = fs.readFileSync('mo-expansion.yml', { encoding: 'utf-8' }) - .trim().split('\\n'); -const regex = /^(?.*): "(?.*)"$/; -const variables = file.map( - (line) => regex.exec(line.trim()).groups -).map( - ({key, value}) => \`export \${key}='\${value}'\` -).join('\n'); - -process.stdout.write(variables); -process.stdout.write('\n'); -EOM - -node --eval "$SOURCE_SCRIPT" | tee expansions.sh -source expansions.sh - -export MONGOOSE_TEST_URI=$MONGODB_URI - # run encryption tests cd .. npx mocha --exit ./test/encryption/*.test.js + +# uninstall encryption dependency +npm uninstall mongodb-client-encryption > /dev/null \ No newline at end of file diff --git a/test/encryption/encryption.test.js b/test/encryption/encryption.test.js index 14e18306d9..a3b562e80a 100644 --- a/test/encryption/encryption.test.js +++ b/test/encryption/encryption.test.js @@ -1,12 +1,31 @@ 'use strict'; const assert = require('assert'); -const mdb = require('mongodb'); +const mongodb = require('mongodb'); +const fs = require('fs'); const isBsonType = require('../../lib/helpers/isBsonType'); const LOCAL_KEY = Buffer.from('Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk', 'base64'); describe('ci', () => { + + const cachedUri = process.env.MONGOOSE_TEST_URI; + const cachedLib = process.env.CRYPT_SHARED_LIB_PATH; + + before(function() { + const cwd = process.cwd(); + const file = fs.readFileSync(cwd + '/data/mo-expansion.yml', { encoding: 'utf-8' }).trim().split('\n'); + const regex = /^(?.*): "(?.*)"$/; + const variables = file.map((line) => regex.exec(line.trim()).groups).reduce((acc, { key, value }) => ({ ...acc, [key]: value }), {}); + process.env.CRYPT_SHARED_LIB_PATH = variables.CRYPT_SHARED_LIB_PATH; + process.env.MONGOOSE_TEST_URI = variables.MONGODB_URI; + }); + + after(function() { + process.env.CRYPT_SHARED_LIB_PATH = cachedLib; + process.env.MONGOOSE_TEST_URI = cachedUri; + }); + describe('environmental variables', () => { it('MONGOOSE_TEST_URI is set', async function() { const uri = process.env.MONGOOSE_TEST_URI; @@ -26,16 +45,16 @@ describe('ci', () => { let unencryptedClient; beforeEach(async function() { - keyVaultClient = new mdb.MongoClient(process.env.MONGOOSE_TEST_URI); + keyVaultClient = new mongodb.MongoClient(process.env.MONGOOSE_TEST_URI); await keyVaultClient.connect(); await keyVaultClient.db('keyvault').collection('datakeys'); - const clientEncryption = new mdb.ClientEncryption(keyVaultClient, { + const clientEncryption = new mongodb.ClientEncryption(keyVaultClient, { keyVaultNamespace: 'keyvault.datakeys', kmsProviders: { local: { key: LOCAL_KEY } } }); dataKey = await clientEncryption.createDataKey('local'); - encryptedClient = new mdb.MongoClient( + encryptedClient = new mongodb.MongoClient( process.env.MONGOOSE_TEST_URI, { autoEncryption: { @@ -66,7 +85,7 @@ describe('ci', () => { } ); - unencryptedClient = new mdb.MongoClient(process.env.MONGOOSE_TEST_URI); + unencryptedClient = new mongodb.MongoClient(process.env.MONGOOSE_TEST_URI); }); afterEach(async function() { From 06ffdead0036e6648bbe8d425f0fe73f671a6cd2 Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Thu, 9 Jan 2025 17:29:47 -0500 Subject: [PATCH 31/33] comments addressed readd dependency readd dependency lint readd dependency readd dependency fix dep fix dep2 fix dep2 fix dep3 fix dep4 fix dep5 fix dep6 --- .github/workflows/encryption-tests.yml | 2 + CONTRIBUTING.md | 9 ++- package.json | 2 +- scripts/configure-cluster-with-encryption.sh | 83 +++++++++++--------- scripts/run-encryption-tests.sh | 23 ------ 5 files changed, 57 insertions(+), 62 deletions(-) delete mode 100755 scripts/run-encryption-tests.sh diff --git a/.github/workflows/encryption-tests.yml b/.github/workflows/encryption-tests.yml index 521aae66ab..263ebaedc1 100644 --- a/.github/workflows/encryption-tests.yml +++ b/.github/workflows/encryption-tests.yml @@ -31,5 +31,7 @@ jobs: node-version: latest - name: Install Dependencies run: npm install + - name: Install mongodb-client-encryption + run: npm install mongodb-client-encryption - name: Run Tests run: npm run test-encryption diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 103d03a6ef..18e99b6657 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,8 +46,13 @@ If you have a question about Mongoose (not a bug report) please post it to eithe * execute `npm run test-tsd` to run the typescript tests * execute `npm run ts-benchmark` to run the typescript benchmark "performance test" for a single time. * execute `npm run ts-benchmark-watch` to run the typescript benchmark "performance test" while watching changes on types folder. Note: Make sure to commit all changes before executing this command. -* in order to run tests that require an cluster with encryption locally, run `npm run test-encryption`. Alternatively, you can start an encrypted cluster using the `scripts/configure-cluster-with-encryption.sh` file. -* These scripts can take a few minutes to run. If a encryption script is exited prematurely, restart the shell and delete the `data/` directory to ensure clean-up. +* in order to run tests that require an cluster with encryption locally, run `npm run test-encryption`/ Alternatively, you can start an encrypted cluster using the `scripts/configure-cluster-with-encryption.sh` file. + * These scripts can take a few minutes to run. + * To change an encryption configuration, it is recommended to follow these steps: + * Edit the variables in `scripts/configure-cluster-with-encryption.sh` with your desired configuration. + * Restart your shell. + * Delete the `data/` directory if it exists. + * Finally, run the configuration script. ## Documentation diff --git a/package.json b/package.json index 39d4451eeb..cdf2556c7c 100644 --- a/package.json +++ b/package.json @@ -104,7 +104,7 @@ "test-deno": "deno run --allow-env --allow-read --allow-net --allow-run --allow-sys --allow-write ./test/deno.js", "test-rs": "START_REPLICA_SET=1 mocha --timeout 30000 --exit ./test/*.test.js", "test-tsd": "node ./test/types/check-types-filename && tsd", - "test-encryption": "bash scripts/run-encryption-tests.sh", + "test-encryption": "bash scripts/configure-cluster-with-encryption.sh && mocha --exit ./test/encryption/*.test.js && npm uninstall mongodb-client-encryption > /dev/null", "tdd": "mocha ./test/*.test.js --inspect --watch --recursive --watch-files ./**/*.{js,ts}", "test-coverage": "nyc --reporter=html --reporter=text npm test", "ts-benchmark": "cd ./benchmarks/typescript/simple && npm install && npm run benchmark | node ../../../scripts/tsc-diagnostics-check" diff --git a/scripts/configure-cluster-with-encryption.sh b/scripts/configure-cluster-with-encryption.sh index 9c4adf330b..8f366bc4bb 100644 --- a/scripts/configure-cluster-with-encryption.sh +++ b/scripts/configure-cluster-with-encryption.sh @@ -5,39 +5,50 @@ # this script downloads all tools required to use FLE with mongodb, then starts a cluster of the provided configuration (sharded on 8.0 server) export CWD=$(pwd); -mkdir data -cd data - -# install encryption dependency -npm install mongodb-client-encryption > /dev/null - -# note: - # we're using drivers-evergreen-tools which is a repo used by MongoDB drivers to start clusters for testing. - # if you'd like to make changes to the cluster settings, edit the exported variables below. - # for configuration options for the exported variables, see here: https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/run-orchestration.sh - # after this script is run, the data/ folder will notably contain the following: - # 'mo-expansion.yml' file which contains for your cluster URI and crypt shared library path - # 'drivers-evergreen-tools/mongodb/bin' which contain executables for other mongodb libraries such as mongocryptd, mongosh, and mongod -if [ ! -d "drivers-evergreen-tools/" ]; then - git clone --depth=1 "https://github.com/mongodb-labs/drivers-evergreen-tools.git" -fi - -# configure cluster settings -export DRIVERS_TOOLS=$CWD/data/drivers-evergreen-tools -export MONGODB_VERSION=8.0 -export AUTH=true -export MONGODB_BINARIES=$DRIVERS_TOOLS/mongodb/bin -export MONGO_ORCHESTRATION_HOME=$DRIVERS_TOOLS/mo -export PROJECT_ORCHESTRATION_HOME=$DRIVERS_TOOLS/.evergreen/orchestration -export TOPOLOGY=sharded_cluster -export SSL=nossl - -cd $DRIVERS_TOOLS -rm -rf mongosh mongodb mo -mkdir mo -cd - - -rm expansions.sh 2> /dev/null - -# start cluster -bash $DRIVERS_TOOLS/.evergreen/run-orchestration.sh + +# install extra dependency +npm install mongodb-client-encryption + +# set up mongodb cluster and encryption configuration if the data/ folder does not exist +if [ ! -d "data" ]; then + + mkdir data + cd data + + # note: + # we're using drivers-evergreen-tools which is a repo used by MongoDB drivers to start clusters for testing. + # if you'd like to make changes to the cluster settings, edit the exported variables below. + # for configuration options for the exported variables, see here: https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/run-orchestration.sh + # after this script is run, the data/ folder will notably contain the following: + # 'mo-expansion.yml' file which contains for your cluster URI and crypt shared library path + # 'drivers-evergreen-tools/mongodb/bin' which contain executables for other mongodb libraries such as mongocryptd, mongosh, and mongod + if [ ! -d "drivers-evergreen-tools/" ]; then + git clone --depth=1 "https://github.com/mongodb-labs/drivers-evergreen-tools.git" + fi + + # configure cluster settings + export DRIVERS_TOOLS=$CWD/data/drivers-evergreen-tools + export MONGODB_VERSION=8.0 + export AUTH=true + export MONGODB_BINARIES=$DRIVERS_TOOLS/mongodb/bin + export MONGO_ORCHESTRATION_HOME=$DRIVERS_TOOLS/mo + export PROJECT_ORCHESTRATION_HOME=$DRIVERS_TOOLS/.evergreen/orchestration + export TOPOLOGY=sharded_cluster + export SSL=nossl + + cd $DRIVERS_TOOLS + rm -rf mongosh mongodb mo + mkdir mo + cd - + + rm expansions.sh 2> /dev/null + + echo 'Configuring Cluster...' + + # start cluster + (bash $DRIVERS_TOOLS/.evergreen/run-orchestration.sh) 1> /dev/null 2> /dev/null + + echo 'Cluster Configuration Finished!' + + cd .. +fi \ No newline at end of file diff --git a/scripts/run-encryption-tests.sh b/scripts/run-encryption-tests.sh deleted file mode 100755 index 60b7dfae24..0000000000 --- a/scripts/run-encryption-tests.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash - -# sets up mongodb cluster and encryption configuration, adds relevant variables to the environment, and runs encryption tests - -export CWD=$(pwd); - -# install encryption dependency -npm install mongodb-client-encryption > /dev/null - -# set up mongodb cluster and encryption configuration if the data/ folder does not exist -# note: for tooling, cluster set-up and configuration look into the 'scripts/configure-cluster-with-encryption.sh' script -if [ -d "data" ]; then - cd data -else - source $CWD/scripts/configure-cluster-with-encryption.sh -fi - -# run encryption tests -cd .. -npx mocha --exit ./test/encryption/*.test.js - -# uninstall encryption dependency -npm uninstall mongodb-client-encryption > /dev/null \ No newline at end of file From 7abbbcc2fbbc518da406cbde0bb7badb1eb76b00 Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Fri, 10 Jan 2025 10:05:42 -0500 Subject: [PATCH 32/33] change version to avert N-API bug --- .github/workflows/encryption-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/encryption-tests.yml b/.github/workflows/encryption-tests.yml index 263ebaedc1..d490320924 100644 --- a/.github/workflows/encryption-tests.yml +++ b/.github/workflows/encryption-tests.yml @@ -28,7 +28,7 @@ jobs: - name: Setup node uses: actions/setup-node@0a44ba7841725637a19e28fa30b79a866c81b0a6 # v4.0.4 with: - node-version: latest + node-version: 22 - name: Install Dependencies run: npm install - name: Install mongodb-client-encryption From da6d7e1334648dc5ad1edbfa73f90f75d0e93f17 Mon Sep 17 00:00:00 2001 From: Aditi Khare Date: Wed, 15 Jan 2025 10:50:33 -0500 Subject: [PATCH 33/33] add separate setup npm command --- .github/workflows/encryption-tests.yml | 2 ++ CONTRIBUTING.md | 2 +- package.json | 4 +++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/encryption-tests.yml b/.github/workflows/encryption-tests.yml index d490320924..3c0fbc65b8 100644 --- a/.github/workflows/encryption-tests.yml +++ b/.github/workflows/encryption-tests.yml @@ -33,5 +33,7 @@ jobs: run: npm install - name: Install mongodb-client-encryption run: npm install mongodb-client-encryption + - name: Setup Tests + run: npm run setup-test-encryption - name: Run Tests run: npm run test-encryption diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 18e99b6657..6efea5d26e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,7 +46,7 @@ If you have a question about Mongoose (not a bug report) please post it to eithe * execute `npm run test-tsd` to run the typescript tests * execute `npm run ts-benchmark` to run the typescript benchmark "performance test" for a single time. * execute `npm run ts-benchmark-watch` to run the typescript benchmark "performance test" while watching changes on types folder. Note: Make sure to commit all changes before executing this command. -* in order to run tests that require an cluster with encryption locally, run `npm run test-encryption`/ Alternatively, you can start an encrypted cluster using the `scripts/configure-cluster-with-encryption.sh` file. +* in order to run tests that require an cluster with encryption locally, run `npm run setup-test-encryption` followed by `npm run test-encryption`. Alternatively, you can start an encrypted cluster using the `scripts/configure-cluster-with-encryption.sh` file. * These scripts can take a few minutes to run. * To change an encryption configuration, it is recommended to follow these steps: * Edit the variables in `scripts/configure-cluster-with-encryption.sh` with your desired configuration. diff --git a/package.json b/package.json index cdf2556c7c..d8a1d01c4a 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "bson": "^6.7.0", "kareem": "2.6.3", "mongodb": "~6.10.0", + "mongodb-client-encryption": "^6.1.1", "mpath": "0.9.0", "mquery": "5.0.0", "ms": "2.1.3", @@ -104,7 +105,8 @@ "test-deno": "deno run --allow-env --allow-read --allow-net --allow-run --allow-sys --allow-write ./test/deno.js", "test-rs": "START_REPLICA_SET=1 mocha --timeout 30000 --exit ./test/*.test.js", "test-tsd": "node ./test/types/check-types-filename && tsd", - "test-encryption": "bash scripts/configure-cluster-with-encryption.sh && mocha --exit ./test/encryption/*.test.js && npm uninstall mongodb-client-encryption > /dev/null", + "setup-test-encryption": "bash scripts/configure-cluster-with-encryption.sh", + "test-encryption": "mocha --exit ./test/encryption/*.test.js", "tdd": "mocha ./test/*.test.js --inspect --watch --recursive --watch-files ./**/*.{js,ts}", "test-coverage": "nyc --reporter=html --reporter=text npm test", "ts-benchmark": "cd ./benchmarks/typescript/simple && npm install && npm run benchmark | node ../../../scripts/tsc-diagnostics-check"