diff --git a/.github/SECURITY.md b/.github/SECURITY.md new file mode 100644 index 0000000..0154e77 --- /dev/null +++ b/.github/SECURITY.md @@ -0,0 +1,4 @@ +If you discover a security issue in this repository, +please submit it through the [GitHub Security Bug Bounty](https://hackerone.com/github). + +Thanks for helping make GitHub safe for everyone. diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..5b6b53a --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: "gomod" + directory: "/" # Location of package manifests + schedule: + interval: "weekly" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..8f21ee4 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,42 @@ +name: CI Build + Unit Test + +on: + push: + branches: ["main"] + pull_request: + branches: ["main"] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: 1.19 + + - name: Build + run: go build -v ./... + + - name: Test + run: go test -v ./... + + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: 1.19 + + - name: Run golangci-lint + uses: golangci/golangci-lint-action@v3 + with: + version: v1.49 + + - name: Run shellcheck + run: shellcheck **/*.sh diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 0000000..828a92d --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,72 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ "main" ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ "main" ] + schedule: + - cron: '22 17 * * 1' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'go' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + + # If the Autobuild fails above, remove it and uncomment the following three lines. + # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. + + # - run: | + # echo "Run, Build Application using script" + # ./location_of_script_within_repo/buildscript.sh + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..8a1da2b --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,31 @@ +name: Release binary builder + +on: + release: + types: [created] + +jobs: + releases-matrix: + name: Release Go Binary + runs-on: ubuntu-latest + strategy: + matrix: + goos: [linux, windows, darwin] + goarch: ["386", amd64, arm64] + exclude: + - goarch: "386" + goos: darwin + - goarch: arm64 + goos: windows + steps: + - uses: actions/checkout@v3 + - uses: wangyoucao577/go-release-action@v1.29 + with: + goversion: 1.19 + github_token: ${{ secrets.GITHUB_TOKEN }} + goos: ${{ matrix.goos }} + goarch: ${{ matrix.goarch }} + binary_name: dependabot + project_path: cmd/dependabot + ldflags: >- + -X github.com/dependabot/cli/cmd/dependabot/internal/cmd.version=${{ github.event.release.tag_name }} diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml new file mode 100644 index 0000000..575cd71 --- /dev/null +++ b/.github/workflows/smoke.yml @@ -0,0 +1,86 @@ +# Runs all ecosystems cached and concurrently. +name: Smoke + +on: + workflow_dispatch: + pull_request: + branches: ["main"] + +env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + +jobs: + smoke: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + suite: + - actions + - bundler + - cargo + - composer + - docker + - elm + - go + - gradle + - hex + - maven + - npm + - nuget + - pip + - pip-compile + - pipenv + - poetry + - pub + - submodules + - terraform + steps: + - uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: 1.19 + + # Download the Proxy cache. The job is ideally 100% cached so no real calls are made. + - name: Download artifacts + run: script/download-cache.sh ${{ matrix.suite }} + + - name: ${{ matrix.suite }} + env: + LOCAL_GITHUB_ACCESS_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -o pipefail + URL=https://api.github.com/repos/dependabot/smoke-tests/contents/tests/smoke-${{ matrix.suite }}.yaml + curl $(gh api $URL --jq .download_url) -o smoke.yaml + go run cmd/dependabot/dependabot.go test -f=smoke.yaml -o=result.yaml --timeout 20m --cache=cache 2>&1 | tee -a log.txt + + - name: Diff + if: always() + continue-on-error: true + run: diff --ignore-space-change smoke.yaml result.yaml && echo "Contents are identical" || exit 0 + + - name: Create summary + run: tail -n100 log.txt | grep -P '\d+/\d+ calls cached \(\d+%\)' >> $GITHUB_STEP_SUMMARY + + # No upload at the end: + # - If a test is uncachable in some regard, the cache would grow unbound. + # - We might want to consider erroring if the cache is changed. + + # Allows us to add a check requirement on allsmoke which covers all in the matrix above + allsmoke: + if: ${{ always() }} + runs-on: ubuntu-latest + name: Smoke result + needs: smoke + steps: + - name: Echo needs + run: echo "${{ toJSON(needs) }}" # for debugging + - name: Check success + run: | + if [ "${{ needs.smoke.result }}" = "success" ]; then + exit 0 + else + exit 1 + fi diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7613027 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +tmp +testdata/caches +cache diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..41ecda8 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,61 @@ +run: + tests: true + skip-dirs: + - test-updater + +linters: + enable: + - depguard + - errcheck + - exportloopref + - gocritic + - gocyclo + - gofmt + - goimports + - gosec + - gosimple + - govet + - ineffassign + - misspell + - nakedret + - prealloc + - revive + - staticcheck + - typecheck + - unconvert + - unused + disable: + - gochecknoglobals # we allow global variables in packages + - gochecknoinits # we allow inits in packages + - goconst # we allow repeated values to go un-const'd + - lll # we allow any line length + - structcheck # structcheck is disabled because of go1.18 + - unparam # we allow function calls to name unused parameters + +linters-settings: + errcheck: + check-type-assertions: true + goconst: + min-len: 2 + min-occurrences: 3 + gocritic: + enabled-tags: + - diagnostic + - experimental + - opinionated + - performance + - style + disabled-checks: + - hugeParam + - octalLiteral + - singleCaseSwitch + govet: + check-shadowing: true + nolintlint: + require-explanation: true + require-specific: true + +issues: + exclude-rules: + - path: internal/infra/proxy.go + text: "G306: Expect WriteFile permissions to be 0600 or less" diff --git a/Brewfile b/Brewfile new file mode 100644 index 0000000..c4e2102 --- /dev/null +++ b/Brewfile @@ -0,0 +1,9 @@ +# Usage: +# $ brew bundle + +tap 'homebrew/core' + +brew 'go' + +tap 'golangci/tap' +brew 'golangci-lint' diff --git a/Brewfile.lock.json b/Brewfile.lock.json new file mode 100644 index 0000000..b18aac6 --- /dev/null +++ b/Brewfile.lock.json @@ -0,0 +1,104 @@ +{ + "entries": { + "tap": { + "homebrew/core": { + "revision": "8c764825c6668815b12df4058bccd6c15495e900" + }, + "golangci/tap": { + "revision": "7e8d1b4b1c31c15d9918a31bcc1a110b2454d29a" + } + }, + "brew": { + "go": { + "version": "1.19", + "bottle": { + "rebuild": 0, + "root_url": "https://ghcr.io/v2/homebrew/core", + "files": { + "arm64_monterey": { + "cellar": "/opt/homebrew/Cellar", + "url": "https://ghcr.io/v2/homebrew/core/go/blobs/sha256:053b3a83d7a9f47ec3b435e3681a2e3ea29ac7d267fe1f97e144b2512c736eb9", + "sha256": "053b3a83d7a9f47ec3b435e3681a2e3ea29ac7d267fe1f97e144b2512c736eb9" + }, + "arm64_big_sur": { + "cellar": "/opt/homebrew/Cellar", + "url": "https://ghcr.io/v2/homebrew/core/go/blobs/sha256:4bc6f293bc5c42caab60cd2e2821e12f3a4f0a76c2afca347efdb2e8715f972a", + "sha256": "4bc6f293bc5c42caab60cd2e2821e12f3a4f0a76c2afca347efdb2e8715f972a" + }, + "monterey": { + "cellar": "/usr/local/Cellar", + "url": "https://ghcr.io/v2/homebrew/core/go/blobs/sha256:0c072d504cbf5ac584d85f46882afc51db91d341e97c9a2c83bc74b980b8409b", + "sha256": "0c072d504cbf5ac584d85f46882afc51db91d341e97c9a2c83bc74b980b8409b" + }, + "big_sur": { + "cellar": "/usr/local/Cellar", + "url": "https://ghcr.io/v2/homebrew/core/go/blobs/sha256:97671397aef87943379ddd6a8f4c56eca31da2d63db57d25fea9e1ce01fbd7be", + "sha256": "97671397aef87943379ddd6a8f4c56eca31da2d63db57d25fea9e1ce01fbd7be" + }, + "catalina": { + "cellar": "/usr/local/Cellar", + "url": "https://ghcr.io/v2/homebrew/core/go/blobs/sha256:0fe6f9f864e16a828c88fef98ea89ed03eb7e5c1bc5c83151d0e952fa276b719", + "sha256": "0fe6f9f864e16a828c88fef98ea89ed03eb7e5c1bc5c83151d0e952fa276b719" + }, + "x86_64_linux": { + "cellar": "/home/linuxbrew/.linuxbrew/Cellar", + "url": "https://ghcr.io/v2/homebrew/core/go/blobs/sha256:5814764a0a39b6ada34106e5775a437c667d9f907041e2a2c0059c780a8a3a1a", + "sha256": "5814764a0a39b6ada34106e5775a437c667d9f907041e2a2c0059c780a8a3a1a" + } + } + } + }, + "golangci-lint": { + "version": "1.49.0", + "bottle": { + "rebuild": 0, + "root_url": "https://ghcr.io/v2/homebrew/core", + "files": { + "arm64_monterey": { + "cellar": ":any_skip_relocation", + "url": "https://ghcr.io/v2/homebrew/core/golangci-lint/blobs/sha256:790eba5d113226ccd2a5fad4ad6d5bf5aafc7a9dda6a32cd08bc032e74b98a9b", + "sha256": "790eba5d113226ccd2a5fad4ad6d5bf5aafc7a9dda6a32cd08bc032e74b98a9b" + }, + "arm64_big_sur": { + "cellar": ":any_skip_relocation", + "url": "https://ghcr.io/v2/homebrew/core/golangci-lint/blobs/sha256:c0239bd2993c5a46f42de9f8bd30bc3700279370e9660e61d4d908a9623cb236", + "sha256": "c0239bd2993c5a46f42de9f8bd30bc3700279370e9660e61d4d908a9623cb236" + }, + "monterey": { + "cellar": ":any_skip_relocation", + "url": "https://ghcr.io/v2/homebrew/core/golangci-lint/blobs/sha256:013ea67d8c8a6bd7276bd689ed44fb7e0072f2b2e93be6029c2582803dd5fe25", + "sha256": "013ea67d8c8a6bd7276bd689ed44fb7e0072f2b2e93be6029c2582803dd5fe25" + }, + "big_sur": { + "cellar": ":any_skip_relocation", + "url": "https://ghcr.io/v2/homebrew/core/golangci-lint/blobs/sha256:f4c69256964d8d727743870f55c7740361566fb32ed45461c3270b0436cebe53", + "sha256": "f4c69256964d8d727743870f55c7740361566fb32ed45461c3270b0436cebe53" + }, + "catalina": { + "cellar": ":any_skip_relocation", + "url": "https://ghcr.io/v2/homebrew/core/golangci-lint/blobs/sha256:72eb2f241f8a43f9dbffbf282108345a8e1c5f41cb78edda625f9d9e0b40f84d", + "sha256": "72eb2f241f8a43f9dbffbf282108345a8e1c5f41cb78edda625f9d9e0b40f84d" + }, + "x86_64_linux": { + "cellar": ":any_skip_relocation", + "url": "https://ghcr.io/v2/homebrew/core/golangci-lint/blobs/sha256:3656e3077a2e9fd6746c1e22c736c5e07b532d2c235829b05d12dac6062fdfe8", + "sha256": "3656e3077a2e9fd6746c1e22c736c5e07b532d2c235829b05d12dac6062fdfe8" + } + } + } + } + } + }, + "system": { + "macos": { + "monterey": { + "HOMEBREW_VERSION": "3.5.10-49-gb2ddb34", + "HOMEBREW_PREFIX": "/usr/local", + "Homebrew/homebrew-core": "8c764825c6668815b12df4058bccd6c15495e900", + "CLT": "12.0.0.32.29", + "Xcode": "dunno", + "macOS": "12.5.1" + } + } + } +} diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 0000000..248dab4 --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1 @@ +* @dependabot/maintainers diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..00e9069 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 GitHub Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..c496f73 --- /dev/null +++ b/README.md @@ -0,0 +1,236 @@ +# Dependabot CLI + +A CLI that pulls in the [updater] and [proxy] containers and runs them. + +## Installation + +You can download a pre-built binary for your system from the [Releases] page. + +If you have the [`gh` CLI][gh] installed, +you can install the latest release of `dependabot` using the following command +([gist source](https://gist.github.com/mattt/e09e1ecd76d5573e0517a7622009f06f)): + +```console +$ gh gist view --raw e09e1ecd76d5573e0517a7622009f06f | bash +``` + +## Requirements + +* Docker + +For development: + +* Go 1.18+ + +## Usage + +```console +$ dependabot update go_modules rsc/quote --dry-run +``` + +
+Output + +``` +time="2022-07-25T11:35:05Z" level=info msg="proxy starting" commit=8bc7edd876c7b566c70dcf22daa1c039912767f9 +time="2022-07-25T11:35:05Z" level=warning msg="initializing metrics client" error="No address passed and autodetection from environment failed" +2022/07/25 11:35:05 Listening (:1080) +I, [2022-07-25T11:35:06.684817 #9] INFO -- sentry: ** [Raven] Raven 3.1.2 configured not to capture errors: DSN not set +INFO Starting job processing +2022/07/25 11:35:28 [002] GET https://api.github.com:443/repos/rsc/quote +2022/07/25 11:35:28 [002] * authenticating github api request +2022/07/25 11:35:28 [002] 200 https://api.github.com:443/repos/rsc/quote +2022/07/25 11:35:28 [004] GET https://api.github.com:443/repos/rsc/quote/git/refs/heads/master +2022/07/25 11:35:28 [004] * authenticating github api request +2022/07/25 11:35:28 [004] 200 https://api.github.com:443/repos/rsc/quote/git/refs/heads/master +2022/07/25 11:35:29 [006] GET https://github.com:443/rsc/quote/info/refs?service=git-upload-pack +2022/07/25 11:35:29 [006] * authenticating git server request (host: github.com) +2022/07/25 11:35:29 [006] 200 https://github.com:443/rsc/quote/info/refs?service=git-upload-pack +2022/07/25 11:35:29 [008] POST https://github.com:443/rsc/quote/git-upload-pack +2022/07/25 11:35:29 [008] * authenticating git server request (host: github.com) +2022/07/25 11:35:29 [008] 200 https://github.com:443/rsc/quote/git-upload-pack +2022/07/25 11:35:29 [010] POST https://github.com:443/rsc/quote/git-upload-pack +2022/07/25 11:35:29 [010] * authenticating git server request (host: github.com) +2022/07/25 11:35:29 [010] 200 https://github.com:443/rsc/quote/git-upload-pack +INFO Finished job processing +I, [2022-07-25T11:35:30.005259 #30] INFO -- sentry: ** [Raven] Raven 3.1.2 configured not to capture errors: DSN not set +INFO Starting job processing +INFO Starting update job for rsc/quote +2022/07/25 11:35:51 [011] POST http://host.docker.internal:8080/update_jobs/cli/update_dependency_list +2022/07/25 11:35:51 [011] 200 http://host.docker.internal:8080/update_jobs/cli/update_dependency_list +INFO Checking if rsc.io/quote/v3 3.0.0 needs updating +2022/07/25 11:35:51 [013] GET https://rsc.io:443/quote/v3?go-get=1 +2022/07/25 11:35:51 [013] 200 https://rsc.io:443/quote/v3?go-get=1 +2022/07/25 11:35:51 [015] GET https://rsc.io:443/quote?go-get=1 +2022/07/25 11:35:51 [015] 200 https://rsc.io:443/quote?go-get=1 +2022/07/25 11:35:51 [017] GET https://github.com:443/rsc/quote/info/refs?service=git-upload-pack +2022/07/25 11:35:51 [017] * authenticating git server request (host: github.com) +2022/07/25 11:35:52 [017] 200 https://github.com:443/rsc/quote/info/refs?service=git-upload-pack +2022/07/25 11:35:52 [019] GET https://github.com:443/rsc/quote/info/refs?service=git-upload-pack +2022/07/25 11:35:52 [019] * authenticating git server request (host: github.com) +2022/07/25 11:35:52 [019] 200 https://github.com:443/rsc/quote/info/refs?service=git-upload-pack +2022/07/25 11:35:52 [021] POST https://github.com:443/rsc/quote/git-upload-pack +2022/07/25 11:35:52 [021] * authenticating git server request (host: github.com) +2022/07/25 11:35:52 [021] 200 https://github.com:443/rsc/quote/git-upload-pack +2022/07/25 11:35:52 [023] POST https://github.com:443/rsc/quote/git-upload-pack +2022/07/25 11:35:52 [023] * authenticating git server request (host: github.com) +2022/07/25 11:35:52 [023] 200 https://github.com:443/rsc/quote/git-upload-pack +INFO Latest version is 3.1.0 +INFO Requirements to unlock own +INFO Requirements update strategy +INFO Updating rsc.io/quote/v3 from 3.0.0 to 3.1.0 +2022/07/25 11:35:52 [026] GET https://rsc.io:443/quote/v3?go-get=1 +2022/07/25 11:35:52 [027] GET https://rsc.io:443/sampler?go-get=1 +2022/07/25 11:35:52 [026] 200 https://rsc.io:443/quote/v3?go-get=1 +2022/07/25 11:35:52 [029] GET https://rsc.io:443/quote?go-get=1 +2022/07/25 11:35:52 [027] 200 https://rsc.io:443/sampler?go-get=1 +2022/07/25 11:35:52 [029] 200 https://rsc.io:443/quote?go-get=1 +2022/07/25 11:35:52 [032] GET https://github.com:443/rsc/sampler/info/refs?service=git-upload-pack +2022/07/25 11:35:52 [032] * authenticating git server request (host: github.com) +2022/07/25 11:35:52 [033] GET https://github.com:443/rsc/quote/info/refs?service=git-upload-pack +2022/07/25 11:35:52 [033] * authenticating git server request (host: github.com) +2022/07/25 11:35:52 [032] 200 https://github.com:443/rsc/sampler/info/refs?service=git-upload-pack +2022/07/25 11:35:52 [033] 200 https://github.com:443/rsc/quote/info/refs?service=git-upload-pack +2022/07/25 11:35:52 [035] GET https://github.com:443/rsc/sampler/info/refs?service=git-upload-pack +2022/07/25 11:35:52 [035] * authenticating git server request (host: github.com) +2022/07/25 11:35:52 [037] GET https://github.com:443/rsc/quote/info/refs?service=git-upload-pack +2022/07/25 11:35:52 [037] * authenticating git server request (host: github.com) +2022/07/25 11:35:52 [035] 200 https://github.com:443/rsc/sampler/info/refs?service=git-upload-pack +2022/07/25 11:35:52 [037] 200 https://github.com:443/rsc/quote/info/refs?service=git-upload-pack +2022/07/25 11:35:52 [039] POST https://github.com:443/rsc/sampler/git-upload-pack +2022/07/25 11:35:52 [039] * authenticating git server request (host: github.com) +2022/07/25 11:35:52 [041] POST https://github.com:443/rsc/quote/git-upload-pack +2022/07/25 11:35:52 [041] * authenticating git server request (host: github.com) +2022/07/25 11:35:52 [039] 200 https://github.com:443/rsc/sampler/git-upload-pack +2022/07/25 11:35:52 [043] POST https://github.com:443/rsc/sampler/git-upload-pack +2022/07/25 11:35:52 [043] * authenticating git server request (host: github.com) +2022/07/25 11:35:52 [041] 200 https://github.com:443/rsc/quote/git-upload-pack +2022/07/25 11:35:52 [045] POST https://github.com:443/rsc/quote/git-upload-pack +2022/07/25 11:35:52 [045] * authenticating git server request (host: github.com) +2022/07/25 11:35:52 [043] 200 https://github.com:443/rsc/sampler/git-upload-pack +2022/07/25 11:35:52 [045] 200 https://github.com:443/rsc/quote/git-upload-pack +2022/07/25 11:35:53 [047] GET https://golang.org:443/x/text?go-get=1 +2022/07/25 11:35:53 [047] 200 https://golang.org:443/x/text?go-get=1 +2022/07/25 11:35:53 [049] GET https://go.googlesource.com:443/text/info/refs?service=git-upload-pack +2022/07/25 11:35:53 [049] 200 https://go.googlesource.com:443/text/info/refs?service=git-upload-pack +2022/07/25 11:35:53 [051] GET https://go.googlesource.com:443/text/info/refs?service=git-upload-pack +2022/07/25 11:35:53 [051] 200 https://go.googlesource.com:443/text/info/refs?service=git-upload-pack +2022/07/25 11:35:53 [053] POST https://go.googlesource.com:443/text/git-upload-pack +2022/07/25 11:35:53 [053] 200 https://go.googlesource.com:443/text/git-upload-pack +2022/07/25 11:35:57 [055] GET https://github.com:443/rsc/sampler/info/refs?service=git-upload-pack +2022/07/25 11:35:57 [055] * authenticating git server request (host: github.com) +2022/07/25 11:35:57 [055] 200 https://github.com:443/rsc/sampler/info/refs?service=git-upload-pack +2022/07/25 11:35:57 [057] POST https://github.com:443/rsc/sampler/git-upload-pack +2022/07/25 11:35:57 [057] * authenticating git server request (host: github.com) +2022/07/25 11:35:57 [057] 200 https://github.com:443/rsc/sampler/git-upload-pack +2022/07/25 11:35:57 [059] POST https://github.com:443/rsc/sampler/git-upload-pack +2022/07/25 11:35:57 [059] * authenticating git server request (host: github.com) +2022/07/25 11:35:57 [059] 200 https://github.com:443/rsc/sampler/git-upload-pack +2022/07/25 11:35:57 [062] GET https://rsc.io:443/quote/v3?go-get=1 +2022/07/25 11:35:57 [063] GET https://golang.org:443/x/text?go-get=1 +2022/07/25 11:35:57 [062] 200 https://rsc.io:443/quote/v3?go-get=1 +2022/07/25 11:35:57 [065] GET https://rsc.io:443/quote?go-get=1 +2022/07/25 11:35:57 [065] 200 https://rsc.io:443/quote?go-get=1 +2022/07/25 11:35:57 [063] 200 https://golang.org:443/x/text?go-get=1 +2022/07/25 11:35:58 [068] GET https://github.com:443/rsc/quote/info/refs?service=git-upload-pack +2022/07/25 11:35:58 [068] * authenticating git server request (host: github.com) +2022/07/25 11:35:58 [069] GET https://go.googlesource.com:443/text/info/refs?service=git-upload-pack +2022/07/25 11:35:58 [069] 200 https://go.googlesource.com:443/text/info/refs?service=git-upload-pack +2022/07/25 11:35:58 [068] 200 https://github.com:443/rsc/quote/info/refs?service=git-upload-pack +2022/07/25 11:35:58 [071] GET https://rsc.io:443/sampler?go-get=1 +2022/07/25 11:35:58 [071] 200 https://rsc.io:443/sampler?go-get=1 +2022/07/25 11:35:58 [073] GET https://github.com:443/rsc/sampler/info/refs?service=git-upload-pack +2022/07/25 11:35:58 [073] * authenticating git server request (host: github.com) +2022/07/25 11:35:58 [073] 200 https://github.com:443/rsc/sampler/info/refs?service=git-upload-pack +INFO Submitting rsc.io/quote/v3 pull request for creation +2022/07/25 11:36:18 [074] POST http://host.docker.internal:8080/update_jobs/cli/create_pull_request +2022/07/25 11:36:18 [074] 200 http://host.docker.internal:8080/update_jobs/cli/create_pull_request +INFO Checking if rsc.io/sampler 1.3.0 needs updating +2022/07/25 11:36:18 [076] GET https://rsc.io:443/sampler?go-get=1 +2022/07/25 11:36:18 [076] 200 https://rsc.io:443/sampler?go-get=1 +2022/07/25 11:36:18 [078] GET https://github.com:443/rsc/sampler/info/refs?service=git-upload-pack +2022/07/25 11:36:18 [078] * authenticating git server request (host: github.com) +2022/07/25 11:36:18 [078] 200 https://github.com:443/rsc/sampler/info/refs?service=git-upload-pack +INFO Latest version is 1.99.99 +INFO Requirements to unlock own +INFO Requirements update strategy +INFO Updating rsc.io/sampler from 1.3.0 to 1.99.99 +2022/07/25 11:36:18 [080] GET https://rsc.io:443/sampler?go-get=1 +2022/07/25 11:36:18 [080] 200 https://rsc.io:443/sampler?go-get=1 +2022/07/25 11:36:18 [082] GET https://golang.org:443/x/text?go-get=1 +2022/07/25 11:36:18 [084] GET https://github.com:443/rsc/sampler/info/refs?service=git-upload-pack +2022/07/25 11:36:18 [084] * authenticating git server request (host: github.com) +2022/07/25 11:36:18 [082] 200 https://golang.org:443/x/text?go-get=1 +2022/07/25 11:36:18 [086] GET https://go.googlesource.com:443/text/info/refs?service=git-upload-pack +2022/07/25 11:36:18 [084] 200 https://github.com:443/rsc/sampler/info/refs?service=git-upload-pack +2022/07/25 11:36:18 [086] 200 https://go.googlesource.com:443/text/info/refs?service=git-upload-pack +2022/07/25 11:36:18 [088] GET https://rsc.io:443/quote/v3?go-get=1 +2022/07/25 11:36:18 [088] 200 https://rsc.io:443/quote/v3?go-get=1 +2022/07/25 11:36:18 [090] GET https://rsc.io:443/quote?go-get=1 +2022/07/25 11:36:18 [090] 200 https://rsc.io:443/quote?go-get=1 +2022/07/25 11:36:18 [092] GET https://golang.org:443/x/text?go-get=1 +2022/07/25 11:36:18 [094] GET https://github.com:443/rsc/quote/info/refs?service=git-upload-pack +2022/07/25 11:36:18 [094] * authenticating git server request (host: github.com) +2022/07/25 11:36:18 [092] 200 https://golang.org:443/x/text?go-get=1 +2022/07/25 11:36:18 [096] GET https://go.googlesource.com:443/text/info/refs?service=git-upload-pack +2022/07/25 11:36:18 [094] 200 https://github.com:443/rsc/quote/info/refs?service=git-upload-pack +2022/07/25 11:36:19 [098] GET https://rsc.io:443/sampler?go-get=1 +2022/07/25 11:36:19 [096] 200 https://go.googlesource.com:443/text/info/refs?service=git-upload-pack +2022/07/25 11:36:19 [098] 200 https://rsc.io:443/sampler?go-get=1 +2022/07/25 11:36:19 [100] GET https://github.com:443/rsc/sampler/info/refs?service=git-upload-pack +2022/07/25 11:36:19 [100] * authenticating git server request (host: github.com) +2022/07/25 11:36:19 [100] 200 https://github.com:443/rsc/sampler/info/refs?service=git-upload-pack +INFO Submitting rsc.io/sampler pull request for creation +2022/07/25 11:36:39 [101] POST http://host.docker.internal:8080/update_jobs/cli/create_pull_request +2022/07/25 11:36:39 [101] 200 http://host.docker.internal:8080/update_jobs/cli/create_pull_request +2022/07/25 11:36:59 [102] PATCH http://host.docker.internal:8080/update_jobs/cli/mark_as_processed +2022/07/25 11:36:59 [102] 200 http://host.docker.internal:8080/update_jobs/cli/mark_as_processed +INFO Finished job processing +INFO Results: ++----------------------------------------------------+ +| Changes to Dependabot Pull Requests | ++---------+------------------------------------------+ +| created | rsc.io/quote/v3 ( from 3.0.0 to 3.1.0 ) | +| created | rsc.io/sampler ( from 1.3.0 to 1.99.99 ) | ++---------+------------------------------------------+ +``` + +
+ +## Troubleshooting + +### Docker daemon not running + +``` +failed to pull ghcr.io/github/dependabot-update-job-proxy/dependabot-update-job-proxy:latest: +Error response from daemon: dial unix docker.raw.sock: connect: no such file or directory +``` + +The CLI requires Docker to be running on your machine. +Follow the instructions on [Docker's website](https://docs.docker.com/get-started/) +to get the latest version of Docker installed and running. + +You can verify that Docker is running locally with the following command: + +```console +$ docker --version +``` + +### Network internet is ambiguous + +``` +failed to start container: Error response from daemon: network internet is ambiguous (2 matches found on name) +``` + +This error can occur when the CLI exits before getting to clean up +(e.g. terminating with ^C). +Run the following command to remove all unused networks: + +```console +$ docker network prune +``` + +[updater]: https://github.com/dependabot/dependabot-core/pkgs/container/dependabot-updater +[proxy]: https://github.com/github/dependabot-update-job-proxy/pkgs/container/dependabot-update-job-proxy%2Fdependabot-update-job-proxy +[gh]: https://github.com/cli/cli +[Releases]: https://github.com/dependabot/cli/releases diff --git a/cmd/dependabot/dependabot.go b/cmd/dependabot/dependabot.go new file mode 100644 index 0000000..0437936 --- /dev/null +++ b/cmd/dependabot/dependabot.go @@ -0,0 +1,9 @@ +package main + +import ( + "github.com/dependabot/cli/cmd/dependabot/internal/cmd" +) + +func main() { + cmd.Execute() +} diff --git a/cmd/dependabot/internal/cmd/root.go b/cmd/dependabot/internal/cmd/root.go new file mode 100644 index 0000000..ff6a1eb --- /dev/null +++ b/cmd/dependabot/internal/cmd/root.go @@ -0,0 +1,48 @@ +package cmd + +import ( + "os" + "time" + + "github.com/dependabot/cli/internal/infra" + + "github.com/MakeNowJust/heredoc" + "github.com/spf13/cobra" +) + +var ( + file string + cache string + debugging bool + output string + pullImages bool + volumes []string + timeout time.Duration + tempDir string +) + +// rootCmd represents the base command when called without any subcommands +var rootCmd = &cobra.Command{ + Use: "dependabot [flags]", + Short: "Dependabot end-to-end runner", + Long: `Run Dependabot jobs from the command line.`, + Example: heredoc.Doc(` + $ dependabot update go_modules rsc/quote --dry-run + $ dependabot test -f input.yml + `), + Version: Version(), +} + +func Execute() { + err := rootCmd.Execute() + if err != nil { + os.Exit(1) + } +} + +func init() { + rootCmd.PersistentFlags().StringVar(&infra.UpdaterImageName, "updater-image", infra.UpdaterImageName, "container image to use for the updater") + rootCmd.PersistentFlags().StringVar(&infra.ProxyImageName, "proxy-image", infra.ProxyImageName, "container image to use for the proxy") + + rootCmd.PersistentFlags().StringVar(&tempDir, "temp-dir", "tmp", "path to the temporary directory for the job") +} diff --git a/cmd/dependabot/internal/cmd/test.go b/cmd/dependabot/internal/cmd/test.go new file mode 100644 index 0000000..3ca9b49 --- /dev/null +++ b/cmd/dependabot/internal/cmd/test.go @@ -0,0 +1,87 @@ +package cmd + +import ( + "encoding/json" + "fmt" + "log" + "os" + + "github.com/dependabot/cli/internal/infra" + "github.com/dependabot/cli/internal/model" + + "github.com/spf13/cobra" + "gopkg.in/yaml.v3" +) + +var ( + jobs int +) + +var testCmd = &cobra.Command{ + Use: "test [-f file]", + Short: "Test scenarios", + RunE: func(cmd *cobra.Command, args []string) error { + if jobs < 1 { + return fmt.Errorf("workers must be greater than or equal to 1") + } + + if file == "" { + return fmt.Errorf("requires a scenario file") + } + + scenario, err := readScenarioFile(file) + if err != nil { + return err + } + + processInput(&scenario.Input) + + if err := infra.Run(infra.RunParams{ + CacheDir: cache, + Creds: scenario.Input.Credentials, + Debug: debugging, + Expected: scenario.Output, + Job: &scenario.Input.Job, + Output: output, + PullImages: pullImages, + TempDir: tempDir, + Timeout: timeout, + Volumes: volumes, + }); err != nil { + log.Fatal(err) + } + + return nil + }, +} + +func readScenarioFile(file string) (*model.Scenario, error) { + var scenario model.Scenario + + data, err := os.ReadFile(file) + if err != nil { + return nil, fmt.Errorf("failed to open scenario file: %w", err) + } + if err = json.Unmarshal(data, &scenario); err != nil { + if err = yaml.Unmarshal(data, &scenario); err != nil { + return nil, fmt.Errorf("failed to decode scenario file: %w", err) + } + } + + return &scenario, nil +} + +func init() { + rootCmd.AddCommand(testCmd) + + testCmd.Flags().StringVarP(&file, "file", "f", "", "path to scenario file") + testCmd.Flags().IntVarP(&jobs, "jobs", "j", 1, "Number of jobs to run simultaneously") + testCmd.MarkFlagsMutuallyExclusive("jobs", "file") + + testCmd.Flags().StringVarP(&output, "output", "o", "", "write scenario to file") + testCmd.Flags().StringVar(&cache, "cache", "", "cache import/export directory") + testCmd.Flags().BoolVar(&pullImages, "pull", true, "pull the image if it isn't present") + testCmd.Flags().BoolVar(&debugging, "debug", false, "run an interactive shell inside the updater") + testCmd.Flags().StringArrayVarP(&volumes, "volume", "v", nil, "mount volumes in Docker") + testCmd.Flags().DurationVarP(&timeout, "timeout", "t", 0, "max time to run an update") +} diff --git a/cmd/dependabot/internal/cmd/update.go b/cmd/dependabot/internal/cmd/update.go new file mode 100644 index 0000000..191111f --- /dev/null +++ b/cmd/dependabot/internal/cmd/update.go @@ -0,0 +1,197 @@ +package cmd + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "log" + "os" + + "github.com/MakeNowJust/heredoc" + "github.com/dependabot/cli/internal/infra" + "github.com/dependabot/cli/internal/model" + "github.com/dependabot/cli/internal/server" + "github.com/spf13/cobra" + "gopkg.in/yaml.v3" +) + +var ( + packageManager string + provider string + repo string + directory string + + dryRun bool + inputServerPort int +) + +var updateCmd = &cobra.Command{ + Use: "update [flags]", + Short: "Perform update job", + Example: heredoc.Doc(` + $ dependabot update go_modules rsc/quote --dry-run + `), + RunE: func(cmd *cobra.Command, args []string) error { + var outFile *os.File + if output != "" { + var err error + outFile, err = os.Create(output) + if err != nil { + return fmt.Errorf("failed to create output file: %w", err) + } + defer outFile.Close() + } + + input := &model.Input{} + + if file != "" { + var err error + input, err = readInputFile(file) + if err != nil { + return err + } + } + + if len(cmd.Flags().Args()) > 0 { + packageManager = cmd.Flags().Args()[0] + if packageManager == "" { + return errors.New("requires a package manager argument") + } + + repo = cmd.Flags().Args()[1] + if repo == "" { + return errors.New("requires a repo argument") + } + + input.Job = model.Job{ + PackageManager: packageManager, + AllowedUpdates: []model.Allowed{{ + UpdateType: "all", + }}, + Dependencies: nil, + ExistingPullRequests: [][]model.ExistingPR{}, + IgnoreConditions: []model.Condition{}, + LockfileOnly: false, + RequirementsUpdateStrategy: nil, + SecurityAdvisories: []model.Advisory{}, + SecurityUpdatesOnly: false, + Source: model.Source{ + Provider: provider, + Repo: repo, + Directory: directory, + Branch: nil, + Hostname: nil, + APIEndpoint: nil, + }, + UpdateSubdependencies: false, + UpdatingAPullRequest: false, + } + } + + if inputServerPort != 0 { + input = server.Input(inputServerPort) + } + + if doesStdinHaveData() { + in := &bytes.Buffer{} + _, err := io.Copy(in, os.Stdin) + if err != nil { + return err + } + data := in.Bytes() + if err = json.Unmarshal(data, &input); err != nil { + if err = yaml.Unmarshal(data, &input); err != nil { + return fmt.Errorf("failed to decode input file: %w", err) + } + } + } + + processInput(input) + + if err := infra.Run(infra.RunParams{ + CacheDir: cache, + Creds: input.Credentials, + Debug: debugging, + Expected: nil, // update subcommand doesn't use expectations + Job: &input.Job, + Output: output, + PullImages: pullImages, + TempDir: tempDir, + Timeout: timeout, + Volumes: volumes, + }); err != nil { + log.Fatalf("failed to run updater: %v", err) + } + + return nil + }, +} + +func readInputFile(file string) (*model.Input, error) { + var input model.Input + + data, err := os.ReadFile(file) + if err != nil { + return nil, fmt.Errorf("failed to open input file: %w", err) + } + if err = json.Unmarshal(data, &input); err != nil { + if err = yaml.Unmarshal(data, &input); err != nil { + return nil, fmt.Errorf("failed to decode input file: %w", err) + } + } + + return &input, nil +} + +func processInput(input *model.Input) { + job := &input.Job + // a few of the fields need to be initialized instead of null, + // it would be nice if the updater didn't care + if job.ExistingPullRequests == nil { + job.ExistingPullRequests = [][]model.ExistingPR{} + } + if job.IgnoreConditions == nil { + job.IgnoreConditions = []model.Condition{} + } + if job.SecurityAdvisories == nil { + job.SecurityAdvisories = []model.Advisory{} + } + + // Process environment variables in the scenario file + for _, cred := range input.Credentials { + for key, value := range cred { + cred[key] = os.ExpandEnv(value) + } + } +} + +func doesStdinHaveData() bool { + file := os.Stdin + fi, err := file.Stat() + if err != nil { + fmt.Println("file.Stat()", err) + } + return fi.Size() > 0 +} + +func init() { + rootCmd.AddCommand(updateCmd) + + updateCmd.Flags().StringVarP(&file, "file", "f", "", "path to scenario file") + + updateCmd.Flags().StringVarP(&provider, "provider", "p", "github", "provider of the repository") + updateCmd.Flags().StringVarP(&directory, "directory", "d", "/", "directory to update") + + updateCmd.Flags().BoolVar(&dryRun, "dry-run", true, "perform update as a dry run") + _ = updateCmd.MarkFlagRequired("dry-run") + + updateCmd.Flags().StringVarP(&output, "output", "o", "", "write scenario to file") + updateCmd.Flags().StringVar(&cache, "cache", "", "cache import/export directory") + updateCmd.Flags().BoolVar(&pullImages, "pull", true, "pull the image if it isn't present") + updateCmd.Flags().BoolVar(&debugging, "debug", false, "run an interactive shell inside the updater") + updateCmd.Flags().StringArrayVarP(&volumes, "volume", "v", nil, "mount volumes in Docker") + updateCmd.Flags().DurationVarP(&timeout, "timeout", "t", 0, "max time to run an update") + updateCmd.Flags().IntVar(&inputServerPort, "input-port", 0, "port to use for securely passing input to the updater") +} diff --git a/cmd/dependabot/internal/cmd/version.go b/cmd/dependabot/internal/cmd/version.go new file mode 100644 index 0000000..eb2777f --- /dev/null +++ b/cmd/dependabot/internal/cmd/version.go @@ -0,0 +1,44 @@ +package cmd + +import ( + "fmt" + "regexp" + "runtime/debug" +) + +// ldflags inserts the version here on release +var version string + +var timestampRegex = regexp.MustCompile("[^a-zA-Z0-9]+") + +func Version() string { + if version == "" { + version = "0.0.0-dev" + commit := "" + timestamp := "" + modified := false + + info, _ := debug.ReadBuildInfo() + for _, entry := range info.Settings { + if entry.Key == "vcs.revision" && len(entry.Value) >= 7 { + commit = entry.Value[:7] // short ref + } + + if entry.Key == "vcs.modified" { + modified = entry.Value == "true" + } + + if entry.Key == "vcs.time" { + timestamp = timestampRegex.ReplaceAllString(entry.Value, "") + } + } + + if modified && timestamp != "" { + return fmt.Sprintf("%s+%s", version, timestamp) + } else if commit != "" { + return fmt.Sprintf("%s+%s", version, commit) + } + } + + return version +} diff --git a/docs/debugging-with-the-cli.md b/docs/debugging-with-the-cli.md new file mode 100644 index 0000000..58dc21a --- /dev/null +++ b/docs/debugging-with-the-cli.md @@ -0,0 +1,72 @@ +# Debugging with the CLI + +Updater is now merged into core, so we want the CLI to become the preferred way to debug issues in core. This is because the CLI runs the proxy and updater just like how Dependabot runs in production. + +To get started, run the dependabot script in core's script directory. This is mostly equivalent to running `bin/dry-run.rb` inside a docker-dev-shell, but without needing to boot a shell in the container first. + +If you want to debug like you do with docker-dev-shell, you can pass the `--debug` flag to `script/dependabot`. This will mount core's ecosystems for editing while debugging just like the docker-dev-shell does. + +For general debugging of an ecosystem, use the update subcommand, and pass the ecosystem and NWO (name with org) like so: + +```console +$ script/dependabot update go_modules rsc/quote --dry-run --debug +``` + +The CLI will use a Personal Access Token (PAT) in the environment variable `LOCAL_GITHUB_ACCESS_TOKEN` and pass that to the Proxy before starting it. + +It will then generate a `job.json` file with `package_manager` of "go_modules" and a `source` of "rsc/quote". + +Unlike the dry-run script and dev-shell, we no longer run the shell while dry-running several repositories. It is recommended you exit the debugging session and start a new one when switching the debugging target. Although you could edit the job.json, the CLI starts very quickly and that's probably not necessary. You may even want to restart after changes to the native helpers. + +Once you are in the debugging session, run `bin/run fetch_files` which is the first step the updater takes when it runs. Once complete, run `bin/run update_files` to perform the update. + +You might notice the proxy output is not logged in the terminal. To see that you'll have to tail the logs in another terminal or using Docker Desktop. Alternatively you can [fix this issue](https://github.com/dependabot/cli/issues/87), and it will work. + +## More complex scenarios + +For debugging of more complex situations, you'll want to use a YAML file for input. + +View the `testdata` directory for examples of inputs. The input section is all you'll need. So for instance you write a YAML file like this: + +```yaml +job: + package_manager: npm_and_yarn + allowed_updates: + - update-type: all + security_advisories: + - dependency-name: express + affected-versions: + - <5.0.0 + patched-versions: [] + unaffected-versions: [] + security_updates_only: true + source: + provider: github + repo: dependabot/e2e-tests + directory: / + commit: 66115359e6f6cc3af6a661c5d5ae803720b98cb8 +credentials: + - type: npm_registry + registry: https://npm.pkg.github.com + token: $GPR_TOKEN +``` + +And then run your debugging session like so: + +```console +$ script/dependabot update -f test.yaml --dry-run --debug +``` + +The CLI will process any `$VARIABLES` in YAML files and replace them with environment variables (for example, `$LOCAL_GITHUB_ACCESS_TOKEN`). + +Or if you just want to run a complete update and view the results, run it without the `--debug` flag + +```console +$ script/dependabot update -f test.yaml -o output.yaml --dry-run +``` + +The `output.yaml` file produced shows all the calls the updater will have made during the course of the update. It also happens to be input to an end-to-end test: + +```console +$ script/dependabot test -f output.yaml +``` diff --git a/docs/design.md b/docs/design.md new file mode 100644 index 0000000..64c7b4d --- /dev/null +++ b/docs/design.md @@ -0,0 +1,96 @@ +# Design + +The CLI is designed to start minimal Dependabot infrastructure on your machine in order to run an update. + +At a high level, the CLI: +- Pulls an Updater image and a Proxy image +- Creates networks so the Updater can make calls only through the Proxy +- Writes the Proxy input file +- Starts the Proxy +- Writes the job file, which is the input to the Updater +- Starts the Updater image +- Records calls for creating pull requests from the Updater +- Waits for the Updater image to finish +- Cleans everything up + +This setup is identical to how [dependabot-action](https://github.com/github/dependabot-action) works, and pretty similar to how our production setup works. So by running with the CLI, we can test essentially what is in production. + +## CLI Goals + +- Create one way to run Dependabot + - Makes it easier to test End-to-End + - Reduce the amount of code the team supports +- Customers can build custom logic downstream from Dependabot CLI by creating adapters +- External users who integrate with other systems can create adapters too + +The CLI also opens a lot of doors around extensibility and maintainability of ecosystems. + +## Sequence Diagrams + +All Updater calls go through the Proxy, I've elided those for brevity. + +### E2E tests + +Generating tests with a --dry-run: + +```mermaid +sequenceDiagram + CLI->>Proxy: Starts the Proxy + CLI->>Updater: Starts the Updater + Updater->>GitHub: Fetch manifests + loop + Updater->>Registry: Get version info, etc + Updater->>CLI: Create/Update PR, etc + end + CLI->>YAML file: CLI was given -o so it outputs the calls made +``` + +Asserting expected behavior, fully cached in the Proxy: + +```mermaid +sequenceDiagram + CLI->>Proxy: Starts the Proxy + CLI->>Updater: Starts the Updater + Updater->>Proxy: Fetch manifests (cached) + loop + Updater->>Proxy: Get version info, etc (cached) + Updater->>CLI: Create/Update PR, etc + end + CLI->>stderr: CLI outputs pass/fail info, exit code +``` + +### On Desktop + +Future phase + +```mermaid +sequenceDiagram + CLI->>Proxy: Starts the Proxy + CLI->>Updater: Starts the Updater + Updater->>GitHub: Fetch manifests + loop + Updater->>Registry: Get version info, etc + Updater->>CLI: Create/Update PR + CLI->>GH Adapter: CLI forwards API calls to an adapter (TBD) + GH Adapter->>GitHub: Adds PAT header to Create/Update PR + end +``` + + +### In GHES + +Future phase + +```mermaid +sequenceDiagram + Workflow->>CLI: Invoke with Job + CLI->>Proxy: Starts the Proxy + CLI->>Updater: Starts the Updater + Updater->>GitHub: Fetch manifests + loop + Updater->>Registry: Get version info, etc + Updater->>CLI: Create/Update PR + CLI->>GHES Adapter: CLI forwards API calls to an adapter (TBD) + GHES Adapter->>Dependabot API: Adds auth headers + end +``` diff --git a/docs/e2e-tests.md b/docs/e2e-tests.md new file mode 100644 index 0000000..9f06c34 --- /dev/null +++ b/docs/e2e-tests.md @@ -0,0 +1,45 @@ +# End-to-End tests + +The Dependabot CLI includes features to produce and run end-to-end (e2e) tests. + +## Producing a test + +- Create a repository with the manifest and files you want to test on GitHub +- Generate the test file by doing a dry-run + - For example: `dependabot update bundler dependabot/e2e-tests -o=testdata/smoke-bundler.yaml --dry-run` +- Test the cache effectiveness by running the `test` command twice with the cache option. Make sure you start with a clean cache initially. + - `dependabot test -f=testdata/smoke-bundler.yaml --cache=tmp/cache` + - At the end of the test you'll see an output like this: `time="2022-08-11T00:34:41Z" level=info msg="71/316 calls cached (22%)"` + - If the cache coverage isn't 100% the test will most likely fail when a new dependency is published +- If you are producing a new smoke test: + - Add entries in the workflows: + - cache-all.yml + - cache-one.yml + - smoke.yml + - Put up the PR + - Run the cache-one workflow on the new ecosystem to fill the cache + - Run the smoke workflow to verify tests are passing, and cache coverage is good + - You may also need to update the smoke workflow in updater or core + +## Debugging a failure + +When an e2e test fails in a workflow, it produces a diff between the test expectations and the actual output: + +![diff output](img/diff.png) + +Use this information to decide if the test failure is just the ecosystem somehow evading caching, or if it's a real failure. + +To test locally: +- Optionally download the exiting cache with, for example `cd tmp/cache && gh run download --repo dependabot/cli --name cache-bundler` +- Run the test: `dependabot test -f=testdata/smoke-bundler.yaml -cache=tmp/cache` + - Overwrite the existing test for easy diffing with git: `dependabot test -f=testdata/smoke-bundler.yaml -o=testdata/smoke-bundler.yaml -cache=tmp/cache` + +## Working with the Actions cache + +The caching in Actions is handled by build artifacts. Each cache has a unique name, like `cache-bundler`. + +To create a new cache, run the `cache-one` workflow. + +To create all new caches, run `cache-all`. It's rare to do this, and takes a long time, so recommended you stick to `cache-one`. + +To download them for local testing, run `gh run download --repo dependabot/cli --name cache-bundler`. diff --git a/docs/img/diff.png b/docs/img/diff.png new file mode 100644 index 0000000..dbb3ed4 Binary files /dev/null and b/docs/img/diff.png differ diff --git a/docs/release.md b/docs/release.md new file mode 100644 index 0000000..726e8b2 --- /dev/null +++ b/docs/release.md @@ -0,0 +1,20 @@ +# Releasing the CLI + +- Go to the [releases] page and note the last release. Decide if this will be major, minor, or patch. + - During our pre-release we are bumping patch + - Choose minor for new features and fixes + - Choose patch if you are back-porting a fix to an older version + - Choose major for breaking changes to command line flags or any other ways the CLI works +- Click [Draft a new release] +- Click "Choose a tag" +- Type in the new version preceded with a `v` +- Click "Create new tag" which is just under where you typed +- Optionally create a Title and add release notes +- Click "Publish release" + +The `release.yml` workflow will kick off and add binaries to the release after a couple of minutes. + +Feel free to automate this in the future! + +[releases]: https://github.com/dependabot/cli/releases +[Draft a new release]: https://github.com/dependabot/cli/releases/new diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..26d7467 --- /dev/null +++ b/go.mod @@ -0,0 +1,36 @@ +module github.com/dependabot/cli + +go 1.19 + +require ( + github.com/MakeNowJust/heredoc v1.0.0 + github.com/docker/cli v20.10.18+incompatible + github.com/docker/docker v20.10.18+incompatible + github.com/moby/sys/signal v0.7.0 + github.com/sergi/go-diff v1.2.0 + github.com/spf13/cobra v1.5.0 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect + github.com/Microsoft/go-winio v0.5.2 // indirect + github.com/docker/distribution v2.8.1+incompatible // indirect + github.com/docker/go-connections v0.4.0 // indirect + github.com/docker/go-units v0.4.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/google/go-cmp v0.5.5 // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae // indirect + github.com/morikuni/aec v1.0.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.0.2 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/sirupsen/logrus v1.7.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/stretchr/testify v1.8.0 // indirect + golang.org/x/net v0.0.0-20201021035429-f5854403a974 // indirect + golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359 // indirect + golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect + gotest.tools/v3 v3.3.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..730015b --- /dev/null +++ b/go.sum @@ -0,0 +1,120 @@ +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= +github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= +github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= +github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= +github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docker/cli v20.10.18+incompatible h1:f/GQLsVpo10VvToRay2IraVA1wHz9KktZyjev3SIVDU= +github.com/docker/cli v20.10.18+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= +github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v20.10.18+incompatible h1:SN84VYXTBNGn92T/QwIRPlum9zfemfitN7pbsp26WSc= +github.com/docker/docker v20.10.18+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= +github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/moby/sys/signal v0.7.0 h1:25RW3d5TnQEoKvRbEKUGay6DCQ46IxAVTT9CUMgmsSI= +github.com/moby/sys/signal v0.7.0/go.mod h1:GQ6ObYZfqacOwTtlXvcmh9A26dVRul/hbOZn88Kg8Tg= +github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae h1:O4SWKdcHVCvYqyDV+9CJA1fcDN2L11Bule0iFy3YlAI= +github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM= +github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= +github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU= +github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359 h1:2B5p2L5IfGiD7+b9BOoRMC6DgObAVZV+Fsp050NqXik= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20220609170525-579cf78fd858 h1:Dpdu/EMxGMFgq0CeYMh4fazTD2vtlZRYE7wyynxJb9U= +golang.org/x/time v0.0.0-20220609170525-579cf78fd858/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= +gotest.tools/v3 v3.3.0 h1:MfDY1b1/0xN1CyMlQDac0ziEy9zJQd9CXBRRDHw2jJo= +gotest.tools/v3 v3.3.0/go.mod h1:Mcr9QNxkg0uMvy/YElmo4SpXgJKWgQvYrT7Kw5RzJ1A= diff --git a/internal/infra/cadetails.go b/internal/infra/cadetails.go new file mode 100644 index 0000000..c0367c1 --- /dev/null +++ b/internal/infra/cadetails.go @@ -0,0 +1,79 @@ +package infra + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "time" +) + +const ( + keySize = 2048 + keyExpiryYears = 2 +) + +var certSubject = pkix.Name{ + CommonName: "Dependabot Internal CA", + OrganizationalUnit: []string{"Dependabot"}, + Organization: []string{"GitHub Inc."}, + Locality: []string{"San Francisco"}, + Province: []string{"California"}, + Country: []string{"US"}, +} + +// GenerateCertificateAuthority generates a new proxy keypair CA +func GenerateCertificateAuthority() (CertificateAuthority, error) { + key, pemKey, err := generateKey() + if err != nil { + return CertificateAuthority{}, err + } + + pemCert, err := generateCert(key) + if err != nil { + return CertificateAuthority{}, err + } + + return CertificateAuthority{ + Cert: pemCert, + Key: pemKey, + }, nil +} + +func generateKey() (*rsa.PrivateKey, string, error) { + key, err := rsa.GenerateKey(rand.Reader, keySize) + if err != nil { + return nil, "", err + } + kb := &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(key), + } + return key, string(pem.EncodeToMemory(kb)), nil +} + +func generateCert(key *rsa.PrivateKey) (string, error) { + notBefore := time.Now() + notAfter := notBefore.AddDate(keyExpiryYears, 0, 0) + + template := x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: certSubject, + NotBefore: notBefore, + NotAfter: notAfter, + SignatureAlgorithm: x509.SHA256WithRSA, + BasicConstraintsValid: true, + IsCA: true, + } + cert, err := x509.CreateCertificate(rand.Reader, &template, &template, key.Public(), key) + if err != nil { + return "", err + } + cb := &pem.Block{ + Type: "CERTIFICATE", + Bytes: cert, + } + return string(pem.EncodeToMemory(cb)), nil +} diff --git a/internal/infra/cadetails_test.go b/internal/infra/cadetails_test.go new file mode 100644 index 0000000..a6155ca --- /dev/null +++ b/internal/infra/cadetails_test.go @@ -0,0 +1,19 @@ +package infra + +import ( + "strings" + "testing" +) + +func TestGenerateCaDetails(t *testing.T) { + ca, err := GenerateCertificateAuthority() + if err != nil { + t.Fatal(err.Error()) + } + if !strings.Contains(ca.Cert, "BEGIN CERTIFICATE") { + t.Errorf("Expected certificate to contain BEGIN CERTIFICATE, got %s", ca.Cert) + } + if !strings.Contains(ca.Key, "BEGIN RSA PRIVATE KEY") { + t.Errorf("Expected certificate to contain BEGIN RSA PRIVATE KEY, got %s", ca.Key) + } +} diff --git a/internal/infra/config.go b/internal/infra/config.go new file mode 100644 index 0000000..cd7cf28 --- /dev/null +++ b/internal/infra/config.go @@ -0,0 +1,44 @@ +package infra + +import ( + "encoding/json" + "fmt" + "os" +) + +// ConfigFilePath is the path to proxy config file. +const ConfigFilePath = "/config.json" + +// Config is the structure of the proxy's config file +type Config struct { + Credentials []map[string]string `json:"all_credentials"` + CA CertificateAuthority `json:"ca"` + ProxyAuth BasicAuthCredentials `json:"proxy_auth"` +} + +// CertificateAuthority includes the MITM CA certificate and private key +type CertificateAuthority struct { + Cert string `json:"cert"` + Key string `json:"key"` +} + +// BasicAuthCredentials represents credentials required for HTTP basic auth +type BasicAuthCredentials struct { + Username string `json:"username"` + Password string `json:"password"` +} + +// StoreProxyConfig saves the config to a temporary file, returning the path +func StoreProxyConfig(tmpPath string, config *Config) (string, error) { + tmp, err := os.CreateTemp(TempDir(tmpPath), "config.json") + if err != nil { + return "", fmt.Errorf("creating proxy config: %w", err) + } + defer tmp.Close() + + if err := json.NewEncoder(tmp).Encode(config); err != nil { + _ = os.RemoveAll(tmp.Name()) + return "", fmt.Errorf("encoding proxy config: %w", err) + } + return tmp.Name(), nil +} diff --git a/internal/infra/network.go b/internal/infra/network.go new file mode 100644 index 0000000..7ecc06a --- /dev/null +++ b/internal/infra/network.go @@ -0,0 +1,57 @@ +package infra + +import ( + "context" + "fmt" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/client" + "github.com/docker/docker/pkg/namesgenerator" +) + +type Networks struct { + NoInternet types.NetworkCreateResponse + Internet types.NetworkCreateResponse + cli *client.Client + noInternetName string + internetName string +} + +func NewNetworks(ctx context.Context, cli *client.Client) (*Networks, error) { + const bridge = "bridge" + + noInternetName := namesgenerator.GetRandomName(1) + noInternet, err := cli.NetworkCreate(ctx, noInternetName, types.NetworkCreate{ + Internal: true, + Driver: bridge, + }) + if err != nil { + return nil, fmt.Errorf("failed to create no-internet network: %w", err) + } + + internetName := namesgenerator.GetRandomName(1) + internet, err := cli.NetworkCreate(ctx, internetName, types.NetworkCreate{ + Driver: bridge, + }) + if err != nil { + return nil, fmt.Errorf("failed to create internet network: %w", err) + } + + return &Networks{ + cli: cli, + NoInternet: noInternet, + Internet: internet, + noInternetName: noInternetName, + internetName: internetName, + }, nil +} + +func (n *Networks) Close() error { + if err := n.cli.NetworkRemove(context.Background(), n.NoInternet.ID); err != nil { + return err + } + if err := n.cli.NetworkRemove(context.Background(), n.Internet.ID); err != nil { + return err + } + return nil +} diff --git a/internal/infra/password.go b/internal/infra/password.go new file mode 100644 index 0000000..7d7f65f --- /dev/null +++ b/internal/infra/password.go @@ -0,0 +1,14 @@ +package infra + +import ( + "crypto/rand" + "encoding/hex" +) + +const passwordLength = 24 + +func generatePassword() string { + b := make([]byte, passwordLength) + _, _ = rand.Read(b) + return hex.EncodeToString(b) +} diff --git a/internal/infra/proxy.go b/internal/infra/proxy.go new file mode 100644 index 0000000..e6245e6 --- /dev/null +++ b/internal/infra/proxy.go @@ -0,0 +1,145 @@ +package infra + +import ( + "context" + "fmt" + "math/rand" + "os" + "path/filepath" + "time" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/mount" + "github.com/docker/docker/api/types/network" + "github.com/docker/docker/client" + "github.com/docker/docker/pkg/namesgenerator" + "github.com/docker/docker/pkg/stdcopy" +) + +func init() { + // needed for namesgenerator.GetRandomName + rand.Seed(time.Now().UnixNano()) +} + +// ProxyImageName is the docker image used by the proxy +var ProxyImageName = "ghcr.io/github/dependabot-update-job-proxy/dependabot-update-job-proxy:latest" + +type Proxy struct { + cli *client.Client + containerID string + CertPath string + proxyConfigPath string + containerName string + url string +} + +func NewProxy(ctx context.Context, cli *client.Client, params *RunParams, nets ...types.NetworkCreateResponse) (*Proxy, error) { + // Generate secrets: + ca, err := GenerateCertificateAuthority() + if err != nil { + return nil, fmt.Errorf("failed to generate cert: %w", err) + } + certPath := filepath.Join(TempDir(params.TempDir), "cert.crt") + os.Remove(certPath) + + err = os.WriteFile(certPath, []byte(ca.Cert), 0777) + if err != nil { + return nil, fmt.Errorf("failed to write cert: %w", err) + } + + // Generate and write configuration to disk: + username := "cli-user" + password := generatePassword() + proxyConfig := &Config{ + Credentials: params.Creds, + ProxyAuth: BasicAuthCredentials{ + Username: username, + Password: password, + }, + CA: ca, + } + proxyConfigPath, err := StoreProxyConfig(params.TempDir, proxyConfig) + if err != nil { + return nil, fmt.Errorf("failed to store proxy config: %w", err) + } + + hostCfg := &container.HostConfig{ + AutoRemove: true, + Mounts: []mount.Mount{{ + Type: mount.TypeBind, + Source: proxyConfigPath, + Target: ConfigFilePath, + ReadOnly: true, + }}, + ExtraHosts: []string{ + "host.docker.internal:host-gateway", + }, + } + if params.CacheDir != "" { + _ = os.MkdirAll(params.CacheDir, 0744) + cacheDir, _ := filepath.Abs(params.CacheDir) + hostCfg.Mounts = append(hostCfg.Mounts, mount.Mount{ + Type: mount.TypeBind, + Source: cacheDir, + Target: "/cache", + }) + } + config := &container.Config{ + Image: ProxyImageName, + Env: []string{ + "JOB_ID=" + jobID, + "PROXY_CACHE=true", + }, + } + hostName := namesgenerator.GetRandomName(1) + proxyContainer, err := cli.ContainerCreate(ctx, config, hostCfg, nil, nil, hostName) + if err != nil { + return nil, fmt.Errorf("failed to create proxy container: %w", err) + } + for _, n := range nets { + if err := cli.NetworkConnect(ctx, n.ID, proxyContainer.ID, &network.EndpointSettings{}); err != nil { + return nil, fmt.Errorf("failed to connect to network: %w", err) + } + } + + if err := cli.ContainerStart(ctx, proxyContainer.ID, types.ContainerStartOptions{}); err != nil { + return nil, fmt.Errorf("failed to start container: %w", err) + } + + return &Proxy{ + cli: cli, + containerID: proxyContainer.ID, + containerName: hostName, + url: fmt.Sprintf("http://%s:%s@%s:1080", username, password, hostName), + CertPath: certPath, + proxyConfigPath: proxyConfigPath, + }, nil +} + +func (p *Proxy) TailLogs(ctx context.Context, cli *client.Client) { + out, err := cli.ContainerLogs(ctx, p.containerID, types.ContainerLogsOptions{ + ShowStdout: true, + ShowStderr: true, + Follow: true, + }) + if err != nil { + return + } + + _, _ = stdcopy.StdCopy(os.Stdout, os.Stderr, out) +} + +func (p *Proxy) Close() error { + defer os.Remove(p.CertPath) + defer os.Remove(p.proxyConfigPath) + + timeout := 5 * time.Second + _ = p.cli.ContainerStop(context.Background(), p.containerID, &timeout) + + err := p.cli.ContainerRemove(context.Background(), p.containerID, types.ContainerRemoveOptions{Force: true}) + if err != nil { + return fmt.Errorf("failed to remove proxy container: %w", err) + } + return nil +} diff --git a/internal/infra/proxy_test.go b/internal/infra/proxy_test.go new file mode 100644 index 0000000..ebe6235 --- /dev/null +++ b/internal/infra/proxy_test.go @@ -0,0 +1,16 @@ +package infra + +import ( + "testing" + + "github.com/docker/docker/pkg/namesgenerator" +) + +func TestSeed(t *testing.T) { + // ensure we're still seeding + a := namesgenerator.GetRandomName(1) + b := namesgenerator.GetRandomName(1) + if a == b { + t.Error("Not seeding math/rand") + } +} diff --git a/internal/infra/run.go b/internal/infra/run.go new file mode 100644 index 0000000..4a9ff3b --- /dev/null +++ b/internal/infra/run.go @@ -0,0 +1,233 @@ +package infra + +import ( + "context" + "encoding/base64" + "fmt" + "io" + "log" + "os" + "os/signal" + "syscall" + "time" + + "github.com/dependabot/cli/internal/server" + + "github.com/dependabot/cli/internal/model" + "github.com/docker/docker/api/types" + "github.com/docker/docker/client" + "gopkg.in/yaml.v3" +) + +type RunParams struct { + // job definition passed to the updater + Job *model.Job + // expectations asserted at the end of a test + Expected []model.Output + // credentials passed to the proxy + Creds []map[string]string + // local directory used for caching + CacheDir string + // write output to a file + Output string + // attempt to pull images if they aren't local? + PullImages bool + // run an interactive shell? + Debug bool + // Volumes are used to mount directories in Docker + Volumes []string + // Timeout specifies an optional maximum duration the CLI will run an update. + // If Timeout is <= 0 it will never time out. + Timeout time.Duration + // TempDir is the path to use as the temporary directory. + TempDir string +} + +func Run(params RunParams) error { + var ctx context.Context + var cancel func() + if params.Timeout > 0 { + ctx, cancel = context.WithTimeout(context.Background(), params.Timeout) + } else { + ctx, cancel = context.WithCancel(context.Background()) + } + signals := make(chan os.Signal, 1) + signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM) + go func() { + <-signals + cancel() + }() + + api := server.NewAPI(params.Expected) + defer api.Stop() + + token := os.Getenv("LOCAL_GITHUB_ACCESS_TOKEN") + if token != "" { + params.Creds = append(params.Creds, map[string]string{ + "type": "git_source", + "host": "github.com", + "username": "x-access-token", + "password": token, + }) + } + + var outFile *os.File + if params.Output != "" { + var err error + // Open a file for writing but don't truncate it yet since an error will delete the test. + // This is done before the test so if the dir isn't writable it doesn't waste time. + outFile, err = os.OpenFile(params.Output, os.O_RDWR|os.O_CREATE, 0666) + if err != nil { + return fmt.Errorf("failed to create output file: %w", err) + } + defer outFile.Close() + } + + if err := runContainers(ctx, params, api); err != nil { + return err + } + + api.Complete() + if outFile != nil { + if err := outFile.Truncate(0); err != nil { + return fmt.Errorf("failed to truncate output file: %w", err) + } + if params.Job.Source.Commit == nil { + // store the SHA we worked with for reproducible tests + params.Job.Source.Commit = api.Actual.Input.Job.Source.Commit + } + api.Actual.Input.Job = *params.Job + + // ignore conditions help make tests reproducible + // so they are generated if there aren't any yet + if len(api.Actual.Input.Job.IgnoreConditions) == 0 && api.Actual.Input.Job.PackageManager != "submodules" { + for _, out := range api.Actual.Output { + if out.Type == "create_pull_request" { + createPR, ok := out.Expect.Data.(model.CreatePullRequest) + if !ok { + return fmt.Errorf("failed to decode CreatePullRequest object") + } + + for _, dep := range createPR.Dependencies { + ignore := model.Condition{ + DependencyName: dep.Name, + VersionRequirement: fmt.Sprintf(">%v", *dep.Version), + Source: params.Output, + } + api.Actual.Input.Job.IgnoreConditions = append(api.Actual.Input.Job.IgnoreConditions, ignore) + } + } + } + } + if err := yaml.NewEncoder(outFile).Encode(api.Actual); err != nil { + return fmt.Errorf("failed to write output: %v", err) + } + } + if len(api.Errors) > 0 { + log.Println("The following errors occurred:") + + for _, e := range api.Errors { + log.Println(e) + } + + return fmt.Errorf("update failed expectations") + } + + return nil +} + +func runContainers(ctx context.Context, params RunParams, api *server.API) error { + cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) + if err != nil { + return fmt.Errorf("failed to create Docker client: %w", err) + } + + if params.PullImages { + err = pullImage(ctx, cli, ProxyImageName) + if err != nil { + return err + } + + err = pullImage(ctx, cli, UpdaterImageName) + if err != nil { + return err + } + } + + networks, err := NewNetworks(ctx, cli) + if err != nil { + return fmt.Errorf("failed to create networks: %w", err) + } + defer networks.Close() + + prox, err := NewProxy(ctx, cli, ¶ms, networks.NoInternet, networks.Internet) + if err != nil { + return err + } + defer prox.Close() + + // proxy logs interfere with debugging output + if !params.Debug { + go prox.TailLogs(ctx, cli) + } + + updater, err := NewUpdater(ctx, cli, networks, ¶ms, prox) + if err != nil { + return err + } + defer updater.Close() + + if err := updater.InstallCertificates(ctx); err != nil { + return err + } + + if params.Debug { + if err := updater.RunShell(ctx, prox.url, api.Port()); err != nil { + return err + } + } else { + if err := updater.RunUpdate(ctx, prox.url, api.Port()); err != nil { + return err + } + } + + return nil +} + +func pullImage(ctx context.Context, cli *client.Client, image string) error { + var inspect types.ImageInspect + + // check if image exists locally + inspect, _, err := cli.ImageInspectWithRaw(ctx, image) + + // pull image if necessary + if err != nil { + var privilegeFunc types.RequestPrivilegeFunc + token := os.Getenv("LOCAL_GITHUB_ACCESS_TOKEN") + if token != "" { + auth := base64.StdEncoding.EncodeToString([]byte("x:" + token)) + privilegeFunc = func() (string, error) { + return "Basic " + auth, nil + } + } + + log.Printf("pulling image: %s\n", image) + out, err := cli.ImagePull(ctx, image, types.ImagePullOptions{ + PrivilegeFunc: privilegeFunc, + }) + if err != nil { + return fmt.Errorf("failed to pull %v: %w", image, err) + } + _, _ = io.Copy(io.Discard, out) + out.Close() + + inspect, _, err = cli.ImageInspectWithRaw(ctx, image) + if err != nil { + return fmt.Errorf("failed to inspect %v: %w", image, err) + } + } + + log.Printf("using image %v at %s\n", image, inspect.ID) + + return nil +} diff --git a/internal/infra/run_test.go b/internal/infra/run_test.go new file mode 100644 index 0000000..0d2496d --- /dev/null +++ b/internal/infra/run_test.go @@ -0,0 +1,169 @@ +package infra + +import ( + "archive/tar" + "bytes" + "context" + "io" + "testing" + + "github.com/dependabot/cli/internal/model" + "github.com/docker/docker/api/types" + "github.com/docker/docker/client" +) + +func TestRun(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test") + } + + cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) + if err != nil { + t.Fatal(err) + } + + ctx := context.Background() + + var buildContext bytes.Buffer + tw := tar.NewWriter(&buildContext) + addFileToArchive(tw, "/Dockerfile", 0644, dockerFile) + addFileToArchive(tw, "/test_main.go", 0644, testMain) + tw.Close() + + UpdaterImageName = "test-updater" + resp, err := cli.ImageBuild(ctx, &buildContext, types.ImageBuildOptions{Tags: []string{UpdaterImageName}}) + if err != nil { + t.Fatal(err) + } + + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + + defer func() { + _, _ = cli.ImageRemove(ctx, UpdaterImageName, types.ImageRemoveOptions{}) + }() + + err = Run(RunParams{ + PullImages: true, + Job: &model.Job{ + PackageManager: "ecosystem", + Source: model.Source{ + Repo: "org/name", + }, + }, + TempDir: "/tmp", + }) + if err != nil { + t.Error(err) + } +} + +func addFileToArchive(tw *tar.Writer, path string, mode int64, content string) { + header := &tar.Header{ + Name: path, + Size: int64(len(content)), + Mode: mode, + } + + err := tw.WriteHeader(header) + if err != nil { + panic(err) + } + + _, err = io.Copy(tw, bytes.NewReader([]byte(content))) + if err != nil { + panic(err) + } +} + +const dockerFile = ` +FROM golang:1.19 + +# needed to run update-ca-certificates +RUN apt-get update && apt-get install -y ca-certificates +# cli will try to start as dependabot +RUN useradd -ms /bin/bash dependabot + +# need to be the user for permissions to work +USER dependabot +WORKDIR /home/dependabot +COPY *.go . +# cli runs bin/run (twice) to do an update, so put exe there +RUN go mod init cli_test && go mod tidy && go build -o bin/run +` + +const testMain = `package main + +import ( + "bytes" + "context" + "encoding/xml" + "log" + "net" + "net/http" + "os" + "os/exec" + "strings" + "sync" + "time" +) + +func main() { + if os.Args[1] == "update_files" { + return + } + var wg sync.WaitGroup + + // print the line that fails + log.SetFlags(log.Lshortfile) + + go connectivityCheck(&wg) + checkIfRoot() + proxyCheck() + + wg.Wait() +} + +func checkIfRoot() { + buf := &bytes.Buffer{} + cmd := exec.Command("id", "-u") + cmd.Stdout = buf + if err := cmd.Run(); err != nil { + log.Fatalln(err) + } + userID := strings.TrimSpace(buf.String()) + if userID == "0" { + log.Fatalln("User is root") + } +} + +func connectivityCheck(wg *sync.WaitGroup) { + wg.Add(1) + + var d net.Dialer + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + _, err := d.DialContext(ctx, "tcp", "1.1.1.1:22") + if err != nil && err.Error() != "dial tcp 1.1.1.1:22: i/o timeout" { + log.Fatalln(err) + } + if err == nil { + log.Fatalln("a connection shouldn't be possible") + } + + wg.Done() +} + +func proxyCheck() { + res, err := http.Get("https://example.com") + if err != nil { + log.Fatalln(err) + } + defer res.Body.Close() + var v any + if err = xml.NewDecoder(res.Body).Decode(&v); err != nil { + log.Fatalln(err) + } +} +` diff --git a/internal/infra/tempdir.go b/internal/infra/tempdir.go new file mode 100644 index 0000000..542bee2 --- /dev/null +++ b/internal/infra/tempdir.go @@ -0,0 +1,30 @@ +package infra + +import ( + "os" + "path" +) + +// TempDir centralizes where the temporary directory is created. +func TempDir(tmpPath string) string { + if path.IsAbs(tmpPath) { + mkdirAll(tmpPath) + return tmpPath + } + + wd, err := os.Getwd() + if err != nil { + panic(err) + } + + tmpPath = path.Join(wd, tmpPath) + mkdirAll(tmpPath) + return tmpPath +} + +func mkdirAll(tmpPath string) { + err := os.MkdirAll(tmpPath, 0700) + if err != nil { + panic(err) + } +} diff --git a/internal/infra/tempdir_test.go b/internal/infra/tempdir_test.go new file mode 100644 index 0000000..6e4f71d --- /dev/null +++ b/internal/infra/tempdir_test.go @@ -0,0 +1,18 @@ +package infra + +import ( + "strings" + "testing" +) + +func TestTempDir(t *testing.T) { + tmp := TempDir("tmp") + if !strings.HasPrefix(tmp, "/") { + t.Errorf("TempDir() = %v, want absolute path", tmp) + } + + tmp = TempDir("/tmp/testing") + if tmp != "/tmp/testing" { + t.Errorf("TempDir() = %v, want /tmp/testing", tmp) + } +} diff --git a/internal/infra/tty.go b/internal/infra/tty.go new file mode 100644 index 0000000..3a2f070 --- /dev/null +++ b/internal/infra/tty.go @@ -0,0 +1,99 @@ +package infra + +import ( + "context" + "log" + "os" + gosignal "os/signal" + "runtime" + "time" + + "github.com/docker/cli/cli/streams" + "github.com/docker/docker/api/types" + "github.com/docker/docker/client" + "github.com/moby/sys/signal" +) + +// NOTE: This file was copied from https://github.com/docker/cli/command/container/tty.go +// and adapted to not pull in the whole docker CLI object. + +// resizeTtyTo resizes tty to specific height and width +func resizeTtyTo(ctx context.Context, c *client.Client, id string, height, width uint, isExec bool) error { + if height == 0 && width == 0 { + return nil + } + + options := types.ResizeOptions{ + Height: height, + Width: width, + } + + var err error + if isExec { + err = c.ContainerExecResize(ctx, id, options) + } else { + err = c.ContainerResize(ctx, id, options) + } + + if err != nil { + log.Printf("Error resize: %s\r", err) + } + return err +} + +// resizeTty is to resize the tty with cli out's tty size +func resizeTty(ctx context.Context, out *streams.Out, cli *client.Client, id string, isExec bool) error { + height, width := out.GetTtySize() + return resizeTtyTo(ctx, cli, id, height, width, isExec) +} + +// initTtySize is to init the tty's size to the same as the window, if there is an error, it will retry 10 times. +func initTtySize(ctx context.Context, out *streams.Out, cli *client.Client, id string, isExec bool, resizeTtyFunc func(context.Context, *streams.Out, *client.Client, string, bool) error) { + rttyFunc := resizeTtyFunc + if rttyFunc == nil { + rttyFunc = resizeTty + } + if err := rttyFunc(ctx, out, cli, id, isExec); err != nil { + go func() { + var err error + for retry := 0; retry < 10; retry++ { + time.Sleep(time.Duration(retry+1) * 10 * time.Millisecond) + if err = rttyFunc(ctx, out, cli, id, isExec); err == nil { + break + } + } + if err != nil { + log.Println("failed to resize tty, using default size") + } + }() + } +} + +// MonitorTtySize updates the container tty size when the terminal tty changes size +func MonitorTtySize(ctx context.Context, out *streams.Out, cli *client.Client, id string, isExec bool) error { + initTtySize(ctx, out, cli, id, isExec, resizeTty) + if runtime.GOOS == "windows" { + go func() { + prevH, prevW := out.GetTtySize() + for { + time.Sleep(time.Millisecond * 250) + h, w := out.GetTtySize() + + if prevW != w || prevH != h { + _ = resizeTty(ctx, out, cli, id, isExec) + } + prevH = h + prevW = w + } + }() + } else { + sigchan := make(chan os.Signal, 1) + gosignal.Notify(sigchan, signal.SIGWINCH) + go func() { + for range sigchan { + _ = resizeTty(ctx, out, cli, id, isExec) + } + }() + } + return nil +} diff --git a/internal/infra/updater.go b/internal/infra/updater.go new file mode 100644 index 0000000..26ac03d --- /dev/null +++ b/internal/infra/updater.go @@ -0,0 +1,323 @@ +package infra + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/dependabot/cli/internal/model" + "github.com/docker/cli/cli/streams" + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/mount" + "github.com/docker/docker/api/types/network" + "github.com/docker/docker/client" + "github.com/docker/docker/pkg/stdcopy" +) + +const jobID = "cli" +const dependabot = "dependabot" + +// UpdaterImageName is the docker image used by the updater +var UpdaterImageName = "ghcr.io/dependabot/dependabot-updater:latest" + +const ( + fetcherOutputFile = "output.json" + fetcherRepoDir = "repo" + guestInputDir = "/home/dependabot/dependabot-updater/job.json" + guestOutputDir = "/home/dependabot/dependabot-updater/output" + guestRepoDir = "/home/dependabot/dependabot-updater/repo" +) + +type Updater struct { + cli *client.Client + containerID string + outputDir string + RepoDir string + inputPath string +} + +const ( + certsPath = "/etc/ssl/certs" + dbotCert = "/usr/local/share/ca-certificates/dbot-ca.crt" +) + +// NewUpdater starts the update container interactively running /bin/sh, so it does not stop. +func NewUpdater(ctx context.Context, cli *client.Client, net *Networks, params *RunParams, prox *Proxy) (*Updater, error) { + f := FileFetcherJobFile{Job: *params.Job} + inputPath, err := WriteContainerInput(params.TempDir, f) + if err != nil { + return nil, fmt.Errorf("failed to write fetcher input: %w", err) + } + + containerCfg := &container.Config{ + User: dependabot, + Image: UpdaterImageName, + Cmd: []string{"/bin/sh"}, + Tty: true, // prevent container from stopping + } + outputDir, err := SetupOutputDir(params.TempDir) + if err != nil { + return nil, fmt.Errorf("failed to setup fetcher output dir: %w", err) + } + + repoDir := filepath.Join(outputDir, fetcherRepoDir) + hostCfg := &container.HostConfig{ + Mounts: []mount.Mount{{ + Type: mount.TypeBind, + Source: inputPath, + Target: guestInputDir, + }, { + Type: mount.TypeBind, + Source: outputDir, + Target: guestOutputDir, + }, { + Type: mount.TypeBind, + Source: prox.CertPath, + Target: dbotCert, + ReadOnly: true, + }}, + } + for _, v := range params.Volumes { + local, remote, _ := strings.Cut(v, ":") + hostCfg.Mounts = append(hostCfg.Mounts, mount.Mount{ + Type: mount.TypeBind, + Source: local, + Target: remote, + }) + } + netCfg := &network.NetworkingConfig{ + EndpointsConfig: map[string]*network.EndpointSettings{ + net.noInternetName: { + NetworkID: net.NoInternet.ID, + }, + }, + } + + updaterContainer, err := cli.ContainerCreate(ctx, containerCfg, hostCfg, netCfg, nil, "") + if err != nil { + return nil, fmt.Errorf("failed to create updater container: %w", err) + } + + if err := cli.ContainerStart(ctx, updaterContainer.ID, types.ContainerStartOptions{}); err != nil { + return nil, fmt.Errorf("failed to start updater container: %w", err) + } + + updater := &Updater{ + cli: cli, + containerID: updaterContainer.ID, + outputDir: outputDir, + RepoDir: repoDir, + inputPath: inputPath, + } + + return updater, nil +} + +// InstallCertificates runs update-ca-certificates as root, blocks until complete. +func (u *Updater) InstallCertificates(ctx context.Context) error { + execCreate, err := u.cli.ContainerExecCreate(ctx, u.containerID, types.ExecConfig{ + AttachStdout: true, + AttachStderr: true, + User: "root", + Cmd: []string{"update-ca-certificates"}, + }) + if err != nil { + return fmt.Errorf("failed to create exec: %w", err) + } + + execResp, err := u.cli.ContainerExecAttach(ctx, execCreate.ID, types.ExecStartCheck{}) + if err != nil { + return fmt.Errorf("failed to start exec: %w", err) + } + defer execResp.Close() + + // block until certs are installed or ctl-c + ch := make(chan struct{}) + go func() { + _, _ = stdcopy.StdCopy(os.Stdout, os.Stderr, execResp.Reader) + ch <- struct{}{} + }() + + select { + case <-ctx.Done(): + return ctx.Err() + case <-ch: + } + + return nil +} + +func userEnv(proxyURL string, apiPort int) []string { + return []string{ + fmt.Sprintf("http_proxy=%s", proxyURL), + fmt.Sprintf("HTTP_PROXY=%s", proxyURL), + fmt.Sprintf("https_proxy=%s", proxyURL), + fmt.Sprintf("HTTPS_PROXY=%s", proxyURL), + fmt.Sprintf("DEPENDABOT_JOB_ID=%v", jobID), + fmt.Sprintf("DEPENDABOT_JOB_TOKEN=%v", ""), + fmt.Sprintf("DEPENDABOT_JOB_PATH=%v", guestInputDir), + fmt.Sprintf("DEPENDABOT_OUTPUT_PATH=%v", filepath.Join(guestOutputDir, fetcherOutputFile)), + fmt.Sprintf("DEPENDABOT_REPO_CONTENTS_PATH=%v", guestRepoDir), + fmt.Sprintf("DEPENDABOT_API_URL=http://host.docker.internal:%v", apiPort), + fmt.Sprintf("SSL_CERT_FILE=%v/ca-certificates.crt", certsPath), + "UPDATER_ONE_CONTAINER=true", + "UPDATER_DETERMINISTIC=true", + } +} + +// RunShell executes an interactive shell, blocks until complete. +func (u *Updater) RunShell(ctx context.Context, proxyURL string, apiPort int) error { + execCreate, err := u.cli.ContainerExecCreate(ctx, u.containerID, types.ExecConfig{ + AttachStdin: true, + AttachStdout: true, + AttachStderr: true, + Tty: true, + User: dependabot, + Env: userEnv(proxyURL, apiPort), + Cmd: []string{"/bin/bash"}, + }) + if err != nil { + return fmt.Errorf("failed to create exec: %w", err) + } + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + execResp, err := u.cli.ContainerExecAttach(ctx, execCreate.ID, types.ExecStartCheck{}) + if err != nil { + return fmt.Errorf("failed to start exec: %w", err) + } + + ch := make(chan struct{}) + + out := streams.NewOut(os.Stdout) + _ = out.SetRawTerminal() + in := streams.NewIn(os.Stdin) + _ = in.SetRawTerminal() + defer func() { + out.RestoreTerminal() + in.RestoreTerminal() + in.Close() + }() + + go func() { + _, _ = stdcopy.StdCopy(out, os.Stderr, execResp.Reader) + ch <- struct{}{} + }() + + go func() { + _, _ = io.Copy(execResp.Conn, in) + ch <- struct{}{} + }() + + _ = MonitorTtySize(ctx, out, u.cli, execCreate.ID, true) + + select { + case <-ctx.Done(): + return ctx.Err() + case <-ch: + cancel() + } + + return nil +} + +// RunUpdate executes the update scripts as the dependabot user, blocks until complete. +func (u *Updater) RunUpdate(ctx context.Context, proxyURL string, apiPort int) error { + execCreate, err := u.cli.ContainerExecCreate(ctx, u.containerID, types.ExecConfig{ + AttachStdout: true, + AttachStderr: true, + User: dependabot, + Env: userEnv(proxyURL, apiPort), + Cmd: []string{"/bin/sh", "-c", "bin/run fetch_files && bin/run update_files"}, + }) + if err != nil { + return fmt.Errorf("failed to create exec: %w", err) + } + + execResp, err := u.cli.ContainerExecAttach(ctx, execCreate.ID, types.ExecStartCheck{}) + if err != nil { + return fmt.Errorf("failed to start exec: %w", err) + } + // blocks until update is complete or ctl-c + ch := make(chan struct{}) + go func() { + _, _ = stdcopy.StdCopy(os.Stdout, os.Stderr, execResp.Reader) + ch <- struct{}{} + }() + + select { + case <-ctx.Done(): + return ctx.Err() + case <-ch: + } + + return nil +} + +// Wait blocks until the condition is true. +func (u *Updater) Wait(ctx context.Context, condition container.WaitCondition) error { + wait, errCh := u.cli.ContainerWait(ctx, u.containerID, condition) + select { + case v := <-wait: + if v.StatusCode != 0 { + return fmt.Errorf("updater exited with code: %v", v.StatusCode) + } + case err := <-errCh: + return fmt.Errorf("updater error while waiting: %w", err) + } + return nil +} + +// Close kills and deletes the container and deletes updater mount paths related to the run. +func (u *Updater) Close() error { + defer os.Remove(u.inputPath) + defer os.RemoveAll(u.outputDir) + + return u.cli.ContainerRemove(context.Background(), u.containerID, types.ContainerRemoveOptions{ + Force: true, + }) +} + +// FileFetcherJobFile is the payload passed to file updater containers. +type FileFetcherJobFile struct { + Job model.Job `json:"job"` +} + +func WriteContainerInput(tempDir string, input interface{}) (string, error) { + // create file: + out, err := os.CreateTemp(TempDir(tempDir), "containers-input-*.json") + if err != nil { + return "", fmt.Errorf("creating container input: %w", err) + } + defer out.Close() + fn := out.Name() + + // TODO why does actions require this? + _ = os.Chmod(fn, 0777) + + // fill with json: + if err := json.NewEncoder(out).Encode(input); err != nil { + _ = os.RemoveAll(fn) + return "", fmt.Errorf("writing container input: %w", err) + } + return fn, nil +} + +func SetupOutputDir(tempDir string) (string, error) { + outputDir, err := os.MkdirTemp(TempDir(tempDir), "fetcher-output-") + if err != nil { + return "", fmt.Errorf("creating output tempdir: %w", err) + } + + repoDir := filepath.Join(outputDir, fetcherRepoDir) + if err := os.Mkdir(repoDir, 0777); err != nil { + return "", fmt.Errorf("creating repo tempdir: %w", err) + } + _ = os.Chmod(outputDir, 0777) + return outputDir, nil +} diff --git a/internal/model/job.go b/internal/model/job.go new file mode 100644 index 0000000..c3d7cb8 --- /dev/null +++ b/internal/model/job.go @@ -0,0 +1,86 @@ +package model + +// Job is the data that is passed to the updater. +type Job struct { + PackageManager string `json:"package-manager" yaml:"package-manager"` + AllowedUpdates []Allowed `json:"allowed-updates" yaml:"allowed-updates,omitempty"` + Dependencies []string `json:"dependencies" yaml:"dependencies,omitempty"` + ExistingPullRequests [][]ExistingPR `json:"existing-pull-requests" yaml:"existing-pull-requests,omitempty"` + Experiments Experiment `json:"experiments" yaml:"experiments,omitempty"` + IgnoreConditions []Condition `json:"ignore-conditions" yaml:"ignore-conditions,omitempty"` + LockfileOnly bool `json:"lockfile-only" yaml:"lockfile-only,omitempty"` + RequirementsUpdateStrategy *string `json:"requirements-update-strategy" yaml:"requirements-update-strategy,omitempty"` + SecurityAdvisories []Advisory `json:"security-advisories" yaml:"security-advisories,omitempty"` + SecurityUpdatesOnly bool `json:"security-updates-only" yaml:"security-updates-only,omitempty"` + Source Source `json:"source" yaml:"source"` + UpdateSubdependencies bool `json:"update-subdependencies" yaml:"update-subdependencies,omitempty"` + UpdatingAPullRequest bool `json:"updating-a-pull-request" yaml:"updating-a-pull-request,omitempty"` + VendorDependencies bool `json:"vendor-dependencies" yaml:"vendor-dependencies,omitempty"` + RejectExternalCode bool `json:"reject-external-code" yaml:"reject-external-code,omitempty"` + CommitMessageOptions *CommitOptions `json:"commit-message-options" yaml:"commit-message-options,omitempty"` + CredentialsMetadata []map[string]string `json:"credentials-metadata" yaml:"credentials-metadata,omitempty"` + MaxUpdaterRunTime int `json:"max-updater-run-time" yaml:"max-updater-run-time,omitempty"` +} + +// Source is a reference to some source code +type Source struct { + Provider string `json:"provider" yaml:"provider,omitempty"` + Repo string `json:"repo" yaml:"repo,omitempty"` + Directory string `json:"directory" yaml:"directory,omitempty"` + Branch *string `json:"branch" yaml:"branch,omitempty"` + Commit *string `json:"commit" yaml:"commit,omitempty"` + + Hostname *string `json:"hostname" yaml:"hostname,omitempty"` // Must be provided if APIEndpoint is + APIEndpoint *string `json:"api-endpoint" yaml:"api-endpoint,omitempty"` // Must be provided if Hostname is +} + +type ExistingPR struct { + DependencyName string `json:"dependency-name" yaml:"dependency-name"` + DependencyVersion string `json:"dependency-version" yaml:"dependency-version"` +} + +type Allowed struct { + DependencyType string `json:"dependency-type,omitempty" yaml:"dependency-type,omitempty"` + DependencyName string `json:"dependency-name,omitempty" yaml:"dependency-name,omitempty"` + UpdateType string `json:"update-type,omitempty" yaml:"update-type,omitempty"` +} + +type Condition struct { + DependencyName string `json:"dependency-name" yaml:"dependency-name"` + Source string `json:"source,omitempty" yaml:"source,omitempty"` + VersionRequirement string `json:"version-requirement,omitempty" yaml:"version-requirement,omitempty"` +} + +type Advisory struct { + DependencyName string `json:"dependency-name" yaml:"dependency-name"` + AffectedVersions []string `json:"affected-versions" yaml:"affected-versions"` + PatchedVersions []string `json:"patched-versions" yaml:"patched-versions"` + UnaffectedVersions []string `json:"unaffected-versions" yaml:"unaffected-versions"` +} + +type Dependency struct { + Name string `json:"name"` + PreviousRequirements *[]Requirement `json:"previous-requirements,omitempty" yaml:"previous-requirements,omitempty"` + PreviousVersion string `json:"previous-version,omitempty" yaml:"previous-version,omitempty"` + Requirements []Requirement `json:"requirements"` + Version *string `json:"version" yaml:"version"` +} + +type Requirement struct { + File string `json:"file" yaml:"file"` + Groups []any `json:"groups" yaml:"groups"` + Metadata *map[string]any `json:"metadata,omitempty" yaml:"metadata,omitempty"` + Requirement *string `json:"requirement" yaml:"requirement"` + Source *RequirementSource `json:"source" yaml:"source"` + Version string `json:"version,omitempty" yaml:"version,omitempty"` + PreviousVersion string `json:"previous-version,omitempty" yaml:"previous-version,omitempty"` +} + +type RequirementSource map[string]any +type Experiment map[string]any + +type CommitOptions struct { + Prefix string `json:"prefix,omitempty" yaml:"prefix,omitempty"` + PrefixDevelopment string `json:"prefix-development,omitempty" yaml:"prefix-development,omitempty"` + IncludeScope *string `json:"include-scope,omitempty" yaml:"include-scope,omitempty"` +} diff --git a/internal/model/job_test.go b/internal/model/job_test.go new file mode 100644 index 0000000..0fe52fb --- /dev/null +++ b/internal/model/job_test.go @@ -0,0 +1,360 @@ +package model + +import ( + "reflect" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +func TestInput(t *testing.T) { + var input Input + if err := yaml.Unmarshal([]byte(exampleJob), &input); err != nil { + t.Fatal(err) + } + var input2 map[string]map[string]any + if err := yaml.Unmarshal([]byte(exampleJob), &input2); err != nil { + t.Fatal(err) + } + compareMap(t, "job", input2["job"], input.Job) +} + +func compareMap(t *testing.T, parent string, expected map[string]any, actual interface{}) { + actualType := reflect.TypeOf(actual) + if actualType.Kind() != reflect.Struct { + // Some fields like Experiments are not modeled as structs + // so there's nothing to do here! + return + } + fields := actualType.NumField() + for key, value := range expected { + // Walk the struct and find the field with the yaml tag that matches the map key name. + fieldIndex := -1 + for i := 0; i < fields; i++ { + field := actualType.Field(i) + name := yamlTagCleaner(field.Tag.Get("yaml")) + if key == name { + fieldIndex = i + break + } + } + if fieldIndex < 0 { + t.Errorf("key is not mapped: %s->%s", parent, key) + } else { + // Now we can compare the values to recur into nested maps. + actualValue := reflect.ValueOf(actual).Field(fieldIndex) + + switch expectedValue := value.(type) { + case map[string]any: + // Recurse to find more mismatches. + structField := actualType.Field(fieldIndex) + name := yamlTagCleaner(structField.Tag.Get("yaml")) + compareMap(t, parent+"->"+name, expectedValue, actualValue.Interface()) + case []any: + // Also check structs that are in arrays. + for _, v := range expectedValue { + switch v := v.(type) { + case map[string]any: + structField := actualType.Field(fieldIndex) + name := yamlTagCleaner(structField.Tag.Get("yaml")) + compareMap(t, parent+"->"+name, v, actualValue.Interface()) + } + } + default: + // Values not matching isn't really a huge concern, but we've come this far. + compareValues(t, parent, key, expectedValue, actualValue) + } + } + } +} + +func compareValues(t *testing.T, parent, key string, expected any, actual reflect.Value) { + if expected == nil && actual.IsNil() { + return + } + if actual.Kind() == reflect.Pointer { + actual = actual.Elem() + } + if !reflect.DeepEqual(expected, actual.Interface()) { + t.Errorf("values are not equal: %s->%s expected %v got %v", parent, key, expected, actual.Interface()) + } +} + +func yamlTagCleaner(tag string) string { + return strings.ReplaceAll(tag, ",omitempty", "") +} + +const exampleJob = `--- +job: + package-manager: npm_and_yarn + source: + provider: github + repo: dependabot/dependabot-core + directory: "/npm_and_yarn/helpers" + branch: + api-endpoint: https://api.github.com/ + hostname: github.com + dependencies: + - got + existing-pull-requests: + - - dependency-name: npm + dependency-version: 6.14.0 + - - dependency-name: prettier + dependency-version: 2.0.1 + - - dependency-name: semver + dependency-version: 7.2.1 + - - dependency-name: semver + dependency-version: 7.2.2 + - - dependency-name: semver + dependency-version: 7.3.0 + - - dependency-name: prettier + dependency-version: 2.0.5 + - - dependency-name: jest + dependency-version: 25.5.0 + - - dependency-name: jest + dependency-version: 25.5.3 + - - dependency-name: jest + dependency-version: 25.5.4 + - - dependency-name: jest + dependency-version: 26.0.0 + - - dependency-name: npm + dependency-version: 6.14.5 + - - dependency-name: jest + dependency-version: 26.0.1 + - - dependency-name: eslint + dependency-version: 7.0.0 + - - dependency-name: eslint + dependency-version: 7.1.0 + - - dependency-name: npm + dependency-version: 6.14.6 + - - dependency-name: lodash + dependency-version: 4.17.19 + - - dependency-name: eslint + dependency-version: 7.6.0 + - - dependency-name: npm + dependency-version: 6.14.7 + - - dependency-name: jest + dependency-version: 26.3.0 + - - dependency-name: jest + dependency-version: 26.4.1 + - - dependency-name: prettier + dependency-version: 2.1.0 + - - dependency-name: eslint + dependency-version: 7.8.0 + - - dependency-name: eslint + dependency-version: 7.13.0 + - - dependency-name: "@npmcli/arborist" + dependency-version: 1.0.11 + - - dependency-name: prettier + dependency-version: 2.2.0 + - - dependency-name: eslint + dependency-version: 7.19.0 + - - dependency-name: "@npmcli/arborist" + dependency-version: 2.2.5 + - - dependency-name: eslint + dependency-version: 7.21.0 + - - dependency-name: npm + dependency-version: 7.6.1 + - - dependency-name: "@npmcli/arborist" + dependency-version: 2.2.7 + - - dependency-name: "@npmcli/arborist" + dependency-version: 2.2.8 + - - dependency-name: semver + dependency-version: 7.3.5 + - - dependency-name: "@npmcli/arborist" + dependency-version: 2.4.4 + - - dependency-name: detect-indent + dependency-version: 6.1.0 + - - dependency-name: prettier + dependency-version: 2.3.1 + - - dependency-name: jest + dependency-version: 27.0.6 + - - dependency-name: detect-indent + dependency-version: 7.0.0 + - - dependency-name: "@npmcli/arborist" + dependency-version: 2.8.1 + - - dependency-name: "@npmcli/arborist" + dependency-version: 2.8.2 + - - dependency-name: npm + dependency-version: 6.14.15 + - - dependency-name: jest + dependency-version: 27.1.0 + - - dependency-name: "@npmcli/arborist" + dependency-version: 2.8.3 + - - dependency-name: jest + dependency-version: 27.1.1 + - - dependency-name: prettier + dependency-version: 2.4.0 + - - dependency-name: jest + dependency-version: 27.2.0 + - - dependency-name: jest + dependency-version: 27.2.1 + - - dependency-name: jest + dependency-version: 27.2.2 + - - dependency-name: "@npmcli/arborist" + dependency-version: 2.9.0 + - - dependency-name: jest + dependency-version: 27.2.3 + - - dependency-name: "@npmcli/arborist" + dependency-version: 2.10.0 + - - dependency-name: npm + dependency-version: 8.0.0 + - - dependency-name: "@npmcli/arborist" + dependency-version: 4.0.0 + - - dependency-name: eslint + dependency-version: 8.0.1 + - - dependency-name: "@npmcli/arborist" + dependency-version: 4.0.1 + - - dependency-name: npm + dependency-version: 8.1.0 + - - dependency-name: jest + dependency-version: 27.3.0 + - - dependency-name: jest + dependency-version: 27.3.1 + - - dependency-name: "@npmcli/arborist" + dependency-version: 4.0.2 + - - dependency-name: npm + dependency-version: 8.1.1 + - - dependency-name: eslint + dependency-version: 8.1.0 + - - dependency-name: "@npmcli/arborist" + dependency-version: 4.0.3 + - - dependency-name: npm + dependency-version: 8.1.2 + - - dependency-name: "@npmcli/arborist" + dependency-version: 4.0.4 + - - dependency-name: npm + dependency-version: 8.1.3 + - - dependency-name: eslint + dependency-version: 8.2.0 + - - dependency-name: npm + dependency-version: 8.1.4 + - - dependency-name: prettier + dependency-version: 2.5.0 + - - dependency-name: jest + dependency-version: 27.4.0 + - - dependency-name: jest + dependency-version: 27.4.2 + - - dependency-name: "@npmcli/arborist" + dependency-version: 4.1.0 + - - dependency-name: eslint + dependency-version: 8.4.0 + - - dependency-name: jest + dependency-version: 27.4.4 + - - dependency-name: jest + dependency-version: 27.4.6 + - - dependency-name: "@npmcli/arborist" + dependency-version: 4.1.2 + - - dependency-name: "@npmcli/arborist" + dependency-version: 4.2.0 + - - dependency-name: "@npmcli/arborist" + dependency-version: 4.3.0 + - - dependency-name: eslint + dependency-version: 8.8.0 + - - dependency-name: jest + dependency-version: 27.5.0 + - - dependency-name: "@npmcli/arborist" + dependency-version: 4.3.1 + - - dependency-name: eslint + dependency-version: 8.9.0 + - - dependency-name: eslint-config-prettier + dependency-version: 8.4.0 + - - dependency-name: "@npmcli/arborist" + dependency-version: 5.0.0 + - - dependency-name: eslint + dependency-version: 8.10.0 + - - dependency-name: "@npmcli/arborist" + dependency-version: 5.0.1 + - - dependency-name: "@npmcli/arborist" + dependency-version: 5.0.2 + - - dependency-name: eslint + dependency-version: 8.11.0 + - - dependency-name: prettier + dependency-version: 2.6.1 + - - dependency-name: eslint + dependency-version: 8.14.0 + - - dependency-name: jest + dependency-version: 28.0.0 + - - dependency-name: jest + dependency-version: 28.0.1 + - - dependency-name: "@npmcli/arborist" + dependency-version: 5.1.1 + - - dependency-name: jest + dependency-version: 28.0.2 + - - dependency-name: jest + dependency-version: 28.0.3 + - - dependency-name: eslint + dependency-version: 8.16.0 + - - dependency-name: "@npmcli/arborist" + dependency-version: 5.2.1 + - - dependency-name: eslint + dependency-version: 8.17.0 + - - dependency-name: prettier + dependency-version: 2.7.0 + - - dependency-name: "@npmcli/arborist" + dependency-version: 5.2.2 + - - dependency-name: "@npmcli/arborist" + dependency-version: 5.3.0 + - - dependency-name: eslint + dependency-version: 8.20.0 + - - dependency-name: "@npmcli/arborist" + dependency-version: 5.4.0 + - - dependency-name: "@npmcli/arborist" + dependency-version: 5.5.0 + - - dependency-name: jest + dependency-version: 29.0.1 + - - dependency-name: eslint + dependency-version: 8.23.0 + - - dependency-name: jest + dependency-version: 29.0.2 + - - dependency-name: jest + dependency-version: 29.0.3 + - - dependency-name: detect-indent + dependency-version: 7.0.1 + - - dependency-name: got + dependency-removed: true + - dependency-name: npm + dependency-version: 8.19.2 + - - dependency-name: "@npmcli/arborist" + dependency-version: 5.6.2 + updating-a-pull-request: false + lockfile-only: false + update-subdependencies: false + ignore-conditions: + - dependency-name: npm + version-requirement: + update-types: + - version-update:semver-major + source: ".github/dependabot.yml" + - dependency-name: npm + version-requirement: ">= 7.a, < 8" + update-types: + source: "@dependabot ignore command" + requirements-update-strategy: + allowed-updates: + - dependency-type: direct + update-type: all + credentials-metadata: + - type: git_source + host: github.com + security-advisories: + - dependency-name: got + patched-versions: [] + unaffected-versions: [] + affected-versions: + - "< 11.8.5" + - ">= 12.0.0 < 12.1.0" + max-updater-run-time: 1800 + vendor-dependencies: false + experiments: + build-pull-request-message: true + npm-transitive-dependency-removal: true + npm-transitive-security-updates: true + reject-external-code: false + commit-message-options: + prefix: + prefix-development: + include-scope: + security-updates-only: true +` diff --git a/internal/model/scenario.go b/internal/model/scenario.go new file mode 100644 index 0000000..8246ade --- /dev/null +++ b/internal/model/scenario.go @@ -0,0 +1,25 @@ +package model + +// Scenario is a way to test a job by asserting the outputs. +type Scenario struct { + // Input is the input parameters + Input Input `yaml:"input"` + // Output is the list of expected outputs + Output []Output `yaml:"output,omitempty"` +} + +// Input is the input to a job +type Input struct { + // Job is the data given to the updater + Job Job `yaml:"job"` + // Credentials is the registry info and tokens to pass to the Proxy + Credentials []map[string]string `yaml:"credentials,omitempty"` +} + +// Output is the expected output given the inputs +type Output struct { + // Type is the kind of data to be checked, e.g. update_dependency_list, create_pull_request, etc + Type string `yaml:"type"` + // Expect is the data expected to be sent + Expect UpdateWrapper `yaml:"expect"` +} diff --git a/internal/model/update.go b/internal/model/update.go new file mode 100644 index 0000000..4d39d27 --- /dev/null +++ b/internal/model/update.go @@ -0,0 +1,53 @@ +package model + +type UpdateWrapper struct { + Data any `json:"data" yaml:"data"` +} + +type UpdateDependencyList struct { + Dependencies []Dependency `json:"dependencies" yaml:"dependencies"` + DependencyFiles []string `json:"dependency_files" yaml:"dependency_files"` +} + +type CreatePullRequest struct { + BaseCommitSha string `json:"base-commit-sha" yaml:"base-commit-sha"` + Dependencies []Dependency `json:"dependencies" yaml:"dependencies"` + UpdatedDependencyFiles []DependencyFile `json:"updated-dependency-files" yaml:"updated-dependency-files"` + PRTitle string `json:"pr-title" yaml:"pr-title,omitempty"` + PRBody string `json:"pr-body" yaml:"pr-body,omitempty"` + CommitMessage string `json:"commit-message" yaml:"commit-message,omitempty"` +} + +type UpdatePullRequest struct { + BaseCommitSha string `json:"base-commit-sha" yaml:"base-commit-sha"` + DependencyNames []string `json:"dependency-names" yaml:"dependency-names"` + UpdatedDependencyFiles []DependencyFile `json:"updated-dependency-files" yaml:"updated-dependency-files"` + PRTitle string `json:"pr-title" yaml:"pr-title,omitempty"` + PRBody string `json:"pr-body" yaml:"pr-body,omitempty"` + CommitMessage string `json:"commit-message" yaml:"commit-message,omitempty"` +} + +type DependencyFile struct { + Content string `json:"content" yaml:"content"` + ContentEncoding string `json:"content_encoding" yaml:"content_encoding"` + Deleted bool `json:"deleted" yaml:"deleted"` + Directory string `json:"directory" yaml:"directory"` + Name string `json:"name" yaml:"name"` + Operation string `json:"operation" yaml:"operation"` + SupportFile bool `json:"support_file" yaml:"support_file"` + Type string `json:"type" yaml:"type"` +} + +type ClosePullRequest struct { + DependencyNames []string `json:"dependency-names" yaml:"dependency-names"` + Reason string `json:"reason" yaml:"reason"` +} + +type MarkAsProcessed struct { + BaseCommitSha string `json:"base-commit-sha" yaml:"base-commit-sha"` +} + +type RecordPackageManagerVersion struct { + Ecosystem string `json:"ecosystem" yaml:"ecosystem"` + PackageManagers map[string]any `json:"package-managers" yaml:"package-managers"` +} diff --git a/internal/server/api.go b/internal/server/api.go new file mode 100644 index 0000000..ad609e8 --- /dev/null +++ b/internal/server/api.go @@ -0,0 +1,247 @@ +package server + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log" + "net" + "net/http" + "os" + "runtime" + "strings" + "time" + + "github.com/sergi/go-diff/diffmatchpatch" + + "github.com/dependabot/cli/internal/model" + "gopkg.in/yaml.v3" +) + +// API intercepts calls to the Dependabot API +type API struct { + // Expectations is the list of expectations that haven't been met yet + Expectations []model.Output + // Errors is the error list populated by doing a Dependabot run + Errors []error + // Actual will contain the scenario output that actually happened after the run is Complete + Actual model.Scenario + + server *http.Server + cursor int + hasExpectations bool + port int +} + +// NewAPI creates a new API instance and starts the server +func NewAPI(expected []model.Output) *API { + fakeAPIHost := "127.0.0.1" + if runtime.GOOS == "linux" { + fakeAPIHost = "0.0.0.0" + } + if os.Getenv("FAKE_API_HOST") != "" { + fakeAPIHost = os.Getenv("FAKE_API_HOST") + } + // Bind to port 0 for arbitrary port assignment + l, err := net.Listen("tcp", fakeAPIHost+":0") + if err != nil { + panic(err) + } + server := &http.Server{ + ReadTimeout: 5 * time.Second, + ReadHeaderTimeout: 5 * time.Second, + WriteTimeout: 10 * time.Second, + IdleTimeout: 60 * time.Second, + } + api := &API{ + server: server, + Expectations: expected, + cursor: 0, + hasExpectations: len(expected) > 0, + port: l.Addr().(*net.TCPAddr).Port, + } + server.Handler = api + + go func() { + if err := server.Serve(l); err != nil && !errors.Is(err, http.ErrServerClosed) { + log.Fatal(err) + } + }() + + return api +} + +// Port returns the port the API is listening on +func (a *API) Port() int { + return a.port +} + +// Stop stops the server +func (a *API) Stop() { + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + _ = a.server.Shutdown(ctx) + cancel() +} + +// Complete adds any remaining expectations to the error queue +func (a *API) Complete() { + for i := a.cursor; i < len(a.Expectations); i++ { + exp := &a.Expectations[i] + a.Errors = append(a.Errors, fmt.Errorf("expectation not met: %v\n%v", exp.Type, exp.Expect)) + } +} + +// ServeHTTP handles requests to the server +func (a *API) ServeHTTP(_ http.ResponseWriter, r *http.Request) { + data, err := io.ReadAll(r.Body) + if err != nil { + err = fmt.Errorf("failed to read body: %w", err) + a.pushError(err) + return + } + if err = r.Body.Close(); err != nil { + err = fmt.Errorf("failed to close body: %w", err) + a.pushError(err) + return + } + + parts := strings.Split(r.URL.String(), "/") + kind := parts[len(parts)-1] + + if err := a.pushResult(kind, data); err != nil { + a.pushError(err) + return + } + + if !a.hasExpectations { + return + } + + a.assertExpectation(kind, data) +} + +func (a *API) assertExpectation(kind string, actualData []byte) { + if len(a.Expectations) <= a.cursor { + err := fmt.Errorf("missing expectation") + a.pushError(err) + return + } + expect := &a.Expectations[a.cursor] + a.cursor++ + if kind != expect.Type { + err := fmt.Errorf("type was unexpected: expected %v got %v", expect.Type, kind) + a.pushError(err) + } + expectJSON, _ := json.Marshal(expect.Expect) + // pretty both for sorting and can use simple comparison + prettyData, _ := pretty(string(expectJSON)) + actual, _ := pretty(string(actualData)) + if actual != prettyData { + err := fmt.Errorf("expected output doesn't match actual data received") + a.pushError(err) + + // print diff to stdout + dmp := diffmatchpatch.New() + + const checklines = false + diffs := dmp.DiffMain(prettyData, actual, checklines) + + diffs = dmp.DiffCleanupSemantic(diffs) + + fmt.Println(dmp.DiffPrettyText(diffs)) + } +} + +func (a *API) pushError(err error) { + escapedError := strings.ReplaceAll(err.Error(), "\n", "") + escapedError = strings.ReplaceAll(escapedError, "\r", "") + log.Println(escapedError) + a.Errors = append(a.Errors, err) +} + +func (a *API) pushResult(kind string, data []byte) error { + actual, err := decodeWrapper(kind, data) + if err != nil { + return err + } + // TODO validate required data + output := model.Output{ + Type: kind, + Expect: *actual, + } + a.Actual.Output = append(a.Actual.Output, output) + + if msg, ok := actual.Data.(model.MarkAsProcessed); ok { + // record the commit SHA so the test is reproducible + a.Actual.Input.Job.Source.Commit = &msg.BaseCommitSha + } + + return nil +} + +// pretty indents and sorts the keys for a consistent comparison +func pretty(jsonString string) (string, error) { + var v map[string]any + if err := json.Unmarshal([]byte(jsonString), &v); err != nil { + return "", err + } + removeNullsFromObjects(v) + // shouldn't be possible to error + b, _ := json.MarshalIndent(v, "", " ") + return string(b), nil +} + +func removeNullsFromObjects(m map[string]any) { + for k, v := range m { + switch assertedVal := v.(type) { + case nil: + delete(m, k) + case map[string]any: + removeNullsFromObjects(assertedVal) + case []any: + for _, item := range assertedVal { + switch assertedItem := item.(type) { + case map[string]any: + removeNullsFromObjects(assertedItem) + } + } + } + } +} + +func decodeWrapper(kind string, data []byte) (*model.UpdateWrapper, error) { + var actual model.UpdateWrapper + switch kind { + case "update_dependency_list": + actual.Data = decode[model.UpdateDependencyList](data) + case "create_pull_request": + actual.Data = decode[model.CreatePullRequest](data) + case "update_pull_request": + actual.Data = decode[model.UpdatePullRequest](data) + case "close_pull_request": + actual.Data = decode[model.ClosePullRequest](data) + case "mark_as_processed": + actual.Data = decode[model.MarkAsProcessed](data) + case "record_package_manager_version": + actual.Data = decode[model.RecordPackageManagerVersion](data) + case "record_update_job_error": + actual.Data = decode[map[string]any](data) + default: + return nil, fmt.Errorf("unexpected output type: %s", kind) + } + return &actual, nil +} + +func decode[T any](data []byte) any { + var wrapper struct { + Data T `json:"data" yaml:"data"` + } + err := yaml.NewDecoder(bytes.NewBuffer(data)).Decode(&wrapper) + if err != nil { + panic(err) + } + return wrapper.Data +} diff --git a/internal/server/input.go b/internal/server/input.go new file mode 100644 index 0000000..4001783 --- /dev/null +++ b/internal/server/input.go @@ -0,0 +1,45 @@ +package server + +import ( + "context" + "encoding/json" + "fmt" + "log" + "net/http" + "time" + + "github.com/dependabot/cli/internal/model" +) + +type credServer struct { + server *http.Server + data *model.Input +} + +// the server receives one payload and shuts itself down +func (s *credServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if err := json.NewDecoder(r.Body).Decode(&s.data); err != nil { + panic(err) + } + w.WriteHeader(200) + _ = r.Body.Close() + go func() { + _ = s.server.Shutdown(context.Background()) + }() +} + +// Input receives configuration via HTTP on the port and returns it decoded +func Input(port int) *model.Input { + server := &http.Server{ + Addr: fmt.Sprintf("127.0.0.1:%d", port), + ReadHeaderTimeout: time.Second, + } + s := &credServer{server: server} + server.Handler = s + // printing so the user doesn't think the cli is hanging + log.Println("waiting for input on port", port) + if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { + panic(err) + } + return s.data +} diff --git a/internal/server/input_test.go b/internal/server/input_test.go new file mode 100644 index 0000000..24e6fc2 --- /dev/null +++ b/internal/server/input_test.go @@ -0,0 +1,37 @@ +package server + +import ( + "bytes" + "net/http" + "sync" + "testing" + + "github.com/dependabot/cli/internal/model" +) + +func TestInput(t *testing.T) { + wg := sync.WaitGroup{} + wg.Add(1) + var input *model.Input + go func() { + input = Input(8080) + wg.Done() + }() + + data := `{"job":{"package-manager":"test"},"credentials":[{"credential":"value"}]}` + resp, err := http.Post("http://localhost:8080", "application/json", bytes.NewReader([]byte(data))) + if err != nil { + t.Fatal(err.Error()) + } + if resp.StatusCode != 200 { + t.Errorf("expected status code 200, got %d", resp.StatusCode) + } + wg.Wait() + + if input.Job.PackageManager != "test" { + t.Errorf("expected package manager to be 'test', got '%s'", input.Job.PackageManager) + } + if input.Credentials[0]["credential"] != "value" { + t.Errorf("expected credential to be 'value', got '%v'", input.Credentials[0]) + } +} diff --git a/script/download-cache.sh b/script/download-cache.sh new file mode 100755 index 0000000..d98feb0 --- /dev/null +++ b/script/download-cache.sh @@ -0,0 +1,16 @@ +#!/bin/bash +if [ $# -eq 0 ] + then + echo "First argument is the ecosystem cache name, e.g. bundler" + exit 1 +fi +retry=0 +until [ "$retry" -ge 5 ] +do + gh run download --repo dependabot/smoke-tests --name cache-"$1" --dir cache && exit 0 + retry=$((retry+1)) + sleep 1 +done + +# Failed to download cache after all retries +exit 1 diff --git a/script/run-all.sh b/script/run-all.sh new file mode 100755 index 0000000..e3b3423 --- /dev/null +++ b/script/run-all.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +# This script is useful for regenerating all of the smoke tests running locally. + +declare -a arr=("actions" "bundler" "cargo" "composer" "docker" "elm" "go" "gradle" "hex" "maven" "npm" "nuget" "pip" "pip-compile" "pipenv" "poetry" "pub" "submodules" "terraform") +for eco in "${arr[@]}" +do + go run cmd/dependabot/dependabot.go test -f "testdata/smoke-$eco.yaml" -o "testdata/smoke-$eco.yaml" +done diff --git a/testdata/go/allow-go.yaml b/testdata/go/allow-go.yaml new file mode 100644 index 0000000..1a9e795 --- /dev/null +++ b/testdata/go/allow-go.yaml @@ -0,0 +1,112 @@ +input: + job: + package-manager: go_modules + allowed-updates: + - dependency-name: github.com/fatih/color + source: + provider: github + repo: dependabot/smoke-tests + directory: / + commit: 832e37c1a7a4ef89feb9dc7cfa06f62205191994 +output: + - type: update_dependency_list + expect: + data: + dependencies: + - name: github.com/fatih/color + requirements: + - file: go.mod + groups: [] + requirement: v1.7.0 + source: + source: github.com/fatih/color + type: default + version: 1.7.0 + - name: rsc.io/qr + requirements: [] + version: 0.1.0 + - name: rsc.io/quote + requirements: + - file: go.mod + groups: [] + requirement: v1.4.0 + source: + source: rsc.io/quote + type: default + version: 1.4.0 + dependency_files: + - /go.mod + - /go.sum + - type: create_pull_request + expect: + data: + base-commit-sha: 832e37c1a7a4ef89feb9dc7cfa06f62205191994 + dependencies: + - name: github.com/fatih/color + previous-requirements: + - file: go.mod + groups: [] + requirement: v1.7.0 + source: + source: github.com/fatih/color + type: default + previous-version: 1.7.0 + requirements: + - file: go.mod + groups: [] + requirement: 1.13.0 + source: + source: github.com/fatih/color + type: default + version: 1.13.0 + updated-dependency-files: + - content: | + module github.com/dependabot/vgotest + + go 1.12 + + require ( + github.com/fatih/color v1.13.0 + golang.org/x/sys v0.0.0-20220731174439-a90be440212d // indirect + rsc.io/qr v0.1.0 + rsc.io/quote v1.4.0 + ) + + replace rsc.io/qr => github.com/rsc/qr v0.2.0 + content_encoding: utf-8 + deleted: false + directory: / + name: go.mod + operation: update + support_file: false + type: file + - content: | + github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= + github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= + github.com/mattn/go-colorable v0.1.9 h1:sqDoxXbdeALODt0DAeJCVp38ps9ZogZEAXjus69YV3U= + github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= + github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= + github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= + github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= + github.com/rsc/qr v0.2.0 h1:tH61+huiZvu+hXL1VUovAu2AnhdG4eJQk2+on3XsDBQ= + github.com/rsc/qr v0.2.0/go.mod h1:IF+uZjkb9fqyeF/4tlBoynqmQxUoPfWEKh921coOuXs= + golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= + golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= + golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= + golang.org/x/sys v0.0.0-20220731174439-a90be440212d h1:Sv5ogFZatcgIMMtBSTTAgMYsicp25MXBubjXNDKwm80= + golang.org/x/sys v0.0.0-20220731174439-a90be440212d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= + rsc.io/quote v1.4.0 h1:tYuJspOzwTRMUOX6qmSDRTEKFVV80GM0/l89OLZuVNg= + rsc.io/quote v1.4.0/go.mod h1:S2vMDfxMfk+OGQ7xf1uNqJCSuSPCW5QC127LHYfOJmQ= + rsc.io/sampler v1.0.0 h1:CZX0Ury6np11Lwls9Jja2rFf3YrNPeUPAWiEVrJ0u/4= + rsc.io/sampler v1.0.0/go.mod h1:cqxpM3ZVz9VtirqxZPmrWzkQ+UkiNiGtkrN+B+i8kx8= + content_encoding: utf-8 + deleted: false + directory: / + name: go.sum + operation: update + support_file: false + type: file + - type: mark_as_processed + expect: + data: + base-commit-sha: 832e37c1a7a4ef89feb9dc7cfa06f62205191994 diff --git a/testdata/go/close-pr.yaml b/testdata/go/close-pr.yaml new file mode 100644 index 0000000..c5cfa85 --- /dev/null +++ b/testdata/go/close-pr.yaml @@ -0,0 +1,51 @@ +input: + job: + package-manager: go_modules + dependencies: + - none + security-updates-only: true + source: + provider: github + repo: dependabot/smoke-tests + directory: / + commit: 832e37c1a7a4ef89feb9dc7cfa06f62205191994 + updating-a-pull-request: true +output: + - type: update_dependency_list + expect: + data: + dependencies: + - name: github.com/fatih/color + requirements: + - file: go.mod + groups: [] + requirement: v1.7.0 + source: + source: github.com/fatih/color + type: default + version: 1.7.0 + - name: rsc.io/qr + requirements: [] + version: 0.1.0 + - name: rsc.io/quote + requirements: + - file: go.mod + groups: [] + requirement: v1.4.0 + source: + source: rsc.io/quote + type: default + version: 1.4.0 + dependency_files: + - /go.mod + - /go.sum + - type: close_pull_request + expect: + data: + dependency-names: + - none + reason: dependency_removed + - type: mark_as_processed + expect: + data: + base-commit-sha: 832e37c1a7a4ef89feb9dc7cfa06f62205191994 diff --git a/testdata/go/security-go.yaml b/testdata/go/security-go.yaml new file mode 100644 index 0000000..a6eae9c --- /dev/null +++ b/testdata/go/security-go.yaml @@ -0,0 +1,117 @@ +input: + job: + package-manager: go_modules + allowed-updates: + - update-type: all + security-advisories: + - dependency-name: github.com/fatih/color + affected-versions: + - <1.10.0 + patched-versions: [] + unaffected-versions: [] + security-updates-only: true + source: + provider: github + repo: dependabot/smoke-tests + directory: / + commit: 832e37c1a7a4ef89feb9dc7cfa06f62205191994 +output: + - type: update_dependency_list + expect: + data: + dependencies: + - name: github.com/fatih/color + requirements: + - file: go.mod + groups: [] + requirement: v1.7.0 + source: + source: github.com/fatih/color + type: default + version: 1.7.0 + - name: rsc.io/qr + requirements: [] + version: 0.1.0 + - name: rsc.io/quote + requirements: + - file: go.mod + groups: [] + requirement: v1.4.0 + source: + source: rsc.io/quote + type: default + version: 1.4.0 + dependency_files: + - /go.mod + - /go.sum + - type: create_pull_request + expect: + data: + base-commit-sha: 832e37c1a7a4ef89feb9dc7cfa06f62205191994 + dependencies: + - name: github.com/fatih/color + previous-requirements: + - file: go.mod + groups: [] + requirement: v1.7.0 + source: + source: github.com/fatih/color + type: default + previous-version: 1.7.0 + requirements: + - file: go.mod + groups: [] + requirement: 1.13.0 + source: + source: github.com/fatih/color + type: default + version: 1.10.0 + updated-dependency-files: + - content: | + module github.com/dependabot/vgotest + + go 1.12 + + require ( + github.com/fatih/color v1.10.0 + golang.org/x/sys v0.0.0-20220731174439-a90be440212d // indirect + rsc.io/qr v0.1.0 + rsc.io/quote v1.4.0 + ) + + replace rsc.io/qr => github.com/rsc/qr v0.2.0 + content_encoding: utf-8 + deleted: false + directory: / + name: go.mod + operation: update + support_file: false + type: file + - content: | + github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= + github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= + github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= + github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= + github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= + github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= + github.com/rsc/qr v0.2.0 h1:tH61+huiZvu+hXL1VUovAu2AnhdG4eJQk2+on3XsDBQ= + github.com/rsc/qr v0.2.0/go.mod h1:IF+uZjkb9fqyeF/4tlBoynqmQxUoPfWEKh921coOuXs= + golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= + golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= + golang.org/x/sys v0.0.0-20220731174439-a90be440212d h1:Sv5ogFZatcgIMMtBSTTAgMYsicp25MXBubjXNDKwm80= + golang.org/x/sys v0.0.0-20220731174439-a90be440212d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= + rsc.io/quote v1.4.0 h1:tYuJspOzwTRMUOX6qmSDRTEKFVV80GM0/l89OLZuVNg= + rsc.io/quote v1.4.0/go.mod h1:S2vMDfxMfk+OGQ7xf1uNqJCSuSPCW5QC127LHYfOJmQ= + rsc.io/sampler v1.0.0 h1:CZX0Ury6np11Lwls9Jja2rFf3YrNPeUPAWiEVrJ0u/4= + rsc.io/sampler v1.0.0/go.mod h1:cqxpM3ZVz9VtirqxZPmrWzkQ+UkiNiGtkrN+B+i8kx8= + content_encoding: utf-8 + deleted: false + directory: / + name: go.sum + operation: update + support_file: false + type: file + - type: mark_as_processed + expect: + data: + base-commit-sha: 832e37c1a7a4ef89feb9dc7cfa06f62205191994 diff --git a/testdata/go/update-pr.yaml b/testdata/go/update-pr.yaml new file mode 100644 index 0000000..a915093 --- /dev/null +++ b/testdata/go/update-pr.yaml @@ -0,0 +1,108 @@ +input: + job: + package-manager: go_modules + allowed-updates: + - update-type: all + dependencies: + - rsc.io/quote + existing-pull-requests: + - - dependency-name: rsc.io/quote + dependency-version: 1.5.2 + ignore-conditions: + - dependency-name: github.com/fatih/color + source: tests/go/update-pr.yaml + - dependency-name: rsc.io/quote + source: tests/go/update-pr.yaml + version-requirement: '>1.9.0' + source: + provider: github + repo: dependabot/smoke-tests + directory: / + commit: 832e37c1a7a4ef89feb9dc7cfa06f62205191994 + updating-a-pull-request: true +output: + - type: update_dependency_list + expect: + data: + dependencies: + - name: github.com/fatih/color + requirements: + - file: go.mod + groups: [] + requirement: v1.7.0 + source: + source: github.com/fatih/color + type: default + version: 1.7.0 + - name: rsc.io/qr + requirements: [] + version: 0.1.0 + - name: rsc.io/quote + requirements: + - file: go.mod + groups: [] + requirement: v1.4.0 + source: + source: rsc.io/quote + type: default + version: 1.4.0 + dependency_files: + - /go.mod + - /go.sum + - type: update_pull_request + expect: + data: + base-commit-sha: 832e37c1a7a4ef89feb9dc7cfa06f62205191994 + dependency-names: + - rsc.io/quote + updated-dependency-files: + - content: | + module github.com/dependabot/vgotest + + go 1.12 + + require ( + github.com/fatih/color v1.7.0 + github.com/mattn/go-colorable v0.0.9 // indirect + github.com/mattn/go-isatty v0.0.4 // indirect + golang.org/x/sys v0.0.0-20220731174439-a90be440212d // indirect + rsc.io/qr v0.1.0 + rsc.io/quote v1.5.2 + ) + + replace rsc.io/qr => github.com/rsc/qr v0.2.0 + content_encoding: utf-8 + deleted: false + directory: / + name: go.mod + operation: update + support_file: false + type: file + - content: | + github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= + github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= + github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= + github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= + github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs= + github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= + github.com/rsc/qr v0.2.0 h1:tH61+huiZvu+hXL1VUovAu2AnhdG4eJQk2+on3XsDBQ= + github.com/rsc/qr v0.2.0/go.mod h1:IF+uZjkb9fqyeF/4tlBoynqmQxUoPfWEKh921coOuXs= + golang.org/x/sys v0.0.0-20220731174439-a90be440212d h1:Sv5ogFZatcgIMMtBSTTAgMYsicp25MXBubjXNDKwm80= + golang.org/x/sys v0.0.0-20220731174439-a90be440212d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= + golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c h1:qgOY6WgZOaTkIIMiVjBQcw93ERBE4m30iBm00nkL0i8= + golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= + rsc.io/quote v1.5.2 h1:w5fcysjrx7yqtD/aO+QwRjYZOKnaM9Uh2b40tElTs3Y= + rsc.io/quote v1.5.2/go.mod h1:LzX7hefJvL54yjefDEDHNONDjII0t9xZLPXsUe+TKr0= + rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4= + rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= + content_encoding: utf-8 + deleted: false + directory: / + name: go.sum + operation: update + support_file: false + type: file + - type: mark_as_processed + expect: + data: + base-commit-sha: 832e37c1a7a4ef89feb9dc7cfa06f62205191994 diff --git a/testdata/private-bundler.json b/testdata/private-bundler.json new file mode 100644 index 0000000..5ebf682 --- /dev/null +++ b/testdata/private-bundler.json @@ -0,0 +1,22 @@ +{ + "job": { + "package-manager": "bundler", + "allowed-updates": [ + { + "update-type": "all" + } + ], + "source": { + "provider": "github", + "repo": "dsp-testing/ruby-bundler-private-registry", + "directory": "/" + } + }, + "credentials": [ + { + "type": "rubygems_server", + "host": "https://rubygems.pkg.github.com/dsp-testing", + "token": "$GPR_TOKEN" + } + ] +} diff --git a/testdata/private-npm.yaml b/testdata/private-npm.yaml new file mode 100644 index 0000000..66f467c --- /dev/null +++ b/testdata/private-npm.yaml @@ -0,0 +1,12 @@ +job: + package-manager: npm_and_yarn + allowed-updates: + - update-type: all + source: + provider: github + repo: dsp-testing/npm-private-registry + directory: /consumer +credentials: + - type: npm_registry + registry: https://npm.pkg.github.com + token: $GPR_TOKEN