From 2d23e04bab915227efb5d3df5055e93fe574a5bb Mon Sep 17 00:00:00 2001 From: "Edgar R. M" Date: Thu, 23 Feb 2023 22:34:53 -0600 Subject: [PATCH] ci: Add integration tests (#33) * ci: Add integration tests * Add missing data file * Reference correct file * Ensure bucket exists * Run minio in the backgroud * Rename release workflow * Fix line break in docker command * Checkout code before runnin minio * Add s3 region --- .github/workflows/release.yml | 61 ++++++++++++------------- .github/workflows/releasev2.yml | 36 --------------- .github/workflows/test.yml | 80 ++++++++++++++++++++++++++++----- .gitignore | 3 -- resources/data/data.jsonl | 13 ++++++ 5 files changed, 114 insertions(+), 79 deletions(-) delete mode 100644 .github/workflows/releasev2.yml create mode 100644 resources/data/data.jsonl diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1f466e4..bce7ff7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,35 +1,36 @@ -on: {} +name: goreleaser + +on: + push: + # run only against tags + tags: + - '*' + +permissions: + contents: write + jobs: - releases-matrix: - name: Release Go Binaries + goreleaser: runs-on: ubuntu-latest - strategy: - matrix: - include: - - goos: linux - goarch: "386" - - goos: linux - goarch: "amd64" - - goos: linux - goarch: "arm64" - - goos: darwin - goarch: "amd64" - - goos: darwin - goarch: "arm64" - - goos: windows - goarch: "386" - - goos: windows - goarch: "amd64" steps: - - name: Checkout code + - name: Checkout uses: actions/checkout@v3 - - name: Release binaries - uses: wangyoucao577/go-release-action@v1.36 with: - github_token: ${{ secrets.GITHUB_TOKEN }} - goos: ${{ matrix.goos }} - goarch: ${{ matrix.goarch }} - goversion: "1.20" - project_path: ./ - binary_name: target-jsonl-blob - compress_assets: false + fetch-depth: 0 + + - run: git fetch --force --tags + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: 1.19 + cache: true + + - name: Run GoReleaser + uses: goreleaser/goreleaser-action@v4 + with: + distribution: goreleaser + version: latest + args: release --clean --skip-validate --debug + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/releasev2.yml b/.github/workflows/releasev2.yml deleted file mode 100644 index bce7ff7..0000000 --- a/.github/workflows/releasev2.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: goreleaser - -on: - push: - # run only against tags - tags: - - '*' - -permissions: - contents: write - -jobs: - goreleaser: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - run: git fetch --force --tags - - - name: Set up Go - uses: actions/setup-go@v3 - with: - go-version: 1.19 - cache: true - - - name: Run GoReleaser - uses: goreleaser/goreleaser-action@v4 - with: - distribution: goreleaser - version: latest - args: release --clean --skip-validate --debug - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 94d0bae..84a07e8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,8 +1,14 @@ name: Test + on: pull_request: {} push: branches: [main] + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + jobs: test: strategy: @@ -11,13 +17,67 @@ jobs: os: [ubuntu-latest, macos-latest, windows-latest] runs-on: ${{ matrix.os }} steps: - - name: Checkout code - uses: actions/checkout@v3 - - name: Install Go - uses: actions/setup-go@v3 - with: - go-version: ${{ matrix.go-version }} - cache: true - cache-dependency-path: go.sum - - name: Test - run: go test ./... -cover + - name: Checkout code + uses: actions/checkout@v3 + - name: Install Go + uses: actions/setup-go@v3 + with: + go-version: ${{ matrix.go-version }} + cache: true + cache-dependency-path: go.sum + - name: Test + run: go test ./... -cover + + integration-test: + name: Integration test - ${{ matrix.fs }} + strategy: + matrix: + include: + - fs: local + config: + bucket: file://./output/my-bucket + - fs: s3 + config: + bucket: s3://my-bucket?endpoint=http://127.0.0.1:9000&disableSSL=true&s3ForcePathStyle=true®ion=us-west-1 + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Start MinIO + if: ${{ matrix.fs == 's3' }} + run: > + docker run -d + --name minio + -p 9000:9000 + -p 9090:9090 + -v $PWD/output:/data + -e "MINIO_REGION_NAME=eu-west-1" + -e "MINIO_ROOT_USER=minioadmin" + -e "MINIO_ROOT_PASSWORD=minioadmin" + quay.io/minio/minio server /data --console-address ":9090" + + - name: Install Go + uses: actions/setup-go@v3 + with: + go-version: 1.20.x + cache: true + cache-dependency-path: go.sum + + - name: Build + run: | + go build + ./target-jsonl-blob --version + + - name: Write config + run: | + echo '${{ toJSON(matrix.config) }}' > config.json + + - run: mkdir output/my-bucket + + - name: Test + env: + AWS_ACCESS_KEY_ID: minioadmin + AWS_SECRET_ACCESS_KEY: minioadmin + run: | + ./target-jsonl-blob --config config.json --input resources/data/data.jsonl diff --git a/.gitignore b/.gitignore index 3081942..edd8a64 100644 --- a/.gitignore +++ b/.gitignore @@ -2,9 +2,6 @@ *.json !*.example.* -# Data files -**/*.jsonl - # Binaries for programs and plugins target-jsonl-blob *.exe diff --git a/resources/data/data.jsonl b/resources/data/data.jsonl new file mode 100644 index 0000000..3683294 --- /dev/null +++ b/resources/data/data.jsonl @@ -0,0 +1,13 @@ +{"type": "BAD"} +{"type": "SCHEMA", "stream": "test", "schema": {"properties": {"id": "integer"}}} +{"type": "RECORD", "stream": "test", "record": {"id": 1}, "time_extracted": "2022-01-01T00:00:00Z"} +{"type": "RECORD", "stream": "test", "record": {"id": 2}, "time_extracted": "2022-01-01T00:00:00Z"} +{"type": "STATE", "value": {"bookmark": "2022-01-01"}} +{"type": "SCHEMA", "stream": "test2", "schema": {"properties": {"name": "string"}}} +{"type": "RECORD", "stream": "test2", "record": {"name": "A"}} +{"type": "RECORD", "stream": "test2", "record": {"name": "B"}} +{"type": "STATE", "value": {"test": "2022-01-01", "test2": "2022-02-01"}} +{"type": "SCHEMA", "stream": "test", "schema": {"properties": {"id": "integer"}}} +{"type": "RECORD", "stream": "test", "record": {"id": 3}} +{"type": "RECORD", "stream": "test", "record": {"id": 4}} +{"type": "STATE", "value": {"test": "2022-03-01", "test2": "2022-02-01"}}