diff --git a/.github/scripts/run-chaos-testing.sh b/.github/scripts/run-chaos-testing.sh new file mode 100755 index 000000000..b7b9bc2b7 --- /dev/null +++ b/.github/scripts/run-chaos-testing.sh @@ -0,0 +1,12 @@ +curl -L https://github.com/alexei-led/pumba/releases/download/0.10.1/pumba_linux_amd64 -o /tmp/pumba +chmod +x /tmp/pumba + + +/tmp/pumba --random --interval 30s \ + restart --timeout 30s \ + re2:^ydb-dynamic & + +/tmp/pumba --random \ + netem --duration 10m --interface docker0 --tc-image gaiadocker/iproute2 \ + delay --time 45 --jitter 30 --distribution normal \ + re2:^ydb-dynamic & diff --git a/.github/workflows/slo.yml b/.github/workflows/slo.yml index 483985d92..f8fade043 100644 --- a/.github/workflows/slo.yml +++ b/.github/workflows/slo.yml @@ -54,7 +54,8 @@ jobs: label: xorm concurrency: - group: slo-${{ github.ref }}-${{matrix.sdk.name}} + group: slo-${{ github.ref }}-${{ matrix.sdk.name }} + cancel-in-progress: true steps: - name: Checkout repository @@ -79,9 +80,17 @@ jobs: github_token: ${{ secrets.GITHUB_TOKEN }} sdk_name: ${{ matrix.sdk.name }} - - name: Run SLO Tests + - name: Prepare SLO Database run: | ./tests/slo/.bin/${{matrix.sdk.id}}_linux_amd64 create grpc://localhost:2135 /Root/testdb + + - name: Run chaos testing + run: | + echo 'Performing chaos testing...' + bash ./.github/scripts/run-chaos-testing.sh & + + - name: Run SLO Tests + run: | ./tests/slo/.bin/${{matrix.sdk.id}}_linux_amd64 run grpc://localhost:2135 /Root/testdb \ -prom-pgw localhost:9091 \ -report-period 250 \ @@ -89,6 +98,8 @@ jobs: -read-rps ${{inputs.slo_workload_read_max_rps || 1000}} \ -write-rps ${{inputs.slo_workload_write_max_rps || 100}} \ -read-timeout 10000 \ - -write-timeout 10000 \ - -shutdown-time 30 + -write-timeout 10000 + + - name: Cleanup SLO Database + run: | ./tests/slo/.bin/${{matrix.sdk.id}}_linux_amd64 cleanup grpc://localhost:2135 /Root/testdb