From 3db263577aa7f0a543a2f8da7cd862b4e9a13d30 Mon Sep 17 00:00:00 2001
From: Vladislav Polyakov <polrk@yandex-team.ru>
Date: Mon, 28 Oct 2024 13:03:30 +0300
Subject: [PATCH] ci: add chaos testing
---
.github/scripts/run-chaos-testing.sh | 12 ++++++++++++
.github/workflows/slo.yml | 19 +++++++++++++++----
2 files changed, 27 insertions(+), 4 deletions(-)
create mode 100755 .github/scripts/run-chaos-testing.sh
diff --git a/.github/scripts/run-chaos-testing.sh b/.github/scripts/run-chaos-testing.sh
new file mode 100755
index 000000000..b7b9bc2b7
--- /dev/null
+++ b/.github/scripts/run-chaos-testing.sh
@@ -0,0 +1,12 @@
+curl -L https://github.com/alexei-led/pumba/releases/download/0.10.1/pumba_linux_amd64 -o /tmp/pumba
+chmod +x /tmp/pumba
+
+
+/tmp/pumba --random --interval 30s \
+ restart --timeout 30s \
+ re2:^ydb-dynamic &
+
+/tmp/pumba --random \
+ netem --duration 10m --interface docker0 --tc-image gaiadocker/iproute2 \
+ delay --time 45 --jitter 30 --distribution normal \
+ re2:^ydb-dynamic &
diff --git a/.github/workflows/slo.yml b/.github/workflows/slo.yml
index 483985d92..f8fade043 100644
--- a/.github/workflows/slo.yml
+++ b/.github/workflows/slo.yml
@@ -54,7 +54,8 @@ jobs:
label: xorm
concurrency:
- group: slo-${{ github.ref }}-${{matrix.sdk.name}}
+ group: slo-${{ github.ref }}-${{ matrix.sdk.name }}
+ cancel-in-progress: true
steps:
- name: Checkout repository
@@ -79,9 +80,17 @@ jobs:
github_token: ${{ secrets.GITHUB_TOKEN }}
sdk_name: ${{ matrix.sdk.name }}
- - name: Run SLO Tests
+ - name: Prepare SLO Database
run: |
./tests/slo/.bin/${{matrix.sdk.id}}_linux_amd64 create grpc://localhost:2135 /Root/testdb
+
+ - name: Run chaos testing
+ run: |
+ echo 'Performing chaos testing...'
+ bash ./.github/scripts/run-chaos-testing.sh &
+
+ - name: Run SLO Tests
+ run: |
./tests/slo/.bin/${{matrix.sdk.id}}_linux_amd64 run grpc://localhost:2135 /Root/testdb \
-prom-pgw localhost:9091 \
-report-period 250 \
@@ -89,6 +98,8 @@ jobs:
-read-rps ${{inputs.slo_workload_read_max_rps || 1000}} \
-write-rps ${{inputs.slo_workload_write_max_rps || 100}} \
-read-timeout 10000 \
- -write-timeout 10000 \
- -shutdown-time 30
+ -write-timeout 10000
+
+ - name: Cleanup SLO Database
+ run: |
./tests/slo/.bin/${{matrix.sdk.id}}_linux_amd64 cleanup grpc://localhost:2135 /Root/testdb