ci: add chaos testing

.github/scripts/run-chaos-testing.sh

@@ -0,0 +1,12 @@
+curl -L https://github.com/alexei-led/pumba/releases/download/0.10.1/pumba_linux_amd64 -o /tmp/pumba
+chmod +x /tmp/pumba
+
+
+/tmp/pumba --random --interval 30s \
+    restart --timeout 30s \
+    re2:^ydb-dynamic &
+
+/tmp/pumba --random \
+    netem --duration 10m --interface docker0 --tc-image gaiadocker/iproute2 \
+    delay --time 45 --jitter 30 --distribution normal \
+    re2:^ydb-dynamic &

.github/workflows/slo.yml

@@ -54,7 +54,8 @@ jobs:
             label: xorm
 
     concurrency:
-      group: slo-${{ github.ref }}-${{matrix.sdk.name}}
+      group: slo-${{ github.ref }}-${{ matrix.sdk.name }}
+      cancel-in-progress: true
 
     steps:
       - name: Checkout repository
@@ -79,16 +80,26 @@ jobs:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           sdk_name: ${{ matrix.sdk.name }}
 
-      - name: Run SLO Tests
+      - name: Prepare SLO Database
         run: |
           ./tests/slo/.bin/${{matrix.sdk.id}}_linux_amd64 create grpc://localhost:2135 /Root/testdb
+
+      - name: Run chaos testing
+        run: |
+            echo 'Performing chaos testing...'
+            bash ./.github/scripts/run-chaos-testing.sh &
+
+      - name: Run SLO Tests
+        run: |
           ./tests/slo/.bin/${{matrix.sdk.id}}_linux_amd64 run grpc://localhost:2135 /Root/testdb \
             -prom-pgw localhost:9091 \
             -report-period 250 \
             -time ${{inputs.slo_workload_duration_seconds || 600}} \
             -read-rps ${{inputs.slo_workload_read_max_rps || 1000}} \
             -write-rps ${{inputs.slo_workload_write_max_rps || 100}} \
             -read-timeout 10000 \
-            -write-timeout 10000 \
-            -shutdown-time 30
+            -write-timeout 10000
+
+      - name: Cleanup SLO Database
+        run: |
           ./tests/slo/.bin/${{matrix.sdk.id}}_linux_amd64 cleanup grpc://localhost:2135 /Root/testdb