ci: add chaos testing

ydb-platform · Oct 28, 2024 · dd35e16 · dd35e16
1 parent 17a901b
commit dd35e16
Show file tree

Hide file tree

Showing 2 changed files with 35 additions and 8 deletions.
diff --git a/.github/scripts/run-chaos-testing.sh b/.github/scripts/run-chaos-testing.sh
@@ -0,0 +1,6 @@
+curl -L https://github.com/alexei-led/pumba/releases/download/0.10.1/pumba_linux_amd64 -o /tmp/pumba
+chmod +x /tmp/pumba
+
+timeout -k 5 3m /tmp/pumba --random --interval 30s \
+    restart --timeout 30s \
+    re2:^ydb-dynamic &
diff --git a/.github/workflows/slo.yml b/.github/workflows/slo.yml
@@ -13,7 +13,7 @@ on:
       github_pull_request_number:
         required: true
       slo_workload_duration_seconds:
-        default: '600'
+        default: '180'
         required: false
       slo_workload_read_max_rps:
         default: '1000'
@@ -54,7 +54,8 @@ jobs:
             label: xorm
 
     concurrency:
-      group: slo-${{ github.ref }}-${{matrix.sdk.name}}
+      group: slo-${{ github.ref }}-${{ matrix.sdk.name }}
+      cancel-in-progress: true
 
     steps:
       - name: Checkout repository
@@ -79,16 +80,36 @@ jobs:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           sdk_name: ${{ matrix.sdk.name }}
 
-      - name: Run SLO Tests
+      - name: Prepare SLO Database
         run: |
           ./tests/slo/.bin/${{matrix.sdk.id}}_linux_amd64 create grpc://localhost:2135 /Root/testdb
+
+      - name: Chaos YDB
+        run: |
+          curl -L https://github.com/alexei-led/pumba/releases/download/0.10.1/pumba_linux_amd64 -o /tmp/pumba
+          chmod +x /tmp/pumba
+
+          /tmp/pumba --random --interval 30s restart --timeout 30s re2:^ydb-dynamic &
+
+      # - name: Chaos Network
+      #   run: |
+      #     sudo tc qdisc add dev lo root netem delay 100ms 50ms loss 5% corrupt 1%
+
+      - name: Run SLO Tests
+        run: |
           ./tests/slo/.bin/${{matrix.sdk.id}}_linux_amd64 run grpc://localhost:2135 /Root/testdb \
             -prom-pgw localhost:9091 \
             -report-period 250 \
-            -time ${{inputs.slo_workload_duration_seconds || 600}} \
+            -time ${{inputs.slo_workload_duration_seconds || 360}} \
             -read-rps ${{inputs.slo_workload_read_max_rps || 1000}} \
             -write-rps ${{inputs.slo_workload_write_max_rps || 100}} \
-            -read-timeout 10000 \
-            -write-timeout 10000 \
-            -shutdown-time 30
-          ./tests/slo/.bin/${{matrix.sdk.id}}_linux_amd64 cleanup grpc://localhost:2135 /Root/testdb
+            -read-timeout 1000 \
+            -write-timeout 1000 || true
+
+      # - if: always()
+      #   run: |
+      #     sudo tc qdisc del dev lo root
+
+      - name: Cleanup SLO Database
+        run: |
+          ./tests/slo/.bin/${{matrix.sdk.id}}_linux_amd64 cleanup grpc://localhost:2135 /Root/testdb || true