From 98b34790fa0670e45b3025d03086ea0dcb482765 Mon Sep 17 00:00:00 2001
From: Vladislav Polyakov <polrk@yandex-team.ru>
Date: Mon, 28 Oct 2024 13:03:30 +0300
Subject: [PATCH] ci: add chaos testing

---
 .github/scripts/run-chaos-testing.sh |  6 +++++
 .github/workflows/slo.yml            | 36 ++++++++++++++++++++++------
 2 files changed, 35 insertions(+), 7 deletions(-)
 create mode 100755 .github/scripts/run-chaos-testing.sh

diff --git a/.github/scripts/run-chaos-testing.sh b/.github/scripts/run-chaos-testing.sh
new file mode 100755
index 000000000..e96f9551c
--- /dev/null
+++ b/.github/scripts/run-chaos-testing.sh
@@ -0,0 +1,6 @@
+curl -L https://github.com/alexei-led/pumba/releases/download/0.10.1/pumba_linux_amd64 -o /tmp/pumba
+chmod +x /tmp/pumba
+
+timeout -k 5 3m /tmp/pumba --random --interval 30s \
+    restart --timeout 30s \
+    re2:^ydb-dynamic &
diff --git a/.github/workflows/slo.yml b/.github/workflows/slo.yml
index 483985d92..19cbc2422 100644
--- a/.github/workflows/slo.yml
+++ b/.github/workflows/slo.yml
@@ -13,7 +13,7 @@ on:
       github_pull_request_number:
         required: true
       slo_workload_duration_seconds:
-        default: '600'
+        default: '180'
         required: false
       slo_workload_read_max_rps:
         default: '1000'
@@ -54,7 +54,8 @@ jobs:
             label: xorm
 
     concurrency:
-      group: slo-${{ github.ref }}-${{matrix.sdk.name}}
+      group: slo-${{ github.ref }}-${{ matrix.sdk.name }}
+      cancel-in-progress: true
 
     steps:
       - name: Checkout repository
@@ -79,16 +80,37 @@ jobs:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           sdk_name: ${{ matrix.sdk.name }}
 
-      - name: Run SLO Tests
+      - name: Prepare SLO Database
         run: |
           ./tests/slo/.bin/${{matrix.sdk.id}}_linux_amd64 create grpc://localhost:2135 /Root/testdb
+
+      # - name: Run chaos containers
+      #   run: |
+      #     curl -L https://github.com/alexei-led/pumba/releases/download/0.10.1/pumba_linux_amd64 -o /tmp/pumba
+      #     chmod +x /tmp/pumba
+
+      #     /tmp/pumba --random --interval 30s restart --timeout 30s re2:^ydb-dynamic &
+
+      - name: Run chaos network
+        run: |
+          sudo tc qdisc add dev lo root netem delay 100ms 50ms loss 5% corrupt 1%
+
+      - name: Run SLO Tests
+        timeout-minutes: 11
+        run: |
           ./tests/slo/.bin/${{matrix.sdk.id}}_linux_amd64 run grpc://localhost:2135 /Root/testdb \
             -prom-pgw localhost:9091 \
             -report-period 250 \
-            -time ${{inputs.slo_workload_duration_seconds || 600}} \
+            -time ${{inputs.slo_workload_duration_seconds || 60}} \
             -read-rps ${{inputs.slo_workload_read_max_rps || 1000}} \
             -write-rps ${{inputs.slo_workload_write_max_rps || 100}} \
             -read-timeout 10000 \
-            -write-timeout 10000 \
-            -shutdown-time 30
-          ./tests/slo/.bin/${{matrix.sdk.id}}_linux_amd64 cleanup grpc://localhost:2135 /Root/testdb
+            -write-timeout 10000
+
+      - if: always()
+        run: |
+          sudo tc qdisc del dev lo root
+
+      - name: Cleanup SLO Database
+        run: |
+          ./tests/slo/.bin/${{matrix.sdk.id}}_linux_amd64 cleanup grpc://localhost:2135 /Root/testdb || true