Merge branch 'AIK-4292' into fix/aikido-security-update-packages-2537…

…517-tTKx
AikidoSec · Jan 22, 2025 · 3fa71e8 · 3fa71e8
2 parents effa57c + a3ab4a7
commit 3fa71e8
Show file tree

Hide file tree

Showing 160 changed files with 12,143 additions and 1,360 deletions.
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
@@ -39,11 +39,13 @@ jobs:
       - name: Start databases
         working-directory:  ./sample-apps/databases
         run: docker compose up --build -d
+      - name: Install dependencies & build
+        run: |
+          python -m pip install --upgrade pip
+          make install && make build
       - name: Start flask-mysql
         working-directory: ./sample-apps/flask-mysql
-        run: |
-          cp .env.benchmark .env.example
-          docker compose -f docker-compose.yml -f docker-compose.benchmark.yml up --build -d
+        run: nohup make runBenchmark & nohup make runZenDisabled &
       - name: Install K6
         uses: grafana/setup-k6-action@v1
       - name: Run flask-mysql k6 Benchmark
@@ -58,11 +60,13 @@ jobs:
       - name: Start databases
         working-directory:  ./sample-apps/databases
         run: docker compose up --build -d
-      - name: Start starlette multi-threaded
-        working-directory: ./sample-apps/starlette-postgres-uvicorn
+      - name: Install dependencies & build
         run: |
-          cp .env.benchmark .env.example
-          docker compose -f docker-compose.yml -f docker-compose.benchmark.yml up --build -d
+          python -m pip install --upgrade pip
+          make install && make build
+      - name: Start starlette
+        working-directory: ./sample-apps/starlette-postgres-uvicorn
+        run: nohup make runBenchmark & nohup make runZenDisabled &
       - name: Install wrk
         run: |
           sudo apt-get update

diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml
@@ -3,126 +3,68 @@ name: End-to-End tests
 on: [pull_request]
 
 jobs:
-  test:
+  build:
     runs-on: ubuntu-latest
     steps:
-    - name: Checkout code
-      uses: actions/checkout@v2
-
-    - name: Start databases
-      working-directory:  ./sample-apps/databases
-      run: docker compose up --build -d
-    - name: Build and start aikido mock server
-      working-directory: ./end2end/server
-      run: docker build -t mock-core . && docker run --name mock_core -d -p 5000:5000 mock-core
+      - uses: actions/checkout@v4
+      - name: Install poetry
+        run: pip install poetry
+      - name: Build
+        run: make build
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: pkg-build
+          path: ./
 
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        make install
-
-    # django-mysql
-    - name: Start django-mysql
-      working-directory: ./sample-apps/django-mysql
-      run: |
-        docker compose -f docker-compose.yml -f docker-compose.benchmark.yml up --build -d
-    - name: Run end2end tests for django-mysql
-      run: sleep 5 && poetry run pytest ./end2end/django_mysql_test.py
-
-    # django-mysql-gunicorn
-    - name: Restart mock server
-      run: docker restart mock_core
-    - name: Start django-mysql-gunicorn
-      working-directory: ./sample-apps/django-mysql-gunicorn
-      run: |
-        docker compose -f docker-compose.yml -f docker-compose.benchmark.yml up --build -d
-    - name: Run end2end tests for django-mysql-gunicorn
-      run: sleep 5 && poetry run pytest ./end2end/django_mysql_gunicorn_test.py
-
-    # django-postgres-gunicorn
-    - name: Restart mock server
-      run: docker restart mock_core
-    - name: Start django-postgres-gunicorn
-      working-directory: ./sample-apps/django-postgres-gunicorn
-      run: |
-        docker compose -f docker-compose.yml -f docker-compose.benchmark.yml up --build -d
-    - name: Run end2end tests for django-postgres-gunicorn
-      run: sleep 5 && poetry run pytest ./end2end/django_postgres_gunicorn_test.py
-
-    # flask-mongo
-    - name: Restart mock server
-      run: docker restart mock_core
-    - name: Start flask-mongo
-      working-directory: ./sample-apps/flask-mongo
-      run: |
-        docker compose -f docker-compose.yml -f docker-compose.benchmark.yml up --build -d
-    - name: Run end2end tests for flask-mongo
-      run: sleep 5 && poetry run pytest ./end2end/flask_mongo_test.py
-
-    # flask-mysql
-    - name: Restart mock server
-      run: docker restart mock_core
-    - name: Start flask-mysql
-      working-directory: ./sample-apps/flask-mysql
-      run: |
-        docker compose -f docker-compose.yml -f docker-compose.benchmark.yml up --build -d
-    - name: Run end2end tests for flask-mysql
-      run: sleep 5 && poetry run pytest ./end2end/flask_mysql_test.py
-
-    # flask-mysql-uwsgi
-    - name: Restart mock server
-      run: docker restart mock_core
-    - name: Start flask-mysql-uwsgi
-      working-directory: ./sample-apps/flask-mysql-uwsgi
-      run: |
-        docker compose -f docker-compose.yml -f docker-compose.benchmark.yml up --build -d
-    - name: Run end2end tests for flask-mysql-uwsgi
-      run: sleep 5 && poetry run pytest ./end2end/flask_mysql_uwsgi_test.py
-
-    # flask-postgres
-    - name: Restart mock server
-      run: docker restart mock_core
-    - name: Start flask-postgres
-      working-directory: ./sample-apps/flask-postgres
-      run: |
-        docker compose -f docker-compose.yml -f docker-compose.benchmark.yml up --build -d
-    - name: Run end2end tests for flask-postgres
-      run: sleep 5 && poetry run pytest ./end2end/flask_postgres_test.py
+  end2end-test:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    needs: build
+    strategy:
+      matrix:
+        app:
+          - { name: django-mysql, testfile: end2end/django_mysql_test.py }
+          - { name: django-mysql-gunicorn, testfile: end2end/django_mysql_gunicorn_test.py }
+          - { name: django-postgres-gunicorn, testfile: end2end/django_postgres_gunicorn_test.py }
+          - { name: flask-mongo, testfile: end2end/flask_mongo_test.py }
+          - { name: flask-mysql, testfile: end2end/flask_mysql_test.py }
+          - { name: flask-mysql-uwsgi, testfile: end2end/flask_mysql_uwsgi_test.py }
+          - { name: flask-postgres, testfile: end2end/flask_postgres_test.py }
+          - { name: flask-postgres-xml, testfile: end2end/flask_postgres_xml_test.py }
+          - { name: flask-postgres-xml, testfile: end2end/flask_postgres_xml_lxml_test.py }
+          - { name: quart-postgres-uvicorn, testfile: end2end/quart_postgres_uvicorn_test.py }
+          - { name: starlette-postgres-uvicorn, testfile: end2end/starlette_postgres_uvicorn_test.py }
+        python-version: ["3.10", "3.11", "3.12"]
+    steps:
+      - name: Install packages
+        run: sudo apt update && sudo apt install python3-dev libmysqlclient-dev
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: pkg-build
 
-    # flask-postgres-xml
-    - name: Restart mock server
-      run: docker restart mock_core
-    - name: Start flask-postgres-xml
-      working-directory: ./sample-apps/flask-postgres-xml
-      run: |
-        docker compose -f docker-compose.yml -f docker-compose.benchmark.yml up --build -d
-    - name: Run end2end tests for flask-postgres-xml
-      run: | 
-        sleep 5
-        poetry run pytest ./end2end/flask_postgres_xml_test.py
-        docker restart mock_core
-        poetry run pytest ./end2end/flask_postgres_xml_lxml_test.py
+      - name: Start databases
+        working-directory: ./sample-apps/databases
+        run: docker compose up --build -d
+      - name: Build and start aikido mock server
+        run: |
+            cd end2end/server && docker build -t mock_core .
+            docker run --name mock_core -d -p 5000:5000 mock_core
 
-    # quart-postgres-uvicorn
-    - name: Restart mock server
-      run: docker restart mock_core
-    - name: Start quart-postgres-uvicorn
-      working-directory: ./sample-apps/quart-postgres-uvicorn
-      run: |
-        docker compose -f docker-compose.yml -f docker-compose.benchmark.yml up --build -d
-    - name: Run end2end tests for quart-postgres-uvicorn
-      run: sleep 5 && poetry run pytest ./end2end/quart_postgres_uvicorn_test.py
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          make install
 
-    # starlette-postgres-uvicorn
-    - name: Restart mock server
-      run: docker restart mock_core
-    - name: Start starlette-postgres-uvicorn
-      working-directory: ./sample-apps/starlette-postgres-uvicorn
-      run: |
-        docker compose -f docker-compose.yml -f docker-compose.benchmark.yml up --build -d
-    - name: Run end2end tests for starlette-postgres-uvicorn
-      run: sleep 5 && poetry run pytest ./end2end/starlette_postgres_uvicorn_test.py
+      - name: Start application
+        working-directory: ./sample-apps/${{ matrix.app.name }}
+        run: |
+          nohup make run > output.log & tail -f output.log & sleep 20
+          nohup make runZenDisabled & sleep 20
+      - name: Run end2end tests for application
+        run: tail -f  ./sample-apps/${{ matrix.app.name }}/output.log & poetry run pytest ./${{ matrix.app.testfile }}
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -48,8 +48,8 @@ jobs:
     - uses: actions/checkout@v4
     - name: Install poetry
       run: pip install poetry
-    - name: Download binaries
-      run: make binaries
+    - name: Download binaries & build
+      run: make build
     - name: Publish to PyPI
       env:
         POETRY_HTTP_BASIC_PYPI_USERNAME: __token__

diff --git a/.github/workflows/smoke-test-ffi.yml b/.github/workflows/smoke-test-ffi.yml
@@ -0,0 +1,44 @@
+name: Smoke Test for FFI
+
+on: [push]
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        # macos-14 is ARM-based
+        # macos-14-large is x86-based
+        # ubuntu-latest (x86) is not included because the full test suite runs on it
+        os: [windows-latest, macos-14, macos-14-large, ubuntu-22.04-arm]
+
+    # Allows all jobs to run even if one fails
+    continue-on-error: true
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.12
+
+      - name: Display system info
+        run: |
+          python -c "import platform; print(f'Machine architecture: {platform.machine()}')"
+          python -c "import platform; print(f'OS: {platform.system()}')"
+
+      - name: Install MySQL
+        if: runner.os == 'macOS'
+        run: brew install mysql
+
+      - name: Install MySQL
+        if: runner.os == 'Linux'
+        run: sudo apt-get install libmysqlclient-dev
+
+      - name: Install dependencies
+        run: make install && cp -r .cache/binaries/* aikido_zen/libs
+
+      - name: Run tests
+        run: poetry run pytest aikido_zen/vulnerabilities/sql_injection/init_test.py
diff --git a/.github/workflows/test-publish.yml b/.github/workflows/test-publish.yml
@@ -49,11 +49,10 @@ jobs:
     - uses: actions/checkout@v4
     - name: Install poetry
       run: pip install poetry
-    - name: Download binaries
-      run: make binaries
     - name: Install dependencies
       run: poetry install
-
+    - name: Download binaries & build
+      run: make build
     - name: Publish to TestPyPI
       env:
         POETRY_HTTP_BASIC_PYPI_USERNAME: __token__

diff --git a/Makefile b/Makefile
@@ -1,42 +1,48 @@
-build: binaries _build
-_build:
-	poetry build
+CACHE_DIR := .cache/binaries
+BUILD_DIR := dist/
 
+# Build/clean/lint/install :
+.PHONY: build
+build: check_binaries
+	@echo "Building using poetry, creates $(BUILD_DIR) folder."
+	@echo "Copying binaries from $(CACHE_DIR) folder : "
+	cp -r .cache/binaries/* aikido_zen/libs
+	poetry build
 .PHONY: clean
 clean:
-	poetry env remove python
-
+	@echo "Cleaning up: Removing build and cache directories, remove poetry env"
+	rm -rf $(BUILD_DIR)
+	rm -rf $(CACHE_DIR)
+	@poetry env remove $(shell poetry env list | grep 'Activated' | awk '{print $$1}')
 .PHONY: lint
 lint:
 	poetry run black aikido_zen/
 	poetry run pylint aikido_zen/
-
-install: binaries
+install: check_binaries
 	pip install poetry
 	poetry install
-
-dev_install: install _dev_install
-_dev_install:
+.PHONY: dev_install
+dev_install: install
 	poetry install --with=dev
 
 
+# Testing/Benchmarks :
 .PHONY: test
-test:
+test: build
 	poetry run pytest aikido_zen/
-
 .PHONY: end2end
 end2end:
-	poetry run pytest end2end/ 	
-
+	poetry run pytest end2end/
 .PHONY: cov
-cov:
+cov: build
 	poetry run pytest aikido_zen/ --cov=aikido_zen --cov-report=xml
-
 .PHONY: benchmark
 benchmark:
 	k6 run -q ./benchmarks/flask-mysql-benchmarks.js
 
-BASE_URL = https://github.com/AikidoSec/zen-internals/releases/download/v0.1.34
+
+# Binaries cache :
+BASE_URL = https://github.com/AikidoSec/zen-internals/releases/download/v0.1.35
 FILES = \
     libzen_internals_aarch64-apple-darwin.dylib \
     libzen_internals_aarch64-apple-darwin.dylib.sha256sum \
@@ -49,10 +55,19 @@ FILES = \
     libzen_internals_x86_64-unknown-linux-gnu.so \
     libzen_internals_x86_64-unknown-linux-gnu.so.sha256sum
 
-binaries: binaries_make_dir $(addprefix aikido_zen/lib/, $(FILES))
+binaries: binaries_make_dir $(addprefix .cache/binaries/, $(FILES))
 binaries_make_dir:
-	rm -rf aikido_zen/lib
-	mkdir -p aikido_zen/lib/
-aikido_zen/lib/%:
+	rm -rf .cache/binaries
+	mkdir -p .cache/binaries/
+.cache/binaries/%:
 	@echo "Downloading $*..."
 	curl -L -o $@ $(BASE_URL)/$*
+
+.PHONY: check_binaries
+check_binaries:
+	@if [ -d "$(CACHE_DIR)" ]; then \
+  		echo "Directory $(CACHE_DIR) exists."; \
+	else \
+		echo "Directory $(CACHE_DIR) is empty. Running 'make binaries'..."; \
+		$(MAKE) binaries; \
+	fi
diff --git a/README.md b/README.md
@@ -83,6 +83,10 @@ To block requests, set the `AIKIDO_BLOCKING` environment variable to `true`.
 
 See [Reporting to Aikido](#reporting-to-your-aikido-security-dashboard) to learn how to send events to Aikido.
 
+## Additional configuration
+
+[Configure Zen using environment variables for authentication, mode settings, debugging, and more.](https://help.aikido.dev/doc/configuration-via-env-vars/docrSItUkeR9)
+
 ## Benchmarks 
 The following table summarizes the performance of both a typical SQL Query and a typical NoSQL Query with and without the Zen, measured in milliseconds :
 | Operation | Avg. time w/o Zen | Avg. time w/ Zen | Delta | Delta in % |