ci: separate type check from integration tests

.github/workflows/playwright.yaml

@@ -53,7 +53,7 @@ jobs:
           enable-cache: true
           cache-dependency-glob: "pyproject.toml"
           github-token: ${{ secrets.GITHUB_TOKEN }}
-      - run: uv pip install --system -q ../. uvloop
+      - run: uv pip install --system ../. uvloop
       - run: uv pip list
       - name: Run Playwright tests
         run: pnpm exec playwright test

.github/workflows/python-CI.yml

@@ -56,8 +56,8 @@ jobs:
     if: ${{ needs.changes.outputs.phoenix_evals == 'true' }}
     strategy:
       matrix:
-        py: [ 3.9, 3.12 ]
-        os: [macos-12, windows-latest, ubuntu-latest]
+        py: [ 3.8, 3.12 ]
+        os: [ ubuntu-latest, windows-latest, macos-13 ]
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
@@ -67,16 +67,17 @@ jobs:
         with:
           version: 0.4.18
       - run: uv tool install tox --with tox-uv
-      - run: tox run -e phoenix_evals
+      - run: tox run -e phoenix_evals -- -ra -x
 
   phoenix-otel:
     name: Phoenix OTel
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
     needs: changes
     if: ${{ needs.changes.outputs.phoenix_otel == 'true' }}
     strategy:
       matrix:
-        py: [ 3.9, 3.12 ]
+        py: [ 3.8, 3.12 ]
+        os: [ ubuntu-latest, windows-latest, macos-13 ]
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
@@ -86,7 +87,7 @@ jobs:
         with:
           version: 0.4.18
       - run: uv tool install tox --with tox-uv
-      - run: tox run -e phoenix_otel
+      - run: tox run -e phoenix_otel -- -ra -x
 
   lint:
     name: Lint Phoenix
@@ -127,50 +128,150 @@ jobs:
           hatch run notebooks:clean
           git diff --exit-code
 
-  unit-test:
+  ruff:
+    name: Ruff
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        py: [ 3.12 ]
+        os: [ ubuntu-latest ]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.py }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.py }}
+      - name: Set up `uv`
+        uses: astral-sh/setup-uv@v3
+        with:
+          version: 0.4.18
+      - name: Install `ruff`
+        run: uv tool install ruff@0.6.8
+      - name: Check format
+        run: uv tool run ruff format --diff
+      - name: Check lint
+        run: uv tool run ruff check --no-fix
+
+  unit-tests:
     name: Unit Tests
     runs-on: ${{ matrix.os }}
     needs: changes
     if: ${{ needs.changes.outputs.phoenix == 'true' }}
     strategy:
       matrix:
-        py: ["3.9"]
-        os: [macos-12, windows-latest, ubuntu-latest]
+        py: [ 3.9, 3.12 ]
+        os: [ ubuntu-latest, windows-latest, macos-13 ]
     steps:
-      - name: Checkout Repository
+      - name: Checkout repository
         uses: actions/checkout@v4
-      - name: Set up python ${{ matrix.py }}
+        with:
+          sparse-checkout: |
+            requirements/
+            src/phoenix/
+            tests/unit/
+            tests/conftest.py
+      - name: Remove symbolic links (non-Windows)
+        run: find . -type l -delete
+        if: runner.os != 'Windows'
+      - name: Remove symbolic links (Windows)
+        run: |
+          Remove-Item src/phoenix/evals
+          Remove-Item src/phoenix/otel
+        shell: powershell
+        if: runner.os == 'Windows'
+      - name: Set up Python ${{ matrix.py }}
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.py }}
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip==${{ env.pip-version }}
-          pip install hatch
-      - name: Install PostgreSQL (Ubuntu)
+      - name: Set up `uv`
+        uses: astral-sh/setup-uv@v3
+        with:
+          version: 0.4.18
+          enable-cache: true
+          cache-dependency-glob: |
+            pyproject.toml
+            requirements/ci.txt
+            requirements/unit-tests.txt
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Set up `tox` with `tox-uv`
+        run: uv tool install tox --with tox-uv
+      - name: Install PostgreSQL (Linux)
         if: runner.os == 'Linux'
-        run: |
-          sudo apt-get -yqq install postgresql
-      - name: Run tests with PostgreSQL
+        run: sudo apt-get -yqq install postgresql
+      - name: Run tests with PostgreSQL (Linux)
         if: runner.os == 'Linux'
-        run: hatch run test:tests --run-postgres
-      - name: Run tests without PostgreSQL
+        run: tox run -e unit_tests -- -ra -x --run-postgres
+      - name: Run tests without PostgreSQL (non-Linux)
         if: runner.os != 'Linux'
-        run: hatch run test:tests
+        run: tox run -e unit_tests -- -ra -x
 
-  integration-test:
+  type-check-integration-tests:
+    name: Type Check Integration Tests
+    runs-on: ${{ matrix.os }}
+    needs: changes
+    if: ${{ needs.changes.outputs.phoenix == 'true' }}
+    strategy:
+      matrix:
+        py: [ 3.9, 3.12 ]
+        os: [ ubuntu-latest ]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: |
+            requirements/
+            src/phoenix/
+            tests/integration/
+      - name: Remove symbolic links (Non-Windows)
+        run: find . -type l -delete
+        if: runner.os != 'Windows'
+      - name: Remove symbolic links (Windows)
+        run: |
+          Remove-Item src/phoenix/evals
+          Remove-Item src/phoenix/otel
+        shell: powershell
+        if: runner.os == 'Windows'
+      - name: Set up Python ${{ matrix.py }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.py }}
+      - name: Set up `uv`
+        uses: astral-sh/setup-uv@v3
+        with:
+          version: 0.4.18
+          enable-cache: true
+          cache-dependency-glob: |
+            pyproject.toml
+            requirements/ci.txt
+            requirements/integration-tests.txt
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Set up `tox` with `tox-uv`
+        run: uv tool install tox --with tox-uv
+      - name: Check types on integration tests
+        run: tox run -e type_check_integration_tests
+
+  integration-tests:
     name: Integration Tests
     runs-on: ${{ matrix.os }}
     needs: changes
     if: ${{ needs.changes.outputs.phoenix == 'true' }}
     strategy:
       matrix:
         py: [ 3.9, 3.12 ]
-        os: [ubuntu-latest, windows-latest, macos-12]
+        db: [ sqlite, postgresql ]
+        os: [ ubuntu-latest, windows-latest, macos-13 ]
+        exclude:
+          - db: postgresql
+            os: windows-latest
+          - db: postgresql
+            os: macos-13
+    env:
+      CI_TEST_DB_BACKEND: ${{ matrix.db }}
     services:
       postgres:
         # Applying this workaround: https://github.com/actions/runner/issues/822
-        image: ${{ (matrix.os == 'ubuntu-latest') && 'postgres:12' || '' }}
+        image: ${{ (matrix.db == 'postgresql') && 'postgres:12' || '' }}
         env:
           POSTGRES_PASSWORD: phoenix
         options: >-
@@ -181,12 +282,37 @@ jobs:
         ports:
           - 5432:5432
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: |
+            requirements/
+            src/phoenix/
+            tests/integration/
+      - name: Remove symbolic links (non-Windows)
+        run: find . -type l -delete
+        if: runner.os != 'Windows'
+      - name: Remove symbolic links (Windows)
+        run: |
+          Remove-Item src/phoenix/evals
+          Remove-Item src/phoenix/otel
+        shell: powershell
+        if: runner.os == 'Windows'
+      - name: Set up Python ${{ matrix.py }}
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.py }}
-      - uses: astral-sh/setup-uv@v3
+      - name: Set up `uv`
+        uses: astral-sh/setup-uv@v3
         with:
           version: 0.4.18
-      - run: uv tool install tox --with tox-uv
-      - run: tox run -e integration_tests
+          enable-cache: true
+          cache-dependency-glob: |
+            pyproject.toml
+            requirements/ci.txt
+            requirements/integration-tests.txt
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Set up `tox` with `tox-uv`
+        run: uv tool install tox --with tox-uv
+      - name: Run integration tests
+        run: tox run -e integration_tests -- -ra -x -n auto

.github/workflows/python-cron.yml

@@ -8,6 +8,8 @@ on:
 jobs:
   integration-test:
     runs-on: ubuntu-latest
+    env:
+      CI_TEST_DB_BACKEND: postgresql
     services:
       postgres:
         image: postgres:12
@@ -30,8 +32,14 @@ jobs:
       - uses: astral-sh/setup-uv@v3
         with:
           version: 0.4.18
+          enable-cache: true
+          cache-dependency-glob: |
+            pyproject.toml
+            requirements/ci.txt
+            requirements/integration-tests.txt
+          github-token: ${{ secrets.GITHUB_TOKEN }}
       - run: uv tool install tox --with tox-uv
-      - run: tox run -e integration_tests
+      - run: tox run -e ci_integration_tests
       - uses: slackapi/slack-github-action@v1
         if: failure()
         env:

packages/phoenix-evals/pyproject.toml

@@ -101,7 +101,7 @@ force-single-line = false
 "*.ipynb" = ["E402", "E501"]
 
 [tool.ruff.format]
-line-ending = "lf"
+line-ending = "native"
 
 [tool.mypy]
 explicit_package_bases = true

packages/phoenix-otel/pyproject.toml

@@ -82,7 +82,7 @@ force-single-line = false
 "*.ipynb" = ["E402", "E501"]
 
 [tool.ruff.format]
-line-ending = "lf"
+line-ending = "native"
 
 [tool.mypy]
 explicit_package_bases = true

pyproject.toml

@@ -75,7 +75,7 @@ dev = [
   "uv==0.4.8",
   "jupyter",
   "nbqa",
-  "ruff==0.6.8",
+  "ruff==0.6.9",
   "mypy==1.11.2",
   "pandas>=1.0",
   "tabulate",  # used by DataFrame.to_markdown()
@@ -244,7 +244,7 @@ python = ["3.9", "3.12"]
 [tool.hatch.envs.style]
 detached = true
 dependencies = [
-  "ruff==0.6.8",
+  "ruff==0.6.9",
 ]
 
 [[tool.hatch.envs.style.matrix]]
@@ -443,7 +443,7 @@ exclude = [
   "docs/",
 ]
 line-length = 100
-target-version = "py38"
+target-version = "py39"
 
 [tool.ruff.lint.per-file-ignores]
 "*.ipynb" = ["E402", "E501"]

dev-requirements.txt → requirements/ci.txt

@@ -1,6 +1,6 @@
 mypy==1.11.2
-ruff==0.6.8
 pytest==8.3.3
 pytest-xdist==3.6.1
-pytest-asyncio==0.23.8
+pytest-asyncio==0.24.0
 uvloop; platform_system != 'Windows'
+pydantic>=1.0,!=2.0.*,<3

tests/integration/requirements.txt → requirements/integration-tests.txt

@@ -1,9 +1,9 @@
+-r ci.txt
 beautifulsoup4
 faker
 httpx
 openinference-semantic-conventions
 opentelemetry-sdk
-pandas-stubs
 portpicker
 psutil
 pyjwt

requirements/unit-tests.txt

@@ -0,0 +1,19 @@
+-r ci.txt
+Faker>=30.1.0
+arize
+asgi-lifespan
+asyncpg
+grpc-interceptor[testing]
+httpx # For OpenAI testing
+litellm>=1.0.3
+nest-asyncio # for executor testing
+numpy
+openai>=1.0.0
+protobuf==3.20  # version minimum (for tests)
+psycopg[binary,pool]
+pytest-postgresql
+responses
+respx # For OpenAI testing
+tenacity
+tiktoken
+typing-extensions==4.7.0

tests/integration/_helpers.py

@@ -656,13 +656,7 @@ def _random_schema(
     url: URL,
 ) -> Iterator[str]:
     engine = create_engine(url.set(drivername="postgresql+psycopg"))
-    try:
-        engine.connect().close()
-    except OperationalError as exc:
-        if "too many clients" in str(exc):
-            pass
-        else:
-            pytest.skip(f"PostgreSQL unavailable: {exc}")
+    engine.connect().close()
     engine.dispose()
     schema = f"_{secrets.token_hex(15)}"
     yield schema