Fix installation process for Serverless

.github/workflows/acceptance.yml

@@ -20,7 +20,10 @@ concurrency:
 
 jobs:
   integration:
-    if: github.event_name == 'pull_request' && github.event.pull_request.draft == false
+    # Only run this job for PRs from branches on the main repository and not from forks.
+    # Workflows triggered by PRs from forks don't have access to the tool environment.
+    # PRs from forks to be tested by the reviewer(s) / maintainer(s) before merging.
+    if: github.event_name == 'pull_request' && !github.event.pull_request.draft && !github.event.pull_request.head.repo.fork
     environment: tool
     runs-on: larger
     steps:
@@ -55,7 +58,10 @@ jobs:
           ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }}
 
   serverless_integration:
-    if: github.event_name == 'pull_request' && github.event.pull_request.draft == false
+    # Only run this job for PRs from branches on the main repository and not from forks.
+    # Workflows triggered by PRs from forks don't have access to the tool environment.
+    # PRs from forks to be tested by the reviewer(s) / maintainer(s) before merging.
+    if: github.event_name == 'pull_request' && !github.event.pull_request.draft && !github.event.pull_request.head.repo.fork
     environment: tool
     runs-on: larger
     env:

docs/dqx/docs/installation.mdx

@@ -10,7 +10,7 @@ The framework can be installed on a Databricks workspace or used as a standalone
 - Python 3.10 or later. See [instructions](https://www.python.org/downloads/).
 - Unity Catalog-enabled [Databricks workspace](https://docs.databricks.com/en/getting-started/index.html).
 - Network access to your Databricks Workspace used for the installation process.
-- (Optional) Databricks CLI v0.213 or later. See [instructions](https://docs.databricks.com/dev-tools/cli/databricks-cli.html).
+- (Optional) Databricks CLI v0.241 or later. See [instructions](https://docs.databricks.com/dev-tools/cli/databricks-cli.html).
 - Databricks Runtime with Spark 3.5.0 or higher. See [instructions](https://docs.databricks.com/clusters/create.html).
 
 ## DQX installation as a Library

labs.yml

@@ -3,6 +3,7 @@ name: dqx
 description: Data Quality Framework for PySpark Workloads
 install:
   script: src/databricks/labs/dqx/installer/install.py
+  extras: cli
 uninstall:
   script: src/databricks/labs/dqx/installer/uninstall.py
 entrypoint: src/databricks/labs/dqx/cli.py

pyproject.toml

@@ -30,9 +30,11 @@ classifiers = [
 dependencies = ["databricks-labs-blueprint>=0.9.1,<0.10",
                 "databricks-sdk~=0.30",
                 "databricks-labs-lsql>=0.5,<0.15",
-                "pyspark~=3.5.0" # required by the CLI to work correctly
 ]
 
+[project.optional-dependencies]
+cli = ["pyspark~=3.5.0"]
+
 [project.entry-points.databricks]
 runtime = "databricks.labs.dqx.runtime:main"
 
@@ -70,6 +72,7 @@ dependencies = [
     "types-PyYAML~=6.0.12",
     "types-requests~=2.31.0",
     "databricks-connect~=15.4",
+    "pyspark~=3.5.0",
 ]
 
 python="3.10"