From fb40b2dac8870601e543c383827526afa0a28e91 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Tue, 4 Feb 2025 10:35:16 +0100 Subject: [PATCH] prepare new release docs update --- CHANGELOG.md | 8 ++++++++ docs/dqx/docs/dev/contributing.mdx | 25 +++++++++++++++---------- docs/dqx/docs/reference.mdx | 26 ++++++++++++++++---------- src/databricks/labs/dqx/__about__.py | 2 +- 4 files changed, 40 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3524593..d6bf707 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Version changelog +## 0.1.10 + +* Support datetime arguments for column range functions (#142) [View](https://github.com/databrickslabs/dqx/pull/142) +* DQX engine refactor and docs update (#138) [View](https://github.com/databrickslabs/dqx/pull/138) +* Add column functions to check for valid date strings (#144) [View](https://github.com/databrickslabs/dqx/pull/144) +* Generate rules for DLT as Python dictionary (#148) [View](https://github.com/databrickslabs/dqx/pull/148) +* Make DQX compatible with Serverless (#147) [View](https://github.com/databrickslabs/dqx/pull/147) + ## 0.1.9 * New dashboard query, Update to demos and docs [#133](https://github.com/databrickslabs/dqx/pull/133) diff --git a/docs/dqx/docs/dev/contributing.mdx b/docs/dqx/docs/dev/contributing.mdx index c111635..3a8f7c2 100644 --- a/docs/dqx/docs/dev/contributing.mdx +++ b/docs/dqx/docs/dev/contributing.mdx @@ -101,23 +101,23 @@ You can use any Unity Catalog enabled Databricks workspace. #### Using terminal -If you want to run integration tests and code coverage from your local machine, you need to set up the following environment variables: +If you want to run the tests from your local machine in the terminal, you need to set up the following environment variables: ```shell export DATABRICKS_HOST=https:// export DATABRICKS_CLUSTER_ID= -# either use PAT token for authentication -export DATABRICKS_TOKEN= - -# or use service principal OAuth token for authentication -export DATABRICKS_CLIENT_ID= -export DATABRICKS_CLIENT_SECRET= +# Authenticate to Databricks using OAuth generated for a service principal (recommended) +export DATABRICKS_CLIENT_ID= +export DATABRICKS_CLIENT_SECRET= # Optionally enable serverless compute to be used for the tests. # Note that we run integration tests on standard and serverless compute clusters as part of the CI/CD pipelines export DATABRICKS_SERVERLESS_COMPUTE_ID=auto ``` +We recommend using [OAuth access token](https://docs.databricks.com/en/dev-tools/auth/oauth-m2m.html) generated for a service principal to authenticate with Databricks as presented above. +Alternatively, you can authenticate using [PAT token](https://docs.databricks.com/en/dev-tools/auth/pat.html) by setting the `DATABRICKS_TOKEN` environment variable. However, we do not recommend this method, as it is less secure than OAuth. + Run integration tests with the following command: ```shell make integration @@ -139,13 +139,17 @@ Create the `~/.databricks/debug-env.json` with the following content, replacing ```json { "ws": { - "DATABRICKS_TOKEN": "", + "DATABRICKS_CLIENT_ID": "", + "DATABRICKS_CLIENT_SECRET": "", "DATABRICKS_HOST": "https://", "DATABRICKS_CLUSTER_ID": "" } } ``` -You must provide an existing cluster, but it will be auto-started for you as part of the tests. +You must provide an existing cluster which will be auto-started for you as part of the tests. + +We recommend using [OAuth access token](https://docs.databricks.com/en/dev-tools/auth/oauth-m2m.html) generated for a service principal to authenticate with Databricks as presented above. +Alternatively, you can authenticate using [PAT token](https://docs.databricks.com/en/dev-tools/auth/pat.html) by providing the `DATABRICKS_TOKEN` field. However, we do not recommend this method, as it is less secure than OAuth. **Running Tests on Serverless Compute** @@ -154,7 +158,8 @@ To run integration tests on serverless compute, add the `DATABRICKS_SERVERLESS_C ```json { "ws": { - "DATABRICKS_TOKEN": "", + "DATABRICKS_CLIENT_ID": "", + "DATABRICKS_CLIENT_SECRET": "", "DATABRICKS_HOST": "https://", "DATABRICKS_CLUSTER_ID": "", "DATABRICKS_SERVERLESS_COMPUTE_ID": "auto" diff --git a/docs/dqx/docs/reference.mdx b/docs/dqx/docs/reference.mdx index 792d56c..fea4848 100644 --- a/docs/dqx/docs/reference.mdx +++ b/docs/dqx/docs/reference.mdx @@ -176,17 +176,17 @@ If you want to run the tests from your local machine in the terminal, you need t export DATABRICKS_HOST=https:// export DATABRICKS_CLUSTER_ID= -# either use PAT token for authentication -export DATABRICKS_TOKEN= - -# or use service principal OAuth token for authentication -export DATABRICKS_CLIENT_ID= -export DATABRICKS_CLIENT_SECRET= +# Authenticate to Databricks using OAuth generated for a service principal (recommended) +export DATABRICKS_CLIENT_ID= +export DATABRICKS_CLIENT_SECRET= # Optionally enable serverless compute to be used for the tests export DATABRICKS_SERVERLESS_COMPUTE_ID=auto ``` +We recommend using [OAuth access token](https://docs.databricks.com/en/dev-tools/auth/oauth-m2m.html) generated for a service principal to authenticate with Databricks as presented above. +Alternatively, you can authenticate using [PAT token](https://docs.databricks.com/en/dev-tools/auth/pat.html) by setting the `DATABRICKS_TOKEN` environment variable. However, we do not recommend this method, as it is less secure than OAuth. + #### Setting up Databricks workspace client authentication in an IDE If you want to run the tests from your IDE, you must setup `.env` or `~/.databricks/debug-env.json` file @@ -199,13 +199,18 @@ Create the `~/.databricks/debug-env.json` with the following content, replacing ```json { "ws": { - "DATABRICKS_TOKEN": "", + "DATABRICKS_CLIENT_ID": "", + "DATABRICKS_CLIENT_SECRET": "", "DATABRICKS_HOST": "https://", "DATABRICKS_CLUSTER_ID": "" } } ``` -You must provide an existing cluster, but it will be auto-started for you as part of the tests. + +You must provide an existing cluster which will be auto-started for you as part of the tests. + +We recommend using [OAuth access token](https://docs.databricks.com/en/dev-tools/auth/oauth-m2m.html) generated for a service principal to authenticate with Databricks as presented above. +Alternatively, you can authenticate using [PAT token](https://docs.databricks.com/en/dev-tools/auth/pat.html) by providing the `DATABRICKS_TOKEN` field. However, we do not recommend this method, as it is less secure than OAuth. **Running Tests on Serverless Compute** @@ -213,7 +218,8 @@ To run the integration tests on serverless compute, add the `DATABRICKS_SERVERLE ```json { "ws": { - "DATABRICKS_TOKEN": "", + "DATABRICKS_CLIENT_ID": "", + "DATABRICKS_CLIENT_SECRET": "", "DATABRICKS_HOST": "https://", "DATABRICKS_CLUSTER_ID": "", "DATABRICKS_SERVERLESS_COMPUTE_ID": "auto" @@ -224,7 +230,7 @@ When `DATABRICKS_SERVERLESS_COMPUTE_ID` is set the `DATABRICKS_CLUSTER_ID` is ig ### Local testing with DQEngine -If workspace-level access is unavailable in your unit testing environment, you can perform local testing by installing the latest `pyspark` package and mocking the workspace client. +If workspace-level access is unavailable in your testing environment, you can perform local testing by installing the latest `pyspark` package and mocking the workspace client. **Note: This approach should be treated as experimental!** It does not offer the same level of testing as the standard approach and it is only applicable to selected methods. We strongly recommend following the standard testing procedure outlined above, which includes proper initialization of the workspace client. diff --git a/src/databricks/labs/dqx/__about__.py b/src/databricks/labs/dqx/__about__.py index c11f861..569b121 100644 --- a/src/databricks/labs/dqx/__about__.py +++ b/src/databricks/labs/dqx/__about__.py @@ -1 +1 @@ -__version__ = "0.1.9" +__version__ = "0.1.10"