diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 06649856..301e875e 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -35,7 +35,7 @@ jobs: env: TOXENV: "integration" - PYTEST_ADDOPTS: "-v --color=yes --csv unit_results.csv" + PYTEST_ADDOPTS: "-v --color=yes --csv integ_results.csv" DBT_AWS_ACCOUNT: ${{ secrets.DBT_AWS_ACCOUNT }} DBT_GLUE_ROLE_ARN: ${{ secrets.DBT_GLUE_ROLE_ARN }} DBT_GLUE_REGION: ${{ secrets.DBT_GLUE_REGION }} @@ -103,7 +103,7 @@ jobs: env: TOXENV: "integration" - PYTEST_ADDOPTS: "-v --color=yes --csv unit_results.csv" + PYTEST_ADDOPTS: "-v --color=yes --csv integ_results.csv -s" DBT_AWS_ACCOUNT: ${{ secrets.DBT_AWS_ACCOUNT }} DBT_GLUE_ROLE_ARN: ${{ secrets.DBT_GLUE_ROLE_ARN }} DBT_GLUE_REGION: ${{ secrets.DBT_GLUE_REGION }} diff --git a/dbt/adapters/glue/connections.py b/dbt/adapters/glue/connections.py index 43cd4423..1399a509 100644 --- a/dbt/adapters/glue/connections.py +++ b/dbt/adapters/glue/connections.py @@ -99,6 +99,7 @@ def get_response(cls, cursor) -> AdapterResponse: @classmethod def get_result_from_cursor(cls, cursor: GlueCursor, limit: Optional[int]) -> agate.Table: + logger.debug("get_result_from_cursor called") data: List[Any] = [] column_names: List[str] = [] if cursor.description is not None: diff --git a/dbt/adapters/glue/gluedbapi/cursor.py b/dbt/adapters/glue/gluedbapi/cursor.py index a8871a09..aff0c8bd 100644 --- a/dbt/adapters/glue/gluedbapi/cursor.py +++ b/dbt/adapters/glue/gluedbapi/cursor.py @@ -207,12 +207,12 @@ def __next__(self): return item def description(self): - logger.debug("GlueCursor get_columns_in_relation called") + logger.debug("GlueCursor description called") if self.response: return [[c["name"], c["type"]] for c in self.response.get("description", [])] def get_response(self) -> AdapterResponse: - logger.debug("GlueCursor get_columns_in_relation called") + logger.debug("GlueCursor get_response called") if self.statement: r = self.statement._get_statement() return AdapterResponse( @@ -222,7 +222,7 @@ def get_response(self) -> AdapterResponse: ) def close(self): - logger.debug("GlueCursor get_columns_in_relation called") + logger.debug("GlueCursor close called") if self._closed: raise Exception("CursorAlreadyClosed") self._closed = True diff --git a/dev-requirements.txt b/dev-requirements.txt index 9c0514bb..ca3becc9 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -14,7 +14,6 @@ flake8 pytz tox>=3.2.0 ipdb -pytest-xdist pytest-dotenv pytest-csv flaky diff --git a/tests/functional/adapter/test_basic.py b/tests/functional/adapter/test_basic.py index f7ea1504..b9ff8bb8 100644 --- a/tests/functional/adapter/test_basic.py +++ b/tests/functional/adapter/test_basic.py @@ -1,5 +1,4 @@ import os - import pytest from dbt.tests.adapter.basic.files import (base_ephemeral_sql, base_table_sql, base_view_sql, ephemeral_table_sql, @@ -16,11 +15,7 @@ from dbt.tests.adapter.basic.test_validate_connection import BaseValidateConnection from dbt.tests.util import (check_relations_equal, check_result_nodes_by_name, get_manifest, relation_from_name, run_dbt) -from tests.util import get_s3_location, get_region, cleanup_s3_location -s3bucket = get_s3_location() -region = get_region() -schema_name = "dbt_functional_test_01" # override schema_base_yml to set missing database schema_base_yml = """ @@ -52,19 +47,10 @@ "before the models are actually run. Not sure how this test is intended to work." ) class TestBaseCachingGlue(BaseAdapterMethod): - @pytest.fixture(scope="class") - def unique_schema(request, prefix) -> str: - return schema_name - pass class TestSimpleMaterializationsGlue(BaseSimpleMaterializations): - # all tests within this test has the same schema - @pytest.fixture(scope="class") - def unique_schema(request, prefix) -> str: - return schema_name - @pytest.fixture(scope="class") def project_config_update(self): return { @@ -83,36 +69,18 @@ def models(self): "schema.yml": schema_base_yml, } - @pytest.fixture(scope='class', autouse=True) - def cleanup(self): - cleanup_s3_location(s3bucket + schema_name, region) - yield - pass class TestSingularTestsGlue(BaseSingularTests): - @pytest.fixture(scope="class") - def unique_schema(request, prefix) -> str: - return schema_name - pass class TestEmptyGlue(BaseEmpty): - @pytest.fixture(scope="class") - def unique_schema(request, prefix) -> str: - return schema_name - pass class TestEphemeralGlue(BaseEphemeral): - # all tests within this test has the same schema - @pytest.fixture(scope="class") - def unique_schema(request, prefix) -> str: - return schema_name - @pytest.fixture(scope="class") def models(self): return { @@ -122,11 +90,6 @@ def models(self): "schema.yml": schema_base_yml, } - @pytest.fixture(scope='class', autouse=True) - def cleanup(self): - cleanup_s3_location(s3bucket + schema_name, region) - yield - # test_ephemeral with refresh table def test_ephemeral(self, project): # seed command @@ -169,19 +132,10 @@ def test_ephemeral(self, project): class TestSingularTestsEphemeralGlue(BaseSingularTestsEphemeral): - @pytest.fixture(scope="class") - def unique_schema(request, prefix) -> str: - return schema_name - pass class TestIncrementalGlue(BaseIncremental): - @pytest.fixture(scope='class', autouse=True) - def cleanup(self): - cleanup_s3_location(s3bucket + schema_name, region) - yield - @pytest.fixture(scope="class") def models(self): model_incremental = """ @@ -190,10 +144,6 @@ def models(self): return {"incremental.sql": model_incremental, "schema.yml": schema_base_yml} - @pytest.fixture(scope="class") - def unique_schema(request, prefix) -> str: - return schema_name - # test_incremental with refresh table def test_incremental(self, project): # seed command @@ -239,15 +189,6 @@ def test_incremental(self, project): class TestGenericTestsGlue(BaseGenericTests): - @pytest.fixture(scope="class") - def unique_schema(request, prefix) -> str: - return schema_name - - @pytest.fixture(scope='class', autouse=True) - def cleanup(self): - cleanup_s3_location(s3bucket + schema_name, region) - yield - def test_generic_tests(self, project): # seed command results = run_dbt(["seed"]) @@ -273,17 +214,9 @@ def test_generic_tests(self, project): class TestTableMatGlue(BaseTableMaterialization): - @pytest.fixture(scope="class") - def unique_schema(request, prefix) -> str: - return schema_name - pass class TestValidateConnectionGlue(BaseValidateConnection): - @pytest.fixture(scope="class") - def unique_schema(request, prefix) -> str: - return schema_name - pass diff --git a/tests/functional/adapter/test_docs.py b/tests/functional/adapter/test_docs.py index aa1bd75c..ea3ad87a 100644 --- a/tests/functional/adapter/test_docs.py +++ b/tests/functional/adapter/test_docs.py @@ -1,18 +1,9 @@ -import os import pytest -from dbt.tests.util import get_artifact from dbt.tests.adapter.basic.test_docs_generate import BaseDocsGenerate, BaseDocsGenReferences from dbt.tests.adapter.basic.expected_catalog import no_stats -schema_name = "dbt_functional_test_01" - class TestDocsGenerate(BaseDocsGenerate): - # all tests within this test has the same schema - @pytest.fixture(scope="class") - def unique_schema(request, prefix) -> str: - return schema_name - @pytest.fixture(scope="class") def expected_catalog(self, project, profile_user): role = None @@ -109,10 +100,6 @@ def case(x): class TestDocsGenReferencesGlue(BaseDocsGenReferences): - @pytest.fixture(scope="class") - def unique_schema(request, prefix) -> str: - return schema_name - @pytest.fixture(scope="class") def expected_catalog(self, project, profile_user): role = None diff --git a/tests/functional/adapter/test_snapshot.py b/tests/functional/adapter/test_snapshot.py index e4601a75..415f5854 100644 --- a/tests/functional/adapter/test_snapshot.py +++ b/tests/functional/adapter/test_snapshot.py @@ -3,11 +3,6 @@ from dbt.tests.adapter.basic.test_snapshot_check_cols import BaseSnapshotCheckCols from dbt.tests.adapter.basic.test_snapshot_timestamp import BaseSnapshotTimestamp from dbt.tests.util import run_dbt, relation_from_name -from tests.util import get_s3_location, get_region, cleanup_s3_location - -s3bucket = get_s3_location() -region = get_region() -schema_name = "dbt_functional_test_01" def check_relation_rows(project, snapshot_name, count): @@ -18,15 +13,6 @@ def check_relation_rows(project, snapshot_name, count): class TestSnapshotCheckColsGlue(BaseSnapshotCheckCols): - @pytest.fixture(scope="class") - def unique_schema(request, prefix) -> str: - return schema_name - - @pytest.fixture(scope='class', autouse=True) - def cleanup(self): - cleanup_s3_location(s3bucket + schema_name, region) - yield - @pytest.fixture(scope="class") def project_config_update(self): return { @@ -79,15 +65,6 @@ def test_snapshot_check_cols(self, project): class TestSnapshotTimestampGlue(BaseSnapshotTimestamp): - @pytest.fixture(scope="class") - def unique_schema(request, prefix) -> str: - return schema_name - - @pytest.fixture(scope='class', autouse=True) - def cleanup(self): - cleanup_s3_location(s3bucket + schema_name, region) - yield - @pytest.fixture(scope="class") def project_config_update(self): return { diff --git a/tests/functional/conftest.py b/tests/functional/conftest.py new file mode 100644 index 00000000..bff84199 --- /dev/null +++ b/tests/functional/conftest.py @@ -0,0 +1,47 @@ +import pytest +import os +import random +import string +from tests.util import get_s3_location, get_region, cleanup_s3_location + +s3bucket = get_s3_location() +region = get_region() + +# Import the standard functional fixtures as a plugin +# Note: fixtures with session scope need to be local +pytest_plugins = ["dbt.tests.fixtures.project"] + +# Use different datatabase for each test class +@pytest.fixture(scope="class") +def unique_schema(request, prefix) -> str: + database_suffix = ''.join(random.choices(string.digits, k=4)) + return f"dbt_functional_test_{database_suffix}" + + +# The profile dictionary, used to write out profiles.yml +# dbt will supply a unique schema per test, so we do not specify 'schema' here +@pytest.fixture(scope="class") +def dbt_profile_target(unique_schema): + return { + 'type': 'glue', + 'query-comment': 'test-glue-adapter', + 'role_arn': os.getenv('DBT_GLUE_ROLE_ARN'), + 'user': os.getenv('DBT_GLUE_ROLE_ARN'), + 'region': get_region(), + 'workers': 2, + 'worker_type': 'G.1X', + 'schema': unique_schema, + 'database': unique_schema, + 'session_provisioning_timeout_in_seconds': 300, + 'location': get_s3_location(), + 'datalake_formats': 'delta', + 'conf': "spark.sql.extensions=io.delta.sql.DeltaSparkSessionExtension --conf spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog --conf spark.sql.legacy.allowNonEmptyLocationInCTAS=true", + 'glue_session_reuse': True + } + + +@pytest.fixture(scope='class', autouse=True) +def cleanup(unique_schema): + cleanup_s3_location(s3bucket + unique_schema, region) + yield + cleanup_s3_location(s3bucket + unique_schema, region)