Skip to content

Commit

Permalink
all basic tests enabled (#278)
Browse files Browse the repository at this point in the history 
* Reorganizing commits

* Added TestSnapshotTimestampGlue

* Added TestDocsGenReferencesGlue

* fix TestTableMatGlue and TestValidateConnectionGlue

* Skip TestBaseCachingGlue

Also this fixes the execution error

---------

Co-authored-by: menuetb <83284881+menuetb@users.noreply.github.com>
Co-authored-by: Akira Ajisaka <akiraaj@amazon.com>
  • Loading branch information
@moomindani @menuetb @aajisaka
3 people authored Nov 14, 2023
1 parent 87359dd commit acadda6
Show file tree
Hide file tree
Showing 4 changed files with 229 additions and 81 deletions.
2 changes: 1 addition & 1 deletion dbt/adapters/glue/connections.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ def get_response(cls, cursor) -> AdapterResponse:
def get_result_from_cursor(cls, cursor: GlueCursor, limit: Optional[int]) -> agate.Table:
data: List[Any] = []
column_names: List[str] = []
if cursor.description is not None:
if not cursor.description:
column_names = [col[0] for col in cursor.description()]
if limit:
rows = cursor.fetchmany(limit)
Expand Down
71 changes: 45 additions & 26 deletions tests/functional/adapter/test_basic.py
View file
Original file line number Diff line number Diff line change
@@ -1,35 +1,23 @@
import pytest

import os

import pytest
from dbt.tests.adapter.basic.files import (base_ephemeral_sql, base_table_sql,
base_view_sql, ephemeral_table_sql,
ephemeral_view_sql)
from dbt.tests.adapter.basic.test_adapter_methods import BaseAdapterMethod
from dbt.tests.adapter.basic.test_base import BaseSimpleMaterializations
from dbt.tests.adapter.basic.test_singular_tests import BaseSingularTests
from dbt.tests.adapter.basic.test_singular_tests_ephemeral import BaseSingularTestsEphemeral
from dbt.tests.adapter.basic.test_empty import BaseEmpty
from dbt.tests.adapter.basic.test_ephemeral import BaseEphemeral
from dbt.tests.adapter.basic.test_incremental import BaseIncremental
from dbt.tests.adapter.basic.test_generic_tests import BaseGenericTests
from dbt.tests.adapter.basic.test_docs_generate import BaseDocsGenerate, BaseDocsGenReferences
from dbt.tests.adapter.basic.test_snapshot_check_cols import BaseSnapshotCheckCols
from dbt.tests.adapter.basic.test_snapshot_timestamp import BaseSnapshotTimestamp
from dbt.tests.adapter.basic.files import (
base_view_sql,
base_table_sql,
base_ephemeral_sql,
ephemeral_view_sql,
ephemeral_table_sql
)

from dbt.tests.util import (
run_dbt,
get_manifest,
check_result_nodes_by_name,
relation_from_name,
check_relations_equal,
)

from dbt.tests.adapter.basic.test_incremental import BaseIncremental
from dbt.tests.adapter.basic.test_singular_tests import BaseSingularTests
from dbt.tests.adapter.basic.test_singular_tests_ephemeral import BaseSingularTestsEphemeral
from dbt.tests.adapter.basic.test_table_materialization import BaseTableMaterialization
from dbt.tests.adapter.basic.test_validate_connection import BaseValidateConnection
from dbt.tests.util import (check_relations_equal, check_result_nodes_by_name,
get_manifest, relation_from_name, run_dbt)
from tests.util import get_s3_location, get_region, cleanup_s3_location


s3bucket = get_s3_location()
region = get_region()
schema_name = "dbt_functional_test_01"
Expand Down Expand Up @@ -59,6 +47,18 @@
base_materialized_var_sql = config_materialized_var + config_incremental_strategy + model_base


@pytest.mark.skip(
reason="Fails because the test tries to fetch the table metadata during the compile step, "
"before the models are actually run. Not sure how this test is intended to work."
)
class TestBaseCachingGlue(BaseAdapterMethod):
@pytest.fixture(scope="class")
def unique_schema(request, prefix) -> str:
return schema_name

pass


class TestSimpleMaterializationsGlue(BaseSimpleMaterializations):
# all tests within this test has the same schema
@pytest.fixture(scope="class")
Expand Down Expand Up @@ -167,13 +167,15 @@ def test_ephemeral(self, project):

pass


class TestSingularTestsEphemeralGlue(BaseSingularTestsEphemeral):
@pytest.fixture(scope="class")
def unique_schema(request, prefix) -> str:
return schema_name

pass


class TestIncrementalGlue(BaseIncremental):
@pytest.fixture(scope='class', autouse=True)
def cleanup(self):
Expand Down Expand Up @@ -267,4 +269,21 @@ def test_generic_tests(self, project):
results = run_dbt(["test"])
assert len(results) == 3

pass
pass


class TestTableMatGlue(BaseTableMaterialization):
@pytest.fixture(scope="class")
def unique_schema(request, prefix) -> str:
return schema_name

pass


class TestValidateConnectionGlue(BaseValidateConnection):
@pytest.fixture(scope="class")
def unique_schema(request, prefix) -> str:
return schema_name

pass

136 changes: 132 additions & 4 deletions tests/functional/adapter/test_docs.py
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
import os
import pytest

from dbt.tests.adapter.basic.test_docs_generate import BaseDocsGenerate
from dbt.tests.util import get_artifact
from dbt.tests.adapter.basic.test_docs_generate import BaseDocsGenerate, BaseDocsGenReferences
from dbt.tests.adapter.basic.expected_catalog import no_stats


schema_name = "dbt_functional_test_docs01"
schema_name = "dbt_functional_test_01"


class TestDocsGenerate(BaseDocsGenerate):
Expand Down Expand Up @@ -107,3 +107,131 @@ def case(x):

pass


class TestDocsGenReferencesGlue(BaseDocsGenReferences):
@pytest.fixture(scope="class")
def unique_schema(request, prefix) -> str:
return schema_name

@pytest.fixture(scope="class")
def expected_catalog(self, project, profile_user):
role = None
id_type = "double"
text_type = "string"
time_type = "string"
view_type = "view"
table_type = "table"
model_stats = no_stats()
bigint_type = "bigint"
seed_stats = None
case = None
case_columns = False
view_summary_stats = None

if case is None:
def case(x):
return x

col_case = case if case_columns else lambda x: x

if seed_stats is None:
seed_stats = model_stats

if view_summary_stats is None:
view_summary_stats = model_stats

my_schema_name = case(project.test_schema)

summary_columns = {
"first_name": {
"name": "first_name",
"index": 0,
"type": text_type,
"comment": None,
},
"ct": {
"name": "ct",
"index": 0,
"type": bigint_type,
"comment": None,
},
}

seed_columns = {
"id": {
"name": col_case("id"),
"index": 0,
"type": id_type,
"comment": None,
},
"first_name": {
"name": col_case("first_name"),
"index": 0,
"type": text_type,
"comment": None,
},
"email": {
"name": col_case("email"),
"index": 0,
"type": text_type,
"comment": None,
},
"ip_address": {
"name": col_case("ip_address"),
"index": 0,
"type": text_type,
"comment": None,
},
"updated_at": {
"name": col_case("updated_at"),
"index": 0,
"type": time_type,
"comment": None,
},
}
return {
"nodes": {
"seed.test.seed": {
"unique_id": "seed.test.seed",
"metadata": {
"schema": my_schema_name,
"database": my_schema_name,
"name": case("seed"),
"type": table_type,
"comment": None,
"owner": role,
},
"stats": seed_stats,
"columns": seed_columns,
},
"model.test.ephemeral_summary": {
"unique_id": "model.test.ephemeral_summary",
"metadata": {
"schema": my_schema_name,
"database": my_schema_name,
"name": case("ephemeral_summary"),
"type": table_type,
"comment": None,
"owner": role,
},
"stats": model_stats,
"columns": summary_columns,
},
"model.test.view_summary": {
"unique_id": "model.test.view_summary",
"metadata": {
"schema": my_schema_name,
"database": my_schema_name,
"name": case("view_summary"),
"type": view_type,
"comment": None,
"owner": role,
},
"stats": view_summary_stats,
"columns": summary_columns,
},
},
"sources": {}
}

pass
101 changes: 51 additions & 50 deletions tests/functional/adapter/test_snapshot.py
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,13 @@
import pytest

from dbt.tests.adapter.basic.test_snapshot_check_cols import BaseSnapshotCheckCols
from dbt.tests.adapter.basic.test_snapshot_timestamp import BaseSnapshotTimestamp
from dbt.tests.util import run_dbt, relation_from_name

from tests.util import get_s3_location, get_region, cleanup_s3_location


s3bucket = get_s3_location()
region = get_region()
schema_name = "dbt_functional_test_snapshot01"
schema_name = "dbt_functional_test_01"


def check_relation_rows(project, snapshot_name, count):
Expand Down Expand Up @@ -48,7 +47,7 @@ def project_config_update(self):
}

def test_snapshot_check_cols(self, project):
# seed command
# seed commandte
results = run_dbt(["seed"])
assert len(results) == 2

Expand Down Expand Up @@ -76,52 +75,54 @@ def test_snapshot_check_cols(self, project):
check_relation_rows(project, "cc_name_snapshot", 20)
check_relation_rows(project, "cc_date_snapshot", 20)

# # update some timestamps in the "added" seed so the snapshot sees 10 more new rows
# update_rows_config = {
# "name": "added",
# "dst_col": "some_date",
# "clause": {"src_col": "some_date", "type": "add_timestamp"},
# "where": "id > 10 and id < 21",
# }
# update_rows(project.adapter, update_rows_config)
#
# # re-run snapshots, using "added'
# results = run_dbt(["snapshot", "--vars", "seed_name: added"])
# for result in results:
# assert result.status == "success"
#
# # check rowcounts for all snapshots
# check_relation_rows(project, "cc_all_snapshot", 30)
# check_relation_rows(project, "cc_date_snapshot", 30)
# # unchanged: only the timestamp changed
# check_relation_rows(project, "cc_name_snapshot", 20)
#
# # Update the name column
# update_rows_config = {
# "name": "added",
# "dst_col": "name",
# "clause": {
# "src_col": "name",
# "type": "add_string",
# "value": "_updated",
# },
# "where": "id < 11",
# }
# update_rows(project.adapter, update_rows_config)
#
# # re-run snapshots, using "added'
# results = run_dbt(["snapshot", "--vars", "seed_name: added"])
# for result in results:
# assert result.status == "success"
#
# # check rowcounts for all snapshots
# check_relation_rows(project, "cc_all_snapshot", 40)
# check_relation_rows(project, "cc_name_snapshot", 30)
# # does not see name updates
# check_relation_rows(project, "cc_date_snapshot", 30)

pass


#class TestSnapshotTimestampGlue(BaseSnapshotTimestamp):
# pass
class TestSnapshotTimestampGlue(BaseSnapshotTimestamp):
@pytest.fixture(scope="class")
def unique_schema(request, prefix) -> str:
return schema_name

@pytest.fixture(scope='class', autouse=True)
def cleanup(self):
cleanup_s3_location(s3bucket + schema_name, region)
yield

@pytest.fixture(scope="class")
def project_config_update(self):
return {
"seeds": {
"+file_format": "delta",
"quote_columns": False,
},
"snapshots": {
"+file_format": "delta",
"+updated_at": "current_timestamp()",
"quote_columns": False,
},
"quoting": {
"database": False,
"schema": False,
"identifier": False
},
}

def test_snapshot_timestamp(self, project):
# seed command
results = run_dbt(["seed"])
assert len(results) == 3

# snapshot command
results = run_dbt(["snapshot"])
assert len(results) == 1

# snapshot has 10 rows
check_relation_rows(project, "ts_snapshot", 10)

# point at the "added" seed so the snapshot sees 10 new rows
results = run_dbt(["snapshot", "--vars", "seed_name: added"])

# snapshot now has 20 rows
check_relation_rows(project, "ts_snapshot", 20)

pass

0 comments on commit acadda6

Please sign in to comment.