diff --git a/pyproject.toml b/pyproject.toml index 945df92..97e7528 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "target-db2" -version = "0.1.3" +version = "0.1.4" description = "`target-db2` is a Singer target for db2, built with the Meltano Singer SDK." readme = "README.md" authors = ["Haleemur Ali "] diff --git a/target_db2/connector.py b/target_db2/connector.py index bce7c62..e9dd130 100644 --- a/target_db2/connector.py +++ b/target_db2/connector.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import math import re import typing as t from random import choice @@ -26,6 +27,7 @@ MAX_VARCHAR_SIZE = 10000 MAX_PK_STRING_SIZE = 1022 +MAX_DECIMAL_PRECISION = 31 sa.dialects.registry.register("ibm_db_sa", "target_db2.ibm_db_sa", "dialect") @@ -278,10 +280,87 @@ def to_sql_type( jsonschema_type["maxLength"] = string_length is_obj = _jsonschema_type_check(jsonschema_type, ("object",)) is_arr = _jsonschema_type_check(jsonschema_type, ("array",)) + if _jsonschema_type_check(jsonschema_type, ("number",)): + multiple_of = jsonschema_type.get("multipleOf") + if multiple_of and multiple_of != int(multiple_of): + _multiple_of = multiple_of - int(multiple_of) + scale = int(math.ceil(math.log10(1 / _multiple_of))) + precision = MAX_DECIMAL_PRECISION - scale + return sa.types.DECIMAL(precision, scale) + return sa.types.DECIMAL() + if is_obj or is_arr: return JSONVARCHAR(varchar_size) return super(DB2Connector, DB2Connector).to_sql_type(jsonschema_type) + def merge_sql_types( + self, + sql_types: t.Sequence[sa.types.TypeEngine], + ) -> sa.types.TypeEngine: + """Return a compatible SQL type for the selected type list. + + Args: + sql_types: List of SQL types. + + Returns: + A SQL type that is compatible with the input types. + + Raises: + ValueError: If sql_types argument has zero members. + """ + if not sql_types: + msg = "Expected at least one member in `sql_types` argument." + raise ValueError(msg) + + if len(sql_types) == 1: + return sql_types[0] + + # Gathering Type to match variables + # sent in _adapt_column_type + current_type = sql_types[0] + cur_len: int = getattr(current_type, "length", 0) + + # Convert the two types given into a sorted list + # containing the best conversion classes + sql_types = self._sort_types(sql_types) + + # If greater than two evaluate the first pair then on down the line + if len(sql_types) > 2: # noqa: PLR2004 + return self.merge_sql_types( + [self.merge_sql_types([sql_types[0], sql_types[1]]), *sql_types[2:]], + ) + + # if all types are DECIMAL, put the one with the biggest scale at the top + # this will ensure that it gets picked as the merged type. + if all(isinstance(opt, sa.types.DECIMAL) for opt in sql_types): + sql_types = sorted(sql_types, reverse=True, key=lambda x: x.scale or 0) # type: ignore[attr-defined] + return sql_types[0] + # Get the generic type class + for opt in sql_types: + # Get the length + opt_len: int = getattr(opt, "length", 0) + generic_type = type(opt.as_generic()) # type: ignore[attr-defined] + + if isinstance(generic_type, type): + if issubclass( + generic_type, + (sa.types.String, sa.types.Unicode), + ): + # If length None or 0 then is varchar max ? + if ( + (opt_len is None) + or (opt_len == 0) + or (cur_len and (opt_len >= cur_len)) + ): + return opt + # If best conversion class is equal to current type + # return the best conversion class + elif str(opt) == str(current_type): + return opt + + msg = f"Unable to merge sql types: {', '.join([str(t) for t in sql_types])}" + raise ValueError(msg) + def create_empty_table( # noqa: PLR0913 self, full_table_name: str, diff --git a/tests/test_core.py b/tests/test_core.py index bd13148..f389b80 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -6,7 +6,10 @@ import typing as t import pytest +from singer_sdk.helpers._compat import importlib_resources from singer_sdk.testing import get_target_test_class +from singer_sdk.testing.suites import TestSuite +from singer_sdk.testing.templates import TargetFileTestTemplate from sqlalchemy import ( Column, Integer, @@ -22,8 +25,10 @@ from target_db2.connector import JSONVARCHAR, DB2Connector from target_db2.target import TargetDb2 +from tests import testdata if t.TYPE_CHECKING: + from singer_sdk.helpers._compat import Traversable from sqlalchemy.engine.base import Engine @@ -90,10 +95,40 @@ def test_column_objectvarchar(db2: Engine) -> None: conn.execute(DropTable(test_table)) # type: ignore[arg-type] -# Run standard built-in target tests from the SDK: +class TargetFileTestTemplateCustomPath(TargetFileTestTemplate): + """Template uses singer messages in testdata folder to generate tests.""" + + @property + def singer_filepath(self) -> Traversable: + """Get path to singer JSONL formatted messages file. + + Files will be sourced from `./target_test_streams/.singer`. + + Returns: + The expected Path to this tests singer file. + """ + return importlib_resources.files(testdata) / f"{self.name}.singer" + + +class TargetSchemaChangeToNumericMultipleOf(TargetFileTestTemplateCustomPath): + """Test Target handles array data.""" + + name = "schema_change_to_numeric_multipleof" + + +class TargetSchemaHasNumericMultipleOf(TargetFileTestTemplateCustomPath): + """Test Target handles array data.""" + + name = "schema_has_numeric_multipleof" + + +custom_tests = TestSuite( + kind="target", + tests=[TargetSchemaChangeToNumericMultipleOf, TargetSchemaHasNumericMultipleOf], +) + StandardTargetTests = get_target_test_class( - target_class=TargetDb2, - config=SAMPLE_CONFIG, + target_class=TargetDb2, config=SAMPLE_CONFIG, custom_suites=[custom_tests] ) diff --git a/tests/testdata/__init__.py b/tests/testdata/__init__.py new file mode 100644 index 0000000..ee83d91 --- /dev/null +++ b/tests/testdata/__init__.py @@ -0,0 +1 @@ +"""Singer output samples, used for testing.""" diff --git a/tests/testdata/schema_change_to_numeric_multipleof.singer b/tests/testdata/schema_change_to_numeric_multipleof.singer new file mode 100644 index 0000000..3d4cbf4 --- /dev/null +++ b/tests/testdata/schema_change_to_numeric_multipleof.singer @@ -0,0 +1,2 @@ +{"type":"SCHEMA","stream":"test_schema_change_numeric_mulipleof","schema":{"properties":{"id":{"type":["integer"]},"val1":{"type":["number","null"]}},"type":"object","required":["id"]},"key_properties":["id"]} +{"type":"SCHEMA","stream":"test_schema_change_numeric_mulipleof","schema":{"properties":{"id":{"type":["integer"]},"val1":{"type":["number","null"], "multipleOf": 0.01}},"type":"object","required":["id"]},"key_properties":["id"]} diff --git a/tests/testdata/schema_has_numeric_multipleof.singer b/tests/testdata/schema_has_numeric_multipleof.singer new file mode 100644 index 0000000..0cf5e6c --- /dev/null +++ b/tests/testdata/schema_has_numeric_multipleof.singer @@ -0,0 +1 @@ +{"type":"SCHEMA","stream":"test_schema_has_numeric_multipleof","schema":{"properties":{"id":{"type":["number"]},"val1":{"type":["number","null"]},"created_at":{"format":"date-time","type":["string","null"]},"amount":{"multipleOf":0.01,"type":["number","null"]}},"type":"object","required":["id"]},"key_properties":["id"],"bookmark_properties":["created_at"]}