Merge pull request #42 from Infostrux-Solutions/add-type-promotion-de…

…cimal

Add type promotion decimal

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "target-db2"
-version = "0.1.3"
+version = "0.1.4"
 description = "`target-db2` is a Singer target for db2, built with the Meltano Singer SDK."
 readme = "README.md"
 authors = ["Haleemur Ali <haleemur@infostrux.com>"]

target_db2/connector.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import json
+import math
 import re
 import typing as t
 from random import choice
@@ -26,6 +27,7 @@
 
 MAX_VARCHAR_SIZE = 10000
 MAX_PK_STRING_SIZE = 1022
+MAX_DECIMAL_PRECISION = 31
 
 sa.dialects.registry.register("ibm_db_sa", "target_db2.ibm_db_sa", "dialect")
 
@@ -278,10 +280,87 @@ def to_sql_type(
                 jsonschema_type["maxLength"] = string_length
         is_obj = _jsonschema_type_check(jsonschema_type, ("object",))
         is_arr = _jsonschema_type_check(jsonschema_type, ("array",))
+        if _jsonschema_type_check(jsonschema_type, ("number",)):
+            multiple_of = jsonschema_type.get("multipleOf")
+            if multiple_of and multiple_of != int(multiple_of):
+                _multiple_of = multiple_of - int(multiple_of)
+                scale = int(math.ceil(math.log10(1 / _multiple_of)))
+                precision = MAX_DECIMAL_PRECISION - scale
+                return sa.types.DECIMAL(precision, scale)
+            return sa.types.DECIMAL()
+
         if is_obj or is_arr:
             return JSONVARCHAR(varchar_size)
         return super(DB2Connector, DB2Connector).to_sql_type(jsonschema_type)
 
+    def merge_sql_types(
+        self,
+        sql_types: t.Sequence[sa.types.TypeEngine],
+    ) -> sa.types.TypeEngine:
+        """Return a compatible SQL type for the selected type list.
+
+        Args:
+            sql_types: List of SQL types.
+
+        Returns:
+            A SQL type that is compatible with the input types.
+
+        Raises:
+            ValueError: If sql_types argument has zero members.
+        """
+        if not sql_types:
+            msg = "Expected at least one member in `sql_types` argument."
+            raise ValueError(msg)
+
+        if len(sql_types) == 1:
+            return sql_types[0]
+
+        # Gathering Type to match variables
+        # sent in _adapt_column_type
+        current_type = sql_types[0]
+        cur_len: int = getattr(current_type, "length", 0)
+
+        # Convert the two types given into a sorted list
+        # containing the best conversion classes
+        sql_types = self._sort_types(sql_types)
+
+        # If greater than two evaluate the first pair then on down the line
+        if len(sql_types) > 2:  # noqa: PLR2004
+            return self.merge_sql_types(
+                [self.merge_sql_types([sql_types[0], sql_types[1]]), *sql_types[2:]],
+            )
+
+        # if all types are DECIMAL, put the one with the biggest scale at the top
+        # this will ensure that it gets picked as the merged type.
+        if all(isinstance(opt, sa.types.DECIMAL) for opt in sql_types):
+            sql_types = sorted(sql_types, reverse=True, key=lambda x: x.scale or 0)  # type: ignore[attr-defined]
+            return sql_types[0]
+        # Get the generic type class
+        for opt in sql_types:
+            # Get the length
+            opt_len: int = getattr(opt, "length", 0)
+            generic_type = type(opt.as_generic())  # type: ignore[attr-defined]
+
+            if isinstance(generic_type, type):
+                if issubclass(
+                    generic_type,
+                    (sa.types.String, sa.types.Unicode),
+                ):
+                    # If length None or 0 then is varchar max ?
+                    if (
+                        (opt_len is None)
+                        or (opt_len == 0)
+                        or (cur_len and (opt_len >= cur_len))
+                    ):
+                        return opt
+                # If best conversion class is equal to current type
+                # return the best conversion class
+                elif str(opt) == str(current_type):
+                    return opt
+
+        msg = f"Unable to merge sql types: {', '.join([str(t) for t in sql_types])}"
+        raise ValueError(msg)
+
     def create_empty_table(  # noqa: PLR0913
         self,
         full_table_name: str,

tests/test_core.py

@@ -6,7 +6,10 @@
 import typing as t
 
 import pytest
+from singer_sdk.helpers._compat import importlib_resources
 from singer_sdk.testing import get_target_test_class
+from singer_sdk.testing.suites import TestSuite
+from singer_sdk.testing.templates import TargetFileTestTemplate
 from sqlalchemy import (
     Column,
     Integer,
@@ -22,8 +25,10 @@
 
 from target_db2.connector import JSONVARCHAR, DB2Connector
 from target_db2.target import TargetDb2
+from tests import testdata
 
 if t.TYPE_CHECKING:
+    from singer_sdk.helpers._compat import Traversable
     from sqlalchemy.engine.base import Engine
 
 
@@ -90,10 +95,40 @@ def test_column_objectvarchar(db2: Engine) -> None:
         conn.execute(DropTable(test_table))  # type: ignore[arg-type]
 
 
-# Run standard built-in target tests from the SDK:
+class TargetFileTestTemplateCustomPath(TargetFileTestTemplate):
+    """Template uses singer messages in testdata folder to generate tests."""
+
+    @property
+    def singer_filepath(self) -> Traversable:
+        """Get path to singer JSONL formatted messages file.
+
+        Files will be sourced from `./target_test_streams/<test name>.singer`.
+
+        Returns:
+            The expected Path to this tests singer file.
+        """
+        return importlib_resources.files(testdata) / f"{self.name}.singer"
+
+
+class TargetSchemaChangeToNumericMultipleOf(TargetFileTestTemplateCustomPath):
+    """Test Target handles array data."""
+
+    name = "schema_change_to_numeric_multipleof"
+
+
+class TargetSchemaHasNumericMultipleOf(TargetFileTestTemplateCustomPath):
+    """Test Target handles array data."""
+
+    name = "schema_has_numeric_multipleof"
+
+
+custom_tests = TestSuite(
+    kind="target",
+    tests=[TargetSchemaChangeToNumericMultipleOf, TargetSchemaHasNumericMultipleOf],
+)
+
 StandardTargetTests = get_target_test_class(
-    target_class=TargetDb2,
-    config=SAMPLE_CONFIG,
+    target_class=TargetDb2, config=SAMPLE_CONFIG, custom_suites=[custom_tests]
 )
 
 

tests/testdata/__init__.py

@@ -0,0 +1 @@
+"""Singer output samples, used for testing."""

tests/testdata/schema_change_to_numeric_multipleof.singer

@@ -0,0 +1,2 @@
+{"type":"SCHEMA","stream":"test_schema_change_numeric_mulipleof","schema":{"properties":{"id":{"type":["integer"]},"val1":{"type":["number","null"]}},"type":"object","required":["id"]},"key_properties":["id"]}
+{"type":"SCHEMA","stream":"test_schema_change_numeric_mulipleof","schema":{"properties":{"id":{"type":["integer"]},"val1":{"type":["number","null"], "multipleOf": 0.01}},"type":"object","required":["id"]},"key_properties":["id"]}

tests/testdata/schema_has_numeric_multipleof.singer

@@ -0,0 +1 @@
+{"type":"SCHEMA","stream":"test_schema_has_numeric_multipleof","schema":{"properties":{"id":{"type":["number"]},"val1":{"type":["number","null"]},"created_at":{"format":"date-time","type":["string","null"]},"amount":{"multipleOf":0.01,"type":["number","null"]}},"type":"object","required":["id"]},"key_properties":["id"],"bookmark_properties":["created_at"]}