map oracle number with scale 0 or none to decimal

Author: anuunchin

dlt/sources/sql_database/schema_types.py

@@ -113,24 +113,23 @@ def sqla_col_to_column_schema(
         # we represent UUID as text by default, see default_table_adapter
         col["data_type"] = "text"
     elif isinstance(sql_t, sqltypes.Numeric):
-        # special handling for oracle NUMBER types
-        if isinstance(sql_t, NUMBER) and sql_t.scale is None or sql_t.scale == 0:
-            col["data_type"] = "bigint"
-            if add_precision and sql_t.precision is not None:
-                col["precision"] = sql_t.precision
-        # all Numeric types that are returned as floats will assume "double" type
-        # and those that are returned as decimals will assume "decimal" type
-        elif sql_t.asdecimal is False:
+        # Oracle NUMBER(p, 0) or NUMBER(p) represents integers but uses decimal arithmetic
+        is_oracle_integer = isinstance(sql_t, NUMBER) and (sql_t.scale is None or sql_t.scale == 0)
+        # "double" for float-returning types, except Oracle integers
+        # "decimal" for all decimal-returning types and Oracle integers
+        if sql_t.asdecimal is False and not is_oracle_integer:
             col["data_type"] = "double"
         else:
             col["data_type"] = "decimal"
             if sql_t.precision is not None:
                 col["precision"] = sql_t.precision
-                # must have a precision for any meaningful scale
-                if sql_t.scale is not None:
-                    col["scale"] = sql_t.scale
-                elif sql_t.decimal_return_scale is not None:
-                    col["scale"] = sql_t.decimal_return_scale
+            # Scale is explicitly 0 for Oracle integers, otherwise from database type
+            if is_oracle_integer:
+                col["scale"] = 0
+            elif sql_t.scale is not None:
+                col["scale"] = sql_t.scale
+            elif sql_t.decimal_return_scale is not None:
+                col["scale"] = sql_t.decimal_return_scale
     elif isinstance(sql_t, sqltypes.SmallInteger):
         col["data_type"] = "bigint"
         if add_precision:

tests/sources/sql_database/test_schema_types.py

@@ -9,6 +9,10 @@
 from dlt.common.data_types import TDataType
 from dlt.sources.sql_database.schema_types import get_table_references, sqla_col_to_column_schema
 
+from dlt.common.libs.pyarrow import get_py_arrow_datatype
+from dlt.common.destination.capabilities import DestinationCapabilitiesContext
+from decimal import Decimal
+
 
 def test_get_table_references() -> None:
     # Test converting foreign keys to reference hints
@@ -128,9 +132,9 @@ def test_get_table_references() -> None:
 @pytest.mark.parametrize(
     "oracle_type,expected_type,expected_precision,expected_scale,test_value",
     [
-        (NUMBER(), "bigint", None, None, 123456789),
-        (NUMBER(precision=17), "bigint", 17, None, 9309935020231023),
-        (NUMBER(precision=17, scale=0), "bigint", 17, None, 9309935020231023),
+        (NUMBER(), "decimal", None, 0, 123456789),
+        (NUMBER(precision=17), "decimal", 17, 0, 9309935020231023),
+        (NUMBER(precision=17, scale=0), "decimal", 17, 0, 9309935020231023),
         (NUMBER(precision=10, scale=2), "decimal", 10, 2, 12345.67),
         (NUMBER(precision=17, scale=2, asdecimal=False), "double", None, None, 12345.67),
     ],
@@ -146,27 +150,37 @@ def test_oracle_number_type_inference(
     """Test Oracle NUMBER type inference to prevent PyArrow conversion errors.
 
     Oracle NUMBER types can represent both integers and decimals based on their scale:
-    - NUMBER with scale=0 or no scale → should be inferred as 'bigint'
-    - NUMBER with scale>0 → should be inferred as 'decimal'
+    - NUMBER with scale=0 or no scale → should be inferred as 'decimal' with scale=0
+    - NUMBER with scale>0 → should be inferred as 'decimal' with appropriate scale
+    - NUMBER with asdecimal=False → should be inferred as 'double'
 
-    Previously, all Oracle NUMBER types were incorrectly inferred as 'double',
-    causing PyArrow conversion failures for large integers like 9309935020231023
-    that cannot be precisely represented as float64.
+    Previously, all Oracle NUMBER types were incorrectly inferred as 'double'.
     """
     sql_col = sa.Column("test_col", oracle_type, nullable=True)
     column_schema = sqla_col_to_column_schema(sql_col, reflection_level="full_with_precision")
 
-    assert column_schema["data_type"] == expected_type
-    if expected_precision is not None:
-        assert column_schema.get("precision") == expected_precision
-    if expected_scale is not None and expected_type == "decimal":
-        assert column_schema.get("scale") == expected_scale
-
-    # the inferred bigint type should work with PyArrow
-    if expected_type == "bigint" and test_value == 9309935020231023:
-        pa_array = pa.array([test_value], type="int64")
+    assert column_schema.get("data_type") == expected_type
+    assert column_schema.get("precision") == expected_precision
+    assert column_schema.get("scale") == expected_scale
+
+    # Use dlt's actual PyArrow type mapping
+    caps = DestinationCapabilitiesContext.generic_capabilities()
+    pa_type = get_py_arrow_datatype(column_schema, caps, tz="UTC")
+
+    if expected_type == "decimal":
+        decimal_value = Decimal(str(test_value))
+        assert pa.types.is_decimal(pa_type)
+        pa_array = pa.array([decimal_value], type=pa_type)
+        assert pa_array[0].as_py() == decimal_value
+
+        # Original bug: float64 cannot precisely represent large integers
+        # This is why mapping Oracle NUMBER(p,0) to 'double' was incorrect and caused 
+        # PyArrow conversion errors. Using 'decimal' preserves exact integer values.
+        if test_value == 9309935020231023:
+            float_val = float(test_value)
+            assert float_val != test_value
+
+    elif expected_type == "double":
+        pa_array = pa.array([test_value], type=pa_type)
+        assert isinstance(pa_array, pa.FloatingPointArray)
         assert pa_array[0].as_py() == test_value
-        with pytest.raises(
-            pa.ArrowInvalid, match="Integer value 9309935020231023 is outside of the range"
-        ):
-            pa.array([test_value], type="float64")