resolve issue #737 fix type conversion issue for bigquery target when doing datacontract import followed by datacontract test after adding server information

LiberationMindset · LiberationMindset · commit 37ffaa874339 · 2025-05-14T16:00:55.000+02:00
diff --git a/datacontract/export/bigquery_converter.py b/datacontract/export/bigquery_converter.py
@@ -98,10 +98,12 @@ def map_type_to_bigquery(field: Field) -> str:
         return "FLOAT64"
     elif field_type.lower() == "boolean":
         return "BOOL"
-    elif field_type.lower() in ["timestamp", "timestamp_tz"]:
+    elif field_type.lower() in "timestamp":
         return "TIMESTAMP"
     elif field_type.lower() == "date":
         return "DATE"
+    elif field_type.lower() ==  "timestamp_tz":
+        return "TIME"
     elif field_type.lower() == "timestamp_ntz":
         return "DATETIME"
     elif field_type.lower() in ["number", "decimal", "numeric"]:
diff --git a/datacontract/imports/bigquery_importer.py b/datacontract/imports/bigquery_importer.py
@@ -127,7 +127,7 @@ def convert_bigquery_schema(
     return data_contract_specification
 
 
-def import_table_fields(table_fields):
+def import_table_fields(table_fields, in_array=False):
     imported_fields = {}
     for field in table_fields:
         field_name = field.get("name")
@@ -136,20 +136,38 @@ def import_table_fields(table_fields):
         imported_fields[field_name].description = field.get("description")
 
         if field.get("type") == "RECORD":
-            imported_fields[field_name].type = "object"
-            imported_fields[field_name].fields = import_table_fields(field.get("fields"))
+            if field.get("mode") == "REPEATED":
+                imported_fields[field_name].type = "array"
+                imported_fields[field_name].items = Field(
+                type="record", fields= import_table_fields(field.get("fields"), in_array=True))
+            else:
+                imported_fields[field_name].type = "object"
+                imported_fields[field_name].fields = import_table_fields(field.get("fields"), in_array=True)
+
         elif field.get("type") == "STRUCT":
             imported_fields[field_name].type = "struct"
-            imported_fields[field_name].fields = import_table_fields(field.get("fields"))
+            imported_fields[field_name].fields = import_table_fields(field.get("fields"), in_array=in_array)
         elif field.get("type") == "RANGE":
             # This is a range of date/datetime/timestamp but multiple values
             # So we map it to an array
             imported_fields[field_name].type = "array"
             imported_fields[field_name].items = Field(
-                type=map_type_from_bigquery(field["rangeElementType"].get("type"))
+                type=map_type_from_bigquery(field["rangeElementType"].get("type"), in_array=True)
             )
-        else:  # primitive type
-            imported_fields[field_name].type = map_type_from_bigquery(field.get("type"))
+        elif field.get("type") == "GEOGRAPHY":
+            imported_fields[field_name].type = map_type_from_bigquery(field.get("type"), in_array=in_array)
+            imported_fields[field_name].config = {"bigqueryType": "GEOGRAPHY"}
+        elif field.get("type") == "JSON":
+            imported_fields[field_name].type = map_type_from_bigquery(field.get("type"), in_array=in_array)
+            imported_fields[field_name].config = {"bigqueryType": "JSON"}
+
+        else:
+            if field.get("type") == "REPEATED": # not a type record meaning type ARRAY<STRING> ARRAY<INTEGER>
+                imported_fields[field_name].type = "array"
+                imported_fields[field_name].items = Field(
+                type= map_type_from_bigquery(field.get("type"), in_array=True))
+            else: # primitive type
+                imported_fields[field_name].type = map_type_from_bigquery(field.get("type"), in_array=in_array)
 
         if field.get("type") == "STRING":
             # in bigquery both string and bytes have maxLength but in the datacontracts
@@ -167,13 +185,13 @@ def import_table_fields(table_fields):
     return imported_fields
 
 
-def map_type_from_bigquery(bigquery_type_str: str):
+def map_type_from_bigquery(bigquery_type_str: str, in_array=False):
     if bigquery_type_str == "STRING":
         return "string"
     elif bigquery_type_str == "BYTES":
         return "bytes"
     elif bigquery_type_str == "INTEGER":
-        return "int"
+        return "bigint" if in_array else "int"
     elif bigquery_type_str == "INT64":
         return "bigint"
     elif bigquery_type_str == "FLOAT":
@@ -187,9 +205,9 @@ def map_type_from_bigquery(bigquery_type_str: str):
     elif bigquery_type_str == "DATE":
         return "date"
     elif bigquery_type_str == "TIME":
-        return "timestamp_ntz"
+        return "timestamp_tz"
     elif bigquery_type_str == "DATETIME":
-        return "timestamp"
+        return "timestamp_ntz"
     elif bigquery_type_str == "NUMERIC":
         return "numeric"
     elif bigquery_type_str == "BIGNUMERIC":
diff --git a/tests/fixtures/bigquery/export/bq_table_schema.json b/tests/fixtures/bigquery/export/bq_table_schema.json
@@ -100,9 +100,9 @@
       },
       {
         "name": "timestamp_tz_field",
-        "type": "TIMESTAMP",
+        "type": "TIME",
         "mode": "NULLABLE",
-        "description": "a simple timestamp_tz field"
+        "description": "a simple time field"
       },
       {
         "name": "timestamp_ntz_field",
@@ -225,7 +225,7 @@
       },
       {
         "name": "int_array_field",
-        "type": "INTEGER",
+        "type": "INT64",
         "mode": "REPEATED",
         "description": "an int array"
       },
diff --git a/tests/fixtures/bigquery/export/datacontract.yaml b/tests/fixtures/bigquery/export/datacontract.yaml
@@ -78,7 +78,7 @@ models:
       timestamp_tz_field:
         type: timestamp_tz
         required: false
-        description: a simple timestamp_tz field
+        description: a simple time field
       timestamp_ntz_field:
         type: timestamp_ntz
         required: false
@@ -165,7 +165,7 @@ models:
         required: false
         description: an int array
         items:
-          type: int
+          type: bigint
       complex_array_field:
         type: array
         required: false
diff --git a/tests/fixtures/bigquery/import/complete_table_schema.json b/tests/fixtures/bigquery/import/complete_table_schema.json
@@ -152,6 +152,27 @@
         "mode": "NULLABLE",
         "name": "JSON_Field",
         "type": "JSON"
+      }, 
+      {
+        "description": "an array of objects that has multiple fields that should carry through",
+        "fields": [
+          {
+            "name": "Id", 
+            "type": "INTEGER",
+            "mode": "NULLABLE", 
+            "description": "an id field"
+          }, 
+          {
+            "name": "Name", 
+            "type": "STRING",
+            "mode": "NULLABLE", 
+            "description": "a name field"
+
+          }
+        ],
+        "mode": "REPEATED", 
+        "name": "Array_of_struct_col", 
+        "type": "RECORD"
       }
     ]
   },
diff --git a/tests/fixtures/bigquery/import/datacontract.yaml b/tests/fixtures/bigquery/import/datacontract.yaml
@@ -51,11 +51,11 @@ models:
         required: false
         description: A Date field
       Time_Field:
-        type: timestamp_ntz
+        type: timestamp_tz
         required: false
         description: A time field
       Datetime_Field:
-        type: timestamp
+        type: timestamp_ntz
         required: false
         description: A Datetime field
       Numeric_Field:
@@ -80,20 +80,40 @@ models:
             required: false
             description: subfield 1 of type string
           subfield_2:
-            type: int
+            type: bigint
             required: false
             description: Subfield 2 of type integer
       Range_field:
         type: array
         required: false
         description: a datetime range
         items:
-          type: timestamp
+          type: timestamp_ntz
       Geography_Field:
         type: object
         required: false
         description: a geography field
+        config: 
+          bigqueryType: GEOGRAPHY
       JSON_Field:
         type: object
         required: false
         description: a json field
+        config: 
+          bigqueryType: JSON
+      Array_of_struct_col:
+        type: array
+        required: false
+        description: an array of objects that has multiple fields that should carry through
+        items: 
+          type: record
+          fields: 
+            Id: 
+              type: bigint
+              required: false
+              description: an id field
+            Name: 
+              type: string
+              required: false
+              description: a name field
+      
