[DAT-17952] Add support for ARRAY, MAP and Struct datatypes (#181)

* chore: Override the default implementation to ARRAY, MAP and STRUCT complex types as Liquibase core does not know how to handle values between <> in the data type.

* chore: adding tests.

* chore: fix tests

* added expected json data

---------

Co-authored-by: KushnirykOleh <[email protected]>

src/main/java/liquibase/ext/databricks/snapshot/jvm/ColumnSnapshotGeneratorDatabricks.java

@@ -0,0 +1,46 @@
+package liquibase.ext.databricks.snapshot.jvm;
+
+import liquibase.database.Database;
+import liquibase.exception.DatabaseException;
+import liquibase.ext.databricks.database.DatabricksDatabase;
+import liquibase.snapshot.CachedRow;
+import liquibase.snapshot.SnapshotGenerator;
+import liquibase.snapshot.jvm.ColumnSnapshotGenerator;
+import liquibase.structure.DatabaseObject;
+import liquibase.structure.core.Column;
+import liquibase.structure.core.DataType;
+
+public class ColumnSnapshotGeneratorDatabricks extends ColumnSnapshotGenerator {
+
+    @Override
+    public int getPriority(Class<? extends DatabaseObject> objectType, Database database) {
+        if (database instanceof DatabricksDatabase) {
+            return super.getPriority(objectType, database) + PRIORITY_DATABASE;
+        } else {
+            return PRIORITY_NONE;
+        }
+    }
+
+    @Override
+    public Class<? extends SnapshotGenerator>[] replaces() {
+        return new Class[] { ColumnSnapshotGenerator.class };
+    }
+
+    /**
+     * Override the default implementation to ARRAY, MAP and STRUCT complex types as
+     * Liquibase core does not know how to handle values between <> in the data type.
+     */
+    @Override
+    protected DataType readDataType(CachedRow columnMetadataResultSet, Column column, Database database) throws DatabaseException {
+        String dataType = (String) columnMetadataResultSet.get("TYPE_NAME");
+        if (dataType != null && database instanceof DatabricksDatabase
+                && (dataType.toUpperCase().startsWith("ARRAY")
+                || dataType.toUpperCase().startsWith("MAP")
+                || dataType.toUpperCase().startsWith("STRUCT"))) {
+            DataType type = new DataType(dataType);
+            type.setDataTypeId(columnMetadataResultSet.getInt("DATA_TYPE"));
+            return type;
+        }
+        return super.readDataType(columnMetadataResultSet, column, database);
+    }
+}

src/main/resources/META-INF/services/liquibase.datatype.LiquibaseDataType

@@ -1,5 +1,3 @@
-liquibase.ext.databricks.datatype.ArrayIntegerDataTypeDatabricks
-liquibase.ext.databricks.datatype.ArrayStringDataTypeDatabricks
 liquibase.ext.databricks.datatype.BigintDatatypeDatabricks
 liquibase.ext.databricks.datatype.BinaryDataTypeDatabricks
 liquibase.ext.databricks.datatype.BooleanDatatypeDatabricks
@@ -10,4 +8,4 @@ liquibase.ext.databricks.datatype.IntegerDatatypeDatabricks
 liquibase.ext.databricks.datatype.SmallintDatatypeDatabricks
 liquibase.ext.databricks.datatype.StringDatatypeDatabricks
 liquibase.ext.databricks.datatype.TimestampDatatypeDatabricks
-liquibase.ext.databricks.datatype.TinyintDatatypeDatabricks
+liquibase.ext.databricks.datatype.TinyintDatatypeDatabricks

src/main/resources/META-INF/services/liquibase.snapshot.SnapshotGenerator

@@ -4,3 +4,5 @@ liquibase.ext.databricks.snapshot.jvm.ForeignKeySnapshotGeneratorDatabricks
 liquibase.ext.databricks.snapshot.jvm.UniqueConstraintSnapshotGeneratorDatabricks
 liquibase.ext.databricks.snapshot.jvm.IndexSnapshotGeneratorDatabricks
 liquibase.ext.databricks.snapshot.jvm.ViewSnapshotGeneratorDatabricks
+liquibase.ext.databricks.snapshot.jvm.ColumnSnapshotGeneratorDatabricks
+

src/test/resources/liquibase/harness/change/changelogs/databricks/createComplexTypesTable.xml

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xmlns:ext="http://www.liquibase.org/xml/ns/dbchangelog-ext"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-latest.xsd http://www.liquibase.org/xml/ns/dbchangelog-ext http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-ext.xsd">
+
+    <changeSet id="1" author="filipe">
+        <ext:createTable tableName="test_table_complex_types">
+            <column name="my_arrs" type="ARRAY&lt;STRING&gt;"/>
+            <column name="my_arrbi" type="ARRAY&lt;BIGINT&gt;"/>
+            <column name="my_map" type="MAP&lt;STRING, BIGINT&gt;"/>
+            <column name="my_struct" type="STRUCT&lt;FIELD1: STRING NOT NULL, FIELD2: INT&gt;"/>
+        </ext:createTable>
+    </changeSet>
+</databaseChangeLog>

src/test/resources/liquibase/harness/change/expectedSnapshot/databricks/createComplexTypesTable.json

@@ -0,0 +1,47 @@
+{
+  "snapshot": {
+    "objects": {
+      "liquibase.structure.core.Table": [
+        {
+          "table": {
+            "name": "test_table_complex_types"
+          }
+        }
+      ],
+      "liquibase.structure.core.Column": [
+        {
+          "column": {
+            "name": "my_arrs",
+            "type": {
+              "typeName": "ARRAY<STRING>"
+            }
+          }
+        },
+        {
+          "column": {
+            "name": "my_arrbi",
+            "type": {
+              "typeName": "ARRAY<BIGINT>"
+            }
+          }
+        },
+        {
+          "column": {
+            "name": "my_map",
+            "type": {
+              "typeName": "MAP<STRING, BIGINT>"
+            }
+          }
+        },
+        {
+          "column": {
+            "name": "my_struct",
+            "type": {
+              "typeName": "STRUCT<FIELD1: STRING NOT NULL, FIELD2: INT>"
+            }
+          }
+        }
+      ]
+    }
+  }
+}

src/test/resources/liquibase/harness/change/expectedSql/databricks/createComplexTypesTable.sql

@@ -0,0 +1 @@
+CREATE TABLE main.liquibase_harness_test_ds.test_table_complex_types (my_arrs ARRAY<STRING>, my_arrbi ARRAY<BIGINT>, my_map MAP<STRING, BIGINT>, my_struct STRUCT<FIELD1: STRING NOT NULL, FIELD2: INT>) USING delta TBLPROPERTIES('delta.feature.allowColumnDefaults' = 'supported', 'delta.columnMapping.mode' = 'name', 'delta.enableDeletionVectors' = true)