Add yet more base test functionality

pom.xml

@@ -70,6 +70,11 @@
             <artifactId>liquibase-core</artifactId>
             <version>${liquibase.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.liquibase</groupId>
+            <artifactId>liquibase-commercial</artifactId>
+            <version>${liquibase.version}</version>
+        </dependency>
         <dependency>
             <groupId>junit</groupId>
             <artifactId>junit</artifactId>

src/main/java/liquibase/ext/databricks/change/addLookupTable/AddLookupTableChangeDatabricks.java

@@ -0,0 +1,273 @@
+package liquibase.ext.databricks.change.addLookupTable;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import liquibase.ext.databricks.database.DatabricksDatabase;
+import liquibase.Scope;
+import liquibase.change.*;
+import liquibase.database.Database;
+import liquibase.database.core.Db2zDatabase;
+import liquibase.exception.ValidationErrors;
+import liquibase.snapshot.SnapshotGeneratorFactory;
+import liquibase.statement.NotNullConstraint;
+import liquibase.statement.SqlStatement;
+import liquibase.statement.core.CreateTableStatement;
+import liquibase.statement.core.RawSqlStatement;
+import liquibase.structure.core.Column;
+import liquibase.structure.core.ForeignKey;
+import liquibase.structure.core.Table;
+import liquibase.change.core.AddLookupTableChange;
+import liquibase.change.core.DropForeignKeyConstraintChange;
+import liquibase.change.core.DropTableChange;
+import liquibase.change.core.AddNotNullConstraintChange;
+import liquibase.change.core.AddPrimaryKeyChange;
+import liquibase.change.core.AddForeignKeyConstraintChange;
+
+import static liquibase.statement.SqlStatement.EMPTY_SQL_STATEMENT;
+
+/**
+ * Extracts data from an existing column to create a lookup table.
+ * A foreign key is created between the old column and the new lookup table.
+ */
+
+@DatabaseChange(name = "addLookupTable", priority = ChangeMetaData.PRIORITY_DATABASE +500, appliesTo = "column",
+        description = "Creates a lookup table containing values stored in a column and creates a foreign key to the new table.")
+public class AddLookupTableChangeDatabricks extends AddLookupTableChange {
+
+    private String existingTableCatalogName;
+    private String existingTableSchemaName;
+    private String existingTableName;
+    private String existingColumnName;
+
+    private String newTableCatalogName;
+    private String newTableSchemaName;
+    private String newTableName;
+    private String newColumnName;
+    private String newColumnDataType;
+    private String constraintName;
+
+    @Override
+    public ValidationErrors validate(Database database) {
+        ValidationErrors errors = super.validate(database);
+        if (database instanceof Db2zDatabase) {
+            if (this.getNewColumnDataType() == null) {
+                errors.addError("newColumnDataType is required for " + Scope.getCurrentScope().getSingleton(ChangeFactory.class).getChangeMetaData(this).getName() + " on " + database.getShortName());
+            }
+        }
+        return errors;
+    }
+
+    @DatabaseChangeProperty(description = "Name of the database catalog of the existing table")
+    public String getExistingTableCatalogName() {
+        return existingTableCatalogName;
+    }
+
+    public void setExistingTableCatalogName(String existingTableCatalogName) {
+        this.existingTableCatalogName = existingTableCatalogName;
+    }
+
+    @DatabaseChangeProperty(mustEqualExisting ="column.relation.schema", description = "Name of the database schema where the table containing data to extract resides")
+    public String getExistingTableSchemaName() {
+        return existingTableSchemaName;
+    }
+
+    public void setExistingTableSchemaName(String existingTableSchemaName) {
+        this.existingTableSchemaName = existingTableSchemaName;
+    }
+
+    @DatabaseChangeProperty(mustEqualExisting = "column.relation", description = "Name of the table containing the data to extract",
+            exampleValue = "address")
+    public String getExistingTableName() {
+        return existingTableName;
+    }
+
+    public void setExistingTableName(String existingTableName) {
+        this.existingTableName = existingTableName;
+    }
+
+    @DatabaseChangeProperty(mustEqualExisting = "column", description = "Name of the column containing the data to extract",
+            exampleValue = "state")
+    public String getExistingColumnName() {
+        return existingColumnName;
+    }
+
+    public void setExistingColumnName(String existingColumnName) {
+        this.existingColumnName = existingColumnName;
+    }
+
+    @DatabaseChangeProperty(since = "3.0", description = "Name of the database catalog for the lookup table")
+    public String getNewTableCatalogName() {
+        return newTableCatalogName;
+    }
+
+    public void setNewTableCatalogName(String newTableCatalogName) {
+        this.newTableCatalogName = newTableCatalogName;
+    }
+
+    @DatabaseChangeProperty(description = "Name of the database schema for the lookup table")
+    public String getNewTableSchemaName() {
+        return newTableSchemaName;
+    }
+
+    public void setNewTableSchemaName(String newTableSchemaName) {
+        this.newTableSchemaName = newTableSchemaName;
+    }
+
+    @DatabaseChangeProperty(description = "Name of lookup table to create", exampleValue = "state")
+    public String getNewTableName() {
+        return newTableName;
+    }
+
+    public void setNewTableName(String newTableName) {
+        this.newTableName = newTableName;
+    }
+
+    @DatabaseChangeProperty(description = "Name of the column in the new table to create", exampleValue = "abbreviation")
+    public String getNewColumnName() {
+        return newColumnName;
+    }
+
+    public void setNewColumnName(String newColumnName) {
+        this.newColumnName = newColumnName;
+    }
+
+    @DatabaseChangeProperty(description = "Data type of the new table column", exampleValue = "char(2)")
+    public String getNewColumnDataType() {
+        return newColumnDataType;
+    }
+
+    public void setNewColumnDataType(String newColumnDataType) {
+        this.newColumnDataType = newColumnDataType;
+    }
+
+    @DatabaseChangeProperty(description = "Name of the foreign key constraint to create between the existing table and the lookup table",
+            exampleValue = "fk_address_state")
+    public String getConstraintName() {
+        return constraintName;
+    }
+
+    public String getFinalConstraintName() {
+        if (constraintName == null) {
+            return ("FK_" + getExistingTableName() + "_" + getNewTableName()).toUpperCase();
+        } else {
+            return constraintName;
+        }
+    }
+
+    public void setConstraintName(String constraintName) {
+        this.constraintName = constraintName;
+    }
+
+    @Override
+    public boolean supports(Database database) {
+        if (database instanceof DatabricksDatabase) {
+            return true;
+        }
+        return super.supports(database);
+    }
+
+    @Override
+    protected Change[] createInverses() {
+        DropForeignKeyConstraintChange dropFK = new DropForeignKeyConstraintChange();
+        dropFK.setBaseTableSchemaName(getExistingTableSchemaName());
+        dropFK.setBaseTableName(getExistingTableName());
+        dropFK.setConstraintName(getFinalConstraintName());
+
+        DropTableChange dropTable = new DropTableChange();
+        dropTable.setSchemaName(getNewTableSchemaName());
+        dropTable.setTableName(getNewTableName());
+
+        return new Change[]{
+                dropFK,
+                dropTable,
+        };
+    }
+
+    @Override
+    public SqlStatement[] generateStatements(Database database) {
+        List<SqlStatement> statements = new ArrayList<>();
+
+        String newTableCatalogName = getNewTableCatalogName();
+        String newTableSchemaName = getNewTableSchemaName();
+
+        String existingTableCatalogName = getExistingTableCatalogName();
+        String existingTableSchemaName = getExistingTableSchemaName();
+
+        // Step 1: Create table statement CTAS as lookup table
+        SqlStatement[] createTablesSQL = {new RawSqlStatement("CREATE TABLE " + database.escapeTableName(newTableCatalogName, newTableSchemaName, getNewTableName()) + " AS SELECT DISTINCT " + database.escapeObjectName(getExistingColumnName(), Column.class) + " AS " + database.escapeObjectName(getNewColumnName(), Column.class) + " FROM " + database.escapeTableName(existingTableCatalogName, existingTableSchemaName, getExistingTableName()) + " WHERE " + database.escapeObjectName(getExistingColumnName(), Column.class) + " IS NOT NULL")};
+
+        statements.addAll(Arrays.asList(createTablesSQL));
+
+        // Step 2: Add not null constraint to lookup table
+        AddNotNullConstraintChange addNotNullChange = new AddNotNullConstraintChange();
+        addNotNullChange.setSchemaName(newTableSchemaName);
+        addNotNullChange.setTableName(getNewTableName());
+        addNotNullChange.setColumnName(getNewColumnName());
+        addNotNullChange.setColumnDataType(getNewColumnDataType());
+        statements.addAll(Arrays.asList(addNotNullChange.generateStatements(database)));
+
+
+        // Step 3: Add Primary Key Constraint to Lookup table
+        // Add a properly named primary key with just the column name + "_pk"
+
+        String inferred_pk_name = "pk_" + getNewColumnName();
+
+        AddPrimaryKeyChange addPKChange = new AddPrimaryKeyChange();
+        addPKChange.setSchemaName(newTableSchemaName);
+        addPKChange.setTableName(getNewTableName());
+        addPKChange.setColumnNames(getNewColumnName());
+        addPKChange.setConstraintName(inferred_pk_name);
+
+        statements.addAll(Arrays.asList(addPKChange.generateStatements(database)));
+
+
+        // Step 4: Add FK constraint to original table, referencing lookup table
+        AddForeignKeyConstraintChange addFKChange = new AddForeignKeyConstraintChange();
+        addFKChange.setBaseTableSchemaName(existingTableSchemaName);
+        addFKChange.setBaseTableName(getExistingTableName());
+        addFKChange.setBaseColumnNames(getExistingColumnName());
+        addFKChange.setReferencedTableSchemaName(newTableSchemaName);
+        addFKChange.setReferencedTableName(getNewTableName());
+        addFKChange.setReferencedColumnNames(getNewColumnName());
+
+        addFKChange.setConstraintName(getFinalConstraintName());
+        statements.addAll(Arrays.asList(addFKChange.generateStatements(database)));
+
+        return statements.toArray(EMPTY_SQL_STATEMENT);
+    }
+
+    @Override
+    public ChangeStatus checkStatus(Database database) {
+        ChangeStatus result = new ChangeStatus();
+        try {
+            Table newTableExample = new Table(getNewTableCatalogName(), getNewTableSchemaName(), getNewTableName());
+            Column newColumnExample = new Column(Table.class, getNewTableCatalogName(), getNewTableSchemaName(), getNewTableName(), getNewColumnName());
+
+            ForeignKey foreignKeyExample = new ForeignKey(getConstraintName(), getExistingTableCatalogName(), getExistingTableSchemaName(), getExistingTableName());
+            foreignKeyExample.setPrimaryKeyTable(newTableExample);
+            foreignKeyExample.setForeignKeyColumns(Column.listFromNames(getExistingColumnName()));
+            foreignKeyExample.setPrimaryKeyColumns(Column.listFromNames(getNewColumnName()));
+
+            result.assertComplete(SnapshotGeneratorFactory.getInstance().has(newTableExample, database), "New table does not exist");
+            result.assertComplete(SnapshotGeneratorFactory.getInstance().has(newColumnExample, database), "New column does not exist");
+            result.assertComplete(SnapshotGeneratorFactory.getInstance().has(foreignKeyExample, database), "Foreign key does not exist");
+
+            return result;
+
+        } catch (Exception e) {
+            return result.unknown(e);
+        }
+    }
+
+    @Override
+    public String getConfirmationMessage() {
+        return "Lookup table added for "+getExistingTableName()+"."+getExistingColumnName();
+    }
+
+    @Override
+    public String getSerializedObjectNamespace() {
+        return STANDARD_CHANGELOG_NAMESPACE;
+    }
+}

src/main/java/liquibase/ext/databricks/snapshot/jvm/ResultSetConstraintsExtractorDatabricks.java

@@ -74,7 +74,7 @@ private String createSql(String catalog, String schema, String table) {
                 ((AbstractJdbcDatabase) this.database).getJdbcSchemaName(catalogAndSchema), Schema.class);
 
         String sql = "select CONSTRAINT_NAME, CONSTRAINT_TYPE, TABLE_NAME from "
-                + this.catalogName + ".INFORMATION_SCHEMA.TABLE_CONSTRAINTS " + "where TABLE_SCHEMA='" + jdbcSchemaName + "'";
+                + catalog + ".INFORMATION_SCHEMA.TABLE_CONSTRAINTS " + "where TABLE_SCHEMA='" + schema + "'";
           //      + "' and CONSTRAINT_TYPE='UNIQUE'";
         if (table != null) {
             sql += " and TABLE_NAME='" + table + "'";

src/main/java/liquibase/ext/databricks/snapshot/jvm/UniqueConstraintSnapshotGeneratorDatabricks.java

@@ -34,9 +34,10 @@ public Class<? extends SnapshotGenerator>[] replaces() {
         return new Class[] { UniqueConstraintSnapshotGenerator.class };
     }
 
+
     @Override
     protected List<CachedRow> listConstraints(Table table, DatabaseSnapshot snapshot, Schema schema) throws DatabaseException, SQLException {
-        Scope.getCurrentScope().getLog(this.getClass()).info("Constraints not supported by Databricks");
+        //Scope.getCurrentScope().getLog(this.getClass()).info("Constraints not supported by Databricks");
         return new ResultSetConstraintsExtractorDatabricks(snapshot, schema.getCatalogName(), schema.getName(), table.getName()).fastFetch();
     }
 

src/main/java/liquibase/ext/databricks/sqlgenerator/RenameViewGeneratorDatabricks.java

@@ -35,7 +35,7 @@ public ValidationErrors validate(RenameViewStatement renameViewStatement, Databa
     public Sql[] generateSql(RenameViewStatement statement, Database database, SqlGeneratorChain sqlGeneratorChain) {
         String sql;
 
-        sql = "ALTER TABLE " + database.escapeViewName(statement.getCatalogName(), statement.getSchemaName(), statement.getOldViewName()) + " RENAME TO " + database.escapeObjectName(statement.getNewViewName(), View.class);
+        sql = "ALTER VIEW " + database.escapeViewName(statement.getCatalogName(), statement.getSchemaName(), statement.getOldViewName()) + " RENAME TO " + database.escapeObjectName(statement.getNewViewName(), View.class);
 
 
         return new Sql[]{

src/main/resources/META-INF/services/liquibase.change.Change

@@ -1,4 +1,5 @@
 liquibase.ext.databricks.change.createTable.CreateTableChangeDatabricks
 liquibase.ext.databricks.change.optimize.OptimizeChange
 liquibase.ext.databricks.change.analyze.AnalyzeChange
-liquibase.ext.databricks.change.vacuum.VacuumChange
+liquibase.ext.databricks.change.vacuum.VacuumChange
+liquibase.ext.databricks.change.addLookupTable.AddLookupTableChangeDatabricks

src/main/resources/META-INF/services/liquibase.snapshot.SnapshotGenerator

@@ -1,4 +1,4 @@
 liquibase.ext.databricks.snapshot.jvm.SchemaSnapshotGeneratorDatabricks
 liquibase.ext.databricks.snapshot.jvm.SequenceSnapshotGeneratorDatabricks
-liquibase.ext.databricks.snapshot.jvm.UniqueConstraintSnapshotGeneratorDatabricks
 liquibase.ext.databricks.snapshot.jvm.ForeignKeySnapshotGeneratorDatabricks
+liquibase.ext.databricks.snapshot.jvm.UniqueConstraintSnapshotGeneratorDatabricks

src/test/resources/liquibase/harness/change/changelogs/databricks/addLookupTable.xml

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<databaseChangeLog
+        xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
+         http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.8.xsd">
+    <changeSet author="codydavis" id="1">
+        <addLookupTable
+                existingColumnName="email"
+                existingTableName="authors"
+                newColumnDataType="STRING"
+                newColumnName="authors_email"
+                newTableName="authors_data"/>
+    </changeSet>
+</databaseChangeLog>

src/test/resources/liquibase/harness/change/expectedSnapshot/databricks/renameColumn.json

@@ -0,0 +1,17 @@
+ {
+   "snapshot": {
+     "objects": {
+       "liquibase.structure.core.Column": [
+         {
+           "column": {
+             "name": "first_name_renameColumn_test",
+             "nullable": false,
+             "type": {
+               "typeName": "STRING"
+             }
+           }
+         }
+       ]
+     }
+   }
+ }

src/test/resources/liquibase/harness/change/expectedSql/databricks/addCheckConstraint.sql

@@ -1,2 +1,2 @@
 INVALID TEST
-Databricks Supports Check Constraints, but Liquibase OSS does not have the change type to extend
+-- Databricks supports check constraints but this change type is only offered in PRO liquibase.

src/test/resources/liquibase/harness/change/expectedSql/databricks/addDefaultValue.sql

@@ -1,2 +1,3 @@
 INVALID TEST
+-- Databricks supports default values, but does not support them in snapshot data json
 --ALTER TABLE main.liquibase_harness_test_ds.posts ALTER COLUMN title SET DEFAULT 'title_test'

src/test/resources/liquibase/harness/change/expectedSql/databricks/addLookupTable.sql

@@ -0,0 +1,4 @@
+CREATE TABLE main.liquibase_harness_test_ds.authors_data AS SELECT DISTINCT email AS authors_email FROM main.liquibase_harness_test_ds.authors WHERE email IS NOT NULL
+ALTER TABLE main.liquibase_harness_test_ds.authors_data ALTER COLUMN  authors_email SET NOT NULL
+ALTER TABLE main.liquibase_harness_test_ds.authors_data ADD CONSTRAINT pk_authors_email PRIMARY KEY (authors_email)
+ALTER TABLE main.liquibase_harness_test_ds.authors ADD CONSTRAINT FK_AUTHORS_AUTHORS_DATA FOREIGN KEY (email) REFERENCES main.liquibase_harness_test_ds.authors_data (authors_email)

src/test/resources/liquibase/harness/change/expectedSql/databricks/createPackage.sql

@@ -0,0 +1,2 @@
+INVALID TEST
+--Databricks does not have the concept of a package

src/test/resources/liquibase/harness/change/expectedSql/databricks/createPackageBody.sql

@@ -0,0 +1,2 @@
+INVALID TEST
+--Databricks does not have the concept of a package / package body

src/test/resources/liquibase/harness/change/expectedSql/databricks/dropDefaultValue.sql

@@ -1,2 +1,4 @@
-ALTER TABLE main.liquibase_harness_test_ds.posts ALTER COLUMN title SET DEFAULT 'title_test'
-ALTER TABLE main.liquibase_harness_test_ds.posts ALTER COLUMN  title SET DEFAULT NULL
+INVALID TEST
+--Databricks supports default values, but does not contain them in snapshot data
+--ALTER TABLE main.liquibase_harness_test_ds.posts ALTER COLUMN title SET DEFAULT 'title_test'
+--ALTER TABLE main.liquibase_harness_test_ds.posts ALTER COLUMN  title SET DEFAULT NULL