feat(migration-scripts) add sql-blueprinting script

Author: FotisVasilopoulos

sql-blueprinting/.env.example

@@ -0,0 +1,8 @@
+DATABASE_BLUEPRINT_NAME=clean_db
+DATABASE_TARGET_NAME=existing_db
+
+DATABASE_HOST=
+DATABASE_PORT=
+DATABASE_USER=
+DATABASE_PASSWORD=
+DATABASE_NAME=

sql-blueprinting/.gitignore

@@ -0,0 +1,3 @@
+node_modules
+*.env
+package-lock.json

sql-blueprinting/README.md

@@ -0,0 +1,26 @@
+# SQL Blueprinting
+
+Compare two databases and drop the tables that are not common in both.
+We run our strapi project in a new sql db so it creates its clean structure.
+This db can be used as a blueprint since it is created by our strapi current state and doesnt have old entries etc.
+
+## Usage example
+
+- DB1 is a blueprint db that contains only a schema, we will use this db as a structure referance.
+- DB2 is a production db that contains the data and a schema.
+- We want to drop from the DB2 (prod) the tables that does not appear in the structure of DB1
+- After cleaning our prod db according to blueprint we can migrate it to v4
+
+## Description
+
+Since we have to cleanup by order keys, columns and finally the tables, the db sets foreign key checks to 0 and after running back to 1.
+
+## Run
+
+- npm i
+- npm run start
+
+## Important Notes
+
+- Please use this script on clone of your production db.
+- This script drops all columns, collections and tables that does not exist in blueprint database, so use it carefully.

sql-blueprinting/index.js

@@ -0,0 +1,190 @@
+import mysql from "mysql2";
+import chalk from "chalk";
+import { config } from "dotenv";
+
+config();
+
+const db1 = process.env.DATABASE_BLUEPRINT_NAME; //reference database
+const db2 = process.env.DATABASE_TARGET_NAME; // target database
+
+const connection = mysql.createConnection({
+  host: process.env.DATABASE_HOST,
+  port: process.env.DATABASE_PORT,
+  user: process.env.DATABASE_USER,
+  password: process.env.DATABASE_PASSWORD,
+  database: process.env.DATABASE_NAME,
+});
+
+connection.connect((err) => {
+  if (err) throw err;
+
+  console.log("Connected to database!");
+
+  const getTables = (db) => {
+    return new Promise((resolve, reject) => {
+      connection.query(
+        "SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = ?",
+        [db],
+        (err, tables) => {
+          if (err) reject(err);
+          return resolve(tables);
+        }
+      );
+    });
+  };
+
+  const dropTable = (table) => {
+    return new Promise((resolve, reject) => {
+      connection.query(
+        `DROP TABLE IF EXISTS ${db2}.??`,
+        [table],
+        (err, res) => {
+          if (err) reject(err);
+          return resolve(res);
+        }
+      );
+    });
+  };
+
+  const getColumns = (db, table) => {
+    return new Promise((resolve, reject) => {
+      connection.query(
+        "SELECT * FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?",
+        [db, table],
+        (err, columns) => {
+          if (err) reject(err);
+          return resolve(columns);
+        }
+      );
+    });
+  };
+
+  const dropColumn = (table, column) => {
+    console.log("droping column: " + column + " from table" + table);
+    return new Promise((resolve, reject) => {
+      connection.query(
+        `ALTER TABLE ${db2}.?? DROP COLUMN ??`,
+        [table, column],
+        (err, res) => {
+          if (err) reject(err);
+          return resolve(res);
+        }
+      );
+    });
+  };
+
+  const getData = (targetTableName) => {
+    return new Promise((resolve, reject) => {
+      connection.query(
+        `SELECT * FROM ${db2}.?`,
+        [targetTableName],
+        (err, columns) => {
+          if (err) reject(err);
+          return resolve(columns);
+        }
+      );
+    });
+  };
+
+  const setData = (columns, targetTableName) => {
+    return new Promise((resolve, reject) => {
+      columns.forEach((entry) => {
+        connection.query(
+          `INSERT INTO ${db1}.? VALUES (?) `,
+          [targetTableName, entry],
+          (err, results) => {
+            if (err) reject(err);
+            return resolve(results);
+          }
+        );
+      });
+    });
+  };
+
+  const toggleForeignKeyCheck = (state) => {
+    return new Promise((resolve, reject) => {
+      connection.query(`SET FOREIGN_KEY_CHECKS = ${state}`, (err, results) => {
+        if (err) reject(err);
+        return resolve("Foreign Key Check is set to " + state + "!");
+      });
+    });
+  };
+
+  (async () => {
+    try {
+      let foreignKeyCheckState = 0;
+      toggleForeignKeyCheck(foreignKeyCheckState).then((res) =>
+        console.log(chalk.bold.yellowBright(res))
+      );
+      const tableNames_db1 = await getTables(db1);
+      const tableNames_db2 = await getTables(db2);
+
+      await new Promise((resolve) => {
+        tableNames_db2.forEach(async (table_db2, tableIndex) => {
+          let tableExistanceFlag = false;
+          let targetTableName = table_db2.TABLE_NAME;
+
+          tableNames_db1.forEach((table_db1) => {
+            if (targetTableName === table_db1.TABLE_NAME) {
+              tableExistanceFlag = true;
+            }
+          });
+          if (tableExistanceFlag) {
+            const columns_db1 = await getColumns(db1, targetTableName);
+            const columns_db2 = await getColumns(db2, targetTableName);
+            let columnsCountEquality = false;
+            if (columns_db1.length === columns_db2.length) {
+              columnsCountEquality = true;
+            } else {
+              columns_db2.forEach(async (column_db2) => {
+                let columnExistanceFlag = false;
+                let columnNameDB2 = column_db2.COLUMN_NAME;
+
+                columns_db1.forEach((column_db1) => {
+                  if (columnNameDB2 === column_db1.COLUMN_NAME) {
+                    columnExistanceFlag = true;
+                  }
+                });
+                if (!columnExistanceFlag) {
+                  await dropColumn(targetTableName, columnNameDB2);
+                }
+              });
+            }
+            console.log(
+              `The table ${chalk.bold.greenBright(
+                targetTableName
+              )} exists in both databases ${
+                columnsCountEquality
+                  ? "and columns count " +
+                    chalk.bold.greenBright("is equal") +
+                    "."
+                  : "but columns count " +
+                    chalk.bold.redBright("is not equal") +
+                    "."
+              }`
+            );
+          } else {
+            console.log(
+              `The table ${chalk.bold.redBright(
+                targetTableName
+              )} does not exists in both databases`
+            );
+            await dropTable(targetTableName);
+          }
+          if (tableIndex === tableNames_db2.length - 1) {
+            resolve();
+          }
+        });
+      });
+      foreignKeyCheckState = 1;
+      toggleForeignKeyCheck(foreignKeyCheckState).then((res) =>
+        console.log(chalk.bold.yellowBright(res))
+      );
+
+      console.log("Database cleanup is done, closing connection...");
+      connection.end();
+    } catch (err) {
+      console.log(err);
+    }
+  })();
+});

sql-blueprinting/package.json

@@ -0,0 +1,22 @@
+{
+  "name": "sql-blueprinting",
+  "version": "0.1.0",
+  "description": "",
+  "main": "index.js",
+  "author": "FotisVasilopoulos",
+  "scripts": {
+    "start": "node index.js"
+  },
+  "dependencies": {
+    "chalk": "5.2.0",
+    "mysql": "2.18.1",
+    "mysql2": "3.2.4",
+    "ora": "6.2.0",
+    "dotenv": "16.0.0"
+  },
+  "type": "module",
+  "engines": {
+    "npm": ">=6.0.0"
+  },
+  "license": "UNLICENSED"
+}