Merge pull request #37 from strapi/feat/mongoSQL

Add support for PG, MySQL, and MariaDB to Mongo migration script

Author: derrickmehaffy

.gitignore

@@ -77,11 +77,8 @@ web_modules/
 .yarn-integrity
 
 # dotenv environment variable files
-.env
-.env.development.local
-.env.test.local
-.env.production.local
-.env.local
+*.env*
+!*.env*.example
 
 # parcel-bundler cache (https://parceljs.org/)
 .cache

.prettierrc.js

@@ -0,0 +1,9 @@
+module.exports = {
+  endOfLine: 'lf',
+  semi: true,
+  singleQuote: true,
+  tabWidth: 2,
+  trailingComma: 'es5',
+  printWidth: 100,
+  arrowParens: 'always',
+};

README.md

@@ -24,7 +24,7 @@ Some scripts may be in various states, *you should pay careful attention to what
 
 ## Scripts
 
-- [Migration from v3 MongoDB to v3 SQL](./v3-mongodb-v3-sql/README.md) - **Currently in Alpha Testing**
+- [Migration from v3 MongoDB to v3 SQL](./v3-mongodb-v3-sql/README.md) - **Currently in Beta Testing**
 - [Migration from v3 SQL to v4 SQL](./v3-sql-v4-sql/README.md) - **Currently in Beta Testing**
 
 ## Callouts

v3-mongodb-v3-sql/.env.mysql.example

@@ -0,0 +1,14 @@
+# MongoDB Settings
+MONGO_USER=root
+MONGO_PASSWORD=strapi
+MONGO_HOST=127.0.0.1
+MONGO_PORT=27017
+MONGO_DB=admin
+
+# SQL Settings
+SQL_CLIENT=mysql
+DATABASE_HOST=127.0.0.1
+DATABASE_PORT=3306
+DATABASE_USER=strapiUser
+DATABASE_PASSWORD=strapiPassword
+DATABASE_DATABASE=strapi

v3-mongodb-v3-sql/.env.pg.example

@@ -0,0 +1,15 @@
+# MongoDB Settings
+MONGO_USER=root
+MONGO_PASSWORD=strapi
+MONGO_HOST=127.0.0.1
+MONGO_PORT=27017
+MONGO_DB=admin
+
+# SQL Settings
+SQL_CLIENT=postgres
+DATABASE_HOST=127.0.0.1
+DATABASE_PORT=5432
+DATABASE_USER=strapiUser
+DATABASE_PASSWORD=strapiPassword
+DATABASE_DATABASE=strapi
+DATABASE_SCHEMA=public

v3-mongodb-v3-sql/README.md

@@ -1,10 +1,9 @@
 # Migration script from Strapi v3 on MongoDB to Strapi v3 on SQL
 
-> For now this works for sqlite only and can then be used to dump into pg or mysql (will automate the other two soon)
-
 ## Preparation steps
 
-1. Preform a backup of your database you wish to migrate and store that backup somewhere secure
+- Preform a backup of your database you wish to migrate and store that backup somewhere secure
+- Create a new database on your choice of database engine (mysql or pg, sqlite will be created automatically)
 
 ## Install
 
@@ -21,11 +20,14 @@ yarn
 
 ## Configuration
 
-WIP
+1. Choose which database you are migrating to (sqlite, mysql, pg)
+2. Copy the corresponding `.env.DBTYPE.example` file to `.env` using something like `cp .env.pg.example .env`
+3. Modify the configuration in the `.env` to match your MongoDB source and your SQL destination
 
 ## Migration
 
 1. Start by following the ["prepare the migration locally"](https://docs.strapi.io/developer-docs/latest/update-migration-guides/migration-guides/v4/data/mongo.html#prepare-the-migration-locally) guide on our documentation to do any needed code changes
 2. Run your SQL Strapi v3 in `develop` mode with an empty DB to generate the DB structure
-3. Turn off / kill the running Strapi v4 server
+3. Turn off / kill the running SQL Strapi v3 server
 4. Run migration script using `yarn start`
+5. Run your SQL Strapi v3 using `yarn develop` mode with the migrated DB to test the migration

v3-mongodb-v3-sql/dialects/mysql.js

@@ -0,0 +1,26 @@
+module.exports = (knex, inspector) => ({
+  async delAllTables() {
+    const tableList = await inspector.tables();
+
+    // clear all tables
+    for (const table of tableList) {
+      await knex(table).del();
+
+      await knex.raw(`
+      ALTER table
+        \`${table}\`
+      AUTO_INCREMENT = 1;
+      `);
+    }
+
+    return tableList;
+  },
+
+  async beforeMigration() {
+    await knex.raw(`SET foreign_key_checks = 0;`);
+  },
+
+  async afterMigration() {
+    await knex.raw(`SET foreign_key_checks = 1;`);
+  },
+});

v3-mongodb-v3-sql/dialects/postgres.js

@@ -0,0 +1,30 @@
+module.exports = (knex, inspector) => ({
+  async delAllTables() {
+    const tableList = await inspector.tables();
+
+    // clear all tables
+    for (const table of tableList) {
+      await knex(table).del();
+    }
+
+    return tableList;
+  },
+
+  async beforeMigration() {
+    // do nothing for postgres
+  },
+
+  async afterMigration() {
+    const tableList = await inspector.tables();
+
+    // restart sequence for tables
+    for (const table of tableList) {
+      let result = await knex.raw('select max(id) from ??', [table]);
+      const max = result.rows[0].max;
+
+      if (max) {
+        await knex.raw(`ALTER SEQUENCE ?? RESTART WITH ??;`, [table + '_id_seq', max + 1]);
+      }
+    }
+  },
+});

v3-mongodb-v3-sql/dialects/sqlite3.js

@@ -1,24 +1,14 @@
-module.exports = (knex) => ({
+module.exports = (knex, inspector) => ({
   async clearSequences(tableList) {
     const hasTable = await knex.schema.hasTable(`sqlite_sequence`);
 
     if (hasTable) {
-      await knex("sqlite_sequence").del().whereIn("name", tableList);
+      await knex('sqlite_sequence').del().whereIn('name', tableList);
     }
   },
 
   async delAllTables() {
-    const res = await knex.raw(`
-    SELECT 
-      name
-    FROM 
-      sqlite_schema
-    WHERE 
-      type ='table' AND 
-      name NOT LIKE 'sqlite_%';
-    `);
-
-    const tableList = res.map((r) => r.name);
+    const tableList = await inspector.tables();
 
     // clear all tables
     for (const table of tableList) {
@@ -27,4 +17,12 @@ module.exports = (knex) => ({
 
     return this.clearSequences(tableList);
   },
+
+  async beforeMigration() {
+    // do nothing because sqlite3 doesn't have foreign key checks
+  },
+
+  async afterMigration() {
+    // do nothing because sqlite3 doesn't have foreign key checks
+  },
 });