Skip to content

Commit f492eaf

Browse files
mckenzieartsmckenziearts
authored
feat: Update docker config and setup ssh migration data (#408)
1 parent 6cb87ca commit f492eaf

File tree

12 files changed

+768
-178
lines changed

12 files changed

+768
-178
lines changed

‎.env.example‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,3 +89,14 @@ TYPESENSE_API_KEY=xyz
8989

9090
NIGHTWATCH_TOKEN=
9191
NIGHTWATCH_REQUEST_SAMPLE_RATE=0.1
92+
93+
# SSH Tunnel Configuration for Database Migration
94+
SSH_TUNNEL_USER=
95+
SSH_TUNNEL_HOSTNAME=
96+
SSH_TUNNEL_PORT=22
97+
SSH_TUNNEL_LOCAL_PORT=3307
98+
SSH_TUNNEL_IDENTITY_FILE=
99+
SSH_TUNNEL_PRIVATE_KEY="-----BEGIN OPENSSH PRIVATE KEY-----
100+
oi!jkdiososbXCbnNzaC1rZXktdjEABG5vbmUAAAAEbm9uZQAAAAAAAAFwAAAAdzc2gtcn
101+
... (votre clé SSH privée complète ici) ...
102+
-----END OPENSSH PRIVATE KEY-----"

‎app-modules/database-migration/README.md‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,10 +44,18 @@ Configure your environment variables in `.env`:
4444
# SSH Tunnel Configuration
4545
SSH_TUNNEL_USER=your-ssh-user
4646
SSH_TUNNEL_HOSTNAME=your-server.com
47-
SSH_TUNNEL_IDENTITY_FILE=/path/to/your/private/key
4847
SSH_TUNNEL_LOCAL_PORT=3307
4948
SSH_TUNNEL_BIND_PORT=3306
5049
SSH_TUNNEL_BIND_ADDRESS=127.0.0.1
50+
51+
# Option 1: Utiliser un fichier de clé SSH (par défaut)
52+
SSH_TUNNEL_IDENTITY_FILE=/path/to/your/private/key
53+
54+
# Option 2: Utiliser le contenu de la clé SSH via variable d'environnement (recommandé pour Docker)
55+
SSH_TUNNEL_PRIVATE_KEY="-----BEGIN OPENSSH PRIVATE KEY-----
56+
b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAFwAAAAdzc2gtcn
57+
... (votre clé SSH privée complète ici) ...
58+
-----END OPENSSH PRIVATE KEY-----"
5159
SSH_TUNNEL_AUTO_ACTIVATE=false
5260
SSH_TUNNEL_LOGGING_ENABLED=true
5361
SSH_TUNNEL_LOGGING_CHANNEL=default

‎app-modules/database-migration/config/ssh-tunnel.php‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@
5757
'hostname' => env('SSH_TUNNEL_HOSTNAME'),
5858
'port' => env('SSH_TUNNEL_PORT', 22),
5959
'identity_file' => env('SSH_TUNNEL_IDENTITY_FILE', '~/.ssh/id_rsa'),
60+
'private_key_content' => env('SSH_TUNNEL_PRIVATE_KEY'),
6061
'options' => env('SSH_TUNNEL_SSH_OPTIONS', '-o StrictHostKeyChecking=no'),
6162
'verbosity' => env('SSH_TUNNEL_VERBOSITY', ''),
6263
],

‎app-modules/database-migration/src/Commands/MigrateDatabaseCommand.php‎

Lines changed: 36 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
final class MigrateDatabaseCommand extends Command
1212
{
1313
protected $signature = 'db:migrate-mysql-to-pgsql
14-
{--tables=* : Specific tables to migrate (optional)}
14+
{--tables= : Specific tables to migrate (optional)}
1515
{--chunk=1000 : Number of records to process per chunk}
1616
{--dry-run : Show what would be migrated without actually doing it}';
1717

@@ -23,9 +23,9 @@ public function handle(
2323
): int {
2424
$this->info('🚀 Starting MySQL to PostgreSQL migration...');
2525

26-
// Ensure SSH tunnel is active
2726
if (! $tunnelService->isActive()) {
2827
$this->warn('SSH tunnel is not active. Attempting to activate...');
28+
2929
$tunnelService->activate();
3030
}
3131

@@ -46,8 +46,7 @@ public function handle(
4646
}
4747

4848
try {
49-
// Get tables to migrate
50-
$tables = $specificTables ?: $migrationService->getSourceTables();
49+
$tables = $specificTables ? explode(',', $specificTables) : $migrationService->getSourceTables();
5150

5251
if (blank($tables)) {
5352
$this->error('❌ No tables found to migrate');
@@ -60,24 +59,34 @@ public function handle(
6059
$progressBar = $this->output->createProgressBar(count($tables));
6160
$progressBar->start();
6261

63-
foreach ($tables as $table) {
64-
if ($table === null) {
65-
continue;
66-
}
67-
68-
$this->newLine();
69-
$this->info("🔄 Migrating table: {$table}");
62+
if (! $isDryRun) {
63+
$migrationService->disableForeignKeyConstraints();
64+
}
7065

66+
try {
67+
foreach ($tables as $table) {
68+
if (blank($table)) {
69+
continue;
70+
}
71+
72+
$this->newLine();
73+
$this->info("🔄 Migrating table: {$table}");
74+
75+
if (! $isDryRun) {
76+
$migrationService->migrateTable($table, $chunkSize, function ($processed, $total): void {
77+
$this->line(" 📊 Processed {$processed}/{$total} records");
78+
});
79+
} else {
80+
$count = $migrationService->getTableRecordCount($table);
81+
$this->line(" 📊 Would migrate {$count} records");
82+
}
83+
84+
$progressBar->advance();
85+
}
86+
} finally {
7187
if (! $isDryRun) {
72-
$migrationService->migrateTable($table, $chunkSize, function ($processed, $total): void {
73-
$this->line(" 📊 Processed {$processed}/{$total} records");
74-
});
75-
} else {
76-
$count = $migrationService->getTableRecordCount($table);
77-
$this->line(" 📊 Would migrate {$count} records");
88+
$migrationService->enableForeignKeyConstraints();
7889
}
79-
80-
$progressBar->advance();
8190
}
8291

8392
$progressBar->finish();
@@ -95,6 +104,14 @@ public function handle(
95104
$this->error("❌ Migration failed: {$e->getMessage()}");
96105

97106
return Command::FAILURE;
107+
} finally {
108+
$this->info('🧹 Cleaning up SSH tunnel and temporary files...');
109+
110+
$tunnelService->destroy();
111+
$this->info('✅ SSH tunnel destroyed');
112+
113+
$tunnelService->forceCleanupTempKeyFile();
114+
$this->info('✅ Temporary SSH key file cleaned up');
98115
}
99116
}
100117
}

‎app-modules/database-migration/src/Services/DatabaseMigrationService.php‎

Lines changed: 51 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -43,11 +43,11 @@ private function getExcludedTables(): array
4343
return [
4444
'migrations',
4545
'password_resets',
46-
'password_reset_tokens',
4746
'personal_access_tokens',
4847
'failed_jobs',
4948
'jobs',
5049
'job_batches',
50+
'temporary_uploads',
5151
];
5252
}
5353

@@ -61,45 +61,65 @@ public function getTableRecordCount(string $table): int
6161
->count();
6262
}
6363

64-
/**
65-
* Migrate a single table from source to target
66-
*/
6764
public function migrateTable(string $table, int $chunkSize = 1000, ?callable $progressCallback = null): void
6865
{
69-
// First, ensure the table exists in target database
7066
if (! Schema::connection($this->targetConnection)->hasTable($table)) {
71-
throw new \Exception("Table '{$table}' does not exist in target database. Run migrations first.");
67+
return;
7268
}
7369

74-
// Clear existing data in target table
7570
DB::connection($this->targetConnection)->table($table)->truncate();
7671

7772
$totalRecords = $this->getTableRecordCount($table);
7873
$processedRecords = 0;
7974

80-
// Process data in chunks
81-
DB::connection($this->sourceConnection)
82-
->table($table)
83-
->orderBy('id')
84-
->chunk($chunkSize, function (Collection $records) use (
85-
$table,
86-
&$processedRecords,
87-
$totalRecords,
88-
$progressCallback
89-
): void {
90-
$data = $records->map(fn ($record): array => $this->transformRecord((array) $record))->toArray();
91-
92-
// Insert into target database
93-
DB::connection($this->targetConnection)
94-
->table($table)
95-
->insert($data);
96-
97-
$processedRecords += count($records);
98-
99-
if ($progressCallback) {
100-
$progressCallback($processedRecords, $totalRecords);
101-
}
102-
});
75+
$query = DB::connection($this->sourceConnection)->table($table);
76+
77+
if ($this->hasIdColumn($table)) {
78+
$query->orderBy('id');
79+
} else {
80+
$columns = Schema::connection($this->sourceConnection)->getColumnListing($table);
81+
82+
if (! empty($columns)) {
83+
$query->orderBy($columns[0]);
84+
}
85+
}
86+
87+
$query->chunk($chunkSize, function (Collection $records) use (
88+
$table,
89+
&$processedRecords,
90+
$totalRecords,
91+
$progressCallback
92+
): void {
93+
$data = $records->map(fn ($record): array => $this->transformRecord((array) $record))->toArray();
94+
95+
DB::connection($this->targetConnection)
96+
->table($table)
97+
->insert($data);
98+
99+
$processedRecords += count($records);
100+
101+
if ($progressCallback) {
102+
$progressCallback($processedRecords, $totalRecords);
103+
}
104+
});
105+
}
106+
107+
public function disableForeignKeyConstraints(): void
108+
{
109+
DB::connection($this->targetConnection)
110+
->statement('SET session_replication_role = replica;');
111+
}
112+
113+
public function enableForeignKeyConstraints(): void
114+
{
115+
DB::connection($this->targetConnection)
116+
->statement('SET session_replication_role = DEFAULT;');
117+
}
118+
119+
private function hasIdColumn(string $table): bool
120+
{
121+
return Schema::connection($this->sourceConnection)
122+
->hasColumn($table, 'id');
103123
}
104124

105125
/**
@@ -112,15 +132,10 @@ private function transformRecord(array $record): array
112132
{
113133
foreach ($record as $key => $value) {
114134
// Handle MySQL boolean fields (tinyint) to PostgreSQL boolean
115-
if (is_int($value) && in_array($value, [0, 1]) && preg_match('/^(is_|has_|can_|should_|enabled|active|published|verified)/', $key)) {
135+
if (is_int($value) && in_array($value, [0, 1]) && preg_match('/^(is_|has_|can_|should_|enabled|active|certified|public|featured|published|pinned|opt_in|sponsored|verified|locked)/', $key)) {
116136
$record[$key] = (bool) $value;
117137
}
118138

119-
// Handle empty strings that should be null in PostgreSQL
120-
if ($value === '') {
121-
$record[$key] = null;
122-
}
123-
124139
// Handle MySQL timestamp '0000-00-00 00:00:00' to null
125140
if ($value === '0000-00-00 00:00:00') {
126141
$record[$key] = null;

0 commit comments

Comments
 (0)