Skip to content

Commit

Permalink
Use altered table schema for tableCols map
Browse files Browse the repository at this point in the history
Fix: #173

---

Pull Request resolved: #178

Co-authored-by: tserakhau <[email protected]>
commit_hash:4a9b80641f7596bba9ca3c144756d62d8775d287
  • Loading branch information
laskoviymishka authored and robot-piglet committed Jan 24, 2025
1 parent 1c8954e commit 93c257d
Show file tree
Hide file tree
Showing 5 changed files with 64 additions and 1 deletion.
3 changes: 3 additions & 0 deletions .mapping.json
Original file line number Diff line number Diff line change
Expand Up @@ -2536,6 +2536,9 @@
"tests/e2e/mysql2ch/snapshot_empty_table/check_db_test.go":"transfer_manager/go/tests/e2e/mysql2ch/snapshot_empty_table/check_db_test.go",
"tests/e2e/mysql2ch/snapshot_empty_table/dump/ch/dump.sql":"transfer_manager/go/tests/e2e/mysql2ch/snapshot_empty_table/dump/ch/dump.sql",
"tests/e2e/mysql2ch/snapshot_empty_table/dump/mysql/dump.sql":"transfer_manager/go/tests/e2e/mysql2ch/snapshot_empty_table/dump/mysql/dump.sql",
"tests/e2e/mysql2ch/snapshot_nofk/ch.sql":"transfer_manager/go/tests/e2e/mysql2ch/snapshot_nofk/ch.sql",
"tests/e2e/mysql2ch/snapshot_nofk/check_db_test.go":"transfer_manager/go/tests/e2e/mysql2ch/snapshot_nofk/check_db_test.go",
"tests/e2e/mysql2ch/snapshot_nofk/dump/dump.sql":"transfer_manager/go/tests/e2e/mysql2ch/snapshot_nofk/dump/dump.sql",
"tests/e2e/mysql2kafka/debezium/replication/canondata/replication.replication.TestReplication/extracted":"transfer_manager/go/tests/e2e/mysql2kafka/debezium/replication/canondata/replication.replication.TestReplication/extracted",
"tests/e2e/mysql2kafka/debezium/replication/canondata/replication.replication.TestReplication/extracted.0":"transfer_manager/go/tests/e2e/mysql2kafka/debezium/replication/canondata/replication.replication.TestReplication/extracted.0",
"tests/e2e/mysql2kafka/debezium/replication/canondata/replication.replication.TestReplication/extracted.1":"transfer_manager/go/tests/e2e/mysql2kafka/debezium/replication/canondata/replication.replication.TestReplication/extracted.1",
Expand Down
2 changes: 1 addition & 1 deletion pkg/providers/mysql/schema.go
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,7 @@ func LoadSchema(tx queryExecutor, useFakePrimaryKey bool, includeViews bool) (ab
tableSchema[i].FakeKey = true
}
}
tableCols[tID] = makeTableSchema(currSchema, uniq(keys))
tableCols[tID] = tableSchema
}
dbSchema := make(abstract.DBSchema)
for tableID, columns := range enrichExpressions(tx, tableCols) {
Expand Down
1 change: 1 addition & 0 deletions tests/e2e/mysql2ch/snapshot_nofk/ch.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
CREATE DATABASE source;
43 changes: 43 additions & 0 deletions tests/e2e/mysql2ch/snapshot_nofk/check_db_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
package snapshotnofk

import (
"testing"

"github.com/doublecloud/transfer/pkg/abstract"
chrecipe "github.com/doublecloud/transfer/pkg/providers/clickhouse/recipe"
"github.com/doublecloud/transfer/tests/e2e/mysql2ch"
"github.com/doublecloud/transfer/tests/e2e/pg2ch"
"github.com/doublecloud/transfer/tests/helpers"
"github.com/stretchr/testify/require"
)

func TestSnapshot(t *testing.T) {
source := helpers.RecipeMysqlSource()
target := chrecipe.MustTarget(chrecipe.WithInitFile("ch.sql"), chrecipe.WithDatabase("source"))

defer func() {
require.NoError(t, helpers.CheckConnections(
helpers.LabeledPort{Label: "MySQL source", Port: source.Port},
helpers.LabeledPort{Label: "CH target", Port: target.NativePort},
))
}()

t.Run("fake_keys", func(t *testing.T) {
source.UseFakePrimaryKey = true
transfer := helpers.MakeTransfer(helpers.TransferID, source, target, abstract.TransferTypeSnapshotAndIncrement)
_, err := helpers.ActivateErr(transfer)
require.NoError(t, err)
require.NoError(t, helpers.CompareStorages(
t,
source,
target,
helpers.NewCompareStorageParams().WithEqualDataTypes(pg2ch.PG2CHDataTypesComparator).WithPriorityComparators(mysql2ch.MySQLBytesToStringOptionalComparator),
))
})
t.Run("no_fake_keys", func(t *testing.T) {
source.UseFakePrimaryKey = false
transfer := helpers.MakeTransfer(helpers.TransferID, source, target, abstract.TransferTypeSnapshotAndIncrement)
_, err := helpers.ActivateErr(transfer)
require.Error(t, err)
})
}
16 changes: 16 additions & 0 deletions tests/e2e/mysql2ch/snapshot_nofk/dump/dump.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
-- Create table without a primary key
CREATE TABLE no_pk (
id INT NOT NULL,
name VARCHAR(100) NOT NULL,
age INT NOT NULL,
city VARCHAR(100) NOT NULL,
email VARCHAR(100) NOT NULL
);

-- Insert 5 unique rows
INSERT INTO no_pk (id, name, age, city, email) VALUES
(1, 'Alice', 30, 'New York', '[email protected]'),
(2, 'Bob', 25, 'Los Angeles', '[email protected]'),
(3, 'Charlie', 35, 'Chicago', '[email protected]'),
(4, 'Diana', 28, 'San Francisco', '[email protected]'),
(5, 'Eve', 40, 'Miami', '[email protected]');

0 comments on commit 93c257d

Please sign in to comment.