diff --git a/Makefile b/Makefile index 61e035c0e..7f505a42e 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ SOURCE ?= file go_bindata github github_ee bitbucket aws_s3 google_cloud_storage godoc_vfs gitlab DATABASE ?= postgres mysql redshift cassandra spanner cockroachdb yugabytedb clickhouse mongodb sqlserver firebird neo4j pgx pgx5 -DATABASE_TEST ?= $(DATABASE) sqlite sqlite3 sqlcipher +DATABASE_TEST ?= $(DATABASE) sqlite sqlite3 sqlcipher oracle VERSION ?= $(shell git describe --tags 2>/dev/null | cut -c 2-) TEST_FLAGS ?= REPO_OWNER ?= $(shell cd .. && basename "$$(pwd)") diff --git a/README.md b/README.md index c26999424..cedae07ec 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,7 @@ Database drivers run migrations. [Add a new database?](database/driver.go) * [ClickHouse](database/clickhouse) * [Firebird](database/firebird) * [MS SQL Server](database/sqlserver) +* [Oracle](database/oracle) ### Database URLs diff --git a/database/oracle/README.md b/database/oracle/README.md new file mode 100644 index 000000000..4540fe0e6 --- /dev/null +++ b/database/oracle/README.md @@ -0,0 +1,105 @@ +# oracle + +The supported oracle specific options can be configured in the query section of the oracle +URL `oracle://user:password@host:port/ServiceName?query` + +| URL Query | WithInstance Config | Description | +|--------------------------|----------------------|-------------------------------------------------------------------------------------------------------------------------| +| `x-migrations-table` | `MigrationsTable` | Name of the migrations table in UPPER case | +| `x-multi-stmt-enabled` | `MultiStmtEnabled` | If the migration files are in multi-statements style | +| `x-multi-stmt-separator` | `MultiStmtSeparator` | a single line which use as the token to spilt multiple statements in single migration file, triple-dash separator `---` | + +## Write migration files + +There are two ways to write the migration files, + +1. Single statement file in which it contains only one SQL statement or one PL/SQL statement(Default) +2. Multi statements file in which it can have multi statements(can be SQL or PL/SQL or mixed) + +### Single statement file + +Oracle godor driver support process one statement at a time, so it is natural to support single statement per file as +the default. +Check the [single statement migration files](examples/migrations) as an example. + +### Multi statements file + +Although the golang oracle driver [godror](https://github.com/godror/godror) does not natively support executing +multiple +statements in a single query, it's more friendly and handy to support multi statements in a single migration file in +some case, +so the multi statements can be separated with a line separator(default to triple-dash separator ---), for example: + +``` +statement 1 +--- +statement 2 +``` + +Check the [multi statements' migration files](examples/migrations-multistmt) as an example. + +## Supported & tested version + +- 18-xe + +## Build cli + +```bash +$ cd /path/to/repo/dir +$ go build -tags 'oracle' -o bin/migrate github.com/golang-migrate/migrate/v4/cli +``` + +## Run test code + +There are two ways to run the test code: + +- Run the test code locally with an existing Oracle Instance(Recommended) +- Run the test code inside a container just like CI, It will require to start an Oracle container every time, and it's + very time expense. + +### Run the test code locally with an existing Oracle Instance + +1. Start the `Oracle Database Instance` via docker first, so that you can reuse whenever you want to run the test code. + +```bash +$ cat docker-compose.yaml +--- +version: '2' +services: + orclxe: + image: container-registry.oracle.com/database/express:18.4.0-xe + ports: + - 1521:1521 + container_name: orclxe + environment: + ORACLE_PWD: oracle + volumes: + - ${HOME}/data/orclxe:/opt/oracle/oradata # permission chown -R 54321:54321 ${HOME}/data/orclxe + +``` + +2. Go into the sqlplus console + +```bash +$ docker exec -it orclxe bash +# su oracle +$ sqlplus / as sysdba +``` + +3. Create a test DB + +```sql +alter session set container=XEPDB1; +create user orcl identified by orcl; +grant dba to orcl; +grant create session to orcl; +grant connect, resource to orcl; +grant all privileges to orcl; +``` + +4. Run the test code + +```bash +$ cd /path/to/repo/database/oracle/dir +$ ORACLE_DSN=oracle://orcl:orcl@localhost:1521/XEPDB1 go test -tags "oracle" -race -v -covermode atomic ./... -coverprofile .coverage -timeout 20m +``` diff --git a/database/oracle/examples/migrations-multistmt/1085649617_create_users_table.down.sql b/database/oracle/examples/migrations-multistmt/1085649617_create_users_table.down.sql new file mode 100644 index 000000000..2cbf04a75 --- /dev/null +++ b/database/oracle/examples/migrations-multistmt/1085649617_create_users_table.down.sql @@ -0,0 +1,8 @@ +BEGIN +EXECUTE IMMEDIATE 'DROP TABLE USERS_MS'; +EXCEPTION + WHEN OTHERS THEN + IF SQLCODE != -942 THEN + RAISE; + END IF; +END; diff --git a/database/oracle/examples/migrations-multistmt/1085649617_create_users_table.up.sql b/database/oracle/examples/migrations-multistmt/1085649617_create_users_table.up.sql new file mode 100644 index 000000000..ecf7d7a66 --- /dev/null +++ b/database/oracle/examples/migrations-multistmt/1085649617_create_users_table.up.sql @@ -0,0 +1,24 @@ +CREATE TABLE USERS_MS ( + USER_ID integer unique, + NAME varchar(40), + EMAIL varchar(40) +); + +--- + +BEGIN +EXECUTE IMMEDIATE 'DROP TABLE USERS_MS'; +EXCEPTION + WHEN OTHERS THEN + IF SQLCODE != -942 THEN + RAISE; + END IF; +END; + +--- + +CREATE TABLE USERS_MS ( + USER_ID integer unique, + NAME varchar(40), + EMAIL varchar(40) +); diff --git a/database/oracle/examples/migrations-multistmt/1185749658_add_city_to_users.down.sql b/database/oracle/examples/migrations-multistmt/1185749658_add_city_to_users.down.sql new file mode 100644 index 000000000..21a4c6049 --- /dev/null +++ b/database/oracle/examples/migrations-multistmt/1185749658_add_city_to_users.down.sql @@ -0,0 +1,14 @@ +DECLARE + v_column_exists number := 0; +BEGIN + SELECT COUNT(*) + INTO v_column_exists + FROM user_tab_cols + WHERE table_name = 'USERS_MS' + AND column_name = 'CITY'; + + IF( v_column_exists = 1 ) + THEN + EXECUTE IMMEDIATE 'ALTER TABLE USERS_MS DROP COLUMN CITY'; + END IF; +END; \ No newline at end of file diff --git a/database/oracle/examples/migrations-multistmt/1185749658_add_city_to_users.up.sql b/database/oracle/examples/migrations-multistmt/1185749658_add_city_to_users.up.sql new file mode 100644 index 000000000..7df7c4bf3 --- /dev/null +++ b/database/oracle/examples/migrations-multistmt/1185749658_add_city_to_users.up.sql @@ -0,0 +1,5 @@ +ALTER TABLE USERS_MS ADD CITY varchar(100); +--- +ALTER TABLE USERS_MS ADD ALIAS varchar(100); + + diff --git a/database/oracle/examples/migrations-multistmt/1285849751_add_index_on_user_emails.down.sql b/database/oracle/examples/migrations-multistmt/1285849751_add_index_on_user_emails.down.sql new file mode 100644 index 000000000..065860237 --- /dev/null +++ b/database/oracle/examples/migrations-multistmt/1285849751_add_index_on_user_emails.down.sql @@ -0,0 +1,12 @@ +DECLARE + COUNT_INDEXES INTEGER; +BEGIN + SELECT COUNT ( * ) + INTO COUNT_INDEXES_MS + FROM USERS_MS_INDEXES + WHERE INDEX_NAME = 'users_ms_email_index'; + IF COUNT_INDEXES > 0 + THEN + EXECUTE IMMEDIATE 'DROP INDEX users_ms_email_index'; + END IF; +END; diff --git a/database/oracle/examples/migrations-multistmt/1285849751_add_index_on_user_emails.up.sql b/database/oracle/examples/migrations-multistmt/1285849751_add_index_on_user_emails.up.sql new file mode 100644 index 000000000..1177ce446 --- /dev/null +++ b/database/oracle/examples/migrations-multistmt/1285849751_add_index_on_user_emails.up.sql @@ -0,0 +1 @@ +CREATE UNIQUE INDEX users_ms_email_index ON users_ms (email); diff --git a/database/oracle/examples/migrations-multistmt/1385949617_create_books_table.down.sql b/database/oracle/examples/migrations-multistmt/1385949617_create_books_table.down.sql new file mode 100644 index 000000000..cad1f5209 --- /dev/null +++ b/database/oracle/examples/migrations-multistmt/1385949617_create_books_table.down.sql @@ -0,0 +1,8 @@ +BEGIN +EXECUTE IMMEDIATE 'DROP TABLE BOOKS_MS'; +EXCEPTION + WHEN OTHERS THEN + IF SQLCODE != -942 THEN + RAISE; + END IF; +END; diff --git a/database/oracle/examples/migrations-multistmt/1385949617_create_books_table.up.sql b/database/oracle/examples/migrations-multistmt/1385949617_create_books_table.up.sql new file mode 100644 index 000000000..02d879e19 --- /dev/null +++ b/database/oracle/examples/migrations-multistmt/1385949617_create_books_table.up.sql @@ -0,0 +1,5 @@ +CREATE TABLE BOOKS_MS ( + USER_ID integer, + NAME varchar(40), + AUTHOR varchar(40) +); diff --git a/database/oracle/examples/migrations-multistmt/1485949617_create_movies_table.down.sql b/database/oracle/examples/migrations-multistmt/1485949617_create_movies_table.down.sql new file mode 100644 index 000000000..7c806cd83 --- /dev/null +++ b/database/oracle/examples/migrations-multistmt/1485949617_create_movies_table.down.sql @@ -0,0 +1,8 @@ +BEGIN +EXECUTE IMMEDIATE 'DROP TABLE MOVIES_MS'; +EXCEPTION + WHEN OTHERS THEN + IF SQLCODE != -942 THEN + RAISE; + END IF; +END; diff --git a/database/oracle/examples/migrations-multistmt/1485949617_create_movies_table.up.sql b/database/oracle/examples/migrations-multistmt/1485949617_create_movies_table.up.sql new file mode 100644 index 000000000..422550e24 --- /dev/null +++ b/database/oracle/examples/migrations-multistmt/1485949617_create_movies_table.up.sql @@ -0,0 +1,5 @@ +CREATE TABLE MOVIES_MS ( + USER_ID integer, + NAME varchar(40), + DIRECTOR varchar(40) +); diff --git a/database/oracle/examples/migrations-multistmt/1585849751_just_a_comment.up.sql b/database/oracle/examples/migrations-multistmt/1585849751_just_a_comment.up.sql new file mode 100644 index 000000000..9b6b57a61 --- /dev/null +++ b/database/oracle/examples/migrations-multistmt/1585849751_just_a_comment.up.sql @@ -0,0 +1 @@ +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/database/oracle/examples/migrations-multistmt/1685849751_another_comment.up.sql b/database/oracle/examples/migrations-multistmt/1685849751_another_comment.up.sql new file mode 100644 index 000000000..9b6b57a61 --- /dev/null +++ b/database/oracle/examples/migrations-multistmt/1685849751_another_comment.up.sql @@ -0,0 +1 @@ +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/database/oracle/examples/migrations-multistmt/1785849751_another_comment.up.sql b/database/oracle/examples/migrations-multistmt/1785849751_another_comment.up.sql new file mode 100644 index 000000000..9b6b57a61 --- /dev/null +++ b/database/oracle/examples/migrations-multistmt/1785849751_another_comment.up.sql @@ -0,0 +1 @@ +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/database/oracle/examples/migrations-multistmt/1885849751_another_comment.up.sql b/database/oracle/examples/migrations-multistmt/1885849751_another_comment.up.sql new file mode 100644 index 000000000..9b6b57a61 --- /dev/null +++ b/database/oracle/examples/migrations-multistmt/1885849751_another_comment.up.sql @@ -0,0 +1 @@ +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/database/oracle/examples/migrations/1085649617_create_users_table.down.sql b/database/oracle/examples/migrations/1085649617_create_users_table.down.sql new file mode 100644 index 000000000..11bb047ce --- /dev/null +++ b/database/oracle/examples/migrations/1085649617_create_users_table.down.sql @@ -0,0 +1,8 @@ +BEGIN +EXECUTE IMMEDIATE 'DROP TABLE USERS'; +EXCEPTION + WHEN OTHERS THEN + IF SQLCODE != -942 THEN + RAISE; + END IF; +END; diff --git a/database/oracle/examples/migrations/1085649617_create_users_table.up.sql b/database/oracle/examples/migrations/1085649617_create_users_table.up.sql new file mode 100644 index 000000000..76ed60b1e --- /dev/null +++ b/database/oracle/examples/migrations/1085649617_create_users_table.up.sql @@ -0,0 +1,5 @@ +CREATE TABLE USERS ( + USER_ID integer unique, + NAME varchar(40), + EMAIL varchar(40) +) \ No newline at end of file diff --git a/database/oracle/examples/migrations/1185749658_add_city_to_users.down.sql b/database/oracle/examples/migrations/1185749658_add_city_to_users.down.sql new file mode 100644 index 000000000..6808a7d66 --- /dev/null +++ b/database/oracle/examples/migrations/1185749658_add_city_to_users.down.sql @@ -0,0 +1,14 @@ +DECLARE + v_column_exists number := 0; +BEGIN + SELECT COUNT(*) + INTO v_column_exists + FROM user_tab_cols + WHERE table_name = 'USERS' + AND column_name = 'CITY'; + + IF( v_column_exists = 1 ) + THEN + EXECUTE IMMEDIATE 'ALTER TABLE users DROP COLUMN CITY'; + END IF; +END; \ No newline at end of file diff --git a/database/oracle/examples/migrations/1185749658_add_city_to_users.up.sql b/database/oracle/examples/migrations/1185749658_add_city_to_users.up.sql new file mode 100644 index 000000000..92a80e9a9 --- /dev/null +++ b/database/oracle/examples/migrations/1185749658_add_city_to_users.up.sql @@ -0,0 +1,3 @@ +ALTER TABLE USERS ADD CITY varchar(100) + + diff --git a/database/oracle/examples/migrations/1285849751_add_index_on_user_emails.down.sql b/database/oracle/examples/migrations/1285849751_add_index_on_user_emails.down.sql new file mode 100644 index 000000000..87e62cb6d --- /dev/null +++ b/database/oracle/examples/migrations/1285849751_add_index_on_user_emails.down.sql @@ -0,0 +1,12 @@ +DECLARE + COUNT_INDEXES INTEGER; +BEGIN + SELECT COUNT ( * ) + INTO COUNT_INDEXES + FROM USER_INDEXES + WHERE INDEX_NAME = 'users_email_index'; + IF COUNT_INDEXES > 0 + THEN + EXECUTE IMMEDIATE 'DROP INDEX users_email_index'; + END IF; +END; diff --git a/database/oracle/examples/migrations/1285849751_add_index_on_user_emails.up.sql b/database/oracle/examples/migrations/1285849751_add_index_on_user_emails.up.sql new file mode 100644 index 000000000..2ce7ea466 --- /dev/null +++ b/database/oracle/examples/migrations/1285849751_add_index_on_user_emails.up.sql @@ -0,0 +1 @@ +CREATE UNIQUE INDEX users_email_index ON users (email) diff --git a/database/oracle/examples/migrations/1385949617_create_books_table.down.sql b/database/oracle/examples/migrations/1385949617_create_books_table.down.sql new file mode 100644 index 000000000..e7c1bad26 --- /dev/null +++ b/database/oracle/examples/migrations/1385949617_create_books_table.down.sql @@ -0,0 +1,8 @@ +BEGIN +EXECUTE IMMEDIATE 'DROP TABLE BOOKS'; +EXCEPTION + WHEN OTHERS THEN + IF SQLCODE != -942 THEN + RAISE; + END IF; +END; diff --git a/database/oracle/examples/migrations/1385949617_create_books_table.up.sql b/database/oracle/examples/migrations/1385949617_create_books_table.up.sql new file mode 100644 index 000000000..774c894d1 --- /dev/null +++ b/database/oracle/examples/migrations/1385949617_create_books_table.up.sql @@ -0,0 +1,5 @@ +CREATE TABLE BOOKS ( + USER_ID integer, + NAME varchar(40), + AUTHOR varchar(40) +) diff --git a/database/oracle/examples/migrations/1485949617_create_movies_table.down.sql b/database/oracle/examples/migrations/1485949617_create_movies_table.down.sql new file mode 100644 index 000000000..d948228e4 --- /dev/null +++ b/database/oracle/examples/migrations/1485949617_create_movies_table.down.sql @@ -0,0 +1,8 @@ +BEGIN +EXECUTE IMMEDIATE 'DROP TABLE MOVIES'; +EXCEPTION + WHEN OTHERS THEN + IF SQLCODE != -942 THEN + RAISE; + END IF; +END; diff --git a/database/oracle/examples/migrations/1485949617_create_movies_table.up.sql b/database/oracle/examples/migrations/1485949617_create_movies_table.up.sql new file mode 100644 index 000000000..0ba07f277 --- /dev/null +++ b/database/oracle/examples/migrations/1485949617_create_movies_table.up.sql @@ -0,0 +1,5 @@ +CREATE TABLE MOVIES ( + USER_ID integer, + NAME varchar(40), + DIRECTOR varchar(40) +) diff --git a/database/oracle/examples/migrations/1585849751_just_a_comment.up.sql b/database/oracle/examples/migrations/1585849751_just_a_comment.up.sql new file mode 100644 index 000000000..9b6b57a61 --- /dev/null +++ b/database/oracle/examples/migrations/1585849751_just_a_comment.up.sql @@ -0,0 +1 @@ +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/database/oracle/examples/migrations/1685849751_another_comment.up.sql b/database/oracle/examples/migrations/1685849751_another_comment.up.sql new file mode 100644 index 000000000..9b6b57a61 --- /dev/null +++ b/database/oracle/examples/migrations/1685849751_another_comment.up.sql @@ -0,0 +1 @@ +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/database/oracle/examples/migrations/1785849751_another_comment.up.sql b/database/oracle/examples/migrations/1785849751_another_comment.up.sql new file mode 100644 index 000000000..9b6b57a61 --- /dev/null +++ b/database/oracle/examples/migrations/1785849751_another_comment.up.sql @@ -0,0 +1 @@ +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/database/oracle/examples/migrations/1885849751_another_comment.up.sql b/database/oracle/examples/migrations/1885849751_another_comment.up.sql new file mode 100644 index 000000000..9b6b57a61 --- /dev/null +++ b/database/oracle/examples/migrations/1885849751_another_comment.up.sql @@ -0,0 +1 @@ +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/database/oracle/oracle.go b/database/oracle/oracle.go new file mode 100644 index 000000000..65dd3ab3e --- /dev/null +++ b/database/oracle/oracle.go @@ -0,0 +1,471 @@ +package oracle + +import ( + "bufio" + "bytes" + "context" + "database/sql" + "fmt" + "io" + nurl "net/url" + "strconv" + "strings" + + "github.com/golang-migrate/migrate/v4" + "github.com/golang-migrate/migrate/v4/database" + "github.com/hashicorp/go-multierror" + _ "github.com/sijms/go-ora/v2" +) + +func init() { + db := Oracle{} + database.Register("oracle", &db) +} + +const ( + migrationsTableQueryKey = "x-migrations-table" + multiStmtEnableQueryKey = "x-multi-stmt-enabled" + multiStmtSeparatorQueryKey = "x-multi-stmt-separator" +) + +var ( + DefaultMigrationsTable = "SCHEMA_MIGRATIONS" + DefaultMultiStmtEnabled = false + DefaultMultiStmtSeparator = "---" +) + +var ( + ErrNilConfig = fmt.Errorf("no config") + ErrNoDatabaseName = fmt.Errorf("no database name") +) + +type Config struct { + MigrationsTable string + MultiStmtEnabled bool + MultiStmtSeparator string + + databaseName string +} + +type Oracle struct { + // Locking and unlocking need to use the same connection + conn *sql.Conn + db *sql.DB + isLocked bool + + // Open and WithInstance need to guarantee that config is never nil + config *Config +} + +func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) { + if config == nil { + return nil, ErrNilConfig + } + + if err := instance.Ping(); err != nil { + return nil, err + } + + query := `SELECT SYS_CONTEXT('USERENV','DB_NAME') FROM DUAL` + var dbName string + if err := instance.QueryRow(query).Scan(&dbName); err != nil { + return nil, &database.Error{OrigErr: err, Query: []byte(query)} + } + + if dbName == "" { + return nil, ErrNoDatabaseName + } + + config.databaseName = dbName + + if config.MigrationsTable == "" { + config.MigrationsTable = DefaultMigrationsTable + } + + if config.MultiStmtSeparator == "" { + config.MultiStmtSeparator = DefaultMultiStmtSeparator + } + + conn, err := instance.Conn(context.Background()) + + if err != nil { + return nil, err + } + + ora := &Oracle{ + conn: conn, + db: instance, + config: config, + } + + if err := ora.ensureVersionTable(); err != nil { + return nil, err + } + + return ora, nil +} + +func (ora *Oracle) Open(url string) (database.Driver, error) { + purl, err := nurl.Parse(url) + if err != nil { + return nil, err + } + db, err := sql.Open("oracle", migrate.FilterCustomQuery(purl).String()) + if err != nil { + return nil, err + } + + migrationsTable := DefaultMigrationsTable + if s := purl.Query().Get(migrationsTableQueryKey); len(s) > 0 { + migrationsTable = strings.ToUpper(s) + } + multiStmtEnabled := DefaultMultiStmtEnabled + if s := purl.Query().Get(multiStmtEnableQueryKey); len(s) > 0 { + multiStmtEnabled, err = strconv.ParseBool(s) + if err != nil { + return nil, fmt.Errorf("unable to parse option %s: %w", multiStmtEnableQueryKey, err) + } + } + multiStmtSeparator := DefaultMultiStmtSeparator + if s := purl.Query().Get(multiStmtSeparatorQueryKey); len(s) > 0 { + multiStmtSeparator = s + } + + oraInst, err := WithInstance(db, &Config{ + databaseName: purl.Path, + MigrationsTable: migrationsTable, + MultiStmtEnabled: multiStmtEnabled, + MultiStmtSeparator: multiStmtSeparator, + }) + + if err != nil { + return nil, err + } + + return oraInst, nil +} + +func (ora *Oracle) Close() error { + connErr := ora.conn.Close() + dbErr := ora.db.Close() + if connErr != nil || dbErr != nil { + return fmt.Errorf("conn: %v, db: %v", connErr, dbErr) + } + return nil +} + +func (ora *Oracle) Lock() error { + if ora.isLocked { + return database.ErrLocked + } + + // https://docs.oracle.com/cd/B28359_01/appdev.111/b28419/d_lock.htm#ARPLS021 + query := ` +declare + v_lockhandle varchar2(200); + v_result number; +begin + + dbms_lock.allocate_unique('control_lock', v_lockhandle); + + v_result := dbms_lock.request(v_lockhandle, dbms_lock.x_mode); + + if v_result <> 0 then + dbms_output.put_line( + case + when v_result=1 then 'Timeout' + when v_result=2 then 'Deadlock' + when v_result=3 then 'Parameter Error' + when v_result=4 then 'Already owned' + when v_result=5 then 'Illegal Lock Handle' + end); + end if; + +end; +` + if _, err := ora.conn.ExecContext(context.Background(), query); err != nil { + return &database.Error{OrigErr: err, Err: "try lock failed", Query: []byte(query)} + } + + ora.isLocked = true + return nil +} + +func (ora *Oracle) Unlock() error { + if !ora.isLocked { + return nil + } + + query := ` +declare + v_lockhandle varchar2(200); + v_result number; +begin + + dbms_lock.allocate_unique('control_lock', v_lockhandle); + + v_result := dbms_lock.release(v_lockhandle); + + if v_result <> 0 then + dbms_output.put_line( + case + when v_result=1 then 'Timeout' + when v_result=2 then 'Deadlock' + when v_result=3 then 'Parameter Error' + when v_result=4 then 'Already owned' + when v_result=5 then 'Illegal Lock Handle' + end); + end if; + +end; +` + if _, err := ora.conn.ExecContext(context.Background(), query); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + ora.isLocked = false + return nil +} + +func (ora *Oracle) Run(migration io.Reader) error { + var queries []string + if !ora.config.MultiStmtEnabled { + // If multi-statements is not enabled explicitly, + // i.e, there is no multi-statement enabled(neither normal multi-statements nor multi-PL/SQL-statements), + // consider the whole migration as a blob. + query, err := removeComments(migration) + if err != nil { + return err + } + if query == "" { + // empty query, do nothing + return nil + } + queries = append(queries, query) + } else { + // If multi-statements is enabled explicitly, + // there could be multi-statements or multi-PL/SQL-statements in a single migration. + var err error + queries, err = parseMultiStatements(migration, ora.config.MultiStmtSeparator) + if err != nil { + return err + } + } + + for _, query := range queries { + if _, err := ora.conn.ExecContext(context.Background(), query); err != nil { + return database.Error{OrigErr: err, Err: "migration failed", Query: []byte(query)} + } + } + + return nil +} + +func (ora *Oracle) SetVersion(version int, dirty bool) error { + tx, err := ora.conn.BeginTx(context.Background(), &sql.TxOptions{}) + if err != nil { + return &database.Error{OrigErr: err, Err: "transaction start failed"} + } + + query := "TRUNCATE TABLE " + ora.config.MigrationsTable + if _, err := tx.Exec(query); err != nil { + if errRollback := tx.Rollback(); errRollback != nil { + err = multierror.Append(err, errRollback) + } + return &database.Error{OrigErr: err, Query: []byte(query)} + } + + if version >= 0 || (version == database.NilVersion && dirty) { + query = `INSERT INTO ` + ora.config.MigrationsTable + ` (VERSION, DIRTY) VALUES (:1, :2)` + if _, err := tx.Exec(query, version, b2i(dirty)); err != nil { + if errRollback := tx.Rollback(); errRollback != nil { + err = multierror.Append(err, errRollback) + } + return &database.Error{OrigErr: err, Query: []byte(query)} + } + } + + if err := tx.Commit(); err != nil { + return &database.Error{OrigErr: err, Err: "transaction commit failed"} + } + + return nil +} + +func (ora *Oracle) Version() (version int, dirty bool, err error) { + query := "SELECT VERSION, DIRTY FROM " + ora.config.MigrationsTable + " WHERE ROWNUM = 1 ORDER BY VERSION desc" + err = ora.conn.QueryRowContext(context.Background(), query).Scan(&version, &dirty) + switch { + case err == sql.ErrNoRows: + return database.NilVersion, false, nil + case err != nil: + return 0, false, &database.Error{OrigErr: err, Query: []byte(query)} + default: + return version, dirty, nil + } +} + +func (ora *Oracle) Drop() (err error) { + // select all tables in current schema + query := `SELECT TABLE_NAME FROM USER_TABLES` + tables, err := ora.conn.QueryContext(context.Background(), query) + if err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + defer func() { + if errClose := tables.Close(); errClose != nil { + err = multierror.Append(err, errClose) + } + }() + + // delete one table after another + tableNames := make([]string, 0) + for tables.Next() { + var tableName string + if err := tables.Scan(&tableName); err != nil { + return err + } + if len(tableName) > 0 { + tableNames = append(tableNames, tableName) + } + } + + query = ` +BEGIN + EXECUTE IMMEDIATE 'DROP TABLE %s'; +EXCEPTION + WHEN OTHERS THEN + IF SQLCODE != -942 THEN + RAISE; + END IF; +END; +` + if len(tableNames) > 0 { + // delete one by one ... + for _, t := range tableNames { + if _, err := ora.conn.ExecContext(context.Background(), fmt.Sprintf(query, t)); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + } + } + + return nil +} + +// ensureVersionTable checks if versions table exists and, if not, creates it. +// Note that this function locks the database, which deviates from the usual +// convention of "caller locks" in the Postgres type. +func (ora *Oracle) ensureVersionTable() (err error) { + if err = ora.Lock(); err != nil { + return err + } + + defer func() { + if e := ora.Unlock(); e != nil { + if err == nil { + err = e + } else { + err = multierror.Append(err, e) + } + } + }() + + query := ` +declare +v_sql LONG; +begin + +v_sql:='create table %s + ( + VERSION NUMBER(20) NOT NULL PRIMARY KEY, + DIRTY NUMBER(1) NOT NULL + )'; +execute immediate v_sql; + +EXCEPTION + WHEN OTHERS THEN + IF SQLCODE = -955 THEN + NULL; -- suppresses ORA-00955 exception + ELSE + RAISE; + END IF; +END; +` + if _, err = ora.conn.ExecContext(context.Background(), fmt.Sprintf(query, ora.config.MigrationsTable)); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + + return nil +} + +func b2i(b bool) int { + if b { + return 1 + } + return 0 +} + +func removeComments(rd io.Reader) (string, error) { + buf := bytes.Buffer{} + scanner := bufio.NewScanner(rd) + for scanner.Scan() { + line := scanner.Text() + // ignore comment + if strings.HasPrefix(line, "--") { + continue + } + if _, err := buf.WriteString(line + "\n"); err != nil { + return "", err + } + } + return buf.String(), nil +} + +func parseMultiStatements(rd io.Reader, plsqlStmtSeparator string) ([]string, error) { + var results []string + var buf bytes.Buffer + scanner := bufio.NewScanner(rd) + for scanner.Scan() { + line := scanner.Text() + if line == plsqlStmtSeparator { + results = append(results, buf.String()) + buf.Reset() + continue + } + if line == "" || strings.HasPrefix(line, "--") { + continue // ignore empty and comment line + } + if _, err := buf.WriteString(line + "\n"); err != nil { + return nil, err + } + } + if buf.Len() > 0 { + // append the final result if it's not empty + results = append(results, buf.String()) + } + + queries := make([]string, 0, len(results)) + for _, result := range results { + result = strings.TrimSpace(result) + result = strings.TrimPrefix(result, "\n") + result = strings.TrimSuffix(result, "\n") + if !isPLSQLTail(result) { + // remove the ";" from the tail if it's not PL/SQL stmt + result = strings.TrimSuffix(result, ";") + } + if result == "" { + continue // skip empty query + } + queries = append(queries, result) + } + return queries, nil +} + +func isPLSQLTail(s string) bool { + plsqlTail := "end;" + if len(s) < len(plsqlTail) { + return false + } + pos := len(s) - len(plsqlTail) + tail := s[pos:] + return strings.EqualFold(tail, plsqlTail) +} diff --git a/database/oracle/oracle_test.go b/database/oracle/oracle_test.go new file mode 100644 index 000000000..f86895259 --- /dev/null +++ b/database/oracle/oracle_test.go @@ -0,0 +1,283 @@ +package oracle + +import ( + "bytes" + "context" + "database/sql" + sqldriver "database/sql/driver" + "fmt" + "io" + "log" + "os" + "path/filepath" + "testing" + "time" + + "github.com/dhui/dktest" + "github.com/docker/docker/api/types/mount" + "github.com/docker/go-connections/nat" + "github.com/golang-migrate/migrate/v4" + dt "github.com/golang-migrate/migrate/v4/database/testing" + "github.com/golang-migrate/migrate/v4/dktesting" + _ "github.com/golang-migrate/migrate/v4/source/file" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +const defaultPort = 1521 + +var ( + specs = []dktesting.ContainerSpec{ + {ImageName: "container-registry.oracle.com/database/express:18.4.0-xe", Options: orclOptions()}, + } +) + +func orclOptions() dktest.Options { + cwd, _ := os.Getwd() + mounts := []mount.Mount{ + { + Type: mount.TypeBind, + Source: filepath.Join(cwd, "testdata/user.sql"), + Target: "/opt/oracle/scripts/setup/user.sql", + }, + } + return dktest.Options{ + PortRequired: true, + PortBindings: map[nat.Port][]nat.PortBinding{ + nat.Port(fmt.Sprintf("%d/tcp", defaultPort)): { + nat.PortBinding{ + HostIP: "0.0.0.0", + HostPort: "0/tcp", + }, + }, + }, + ReadyFunc: isReady, + PullTimeout: time.Minute * 10, + Timeout: time.Minute * 30, + Mounts: mounts, + } +} + +func orclConnectionString(host, port string) string { + return fmt.Sprintf("oracle://orcl:orcl@%s:%s/XEPDB1", host, port) +} + +func isReady(ctx context.Context, c dktest.ContainerInfo) bool { + ip, port, err := c.Port(defaultPort) + if err != nil { + log.Println("get port error", err) + return false + } + + db, err := sql.Open("oracle", orclConnectionString(ip, port)) + if err != nil { + return false + } + defer func() { + if err := db.Close(); err != nil { + log.Println("close error:", err) + } + }() + if err = db.PingContext(ctx); err != nil { + switch err { + case sqldriver.ErrBadConn, io.EOF: + log.Println(err) + return false + default: + log.Println(err) + } + log.Println(ip, port) + return false + } + + return true +} + +type oracleSuite struct { + dsn string + suite.Suite +} + +// In order for 'go test' to run this suite, we need to create +// a normal test function and pass our suite to suite.Run +func TestOracleTestSuite(t *testing.T) { + if dsn := os.Getenv("ORACLE_DSN"); dsn != "" { + s := oracleSuite{dsn: dsn} + suite.Run(t, &s) + return + } + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.Port(defaultPort) + if err != nil { + t.Fatal(err) + } + + dsn := orclConnectionString(ip, port) + s := oracleSuite{dsn: dsn} + + suite.Run(t, &s) + }) +} + +func (s *oracleSuite) TestOpen() { + ora := &Oracle{} + d, err := ora.Open(s.dsn) + s.Require().Nil(err) + s.Require().NotNil(d) + defer func() { + if err := d.Close(); err != nil { + s.Error(err) + } + }() + ora = d.(*Oracle) + s.Require().Equal(DefaultMigrationsTable, ora.config.MigrationsTable) + + tbName := "" + err = ora.conn.QueryRowContext( + context.Background(), + `SELECT tname FROM tab WHERE tname = :1`, + ora.config.MigrationsTable, + ).Scan(&tbName) + s.Require().Nil(err) + s.Require().Equal(ora.config.MigrationsTable, tbName) + + dt.Test(s.T(), d, []byte(`BEGIN DBMS_OUTPUT.PUT_LINE('hello'); END;`)) +} + +func (s *oracleSuite) TestMigrate() { + ora := &Oracle{} + d, err := ora.Open(s.dsn) + s.Require().Nil(err) + s.Require().NotNil(d) + defer func() { + if err := d.Close(); err != nil { + s.Error(err) + } + }() + m, err := migrate.NewWithDatabaseInstance("file://./examples/migrations", "", d) + s.Require().Nil(err) + dt.TestMigrate(s.T(), m) +} + +func (s *oracleSuite) TestMultiStmtMigrate() { + ora := &Oracle{} + dsn := fmt.Sprintf("%s?%s=%s&&%s=%s", s.dsn, multiStmtEnableQueryKey, "true", multiStmtSeparatorQueryKey, "---") + d, err := ora.Open(dsn) + s.Require().Nil(err) + s.Require().NotNil(d) + defer func() { + if err := d.Close(); err != nil { + s.Error(err) + } + }() + m, err := migrate.NewWithDatabaseInstance("file://./examples/migrations-multistmt", "", d) + s.Require().Nil(err) + dt.TestMigrate(s.T(), m) +} + +func (s *oracleSuite) TestLockWorks() { + ora := &Oracle{} + d, err := ora.Open(s.dsn) + s.Require().Nil(err) + s.Require().NotNil(d) + defer func() { + if err := d.Close(); err != nil { + s.Error(err) + } + }() + + dt.Test(s.T(), d, []byte(`BEGIN DBMS_OUTPUT.PUT_LINE('hello'); END;`)) + + ora = d.(*Oracle) + err = ora.Lock() + s.Require().Nil(err) + + err = ora.Unlock() + s.Require().Nil(err) + + err = ora.Lock() + s.Require().Nil(err) + + err = ora.Unlock() + s.Require().Nil(err) +} + +func TestParseStatements(t *testing.T) { + cases := []struct { + migration string + expectedQueries []string + }{ + {migration: ` +CREATE TABLE USERS ( + USER_ID integer unique, + NAME varchar(40), + EMAIL varchar(40) +); + +--- +-- +BEGIN +EXECUTE IMMEDIATE 'DROP TABLE USERS'; +EXCEPTION + WHEN OTHERS THEN + IF SQLCODE != -942 THEN + RAISE; + END IF; +END; + +--- +-- comment +-- +CREATE TABLE USERS ( + USER_ID integer unique, + NAME varchar(40), + EMAIL varchar(40) +); +--- +--`, + expectedQueries: []string{ + `CREATE TABLE USERS ( + USER_ID integer unique, + NAME varchar(40), + EMAIL varchar(40) +)`, + `BEGIN +EXECUTE IMMEDIATE 'DROP TABLE USERS'; +EXCEPTION + WHEN OTHERS THEN + IF SQLCODE != -942 THEN + RAISE; + END IF; +END;`, + `CREATE TABLE USERS ( + USER_ID integer unique, + NAME varchar(40), + EMAIL varchar(40) +)`, + }}, + {migration: ` +-- comment +CREATE TABLE USERS ( + USER_ID integer unique, + NAME varchar(40), + EMAIL varchar(40) +); +-- this is comment +--- +ALTER TABLE USERS ADD CITY varchar(100); +`, + expectedQueries: []string{ + `CREATE TABLE USERS ( + USER_ID integer unique, + NAME varchar(40), + EMAIL varchar(40) +)`, + `ALTER TABLE USERS ADD CITY varchar(100)`, + }}, + } + for _, c := range cases { + queries, err := parseMultiStatements(bytes.NewBufferString(c.migration), DefaultMultiStmtSeparator) + require.Nil(t, err) + require.Equal(t, c.expectedQueries, queries) + } +} diff --git a/database/oracle/testdata/user.sql b/database/oracle/testdata/user.sql new file mode 100644 index 000000000..541174140 --- /dev/null +++ b/database/oracle/testdata/user.sql @@ -0,0 +1,7 @@ +alter session set container=XEPDB1; +create user orcl identified by orcl; +grant dba to orcl; +grant create session to orcl; +grant connect, resource to orcl; +grant all privileges to orcl; + diff --git a/go.mod b/go.mod index da117acd3..883dd3bc8 100644 --- a/go.mod +++ b/go.mod @@ -29,6 +29,7 @@ require ( github.com/mutecomm/go-sqlcipher/v4 v4.4.0 github.com/nakagami/firebirdsql v0.0.0-20190310045651-3c02a58cfed8 github.com/neo4j/neo4j-go-driver v1.8.1-0.20200803113522-b626aa943eba + github.com/sijms/go-ora/v2 v2.7.13 github.com/snowflakedb/gosnowflake v1.6.19 github.com/stretchr/testify v1.8.1 github.com/xanzy/go-gitlab v0.15.0 diff --git a/go.sum b/go.sum index 700aff24e..6981fcf87 100644 --- a/go.sum +++ b/go.sum @@ -535,6 +535,8 @@ github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9Nz github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/sijms/go-ora/v2 v2.7.13 h1:f+B+udrahHFIkxeAaSw1M4b6hW/m4RUm7QWsMQZVavo= +github.com/sijms/go-ora/v2 v2.7.13/go.mod h1:EHxlY6x7y9HAsdfumurRfTd+v8NrEOTR3Xl4FWlH6xk= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= diff --git a/internal/cli/build_oracle.go b/internal/cli/build_oracle.go new file mode 100644 index 000000000..067dba07d --- /dev/null +++ b/internal/cli/build_oracle.go @@ -0,0 +1,7 @@ +// +build oracle + +package cli + +import ( + _ "github.com/golang-migrate/migrate/v4/database/oracle" +)