[GOBBLIN-2098]Update the test Mysql version, and getting rid of com.w…

…ix:wix-embeded-mysql(deprecated) in favour of org.testcontainers (#3986) Updating the test Mysql version, and getting rid of com.wix:wix-embedded-mysql(deprected) in favour of org.testcontainers --------- Co-authored-by: Aditya Pratap Singh <[email protected]>
apache · Jun 26, 2024 · c64704e · c64704e
1 parent 26cf6bc
commit c64704e
Show file tree

Hide file tree

Showing 6 changed files with 50 additions and 52 deletions.
diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml
@@ -101,12 +101,12 @@ jobs:
     needs: build
     services:
       mysql:
-        image: mysql:5.7.32
+        image: mysql:8.0.20
         env:
           MYSQL_USER: testUser
           MYSQL_PASSWORD: testPassword
           MYSQL_DATABASE: test
-          MYSQL_ROOT_PASSWORD: password
+          MYSQL_ROOT_PASSWORD: testPassword
         ports:
           - 3306:3306
         options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=5
@@ -134,8 +134,8 @@ jobs:
             sudo dpkg -l | grep -i mysql
             sudo apt-get clean
             sudo apt-get install -y mysql-client
-            mysql --host 127.0.0.1 --port 3306 -uroot -ppassword -e "SET GLOBAL max_connections=2000"
-            mysql --host 127.0.0.1 --port 3306 -uroot -ppassword -e "SHOW DATABASES"
+            mysql --host 127.0.0.1 --port 3306 -uroot -ptestPassword -e "SHOW DATABASES"
+            mysql --host 127.0.0.1 --port 3306 -uroot -ptestPassword -e "SET GLOBAL max_connections=2000"
       - name: Cache Gradle Dependencies
         uses: actions/cache@v2
         with:

diff --git a/gobblin-metastore/build.gradle b/gobblin-metastore/build.gradle
@@ -40,7 +40,8 @@ dependencies {
 
     testCompile externalDependency.testng
     testCompile externalDependency.embeddedProcess
-    testCompile externalDependency.testMysqlServer
+    testCompile externalDependency.testContainers
+    testCompile externalDependency.testContainersMysql
     testCompile externalDependency.slf4jToLog4j
 }
 

diff --git a/...lin-metastore/src/test/java/org/apache/gobblin/metastore/DatabaseJobHistoryStoreTest.java b/...lin-metastore/src/test/java/org/apache/gobblin/metastore/DatabaseJobHistoryStoreTest.java
@@ -187,6 +187,7 @@ private JobExecutionInfo create(int index, boolean differentTableType) {
     jobExecutionInfo.setJobName("TestJob" + index);
     jobExecutionInfo.setJobId(jobExecutionInfo.getJobName() + "_" + System.currentTimeMillis());
     jobExecutionInfo.setStartTime(System.currentTimeMillis());
+    jobExecutionInfo.setEndTime(System.currentTimeMillis());
     jobExecutionInfo.setState(JobStateEnum.PENDING);
     jobExecutionInfo.setLaunchedTasks(2);
     jobExecutionInfo.setCompletedTasks(0);
@@ -212,6 +213,7 @@ private JobExecutionInfo create(int index, boolean differentTableType) {
     taskExecutionInfo1.setJobId(jobExecutionInfo.getJobId());
     taskExecutionInfo1.setTaskId(jobExecutionInfo.getJobId() + "_0");
     taskExecutionInfo1.setStartTime(System.currentTimeMillis());
+    taskExecutionInfo1.setEndTime(System.currentTimeMillis());
     taskExecutionInfo1.setState(TaskStateEnum.PENDING);
     taskExecutionInfo1.setLowWatermark(0L);
     taskExecutionInfo1.setHighWatermark(1000L);
@@ -237,6 +239,7 @@ private JobExecutionInfo create(int index, boolean differentTableType) {
     taskExecutionInfo2.setJobId(jobExecutionInfo.getJobId());
     taskExecutionInfo2.setTaskId(jobExecutionInfo.getJobId() + "_1");
     taskExecutionInfo2.setStartTime(System.currentTimeMillis());
+    taskExecutionInfo2.setEndTime(System.currentTimeMillis());
     taskExecutionInfo2.setState(TaskStateEnum.PENDING);
     taskExecutionInfo2.setLowWatermark(0L);
     taskExecutionInfo2.setHighWatermark(2000L);

diff --git a/...store/src/test/java/org/apache/gobblin/metastore/testing/TestMetastoreDatabaseServer.java b/...store/src/test/java/org/apache/gobblin/metastore/testing/TestMetastoreDatabaseServer.java
@@ -25,53 +25,49 @@
 import java.sql.SQLException;
 import java.util.Properties;
 
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.testcontainers.containers.MySQLContainer;
+
 
 import com.google.common.base.Optional;
 import com.google.common.collect.ImmutableMap;
-import com.google.inject.Guice;
-import com.google.inject.Injector;
 import com.typesafe.config.Config;
 import com.typesafe.config.ConfigFactory;
-import com.wix.mysql.EmbeddedMysql;
-import com.wix.mysql.config.MysqldConfig;
-import com.wix.mysql.distribution.Version;
-
-import javax.sql.DataSource;
+import com.zaxxer.hikari.HikariDataSource;
 
 import org.apache.gobblin.configuration.ConfigurationKeys;
-import org.apache.gobblin.metastore.MetaStoreModule;
 import org.apache.gobblin.metastore.util.DatabaseJobHistoryStoreSchemaManager;
 import org.apache.gobblin.metastore.util.MySqlJdbcUrl;
-import org.apache.gobblin.util.PortUtils;
 
 
 class TestMetastoreDatabaseServer implements Closeable {
 
   private static final String INFORMATION_SCHEMA = "information_schema";
   private static final String ROOT_USER = "root";
-  private static final String ROOT_PASSWORD = "password";
   private static final String DROP_DATABASE_TEMPLATE = "DROP DATABASE IF EXISTS %s;";
   private static final String CREATE_DATABASE_TEMPLATE = "CREATE DATABASE %s CHARACTER SET = %s COLLATE = %s;";
   private static final String ADD_USER_TEMPLATE = "GRANT ALL ON %s.* TO '%s'@'%%';";
 
   public static final String CONFIG_PREFIX = "gobblin.metastore.testing";
   public static final String EMBEDDED_MYSQL_ENABLED_KEY = "embeddedMysqlEnabled";
-  public static final String EMBEDDED_MYSQL_ENABLED_FULL_KEY =
-      CONFIG_PREFIX + "." + EMBEDDED_MYSQL_ENABLED_KEY;
+  public static final String EMBEDDED_MYSQL_ENABLED_FULL_KEY = CONFIG_PREFIX + "." + EMBEDDED_MYSQL_ENABLED_KEY;
   public static final String DBUSER_NAME_KEY = "testUser";
-  public static final String DBUSER_NAME_FULL_KEY =  CONFIG_PREFIX + "." + DBUSER_NAME_KEY;
+  public static final String DBUSER_NAME_FULL_KEY = CONFIG_PREFIX + "." + DBUSER_NAME_KEY;
   public static final String DBUSER_PASSWORD_KEY = "testPassword";
-  public static final String DBUSER_PASSWORD_FULL_KEY =  CONFIG_PREFIX + "." + DBUSER_PASSWORD_KEY;
+  public static final String DBUSER_PASSWORD_FULL_KEY = CONFIG_PREFIX + "." + DBUSER_PASSWORD_KEY;
   public static final String DBHOST_KEY = "dbHost";
-  public static final String DBHOST_FULL_KEY =  CONFIG_PREFIX + "." + DBHOST_KEY;
+  public static final String DBHOST_FULL_KEY = CONFIG_PREFIX + "." + DBHOST_KEY;
   public static final String DBPORT_KEY = "dbPort";
-  public static final String DBPORT_FULL_KEY =  CONFIG_PREFIX + "." + DBPORT_KEY;
+  public static final String DBPORT_FULL_KEY = CONFIG_PREFIX + "." + DBPORT_KEY;
+  public static final String MYSQL_VERSION = "8.0.20";
+  public static final String MYSQL_ROOT_PASSWORD_KEY = "MYSQL_ROOT_PASSWORD";
+  private static final String CHARSET_UTF_8_MB_4 = "utf8mb4";
+  private static final String COLLATE_UTF_8_MB_4_UNICODE_CI = "utf8mb4_unicode_ci";
 
   private final Logger log = LoggerFactory.getLogger(TestMetastoreDatabaseServer.class);
-  private final MysqldConfig config;
-  private final EmbeddedMysql testingMySqlServer;
+  private final MySQLContainer mySQLContainer;
   private final boolean embeddedMysqlEnabled;
   private final String dbUserName;
   private final String dbUserPassword;
@@ -83,28 +79,23 @@ class TestMetastoreDatabaseServer implements Closeable {
     this.embeddedMysqlEnabled = realConfig.getBoolean(EMBEDDED_MYSQL_ENABLED_KEY);
     this.dbUserName = realConfig.getString(DBUSER_NAME_KEY);
     this.dbUserPassword = realConfig.getString(DBUSER_PASSWORD_KEY);
-    this.dbHost = this.embeddedMysqlEnabled ? "localhost" : realConfig.getString(DBHOST_KEY);
-    this.dbPort = this.embeddedMysqlEnabled ? new PortUtils.ServerSocketPortLocator().random() : realConfig.getInt(DBPORT_KEY);
-
-    this.log.info("Starting with config: embeddedMysqlEnabled={} dbUserName={} dbHost={} dbPort={}",
-                  this.embeddedMysqlEnabled,
-                  this.dbUserName,
-                  this.dbHost,
-                  this.dbPort);
-
-    config = MysqldConfig.aMysqldConfig(Version.v8_latest)
-        .withPort(this.dbPort)
-        .withUser(this.dbUserName, this.dbUserPassword)
-        .withServerVariable("explicit_defaults_for_timestamp", "off")
-        // default `max_connections` is apparently 151 - see: https://dev.mysql.com/doc/refman/8.0/en/server-system-variables.html#sysvar_max_connections
-        .withServerVariable("max_connections", "501")
-        .build();
     if (this.embeddedMysqlEnabled) {
-      testingMySqlServer = EmbeddedMysql.anEmbeddedMysql(config).start();
-    }
-    else {
-      testingMySqlServer = null;
+      mySQLContainer = new MySQLContainer<>("mysql:" + MYSQL_VERSION)
+          .withUsername(this.dbUserName)
+          .withPassword(this.dbUserPassword);
+      mySQLContainer.withEnv(MYSQL_ROOT_PASSWORD_KEY, this.dbUserPassword);
+      mySQLContainer.start();
+
+      this.dbHost = mySQLContainer.getHost();
+      this.dbPort = mySQLContainer.getFirstMappedPort();
+
+    } else {
+      mySQLContainer = null;
+      this.dbHost =  realConfig.getString(DBHOST_KEY);
+      this.dbPort =  realConfig.getInt(DBPORT_KEY);
     }
+    this.log.info("Started with config: embeddedMysqlEnabled={} dbUserName={} dbHost={} dbPort={}",
+        this.embeddedMysqlEnabled, this.dbUserName, this.dbHost, this.dbPort);
   }
 
   static Config getDefaultConfig() {
@@ -132,8 +123,8 @@ public void drop(String database) throws SQLException, URISyntaxException {
 
   @Override
   public void close() throws IOException {
-    if (testingMySqlServer != null) {
-      testingMySqlServer.stop();
+    if (mySQLContainer != null) {
+      mySQLContainer.stop();
     }
   }
 
@@ -154,7 +145,7 @@ private MySqlJdbcUrl getBaseJdbcUrl() throws URISyntaxException {
 
   private MySqlJdbcUrl getInformationSchemaJdbcUrl() throws URISyntaxException {
     // embedded mysql has an empty password by default
-    String password =  this.embeddedMysqlEnabled ? "" : ROOT_PASSWORD;
+    String password =  this.embeddedMysqlEnabled ? "" : this.dbUserPassword;
     return getBaseJdbcUrl()
         .setPath(INFORMATION_SCHEMA)
         .setUser(ROOT_USER)
@@ -164,8 +155,9 @@ private MySqlJdbcUrl getInformationSchemaJdbcUrl() throws URISyntaxException {
   private Optional<Connection> getConnector(MySqlJdbcUrl jdbcUrl) throws SQLException {
     Properties properties = new Properties();
     properties.setProperty(ConfigurationKeys.JOB_HISTORY_STORE_URL_KEY, jdbcUrl.toString());
-    Injector injector = Guice.createInjector(new MetaStoreModule(properties));
-    DataSource dataSource = injector.getInstance(DataSource.class);
+    HikariDataSource dataSource = new HikariDataSource();
+    dataSource.setJdbcUrl(jdbcUrl.toString());
+    dataSource.setPassword(this.dbUserPassword);
     return Optional.of(dataSource.getConnection());
   }
 
@@ -177,8 +169,8 @@ private void ensureDatabaseExists(String database) throws SQLException, URISynta
       executeStatements(connection,
           String.format(DROP_DATABASE_TEMPLATE, database),
           String.format(CREATE_DATABASE_TEMPLATE, database,
-                 config.getCharset().getCharset(), config.getCharset().getCollate()),
-          String.format(ADD_USER_TEMPLATE, database, config.getUsername()));
+                 this.CHARSET_UTF_8_MB_4, this.COLLATE_UTF_8_MB_4_UNICODE_CI),
+          String.format(ADD_USER_TEMPLATE, database, this.dbUserName));
     } finally {
       if (connectionOptional.isPresent()) {
         connectionOptional.get().close();

diff --git a/...gobblin-rest-server/src/test/java/org/apache/gobblin/rest/JobExecutionInfoServerTest.java b/...gobblin-rest-server/src/test/java/org/apache/gobblin/rest/JobExecutionInfoServerTest.java
@@ -157,6 +157,7 @@ private static JobExecutionInfo createJobExecutionInfo(int index) {
     jobExecutionInfo.setJobName("TestJob" + index);
     jobExecutionInfo.setJobId(jobExecutionInfo.getJobName() + "_" + System.currentTimeMillis());
     jobExecutionInfo.setStartTime(System.currentTimeMillis());
+    jobExecutionInfo.setEndTime(System.currentTimeMillis());
     jobExecutionInfo.setState(JobStateEnum.PENDING);
     jobExecutionInfo.setLaunchedTasks(2);
     jobExecutionInfo.setCompletedTasks(0);
@@ -180,6 +181,7 @@ private static JobExecutionInfo createJobExecutionInfo(int index) {
     taskExecutionInfo1.setJobId(jobExecutionInfo.getJobId());
     taskExecutionInfo1.setTaskId(jobExecutionInfo.getJobId() + "_0");
     taskExecutionInfo1.setStartTime(System.currentTimeMillis());
+    taskExecutionInfo1.setEndTime(System.currentTimeMillis());
     taskExecutionInfo1.setState(TaskStateEnum.PENDING);
     taskExecutionInfo1.setLowWatermark(0L);
     taskExecutionInfo1.setHighWatermark(1000L);
@@ -205,6 +207,7 @@ private static JobExecutionInfo createJobExecutionInfo(int index) {
     taskExecutionInfo2.setJobId(jobExecutionInfo.getJobId());
     taskExecutionInfo2.setTaskId(jobExecutionInfo.getJobId() + "_1");
     taskExecutionInfo2.setStartTime(System.currentTimeMillis());
+    taskExecutionInfo2.setEndTime(System.currentTimeMillis());
     taskExecutionInfo2.setState(TaskStateEnum.PENDING);
     taskExecutionInfo2.setLowWatermark(0L);
     taskExecutionInfo2.setHighWatermark(2000L);

diff --git a/gradle/scripts/dependencyDefinitions.gradle b/gradle/scripts/dependencyDefinitions.gradle
@@ -126,7 +126,7 @@ ext.externalDependency = [
     "commonsVfs": "org.apache.commons:commons-vfs2:2.0",
     "mysqlConnector": "mysql:mysql-connector-java:8.0.24",
     "javaxInject": "javax.inject:javax.inject:1",
-    "guice": "com.google.inject:guice:4.0",
+    "guice": "com.google.inject:guice:4.2.2",
     "guiceMultibindings": "com.google.inject.extensions:guice-multibindings:4.0",
     "guiceServlet": "com.google.inject.extensions:guice-servlet:4.0",
     "derby": "org.apache.derby:derby:10.12.1.1",
@@ -182,7 +182,6 @@ ext.externalDependency = [
     "jsonAssert": "org.skyscreamer:jsonassert:1.3.0",
     "reflections" : "org.reflections:reflections:0.9.10",
     "embeddedProcess": "de.flapdoodle.embed:de.flapdoodle.embed.process:1.50.2",
-    "testMysqlServer": "com.wix:wix-embedded-mysql:4.6.1",
     "flyway": "org.flywaydb:flyway-core:7.9.2",
     "oltu": "org.apache.oltu.oauth2:org.apache.oltu.oauth2.client:1.0.2",
     "googleAnalytics": "com.google.apis:google-api-services-analytics:v3-rev134-1.22.0",