Container Testing Updates

Summary:
ericvicenti Here is the latest updates and fixes for the container testing. Everything should be good to go now, the update for `inotify.max_user_watches` should fix the E2E test issues you were seeing locally.

The update to the packer prevents excessively large filename lengths due to the hash used in the name and splits them into directories instead. I was getting errors locally on the E2E because the hash filename was over 248 characters which was causing issues on the base image file system. It might not have appeared in circle due to another file system being used with different limits. I can separate it out into another PR if you want though.

* Turned jenkins instrumentation parallelism up to 3
* Disabled various instrumentation tests that seemed to have inconsistent results from
Jenkins by default
* Install google-chrome in the android base image so the chrome debug E2E test does not fail
* Turned back on E2E tests

cc normanjoyner
Closes facebook#13417

Differential Revision: D4876137

Pulled By: ericvicenti

fbshipit-source-id: 31e033c1e34b02acb5484478414197ac9eb11f95

Author: nicktate

ContainerShip/Dockerfile.android-base

@@ -86,10 +86,16 @@ RUN echo "y" | android update sdk -u -a -t $(android list sdk -a | grep "Intel x
 RUN echo "y" | android update sdk -u -a -t $(android list sdk -a | grep "Google APIs, Android API 23, revision 1" | awk '{ print $1 }' | sed 's/.$//')
 
 # Android Support Repository, revision 45
-RUN echo "y" | android update sdk -u -a -t $(android list sdk -a | grep "Android Support Repository, revision 45" | awk '{ print $1 }' | sed 's/.$//')
+RUN echo "y" | android update sdk -u -a -t $(android list sdk -a | grep "Android Support Repository" | awk '{ print $1 }' | sed 's/.$//')
 
 # Link adb executable
 RUN ln -s /opt/android/platform-tools/adb /usr/bin/adb
 
+# Install google-chrome
+RUN curl -fsSL https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \
+     && echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google-chrome.list \
+     && apt-get update \
+     && apt-get install -y google-chrome-stable
+
 # clean up unnecessary directories
 RUN rm -rf /opt/android/system-images/android-19/default/x86

ContainerShip/scripts/run-android-ci-instrumentation-tests.js

@@ -28,12 +28,6 @@ const child_process = require('child_process');
 const fs = require('fs');
 const path = require('path');
 
-// Flaky tests ignored on Circle CI. They still run internally at fb.
-const ignoredTests = [
-  'ReactScrollViewTestCase',
-  'ReactHorizontalScrollViewTestCase'
-];
-
 const colors = {
     GREEN: '\x1b[32m',
     RED: '\x1b[31m',
@@ -42,6 +36,7 @@ const colors = {
 
 const test_opts = {
     FILTER: new RegExp(argv.filter || '.*', 'i'),
+    IGNORE: argv.ignore || null,
     PACKAGE: argv.package || 'com.facebook.react.tests',
     PATH: argv.path || './ReactAndroid/src/androidTest/java/com/facebook/react/tests',
     RETRIES: parseInt(argv.retries || 2, 10),
@@ -59,14 +54,21 @@ let testClasses = fs.readdirSync(path.resolve(process.cwd(), test_opts.PATH))
         return file.endsWith('.java');
     }).map((clazz) => {
         return path.basename(clazz, '.java');
-    }).filter(className => {
-        return ignoredTests.indexOf(className) === -1;
-    }).map((clazz) => {
-        return test_opts.PACKAGE + '.' + clazz;
-    }).filter((clazz) => {
-        return test_opts.FILTER.test(clazz);
     });
 
+if (test_opts.IGNORE) {
+    test_opts.IGNORE = new RegExp(test_opts.IGNORE, 'i');
+    testClasses = testClasses.filter(className => {
+        return !test_opts.IGNORE.test(className);
+    });
+}
+
+testClasses = testClasses.map((clazz) => {
+    return test_opts.PACKAGE + '.' + clazz;
+}).filter((clazz) => {
+    return test_opts.FILTER.test(clazz);
+});
+
 // only process subset of the tests at corresponding offset and count if args provided
 if (test_opts.COUNT != null && test_opts.OFFSET != null) {
     const testCount = testClasses.length;

ContainerShip/scripts/run-ci-e2e-tests.sh

@@ -11,14 +11,18 @@ RUN_CLI_INSTALL=1
 RUN_IOS=0
 RUN_JS=0
 
-RETRY_COUNT=${RETRY_COUNT:-1}
+RETRY_COUNT=${RETRY_COUNT:-2}
 AVD_UUID=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 8 | head -n 1)
 
 ANDROID_NPM_DEPS="appium@1.5.1 mocha@2.4.5 wd@0.3.11 colors@1.0.3 pretty-data2@0.40.1"
 CLI_PACKAGE=$ROOT/react-native-cli/react-native-cli-*.tgz
 PACKAGE=$ROOT/react-native-*.tgz
 REACT_NATIVE_MAX_WORKERS=1
 
+# solve issue with max user watches limit
+echo 65536 | tee -a /proc/sys/fs/inotify/max_user_watches
+watchman shutdown-server
+
 # retries command on failure
 # $1 -- max attempts
 # $2 -- command to run
@@ -177,7 +181,6 @@ function e2e_suite() {
       cd ..
       keytool -genkey -v -keystore android/keystores/debug.keystore -storepass android -alias androiddebugkey -keypass android -dname "CN=Android Debug,O=Android,C=US"
 
-      echo "Starting packager server"
       node ./node_modules/.bin/appium >> /dev/null &
       APPIUM_PID=$!
       echo "Starting appium server $APPIUM_PID"
@@ -193,6 +196,7 @@ function e2e_suite() {
         return 1
       fi
 
+      echo "Starting packager server"
       npm start >> /dev/null &
       SERVER_PID=$!
       sleep 15

Jenkinsfile

@@ -52,11 +52,13 @@ def getParallelInstrumentationTests(testDir, parallelCount, imageName) {
     def testCount = sh(script: "ls ${testDir} | wc -l", returnStdout: true).trim().toInteger()
     def testPerParallel = testCount.intdiv(parallelCount) + 1
 
+    def ignoredTests = 'CatalystNativeJavaToJSReturnValuesTestCase|CatalystUIManagerTestCase|CatalystMeasureLayoutTest|CatalystNativeJavaToJSArgumentsTestCase|CatalystNativeJSToJavaParametersTestCase|ReactScrollViewTestCase|ReactHorizontalScrollViewTestCase|ViewRenderingTestCase';
+
     for (def x = 0; (x*testPerParallel) < testCount; x++) {
         def offset = x
         integrationTests["android integration tests: ${offset}"] = {
             run: {
-                runCmdOnDockerImage(imageName, "bash /app/ContainerShip/scripts/run-android-docker-instrumentation-tests.sh --offset=${offset} --count=${testPerParallel}", '--privileged --rm')
+                runCmdOnDockerImage(imageName, "bash /app/ContainerShip/scripts/run-android-docker-instrumentation-tests.sh --offset=${offset} --count=${testPerParallel} --ignore=\"${ignoredTests}\"", '--privileged --rm')
             }
         }
     }
@@ -106,7 +108,7 @@ def runStages() {
                 jsImageName = "${buildInfo.image.name}-js:${jsTag}"
                 androidImageName = "${buildInfo.image.name}-android:${androidTag}"
 
-                parallelInstrumentationTests = getParallelInstrumentationTests('./ReactAndroid/src/androidTest/java/com/facebook/react/tests', 1, androidImageName)
+                parallelInstrumentationTests = getParallelInstrumentationTests('./ReactAndroid/src/androidTest/java/com/facebook/react/tests', 3, androidImageName)
 
                 parallel(
                     'javascript build': {
@@ -148,9 +150,7 @@ def runStages() {
                             runCmdOnDockerImage(androidImageName, 'bash /app/ContainerShip/scripts/run-android-docker-unit-tests.sh', '--privileged --rm')
                         },
                         'android e2e tests': {
-                            // temporarily disable e2e tests, they have a high transient failure rate
-                            // runCmdOnDockerImage(androidImageName, 'bash /app/ContainerShip/scripts/run-ci-e2e-tests.sh --android --js', '--rm')
-                            echo "Android E2E tests have been temporarily disabled"
+                            runCmdOnDockerImage(androidImageName, 'bash /app/ContainerShip/scripts/run-ci-e2e-tests.sh --android --js', '--privileged --rm')
                         }
                     )
                 } catch(e) {

package.json

@@ -118,7 +118,7 @@
     "test-android-build": "docker build -t react/android -f ContainerShip/Dockerfile.android .",
     "test-android-run-instrumentation": "docker run --cap-add=SYS_ADMIN -it react/android bash ContainerShip/scripts/run-android-docker-instrumentation-tests.sh",
     "test-android-run-unit": "docker run --cap-add=SYS_ADMIN -it react/android bash ContainerShip/scripts/run-android-docker-unit-tests.sh",
-    "test-android-run-e2e": "docker run -it react/android bash ContainerShip/scripts/run-ci-e2e-tests.sh --android --js",
+    "test-android-run-e2e": "docker run --privileged -it react/android bash ContainerShip/scripts/run-ci-e2e-tests.sh --android --js",
     "test-android-all": "npm run test-android-build && npm run test-android-run-unit && npm run test-android-run-instrumentation && npm run test-android-run-e2e",
     "test-android-instrumentation": "npm run test-android-build && npm run test-android-run-instrumentation",
     "test-android-unit": "npm run test-android-build && npm run test-android-run-unit",