Refactor to make testing easier (#90)

* minor: refactor to make testing easier

* patch: retrieve inputs into object rather than globals

* test: run more "integration" tests in parallel

* test: fix needs and rearrange ci_integration_* jobs

* test: forgot comma

* test: fix sad_path_timeout_minutes assertions

* test: add single ci_all_tests_passed job that can be required for CI rather than each individual job

* test: add single ci_all_tests_passed job that can be required for CI rather than each individual job

Author: nick-fields

.github/workflows/ci_cd.yml

@@ -24,6 +24,117 @@ jobs:
           directory: ./coverage/
           verbose: true
 
+  ci_integration:
+    name: Run Integration Tests
+    if: startsWith(github.ref, 'refs/heads')
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup Node.js
+        uses: actions/setup-node@v1
+        with:
+          node-version: 16
+      - name: Install dependencies
+        run: npm ci
+
+      - name: happy-path
+        id: happy_path
+        uses: ./
+        with:
+          timeout_minutes: 1
+          max_attempts: 2
+          command: npm -v
+      - uses: nick-invision/assert-action@v1
+        with:
+          expected: true
+          actual: ${{ steps.happy_path.outputs.total_attempts == '1' && steps.happy_path.outputs.exit_code == '0' }}
+
+      - name: log examples
+        uses: ./
+        with:
+          command: node ./.github/scripts/log-examples.js
+          timeout_minutes: 1
+
+      - name: sad-path (error)
+        id: sad_path_error
+        uses: ./
+        continue-on-error: true
+        with:
+          timeout_minutes: 1
+          max_attempts: 2
+          command: node -e "process.exit(1)"
+      - uses: nick-invision/assert-action@v1
+        with:
+          expected: 2
+          actual: ${{ steps.sad_path_error.outputs.total_attempts }}
+      - uses: nick-invision/assert-action@v1
+        with:
+          expected: failure
+          actual: ${{ steps.sad_path_error.outcome }}
+
+      - name: retry_on (timeout) fails early if error encountered
+        id: retry_on_timeout_fail
+        uses: ./
+        continue-on-error: true
+        with:
+          timeout_minutes: 1
+          max_attempts: 3
+          retry_on: timeout
+          command: node -e "process.exit(2)"
+      - uses: nick-invision/assert-action@v1
+        with:
+          expected: 1
+          actual: ${{ steps.retry_on_timeout_fail.outputs.total_attempts }}
+      - uses: nick-invision/assert-action@v1
+        with:
+          expected: failure
+          actual: ${{ steps.retry_on_timeout_fail.outcome }}
+      - uses: nick-invision/assert-action@v1
+        with:
+          expected: 2
+          actual: ${{ steps.retry_on_timeout_fail.outputs.exit_code }}
+
+      - name: retry_on (error)
+        id: retry_on_error
+        uses: ./
+        continue-on-error: true
+        with:
+          timeout_minutes: 1
+          max_attempts: 2
+          retry_on: error
+          command: node -e "process.exit(2)"
+      - uses: nick-invision/assert-action@v1
+        with:
+          expected: 2
+          actual: ${{ steps.retry_on_error.outputs.total_attempts }}
+      - uses: nick-invision/assert-action@v1
+        with:
+          expected: failure
+          actual: ${{ steps.retry_on_error.outcome }}
+      - uses: nick-invision/assert-action@v1
+        with:
+          expected: 2
+          actual: ${{ steps.retry_on_error.outputs.exit_code }}
+
+      - name: sad-path (wrong shell for OS)
+        id: wrong_shell
+        uses: ./
+        continue-on-error: true
+        with:
+          timeout_minutes: 1
+          max_attempts: 2
+          shell: cmd
+          command: 'dir'
+      - uses: nick-invision/assert-action@v1
+        with:
+          expected: 2
+          actual: ${{ steps.wrong_shell.outputs.total_attempts }}
+      - uses: nick-invision/assert-action@v1
+        with:
+          expected: failure
+          actual: ${{ steps.wrong_shell.outcome }}
+
   ci_integration_envvar:
     name: Run Integration Env Var Tests
     if: startsWith(github.ref, 'refs/heads')
@@ -175,8 +286,8 @@ jobs:
           expected: success
           actual: ${{ steps.sad_path_continue_on_error.outcome }}
 
-  ci_integration:
-    name: Run Integration Tests
+  ci_integration_retry_wait_seconds:
+    name: Run Integration Tests (retry_wait_seconds)
     if: startsWith(github.ref, 'refs/heads')
     runs-on: ubuntu-latest
     steps:
@@ -189,24 +300,6 @@ jobs:
       - name: Install dependencies
         run: npm ci
 
-      - name: happy-path
-        id: happy_path
-        uses: ./
-        with:
-          timeout_minutes: 1
-          max_attempts: 2
-          command: npm -v
-      - uses: nick-invision/assert-action@v1
-        with:
-          expected: true
-          actual: ${{ steps.happy_path.outputs.total_attempts == '1' && steps.happy_path.outputs.exit_code == '0' }}
-
-      - name: log examples
-        uses: ./
-        with:
-          command: node ./.github/scripts/log-examples.js
-          timeout_minutes: 1
-
       - name: sad-path (retry_wait_seconds)
         id: sad_path_wait_sec
         uses: ./
@@ -230,6 +323,20 @@ jobs:
           actual: ${{ steps.sad_path_wait_sec.outputs.exit_error }}
           comparison: contains
 
+  ci_integration_on_retry_cmd:
+    name: Run Integration Tests (on_retry_command)
+    if: startsWith(github.ref, 'refs/heads')
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup Node.js
+        uses: actions/setup-node@v1
+        with:
+          node-version: 16
+      - name: Install dependencies
+        run: npm ci
+
       - name: new-command-on-retry
         id: new-command-on-retry
         uses: ./
@@ -259,88 +366,9 @@ jobs:
           command: node -e "process.exit(1)"
           on_retry_command: node -e "throw new Error('This is an on-retry command error')"
 
-      - name: sad-path (error)
-        id: sad_path_error
-        uses: ./
-        continue-on-error: true
-        with:
-          timeout_minutes: 1
-          max_attempts: 2
-          command: node -e "process.exit(1)"
-      - uses: nick-invision/assert-action@v1
-        with:
-          expected: 2
-          actual: ${{ steps.sad_path_error.outputs.total_attempts }}
-      - uses: nick-invision/assert-action@v1
-        with:
-          expected: failure
-          actual: ${{ steps.sad_path_error.outcome }}
-
-      - name: retry_on (timeout) fails early if error encountered
-        id: retry_on_timeout_fail
-        uses: ./
-        continue-on-error: true
-        with:
-          timeout_minutes: 1
-          max_attempts: 3
-          retry_on: timeout
-          command: node -e "process.exit(2)"
-      - uses: nick-invision/assert-action@v1
-        with:
-          expected: 1
-          actual: ${{ steps.retry_on_timeout_fail.outputs.total_attempts }}
-      - uses: nick-invision/assert-action@v1
-        with:
-          expected: failure
-          actual: ${{ steps.retry_on_timeout_fail.outcome }}
-      - uses: nick-invision/assert-action@v1
-        with:
-          expected: 2
-          actual: ${{ steps.retry_on_timeout_fail.outputs.exit_code }}
-
-      - name: retry_on (error)
-        id: retry_on_error
-        uses: ./
-        continue-on-error: true
-        with:
-          timeout_minutes: 1
-          max_attempts: 2
-          retry_on: error
-          command: node -e "process.exit(2)"
-      - uses: nick-invision/assert-action@v1
-        with:
-          expected: 2
-          actual: ${{ steps.retry_on_error.outputs.total_attempts }}
-      - uses: nick-invision/assert-action@v1
-        with:
-          expected: failure
-          actual: ${{ steps.retry_on_error.outcome }}
-      - uses: nick-invision/assert-action@v1
-        with:
-          expected: 2
-          actual: ${{ steps.retry_on_error.outputs.exit_code }}
-
-      - name: sad-path (wrong shell for OS)
-        id: wrong_shell
-        uses: ./
-        continue-on-error: true
-        with:
-          timeout_minutes: 1
-          max_attempts: 2
-          shell: cmd
-          command: 'dir'
-      - uses: nick-invision/assert-action@v1
-        with:
-          expected: 2
-          actual: ${{ steps.wrong_shell.outputs.total_attempts }}
-      - uses: nick-invision/assert-action@v1
-        with:
-          expected: failure
-          actual: ${{ steps.wrong_shell.outcome }}
-
   # timeout tests take longer to run so run in parallel
-  ci_integration_timeout:
-    name: Run Integration Timeout Tests
+  ci_integration_timeout_seconds:
+    name: Run Integration Timeout Tests (seconds)
     if: startsWith(github.ref, 'refs/heads')
     runs-on: ubuntu-latest
     steps:
@@ -370,6 +398,20 @@ jobs:
           expected: failure
           actual: ${{ steps.sad_path_timeout.outcome }}
 
+  ci_integration_timeout_retry_on_timeout:
+    name: Run Integration Timeout Tests (retry_on timeout)
+    if: startsWith(github.ref, 'refs/heads')
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup Node.js
+        uses: actions/setup-node@v1
+        with:
+          node-version: 16
+      - name: Install dependencies
+        run: npm ci
+
       - name: retry_on (timeout)
         id: retry_on_timeout
         uses: ./
@@ -388,6 +430,20 @@ jobs:
           expected: failure
           actual: ${{ steps.retry_on_timeout.outcome }}
 
+  ci_integration_timeout_retry_on_error:
+    name: Run Integration Timeout Tests (retry_on error)
+    if: startsWith(github.ref, 'refs/heads')
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup Node.js
+        uses: actions/setup-node@v1
+        with:
+          node-version: 16
+      - name: Install dependencies
+        run: npm ci
+
       - name: retry_on (error) fails early if timeout encountered
         id: retry_on_error_fail
         uses: ./
@@ -410,6 +466,20 @@ jobs:
           expected: 1
           actual: ${{ steps.retry_on_error_fail.outputs.exit_code }}
 
+  ci_integration_timeout_minutes:
+    name: Run Integration Timeout Tests (minutes)
+    if: startsWith(github.ref, 'refs/heads')
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup Node.js
+        uses: actions/setup-node@v1
+        with:
+          node-version: 16
+      - name: Install dependencies
+        run: npm ci
+
       - name: sad-path (timeout minutes)
         id: sad_path_timeout_minutes
         uses: ./
@@ -421,11 +491,11 @@ jobs:
       - uses: nick-invision/assert-action@v1
         with:
           expected: 2
-          actual: ${{ steps.sad_path_timeout.outputs.total_attempts }}
+          actual: ${{ steps.sad_path_timeout_minutes.outputs.total_attempts }}
       - uses: nick-invision/assert-action@v1
         with:
           expected: failure
-          actual: ${{ steps.sad_path_timeout.outcome }}
+          actual: ${{ steps.sad_path_timeout_minutes.outcome }}
 
   ci_windows:
     name: Run Windows Tests
@@ -479,10 +549,32 @@ jobs:
             echo "this is
             a test"
 
+  ci_all_tests_passed:
+    name: All tests passed
+    needs:
+      [
+        ci_unit,
+        ci_integration,
+        ci_integration_envvar,
+        ci_integration_large_output,
+        ci_integration_on_retry_cmd,
+        ci_integration_retry_wait_seconds,
+        ci_integration_continue_on_error,
+        ci_integration_retry_on_exit_code,
+        ci_integration_timeout_seconds,
+        ci_integration_timeout_minutes,
+        ci_integration_timeout_retry_on_timeout,
+        ci_integration_timeout_retry_on_error,
+        ci_windows,
+      ]
+    runs-on: ubuntu-latest
+    steps:
+      - run: echo "If this is hit, all tests successfully passed"
+
   # runs on push to master only
   cd:
     name: Publish Action
-    needs: [ci_integration, ci_integration_timeout, ci_windows]
+    needs: [ci_all_tests_passed]
     if: github.ref == 'refs/heads/master'
     runs-on: ubuntu-latest
     steps: