Support parallel builds (via buildx) (#39)

This adjusts the build process to use `buildx` (which may or may not be
the best approach for parallel builds) AND adjusts some of the
`--cache-from`/`--cache-to` flags so the pre-pulled caches should
actually work (historically, enabling buildkit would miss cache and
build from scratch most of the time)

https://docs.docker.com/build/cache/backends/

Fixes #36

.github/workflows/self-test-multistage-docker-build.yml

@@ -13,6 +13,62 @@ permissions:
   packages: write
 
 jobs:
+  test-build-parallel-setting:
+    name: Parallel via setting
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        parallel: [true, false]
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Auth to GH registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: generate file change
+        run: echo $GITHUB_SHA > file.txt
+
+      - uses: ./
+        id: build
+        with:
+          build-args: BUILD_ARG_1=hello
+          dockerfile: examples/Dockerfile
+          repository: ghcr.io/firehed/actions
+          parallel: ${{ matrix.parallel }}
+          server-stage: server
+          quiet: false
+
+  test-build-parallel-buildkit:
+    name: Parallel via DOCKER_BUILDKIT
+    runs-on: ubuntu-latest
+    env:
+      DOCKER_BUILDKIT: "1"
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Auth to GH registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: generate file change
+        run: echo $GITHUB_SHA > file.txt
+
+      - uses: ./
+        id: build
+        with:
+          build-args: BUILD_ARG_1=hello
+          dockerfile: examples/Dockerfile
+          repository: ghcr.io/firehed/actions
+          server-stage: server
+          quiet: false
+
   build:
     name: Build docker images
     runs-on: ubuntu-latest

README.md

@@ -24,8 +24,25 @@ While the action allows many stages to be pushed to the registry for future re-u
 | `context` | no | `.` | Build context |
 | `dockerfile` | no | `Dockerfile` | Path to the Dockerfile |
 | `quiet` | no | `true` | Should docker commands be passed `--quiet` |
+| `parallel` | no | `false` | Should stages be built in parallel (via BuildX) |
 | `build-args` | no | | Comma-separated list of `--build-arg` flags. |
 
+### Parallel builds
+The new `parallel` option, added in `v1.7`, defaults to off.
+In the next major version (v2), it will default to on.
+
+Changing to the opposite build mode, either implicitly or explicitly, *will break your layer cache for the first build*.
+The internal image formats are incompatible, and are tagged accordingly to avoid conflicts.
+This is a Docker limitation at this time.
+Please note that all images not produced in `outputs` (see below) are considered internal implementation details, subject to change, and **should never be deployed**.
+
+The current parallel build implementation uses `docker buildx` with very specific `--cache-from` flags to encourage layer reuse.
+Note that this is considered an internal implementation detail, and is subject to change during a minor and/or point release.
+However such a change is unlikely and will be documented.
+
+If you have explicly set `DOCKER_BUILDKIT=1` or `DOCKER_BUILDKIT=0`, it will override the input setting.
+Use of this is **not recommended**.
+
 ## Outputs
 
 | Output | Description |
@@ -106,7 +123,4 @@ tl:dr: If it comes from one of the `outputs` of this action, go ahead and use it
 
 ## Known issues/Future features
 
-- Use with Docker Buildkit (via `DOCKER_BUILDKIT=1`) does not consistently use the layer caches.
-  This seems to be a Buildkit issue.
-  It's recommended to leave Buildkit disabled at this time.
 - Make a straightforward mechanism to do cleanup

action.yml

@@ -17,6 +17,10 @@ inputs:
     description: Path to Dockerfile
     required: false
     default: ''
+  parallel:
+    description: Build in parallel
+    required: false
+    default: false
   repository:
     required: true
     description: Repository that all of the images and tags will pull from and push to

examples/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.13 AS env
+FROM alpine:3.13@sha256:469b6e04ee185740477efa44ed5bdd64a07bbdd6c7e5f5d169e540889597b911 AS env
 WORKDIR app
 RUN apk add --update --no-cache icu-dev
 

src/helpers.ts

@@ -4,10 +4,22 @@ import * as github from '@actions/github'
 
 // Returns a string like "refs_pull_1_merge-bk1"
 export function getTagForRun(): string {
-  const usingBuildkit = process.env.DOCKER_BUILDKIT === '1'
+  const parallel = shouldBuildInParallel()
   const tagFriendlyRef = process.env.GITHUB_REF?.replace(/\//g, '_') as unknown as string
 
-  return `${tagFriendlyRef}-bk${usingBuildkit ? '1' : '0'}`
+  return `${tagFriendlyRef}-bk${parallel ? '1' : '0'}`
+}
+
+export function shouldBuildInParallel(): boolean {
+  // Respect DOCKER_BUILDKIT, if set.
+  if (process.env.DOCKER_BUILDKIT === '1') {
+    core.debug('Building in parallel due to DOCKER_BUILDKIT=1')
+    return true
+  } else if (process.env.DOCKER_BUILDKIT === '0') {
+    core.debug('Not building in parallel due to DOCKER_BUILDKIT=0')
+    return false
+  }
+  return core.getBooleanInput('parallel')
 }
 
 export function isDefaultBranch(): boolean {
@@ -94,10 +106,14 @@ interface ExecResult {
  */
 export async function runDockerCommand(command: DockerCommand, ...args: string[]): Promise<ExecResult> {
   const rest: string[] = [command]
+  if (command === 'build' && shouldBuildInParallel()) {
+    rest.unshift('buildx')
+  }
   if (core.getBooleanInput('quiet') && command !== 'tag') {
     rest.push('--quiet')
   }
   rest.push(...args)
+  core.info(JSON.stringify(rest))
 
   let stdout = ''
   let stderr = ''

src/index.ts

@@ -9,6 +9,7 @@ import {
   getAllStages,
   getTaggedImageForStage,
   runDockerCommand,
+  shouldBuildInParallel,
   time,
 } from './helpers'
 
@@ -97,20 +98,27 @@ async function buildStage(stage: string, extraTags: string[]): Promise<string> {
   return time(`Build ${stage}`, async () => {
     core.startGroup(`Building stage: ${stage}`)
 
+    const useBuildx = shouldBuildInParallel()
+
     const dockerfile = core.getInput('dockerfile')
     const dockerfileArg = (dockerfile === '') ? [] : ['--file', dockerfile]
 
     const targetTag = getTaggedImageForStage(stage, getTagForRun())
 
     const cacheFromArg = getAllPossibleCacheTargets()
-      .flatMap(target => ['--cache-from', target])
+      .flatMap(target => ['--cache-from', useBuildx
+        ? `type=registry,ref=${target}`
+        : target
+      ])
 
     const buildArgs = getBuildArgs()
       .flatMap(arg => ['--build-arg', arg])
+    if (useBuildx) {
+      buildArgs.push('--build-arg', 'BUILDKIT_INLINE_CACHE=1')
+    }
 
     const result = await runDockerCommand(
       'build',
-      // '--build-arg', 'BUILDKIT_INLINE_CACHE="1"',
       ...buildArgs,
       ...cacheFromArg,
       ...dockerfileArg,