Skip to content

Commit

Permalink
Try adding more input caches to maybe avoid double build of build target
Browse files Browse the repository at this point in the history
  • Loading branch information
@adamnovak
adamnovak committed Sep 6, 2023
1 parent 8d4c748 commit 56877a1
Showing 1 changed file with 22 additions and 22 deletions.
44 changes: 22 additions & 22 deletions .gitlab-ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,20 +129,20 @@ build-job:
# Note that A LOCAL CACHE CAN ONLY HOLD ONE TAG/TARGET AT A TIME!
# And Quay can't use mode=max registry caching to cache the intermediate targets with the final image, just inline caching.
# So we have to do the Complicated Cache Shuffle.
# Build base image from branch and mainline base caches to local cache
- docker buildx build --cache-from type=registry,ref=${CI_REPO}:${MAINLINE_CACHE_TAG}-base --cache-from type=registry,ref=${CI_REPO}:${CACHE_TAG}-base --cache-to type=local,dest=${HOME}/docker-cache --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target base -t ${CI_REPO}:${CACHE_TAG}-base -f Dockerfile .
# Push base image from local cache to registry cache for branch.
- docker buildx build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target base -t ${CI_REPO}:${CACHE_TAG}-base -f Dockerfile --push .
# Build build image from local base cache and branch and mainline build caches to local cache
- docker buildx build --cache-from type=registry,ref=${CI_REPO}:${MAINLINE_CACHE_TAG}-build --cache-from type=registry,ref=${CI_REPO}:${CACHE_TAG}-build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=local,dest=${HOME}/docker-cache --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target build -t ${CI_REPO}:${CACHE_TAG}-build -f Dockerfile .
# Push build image to registry cache for branch
- docker buildx build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target build -t ${CI_REPO}:${CACHE_TAG}-build -f Dockerfile --push .
# Build run image from local build cache and branch and mainline run caches to local cache
- docker buildx build --cache-from type=registry,ref=${CI_REPO}:${MAINLINE_CACHE_TAG}-run --cache-from type=registry,ref=${CI_REPO}:${CACHE_TAG}-run --cache-from type=local,src=${HOME}/docker-cache --cache-to type=local,dest=${HOME}/docker-cache --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target run -t ${CI_REPO}:${CACHE_TAG}-run -f Dockerfile .
# Push run image to registry cache for branch
- docker buildx build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target run -t ${CI_REPO}:${CACHE_TAG}-run -f Dockerfile --push .
# Finally, push run image to where we actually want it.
- docker buildx build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target run -t ${CI_REPO}:${DOCKER_TAG} -f Dockerfile --push .
- echo " Build base image from branch and mainline base caches to local cache"
- docker buildx build --cache-from type=registry,ref=${CI_REPO}:${MAINLINE_CACHE_TAG}-base --cache-from type=registry,ref=${CI_REPO}:${CACHE_TAG}-base --cache-from type=local,src=${HOME}/docker-cache/base --cache-to type=local,dest=${HOME}/docker-cache/base --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target base -t ${CI_REPO}:${CACHE_TAG}-base -f Dockerfile .
- echo "Push base image from local cache to registry cache for branch."
- docker buildx build --cache-from type=local,src=${HOME}/docker-cache/base --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target base -t ${CI_REPO}:${CACHE_TAG}-base -f Dockerfile --push .
- echo "Build build image from local base cache and branch and mainline build caches to local cache"
- docker buildx build --cache-from type=registry,ref=${CI_REPO}:${MAINLINE_CACHE_TAG}-build --cache-from type=registry,ref=${CI_REPO}:${CACHE_TAG}-build --cache-from type=local,src=${HOME}/docker-cache/base --cache-from type=local,src=${HOME}/docker-cache/build --cache-to type=local,dest=${HOME}/docker-cache/build --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target build -t ${CI_REPO}:${CACHE_TAG}-build -f Dockerfile .
- echo "Push build image to registry cache for branch"
- docker buildx build --cache-from type=local,src=${HOME}/docker-cache/base --cache-from type=local,src=${HOME}/docker-cache/build --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target build -t ${CI_REPO}:${CACHE_TAG}-build -f Dockerfile --push .
- echo "Build run image from local build cache and branch and mainline run caches to local cache"
- docker buildx build --cache-from type=registry,ref=${CI_REPO}:${MAINLINE_CACHE_TAG}-run --cache-from type=registry,ref=${CI_REPO}:${CACHE_TAG}-run --cache-from type=local,src=${HOME}/docker-cache/base --cache-from type=local,src=${HOME}/docker-cache/build --cache-from type=local,src=${HOME}/docker-cache/run --cache-to type=local,dest=${HOME}/docker-cache/run --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target run -t ${CI_REPO}:${CACHE_TAG}-run -f Dockerfile .
- echo "Push run image to registry cache for branch"
- docker buildx build --cache-from type=local,src=${HOME}/docker-cache/base --cache-from type=local,src=${HOME}/docker-cache/build --cache-from type=local,src=${HOME}/docker-cache/run --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target run -t ${CI_REPO}:${CACHE_TAG}-run -f Dockerfile --push .
- echo "Finally, push run image to where we actually want it."
- docker buildx build --cache-from type=local,src=${HOME}/docker-cache/base --cache-from type=local,src=${HOME}/docker-cache/build --cache-from type=local,src=${HOME}/docker-cache/run --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target run -t ${CI_REPO}:${DOCKER_TAG} -f Dockerfile --push .
variables:
GIT_SUBMODULE_STRATEGY: recursive

Expand Down Expand Up @@ -173,21 +173,21 @@ production-build-job:
# And Quay can't use mode=max registry caching to cache the intermediate targets with the final image, just inline caching.
# So we have to do the Complicated Cache Shuffle.
# Build base image from branch and mainline base caches to local cache
- docker buildx build --cache-from type=registry,ref=${CI_REPO}:${MAINLINE_CACHE_TAG}-base --cache-from type=registry,ref=${CI_REPO}:${CACHE_TAG}-base --cache-to type=local,dest=${HOME}/docker-cache --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target base -t ${CI_REPO}:${CACHE_TAG}-base -f Dockerfile .
- docker buildx build --cache-from type=registry,ref=${CI_REPO}:${MAINLINE_CACHE_TAG}-base --cache-from type=registry,ref=${CI_REPO}:${CACHE_TAG}-base --cache-from type=local,src=${HOME}/docker-cache/base --cache-to type=local,dest=${HOME}/docker-cache/base --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target base -t ${CI_REPO}:${CACHE_TAG}-base -f Dockerfile .
# Push base image from local cache to registry cache for branch.
- docker buildx build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target base -t ${CI_REPO}:${CACHE_TAG}-base -f Dockerfile --push .
- docker buildx build --cache-from type=local,src=${HOME}/docker-cache/base --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target base -t ${CI_REPO}:${CACHE_TAG}-base -f Dockerfile --push .
# Build build image from local base cache and branch and mainline build caches to local cache
- docker buildx build --cache-from type=registry,ref=${CI_REPO}:${MAINLINE_CACHE_TAG}-build --cache-from type=registry,ref=${CI_REPO}:${CACHE_TAG}-build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=local,dest=${HOME}/docker-cache --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target build -t ${CI_REPO}:${CACHE_TAG}-build -f Dockerfile .
- docker buildx build --cache-from type=registry,ref=${CI_REPO}:${MAINLINE_CACHE_TAG}-build --cache-from type=registry,ref=${CI_REPO}:${CACHE_TAG}-build --cache-from type=local,src=${HOME}/docker-cache/base --cache-from type=local,src=${HOME}/docker-cache/build --cache-to type=local,dest=${HOME}/docker-cache/build --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target build -t ${CI_REPO}:${CACHE_TAG}-build -f Dockerfile .
# Push build image to registry cache for branch
- docker buildx build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target build -t ${CI_REPO}:${CACHE_TAG}-build -f Dockerfile --push .
- docker buildx build --cache-from type=local,src=${HOME}/docker-cache/base --cache-from type=local,src=${HOME}/docker-cache/build --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target build -t ${CI_REPO}:${CACHE_TAG}-build -f Dockerfile --push .
# Build run image from local build cache and branch and mainline run caches to local cache
- docker buildx build --cache-from type=registry,ref=${CI_REPO}:${MAINLINE_CACHE_TAG}-run --cache-from type=registry,ref=${CI_REPO}:${CACHE_TAG}-run --cache-from type=local,src=${HOME}/docker-cache --cache-to type=local,dest=${HOME}/docker-cache --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target run -t ${CI_REPO}:${CACHE_TAG}-run -f Dockerfile .
- docker buildx build --cache-from type=registry,ref=${CI_REPO}:${MAINLINE_CACHE_TAG}-run --cache-from type=registry,ref=${CI_REPO}:${CACHE_TAG}-run --cache-from type=local,src=${HOME}/docker-cache/base --cache-from type=local,src=${HOME}/docker-cache/build --cache-from type=local,src=${HOME}/docker-cache/run --cache-to type=local,dest=${HOME}/docker-cache/run --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target run -t ${CI_REPO}:${CACHE_TAG}-run -f Dockerfile .
# Push run image to registry cache for branch
- docker buildx build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target run -t ${CI_REPO}:${CACHE_TAG}-run -f Dockerfile --push .
- docker buildx build --cache-from type=local,src=${HOME}/docker-cache/base --cache-from type=local,src=${HOME}/docker-cache/build --cache-from type=local,src=${HOME}/docker-cache/run --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target run -t ${CI_REPO}:${CACHE_TAG}-run -f Dockerfile --push .
# Finally, push run image to where we actually want it.
- docker buildx build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target run -t ${CI_REPO}:${DOCKER_TAG} -f Dockerfile --push .
- docker buildx build --cache-from type=local,src=${HOME}/docker-cache/base --cache-from type=local,src=${HOME}/docker-cache/build --cache-from type=local,src=${HOME}/docker-cache/run --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target run -t ${CI_REPO}:${DOCKER_TAG} -f Dockerfile --push .
# Tag it latest if we pushed a real release tag
- if [[ ! -z "${CI_COMMIT_TAG}" ]]; then docker buildx build --cache-from type=local,src=${HOME}/docker-cache --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target run -t ${CI_REPO}:latest -f Dockerfile --push .; fi
- if [[ ! -z "${CI_COMMIT_TAG}" ]]; then docker buildx build --cache-from type=local,src=${HOME}/docker-cache/base --cache-from type=local,src=${HOME}/docker-cache/build --cache-from type=local,src=${HOME}/docker-cache/run --cache-to type=inline --platform="${PLATFORMS}" --build-arg THREADS=${THREADS} --target run -t ${CI_REPO}:latest -f Dockerfile --push .; fi
# If we wanted to run the tests under ARM emulation, we could do:
# docker buildx build --platform=linux/arm64 --build-arg THREADS=${THREADS} --target test -f Dockerfile .
# But we don't, because they both don't actually pass yet on ARM and also
Expand Down

1 comment on commit 56877a1

@adamnovak
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

vg CI tests complete for branch arm-ci-build. View the full report here.

11 tests passed, 5 tests failed and 0 tests skipped in 16992 seconds

Failed tests:

  • test_full_brca2_cactus (124 seconds)
  • test_full_brca2_primary (118 seconds)
  • test_full_brca2_snp1kg (136 seconds)
  • test_map_brca1_cactus (110 seconds)
  • test_sim_mhc_cactus (159 seconds)

Please sign in to comment.