From f7385133e030662a6ae71daca6ef56b9f98428b4 Mon Sep 17 00:00:00 2001 From: Marco Castignoli Date: Wed, 17 Dec 2025 17:35:48 +0100 Subject: [PATCH 1/8] Speed up CI builds with ccache --- .circleci/config.yml | 39 ++++++++++++++++++- .circleci/osx_install_dependencies.sh | 1 + .../buildpack-deps/Dockerfile.emscripten | 1 + .../Dockerfile.ubuntu.clang.ossfuzz | 1 + .../buildpack-deps/Dockerfile.ubuntu2404 | 1 + .../buildpack-deps/Dockerfile.ubuntu2404.arm | 1 + .../Dockerfile.ubuntu2404.clang | 1 + 7 files changed, 43 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3cab7fbadb58..bbeb37f70ff4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -208,9 +208,44 @@ commands: run_build: steps: + # TODO: remove the line below + # Before merging this PR we need to use an alternative develop branch to test the functionality, `develop-cache-test` + + # On `develop-cache-test` we intentionally do NOT restore any ccache, so every run starts from scratch. + # Other branches restore the most recent cache produced on `develop-cache-test`. + - when: + condition: + not: + equal: ["develop-cache-test", << pipeline.git.branch >>] + steps: + - restore_cache: + keys: + # Reuse cache produced on develop-cache-test (prefix match; restores the most recent). + - v1-ccache-{{ arch }}-develop-cache-test- + - v1-ccache-{{ arch }}- - run: name: Build - command: scripts/ci/build.sh + command: | + export CCACHE_DIR="$HOME/.ccache" + export CCACHE_BASEDIR="$(pwd)" + export CCACHE_NOHASHDIR=1 + mkdir -p "$CCACHE_DIR" + export CMAKE_OPTIONS="${CMAKE_OPTIONS:-} -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" + ccache -z || true + + scripts/ci/build.sh + + ccache -s || true + + # On `develop-cache-test` we always generate a fresh cache by never restoring and by saving under a unique key. + - when: + condition: + equal: ["develop-cache-test", << pipeline.git.branch >>] + steps: + - save_cache: + key: v1-ccache-{{ arch }}-develop-cache-test-{{ .Revision }} + paths: + - ~/.ccache run_build_ossfuzz: steps: @@ -1197,7 +1232,7 @@ jobs: - run: name: Install build dependencies command: | - pacman --noconfirm -Syu --noprogressbar --needed base-devel boost cmake git openssh tar + pacman --noconfirm -Syu --noprogressbar --needed base-devel boost cmake ccache git openssh tar - checkout - run_build - store_artifacts_solc diff --git a/.circleci/osx_install_dependencies.sh b/.circleci/osx_install_dependencies.sh index dd5a8397b168..0666211b76b3 100755 --- a/.circleci/osx_install_dependencies.sh +++ b/.circleci/osx_install_dependencies.sh @@ -51,6 +51,7 @@ then brew update brew upgrade brew install cmake + brew install ccache brew install wget brew install coreutils brew install diffutils diff --git a/scripts/docker/buildpack-deps/Dockerfile.emscripten b/scripts/docker/buildpack-deps/Dockerfile.emscripten index 007729d4113a..d257e7425c0d 100644 --- a/scripts/docker/buildpack-deps/Dockerfile.emscripten +++ b/scripts/docker/buildpack-deps/Dockerfile.emscripten @@ -41,6 +41,7 @@ RUN <<-EOF apt-get update apt-get install -qqy --no-install-recommends \ lsof \ + ccache \ lz4 \ python3 \ python3-pip \ diff --git a/scripts/docker/buildpack-deps/Dockerfile.ubuntu.clang.ossfuzz b/scripts/docker/buildpack-deps/Dockerfile.ubuntu.clang.ossfuzz index 5a96e6889dd0..0536d98fa5e7 100644 --- a/scripts/docker/buildpack-deps/Dockerfile.ubuntu.clang.ossfuzz +++ b/scripts/docker/buildpack-deps/Dockerfile.ubuntu.clang.ossfuzz @@ -31,6 +31,7 @@ RUN apt-get update; \ automake \ bison \ build-essential \ + ccache \ curl \ git \ jq \ diff --git a/scripts/docker/buildpack-deps/Dockerfile.ubuntu2404 b/scripts/docker/buildpack-deps/Dockerfile.ubuntu2404 index 9cf381f760ef..e605af7aa267 100644 --- a/scripts/docker/buildpack-deps/Dockerfile.ubuntu2404 +++ b/scripts/docker/buildpack-deps/Dockerfile.ubuntu2404 @@ -39,6 +39,7 @@ RUN set -ex; \ apt-get install -qqy --no-install-recommends \ build-essential \ cmake \ + ccache \ jq \ libboost-filesystem-dev \ libboost-program-options-dev \ diff --git a/scripts/docker/buildpack-deps/Dockerfile.ubuntu2404.arm b/scripts/docker/buildpack-deps/Dockerfile.ubuntu2404.arm index 663928220a0e..18cf787d455f 100644 --- a/scripts/docker/buildpack-deps/Dockerfile.ubuntu2404.arm +++ b/scripts/docker/buildpack-deps/Dockerfile.ubuntu2404.arm @@ -40,6 +40,7 @@ RUN <<-EOF apt-get install --quiet=2 --no-install-recommends \ build-essential \ cmake \ + ccache \ jq \ libboost-filesystem-dev \ libboost-program-options-dev \ diff --git a/scripts/docker/buildpack-deps/Dockerfile.ubuntu2404.clang b/scripts/docker/buildpack-deps/Dockerfile.ubuntu2404.clang index aec6fe6e26dc..888219ec96a0 100644 --- a/scripts/docker/buildpack-deps/Dockerfile.ubuntu2404.clang +++ b/scripts/docker/buildpack-deps/Dockerfile.ubuntu2404.clang @@ -40,6 +40,7 @@ RUN set -ex; \ build-essential \ clang \ cmake \ + ccache \ jq \ libboost-filesystem-dev \ libboost-program-options-dev \ From 51177ce1230ddc7ce551156147b5b7c197717d80 Mon Sep 17 00:00:00 2001 From: Marco Castignoli Date: Mon, 22 Dec 2025 08:47:49 +0100 Subject: [PATCH 2/8] Address PR comments - improve comments and warnings - treat ccache as always existing - add CCACHE_KEY to easily invalidate cache - skip cache also on "breaking" branch --- .circleci/config.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bbeb37f70ff4..2ad4488d9bd5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -211,18 +211,20 @@ commands: # TODO: remove the line below # Before merging this PR we need to use an alternative develop branch to test the functionality, `develop-cache-test` - # On `develop-cache-test` we intentionally do NOT restore any ccache, so every run starts from scratch. + # On `develop-cache-test` and `breaking` we intentionally do NOT restore any ccache, so every run starts from scratch. # Other branches restore the most recent cache produced on `develop-cache-test`. - when: condition: not: - equal: ["develop-cache-test", << pipeline.git.branch >>] + or: + - equal: ["develop-cache-test", << pipeline.git.branch >>] + - equal: ["breaking", << pipeline.git.branch >>] steps: - restore_cache: keys: # Reuse cache produced on develop-cache-test (prefix match; restores the most recent). - v1-ccache-{{ arch }}-develop-cache-test- - - v1-ccache-{{ arch }}- + # WARNING! If you edit anything between here and save_cache, remember to bump version or invalidate the cache manually. - run: name: Build command: | @@ -231,11 +233,11 @@ commands: export CCACHE_NOHASHDIR=1 mkdir -p "$CCACHE_DIR" export CMAKE_OPTIONS="${CMAKE_OPTIONS:-} -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" - ccache -z || true + ccache -z scripts/ci/build.sh - ccache -s || true + ccache -s # On `develop-cache-test` we always generate a fresh cache by never restoring and by saving under a unique key. - when: @@ -243,7 +245,7 @@ commands: equal: ["develop-cache-test", << pipeline.git.branch >>] steps: - save_cache: - key: v1-ccache-{{ arch }}-develop-cache-test-{{ .Revision }} + key: v1-{{ .Environment.CCACHE_KEY }}-ccache-{{ arch }}-develop-cache-test-{{ .Revision }} paths: - ~/.ccache From 4df4fd0fc8eea07059db339315e064e6370fcddc Mon Sep 17 00:00:00 2001 From: Marco Castignoli Date: Mon, 22 Dec 2025 08:52:36 +0100 Subject: [PATCH 3/8] Bumo versions of buildpack-deps dockerfiles --- scripts/docker/buildpack-deps/Dockerfile.emscripten | 2 +- scripts/docker/buildpack-deps/Dockerfile.ubuntu.clang.ossfuzz | 2 +- scripts/docker/buildpack-deps/Dockerfile.ubuntu2404 | 2 +- scripts/docker/buildpack-deps/Dockerfile.ubuntu2404.arm | 2 +- scripts/docker/buildpack-deps/Dockerfile.ubuntu2404.clang | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/docker/buildpack-deps/Dockerfile.emscripten b/scripts/docker/buildpack-deps/Dockerfile.emscripten index d257e7425c0d..67c9ec61a162 100644 --- a/scripts/docker/buildpack-deps/Dockerfile.emscripten +++ b/scripts/docker/buildpack-deps/Dockerfile.emscripten @@ -33,7 +33,7 @@ # Using $(em-config CACHE)/sysroot/usr seems to work, though, and still has cmake find the # dependencies automatically. FROM emscripten/emsdk:3.1.19 AS base -LABEL version="21" +LABEL version="22" ADD emscripten.jam /usr/src RUN <<-EOF diff --git a/scripts/docker/buildpack-deps/Dockerfile.ubuntu.clang.ossfuzz b/scripts/docker/buildpack-deps/Dockerfile.ubuntu.clang.ossfuzz index 0536d98fa5e7..08c091cd7726 100644 --- a/scripts/docker/buildpack-deps/Dockerfile.ubuntu.clang.ossfuzz +++ b/scripts/docker/buildpack-deps/Dockerfile.ubuntu.clang.ossfuzz @@ -22,7 +22,7 @@ # (c) 2016-2021 solidity contributors. #------------------------------------------------------------------------------ FROM gcr.io/oss-fuzz-base/base-builder AS base -LABEL version="12" +LABEL version="13" ARG DEBIAN_FRONTEND=noninteractive diff --git a/scripts/docker/buildpack-deps/Dockerfile.ubuntu2404 b/scripts/docker/buildpack-deps/Dockerfile.ubuntu2404 index e605af7aa267..8f5c1aec8426 100644 --- a/scripts/docker/buildpack-deps/Dockerfile.ubuntu2404 +++ b/scripts/docker/buildpack-deps/Dockerfile.ubuntu2404 @@ -22,7 +22,7 @@ # (c) 2016-2024 solidity contributors. #------------------------------------------------------------------------------ FROM buildpack-deps:noble AS base -LABEL version="6" +LABEL version="7" ARG DEBIAN_FRONTEND=noninteractive diff --git a/scripts/docker/buildpack-deps/Dockerfile.ubuntu2404.arm b/scripts/docker/buildpack-deps/Dockerfile.ubuntu2404.arm index 18cf787d455f..cc58ed67963e 100644 --- a/scripts/docker/buildpack-deps/Dockerfile.ubuntu2404.arm +++ b/scripts/docker/buildpack-deps/Dockerfile.ubuntu2404.arm @@ -22,7 +22,7 @@ # (c) 2016-2025 solidity contributors. #------------------------------------------------------------------------------ FROM --platform=linux/arm64 buildpack-deps:noble AS base -LABEL version="2" +LABEL version="3" ARG DEBIAN_FRONTEND=noninteractive diff --git a/scripts/docker/buildpack-deps/Dockerfile.ubuntu2404.clang b/scripts/docker/buildpack-deps/Dockerfile.ubuntu2404.clang index 888219ec96a0..9a9928514088 100644 --- a/scripts/docker/buildpack-deps/Dockerfile.ubuntu2404.clang +++ b/scripts/docker/buildpack-deps/Dockerfile.ubuntu2404.clang @@ -22,7 +22,7 @@ # (c) 2016-2024 solidity contributors. #------------------------------------------------------------------------------ FROM buildpack-deps:noble AS base -LABEL version="7" +LABEL version="8" ARG DEBIAN_FRONTEND=noninteractive From 456a48c610f2fbd080327c638c2dbd66b474a06f Mon Sep 17 00:00:00 2001 From: Marco Castignoli Date: Tue, 23 Dec 2025 09:42:54 +0100 Subject: [PATCH 4/8] Skip ccache build when pushing tags --- .circleci/config.yml | 48 +++++++++++++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2ad4488d9bd5..45b68f05454d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -212,32 +212,48 @@ commands: # Before merging this PR we need to use an alternative develop branch to test the functionality, `develop-cache-test` # On `develop-cache-test` and `breaking` we intentionally do NOT restore any ccache, so every run starts from scratch. - # Other branches restore the most recent cache produced on `develop-cache-test`. + # Tags always skip ccache. Other branches restore the most recent cache produced on `develop-cache-test`. - when: condition: - not: - or: - - equal: ["develop-cache-test", << pipeline.git.branch >>] - - equal: ["breaking", << pipeline.git.branch >>] + and: + - equal: ["", << pipeline.git.tag >>] + - not: + or: + - equal: ["develop-cache-test", << pipeline.git.branch >>] + - equal: ["breaking", << pipeline.git.branch >>] steps: - restore_cache: keys: # Reuse cache produced on develop-cache-test (prefix match; restores the most recent). - v1-ccache-{{ arch }}-develop-cache-test- # WARNING! If you edit anything between here and save_cache, remember to bump version or invalidate the cache manually. - - run: - name: Build - command: | - export CCACHE_DIR="$HOME/.ccache" - export CCACHE_BASEDIR="$(pwd)" - export CCACHE_NOHASHDIR=1 - mkdir -p "$CCACHE_DIR" - export CMAKE_OPTIONS="${CMAKE_OPTIONS:-} -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" - ccache -z + - when: + condition: + and: + - equal: ["", << pipeline.git.tag >>] + steps: + - run: + name: Build (ccache) + command: | + export CCACHE_DIR="$HOME/.ccache" + export CCACHE_BASEDIR="$(pwd)" + export CCACHE_NOHASHDIR=1 + mkdir -p "$CCACHE_DIR" + export CMAKE_OPTIONS="${CMAKE_OPTIONS:-} -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" + ccache -z - scripts/ci/build.sh + scripts/ci/build.sh - ccache -s + ccache -s + - when: + condition: + or: + - not: + equal: ["", << pipeline.git.tag >>] + steps: + - run: + name: Build (no ccache) + command: scripts/ci/build.sh # On `develop-cache-test` we always generate a fresh cache by never restoring and by saving under a unique key. - when: From 321465f65f5a1d978968178e63a2e8f453a2513a Mon Sep 17 00:00:00 2001 From: Marco Castignoli Date: Tue, 23 Dec 2025 10:17:31 +0100 Subject: [PATCH 5/8] Fix ccache permission error while testing buildpack-deps --- scripts/ci/docker_upgrade.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/ci/docker_upgrade.sh b/scripts/ci/docker_upgrade.sh index c2077239a5b6..533982a3759a 100755 --- a/scripts/ci/docker_upgrade.sh +++ b/scripts/ci/docker_upgrade.sh @@ -55,6 +55,7 @@ docker run \ --rm \ --volume "${PWD}:/project" \ -u "$(id -u "${USER}"):$(id -g "${USER}")" \ + -e CCACHE_DIR=/tmp/ccache \ "${IMAGE_NAME}" \ bash -c "/project/scripts/ci/${IMAGE_NAME}_test_${IMAGE_VARIANT}.sh" From 25d5d8b93237f10197a8e761a26858fdb11ec88c Mon Sep 17 00:00:00 2001 From: Marco Castignoli Date: Tue, 23 Dec 2025 12:31:22 +0100 Subject: [PATCH 6/8] Update docker images to use version with ccache --- .circleci/config.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 45b68f05454d..863160d4c414 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -13,25 +13,25 @@ parameters: default: "ghcr.io/argotorg/solidity-buildpack-deps@sha256:1f387a77be889f65a2a25986a5c5eccc88cec23fabe6aeaf351790751145c81e" ubuntu-2404-docker-image: type: string - # ghcr.io/argotorg/solidity-buildpack-deps:ubuntu2404-6 - default: "ghcr.io/argotorg/solidity-buildpack-deps@sha256:c0412c53e59ce0c96bde4c08e7332ea12e4cadba9bbac829621947897fa21272" + # ghcr.io/argotorg/solidity-buildpack-deps:ubuntu2404-7 + default: "ghcr.io/argotorg/solidity-buildpack-deps@sha256:bf3c7dac5f613d220757d9e32ee6855648b98bdb460a984ade214f8fe6920d63" ubuntu-2404-arm-docker-image: type: string - # ghcr.io/argotorg/solidity-buildpack-deps:ubuntu2404.arm-2 - default: "ghcr.io/argotorg/solidity-buildpack-deps@sha256:27b03c1c4688e5d69b10e0539460346a3dee3b10b0e04fb406b9707c6f0dd95e" + # ghcr.io/argotorg/solidity-buildpack-deps:ubuntu2404.arm-3 + default: "ghcr.io/argotorg/solidity-buildpack-deps@sha256:c4f7da98bc922e8ff2e58d5cfc0216b698ca1ee3ed84f1a0561bbf3641905b4a" ubuntu-2404-clang-docker-image: type: string - # ghcr.io/argotorg/solidity-buildpack-deps:ubuntu2404.clang-7 - default: "ghcr.io/argotorg/solidity-buildpack-deps@sha256:7466ed23590cf14e60da884894723bcdab064742f017dcfcce015999b4f9143b" + # ghcr.io/argotorg/solidity-buildpack-deps:ubuntu2404.clang-8 + default: "ghcr.io/argotorg/solidity-buildpack-deps@sha256:4c936d31b0eaebb770ea72e6fb8af002dfbdb920604858afa0c8cd055aa49664" ubuntu-clang-ossfuzz-docker-image: type: string - # ghcr.io/argotorg/solidity-buildpack-deps:ubuntu.clang.ossfuzz-12 - default: "ghcr.io/argotorg/solidity-buildpack-deps@sha256:c7088af5082bf764244a6251ecf3dd29d280d2cd2cd071f011f7f32e0326f426" + # ghcr.io/argotorg/solidity-buildpack-deps:ubuntu.clang.ossfuzz-13 + default: "ghcr.io/argotorg/solidity-buildpack-deps@sha256:ed197b3d083aa05526ec40d6d3d1f2bc3e8856bd23c1a7037becb4d3a15b3529" emscripten-docker-image: type: string # NOTE: Please remember to update the `scripts/build_emscripten.sh` whenever the hash of this image changes. - # ghcr.io/argotorg/solidity-buildpack-deps:emscripten-21 - default: "ghcr.io/argotorg/solidity-buildpack-deps@sha256:fc53d68a4680ffa7d5f70164e13a903478964f15bcc07434d74833a05f4fbc19" + # ghcr.io/argotorg/solidity-buildpack-deps:emscripten-22 + default: "ghcr.io/argotorg/solidity-buildpack-deps@sha256:e7ca70baa048d4307124f22670f38db4205d57689661feb8e92ea50a31d8fdbe" evm-version: type: string default: osaka From b50f7d43ce45d329fca1e2d2e1767324c9875e2a Mon Sep 17 00:00:00 2001 From: Marco Castignoli Date: Tue, 23 Dec 2025 13:05:52 +0100 Subject: [PATCH 7/8] Fix cache key in restore_cache --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 863160d4c414..19b82081692d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -225,7 +225,7 @@ commands: - restore_cache: keys: # Reuse cache produced on develop-cache-test (prefix match; restores the most recent). - - v1-ccache-{{ arch }}-develop-cache-test- + - v1-{{ .Environment.CCACHE_KEY }}-ccache-{{ arch }}-develop-cache-test- # WARNING! If you edit anything between here and save_cache, remember to bump version or invalidate the cache manually. - when: condition: From f55d65e1cd6f9d6c6e832ae4a8b5b4e966a9dccc Mon Sep 17 00:00:00 2001 From: Marco Castignoli Date: Tue, 23 Dec 2025 14:49:47 +0100 Subject: [PATCH 8/8] This commit is to test if ccache is being used --- test/libsolidity/SolidityParser.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/test/libsolidity/SolidityParser.cpp b/test/libsolidity/SolidityParser.cpp index a0fd1fe94bec..f1320629e863 100644 --- a/test/libsolidity/SolidityParser.cpp +++ b/test/libsolidity/SolidityParser.cpp @@ -136,6 +136,7 @@ BOOST_AUTO_TEST_CASE(function_natspec_documentation) } )"; BOOST_CHECK(successParse(text)); + BOOST_CHECK(successParse(text)); ErrorList errors; ASTPointer contract = parseText(text, errors); FunctionDefinition const* function = nullptr;