diff --git a/.circleci/config.yml b/.circleci/config.yml index 2276350e0469..1b6a309fc1da 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -13,25 +13,25 @@ parameters: default: "ghcr.io/argotorg/solidity-buildpack-deps@sha256:1f387a77be889f65a2a25986a5c5eccc88cec23fabe6aeaf351790751145c81e" ubuntu-2404-docker-image: type: string - # ghcr.io/argotorg/solidity-buildpack-deps:ubuntu2404-6 - default: "ghcr.io/argotorg/solidity-buildpack-deps@sha256:c0412c53e59ce0c96bde4c08e7332ea12e4cadba9bbac829621947897fa21272" + # ghcr.io/argotorg/solidity-buildpack-deps:ubuntu2404-7 + default: "ghcr.io/argotorg/solidity-buildpack-deps@sha256:e52bd8fbb38908f203c6866e39562548faf26e5b7a5174f9860b44aae2b8f0cd" ubuntu-2404-arm-docker-image: type: string - # ghcr.io/argotorg/solidity-buildpack-deps:ubuntu2404.arm-2 - default: "ghcr.io/argotorg/solidity-buildpack-deps@sha256:27b03c1c4688e5d69b10e0539460346a3dee3b10b0e04fb406b9707c6f0dd95e" + # ghcr.io/argotorg/solidity-buildpack-deps:ubuntu2404.arm-3 + default: "ghcr.io/argotorg/solidity-buildpack-deps@sha256:1e6dbf9a9173f2645449281a1e11918a95ca6dc04e59aa9d70824cffc44697ed" ubuntu-2404-clang-docker-image: type: string - # ghcr.io/argotorg/solidity-buildpack-deps:ubuntu2404.clang-7 - default: "ghcr.io/argotorg/solidity-buildpack-deps@sha256:7466ed23590cf14e60da884894723bcdab064742f017dcfcce015999b4f9143b" + # ghcr.io/argotorg/solidity-buildpack-deps:ubuntu2404.clang-8 + default: "ghcr.io/argotorg/solidity-buildpack-deps@sha256:e283c8ec6a59c76565c89b856d8019af2fa5beb96f27d3064fc4dda42bf3524d" ubuntu-clang-ossfuzz-docker-image: type: string - # ghcr.io/argotorg/solidity-buildpack-deps:ubuntu.clang.ossfuzz-12 - default: "ghcr.io/argotorg/solidity-buildpack-deps@sha256:c7088af5082bf764244a6251ecf3dd29d280d2cd2cd071f011f7f32e0326f426" + # ghcr.io/argotorg/solidity-buildpack-deps:ubuntu.clang.ossfuzz-13 + default: "ghcr.io/argotorg/solidity-buildpack-deps@sha256:2acb5d6f254ece25a96096f26960b5fe114eb86ceca92312cd20d59c096bf6e4" emscripten-docker-image: type: string # NOTE: Please remember to update the `scripts/build_emscripten.sh` whenever the hash of this image changes. - # ghcr.io/argotorg/solidity-buildpack-deps:emscripten-21 - default: "ghcr.io/argotorg/solidity-buildpack-deps@sha256:fc53d68a4680ffa7d5f70164e13a903478964f15bcc07434d74833a05f4fbc19" + # ghcr.io/argotorg/solidity-buildpack-deps:emscripten-22 + default: "ghcr.io/argotorg/solidity-buildpack-deps@sha256:b9c953144d82cca5553f65626bc43b8af456dbc8966f8732ff9522f33ca8d722" evm-version: type: string default: osaka @@ -75,6 +75,55 @@ commands: event: release condition: on_success + ensure_ccache_key: + description: "Fail fast when CCACHE_KEY is missing." + steps: + - run: + name: Check CCACHE_KEY + command: | + if [ -z "${CCACHE_KEY:-}" ]; then + echo "CCACHE_KEY is not set. Please set it in CircleCI environment settings." + exit 1 + fi + + restore_ccache_if_allowed: + description: "Restore ccache unless on a tag or a skipped branch." + parameters: + # TODO: remove the line below + # Before merging this PR we need to use an alternative develop branch to test the functionality, `ci-ccache` + skip_branches_regex: + description: "Regex of branches that should skip restoring the cache." + type: string + default: "^(ci-ccache|breaking)$" + steps: + # Tags always skip ccache. Selected branches intentionally start from scratch. + - when: + condition: + and: + - equal: ["", << pipeline.git.tag >>] + - not: + matches: + pattern: "<< parameters.skip_branches_regex >>" + value: << pipeline.git.branch >> + steps: + - restore_cache: + keys: + # Reuse cache produced on ci-ccache (prefix match; restores the most recent). + - v1-{{ .Environment.CCACHE_KEY }}-ccache-{{ .Environment.CIRCLE_JOB }}-ci-ccache- + + save_ccache_if_develop: + description: "Save a fresh ccache only on the ci-ccache branch." + steps: + # On `ci-ccache` we always generate a fresh cache by never restoring and by saving under a unique key. + - when: + condition: + equal: ["ci-ccache", << pipeline.git.branch >>] + steps: + - save_cache: + key: v1-{{ .Environment.CCACHE_KEY }}-ccache-{{ .Environment.CIRCLE_JOB }}-ci-ccache-{{ .Revision }} + paths: + - ~/.ccache + prepare_bytecode_report: description: "Generate bytecode report and upload it as an artifact." parameters: @@ -208,15 +257,61 @@ commands: run_build: steps: - - run: - name: Build - command: scripts/ci/build.sh + - when: + condition: + equal: ["", << pipeline.git.tag >>] + steps: + - ensure_ccache_key + - restore_ccache_if_allowed + # WARNING! If you edit anything between here and save_cache, remember to bump the CCACHE_KEY version in CircleCI environment settings + - when: + condition: + and: + - equal: ["", << pipeline.git.tag >>] + steps: + - run: + name: Build (ccache) + command: CCACHE_ENABLED=${CCACHE_ENABLED:-1} scripts/ci/build.sh + - when: + condition: + or: + - not: + equal: ["", << pipeline.git.tag >>] + steps: + - run: + name: Build (no ccache) + command: scripts/ci/build.sh + + - save_ccache_if_develop run_build_ossfuzz: steps: - - run: - name: Build_ossfuzz - command: scripts/ci/build_ossfuzz.sh + - when: + condition: + equal: ["", << pipeline.git.tag >>] + steps: + - ensure_ccache_key + - restore_ccache_if_allowed + # WARNING! If you edit anything between here and save_cache, remember to bump the CCACHE_KEY version in CircleCI environment settings + - when: + condition: + and: + - equal: ["", << pipeline.git.tag >>] + steps: + - run: + name: Build_ossfuzz (ccache) + command: CCACHE_ENABLED=${CCACHE_ENABLED:-1} scripts/ci/build_ossfuzz.sh + - when: + condition: + or: + - not: + equal: ["", << pipeline.git.tag >>] + steps: + - run: + name: Build_ossfuzz (no ccache) + command: scripts/ci/build_ossfuzz.sh + + - save_ccache_if_develop run_proofs: steps: @@ -1197,7 +1292,7 @@ jobs: - run: name: Install build dependencies command: | - pacman --noconfirm -Syu --noprogressbar --needed base-devel boost cmake git openssh tar + pacman --noconfirm -Syu --noprogressbar --needed base-devel boost cmake ccache git openssh tar - checkout - run_build - store_artifacts_solc @@ -1257,10 +1352,33 @@ jobs: MAKEFLAGS: -j 10 steps: - checkout - - run: - name: Build - command: | - scripts/ci/build_emscripten.sh + - when: + condition: + equal: ["", << pipeline.git.tag >>] + steps: + - ensure_ccache_key + - restore_ccache_if_allowed + # WARNING! If you edit anything between here and save_cache, remember to bump the CCACHE_KEY version in CircleCI environment settings + - when: + condition: + and: + - equal: ["", << pipeline.git.tag >>] + steps: + - run: + name: Build (ccache) + command: CCACHE_ENABLED=${CCACHE_ENABLED:-1} scripts/ci/build_emscripten.sh + - when: + condition: + or: + - not: + equal: ["", << pipeline.git.tag >>] + steps: + - run: + name: Build (no ccache) + command: | + scripts/ci/build_emscripten.sh + + - save_ccache_if_develop - store_artifacts: path: upload/soljson.js destination: soljson.js @@ -1737,9 +1855,39 @@ jobs: key: dependencies-win-{{ arch }}-{{ checksum "scripts/install_deps.ps1" }} paths: - .\deps - - run: - name: "Building solidity" - command: scripts/ci/build_win.sh + - when: + condition: + equal: ["", << pipeline.git.tag >>] + steps: + - ensure_ccache_key + - restore_ccache_if_allowed + # WARNING! If you edit anything between here and save_cache, remember to bump the CCACHE_KEY version in CircleCI environment settings + - when: + condition: + and: + - equal: ["", << pipeline.git.tag >>] + steps: + - run: + name: "Building solidity (ccache)" + command: CCACHE_ENABLED=${CCACHE_ENABLED:-1} scripts/ci/build_win.sh + - when: + condition: + or: + - not: + equal: ["", << pipeline.git.tag >>] + steps: + - run: + name: "Building solidity (no ccache)" + command: scripts/ci/build_win.sh + # On `ci-ccache` we always generate a fresh cache by never restoring and by saving under a unique key. + - when: + condition: + equal: ["ci-ccache", << pipeline.git.branch >>] + steps: + - save_cache: + key: v1-{{ .Environment.CCACHE_KEY }}-ccache-{{ .Environment.CIRCLE_JOB }}-ci-ccache + paths: + - ~/.ccache - run: name: "Run solc.exe to make sure build was successful." command: .\build\solc\Release\solc.exe --version diff --git a/.circleci/osx_install_dependencies.sh b/.circleci/osx_install_dependencies.sh index dd5a8397b168..0666211b76b3 100755 --- a/.circleci/osx_install_dependencies.sh +++ b/.circleci/osx_install_dependencies.sh @@ -51,6 +51,7 @@ then brew update brew upgrade brew install cmake + brew install ccache brew install wget brew install coreutils brew install diffutils diff --git a/scripts/ci/build.sh b/scripts/ci/build.sh index b7f7e92f5708..385d2d898915 100755 --- a/scripts/ci/build.sh +++ b/scripts/ci/build.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -ex -ROOTDIR="$(dirname "$0")/../.." +ROOTDIR="$(realpath "$(dirname "$0")/../..")" # shellcheck source=scripts/common.sh source "${ROOTDIR}/scripts/common.sh" @@ -16,7 +16,23 @@ cd build [[ -n $COVERAGE && -z $CIRCLE_TAG ]] && CMAKE_OPTIONS="$CMAKE_OPTIONS -DCOVERAGE=ON" +if [[ "${CCACHE_ENABLED:-}" == "1" ]]; then + export CCACHE_DIR="$HOME/.ccache" + export CCACHE_BASEDIR="$ROOTDIR" + export CCACHE_NOHASHDIR=1 + CMAKE_OPTIONS="${CMAKE_OPTIONS:-} -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" + mkdir -p "$CCACHE_DIR" +fi + # shellcheck disable=SC2086 cmake .. -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE:-Release}" $CMAKE_OPTIONS +if [[ "${CCACHE_ENABLED:-}" == "1" ]]; then + ccache -z +fi + cmake --build . + +if [[ "${CCACHE_ENABLED:-}" == "1" ]]; then + ccache -s +fi diff --git a/scripts/ci/build_emscripten.sh b/scripts/ci/build_emscripten.sh index 50c851af8448..bf19bad3600d 100755 --- a/scripts/ci/build_emscripten.sh +++ b/scripts/ci/build_emscripten.sh @@ -54,6 +54,15 @@ function build() { # TODO: This can be removed if and when all usages of `move()` in our codebase use the `std::` qualifier. CMAKE_CXX_FLAGS="-Wno-unqualified-std-cast-call" + if [[ "${CCACHE_ENABLED:-}" == "1" ]]; then + export CCACHE_DIR="$HOME/.ccache" + CCACHE_BASEDIR="$(pwd)" + export CCACHE_BASEDIR + export CCACHE_NOHASHDIR=1 + mkdir -p "$CCACHE_DIR" + ccache -z + fi + mkdir -p "$build_dir" cd "$build_dir" emcmake cmake \ @@ -64,6 +73,9 @@ function build() { -DTESTS=0 \ .. make soljson + if [[ "${CCACHE_ENABLED:-}" == "1" ]]; then + ccache -s + fi cd .. mkdir -p upload diff --git a/scripts/ci/build_ossfuzz.sh b/scripts/ci/build_ossfuzz.sh index 64d992994ed6..883c939b89ea 100755 --- a/scripts/ci/build_ossfuzz.sh +++ b/scripts/ci/build_ossfuzz.sh @@ -18,9 +18,24 @@ function generate_protobuf_bindings function build_fuzzers { cd "${BUILDDIR}" - cmake .. -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE:-Release}" -DCCACHE=OFF \ - -DCMAKE_TOOLCHAIN_FILE="${ROOTDIR}"/cmake/toolchains/libfuzzer.cmake + if [[ "${CCACHE_ENABLED:-}" == "1" ]]; then + export CCACHE_DIR="$HOME/.ccache" + export CCACHE_BASEDIR="$ROOTDIR" + export CCACHE_NOHASHDIR=1 + CMAKE_OPTIONS="${CMAKE_OPTIONS:-} -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" + mkdir -p "$CCACHE_DIR" + fi + # shellcheck disable=SC2086 + cmake .. -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE:-Release}" \ + -DCMAKE_TOOLCHAIN_FILE="${ROOTDIR}"/cmake/toolchains/libfuzzer.cmake \ + $CMAKE_OPTIONS + if [[ "${CCACHE_ENABLED:-}" == "1" ]]; then + ccache -z + fi make ossfuzz ossfuzz_proto ossfuzz_abiv2 -j 4 + if [[ "${CCACHE_ENABLED:-}" == "1" ]]; then + ccache -s + fi } generate_protobuf_bindings diff --git a/scripts/ci/build_win.sh b/scripts/ci/build_win.sh index 64d63c58f4ef..07604c8eb854 100755 --- a/scripts/ci/build_win.sh +++ b/scripts/ci/build_win.sh @@ -6,22 +6,53 @@ prerelease_source="${1:-ci}" ROOTDIR="$(realpath "$(dirname "$0")/../../")" FORCE_RELEASE="${FORCE_RELEASE:-}" CIRCLE_TAG="${CIRCLE_TAG:-}" +CMAKE_VS_GLOBALS_ARG=() cd "$ROOTDIR" "${ROOTDIR}/scripts/prerelease_suffix.sh" "$prerelease_source" "$CIRCLE_TAG" > prerelease.txt +if [[ "${CCACHE_ENABLED:-}" == "1" ]]; then + export CCACHE_DIR="$HOME/.ccache" + export CCACHE_BASEDIR="$ROOTDIR" + export CCACHE_NOHASHDIR=1 + export CCACHE_COMPILERTYPE=msvc + VSWHERE="${VSWHERE:-C:/Program Files (x86)/Microsoft Visual Studio/Installer/vswhere.exe}" + VS_INSTALL="$("$VSWHERE" -latest -products "*" \ + -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 \ + -property installationPath | tr -d '\r')" + VCTOOLS_VERSION="$(tr -d '\r' < "$VS_INSTALL/VC/Auxiliary/Build/Microsoft.VCToolsVersion.default.txt")" + CCACHE_COMPILER="$VS_INSTALL/VC/Tools/MSVC/$VCTOOLS_VERSION/bin/Hostx64/x64/cl.exe" + CCACHE_COMPILER="${CCACHE_COMPILER//\\//}" + export CCACHE_COMPILER + PATH="$ROOTDIR/deps/ccache:$(dirname "$CCACHE_COMPILER"):$PATH" + export PATH + # Use a Windows-style path for Visual Studio globals. + WIN_PWD="$(pwd -W)" + # Visual Studio generator doesn't use compiler launchers; use ccache masquerade. + # See https://github.com/ccache/ccache/wiki/MS-Visual-Studio + CMAKE_VS_GLOBALS="CLToolPath=${WIN_PWD}/deps/ccache;UseMultiToolTask=true" + CMAKE_VS_GLOBALS_ARG=(-DCMAKE_VS_GLOBALS="$CMAKE_VS_GLOBALS") + mkdir -p "$CCACHE_DIR" +fi + mkdir -p build/ cd build/ # NOTE: Using an array to force Bash to do wildcard expansion boost_dir=("${ROOTDIR}/deps/boost/lib/cmake/Boost-"*) -"${ROOTDIR}/deps/cmake/bin/cmake" \ + # shellcheck disable=SC2086 + "${ROOTDIR}/deps/cmake/bin/cmake" \ -G "Visual Studio 16 2019" \ -DBoost_DIR="${boost_dir[*]}" \ -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded \ -DCMAKE_INSTALL_PREFIX="${ROOTDIR}/uploads/" \ + "${CMAKE_VS_GLOBALS_ARG[@]}" \ + ${CMAKE_OPTIONS:-} \ .. +if [[ "${CCACHE_ENABLED:-}" == "1" ]]; then + ccache -z +fi MSBuild.exe solidity.sln \ -property:Configuration=Release \ -maxCpuCount:10 \ @@ -31,3 +62,6 @@ MSBuild.exe solidity.sln \ -j 10 \ --target install \ --config Release +if [[ "${CCACHE_ENABLED:-}" == "1" ]]; then + ccache -s +fi diff --git a/scripts/install_deps.ps1 b/scripts/install_deps.ps1 index ad4f5bdadccc..bf86735811e9 100644 --- a/scripts/install_deps.ps1 +++ b/scripts/install_deps.ps1 @@ -14,6 +14,17 @@ if ( -not (Test-Path "$PSScriptRoot\..\deps\boost") ) { mv cmake-3.27.4-windows-x86_64 "$PSScriptRoot\..\deps\cmake" Remove-Item cmake.zip + Invoke-WebRequest -URI "https://github.com/ccache/ccache/releases/download/v4.12.2/ccache-4.12.2-windows-x86_64.zip" -OutFile ccache.zip + if ((Get-FileHash ccache.zip).Hash -ne "82c1b130b25cc162531dc7a062dc5ea99349cd536bc9eba8a66d976802d66516") { + throw 'Downloaded ccache package has wrong checksum.' + } + tar -xf ccache.zip + mv ccache-4.12.2-windows-x86_64 "$PSScriptRoot\..\deps\ccache" + # ccache MSVC guide: https://github.com/ccache/ccache/wiki/MS-Visual-Studio + # Replace ccache.exe as cl.exe so MSBuild still sees "cl.exe" while ccache wraps the real compiler. + Copy-Item -Force "$PSScriptRoot\..\deps\ccache\ccache.exe" "$PSScriptRoot\..\deps\ccache\cl.exe" + Remove-Item ccache.zip + # FIXME: The default user agent results in Artifactory treating Invoke-WebRequest as a browser # and serving it a page that requires JavaScript. Invoke-WebRequest -URI "https://archives.boost.io/release/1.83.0/source/boost_1_83_0.zip" -OutFile boost.zip -UserAgent ""