[CI] Use L2 ccache reduce build time in all workflows

.github/workflows/Coverage.yml

@@ -33,7 +33,7 @@ jobs:
     name: Coverage clone
     uses: ./.github/workflows/_Clone-linux.yml
     with:
-      workflow-name: 'coverage'
+      workflow-name: "coverage"
       clone_dir: Paddle-coverage
 
   build-docker:
@@ -57,7 +57,8 @@ jobs:
       - name: Check docker image and run container
         env:
           CACHE_DIR: "/root/.cache/coverage"
-          CCACHE_DIR: "/root/.ccache/coverage"
+          CCACHE_DIR: "/home/data/shared/.ccache/l1" # L1 cache on machine shared dir
+          CCACHE_SECONDARY_STORAGE: "file:///home/data/cfs/.ccache/l2" # L2 cache on cfs
           FLAGS_fraction_of_gpu_memory_to_use: 0.15
           CTEST_PARALLEL_LEVEL: 2
           WITH_GPU: "ON"
@@ -76,7 +77,7 @@ jobs:
           WITH_SHARED_PHI: "ON"
           WITH_CINN: "ON"
           INFERENCE_DEMO_INSTALL_DIR: /root/.cache/coverage
-          CCACHE_MAXSIZE: 200G
+          CCACHE_MAXSIZE: 50G
           CCACHE_LIMIT_MULTIPLE: 0.8
           ON_INFER: "ON"
           PADDLE_CUDA_INSTALL_REQUIREMENTS: "ON"
@@ -90,7 +91,7 @@ jobs:
           docker run -d -t --name ${container_name} \
             -v "/home/data/cfs:/home/data/cfs" \
             -v "/home/data/cfs/.cache:/root/.cache" \
-            -v "/home/data/cfs/.ccache:/root/.ccache" \
+            -v "/home/data/shared:/home/data/shared" \
             -v "/dev/shm:/dev/shm" \
             -v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
             -v ${{ github.workspace }}:/paddle \
@@ -103,6 +104,7 @@ jobs:
             -e GIT_PR_ID \
             -e CACHE_DIR \
             -e CCACHE_DIR \
+            -e CCACHE_SECONDARY_STORAGE \
             -e ci_scripts \
             -e FLAGS_fraction_of_gpu_memory_to_use \
             -e CTEST_PARALLEL_LEVEL \
@@ -250,7 +252,6 @@ jobs:
           docker stop ${{ env.container_name }}
           docker rm ${{ env.container_name }}
 
-
   test:
     name: Coverage test
     needs: [build, build-docker]

.github/workflows/H-Coverage.yml

@@ -33,7 +33,7 @@ jobs:
     name: Coverage clone
     uses: ./.github/workflows/_Clone-linux.yml
     with:
-      workflow-name: 'coverage'
+      workflow-name: "coverage"
       clone_dir: h-ci
 
   build:
@@ -49,7 +49,8 @@ jobs:
       - name: Check docker image and run container
         env:
           CACHE_DIR: "/root/.cache/coverage"
-          CCACHE_DIR: "/root/.ccache/h-coverage"
+          CCACHE_DIR: "/home/data/shared/.ccache/l1" # L1 cache on machine shared dir
+          CCACHE_SECONDARY_STORAGE: "file:///home/data/cfs/.ccache/l2" # L2 cache on cfs
           FLAGS_fraction_of_gpu_memory_to_use: 0.15
           CTEST_PARALLEL_LEVEL: 2
           WITH_GPU: "ON"
@@ -63,7 +64,7 @@ jobs:
           WITH_FA_BUILD_WITH_CACHE: "ON"
           PY_VERSION: "3.10"
           INFERENCE_DEMO_INSTALL_DIR: /root/.cache/coverage
-          CCACHE_MAXSIZE: 200G
+          CCACHE_MAXSIZE: 50G
           CCACHE_LIMIT_MULTIPLE: 0.8
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -74,7 +75,7 @@ jobs:
           docker run -d -t --name ${container_name} \
             -v "/home/data/cfs:/home/data/cfs" \
             -v "/home/data/cfs/.cache:/root/.cache" \
-            -v "/home/data/cfs/.ccache:/root/.ccache" \
+            -v "/home/data/shared:/home/data/shared" \
             -v "/dev/shm:/dev/shm" \
             -v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
             -v ${{ github.workspace }}:/paddle \
@@ -87,6 +88,7 @@ jobs:
             -e GIT_PR_ID \
             -e CACHE_DIR \
             -e CCACHE_DIR \
+            -e CCACHE_SECONDARY_STORAGE \
             -e ci_scripts \
             -e FLAGS_fraction_of_gpu_memory_to_use \
             -e CTEST_PARALLEL_LEVEL \
@@ -155,10 +157,13 @@ jobs:
           mkdir -p ${CFS_DIR}/.ccache/coverage
           export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda-12.9/compat
           source ${{ github.workspace }}/../../../proxy
+          bash ${ci_scripts}/cmake-predownload.sh
           pip install -r python/requirements.txt
-          mkdir build && cd build
+          mkdir -p build && cd build
+          ccache -z
           cmake .. -DPY_VERSION=3.10 -DWITH_GPU=ON -DWITH_DISTRIBUTE=ON -DWITH_TESTING=ON -DCUDA_ARCH_NAME=Manual -DCUDA_ARCH_BIN="90" -DFA_JOB_POOLS_COMPILE=1 -DWITH_CUDNN_FRONTEND=ON -DON_INFER=OFF -DWITH_NVSHMEM=ON
           make -j20
+          ccache -s
           '
 
       - name: Clean up env

.github/workflows/_Distribute-stable.yml

@@ -61,8 +61,9 @@ jobs:
           WITH_CUDNN_FRONTEND: "ON"
           FLAGS_enable_cudnn_frontend: 1
           CACHE_DIR: /root/.cache/build
-          CCACHE_DIR: /root/.ccache/gpubox
-          CCACHE_MAXSIZE: 150G
+          CCACHE_DIR: "/home/data/shared/.ccache/l1" # L1 cache on machine shared dir
+          CCACHE_SECONDARY_STORAGE: "file:///home/data/cfs/.ccache/l2" # L2 cache on cfs
+          CCACHE_MAXSIZE: 50G
           CCACHE_LIMIT_MULTIPLE: 0.8
           CCACHE_STATSLOG: /paddle/build/.stats.log
           CCACHE_SLOPPINESS: clang_index_store,time_macros,include_file_mtime
@@ -72,7 +73,7 @@ jobs:
           docker run -d -t --name ${container_name} \
             -v "/home/data/cfs:/home/data/cfs" \
             -v "/home/data/cfs/.cache/:/root/.cache" \
-            -v "/home/data/cfs/.ccache:/root/.ccache" \
+            -v "/home/data/shared:/home/data/shared" \
             -v "/dev/shm:/dev/shm"  \
             -v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
             -v ${{ github.workspace }}:/paddle \
@@ -98,6 +99,7 @@ jobs:
             -e WITH_PYTHON \
             -e CACHE_DIR \
             -e CCACHE_DIR \
+            -e CCACHE_SECONDARY_STORAGE \
             -e CCACHE_STATSLOG \
             -e CCACHE_SLOPPINESS \
             -e FLAGS_fraction_of_gpu_memory_to_use \

.github/workflows/_Inference.yml

@@ -32,7 +32,7 @@ jobs:
     if: ${{ inputs.clone-can-skip != 'true' }}
     uses: ./.github/workflows/check-bypass.yml
     with:
-      workflow-name: 'inference'
+      workflow-name: "inference"
     secrets:
       github-token: ${{ secrets.GITHUB_TOKEN }}
 
@@ -66,8 +66,9 @@ jobs:
           WITH_NVCC_LAZY: "OFF"
           PYTHON_ABI: cp310-cp310
           CACHE_DIR: /root/.cache/inference
-          CCACHE_DIR: /root/.ccache/inference
-          CCACHE_MAXSIZE: 150G
+          CCACHE_DIR: "/home/data/shared/.ccache/l1" # L1 cache on machine shared dir
+          CCACHE_SECONDARY_STORAGE: "file:///home/data/cfs/.ccache/l2" # L2 cache on cfs
+          CCACHE_MAXSIZE: 50G
           CCACHE_LIMIT_MULTIPLE: "0.8"
           CUDA_ARCH_NAME: Pascal
           PADDLE_CUDA_INSTALL_REQUIREMENTS: "ON"
@@ -226,8 +227,7 @@ jobs:
           docker run -d -t --gpus all --name ${container_name} \
           -v "/home/data/cfs:/home/data/cfs" \
           -v "/home/data/cfs/.cache:/root/.cache" \
-          -v "/home/data/cfs/.ccache:/root/.ccache" \
-          -v "/home/data/cfs/.ccache/inference:/root/.ccache/inference" \
+          -v "/home/data/shared:/home/data/shared" \
           -v "/dev/shm:/dev/shm" \
           -v "${{ github.workspace }}/../../..:/action-runner" \
           -e BRANCH \
@@ -254,6 +254,7 @@ jobs:
           -e WITH_UNITY_BUILD \
           -e CACHE_DIR \
           -e CCACHE_DIR \
+          -e CCACHE_SECONDARY_STORAGE \
           -e CCACHE_MAXSIZE \
           -e CCACHE_LIMIT_MULTIPLE \
           -e ci_scripts \

.github/workflows/_Linux-CPU.yml

@@ -52,19 +52,20 @@ jobs:
           FLAGS_enable_eager_mode: 1
           WITH_TENSORRT: "OFF"
           GENERATOR: "Ninja"
-          CCACHE_MAXSIZE: 150G
+          CCACHE_MAXSIZE: 50G
           CCACHE_LIMIT_MULTIPLE: 0.8
           WITH_AVX: "OFF"
           CACHE_DIR: "/root/.cache/cpu"
-          CCACHE_DIR: "/root/.ccache/cpu"
+          CCACHE_DIR: "/home/data/shared/.ccache/l1" # L1 cache on machine shared dir
+          CCACHE_SECONDARY_STORAGE: "file:///home/data/cfs/.ccache/l2" # L2 cache on cfs
         run: |
           container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
           echo "container_name=${container_name}" >> ${{ github.env }}
           docker_image=${{ inputs.docker_cpu_image }}
           docker run -d -t --name ${container_name} \
             -v "/home/data/cfs:/home/data/cfs" \
             -v "/home/data/cfs/.cache/python35-cpu:/root/.cache" \
-            -v "/home/data/cfs/.ccache:/root/.ccache" \
+            -v "/home/data/shared:/home/data/shared" \
             -v "/dev/shm:/dev/shm"  \
             -v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
             -v ${{ github.workspace }}:/paddle \
@@ -95,6 +96,7 @@ jobs:
             -e WITH_AVX \
             -e CACHE_DIR \
             -e CCACHE_DIR \
+            -e CCACHE_SECONDARY_STORAGE \
             -e CI_name \
             -e no_proxy \
             -w /paddle --network host ${docker_image}

.github/workflows/_Linux-build.yml

@@ -9,7 +9,7 @@ on:
       is_pr:
         type: string
         required: false
-        default: 'true'
+        default: "true"
     outputs:
       can-skip:
         description: "Whether to skip the job"
@@ -65,20 +65,21 @@ jobs:
           WITH_UNITY_BUILD: "ON"
           PY_VERSION: "3.10"
           WITH_SHARED_PHI: "ON"
-          CCACHE_MAXSIZE: 150G
+          CCACHE_MAXSIZE: 50G
           CCACHE_LIMIT_MULTIPLE: 0.8
           CCACHE_STATSLOG: /paddle/build/.stats.log
           CCACHE_SLOPPINESS: clang_index_store,time_macros,include_file_mtime
           CACHE_DIR: /home/data/cfs/.cache/build
-          CCACHE_DIR: /home/data/cfs/.ccache/build
+          CCACHE_DIR: "/home/data/shared/.ccache/l1" # L1 cache on machine shared dir
+          CCACHE_SECONDARY_STORAGE: "file:///home/data/cfs/.ccache/l2" # L2 cache on cfs
         run: |
           container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
           echo "container_name=${container_name}" >> ${{ github.env }}
           docker_image=${{ inputs.docker_build_image }}
           docker run -d -t --name ${container_name} \
             -v "/home/data/cfs:/home/data/cfs" \
             -v "/home/data/cfs/.cache:/root/.cache" \
-            -v "/home/data/cfs/.ccache:/root/.ccache" \
+            -v "/home/data/shared:/home/data/shared" \
             -v "/dev/shm:/dev/shm"  \
             -v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
             -v ${{ github.workspace }}:/paddle \
@@ -115,6 +116,7 @@ jobs:
             -e WITH_AVX \
             -e CACHE_DIR \
             -e CCACHE_DIR \
+            -e CCACHE_SECONDARY_STORAGE \
             -e CCACHE_SLOPPINESS \
             -e CCACHE_STATSLOG \
             -e CFS_DIR \

.github/workflows/_SOT.yml

@@ -59,12 +59,12 @@ jobs:
           WITH_TENSORRT: "OFF"
           GENERATOR: Ninja
           WITH_INFERENCE_API_TEST: "OFF"
-          CCACHE_MAXSIZE: 150G
+          CCACHE_MAXSIZE: 50G
           CCACHE_LIMIT_MULTIPLE: 0.8
           WITH_AVX: "OFF"
           CACHE_DIR: "/root/.cache/sot"
-          CCACHE_DIR: "/root/.ccache/sot"
-          CCACHE_SECONDARY_STORAGE: "file:///home/data/cfs/.ccache/sot"
+          CCACHE_DIR: "/home/data/shared/.ccache/l1" # L1 cache on machine shared dir
+          CCACHE_SECONDARY_STORAGE: "file:///home/data/cfs/.ccache/l2" # L2 cache on cfs
         run: |
           set -x
           container_name=${TASK}-$(date +%s)
@@ -73,6 +73,7 @@ jobs:
           docker run -d -t --name ${container_name} \
             -v "/home/data/cfs:/home/data/cfs" \
             -v "/home/data/cfs/.cache:/root/.cache" \
+            -v "/home/data/shared:/home/data/shared" \
             -v "/dev/shm:/dev/shm" \
             -v "${{ github.workspace }}/../../..:${{ github.workspace }}/../../.." \
             -v ${{ github.workspace }}:/paddle \