Skip to content

Commit fb17c4c

Browse files
hjmjohnsonhjmjohnson
committed
COMP: Fix near-zero ccache hit rate on all Azure DevOps pipelines
Remove CCACHE_NODIRECT=1 which disabled ccache's fast direct mode, forcing preprocessor-mode hashing on every compilation unit. The ARM CI (which never had NODIRECT) achieves 98.5% hit rate; Azure DevOps was at 0.02%. Applied to all 7 Azure DevOps pipelines: - AzurePipelinesLinux.yml (3 jobs: Linux, LinuxLegacyRemoved, LinuxCxx20) - AzurePipelinesLinuxPython.yml - AzurePipelinesMacOS.yml - AzurePipelinesMacOSPython.yml - AzurePipelinesWindows.yml - AzurePipelinesWindowsPython.yml - AzurePipelinesBatch.yml Changes per pipeline: - Remove CCACHE_NODIRECT=1 (enable direct mode) - Add CCACHE_SLOPPINESS=pch_defines,time_macros - Add per-job name to cache key (ccache-v4 | OS | JobName | SHA) - Remove ITK_USE_CCACHE:BOOL=ON (redundant with CMAKE_*_LAUNCHER) - Add ccache --zero-stats + --show-config maintenance step - Add ccache --show-stats step after build for monitoring Tested in PR #6047: Run 2 achieved 99.95% hit rate (2166/2167 direct hits) after these fixes on the Linux pipeline.
1 parent 9e00915 commit fb17c4c

File tree

7 files changed

+115
-50
lines changed

7 files changed

+115
-50
lines changed

‎Testing/ContinuousIntegration/AzurePipelinesBatch.yml‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ variables:
1616
CCACHE_BASEDIR: $(Build.SourcesDirectory)
1717
CCACHE_COMPILERCHECK: content
1818
CCACHE_NOHASHDIR: 'true'
19-
CCACHE_NODIRECT: '1'
19+
CCACHE_SLOPPINESS: pch_defines,time_macros
2020
jobs:
2121
- job: Windows
2222
timeoutInMinutes: 0
@@ -70,14 +70,18 @@ jobs:
7070
7171
- task: Cache@2
7272
inputs:
73-
key: '"ccache" | "$(Agent.OS)" | "$(Build.SourceVersion)"'
73+
key: '"ccache-v4" | "$(Agent.OS)" | "WindowsBatch" | "$(Build.SourceVersion)"'
7474
restoreKeys: |
75-
"ccache" | "$(Agent.OS)"
75+
"ccache-v4" | "$(Agent.OS)" | "WindowsBatch"
7676
path: $(CCACHE_DIR)
7777
displayName: 'Restore ccache'
7878

79-
- bash: ccache --evict-older-than 7d
80-
displayName: 'Evict old ccache entries'
79+
- bash: |
80+
set -x
81+
ccache --zero-stats
82+
ccache --evict-older-than 7d
83+
ccache --show-config
84+
displayName: 'ccache config and maintenance'
8185
8286
- bash: |
8387
cat > dashboard.cmake << EOF
@@ -89,7 +93,6 @@ jobs:
8993
BUILD_SHARED_LIBS:BOOL=ON
9094
BUILD_EXAMPLES:BOOL=OFF
9195
ITK_WRAP_PYTHON:BOOL=OFF
92-
ITK_USE_CCACHE:BOOL=ON
9396
CMAKE_C_COMPILER_LAUNCHER:STRING=ccache
9497
CMAKE_CXX_COMPILER_LAUNCHER:STRING=ccache
9598
ITK_COMPUTER_MEMORY_SIZE:STRING=4.5
@@ -109,6 +112,10 @@ jobs:
109112
CTEST_OUTPUT_ON_FAILURE: 1
110113
CCACHE_MAXSIZE: 2.4G
111114
115+
- bash: ccache --show-stats
116+
condition: always()
117+
displayName: 'ccache stats'
118+
112119
- script: python $(Build.SourcesDirectory)/Testing/ContinuousIntegration/report_build_diagnostics.py $(Build.SourcesDirectory)-build
113120
condition: succeededOrFailed()
114121
displayName: 'Report build warnings and errors'

‎Testing/ContinuousIntegration/AzurePipelinesLinux.yml‎

Lines changed: 37 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ variables:
3232
CCACHE_BASEDIR: $(Build.SourcesDirectory)
3333
CCACHE_COMPILERCHECK: content
3434
CCACHE_NOHASHDIR: 'true'
35-
CCACHE_NODIRECT: '1'
35+
CCACHE_SLOPPINESS: pch_defines,time_macros
3636
jobs:
3737
- job: Linux
3838
timeoutInMinutes: 0
@@ -76,14 +76,18 @@ jobs:
7676
7777
- task: Cache@2
7878
inputs:
79-
key: '"ccache" | "$(Agent.OS)" | "$(Build.SourceVersion)"'
79+
key: '"ccache-v4" | "$(Agent.OS)" | "Linux" | "$(Build.SourceVersion)"'
8080
restoreKeys: |
81-
"ccache" | "$(Agent.OS)"
81+
"ccache-v4" | "$(Agent.OS)" | "Linux"
8282
path: $(CCACHE_DIR)
8383
displayName: 'Restore ccache'
8484

85-
- bash: ccache --evict-older-than 7d
86-
displayName: 'Evict old ccache entries'
85+
- bash: |
86+
set -x
87+
ccache --zero-stats
88+
ccache --evict-older-than 7d
89+
ccache --show-config
90+
displayName: 'ccache config and maintenance'
8791
8892
- bash: |
8993
cat > dashboard.cmake << EOF
@@ -93,7 +97,6 @@ jobs:
9397
BUILD_SHARED_LIBS:BOOL=OFF
9498
BUILD_EXAMPLES:BOOL=OFF
9599
ITK_WRAP_PYTHON:BOOL=OFF
96-
ITK_USE_CCACHE:BOOL=ON
97100
CMAKE_C_COMPILER_LAUNCHER:STRING=ccache
98101
CMAKE_CXX_COMPILER_LAUNCHER:STRING=ccache
99102
ITK_COMPUTER_MEMORY_SIZE:STRING=4.5
@@ -117,6 +120,10 @@ jobs:
117120
CTEST_OUTPUT_ON_FAILURE: 1
118121
CCACHE_MAXSIZE: 2.4G
119122
123+
- bash: ccache --show-stats
124+
condition: always()
125+
displayName: 'ccache stats'
126+
120127
- bash: python3 $(Build.SourcesDirectory)/Testing/ContinuousIntegration/report_build_diagnostics.py $(Build.SourcesDirectory)-build
121128
condition: succeededOrFailed()
122129
displayName: 'Report build warnings and errors'
@@ -170,14 +177,18 @@ jobs:
170177
171178
- task: Cache@2
172179
inputs:
173-
key: '"ccache" | "$(Agent.OS)" | "$(Build.SourceVersion)"'
180+
key: '"ccache-v4" | "$(Agent.OS)" | "LinuxLegacyRemoved" | "$(Build.SourceVersion)"'
174181
restoreKeys: |
175-
"ccache" | "$(Agent.OS)"
182+
"ccache-v4" | "$(Agent.OS)" | "LinuxLegacyRemoved"
176183
path: $(CCACHE_DIR)
177184
displayName: 'Restore ccache'
178185

179-
- bash: ccache --evict-older-than 7d
180-
displayName: 'Evict old ccache entries'
186+
- bash: |
187+
set -x
188+
ccache --zero-stats
189+
ccache --evict-older-than 7d
190+
ccache --show-config
191+
displayName: 'ccache config and maintenance'
181192
182193
- bash: |
183194
cat > dashboard.cmake << EOF
@@ -212,6 +223,10 @@ jobs:
212223
CTEST_OUTPUT_ON_FAILURE: 1
213224
CCACHE_MAXSIZE: 2.4G
214225
226+
- bash: ccache --show-stats
227+
condition: always()
228+
displayName: 'ccache stats'
229+
215230
- bash: python3 $(Build.SourcesDirectory)/Testing/ContinuousIntegration/report_build_diagnostics.py $(Build.SourcesDirectory)-build
216231
condition: succeededOrFailed()
217232
displayName: 'Report build warnings and errors'
@@ -265,14 +280,18 @@ jobs:
265280
displayName: "Download dashboard script and testing data"
266281
- task: Cache@2
267282
inputs:
268-
key: '"ccache" | "$(Agent.OS)" | "$(Build.SourceVersion)"'
283+
key: '"ccache-v4" | "$(Agent.OS)" | "LinuxCxx20" | "$(Build.SourceVersion)"'
269284
restoreKeys: |
270-
"ccache" | "$(Agent.OS)"
285+
"ccache-v4" | "$(Agent.OS)" | "LinuxCxx20"
271286
path: $(CCACHE_DIR)
272287
displayName: 'Restore ccache'
273288

274-
- bash: ccache --evict-older-than 7d
275-
displayName: 'Evict old ccache entries'
289+
- bash: |
290+
set -x
291+
ccache --zero-stats
292+
ccache --evict-older-than 7d
293+
ccache --show-config
294+
displayName: 'ccache config and maintenance'
276295
277296
- bash: |
278297
cat > dashboard.cmake << EOF
@@ -305,6 +324,10 @@ jobs:
305324
CTEST_OUTPUT_ON_FAILURE: 1
306325
CCACHE_MAXSIZE: 2.4G
307326
327+
- bash: ccache --show-stats
328+
condition: always()
329+
displayName: 'ccache stats'
330+
308331
- bash: python3 $(Build.SourcesDirectory)/Testing/ContinuousIntegration/report_build_diagnostics.py $(Build.SourcesDirectory)-build
309332
condition: succeededOrFailed()
310333
displayName: 'Report build warnings and errors'

‎Testing/ContinuousIntegration/AzurePipelinesLinuxPython.yml‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ variables:
3232
CCACHE_BASEDIR: $(Build.SourcesDirectory)
3333
CCACHE_COMPILERCHECK: content
3434
CCACHE_NOHASHDIR: 'true'
35-
CCACHE_NODIRECT: '1'
35+
CCACHE_SLOPPINESS: pch_defines,time_macros
3636
jobs:
3737
- job: Linux
3838
timeoutInMinutes: 0
@@ -78,14 +78,18 @@ jobs:
7878
7979
- task: Cache@2
8080
inputs:
81-
key: '"ccache" | "$(Agent.OS)" | "$(Build.SourceVersion)"'
81+
key: '"ccache-v4" | "$(Agent.OS)" | "LinuxPython" | "$(Build.SourceVersion)"'
8282
restoreKeys: |
83-
"ccache" | "$(Agent.OS)"
83+
"ccache-v4" | "$(Agent.OS)" | "LinuxPython"
8484
path: $(CCACHE_DIR)
8585
displayName: 'Restore ccache'
8686

87-
- bash: ccache --evict-older-than 7d
88-
displayName: 'Evict old ccache entries'
87+
- bash: |
88+
set -x
89+
ccache --zero-stats
90+
ccache --evict-older-than 7d
91+
ccache --show-config
92+
displayName: 'ccache config and maintenance'
8993
9094
- bash: |
9195
python_executable=`which python3`
@@ -101,7 +105,6 @@ jobs:
101105
BUILD_SHARED_LIBS:BOOL=OFF
102106
BUILD_EXAMPLES:BOOL=OFF
103107
ITK_WRAP_PYTHON:BOOL=ON
104-
ITK_USE_CCACHE:BOOL=ON
105108
CMAKE_C_COMPILER_LAUNCHER:STRING=ccache
106109
CMAKE_CXX_COMPILER_LAUNCHER:STRING=ccache
107110
ITK_COMPUTER_MEMORY_SIZE:STRING=4.5
@@ -124,6 +127,10 @@ jobs:
124127
CTEST_OUTPUT_ON_FAILURE: 1
125128
CCACHE_MAXSIZE: 2.4G
126129
130+
- bash: ccache --show-stats
131+
condition: always()
132+
displayName: 'ccache stats'
133+
127134
- bash: python3 $(Build.SourcesDirectory)/Testing/ContinuousIntegration/report_build_diagnostics.py $(Build.SourcesDirectory)-build
128135
condition: succeededOrFailed()
129136
displayName: 'Report build warnings and errors'

‎Testing/ContinuousIntegration/AzurePipelinesMacOS.yml‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ variables:
3232
CCACHE_BASEDIR: $(Build.SourcesDirectory)
3333
CCACHE_COMPILERCHECK: content
3434
CCACHE_NOHASHDIR: 'true'
35-
CCACHE_NODIRECT: '1'
35+
CCACHE_SLOPPINESS: pch_defines,time_macros
3636
jobs:
3737
- job: macOS
3838
timeoutInMinutes: 0
@@ -83,14 +83,18 @@ jobs:
8383
8484
- task: Cache@2
8585
inputs:
86-
key: '"ccache" | "$(Agent.OS)" | "$(Build.SourceVersion)"'
86+
key: '"ccache-v4" | "$(Agent.OS)" | "macOS" | "$(Build.SourceVersion)"'
8787
restoreKeys: |
88-
"ccache" | "$(Agent.OS)"
88+
"ccache-v4" | "$(Agent.OS)" | "macOS"
8989
path: $(CCACHE_DIR)
9090
displayName: 'Restore ccache'
9191

92-
- bash: ccache --evict-older-than 7d
93-
displayName: 'Evict old ccache entries'
92+
- bash: |
93+
set -x
94+
ccache --zero-stats
95+
ccache --evict-older-than 7d
96+
ccache --show-config
97+
displayName: 'ccache config and maintenance'
9498
9599
- bash: |
96100
cat > dashboard.cmake << EOF
@@ -100,7 +104,6 @@ jobs:
100104
BUILD_SHARED_LIBS:BOOL=ON
101105
BUILD_EXAMPLES:BOOL=ON
102106
ITK_WRAP_PYTHON:BOOL=OFF
103-
ITK_USE_CCACHE:BOOL=ON
104107
CMAKE_C_COMPILER_LAUNCHER:STRING=ccache
105108
CMAKE_CXX_COMPILER_LAUNCHER:STRING=ccache
106109
ITK_COMPUTER_MEMORY_SIZE:STRING=11
@@ -123,6 +126,10 @@ jobs:
123126
CTEST_OUTPUT_ON_FAILURE: 1
124127
CCACHE_MAXSIZE: 2.4G
125128
129+
- bash: ccache --show-stats
130+
condition: always()
131+
displayName: 'ccache stats'
132+
126133
- bash: python3 $(Build.SourcesDirectory)/Testing/ContinuousIntegration/report_build_diagnostics.py $(Build.SourcesDirectory)-build
127134
condition: succeededOrFailed()
128135
displayName: 'Report build warnings and errors'

‎Testing/ContinuousIntegration/AzurePipelinesMacOSPython.yml‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ variables:
3232
CCACHE_BASEDIR: $(Build.SourcesDirectory)
3333
CCACHE_COMPILERCHECK: content
3434
CCACHE_NOHASHDIR: 'true'
35-
CCACHE_NODIRECT: '1'
35+
CCACHE_SLOPPINESS: pch_defines,time_macros
3636
jobs:
3737
- job: macOS
3838
timeoutInMinutes: 0
@@ -86,14 +86,18 @@ jobs:
8686
8787
- task: Cache@2
8888
inputs:
89-
key: '"ccache" | "$(Agent.OS)" | "$(Build.SourceVersion)"'
89+
key: '"ccache-v4" | "$(Agent.OS)" | "macOSPython" | "$(Build.SourceVersion)"'
9090
restoreKeys: |
91-
"ccache" | "$(Agent.OS)"
91+
"ccache-v4" | "$(Agent.OS)" | "macOSPython"
9292
path: $(CCACHE_DIR)
9393
displayName: 'Restore ccache'
9494

95-
- bash: ccache --evict-older-than 7d
96-
displayName: 'Evict old ccache entries'
95+
- bash: |
96+
set -x
97+
ccache --zero-stats
98+
ccache --evict-older-than 7d
99+
ccache --show-config
100+
displayName: 'ccache config and maintenance'
97101
98102
- bash: |
99103
python_executable=`which python3`
@@ -109,7 +113,6 @@ jobs:
109113
BUILD_SHARED_LIBS:BOOL=OFF
110114
BUILD_EXAMPLES:BOOL=OFF
111115
ITK_WRAP_PYTHON:BOOL=ON
112-
ITK_USE_CCACHE:BOOL=ON
113116
CMAKE_C_COMPILER_LAUNCHER:STRING=ccache
114117
CMAKE_CXX_COMPILER_LAUNCHER:STRING=ccache
115118
ITK_COMPUTER_MEMORY_SIZE:STRING=11
@@ -132,6 +135,10 @@ jobs:
132135
CTEST_OUTPUT_ON_FAILURE: 1
133136
CCACHE_MAXSIZE: 2.4G
134137
138+
- bash: ccache --show-stats
139+
condition: always()
140+
displayName: 'ccache stats'
141+
135142
- bash: python3 $(Build.SourcesDirectory)/Testing/ContinuousIntegration/report_build_diagnostics.py $(Build.SourcesDirectory)-build
136143
condition: succeededOrFailed()
137144
displayName: 'Report build warnings and errors'

‎Testing/ContinuousIntegration/AzurePipelinesWindows.yml‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ variables:
3232
CCACHE_BASEDIR: $(Build.SourcesDirectory)
3333
CCACHE_COMPILERCHECK: content
3434
CCACHE_NOHASHDIR: 'true'
35-
CCACHE_NODIRECT: '1'
35+
CCACHE_SLOPPINESS: pch_defines,time_macros
3636
jobs:
3737
- job: Windows
3838
timeoutInMinutes: 0
@@ -71,14 +71,18 @@ jobs:
7171
7272
- task: Cache@2
7373
inputs:
74-
key: '"ccache" | "$(Agent.OS)" | "$(Build.SourceVersion)"'
74+
key: '"ccache-v4" | "$(Agent.OS)" | "Windows" | "$(Build.SourceVersion)"'
7575
restoreKeys: |
76-
"ccache" | "$(Agent.OS)"
76+
"ccache-v4" | "$(Agent.OS)" | "Windows"
7777
path: $(CCACHE_DIR)
7878
displayName: 'Restore ccache'
7979

80-
- bash: ccache --evict-older-than 7d
81-
displayName: 'Evict old ccache entries'
80+
- bash: |
81+
set -x
82+
ccache --zero-stats
83+
ccache --evict-older-than 7d
84+
ccache --show-config
85+
displayName: 'ccache config and maintenance'
8286
8387
- bash: |
8488
cat > dashboard.cmake << EOF
@@ -90,7 +94,6 @@ jobs:
9094
BUILD_SHARED_LIBS:BOOL=ON
9195
BUILD_EXAMPLES:BOOL=OFF
9296
ITK_WRAP_PYTHON:BOOL=OFF
93-
ITK_USE_CCACHE:BOOL=ON
9497
CMAKE_C_COMPILER_LAUNCHER:STRING=ccache
9598
CMAKE_CXX_COMPILER_LAUNCHER:STRING=ccache
9699
ITK_COMPUTER_MEMORY_SIZE:STRING=4.5
@@ -110,6 +113,10 @@ jobs:
110113
CTEST_OUTPUT_ON_FAILURE: 1
111114
CCACHE_MAXSIZE: 2.4G
112115
116+
- bash: ccache --show-stats
117+
condition: always()
118+
displayName: 'ccache stats'
119+
113120
- script: python $(Build.SourcesDirectory)/Testing/ContinuousIntegration/report_build_diagnostics.py $(Build.SourcesDirectory)-build
114121
condition: succeededOrFailed()
115122
displayName: 'Report build warnings and errors'

0 commit comments

Comments
 (0)