Skip to content

Commit 613a539

Browse files
dfedermdfederm
authored
Add Microsoft.MSBuildCache (#17393)
Add Microsoft.MSBuildCache This change adds a new pipeline which enables caching in the build. This is added as a separate pipeline for now with the eventual goal of enabling for PR and/or CI builds. Documentation for Microsoft.MSBuildCache can be found in the GitHub repo: https://github.com/microsoft/MSBuildCache Preliminary numbers below. * [Baseline](https://dev.azure.com/ms/terminal/_build/results?buildId=579399&view=results): 12 min * [0% Cache hits](https://dev.azure.com/ms/terminal/_build/results?buildId=579419&view=results): 16 mins * [100% cache hits](https://dev.azure.com/ms/terminal/_build/results?buildId=579427&view=results): 3 mins
1 parent bd116e3 commit 613a539

File tree

7 files changed

+139
-20
lines changed

7 files changed

+139
-20
lines changed

‎.github/actions/spelling/expect/expect.txt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -738,6 +738,7 @@ HABCDEF
738738
Hackathon
739739
HALTCOND
740740
HANGEUL
741+
hardlinks
741742
hashalg
742743
HASSTRINGS
743744
hbitmap
@@ -1080,6 +1081,7 @@ MOUSEFIRST
10801081
MOUSEHWHEEL
10811082
MOVESTART
10821083
msb
1084+
msbuildcache
10831085
msctf
10841086
msctls
10851087
msdata
@@ -1489,6 +1491,7 @@ reparented
14891491
reparenting
14901492
replatformed
14911493
Replymessage
1494+
reportfileaccesses
14921495
repositorypath
14931496
Requiresx
14941497
rerasterize
@@ -2005,6 +2008,7 @@ wincontypes
20052008
WINCORE
20062009
windbg
20072010
WINDEF
2011+
windir
20082012
windll
20092013
WINDOWALPHA
20102014
windowdpiapi

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -283,3 +283,6 @@ MSG*.bin
283283
profiles.json
284284
*.metaproj
285285
*.swp
286+
287+
# MSBuildCache
288+
/MSBuildCacheLogs/

‎Directory.Build.props‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
<Project>
2+
<!--
3+
NOTE! This file gets written-over entirely by the release builds.
4+
Any build logic in this file must be optional and only apply to non-release builds.
5+
-->
6+
7+
<!-- MsBuildCache -->
8+
<PropertyGroup>
9+
<!-- Off by default -->
10+
<MsBuildCacheEnabled Condition="'$(MsBuildCacheEnabled)' == ''">false</MsBuildCacheEnabled>
11+
12+
<!-- Always off during package restore -->
13+
<MsBuildCacheEnabled Condition=" '$(ExcludeRestorePackageImports)' == 'true' ">false</MsBuildCacheEnabled>
14+
15+
<!-- In Azure pipelines, use Pipeline Caching as the cache storage backend. Otherwise, use the local cache. -->
16+
<MSBuildCachePackageName Condition="'$(TF_BUILD)' != ''">Microsoft.MSBuildCache.AzurePipelines</MSBuildCachePackageName>
17+
<MSBuildCachePackageName Condition="'$(MSBuildCachePackageName)' == ''">Microsoft.MSBuildCache.Local</MSBuildCachePackageName>
18+
</PropertyGroup>
19+
20+
<PropertyGroup Condition="'$(MSBuildCacheEnabled)' == 'true'">
21+
<!-- Change this to bust the cache -->
22+
<MSBuildCacheCacheUniverse Condition="'$(MSBuildCacheCacheUniverse)' == ''">202310210737</MSBuildCacheCacheUniverse>
23+
24+
<!--
25+
Visual Studio telemetry reads various ApplicationInsights.config files and other files after the project is finished, likely in a detached process.
26+
This is acceptable and should not impact cache correctness.
27+
-->
28+
<MSBuildCacheAllowFileAccessAfterProjectFinishFilePatterns>
29+
$(MSBuildCacheAllowFileAccessAfterProjectFinishFilePatterns);
30+
\**\ApplicationInsights.config;
31+
$(LocalAppData)\Microsoft\VSApplicationInsights\**;
32+
$(LocalAppData)\Microsoft\Windows\INetCache\**;
33+
A:\;
34+
E:\;
35+
$(windir)\**;
36+
</MSBuildCacheAllowFileAccessAfterProjectFinishFilePatterns>
37+
38+
<!--
39+
This repo uses a common output directory with many projects writing duplicate outputs. Allow everything, but note this costs some performance in the form of requiring
40+
the cache to use copies instead of hardlinks when pulling from cache.
41+
-->
42+
<MSBuildCacheIdenticalDuplicateOutputPatterns>$(MSBuildCacheIdenticalDuplicateOutputPatterns);bin\**</MSBuildCacheIdenticalDuplicateOutputPatterns>
43+
44+
<!-- version of MSBuildCache is not part of the cache key -->
45+
<PackagesConfigFile>$(MSBuildThisFileDirectory)\dep\nuget\packages.config</PackagesConfigFile>
46+
<MSBuildCacheIgnoredInputPatterns>$(MSBuildCacheIgnoredInputPatterns);$(PackagesConfigFile)</MSBuildCacheIgnoredInputPatterns>
47+
</PropertyGroup>
48+
49+
<PropertyGroup Condition="'$(MSBuildCacheEnabled)' == 'true' and '$(MSBuildCachePackageRoot)' == ''">
50+
<PackagesConfigContents>$([System.IO.File]::ReadAllText("$(PackagesConfigFile)"))</PackagesConfigContents>
51+
<MSBuildCachePackageVersion>$([System.Text.RegularExpressions.Regex]::Match($(PackagesConfigContents), 'Microsoft.MSBuildCache.*?version="(.*?)"').Groups[1].Value)</MSBuildCachePackageVersion>
52+
<MSBuildCachePackageRoot>$(MSBuildThisFileDirectory)packages\$(MSBuildCachePackageName).$(MSBuildCachePackageVersion)</MSBuildCachePackageRoot>
53+
<MSBuildCacheSharedCompilationPackageRoot>$(MSBuildThisFileDirectory)packages\Microsoft.MSBuildCache.SharedCompilation.$(MSBuildCachePackageVersion)</MSBuildCacheSharedCompilationPackageRoot>
54+
</PropertyGroup>
55+
56+
<ImportGroup Condition="'$(MSBuildCacheEnabled)' == 'true'">
57+
<Import Project="$(MSBuildCachePackageRoot)\build\$(MSBuildCachePackageName).props" />
58+
<Import Project="$(MSBuildCacheSharedCompilationPackageRoot)\build\Microsoft.MSBuildCache.SharedCompilation.props" />
59+
</ImportGroup>
60+
</Project>

‎Directory.Build.targets‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
<Project>
2+
<!--
3+
NOTE! This file gets written-over entirely by the release builds.
4+
Any build logic in this file must be optional and only apply to non-release builds.
5+
-->
6+
7+
<ImportGroup Condition="'$(MSBuildCacheEnabled)' == 'true'">
8+
<Import Project="$(MSBuildCachePackageRoot)\build\$(MSBuildCachePackageName).targets" />
9+
<Import Project="$(MSBuildCacheSharedCompilationPackageRoot)\build\Microsoft.MSBuildCache.SharedCompilation.targets" />
10+
</ImportGroup>
11+
</Project>

‎build/pipelines/ci-caching.yml‎

Lines changed: 24 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,32 @@
11
trigger:
22
batch: true
3-
# branches:
4-
# include:
5-
# - main
6-
# - feature/*
7-
# - gh-readonly-queue/*
8-
# paths:
9-
# exclude:
10-
# - doc/*
11-
# - samples/*
12-
# - tools/*
3+
branches:
4+
include:
5+
- main
6+
- feature/*
7+
- gh-readonly-queue/*
8+
paths:
9+
exclude:
10+
- doc/*
11+
- samples/*
12+
- tools/*
1313

14-
#pr:
15-
# branches:
16-
# include:
17-
# - main
18-
# - feature/*
19-
# paths:
20-
# exclude:
21-
# - doc/*
22-
# - samples/*
23-
# - tools/*
14+
pr:
15+
branches:
16+
include:
17+
- main
18+
- feature/*
19+
paths:
20+
exclude:
21+
- doc/*
22+
- samples/*
23+
- tools/*
2424

2525
variables:
2626
- name: runCodesignValidationInjectionBG
2727
value: false
28+
- name: EnablePipelineCache
29+
value: true
2830

2931
# 0.0.yyMM.dd##
3032
# 0.0.1904.0900
@@ -81,6 +83,8 @@ stages:
8183
buildConfigurations: [Release]
8284
buildEverything: true
8385
keepAllExpensiveBuildOutputs: false
86+
${{ if eq(variables['System.PullRequest.IsFork'], 'False') }}:
87+
enableCaching: true
8488

8589
- ${{ if eq(parameters.runTests, true) }}:
8690
- stage: Test_${{ platform }}

‎build/pipelines/templates-v2/job-build-project.yml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,9 @@ parameters:
6868
- name: signingIdentity
6969
type: object
7070
default: {}
71+
- name: enableCaching
72+
type: boolean
73+
default: false
7174

7275
jobs:
7376
- job: ${{ parameters.jobName }}
@@ -95,6 +98,7 @@ jobs:
9598
# Yup.
9699
BuildTargetParameter: ' '
97100
SelectedSigningFragments: ' '
101+
MSBuildCacheParameters: ' '
98102
# When building the unpackaged distribution, build it in portable mode if it's Canary-branded
99103
${{ if eq(parameters.branding, 'Canary') }}:
100104
UnpackagedBuildArguments: -PortableMode
@@ -111,6 +115,7 @@ jobs:
111115
clean: true
112116
submodules: true
113117
persistCredentials: True
118+
114119
# This generates either nothing for BuildTargetParameter, or /t:X;Y;Z, to control targets later.
115120
- pwsh: |-
116121
If (-Not [bool]::Parse("${{ parameters.buildEverything }}")) {
@@ -139,6 +144,17 @@ jobs:
139144
}
140145
displayName: Prepare Build and Sign Targets
141146

147+
- ${{ if eq(parameters.enableCaching, true) }}:
148+
- pwsh: |-
149+
$MSBuildCacheParameters = ""
150+
$MSBuildCacheParameters += " -graph"
151+
$MSBuildCacheParameters += " -reportfileaccesses"
152+
$MSBuildCacheParameters += " -p:MSBuildCacheEnabled=true"
153+
$MSBuildCacheParameters += " -p:MSBuildCacheLogDirectory=$(Build.SourcesDirectory)\MSBuildCacheLogs"
154+
Write-Host "MSBuildCacheParameters: $MSBuildCacheParameters"
155+
Write-Host "##vso[task.setvariable variable=MSBuildCacheParameters]$MSBuildCacheParameters"
156+
displayName: Prepare MSBuildCache variables
157+
142158
- pwsh: |-
143159
.\build\scripts\Generate-ThirdPartyNotices.ps1 -MarkdownNoticePath .\NOTICE.md -OutputPath .\src\cascadia\CascadiaPackage\NOTICE.html
144160
displayName: Generate NOTICE.html from NOTICE.md
@@ -160,21 +176,37 @@ jobs:
160176
${{ parameters.additionalBuildOptions }}
161177
/bl:$(Build.SourcesDirectory)\msbuild.binlog
162178
$(BuildTargetParameter)
179+
$(MSBuildCacheParameters)
163180
platform: $(BuildPlatform)
164181
configuration: $(BuildConfiguration)
182+
msbuildArchitecture: x64
165183
maximumCpuCount: true
184+
${{ if eq(parameters.enableCaching, true) }}:
185+
env:
186+
SYSTEM_ACCESSTOKEN: $(System.AccessToken)
166187

167188
- ${{ if eq(parameters.publishArtifacts, true) }}:
168189
- publish: $(Build.SourcesDirectory)/msbuild.binlog
169190
artifact: logs-$(BuildPlatform)-$(BuildConfiguration)${{ parameters.artifactStem }}
170191
condition: always()
171192
displayName: Publish Build Log
193+
- ${{ if eq(parameters.enableCaching, true) }}:
194+
- publish: $(Build.SourcesDirectory)\MSBuildCacheLogs
195+
artifact: logs-msbuildcache-$(BuildPlatform)-$(BuildConfiguration)${{ parameters.artifactStem }}
196+
condition: always()
197+
displayName: Publish MSBuildCache Logs
172198
- ${{ else }}:
173199
- task: CopyFiles@2
174200
displayName: Copy Build Log
175201
inputs:
176202
contents: $(Build.SourcesDirectory)/msbuild.binlog
177203
TargetFolder: $(Terminal.BinDir)
204+
- ${{ if eq(parameters.enableCaching, true) }}:
205+
- task: CopyFiles@2
206+
displayName: Copy MSBuildCache Logs
207+
inputs:
208+
contents: $(Build.SourcesDirectory)/MSBuildCacheLogs/**
209+
TargetFolder: $(Terminal.BinDir)/MSBuildCacheLogs
178210

179211
# This saves ~2GiB per architecture. We won't need these later.
180212
# Removes:

‎dep/nuget/packages.config‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,4 +17,9 @@
1717
<package id="Newtonsoft.Json" version="13.0.1" targetFramework="net45" />
1818
<package id="Selenium.Support" version="3.5.0" targetFramework="net45" />
1919
<package id="Selenium.WebDriver" version="3.5.0" targetFramework="net45" />
20+
21+
<!-- MSBuildCache -->
22+
<package id="Microsoft.MSBuildCache.AzurePipelines" version="0.1.273-preview" />
23+
<package id="Microsoft.MSBuildCache.Local" version="0.1.273-preview" />
24+
<package id="Microsoft.MSBuildCache.SharedCompilation" version="0.1.273-preview" />
2025
</packages>

0 commit comments

Comments
 (0)