Skip to content

Commit 952e162

Browse files
dseefelddseefeld
authored
[master] Add incremental build (#1740)
* Add incremental build (#1700) * Add semaphores for incremental build * Updates based on PR review comments * One more PR comment update Conflicts: build.proj repos/Directory.Build.targets repos/core-sdk.proj repos/core-setup.common.targets repos/coreclr-portable.proj repos/toolset.proj support/tarball/build.sh tools-local/init-build.proj * Trim trailing whitespace * Fix spacing issue
1 parent 4cd1226 commit 952e162

16 files changed

+231
-71
lines changed

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -281,3 +281,4 @@ testing-smoke/
281281
# UpdateDependenciesAndSubmitPullRequest: if no upgrades are performed, the
282282
# sanity check fails when "git status" shows this file.
283283
msbuild.ProjectImports.zip
284+
Build_*.ProjectImports.zip

‎Directory.Build.props

+6-3
Original file line numberDiff line numberDiff line change
@@ -211,9 +211,12 @@
211211
<RootRepoTests>known-good-tests</RootRepoTests>
212212
</PropertyGroup>
213213

214-
<ItemGroup>
215-
<SourceBuiltPackage Include="$(SourceBuiltPackagesPath)/*.nupkg" />
216-
</ItemGroup>
214+
<!--
215+
Semaphore path for incremental builds
216+
-->
217+
<PropertyGroup>
218+
<CompletedSemaphorePath>$(BaseIntermediatePath)semaphores/</CompletedSemaphorePath>
219+
</PropertyGroup>
217220

218221
<!-- CLI internal version is statically set by us to a version that will never show up in the wild.
219222
This ensures we will never restore a public version instead of our source-built version. We

‎Directory.Build.targets

+5-1
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,14 @@
1313
<Target Name="AcquireDarc"
1414
Condition="
1515
'$(OfflineBuild)' != 'true' and
16-
'$(SkipAcquireDarc)' != 'true'">
16+
'$(SkipAcquireDarc)' != 'true'"
17+
Inputs="$(MSBuildThisFileDirectory)tools-local\acquire-darc\acquire-darc.proj"
18+
Outputs="$(CompletedSemaphorePath)AcquireDarc.complete">
1719
<MSBuild
1820
Projects="$(MSBuildThisFileDirectory)tools-local\acquire-darc\acquire-darc.proj"
1921
Targets="AcquireDarcCore" />
22+
23+
<WriteLinesToFile File="$(CompletedSemaphorePath)AcquireDarc.complete" Overwrite="true" />
2024
</Target>
2125

2226
<Target Name="GetCommonDarcCloneCommand" DependsOnTargets="AcquireDarc">

‎build-source-tarball.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,6 @@ mkdir -p "$FULL_TARBALL_ROOT"
143143
MAIN_BUILD_ARGS+=( "/p:TarballRoot=$FULL_TARBALL_ROOT" )
144144
MAIN_BUILD_ARGS+=( "/p:PackSourceBuildTarball=true" )
145145

146-
$SCRIPT_ROOT/build.sh ${MAIN_BUILD_ARGS[@]} "$@"
146+
$SCRIPT_ROOT/build.sh ${MAIN_BUILD_ARGS[@]} /bl:$SCRIPT_ROOT/artifacts/log/Debug/BuildTarball_$(date +"%m%d%H%M%S").binlog "$@"
147147

148148
echo "Done. Tarball created: $TARBALL_ROOT"

‎build.proj

+35-9
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@
4444
<Target Name="InitBuild">
4545
<MSBuild Projects="tools-local/init-build.proj" />
4646
</Target>
47-
47+
4848
<Target Name="Clone">
4949
<MSBuild Projects="tools-local/init-build.proj" Targets="FetchSources" />
5050
</Target>
@@ -55,18 +55,36 @@
5555

5656
<Target Name="DownloadSourceBuildReferencePackages"
5757
AfterTargets="Build"
58-
Condition="'$(OfflineBuild)' != 'true' and '$(OS)' != 'Windows_NT' and '$(SkipDownloadingReferencePackages)' != 'true'">
59-
<DownloadFileSB
58+
Condition="'$(OfflineBuild)' != 'true' and '$(OS)' != 'Windows_NT' and '$(SkipDownloadingReferencePackages)' != 'true'"
59+
Inputs="$(MSBuildProjectFullPath)"
60+
Outputs="$(CompletedSemaphorePath)DownloadSourceBuildReferencePackages.complete" >
61+
<PropertyGroup Condition="'$(DownloadSourceBuildReferencePackagesTimeoutSeconds)' == ''">
62+
<DownloadSourceBuildReferencePackagesTimeoutSeconds>600</DownloadSourceBuildReferencePackagesTimeoutSeconds>
63+
</PropertyGroup>
64+
65+
<DownloadFileSB
6066
SourceUrl="$(ReferencePackagesTarballUrl)$(ReferencePackagesTarballName).$(PrivateSourceBuildReferencePackagesPackageVersion).tar.gz"
61-
DestinationFolder="$(ExternalTarballsDir)" />
67+
DestinationFolder="$(ExternalTarballsDir)"
68+
TimeoutSeconds="$(DownloadSourceBuildReferencePackagesTimeoutSeconds)" />
69+
70+
<WriteLinesToFile File="$(CompletedSemaphorePath)DownloadSourceBuildReferencePackages.complete" Overwrite="true" />
6271
</Target>
6372

6473
<Target Name="DownloadSourceBuiltArtifacts"
6574
AfterTargets="Build"
66-
Condition="'$(OfflineBuild)' != 'true' and '$(OS)' != 'Windows_NT' and '$(SkipDownloadingPreviouslySourceBuiltPackages)' != 'true'">
67-
<DownloadFileSB
75+
Condition="'$(OfflineBuild)' != 'true' and '$(OS)' != 'Windows_NT' and '$(SkipDownloadingPreviouslySourceBuiltPackages)' != 'true'"
76+
Inputs="$(MSBuildProjectFullPath)"
77+
Outputs="$(CompletedSemaphorePath)DownloadSourceBuiltArtifacts.complete" >
78+
<PropertyGroup Condition="'$(DownloadSourceBuiltArtifactsTimeoutSeconds)' == ''">
79+
<DownloadSourceBuiltArtifactsTimeoutSeconds>1500</DownloadSourceBuiltArtifactsTimeoutSeconds>
80+
</PropertyGroup>
81+
82+
<DownloadFileSB
6883
SourceUrl="$(SourceBuiltArtifactsTarballUrl)$(SourceBuiltArtifactsTarballName).$(PrivateSourceBuiltArtifactsPackageVersion).tar.gz"
69-
DestinationFolder="$(ExternalTarballsDir)" />
84+
DestinationFolder="$(ExternalTarballsDir)"
85+
TimeoutSeconds="$(DownloadSourceBuiltArtifactsTimeoutSeconds)" />
86+
87+
<WriteLinesToFile File="$(CompletedSemaphorePath)DownloadSourceBuiltArtifacts.complete" Overwrite="true" />
7088
</Target>
7189

7290
<!-- After building, generate a prebuilt usage report. -->
@@ -88,23 +106,31 @@
88106

89107
<Target Name="ReportPoisonUsage"
90108
AfterTargets="Build"
91-
Condition="'$(EnablePoison)' == 'true' and '$(OfflineBuild)' == 'true'">
109+
Condition="'$(EnablePoison)' == 'true' and '$(OfflineBuild)' == 'true'"
110+
Inputs="$(MSBuildProjectFullPath)"
111+
Outputs="$(CompletedSemaphorePath)ReportPoisonUsage.complete" >
92112
<ItemGroup>
93113
<FinalCliTarball Include="$(SourceBuiltTarBallPath)**/*$(TarBallExtension)" />
94114
</ItemGroup>
95115
<CheckForPoison FilesToCheck="@(FinalCliTarball)"
96116
HashCatalogFilePath="$(PoisonReportDataFile)"
97117
MarkerFileName="$(PoisonMarkerFile)"
98118
PoisonReportOutputFilePath="$(PoisonUsageReportFile)" />
119+
120+
<WriteLinesToFile File="$(CompletedSemaphorePath)ReportPoisonUsage.complete" Overwrite="true" />
99121
</Target>
100122

101-
<Target Name="GeneratePrebuiltBurndownData">
123+
<Target Name="GeneratePrebuiltBurndownData"
124+
Inputs="$(MSBuildProjectFullPath)"
125+
Outputs="$(CompletedSemaphorePath)GeneratePrebuiltBurndownData.complete" >
102126
<WriteUsageBurndownData RootDirectory="$(ProjectDir)"
103127
PrebuiltBaselineFile="$(OnlineBaselineDataFile)"
104128
OutputFilePath="$(OnlinePrebuiltBurndownDataFile)" />
105129
<WriteUsageBurndownData RootDirectory="$(ProjectDir)"
106130
PrebuiltBaselineFile="$(OfflineBaselineDataFile)"
107131
OutputFilePath="$(OfflinePrebuiltBurndownDataFile)" />
132+
133+
<WriteLinesToFile File="$(CompletedSemaphorePath)GeneratePrebuiltBurndownData.complete" Overwrite="true" />
108134
</Target>
109135

110136
<Target Name="RunSmokeTest" DependsOnTargets="GetProdConBlobFeedUrl">

‎build.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -111,5 +111,5 @@ scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
111111
if [ "$alternateTarget" == "true" ]; then
112112
"$CLIPATH/dotnet" $SDKPATH/MSBuild.dll "$scriptroot/build.proj" /bl:source-build-test.binlog /flp:v=diag /clp:v=m "$@"
113113
else
114-
"$scriptroot/eng/common/build.sh" --restore --build -c Release --warnaserror false -bl /flp:v=diag "$@"
114+
"$scriptroot/eng/common/build.sh" --restore --build -c Release --warnaserror false /bl:$scriptroot/artifacts/log/Debug/Build_$(date +"%m%d%H%M%S").binlog /flp:v=diag "$@"
115115
fi

‎repos/Directory.Build.props

+7-2
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,11 @@
1717
<RepoConsoleLogFile>$(LoggingDir)$(RepositoryName).log</RepoConsoleLogFile>
1818
<RedirectRepoOutputToLog Condition="'$(MinimalConsoleLogOutput)' == 'true'">&gt;&gt; $(RepoConsoleLogFile) 2&gt;&amp;1</RedirectRepoOutputToLog>
1919
<IsStable Condition="'$(UseStableVersions)' == 'true'">true</IsStable>
20+
21+
<!--
22+
Repo specific semaphore path for incremental build
23+
-->
24+
<RepoCompletedSemaphorePath>$(CompletedSemaphorePath)$(RepositoryName)/</RepoCompletedSemaphorePath>
2025
</PropertyGroup>
2126

2227
<PropertyGroup Condition="'$(OS)' == 'Windows_NT'">
@@ -77,8 +82,8 @@
7782
<!-- We pass '-ci', but also apply ci mode via env var for edge cases. (E.g. misbehaving inner builds.) -->
7883
<EnvironmentVariables Include="ContinuousIntegrationBuild=true" />
7984

80-
<!-- Turn off node reuse for source build because repos use conflicting versions
81-
of compilers which cause assembly load errors.
85+
<!-- Turn off node reuse for source build because repos use conflicting versions
86+
of compilers which cause assembly load errors.
8287
See https://github.com/dotnet/source-build/issues/541 -->
8388
<EnvironmentVariables Include="MSBUILDDISABLENODEREUSE=1" />
8489

0 commit comments

Comments
 (0)