Skip to content

Commit 0eb8d7d

Browse files
freddydkfreddydk
authored
Use artifact index for Get-BcArtifactUrl (#3859)
Co-authored-by: freddydk <[email protected]>
1 parent 88631bd commit 0eb8d7d

5 files changed

+227
-126
lines changed

‎Artifacts/Get-BCArtifactUrl.ps1

+25-103
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@ try {
135135
if (-not $storageAccount.Contains('.')) {
136136
$storageAccount += ".blob.core.windows.net"
137137
}
138-
$BaseUrl = ReplaceCDN -sourceUrl "https://$storageAccount/$($Type.ToLowerInvariant())/" -useBlobUrl:($bcContainerHelperConfig.DoNotUseCdnForArtifacts)
138+
$BaseUrl = ReplaceCDN -sourceUrl "https://$storageAccount/$($Type.ToLowerInvariant())/"
139139
$storageAccount = ReplaceCDN -sourceUrl $storageAccount -useBlobUrl
140140

141141
if ($storageAccount -eq 'bcinsider.blob.core.windows.net' -and !$accept_insiderEULA) {
@@ -181,7 +181,7 @@ try {
181181
$artifactUrl
182182
}
183183
else {
184-
$GetListUrl = "https://$storageAccount/$($Type.ToLowerInvariant())/?comp=list&restype=container"
184+
$versionPrefix = ''
185185
if ($select -eq 'SecondToLastMajor') {
186186
if ($version) {
187187
throw "You cannot specify a version when asking for the Second To Last Major version"
@@ -196,138 +196,60 @@ try {
196196
if ($dots -ne 3 -or !([Version]::TryParse($version, [ref] $closestToVersion))) {
197197
throw "Version number must be in the format 1.2.3.4 when you want to get the closest artifact Url"
198198
}
199-
$GetListUrl += "&prefix=$($closestToVersion.Major).$($closestToVersion.Minor)."
199+
$versionPrefix = "$($closestToVersion.Major).$($closestToVersion.Minor)."
200200
}
201201
elseif (!([string]::IsNullOrEmpty($version))) {
202202
$dots = ($version.ToCharArray() -eq '.').Count
203203
if ($dots -lt 3) {
204204
# avoid 14.1 returning 14.10, 14.11 etc.
205205
$version = "$($version.TrimEnd('.'))."
206206
}
207-
$GetListUrl += "&prefix=$($Version)"
207+
$versionPrefix = $Version
208208
}
209-
210-
$Artifacts = @()
211-
$nextMarker = ''
212-
$currentMarker = ''
213-
$downloadAttempt = 1
214-
$downloadRetryAttempts = 10
215-
do {
216-
if ($currentMarker -ne $nextMarker)
217-
{
218-
$currentMarker = $nextMarker
219-
$downloadAttempt = 1
220-
}
221-
Write-Verbose "Download String $GetListUrl$nextMarker"
222-
try
223-
{
224-
$Response = Invoke-RestMethod -UseBasicParsing -ContentType "application/json; charset=UTF8" -Uri "$GetListUrl$nextMarker"
225-
if (([int]$Response[0]) -eq 239 -and ([int]$Response[1]) -eq 187 -and ([int]$Response[2]) -eq 191) {
226-
# Remove UTF8 BOM
227-
$response = $response.Substring(3)
228-
}
229-
if (([int]$Response[0]) -eq 65279) {
230-
# Remove Unicode BOM (PowerShell 7.4)
231-
$response = $response.Substring(1)
232-
}
233-
$enumerationResults = ([xml]$Response).EnumerationResults
234-
235-
if ($enumerationResults.Blobs) {
236-
if (($After) -or ($Before)) {
237-
$artifacts += $enumerationResults.Blobs.Blob | % {
238-
if ($after) {
239-
$blobModifiedDate = [DateTime]::Parse($_.Properties."Last-Modified")
240-
if ($before) {
241-
if ($blobModifiedDate -lt $before -and $blobModifiedDate -gt $after) {
242-
$_.Name
243-
}
244-
}
245-
elseif ($blobModifiedDate -gt $after) {
246-
$_.Name
247-
}
248-
}
249-
else {
250-
$blobModifiedDate = [DateTime]::Parse($_.Properties."Last-Modified")
251-
if ($blobModifiedDate -lt $before) {
252-
$_.Name
253-
}
254-
}
255-
}
256-
}
257-
else {
258-
$artifacts += $enumerationResults.Blobs.Blob.Name
259-
}
260-
}
261-
$nextMarker = $enumerationResults.NextMarker
262-
if ($nextMarker) {
263-
$nextMarker = "&marker=$nextMarker"
264-
}
265-
}
266-
catch
267-
{
268-
$downloadAttempt += 1
269-
Write-Host "Error querying artifacts. Error message was $($_.Exception.Message)"
270-
Write-Host
271-
272-
if ($downloadAttempt -le $downloadRetryAttempts)
273-
{
274-
Write-Host "Repeating download attempt (" $downloadAttempt.ToString() " of " $downloadRetryAttempts.ToString() ")..."
275-
Write-Host
276-
}
277-
else
278-
{
279-
throw
280-
}
281-
}
282-
} while ($nextMarker)
283209

284-
if (!([string]::IsNullOrEmpty($country))) {
285-
# avoid confusion between base and se
286-
$countryArtifacts = $Artifacts | Where-Object { $_.EndsWith("/$country", [System.StringComparison]::InvariantCultureIgnoreCase) -and ($doNotCheckPlatform -or ($Artifacts.Contains("$($_.Split('/')[0])/platform"))) }
287-
if (!$countryArtifacts) {
288-
if (($type -eq "sandbox") -and ($bcContainerHelperConfig.mapCountryCode.PSObject.Properties.Name -eq $country)) {
289-
$country = $bcContainerHelperConfig.mapCountryCode."$country"
290-
$countryArtifacts = $Artifacts | Where-Object { $_.EndsWith("/$country", [System.StringComparison]::InvariantCultureIgnoreCase) -and ($doNotCheckPlatform -or ($Artifacts.Contains("$($_.Split('/')[0])/platform"))) }
291-
}
292-
}
293-
$Artifacts = $countryArtifacts
210+
$parameters = @{
211+
"storageAccount" = $storageAccount
212+
"Type" = $Type
213+
"VersionPrefix" = $versionPrefix
214+
"Country" = $country
215+
"doNotCheckPlatform" = $doNotCheckPlatform
216+
}
217+
if ($after) {
218+
$parameters["after"] = $after
219+
}
220+
if ($before) {
221+
$parameters["before"] = $before
222+
}
223+
if ($bcContainerHelperConfig.useIndexForArtifacts) {
224+
$artifacts = QueryArtifactsFromIndex @parameters
294225
}
295226
else {
296-
$Artifacts = $Artifacts | Where-Object { !($_.EndsWith("/platform", [System.StringComparison]::InvariantCultureIgnoreCase)) }
227+
$artifacts = QueryArtifactsFromStorage @parameters
297228
}
298229

299230
switch ($Select) {
300231
'All' {
301-
$Artifacts = $Artifacts |
302-
Sort-Object { [Version]($_.Split('/')[0]) }
232+
# Artifacts are sorted
303233
}
304234
'Latest' {
305-
$Artifacts = $Artifacts |
306-
Sort-Object { [Version]($_.Split('/')[0]) } |
307-
Select-Object -Last 1
235+
$Artifacts = $Artifacts | Select-Object -Last 1
308236
}
309237
'First' {
310-
$Artifacts = $Artifacts |
311-
Sort-Object { [Version]($_.Split('/')[0]) } |
312-
Select-Object -First 1
238+
$Artifacts = $Artifacts | Select-Object -First 1
313239
}
314240
'SecondToLastMajor' {
315-
$Artifacts = $Artifacts |
316-
Sort-Object -Descending { [Version]($_.Split('/')[0]) }
317-
$latest = $Artifacts | Select-Object -First 1
241+
$latest = $Artifacts | Select-Object -Last 1
318242
if ($latest) {
319243
$latestversion = [Version]($latest.Split('/')[0])
320244
$Artifacts = $Artifacts |
321245
Where-Object { ([Version]($_.Split('/')[0])).Major -ne $latestversion.Major } |
322-
Select-Object -First 1
246+
Select-Object -Last 1
323247
}
324248
else {
325249
$Artifacts = @()
326250
}
327251
}
328252
'Closest' {
329-
$Artifacts = $Artifacts |
330-
Sort-Object { [Version]($_.Split('/')[0]) }
331253
$closest = $Artifacts |
332254
Where-Object { [Version]($_.Split('/')[0]) -ge $closestToVersion } |
333255
Select-Object -First 1

‎BC.HelperFunctions.ps1

+1-1
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ function Get-ContainerHelperConfig {
108108
"NoOfSecondsToSleepAfterPublishBcContainerApp" = 1
109109
"RenewClientContextBetweenTests" = $false
110110
"DebugMode" = $false
111-
"DoNotUseCdnForArtifacts" = $false
111+
"UseIndexForArtifacts" = $false
112112
"MinimumDotNetRuntimeVersionStr" = "6.0.16"
113113
"MinimumDotNetRuntimeVersionUrl" = 'https://download.visualstudio.microsoft.com/download/pr/ca13c6f1-3107-4cf8-991c-f70edc1c1139/a9f90579d827514af05c3463bed63c22/dotnet-sdk-6.0.408-win-x64.zip'
114114
"MSSymbolsNuGetFeedUrl" = 'https://dynamicssmb2.pkgs.visualstudio.com/DynamicsBCPublicFeeds/_packaging/MSSymbols/nuget/v3/index.json'

‎Common/Download-File.ps1

+11-21
Original file line numberDiff line numberDiff line change
@@ -70,30 +70,20 @@ function Download-File {
7070
Invoke-WebRequest -UseBasicParsing -Uri $sourceUrl -OutFile $destinationFile
7171
}
7272
else {
73-
if ($bcContainerHelperConfig.DoNotUseCdnForArtifacts -or $sourceUrl -like 'https://bcinsider*.net/*') {
74-
# Do not use CDN when configured or bcinsider
75-
$sourceUrl = ReplaceCDN -sourceUrl $sourceUrl -useBlobUrl
76-
$timeout += $timeout
77-
}
78-
try {
79-
DownloadFileLow -sourceUrl (ReplaceCDN -sourceUrl $sourceUrl) -destinationFile $destinationFile -dontOverwrite:$dontOverwrite -timeout $timeout -headers $headers
80-
}
81-
catch {
73+
$waitTime = 2
74+
$sourceUrl = ReplaceCDN -sourceUrl $sourceUrl
75+
while ($true) {
8276
try {
83-
$waittime = 2 + (Get-Random -Maximum 5 -Minimum 0)
84-
$newSourceUrl = ReplaceCDN -sourceUrl $sourceUrl -useBlobUrl
85-
if ($sourceUrl -eq $newSourceUrl) {
86-
Write-Host "Error downloading..., retrying in $waittime seconds..."
87-
}
88-
else {
89-
Write-Host "Could not download from CDN..., retrying from blob storage in $waittime seconds..."
90-
$timeout += $timeout
91-
}
92-
Start-Sleep -Seconds $waittime
93-
DownloadFileLow -sourceUrl $newSourceUrl -destinationFile $destinationFile -dontOverwrite:$dontOverwrite -timeout $timeout -headers $headers
77+
DownloadFileLow -sourceUrl $sourceUrl -destinationFile $destinationFile -dontOverwrite:$dontOverwrite -timeout $timeout -headers $headers
78+
break
9479
}
9580
catch {
96-
throw (GetExtendedErrorMessage $_)
81+
$waitTime += $waitTime
82+
if ($_.Exception.Message -like '*404*' -or $waitTime -gt 60) {
83+
throw (GetExtendedErrorMessage $_)
84+
}
85+
Write-Host "Error downloading..., retrying in $waitTime seconds..."
86+
Start-Sleep -Seconds $waitTime
9787
}
9888
}
9989
}

0 commit comments

Comments
 (0)