Reindex testing improvements and fixes

build/build-variables.yml

@@ -16,6 +16,14 @@ variables:
   DeploymentEnvironmentNameR4BSql: '$(DeploymentEnvironmentNameR4B)-sql'
   DeploymentEnvironmentNameR5: '$(DeploymentEnvironmentName)-r5'
   DeploymentEnvironmentNameR5Sql: '$(DeploymentEnvironmentNameR5)-sql'
+  # Key Vault names (shorter due to 24 character limit)
+  KeyVaultNameSql: '$(KeyVaultBaseName)-sql'
+  KeyVaultNameR4: '$(KeyVaultBaseName)-r4'
+  KeyVaultNameR4Sql: '$(KeyVaultNameR4)-sql'
+  KeyVaultNameR4B: '$(KeyVaultBaseName)-r4b'
+  KeyVaultNameR4BSql: '$(KeyVaultNameR4B)-sql'
+  KeyVaultNameR5: '$(KeyVaultBaseName)-r5'
+  KeyVaultNameR5Sql: '$(KeyVaultNameR5)-sql'
   TestEnvironmentUrl: 'https://$(DeploymentEnvironmentName).azurewebsites.net'
   # These variables are not used in the deployment scripts, but are used in the E2E tests files.
   TestEnvironmentUrl_Sql: 'https://$(DeploymentEnvironmentName)-sql.azurewebsites.net'

build/ci-variables.yml

@@ -4,6 +4,8 @@ variables:
     resourceGroupRoot: 'msh-fhir-ci4'
     appServicePlanName: '$(resourceGroupRoot)-linux'
     DeploymentEnvironmentName: '$(resourceGroupRoot)'
+    # CI uses the same name for Key Vaults as the deployment environment (no BuildId suffix needed)
+    KeyVaultBaseName: '$(DeploymentEnvironmentName)'
     ResourceGroupName: '$(resourceGroupRoot)'
     CrucibleEnvironmentUrl: 'https://crucible.mshapis.com/'
     TestEnvironmentName: 'OSS CI'

build/jobs/add-aad-test-environment.yml

@@ -32,22 +32,10 @@ steps:
       $password =  ConvertTo-SecureString -AsPlainText $password_raw -Force
       $clientSecretCredential = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $clientId, $password
 
-      # If a deleted keyvault with purge protection exists, try to restore it.
       $environmentName = "$(DeploymentEnvironmentName)".ToLower() -replace "\.", "" 
+      $keyVaultName = "$(KeyVaultBaseName)-ts".ToLower() -replace "\.", ""
       Write-Host "Installed module and set variables"
 
-      $vaultName = "${environmentName}-ts"
-      $vaultLocation = "westus"
-      $vaultResourceGroupName = "$(ResourceGroupName)"
-      if (Get-AzKeyVault -VaultName $vaultName -Location $vaultLocation -InRemovedState)
-      {
-          Write-Host "Attempting to restore vault ${vaultName}"
-
-          Undo-AzKeyVaultRemoval -VaultName $vaultName -ResourceGroupName $vaultResourceGroupName -Location $vaultLocation
-          Write-Host "KeyVault $vaultName is restored"
-      }
-      Write-Host "Restored keyvaults"
-
       # Connect to Microsoft Graph using client credentials
       Write-Host "Connecting to Microsoft Graph"
       Connect-MgGraph -TenantId $tenantId -ClientSecretCredential $clientSecretCredential
@@ -57,4 +45,4 @@ steps:
       Import-Module $(System.DefaultWorkingDirectory)/release/scripts/PowerShell/FhirServerRelease/FhirServerRelease.psd1
 
       Write-Host "Imported modules"
-      $output = Add-AadTestAuthEnvironment -TestAuthEnvironmentPath $(System.DefaultWorkingDirectory)/testauthenvironment.json -EnvironmentName $environmentName -ResourceGroupName $(ResourceGroupName) -TenantAdminCredential $clientSecretCredential -TenantId $tenantId -ClientId $clientId -ClientSecret $password
+      $output = Add-AadTestAuthEnvironment -TestAuthEnvironmentPath $(System.DefaultWorkingDirectory)/testauthenvironment.json -EnvironmentName $environmentName -KeyVaultName $keyVaultName -ResourceGroupName $(UniqueResourceGroupName) -EnvironmentLocation $(ResourceGroupRegion) -TenantAdminCredential $clientSecretCredential -TenantId $tenantId -ClientId $clientId -ClientSecret $password

build/jobs/cleanup.yml

@@ -12,12 +12,71 @@ jobs:
       azurePowerShellVersion: latestVersion
       ScriptType: InlineScript
       Inline: |
-        Get-AzResourceGroup -Name $(ResourceGroupName) -ErrorVariable notPresent -ErrorAction SilentlyContinue 
+        $deploymentEnvName = "$(DeploymentEnvironmentName)"
+        $keyVaultBaseName = "$(KeyVaultBaseName)"
+
+        Get-AzResourceGroup -Name $(UniqueResourceGroupName) -ErrorVariable notPresent -ErrorAction SilentlyContinue 
         if ($notPresent) {
-          Write-Host "Resource group $(ResourceGroupName) not found"
+          Write-Host "Resource group $(UniqueResourceGroupName) not found"
         } else {
-          Write-Host "Deleting resource group $(ResourceGroupName)"
-          Remove-AzResourceGroup -Name $(ResourceGroupName) -Force -Verbose
+          Write-Host "Deleting resource group $(UniqueResourceGroupName)"
+          $maxRetries = 3
+          $retryCount = 0
+          $deleted = $false
+
+          while (-not $deleted -and $retryCount -lt $maxRetries) {
+            try {
+              Remove-AzResourceGroup -Name $(UniqueResourceGroupName) -Force -Verbose -ErrorAction Stop
+              $deleted = $true
+              Write-Host "Successfully deleted resource group $(UniqueResourceGroupName)"
+            }
+            catch {
+              $retryCount++
+              if ($retryCount -lt $maxRetries) {
+                Write-Warning "Failed to delete resource group (attempt $retryCount of $maxRetries). Retrying in 30 seconds... Error: $($_.Exception.Message)"
+                Start-Sleep -Seconds 30
+              } else {
+                Write-Warning "Failed to delete resource group after $maxRetries attempts. Error: $($_.Exception.Message)"
+              }
+            }
+          }
+        }
+
+        # Purge any soft-deleted Key Vaults from this build
+        # Vault names use KeyVaultBaseName (e.g., f20585pr5240-ts, f20585pr5240-r4, etc.)
+        $vaultPattern = "$keyVaultBaseName*"
+        Write-Host "Checking for soft-deleted Key Vaults matching pattern: $vaultPattern"
+        $softDeletedVaults = Get-AzKeyVault -InRemovedState -ErrorAction SilentlyContinue | Where-Object {
+          $_.VaultName -like $vaultPattern
+        }
+
+        if ($softDeletedVaults.Count -eq 0) {
+          Write-Host "No soft-deleted vaults found to purge"
+        } else {
+          Write-Host "Found $($softDeletedVaults.Count) soft-deleted vault(s) to purge"
+
+          $results = $softDeletedVaults | ForEach-Object -Parallel {
+            $vaultName = $_.VaultName
+            $vaultLocation = $_.Location
+            try {
+              Remove-AzKeyVault -VaultName $vaultName -Location $vaultLocation -InRemovedState -Force -ErrorAction Stop
+              [PSCustomObject]@{ Vault = $vaultName; Status = "Success" }
+            }
+            catch {
+              [PSCustomObject]@{ Vault = $vaultName; Status = "Failed: $($_.Exception.Message)" }
+            }
+          } -ThrottleLimit 10
+
+          $results | ForEach-Object {
+            if ($_.Status -eq "Success") {
+              Write-Host "Successfully purged vault: $($_.Vault)"
+            } else {
+              Write-Warning "Failed to purge vault $($_.Vault). $($_.Status)"
+            }
+          }
+
+          $successCount = ($results | Where-Object { $_.Status -eq "Success" }).Count
+          Write-Host "Completed purging: $successCount of $($softDeletedVaults.Count) vaults"
         }
 
 

build/jobs/e2e-setup.yml

@@ -17,7 +17,7 @@ steps:
       KeyVaultName: 'resolute-oss-tenant-info'
 
   - task: AzureKeyVault@1
-    displayName: 'Azure Key Vault: $(DeploymentEnvironmentName)-ts'
+    displayName: 'Azure Key Vault: $(KeyVaultBaseName)-ts'
     inputs:
       azureSubscription: $(ConnectedServiceName)
-      KeyVaultName: '$(DeploymentEnvironmentName)-ts'
+      KeyVaultName: '$(KeyVaultBaseName)-ts'

build/jobs/e2e-tests.yml

@@ -5,92 +5,29 @@ parameters:
   type: string
 - name: appServiceType
   type: string
+- name: categoryFilter
+  type: string
+  default: 'Category!=ExportLongRunning'
+- name: testRunTitleSuffix
+  type: string
+  default: ''
 
 steps:
   - template: e2e-tests-extract.yml
     parameters:
       version: ${{parameters.version}}
 
-  - task: AzurePowerShell@5
-    displayName: 'Set Variables'
-    inputs:
-      azureSubscription: $(ConnectedServiceName)
-      azurePowerShellVersion: latestVersion
-      ScriptType: inlineScript
-      Inline: |
-        $keyVault = "$(DeploymentEnvironmentName)-ts"
-        $secrets = Get-AzKeyVaultSecret -VaultName $keyVault
-
-        foreach($secret in $secrets)
-        {
-            $environmentVariableName = $secret.Name.Replace("--","_")
-
-            $secretValue = Get-AzKeyVaultSecret -VaultName $keyVault -Name $secret.Name
-            # Replace with -AsPlainText flag when v5.3 of the Az Module is supported
-            $plainValue = ([System.Net.NetworkCredential]::new("", $secretValue.SecretValue).Password).ToString()
-            if([string]::IsNullOrEmpty($plainValue))
-            {
-                throw "$($secret.Name) is empty"
-            }
-            Write-Host "##vso[task.setvariable variable=$($environmentVariableName)]$($plainValue)"
-        }
-
-        $appServiceName = "${{ parameters.appServiceName }}"
-        $appSettings = (Get-AzWebApp -ResourceGroupName $(ResourceGroupName) -Name $appServiceName).SiteConfig.AppSettings
-        $acrSettings = $appSettings | where {$_.Name -eq "FhirServer__Operations__ConvertData__ContainerRegistryServers__0"}
-        $acrLoginServer = $acrSettings[0].Value
-        $acrAccountName = ($acrLoginServer -split '\.')[0]
-
-        ## This needs to be moved to MI, WI #125246  
-        # $acrPassword = (Get-AzContainerRegistryCredential -ResourceGroupName $(ResourceGroupName) -Name $acrAccountName).Password
-        # Write-Host "##vso[task.setvariable variable=TestContainerRegistryServer]$($acrLoginServer)"
-        # Write-Host "##vso[task.setvariable variable=TestContainerRegistryPassword]$($acrPassword)"
-
-        $exportStoreSettings = $appSettings | where {$_.Name -eq "FhirServer__Operations__Export__StorageAccountUri"}
-        $exportStoreUri = $exportStoreSettings[0].Value
-        Write-Host "$exportStoreUri"
-        $exportStoreAccountName = [System.Uri]::new("$exportStoreUri").Host.Split('.')[0]
-        $exportStoreKey = Get-AzStorageAccountKey -ResourceGroupName $(ResourceGroupName) -Name "$exportStoreAccountName" | Where-Object {$_.KeyName -eq "key1"}
-
-        Write-Host "##vso[task.setvariable variable=TestExportStoreUri]$($exportStoreUri)"
-
-        $integrationStoreSettings = $appSettings | where {$_.Name -eq "FhirServer__Operations__IntegrationDataStore__StorageAccountUri"}
-        $integrationStoreUri = $integrationStoreSettings[0].Value
-        Write-Host "$integrationStoreUri"
-        $integrationStoreAccountName = [System.Uri]::new("$integrationStoreUri").Host.Split('.')[0]
-        Write-Host "##vso[task.setvariable variable=TestIntegrationStoreUri]$($integrationStoreUri)"
-
-        Write-Host "##vso[task.setvariable variable=Resource]$(TestApplicationResource)"
-
-        $secrets = Get-AzKeyVaultSecret -VaultName resolute-oss-tenant-info
-        foreach($secret in $secrets)
-        {
-            $environmentVariableName = $secret.Name.Replace("--","_")
-
-            $secretValue = Get-AzKeyVaultSecret -VaultName resolute-oss-tenant-info -Name $secret.Name
-            # Replace with -AsPlainText flag when v5.3 of the Az Module is supported
-            $plainValue = ([System.Net.NetworkCredential]::new("", $secretValue.SecretValue).Password).ToString()
-            if([string]::IsNullOrEmpty($plainValue))
-            {
-                throw "$($secret.Name) is empty"
-            }
-            Write-Host "##vso[task.setvariable variable=$($environmentVariableName)]$($plainValue)"
-        }
-        # ----------------------------------------
-
-        dotnet dev-certs https
-
-        Write-Host "##vso[task.setvariable variable=AZURESUBSCRIPTION_CLIENT_ID]$env:AZURESUBSCRIPTION_CLIENT_ID"
-        Write-Host "##vso[task.setvariable variable=AZURESUBSCRIPTION_TENANT_ID]$env:AZURESUBSCRIPTION_TENANT_ID"
-        Write-Host "##vso[task.setvariable variable=AZURESUBSCRIPTION_SERVICE_CONNECTION_ID]$env:AZURESUBSCRIPTION_SERVICE_CONNECTION_ID"
+  - template: ../tasks/e2e-set-variables.yml
+    parameters:
+      appServiceName: ${{ parameters.appServiceName }}
 
   - task: DotNetCoreCLI@2
-    displayName: 'E2E ${{ parameters.version }} ${{parameters.appServiceType}}'
+    displayName: 'E2E ${{ parameters.version }} ${{parameters.appServiceType}}${{ parameters.testRunTitleSuffix }}'
     inputs:
       command: test
-      arguments: '"$(Agent.TempDirectory)/E2ETests/**/*${{ parameters.version }}.Tests.E2E*.dll" --blame-hang-timeout 7m --filter "FullyQualifiedName~${{parameters.appServiceType}}&Category!=ExportLongRunning"'
+      arguments: '"$(Agent.TempDirectory)/E2ETests/**/*${{ parameters.version }}.Tests.E2E*.dll" --blame-hang-timeout 15m --filter "FullyQualifiedName~${{parameters.appServiceType}}&${{ parameters.categoryFilter }}"'
       workingDirectory: "$(System.ArtifactsDirectory)"
-      testRunTitle: '${{ parameters.version }} ${{parameters.appServiceType}}'
+      testRunTitle: '${{ parameters.version }} ${{parameters.appServiceType}}${{ parameters.testRunTitleSuffix }}'
     env:
       'TestEnvironmentUrl': $(TestEnvironmentUrl)
       'TestEnvironmentUrl_${{ parameters.version }}': $(TestEnvironmentUrl_${{ parameters.version }})

build/jobs/provision-deploy.yml

@@ -30,6 +30,9 @@ parameters:
 - name: reindexEnabled
   type: boolean
   default: true
+- name: keyVaultName
+  type: string
+  default: ''
 
 jobs:
 - job: provisionEnvironment
@@ -73,6 +76,7 @@ jobs:
             appServicePlanSku = "P2V3"
             numberOfInstances = 2
             serviceName = $webAppName
+            keyVaultName = "${{ parameters.keyVaultName }}".ToLower()
             securityAuthenticationAuthority = "https://login.microsoftonline.com/$(tenant-id)"
             securityAuthenticationAudience = "${{ parameters.testEnvironmentUrl }}"
             additionalFhirServerConfigProperties = $additionalProperties
@@ -99,13 +103,6 @@ jobs:
         $deploymentName = $webAppName
         $resourceGroupName = "${{ parameters.resourceGroup }}"
 
-        Write-Host "Check for keyvaults in removed state..."
-        if (Get-AzKeyVault -VaultName $webAppName -Location $(ResourceGroupRegion) -InRemovedState)
-        {
-            Undo-AzKeyVaultRemoval -VaultName $webAppName -ResourceGroupName $resourceGroupName -Location $(ResourceGroupRegion)
-            Write-Host "KeyVault $webAppName is restored"
-        }
-
         Write-Host "Provisioning Resource Group"
         Write-Host "ResourceGroupName: ${{ parameters.resourceGroup }}"
 

build/jobs/provision-sqlServer.yml

@@ -114,13 +114,34 @@ jobs:
               'nspProfile' = 'default'
           }
 
-          # Deploy using external template
-          $nspDeploymentResult = New-AzResourceGroupDeployment `
-            -ResourceGroupName $resourceGroup `
-            -Name "$sqlServerName-nsp-association" `
-            -TemplateFile $(System.DefaultWorkingDirectory)/samples/templates/nsp-resource-association.json `
-            -TemplateParameterObject $templateParameters `
-            -Verbose
+          # Deploy using external template with retry logic for transient conflicts
+          $maxRetries = 3
+          $retryDelaySeconds = 15
+          $attempt = 0
+          $success = $false
+
+          do {
+            try {
+              $attempt++
+              Write-Host "Attempt $attempt of $maxRetries to create NSP association..."
+
+              $nspDeploymentResult = New-AzResourceGroupDeployment `
+                -ResourceGroupName $resourceGroup `
+                -Name "$sqlServerName-nsp-association" `
+                -TemplateFile $(System.DefaultWorkingDirectory)/samples/templates/nsp-resource-association.json `
+                -TemplateParameterObject $templateParameters `
+                -Verbose
+
+              $success = $true
+            } catch {
+              if ($_.Exception.Message -like "*Conflict*" -and $attempt -lt $maxRetries) {
+                Write-Host "Conflict detected. Retrying in $retryDelaySeconds seconds..."
+                Start-Sleep -Seconds $retryDelaySeconds
+              } else {
+                throw
+              }
+            }
+          } while (-not $success -and $attempt -lt $maxRetries)
 
           Write-Host "Successfully associated SQL Server with NSP using external ARM template"
           Write-Host "Association Resource ID: $($nspDeploymentResult.Outputs.associationResourceId.Value)"

build/jobs/run-export-tests.yml

@@ -23,7 +23,7 @@ jobs:
       azurePowerShellVersion: latestVersion
       ScriptType: inlineScript
       Inline: |
-        $keyVault = "$(DeploymentEnvironmentName)-ts"
+        $keyVault = "$(KeyVaultBaseName)-ts"
         $secrets = Get-AzKeyVaultSecret -VaultName $keyVault
 
         foreach($secret in $secrets)
@@ -116,7 +116,7 @@ jobs:
       azurePowerShellVersion: latestVersion
       ScriptType: inlineScript
       Inline: |
-        $keyVault = "$(DeploymentEnvironmentName)-ts"
+        $keyVault = "$(KeyVaultBaseName)-ts"
         $secrets = Get-AzKeyVaultSecret -VaultName $keyVault
 
         foreach($secret in $secrets)