feat: Enable skip of CI deployment tests

.github/CODEOWNERS

@@ -197,3 +197,4 @@
 /avm/res/web/static-site/ @Azure/avm-res-web-staticsite-module-owners-bicep @Azure/avm-module-reviewers-bicep
 /avm/utl/types/avm-common-types/ @Azure/avm-utl-types-avmcommontypes-module-owners-bicep @Azure/avm-module-reviewers-bicep
 *avm.core.team.tests.ps1 @Azure/avm-core-team-technical-bicep
+*.e2eignore @Azure/avm-core-team-technical-bicep

.github/actions/templates/avm-getModuleTestFiles/action.yml

@@ -44,6 +44,8 @@ runs:
           @{
             path = $_
             name = Split-Path (Split-Path $_) -Leaf
+            # if an .e2eignore file exists, the testcase will not be deployed
+            e2eIgnore = Test-Path (Join-Path $moduleFolderPath $_).Replace('main.test.bicep', '.e2eignore')
           }
         } | ConvertTo-Json -Compress
 

.github/actions/templates/avm-validateModuleDeployment/action.yml

@@ -55,14 +55,47 @@ inputs:
   customLocation:
     description: "Custom location overwrite, if needed"
     required: false
+  e2eIgnore:
+    description: "A test case will not be deployed if an .e2eignore file exists"
+    required: false
 
 runs:
   using: "composite"
   steps:
+    # [Deployment validation] task(s)
+    # -------------------------------
+    - name: "Validate Test Execution"
+      uses: azure/powershell@v2
+      with:
+        azPSVersion: "latest"
+        errorActionPreference: "SilentlyContinue"
+        inlineScript: |
+          # the first step is to check, if the deployment should be bypassed
+
+          # Resolve template file path
+          $moduleTestFilePath = Join-Path $env:GITHUB_WORKSPACE '${{ inputs.templateFilePath }}'
+
+          # Check if the deployment test should be bypassed
+          if ('${{ inputs.e2eIgnore }}' -eq 'true') {
+            Write-Warning -Message "File '.e2eignore' exists in the folder: $moduleTestFilePath"
+            Write-Output 'skip_deployment_ci=true' >> $env:GITHUB_ENV
+
+            # get the reason for skipping the deployment
+            $excludeReason = Get-Content Join-Path $moduleTestFilePath '.e2eignore'
+            excludeMessage = "Skipping deployment for ${{ inputs.templateFilePath }}: $excludeReason"
+
+            Write-Output "Skipping deployment for ${{ inputs.templateFilePath }}: $excludeMessage" >> $env:VALIDATE_TEST_EXECUTION_RESULT
+            Write-Warning "Skipping deployment for ${{ inputs.templateFilePath }}: $excludeMessage"
+          } else {
+            Write-Output "File '.e2eignore' does not exist in the folder: $moduleTestFilePath" -Verbose
+            Write-Output "skip_deployment_ci=false" >> $env:GITHUB_ENV
+          }
+
     # [Azure login] task(s)
     # ------------------------------
     - name: "Set OIDC temporary exception"
       id: set-oidc-exception
+      if: env.skip_deployment_ci == 'false'
       uses: azure/powershell@v2
       with:
         azPSVersion: "latest"
@@ -97,7 +130,7 @@ runs:
     # Default: support both OIDC and service principal with secret
     # 'creds' will be ignored if 'client-id', 'subscription-id' or 'tenant-id' is set
     - name: "Azure Login - Default"
-      if: ${{ steps.set-oidc-exception.outputs.oidcException == 'false' }}
+      if: ${{ steps.set-oidc-exception.outputs.oidcException == 'false' && env.skip_deployment_ci == 'false' }}
       uses: azure/login@v2
       with:
         creds: ${{ env.AZURE_CREDENTIALS }}
@@ -109,7 +142,7 @@ runs:
     # Exception: module requires login by using service principal with secret
     # Should only be leveraged by modules listed in $exceptionModulePaths above
     - name: "Azure Login - Exception"
-      if: ${{ steps.set-oidc-exception.outputs.oidcException == 'true' }}
+      if: ${{ steps.set-oidc-exception.outputs.oidcException == 'true'  && env.skip_deployment_ci == 'false'}}
       uses: azure/login@v2
       with:
         creds: ${{ env.AZURE_CREDENTIALS }}
@@ -119,6 +152,7 @@ runs:
     # ---------------------------
     - name: "Get Resource Location"
       id: get-resource-location
+      if: env.skip_deployment_ci == 'false'
       uses: azure/powershell@v2
       with:
         azPSVersion: "latest"
@@ -153,6 +187,7 @@ runs:
     # [Token replacement] task(s)
     # ---------------------------
     - name: "Replace tokens in template file"
+      if: env.skip_deployment_ci == 'false'
       uses: azure/powershell@v2
       with:
         azPSVersion: "latest"
@@ -216,9 +251,8 @@ runs:
 
           Write-Output '::endgroup::'
 
-    # [Deployment validation] task(s)
-    # -------------------------------
     - name: "Validate template file"
+      if: env.skip_deployment_ci == 'false'
       uses: azure/powershell@v2
       with:
         azPSVersion: "latest"
@@ -298,6 +332,7 @@ runs:
     # [Deployment execution] task(s)
     # ------------------------------
     - name: "Deploy template file"
+      if: env.skip_deployment_ci == 'false'
       id: deploy_step
       uses: azure/powershell@v2
       with:
@@ -392,6 +427,7 @@ runs:
     # [Post-Deployment test] task(s)
     # ------------------------------
     - name: "Run post-deployment Pester tests"
+      if: env.skip_deployment_ci == 'false'
       id: pester_run_step
       shell: pwsh
       run: |
@@ -462,27 +498,30 @@ runs:
         }
 
     - name: "Output to GitHub job summaries"
-      if: steps.pester_run_step.outputs.formattedPesterResultsPath != ''
+      if: env.skip_deployment_ci == 'false' && steps.pester_run_step.outputs.formattedPesterResultsPath != ''
       shell: pwsh
       run: |
         # Grouping task logs
         Write-Output '::group::Output to GitHub job summaries'
 
         $mdPesterOutputFilePath = '${{ steps.pester_run_step.outputs.formattedPesterResultsPath }}'
+        $validateTestExecutionResult = $env:VALIDATE_TEST_EXECUTION_RESULT
 
         if (-not (Test-Path $mdPesterOutputFilePath)) {
           Write-Warning ('Input file [{0}] not found. Please check if the previous task threw an error and try again.' -f $mdPesterOutputFilePath)
+          $validateTestExecutionResult >> $env:GITHUB_STEP_SUMMARY
         } else {
-          Get-Content $mdPesterOutputFilePath >> $env:GITHUB_STEP_SUMMARY
+          $jobOutput = Get-Content $mdPesterOutputFilePath '\n' $validateTestExecutionResult
           Write-Verbose ('Successfully printed out file [{0}] to Job Summaries' -f $mdPesterOutputFilePath) -Verbose
+          $jobOutput >> $env:GITHUB_STEP_SUMMARY
         }
 
         Write-Output '::endgroup::'
 
     # [Deployment removal] task(s)
     # ----------------------------
     - name: "Remove deployed resources"
-      if: ${{ (success() || failure()) && inputs.removeDeployment == 'true' && steps.deploy_step.outputs.deploymentNames != '' }}
+      if: ${{ (success() || failure()) && inputs.removeDeployment == 'true' && steps.deploy_step.outputs.deploymentNames != '' && env.skip_deployment_ci == 'false' }}
       uses: azure/powershell@v2
       with:
         azPSVersion: "latest"

.github/workflows/avm.template.module.yml

@@ -178,6 +178,7 @@ jobs:
           managementGroupId: "${{ secrets.ARM_MGMTGROUP_ID }}"
           removeDeployment: "${{ fromJson(inputs.workflowInput).removeDeployment }}"
           customLocation: "${{ fromJson(inputs.workflowInput).customLocation }}"
+          e2eIgnore: "${{ matrix.testCases.e2eIgnore }}"
         env:
           AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }}
           VALIDATE_CLIENT_ID: ${{ secrets.VALIDATE_CLIENT_ID }}

utilities/pipelines/staticValidation/compliance/module.tests.ps1

@@ -247,7 +247,9 @@ Describe 'File/folder tests' -Tag 'Modules' {
                 [string] $moduleFolderPath
             )
 
-            $wafAlignedFolder = Get-ChildItem -Directory (Join-Path -Path $moduleFolderPath 'tests' 'e2e') -Filter '*waf-aligned'
+            $wafAlignedFolder = Get-ChildItem -Directory (Join-Path -Path $moduleFolderPath 'tests' 'e2e') | Where-Object {
+                $_.Name -match '^(waf-aligned)|.*(waf-aligned)$' # the spec BCPRMNFR1 states, that the folder names should start with defaults|waf-aligned. Since it is a should and not a must, need to check for both cases.
+            }
             $wafAlignedFolder | Should -Not -BeNullOrEmpty
         }
 
@@ -263,7 +265,9 @@ Describe 'File/folder tests' -Tag 'Modules' {
                 return
             }
 
-            $defaultsFolder = Get-ChildItem -Directory (Join-Path -Path $moduleFolderPath 'tests' 'e2e') -Filter '*defaults'
+            $defaultsFolder = Get-ChildItem -Directory (Join-Path -Path $moduleFolderPath 'tests' 'e2e') | Where-Object {
+                $_.Name -match '^(defaults)|.*(defaults)$' # the spec BCPRMNFR1 states, that the folder names should start with defaults|waf-aligned. Since it is a should and not a must, need to check for both cases.
+            }
             $defaultsFolder | Should -Not -BeNullOrEmpty
         }
 
@@ -281,6 +285,48 @@ Describe 'File/folder tests' -Tag 'Modules' {
                 $pathExisting | Should -Be $true -Because "path [$filePath] is expected to exist."
             }
         }
+
+        It '[<moduleFolderName>] Resource Modules must not skip the "defaults" or "waf-aligned" tests with a [` .e2eignore `] file.' -TestCases ($topLevelModuleTestCases | Where-Object { $_.moduleType -eq 'res' }) {
+
+            param(
+                [string] $moduleFolderName,
+                [string] $moduleFolderPath
+            )
+
+            $incorrectFolders = @()
+            $e2eTestFolderPathList = Get-ChildItem -Directory (Join-Path -Path $moduleFolderPath 'tests' 'e2e') | Where-Object {
+                $_.Name -match '^(defaults|waf-aligned)|.*(defaults|waf-aligned)$' # the spec BCPRMNFR1 states, that the folder names should start with defaults|waf-aligned. Since it is a should and not a must, need to check for both cases.
+            }
+            foreach ($e2eTestFolderPath in $e2eTestFolderPathList) {
+                $filePath = Join-Path -Path $e2eTestFolderPath '.e2eignore'
+                if (Test-Path $filePath) {
+                    $incorrectFolders += $e2eTestFolderPath.Name
+                }
+            }
+            $incorrectFolders | Should -BeNullOrEmpty -Because ('skipping this test is not allowed. Found incorrect items: [{0}].' -f ($incorrectFolders -join ', '))
+        }
+
+        It '[<moduleFolderName>] a [` .e2eignore `] file must contain text, which states the exception reason for not executing the deployment test.' -TestCases $topLevelModuleTestCases {
+
+            param(
+                [string] $moduleFolderName,
+                [string] $moduleFolderPath
+            )
+
+            $incorrectFolders = @()
+            $e2eTestFolderPathList = Get-ChildItem -Directory (Join-Path -Path $moduleFolderPath 'tests' 'e2e')
+            foreach ($e2eTestFolderPath in $e2eTestFolderPathList) {
+                $filePath = Join-Path -Path $e2eTestFolderPath '.e2eignore'
+                $pathExisting = Test-Path $filePath
+                if ($pathExisting) {
+                    $fileContent = Get-Content -Path $filePath
+                    if (-not $fileContent) {
+                        $incorrectFolders += $e2eTestFolderPath.Name + '\.e2eignore'
+                    }
+                }
+            }
+            $incorrectFolders | Should -BeNullOrEmpty -Because ('the file should contain a reason for skipping the test. Found incorrect items: [{0}].' -f ($incorrectFolders -join ', '))
+        }
     }
 }
 

utilities/tools/Test-ModuleLocally.ps1

@@ -19,14 +19,17 @@ Optional. A string array that can be specified to run only Pester tests with the
 Optional. A switch parameter that triggers a Pester test for the module
 
 .PARAMETER ValidateOrDeployParameters
-Optional. An object consisting of the components that are required when using the Validate test or DeploymentTest switch parameter.  Mandatory if the DeploymentTest/ValidationTest switches are set.
+Optional. An object consisting of the components that are required when using the Validate test or DeploymentTest switch parameter. Mandatory if the DeploymentTest/ValidationTest switches are set. The e2eIgnore file in test cases will not execute a deployment.
 
 .PARAMETER DeploymentTest
 Optional. A switch parameter that triggers the deployment of the module
 
 .PARAMETER ValidationTest
 Optional. A switch parameter that triggers the validation of the module only without deployment
 
+.PARAMETER RespectE2eIgnoreFlag
+Optional. A switch parameter that enables you to respect the e2eIgnore file in the test cases. If set to $true, the deployment tasks will not be executed, if a test-case is excluded by an .e2eignore file.
+
 .PARAMETER SkipParameterFileTokens
 Optional. A switch parameter that enables you to skip the search for local custom parameter file tokens.
 
@@ -115,6 +118,7 @@ $TestModuleLocallyInput = @{
     AdditionalTokens           = @{
         tenantId = '00000000-0000-0000-0000-000000000000'
     }
+    RespectE2eIgnoreFlag       = $true
 }
 Test-ModuleLocally @TestModuleLocallyInput -Verbose
 Get What-If deployment result using a test file with the provided tokens
@@ -161,6 +165,9 @@ function Test-ModuleLocally {
         [Parameter(Mandatory = $false)]
         [switch] $ValidationTest,
 
+        [Parameter(Mandatory = $false)]
+        [switch] $RespectE2eIgnoreFlag,
+
         [Parameter(Mandatory = $false)]
         [switch] $WhatIfTest
     )
@@ -285,8 +292,13 @@ function Test-ModuleLocally {
                 if ($ValidationTest) {
                     # Loop through test files
                     foreach ($moduleTestFile in $moduleTestFiles) {
-                        Write-Verbose ('Validating module [{0}] with test file [{1}]' -f $ModuleName, (Split-Path $moduleTestFile -Leaf)) -Verbose
-                        Test-TemplateDeployment @functionInput -TemplateFilePath $moduleTestFile
+                        $ignoreFilePath = Join-Path (Split-Path $moduleTestFile) '.e2eignore'
+                        if ((Test-Path $ignoreFilePath) -and ($RespectE2eIgnoreFlag -eq $true)) {
+                            Write-Output "File '.e2eignore' exists in the folder: $moduleTestFile"
+                        } else {
+                            Write-Verbose ('Validating module [{0}] with test file [{1}]' -f $ModuleName, (Split-Path $moduleTestFile -Leaf)) -Verbose
+                            Test-TemplateDeployment @functionInput -TemplateFilePath $moduleTestFile
+                        }
                     }
                 }
 
@@ -295,8 +307,13 @@ function Test-ModuleLocally {
                 if ($WhatIfTest) {
                     # Loop through test files
                     foreach ($moduleTestFile in $moduleTestFiles) {
-                        Write-Verbose ('Get Deployment What-If result for module [{0}] with test file [{1}]' -f $ModuleName, (Split-Path $moduleTestFile -Leaf)) -Verbose
-                        Get-TemplateDeploymentWhatIf @functionInput -TemplateFilePath $moduleTestFile
+                        $ignoreFilePath = Join-Path (Split-Path $moduleTestFile) '.e2eignore'
+                        if ((Test-Path $ignoreFilePath) -and ($RespectE2eIgnoreFlag -eq $true)) {
+                            Write-Output "File '.e2eignore' exists in the folder: $moduleTestFile"
+                        } else {
+                            Write-Verbose ('Get Deployment What-If result for module [{0}] with test file [{1}]' -f $ModuleName, (Split-Path $moduleTestFile -Leaf)) -Verbose
+                            Get-TemplateDeploymentWhatIf @functionInput -TemplateFilePath $moduleTestFile
+                        }
                     }
                 }
 
@@ -306,9 +323,14 @@ function Test-ModuleLocally {
                     $functionInput['retryLimit'] = 1 # Overwrite default of 3
                     # Loop through test files
                     foreach ($moduleTestFile in $moduleTestFiles) {
-                        Write-Verbose ('Deploy Module [{0}] with test file [{1}]' -f $ModuleName, (Split-Path $moduleTestFile -Leaf)) -Verbose
-                        if ($PSCmdlet.ShouldProcess(('Module [{0}] with test file [{1}]' -f $ModuleName, (Split-Path $moduleTestFile -Leaf)), 'Deploy')) {
-                            New-TemplateDeployment @functionInput -TemplateFilePath $moduleTestFile
+                        $ignoreFilePath = Join-Path (Split-Path $moduleTestFile) '.e2eignore'
+                        if ((Test-Path $ignoreFilePath) -and ($RespectE2eIgnoreFlag -eq $true)) {
+                            Write-Output "File '.e2eignore' exists in the folder: $moduleTestFile"
+                        } else {
+                            Write-Verbose ('Deploy Module [{0}] with test file [{1}]' -f $ModuleName, (Split-Path $moduleTestFile -Leaf)) -Verbose
+                            if ($PSCmdlet.ShouldProcess(('Module [{0}] with test file [{1}]' -f $ModuleName, (Split-Path $moduleTestFile -Leaf)), 'Deploy')) {
+                                New-TemplateDeployment @functionInput -TemplateFilePath $moduleTestFile
+                            }
                         }
                     }
                 }