initial implementation of macOS crypto backend

Author: gdams

.github/workflows/codeql.yml

@@ -23,16 +23,16 @@ jobs:
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@48ab28a6f5dbc2a99bf1e0131198dd8f1df78169 # v3.28.0
+        uses: github/codeql-action/init@b6a472f63d85b9c78a3ac5e89422239fc15e9b3c # v3.28.1
         with:
           languages: go
 
       - name: Autobuild
-        uses: github/codeql-action/autobuild@48ab28a6f5dbc2a99bf1e0131198dd8f1df78169 # v3.28.0
+        uses: github/codeql-action/autobuild@b6a472f63d85b9c78a3ac5e89422239fc15e9b3c # v3.28.1
         with:
           working-directory: eng/_util
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@48ab28a6f5dbc2a99bf1e0131198dd8f1df78169 # v3.28.0
+        uses: github/codeql-action/analyze@b6a472f63d85b9c78a3ac5e89422239fc15e9b3c # v3.28.1
         with:
           category: /language:go

eng/pipeline/stages/builders-to-stages.yml

@@ -70,11 +70,10 @@ stages:
     - template: pool.yml
       parameters:
         inner:
-          template: publish-stage.yml
+          template: internal-publish-stage.yml
           parameters:
             builder: { os: windows, arch: amd64 }
             official: true
-            public: false
             builders:
               - ${{ each builder in parameters.builders }}:
                 - ${{ if eq(builder.config, 'buildandpack') }}:

eng/pipeline/stages/publish-stage.yml renamed to eng/pipeline/stages/internal-publish-stage.yml

@@ -5,9 +5,6 @@
 # Create a build asset JSON file as a pipeline artifact and publish build artifacts to blob storage.
 
 parameters:
-  - name: public
-    type: boolean
-
   - name: pool
     type: object
 
@@ -28,34 +25,19 @@ parameters:
     default: false
 
 stages:
-  - stage: Publish${{ parameters.public }}
-    ${{ if parameters.public }}:
-      displayName: Publish Public
-    ${{ else }}:
-      displayName: Publish Internal
+  - stage: PrePublish
+    displayName: Pre Internal Publish
     ${{ if eq(parameters.publishExistingRunID, 'nil') }}:
       dependsOn: Sign
     ${{ else }}:
       dependsOn: []
+
     jobs:
-      - job: Publish
+      - job: PrePublish
         pool: ${{ parameters.pool }}
 
         variables:
-          - name: blobBackupAccount
-            value: golangartifactsbackup
-          - name: blobContainer
-            ${{ if parameters.public }}:
-              value: 'https://dotnetbuildoutput.blob.core.windows.net/golang/microsoft'
-            ${{ else }}:
-              value: 'https://$(blobBackupAccount).blob.core.windows.net/microsoft'
-
-          - name: blobPrefix
-            value: '$(PublishBranchAlias)/$(Build.BuildNumber)'
-          - name: blobDestinationUrl
-            value: '$(blobContainer)/$(blobPrefix)'
-
-          - group: go-storage
+          - template: ../variables/publish-internal.yml
 
         workspace:
           clean: all
@@ -68,17 +50,11 @@ stages:
               # Single file publish requires folder to be specified.
               # https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/1es-pipeline-templates/features/sbom#specifying-sbom-build-drop-path-build-component-path-package-name-and-package-version-per-1es-pt-output
               sbomBuildDropPath: $(Pipeline.Workspace)/Binaries Signed
-              ${{ if parameters.public }}:
-                artifact: BuildAssets
-              ${{ else }}:
-                artifact: BuildAssetsInternal
+              artifact: BuildAssetsInternal
             - ${{ if parameters.publishSymbols }}:
               - output: pipelineArtifact
                 path: $(Pipeline.Workspace)/Symbols
-                ${{ if parameters.public }}:
-                  artifact: Symbols
-                ${{ else }}:
-                  artifact: SymbolsInternal
+                artifact: SymbolsInternal
 
         steps:
           - template: ../steps/checkout-windows-task.yml
@@ -96,40 +72,9 @@ stages:
                 -destination-url '$(blobDestinationUrl)' `
                 -branch '$(PublishBranchAlias)' `
                 -o '$(Pipeline.Workspace)/Binaries Signed/assets.json'
-            displayName: 'Create build asset JSON'
-
-          - ${{ if parameters.public }}:
-            - task: AzureCLI@2
-              displayName: Upload to blob storage
-              inputs:
-                azureSubscription: GoLang
-                scriptType: bash
-                scriptLocation: inlineScript
-                # Send literal '*' to az: it handles the wildcard itself. Az copy only accepts one
-                # "from" argument, so we can't use the shell's wildcard expansion.
-                inlineScript: |
-                  az storage copy -s '*' -d '$(blobDestinationUrl)' --sas-token '$(dotnetbuildoutput-golang-write-sas-query)'
-                workingDirectory: '$(Pipeline.Workspace)/Binaries Signed/'
-          - ${{ else }}:
-            - task: AzureFileCopy@6
-              displayName: Upload to blob storage
-              inputs:
-                Destination: AzureBlob
-                azureSubscription: golang-pme-storage
-                storage: $(blobBackupAccount)
-                ContainerName: microsoft
-                SourcePath: '$(Pipeline.Workspace)/Binaries Signed/*'
-                BlobPrefix: $(blobPrefix)
-
-          - pwsh: |
-              Write-Host 'Generated links to artifacts in blob storage:'
-              Write-Host ''
-              Get-ChildItem -File -Path '.' | %{
-                Write-Host "$(blobDestinationUrl)/$($_.Name)"
-              }
-            displayName: Show expected uploaded URLs
-            workingDirectory: '$(Pipeline.Workspace)/Binaries Signed/'
+            displayName: '🧾 Create build asset JSON'
 
+          # Gather symbols from all builders.
           - ${{ if eq(parameters.publishSymbols, true) }}:
             - ${{ each builder in parameters.builders }}:
               - ${{ if eq(parameters.publishExistingRunID, 'nil') }}:
@@ -164,8 +109,62 @@ stages:
                     }
                     Copy-Item $_.FullName $flatDir
                   }
-                displayName: 'Flatten: Symbols ${{ builder.id }}'
+                displayName: '🫓 Flatten: Symbols ${{ builder.id }}'
                 workingDirectory: '$(Pipeline.Workspace)'
+
+  - stage: Publish
+    displayName: Publish Internal
+    dependsOn: PrePublish
+
+    jobs:
+      - job: Publish
+        pool: ${{ parameters.pool }}
+
+        variables:
+          - template: ../variables/publish-internal.yml
+
+        workspace:
+          clean: all
+
+        templateContext:
+          type: releaseJob # https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/1es-pipeline-templates/features/releasepipelines/releaseworkflows/releasejob
+          isProduction: true
+          inputs:
+            - input: pipelineArtifact
+              artifactName: BuildAssetsInternal
+            - ${{ if parameters.publishSymbols }}:
+              - input: pipelineArtifact
+                artifactName: SymbolsInternal
+                path: $(Pipeline.Workspace)/Symbols
+
+        steps:
+          - template: ../steps/find-PublishBranchAlias-task.yml
+
+          - template: ../steps/download-signed-binaries-task.yml
+            parameters:
+              runID: ${{ parameters.publishExistingRunID }}
+              ReleaseJob: true
+
+          - task: AzureFileCopy@6
+            displayName: ↗️ Upload to blob storage
+            inputs:
+              Destination: AzureBlob
+              azureSubscription: golang-pme-storage
+              storage: $(blobBackupAccount)
+              ContainerName: microsoft
+              SourcePath: '$(Pipeline.Workspace)/Binaries Signed/*'
+              BlobPrefix: $(blobPrefix)
+
+          - pwsh: |
+              Write-Host 'Generated links to artifacts in blob storage:'
+              Write-Host ''
+              Get-ChildItem -File -Path '.' | %{
+                Write-Host "$(blobDestinationUrl)/$($_.Name)"
+              }
+            displayName: 🧾 Show expected uploaded URLs
+            workingDirectory: '$(Pipeline.Workspace)/Binaries Signed/'
+
+          - ${{ if parameters.publishSymbols }}:
             - task: PublishSymbols@2
               inputs:
                 SymbolsFolder: $(Pipeline.Workspace)/Symbols
@@ -175,4 +174,4 @@ stages:
                 # in the AzDO repo, but we pull them at build time using a git submodule.
                 # See https://github.com/microsoft/go-lab/issues/67.
                 IndexSources: false
-              displayName: Publish symbols
+              displayName: ↗️ Publish symbols

eng/pipeline/steps/download-signed-binaries-task.yml

@@ -9,14 +9,43 @@ parameters:
     type: string
     default: 'nil'
 
+  - name: ReleaseJob
+    type: boolean
+    default: false
+
 steps:
-  - ${{ if eq(parameters.runID, 'nil') }}:
+  - ${{ if parameters.ReleaseJob }}:
+
+    - task: 1ES.DownloadPipelineArtifact@1
+      displayName: 'Download: Binaries Signed'
+      inputs:
+        ${{ if eq(parameters.runID, 'nil') }}:
+          buildType: current
+        ${{ else }}:
+          buildType: specific
+          runVersion: 'specific'
+          runId: ${{ parameters.runID }}
+        project: $(System.TeamProject)
+        definition: $(System.DefinitionId)
+        artifactName: Binaries Signed
+        targetPath: '$(Pipeline.Workspace)/Binaries Signed'
+
+    # The 1ES step tries to validate the SBOM manifests after download, so we can't assign
+    # 'patterns' to filter them out. Instead, remove them after validation is done.
+    - pwsh: |
+        Remove-Item -Path '$(Pipeline.Workspace)/Binaries Signed/_manifest' -Recurse -Force
+      displayName: Delete SBOM manifests
+
+  - ${{ elseif eq(parameters.runID, 'nil') }}:
+
     - download: current
       artifact: Binaries Signed
       # Filter out manifests added by 1ES pipeline template.
       patterns: '!_manifest/**'
       displayName: 'Download: Binaries Signed'
+
   - ${{ else }}:
+
     - task: DownloadPipelineArtifact@2
       displayName: 'Download: Binaries Signed (Specific)'
       inputs:

eng/pipeline/variables/publish-internal.yml

@@ -0,0 +1,15 @@
+# Copyright (c) Microsoft Corporation.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+# Variables that determine where to publish internal artifacts.
+
+variables:
+  - name: blobBackupAccount
+    value: golangartifactsbackup
+  - name: blobContainer
+    value: 'https://$(blobBackupAccount).blob.core.windows.net/microsoft'
+  - name: blobPrefix
+    value: '$(PublishBranchAlias)/$(Build.BuildNumber)'
+  - name: blobDestinationUrl
+    value: '$(blobContainer)/$(blobPrefix)'