[RSDK-7007] Add Manifest Generation in CI (#78)

Author: Otterverse

.github/workflows/workflow.yaml

@@ -35,7 +35,7 @@ jobs:
     - name: Test
       run: make test
     - name: Build
-      run: make debug-workflow all
+      run: make all
 
   build:
     name: Build and Upload
@@ -50,13 +50,18 @@ jobs:
       with:
         go-version-file: go.mod
     - name: Build
-      run: make debug-workflow all
-    - uses: google-github-actions/auth@v2 # This BREAKS the git checkout/tags/history, so must run after builds!
+      run: make all manifest
+    - uses: google-github-actions/auth@v2
       with:
         credentials_json: ${{ secrets.GCP_CREDENTIALS }}
     - uses: google-github-actions/setup-gcloud@v2
-    - name: Upload binaries to GCS
-      run: gsutil -h "Cache-Control:no-cache" cp bin/viam-agent-* gs://packages.viam.com/temp/prerelease/ #TODO use real target after live testing
-    - name: "Upload install scripts to GCS"
+    - name: Upload binaries to GCS (Release)
       if: github.event_name == 'release'
-      run: gsutil -h "Cache-Control:no-cache" cp preinstall.sh install.sh uninstall.sh gs://packages.viam.com/temp/prerelease/ #TODO use real target after live testing
+      run: |
+        gsutil -h "Cache-Control:no-cache" cp bin/viam-agent-* install.sh uninstall.sh preinstall.sh gs://packages.viam.com/apps/viam-agent/
+        gsutil cp etc/viam-agent-*.json gs://packages.viam.com/apps/viam-subsystems/
+    - name: Upload binaries to GCS (Prerelease)
+      if: github.event_name != 'release'
+      run: |
+        gsutil -h "Cache-Control:no-cache" cp bin/viam-agent-* gs://packages.viam.com/apps/viam-agent/prerelease/
+        gsutil cp etc/viam-agent-*.json gs://packages.viam.com/apps/viam-subsystems/

.gitignore

@@ -1 +1,5 @@
 bin/
+etc/
+
+# GCP's auth action adds this in CI, dirtying the tree
+gha-creds-*.json

Makefile

@@ -6,23 +6,8 @@ else ifeq ($(GOARCH),arm64)
 LINUX_ARCH = aarch64
 endif
 
-GITHUB_REF_NAME ?= $(shell git branch --show-current)
-SHOULD_PUBLISH = $(shell echo $(GITHUB_REF_NAME) | grep -qE '^(main|v[0-9]+\.[0-9]+\.[0-9]+)$$' && echo true)
-
-ifeq ($(shell git status -s),)
-	ifeq ($(SHOULD_PUBLISH),true)
-		LAST_TAG := $(shell git describe --tags --abbrev=0 2>/dev/null)
-		COMMITS_SINCE_TAG := $(shell git rev-list $(LAST_TAG)..HEAD --count 2>/dev/null)
-		BASE_VERSION := $(shell echo $(LAST_TAG) | cut -c2-)
-		NEXT_VERSION := $(shell echo $(BASE_VERSION) | awk -F. '{$$3+=1}1' OFS=.)
-		ifeq ($(COMMITS_SINCE_TAG),0)
-			TAG_VERSION ?= $(BASE_VERSION)
-		else
-			TAG_VERSION ?= $(NEXT_VERSION)-dev.$(COMMITS_SINCE_TAG)
-		endif
-	endif
-	GIT_REVISION = $(shell git rev-parse HEAD | tr -d '\n')
-endif
+GIT_REVISION = $(shell git rev-parse HEAD)
+TAG_VERSION ?= $(shell ./dev-version.sh | sed 's/^v//')
 ifeq ($(TAG_VERSION),)
 PATH_VERSION = custom
 else
@@ -35,16 +20,6 @@ TAGS = osusergo,netgo
 
 .DEFAULT_GOAL := bin/viam-agent-$(PATH_VERSION)-$(LINUX_ARCH)
 
-.PHONY: debug-workflow
-debug-workflow:
-	echo GITHUB_REF_NAME $(GITHUB_REF_NAME)
-	echo SHOULD_PUBLISH $(SHOULD_PUBLISH)
-	echo TAG_VERSION $(TAG_VERSION)
-	echo PATH_VERSION $(PATH_VERSION)
-	echo GIT_REVISION $(GIT_REVISION)
-	echo LAST_TAG $(LAST_TAG)
-	echo COMMITS_SINCE_TAG $(COMMITS_SINCE_TAG)
-
 .PHONY: all
 all: amd64 arm64
 
@@ -76,10 +51,17 @@ lint: bin/golangci-lint
 test:
 	go test -race ./...
 
+.PHONY: manifest
+manifest: bin/viam-agent-$(PATH_VERSION)-x86_64 bin/viam-agent-$(PATH_VERSION)-aarch64
+	echo $(PATH_VERSION) | grep -qE '^v[0-9]+\.[0-9]+\.[0-9]+' || exit 1
+	./manifest.sh bin/viam-agent-$(PATH_VERSION)-x86_64
+	./manifest.sh bin/viam-agent-$(PATH_VERSION)-aarch64
+
 .PHONY: upload-stable
-upload-stable: bin/viam-agent-$(PATH_VERSION)-x86_64 bin/viam-agent-$(PATH_VERSION)-aarch64 bin/viam-agent-stable-x86_64 bin/viam-agent-stable-aarch64
+upload-stable: bin/viam-agent-$(PATH_VERSION)-x86_64 bin/viam-agent-$(PATH_VERSION)-aarch64 bin/viam-agent-stable-x86_64 bin/viam-agent-stable-aarch64 manifest
 	echo $(PATH_VERSION) | grep -qE '^v[0-9]+\.[0-9]+\.[0-9]+$$' || exit 1
 	gsutil -h "Cache-Control:no-cache" cp bin/viam-agent-$(PATH_VERSION)-x86_64 bin/viam-agent-$(PATH_VERSION)-aarch64 bin/viam-agent-stable-x86_64 bin/viam-agent-stable-aarch64 gs://packages.viam.com/apps/viam-agent/
+	gsutil cp etc/viam-agent-$(PATH_VERSION)-x86_64.json etc/viam-agent-$(PATH_VERSION)-aarch64.json gs://packages.viam.com/apps/viam-subsystems/
 
 .PHONY: upload-installer
 upload-installer:

dev-version.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+# Exit with a blank if tree is dirty
+if [ -n "$(git status -s)" ]; then
+    exit 0
+fi
+
+# See if we have a direct tag
+DIRECT_TAG=$(git tag --points-at | sort -Vr | head -n1)
+if [ -n "$DIRECT_TAG" ]; then
+    echo ${DIRECT_TAG}
+    exit 0
+fi
+
+if [ -z "$GITHUB_REF_NAME" ]; then
+    GITHUB_REF_NAME=$(git rev-parse --abbrev-ref HEAD)
+fi
+
+# If we're not on main, we have no (automated) version to create
+if [ "$GITHUB_REF_NAME" != "main" ]; then
+    exit 0
+fi
+
+# If we don't have a direct tag, use the most recent non-RC tag
+DESC=$(git describe --tags --match="v*" --exclude="*-rc*" --long | sed 's/^v//')
+
+BASE_VERSION=$(echo "$DESC" | cut -d'-' -f1)
+COMMITS_SINCE_TAG=$(echo "$DESC" | cut -d'-' -f2)
+
+# Calculate next version by incrementing patch number
+NEXT_VERSION=$(echo "$BASE_VERSION" | awk -F. '{$3+=1}1' OFS=.)
+
+# Set TAG_VERSION based on commits since last tag
+if [ "$COMMITS_SINCE_TAG" -eq 0 ]; then
+    TAG_VERSION="$BASE_VERSION"
+else
+    TAG_VERSION="${NEXT_VERSION}-dev.${COMMITS_SINCE_TAG}"
+fi
+
+# Set PATH_VERSION based on TAG_VERSION
+PATH_VERSION="v${TAG_VERSION}"
+
+echo ${PATH_VERSION}

manifest.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+# Check if binary path is provided
+if [ $# -ne 1 ]; then
+    echo "Usage: $0 <path-to-binary>"
+    exit 1
+fi
+
+BINARY_PATH="$1"
+BINARY_NAME=$(basename "$BINARY_PATH")
+
+# Extract version from binary name
+VERSION=$(echo "$BINARY_NAME" | sed -E 's/viam-agent-v([0-9]+\.[0-9]+\.[0-9]+(-dev\.[0-9]+)?).*/\1/')
+
+# Determine if this is a prerelease
+if [[ "$VERSION" == *"-dev"* ]]; then
+    PRERELEASE="prerelease/"
+fi
+
+# Set platform based on architecture in binary name
+if [[ "$BINARY_NAME" == *"x86_64"* ]]; then
+    ARCH="amd64"
+elif [[ "$BINARY_NAME" == *"aarch64"* ]]; then
+    ARCH="arm64"
+else
+    echo "Unknown architecture in binary name"
+    exit 1
+fi
+
+# Set OS based on binary name
+OS="linux"
+if [[ "$BINARY_NAME" == *"windows"* ]]; then
+    OS="windows"
+fi
+
+# Set platform
+PLATFORM="${OS}/${ARCH}"
+
+# Calculate SHA256
+SHA256=$(sha256sum "$BINARY_PATH" | cut -d' ' -f1)
+
+# Generate json manifest with binary-specific name
+mkdir -p etc
+cat > "etc/${BINARY_NAME}.json" << EOF
+{
+	"subsystem": "viam-agent",
+	"version": "$VERSION",
+	"platform": "$PLATFORM",
+	"upload-path": "packages.viam.com/apps/viam-agent/${PRERELEASE}${BINARY_NAME}",
+	"sha256": "$SHA256"
+}
+EOF