🚀 Feat run retry policy

.changes/unreleased/ENHANCEMENTS-594-20250424-164645.yaml

@@ -0,0 +1,5 @@
+kind: ENHANCEMENTS
+body: Relink to more recent destroy runs when destroying a workspace
+time: 2025-04-24T16:46:45.710081284+02:00
+custom:
+    PR: "594"

.changes/unreleased/ENHANCEMENTS-602-20250429-154503.yaml

@@ -0,0 +1,5 @@
+kind: ENHANCEMENTS
+body: Workspace CRD option for the operator to retry failed runs
+time: 2025-04-29T15:45:03.031933484+02:00
+custom:
+    PR: "602"

api/v1alpha2/workspace_types.go

@@ -65,6 +65,17 @@ type RemoteStateSharing struct {
 	Workspaces []*ConsumerWorkspace `json:"workspaces,omitempty"`
 }
 
+// RetryPolicy allows you to configure retry behavior for failed runs on the workspace.
+// It will apply for the latest current run of the operator.
+type RetryPolicy struct {
+	// Limit is the maximum number of retries for failed runs. If set to a negative number, no limit will be applied.
+	// Default: `0`.
+	//
+	//+kubebuilder:default:=0
+	//+optional
+	BackoffLimit int64 `json:"backoffLimit,omitempty"`
+}
+
 // Run tasks allow HCP Terraform to interact with external systems at specific points in the HCP Terraform run lifecycle.
 // Only one of the fields `ID` or `Name` is allowed.
 // At least one of the fields `ID` or `Name` is mandatory.
@@ -592,12 +603,16 @@ type WorkspaceSpec struct {
 	//
 	//+optional
 	RemoteStateSharing *RemoteStateSharing `json:"remoteStateSharing,omitempty"`
+	// Retry Policy allows you to specify how the operator should retry failed runs automatically.
+	//
+	//+optional
+	RetryPolicy *RetryPolicy `json:"retryPolicy,omitempty"`
 	// Run triggers allow you to connect this workspace to one or more source workspaces.
 	// These connections allow runs to queue automatically in this workspace on successful apply of runs in any of the source workspaces.
 	// More information:
 	//   - https://developer.hashicorp.com/terraform/cloud-docs/workspaces/settings/run-triggers
 	//
-	//+kubebuilder:validation:MinItems:=1
+	//+kubebuilder:validation:MinItems:=2
 	//+optional
 	RunTriggers []RunTrigger `json:"runTriggers,omitempty"`
 	// Settings for the workspace's VCS repository, enabling the UI/VCS-driven run workflow.
@@ -742,13 +757,25 @@ type WorkspaceStatus struct {
 	//
 	//+optional
 	VariableSets []VariableSetStatus `json:"variableSet,omitempty"`
+
+	// Retry status of the latest run on the workspace.
+	//
+	//+optional
+	Retry *RetryStatus `json:"retry,omitempty"`
 }
 
 type VariableSetStatus struct {
 	ID   string `json:"id,omitempty"`
 	Name string `json:"name,omitempty"`
 }
 
+// RetryStatus contains the status of the retry of the latest run on the workspace. How many attempts are left and
+// possibly a time to wait for the next attempt.
+type RetryStatus struct {
+	// Failed is the number of failed attempts, counting the initial one.
+	Failed int64 `json:"failed,omitempty"`
+}
+
 //+kubebuilder:object:root=true
 //+kubebuilder:subresource:status
 //+kubebuilder:printcolumn:name="Workspace ID",type=string,JSONPath=`.status.workspaceID`

charts/hcp-terraform-operator/crds/app.terraform.io_workspaces.yaml

@@ -374,6 +374,18 @@ spec:
                     minItems: 1
                     type: array
                 type: object
+              retryPolicy:
+                description: Retry Policy allows you to specify how the operator should
+                  retry failed runs automatically.
+                properties:
+                  backoffLimit:
+                    default: 0
+                    description: |-
+                      Limit is the maximum number of retries for failed runs. If set to a negative number, no limit will be applied.
+                      Default: `0`.
+                    format: int64
+                    type: integer
+                type: object
               runTasks:
                 description: |-
                   Run tasks allow HCP Terraform to interact with external systems at specific points in the HCP Terraform run lifecycle.
@@ -442,7 +454,7 @@ spec:
                       minLength: 1
                       type: string
                   type: object
-                minItems: 1
+                minItems: 2
                 type: array
               sshKey:
                 description: |-
@@ -836,6 +848,15 @@ spec:
                     pattern: ^\d{1}\.\d{1,2}\.\d{1,2}$
                     type: string
                 type: object
+              retry:
+                description: Retry status of the latest run on the workspace.
+                properties:
+                  failed:
+                    description: Failed is the number of failed attempts, counting
+                      the initial one.
+                    format: int64
+                    type: integer
+                type: object
               runStatus:
                 description: Workspace Runs status.
                 properties:

config/crd/bases/app.terraform.io_workspaces.yaml

@@ -371,6 +371,18 @@ spec:
                     minItems: 1
                     type: array
                 type: object
+              retryPolicy:
+                description: Retry Policy allows you to specify how the operator should
+                  retry failed runs automatically.
+                properties:
+                  backoffLimit:
+                    default: 0
+                    description: |-
+                      Limit is the maximum number of retries for failed runs. If set to a negative number, no limit will be applied.
+                      Default: `0`.
+                    format: int64
+                    type: integer
+                type: object
               runTasks:
                 description: |-
                   Run tasks allow HCP Terraform to interact with external systems at specific points in the HCP Terraform run lifecycle.
@@ -439,7 +451,7 @@ spec:
                       minLength: 1
                       type: string
                   type: object
-                minItems: 1
+                minItems: 2
                 type: array
               sshKey:
                 description: |-
@@ -833,6 +845,15 @@ spec:
                     pattern: ^\d{1}\.\d{1,2}\.\d{1,2}$
                     type: string
                 type: object
+              retry:
+                description: Retry status of the latest run on the workspace.
+                properties:
+                  failed:
+                    description: Failed is the number of failed attempts, counting
+                      the initial one.
+                    format: int64
+                    type: integer
+                type: object
               runStatus:
                 description: Workspace Runs status.
                 properties:

docs/api-reference.md

@@ -549,6 +549,36 @@ _Appears in:_
 | `workspaces` _[ConsumerWorkspace](#consumerworkspace) array_ | Allow access to the state for specific workspaces within the same organization. |
 
 
+#### RetryPolicy
+
+
+
+RetryPolicy allows you to configure retry behavior for failed runs on the workspace.
+It will apply for the latest current run of the operator.
+
+_Appears in:_
+- [WorkspaceSpec](#workspacespec)
+
+| Field | Description |
+| --- | --- |
+| `backoffLimit` _integer_ | Limit is the maximum number of retries for failed runs. If set to a negative number, no limit will be applied.<br />Default: `0`. |
+
+
+#### RetryStatus
+
+
+
+RetryStatus contains the status of the retry of the latest run on the workspace. How many attempts are left and
+possibly a time to wait for the next attempt.
+
+_Appears in:_
+- [WorkspaceStatus](#workspacestatus)
+
+| Field | Description |
+| --- | --- |
+| `failed` _integer_ | Failed is the number of failed attempts, counting the initial one. |
+
+
 #### RunStatus
 
 
@@ -894,6 +924,7 @@ _Appears in:_
 | `environmentVariables` _[Variable](#variable) array_ | Terraform Environment variables for all plans and applies in this workspace.<br />Variables defined within a workspace always overwrite variables from variable sets that have the same type and the same key.<br />More information:<br />  - https://developer.hashicorp.com/terraform/cloud-docs/workspaces/variables<br />  - https://developer.hashicorp.com/terraform/cloud-docs/workspaces/variables#environment-variables |
 | `terraformVariables` _[Variable](#variable) array_ | Terraform variables for all plans and applies in this workspace.<br />Variables defined within a workspace always overwrite variables from variable sets that have the same type and the same key.<br />More information:<br />  - https://developer.hashicorp.com/terraform/cloud-docs/workspaces/variables<br />  - https://developer.hashicorp.com/terraform/cloud-docs/workspaces/variables#terraform-variables |
 | `remoteStateSharing` _[RemoteStateSharing](#remotestatesharing)_ | Remote state access between workspaces.<br />By default, new workspaces in HCP Terraform do not allow other workspaces to access their state.<br />More information:<br />  - https://developer.hashicorp.com/terraform/cloud-docs/workspaces/state#accessing-state-from-other-workspaces |
+| `retryPolicy` _[RetryPolicy](#retrypolicy)_ | Retry Policy allows you to specify how the operator should retry failed runs automatically. |
 | `runTriggers` _[RunTrigger](#runtrigger) array_ | Run triggers allow you to connect this workspace to one or more source workspaces.<br />These connections allow runs to queue automatically in this workspace on successful apply of runs in any of the source workspaces.<br />More information:<br />  - https://developer.hashicorp.com/terraform/cloud-docs/workspaces/settings/run-triggers |
 | `versionControl` _[VersionControl](#versioncontrol)_ | Settings for the workspace's VCS repository, enabling the UI/VCS-driven run workflow.<br />Omit this argument to utilize the CLI-driven and API-driven workflows, where runs are not driven by webhooks on your VCS provider.<br />More information:<br />  - https://www.terraform.io/cloud-docs/run/ui<br />  - https://www.terraform.io/cloud-docs/vcs |
 | `sshKey` _[SSHKey](#sshkey)_ | SSH key used to clone Terraform modules.<br />More information:<br />  - https://developer.hashicorp.com/terraform/cloud-docs/workspaces/settings/ssh-keys |

internal/controller/agentpool_controller_autoscaling_test.go

@@ -92,7 +92,7 @@ var _ = Describe("Agent Pool controller", Ordered, func() {
 				Expect(err).Should(Succeed())
 				Expect(ws).ShouldNot(BeNil())
 				// Create a new Run and execute it
-				_ = createAndUploadConfigurationVersion(ws.ID, "hoi")
+				_ = createAndUploadConfigurationVersion(ws.ID, "hoi", true)
 				Eventually(func() bool {
 					ws, err = tfClient.Workspaces.ReadByID(ctx, ws.ID)
 					Expect(err).Should(Succeed())
@@ -147,7 +147,7 @@ var _ = Describe("Agent Pool controller", Ordered, func() {
 			Expect(err).Should(Succeed())
 			Expect(ws).ShouldNot(BeNil())
 			// New Run
-			_ = createAndUploadConfigurationVersion(ws.ID, "hoi")
+			_ = createAndUploadConfigurationVersion(ws.ID, "hoi", true)
 			Eventually(func() bool {
 				ws, err = tfClient.Workspaces.ReadByID(ctx, ws.ID)
 				Expect(err).Should(Succeed())

internal/controller/workspace_controller_deletion_policy.go

@@ -101,6 +101,39 @@ func (r *WorkspaceReconciler) deleteWorkspace(ctx context.Context, w *workspaceI
 
 		if _, ok := runStatusUnsuccessful[run.Status]; ok {
 			w.log.Info("Destroy Run", "msg", fmt.Sprintf("destroy run %s is unsuccessful: %s", run.ID, run.Status))
+
+			workspace, err := w.tfClient.Client.Workspaces.ReadByID(ctx, w.instance.Status.WorkspaceID)
+			if err != nil {
+				return r.handleWorkspaceErrorNotFound(ctx, w, err)
+			}
+
+			w.log.Info("Destroy Run", "msg", fmt.Sprintf("CurrentRun: %s %s %v", workspace.CurrentRun.ID, workspace.CurrentRun.Status, workspace.CurrentRun.IsDestroy))
+
+			if workspace.CurrentRun != nil && workspace.CurrentRun.ID != w.instance.Status.DestroyRunID {
+
+				run, err := w.tfClient.Client.Runs.Read(ctx, w.instance.Status.DestroyRunID)
+				if err != nil {
+					// ignore this run id, and let the next reconcile loop handle the error
+					return nil
+				}
+				if run.IsDestroy {
+					w.log.Info("Destroy Run", "msg", fmt.Sprintf("found more recent destroy run %s, updating DestroyRunID", workspace.CurrentRun.ID))
+
+					w.instance.Status.DestroyRunID = workspace.CurrentRun.ID
+					w.updateWorkspaceStatusRun(run)
+					return r.Status().Update(ctx, &w.instance)
+				}
+			}
+			if isRetryEnabled(w) {
+				w.log.Info("Destroy Run", "msg", fmt.Sprintf("ongoing destroy run %s is unsuccessful, retrying it", run.ID))
+				err := r.retryFailedDestroyRun(ctx, w, workspace, run)
+				if err != nil {
+					w.log.Info("Destroy Run", "msg", fmt.Sprintf("ongoing destroy run %s is unsuccessful, retrying it", run.ID))
+					return err
+				}
+				return r.Status().Update(ctx, &w.instance)
+			}
+
 			return nil
 		}
 		w.log.Info("Destroy Run", "msg", fmt.Sprintf("destroy run %s is not finished", run.ID))