diff --git a/.changes/unreleased/ENHANCEMENTS-594-20250424-164645.yaml b/.changes/unreleased/ENHANCEMENTS-594-20250424-164645.yaml new file mode 100644 index 00000000..d217e400 --- /dev/null +++ b/.changes/unreleased/ENHANCEMENTS-594-20250424-164645.yaml @@ -0,0 +1,5 @@ +kind: ENHANCEMENTS +body: Relink to more recent destroy runs when destroying a workspace +time: 2025-04-24T16:46:45.710081284+02:00 +custom: + PR: "594" diff --git a/internal/controller/workspace_controller_deletion_policy.go b/internal/controller/workspace_controller_deletion_policy.go index 47e3764a..badf222c 100644 --- a/internal/controller/workspace_controller_deletion_policy.go +++ b/internal/controller/workspace_controller_deletion_policy.go @@ -101,6 +101,30 @@ func (r *WorkspaceReconciler) deleteWorkspace(ctx context.Context, w *workspaceI if _, ok := runStatusUnsuccessful[run.Status]; ok { w.log.Info("Destroy Run", "msg", fmt.Sprintf("destroy run %s is unsuccessful: %s", run.ID, run.Status)) + + workspace, err := w.tfClient.Client.Workspaces.ReadByID(ctx, w.instance.Status.WorkspaceID) + if err != nil { + return r.handleWorkspaceErrorNotFound(ctx, w, err) + } + + w.log.Info("Destroy Run", "msg", fmt.Sprintf("CurrentRun: %s %s %v", workspace.CurrentRun.ID, workspace.CurrentRun.Status, workspace.CurrentRun.IsDestroy)) + + if workspace.CurrentRun != nil && workspace.CurrentRun.ID != w.instance.Status.DestroyRunID { + + run, err := w.tfClient.Client.Runs.Read(ctx, w.instance.Status.DestroyRunID) + if err != nil { + // ignore this run id, and let the next reconcile loop handle the error + return nil + } + if run.IsDestroy { + w.log.Info("Destroy Run", "msg", fmt.Sprintf("found more recent destroy run %s, updating DestroyRunID", workspace.CurrentRun.ID)) + + w.instance.Status.DestroyRunID = workspace.CurrentRun.ID + w.updateWorkspaceStatusRun(run) + return r.Status().Update(ctx, &w.instance) + } + } + return nil } w.log.Info("Destroy Run", "msg", fmt.Sprintf("destroy run %s is not finished", run.ID)) diff --git a/internal/controller/workspace_controller_deletion_policy_test.go b/internal/controller/workspace_controller_deletion_policy_test.go index 68e54dfe..a3e13e36 100644 --- a/internal/controller/workspace_controller_deletion_policy_test.go +++ b/internal/controller/workspace_controller_deletion_policy_test.go @@ -195,6 +195,97 @@ var _ = Describe("Workspace controller", Ordered, func() { return err == tfc.ErrResourceNotFound }).Should(BeTrue()) }) + It("can destroy delete a workspace when the destroy was retried manually after failing", func() { + if cloudEndpoint != tfcDefaultAddress { + Skip("Does not run against TFC, skip this test") + } + instance.Spec.AllowDestroyPlan = true + instance.Spec.DeletionPolicy = appv1alpha2.DeletionPolicyDestroy + createWorkspace(instance) + workspaceID := instance.Status.WorkspaceID + + cv := createAndUploadConfigurationVersion(instance.Status.WorkspaceID, "hoi") + Eventually(func() bool { + listOpts := tfc.ListOptions{ + PageNumber: 1, + PageSize: maxPageSize, + } + for listOpts.PageNumber != 0 { + runs, err := tfClient.Runs.List(ctx, workspaceID, &tfc.RunListOptions{ + ListOptions: listOpts, + }) + Expect(err).To(Succeed()) + for _, r := range runs.Items { + if r.ConfigurationVersion.ID == cv.ID { + return r.Status == tfc.RunApplied + } + } + listOpts.PageNumber = runs.NextPage + } + return false + }).Should(BeTrue()) + + // create an errored ConfigurationVersion for the delete to fail + cv = createAndUploadErroredConfigurationVersion(instance.Status.WorkspaceID, false) + + Expect(k8sClient.Delete(ctx, instance)).To(Succeed()) + + var destroyRunID string + Eventually(func() bool { + ws, err := tfClient.Workspaces.ReadByID(ctx, workspaceID) + Expect(err).To(Succeed()) + Expect(ws).ToNot(BeNil()) + Expect(ws.CurrentRun).ToNot(BeNil()) + run, err := tfClient.Runs.Read(ctx, ws.CurrentRun.ID) + Expect(err).To(Succeed()) + Expect(run).ToNot(BeNil()) + destroyRunID = run.ID + + return run.IsDestroy + }).Should(BeTrue()) + + Eventually(func() bool { + run, _ := tfClient.Runs.Read(ctx, destroyRunID) + if run.Status == tfc.RunErrored { + return true + } + + return false + }).Should(BeTrue()) + + // put back a working configuration + cv = createAndUploadConfigurationVersion(instance.Status.WorkspaceID, "hoi") + + // start a new destroy run manually + run, err := tfClient.Runs.Create(ctx, tfc.RunCreateOptions{ + IsDestroy: tfc.Bool(true), + Message: tfc.String(runMessage), + Workspace: &tfc.Workspace{ + ID: workspaceID, + }, + }) + Expect(err).To(Succeed()) + Expect(run).ToNot(BeNil()) + + var newDestroyRunID string + Eventually(func() bool { + ws, err := tfClient.Workspaces.ReadByID(ctx, workspaceID) + Expect(err).To(Succeed()) + Expect(ws).ToNot(BeNil()) + Expect(ws.CurrentRun).ToNot(BeNil()) + run, err := tfClient.Runs.Read(ctx, ws.CurrentRun.ID) + Expect(err).To(Succeed()) + Expect(run).ToNot(BeNil()) + newDestroyRunID = run.ID + + return run.IsDestroy && newDestroyRunID != destroyRunID + }).Should(BeTrue()) + + Eventually(func() bool { + _, err := tfClient.Workspaces.ReadByID(ctx, workspaceID) + return err == tfc.ErrResourceNotFound + }).Should(BeTrue()) + }) It("can force delete a workspace", func() { instance.Spec.DeletionPolicy = appv1alpha2.DeletionPolicyForce createWorkspace(instance) diff --git a/internal/controller/workspace_controller_outputs_test.go b/internal/controller/workspace_controller_outputs_test.go index e9a3be34..f8a611b2 100644 --- a/internal/controller/workspace_controller_outputs_test.go +++ b/internal/controller/workspace_controller_outputs_test.go @@ -180,3 +180,46 @@ func createAndUploadConfigurationVersion(workspaceID string, outputValue string) return cv } + +func createAndUploadErroredConfigurationVersion(workspaceID string, autoQueueRuns bool) *tfc.ConfigurationVersion { + GinkgoHelper() + // Create a temporary dir in the current one + cd, err := os.Getwd() + Expect(err).Should(Succeed()) + td, err := os.MkdirTemp(cd, "tf-*") + Expect(err).Should(Succeed()) + defer os.RemoveAll(td) + // Create a temporary file in the temporary dir + f, err := os.CreateTemp(td, "*.tf") + Expect(err).Should(Succeed()) + defer os.Remove(f.Name()) + // Terraform code to upload + tf := fmt.Sprint(` + resource "test_non_existent_resource" "this" {} + `) + // Save the Terraform code to the temporary file + _, err = f.WriteString(tf) + Expect(err).Should(Succeed()) + + cv, err := tfClient.ConfigurationVersions.Create(ctx, workspaceID, tfc.ConfigurationVersionCreateOptions{ + AutoQueueRuns: tfc.Bool(autoQueueRuns), + Speculative: tfc.Bool(false), + }) + Expect(err).Should(Succeed()) + Expect(cv).ShouldNot(BeNil()) + + Expect(tfClient.ConfigurationVersions.Upload(ctx, cv.UploadURL, td)).Should(Succeed()) + + Eventually(func() bool { + c, err := tfClient.ConfigurationVersions.Read(ctx, cv.ID) + if err != nil { + return false + } + if c.Status == tfc.ConfigurationUploaded { + return true + } + return false + }).Should(BeTrue()) + + return cv +}