feat: Add support for online repair of Machines, allowing repair without Instance deletion

api/pkg/api/handler/instance_test.go

@@ -8844,7 +8844,7 @@ func TestDeleteInstanceHandler_Handle(t *testing.T) {
 				cfg:       cfg,
 			},
 			args: args{
-				reqData: &model.APIInstanceDeleteRequest{MachineHealthIssue: &model.APIMachineHealthIssueReport{
+				reqData: &model.APIInstanceDeleteRequest{MachineHealthIssue: &model.APIMachineHealthIssue{
 					Category: "Hardware", Summary: cdb.GetStrPtr("Some summary"), Details: cdb.GetStrPtr("Some details"),
 				}},
 				reqInstance: inst1.ID.String(),
@@ -8864,7 +8864,7 @@ func TestDeleteInstanceHandler_Handle(t *testing.T) {
 				cfg:       cfg,
 			},
 			args: args{
-				reqData: &model.APIInstanceDeleteRequest{MachineHealthIssue: &model.APIMachineHealthIssueReport{
+				reqData: &model.APIInstanceDeleteRequest{MachineHealthIssue: &model.APIMachineHealthIssue{
 					Category: "UNSPECIFIED", Summary: cdb.GetStrPtr("Some summary"), Details: cdb.GetStrPtr("Some details"),
 				}},
 				reqInstance: inst1.ID.String(),

api/pkg/api/handler/machine_test.go

@@ -1616,6 +1616,7 @@ func TestMachineHandler_Update(t *testing.T) {
 
 	tenant := testMachineBuildTenant(t, dbSession, ipOrg1, "testTenant1")
 	tenant2 := testMachineBuildTenant(t, dbSession, tnOrg2, "testTenant2")
+	_ = testMachineUpdateTenantCapability(t, dbSession, tenant2)
 	_ = common.TestBuildTenantAccount(t, dbSession, ip, &tenant2.ID, tnOrg2, cdbm.TenantAccountStatusReady, tnu2)
 
 	instanceType1 := testMachineBuildInstanceType(t, dbSession, ip, site, "testInstanceType1")
@@ -1702,6 +1703,104 @@ func TestMachineHandler_Update(t *testing.T) {
 	assert.Nil(t, err)
 	assert.NotNil(t, i1)
 
+	mOnlineRepairReady := testMachineBuildMachine(t, dbSession, ip.ID, site.ID, cdb.GetUUIDPtr(instanceType2.ID), cdb.GetStrPtr("mcType"), true, false, cdbm.MachineStatusReady)
+	_ = common.TestBuildMachineInstanceType(t, dbSession, mOnlineRepairReady, instanceType2)
+	iOnlineRepairReady, err := isd.Create(
+		context.Background(), nil,
+		cdbm.InstanceCreateInput{
+			Name:                     "testOnlineRepairReady",
+			TenantID:                 tenant.ID,
+			InfrastructureProviderID: ip.ID,
+			SiteID:                   site.ID,
+			InstanceTypeID:           &instanceType2.ID,
+			VpcID:                    vpc.ID,
+			MachineID:                &mOnlineRepairReady.ID,
+			Hostname:                 cdb.GetStrPtr("or-ready.example.com"),
+			OperatingSystemID:        cdb.GetUUIDPtr(operatingSystem.ID),
+			IpxeScript:               cdb.GetStrPtr("ipxe"),
+			AlwaysBootWithCustomIpxe: true,
+			UserData:                 cdb.GetStrPtr("userdata"),
+			Status:                   cdbm.InstanceStatusReady,
+			CreatedBy:                tnu.ID,
+		},
+	)
+	require.NoError(t, err)
+
+	mOnlineRepairMissing := testMachineBuildMachine(t, dbSession, ip.ID, site.ID, cdb.GetUUIDPtr(instanceType2.ID), cdb.GetStrPtr("mcType"), true, true, cdbm.MachineStatusReady)
+	_ = common.TestBuildMachineInstanceType(t, dbSession, mOnlineRepairMissing, instanceType2)
+
+	mOnlineRepairPending := testMachineBuildMachine(t, dbSession, ip.ID, site.ID, cdb.GetUUIDPtr(instanceType2.ID), cdb.GetStrPtr("mcType"), true, false, cdbm.MachineStatusReady)
+	_ = common.TestBuildMachineInstanceType(t, dbSession, mOnlineRepairPending, instanceType2)
+	_, err = isd.Create(
+		context.Background(), nil,
+		cdbm.InstanceCreateInput{
+			Name:                     "testOnlineRepairPending",
+			TenantID:                 tenant.ID,
+			InfrastructureProviderID: ip.ID,
+			SiteID:                   site.ID,
+			InstanceTypeID:           &instanceType2.ID,
+			VpcID:                    vpc.ID,
+			MachineID:                &mOnlineRepairPending.ID,
+			Hostname:                 cdb.GetStrPtr("or-pending.example.com"),
+			OperatingSystemID:        cdb.GetUUIDPtr(operatingSystem.ID),
+			IpxeScript:               cdb.GetStrPtr("ipxe"),
+			AlwaysBootWithCustomIpxe: true,
+			UserData:                 cdb.GetStrPtr("userdata"),
+			Status:                   cdbm.InstanceStatusPending,
+			CreatedBy:                tnu.ID,
+		},
+	)
+	require.NoError(t, err)
+
+	buildOnlineRepairEnterRequest := func(allowAutoInstanceDeletion bool) *model.APIMachineUpdateRequest {
+		return &model.APIMachineUpdateRequest{
+			OnlineRepair: &model.APIMachineOnlineRepair{
+				Enabled: cdb.GetBoolPtr(true),
+				Policy: &model.APIMachineOnlineRepairPolicy{
+					AllowAutoInstanceDeletionOnFailure: cdb.GetBoolPtr(allowAutoInstanceDeletion),
+				},
+				Acknowledgments: &model.APIMachineOnlineRepairAcknowledgments{
+					AcceptDataCorruptionRisk:   cdb.GetBoolPtr(true),
+					AcceptRepairTeamAccess:     cdb.GetBoolPtr(true),
+					AcceptInstanceDeletionRisk: cdb.GetBoolPtr(true),
+				},
+			},
+			HealthIssue: &model.APIMachineHealthIssue{
+				Category: model.HealthIssueStorage,
+				Summary:  cdb.GetStrPtr("tenant summary"),
+				Details:  cdb.GetStrPtr("tenant details"),
+			},
+		}
+	}
+
+	buildOnlineRepairExitRequest := func() *model.APIMachineUpdateRequest {
+		return &model.APIMachineUpdateRequest{
+			OnlineRepair: &model.APIMachineOnlineRepair{
+				Enabled: cdb.GetBoolPtr(false),
+			},
+		}
+	}
+
+	machineRepairResetReady := func(t *testing.T) {
+		t.Helper()
+		_, uerr := isd.Update(context.Background(), nil, cdbm.InstanceUpdateInput{
+			InstanceID: iOnlineRepairReady.ID,
+			Status:     cdb.GetStrPtr(cdbm.InstanceStatusReady),
+			Labels:     map[string]string{},
+		})
+		require.NoError(t, uerr)
+	}
+
+	machineRepairSetRepairing := func(t *testing.T) {
+		t.Helper()
+		_, uerr := isd.Update(context.Background(), nil, cdbm.InstanceUpdateInput{
+			InstanceID: iOnlineRepairReady.ID,
+			Status:     cdb.GetStrPtr(cdbm.InstanceStatusRepairing),
+			Labels:     map[string]string{model.InstanceLabelOnlineRepairAllowAutoDeletion: "false"},
+		})
+		require.NoError(t, uerr)
+	}
+
 	mit1 := common.TestBuildMachineInstanceType(t, dbSession, m, instanceType1)
 	assert.NotNil(t, mit1)
 
@@ -1733,6 +1832,8 @@ func TestMachineHandler_Update(t *testing.T) {
 	tsc.Mock.On("ExecuteWorkflow", mock.Anything, mock.AnythingOfType("internal.StartWorkflowOptions"), "AssociateMachinesWithInstanceType", mock.Anything).Return(wrun, nil)
 	tsc.Mock.On("ExecuteWorkflow", mock.Anything, mock.AnythingOfType("internal.StartWorkflowOptions"), "RemoveMachineInstanceTypeAssociation", mock.Anything).Return(wrun, nil)
 	tsc.Mock.On("ExecuteWorkflow", mock.Anything, mock.AnythingOfType("internal.StartWorkflowOptions"), "UpdateMachineMetadata", mock.Anything).Return(wrun, nil)
+	tsc.Mock.On("ExecuteWorkflow", mock.Anything, mock.AnythingOfType("internal.StartWorkflowOptions"), "CreateMachineHealthReportOverride", mock.Anything).Return(wrun, nil)
+	tsc.Mock.On("ExecuteWorkflow", mock.Anything, mock.AnythingOfType("internal.StartWorkflowOptions"), "DeleteMachineHealthReportOverride", mock.Anything).Return(wrun, nil)
 
 	// Mock timeout error
 	wruntimeout := &tmocks.WorkflowRun{}
@@ -1744,6 +1845,8 @@ func TestMachineHandler_Update(t *testing.T) {
 	tsc1.Mock.On("ExecuteWorkflow", mock.Anything, mock.AnythingOfType("internal.StartWorkflowOptions"), "AssociateMachinesWithInstanceType", mock.Anything).Return(wruntimeout, nil)
 	tsc1.Mock.On("ExecuteWorkflow", mock.Anything, mock.AnythingOfType("internal.StartWorkflowOptions"), "RemoveMachineInstanceTypeAssociation", mock.Anything).Return(wruntimeout, nil)
 	tsc1.Mock.On("ExecuteWorkflow", mock.Anything, mock.AnythingOfType("internal.StartWorkflowOptions"), "UpdateMachineMetadata", mock.Anything).Return(wruntimeout, nil)
+	tsc1.Mock.On("ExecuteWorkflow", mock.Anything, mock.AnythingOfType("internal.StartWorkflowOptions"), "CreateMachineHealthReportOverride", mock.Anything).Return(wruntimeout, nil)
+	tsc1.Mock.On("ExecuteWorkflow", mock.Anything, mock.AnythingOfType("internal.StartWorkflowOptions"), "DeleteMachineHealthReportOverride", mock.Anything).Return(wruntimeout, nil)
 
 	tsc1.Mock.On("TerminateWorkflow", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil)
 
@@ -1774,6 +1877,8 @@ func TestMachineHandler_Update(t *testing.T) {
 		reqOldMachineInstanceType   *cdbm.MachineInstanceType
 		reqClearInstanceType        bool
 		reqLabels                   map[string]string
+		beforeHandle                func(t *testing.T)
+		verifyOnlineRepair          func(t *testing.T)
 	}
 
 	machineDAO := cdbm.NewMachineDAO(dbSession)
@@ -2263,6 +2368,173 @@ func TestMachineHandler_Update(t *testing.T) {
 				reqLabels:  map[string]string{"test-2": "test-2"},
 			},
 		},
+		{
+			name: "test Machine update API online repair failure, Machine is missing on Site",
+			fields: fields{
+				dbSession: dbSession,
+				tc:        tc,
+				scp:       scp,
+				cfg:       cfg,
+			},
+			args: args{
+				reqData:    buildOnlineRepairEnterRequest(false),
+				reqMachine: mOnlineRepairMissing,
+				reqOrg:     ipOrg1,
+				reqUser:    ipu,
+				respCode:   http.StatusBadRequest,
+			},
+		},
+		{
+			name: "test Machine update API online repair failure, Machine has no assigned Instance",
+			fields: fields{
+				dbSession: dbSession,
+				tc:        tc,
+				scp:       scp,
+				cfg:       cfg,
+			},
+			args: args{
+				reqData:    buildOnlineRepairEnterRequest(false),
+				reqMachine: m7,
+				reqOrg:     ipOrg1,
+				reqUser:    ipu,
+				respCode:   http.StatusBadRequest,
+			},
+		},
+		{
+			name: "test Machine update API online repair failure, Instance must be Ready to enter",
+			fields: fields{
+				dbSession: dbSession,
+				tc:        tc,
+				scp:       scp,
+				cfg:       cfg,
+			},
+			args: args{
+				reqData:    buildOnlineRepairEnterRequest(false),
+				reqMachine: mOnlineRepairPending,
+				reqOrg:     ipOrg1,
+				reqUser:    ipu,
+				respCode:   http.StatusBadRequest,
+			},
+		},
+		{
+			name: "test Machine update API online repair failure, Instance must be Repairing to exit",
+			fields: fields{
+				dbSession: dbSession,
+				tc:        tc,
+				scp:       scp,
+				cfg:       cfg,
+			},
+			args: args{
+				reqData:    buildOnlineRepairExitRequest(),
+				reqMachine: mOnlineRepairReady,
+				reqOrg:     ipOrg1,
+				reqUser:    ipu,
+				respCode:   http.StatusBadRequest,
+				beforeHandle: func(t *testing.T) {
+					machineRepairResetReady(t)
+				},
+			},
+		},
+		{
+			name: "test Machine update API online repair success, enter repair (Provider Admin)",
+			fields: fields{
+				dbSession: dbSession,
+				tc:        tc,
+				scp:       scp,
+				cfg:       cfg,
+			},
+			args: args{
+				reqData:    buildOnlineRepairEnterRequest(false),
+				reqMachine: mOnlineRepairReady,
+				reqOrg:     ipOrg1,
+				reqUser:    ipu,
+				respCode:   http.StatusOK,
+				beforeHandle: func(t *testing.T) {
+					machineRepairResetReady(t)
+				},
+				verifyOnlineRepair: func(t *testing.T) {
+					inst, gerr := isd.GetByID(context.Background(), nil, iOnlineRepairReady.ID, nil)
+					require.NoError(t, gerr)
+					assert.Equal(t, cdbm.InstanceStatusRepairing, inst.Status)
+					assert.Equal(t, "false", inst.Labels[model.InstanceLabelOnlineRepairAllowAutoDeletion])
+				},
+			},
+		},
+		{
+			name: "test Machine update API online repair success, enter repair with allowAutoInstanceDeletionOnFailure",
+			fields: fields{
+				dbSession: dbSession,
+				tc:        tc,
+				scp:       scp,
+				cfg:       cfg,
+			},
+			args: args{
+				reqData:    buildOnlineRepairEnterRequest(true),
+				reqMachine: mOnlineRepairReady,
+				reqOrg:     ipOrg1,
+				reqUser:    ipu,
+				respCode:   http.StatusOK,
+				beforeHandle: func(t *testing.T) {
+					machineRepairResetReady(t)
+				},
+				verifyOnlineRepair: func(t *testing.T) {
+					inst, gerr := isd.GetByID(context.Background(), nil, iOnlineRepairReady.ID, nil)
+					require.NoError(t, gerr)
+					assert.Equal(t, cdbm.InstanceStatusRepairing, inst.Status)
+					assert.Equal(t, "true", inst.Labels[model.InstanceLabelOnlineRepairAllowAutoDeletion])
+				},
+			},
+		},
+		{
+			name: "test Machine update API online repair success, exit repair",
+			fields: fields{
+				dbSession: dbSession,
+				tc:        tc,
+				scp:       scp,
+				cfg:       cfg,
+			},
+			args: args{
+				reqData:    buildOnlineRepairExitRequest(),
+				reqMachine: mOnlineRepairReady,
+				reqOrg:     ipOrg1,
+				reqUser:    ipu,
+				respCode:   http.StatusOK,
+				beforeHandle: func(t *testing.T) {
+					machineRepairSetRepairing(t)
+				},
+				verifyOnlineRepair: func(t *testing.T) {
+					inst, gerr := isd.GetByID(context.Background(), nil, iOnlineRepairReady.ID, nil)
+					require.NoError(t, gerr)
+					assert.Equal(t, cdbm.InstanceStatusReady, inst.Status)
+					_, has := inst.Labels[model.InstanceLabelOnlineRepairAllowAutoDeletion]
+					assert.False(t, has)
+				},
+			},
+		},
+		{
+			name: "test Machine update API online repair success, enter repair (privileged Tenant Admin)",
+			fields: fields{
+				dbSession: dbSession,
+				tc:        tc,
+				scp:       scp,
+				cfg:       cfg,
+			},
+			args: args{
+				reqData:    buildOnlineRepairEnterRequest(false),
+				reqMachine: mOnlineRepairReady,
+				reqOrg:     tnOrg2,
+				reqUser:    tnu2,
+				respCode:   http.StatusOK,
+				beforeHandle: func(t *testing.T) {
+					machineRepairResetReady(t)
+				},
+				verifyOnlineRepair: func(t *testing.T) {
+					inst, gerr := isd.GetByID(context.Background(), nil, iOnlineRepairReady.ID, nil)
+					require.NoError(t, gerr)
+					assert.Equal(t, cdbm.InstanceStatusRepairing, inst.Status)
+				},
+			},
+		},
 		{
 			name: "test Machine update API endpoint failure, Instance Type is being cleared by Tenant Admin with TargetedInstanceCreation capability",
 			fields: fields{
@@ -2308,6 +2580,10 @@ func TestMachineHandler_Update(t *testing.T) {
 			ctx = context.WithValue(ctx, otelecho.TracerKey, tracer)
 			ec.SetRequest(ec.Request().WithContext(ctx))
 
+			if tt.args.beforeHandle != nil {
+				tt.args.beforeHandle(t)
+			}
+
 			err := umh.Handle(ec)
 			require.NoError(t, err)
 
@@ -2351,7 +2627,10 @@ func TestMachineHandler_Update(t *testing.T) {
 					}
 				}
 
-				assert.NotEqual(t, rst.Updated.String(), tt.args.reqMachine.Updated.String())
+				// Online repair updates Instance (and Site workflow); Machine row may not be touched, so Updated can match.
+				if tt.args.verifyOnlineRepair == nil {
+					assert.NotEqual(t, rst.Updated.String(), tt.args.reqMachine.Updated.String())
+				}
 
 				mitDAO := cdbm.NewMachineInstanceTypeDAO(dbSession)
 
@@ -2409,6 +2688,10 @@ func TestMachineHandler_Update(t *testing.T) {
 				if tt.args.reqLabels != nil {
 					assert.Equal(t, rst.Labels, tt.args.reqLabels)
 				}
+
+				if tt.args.verifyOnlineRepair != nil {
+					tt.args.verifyOnlineRepair(t)
+				}
 			}
 
 			if tt.verifyChildSpanner {

api/pkg/api/model/instance.go

@@ -1476,18 +1476,8 @@ func (iur APIInstanceUpdateRequest) Validate() error {
 // APIInstanceDeleteRequest is the data structure to capture request to delete an Instance
 type APIInstanceDeleteRequest struct {
 	// MachineHealthIssue is the report of a machine health issue
-	MachineHealthIssue *APIMachineHealthIssueReport `json:"machineHealthIssue"`
-	IsRepairTenant     *bool                        `json:"isRepairTenant"`
-}
-
-// APIMachineHealthIssueReport is the data structure to capture a machine health issue report
-type APIMachineHealthIssueReport struct {
-	// Category is the type of the issue
-	Category string `json:"category"`
-	// Summary is the summary of the issue
-	Summary *string `json:"summary"`
-	// Details is the message of the issue
-	Details *string `json:"details"`
+	MachineHealthIssue *APIMachineHealthIssue `json:"machineHealthIssue"`
+	IsRepairTenant     *bool                  `json:"isRepairTenant"`
 }
 
 // Validate ensures the values passed in request are acceptable