upgrade: Add support for retrying automatically without remediations

Signed-off-by: Matheus Pimenta <[email protected]>

Author: matheuscscp

api/v2/helmrelease_types.go

@@ -437,6 +437,21 @@ type Remediation interface {
 	RetriesExhausted(hr *HelmRelease) bool
 }
 
+// Strategy defines a consistent interface for InstallStrategy and
+// UpgradeStrategy.
+// +kubebuilder:object:generate=false
+type Strategy interface {
+	GetRetry() Retry
+}
+
+// Retry defines a consistent interface for retry strategies from
+// InstallStrategy and UpgradeStrategy.
+// +kubebuilder:object:generate=false
+type Retry interface {
+	GetRetryInterval() time.Duration
+	GetAction() ReleaseAction
+}
+
 // Install holds the configuration for Helm install actions performed for this
 // HelmRelease.
 type Install struct {
@@ -631,6 +646,11 @@ type Upgrade struct {
 	// +optional
 	Timeout *metav1.Duration `json:"timeout,omitempty"`
 
+	// Strategy defines the upgrade strategy to use for this HelmRelease.
+	// Defaults to 'RemediateOnFailure'.
+	// +optional
+	Strategy *UpgradeStrategy `json:"strategy,omitempty"`
+
 	// Remediation holds the remediation configuration for when the Helm upgrade
 	// action for the HelmRelease fails. The default is to not perform any action.
 	// +optional
@@ -719,6 +739,46 @@ func (in Upgrade) GetRemediation() Remediation {
 	return *in.Remediation
 }
 
+// GetRetry returns the configured retry strategy for the Helm upgrade
+// action.
+func (in Upgrade) GetRetry() Retry {
+	if in.Strategy == nil || in.Strategy.Name != string(ActionStrategyRetryOnFailure) {
+		return nil
+	}
+	return in.Strategy
+}
+
+// UpgradeStrategy holds the configuration for Helm upgrade strategy.
+// +kubebuilder:validation:XValidation:rule="!has(self.retryInterval) || self.name == 'RetryOnFailure'", message=".retryInterval can only be set when .name is 'RetryOnFailure'"
+type UpgradeStrategy struct {
+	// Name of the upgrade strategy.
+	// +kubebuilder:validation:Enum=RemediateOnFailure;RetryOnFailure
+	// +required
+	Name string `json:"name"`
+
+	// RetryInterval is the interval at which to retry a failed upgrade.
+	// Can be used only when Name is set to RetryOnFailure.
+	// Defaults to '5m'.
+	// +kubebuilder:validation:Type=string
+	// +kubebuilder:validation:Pattern="^([0-9]+(\\.[0-9]+)?(ms|s|m|h))+$"
+	// +optional
+	RetryInterval *metav1.Duration `json:"retryInterval,omitempty"`
+}
+
+// GetRetryInterval returns the configured retry interval for the Helm upgrade
+// action, or the default.
+func (in UpgradeStrategy) GetRetryInterval() time.Duration {
+	if in.RetryInterval == nil {
+		return 5 * time.Minute
+	}
+	return in.RetryInterval.Duration
+}
+
+// GetAction returns the retry action for the Helm upgrade action.
+func (in UpgradeStrategy) GetAction() ReleaseAction {
+	return ReleaseActionUpgrade
+}
+
 // UpgradeRemediation holds the configuration for Helm upgrade remediation.
 type UpgradeRemediation struct {
 	// Retries is the number of retries that should be attempted on failures before
@@ -791,6 +851,19 @@ func (in UpgradeRemediation) RetriesExhausted(hr *HelmRelease) bool {
 	return in.Retries >= 0 && in.GetFailureCount(hr) > int64(in.Retries)
 }
 
+// ActionStrategyName is a valid name for an action strategy.
+type ActionStrategyName string
+
+const (
+	// ActionStrategyRemediateOnFailure is the action strategy name for
+	// remediate on failure.
+	ActionStrategyRemediateOnFailure ActionStrategyName = "RemediateOnFailure"
+
+	// ActionStrategyRetryOnFailure is the action strategy name for retry on
+	// failure.
+	ActionStrategyRetryOnFailure ActionStrategyName = "RetryOnFailure"
+)
+
 // RemediationStrategy returns the strategy to use to remediate a failed install
 // or upgrade.
 type RemediationStrategy string
@@ -1012,7 +1085,8 @@ type HelmReleaseStatus struct {
 	History Snapshots `json:"history,omitempty"`
 
 	// LastAttemptedReleaseAction is the last release action performed for this
-	// HelmRelease. It is used to determine the active remediation strategy.
+	// HelmRelease. It is used to determine the active retry or remediation
+	// strategy.
 	// +kubebuilder:validation:Enum=install;upgrade
 	// +optional
 	LastAttemptedReleaseAction ReleaseAction `json:"lastAttemptedReleaseAction,omitempty"`
@@ -1189,6 +1263,17 @@ func (in HelmRelease) GetActiveRemediation() Remediation {
 	}
 }
 
+// GetActiveRetry returns the active retry configuration for the
+// HelmRelease.
+func (in HelmRelease) GetActiveRetry() Retry {
+	switch in.Status.LastAttemptedReleaseAction {
+	case ReleaseActionUpgrade:
+		return in.GetUpgrade().GetRetry()
+	default:
+		return nil
+	}
+}
+
 // GetRequeueAfter returns the duration after which the HelmRelease
 // must be reconciled again.
 func (in HelmRelease) GetRequeueAfter() time.Duration {

api/v2/zz_generated.deepcopy.go

@@ -912,6 +912,30 @@ spec:
                         - uninstall
                         type: string
                     type: object
+                  strategy:
+                    description: |-
+                      Strategy defines the upgrade strategy to use for this HelmRelease.
+                      Defaults to 'RemediateOnFailure'.
+                    properties:
+                      name:
+                        description: Name of the upgrade strategy.
+                        enum:
+                        - RemediateOnFailure
+                        - RetryOnFailure
+                        type: string
+                      retryInterval:
+                        description: |-
+                          RetryInterval is the interval at which to retry a failed upgrade.
+                          Can be used only when Name is set to RetryOnFailure.
+                          Defaults to '5m'.
+                        pattern: ^([0-9]+(\.[0-9]+)?(ms|s|m|h))+$
+                        type: string
+                    required:
+                    - name
+                    type: object
+                    x-kubernetes-validations:
+                    - message: .retryInterval can only be set when .name is 'RetryOnFailure'
+                      rule: '!has(self.retryInterval) || self.name == ''RetryOnFailure'''
                   timeout:
                     description: |-
                       Timeout is the time to wait for any individual Kubernetes operation (like
@@ -1178,7 +1202,8 @@ spec:
               lastAttemptedReleaseAction:
                 description: |-
                   LastAttemptedReleaseAction is the last release action performed for this
-                  HelmRelease. It is used to determine the active remediation strategy.
+                  HelmRelease. It is used to determine the active retry or remediation
+                  strategy.
                 enum:
                 - install
                 - upgrade

config/crd/bases/helm.toolkit.fluxcd.io_helmreleases.yaml

@@ -424,6 +424,9 @@ HelmReleaseStatus
 </table>
 </div>
 </div>
+<h3 id="helm.toolkit.fluxcd.io/v2.ActionStrategyName">ActionStrategyName
+(<code>string</code> alias)</h3>
+<p>ActionStrategyName is a valid name for an action strategy.</p>
 <h3 id="helm.toolkit.fluxcd.io/v2.CRDsPolicy">CRDsPolicy
 (<code>string</code> alias)</h3>
 <p>
@@ -1676,7 +1679,8 @@ ReleaseAction
 <td>
 <em>(Optional)</em>
 <p>LastAttemptedReleaseAction is the last release action performed for this
-HelmRelease. It is used to determine the active remediation strategy.</p>
+HelmRelease. It is used to determine the active retry or remediation
+strategy.</p>
 </td>
 </tr>
 <tr>
@@ -2247,6 +2251,10 @@ UpgradeRemediation.</p>
 </p>
 <p>RemediationStrategy returns the strategy to use to remediate a failed install
 or upgrade.</p>
+<h3 id="helm.toolkit.fluxcd.io/v2.Retry">Retry
+</h3>
+<p>Retry defines a consistent interface for retry strategies from
+InstallStrategy and UpgradeStrategy.</p>
 <h3 id="helm.toolkit.fluxcd.io/v2.Rollback">Rollback
 </h3>
 <p>
@@ -2570,6 +2578,10 @@ string
 <a href="#helm.toolkit.fluxcd.io/v2.HelmReleaseStatus">HelmReleaseStatus</a>)
 </p>
 <p>Snapshots is a list of Snapshot objects.</p>
+<h3 id="helm.toolkit.fluxcd.io/v2.Strategy">Strategy
+</h3>
+<p>Strategy defines a consistent interface for InstallStrategy and
+UpgradeStrategy.</p>
 <h3 id="helm.toolkit.fluxcd.io/v2.Test">Test
 </h3>
 <p>
@@ -2833,6 +2845,21 @@ Jobs for hooks) during the performance of a Helm upgrade action. Defaults to
 </tr>
 <tr>
 <td>
+<code>strategy</code><br>
+<em>
+<a href="#helm.toolkit.fluxcd.io/v2.UpgradeStrategy">
+UpgradeStrategy
+</a>
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>Strategy defines the upgrade strategy to use for this HelmRelease.
+Defaults to &lsquo;RemediateOnFailure&rsquo;.</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>remediation</code><br>
 <em>
 <a href="#helm.toolkit.fluxcd.io/v2.UpgradeRemediation">
@@ -3066,6 +3093,54 @@ RemediationStrategy
 </table>
 </div>
 </div>
+<h3 id="helm.toolkit.fluxcd.io/v2.UpgradeStrategy">UpgradeStrategy
+</h3>
+<p>
+(<em>Appears on:</em>
+<a href="#helm.toolkit.fluxcd.io/v2.Upgrade">Upgrade</a>)
+</p>
+<p>UpgradeStrategy holds the configuration for Helm upgrade strategy.</p>
+<div class="md-typeset__scrollwrap">
+<div class="md-typeset__table">
+<table>
+<thead>
+<tr>
+<th>Field</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>
+<code>name</code><br>
+<em>
+string
+</em>
+</td>
+<td>
+<p>Name of the upgrade strategy.</p>
+</td>
+</tr>
+<tr>
+<td>
+<code>retryInterval</code><br>
+<em>
+<a href="https://godoc.org/k8s.io/apimachinery/pkg/apis/meta/v1#Duration">
+Kubernetes meta/v1.Duration
+</a>
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>RetryInterval is the interval at which to retry a failed upgrade.
+Can be used only when Name is set to RetryOnFailure.
+Defaults to &lsquo;5m&rsquo;.</p>
+</td>
+</tr>
+</tbody>
+</table>
+</div>
+</div>
 <div class="admonition note">
 <p class="last">This page was automatically generated with <code>gen-crd-api-reference-docs</code></p>
 </div>

docs/api/v2/helm.md

@@ -606,6 +606,30 @@ The field offers the following subfields:
   last release while merging in overrides from [values](#values). Setting
   this flag makes the HelmRelease non-declarative. Defaults to `false`.
 
+#### Upgrade strategy
+
+`.spec.upgrade.strategy` is an optional field to specify the strategy
+to use when running a Helm upgrade action.
+
+The field offers the following subfields:
+
+- `.name` (Required): The name of the upgrade strategy to use. One of
+  `RemediateOnFailure` or `RetryOnFailure`. If the `.spec.upgrade.strategy`
+  field is not specified, the HelmRelease reconciliation behaves as if
+  `.spec.upgrade.strategy.name` was set to `RemediateOnFailure`.
+- `.retryInterval` (Optional): The time to wait between retries of failed
+  releases when the upgrade strategy is set to `RetryOnFailure`. Defaults
+  to `5m`. Cannot be used with `RemediateOnFailure`.
+
+The default `RemediateOnFailure` strategy applies the rules defined by the
+`.spec.upgrade.remediation` field to the upgrade action, i.e. the same
+behavior of the controller prior to the introduction of the `RetryOnFailure`
+strategy.
+
+The `RetryOnFailure` strategy will retry failed releases in a regular
+interval defined by the `.spec.upgrade.strategy.retryInterval` field,
+without applying any remediation.
+
 #### Upgrade remediation
 
 `.spec.upgrade.remediation` is an optional field to configure the remediation

docs/spec/v2/helmreleases.md

@@ -392,10 +392,12 @@ func (r *HelmReleaseReconciler) reconcileRelease(ctx context.Context, patchHelpe
 		Chart:  loadedChart,
 		Values: values,
 	}); err != nil {
-		if errors.Is(err, intreconcile.ErrMustRequeue) {
+		switch {
+		case errors.Is(err, intreconcile.ErrRetryAfterInterval):
+			return jitter.JitteredRequeueInterval(ctrl.Result{RequeueAfter: obj.GetActiveRetry().GetRetryInterval()}), nil
+		case errors.Is(err, intreconcile.ErrMustRequeue):
 			return ctrl.Result{Requeue: true}, nil
-		}
-		if interrors.IsOneOf(err, intreconcile.ErrExceededMaxRetries, intreconcile.ErrMissingRollbackTarget) {
+		case interrors.IsOneOf(err, intreconcile.ErrExceededMaxRetries, intreconcile.ErrMissingRollbackTarget):
 			err = reconcile.TerminalError(err)
 		}
 		return ctrl.Result{}, err