tektoncd#7617 support k8s native sidecar

Author: kgcarr

config/config-feature-flags.yaml

@@ -138,3 +138,5 @@ data:
   disable-inline-spec: ""
   # Setting this flag to "true" will enable the use of concise resolver syntax
   enable-concise-resolver-syntax: "false"
+  # Setthing this flag to "true" will enable native Kubernetes Sidecar support
+  enable-kubernetes-sidecar: "false"

docs/additional-configs.md

@@ -333,6 +333,8 @@ enables [beta features](#beta-features). When using v1 APIs, setting this field
 allows only stable features, and setting it to "beta" allows only beta features.
 Set this field to "alpha" to allow [alpha features](#alpha-features) to be used.
 
+- `enable-kubernetes-sidecar`: Set this flag to `"true"` to enable native kubernetes sidecar support. This will allow Tekton sidecars to run as Kubernetes sidecars. Must be using Kubernetes v1.29 or greater.
+
 For example:
 
 ```yaml

docs/pipeline-api.md

@@ -4135,6 +4135,23 @@ other Step or Sidecar that does not also request this Workspace will
 not have access to it.</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>restartPolicy</code><br/>
+<em>
+<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#containerrestartpolicy-v1-core">
+Kubernetes core/v1.ContainerRestartPolicy
+</a>
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>RestartPolicy refers to kubernetes RestartPolicy. It can only be set for an
+initContainer and must have it&rsquo;s policy set to &ldquo;Always&rdquo;. It is currently
+left optional to help support Kubernetes versions prior to 1.29 when this feature
+was introduced.</p>
+</td>
+</tr>
 </tbody>
 </table>
 <h3 id="tekton.dev/v1.SidecarState">SidecarState
@@ -13459,6 +13476,23 @@ other Step or Sidecar that does not also request this Workspace will
 not have access to it.</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>restartPolicy</code><br/>
+<em>
+<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#containerrestartpolicy-v1-core">
+Kubernetes core/v1.ContainerRestartPolicy
+</a>
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>RestartPolicy refers to kubernetes RestartPolicy. It can only be set for an
+initContainer and must have it&rsquo;s policy set to &ldquo;Always&rdquo;. It is currently
+left optional to help support Kubernetes versions prior to 1.29 when this feature
+was introduced.</p>
+</td>
+</tr>
 </tbody>
 </table>
 <h3 id="tekton.dev/v1beta1.SidecarState">SidecarState

docs/tasks.md

@@ -1113,6 +1113,11 @@ to run alongside the `Steps` in your `Task`. You can use `Sidecars` to provide a
 `Sidecars` spin up before your `Task` executes and are deleted after the `Task` execution completes.
 For further information, see [`Sidecars` in `TaskRuns`](taskruns.md#specifying-sidecars).
 
+**Note**: Starting in v0.62 you can enable native Kubernetes sidecar support using the `enable-kubernetes-sidecar` feature flag ([see instructions](./additional-configs.md#customizing-the-pipelines-controller-behavior)). If kubernetes does not wait for your sidecar application to be ready, use a `startupProbe` to help kubernetes identify when it is ready.
+
+Refer to the detailed instructions listed in [additional config](additional-configs.md#enabling-larger-results-using-sidecar-logs)
+to learn how to enable this feature.
+
 In the example below, a `Step` uses a Docker-in-Docker `Sidecar` to build a Docker image:
 
 ```yaml

examples/v1/taskruns/sidecar-ready-script.yaml

@@ -9,6 +9,7 @@ spec:
       image: docker.io/library/ubuntu
       script: |
         echo "hello from sidecar" > /shared/message
+        sleep 2
       volumeMounts:
       - name: shared
         mountPath: /shared

examples/v1/taskruns/sidecar-ready.yaml

@@ -18,6 +18,15 @@ spec:
           - -c
           - sleep 5 && touch /shared/ready
         timeoutSeconds: 10
+      # Adding startup probe for k8s native sidecar support
+      # Readiness Probe is not honored for k8s native sidecar support
+      startupProbe:
+        exec:
+          command:
+          - sh
+          - -c
+          - sleep 5 && touch /shared/ready
+        timeoutSeconds: 10
       volumeMounts:
       - name: shared
         mountPath: /shared

go.mod

@@ -1,6 +1,7 @@
 module github.com/tektoncd/pipeline
 
 go 1.22
+
 toolchain go1.22.4
 
 require (

pkg/apis/config/feature_flags.go

@@ -108,6 +108,10 @@ const (
 	EnableParamEnum = "enable-param-enum"
 	// EnableConciseResolverSyntax is the flag to enable concise resolver syntax
 	EnableConciseResolverSyntax = "enable-concise-resolver-syntax"
+	// EnableKubernetesSidecar is the flag to enable kubernetes sidecar support
+	EnableKubernetesSidecar = "enable-kubernetes-sidecar"
+	// DefaultEnableKubernetesSidecar is the default value for EnableKubernetesSidecar
+	DefaultEnableKubernetesSidecar = false
 
 	// DisableInlineSpec is the flag to disable embedded spec
 	// in Taskrun or Pipelinerun
@@ -208,6 +212,7 @@ type FeatureFlags struct {
 	EnableArtifacts             bool
 	DisableInlineSpec           string
 	EnableConciseResolverSyntax bool
+	EnableKubernetesSidecar     bool
 }
 
 // GetFeatureFlagsConfigName returns the name of the configmap containing all
@@ -312,6 +317,9 @@ func NewFeatureFlagsFromMap(cfgMap map[string]string) (*FeatureFlags, error) {
 	if err := setPerFeatureFlag(EnableConciseResolverSyntax, DefaultEnableConciseResolverSyntax, &tc.EnableConciseResolverSyntax); err != nil {
 		return nil, err
 	}
+	if err := setFeature(EnableKubernetesSidecar, DefaultEnableKubernetesSidecar, &tc.EnableKubernetesSidecar); err != nil {
+		return nil, err
+	}
 
 	return &tc, nil
 }

pkg/apis/config/feature_flags_test.go

@@ -83,6 +83,7 @@ func TestNewFeatureFlagsFromConfigMap(t *testing.T) {
 				EnableParamEnum:                  true,
 				DisableInlineSpec:                "pipeline,pipelinerun,taskrun",
 				EnableConciseResolverSyntax:      true,
+				EnableKubernetesSidecar:          true,
 			},
 			fileName: "feature-flags-all-flags-set",
 		},
@@ -314,6 +315,9 @@ func TestNewFeatureFlagsConfigMapErrors(t *testing.T) {
 	}, {
 		fileName: "feature-flags-invalid-enable-concise-resolver-syntax",
 		want:     `failed parsing feature flags config "invalid": strconv.ParseBool: parsing "invalid": invalid syntax for feature enable-concise-resolver-syntax`,
+	}, {
+		fileName: "feature-flags-invalid-enable-kubernetes-sidecar",
+		want:     `failed parsing feature flags config "invalid": strconv.ParseBool: parsing "invalid": invalid syntax`,
 	}} {
 		t.Run(tc.fileName, func(t *testing.T) {
 			cm := test.ConfigMapFromTestFile(t, tc.fileName)

pkg/apis/config/testdata/feature-flags-all-flags-set.yaml

@@ -38,3 +38,4 @@ data:
   enable-artifacts: "true"
   disable-inline-spec: "pipeline,pipelinerun,taskrun"
   enable-concise-resolver-syntax: "true"
+  enable-kubernetes-sidecar: "true"

pkg/apis/config/testdata/feature-flags-invalid-enable-kubernetes-sidecar.yaml

@@ -0,0 +1,21 @@
+# Copyright 2024 The Tekton Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: feature-flags
+  namespace: tekton-pipelines
+data:
+  enable-kubernetes-sidecar: "invalid"

pkg/apis/pipeline/v1/container_types.go

@@ -543,10 +543,43 @@ type Sidecar struct {
 	// +optional
 	// +listType=atomic
 	Workspaces []WorkspaceUsage `json:"workspaces,omitempty"`
+
+	// RestartPolicy refers to kubernetes RestartPolicy. It can only be set for an
+	// initContainer and must have it's policy set to "Always". It is currently
+	// left optional to help support Kubernetes versions prior to 1.29 when this feature
+	// was introduced.
+	// +optional
+	RestartPolicy *corev1.ContainerRestartPolicy `json:"restartPolicy,omitempty"`
 }
 
 // ToK8sContainer converts the Sidecar to a Kubernetes Container struct
 func (s *Sidecar) ToK8sContainer() *corev1.Container {
+	if s.RestartPolicy == nil {
+		return &corev1.Container{
+			Name:                     s.Name,
+			Image:                    s.Image,
+			Command:                  s.Command,
+			Args:                     s.Args,
+			WorkingDir:               s.WorkingDir,
+			Ports:                    s.Ports,
+			EnvFrom:                  s.EnvFrom,
+			Env:                      s.Env,
+			Resources:                s.ComputeResources,
+			VolumeMounts:             s.VolumeMounts,
+			VolumeDevices:            s.VolumeDevices,
+			LivenessProbe:            s.LivenessProbe,
+			ReadinessProbe:           s.ReadinessProbe,
+			StartupProbe:             s.StartupProbe,
+			Lifecycle:                s.Lifecycle,
+			TerminationMessagePath:   s.TerminationMessagePath,
+			TerminationMessagePolicy: s.TerminationMessagePolicy,
+			ImagePullPolicy:          s.ImagePullPolicy,
+			SecurityContext:          s.SecurityContext,
+			Stdin:                    s.Stdin,
+			StdinOnce:                s.StdinOnce,
+			TTY:                      s.TTY,
+		}
+	}
 	return &corev1.Container{
 		Name:                     s.Name,
 		Image:                    s.Image,
@@ -561,6 +594,7 @@ func (s *Sidecar) ToK8sContainer() *corev1.Container {
 		VolumeDevices:            s.VolumeDevices,
 		LivenessProbe:            s.LivenessProbe,
 		ReadinessProbe:           s.ReadinessProbe,
+		RestartPolicy:            s.RestartPolicy,
 		StartupProbe:             s.StartupProbe,
 		Lifecycle:                s.Lifecycle,
 		TerminationMessagePath:   s.TerminationMessagePath,
@@ -597,6 +631,7 @@ func (s *Sidecar) SetContainerFields(c corev1.Container) {
 	s.Stdin = c.Stdin
 	s.StdinOnce = c.StdinOnce
 	s.TTY = c.TTY
+	s.RestartPolicy = c.RestartPolicy
 }
 
 // GetVarSubstitutionExpressions walks all the places a substitution reference can be used

pkg/apis/pipeline/v1/container_types_test.go

@@ -120,3 +120,35 @@ func TestSidecarGetVarSubstitutionExpressions(t *testing.T) {
 		t.Fatalf("Unexpected result (-want, +got): %s", d)
 	}
 }
+
+func TestSidecarRestartPolicyToK8sContainer(t *testing.T) {
+	always := corev1.ContainerRestartPolicyAlways
+	s := Sidecar{
+		Name:          "sidecarName",
+		RestartPolicy: &always,
+	}
+
+	expectedContainer := corev1.Container{
+		Name:          "sidecarName",
+		RestartPolicy: &always,
+	}
+
+	c := s.ToK8sContainer()
+
+	if !(c.RestartPolicy == expectedContainer.RestartPolicy) {
+		t.Fatalf("Unexpected result with RestartPolicy")
+	}
+
+	s = Sidecar{
+		Name: "sidecarName",
+	}
+
+	expectedContainer = corev1.Container{
+		Name: "sidecarName",
+	}
+
+	c = s.ToK8sContainer()
+	if !(c.RestartPolicy == expectedContainer.RestartPolicy) {
+		t.Fatalf("Unexpected result without RestartPolicy")
+	}
+}

pkg/apis/pipeline/v1/openapi_generated.go

@@ -1325,6 +1325,10 @@
           "description": "Periodic probe of Sidecar service readiness. Container will be removed from service endpoints if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes",
           "$ref": "#/definitions/v1.Probe"
         },
+        "restartPolicy": {
+          "description": "RestartPolicy refers to kubernetes RestartPolicy. It can only be set for an initContainer and must have it's policy set to \"Always\". It is currently left optional to help support Kubernetes versions prior to 1.29 when this feature was introduced.",
+          "type": "string"
+        },
         "script": {
           "description": "Script is the contents of an executable file to execute.\n\nIf Script is not empty, the Step cannot have an Command or Args.",
           "type": "string"

pkg/apis/pipeline/v1/swagger.json

@@ -747,10 +747,43 @@ type Sidecar struct {
 	// +optional
 	// +listType=atomic
 	Workspaces []WorkspaceUsage `json:"workspaces,omitempty"`
+
+	// RestartPolicy refers to kubernetes RestartPolicy. It can only be set for an
+	// initContainer and must have it's policy set to "Always". It is currently
+	// left optional to help support Kubernetes versions prior to 1.29 when this feature
+	// was introduced.
+	// +optional
+	RestartPolicy *corev1.ContainerRestartPolicy `json:"restartPolicy,omitempty"`
 }
 
 // ToK8sContainer converts the Sidecar to a Kubernetes Container struct
 func (s *Sidecar) ToK8sContainer() *corev1.Container {
+	if s.RestartPolicy == nil {
+		return &corev1.Container{
+			Name:                     s.Name,
+			Image:                    s.Image,
+			Command:                  s.Command,
+			Args:                     s.Args,
+			WorkingDir:               s.WorkingDir,
+			Ports:                    s.Ports,
+			EnvFrom:                  s.EnvFrom,
+			Env:                      s.Env,
+			Resources:                s.Resources,
+			VolumeMounts:             s.VolumeMounts,
+			VolumeDevices:            s.VolumeDevices,
+			LivenessProbe:            s.LivenessProbe,
+			ReadinessProbe:           s.ReadinessProbe,
+			StartupProbe:             s.StartupProbe,
+			Lifecycle:                s.Lifecycle,
+			TerminationMessagePath:   s.TerminationMessagePath,
+			TerminationMessagePolicy: s.TerminationMessagePolicy,
+			ImagePullPolicy:          s.ImagePullPolicy,
+			SecurityContext:          s.SecurityContext,
+			Stdin:                    s.Stdin,
+			StdinOnce:                s.StdinOnce,
+			TTY:                      s.TTY,
+		}
+	}
 	return &corev1.Container{
 		Name:                     s.Name,
 		Image:                    s.Image,
@@ -765,6 +798,7 @@ func (s *Sidecar) ToK8sContainer() *corev1.Container {
 		VolumeDevices:            s.VolumeDevices,
 		LivenessProbe:            s.LivenessProbe,
 		ReadinessProbe:           s.ReadinessProbe,
+		RestartPolicy:            s.RestartPolicy,
 		StartupProbe:             s.StartupProbe,
 		Lifecycle:                s.Lifecycle,
 		TerminationMessagePath:   s.TerminationMessagePath,
@@ -801,4 +835,5 @@ func (s *Sidecar) SetContainerFields(c corev1.Container) {
 	s.Stdin = c.Stdin
 	s.StdinOnce = c.StdinOnce
 	s.TTY = c.TTY
+	s.RestartPolicy = c.RestartPolicy
 }

pkg/apis/pipeline/v1/zz_generated.deepcopy.go

@@ -1932,6 +1932,10 @@
           "default": {},
           "$ref": "#/definitions/v1.ResourceRequirements"
         },
+        "restartPolicy": {
+          "description": "RestartPolicy refers to kubernetes RestartPolicy. It can only be set for an initContainer and must have it's policy set to \"Always\". It is currently left optional to help support Kubernetes versions prior to 1.29 when this feature was introduced.",
+          "type": "string"
+        },
         "script": {
           "description": "Script is the contents of an executable file to execute.\n\nIf Script is not empty, the Step cannot have an Command or Args.",
           "type": "string"