feat: template background controller (kyverno#6157)

* add make targets for background controller Signed-off-by: ShutingZhao <[email protected]> * template background controller Signed-off-by: ShutingZhao <[email protected]> * fix code-gen manifests Signed-off-by: ShutingZhao <[email protected]> * remove unused code Signed-off-by: ShutingZhao <[email protected]> * trim spaces Signed-off-by: ShutingZhao <[email protected]> * Update Makefile Co-authored-by: Charles-Edouard Brétéché <[email protected]> Signed-off-by: shuting <[email protected]> * Update Makefile Signed-off-by: shuting <[email protected]> * fix Signed-off-by: ShutingZhao <[email protected]> * refactor: move resolvers into engine api package (kyverno#6162) Signed-off-by: Charles-Edouard Brétéché <[email protected]> * add rbac Signed-off-by: ShutingZhao <[email protected]> * add more rbac Signed-off-by: ShutingZhao <[email protected]> * add debug info Signed-off-by: ShutingZhao <[email protected]> * add notes to helm install Signed-off-by: ShutingZhao <[email protected]> * fix permissions Signed-off-by: ShutingZhao <[email protected]> --------- Signed-off-by: ShutingZhao <[email protected]> Signed-off-by: shuting <[email protected]> Signed-off-by: Charles-Edouard Brétéché <[email protected]> Co-authored-by: Charles-Edouard Brétéché <[email protected]> Co-authored-by: kyverno-bot <[email protected]> Co-authored-by: Charles-Edouard Brétéché <[email protected]>
Mohdcode · Jan 31, 2023 · b4c1590 · b4c1590
1 parent 8290112
commit b4c1590
Show file tree

Hide file tree

Showing 20 changed files with 919 additions and 58 deletions.
diff --git a/.github/actions/kyverno-logs/action.yaml b/.github/actions/kyverno-logs/action.yaml
@@ -17,7 +17,9 @@ runs:
       run: |
         kubectl -n kyverno logs deploy/kyverno --all-containers -p || true
         kubectl -n kyverno logs deploy/kyverno-cleanup-controller --all-containers -p || true
+        kubectl -n kyverno logs deploy/kyverno-background-controller --all-containers -p || true
     - shell: bash
       run: |
         kubectl -n kyverno logs deploy/kyverno --all-containers
         kubectl -n kyverno logs deploy/kyverno-cleanup-controller --all-containers
+        kubectl -n kyverno logs deploy/kyverno-background-controller --all-containers
diff --git a/.gitignore b/.gitignore
@@ -9,6 +9,7 @@ cmd/kyverno/kyverno
 cmd/kyverno-init/kyvernopre
 cmd/cleanup-controller/cleanup-controller
 cmd/reports-controller/reports-controller
+cmd/background-controller/background-controller
 /release
 .DS_Store
 .tools

diff --git a/.ko.yaml b/.ko.yaml
@@ -24,7 +24,7 @@ builds:
   ldflags:
   - '{{ if index .Env "LD_FLAGS" }}{{ .Env.LD_FLAGS }}{{ end }}'
 
-- id: updaterequest-controller
-  main: ./cmd/updaterequest-controller
+- id: background-controller
+  main: ./cmd/background-controller
   ldflags:
   - '{{ if index .Env "LD_FLAGS" }}{{ .Env.LD_FLAGS }}{{ end }}'
diff --git a/Makefile b/Makefile
@@ -27,11 +27,13 @@ KYVERNO_IMAGE        := kyverno
 CLI_IMAGE            := kyverno-cli
 CLEANUP_IMAGE        := cleanup-controller
 REPORTS_IMAGE        := reports-controller
+BACKGROUND_IMAGE     := background-controller
 REPO_KYVERNOPRE      := $(REGISTRY)/$(REPO)/$(KYVERNOPRE_IMAGE)
 REPO_KYVERNO         := $(REGISTRY)/$(REPO)/$(KYVERNO_IMAGE)
 REPO_CLI             := $(REGISTRY)/$(REPO)/$(CLI_IMAGE)
 REPO_CLEANUP         := $(REGISTRY)/$(REPO)/$(CLEANUP_IMAGE)
 REPO_REPORTS         := $(REGISTRY)/$(REPO)/$(REPORTS_IMAGE)
+REPO_BACKGROUND      := $(REGISTRY)/$(REPO)/$(BACKGROUND_IMAGE)
 USE_CONFIG           ?= standard
 
 #########
@@ -139,11 +141,13 @@ KYVERNOPRE_DIR := $(CMD_DIR)/kyverno-init
 CLI_DIR        := $(CMD_DIR)/cli/kubectl-kyverno
 CLEANUP_DIR    := $(CMD_DIR)/cleanup-controller
 REPORTS_DIR    := $(CMD_DIR)/reports-controller
+BACKGROUND_DIR := $(CMD_DIR)/background-controller
 KYVERNO_BIN    := $(KYVERNO_DIR)/kyverno
 KYVERNOPRE_BIN := $(KYVERNOPRE_DIR)/kyvernopre
 CLI_BIN        := $(CLI_DIR)/kubectl-kyverno
 CLEANUP_BIN    := $(CLEANUP_DIR)/cleanup-controller
 REPORTS_BIN    := $(REPORTS_DIR)/reports-controller
+BACKGROUND_BIN := $(BACKGROUND_DIR)/background-controller
 PACKAGE        ?= github.com/kyverno/kyverno
 CGO_ENABLED    ?= 0
 LD_FLAGS        = "-s -w -X $(PACKAGE)/pkg/version.BuildVersion=$(GIT_VERSION) -X $(PACKAGE)/pkg/version.BuildHash=$(GIT_HASH) -X $(PACKAGE)/pkg/version.BuildTime=$(TIMESTAMP)"
@@ -207,6 +211,10 @@ $(REPORTS_BIN): fmt vet
 	@echo Build reports controller binary... >&2
 	@CGO_ENABLED=$(CGO_ENABLED) GOOS=$(GOOS) go build -o ./$(REPORTS_BIN) -ldflags=$(LD_FLAGS) ./$(REPORTS_DIR)
 
+$(REPORTS_BIN): fmt vet
+	@echo Build background controller binary... >&2
+	@CGO_ENABLED=$(CGO_ENABLED) GOOS=$(GOOS) go build -o $(BACKGROUND_BIN) -ldflags=$(LD_FLAGS) $(BACKGROUND_DIR)
+
 .PHONY: build-kyverno-init
 build-kyverno-init: $(KYVERNOPRE_BIN) ## Build kyvernopre binary
 
@@ -222,7 +230,10 @@ build-cleanup-controller: $(CLEANUP_BIN) ## Build cleanup controller binary
 .PHONY: build-reports-controller
 build-reports-controller: $(REPORTS_BIN) ## Build reports controller binary
 
-build-all: build-kyverno-init build-kyverno build-cli build-cleanup-controller build-reports-controller ## Build all binaries
+.PHONY: build-background-controller
+build-background-controller: $(BACKGROUND_BIN) ## Build background controller binary
+
+build-all: build-kyverno-init build-kyverno build-cli build-cleanup-controller build-reports-controller build-background-controller ## Build all binaries
 
 ##############
 # BUILD (KO) #
@@ -263,14 +274,24 @@ ko-build-reports-controller: $(KO) ## Build reports controller local image (with
 	@echo Build reports controller local image with ko... >&2
 	@LD_FLAGS=$(LD_FLAGS_DEV) KOCACHE=$(KOCACHE) KO_DOCKER_REPO=$(KO_REGISTRY) $(KO) build ./$(REPORTS_DIR) --preserve-import-paths --tags=$(KO_TAGS_DEV) --platform=$(LOCAL_PLATFORM)
 
+.PHONY: ko-build-background-controller
+ko-build-background-controller: $(KO) ## Build background controller local image (with ko)
+	@echo Build background controller local image with ko... >&2
+	@LD_FLAGS=$(LD_FLAGS_DEV) KOCACHE=$(KOCACHE) KO_DOCKER_REPO=ko.local $(KO) build ./$(BACKGROUND_DIR) --preserve-import-paths --tags=$(KO_TAGS_DEV) --platform=$(LOCAL_PLATFORM)
+
 .PHONY: ko-build-all
-ko-build-all: ko-build-kyverno-init ko-build-kyverno ko-build-cli ko-build-cleanup-controller ko-build-reports-controller ## Build all local images (with ko)
+ko-build-all: ko-build-kyverno-init ko-build-kyverno ko-build-cli ko-build-cleanup-controller ko-build-reports-controller ko-build-background-controller ## Build all local images (with ko)
 
 ################
 # PUBLISH (KO) #
 ################
 
 REGISTRY_USERNAME   ?= dummy
+KO_KYVERNOPRE_IMAGE := ko.local/github.com/kyverno/kyverno/cmd/kyverno-init
+KO_KYVERNO_IMAGE    := ko.local/github.com/kyverno/kyverno/cmd/kyverno
+KO_CLEANUP_IMAGE    := ko.local/github.com/kyverno/kyverno/cmd/cleanup-controller
+KO_REPORTS_IMAGE    := ko.local/github.com/kyverno/kyverno/cmd/reports-controller
+KO_BACKGROUND_IMAGE := ko.local/github.com/kyverno/kyverno/cmd/background-controller
 PLATFORMS           := linux/amd64,linux/arm64,linux/s390x
 
 .PHONY: ko-login
@@ -297,6 +318,10 @@ ko-publish-cleanup-controller: ko-login ## Build and publish cleanup controller
 ko-publish-reports-controller: ko-login ## Build and publish reports controller image (with ko)
 	@LD_FLAGS=$(LD_FLAGS) KOCACHE=$(KOCACHE) KO_DOCKER_REPO=$(REPO_REPORTS) $(KO) build $(REPORTS_DIR) --bare --tags=$(KO_TAGS) --platform=$(PLATFORMS)
 
+.PHONY: ko-publish-backgound-controller
+ko-publish-backgound-controller: ko-login ## Build and publish background controller image (with ko)
+	@LD_FLAGS=$(LD_FLAGS) KOCACHE=$(KOCACHE) KO_DOCKER_REPO=$(REPO_REPORTS) $(KO) build $(BACKGROUND_DIR) --bare --tags=$(KO_TAGS) --platform=$(PLATFORMS)
+
 .PHONY: ko-publish-kyverno-init-dev
 ko-publish-kyverno-init-dev: ko-login ## Build and publish kyvernopre dev image (with ko)
 	@LD_FLAGS=$(LD_FLAGS_DEV) KOCACHE=$(KOCACHE) KO_DOCKER_REPO=$(REPO_KYVERNOPRE) $(KO) build $(KYVERNOPRE_DIR) --bare --tags=$(KO_TAGS_DEV) --platform=$(PLATFORMS)
@@ -317,11 +342,16 @@ ko-publish-cleanup-controller-dev: ko-login ## Build and publish cleanup control
 ko-publish-reports-controller-dev: ko-login ## Build and publish reports controller dev image (with ko)
 	@LD_FLAGS=$(LD_FLAGS_DEV) KOCACHE=$(KOCACHE) KO_DOCKER_REPO=$(REPO_REPORTS) $(KO) build $(REPORTS_DIR) --bare --tags=$(KO_TAGS_DEV) --platform=$(PLATFORMS)
 
+.PHONY: ko-publish-background-controller-dev
+ko-publish-background-controller-dev: ko-login ## Build and publish background controller dev image (with ko)
+	@LD_FLAGS=$(LD_FLAGS_DEV) KOCACHE=$(KOCACHE) KO_DOCKER_REPO=$(REPO_REPORTS) $(KO) build $(BACKGROUND_DIR) --bare --tags=$(KO_TAGS_DEV) --platform=$(PLATFORMS)
+
+
 .PHONY: ko-publish-all
-ko-publish-all: ko-publish-kyverno-init ko-publish-kyverno ko-publish-cli ko-publish-cleanup-controller ko-publish-reports-controller ## Build and publish all images (with ko)
+ko-publish-all: ko-publish-kyverno-init ko-publish-kyverno ko-publish-cli ko-publish-cleanup-controller ko-publish-reports-controller ko-publish-backgound-controller ## Build and publish all images (with ko)
 
 .PHONY: ko-publish-all-dev
-ko-publish-all-dev: ko-publish-kyverno-init-dev ko-publish-kyverno-dev ko-publish-cli-dev ko-publish-cleanup-controller-dev ko-publish-reports-controller-dev ## Build and publish all dev images (with ko)
+ko-publish-all-dev: ko-publish-kyverno-init-dev ko-publish-kyverno-dev ko-publish-cli-dev ko-publish-cleanup-controller-dev ko-publish-reports-controller-dev ko-publish-background-controller-dev ## Build and publish all dev images (with ko)
 
 #################
 # BUILD (IMAGE) #
@@ -333,6 +363,7 @@ LOCAL_KYVERNOPRE_REPO  := $($(shell echo $(BUILD_WITH) | tr '[:lower:]' '[:upper
 LOCAL_KYVERNO_REPO     := $($(shell echo $(BUILD_WITH) | tr '[:lower:]' '[:upper:]')_KYVERNO_REPO)
 LOCAL_CLEANUP_REPO     := $($(shell echo $(BUILD_WITH) | tr '[:lower:]' '[:upper:]')_CLEANUP_REPO)
 LOCAL_REPORTS_REPO     := $($(shell echo $(BUILD_WITH) | tr '[:lower:]' '[:upper:]')_REPORTS_REPO)
+LOCAL_BACKGROUND_IMAGE := $($(shell echo $(BUILD_WITH) | tr '[:lower:]' '[:upper:]')_BACKGROUND_IMAGE)
 
 .PHONY: image-build-kyverno-init
 image-build-kyverno-init: $(BUILD_WITH)-build-kyverno-init
@@ -349,6 +380,9 @@ image-build-cleanup-controller: $(BUILD_WITH)-build-cleanup-controller
 .PHONY: image-build-reports-controller
 image-build-reports-controller: $(BUILD_WITH)-build-reports-controller
 
+.PHONY: image-build-background-controller
+image-build-background-controller: $(BUILD_WITH)-build-background-controller
+
 .PHONY: image-build-all
 image-build-all: $(BUILD_WITH)-build-all
 
@@ -467,6 +501,7 @@ codegen-manifest-install: $(HELM) ## Create install manifest
 		--set initImage.tag=latest \
 		--set cleanupController.image.tag=latest \
 		--set reportsController.image.tag=latest \
+		--set backgroundController.image.tag=latest \
  		| $(SED) -e '/^#.*/d' \
 		> ./config/install.yaml
 
@@ -734,8 +769,13 @@ kind-load-reports-controller: $(KIND) image-build-reports-controller ## Build re
 	@echo Load reports controller image... >&2
 	@$(KIND) load docker-image --name $(KIND_NAME) $(LOCAL_REGISTRY)/$(LOCAL_REPORTS_REPO):$(IMAGE_TAG_DEV)
 
+.PHONY: kind-load-background-controller
+kind-load-background-controller: $(KIND) image-build-background-controller ## Build background controller image and load it in kind cluster
+	@echo Load background controller image... >&2
+	@$(KIND) load docker-image --name $(KIND_NAME) $(LOCAL_BACKGROUND_IMAGE):$(IMAGE_TAG_DEV)
+
 .PHONY: kind-load-all
-kind-load-all: kind-load-kyverno-init kind-load-kyverno kind-load-cleanup-controller kind-load-reports-controller ## Build images and load them in kind cluster
+kind-load-all: kind-load-kyverno-init kind-load-kyverno kind-load-cleanup-controller kind-load-reports-controller kind-load-background-controller ## Build images and load them in kind cluster
 
 .PHONY: kind-deploy-kyverno
 kind-deploy-kyverno: $(HELM) kind-load-all ## Build images, load them in kind cluster and deploy kyverno helm chart
@@ -753,6 +793,8 @@ kind-deploy-kyverno: $(HELM) kind-load-all ## Build images, load them in kind cl
 		--set reportsController.image.registry=$(LOCAL_REGISTRY) \
 		--set reportsController.image.repository=$(LOCAL_REPORTS_REPO) \
 		--set reportsController.image.tag=$(IMAGE_TAG_DEV) \
+		--set backgroundController.image.repository=$(LOCAL_BACKGROUND_IMAGE) \
+		--set backgroundController.image.tag=$(IMAGE_TAG_DEV) \
 		--values ./scripts/config/$(USE_CONFIG)/kyverno.yaml
 
 .PHONY: kind-deploy-kyverno-policies

diff --git a/charts/kyverno/README.md b/charts/kyverno/README.md
@@ -354,6 +354,56 @@ The command removes all the Kubernetes components associated with the chart and
 | reportsController.metering.port | int | `8000` | Prometheus endpoint port |
 | reportsController.metering.collector | string | `""` | Otel collector endpoint |
 | reportsController.metering.creds | string | `""` | Otel collector credentials |
+| backgroundController.enabled | bool | `true` | Enable background controller. |
+| backgroundController.rbac.create | bool | `true` | Create RBAC resources |
+| backgroundController.rbac.serviceAccount.name | string | `nil` | Service account name |
+| backgroundController.rbac.clusterRole.extraResources | list | `[]` | Extra resource permissions to add in the cluster role |
+| backgroundController.image.registry | string | `nil` | Image registry |
+| backgroundController.image.repository | string | `"ghcr.io/kyverno/background-controller"` | Image repository |
+| backgroundController.image.tag | string | `nil` | Image tag Defaults to appVersion in Chart.yaml if omitted |
+| backgroundController.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy |
+| backgroundController.image.pullSecrets | list | `[]` | Image pull secrets |
+| backgroundController.replicas | int | `nil` | Desired number of pods |
+| backgroundController.updateStrategy | object | See [values.yaml](values.yaml) | Deployment update strategy. Ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy |
+| backgroundController.priorityClassName | string | `""` | Optional priority class |
+| backgroundController.hostNetwork | bool | `false` | Change `hostNetwork` to `true` when you want the pod to share its host's network namespace. Useful for situations like when you end up dealing with a custom CNI over Amazon EKS. Update the `dnsPolicy` accordingly as well to suit the host network mode. |
+| backgroundController.dnsPolicy | string | `"ClusterFirst"` | `dnsPolicy` determines the manner in which DNS resolution happens in the cluster. In case of `hostNetwork: true`, usually, the `dnsPolicy` is suitable to be `ClusterFirstWithHostNet`. For further reference: https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy. |
+| backgroundController.extraArgs | list | `[]` | Extra arguments passed to the container on the command line |
+| backgroundController.resources.limits | object | `{"memory":"128Mi"}` | Pod resource limits |
+| backgroundController.resources.requests | object | `{"cpu":"100m","memory":"64Mi"}` | Pod resource requests |
+| backgroundController.nodeSelector | object | `{}` | Node labels for pod assignment |
+| backgroundController.tolerations | list | `[]` | List of node taints to tolerate |
+| backgroundController.antiAffinity.enabled | bool | `true` | Pod antiAffinities toggle. Enabled by default but can be disabled if you want to schedule pods to the same node. |
+| backgroundController.podAntiAffinity | object | See [values.yaml](values.yaml) | Pod anti affinity constraints. |
+| backgroundController.podAffinity | object | `{}` | Pod affinity constraints. |
+| backgroundController.nodeAffinity | object | `{}` | Node affinity constraints. |
+| backgroundController.topologySpreadConstraints | list | `[]` | Topology spread constraints. |
+| backgroundController.podSecurityContext | object | `{}` | Security context for the pod |
+| backgroundController.securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"privileged":false,"readOnlyRootFilesystem":true,"runAsNonRoot":true,"seccompProfile":{"type":"RuntimeDefault"}}` | Security context for the containers |
+| backgroundController.podDisruptionBudget.minAvailable | int | `1` | Configures the minimum available pods for disruptions. Cannot be used if `maxUnavailable` is set. |
+| backgroundController.podDisruptionBudget.maxUnavailable | string | `nil` | Configures the maximum unavailable pods for disruptions. Cannot be used if `minAvailable` is set. |
+| backgroundController.metricsService.create | bool | `true` | Create service. |
+| backgroundController.metricsService.port | int | `8000` | Service port. Metrics server will be exposed at this port. |
+| backgroundController.metricsService.type | string | `"ClusterIP"` | Service type. |
+| backgroundController.metricsService.nodePort | string | `nil` | Service node port. Only used if `metricsService.type` is `NodePort`. |
+| backgroundController.metricsService.annotations | object | `{}` | Service annotations. |
+| backgroundController.serviceMonitor.enabled | bool | `false` | Create a `ServiceMonitor` to collect Prometheus metrics. |
+| backgroundController.serviceMonitor.additionalLabels | string | `nil` | Additional labels |
+| backgroundController.serviceMonitor.namespace | string | `nil` | Override namespace (default is the same as kyverno) |
+| backgroundController.serviceMonitor.interval | string | `"30s"` | Interval to scrape metrics |
+| backgroundController.serviceMonitor.scrapeTimeout | string | `"25s"` | Timeout if metrics can't be retrieved in given time interval |
+| backgroundController.serviceMonitor.secure | bool | `false` | Is TLS required for endpoint |
+| backgroundController.serviceMonitor.tlsConfig | object | `{}` | TLS Configuration for endpoint |
+| backgroundController.tracing.enabled | bool | `false` | Enable tracing |
+| backgroundController.tracing.address | string | `nil` | Traces receiver address |
+| backgroundController.tracing.port | string | `nil` | Traces receiver port |
+| backgroundController.tracing.creds | string | `""` | Traces receiver credentials |
+| backgroundController.logging.format | string | `"text"` | Logging format |
+| backgroundController.metering.disabled | bool | `false` | Disable metrics export |
+| backgroundController.metering.config | string | `"prometheus"` | Otel configuration, can be `prometheus` or `grpc` |
+| backgroundController.metering.port | int | `8000` | Prometheus endpoint port |
+| backgroundController.metering.collector | string | `""` | Otel collector endpoint |
+| backgroundController.metering.creds | string | `""` | Otel collector credentials |
 
 ## TLS Configuration
 

diff --git a/charts/kyverno/templates/NOTES.txt b/charts/kyverno/templates/NOTES.txt
@@ -14,6 +14,9 @@ The following components have been installed in your cluster:
 {{- if .Values.cleanupController.enabled }}
 - Cleanup controller
 {{- end }}
+{{- if .Values.backgroundController.enabled }}
+- Background controller
+{{- end }}
 {{- if .Values.grafana.enabled }}
 - Grafana dashboard
 {{- end }}

diff --git a/charts/kyverno/templates/background-controller/_helpers.tpl b/charts/kyverno/templates/background-controller/_helpers.tpl
@@ -0,0 +1,39 @@
+{{/* vim: set filetype=mustache: */}}
+
+{{- define "kyverno.background-controller.name" -}}
+{{ template "kyverno.name" . }}-background-controller
+{{- end -}}
+
+{{- define "kyverno.background-controller.labels" -}}
+{{- template "kyverno.labels.merge" (list
+  (include "kyverno.labels.common" .)
+  (include "kyverno.background-controller.matchLabels" .)
+) -}}
+{{- end -}}
+
+{{- define "kyverno.background-controller.matchLabels" -}}
+{{- template "kyverno.labels.merge" (list
+  (include "kyverno.matchLabels.common" .)
+  (include "kyverno.labels.component" "background-controller")
+) -}}
+{{- end -}}
+
+{{- define "kyverno.background-controller.image" -}}
+{{- if .image.registry -}}
+  {{ .image.registry }}/{{ required "An image repository is required" .image.repository }}:{{ default .defaultTag .image.tag }}
+{{- else -}}
+  {{ required "An image repository is required" .image.repository }}:{{ default .defaultTag .image.tag }}
+{{- end -}}
+{{- end -}}
+
+{{- define "kyverno.background-controller.roleName" -}}
+{{ .Release.Name }}:background-controller
+{{- end -}}
+
+{{- define "kyverno.background-controller.serviceAccountName" -}}
+{{- if .Values.backgroundController.rbac.create -}}
+    {{ default (include "kyverno.background-controller.name" .) .Values.backgroundController.rbac.serviceAccount.name }}
+{{- else -}}
+    {{ required "A service account name is required when `rbac.create` is set to `false`" .Values.backgroundController.rbac.serviceAccount.name }}
+{{- end -}}
+{{- end -}}