CP/DP Split: remove unneeded provisioner mode (#3180)

With the new deployment model, the provisioner mode for conformance tests is no longer needed. This code is removed, and at a later date the conformance tests will be updated to work with the new model. Renamed the "static-mode" to "controller".

Also removed some unneeded metrics collection.

Author: sjberman

.github/workflows/conformance.yml

@@ -76,13 +76,6 @@ jobs:
             type=ref,event=pr
             type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }}
 
-      - name: Generate static deployment
-        run: |
-          ngf_prefix=ghcr.io/nginx/nginx-gateway-fabric
-          ngf_tag=${{ steps.ngf-meta.outputs.version }}
-          make generate-static-deployment PLUS_ENABLED=${{ inputs.image == 'plus' && 'true' || 'false' }} PREFIX=${ngf_prefix} TAG=${ngf_tag}
-        working-directory: ./tests
-
       - name: Build binary
         uses: goreleaser/goreleaser-action@90a3faa9d0182683851fbfa97ca1a2cb983bfca3 # v6.2.1
         with:
@@ -151,7 +144,6 @@ jobs:
           ngf_tag=${{ steps.ngf-meta.outputs.version }}
           if [ ${{ github.event_name }} == "schedule" ]; then export GW_API_VERSION=main; fi
           make helm-install-local${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag}
-          make deploy-updated-provisioner PREFIX=${ngf_prefix} TAG=${ngf_tag}
         working-directory: ./tests
 
       - name: Run conformance tests

charts/nginx-gateway-fabric/templates/deployment.yaml

@@ -31,7 +31,7 @@ spec:
     spec:
       containers:
       - args:
-        - static-mode
+        - controller
         - --gateway-ctlr-name={{ .Values.nginxGateway.gatewayControllerName }}
         - --gatewayclass={{ .Values.nginxGateway.gatewayClassName }}
         - --config={{ include "nginx-gateway.config-name" . }}

cmd/gateway/commands.go

@@ -21,7 +21,6 @@ import (
 	ctlrZap "sigs.k8s.io/controller-runtime/pkg/log/zap"
 
 	"github.com/nginx/nginx-gateway-fabric/internal/framework/file"
-	"github.com/nginx/nginx-gateway-fabric/internal/mode/provisioner"
 	"github.com/nginx/nginx-gateway-fabric/internal/mode/static"
 	"github.com/nginx/nginx-gateway-fabric/internal/mode/static/config"
 	"github.com/nginx/nginx-gateway-fabric/internal/mode/static/licensing"
@@ -53,7 +52,7 @@ func createRootCommand() *cobra.Command {
 	return rootCmd
 }
 
-func createStaticModeCommand() *cobra.Command {
+func createControllerCommand() *cobra.Command {
 	// flag names
 	const (
 		gatewayFlag                    = "gateway"
@@ -145,8 +144,8 @@ func createStaticModeCommand() *cobra.Command {
 	)
 
 	cmd := &cobra.Command{
-		Use:   "static-mode",
-		Short: "Configure NGINX in the scope of a single Gateway resource",
+		Use:   "controller",
+		Short: "Run the NGINX Gateway Fabric control plane",
 		RunE: func(cmd *cobra.Command, _ []string) error {
 			atom := zap.NewAtomicLevel()
 
@@ -155,7 +154,7 @@ func createStaticModeCommand() *cobra.Command {
 
 			commit, date, dirty := getBuildInfo()
 			logger.Info(
-				"Starting NGINX Gateway Fabric in static mode",
+				"Starting the NGINX Gateway Fabric control plane",
 				"version", version,
 				"commit", commit,
 				"date", date,
@@ -443,56 +442,6 @@ func createStaticModeCommand() *cobra.Command {
 	return cmd
 }
 
-func createProvisionerModeCommand() *cobra.Command {
-	var (
-		gatewayCtlrName = stringValidatingValue{
-			validator: validateGatewayControllerName,
-		}
-		gatewayClassName = stringValidatingValue{
-			validator: validateResourceName,
-		}
-	)
-
-	cmd := &cobra.Command{
-		Use:    "provisioner-mode",
-		Short:  "Provision a static-mode NGINX Gateway Fabric Deployment per Gateway resource",
-		Hidden: true,
-		RunE: func(_ *cobra.Command, _ []string) error {
-			logger := ctlrZap.New()
-			commit, date, dirty := getBuildInfo()
-			logger.Info(
-				"Starting NGINX Gateway Fabric Provisioner",
-				"version", version,
-				"commit", commit,
-				"date", date,
-				"dirty", dirty,
-			)
-
-			return provisioner.StartManager(provisioner.Config{
-				Logger:           logger,
-				GatewayClassName: gatewayClassName.value,
-				GatewayCtlrName:  gatewayCtlrName.value,
-			})
-		},
-	}
-
-	cmd.Flags().Var(
-		&gatewayCtlrName,
-		gatewayCtlrNameFlag,
-		fmt.Sprintf(gatewayCtlrNameUsageFmt, domain),
-	)
-	utilruntime.Must(cmd.MarkFlagRequired(gatewayCtlrNameFlag))
-
-	cmd.Flags().Var(
-		&gatewayClassName,
-		gatewayClassFlag,
-		gatewayClassNameUsage,
-	)
-	utilruntime.Must(cmd.MarkFlagRequired(gatewayClassFlag))
-
-	return cmd
-}
-
 // FIXME(pleshakov): Remove this command once NGF min supported Kubernetes version supports sleep action in
 // preStop hook.
 // See https://github.com/kubernetes/enhancements/tree/4ec371d92dcd4f56a2ab18c8ba20bb85d8d20efe/keps/sig-node/3960-pod-lifecycle-sleep-action

cmd/gateway/commands_test.go

@@ -122,13 +122,9 @@ func TestCommonFlagsValidation(t *testing.T) {
 	}
 
 	for _, test := range tests {
-		t.Run(test.name+"_static_mode", func(t *testing.T) {
+		t.Run(test.name+"_controller", func(t *testing.T) {
 			t.Parallel()
-			testFlag(t, createStaticModeCommand(), test)
-		})
-		t.Run(test.name+"_provisioner_mode", func(t *testing.T) {
-			t.Parallel()
-			testFlag(t, createProvisionerModeCommand(), test)
+			testFlag(t, createControllerCommand(), test)
 		})
 	}
 }
@@ -439,28 +435,12 @@ func TestStaticModeCmdFlagValidation(t *testing.T) {
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			t.Parallel()
-			cmd := createStaticModeCommand()
+			cmd := createControllerCommand()
 			testFlag(t, cmd, test)
 		})
 	}
 }
 
-func TestProvisionerModeCmdFlagValidation(t *testing.T) {
-	t.Parallel()
-	testCase := flagTestCase{
-		name: "valid flags",
-		args: []string{
-			"--gateway-ctlr-name=gateway.nginx.org/nginx-gateway", // common and required flag
-			"--gatewayclass=nginx",                                // common and required flag
-		},
-		wantErr: false,
-	}
-
-	// common flags validation is tested separately
-
-	testFlag(t, createProvisionerModeCommand(), testCase)
-}
-
 func TestSleepCmdFlagValidation(t *testing.T) {
 	t.Parallel()
 	tests := []flagTestCase{

cmd/gateway/main.go

@@ -21,8 +21,7 @@ func main() {
 	rootCmd := createRootCommand()
 
 	rootCmd.AddCommand(
-		createStaticModeCommand(),
-		createProvisionerModeCommand(),
+		createControllerCommand(),
 		createInitializeCommand(),
 		createSleepCommand(),
 	)

config/tests/static-deployment.yaml

@@ -200,7 +200,7 @@ spec:
     spec:
       containers:
       - args:
-        - static-mode
+        - controller
         - --gateway-ctlr-name=gateway.nginx.org/nginx-gateway-controller
         - --gatewayclass=nginx
         - --config=nginx-gateway-config

deploy/aws-nlb/deploy.yaml

@@ -200,7 +200,7 @@ spec:
     spec:
       containers:
       - args:
-        - static-mode
+        - controller
         - --gateway-ctlr-name=gateway.nginx.org/nginx-gateway-controller
         - --gatewayclass=nginx
         - --config=nginx-gateway-config

deploy/azure/deploy.yaml

@@ -200,7 +200,7 @@ spec:
     spec:
       containers:
       - args:
-        - static-mode
+        - controller
         - --gateway-ctlr-name=gateway.nginx.org/nginx-gateway-controller
         - --gatewayclass=nginx
         - --config=nginx-gateway-config

deploy/default/deploy.yaml

@@ -204,7 +204,7 @@ spec:
     spec:
       containers:
       - args:
-        - static-mode
+        - controller
         - --gateway-ctlr-name=gateway.nginx.org/nginx-gateway-controller
         - --gatewayclass=nginx
         - --config=nginx-gateway-config

deploy/experimental-nginx-plus/deploy.yaml

@@ -204,7 +204,7 @@ spec:
     spec:
       containers:
       - args:
-        - static-mode
+        - controller
         - --gateway-ctlr-name=gateway.nginx.org/nginx-gateway-controller
         - --gatewayclass=nginx
         - --config=nginx-gateway-config

deploy/experimental/deploy.yaml

@@ -200,7 +200,7 @@ spec:
     spec:
       containers:
       - args:
-        - static-mode
+        - controller
         - --gateway-ctlr-name=gateway.nginx.org/nginx-gateway-controller
         - --gatewayclass=nginx
         - --config=nginx-gateway-config

deploy/nginx-plus/deploy.yaml

@@ -200,7 +200,7 @@ spec:
     spec:
       containers:
       - args:
-        - static-mode
+        - controller
         - --gateway-ctlr-name=gateway.nginx.org/nginx-gateway-controller
         - --gatewayclass=nginx
         - --config=nginx-gateway-config

deploy/nodeport/deploy.yaml

@@ -210,7 +210,7 @@ spec:
     spec:
       containers:
       - args:
-        - static-mode
+        - controller
         - --gateway-ctlr-name=gateway.nginx.org/nginx-gateway-controller
         - --gatewayclass=nginx
         - --config=nginx-gateway-config

deploy/openshift/deploy.yaml

@@ -202,7 +202,7 @@ spec:
     spec:
       containers:
       - args:
-        - static-mode
+        - controller
         - --gateway-ctlr-name=gateway.nginx.org/nginx-gateway-controller
         - --gatewayclass=nginx
         - --config=nginx-gateway-config

deploy/snippets-filters-nginx-plus/deploy.yaml

@@ -202,7 +202,7 @@ spec:
     spec:
       containers:
       - args:
-        - static-mode
+        - controller
         - --gateway-ctlr-name=gateway.nginx.org/nginx-gateway-controller
         - --gatewayclass=nginx
         - --config=nginx-gateway-config

deploy/snippets-filters/deploy.yaml

@@ -44,9 +44,8 @@ To create a new release, follow these steps:
    1. Kick off the [longevity tests](https://github.com/nginx/nginx-gateway-fabric/blob/main/tests/README.md#longevity-testing) for both OSS and Plus. You'll need to create two clusters and VMs for this. Before running, update your `vars.env` file with the proper image tag and prefixes. NGF and nginx images will be available from `ghcr.io`, and nginx plus will be available in GCP (`us-docker.pkg.dev/<GCP_PROJECT_ID>/nginx-gateway-fabric/nginx-plus`). These tests need to run for 4 days before releasing. The results should be committed to the main branch and then cherry-picked to the release branch.
    2. Kick off the [NFR workflow](https://github.com/nginx/nginx-gateway-fabric/actions/workflows/nfr.yml) in the browser. For `image_tag`, use `release-X.X-rc`, and for `version`, use the upcoming `X.Y.Z` NGF version. Run the workflow on the new release branch. This will run all of the NFR tests which are automated and open a PR with the results files when it is complete. Review this PR and make any necessary changes before merging. Once merged, be sure to cherry-pick the commit to the main branch as well (the original PR targets the release branch).
 5. Run the [Release PR](https://github.com/nginx/nginx-gateway-fabric/actions/workflows/release-pr.yml) workflow to update the repo files for the release. Then there are a few manual steps to complete:
-   1. Update the version tag used in the [provisioner manifest](/tests/conformance/provisioner/provisioner.yaml) and [getting started guide](/site/content/get-started.md).
-   2. Update the [README](/README.md) to include information about the release.
-   3. Update the [changelog](/CHANGELOG.md). There is going to be a new blank section generated by the automation that needs to be adjusted accordingly.
+   1. Update the [README](/README.md) to include information about the release.
+   2. Update the [changelog](/CHANGELOG.md). There is going to be a new blank section generated by the automation that needs to be adjusted accordingly.
       - At the top there will be a list of all PRs that are labeled with `release-notes`.
       The changelog includes only important (from the user perspective)
       changes to NGF. This is in contrast with the autogenerated full changelog, which is created in the next

docs/developer/release-process.md

@@ -155,7 +155,7 @@ spec:
       name: my-annotation
 ```
 
-Infrastructure labels and annotations should be applied to all resources created in response to the Gateway. This only applies to _automated deployments_ (i.e., provisioner mode), implementations that automatically deploy the data plane based on a Gateway.
+Infrastructure labels and annotations should be applied to all resources created in response to the Gateway.
 Other use cases for this API are Service type, Service IP, CPU memory requests, affinity rules, and Gateway routability (public, private, and cluster).
 
 ### TLS Options

docs/proposals/nginx-extensions.md

@@ -150,18 +150,13 @@ func StartManager(cfg config.Config) error {
 	if cfg.MetricsConfig.Enabled {
 		constLabels := map[string]string{"class": cfg.GatewayClassName}
 
-		ngxruntimeCollector := collectors.NewManagerMetricsCollector(constLabels)
 		handlerCollector = collectors.NewControllerCollector(constLabels)
-
 		handlerCollector, ok := handlerCollector.(prometheus.Collector)
 		if !ok {
 			return fmt.Errorf("handlerCollector is not a prometheus.Collector: %w", frameworkStatus.ErrFailedAssert)
 		}
 
-		metrics.Registry.MustRegister(
-			ngxruntimeCollector,
-			handlerCollector,
-		)
+		metrics.Registry.MustRegister(handlerCollector)
 	}
 
 	statusUpdater := frameworkStatus.NewUpdater(

embedded.go

@@ -30,10 +30,3 @@ done
 
 # For OpenShift, we don't need a Helm example so we generate the manifests from the default values.yaml
 generate_manifests openshift
-
-# FIXME(lucacome): Implement a better way to generate the static deployment file
-# https://github.com/nginx/nginx-gateway-fabric/issues/2326
-helm template nginx-gateway charts/nginx-gateway-fabric --set nameOverride=nginx-gateway --set nginxGateway.metrics.enable=false --set nginxGateway.productTelemetry.enable=false -n nginx-gateway -s templates/deployment.yaml >config/tests/static-deployment.yaml
-sed -i.bak '/app.kubernetes.io\/managed-by: Helm/d' config/tests/static-deployment.yaml
-sed -i.bak '/helm.sh/d' config/tests/static-deployment.yaml
-rm -f config/tests/static-deployment.yaml.bak

internal/mode/provisioner/deployment.go

@@ -13,7 +13,6 @@ GW_SVC_GKE_INTERNAL = false
 NGF_VERSION ?= edge## NGF version to be tested
 PULL_POLICY = Never## Pull policy for the images
 NGINX_CONF_DIR = internal/mode/static/nginx/conf
-PROVISIONER_MANIFEST = conformance/provisioner/provisioner.yaml
 SUPPORTED_EXTENDED_FEATURES = HTTPRouteQueryParamMatching,HTTPRouteMethodMatching,HTTPRoutePortRedirect,HTTPRouteSchemeRedirect,HTTPRouteHostRewrite,HTTPRoutePathRewrite,GatewayPort8080,HTTPRouteResponseHeaderModification,HTTPRoutePathRedirect,GatewayHTTPListenerIsolation,GatewayInfrastructurePropagation
 STANDARD_CONFORMANCE_PROFILES = GATEWAY-HTTP,GATEWAY-GRPC
 EXPERIMENTAL_CONFORMANCE_PROFILES = GATEWAY-TLS
@@ -74,9 +73,6 @@ cleanup-conformance-tests: ## Clean up conformance tests fixtures
 	kubectl delete pod conformance
 	kubectl delete -f conformance/conformance-rbac.yaml
 
-.PHONY: build
-build: generate-static-deployment
-
 .PHONY: reset-go-modules
 reset-go-modules: ## Reset the go modules changes
 	git checkout -- ../go.mod ../go.sum
@@ -169,16 +165,7 @@ delete-gke-cluster: ## Delete the GKE cluster
 add-local-ip-to-cluster: ## Add local IP to the GKE cluster master-authorized-networks
 	./scripts/add-local-ip-auth-networks.sh
 
-HELM_PARAMETERS += --set nameOverride=nginx-gateway --set nginxGateway.kind=skip --set nginx.service.type=ClusterIP --skip-schema-validation
-
-.PHONY: deploy-updated-provisioner
-deploy-updated-provisioner: ## Update provisioner manifest and deploy to the configured kind cluster
-	yq '(select(di != 3))' $(PROVISIONER_MANIFEST) | kubectl apply -f -
-	yq '(select(.spec.template.spec.containers[].image) | .spec.template.spec.containers[].image="$(PREFIX):$(TAG)" | .spec.template.spec.containers[].imagePullPolicy = "Never")' $(PROVISIONER_MANIFEST) | kubectl apply -f -
-
-.PHONY: generate-static-deployment
-generate-static-deployment:
-	helm template nginx-gateway $(CHART_DIR) --set nameOverride=nginx-gateway --set metrics.enable=false --set nginxGateway.productTelemetry.enable=false -n nginx-gateway -s templates/deployment.yaml --set nginxGateway.image.repository=$(PREFIX) --set nginxGateway.image.tag=$(TAG) --set nginxGateway.image.pullPolicy=Never --set nginx.image.repository=$(NGINX_PREFIX) --set nginx.image.tag=$(TAG) --set nginx.image.pullPolicy=Never --set nginxGateway.gwAPIExperimentalFeatures.enable=$(ENABLE_EXPERIMENTAL) --set nginx.plus=$(PLUS_ENABLED) --set nginx.usage.endpoint=$(PLUS_USAGE_ENDPOINT) > $(SELF_DIR)config/tests/static-deployment.yaml
+HELM_PARAMETERS += --set nameOverride=nginx-gateway --set nginx.service.type=ClusterIP --skip-schema-validation
 
 # this target is used to install the gateway-api CRDs from the main branch (only used in the nightly CI job)
 # it overrides the target in the main Makefile when the GW_API_VERSION is set to main
@@ -187,27 +174,15 @@ install-gateway-crds:
 	kubectl kustomize "https://github.com/kubernetes-sigs/gateway-api/config/crd/$(if $(filter true,$(ENABLE_EXPERIMENTAL)),experimental,)?timeout=120&ref=main" | kubectl apply -f -
 endif
 
-.PHONY: install-ngf-local-build
-install-ngf-local-build: deploy-updated-provisioner
-
 .PHONY: install-ngf-local-no-build
-install-ngf-local-no-build: load-images helm-install-local deploy-updated-provisioner ## Install NGF from local build with provisioner on configured kind cluster but do not build the NGF image
-
-.PHONY: install-ngf-local-build-with-plus
-install-ngf-local-build-with-plus: deploy-updated-provisioner
+install-ngf-local-no-build: load-images helm-install-local ## Install NGF from local build on configured kind cluster but do not build the NGF image
 
 .PHONY: install-ngf-local-no-build-with-plus
-install-ngf-local-no-build-with-plus: load-images-with-plus helm-install-local-with-plus deploy-updated-provisioner ## Install NGF with Plus from local build with provisioner on configured kind cluster but do not build the NGF image
-
-.PHONY: install-ngf-edge
-install-ngf-edge: load-images helm-install-local ## Install NGF with provisioner from edge on configured kind cluster
-	kubectl apply -f $(PROVISIONER_MANIFEST)
+install-ngf-local-no-build-with-plus: load-images-with-plus helm-install-local-with-plus ## Install NGF with Plus from local build on configured kind cluster but do not build the NGF image
 
 .PHONY: uninstall-ngf
-uninstall-ngf: ## Uninstall NGF on configured kind cluster and undo manifest changes
+uninstall-ngf: ## Uninstall NGF on configured kind cluster
 	-helm uninstall nginx-gateway -n nginx-gateway
 	-make uninstall-gateway-crds
-	-kubectl delete clusterrole nginx-gateway-provisioner
-	-kubectl delete clusterrolebinding nginx-gateway-provisioner
 	-kubectl delete namespace nginx-gateway
 	-kubectl kustomize ../config/crd | kubectl delete -f -

internal/mode/provisioner/doc.go

@@ -19,7 +19,6 @@ This directory contains the tests for NGINX Gateway Fabric. The tests are divide
   - [Step 1 - Install NGINX Gateway Fabric to configured kind cluster](#step-1---install-nginx-gateway-fabric-to-configured-kind-cluster)
     - [Option 1 - Build and install NGINX Gateway Fabric from local to configured kind cluster](#option-1---build-and-install-nginx-gateway-fabric-from-local-to-configured-kind-cluster)
     - [Option 2 - Install NGINX Gateway Fabric from local already built image to configured kind cluster](#option-2---install-nginx-gateway-fabric-from-local-already-built-image-to-configured-kind-cluster)
-    - [Option 3 - Install NGINX Gateway Fabric from edge to configured kind cluster](#option-3---install-nginx-gateway-fabric-from-edge-to-configured-kind-cluster)
   - [Step 2 - Build conformance test runner image](#step-2---build-conformance-test-runner-image)
   - [Step 3 - Run Gateway conformance tests](#step-3---run-gateway-conformance-tests)
   - [Step 4 - Cleanup the conformance test fixtures and uninstall NGINX Gateway Fabric](#step-4---cleanup-the-conformance-test-fixtures-and-uninstall-nginx-gateway-fabric)
@@ -158,15 +157,6 @@ Or, to install NGF with NGINX Plus enabled:
 make install-ngf-local-no-build-with-plus
 ```
 
-#### Option 3 - Install NGINX Gateway Fabric from edge to configured kind cluster
-
-You can also skip the build NGF image step and prepare the environment to instead use the `edge` image. Note that this
-option does not currently support installing with NGINX Plus enabled.
-
-```makefile
-make install-ngf-edge
-```
-
 ### Step 2 - Build conformance test runner image
 
 > Note: If you want to run the latest conformance tests from the Gateway API `main` branch, run the following