From 90cd1ba3632ea2300ba74a8de7be3d34d3dca740 Mon Sep 17 00:00:00 2001 From: gotti <18141824+gotti@users.noreply.github.com> Date: Tue, 27 Aug 2024 13:50:40 +0900 Subject: [PATCH] Implement NAT Gateway (#6) * add pod_watcher Signed-off-by: walnuts1018 * add pod_watcher Signed-off-by: walnuts1018 * add shouldHandle Signed-off-by: walnuts1018 * [WIP] implementing pod watcher Signed-off-by: gotti <18141824+gotti@users.noreply.github.com> * implement pod watcher Signed-off-by: walnuts1018 * change test dependencies Signed-off-by: walnuts1018 * Implement pod_watcher Signed-off-by: walnuts1018 * Implement pod_watcher_test Signed-off-by: walnuts1018 * [WIP] implementing fou tunnel Signed-off-by: gotti <18141824+gotti@users.noreply.github.com> * [WIP] Addpeer Signed-off-by: walnuts1018 * [WIP] writing addpeer Signed-off-by: gotti <18141824+gotti@users.noreply.github.com> * [WIP] delete unnecessary mutex Signed-off-by: gotti <18141824+gotti@users.noreply.github.com> * [WIP] setupDevice Signed-off-by: walnuts1018 * [WIP] implement nat server Signed-off-by: gotti <18141824+gotti@users.noreply.github.com> * [WIP] implement nat.AddClient Signed-off-by: walnuts1018 * fou.ConvNetIP -> netiputil.ConvNetIP Signed-off-by: walnuts1018 * [WIP] writing fou tunnel Signed-off-by: gotti <18141824+gotti@users.noreply.github.com> * init pod_watcher Signed-off-by: walnuts1018 * implement tests for pod_watcher Signed-off-by: gotti <18141824+gotti@users.noreply.github.com> * add dockerfiles and change Makefile to use them Signed-off-by: gotti <18141824+gotti@users.noreply.github.com> * change natpod image Signed-off-by: walnuts1018 * [WIP] testing nat gateway Signed-off-by: gotti <18141824+gotti@users.noreply.github.com> * rename dummy device Signed-off-by: walnuts1018 * rename nat pod to nat gateway Signed-off-by: walnuts1018 * rename test image name Signed-off-by: walnuts1018 * expect egressContainer.command == nil Signed-off-by: walnuts1018 * change package structure Signed-off-by: walnuts1018 * fix test failing Signed-off-by: gotti <18141824+gotti@users.noreply.github.com> --------- Signed-off-by: walnuts1018 Signed-off-by: gotti <18141824+gotti@users.noreply.github.com> Co-authored-by: walnuts1018 --- .github/workflows/ci.yaml | 33 +- Makefile | 18 +- api/v1beta1/egress_types.go | 2 +- cmd/egress-controller/main.go | 17 +- cmd/nat-gateway/main.go | 225 ++++++++++ .../crd/bases/pona.cybozu.com_egresses.yaml | 2 +- config/manager/egress-controller.yaml | 3 +- config/manager/kustomization.yaml | 6 +- config/samples/nat-client.yaml | 28 ++ .../Dockerfile.egress-controller | 5 +- dockerfiles/Dockerfile.nat-gateway | 31 ++ docs/design.md | 65 +-- go.mod | 41 +- go.sum | 156 +++++-- internal/controller/egress_controller.go | 14 +- internal/controller/egress_controller_test.go | 7 +- internal/controller/pod_watcher.go | 268 ++++++++++++ internal/controller/pod_watcher_test.go | 154 +++++++ internal/nat/mock/mock.go | 32 ++ internal/nat/nat.go | 158 +++++++ internal/tunnel/fou/fou.go | 389 ++++++++++++++++++ internal/tunnel/mock/mock.go | 41 ++ internal/tunnel/tunnel.go | 26 ++ internal/util/netiputil/netiputil.go | 10 + 24 files changed, 1596 insertions(+), 135 deletions(-) create mode 100644 cmd/nat-gateway/main.go create mode 100644 config/samples/nat-client.yaml rename Dockerfile => dockerfiles/Dockerfile.egress-controller (83%) create mode 100644 dockerfiles/Dockerfile.nat-gateway create mode 100644 internal/controller/pod_watcher.go create mode 100644 internal/controller/pod_watcher_test.go create mode 100644 internal/nat/mock/mock.go create mode 100644 internal/nat/nat.go create mode 100644 internal/tunnel/fou/fou.go create mode 100644 internal/tunnel/mock/mock.go create mode 100644 internal/tunnel/tunnel.go create mode 100644 internal/util/netiputil/netiputil.go diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a603b76..714d14e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -3,27 +3,28 @@ on: pull_request: push: branches: - - 'main' + - "main" jobs: test: strategy: matrix: - go-version: [ 1.22 ] - platform: [ ubuntu-22.04 ] + go-version: [1.22] + platform: [ubuntu-22.04] name: Small tests runs-on: ${{ matrix.platform }} steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 - with: - go-version: ${{ matrix.go-version }} - - uses: actions/cache@v4 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go- - - run: make test + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go-version }} + - uses: actions/cache@v4 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + - run: make check-generate + - run: make test diff --git a/Makefile b/Makefile index 808d1ab..7ea3294 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,7 @@ # Image URL to use all building/pushing image targets -IMG ?= controller:latest +IMG_TAG ?= dev +IMG_CONTROLLER ?= egress-controller:$(IMG_TAG) +IMG_GATEWAY ?= nat-gateway:$(IMG_TAG) # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. ENVTEST_K8S_VERSION = 1.30.0 @@ -65,11 +67,11 @@ mod: ## Run go mod tidy against code. go mod tidy .PHONY: test -test: vet envtest check-generate ## Run tests. +test: envtest manifests generate fmt vet mod ## Run tests. KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out .PHONY: check-generate -check-generate: manifests generate fmt mod +check-generate: manifests generate fmt mod git diff --exit-code --name-only # Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors. @@ -100,11 +102,13 @@ run: manifests generate fmt vet mod ## Run a controller from your host. # More info: https://docs.docker.com/develop/develop-images/build_enhancements/ .PHONY: docker-build docker-build: ## Build docker image with the manager. - $(CONTAINER_TOOL) build -t ${IMG} . + $(CONTAINER_TOOL) build -t ${IMG_CONTROLLER} -f ./dockerfiles/Dockerfile.egress-controller . + $(CONTAINER_TOOL) build -t ${IMG_GATEWAY} -f ./dockerfiles/Dockerfile.nat-gateway . .PHONY: docker-push docker-push: ## Push docker image with the manager. - $(CONTAINER_TOOL) push ${IMG} + $(CONTAINER_TOOL) push ${IMG_CONTROLLER} + $(CONTAINER_TOOL) push ${IMG_GATEWAY} # PLATFORMS defines the target platforms for the manager image be built to provide support to multiple # architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to: @@ -126,7 +130,7 @@ docker-buildx: ## Build and push docker image for the manager for cross-platform .PHONY: build-installer build-installer: manifests generate kustomize ## Generate a consolidated YAML with CRDs and deployment. mkdir -p dist - cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} + cd config/manager && $(KUSTOMIZE) edit set image egress-controller=${IMG_CONTROLLER} $(KUSTOMIZE) build config/default > dist/install.yaml ##@ Deployment @@ -145,7 +149,7 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified .PHONY: deploy deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. - cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} + cd config/manager && $(KUSTOMIZE) edit set image egress-controller=${IMG_CONTROLLER} $(KUSTOMIZE) build config/default | $(KUBECTL) apply -f - .PHONY: undeploy diff --git a/api/v1beta1/egress_types.go b/api/v1beta1/egress_types.go index b0f4010..e633876 100644 --- a/api/v1beta1/egress_types.go +++ b/api/v1beta1/egress_types.go @@ -50,7 +50,7 @@ type EgressSpec struct { // +optional SessionAffinityConfig *corev1.SessionAffinityConfig `json:"sessionAffinityConfig,omitempty"` - // PodDisruptionBudget is an optional PodDisruptionBudget for Egress NAT pods. + // PodDisruptionBudget is an optional PodDisruptionBudget for Egress NAT Gateways. // +optional PodDisruptionBudget *EgressPDBSpec `json:"podDisruptionBudget,omitempty"` } diff --git a/cmd/egress-controller/main.go b/cmd/egress-controller/main.go index 82446a7..b60709c 100644 --- a/cmd/egress-controller/main.go +++ b/cmd/egress-controller/main.go @@ -3,6 +3,7 @@ package main import ( "crypto/tls" "flag" + "fmt" "os" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) @@ -37,7 +38,8 @@ func init() { } type Config struct { - FoUPort int + FoUPort int + NatGatewayImage string } func main() { @@ -61,6 +63,7 @@ func main() { flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") flag.IntVar(&config.FoUPort, "fou-port", 5555, "port number for foo-over-udp tunnels") + flag.StringVar(&config.NatGatewayImage, "natgateway-image", "", "default image name for nat-gateway pods") opts := zap.Options{ Development: true, @@ -68,6 +71,11 @@ func main() { opts.BindFlags(flag.CommandLine) flag.Parse() + if config.NatGatewayImage == "" { + setupLog.Error(fmt.Errorf("--natgateway-image is required"), "failed to parse flags") + os.Exit(1) + } + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) // if the enable-http2 flag is false (the default), http/2 should be disabled @@ -138,9 +146,10 @@ func main() { } if err = (&controller.EgressReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Port: int32(config.FoUPort), + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Port: int32(config.FoUPort), + DefaultImage: config.NatGatewayImage, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Egress") os.Exit(1) diff --git a/cmd/nat-gateway/main.go b/cmd/nat-gateway/main.go new file mode 100644 index 0000000..21ad7d9 --- /dev/null +++ b/cmd/nat-gateway/main.go @@ -0,0 +1,225 @@ +package main + +import ( + "crypto/tls" + "errors" + "flag" + "net/netip" + "os" + "strings" + + // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) + // to ensure that exec-entrypoint and run can make use of them. + _ "k8s.io/client-go/plugin/pkg/client/auth" + + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/metrics/filters" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/controller-runtime/pkg/webhook" + + ponav1beta1 "github.com/cybozu-go/pona/api/v1beta1" + "github.com/cybozu-go/pona/internal/controller" + "github.com/cybozu-go/pona/internal/nat" + "github.com/cybozu-go/pona/internal/tunnel/fou" + // +kubebuilder:scaffold:imports +) + +var ( + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") +) + +func init() { + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + + utilruntime.Must(ponav1beta1.AddToScheme(scheme)) + // +kubebuilder:scaffold:scheme +} + +type Config struct { + FoUPort int +} + +func main() { + var metricsAddr string + var enableLeaderElection bool + var probeAddr string + var secureMetrics bool + var enableHTTP2 bool + var tlsOpts []func(*tls.Config) + + var config Config + + flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+ + "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") + flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") + flag.BoolVar(&enableLeaderElection, "leader-elect", false, + "Enable leader election for controller manager. "+ + "Enabling this will ensure there is only one active controller manager.") + flag.BoolVar(&secureMetrics, "metrics-secure", true, + "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + flag.BoolVar(&enableHTTP2, "enable-http2", false, + "If set, HTTP/2 will be enabled for the metrics and webhook servers") + flag.IntVar(&config.FoUPort, "fou-port", 5555, "port number for foo-over-udp tunnels") + + opts := zap.Options{ + Development: true, + } + opts.BindFlags(flag.CommandLine) + flag.Parse() + + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + + // if the enable-http2 flag is false (the default), http/2 should be disabled + // due to its vulnerabilities. More specifically, disabling http/2 will + // prevent from being vulnerable to the HTTP/2 Stream Cancellation and + // Rapid Reset CVEs. For more information see: + // - https://github.com/advisories/GHSA-qppj-fm5r-hxr3 + // - https://github.com/advisories/GHSA-4374-p667-p6c8 + disableHTTP2 := func(c *tls.Config) { + setupLog.Info("disabling http/2") + c.NextProtos = []string{"http/1.1"} + } + + if !enableHTTP2 { + tlsOpts = append(tlsOpts, disableHTTP2) + } + + webhookServer := webhook.NewServer(webhook.Options{ + TLSOpts: tlsOpts, + }) + + // Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server. + // More info: + // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.18.4/pkg/metrics/server + // - https://book.kubebuilder.io/reference/metrics.html + metricsServerOptions := metricsserver.Options{ + BindAddress: metricsAddr, + SecureServing: secureMetrics, + // TODO(user): TLSOpts is used to allow configuring the TLS config used for the server. If certificates are + // not provided, self-signed certificates will be generated by default. This option is not recommended for + // production environments as self-signed certificates do not offer the same level of trust and security + // as certificates issued by a trusted Certificate Authority (CA). The primary risk is potentially allowing + // unauthorized access to sensitive metrics data. Consider replacing with CertDir, CertName, and KeyName + // to provide certificates, ensuring the server communicates using trusted and secure certificates. + TLSOpts: tlsOpts, + } + + if secureMetrics { + // FilterProvider is used to protect the metrics endpoint with authn/authz. + // These configurations ensure that only authorized users and service accounts + // can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info: + // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.18.4/pkg/metrics/filters#WithAuthenticationAndAuthorization + metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization + } + + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + Scheme: scheme, + Metrics: metricsServerOptions, + WebhookServer: webhookServer, + HealthProbeBindAddress: probeAddr, + LeaderElection: enableLeaderElection, + LeaderElectionID: "7e4aaa0a.pona.cybozu.com", + // LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily + // when the Manager ends. This requires the binary to immediately end when the + // Manager is stopped, otherwise, this setting is unsafe. Setting this significantly + // speeds up voluntary leader transitions as the new leader don't have to wait + // LeaseDuration time first. + // + // In the default scaffold provided, the program ends immediately after + // the manager stops, so would be fine to enable this option. However, + // if you are doing or is intended to do any operation such as perform cleanups + // after the manager stops then its usage might be unsafe. + // LeaderElectionReleaseOnCancel: true, + }) + if err != nil { + setupLog.Error(err, "unable to start manager") + os.Exit(1) + } + + myNS := os.Getenv(controller.EnvPodNamespace) + if myNS == "" { + setupLog.Error(errors.New(controller.EnvPodNamespace+" environment variable must be set"), "unable to get env") + os.Exit(1) + } + + myName := os.Getenv(controller.EnvEgressName) + if myName == "" { + setupLog.Error(errors.New(controller.EnvEgressName+" environment variable must be set"), "unable to get env") + os.Exit(1) + } + + myAddresses := strings.Split(os.Getenv(controller.EnvAddresses), ",") + if len(myAddresses) == 0 { + setupLog.Error(errors.New(controller.EnvAddresses+" environment variable must be set"), "unable to get env") + os.Exit(1) + } + + var ipv4, ipv6 *netip.Addr + for _, addr := range myAddresses { + n, err := netip.ParseAddr(addr) + if err != nil { + setupLog.Error(errors.New(controller.EnvAddresses+" contains invalid address"), "unable to parse address", + "address", addr, + ) + os.Exit(1) + } + if n.Is4() { + ipv4 = &n + } else { + ipv6 = &n + } + } + + fc, err := fou.NewFoUTunnelController(config.FoUPort, ipv4, ipv6) + if err != nil { + setupLog.Error(err, "unable to create FouTunnelController") + os.Exit(1) + } + if err := fc.Init(); err != nil { + setupLog.Error(err, "failed to Initialize FoUTunnelController") + os.Exit(1) + } + nc, err := nat.NewController("eth0", ipv4, ipv6) + if err != nil { + setupLog.Error(err, "unable to create nat.Controller") + os.Exit(1) + } + if err := nc.Init(); err != nil { + setupLog.Error(err, "failed to Initialize nat.Controller") + os.Exit(1) + } + + if err = controller.NewPodWatcher( + mgr.GetClient(), + mgr.GetScheme(), + myName, + myNS, + fc, + nc, + ).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "Pod") + os.Exit(1) + } + // +kubebuilder:scaffold:builder + + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up health check") + os.Exit(1) + } + if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up ready check") + os.Exit(1) + } + + setupLog.Info("starting manager") + if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { + setupLog.Error(err, "problem running manager") + os.Exit(1) + } +} diff --git a/config/crd/bases/pona.cybozu.com_egresses.yaml b/config/crd/bases/pona.cybozu.com_egresses.yaml index 22647ca..3b5c356 100644 --- a/config/crd/bases/pona.cybozu.com_egresses.yaml +++ b/config/crd/bases/pona.cybozu.com_egresses.yaml @@ -48,7 +48,7 @@ spec: minItems: 1 type: array podDisruptionBudget: - description: PodDisruptionBudget is an optional PodDisruptionBudget for Egress NAT pods. + description: PodDisruptionBudget is an optional PodDisruptionBudget for Egress NAT Gateways. properties: maxUnavailable: anyOf: diff --git a/config/manager/egress-controller.yaml b/config/manager/egress-controller.yaml index 5e22dae..2869854 100644 --- a/config/manager/egress-controller.yaml +++ b/config/manager/egress-controller.yaml @@ -61,7 +61,8 @@ spec: - args: - --leader-elect - --health-probe-bind-address=:8081 - image: controller:latest + - --natgateway-image=nat-gateway:dev # to be changed + image: egress-controller:dev imagePullPolicy: IfNotPresent name: manager securityContext: diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index ce2158e..e639dea 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -3,6 +3,6 @@ resources: apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization images: -- name: controller - newName: controller - newTag: latest +- name: egress-controller + newName: egress-controller + newTag: dev diff --git a/config/samples/nat-client.yaml b/config/samples/nat-client.yaml new file mode 100644 index 0000000..f194eb9 --- /dev/null +++ b/config/samples/nat-client.yaml @@ -0,0 +1,28 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nat-client + annotations: + egress.pona.cybozu.com/egress: egress + labels: + app.kubernetes.io/component: nat-client +spec: + selector: + matchLabels: + app.kubernetes.io/component: nat-client + replicas: 1 + template: + metadata: + labels: + app.kubernetes.io/component: nat-client + annotations: + egress.pona.cybozu.com/egress: egress + spec: + containers: + - name: nat-client + image: ghcr.io/cybozu/ubuntu-debug:22.04 + command: ["sleep"] + securityContext: + privileged: true + args: + - infinity diff --git a/Dockerfile b/dockerfiles/Dockerfile.egress-controller similarity index 83% rename from Dockerfile rename to dockerfiles/Dockerfile.egress-controller index 7c3ff2f..bdf1001 100644 --- a/Dockerfile +++ b/dockerfiles/Dockerfile.egress-controller @@ -14,9 +14,10 @@ RUN go mod download # Copy the go source COPY cmd/egress-controller/main.go cmd/egress-controller/main.go COPY api/ api/ -COPY internal/controller/ internal/controller/ +COPY internal/ internal/ -RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o egress-controller cmd/egress-controller/main.go +RUN --mount=type=cache,target=/go/pkg/mod/ \ + CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o egress-controller cmd/egress-controller/main.go FROM ghcr.io/cybozu/ubuntu:22.04 WORKDIR / diff --git a/dockerfiles/Dockerfile.nat-gateway b/dockerfiles/Dockerfile.nat-gateway new file mode 100644 index 0000000..2e10461 --- /dev/null +++ b/dockerfiles/Dockerfile.nat-gateway @@ -0,0 +1,31 @@ +# Build the manager binary +FROM ghcr.io/cybozu/golang:1.22-jammy AS builder +ARG TARGETOS +ARG TARGETARCH + +WORKDIR /workspace +# Copy the Go Modules manifests +COPY go.mod go.mod +COPY go.sum go.sum +# cache deps before building and copying source so that we don't need to re-download as much +# and so that source changes don't invalidate our downloaded layer +RUN go mod download + +# Copy the go source +COPY cmd/nat-gateway/main.go cmd/nat-gateway/main.go +COPY api/ api/ +COPY internal/ internal/ + +RUN --mount=type=cache,target=/go/pkg/mod/ \ + CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o nat-gateway cmd/nat-gateway/main.go + +FROM ghcr.io/cybozu/ubuntu:22.04 +WORKDIR / +RUN --mount=type=cache,target=/var/lib/apt,sharing=locked \ + --mount=type=cache,target=/var/cache/apt,sharing=locked \ + apt-get -yy update && apt-get install -yy netbase kmod iptables iproute2 + +COPY --from=builder /workspace/nat-gateway . +USER 0:0 + +ENTRYPOINT ["/nat-gateway"] diff --git a/docs/design.md b/docs/design.md index 05ee066..453f25e 100644 --- a/docs/design.md +++ b/docs/design.md @@ -34,14 +34,14 @@ flowchart LR end subgraph ServerNode[Node] - natServer[NAT Pod] + natServer[NAT Gateway] end controller[Egress Controller] external[External Host] controller -- a - 1 watch Egress resource --> egress controller -- a - 2 apply ClusterIP Service resource --> svc - controller -- a - 2 apply NAT Pod resource --> natServer + controller -- a - 2 apply NAT Gateway resource --> natServer linkStyle 0,1,2 stroke: indianred; kubelet -- " b-1 create pod " --> natClient @@ -60,62 +60,65 @@ flowchart LR ### How Pona works -Users can create a NAT Pod with a Egress Custom Resource. -When a Egress resource is created, the Egress Controller creates a NAT Pod and a ClusterIP Service (`a-1`, `a-2`). +Users can create a NAT Gateway with a Egress Custom Resource. +When a Egress resource is created, the Egress Controller creates a NAT Gateway and a ClusterIP Service (`a-1`, `a-2`). When a user want to send a request with NAT, the user creates a NAT client Pod with an annotation. Ponad which is deployed as daemonset, watches Pod resources. -When a Pod with the annotation is created, Ponad configures the Pod to be routed to NAT Pod via the ClusterIP Service. -When a NAT client Pod is deleted, Pona CNI plugin is received a CNI Del and Ponad removes the configuration and NAT Pod removes the configuration corresponding to the NAT client (`b-1`, `b-2`, `b-3`, `b-4`). +When a Pod with the annotation is created, Ponad configures the Pod to be routed to NAT Gateway via the ClusterIP Service. +When a NAT client Pod is deleted, Pona CNI plugin is received a CNI Del and Ponad removes the configuration and NAT Gateway removes the configuration corresponding to the NAT client (`b-1`, `b-2`, `b-3`, `b-4`). -A request from a NAT client Pod is routed to the NAT Pod via the ClusterIP Service, and the NAT Pod performs SNAT and sends the request to the external host. +A request from a NAT client Pod is routed to the NAT Gateway via the ClusterIP Service, and the NAT Gateway performs SNAT and sends the request to the external host. -Pona uses FoU (Foo-over-UDP) to route packets between NAT client Pod and NAT Pod as. +Pona uses FoU (Foo-over-UDP) to route packets between NAT client Pod and NAT Gateway as. See [Coil's design docs](https://github.com/cybozu-go/coil/blob/main/docs/design.md#foo-over-udp-tunnel) for the reasons why FoU is adopted. However, Pona does not have IPAM functionality. -NAT Pods' IP addresses can be managed by using CNI plugins' feature such as [Coil AddressPool](https://github.com/cybozu-go/coil/blob/main/docs/usage.md#address-pools) or [Cilium Multi-Pool IPAM](https://docs.cilium.io/en/latest/network/concepts/ipam/multi-pool/). +NAT Gateways' IP addresses can be managed by using CNI plugins' feature such as [Coil AddressPool](https://github.com/cybozu-go/coil/blob/main/docs/usage.md#address-pools) or [Cilium Multi-Pool IPAM](https://docs.cilium.io/en/latest/network/concepts/ipam/multi-pool/). ### Components #### Egress Controller -- It Watches Egress resources and creates NAT Pods and ClusterIP Services. -#### NAT Pod +- It Watches Egress resources and creates NAT Gateways and ClusterIP Services. + +#### NAT Gateway + - It is a Pod that performs SNAT for NAT client Pods. - It configures MASQUERADE in iptables and FoU device at start-up #### Pona CNI Plugin + - It is a CLI tool that is satified CNI spec interface. - It delegates CNI calls to Ponad on the same node via an RPC call. - It is designed to be used in CNI chains and does not have IPAM functionality. #### Ponad + - It is deployed as a daemonset pod and runs on each node. - It configures network device and routing in the network namespace of a NAT client Pod via RPC calls from the Pona CNI Plugin. - It also watches Pod and Egress resources and configures NAT client Pods when those resources are changed. -### API +### API -#### Egress Custom Resource +#### Egress Custom Resource Egress resources have the following fields as well as Coil's Egress. -| Field | Type | required | Description | -|-------------------------|---------------------------|----------|-----------------------------------------------------------------| -| `destinations` | `[]string` | true | IP subnets where the packets are SNATed and sent. | -| `replicas` | `int` | false | Copied to Deployment's `spec.replicas`. Default is 1. | -| `strategy` | [DeploymentStrategy][] | false | Copied to Deployment's `spec.strategy`. | -| `template` | [PodTemplateSpec][] | false | Copied to Deployment's `spec.template`. | -| `sessionAffinity` | `ClusterIP` or `None` | false | Copied to Service's `spec.sessionAffinity`. Default is `None`. | -| `sessionAffinityConfig` | [SessionAffinityConfig][] | false | Copied to Service's `spec.sessionAffinityConfig`. | -| `podDisruptionBudget` | `EgressPDBSpec` | false | `minAvailable` and `maxUnavailable` are copied to PDB's spec. | +| Field | Type | required | Description | +| ----------------------- | ------------------------- | -------- | -------------------------------------------------------------- | +| `destinations` | `[]string` | true | IP subnets where the packets are SNATed and sent. | +| `replicas` | `int` | false | Copied to Deployment's `spec.replicas`. Default is 1. | +| `strategy` | [DeploymentStrategy][] | false | Copied to Deployment's `spec.strategy`. | +| `template` | [PodTemplateSpec][] | false | Copied to Deployment's `spec.template`. | +| `sessionAffinity` | `ClusterIP` or `None` | false | Copied to Service's `spec.sessionAffinity`. Default is `None`. | +| `sessionAffinityConfig` | [SessionAffinityConfig][] | false | Copied to Service's `spec.sessionAffinityConfig`. | +| `podDisruptionBudget` | `EgressPDBSpec` | false | `minAvailable` and `maxUnavailable` are copied to PDB's spec. | [DeploymentStrategy]: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#deploymentstrategy-v1-apps [PodTemplateSpec]: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#podtemplatespec-v1-core [SessionAffinityConfig]: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#sessionaffinityconfig-v1-core - Here is an example of Egress resource. ```yaml @@ -126,8 +129,8 @@ metadata: name: egress spec: destinations: - - 172.20.0.0/16 - - fd04::/64 + - 172.20.0.0/16 + - fd04::/64 replicas: 3 strategy: type: RollingUpdate @@ -142,10 +145,10 @@ spec: label1: bar spec: containers: - - name: egress - resources: - limits: - memory: 400Mi + - name: egress + resources: + limits: + memory: 400Mi sessionAffinity: ClientIP sessionAffinityConfig: clientIP: @@ -155,7 +158,8 @@ spec: ``` #### Annotations -To use NAT pod, users have to add an annotation to the Pod. + +To use NAT Gateway, users have to add an annotation to the Pod. Egress annotation's key is `egress.pona.cybozu.com/NAMESPACE` and its value is Egress resource's name which you want to use. Here is an example of Pod with Egress annotation. @@ -171,4 +175,3 @@ metadata: spec: # ... ``` - diff --git a/go.mod b/go.mod index b61af3f..23e4161 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,10 @@ module github.com/cybozu-go/pona go 1.22.6 require ( - github.com/onsi/ginkgo/v2 v2.17.1 - github.com/onsi/gomega v1.32.0 + github.com/containernetworking/plugins v1.5.1 + github.com/coreos/go-iptables v0.7.0 + github.com/onsi/ginkgo/v2 v2.19.0 + github.com/onsi/gomega v1.33.1 k8s.io/api v0.30.1 k8s.io/apimachinery v0.30.1 k8s.io/client-go v0.30.1 @@ -12,6 +14,13 @@ require ( sigs.k8s.io/controller-runtime v0.18.4 ) +require ( + github.com/containernetworking/cni v1.1.2 // indirect + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect + github.com/safchain/ethtool v0.4.0 // indirect + github.com/vishvananda/netns v0.0.4 // indirect +) + require ( github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -29,7 +38,6 @@ require ( github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.3 // indirect - github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect @@ -37,8 +45,8 @@ require ( github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect - github.com/google/uuid v1.3.0 // indirect + github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect github.com/imdario/mergo v0.3.6 // indirect github.com/josharian/intern v1.0.0 // indirect @@ -55,6 +63,7 @@ require ( github.com/prometheus/procfs v0.12.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stoewer/go-strcase v1.2.0 // indirect + github.com/vishvananda/netlink v1.2.1 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 // indirect go.opentelemetry.io/otel v1.19.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 // indirect @@ -66,19 +75,19 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect - golang.org/x/net v0.23.0 // indirect - golang.org/x/oauth2 v0.12.0 // indirect - golang.org/x/sync v0.6.0 // indirect - golang.org/x/sys v0.18.0 // indirect - golang.org/x/term v0.18.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/oauth2 v0.16.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.21.0 // indirect + golang.org/x/term v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.18.0 // indirect + golang.org/x/tools v0.21.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect - google.golang.org/grpc v1.58.3 // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 // indirect + google.golang.org/grpc v1.62.0 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 9ce9c0d..5171c35 100644 --- a/go.sum +++ b/go.sum @@ -11,6 +11,12 @@ github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/containernetworking/cni v1.1.2 h1:wtRGZVv7olUHMOqouPpn3cXJWpJgM6+EUl31EQbXALQ= +github.com/containernetworking/cni v1.1.2/go.mod h1:sDpYKmGVENF3s6uvMvGgldDWeG8dMxakj/u+i9ht9vw= +github.com/containernetworking/plugins v1.5.1 h1:T5ji+LPYjjgW0QM+KyrigZbLsZ8jaX+E5J/EcKOE4gQ= +github.com/containernetworking/plugins v1.5.1/go.mod h1:MIQfgMayGuHYs0XdNudf31cLLAC+i242hNm6KuDGqCM= +github.com/coreos/go-iptables v0.7.0 h1:XWM3V+MPRr5/q51NuWSgU0fqMad64Zyxs8ZUoMsamr8= +github.com/coreos/go-iptables v0.7.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -23,6 +29,8 @@ github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0 github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -38,34 +46,48 @@ github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2Kv github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= -github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= +github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68= +github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/cel-go v0.17.8 h1:j9m730pMZt1Fc4oKhCLUHfjj6527LuhYcYw0Rl8gqto= github.com/google/cel-go v0.17.8/go.mod h1:HXZKzB0LXqer5lHHgfWAnlYwJaQBDKMjxjulNQzhwhY= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 h1:k7nVchz72niMH6YLQNvHSdIE7iqsQxK1P41mySCvssg= +github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= @@ -93,10 +115,19 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onsi/ginkgo/v2 v2.17.1 h1:V++EzdbhI4ZV4ev0UTIj0PzhzOcReJFyJaLjtSF55M8= -github.com/onsi/ginkgo/v2 v2.17.1/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs= -github.com/onsi/gomega v1.32.0 h1:JRYU78fJ1LPxlckP6Txi/EYqJvjtMrDC04/MM5XRHPk= -github.com/onsi/gomega v1.32.0/go.mod h1:a4x4gW6Pz2yK1MAmvluYme5lvYTn61afQ2ETw/8n4Lg= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA= +github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= +github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -111,6 +142,8 @@ github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/safchain/ethtool v0.4.0 h1:vq1i2HCjshJNywOXFZ1BpwIjyeFR/kvNdHiRzqSElDI= +github.com/safchain/ethtool v0.4.0/go.mod h1:XLLnZmy4OCRTkksP/UiMjij96YmIsBfmBQcs7H6tA48= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU= @@ -120,14 +153,18 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/vishvananda/netlink v1.2.1 h1:pfLv/qlJUwOTPvtWREA7c3PI4u81YkqZw1DYhI2HmLA= +github.com/vishvananda/netlink v1.2.1/go.mod h1:i6NetklAujEcC6fK0JPjT8qSwWyO0HLn4UKG+hGqeJs= +github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8= +github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 h1:KfYpVmrjI7JuToy5k8XV3nkapjWx48k4E4JOtVstzQI= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0/go.mod h1:SeQhzAEccGVZVEy7aH87Nh0km+utSpo1pTv6eMMop48= go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs= @@ -153,71 +190,110 @@ go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA= golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= -golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= -golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= +golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= -golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.18.0 h1:k8NLag8AGHnn+PHbl7g43CtqZAwG60vZkLqgyZgIHgQ= -golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.21.0 h1:qc0xYgIbsSDt9EyWz05J5wfa7LOVW0YTLOXrqdLAWIw= +golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 h1:L6iMMGrtzgHsWofoFcihmDEMYeDR9KN/ThbPWGrh++g= -google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5/go.mod h1:oH/ZOT02u4kWEp7oYBGYFFkCdKS/uYR9Z7+0/xuuFp8= -google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e h1:z3vDksarJxsAKM5dmEGv0GHwE2hKJ096wZra71Vs4sw= -google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= -google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= -google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= +google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 h1:KAeGQVN3M9nD0/bQXnr/ClcEMJ968gUXJQ9pwfSynuQ= +google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80/go.mod h1:cc8bqMqtv9gMOr0zHg2Vzff5ULhhL2IXP4sbcn32Dro= +google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 h1:Lj5rbfG876hIAYFjqiJnPHfhXbv+nzTWfm04Fg/XSVU= +google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80/go.mod h1:4jWUdICTdgc3Ibxmr8nAJiiLHwQBY0UI0XZcEMaFKaA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 h1:AjyfHzEPEFp/NpvfN5g+KDla3EMojjhRVZc1i7cj+oM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80/go.mod h1:PAREbraiVEVGVdTZsVWjSbbTtSyGbAgIIvni8a8CD5s= +google.golang.org/grpc v1.62.0 h1:HQKZ/fa1bXkX1oFOvSjmZEUL8wLSaZTjCcLAlmZRtdk= +google.golang.org/grpc v1.62.0/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/controller/egress_controller.go b/internal/controller/egress_controller.go index a5dac20..fc34416 100644 --- a/internal/controller/egress_controller.go +++ b/internal/controller/egress_controller.go @@ -30,7 +30,6 @@ const ( ) const ( - egressImage = "ghcr.io/cybozu-go/coil:2.7.2" egressDefaultCpuRequest = "100m" egressDefaultMemRequest = "200Mi" egressServiceAccountName = "egress" @@ -52,7 +51,8 @@ type EgressReconciler struct { client.Client Scheme *runtime.Scheme - Port int32 + Port int32 + DefaultImage string } // +kubebuilder:rbac:groups=pona.cybozu.com,resources=egresses,verbs=get;list;watch;create;update;patch;delete @@ -469,16 +469,10 @@ func (r *EgressReconciler) reconcilePodTemplate(eg *ponav1beta1.Egress, deploy * } egressContainer.Name = "egress" - //TODO: Change image name and others from coil if egressContainer.Image == "" { - egressContainer.Image = egressImage - } - if len(egressContainer.Command) == 0 { - egressContainer.Command = []string{"coil-egress"} - } - if len(egressContainer.Args) == 0 { - egressContainer.Args = []string{"--zap-stacktrace-level=panic"} + egressContainer.Image = r.DefaultImage } + egressContainer.Env = append(egressContainer.Env, corev1.EnvVar{ Name: EnvPodNamespace, diff --git a/internal/controller/egress_controller_test.go b/internal/controller/egress_controller_test.go index 4b7a861..3130adf 100644 --- a/internal/controller/egress_controller_test.go +++ b/internal/controller/egress_controller_test.go @@ -107,7 +107,8 @@ var _ = Describe("Egress Controller", func() { Client: k8sClient, Scheme: k8sClient.Scheme(), - Port: port, + Port: port, + DefaultImage: "test-image", } _, err := controllerReconciler.Reconcile(ctx, reconcile.Request{ @@ -169,8 +170,8 @@ var _ = Describe("Egress Controller", func() { egressContainer := dep.Spec.Template.Spec.Containers[0] Expect(egressContainer).NotTo(BeNil()) - Expect(egressContainer.Image).To(Equal(egressImage)) - Expect(egressContainer.Command).To(Equal([]string{"coil-egress"})) //TODO: Change this when use another container image + Expect(egressContainer.Image).To(Equal(controllerReconciler.DefaultImage)) + Expect(egressContainer.Command).To(BeNil()) Expect(egressContainer.Env).To(HaveLen(3)) Expect(egressContainer.VolumeMounts).To(HaveLen(2)) Expect(egressContainer.SecurityContext).NotTo(BeNil()) diff --git a/internal/controller/pod_watcher.go b/internal/controller/pod_watcher.go new file mode 100644 index 0000000..8a56837 --- /dev/null +++ b/internal/controller/pod_watcher.go @@ -0,0 +1,268 @@ +package controller + +import ( + "context" + "errors" + "fmt" + "net/netip" + "slices" + "strings" + "sync" + + "github.com/cybozu-go/pona/internal/nat" + "github.com/cybozu-go/pona/internal/tunnel" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +const ( + EgressAnnotationPrefix = "egress.pona.cybozu.com/" +) + +// PodWatcher reconciles a Pod object +type PodWatcher struct { + client.Client + Scheme *runtime.Scheme + + EgressName string + EgressNamespace string + + linkMutex sync.Mutex + + podToPodIPs map[types.NamespacedName][]netip.Addr + podIPToPod map[netip.Addr]Set[types.NamespacedName] + + tun tunnel.Controller + nat nat.Controller +} + +type Set[T comparable] map[T]struct{} + +func NewPodWatcher(client client.Client, scheme *runtime.Scheme, egressName, egressNamespace string, t tunnel.Controller, n nat.Controller) *PodWatcher { + return &PodWatcher{ + Client: client, + Scheme: scheme, + EgressName: egressName, + EgressNamespace: egressNamespace, + + podToPodIPs: make(map[types.NamespacedName][]netip.Addr), + podIPToPod: make(map[netip.Addr]Set[types.NamespacedName]), + + tun: t, + nat: n, + } +} + +// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// TODO(user): Modify the Reconcile function to compare the state specified by +// the Pod object against the actual cluster state, and then +// perform operations to make the cluster state reflect the state specified by +// the user. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.0/pkg/reconcile +func (r *PodWatcher) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + logger := log.FromContext(ctx) + + pod := &corev1.Pod{} + if err := r.Get(ctx, req.NamespacedName, pod); err != nil { + if apierrors.IsNotFound(err) { + if err := r.handlePodDeletion(ctx, req.NamespacedName); err != nil { + logger.Error(err, "failed to remove tunnel") + return ctrl.Result{}, fmt.Errorf("failed to remove tunnel: %w", err) + } + return ctrl.Result{}, nil + } + return ctrl.Result{}, fmt.Errorf("failed to get Pod: %w", err) + } + + if !r.shouldHandle(pod) { + return ctrl.Result{}, nil + } + + // Pod is terminated or terminating + if isTerminated(pod) || pod.DeletionTimestamp != nil { + if err := r.handlePodDeletion(ctx, req.NamespacedName); err != nil { + logger.Error(err, "failed to remove tunnel for terminated pod") + return ctrl.Result{}, fmt.Errorf("failed to remove tunnel for terminated pod: %w", err) + } + return ctrl.Result{}, nil + } + + if err := r.handlePodRunning(ctx, pod); err != nil { + logger.Error(err, "failed to setup tunnel") + return ctrl.Result{}, fmt.Errorf("failed to setup tunnel: %w", err) + } + + return ctrl.Result{}, nil +} + +func isTerminated(pod *corev1.Pod) bool { + return pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed +} + +func (r *PodWatcher) shouldHandle(pod *corev1.Pod) bool { + if pod.Spec.HostNetwork { + // Egress feature is not available for Pods running in the host network. + return false + } + + return r.hasEgressAnnotation(pod) +} + +func (r *PodWatcher) handlePodRunning(ctx context.Context, pod *corev1.Pod) error { + logger := log.FromContext(ctx) + + r.linkMutex.Lock() + defer r.linkMutex.Unlock() + + podKey := types.NamespacedName{ + Name: pod.Name, + Namespace: pod.Namespace, + } + + existing := r.podToPodIPs[podKey] + statusPodIPs := make([]netip.Addr, len(pod.Status.PodIPs)) + for i, v := range pod.Status.PodIPs { + addr, err := netip.ParseAddr(v.IP) + if err != nil { + return err + } + statusPodIPs[i] = addr + } + + for _, ip := range statusPodIPs { + if slices.Contains(existing, ip) { + continue + } + + link, err := r.tun.AddPeer(ip) + if err != nil { + if errors.Is(err, tunnel.ErrIPFamilyMismatch) { + logger.Info("skipping unsupported pod IP", "pod", podKey, "ip", ip.String()) + continue + } + return err + } + + if err := r.nat.AddClient(ip, link); err != nil { + return fmt.Errorf("failed to setup NAT for ip=%s; %w", ip, err) + } + + } + + for _, eip := range existing { + if slices.Contains(statusPodIPs, eip) { + continue + } + + if err := r.tun.DelPeer(eip); err != nil { + return err + } + logger.Info("tunnel has been deleted", + "caller", "addPod", + "pod", podKey, + "ip", eip.String(), + ) + } + + r.podToPodIPs[podKey] = statusPodIPs + for _, ip := range statusPodIPs { + keySet, ok := r.podIPToPod[ip] + if !ok { + r.podIPToPod[ip] = Set[types.NamespacedName]{ + podKey: struct{}{}, + } + } else { + keySet[podKey] = struct{}{} + } + } + + return nil +} + +func (r *PodWatcher) handlePodDeletion(ctx context.Context, namespacedName types.NamespacedName) error { + logger := log.FromContext(ctx) + + r.linkMutex.Lock() + defer r.linkMutex.Unlock() + for _, ip := range r.podToPodIPs[namespacedName] { + exists, err := r.existsOtherLiveTunnels(namespacedName, ip) + if err != nil { + return err + } + + if !exists { + if err := r.tun.DelPeer(ip); err != nil { + return err + } + + logger.Info("tunnel has been deleted", + "caller", "addPod", + "pod", namespacedName, + "ip", ip.String(), + ) + } + + if keySet, ok := r.podIPToPod[ip]; ok { + delete(keySet, namespacedName) + if len(keySet) == 0 { + delete(r.podIPToPod, ip) + } + } + } + + delete(r.podToPodIPs, namespacedName) + + return nil +} + +func (r *PodWatcher) existsOtherLiveTunnels(namespacedName types.NamespacedName, ip netip.Addr) (bool, error) { + if keySet, ok := r.podIPToPod[ip]; ok { + if _, ok := keySet[namespacedName]; ok { + return len(keySet) > 1, nil + } + return false, fmt.Errorf("keySet in the podIPToPod doesn't contain my key. key: %s ip: %s", namespacedName, ip) + } + + return false, fmt.Errorf("podIPToPod doesn't contain my IP. key: %s ip: %s", namespacedName, ip) +} + +func (r *PodWatcher) hasEgressAnnotation(pod *corev1.Pod) bool { + for k, name := range pod.Annotations { + if !strings.HasPrefix(k, EgressAnnotationPrefix) { + continue + } + + if k[len(EgressAnnotationPrefix):] != r.EgressNamespace { + continue + } + + // shortcut for the most typical case + if name == r.EgressName { + return true + } + + for _, n := range strings.Split(name, ",") { + if n == r.EgressNamespace { + return true + } + } + } + return false +} + +// SetupWithManager sets up the controller with the Manager. +func (r *PodWatcher) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&corev1.Pod{}). + Complete(r) +} diff --git a/internal/controller/pod_watcher_test.go b/internal/controller/pod_watcher_test.go new file mode 100644 index 0000000..1db774c --- /dev/null +++ b/internal/controller/pod_watcher_test.go @@ -0,0 +1,154 @@ +package controller + +import ( + "context" + "net/netip" + "path/filepath" + + natmock "github.com/cybozu-go/pona/internal/nat/mock" + tunnelmock "github.com/cybozu-go/pona/internal/tunnel/mock" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +type pod struct { + NamespacedName types.NamespacedName + PodIPs []netip.Addr +} + +var _ = Describe("Pod Watcher", func() { + Context("When reconciling a resource", func() { + ctx := context.Background() + const ( + egressName = "egress" + egressNamespace = "default" + ) + + podInfo := pod{ + NamespacedName: types.NamespacedName{ + Name: "pod", + Namespace: "default", + }, + PodIPs: []netip.Addr{netip.MustParseAddr("192.168.0.1")}, + } + pod := &corev1.Pod{} + + BeforeEach(func() { + pod.SetName(podInfo.NamespacedName.Name) + pod.SetNamespace(podInfo.NamespacedName.Namespace) + pod.Spec.Containers = []corev1.Container{ + { + Name: "container", + Image: "image", + }, + } + + pod.Annotations = map[string]string{ + filepath.Join(EgressAnnotationPrefix, egressNamespace): egressName, + } + + By("create pod") + err := k8sClient.Create(ctx, pod) + Expect(err).NotTo(HaveOccurred()) + + By("set pod status") + Eventually(func() error { + return k8sClient.Get(ctx, client.ObjectKey(podInfo.NamespacedName), pod) + }).Should(Succeed()) + + pod.Status.PodIPs = make([]corev1.PodIP, 0, len(podInfo.PodIPs)) + for _, ip := range podInfo.PodIPs { + pod.Status.PodIPs = append(pod.Status.PodIPs, corev1.PodIP{IP: ip.String()}) + } + + err = k8sClient.Status().Update(ctx, pod) + Expect(err).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + }) + + It("should successfully reconcile the resource", func() { + By("Reconcile the created resource") + t := tunnelmock.NewMockTunnel() + n := natmock.NewMockNat() + w := &PodWatcher{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + tun: t, + nat: n, + + EgressName: egressName, + EgressNamespace: egressNamespace, + + podToPodIPs: make(map[types.NamespacedName][]netip.Addr), + podIPToPod: make(map[netip.Addr]Set[types.NamespacedName]), + } + + _, err := w.Reconcile(ctx, reconcile.Request{ + NamespacedName: podInfo.NamespacedName, + }) + Expect(err).NotTo(HaveOccurred()) + + By("Check if mockTunnel.AddPeer() is called") + for _, ip := range podInfo.PodIPs { + _, ok := t.Tunnels[ip] + Expect(ok).To(BeTrue()) + } + + By("Check if mockNAT.AddClient() is called") + for _, ip := range podInfo.PodIPs { + _, ok := n.Clients[ip] + Expect(ok).To(BeTrue()) + } + + By("Check podToPodIPs, podIPsToPod") + Expect(w.podToPodIPs).To(Equal(map[types.NamespacedName][]netip.Addr{ + podInfo.NamespacedName: podInfo.PodIPs, + })) + Expect(w.podIPToPod).To(Equal(podIPToPod(podInfo))) + + By("Delete Pod") + + err = k8sClient.Delete(ctx, pod) + Expect(err).NotTo(HaveOccurred()) + + By("Reconcile the created resource") + _, err = w.Reconcile(ctx, reconcile.Request{ + NamespacedName: podInfo.NamespacedName, + }) + Expect(err).NotTo(HaveOccurred()) + + By("Check if mockTunnel.DelPeer() is called") + for _, ip := range podInfo.PodIPs { + _, ok := t.Tunnels[ip] + Expect(ok).To(BeFalse()) + } + + By("Check podToPodIPs, podIPsToPod") + Expect(w.podToPodIPs).To(Equal(map[types.NamespacedName][]netip.Addr{})) + Expect(w.podIPToPod).To(Equal(map[netip.Addr]Set[types.NamespacedName]{})) + + }) + }) +}) + +func podIPToPod(pod pod) map[netip.Addr]Set[types.NamespacedName] { + m := make(map[netip.Addr]Set[types.NamespacedName]) + for _, v := range pod.PodIPs { + nns, ok := m[v] + if !ok { + m[v] = Set[types.NamespacedName]{ + pod.NamespacedName: struct{}{}, + } + } else { + nns[pod.NamespacedName] = struct{}{} + } + } + return m +} diff --git a/internal/nat/mock/mock.go b/internal/nat/mock/mock.go new file mode 100644 index 0000000..5d7cc94 --- /dev/null +++ b/internal/nat/mock/mock.go @@ -0,0 +1,32 @@ +package mock + +import ( + "fmt" + "net/netip" + + "github.com/vishvananda/netlink" +) + +type linkName string + +type mockNAT struct { + Clients map[netip.Addr]linkName +} + +func NewMockNat() *mockNAT { + return &mockNAT{ + Clients: make(map[netip.Addr]linkName), + } +} + +func (m *mockNAT) Init() error { + return nil +} + +func (m *mockNAT) AddClient(addr netip.Addr, link netlink.Link) error { + if link.Attrs() == nil { + return fmt.Errorf("link.Attrs() returns nil") + } + m.Clients[addr] = linkName(link.Attrs().Name) + return nil +} diff --git a/internal/nat/nat.go b/internal/nat/nat.go new file mode 100644 index 0000000..5dd8c37 --- /dev/null +++ b/internal/nat/nat.go @@ -0,0 +1,158 @@ +package nat + +import ( + "errors" + "fmt" + "net/netip" + + "github.com/coreos/go-iptables/iptables" + "github.com/cybozu-go/pona/internal/util/netiputil" + "github.com/vishvananda/netlink" +) + +const ( + egressTableID = 118 + egressProtocolID = 30 + egressRulePrio = 2000 + + egressDummy = "nat-dummy" +) + +type Controller interface { + Init() error + AddClient(netip.Addr, netlink.Link) error +} + +type controller struct { + iface string + ipv4 *netip.Addr + ipv6 *netip.Addr +} + +var ErrIPFamilyMismatch = errors.New("no matching IP family") + +func NewController(iface string, ipv4, ipv6 *netip.Addr) (Controller, error) { + if ipv4 != nil && !ipv4.Is4() { + return nil, fmt.Errorf("invalid IPv4 address, ip=%s", ipv4.String()) + } + if ipv6 != nil && !ipv6.Is6() { + return nil, fmt.Errorf("invalid IPv6 address, ip=%s", ipv6.String()) + } + + return &controller{ + iface: iface, + ipv4: ipv4, + ipv6: ipv6, + }, nil +} + +func (c *controller) newRule(family int) *netlink.Rule { + r := netlink.NewRule() + r.Family = family + r.IifName = c.iface + r.Table = egressTableID + r.Priority = egressRulePrio + return r +} + +func (c *controller) Init() error { + // avoid double initialization in case the program restarts + _, err := netlink.LinkByName(egressDummy) + if err == nil { + return nil + } + if _, ok := err.(netlink.LinkNotFoundError); !ok { + return err + } + + if c.ipv4 != nil { + ipt, err := iptables.NewWithProtocol(iptables.ProtocolIPv4) + if err != nil { + return err + } + ipn := netlink.NewIPNet(netiputil.ConvNetIP(*c.ipv4)) + err = ipt.Append("nat", "POSTROUTING", "!", "-s", ipn.String(), "-o", c.iface, "-j", "MASQUERADE") + if err != nil { + return fmt.Errorf("failed to setup masquerade rule for IPv4: %w", err) + } + + rule := c.newRule(netlink.FAMILY_V4) + if err := netlink.RuleAdd(rule); err != nil { + return fmt.Errorf("netlink: failed to add egress rule for IPv4: %w", err) + } + } + if c.ipv6 != nil { + ipt, err := iptables.NewWithProtocol(iptables.ProtocolIPv6) + if err != nil { + return err + } + ipn := netlink.NewIPNet(netiputil.ConvNetIP(*c.ipv6)) + err = ipt.Append("nat", "POSTROUTING", "!", "-s", ipn.String(), "-o", c.iface, "-j", "MASQUERADE") + if err != nil { + return fmt.Errorf("failed to setup masquerade rule for IPv6: %w", err) + } + + rule := c.newRule(netlink.FAMILY_V6) + if err := netlink.RuleAdd(rule); err != nil { + return fmt.Errorf("netlink: failed to add egress rule for IPv6: %w", err) + } + } + + attrs := netlink.NewLinkAttrs() + attrs.Name = egressDummy + + if err := netlink.LinkAdd(&netlink.Dummy{LinkAttrs: attrs}); err != nil { + return fmt.Errorf("failed to add dummy device: %w", err) + } + return nil +} + +func (c *controller) AddClient(addr netip.Addr, link netlink.Link) error { + // Note: + // The following checks are not necessary in fact because, + // prior to this point, the support for the IP family is tested + // by FouTunnel.AddPeer(). If the test fails, then no `link` + // is created and this method will not be called. + // Just as a safeguard. + if addr.Is4() && c.ipv4 == nil { + return ErrIPFamilyMismatch + } + if addr.Is6() && c.ipv6 == nil { + return ErrIPFamilyMismatch + } + + family := netlink.FAMILY_V4 + if addr.Is6() { + family = netlink.FAMILY_V6 + } + + routes, err := netlink.RouteListFiltered(family, &netlink.Route{Table: egressTableID}, netlink.RT_FILTER_TABLE) + if err != nil { + return fmt.Errorf("netlink: failed to list routes in table %d: %w", egressTableID, err) + } + + for _, r := range routes { + if r.Dst == nil { + continue + } + if r.Dst.IP.Equal(netiputil.ConvNetIP(addr)) { + return nil + } + } + + // link up here to minimize the down time + // See https://github.com/cybozu-go/coil/issues/287. + if err := netlink.LinkSetUp(link); err != nil { + return fmt.Errorf("netlink: failed to link up %s: %w", link.Attrs().Name, err) + } + if err := netlink.RouteAdd(&netlink.Route{ + Dst: netlink.NewIPNet(netiputil.ConvNetIP(addr)), + LinkIndex: link.Attrs().Index, + Table: egressTableID, + Protocol: egressProtocolID, + }); err != nil { + return fmt.Errorf("netlink: failed to add %s to table %d: %w", addr.String(), egressTableID, err) + } + + return nil +} diff --git a/internal/tunnel/fou/fou.go b/internal/tunnel/fou/fou.go new file mode 100644 index 0000000..2013d03 --- /dev/null +++ b/internal/tunnel/fou/fou.go @@ -0,0 +1,389 @@ +package fou + +import ( + "crypto/sha1" + "errors" + "fmt" + "net/netip" + "os/exec" + "strconv" + + "github.com/containernetworking/plugins/pkg/ip" + "github.com/containernetworking/plugins/pkg/utils/sysctl" + "github.com/coreos/go-iptables/iptables" + "github.com/cybozu-go/pona/internal/tunnel" + "github.com/cybozu-go/pona/internal/util/netiputil" + "github.com/vishvananda/netlink" +) + +// Prefixes for Foo-over-UDP tunnel link names +const ( + FoU4LinkPrefix = "fou4_" + FoU6LinkPrefix = "fou6_" +) + +const fouDummy = "fou-dummy" + +func fouName(addr netip.Addr) (string, error) { + if addr.Is4() { + return fmt.Sprintf("%s%x", FoU4LinkPrefix, addr.As4()), nil + } else if addr.Is6() { + addrSlice := addr.As16() + hash := sha1.Sum(addrSlice[:]) + return fmt.Sprintf("%s%x", FoU6LinkPrefix, hash[:4]), nil + } + return "", fmt.Errorf("unknown ip families ip=%s", addr.String()) +} + +func modProbe(module string) error { + out, err := exec.Command("/sbin/modprobe", module).CombinedOutput() + if err != nil { + return fmt.Errorf("modprobe %s failed with %w: %s", module, err, string(out)) + } + return nil +} + +func disableRPFilter() error { + if _, err := sysctl.Sysctl("net.ipv4.conf.default.rp_filter", "0"); err != nil { + return fmt.Errorf("setting net.ipv4.conf.default.rp_filter=0 failed: %w", err) + } + if _, err := sysctl.Sysctl("net.ipv4.conf.all.rp_filter", "0"); err != nil { + return fmt.Errorf("setting net.ipv4.conf.all.rp_filter=0 failed: %w", err) + } + return nil +} + +type FouTunnelController struct { + port int + local4 *netip.Addr + local6 *netip.Addr +} + +// NewFoUTunnel creates a new fouTunnel. +// port is the UDP port to receive FoU packets. +// localIPv4 is the local IPv4 address of the IPIP tunnel. This can be nil. +// localIPv6 is the same as localIPv4 for IPv6. +func NewFoUTunnelController(port int, localIPv4, localIPv6 *netip.Addr) (*FouTunnelController, error) { + if localIPv4 != nil && !localIPv4.Is4() { + return nil, tunnel.ErrIPFamilyMismatch + } + if localIPv6 != nil && !localIPv6.Is6() { + return nil, tunnel.ErrIPFamilyMismatch + } + return &FouTunnelController{ + port: port, + local4: localIPv4, + local6: localIPv6, + }, nil +} + +func (t *FouTunnelController) Init() error { + _, err := netlink.LinkByName(fouDummy) + if err == nil { + return nil + } + var linkNotFoundError netlink.LinkNotFoundError + if !errors.As(err, &linkNotFoundError) { + return fmt.Errorf("failed to initialize fou tunnel: %w", err) + } + + if t.local4 != nil { + if err := disableRPFilter(); err != nil { + return fmt.Errorf("failed to disable RP Filter: %w", err) + } + if err := ip.EnableIP4Forward(); err != nil { + return fmt.Errorf("failed to enable IPv4 forwarding: %w", err) + } + + if err := modProbe("fou"); err != nil { + return fmt.Errorf("failed to load fou module: %w", err) + } + err := netlink.FouAdd(netlink.Fou{ + Family: netlink.FAMILY_V4, + Protocol: 4, // IPv4 over IPv4 + Port: t.port, + EncapType: netlink.FOU_ENCAP_DIRECT, + }) + if err != nil { + return fmt.Errorf("netlink: fou addlink failed: %w", err) + } + + if err := t.initIPTables(iptables.ProtocolIPv4); err != nil { + return err + } + } + if t.local6 != nil { + if err := ip.EnableIP6Forward(); err != nil { + return fmt.Errorf("failed to enable IPv6 forwarding: %w", err) + } + + if err := modProbe("fou6"); err != nil { + return fmt.Errorf("failed to load fou module: %w", err) + } + err := netlink.FouAdd(netlink.Fou{ + Family: netlink.FAMILY_V6, + Protocol: 41, // IPv6 over IPv6 + Port: t.port, + EncapType: netlink.FOU_ENCAP_DIRECT, + }) + if err != nil { + return fmt.Errorf("netlink: fou addlink failed: %w", err) + } + + if err := t.initIPTables(iptables.ProtocolIPv6); err != nil { + return err + } + } + + attrs := netlink.NewLinkAttrs() + attrs.Name = fouDummy + + if err := netlink.LinkAdd(&netlink.Dummy{LinkAttrs: attrs}); err != nil { + return fmt.Errorf("failed to add dummy device: %w", err) + } + return nil +} + +func (t *FouTunnelController) initIPTables(p iptables.Protocol) error { + ipt, err := iptables.NewWithProtocol(p) + if err != nil { + return err + } + // workaround for kube-proxy's double NAT problem + rulespec := []string{ + "-p", "udp", "--dport", strconv.Itoa(t.port), "-j", "CHECKSUM", "--checksum-fill", + } + if err := ipt.Insert("mangle", "POSTROUTING", 1, rulespec...); err != nil { + return fmt.Errorf("failed to setup mangle table: %w", err) + } + + return nil +} + +func (t *FouTunnelController) IsInitialized() bool { + _, err := netlink.LinkByName(fouDummy) + return err == nil +} + +func (t *FouTunnelController) AddPeer(addr netip.Addr) (netlink.Link, error) { + if addr.Is4() { + return t.addPeer4(addr) + } else if addr.Is6() { + return t.addPeer6(addr) + } + return nil, fmt.Errorf("unknown ip families ip=%s", addr.String()) +} + +func (t *FouTunnelController) addPeer4(addr netip.Addr) (netlink.Link, error) { + if t.local4 == nil { + return nil, tunnel.ErrIPFamilyMismatch + } + + linkname, err := fouName(addr) + if err != nil { + return nil, fmt.Errorf("failed to generate fou name: %w", err) + } + link, err := netlink.LinkByName(linkname) + if err == nil { + // if already exists, return old link + return link, nil + } else { + var linkNotFoundError netlink.LinkNotFoundError + if !errors.As(err, &linkNotFoundError) { + return nil, fmt.Errorf("netlink: failed to get link by name: %w", err) + } + } + + attrs := netlink.NewLinkAttrs() + attrs.Name = linkname + link = &netlink.Iptun{ + LinkAttrs: attrs, + Ttl: 64, + EncapType: netlink.FOU_ENCAP_DIRECT, + EncapDport: uint16(t.port), + EncapSport: 0, // sportauto is always on + Remote: netiputil.ConvNetIP(addr), + Local: netiputil.ConvNetIP(*t.local4), + } + if err := netlink.LinkAdd(link); err != nil { + return nil, fmt.Errorf("netlink: failed to add fou link: %w", err) + } + + if err := setupFlowBasedIP4TunDevice(); err != nil { + return nil, fmt.Errorf("netlink: failed to setup ipip device: %w", err) + } + + return link, nil +} + +func (t *FouTunnelController) addPeer6(addr netip.Addr) (netlink.Link, error) { + if t.local6 == nil { + return nil, tunnel.ErrIPFamilyMismatch + } + + linkname, err := fouName(addr) + if err != nil { + return nil, fmt.Errorf("failed to generate fou name: %w", err) + } + link, err := netlink.LinkByName(linkname) + if err == nil { + // if already exists, return old link + return link, nil + } else { + var linkNotFoundError netlink.LinkNotFoundError + if !errors.As(err, &linkNotFoundError) { + return nil, fmt.Errorf("netlink: failed to get link by name: %w", err) + } + } + + attrs := netlink.NewLinkAttrs() + attrs.Name = linkname + link = &netlink.Iptun{ + LinkAttrs: attrs, + Ttl: 64, + EncapType: netlink.FOU_ENCAP_DIRECT, + EncapDport: uint16(t.port), + EncapSport: 0, // sportauto is always on + Remote: netiputil.ConvNetIP(addr), + Local: netiputil.ConvNetIP(*t.local6), + } + if err := netlink.LinkAdd(link); err != nil { + return nil, fmt.Errorf("netlink: failed to add fou link: %w", err) + } + + if err := setupFlowBasedIP6TunDevice(); err != nil { + return nil, fmt.Errorf("netlink: failed to setup ipip device: %w", err) + } + + return link, nil +} + +func (t *FouTunnelController) DelPeer(addr netip.Addr) error { + linkName, err := fouName(addr) + if err != nil { + return fmt.Errorf("failed to generate fou name: %w", err) + } + + link, err := netlink.LinkByName(linkName) + if err != nil { + var linkNotFoundError netlink.LinkNotFoundError + if errors.As(err, &linkNotFoundError) { + return nil + } + return fmt.Errorf("failed to delete interface: %w", err) + } + return netlink.LinkDel(link) +} + +// setupFlowBasedIP[4,6]TunDevice creates an IPv4 or IPv6 tunnel device +// +// This flow based IPIP tunnel device is used to decapsulate packets from +// the router Pods. +// +// Calling this function may result in tunl0 (v4) or ip6tnl0 (v6) +// fallback interface being renamed to coil_tunl or coil_ip6tnl. +// This is to communicate to the user that this plugin has taken +// control of the encapsulation stack on the netns, as it currently +// doesn't explicitly support sharing it with other tools/CNIs. +// Fallback devices are left unused for production traffic. +// Only devices that were explicitly created are used. +// +// This fallback interface is present as a result of loading the +// ipip and ip6_tunnel kernel modules by fou tunnel interfaces. +// These are catch-all interfaces for the ipip decapsulation stack. +// By default, these interfaces will be created in new network namespaces, +// but this behavior can be disabled by setting net.core.fb_tunnels_only_for_init_net = 2. +func setupFlowBasedIP4TunDevice() error { + ipip4Device := "coil_ipip4" + // Set up IPv4 tunnel device if requested. + if err := setupDevice(&netlink.Iptun{ + LinkAttrs: netlink.LinkAttrs{Name: ipip4Device}, + FlowBased: true, + }); err != nil { + return fmt.Errorf("creating %s: %w", ipip4Device, err) + } + + // Rename fallback device created by potential kernel module load after + // creating tunnel interface. + if err := renameDevice("tunl0", "pona_tunl"); err != nil { + return fmt.Errorf("renaming fallback device %s: %w", "tunl0", err) + } + + return nil +} + +// See setupFlowBasedIP4TunDevice +func setupFlowBasedIP6TunDevice() error { + ipip6Device := "coil_ipip6" + + // Set up IPv6 tunnel device if requested. + if err := setupDevice(&netlink.Ip6tnl{ + LinkAttrs: netlink.LinkAttrs{Name: ipip6Device}, + FlowBased: true, + }); err != nil { + return fmt.Errorf("creating %s: %w", ipip6Device, err) + } + + // Rename fallback device created by potential kernel module load after + // creating tunnel interface. + if err := renameDevice("ip6tnl0", "coil_ip6tnl"); err != nil { + return fmt.Errorf("renaming fallback device %s: %w", "tunl0", err) + } + + return nil +} + +// setupDevice creates and configures a device based on the given netlink attrs. +func setupDevice(link netlink.Link) error { + name := link.Attrs().Name + + // Reuse existing tunnel interface created by previous runs. + l, err := netlink.LinkByName(name) + if err != nil { + var linkNotFoundError netlink.LinkNotFoundError + if !errors.As(err, &linkNotFoundError) { + return err + } + + if err := netlink.LinkAdd(link); err != nil { + return fmt.Errorf("netlink: failed to create device %s: %w", name, err) + } + + // Fetch the link we've just created. + l, err = netlink.LinkByName(name) + if err != nil { + return fmt.Errorf("netlink: failed to retrieve created device %s: %w", name, err) + } + } + + if err := configureDevice(l); err != nil { + return fmt.Errorf("failed to set up device %s: %w", l.Attrs().Name, err) + } + + return nil +} + +// configureDevice puts the given link into the up state +func configureDevice(link netlink.Link) error { + ifName := link.Attrs().Name + + if err := netlink.LinkSetUp(link); err != nil { + return fmt.Errorf("netlink: failed to set link %s up: %w", ifName, err) + } + return nil +} + +// renameDevice renames a network device from and to a given value. Returns nil +// if the device does not exist. +func renameDevice(from, to string) error { + link, err := netlink.LinkByName(from) + if err != nil { + return nil + } + + if err := netlink.LinkSetName(link, to); err != nil { + return fmt.Errorf("netlink: failed to rename device %s to %s: %w", from, to, err) + } + + return nil +} diff --git a/internal/tunnel/mock/mock.go b/internal/tunnel/mock/mock.go new file mode 100644 index 0000000..ce4e00b --- /dev/null +++ b/internal/tunnel/mock/mock.go @@ -0,0 +1,41 @@ +package mock + +import ( + "net/netip" + + "github.com/vishvananda/netlink" +) + +type mockTunnel struct { + Tunnels map[netip.Addr]struct{} +} + +func NewMockTunnel() mockTunnel { + return mockTunnel{ + Tunnels: make(map[netip.Addr]struct{}), + } +} + +func (m mockTunnel) AddPeer(addr netip.Addr) (netlink.Link, error) { + m.Tunnels[addr] = struct{}{} + link := &netlink.Dummy{ + LinkAttrs: netlink.LinkAttrs{ + Name: "dummy", + Index: 1, + }, + } + return link, nil +} + +func (m mockTunnel) DelPeer(addr netip.Addr) error { + delete(m.Tunnels, addr) + return nil +} + +func (m mockTunnel) Init() error { + return nil +} + +func (m mockTunnel) IsInitialized() bool { + return true +} diff --git a/internal/tunnel/tunnel.go b/internal/tunnel/tunnel.go new file mode 100644 index 0000000..be3d3fb --- /dev/null +++ b/internal/tunnel/tunnel.go @@ -0,0 +1,26 @@ +package tunnel + +import ( + "errors" + "net/netip" + + "github.com/vishvananda/netlink" +) + +type Controller interface { + // Init starts FoU listening socket. + Init() error + + // IsInitialized checks if this Controller has been initialized + IsInitialized() bool + + // Add setups tunnel devices to the given peer and returns them. + // If Controller does not setup for the IP family of the given address, + // this returns ErrIPFamilyMismatch error. + AddPeer(netip.Addr) (netlink.Link, error) + + // Del deletes tunnel for the peer, if any. + DelPeer(netip.Addr) error +} + +var ErrIPFamilyMismatch = errors.New("no matching IP family") diff --git a/internal/util/netiputil/netiputil.go b/internal/util/netiputil/netiputil.go new file mode 100644 index 0000000..5f78955 --- /dev/null +++ b/internal/util/netiputil/netiputil.go @@ -0,0 +1,10 @@ +package netiputil + +import ( + "net" + "net/netip" +) + +func ConvNetIP(addr netip.Addr) net.IP { + return net.IP(addr.AsSlice()) +}