Merge pull request #26 from aojea/docker2

Run CCM inside a container

Author: aojea

Dockerfile

@@ -8,6 +8,6 @@ COPY . .
 RUN make build
 
 # build real cloud-provider-kind image
-FROM gcr.io/distroless/static-debian12
+FROM docker:25.0-dind
 COPY --from=0 --chown=root:root ./go/src/bin/cloud-provider-kind /bin/cloud-provider-kind
 ENTRYPOINT ["/bin/cloud-provider-kind"]

README.md

@@ -35,6 +35,17 @@ cd cloud-provider-kind && make
 sudo mv ./bin/cloud-provider-kind  /usr/local/bin/cloud-provider-kind
 ```
 
+Another alternative is to run it as a container, but this will require to mount
+the docker socket inside the container:
+
+```sh
+docker build . -t aojea/cloud-provider-kind:v0.1
+# using the host network
+docker run --rm --network host -v /var/run/docker.sock:/var/run/docker.sock aojea/cloud-provider-kind:v0.1
+# or the kind network
+docker run --rm --network kind  -v /var/run/docker.sock:/var/run/docker.sock aojea/cloud-provider-kind:v0.1
+```
+
 ## How to use it
 
 Run a KIND cluster:

pkg/controller/controller.go

@@ -2,6 +2,9 @@ package controller
 
 import (
 	"context"
+	"crypto/tls"
+	"fmt"
+	"io"
 	"net/http"
 	"time"
 
@@ -47,20 +50,10 @@ func New(logger log.Logger) *Controller {
 }
 
 func (c *Controller) Run(ctx context.Context) {
+	defer cleanup()
 	for {
 		select {
 		case <-ctx.Done():
-			// cleanup
-			containers, err := container.ListByLabel(constants.NodeCCMLabelKey)
-			if err != nil {
-				klog.Errorf("can't list containers: %v", err)
-				return
-			}
-			for _, id := range containers {
-				if err := container.Delete(id); err != nil {
-					klog.Errorf("can't delete container %s: %v", id, err)
-				}
-			}
 			return
 		default:
 		}
@@ -72,27 +65,20 @@ func (c *Controller) Run(ctx context.Context) {
 
 		// add new ones
 		for _, cluster := range clusters {
+			select {
+			case <-ctx.Done():
+				return
+			default:
+			}
+
 			klog.V(3).Infof("processing cluster %s", cluster)
 			_, ok := c.clusters[cluster]
 			if ok {
 				klog.V(3).Infof("cluster %s already exist", cluster)
 				continue
 			}
 
-			// get kubeconfig
-			kconfig, err := c.kind.KubeConfig(cluster, false)
-			if err != nil {
-				klog.Errorf("Failed to get kubeconfig for cluster %s: %v", cluster, err)
-				continue
-			}
-
-			config, err := clientcmd.RESTConfigFromKubeConfig([]byte(kconfig))
-			if err != nil {
-				klog.Errorf("Failed to convert kubeconfig for cluster %s: %v", cluster, err)
-				continue
-			}
-
-			kubeClient, err := kubernetes.NewForConfig(config)
+			kubeClient, err := c.getKubeClient(ctx, cluster)
 			if err != nil {
 				klog.Errorf("Failed to create kubeClient for cluster %s: %v", cluster, err)
 				continue
@@ -122,6 +108,75 @@ func (c *Controller) Run(ctx context.Context) {
 	}
 }
 
+// getKubeClient returns a kubeclient depending if the ccm runs inside a container
+// inside the same docker network that the kind cluster or run externally in the host
+// It tries first to connect to the external endpoint
+func (c *Controller) getKubeClient(ctx context.Context, cluster string) (kubernetes.Interface, error) {
+	httpClient := &http.Client{
+		Timeout: 5 * time.Second,
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		},
+	}
+	// try internal first
+	for _, internal := range []bool{false, true} {
+		kconfig, err := c.kind.KubeConfig(cluster, internal)
+		if err != nil {
+			klog.Errorf("Failed to get kubeconfig for cluster %s: %v", cluster, err)
+			continue
+		}
+
+		config, err := clientcmd.RESTConfigFromKubeConfig([]byte(kconfig))
+		if err != nil {
+			klog.Errorf("Failed to convert kubeconfig for cluster %s: %v", cluster, err)
+			continue
+		}
+
+		// check that the apiserver is reachable before continue
+		// to fail fast and avoid waiting until the client operations timeout
+		var ok bool
+		for i := 0; i < 5; i++ {
+			select {
+			case <-ctx.Done():
+				return nil, ctx.Err()
+			default:
+			}
+			if probeHTTP(httpClient, config.Host) {
+				ok = true
+				break
+			}
+			time.Sleep(time.Second * time.Duration(i))
+		}
+		if !ok {
+			klog.Errorf("Failed to connect to apiserver %s: %v", cluster, err)
+			continue
+		}
+
+		kubeClient, err := kubernetes.NewForConfig(config)
+		if err != nil {
+			klog.Errorf("Failed to create kubeClient for cluster %s: %v", cluster, err)
+			continue
+		}
+		return kubeClient, err
+	}
+	return nil, fmt.Errorf("can not find a working kubernetes clientset")
+}
+
+func probeHTTP(client *http.Client, address string) bool {
+	klog.Infof("probe HTTP address %s", address)
+	resp, err := client.Get(address)
+	if err != nil {
+		klog.Infof("Failed to connect to HTTP address %s: %v", address, err)
+		return false
+	}
+	defer resp.Body.Close()
+	// drain the body
+	io.ReadAll(resp.Body) // nolint:errcheck
+	// we only want to verify connectivity so don't need to check the http status code
+	// as the apiserver may not be ready
+	return true
+}
+
 // TODO: implement leader election to not have problems with  multiple providers
 // ref: https://github.com/kubernetes/kubernetes/blob/d97ea0f705847f90740cac3bc3dd8f6a4026d0b5/cmd/kube-scheduler/app/server.go#L211
 func startCloudControllerManager(ctx context.Context, clusterName string, kubeClient kubernetes.Interface, cloud cloudprovider.Interface) (*ccm, error) {
@@ -194,3 +249,16 @@ func startCloudControllerManager(ctx context.Context, clusterName string, kubeCl
 		nodeController:    nodeController,
 		cancelFn:          cancel}, nil
 }
+
+func cleanup() {
+	containers, err := container.ListByLabel(constants.NodeCCMLabelKey)
+	if err != nil {
+		klog.Errorf("can't list containers: %v", err)
+		return
+	}
+	for _, id := range containers {
+		if err := container.Delete(id); err != nil {
+			klog.Errorf("can't delete container %s: %v", id, err)
+		}
+	}
+}

pkg/loadbalancer/proxy.go

@@ -17,7 +17,7 @@ import (
 )
 
 // proxyImage defines the loadbalancer image:tag
-const proxyImage = "kindest/haproxy:v20230330-2f738c2"
+const proxyImage = "docker.io/kindest/haproxy:v20230606-42a2262b"
 
 // proxyConfigPath defines the path to the config file in the image
 const proxyConfigPath = "/usr/local/etc/haproxy/haproxy.cfg"