feat: repository cloner

This feature allows users to specify a list of repositories to clone when starting their environment.

Each repository can come from a different provider and use different protocols (e.g. https, git).

Username + password or certificate + unlock password can be used to clone private repositories.

---------

Co-authored-by: Tasko Olevski <[email protected]>

Author: sgaist

api/v1alpha1/amaltheasession_children.go

@@ -86,6 +86,16 @@ func (cr *AmaltheaSession) StatefulSet() appsv1.StatefulSet {
 		)
 	}
 
+	initContainers := []v1.Container{}
+
+	if len(cr.Spec.CodeRepositories) > 0 {
+		gitCloneContainers, gitCloneVols := cr.initClones()
+		initContainers = append(initContainers, gitCloneContainers...)
+		volumes = append(volumes, gitCloneVols...)
+	}
+
+	initContainers = append(initContainers, cr.Spec.ExtraInitContainers...)
+
 	// NOTE: ports on a container are for information purposes only, so they are removed because the port specified
 	// in the CR can point to either the session container or another container.
 	sessionContainer := v1.Container{
@@ -197,7 +207,7 @@ func (cr *AmaltheaSession) StatefulSet() appsv1.StatefulSet {
 				},
 				Spec: v1.PodSpec{
 					Containers:     containers,
-					InitContainers: cr.Spec.ExtraInitContainers,
+					InitContainers: initContainers,
 					Volumes:        volumes,
 				},
 			},
@@ -354,6 +364,51 @@ func (cr *AmaltheaSession) Pod(ctx context.Context, clnt client.Client) (*v1.Pod
 	return &pod, err
 }
 
+// Generates the init containers that clones the specified Git repositories
+func (cr *AmaltheaSession) initClones() ([]v1.Container, []v1.Volume) {
+	envVars := []v1.EnvVar{}
+	volMounts := []v1.VolumeMount{{Name: sessionVolumeName, MountPath: cr.Spec.Session.Storage.MountPath}}
+	vols := []v1.Volume{}
+	containers := []v1.Container{}
+
+	for irepo, repo := range cr.Spec.CodeRepositories {
+		args := []string{"clone", "--remote", repo.Remote, "--path", cr.Spec.Session.Storage.MountPath + "/" + repo.ClonePath}
+
+		if repo.CloningConfigSecretRef != nil {
+			secretVolName := fmt.Sprintf("git-clone-cred-volume-%d", irepo)
+			secretMountPath := "/git-clone-secrets"
+			secretFilePath := fmt.Sprintf("%s/%s", secretMountPath, repo.CloningConfigSecretRef.Key)
+			vols = append(
+				vols,
+				v1.Volume{
+					Name:         secretVolName,
+					VolumeSource: v1.VolumeSource{Secret: &v1.SecretVolumeSource{SecretName: repo.CloningConfigSecretRef.Name}},
+				},
+			)
+			volMounts = append(volMounts, v1.VolumeMount{Name: secretVolName, MountPath: secretMountPath})
+
+			args = append(args, []string{"--config", secretFilePath}...)
+		}
+
+		if repo.Revision != "" {
+			args = append(args, []string{"--revision", repo.Revision}...)
+		}
+
+		gitCloneContainerName := fmt.Sprintf("git-clone-%d", irepo)
+		containers = append(containers, v1.Container{
+			Name:            gitCloneContainerName,
+			Image:           "renku/cloner:0.0.1",
+			VolumeMounts:    volMounts,
+			WorkingDir:      cr.Spec.Session.Storage.MountPath,
+			Env:             envVars,
+			SecurityContext: &v1.SecurityContext{RunAsUser: &cr.Spec.Session.RunAsUser, RunAsGroup: &cr.Spec.Session.RunAsGroup},
+			Args:            args,
+		})
+	}
+
+	return containers, vols
+}
+
 // Returns the list of all the secrets used in this CR
 func (cr *AmaltheaSession) AllSecrets() v1.SecretList {
 	secrets := v1.SecretList{}

api/v1alpha1/amaltheasession_types.go

@@ -31,6 +31,7 @@ type AmaltheaSessionSpec struct {
 	Session Session `json:"session"`
 
 	// +optional
+	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="CodeRepositories is immutable"
 	// A list of code repositories and associated configuration that will be cloned in the session
 	CodeRepositories []CodeRepository `json:"codeRepositories,omitempty"`
 
@@ -160,13 +161,16 @@ type CodeRepository struct {
 	// The tag, branch or commit SHA to checkout, if omitted then will be the tip of the default branch of the repo
 	Revision string `json:"revision,omitempty"`
 	// The Kubernetes secret that contains the code repository configuration to be used during cloning.
-	// For 'git' this is the git configuration which can be used to inject credentials in addition to any other repo-specific Git configuration.
+	// For 'git' this should contain either:
+	// The username and password
+	// The private key and its corresponding password
+	// An empty value can be used when cloning from public repositories using the http protocol
 	// NOTE: you have to specify the whole config in a single key in the secret.
-	CloningConfigSecretRef *SessionSecretRef `json:"cloningGitConfigSecretRef,omitempty"`
+	CloningConfigSecretRef *SessionSecretRef `json:"cloningConfigSecretRef,omitempty"`
 	// The Kubernetes secret that contains the code repository configuration to be used when the session is running.
 	// For 'git' this is the git configuration which can be used to inject credentials in addition to any other repo-specific Git configuration.
 	// NOTE: you have to specify the whole config in a single key in the secret.
-	ConfigSecretRef *SessionSecretRef `json:"gitConfigSecretRef,omitempty"`
+	ConfigSecretRef *SessionSecretRef `json:"configSecretRef,omitempty"`
 }
 
 // +kubebuilder:validation:Enum={rclone}

cloner/.gitignore

@@ -0,0 +1,42 @@
+#
+# Copyright 2024.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+
+# Test binary, built with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Dependency directories (remove the comment below to include it)
+# vendor/
+
+# Go workspace file
+go.work
+go.work.sum
+
+# env file
+.env
+
+# Output folder
+bin/

cloner/Dockerfile

@@ -0,0 +1,27 @@
+# Build the application from source
+FROM golang:1.22 AS build-stage
+
+WORKDIR /app
+
+COPY go.mod go.sum ./
+RUN go mod download
+
+COPY *.go ./
+COPY cmd ./cmd
+
+RUN CGO_ENABLED=0 GOOS=linux go build -o /cloner
+
+# Run the tests in the container
+FROM build-stage AS run-test-stage
+RUN go test -v ./...
+
+# Deploy the application binary into a lean image
+FROM gcr.io/distroless/base-debian12 AS build-release-stage
+
+WORKDIR /
+
+COPY --from=build-stage /cloner /cloner
+
+USER nonroot:nonroot
+
+ENTRYPOINT ["/cloner"]

cloner/Makefile

@@ -0,0 +1,122 @@
+#
+# Copyright 2024.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# VERSION defines the project version for the bundle.
+# Update this value when you upgrade the version of your project.
+# To re-generate a bundle for another specific version without changing the standard setup, you can:
+# - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2)
+# - use environment variables to overwrite this value (e.g export VERSION=0.0.2)
+VERSION ?= 0.0.1
+LDFLAGS="-X 'cloner/cmd.Version=v$(VERSION)'"
+
+# IMAGE_TAG_BASE defines the docker.io namespace and part of the image name for remote images.
+# This variable is used to construct full image tags for bundle and catalog images.
+#
+# For example, running 'make bundle-build bundle-push catalog-build catalog-push' will build and push both
+# amalthea.dev/amalthea-bundle:$VERSION and amalthea.dev/amalthea-catalog:$VERSION.
+IMAGE_TAG_BASE ?= renku/cloner
+IMG ?= $(IMAGE_TAG_BASE):$(VERSION)
+
+# CONTAINER_TOOL defines the container tool to be used for building images.
+# Be aware that the target commands are only tested with Docker which is
+# scaffolded by default. However, you might want to replace it to use other
+# tools. (i.e. podman)
+CONTAINER_TOOL ?= docker
+
+# Setting SHELL to bash allows bash commands to be executed by recipes.
+# Options are set to exit when a recipe line exits non-zero or a piped command fails.
+SHELL = /usr/bin/env bash -o pipefail
+.SHELLFLAGS = -ec
+
+.PHONY: help
+help: ## Display this help.
+	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
+
+.PHONY: all
+all: mod test vet fmt build run
+
+##@ Run
+
+.PHONY:
+run: fmt vet build run ## Run the proxy
+	chmod +x bin/cloner
+	./bin/cloner
+
+.PHONY: install
+install: run ## Install the proxy
+	go install -v ./...
+
+##@ QA
+
+.PHONY: audit
+audit: ## Run quality control checks
+	go mod verify
+	go vet ./...
+	go run honnef.co/go/tools/cmd/staticcheck@latest -checks=all,-ST1000,-U1000 ./...
+	go run golang.org/x/vuln/cmd/govulncheck@latest ./...
+
+.PHONY: test
+test: build ## Run tests
+	go test $$(go list ./... | grep -v /e2e) -race -buildvcs
+
+.PHONY: vet
+vet: ## Run go vet against code.
+	go vet ./...
+
+.PHONY: fmt
+fmt: ## Run go fmt against code.
+	go fmt ./...
+
+.PHONY: mod
+mod: ## Tidy mods
+	go mod tidy
+
+.PHONY: code-cleanup
+code-cleanup: vet fmt mod ## Code cleanup
+
+##@ Build
+
+.PHONY: build
+build: fmt vet
+	go build -ldflags=$(LDFLAGS) -o bin/cloner main.go
+
+# If you wish to build the manager image targeting other platforms you can use the --platform flag.
+# (i.e. docker build --platform linux/arm64). However, you must enable docker buildKit for it.
+# More info: https://docs.docker.com/develop/develop-images/build_enhancements/
+.PHONY: docker-build
+docker-build: ## Build docker image with the manager.
+	$(CONTAINER_TOOL) build -t ${IMG} .
+
+.PHONY: docker-push
+docker-push: ## Push docker image with the manager.
+	$(CONTAINER_TOOL) push ${IMG}
+
+# PLATFORMS defines the target platforms for the manager image be built to provide support to multiple
+# architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to:
+# - be able to use docker buildx. More info: https://docs.docker.com/build/buildx/
+# - have enabled BuildKit. More info: https://docs.docker.com/develop/develop-images/build_enhancements/
+# - be able to push the image to your registry (i.e. if you do not set a valid value via IMG=<myregistry/image:<tag>> then the export will fail)
+# To adequately provide solutions that are compatible with multiple platforms, you should consider using this option.
+PLATFORMS ?= linux/arm64,linux/amd64,linux/s390x,linux/ppc64le
+.PHONY: docker-buildx
+docker-buildx: ## Build and push docker image for the manager for cross-platform support
+	# copy existing Dockerfile and insert --platform=${BUILDPLATFORM} into Dockerfile.cross, and preserve the original Dockerfile
+	sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross
+	- $(CONTAINER_TOOL) buildx create --name project-v3-builder
+	$(CONTAINER_TOOL) buildx use project-v3-builder
+	- $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile.cross .
+	- $(CONTAINER_TOOL) buildx rm project-v3-builder
+	rm Dockerfile.cross