E2e test scaffolding

Makefile

@@ -1,6 +1,10 @@
+REPO?=quay.io/quay/dba-operator
+TAG?=$(shell git rev-parse --short HEAD)
 
 # Image URL to use all building/pushing image targets
-IMG ?= controller:latest
+IMG ?= ${REPO}:${TAG}
+DB_IMG ?= mysql/mysql-server:latest
+
 # Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
 CRD_OPTIONS ?= "crd:trivialVersions=true"
 
@@ -17,6 +21,17 @@ all: manager
 test: generate fmt vet manifests
 	go test ./api/... ./controllers/... -coverprofile cover.out
 
+# Run go test: e2e tests
+.PHONY: test-e2e-local
+test-e2e-local: KUBECONFIG?=$(HOME)/.kube/config
+test-e2e-local:
+	go test ./test/e2e/... --kubeconfig=${KUBECONFIG} --operator-image=${REPO}:${TAG} --db-image=${DB_IMG}
+
+# Run e2e tests: Setup/teardown minikube, build image and go test
+.PHONY: run-e2e-local
+run-e2e-local:
+	REPO=${REPO} TAG=${TAG} ./scripts/run-e2e-local.sh
+
 # Build manager binary
 manager: generate fmt vet
 	go build -o bin/manager main.go

api/v1alpha1/databasemigration_types.go

@@ -43,6 +43,7 @@ type DatabaseMigrationStatus struct {
 // +kubebuilder:object:root=true
 
 // DatabaseMigration is the Schema for the databasemigrations API
+// +kubebuilder:resource:scope="Namespaced"
 type DatabaseMigration struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty"`

api/v1alpha1/manageddatabase_types.go

@@ -53,6 +53,7 @@ type ManagedDatabaseStatus struct {
 // +kubebuilder:object:root=true
 
 // ManagedDatabase is the Schema for the manageddatabases API
+// +kubebuilder:resource:scope="Namespaced"
 type ManagedDatabase struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty"`

scripts/minikube.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+set -e
+set -o pipefail
+set -u
+
+export MINIKUBE_VERSION=${MINIKUBE_VERSION:-v1.2.0}
+export KUBERNETES_VERSION=${KUBERNETES_VERSION:-v1.15.2}
+export KUBECONFIG=${KUBECONFIG:-$HOME/.kube/config}
+
+export REPO=${REPO:-quay.io/quay/dba-operator}
+export TAG=${TAG:-$(git rev-parse --short HEAD)}
+
+
+kubectl > /dev/null || { curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/$KUBERNETES_VERSION/bin/linux/amd64/kubectl && \
+        chmod +x kubectl &&  \
+        sudo mv kubectl /usr/local/bin/ ;}
+
+minikube > /dev/null || { curl -Lo minikube https://storage.googleapis.com/minikube/releases/$MINIKUBE_VERSION/minikube-linux-amd64 && \
+        chmod +x minikube && \
+        sudo mv minikube /usr/local/bin/ ;}
+
+
+# Create a minikube instance
+minikube_create() {
+    minikube version | awk '{ print $3 }' | grep -q "$MINIKUBE_VERSION" || { echo "Wrong minikube version"; exit 1; }
+
+    pgrep -f "[m]inikube" >/dev/null || minikube start --cpus 2 --memory 2048 --kubernetes-version="$KUBERNETES_VERSION" || { echo 'Cannot start minikube.'; exit 1; }
+
+    kubectl version --output json | jq .serverVersion.gitVersion | grep -q "$KUBERNETES_VERSION" && \
+        kubectl version --output json | jq .clientVersion.gitVersion | grep -q "$KUBERNETES_VERSION" || { echo "Kubectl version does not match k8s version"; exit 1; }
+
+    kubectl config use-context minikube
+}
+
+# Delete the minikube instance
+minikube_delete() {
+    minikube delete
+}
+
+# Build the operator image in the minikube's docker context
+build_image() {
+    eval "$(minikube docker-env)" || { echo 'Cannot switch to minikube docker'; exit 1; }
+    GOOS=linux GOARCH=amd64 make manager
+    docker build -f Dockerfile.e2e -t $REPO:$TAG .
+}
+
+
+if [ $1 == "create-minikube" ]; then
+    minikube_create
+elif [ $1 == "delete-minikube" ]; then
+    minikube_delete
+elif [ $1 == "build-image" ]; then
+    build_image
+else
+    echo "Specify create-minikube, delete-minikube, or build-image"
+    exit 1
+fi

scripts/run-e2e-local.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+set -e
+set -o pipefail
+set -u
+set -x
+
+
+export REPO=${REPO:-quay.io/quay/dba-operator}
+export TAG=${TAG:-$(git rev-parse --short HEAD)}
+
+SCRIPTS_DIR=$(dirname "${BASH_SOURCE[0]}")
+
+
+"${SCRIPTS_DIR}"/minikube.sh create-minikube
+
+"${SCRIPTS_DIR}"/minikube.sh build-image
+
+make install || true # Install the genereated CRDs manifest in the cluster
+make test-e2e-local # Run the tests
+
+"${SCRIPTS_DIR}"/minikube.sh delete-minikube

test/e2e/framework_test.go

@@ -0,0 +1,138 @@
+package e2e
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"golang.org/x/sync/errgroup"
+
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/config"
+
+	"github.com/app-sre/dba-operator/pkg/dbadmin"
+)
+
+var ()
+
+type finalizerFn func() error
+
+type TestFramework struct {
+	client.Client
+	admin dbadmin.DbAdmin
+}
+
+func New(operatorImage string) (*TestFramework, error) {
+	c, err := client.New(config.GetConfigOrDie(), client.Options{})
+	if err != nil {
+		return nil, nil
+	}
+
+	return &TestFramework{
+		Client: c,
+	}, nil
+}
+
+// Create a new test context with the test's name and the current time as its ID.
+// The ID will be used for creating objects, such as creating a namespace for the test.
+func (tf *TestFramework) NewTestCtx(t *testing.T) TestCtx {
+	prefix := strings.TrimPrefix(
+		strings.ReplaceAll(
+			strings.ToLower(t.Name()),
+			"/",
+			"-",
+		),
+		"test",
+	)
+
+	id := prefix + "-" + strconv.FormatInt(time.Now().Unix(), 36)
+	return TestCtx{
+		ID: id,
+	}
+}
+
+type TestCtx struct {
+	ID         string
+	cleanupFns []finalizerFn
+	ctx        context.Context
+}
+
+func (tctx *TestCtx) Cleanup(t *testing.T) {
+	var eg errgroup.Group
+
+	for i := len(tctx.cleanupFns) - 1; i >= 0; i-- {
+		eg.Go(tctx.cleanupFns[i])
+	}
+
+	if err := eg.Wait(); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func (tctx *TestCtx) AddFinalizerFn(fn finalizerFn) {
+	tctx.cleanupFns = append(tctx.cleanupFns, fn)
+}
+
+func (tctx *TestCtx) CreateNamespace(t *testing.T, c client.Client) string {
+	name := tctx.ID
+	if err := CreateNamespace(tctx.ctx, c, name); err != nil {
+		t.Fatal(err)
+	}
+
+	namespaceCleanupFn := func() error {
+		return DeleteNamespace(tctx.ctx, c, name)
+	}
+
+	tctx.AddFinalizerFn(namespaceCleanupFn)
+
+	return name
+}
+
+func (tctx *TestCtx) CreateDbaOperator(c client.Client, operatorImage string) error {
+	deployment, err := MakeDeployment("../../deploy/dba-operator.yaml")
+	if err != nil {
+		return err
+	}
+
+	if operatorImage != "" {
+		repoTag := strings.Split(operatorImage, ":")
+		if len(repoTag) != 2 {
+			return fmt.Errorf("invalid operator image '%s'", operatorImage)
+		}
+
+		deployment.Spec.Template.Spec.Containers[0].Image = operatorImage
+	}
+
+	err = CreateDeployment(tctx.ctx, c, tctx.ID, deployment)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (tctx *TestCtx) CreateDB(c client.Client, image string) error {
+	deployment, err := MakeDeployment("../../deploy/debug.yaml")
+	if err != nil {
+		return err
+	}
+
+	if image != "" {
+		repoTag := strings.Split(image, ":")
+		if len(repoTag) != 2 {
+			return fmt.Errorf("invalid operator image '%s'", image)
+		}
+
+		deployment.Spec.Template.Spec.Containers[0].Image = image
+	}
+
+	err = CreateDeployment(tctx.ctx, c, tctx.ID, deployment)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

test/e2e/migration_test.go

@@ -0,0 +1,22 @@
+package e2e
+
+import "testing"
+
+func TestBasicMigration(t *testing.T) {
+	ctx := testFramework.NewTestCtx(t)
+	// defer ctx.Cleanup(t)
+
+	ctx.CreateNamespace(t, testFramework.Client)
+
+	err := ctx.CreateDB(testFramework.Client, *dbImage)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = ctx.CreateDbaOperator(testFramework.Client, *operatorImage)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// TODO
+}

test/e2e/setup_test.go

@@ -0,0 +1,37 @@
+package e2e
+
+import (
+	"flag"
+	"log"
+	"os"
+	"testing"
+)
+
+var (
+	testFramework *TestFramework
+	operatorImage *string
+	dbImage       *string
+)
+
+func TestMain(m *testing.M) {
+	operatorImage = flag.String(
+		"operator-image",
+		"",
+		"operator image, e.g. quay.io/quay/dba-operator:v1.0.0",
+	)
+	dbImage = flag.String(
+		"db-image",
+		"",
+		"db image, e.g. mysql/mysql-server:latest",
+	)
+	flag.Parse()
+
+	var err error
+	if testFramework, err = New(*operatorImage); err != nil {
+		log.Printf("failed to setup testFramework: %s", err)
+		os.Exit(1)
+	}
+
+	exitCode := m.Run()
+	os.Exit(exitCode)
+}