Define pr and nightly tests (FoundationDB#1635)

* Define a set of tests that should run per pr and on a nightly basis

* Update examples on how to run a subset of tests

Author: johscheuer

e2e/Makefile

@@ -32,6 +32,9 @@ FEATURE_LOCALITIES?=false
 # Make bash pickier about errors.
 SHELL=/bin/bash -euo pipefail
 
+# Defines a variable that has the directory of this Makefile as value.
+BASE_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
+
 clean:
 	@rm -f TEST-ginkgo*.xml
 	@find . -name '*~' | xargs rm -f
@@ -60,7 +63,7 @@ kind-update-operator:
 # This makefile reimplements that, and works around a few bugs in ginkgo itself.
 ifndef QUIET
   VERBOSE=-v
-  GINKGO_VERBOSE=-ginkgo.v
+  GINKGO_VERBOSE=--ginkgo.v
 else
   VERBOSE=
   GINKGO_VERBOSE=
@@ -78,9 +81,26 @@ TARGETS=$(patsubst test_%,test_%.run,${SUITES})
 
 run: ${TARGETS}
 
+# This variable can be used to define any label-filter for ginkgo to run a subset of tests.
+# For more information see the Ginkgo Spec labels documentation: https://onsi.github.io/ginkgo/#spec-labels.
+GINKGO_LABEL_FILTER?=
+
+# Only run tests that are labeled with the "pr" label.
+pr-tests: GINKGO_LABEL_FILTER=--ginkgo.label-filter="pr"
+
+# Run the actual pr tests.
+pr-tests: run
+
+# Only run tests that are labeled with the "pr" or the "nightly" label.
+nightly-tests: GINKGO_LABEL_FILTER=--ginkgo.label-filter="pr || nightly"
+
+# Run the actual nightly tests
+nightly-tests: run
+
 %.run: %
 	go test -timeout=$(TIMEOUT) $(VERBOSE) ./$< \
 	${NO_COLOR} ${GINKGO_VERBOSE} \
+	  $(GINKGO_LABEL_FILTER) \
 	  --ginkgo.junit-report=$@.xml \
 	  --ginkgo.timeout=$(TIMEOUT) \
 	  --timeout=$(TIMEOUT) \
@@ -100,4 +120,4 @@ run: ${TARGETS}
 	  --feature-unified-image=$(FEATURE_UNIFIED_IMAGE) \
 	  --feature-localities=$(FEATURE_LOCALITIES) \
 	  --feature-dns=$(FEATURE_DNS) \
-	  | grep -v 'constructing many client instances from the same exec auth config can cause performance problems during cert rotation'
+	  | grep -v 'constructing many client instances from the same exec auth config can cause performance problems during cert rotation' &> $(BASE_DIR)/../logs/$<.log

e2e/README.md

@@ -8,13 +8,13 @@ Every test suite has a head in the `*_test.go` file that describes the test case
 The following command will run all the operator related tests with the default values:
 
 ```bash
-make -C e2e -kj run
+make -kj -C e2e run
 ```
 
 You can also run a specific test suite by providing the name of the test suite:
 
 ```bash
-make -C e2e -kj test_operator.run
+make -C e2e test_operator.run
 ```
 
 Every test suite will create at least one namespace, HA cluster tests will create all the required namespaces.
@@ -25,7 +25,7 @@ The e2e tests assume that at least one `StorageClass` is present in the target K
 You can provide the targeted `StorageClass` as an environment variable:
 
 ```bash
-STORAGE_CLASS='my-fancy-storage' make -C e2e -kj test_operator.run
+STORAGE_CLASS='my-fancy-storage' make -kj -C e2e test_operator.run
 ```
 
 If the `STORAGE_CLASS` is not set, the operator will take the default `StorageClass` in this cluster.
@@ -67,3 +67,45 @@ You just can rebuild the operator image and push the new image inside the kind c
 ```bash
 make -C e2e kind-update-operator
 ```
+
+## What is tested
+
+The following section will cover which test suites are run when.
+
+### Test running for PRs
+
+The following tests will be running for each PR:
+
+- [test_operator](./test_operator)
+- [test_operator_upgrades](./test_operator_upgrades)
+
+Those are all test suites labeled with the `pr` label.
+You can run those test suites with: `make -kj -C e2e pr-tests`
+
+### Tests running regularly
+
+The following tests will be running on a nightly base:
+
+- [test_operator](./test_operator)
+- [test_operator_upgrades](./test_operator_upgrades)
+- [test_operator_ha_upgrades](./test_operator_ha_upgrades)
+- [test_operator_velocity](./test_operator_velocity)
+
+Those are all test suites labeled with the `pr` or the `nightly` label.
+You can run those test suites with: `make -kj -C e2e nightly-tests`
+
+### Tests that are not run regularly
+
+Those tests are only run manually:
+
+- [test_operator_creation_velocity](./test_operator_creation_velocity)
+- [test_operator_stress](./test_operator_stress)
+
+e.g. these tests will be used to qualify a new release.
+You can run all tests with `make -kj -C e2e run`
+
+## How to debug failed PR tests
+
+All test suited will be logging to the `logs` folder in the root of this repository.
+If you want to see the current state of a running test you can use `tail`, e.g. `tail -f ./logs/test_operator.log`, to see the progress of the operator tests, the command assumes you are running it from the project directory.
+All tests that are started by our CI pipelines will report in the PR with the test status.

e2e/test_operator/operator_test.go

@@ -106,7 +106,7 @@ var _ = AfterSuite(func() {
 	factory.Shutdown()
 })
 
-var _ = Describe("Operator", Label("e2e"), func() {
+var _ = Describe("Operator", Label("e2e", "pr"), func() {
 	var availabilityCheck bool
 
 	AfterEach(func() {

e2e/test_operator_ha_upgrades/operator_ha_upgrade_test.go

@@ -139,7 +139,7 @@ func verifyBouncingIsBlocked() {
 	}).WithTimeout(10 * time.Minute).WithPolling(5 * time.Second).MustPassRepeatedly(30).Should(BeTrue())
 }
 
-var _ = Describe("Operator HA Upgrades", Label("e2e"), func() {
+var _ = Describe("Operator HA Upgrades", Label("e2e", "nightly"), func() {
 	BeforeEach(func() {
 		factory = fixtures.CreateFactory(testOptions)
 	})

e2e/test_operator_upgrades/operator_upgrades_test.go

@@ -96,7 +96,7 @@ func upgradeAndVerify(cluster *fixtures.FdbCluster, expectedVersion string) {
 	log.Println("Upgrade took:", time.Since(startTime).String())
 }
 
-var _ = Describe("Operator Upgrades", Label("e2e"), func() {
+var _ = Describe("Operator Upgrades", Label("e2e", "pr"), func() {
 	BeforeEach(func() {
 		factory = fixtures.CreateFactory(testOptions)
 	})

e2e/test_operator_velocity/operator_velocity_test.go

@@ -161,7 +161,7 @@ var _ = AfterSuite(func() {
 	factory.Shutdown()
 })
 
-var _ = Describe("Test Operator Velocity", Label("e2e"), func() {
+var _ = Describe("Test Operator Velocity", Label("e2e", "nightly"), func() {
 	BeforeEach(func() {
 		Expect(fdbCluster.WaitForReconciliation()).ToNot(HaveOccurred())
 	})

logs/.gitignore

@@ -0,0 +1 @@
+*.log