diff --git a/.github/scripts/js/ci.js b/.github/scripts/js/ci.js index 91c11372f7..261e9eaa03 100644 --- a/.github/scripts/js/ci.js +++ b/.github/scripts/js/ci.js @@ -65,6 +65,29 @@ const reactToComment = async ({github, context, comment_id, content}) => { }); }; module.exports.reactToComment = reactToComment; + +const checkUserClusterLabel = async ({prLabels, userClusterLabels}) => { + const userLabelsInPR = prLabels + .map(label => label.name) + .filter(labelName => userClusterLabels[labelName]); + return userLabelsInPR; +}; +module.exports.checkUserClusterLabel = checkUserClusterLabel; + +const getClusterUser = async ({context, core, userClusterLabels}) => { + const prLabels = context.payload.pull_request.labels; + let userLabelsInPR = await checkUserClusterLabel({prLabels, userClusterLabels}); + if (userLabelsInPR.length === 0) { + core.info('No user labels found in PR, using PR author\'s cluster'); + const prAuthorId = context.payload.pull_request.user.id; + core.info(`PR author: ${prAuthorId}`); + return prAuthorId.toString(); + } else if (userLabelsInPR.length > 1) { + return core.setFailed(`Error: PR has multiple user labels: ${userLabelsInPR.join(', ')}`); + } + return userClusterLabels[userLabelsInPR].id +}; +module.exports.getClusterUser = getClusterUser; /** * Start workflow using workflow_dispatch event. diff --git a/.github/scripts/js/constants.js b/.github/scripts/js/constants.js new file mode 100644 index 0000000000..7515864230 --- /dev/null +++ b/.github/scripts/js/constants.js @@ -0,0 +1,25 @@ +// Copyright 2022 Flant JSC +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//@ts-check + +const skipE2eLabel = 'skip/e2e'; +module.exports.skipE2eLabel = skipE2eLabel; + +// Labels available for pull requests. +const labels = { + // E2E + 'e2e/run': { type: 'e2e-run', provider: 'static' }, + // Allow running workflows for external PRs. + 'status/ok-to-test': { type: 'ok-to-test' }, + +}; +module.exports.knownLabels = labels; diff --git a/.github/scripts/js/e2e-commit-status.js b/.github/scripts/js/e2e-commit-status.js new file mode 100644 index 0000000000..41464b4f80 --- /dev/null +++ b/.github/scripts/js/e2e-commit-status.js @@ -0,0 +1,264 @@ +// Copyright 2022 Flant JSC +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +const { + sleep +} = require('./time'); + +/** + * Build workflow run url for add it to commit status target url + * @param {object} inputs + * @param {object} inputs.core - A reference to the '@actions/core' package. + * @param {object} inputs.context - A reference to context https://github.com/actions/toolkit/blob/main/packages/github/src/context.ts#L6 + * @returns string + */ +function workflowUrl({core, context}) { + core.debug(`workflowUrl context: ${JSON.stringify(context)}`); + const {serverUrl, repo, runId} = context; + const repository = repo.repo; + const owner = repo.owner; + const url = `${serverUrl}/${owner}/${repository}/actions/runs/${runId}`; + core.debug(`workflowUrl url: ${url}`); + return url +} + +/** + * Wrap github.rest.repos.createCommitStatus. Returns true if status was set + * Use STATUS_TARGET_COMMIT env var as target commit sha + * @param {object} inputs + * @param {object} inputs.core - A reference to the '@actions/core' package. + * @param {object} inputs.github - A pre-authenticated octokit/rest.js client with pagination plugins. + * @param {object} inputs.context - A reference to context https://github.com/actions/toolkit/blob/main/packages/github/src/context.ts#L6 + * @param {object} inputs.status - A state object for send + * @param {string} inputs.status.state - A state type as 'success' (it is mark in GitHub ui) + * @param {string} inputs.status.description - A description for commit status + * @param {string|undefined} inputs.status.url - A target url for commit status (Details link in GitHub ui) + * @param {string} inputs.status.commitSha - A commit for set status + * @returns Promise + */ +async function sendCreateCommitStatus({github, context, core, status}) { + const {state, description, url, commitSha} = status + core.debug(`sendCreateCommitStatus target commit: ${commitSha}`); + + for(let i = 0; i < 3; i++) { + const response = await github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: commitSha, + state: state, + description: description, + target_url: url, + context: 'E2e test' + }); + + core.debug(`rest.repos.createCommitStatus response: ${JSON.stringify(response)}`); + if (response.status === 201) { + core.debug(`rest.repos.createCommitStatus response status is 201. Returns true`); + return true; + } + + // wait 3s for retry request + await sleep(3000); + } + + return false +} + +/** + * Set `waiting for start e2e` status (pending) Uses with push commit + * Use STATUS_TARGET_COMMIT env var as target commit sha + * @param {object} inputs + * @param {object} inputs.core - A reference to the '@actions/core' package. + * @param {object} inputs.github - A pre-authenticated octokit/rest.js client with pagination plugins. + * @param {object} inputs.context - A reference to context https://github.com/actions/toolkit/blob/main/packages/github/src/context.ts#L6 + * @param {string} inputs.commitSha - sha commit for set status + * @returns Promise + */ +async function setWait ({github, context, core, commitSha}) { + return sendCreateCommitStatus({ + github, + context, + core, + status: { + commitSha, + state: 'pending', + description: 'Waiting for run e2e test' + } + }) +} + +/** + * Set `e2e was failed` status (failed) when e2e was failed + * Use STATUS_TARGET_COMMIT env var as target commit sha + * @param {object} inputs + * @param {object} inputs.core - A reference to the '@actions/core' package. + * @param {object} inputs.github - A pre-authenticated octokit/rest.js client with pagination plugins. + * @param {object} inputs.context - A reference to context https://github.com/actions/toolkit/blob/main/packages/github/src/context.ts#L6 + * @param {string} inputs.commitSha - sha commit for set status + * @returns Promise + */ +async function setFail({github, context, core, commitSha}){ + return sendCreateCommitStatus({ + github, + context, + core, + status: { + commitSha, + state: 'failure', + description: 'E2e test was failed', + url: workflowUrl({core, context}), + } + }) +} + +/** + * Set `e2e was passed` status (failed) when e2e was failed + * Use STATUS_TARGET_COMMIT env var as target commit sha + * @param {object} inputs + * @param {object} inputs.core - A reference to the '@actions/core' package. + * @param {object} inputs.github - A pre-authenticated octokit/rest.js client with pagination plugins. + * @param {object} inputs.context - A reference to context https://github.com/actions/toolkit/blob/main/packages/github/src/context.ts#L6 + * @param {string} inputs.commitSha - sha commit for set status + * @returns Promise + */ +function setSuccess ({github, context, core, commitSha}) { + return sendCreateCommitStatus({ + github, + context, + core, + status: { + commitSha, + state: 'success', + description: 'E2e test was passed', + url: workflowUrl({core, context}), + } + }) +} + +/** + * Set `e2e was failed` status (success) when e2e was failed + * Unfortunately we do not have 'skip' status and use 'success' + * Use STATUS_TARGET_COMMIT env var as target commit sha + * @param {object} inputs + * @param {object} inputs.core - A reference to the '@actions/core' package. + * @param {object} inputs.github - A pre-authenticated octokit/rest.js client with pagination plugins. + * @param {object} inputs.context - A reference to context https://github.com/actions/toolkit/blob/main/packages/github/src/context.ts#L6 + * @param {string} inputs.commitSha - sha commit for set status + * @returns Promise + */ +async function setSkip({github, context, core, commitSha}){ + return sendCreateCommitStatus({ + github, + context, + core, + status: { + commitSha, + state: 'success', + description: 'E2e test was skipped', + }, + }) +} + +/** + * Set commit status when label set/unset. + * Check label for skipping e2e test and e2e tests should skip set success status + * If status was not set then fail job + * + * Used in build-and-test_dev workflow + * + * Use STATUS_TARGET_COMMIT env var as target commit sha + * @param {object} inputs + * @param {object} inputs.core - A reference to the '@actions/core' package. + * @param {object} inputs.github - A pre-authenticated octokit/rest.js client with pagination plugins. + * @param {object} inputs.context - A reference to context https://github.com/actions/toolkit/blob/main/packages/github/src/context.ts#L6 + * @param {boolean} inputs.labeled - true - PR was labeled, false - unlabeled + * @param {string} inputs.commitSha - sha commit for set status + * @returns Promise + */ +async function onLabeledForSkip({github, context, core, labeled, commitSha}) { + const statusSetFunc = (labeled) ? setSkip : setWait; + + const done = await statusSetFunc({github, context, core, commitSha}); + if (!done) { + core.setFailed('e2e requirement status was not set.'); + } +} + +/** + * Set commit status when commit was pushed. + * Check label for skipping e2e test and e2e tests should skip set success status + * If status was not set then fail job + * + * Used in e2e_run* workflow + * + * Use STATUS_TARGET_COMMIT env var as target commit sha + * Use STATUS_TARGET_COMMIT env var as job status + * Use env var as target commit sha + * @param {object} inputs + * @param {object} inputs.core - A reference to the '@actions/core' package. + * @param {object} inputs.github - A pre-authenticated octokit/rest.js client with pagination plugins. + * @param {object} inputs.context - A reference to context https://github.com/actions/toolkit/blob/main/packages/github/src/context.ts#L6 + * @returns Promise + */ +async function setStatusAfterE2eRun({github, context, core}) { + const jobStatus = process.env.JOB_STATUS; + const commitSha = process.env.STATUS_TARGET_COMMIT; + + let setStateFunc = null; + if (jobStatus === 'failure' || jobStatus === 'cancelled') { + setStateFunc = setFail; + } else if (jobStatus === 'success') { + setStateFunc = await setSuccess; + } else { + core.setFailed(`e2e requirement status was not set. Job status ${jobStat}`) + return + } + + const success = setStateFunc({github, context, core, commitSha}) + if (!success) { + core.setFailed(`e2e requirement status was not set. Job status ${jobStat}`) + } +} + +/** + * Set commit status when commit was pushed. + * Check label for skipping e2e test and e2e tests should skip set success status + * If status was not set then fail job + * + * Used in build-and-test_dev workflow + * + * Use STATUS_TARGET_COMMIT env var as target commit sha + * Use PR_LABELS env var as list of PR labels + * @param {object} inputs + * @param {object} inputs.core - A reference to the '@actions/core' package. + * @param {object} inputs.github - A pre-authenticated octokit/rest.js client with pagination plugins. + * @param {object} inputs.context - A reference to context https://github.com/actions/toolkit/blob/main/packages/github/src/context.ts#L6 + * @returns Promise + */ +async function setInitialStatus ({github, context, core}) { + core.info(`Labels json: ${process.env.PR_LABELS}`); + + const labels = JSON.parse(process.env.PR_LABELS); + const commitSha = process.env.STATUS_TARGET_COMMIT; + + core.debug(`Labels: ${labels ? JSON.stringify(labels.map((l) => l.name)) : 'no labels'}`); + + const shouldSkip = labels ? labels.some((l) => l.name === "skip/e2e") : false; + core.debug(`Should skip e2e: ${shouldSkip}`); + + return onLabeledForSkip({github, context, core, labeled: shouldSkip, commitSha}) +} + +module.exports = { + setStatusAfterE2eRun, + setInitialStatus, + onLabeledForSkip +} diff --git a/.github/scripts/js/time.js b/.github/scripts/js/time.js new file mode 100644 index 0000000000..ac3df2b4f8 --- /dev/null +++ b/.github/scripts/js/time.js @@ -0,0 +1,14 @@ +// Copyright 2022 Flant JSC +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +module.exports.sleep = async (ms) => { + return new Promise(resolve => setTimeout(resolve, ms)) +} diff --git a/.github/workflows/dev_module_build.yml b/.github/workflows/dev_module_build.yml index cc1c4ecb8a..175e5ee699 100644 --- a/.github/workflows/dev_module_build.yml +++ b/.github/workflows/dev_module_build.yml @@ -35,7 +35,7 @@ on: required: false type: number pull_request: - types: [opened, reopened, synchronize, labeled, unlabeled] + types: [opened, reopened, synchronize, labeled] push: branches: - main @@ -337,6 +337,45 @@ jobs: module_name: ${{ vars.MODULE_NAME }} module_tag: "$MODULES_MODULE_TAG" + pull_request_info: + name: Get PR info + runs-on: ubuntu-latest + outputs: + labels: ${{ steps.pr_labels.outputs.labels }} + steps: + - name: Get PR labels + id: pr_labels + uses: actions/github-script@v6.4.1 + with: + script: | + const prNumber = context.payload.pull_request.number; + const { data: labels } = await github.rest.issues.listLabelsOnIssue({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber + }); + core.setOutput('labels', JSON.stringify(labels)); + + set_e2e_requirement_status: + name: Set 'waiting for e2e' commit status + needs: + - pull_request_info + runs-on: ubuntu-latest + steps: + - name: Checkout sources + uses: actions/checkout@v3.5.2 + - name: Set commit status after e2e run + id: set_e2e_requirement_status + uses: actions/github-script@v6.4.1 + env: + STATUS_TARGET_COMMIT: ${{ github.event.pull_request.head.sha }} + PR_LABELS: ${{ needs.pull_request_info.outputs.labels }} + with: + github-token: ${{secrets.RELEASE_PLEASE_TOKEN}} + script: | + const e2eStatus = require('./.github/scripts/js/e2e-commit-status'); + await e2eStatus.setInitialStatus({github, context, core}); + cve_scan_on_pr: name: Trivy images check runs-on: ${{ fromJSON(needs.set_vars.outputs.runner_type)}} @@ -356,3 +395,173 @@ jobs: trivy_registry_user: ${{ vars.PROD_MODULES_REGISTRY_LOGIN }} trivy_registry_password: ${{ secrets.PROD_MODULES_REGISTRY_PASSWORD }} deckhouse_private_repo: ${{vars.DECKHOUSE_PRIVATE_REPO}} + + skip_e2e: + if: ${{ github.event.label.name == 'skip/e2e' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Skip E2E tests + id: skip_e2e + uses: actions/github-script@v6 + with: + script: | + const e2eStatus = require('./.github/scripts/js/e2e-commit-status'); + e2eStatus.onLabeledForSkip({ + github, + context, + core, + labeled: true, + commitSha: context.payload.pull_request.head.sha + }) + + run_e2e: + if: ${{ github.event_name == 'pull_request' && github.event.action == 'labeled' && github.event.label.name == 'e2e/run' }} + name: Run E2E tests + runs-on: ubuntu-latest + needs: + - dev_setup_build + - set_e2e_requirement_status + - set_vars + steps: + - uses: actions/checkout@v4 + - name: Select user + id: select_user + uses: actions/github-script@v6 + env: + KUBECONFIGS: ${{ secrets.K8S_CLUSTER_SECRET }} + USER_CLUSTER_LABELS: ${{ vars.USER_CLUSTER_LABELS }} + with: + script: | + const ci = require('./.github/scripts/js/ci'); + let userClusterLabels; + try { + if (!process.env.USER_CLUSTER_LABELS || process.env.USER_CLUSTER_LABELS.trim() === '') { + throw new Error('USER_CLUSTER_LABELS is empty or not provided.'); + } + userClusterLabels = JSON.parse(process.env.USER_CLUSTER_LABELS); + } catch (error) { + core.setFailed(`Failed to parse USER_CLUSTER_LABELS: ${error.message}`); + return; + } + const userId = await ci.getClusterUser({context, core, userClusterLabels}); + const fs = require('fs'); + const path = require('path'); + const kubeconfigs = JSON.parse(process.env.KUBECONFIGS); + const kubeconfig = kubeconfigs.find(config => config.id === userId)?.kubeconfig; + if (!kubeconfig) { + core.setFailed(`No kubeconfig found for user with ID ${userId}.`); + } else { + core.info(`Found kubeconfig for user with ID ${userId}`); + const runnerTempDir = process.env['RUNNER_TEMP']; + const kubeconfigFile = path.join(runnerTempDir, `kubeconfig_${Date.now()}`); + fs.writeFileSync(kubeconfigFile, kubeconfig); + fs.chmodSync(kubeconfigFile, '600'); + core.exportVariable('KUBECONFIG', kubeconfigFile) + } + + - name: Install Deckhouse-cli + run: | + echo "Install d8" + curl -fsSL -o d8-install.sh https://raw.githubusercontent.com/deckhouse/deckhouse-cli/main/d8-install.sh + bash d8-install.sh + + - name: Set up Go ${{ env.GO_VERSION }} + uses: actions/setup-go@v5 + with: + go-version: "${{ env.GO_VERSION }}" + + - name: Install Task + uses: arduino/setup-task@v2 + + - name: Install ginkgo + working-directory: ./tests/e2e/ + run: | + echo "Install ginkgo" + GINKGO_VERSION=$(go list -f '{{.Version}}' -m github.com/onsi/ginkgo/v2) + go install "github.com/onsi/ginkgo/v2/ginkgo@${GINKGO_VERSION}" + + - uses: deckhouse/modules-actions/setup@v2 + with: + registry: ${{ vars.DEV_REGISTRY }} + registry_login: ${{ vars.DEV_MODULES_REGISTRY_LOGIN }} + registry_password: ${{ secrets.DEV_MODULES_REGISTRY_PASSWORD }} + + - name: Checkout cluster to revision + env: + v12n_tag: pr${{ github.event.pull_request.number }} + working-directory: ./tests/e2e/ + run: | + task checkout-to-mpo + + - name: Download dependencies + working-directory: ./tests/e2e/ + run: | + echo "Download dependencies" + go mod download + + - name: Run E2E + id: e2e-tests + working-directory: ./tests/e2e/ + run: | + task run -v + + - name: Remove label + if: ${{ always() }} + id: remove-label + uses: actions/github-script@v6 + with: + github-token: ${{secrets.RELEASE_PLEASE_TOKEN}} + script: | + const issueNumber = context.issue.number; + const owner = context.repo.owner; + const repo = context.repo.repo; + const labelToRemove = 'e2e/run'; + + const { data: labels } = await github.rest.issues.listLabelsOnIssue({ + owner, + repo, + issue_number: issueNumber, + }); + + if (labels.some(label => label.name === labelToRemove)) { + await github.rest.issues.removeLabel({ + owner, + repo, + issue_number: issueNumber, + name: labelToRemove, + }); + console.log(`Removed label '${labelToRemove}' from PR #${issueNumber}`); + } else { + console.log(`Label '${labelToRemove}' not found on PR #${issueNumber}`); + } + + - name: Cleanup E2E resources on cancel + if: always() && steps.e2e-tests.outcome == 'cancelled' + id: e2e-tests-cleanup + working-directory: ./tests/e2e/ + run: | + task cleanup + + update_comment_on_finish: + name: Update comment on finish + needs: + - run_e2e + runs-on: ubuntu-latest + steps: + - name: Checkout sources + uses: actions/checkout@v3.5.2 + + - name: Set commit status after e2e run and remove label + if: ${{ always() }} + id: set_e2e_requirement_status + uses: actions/github-script@v6.4.1 + env: + JOB_STATUS: ${{ job.status }} + STATUS_TARGET_COMMIT: ${{ github.event.pull_request.head.sha }} + with: + github-token: ${{secrets.RELEASE_PLEASE_TOKEN}} + script: | + const e2eStatus = require('./.github/scripts/js/e2e-commit-status'); + + await e2eStatus.setStatusAfterE2eRun({github, context, core}); diff --git a/tests/e2e/Taskfile.yaml b/tests/e2e/Taskfile.yaml index 0f6aa48665..c95334ff23 100644 --- a/tests/e2e/Taskfile.yaml +++ b/tests/e2e/Taskfile.yaml @@ -110,3 +110,93 @@ tasks: {{if .FOCUS -}} --focus "{{ .FOCUS }}" {{end -}} + cleanup: + desc: "Cleanup namespaces & resources left from e2e tests" + deps: + - kubectl + - d8 + cmds: + - | + E2E_PREFIX="head-$(git rev-parse --short HEAD)" + + delete_resources_with_prefix() { + local RESOURCE_TYPE=$1 + local RESOURCES=$(kubectl get "$RESOURCE_TYPE" --no-headers 2>/dev/null | awk "/$E2E_PREFIX/ {print \$1}") + + if [[ -n "$RESOURCES" ]]; then + echo "$RESOURCES" | xargs -r kubectl delete "$RESOURCE_TYPE" + else + echo "No $RESOURCE_TYPE found with prefix $E2E_PREFIX" + fi + } + + delete_resources_with_prefix "namespaces" + + delete_resources_with_prefix "projects" + + readarray -t CLEANUP_RESOURCES < <(yq '.cleanupResources[]' default_config.yaml) + for RESOURCE in "${CLEANUP_RESOURCES[@]}"; do + delete_resources_with_prefix "$RESOURCE" + done + + checkout-to-mpo: + deps: + - d8 + cmds: + - | + if [ -z "$v12n_tag" ]; then + echo "Error: v12n_tag is not set." + exit 1 + fi + DECKHOUSE_READY_STATUS=$(kubectl get po -n d8-system -l app=deckhouse -o json | jq -r '.items[0].status.conditions[] | select(.type=="ContainersReady") | .status') + if [ "$DECKHOUSE_READY_STATUS" != "True" ]; then + echo "Error: Deckhouse is not ready." + exit 1 + fi + d8 k patch mpo virtualization --type merge -p "{\"spec\":{\"imageTag\":\"$v12n_tag\"}}" + images_hash=$(crane export "dev-registry.deckhouse.io/sys/deckhouse-oss/modules/virtualization:$v12n_tag" - | tar -Oxf - images_digests.json) + v12n_pods=$(kubectl -n d8-virtualization get pods -o json | jq -c) + retry_count=0 + max_retries=120 + sleep_interval=5 + + while true; do + all_hashes_found=true + + # Fetch current pods information + v12n_pods=$(kubectl -n d8-virtualization get pods -o json | jq -c) + + # Process each image entry + while IFS= read -r image_entry; do + image=$(echo "$image_entry" | jq -r '.key') + hash=$(echo "$image_entry" | jq -r '.value') + + if [[ "${image,,}" =~ (libguestfs|predeletehook) ]]; then + continue + fi + + if echo "$v12n_pods" | grep -q "$hash"; then + echo "- ✅ $image $hash" + else + echo "- 🟥 $image $hash" + all_hashes_found=false + fi + done < <(echo "$images_hash" | jq -c '. | to_entries | sort_by(.key)[]') + + # If all hashes are found, break the loop + if [ "$all_hashes_found" = true ]; then + echo "All image hashes found in pods." + break + fi + + retry_count=$((retry_count + 1)) + echo "Some hashes are missing, rechecking... Attempt: $retry_count" + + # Check if the retry limit has been reached + if [ "$retry_count" -ge "$max_retries" ]; then + echo "Error: Timeout reached after $((retry_count * sleep_interval)) seconds. Some image hashes are still missing." + exit 1 + fi + # Wait for the specified interval before retrying + sleep "$sleep_interval" + done diff --git a/tests/e2e/config/config.go b/tests/e2e/config/config.go index 6d335b117d..88fde62036 100644 --- a/tests/e2e/config/config.go +++ b/tests/e2e/config/config.go @@ -131,6 +131,7 @@ type Config struct { Namespace string `yaml:"namespaceSuffix"` TestData TestData `yaml:"testData"` LogFilter []string `yaml:"logFilter"` + CleanupResources []string `yaml:"cleanupResources"` RegexpLogFilter []regexp.Regexp `yaml:"regexpLogFilter"` StorageClass StorageClass } diff --git a/tests/e2e/default_config.yaml b/tests/e2e/default_config.yaml index adaafe08e3..f0e32fa5c3 100644 --- a/tests/e2e/default_config.yaml +++ b/tests/e2e/default_config.yaml @@ -55,3 +55,9 @@ logFilter: regexpLogFilter: - "failed to detach: .* not found" # "err" "failed to detach: virtualmachine.kubevirt.io \"head-497d17b-vm-automatic-with-hotplug\" not found", - "error patching .* not found" # "err" "error patching *** virtualimages.virtualization.deckhouse.io \"head-497d17b-vi-pvc-oref-vi-oref-vd\" not found", + +cleanupResources: + - clustervirtualimages.virtualization.deckhouse.io + - virtualmachineclasses.virtualization.deckhouse.io + - replicatedstorageclasses.storage.deckhouse.io + - virtualmachineipaddressleases.virtualization.deckhouse.io diff --git a/tests/e2e/tests_suite_test.go b/tests/e2e/tests_suite_test.go index 80f83ccfa5..70b43af3e3 100644 --- a/tests/e2e/tests_suite_test.go +++ b/tests/e2e/tests_suite_test.go @@ -210,26 +210,18 @@ func Cleanup() []error { continue } } - - res = kubectl.Delete(kc.DeleteOptions{ - IgnoreNotFound: true, - Labels: map[string]string{"id": namePrefix}, - Resource: kc.ResourceCVI, - }) - if res.Error() != nil { - cleanupErrs = append( - cleanupErrs, fmt.Errorf("cmd: %s\nstderr: %s", res.GetCmd(), res.StdErr()), - ) - } - res = kubectl.Delete(kc.DeleteOptions{ - IgnoreNotFound: true, - Labels: map[string]string{"id": namePrefix}, - Resource: kc.ResourceVMClass, - }) - if res.Error() != nil { - cleanupErrs = append( - cleanupErrs, fmt.Errorf("cmd: %s\nstderr: %s", res.GetCmd(), res.StdErr()), - ) + + for _, r := range conf.CleanupResources { + res = kubectl.Delete(kc.DeleteOptions{ + IgnoreNotFound: true, + Labels: map[string]string{"id": namePrefix}, + Resource: kc.Resource(r), + }) + if res.Error() != nil { + cleanupErrs = append( + cleanupErrs, fmt.Errorf("cmd: %s\nstderr: %s", res.GetCmd(), res.StdErr()), + ) + } } return cleanupErrs