diff --git a/ci/Jenkinsfile.combined b/ci/Jenkinsfile.combined index 7dc9a1117de..aad93635f93 100644 --- a/ci/Jenkinsfile.combined +++ b/ci/Jenkinsfile.combined @@ -86,23 +86,43 @@ pipeline { } } } - stage('E2E') { + stage('E2E Tests') { when { expression { env.JOB_NAME.toLowerCase().contains('nightly') } } - steps { - script { - windows_e2e = build( - job: 'status-app/systems/windows/x86_64/tests-e2e', - parameters: jenkins.mapToParams([ - BUILD_SOURCE: windows_x86_64.fullProjectName, - TESTRAIL_RUN_NAME: utils.pkgFilename(), - TEST_SCOPE_FLAG: utils.isReleaseBuild() ? '-m=critical' : '', - GIT_REF: env.BRANCH_NAME, - ]), - ) + parallel { + stage('E2E') { + steps { + script { + windows_e2e = build( + job: 'status-app/systems/windows/x86_64/tests-e2e', + parameters: jenkins.mapToParams([ + BUILD_SOURCE: windows_x86_64.fullProjectName, + TESTRAIL_RUN_NAME: utils.pkgFilename(), + TEST_SCOPE_FLAG: utils.isReleaseBuild() ? '-m=critical' : '', + GIT_REF: env.BRANCH_NAME, + ]), + ) + } + } + } + stage('Benchmark E2E') { + steps { + script { + benchmark_windows_e2e = build( + job: 'status-app/e2e/nightly-benchmark-windows-master', + propagate: false, + parameters: jenkins.mapToParams([ + BUILD_SOURCE: windows_x86_64.fullProjectName, + TESTRAIL_RUN_NAME: utils.pkgFilename(), + TEST_SCOPE_FLAG: utils.isReleaseBuild() ? '-m=critical' : '', + GIT_REF: env.BRANCH_NAME + ]), + ) + } + } } } } diff --git a/ci/Jenkinsfile.tests-e2e.windows-benchmark b/ci/Jenkinsfile.tests-e2e.windows-benchmark new file mode 100644 index 00000000000..3841c37ed24 --- /dev/null +++ b/ci/Jenkinsfile.tests-e2e.windows-benchmark @@ -0,0 +1,247 @@ +#!/usr/bin/env groovy +library 'status-jenkins-lib@v1.9.31' + +pipeline { + agent { + label 'windows && x86_64 && qt-6.9.2 && windows-e2e-benchmark' + } + + parameters { + string( + name: 'GIT_REF', + description: 'Git branch to checkout.', + defaultValue: 'master' + ) + string( + name: 'BUILD_SOURCE', + description: 'URL to tar.gz file OR path to Jenkins build.', + defaultValue: 'status-app/systems/windows/x86_64/package' + ) + string( + name: 'TEST_NAME', + description: 'Paste test name/part of test name to run specific test.', + defaultValue: '' + ) + string( + name: 'TEST_SCOPE_FLAG', + description: 'Paste a known mark to run tests labeled with this mark', + defaultValue: '' + ) + string( + name: 'TESTRAIL_RUN_NAME', + description: 'Test run name in Test Rail.', + defaultValue: '' + ) + choice( + name: 'LOG_LEVEL', + description: 'Log level for pytest.', + choices: ['INFO', 'DEBUG', 'TRACE', 'WARNING', 'CRITICAL'] + ) + } + + options { + timestamps() + /* Prevent Jenkins jobs from running forever */ + timeout(time: 120, unit: 'MINUTES') + disableConcurrentBuilds() + disableRestartFromStage() + /* manage how many builds we keep */ + buildDiscarder(logRotator( + numToKeepStr: '10', + daysToKeepStr: '30', + artifactNumToKeepStr: '1', + )) + } + + environment { + PLATFORM = 'tests/e2e-windows' + + SQUISH_DIR = 'C:\\squish-runner-9.1.0-qt-6.9' + PYTHONPATH = "${SQUISH_DIR}\\lib;${SQUISH_DIR}\\bin;${SQUISH_DIR}\\lib\\python;${PYTHONPATH}" + + /* To stop e2e tests using port 8545 */ + STATUS_RUNTIME_HTTP_API = 'False' + STATUS_RUNTIME_WS_API = 'False' + + /* Avoid race conditions with other builds using virtualenv. */ + VIRTUAL_ENV = "${WORKSPACE_TMP}\\venv" + PATH = "${VIRTUAL_ENV}\\bin;C:\\Qt\\6.9.2\\msvc2022_64\\bin;${PATH}" + + /* To store user configuratiin files in temp dir */ + XDG_CONFIG_HOME = "${WORKSPACE_TMP}/config" + + TESTRAIL_URL = 'https://ethstatus.testrail.net' + TESTRAIL_PROJECT_ID = 18 + + /* Runtime flag to make testing of the app easier. Switched off: unpredictable app behavior under new tests */ + STATUS_RUNTIME_TEST_MODE = 1 + + /*To enable Home tab */ + FLAG_HOMEPAGE_ENABLED = 1 + + /* Hack fix for params not being set in job on first run */ + BUILD_SOURCE = "${params.BUILD_SOURCE}" + TEST_NAME = "${params.TEST_NAME}" + TEST_SCOPE_FLAG = "${params.TEST_SCOPE_FLAG}" + TESTRAIL_RUN_NAME = "${params.TESTRAIL_RUN_NAME}" + LOG_LEVEL = "${params.LOG_LEVEL}" + + /* Forces QT to use OpenGL for rendering instead of default Direct3D 11 */ + QSG_RHI_BACKEND = 'opengl' + } + + + stages { + stage('Cleanup Workspace') { + steps { + sh './scripts/clean-git.sh' + } + } + + stage('Deps') { + steps { script { dir('test/e2e') { + bat """ + python310 -m venv ${VIRTUAL_ENV} + ${VIRTUAL_ENV}\\Scripts\\python.exe -m pip install --upgrade pip + ${VIRTUAL_ENV}\\Scripts\\python.exe -m pip install -r requirements.txt + """ + } } } + } + + stage('Download') { + when { expression { params.BUILD_SOURCE.startsWith('http') } } + steps { timeout(5) { script { dir('test/e2e') { + sh 'mkdir -p ./pkg/' + setBuildDescFromFile(params.BUILD_SOURCE) + fileOperations([ + fileDownloadOperation( + url: params.BUILD_SOURCE, + targetFileName: 'StatusIm-Desktop.7z', + targetLocation: './pkg/', + ) + ]) + } } } } + } + + stage('Copy') { + when { expression { ! params.BUILD_SOURCE.startsWith('http') } } + steps { timeout(5) { script { dir('test/e2e') { + copyArtifacts( + projectName: params.BUILD_SOURCE, + filter: 'pkg/*-x86_64.7z', + selector: lastWithArtifacts(), + target: './' + ) + setBuildDescFromFile(utils.findFile('pkg/*7z')) + } } } } + } + + stage('Unpack') { + steps { timeout(5) { script { dir('test/e2e') { + sh 'mkdir aut' + sh "7z x '${utils.findFile('pkg/*.7z')}' -o'./aut'" + env.AUT_PATH = utils.findFile('aut/Status/bin/Status.exe').replace('\\','/') + } } } } + } + + stage('Test') { + steps { + timeout(time: 120) { + dir('test/e2e') { + /* Lock the agent so we run only one e2e build at a time */ + lock(resource: "e2e-windows-${env.NODE_NAME}", quantity: 1) { + script { + def flags = [] + if (params.TEST_NAME) { flags.add("-k=${params.TEST_NAME}") } + if (params.TEST_SCOPE_FLAG) { flags.add(params.TEST_SCOPE_FLAG) } + if (params.LOG_LEVEL) { flags.addAll(["--log-level=${params.LOG_LEVEL}", "--log-cli-level=${params.LOG_LEVEL}"]) } + def flagStr = flags.join(' ') + + withCredentials([ + usernamePassword(credentialsId: 'test-rail-api-devops', usernameVariable: 'TESTRAIL_USR', passwordVariable: 'TESTRAIL_PSW'), + string(credentialsId: 'wallet-test-user-seed', variable: 'WALLET_TEST_USER_SEED') + ]) { + sh""" + pushd configs + ln -sf _local.ci.py _local.py || cp _local.ci.py _local.py + popd + + """ + bat""" + ${VIRTUAL_ENV}\\Scripts\\python.exe -m pytest -m "not keycard" -v --reruns=1 --timeout=300 ${flagStr} --disable-warnings --alluredir=./allure-results -o timeout_func_only=true + """ + } + } + } + } + } + } + } + } + + + post { + always { + script { + dir('test/e2e') { + archiveArtifacts('aut/Status/bin/*.log') + + sh 'cp ext/allure_files/categories.json allure-results' + sh 'cp ext/allure_files/environment.properties allure-results' + + allure([ + results: [[path: 'allure-results']], + reportBuildPolicy: 'ALWAYS', + properties: [], + jdk: '', + ]) + } + } + script { + withCredentials([sshUserPrivateKey( + credentialsId: 'status-app-benchmarks-deploy-key', + keyFileVariable: 'SSH_KEY_FILE' + )]) { + sh ''' + eval $(ssh-agent -s) + ssh-add $SSH_KEY_FILE + ./scripts/push_benchmark.sh + ''' + } + } + } + cleanup { + script { + powershell ''' + if (Test-Path "C:\\Users\\jenkins\\AppData\\Local\\Status") { + Remove-Item "C:\\Users\\jenkins\\AppData\\Local\\Status" -Recurse -Force -ErrorAction SilentlyContinue + } + # Clean-up registry entries + $registryPath = "HKCU:\\Software\\Status\\Status Desktop" + if (Test-Path $registryPath) { + Remove-Item $registryPath -Recurse -Force -ErrorAction SilentlyContinue + } + ''' + } + cleanWs(disableDeferredWipeout: true) + powershell "Remove-Item -Path '${env.WORKSPACE_TMP}' -Recurse -Force" + } + } +} + +def setBuildDescFromFile(fileNameOrPath) { + def tokens = utils.parseFilename(utils.baseName(fileNameOrPath)) + if (tokens == null) { /* Fallback for regex fail. */ + currentBuild.description = utils.baseName(fileNameOrPath) + return + } + if (tokens.build && tokens.build.startsWith('pr')) { + currentBuild.displayName = tokens.build.replace(/^pr/, 'PR-') + } + currentBuild.description = formatMap([ + Node: NODE_NAME, + Build: tokens.build, + Commit: tokens.commit, + Version: (tokens.tstamp ?: tokens.version), + ]) +} \ No newline at end of file diff --git a/scripts/push_benchmark.sh b/scripts/push_benchmark.sh new file mode 100755 index 00000000000..5edd72c3e6a --- /dev/null +++ b/scripts/push_benchmark.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +export YLW='\033[1;33m' +export RED='\033[0;31m' +export GRN='\033[0;32m' +export BLU='\033[0;34m' +export BLD='\033[1m' +export RST='\033[0m' + +# Clear line +export CLR='\033[2K' + +set -o nounset +set -o errexit +set -o pipefail + +REPO_URL="git@github.com:status-im/status-app-benchmarks.git" + +GIT_ROOT=$(cd "${BASH_SOURCE%/*}" && git rev-parse --show-toplevel) + +echo -e "${GRN}Pushing benchmark results${RST}" + +cd "${GIT_ROOT}" +# Get the commit SHA from the status-app repo BEFORE cloning benchmarks-repo +commit_sha=$(git rev-parse --short HEAD) + +git clone "${REPO_URL}" benchmarks-repo +cd benchmarks-repo + +date_time=$(date -u '+%Y-%m-%dT%H:%M:%S') + +echo -e "${GRN}Creating virtual environment${RST}" +python3 -m venv .venv +source .venv/Scripts/activate +PYTHON_CMD=".venv/Scripts/python.exe" + +echo -e "${GRN}Installing dependencies${RST}" +${PYTHON_CMD} -m pip install --upgrade pip +${PYTHON_CMD} -m pip install -r requirements.txt + +echo -e "${GRN}Updating data in repo${RST}" +${PYTHON_CMD} scripts/parse-results.py --data-dir ./data ../test/e2e/allure-report/data/ --commit-hash "${commit_sha}" --date "${date_time}" + +echo -e "${GRN}Generating new visualizations from data${RST}" +${PYTHON_CMD} scripts/visualize-data.py --data-dir ./data/ --output-dir ./docs/ + +echo -e "${GRN}Committing changes${RST}" +git add . +git commit -m "Add benchmark results for commit ${commit_sha}" + +echo -e "${GRN}Pushing changes${RST}" +git push "${REPO_URL}" + +echo -e "${GRN}Push finished${RST}"