Skip to content

Commit 1f541b3

Browse files
markoburculmarkoburcul
committed
ci: add windows benchmark ci and script to push result
Referenced issue: * status-im/infra-ci#233
1 parent 2966e3b commit 1f541b3

File tree

3 files changed

+344
-12
lines changed

3 files changed

+344
-12
lines changed

‎ci/Jenkinsfile.combined‎

Lines changed: 32 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -86,23 +86,43 @@ pipeline {
8686
}
8787
}
8888
}
89-
stage('E2E') {
89+
stage('E2E Tests') {
9090
when {
9191
expression {
9292
env.JOB_NAME.toLowerCase().contains('nightly')
9393
}
9494
}
95-
steps {
96-
script {
97-
windows_e2e = build(
98-
job: 'status-app/systems/windows/x86_64/tests-e2e',
99-
parameters: jenkins.mapToParams([
100-
BUILD_SOURCE: windows_x86_64.fullProjectName,
101-
TESTRAIL_RUN_NAME: utils.pkgFilename(),
102-
TEST_SCOPE_FLAG: utils.isReleaseBuild() ? '-m=critical' : '',
103-
GIT_REF: env.BRANCH_NAME,
104-
]),
105-
)
95+
parallel {
96+
stage('E2E') {
97+
steps {
98+
script {
99+
windows_e2e = build(
100+
job: 'status-app/systems/windows/x86_64/tests-e2e',
101+
parameters: jenkins.mapToParams([
102+
BUILD_SOURCE: windows_x86_64.fullProjectName,
103+
TESTRAIL_RUN_NAME: utils.pkgFilename(),
104+
TEST_SCOPE_FLAG: utils.isReleaseBuild() ? '-m=critical' : '',
105+
GIT_REF: env.BRANCH_NAME,
106+
]),
107+
)
108+
}
109+
}
110+
}
111+
stage('Benchmark E2E') {
112+
steps {
113+
script {
114+
benchmark_windows_e2e = build(
115+
job: 'status-app/e2e/nightly-benchmark-windows-master',
116+
propagate: false,
117+
parameters: jenkins.mapToParams([
118+
BUILD_SOURCE: windows_x86_64.fullProjectName,
119+
TESTRAIL_RUN_NAME: utils.pkgFilename(),
120+
TEST_SCOPE_FLAG: utils.isReleaseBuild() ? '-m=critical' : '',
121+
GIT_REF: env.BRANCH_NAME
122+
]),
123+
)
124+
}
125+
}
106126
}
107127
}
108128
}

‎ci/Jenkinsfile.tests-e2e.windows-benchmark‎

Lines changed: 258 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,258 @@
1+
#!/usr/bin/env groovy
2+
library '[email protected]'
3+
4+
pipeline {
5+
agent {
6+
label 'windows && x86_64 && qt-6.9.2 && windows-e2e-benchmark'
7+
}
8+
9+
parameters {
10+
string(
11+
name: 'GIT_REF',
12+
description: 'Git branch to checkout.',
13+
defaultValue: 'master'
14+
)
15+
string(
16+
name: 'BUILD_SOURCE',
17+
description: 'URL to tar.gz file OR path to Jenkins build.',
18+
defaultValue: 'status-app/systems/windows/x86_64/package'
19+
)
20+
string(
21+
name: 'TEST_NAME',
22+
description: 'Paste test name/part of test name to run specific test.',
23+
defaultValue: ''
24+
)
25+
string(
26+
name: 'TEST_SCOPE_FLAG',
27+
description: 'Paste a known mark to run tests labeled with this mark',
28+
defaultValue: ''
29+
)
30+
string(
31+
name: 'TESTRAIL_RUN_NAME',
32+
description: 'Test run name in Test Rail.',
33+
defaultValue: ''
34+
)
35+
choice(
36+
name: 'LOG_LEVEL',
37+
description: 'Log level for pytest.',
38+
choices: ['INFO', 'DEBUG', 'TRACE', 'WARNING', 'CRITICAL']
39+
)
40+
}
41+
42+
options {
43+
timestamps()
44+
/* Prevent Jenkins jobs from running forever */
45+
timeout(time: 120, unit: 'MINUTES')
46+
disableConcurrentBuilds()
47+
disableRestartFromStage()
48+
/* manage how many builds we keep */
49+
buildDiscarder(logRotator(
50+
numToKeepStr: '10',
51+
daysToKeepStr: '30',
52+
artifactNumToKeepStr: '1',
53+
))
54+
}
55+
56+
environment {
57+
PLATFORM = 'tests/e2e-windows'
58+
59+
SQUISH_DIR = 'C:\\squish-runner-9.1.0-qt-6.9'
60+
PYTHONPATH = "${SQUISH_DIR}\\lib;${SQUISH_DIR}\\bin;${SQUISH_DIR}\\lib\\python;${PYTHONPATH}"
61+
62+
/* To stop e2e tests using port 8545 */
63+
STATUS_RUNTIME_HTTP_API = 'False'
64+
STATUS_RUNTIME_WS_API = 'False'
65+
66+
/* Avoid race conditions with other builds using virtualenv. */
67+
VIRTUAL_ENV = "${WORKSPACE_TMP}\\venv"
68+
PATH = "${VIRTUAL_ENV}\\bin;C:\\Qt\\6.9.2\\msvc2022_64\\bin;${PATH}"
69+
70+
/* To store user configuratiin files in temp dir */
71+
XDG_CONFIG_HOME = "${WORKSPACE_TMP}/config"
72+
73+
TESTRAIL_URL = 'https://ethstatus.testrail.net'
74+
TESTRAIL_PROJECT_ID = 18
75+
/* Override QT xcb plugin with linux to avoid errors like:
76+
* "Could not load the Qt platform plugin "xcb" in "" even though it was found."
77+
QT_QPA_PLATFORM = "linuxfb"*/
78+
79+
/* Runtime flag to make testing of the app easier. Switched off: unpredictable app behavior under new tests */
80+
STATUS_RUNTIME_TEST_MODE = 1
81+
82+
/*To enable Home tab */
83+
FLAG_HOMEPAGE_ENABLED = 1
84+
85+
/* Logging rules let you enable or disable logging for categories */
86+
/* QT_LOGGING_RULES = '*.warning=true' */
87+
88+
/* Set to a non-zero value to make Qt print out diagnostic information about the each (C++) plugin it tries to load. */
89+
/* QT_DEBUG_PLUGINS = 0 */
90+
91+
/* Hack fix for params not being set in job on first run */
92+
BUILD_SOURCE = "${params.BUILD_SOURCE}"
93+
TEST_NAME = "${params.TEST_NAME}"
94+
TEST_SCOPE_FLAG = "${params.TEST_SCOPE_FLAG}"
95+
TESTRAIL_RUN_NAME = "${params.TESTRAIL_RUN_NAME}"
96+
LOG_LEVEL = "${params.LOG_LEVEL}"
97+
98+
/* Forces QT to use OpenGL for rendering instead of default Direct3D 11 */
99+
QSG_RHI_BACKEND = 'opengl'
100+
}
101+
102+
103+
stages {
104+
stage('Cleanup Workspace') {
105+
steps {
106+
sh './scripts/clean-git.sh'
107+
}
108+
}
109+
110+
stage('Deps') {
111+
steps { script { dir('test/e2e') {
112+
bat """
113+
python310 -m venv ${VIRTUAL_ENV}
114+
${VIRTUAL_ENV}\\Scripts\\python.exe -m pip install --upgrade pip
115+
${VIRTUAL_ENV}\\Scripts\\python.exe -m pip install -r requirements.txt
116+
"""
117+
} } }
118+
}
119+
120+
stage('Download') {
121+
when { expression { params.BUILD_SOURCE.startsWith('http') } }
122+
steps { timeout(5) { script { dir('test/e2e') {
123+
sh 'mkdir -p ./pkg/'
124+
setBuildDescFromFile(params.BUILD_SOURCE)
125+
fileOperations([
126+
fileDownloadOperation(
127+
url: params.BUILD_SOURCE,
128+
targetFileName: 'StatusIm-Desktop.7z',
129+
targetLocation: './pkg/',
130+
userName: '',
131+
password: '',
132+
)
133+
])
134+
} } } }
135+
}
136+
137+
stage('Copy') {
138+
when { expression { ! params.BUILD_SOURCE.startsWith('http') } }
139+
steps { timeout(5) { script { dir('test/e2e') {
140+
copyArtifacts(
141+
projectName: params.BUILD_SOURCE,
142+
filter: 'pkg/*-x86_64.7z',
143+
selector: lastWithArtifacts(),
144+
target: './'
145+
)
146+
setBuildDescFromFile(utils.findFile('pkg/*7z'))
147+
} } } }
148+
}
149+
150+
stage('Unpack') {
151+
steps { timeout(5) { script { dir('test/e2e') {
152+
sh 'mkdir aut'
153+
sh "7z x '${utils.findFile('pkg/*.7z')}' -o'./aut'"
154+
env.AUT_PATH = utils.findFile('aut/Status/bin/Status.exe').replace('\\','/')
155+
} } } }
156+
}
157+
158+
stage('Test') {
159+
steps {
160+
timeout(time: 120) {
161+
dir('test/e2e') {
162+
/* Lock the agent so we run only one e2e build at a time */
163+
lock(resource: "e2e-windows-${env.NODE_NAME}", quantity: 1) {
164+
script {
165+
def flags = []
166+
if (params.TEST_NAME) { flags.add("-k=${params.TEST_NAME}") }
167+
if (params.TEST_SCOPE_FLAG) { flags.add(params.TEST_SCOPE_FLAG) }
168+
if (params.LOG_LEVEL) { flags.addAll(["--log-level=${params.LOG_LEVEL}", "--log-cli-level=${params.LOG_LEVEL}"]) }
169+
def flagStr = flags.join(' ')
170+
171+
withCredentials([
172+
usernamePassword(credentialsId: 'test-rail-api-devops', usernameVariable: 'TESTRAIL_USR', passwordVariable: 'TESTRAIL_PSW'),
173+
string(credentialsId: 'wallet-test-user-seed', variable: 'WALLET_TEST_USER_SEED')
174+
]) {
175+
sh"""
176+
pushd configs
177+
ln -sf _local.ci.py _local.py || cp _local.ci.py _local.py
178+
popd
179+
180+
"""
181+
bat"""
182+
${VIRTUAL_ENV}\\Scripts\\python.exe -m pytest -m "not keycard" -v --reruns=1 --timeout=300 ${flagStr} --disable-warnings --alluredir=./allure-results -o timeout_func_only=true
183+
"""
184+
}
185+
}
186+
}
187+
}
188+
}
189+
}
190+
}
191+
}
192+
193+
194+
post {
195+
always {
196+
script {
197+
dir('test/e2e') {
198+
archiveArtifacts('aut/Status/bin/*.log')
199+
200+
sh 'cp ext/allure_files/categories.json allure-results'
201+
sh 'cp ext/allure_files/environment.properties allure-results'
202+
203+
allure([
204+
results: [[path: 'allure-results']],
205+
reportBuildPolicy: 'ALWAYS',
206+
properties: [],
207+
jdk: '',
208+
])
209+
}
210+
}
211+
script {
212+
withCredentials([sshUserPrivateKey(
213+
credentialsId: 'status-app-benchmarks-deploy-key',
214+
keyFileVariable: 'SSH_KEY_FILE'
215+
)]) {
216+
sh '''
217+
eval $(ssh-agent -s)
218+
ssh-add $SSH_KEY_FILE
219+
./scripts/push_benchmark.sh
220+
'''
221+
}
222+
}
223+
}
224+
cleanup {
225+
script {
226+
powershell '''
227+
if (Test-Path "C:\\Users\\jenkins\\AppData\\Local\\Status") {
228+
Remove-Item "C:\\Users\\jenkins\\AppData\\Local\\Status" -Recurse -Force -ErrorAction SilentlyContinue
229+
}
230+
# Clean-up registry entries
231+
$registryPath = "HKCU:\\Software\\Status\\Status Desktop"
232+
if (Test-Path $registryPath) {
233+
Remove-Item $registryPath -Recurse -Force -ErrorAction SilentlyContinue
234+
}
235+
'''
236+
}
237+
cleanWs(disableDeferredWipeout: true)
238+
powershell "Remove-Item -Path '${env.WORKSPACE_TMP}' -Recurse -Force"
239+
}
240+
}
241+
}
242+
243+
def setBuildDescFromFile(fileNameOrPath) {
244+
def tokens = utils.parseFilename(utils.baseName(fileNameOrPath))
245+
if (tokens == null) { /* Fallback for regex fail. */
246+
currentBuild.description = utils.baseName(fileNameOrPath)
247+
return
248+
}
249+
if (tokens.build && tokens.build.startsWith('pr')) {
250+
currentBuild.displayName = tokens.build.replace(/^pr/, 'PR-')
251+
}
252+
currentBuild.description = formatMap([
253+
Node: NODE_NAME,
254+
Build: tokens.build,
255+
Commit: tokens.commit,
256+
Version: (tokens.tstamp ?: tokens.version),
257+
])
258+
}

‎scripts/push_benchmark.sh‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
#!/usr/bin/env bash
2+
3+
export YLW='\033[1;33m'
4+
export RED='\033[0;31m'
5+
export GRN='\033[0;32m'
6+
export BLU='\033[0;34m'
7+
export BLD='\033[1m'
8+
export RST='\033[0m'
9+
10+
# Clear line
11+
export CLR='\033[2K'
12+
13+
set -o nounset
14+
set -o errexit
15+
set -o pipefail
16+
17+
REPO_URL="[email protected]:status-im/status-app-benchmarks.git"
18+
19+
GIT_ROOT=$(cd "${BASH_SOURCE%/*}" && git rev-parse --show-toplevel)
20+
21+
echo -e "${GRN}Pushing benchmark results${RST}"
22+
23+
cd "${GIT_ROOT}"
24+
# Get the commit SHA from the status-app repo BEFORE cloning benchmarks-repo
25+
commit_sha=$(git rev-parse --short HEAD)
26+
27+
git clone "${REPO_URL}" benchmarks-repo
28+
cd benchmarks-repo
29+
30+
date_time=$(date -u '+%Y-%m-%dT%H:%M:%S')
31+
32+
echo -e "${GRN}Creating virtual environment${RST}"
33+
python3 -m venv .venv
34+
source .venv/Scripts/activate
35+
PYTHON_CMD=".venv/Scripts/python.exe"
36+
37+
echo -e "${GRN}Installing dependencies${RST}"
38+
${PYTHON_CMD} -m pip install --upgrade pip
39+
${PYTHON_CMD} -m pip install -r requirements.txt
40+
41+
echo -e "${GRN}Updating data in repo${RST}"
42+
${PYTHON_CMD} scripts/parse-results.py --data-dir ./data ../test/e2e/allure-report/data/ --commit-hash "${commit_sha}" --date "${date_time}"
43+
44+
echo -e "${GRN}Generating new visualizations from data${RST}"
45+
${PYTHON_CMD} scripts/visualize-data.py --data-dir ./data/ --output-dir ./docs/
46+
47+
echo -e "${GRN}Committing changes${RST}"
48+
git add .
49+
git commit -m "Add benchmark results ${benchmark_dir}"
50+
51+
echo -e "${GRN}Pushing changes${RST}"
52+
git push "${REPO_URL}"
53+
54+
echo -e "${GRN}Push finished${RST}"

0 commit comments

Comments
 (0)