Skip to content

feat(smoketests): run tests on readonly and isolated too COMPASS-8128 #6702

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 6 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .evergreen/buildvariants-and-tasks.in.yml
Original file line number Diff line number Diff line change
Expand Up @@ -217,7 +217,7 @@ buildvariants:
<% } %>

<% for (const buildVariant of SMOKETEST_BUILD_VARIANTS) { %>
<% for (const distribution of ['compass']) { %>
<% for (const distribution of COMPASS_DISTRIBUTIONS) { %>
- name: <%= buildVariant.name %>-<%= distribution %>
display_name: <%= buildVariant.display_name %> (<%= distribution %>)
run_on: <%= buildVariant.run_on %>
Expand Down Expand Up @@ -468,7 +468,7 @@ tasks:
compass_distribution: <%= distribution %>
<% } %>

<% for (const distribution of ['compass']) { %>
<% for (const distribution of COMPASS_DISTRIBUTIONS) { %>
- name: smoketest-<%= distribution %>
tags: ['required-for-publish', 'run-on-pr']
commands:
Expand Down
110 changes: 110 additions & 0 deletions .evergreen/buildvariants-and-tasks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,22 @@ buildvariants:
variant: package-ubuntu
tasks:
- name: smoketest-compass
- name: smoketest-ubuntu-compass-isolated
display_name: Smoketest Ubuntu (compass-isolated)
run_on: ubuntu2004-large
depends_on:
- name: package-compass-isolated
variant: package-ubuntu
tasks:
- name: smoketest-compass-isolated
- name: smoketest-ubuntu-compass-readonly
display_name: Smoketest Ubuntu (compass-readonly)
run_on: ubuntu2004-large
depends_on:
- name: package-compass-readonly
variant: package-ubuntu
tasks:
- name: smoketest-compass-readonly
- name: smoketest-windows-compass
display_name: Smoketest Windows (compass)
run_on: windows-vsCurrent-large
Expand All @@ -92,6 +108,22 @@ buildvariants:
variant: package-windows
tasks:
- name: smoketest-compass
- name: smoketest-windows-compass-isolated
display_name: Smoketest Windows (compass-isolated)
run_on: windows-vsCurrent-large
depends_on:
- name: package-compass-isolated
variant: package-windows
tasks:
- name: smoketest-compass-isolated
- name: smoketest-windows-compass-readonly
display_name: Smoketest Windows (compass-readonly)
run_on: windows-vsCurrent-large
depends_on:
- name: package-compass-readonly
variant: package-windows
tasks:
- name: smoketest-compass-readonly
- name: smoketest-rhel-compass
display_name: Smoketest RHEL (compass)
run_on: rhel80-large
Expand All @@ -100,6 +132,22 @@ buildvariants:
variant: package-rhel
tasks:
- name: smoketest-compass
- name: smoketest-rhel-compass-isolated
display_name: Smoketest RHEL (compass-isolated)
run_on: rhel80-large
depends_on:
- name: package-compass-isolated
variant: package-rhel
tasks:
- name: smoketest-compass-isolated
- name: smoketest-rhel-compass-readonly
display_name: Smoketest RHEL (compass-readonly)
run_on: rhel80-large
depends_on:
- name: package-compass-readonly
variant: package-rhel
tasks:
- name: smoketest-compass-readonly
- name: smoketest-macos-x64-compass
display_name: Smoketest MacOS Intel (compass)
run_on: macos-14-gui
Expand All @@ -108,6 +156,22 @@ buildvariants:
variant: package-macos-x64
tasks:
- name: smoketest-compass
- name: smoketest-macos-x64-compass-isolated
display_name: Smoketest MacOS Intel (compass-isolated)
run_on: macos-14-gui
depends_on:
- name: package-compass-isolated
variant: package-macos-x64
tasks:
- name: smoketest-compass-isolated
- name: smoketest-macos-x64-compass-readonly
display_name: Smoketest MacOS Intel (compass-readonly)
run_on: macos-14-gui
depends_on:
- name: package-compass-readonly
variant: package-macos-x64
tasks:
- name: smoketest-compass-readonly
- name: smoketest-macos-arm-compass
display_name: Smoketest MacOS Arm64 (compass)
run_on: macos-14-arm64-gui
Expand All @@ -116,6 +180,22 @@ buildvariants:
variant: package-macos-arm
tasks:
- name: smoketest-compass
- name: smoketest-macos-arm-compass-isolated
display_name: Smoketest MacOS Arm64 (compass-isolated)
run_on: macos-14-arm64-gui
depends_on:
- name: package-compass-isolated
variant: package-macos-arm
tasks:
- name: smoketest-compass-isolated
- name: smoketest-macos-arm-compass-readonly
display_name: Smoketest MacOS Arm64 (compass-readonly)
run_on: macos-14-arm64-gui
depends_on:
- name: package-compass-readonly
variant: package-macos-arm
tasks:
- name: smoketest-compass-readonly
- name: test-eol-servers
display_name: Test EoL Servers
run_on: ubuntu1804-large
Expand Down Expand Up @@ -523,6 +603,36 @@ tasks:
mongodb_version: 8.0.x-enterprise
compass_distribution: compass
debug: compass-e2e-tests*,electron*,hadron*,mongo*
- name: smoketest-compass-isolated
tags:
- required-for-publish
- run-on-pr
commands:
- func: prepare
- func: install
- func: bootstrap
vars:
scope: compass-e2e-tests
- func: smoketest-packaged-app
vars:
mongodb_version: 8.0.x-enterprise
compass_distribution: compass-isolated
debug: compass-e2e-tests*,electron*,hadron*,mongo*
- name: smoketest-compass-readonly
tags:
- required-for-publish
- run-on-pr
commands:
- func: prepare
- func: install
- func: bootstrap
vars:
scope: compass-e2e-tests
- func: smoketest-packaged-app
vars:
mongodb_version: 8.0.x-enterprise
compass_distribution: compass-readonly
debug: compass-e2e-tests*,electron*,hadron*,mongo*
- name: test-server-40x-community-1
tags:
- required-for-publish
Expand Down
43 changes: 34 additions & 9 deletions .evergreen/functions.yml
Original file line number Diff line number Diff line change
Expand Up @@ -677,9 +677,19 @@ functions:
npm i -w packages/compass-smoke-tests https://x-access-token:${generated_token}@github.com/10gen/compass-mongodb-com --engine-strict=false

if [[ "$IS_WINDOWS" == "true" ]]; then
npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=windows_setup --tests time-to-first-query
npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=windows_zip --tests auto-update-from
npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=windows_msi --tests auto-update-from
if [[ "$HADRON_DISTRIBUTION" == "compass-readonly" ]]; then
npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=windows_setup --tests read-only
else
npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=windows_setup --tests time-to-first-query
fi
if [[ "$HADRON_DISTRIBUTION" == "compass-isolated" ]]; then
# auto-update is not supported on isolated
npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=windows_zip --tests time-to-first-query
npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=windows_msi --tests time-to-first-query
else
npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=windows_zip --tests auto-update-from
npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=windows_msi --tests auto-update-from
fi
fi

if [[ "$IS_OSX" == "true" ]]; then
Expand All @@ -688,18 +698,33 @@ functions:
# NOTE: We're also skipping auto-update of the macOS app in CI
# because it doesn't work. Running a different test to make sure it
# can install and run successfully at least.
npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=osx_zip --tests=time-to-first-query
npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=osx_dmg --tests=time-to-first-query
if [[ "$HADRON_DISTRIBUTION" == "compass-readonly" ]]; then
npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=osx_zip --tests=read-only
npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=osx_dmg --tests=read-only
else
npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=osx_zip --tests=time-to-first-query
npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=osx_dmg --tests=time-to-first-query
fi
fi

if [[ "$IS_UBUNTU" == "true" ]]; then
npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=linux_deb --tests=time-to-first-query
npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=linux_tar --tests=time-to-first-query
if [[ "$HADRON_DISTRIBUTION" == "compass-readonly" ]]; then
npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=linux_deb --tests=read-only
npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=linux_tar --tests=read-only
else
npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=linux_deb --tests=time-to-first-query
npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=linux_tar --tests=time-to-first-query
fi
fi

if [[ "$IS_RHEL" == "true" ]]; then
npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=linux_rpm --tests=time-to-first-query
npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=linux_tar --tests=time-to-first-query
if [[ "$HADRON_DISTRIBUTION" == "compass-readonly" ]]; then
npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=linux_rpm --tests=read-only
npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=linux_tar --tests=read-only
else
npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=linux_rpm --tests=time-to-first-query
npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=linux_tar --tests=time-to-first-query
fi
fi

test-web-sandbox:
Expand Down
3 changes: 3 additions & 0 deletions packages/compass-smoke-tests/src/cli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import { SUPPORTED_TESTS } from './tests/types';
import { type SmokeTestsContext } from './context';
import { SUPPORTED_PACKAGES } from './packages';
import { testTimeToFirstQuery } from './tests/time-to-first-query';
import { testReadOnly } from './tests/read-only';
import { testAutoUpdateFrom } from './tests/auto-update-from';
import { testAutoUpdateTo } from './tests/auto-update-to';

Expand Down Expand Up @@ -126,6 +127,8 @@ async function run() {

if (testName === 'time-to-first-query') {
await testTimeToFirstQuery(context);
} else if (testName === 'read-only') {
await testReadOnly(context);
} else if (testName === 'auto-update-from') {
await testAutoUpdateFrom(context);
} else if (testName === 'auto-update-to') {
Expand Down
57 changes: 57 additions & 0 deletions packages/compass-smoke-tests/src/tests/read-only.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
import fs from 'node:fs';
import { type SmokeTestsContext } from '../context';
import { execute } from '../execute';
import { getInstaller } from '../installers';
import { createSandbox } from '../directories';
import { getTestSubject } from '../test-subject';

export async function testReadOnly(context: SmokeTestsContext) {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This whole file is basically a copy/paste of time-to-first-query.ts. We can probably refactor it.

const sandboxPath = createSandbox();
const { kind, appName, filepath } = await getTestSubject({
...context,
sandboxPath,
});

try {
const install = getInstaller(kind);

const { appPath, uninstall } = install({
appName,
filepath,
destinationPath: sandboxPath,
});

try {
execute(
'npm',
[
'run',
'--unsafe-perm',
'test-packaged',
'--workspace',
'compass-e2e-tests',
'--',
'--test-filter=read-only',
],
{
// We need to use a shell to get environment variables setup correctly
shell: true,
env: {
...process.env,
COMPASS_APP_NAME: appName,
COMPASS_APP_PATH: appPath,
},
}
);
} finally {
await uninstall();
}
} finally {
if (context.skipCleanup) {
console.log(`Skipped cleaning up sandbox: ${sandboxPath}`);
} else {
console.log(`Cleaning up sandbox: ${sandboxPath}`);
fs.rmSync(sandboxPath, { recursive: true });
}
}
}
1 change: 1 addition & 0 deletions packages/compass-smoke-tests/src/tests/types.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
export const SUPPORTED_TESTS = [
'time-to-first-query',
'read-only',
'auto-update-from',
'auto-update-to',
] as const;
Expand Down
Loading