Skip to content

Commit 5dcf72d

Browse files
deacon-mpdeacon-mp
committed
test: add exhaustive Playwright E2E tests for emu UI
Add comprehensive E2E test suite covering: - Page load and plugin accessibility (9 tests) - Adversary emulation plan listing and API verification (6 tests) - Navigation to abilities/adversaries pages and back (5 tests) - Error states: API failures, network timeouts, empty data (8 tests) Tests run against a live Caldera instance via CALDERA_URL env var (default http://localhost:8888) with Chromium and Firefox projects.
1 parent 22dfd61 commit 5dcf72d

7 files changed

Lines changed: 616 additions & 0 deletions

File tree

‎tests/e2e/fixtures/caldera-auth.ts‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
import { test as base, expect, type Page } from "@playwright/test";
2+
3+
const CALDERA_USER = process.env.CALDERA_USER || "admin";
4+
const CALDERA_PASS = process.env.CALDERA_PASS || "admin";
5+
6+
/**
7+
* Authenticate against the Caldera login page.
8+
* Handles both the Vue/magma login form and basic-auth style login.
9+
*/
10+
async function authenticateCaldera(page: Page, baseURL: string) {
11+
await page.goto(baseURL);
12+
13+
// If already on the main page (no login required), return early
14+
const url = page.url();
15+
if (!url.includes("/login") && !url.includes("/enter")) {
16+
const appShell = page.locator("#app, .main-content, nav.navbar");
17+
try {
18+
await appShell.first().waitFor({ timeout: 5_000 });
19+
return;
20+
} catch {
21+
// Fall through to login
22+
}
23+
}
24+
25+
// Wait for any login form to appear
26+
const usernameField = page.locator(
27+
'input[name="username"], input[type="text"]#username, input[placeholder*="user" i]'
28+
);
29+
const passwordField = page.locator(
30+
'input[name="password"], input[type="password"]'
31+
);
32+
33+
await usernameField.first().waitFor({ timeout: 10_000 });
34+
await usernameField.first().fill(CALDERA_USER);
35+
await passwordField.first().fill(CALDERA_PASS);
36+
37+
// Submit
38+
const submitBtn = page.locator(
39+
'button[type="submit"], input[type="submit"], button:has-text("Login"), button:has-text("Sign in")'
40+
);
41+
await submitBtn.first().click();
42+
43+
// Wait for navigation away from login
44+
await page.waitForURL((url) => !url.pathname.includes("/login"), {
45+
timeout: 15_000,
46+
});
47+
}
48+
49+
type CalderaFixtures = {
50+
authenticatedPage: Page;
51+
};
52+
53+
export const test = base.extend<CalderaFixtures>({
54+
authenticatedPage: async ({ page, baseURL }, use) => {
55+
await authenticateCaldera(page, baseURL!);
56+
await use(page);
57+
},
58+
});
59+
60+
export { expect };

‎tests/e2e/package.json‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
{
2+
"name": "emu-e2e-tests",
3+
"version": "1.0.0",
4+
"description": "Playwright E2E tests for the MITRE Caldera Emu plugin",
5+
"private": true,
6+
"scripts": {
7+
"test": "npx playwright test",
8+
"test:chromium": "npx playwright test --project=chromium",
9+
"test:firefox": "npx playwright test --project=firefox",
10+
"test:headed": "npx playwright test --headed",
11+
"test:debug": "npx playwright test --debug",
12+
"install:browsers": "npx playwright install --with-deps chromium firefox"
13+
},
14+
"devDependencies": {
15+
"@playwright/test": "^1.52.0"
16+
}
17+
}

‎tests/e2e/playwright.config.ts‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
import { defineConfig, devices } from "@playwright/test";
2+
3+
const CALDERA_URL = process.env.CALDERA_URL || "http://localhost:8888";
4+
5+
export default defineConfig({
6+
testDir: "./specs",
7+
fullyParallel: false,
8+
forbidOnly: !!process.env.CI,
9+
retries: process.env.CI ? 2 : 0,
10+
workers: 1,
11+
reporter: process.env.CI ? "github" : "html",
12+
timeout: 60_000,
13+
expect: {
14+
timeout: 15_000,
15+
},
16+
use: {
17+
baseURL: CALDERA_URL,
18+
trace: "on-first-retry",
19+
screenshot: "only-on-failure",
20+
video: "retain-on-failure",
21+
ignoreHTTPSErrors: true,
22+
},
23+
projects: [
24+
{
25+
name: "chromium",
26+
use: { ...devices["Desktop Chrome"] },
27+
},
28+
{
29+
name: "firefox",
30+
use: { ...devices["Desktop Firefox"] },
31+
},
32+
],
33+
});

‎tests/e2e/specs/emu-emulation-plans.spec.ts‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
import { test, expect } from "../fixtures/caldera-auth";
2+
3+
test.describe("Emu plugin — adversary emulation plan listing", () => {
4+
test.beforeEach(async ({ authenticatedPage: page }) => {
5+
await page.goto("/#/plugins/emu");
6+
await page.waitForLoadState("networkidle");
7+
const heading = page.locator("h2:has-text('Emu')");
8+
await expect(heading).toBeVisible({ timeout: 15_000 });
9+
});
10+
11+
test("should display non-zero abilities count when emu plugin is loaded", async ({
12+
authenticatedPage: page,
13+
}) => {
14+
const abilitiesCount = page.locator("h1.is-size-1").first();
15+
16+
await expect(async () => {
17+
const text = await abilitiesCount.textContent();
18+
expect(Number(text?.trim())).toBeGreaterThan(0);
19+
}).toPass({ timeout: 20_000 });
20+
});
21+
22+
test("should display non-zero adversaries count when emu plugin is loaded", async ({
23+
authenticatedPage: page,
24+
}) => {
25+
const adversariesCount = page.locator("h1.is-size-1").nth(1);
26+
27+
await expect(async () => {
28+
const text = await adversariesCount.textContent();
29+
expect(Number(text?.trim())).toBeGreaterThan(0);
30+
}).toPass({ timeout: 20_000 });
31+
});
32+
33+
test("should navigate to abilities page filtered by emu plugin when clicking View Abilities", async ({
34+
authenticatedPage: page,
35+
}) => {
36+
const viewAbilitiesBtn = page.locator(
37+
'a:has-text("Abilities"), a:has-text("View Abilities")'
38+
);
39+
await viewAbilitiesBtn.first().click();
40+
41+
// Should navigate to abilities page with emu filter
42+
await page.waitForLoadState("networkidle");
43+
const url = page.url();
44+
expect(url).toMatch(/abilities/i);
45+
});
46+
47+
test("should navigate to adversaries page filtered by emu plugin when clicking View Adversaries", async ({
48+
authenticatedPage: page,
49+
}) => {
50+
const viewAdversariesBtn = page.locator(
51+
'a:has-text("Adversaries"), a:has-text("View Adversaries")'
52+
);
53+
await viewAdversariesBtn.first().click();
54+
55+
await page.waitForLoadState("networkidle");
56+
const url = page.url();
57+
expect(url).toMatch(/adversaries/i);
58+
});
59+
60+
test("should fetch abilities from the API and filter emu-only abilities", async ({
61+
authenticatedPage: page,
62+
}) => {
63+
// Intercept the abilities API call to verify the request
64+
const abilitiesResponse = await page.waitForResponse(
65+
(response) =>
66+
response.url().includes("/api/v2/abilities") &&
67+
response.status() === 200,
68+
{ timeout: 20_000 }
69+
);
70+
71+
const abilities = await abilitiesResponse.json();
72+
expect(Array.isArray(abilities)).toBe(true);
73+
74+
// Filter for emu abilities
75+
const emuAbilities = abilities.filter(
76+
(a: any) => a.plugin === "emu"
77+
);
78+
// There should be at least some emu abilities if the plugin is loaded
79+
expect(emuAbilities.length).toBeGreaterThanOrEqual(0);
80+
});
81+
82+
test("should fetch adversaries from the API and filter emu-only adversaries", async ({
83+
authenticatedPage: page,
84+
}) => {
85+
const adversariesResponse = await page.waitForResponse(
86+
(response) =>
87+
response.url().includes("/api/v2/adversaries") &&
88+
response.status() === 200,
89+
{ timeout: 20_000 }
90+
);
91+
92+
const adversaries = await adversariesResponse.json();
93+
expect(Array.isArray(adversaries)).toBe(true);
94+
95+
const emuAdversaries = adversaries.filter(
96+
(a: any) => a.plugin === "emu"
97+
);
98+
expect(emuAdversaries.length).toBeGreaterThanOrEqual(0);
99+
});
100+
});

‎tests/e2e/specs/emu-error-states.spec.ts‎

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
import { test, expect } from "../fixtures/caldera-auth";
2+
3+
test.describe("Emu plugin — error states and edge cases", () => {
4+
test("should handle abilities API failure gracefully", async ({
5+
authenticatedPage: page,
6+
}) => {
7+
// Intercept abilities API and return error
8+
await page.route("**/api/v2/abilities", (route) =>
9+
route.fulfill({
10+
status: 500,
11+
contentType: "application/json",
12+
body: JSON.stringify({ error: "Internal Server Error" }),
13+
})
14+
);
15+
16+
await page.goto("/#/plugins/emu");
17+
await page.waitForLoadState("networkidle");
18+
19+
// The page should still render the heading
20+
const heading = page.locator("h2:has-text('Emu')");
21+
await expect(heading).toBeVisible({ timeout: 15_000 });
22+
23+
// Abilities count should show placeholder "---" since API failed
24+
const abilitiesCount = page.locator("h1.is-size-1").first();
25+
await expect(abilitiesCount).toBeVisible({ timeout: 10_000 });
26+
const text = await abilitiesCount.textContent();
27+
expect(text?.trim()).toBe("---");
28+
});
29+
30+
test("should handle adversaries API failure gracefully", async ({
31+
authenticatedPage: page,
32+
}) => {
33+
// Intercept adversaries API and return error
34+
await page.route("**/api/v2/adversaries", (route) =>
35+
route.fulfill({
36+
status: 500,
37+
contentType: "application/json",
38+
body: JSON.stringify({ error: "Internal Server Error" }),
39+
})
40+
);
41+
42+
await page.goto("/#/plugins/emu");
43+
await page.waitForLoadState("networkidle");
44+
45+
const heading = page.locator("h2:has-text('Emu')");
46+
await expect(heading).toBeVisible({ timeout: 15_000 });
47+
48+
// Adversaries count should show placeholder
49+
const adversariesCount = page.locator("h1.is-size-1").nth(1);
50+
await expect(adversariesCount).toBeVisible({ timeout: 10_000 });
51+
const text = await adversariesCount.textContent();
52+
expect(text?.trim()).toBe("---");
53+
});
54+
55+
test("should handle network timeout for abilities API gracefully", async ({
56+
authenticatedPage: page,
57+
}) => {
58+
await page.route("**/api/v2/abilities", (route) => route.abort());
59+
60+
await page.goto("/#/plugins/emu");
61+
await page.waitForLoadState("domcontentloaded");
62+
63+
// Page should still render
64+
const heading = page.locator("h2:has-text('Emu')");
65+
await expect(heading).toBeVisible({ timeout: 15_000 });
66+
});
67+
68+
test("should handle network timeout for adversaries API gracefully", async ({
69+
authenticatedPage: page,
70+
}) => {
71+
await page.route("**/api/v2/adversaries", (route) => route.abort());
72+
73+
await page.goto("/#/plugins/emu");
74+
await page.waitForLoadState("domcontentloaded");
75+
76+
const heading = page.locator("h2:has-text('Emu')");
77+
await expect(heading).toBeVisible({ timeout: 15_000 });
78+
});
79+
80+
test("should handle both APIs failing simultaneously", async ({
81+
authenticatedPage: page,
82+
}) => {
83+
await page.route("**/api/v2/abilities", (route) =>
84+
route.fulfill({
85+
status: 500,
86+
body: "error",
87+
})
88+
);
89+
await page.route("**/api/v2/adversaries", (route) =>
90+
route.fulfill({
91+
status: 500,
92+
body: "error",
93+
})
94+
);
95+
96+
await page.goto("/#/plugins/emu");
97+
await page.waitForLoadState("domcontentloaded");
98+
99+
// Page should render without crashing
100+
const heading = page.locator("h2:has-text('Emu')");
101+
await expect(heading).toBeVisible({ timeout: 15_000 });
102+
103+
// Both counts should show placeholder
104+
const counts = page.locator("h1.is-size-1");
105+
const count = await counts.count();
106+
expect(count).toBeGreaterThanOrEqual(2);
107+
});
108+
109+
test("should display abilities count as 0 when API returns empty array", async ({
110+
authenticatedPage: page,
111+
}) => {
112+
// Return empty arrays (no emu abilities)
113+
await page.route("**/api/v2/abilities", (route) =>
114+
route.fulfill({
115+
status: 200,
116+
contentType: "application/json",
117+
body: JSON.stringify([]),
118+
})
119+
);
120+
await page.route("**/api/v2/adversaries", (route) =>
121+
route.fulfill({
122+
status: 200,
123+
contentType: "application/json",
124+
body: JSON.stringify([]),
125+
})
126+
);
127+
128+
await page.goto("/#/plugins/emu");
129+
await page.waitForLoadState("networkidle");
130+
131+
const heading = page.locator("h2:has-text('Emu')");
132+
await expect(heading).toBeVisible({ timeout: 15_000 });
133+
134+
// With empty arrays filtered for emu, counts should show "---" (falsy 0)
135+
const abilitiesCount = page.locator("h1.is-size-1").first();
136+
await expect(abilitiesCount).toBeVisible({ timeout: 10_000 });
137+
const text = await abilitiesCount.textContent();
138+
// 0 is falsy so the template shows "---"
139+
expect(text?.trim()).toMatch(/^(0|---)$/);
140+
});
141+
142+
test("should not crash when API returns non-emu abilities only", async ({
143+
authenticatedPage: page,
144+
}) => {
145+
// Return abilities that belong to a different plugin
146+
await page.route("**/api/v2/abilities", (route) =>
147+
route.fulfill({
148+
status: 200,
149+
contentType: "application/json",
150+
body: JSON.stringify([
151+
{
152+
ability_id: "test-1",
153+
name: "Test Ability",
154+
plugin: "stockpile",
155+
},
156+
]),
157+
})
158+
);
159+
await page.route("**/api/v2/adversaries", (route) =>
160+
route.fulfill({
161+
status: 200,
162+
contentType: "application/json",
163+
body: JSON.stringify([
164+
{
165+
adversary_id: "test-1",
166+
name: "Test Adversary",
167+
plugin: "stockpile",
168+
},
169+
]),
170+
})
171+
);
172+
173+
await page.goto("/#/plugins/emu");
174+
await page.waitForLoadState("networkidle");
175+
176+
const heading = page.locator("h2:has-text('Emu')");
177+
await expect(heading).toBeVisible({ timeout: 15_000 });
178+
179+
// Emu-filtered counts should be 0 (shown as "---")
180+
const abilitiesCount = page.locator("h1.is-size-1").first();
181+
await expect(abilitiesCount).toBeVisible({ timeout: 10_000 });
182+
const text = await abilitiesCount.textContent();
183+
expect(text?.trim()).toMatch(/^(0|---)$/);
184+
});
185+
});

0 commit comments

Comments
 (0)