Merge pull request #1339 from supertokens/agent/issue-1335-stress-assertions-dispatch

fix: re-enable stress-test seeding assertions and allow manual dispatch

Author: porcellus

.github/workflows/stress-tests.yml

@@ -7,6 +7,12 @@ on:
         description: 'Docker image tag to use'
         required: true
         type: string
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: 'Docker image tag to use'
+        required: true
+        type: string
 
 jobs:
   stress-tests:

CHANGELOG.md

@@ -7,6 +7,9 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
 
+- Test-only: the 1M-user stress-test suite now records non-OK results per seeding step and fails the run at the end if
+  any step errored, and its workflow can be triggered manually via `workflow_dispatch`
+
 ## [12.0.8]
 
 - Security improvements around api-key/ip allow list handling

stress-tests/src/common/utils.ts

@@ -132,6 +132,57 @@ export class StatsCollector {
   }
 }
 
+/**
+ * Tracks non-OK results produced by the seeding steps. A seeding step that
+ * silently errors would otherwise still "pass" and invalidate every
+ * measurement downstream of it, so we count the non-OK results per step and
+ * fail the run at the end if any step had failures.
+ */
+export class FailureTracker {
+  private static instance: FailureTracker;
+  private failures: Map<string, { count: number; statuses: Record<string, number> }> = new Map();
+
+  private constructor() {}
+
+  public static getInstance(): FailureTracker {
+    if (!FailureTracker.instance) {
+      FailureTracker.instance = new FailureTracker();
+    }
+    return FailureTracker.instance;
+  }
+
+  public recordFailure(step: string, status: string) {
+    const entry = this.failures.get(step) ?? { count: 0, statuses: {} };
+    entry.count++;
+    entry.statuses[status] = (entry.statuses[status] ?? 0) + 1;
+    this.failures.set(step, entry);
+  }
+
+  public hasFailures(): boolean {
+    return this.failures.size > 0;
+  }
+
+  /**
+   * Prints a per-step summary of any non-OK results and throws if there were
+   * any, so the run fails at the end without aborting the seeding mid-way.
+   */
+  public throwIfAnyFailures() {
+    if (!this.hasFailures()) {
+      console.log('\nAll seeding steps completed with no non-OK results.');
+      return;
+    }
+    console.error('\nSeeding step failures detected (non-OK results):');
+    let total = 0;
+    for (const [step, info] of this.failures) {
+      total += info.count;
+      console.error(`  ${step}: ${info.count} non-OK result(s) — ${JSON.stringify(info.statuses)}`);
+    }
+    throw new Error(
+      `${total} non-OK result(s) across ${this.failures.size} seeding step(s); measurements are not trustworthy.`
+    );
+  }
+}
+
 export const measureTime = async <T>(title: string, fn: () => Promise<T>): Promise<T> => {
   const st = Date.now();
   const result = await fn();

stress-tests/src/oneMillionUsers/createUsers.ts

@@ -2,7 +2,7 @@ import EmailPassword from 'supertokens-node/recipe/emailpassword';
 import Passwordless from 'supertokens-node/recipe/passwordless';
 import ThirdParty from 'supertokens-node/recipe/thirdparty';
 
-import { workInBatches, measureTime } from '../common/utils';
+import { workInBatches, measureTime, FailureTracker } from '../common/utils';
 
 const TOTAL_USERS = 10000;
 
@@ -16,13 +16,13 @@ const createEmailPasswordUsers = async () => {
         .map(() => String.fromCharCode(97 + Math.floor(Math.random() * 26)))
         .join('') + '@example.com';
     const createdUser = await EmailPassword.signUp('public', email, 'password');
-    // expect(createdUser.status).toBe("OK");
     if (createdUser.status === 'OK') {
       return {
         recipeUserId: createdUser.recipeUserId.getAsString(),
         email: email,
       };
     }
+    FailureTracker.getInstance().recordFailure('EmailPassword users creation', createdUser.status);
   });
 };
 
@@ -38,13 +38,16 @@ const createPasswordlessUsersWithEmail = async () => {
       tenantId: 'public',
       email,
     });
-    // expect(createdUser.status).toBe("OK");
     if (createdUser.status === 'OK') {
       return {
         recipeUserId: createdUser.recipeUserId.getAsString(),
         email,
       };
     }
+    FailureTracker.getInstance().recordFailure(
+      'Passwordless users (with email) creation',
+      createdUser.status
+    );
   });
 };
 
@@ -56,13 +59,16 @@ const createPasswordlessUsersWithPhone = async () => {
       tenantId: 'public',
       phoneNumber,
     });
-    // expect(createdUser.status).toBe("OK");
     if (createdUser.status === 'OK') {
       return {
         recipeUserId: createdUser.recipeUserId.getAsString(),
         phoneNumber,
       };
     }
+    FailureTracker.getInstance().recordFailure(
+      'Passwordless users (with phone) creation',
+      createdUser.status
+    );
   });
 };
 
@@ -85,13 +91,16 @@ const createThirdPartyUsers = async (thirdPartyId: string) => {
       email,
       true
     );
-    // expect(createdUser.status).toBe("OK");
     if (createdUser.status === 'OK') {
       return {
         recipeUserId: createdUser.recipeUserId.getAsString(),
         email,
       };
     }
+    FailureTracker.getInstance().recordFailure(
+      `ThirdParty users (${thirdPartyId}) creation`,
+      createdUser.status
+    );
   });
 };
 

stress-tests/src/oneMillionUsers/index.ts

@@ -3,6 +3,7 @@ import {
   deleteStInstance,
   setupLicense,
   StatsCollector,
+  FailureTracker,
 } from '../common/utils';
 
 import SuperTokens from 'supertokens-node';
@@ -147,6 +148,11 @@ async function main() {
     // Write stats to file
     StatsCollector.getInstance().writeToFile();
     console.log('\nStats written to stats.json');
+
+    // Fail the run if any seeding step produced non-OK results, so silently
+    // errored steps don't leave the run looking green with untrustworthy
+    // measurements.
+    FailureTracker.getInstance().throwIfAnyFailures();
   } catch (error) {
     console.error('An error occurred during execution:', error);
     throw error;