feat: pause on --debug

packages/playwright/src/common/ipc.ts

@@ -84,11 +84,15 @@ export type AttachmentPayload = {
   stepId?: string;
 };
 
+export type TestErrorsPayload = {
+  testId: string;
+  errors: TestInfoErrorImpl[];
+};
+
 export type TestInfoErrorImpl = TestInfoError;
 
 export type TestPausedPayload = {
   testId: string;
-  errors: TestInfoErrorImpl[];
 };
 
 export type CustomMessageRequestPayload = {
@@ -105,7 +109,6 @@ export type TestEndPayload = {
   testId: string;
   duration: number;
   status: TestStatus;
-  errors: TestInfoErrorImpl[];
   hasNonRetriableError: boolean;
   expectedStatus: TestStatus;
   annotations: { type: string, description?: string }[];

packages/playwright/src/isomorphic/teleReceiver.ts

@@ -96,7 +96,8 @@ export type JsonTestResultEnd = {
   id: string;
   duration: number;
   status: reporterTypes.TestStatus;
-  errors: reporterTypes.TestError[];
+  /** No longer emitted, but kept for backwards compatibility */
+  errors?: reporterTypes.TestError[];
   /** No longer emitted, but kept for backwards compatibility */
   attachments?: JsonAttachment[];
   annotations?: TestAnnotation[];
@@ -132,7 +133,7 @@ export type JsonFullResult = {
 };
 
 export type JsonEvent = JsonOnConfigureEvent | JsonOnBlobReportMetadataEvent | JsonOnEndEvent | JsonOnExitEvent | JsonOnProjectEvent | JsonOnBeginEvent | JsonOnTestBeginEvent
-  | JsonOnTestEndEvent | JsonOnStepBeginEvent | JsonOnStepEndEvent | JsonOnAttachEvent | JsonOnErrorEvent | JsonOnStdIOEvent;
+  | JsonOnTestEndEvent | JsonOnStepBeginEvent | JsonOnStepEndEvent | JsonOnAttachEvent | JsonOnTestErrorEvent | JsonOnErrorEvent | JsonOnStdIOEvent;
 
 export type JsonOnConfigureEvent = {
   method: 'onConfigure';
@@ -198,6 +199,15 @@ export type JsonOnAttachEvent = {
   params: JsonTestResultOnAttach;
 };
 
+export type JsonOnTestErrorEvent = {
+  method: 'onTestError';
+  params: {
+    testId: string;
+    resultId: string;
+    error: reporterTypes.TestError;
+  }
+};
+
 export type JsonOnErrorEvent = {
   method: 'onError';
   params: {
@@ -294,6 +304,10 @@ export class TeleReporterReceiver {
       this._onAttach(params.testId, params.resultId, params.attachments);
       return;
     }
+    if (method === 'onTestError') {
+      this._onTestError(params.testId, params.resultId, params.error);
+      return;
+    }
     if (method === 'onStepEnd') {
       this._onStepEnd(params.testId, params.resultId, params.step);
       return;
@@ -353,8 +367,11 @@ export class TeleReporterReceiver {
     const result = test.results.find(r => r._id === payload.id)!;
     result.duration = payload.duration;
     result.status = payload.status;
-    result.errors = payload.errors;
-    result.error = result.errors?.[0];
+    // Errors are only present here from legacy blobs. These override all _onTestErrors events
+    if (!!payload.errors) {
+      result.errors = payload.errors;
+      result.error = result.errors[0];
+    }
     // Attachments are only present here from legacy blobs. These override all _onAttach events
     if (!!payload.attachments)
       result.attachments = this._parseAttachments(payload.attachments);
@@ -404,6 +421,13 @@ export class TeleReporterReceiver {
     })));
   }
 
+  private _onTestError(testId: string, resultId: string, error: reporterTypes.TestError) {
+    const test = this._tests.get(testId)!;
+    const result = test.results.find(r => r._id === resultId)!;
+    result.errors.push(error);
+    result.error = result.errors[0];
+  }
+
   private _onError(error: reporterTypes.TestError) {
     this._reporter.onError?.(error);
   }

packages/playwright/src/reporters/internalReporter.ts

@@ -67,8 +67,11 @@ export class InternalReporter implements ReporterV2 {
     this._reporter.onStdErr?.(chunk, test, result);
   }
 
+  onTestError(test: TestCase, result: TestResult, error: TestError): void {
+    addLocationAndSnippetToError(this._config, error, test.location.file);
+  }
+
   onTestEnd(test: TestCase, result: TestResult) {
-    this._addSnippetToTestErrors(test, result);
     this._reporter.onTestEnd?.(test, result);
   }
 
@@ -89,7 +92,7 @@ export class InternalReporter implements ReporterV2 {
   }
 
   onError(error: TestError) {
-    addLocationAndSnippetToError(this._config, error);
+    addLocationAndSnippetToError(this._config, error, undefined);
     this._reporter.onError?.(error);
   }
 
@@ -98,26 +101,17 @@ export class InternalReporter implements ReporterV2 {
   }
 
   onStepEnd(test: TestCase, result: TestResult, step: TestStep) {
-    this._addSnippetToStepError(test, step);
+    if (step.error)
+      addLocationAndSnippetToError(this._config, step.error, test.location.file);
     this._reporter.onStepEnd?.(test, result, step);
   }
 
   printsToStdio() {
     return this._reporter.printsToStdio ? this._reporter.printsToStdio() : true;
   }
-
-  private _addSnippetToTestErrors(test: TestCase, result: TestResult) {
-    for (const error of result.errors)
-      addLocationAndSnippetToError(this._config, error, test.location.file);
-  }
-
-  private _addSnippetToStepError(test: TestCase, step: TestStep) {
-    if (step.error)
-      addLocationAndSnippetToError(this._config, step.error, test.location.file);
-  }
 }
 
-export function addLocationAndSnippetToError(config: FullConfig, error: TestError, file?: string) {
+export function addLocationAndSnippetToError(config: FullConfig, error: TestError, file: string | undefined) {
   if (error.stack && !error.location)
     error.location = prepareErrorStack(error.stack).location;
   const location = error.location;

packages/playwright/src/reporters/merge.ts

@@ -404,6 +404,7 @@ class IdsPatcher {
       case 'onProject':
         this._onProject(params.project);
         return;
+      case 'onTestError':
       case 'onAttach':
       case 'onTestBegin':
       case 'onStepBegin':
@@ -498,7 +499,7 @@ class PathSeparatorPatcher {
       test.annotations?.forEach(annotation => this._updateAnnotationLocation(annotation));
       const testResult = jsonEvent.params.result;
       testResult.annotations?.forEach(annotation => this._updateAnnotationLocation(annotation));
-      testResult.errors.forEach(error => this._updateErrorLocations(error));
+      testResult.errors?.forEach(error => this._updateErrorLocations(error));
       (testResult.attachments ?? []).forEach(attachment => {
         if (attachment.path)
           attachment.path = this._updatePath(attachment.path);
@@ -516,6 +517,10 @@ class PathSeparatorPatcher {
       step.annotations?.forEach(annotation => this._updateAnnotationLocation(annotation));
       return;
     }
+    if (jsonEvent.method === 'onTestError') {
+      this._updateErrorLocations(jsonEvent.params.error);
+      return;
+    }
     if (jsonEvent.method === 'onAttach') {
       const attach = jsonEvent.params;
       attach.attachments.forEach(attachment => {

packages/playwright/src/reporters/reporterV2.ts

@@ -22,6 +22,7 @@ export interface ReporterV2 {
   onTestBegin?(test: TestCase, result: TestResult): void;
   onStdOut?(chunk: string | Buffer, test?: TestCase, result?: TestResult): void;
   onStdErr?(chunk: string | Buffer, test?: TestCase, result?: TestResult): void;
+  onTestError?(test: TestCase, result: TestResult, error: TestError): void;
   onTestEnd?(test: TestCase, result: TestResult): void;
   onEnd?(result: FullResult): Promise<{ status?: FullResult['status'] } | undefined | void> | void;
   onExit?(): void | Promise<void>;

packages/playwright/src/reporters/teleEmitter.ts

@@ -71,6 +71,17 @@ export class TeleReporterEmitter implements ReporterV2 {
     });
   }
 
+  onTestError(test: reporterTypes.TestCase, result: reporterTypes.TestResult, error: reporterTypes.TestError): void {
+    this._messageSink({
+      method: 'onTestError',
+      params: {
+        testId: test.id,
+        resultId: (result as any)[this._idSymbol],
+        error,
+      }
+    });
+  }
+
   onTestEnd(test: reporterTypes.TestCase, result: reporterTypes.TestResult): void {
     const testEnd: teleReceiver.JsonTestEnd = {
       testId: test.id,
@@ -248,7 +259,6 @@ export class TeleReporterEmitter implements ReporterV2 {
       id: (result as any)[this._idSymbol],
       duration: result.duration,
       status: result.status,
-      errors: result.errors,
       annotations: result.annotations?.length ? this._relativeAnnotationLocations(result.annotations) : undefined,
     };
   }

packages/playwright/src/runner/dispatcher.ts

@@ -28,7 +28,7 @@ import type { ProcessExitData } from './processHost';
 import type { TestGroup } from './testGroups';
 import type { TestError, TestResult, TestStep } from '../../types/testReporter';
 import type { FullConfigInternal } from '../common/config';
-import type { AttachmentPayload, DonePayload, RunPayload, SerializedConfig, StepBeginPayload, StepEndPayload, TeardownErrorsPayload, TestBeginPayload, TestEndPayload, TestOutputPayload, TestPausedPayload } from '../common/ipc';
+import type { AttachmentPayload, DonePayload, RunPayload, SerializedConfig, StepBeginPayload, StepEndPayload, TeardownErrorsPayload, TestBeginPayload, TestEndPayload, TestErrorsPayload, TestOutputPayload, TestPausedPayload } from '../common/ipc';
 import type { Suite } from '../common/test';
 import type { TestCase } from '../common/test';
 import type { ReporterV2 } from '../reporters/reporterV2';
@@ -322,7 +322,6 @@ class JobDispatcher {
       // Do not show more than one error to avoid confusion, but report
       // as interrupted to indicate that we did actually start the test.
       params.status = 'interrupted';
-      params.errors = [];
     }
     const data = this._dataByTestId.get(params.testId);
     if (!data) {
@@ -333,8 +332,6 @@ class JobDispatcher {
     this._remainingByTestId.delete(params.testId);
     const { result, test } = data;
     result.duration = params.duration;
-    result.errors = params.errors;
-    result.error = result.errors[0];
     result.status = params.status;
     result.annotations = params.annotations;
     test.annotations = [...params.annotations]; // last test result wins
@@ -429,6 +426,23 @@ class JobDispatcher {
     }
   }
 
+  private _onTestErrors(params: TestErrorsPayload) {
+    if (this._failureTracker.hasReachedMaxFailures()) {
+      // Do not show more than one error to avoid confusion.
+      return;
+    }
+
+    const data = this._dataByTestId.get(params.testId)!;
+    if (!data)
+      return;
+    const { test, result } = data;
+    for (const error of params.errors) {
+      result.errors.push(error);
+      result.error = result.errors[0];
+      this._reporter.onTestError?.(test, result, error);
+    }
+  }
+
   private _failTestWithErrors(test: TestCase, errors: TestError[]) {
     const runData = this._dataByTestId.get(test.id);
     // There might be a single test that has started but has not finished yet.
@@ -439,8 +453,11 @@ class JobDispatcher {
       result = test._appendTestResult();
       this._reporter.onTestBegin?.(test, result);
     }
-    result.errors = [...errors];
-    result.error = result.errors[0];
+    for (const error of errors) {
+      result.errors.push(error);
+      result.error = result.errors[0];
+      this._reporter.onTestError?.(test, result, error);
+    }
     result.status = errors.length ? 'failed' : 'skipped';
     this._reportTestEnd(test, result);
     this._failedTests.add(test);
@@ -578,31 +595,36 @@ class JobDispatcher {
       eventsHelper.addEventListener(worker, 'stepBegin', this._onStepBegin.bind(this)),
       eventsHelper.addEventListener(worker, 'stepEnd', this._onStepEnd.bind(this)),
       eventsHelper.addEventListener(worker, 'attach', this._onAttach.bind(this)),
+      eventsHelper.addEventListener(worker, 'testErrors', this._onTestErrors.bind(this)),
       eventsHelper.addEventListener(worker, 'testPaused', this._onTestPaused.bind(this, worker)),
       eventsHelper.addEventListener(worker, 'done', this._onDone.bind(this)),
       eventsHelper.addEventListener(worker, 'exit', this.onExit.bind(this)),
     ];
   }
 
   private _onTestPaused(worker: WorkerHost, params: TestPausedPayload) {
+    const data = this._dataByTestId.get(params.testId);
+    if (!data)
+      return;
+
+    const { test, result } = data;
+
     const sendMessage = async (message: { request: any }) => {
       try {
         if (this.jobResult.isDone())
           throw new Error('Test has already stopped');
         const response = await worker.sendCustomMessage({ testId: params.testId, request: message.request });
         if (response.error)
-          addLocationAndSnippetToError(this._config.config, response.error);
+          addLocationAndSnippetToError(this._config.config, response.error, test.location.file);
         return response;
       } catch (e) {
         const error = serializeError(e);
-        addLocationAndSnippetToError(this._config.config, error);
+        addLocationAndSnippetToError(this._config.config, error, test.location.file);
         return { response: undefined, error };
       }
     };
 
-    for (const error of params.errors)
-      addLocationAndSnippetToError(this._config.config, error);
-    this._failureTracker.onTestPaused?.({ ...params, sendMessage });
+    this._failureTracker.onTestPaused?.({ errors: result.errors, sendMessage });
   }
 
   skipWholeJob(): boolean {

packages/playwright/src/runner/testRunner.ts

@@ -470,7 +470,9 @@ export async function runAllTestsWithConfig(config: FullConfigInternal): Promise
     createLoadTask('in-process', { filterOnly: true, failOnLoadErrors: true }),
     ...createRunTestsTasks(config),
   ];
-  const status = await runTasks(new TestRun(config, reporter), tasks, config.config.globalTimeout);
+
+  const testRun = new TestRun(config, reporter, { pauseAtEnd: config.configCLIOverrides.debug, pauseOnError: config.configCLIOverrides.debug });
+  const status = await runTasks(testRun, tasks, config.config.globalTimeout);
 
   // Calling process.exit() might truncate large stdout/stderr output.
   // See https://github.com/nodejs/node/issues/6456.

packages/playwright/src/transform/babelHighlightUtils.ts

@@ -0,0 +1,45 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import path from 'path';
+import { traverse, babelParse, T, types as t } from './babelBundle';
+import type { Location } from '../../types/testReporter';
+
+function containsLocation(range: T.SourceLocation, location: Location): boolean {
+  if (location.line < range.start.line || location.line > range.end.line)
+    return false;
+  if (location.line === range.start.line && location.column < range.start.column)
+    return false;
+  if (location.line === range.end.line && location.column > range.end.column)
+    return false;
+  return true;
+}
+
+export function findTestEndLocation(text: string, testStartLocation: Location): Location | undefined {
+  const ast = babelParse(text, path.basename(testStartLocation.file), false);
+  let result: Location | undefined;
+  traverse(ast, {
+    enter(path) {
+      if (t.isCallExpression(path.node) && path.node.loc && containsLocation(path.node.loc, testStartLocation)) {
+        const callNode = path.node;
+        const funcNode = callNode.arguments[callNode.arguments.length - 1];
+        if (callNode.arguments.length >= 2 && t.isFunction(funcNode) && funcNode.body.loc)
+          result = { file: testStartLocation.file, ...funcNode.body.loc.end };
+      }
+    }
+  });
+  return result;
+}