From fd492ce39f2f0c78d26509baf3dbbbf805f3e7f2 Mon Sep 17 00:00:00 2001 From: Yashodhan Joshi Date: Mon, 25 Nov 2024 20:34:41 +0530 Subject: [PATCH 01/16] fix: use getPath to prevent video file overwrite Signed-off-by: Yashodhan Joshi --- packages/server/lib/modes/run.ts | 30 +++++++--- packages/server/lib/screenshots.js | 92 +--------------------------- packages/server/lib/util/fs.ts | 96 ++++++++++++++++++++++++++++++ 3 files changed, 120 insertions(+), 98 deletions(-) diff --git a/packages/server/lib/modes/run.ts b/packages/server/lib/modes/run.ts index db70e790eb7a..6dc14177f1d8 100644 --- a/packages/server/lib/modes/run.ts +++ b/packages/server/lib/modes/run.ts @@ -13,7 +13,7 @@ import Reporter from '../reporter' import browserUtils from '../browsers' import { openProject } from '../open_project' import * as videoCapture from '../video_capture' -import { fs } from '../util/fs' +import { fs, getPath } from '../util/fs' import runEvents from '../plugins/run_events' import env from '../util/env' import trash from '../util/trash' @@ -224,15 +224,24 @@ async function trashAssets (config: Cfg) { } } -async function startVideoRecording (options: { previous?: VideoRecording, project: Project, spec: SpecWithRelativeRoot, videosFolder: string }): Promise { +async function startVideoRecording (options: { previous?: VideoRecording, project: Project, spec: SpecWithRelativeRoot, videosFolder: string, overwrite: boolean }): Promise { if (!options.videosFolder) throw new Error('Missing videoFolder for recording') - function videoPath (suffix: string) { - return path.join(options.videosFolder, options.spec.relativeToCommonRoot + suffix) + async function videoPath (suffix: string, ext: string) { + const specPath = options.spec.relativeToCommonRoot + suffix + const data = { + name: specPath, + testFailure: false, + testAttemptIndex: 0, + titles: [], + } + + // getPath returns a Promise!!! + return await getPath(data, ext, options.videosFolder, options.overwrite) } - const videoName = videoPath('.mp4') - const compressedVideoName = videoPath('-compressed.mp4') + const videoName = await videoPath('', 'mp4') + const compressedVideoName = await videoPath('-compressed', 'mp4') const outputDir = path.dirname(videoName) @@ -333,6 +342,13 @@ async function compressRecording (options: { quiet: boolean, videoCompression: n if (options.videoCompression === false || options.videoCompression === 0) { debug('skipping compression') + // the getSafePath used to get the compressedVideoName creates the file + // in order to check if the path is safe or not. So here, if the compressed + // file exists, we remove it as compression is not enabled + if (fs.existsSync(options.processOptions.compressedVideoName)) { + await fs.remove(options.processOptions.compressedVideoName) + } + return } @@ -945,7 +961,7 @@ async function runSpec (config, spec: SpecWithRelativeRoot, options: { project: async function getVideoRecording () { if (!options.video) return undefined - const opts = { project, spec, videosFolder: options.videosFolder } + const opts = { project, spec, videosFolder: options.videosFolder, overwrite: options.config.trashAssetsBeforeRuns } telemetry.startSpan({ name: 'video:capture' }) diff --git a/packages/server/lib/screenshots.js b/packages/server/lib/screenshots.js index 79730f952a3f..80e88ffc97ee 100644 --- a/packages/server/lib/screenshots.js +++ b/packages/server/lib/screenshots.js @@ -1,30 +1,17 @@ const _ = require('lodash') const mime = require('mime') -const path = require('path') const Promise = require('bluebird') const dataUriToBuffer = require('data-uri-to-buffer') const Jimp = require('jimp') const sizeOf = require('image-size') const colorString = require('color-string') -const sanitize = require('sanitize-filename') let debug = require('debug')('cypress:server:screenshot') const plugins = require('./plugins') -const { fs } = require('./util/fs') - -const RUNNABLE_SEPARATOR = ' -- ' -const pathSeparatorRe = /[\\\/]/g +const { fs, getPath } = require('./util/fs') // internal id incrementor let __ID__ = null -// many filesystems limit filename length to 255 bytes/characters, so truncate the filename to -// the smallest common denominator of safe filenames, which is 255 bytes. when ENAMETOOLONG -// errors are encountered, `maxSafeBytes` will be decremented to at most `MIN_PREFIX_BYTES`, at -// which point the latest ENAMETOOLONG error will be emitted. -// @see https://en.wikipedia.org/wiki/Comparison_of_file_systems#Limits -let maxSafeBytes = Number(process.env.CYPRESS_MAX_SAFE_FILENAME_BYTES) || 254 -const MIN_PREFIX_BYTES = 64 - // TODO: when we parallelize these builds we'll need // a semaphore to access the file system when we write // screenshots since its possible two screenshots with @@ -293,83 +280,6 @@ const getDimensions = function (details) { return pick(details.image.bitmap) } -const ensureSafePath = function (withoutExt, extension, overwrite, num = 0) { - const suffix = `${(num && !overwrite) ? ` (${num})` : ''}.${extension}` - - const maxSafePrefixBytes = maxSafeBytes - suffix.length - const filenameBuf = Buffer.from(path.basename(withoutExt)) - - if (filenameBuf.byteLength > maxSafePrefixBytes) { - const truncated = filenameBuf.slice(0, maxSafePrefixBytes).toString() - - withoutExt = path.join(path.dirname(withoutExt), truncated) - } - - const fullPath = [withoutExt, suffix].join('') - - debug('ensureSafePath %o', { withoutExt, extension, num, maxSafeBytes, maxSafePrefixBytes }) - - return fs.pathExists(fullPath) - .then((found) => { - if (found && !overwrite) { - return ensureSafePath(withoutExt, extension, overwrite, num + 1) - } - - // path does not exist, attempt to create it to check for an ENAMETOOLONG error - return fs.outputFileAsync(fullPath, '') - .then(() => fullPath) - .catch((err) => { - debug('received error when testing path %o', { err, fullPath, maxSafePrefixBytes, maxSafeBytes }) - - if (err.code === 'ENAMETOOLONG' && maxSafePrefixBytes >= MIN_PREFIX_BYTES) { - maxSafeBytes -= 1 - - return ensureSafePath(withoutExt, extension, overwrite, num) - } - - throw err - }) - }) -} - -const sanitizeToString = (title) => { - // test titles may be values which aren't strings like - // null or undefined - so convert before trying to sanitize - return sanitize(_.toString(title)) -} - -const getPath = function (data, ext, screenshotsFolder, overwrite) { - let names - const specNames = (data.specName || '') - .split(pathSeparatorRe) - - if (data.name) { - names = data.name.split(pathSeparatorRe).map(sanitize) - } else { - names = _ - .chain(data.titles) - .map(sanitizeToString) - .join(RUNNABLE_SEPARATOR) - .concat([]) - .value() - } - - const index = names.length - 1 - - // append (failed) to the last name - if (data.testFailure) { - names[index] = `${names[index]} (failed)` - } - - if (data.testAttemptIndex > 0) { - names[index] = `${names[index]} (attempt ${data.testAttemptIndex + 1})` - } - - const withoutExt = path.join(screenshotsFolder, ...specNames, ...names) - - return ensureSafePath(withoutExt, ext, overwrite) -} - const getPathToScreenshot = function (data, details, screenshotsFolder) { const ext = mime.getExtension(getType(details)) diff --git a/packages/server/lib/util/fs.ts b/packages/server/lib/util/fs.ts index 5846acecc837..fb67a8f1e080 100644 --- a/packages/server/lib/util/fs.ts +++ b/packages/server/lib/util/fs.ts @@ -2,6 +2,20 @@ import Bluebird from 'bluebird' import fsExtra from 'fs-extra' +import sanitize from 'sanitize-filename' +import path from 'path' +import _ from 'lodash' + +const RUNNABLE_SEPARATOR = ' -- ' +const pathSeparatorRe = /[\\\/]/g + +// many filesystems limit filename length to 255 bytes/characters, so truncate the filename to +// the smallest common denominator of safe filenames, which is 255 bytes. when ENAMETOOLONG +// errors are encountered, `maxSafeBytes` will be decremented to at most `MIN_PREFIX_BYTES`, at +// which point the latest ENAMETOOLONG error will be emitted. +// @see https://en.wikipedia.org/wiki/Comparison_of_file_systems#Limits +let maxSafeBytes = Number(process.env.CYPRESS_MAX_SAFE_FILENAME_BYTES) || 254 +const MIN_PREFIX_BYTES = 64 type Promisified any> = (...params: Parameters) => Bluebird> @@ -12,6 +26,88 @@ interface PromisifiedFsExtra { readFileAsync: Promisified writeFileAsync: Promisified pathExistsAsync: Promisified + outputFileAsync: Promisified } +type PathOptions = { + specName?: string + name: string + testFailure: boolean + testAttemptIndex: number + titles: Array +}; + export const fs = Bluebird.promisifyAll(fsExtra) as PromisifiedFsExtra & typeof fsExtra + +const ensureSafePath = async function (withoutExt: string, extension: string, overwrite: boolean, num: number = 0): Promise { + const suffix = `${(num && !overwrite) ? ` (${num})` : ''}.${extension}` + + const maxSafePrefixBytes = maxSafeBytes - suffix.length + const filenameBuf = Buffer.from(path.basename(withoutExt)) + + if (filenameBuf.byteLength > maxSafePrefixBytes) { + const truncated = filenameBuf.slice(0, maxSafePrefixBytes).toString() + + withoutExt = path.join(path.dirname(withoutExt), truncated) + } + + const fullPath = [withoutExt, suffix].join('') + + return fs.pathExists(fullPath) + .then((found) => { + if (found && !overwrite) { + return ensureSafePath(withoutExt, extension, overwrite, num + 1) + } + + // path does not exist, attempt to create it to check for an ENAMETOOLONG error + return fs.outputFileAsync(fullPath, '') + .then(() => fullPath) + .catch((err) => { + if (err.code === 'ENAMETOOLONG' && maxSafePrefixBytes >= MIN_PREFIX_BYTES) { + maxSafeBytes -= 1 + + return ensureSafePath(withoutExt, extension, overwrite, num) + } + + throw err + }) + }) +} + +const sanitizeToString = (title: any, idx: number, arr: Array) => { + // test titles may be values which aren't strings like + // null or undefined - so convert before trying to sanitize + return sanitize(_.toString(title)) +} + +export const getPath = async function (data: PathOptions, ext: string, screenshotsFolder: string, overwrite: boolean): Promise { + let names + const specNames = (data.specName || '') + .split(pathSeparatorRe) + + if (data.name) { + names = data.name.split(pathSeparatorRe).map(sanitizeToString) + } else { + // we put this in array so to match with type of the if branch above + names = [_ + .chain(data.titles) + .map(sanitizeToString) + .join(RUNNABLE_SEPARATOR) + .value()] + } + + const index = names.length - 1 + + // append '(failed)' to the last name + if (data.testFailure) { + names[index] = `${names[index]} (failed)` + } + + if (data.testAttemptIndex > 0) { + names[index] = `${names[index]} (attempt ${data.testAttemptIndex + 1})` + } + + const withoutExt = path.join(screenshotsFolder, ...specNames, ...names) + + return await ensureSafePath(withoutExt, ext, overwrite) +} From 49e69f158433c6becf023736f3ebef62f4b5087d Mon Sep 17 00:00:00 2001 From: Yashodhan Joshi Date: Mon, 25 Nov 2024 20:39:11 +0530 Subject: [PATCH 02/16] chore: update changelog Signed-off-by: Yashodhan Joshi --- cli/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/CHANGELOG.md b/cli/CHANGELOG.md index 6747c81aa385..b0ace64c99fc 100644 --- a/cli/CHANGELOG.md +++ b/cli/CHANGELOG.md @@ -1,5 +1,4 @@ - ## 13.16.1 _Released 11/26/2024 (PENDING)_ @@ -7,6 +6,7 @@ _Released 11/26/2024 (PENDING)_ **Bugfixes:** - Support multiple imports of one module with multiple lines. Addressed in [#30314](https://github.com/cypress-io/cypress/pull/30314). +- Prevent overwriting of video files on across multiple runs. Addresses [#8280](https://github.com/cypress-io/cypress/issues/8280). Addressed in [#30673](https://github.com/cypress-io/cypress/pull/30673). ## 13.16.0 From 170988421bcad6315ddc794b3d6f657469f07985 Mon Sep 17 00:00:00 2001 From: Yashodhan Joshi Date: Tue, 10 Dec 2024 12:47:29 +0530 Subject: [PATCH 03/16] test: add system e2e for the retain videos fix Signed-off-by: Yashodhan Joshi --- system-tests/README.md | 10 +- .../issue-8280-retain-video/cypress.config.js | 6 ++ .../cypress/e2e/spec1.cy.js | 13 +++ .../cypress/e2e/spec2.cy.js | 13 +++ system-tests/test/issue_8280_spec.js | 91 +++++++++++++++++++ 5 files changed, 128 insertions(+), 5 deletions(-) create mode 100644 system-tests/projects/issue-8280-retain-video/cypress.config.js create mode 100644 system-tests/projects/issue-8280-retain-video/cypress/e2e/spec1.cy.js create mode 100644 system-tests/projects/issue-8280-retain-video/cypress/e2e/spec2.cy.js create mode 100644 system-tests/test/issue_8280_spec.js diff --git a/system-tests/README.md b/system-tests/README.md index 1d9d5b278cc0..78eb297a6d62 100644 --- a/system-tests/README.md +++ b/system-tests/README.md @@ -10,23 +10,23 @@ These tests run in CI in Electron, Chrome, Firefox, and WebKit under the `system ## Running System Tests ```bash -yarn test # runs all tests +yarn test-system # runs all tests ## or use globbing to find spec in folders as defined in "glob-in-dir" param in package.json -yarn test screenshot*element # runs screenshot_element_capture_spec.js -yarn test screenshot # runs screenshot_element_capture_spec.js, screenshot_fullpage_capture_spec.js, ..., etc. +yarn test-system screenshot*element # runs screenshot_element_capture_spec.js +yarn test-system screenshot # runs screenshot_element_capture_spec.js, screenshot_fullpage_capture_spec.js, ..., etc. ``` To keep the browser open after a spec run (for easier debugging and iterating on specs), you can pass the `--no-exit` flag to the test command. Live reloading due to spec changes should also work: ```sh -yarn test go_spec.js --browser chrome --no-exit +yarn test-system go_spec.js --browser chrome --no-exit ``` To debug the Cypress process under test, you can pass `--cypress-inspect-brk`: ```sh -yarn test go_spec.js --browser chrome --no-exit --cypress-inspect-brk +yarn test-system go_spec.js --browser chrome --no-exit --cypress-inspect-brk ``` ## Developing Tests diff --git a/system-tests/projects/issue-8280-retain-video/cypress.config.js b/system-tests/projects/issue-8280-retain-video/cypress.config.js new file mode 100644 index 000000000000..56bd22847ce1 --- /dev/null +++ b/system-tests/projects/issue-8280-retain-video/cypress.config.js @@ -0,0 +1,6 @@ +module.exports = { + e2e: { + supportFile: false, + video: true, + }, +} diff --git a/system-tests/projects/issue-8280-retain-video/cypress/e2e/spec1.cy.js b/system-tests/projects/issue-8280-retain-video/cypress/e2e/spec1.cy.js new file mode 100644 index 000000000000..8908e841df82 --- /dev/null +++ b/system-tests/projects/issue-8280-retain-video/cypress/e2e/spec1.cy.js @@ -0,0 +1,13 @@ +// here the delays are just so there is something in the screenshots and recordings. + +describe('spec1', () => { + it('testCase1', () => { + cy.wait(500) + assert(false) + }) + + it('testCase2', () => { + cy.wait(500) + assert(true) + }) +}) diff --git a/system-tests/projects/issue-8280-retain-video/cypress/e2e/spec2.cy.js b/system-tests/projects/issue-8280-retain-video/cypress/e2e/spec2.cy.js new file mode 100644 index 000000000000..4a7edb96217c --- /dev/null +++ b/system-tests/projects/issue-8280-retain-video/cypress/e2e/spec2.cy.js @@ -0,0 +1,13 @@ +// here the delays are just so there is something in the screenshots and recordings. + +describe('spec2', () => { + it('testCase1', () => { + cy.wait(500) + assert(false) + }) + + it('testCase2', () => { + cy.wait(500) + assert(true) + }) +}) diff --git a/system-tests/test/issue_8280_spec.js b/system-tests/test/issue_8280_spec.js new file mode 100644 index 000000000000..a113ee843c44 --- /dev/null +++ b/system-tests/test/issue_8280_spec.js @@ -0,0 +1,91 @@ +const { fs } = require('@packages/server/lib/util/fs') +const Fixtures = require('../lib/fixtures') +const systemTests = require('../lib/system-tests').default + +const PROJECT_NAME = 'issue-8280-retain-video' + +describe('e2e issue 8280', () => { + systemTests.setup() + + // https://github.com/cypress-io/cypress/issues/8280 + + it('should retain the videos from previous runs if trashAssetsBeforeRuns=false', function () { + return systemTests.exec(this, { + project: PROJECT_NAME, + snapshot: false, + expectedExitCode: 2, + processEnv: { + 'CYPRESS_trashAssetsBeforeRuns': 'false', + }, + }) + .then(() => { + return systemTests.exec(this, { + project: PROJECT_NAME, + snapshot: false, + expectedExitCode: 2, + processEnv: { + 'CYPRESS_trashAssetsBeforeRuns': 'false', + }, + }) + }).then(() => { + return fs.readdir(Fixtures.projectPath(`${PROJECT_NAME}/cypress/screenshots/spec1.cy.js`)) + }).then((screenshots) => { + expect(screenshots.length).to.eq(2) + expect(screenshots).to.include('spec1 -- testCase1 (failed).png') + expect(screenshots).to.include('spec1 -- testCase1 (failed) (1).png') + }).then(() => { + return fs.readdir(Fixtures.projectPath(`${PROJECT_NAME}/cypress/screenshots/spec2.cy.js`)) + }).then((screenshots) => { + expect(screenshots.length).to.eq(2) + expect(screenshots).to.include('spec2 -- testCase1 (failed).png') + expect(screenshots).to.include('spec2 -- testCase1 (failed) (1).png') + }) + .then(() => { + return fs.readdir(Fixtures.projectPath(`${PROJECT_NAME}/cypress/videos`)) + }).then((videos) => { + expect(videos.length).to.eq(4) + expect(videos).to.include('spec1.cy.js.mp4') + expect(videos).to.include('spec1.cy.js (1).mp4') + expect(videos).to.include('spec2.cy.js.mp4') + expect(videos).to.include('spec2.cy.js (1).mp4') + }) + }) + + // if trash assets = true, then there will be no retention of screenshots or videos + it('should not retain the videos from previous runs if trashAssetsBeforeRuns=true', function () { + return systemTests.exec(this, { + project: PROJECT_NAME, + snapshot: false, + expectedExitCode: 2, + processEnv: { + 'CYPRESS_trashAssetsBeforeRuns': 'true', + }, + }) + .then(() => { + return systemTests.exec(this, { + project: PROJECT_NAME, + snapshot: false, + expectedExitCode: 2, + processEnv: { + 'CYPRESS_trashAssetsBeforeRuns': 'true', + }, + }) + }).then(() => { + return fs.readdir(Fixtures.projectPath(`${PROJECT_NAME}/cypress/screenshots/spec1.cy.js`)) + }).then((screenshots) => { + expect(screenshots.length).to.eq(1) + expect(screenshots).to.include('spec1 -- testCase1 (failed).png') + }).then(() => { + return fs.readdir(Fixtures.projectPath(`${PROJECT_NAME}/cypress/screenshots/spec2.cy.js`)) + }).then((screenshots) => { + expect(screenshots.length).to.eq(1) + expect(screenshots).to.include('spec2 -- testCase1 (failed).png') + }).then(() => { + return fs.readdir(Fixtures.projectPath(`${PROJECT_NAME}/cypress/videos`)) + }).then((videos) => { + expect(videos.length).to.eq(2) + expect(videos).to.include('spec1.cy.js.mp4') + expect(videos).to.include('spec2.cy.js.mp4') + }) + }) +}) From 4415974551c778236b50e9c7ab57325e2d639c97 Mon Sep 17 00:00:00 2001 From: Jennifer Shehane Date: Fri, 17 Jan 2025 09:16:47 -0500 Subject: [PATCH 04/16] Update cli/CHANGELOG.md Co-authored-by: Mike McCready <66998419+MikeMcC399@users.noreply.github.com> --- cli/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/CHANGELOG.md b/cli/CHANGELOG.md index 227d85b7bd18..6d50ba7108f1 100644 --- a/cli/CHANGELOG.md +++ b/cli/CHANGELOG.md @@ -6,7 +6,7 @@ _Released 12/17/2024 (PENDING)_ **Bugfixes:** - Fixed an issue where targets may hang if `Network.enable` is not implemented for the target. Addresses [#29876](https://github.com/cypress-io/cypress/issues/29876). -- Prevent overwriting of video files across multiple runs. Addresses [#8280](https://github.com/cypress-io/cypress/issues/8280). Addressed in [#30673](https://github.com/cypress-io/cypress/pull/30673). +- Fixed an issue where the configuration setting `trashAssetsBeforeRuns=false` was ignored for assets in the `videosfolder` and these assets were incorrectly deleted before running tests with `cypress run`. Addresses [#8280](https://github.com/cypress-io/cypress/issues/8280). - Updated Firefox `userChrome.css` to correctly hide the toolbox during headless mode. Addresses [#30721](https://github.com/cypress-io/cypress/issues/30721). ## 13.16.1 From 17a25071f84231b9eb1bcf78bddd9abd2d9d5729 Mon Sep 17 00:00:00 2001 From: Jennifer Shehane Date: Thu, 23 Jan 2025 15:08:03 -0500 Subject: [PATCH 05/16] Update types for screenshots to be in util/fs --- packages/server/lib/screenshots.ts | 30 +----------------------- packages/server/lib/util/fs.ts | 37 +++++++++++++++++++++++------- 2 files changed, 30 insertions(+), 37 deletions(-) diff --git a/packages/server/lib/screenshots.ts b/packages/server/lib/screenshots.ts index 69e596447aea..8eb90b5a13c9 100644 --- a/packages/server/lib/screenshots.ts +++ b/packages/server/lib/screenshots.ts @@ -8,6 +8,7 @@ import sizeOf from 'image-size' import colorString from 'color-string' import * as plugins from './plugins' import { fs, getPath } from './util/fs' +import type { Data } from './util/fs' let debug = Debug('cypress:server:screenshot') @@ -16,35 +17,6 @@ let __ID__: string | null = null type ScreenshotsFolder = string | false | undefined -interface Clip { - x: number - y: number - width: number - height: number -} - -// TODO: This is likely not representative of the entire Type and should be updated -interface Data { - specName: string - name: string - startTime: Date - viewport: { - width: number - height: number - } - titles?: string[] - testFailure?: boolean - overwrite?: boolean - simple?: boolean - current?: number - total?: number - testAttemptIndex?: number - appOnly?: boolean - hideRunnerUi?: boolean - clip?: Clip - userClip?: Clip -} - // TODO: This is likely not representative of the entire Type and should be updated interface Details { image: any diff --git a/packages/server/lib/util/fs.ts b/packages/server/lib/util/fs.ts index fb67a8f1e080..367c90125420 100644 --- a/packages/server/lib/util/fs.ts +++ b/packages/server/lib/util/fs.ts @@ -29,13 +29,34 @@ interface PromisifiedFsExtra { outputFileAsync: Promisified } -type PathOptions = { - specName?: string +interface Clip { + x: number + y: number + width: number + height: number +} + +// TODO: This is likely not representative of the entire Type and should be updated +export interface Data { + specName: string name: string - testFailure: boolean - testAttemptIndex: number - titles: Array -}; + startTime: Date + viewport: { + width: number + height: number + } + titles?: string[] + testFailure?: boolean + overwrite?: boolean + simple?: boolean + current?: number + total?: number + testAttemptIndex?: number + appOnly?: boolean + hideRunnerUi?: boolean + clip?: Clip + userClip?: Clip +} export const fs = Bluebird.promisifyAll(fsExtra) as PromisifiedFsExtra & typeof fsExtra @@ -80,7 +101,7 @@ const sanitizeToString = (title: any, idx: number, arr: Array) => { return sanitize(_.toString(title)) } -export const getPath = async function (data: PathOptions, ext: string, screenshotsFolder: string, overwrite: boolean): Promise { +export const getPath = async function (data: Data, ext: string, screenshotsFolder: string, overwrite: boolean): Promise { let names const specNames = (data.specName || '') .split(pathSeparatorRe) @@ -103,7 +124,7 @@ export const getPath = async function (data: PathOptions, ext: string, screensho names[index] = `${names[index]} (failed)` } - if (data.testAttemptIndex > 0) { + if (data.testAttemptIndex && data.testAttemptIndex > 0) { names[index] = `${names[index]} (attempt ${data.testAttemptIndex + 1})` } From 436569a2edd8d824af24d0c24218631b74098b6c Mon Sep 17 00:00:00 2001 From: Jennifer Shehane Date: Thu, 23 Jan 2025 15:10:57 -0500 Subject: [PATCH 06/16] Fix changelog entry placement --- cli/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/CHANGELOG.md b/cli/CHANGELOG.md index 498883c18b66..b7a21bbe3a59 100644 --- a/cli/CHANGELOG.md +++ b/cli/CHANGELOG.md @@ -6,6 +6,7 @@ _Released 1/28/2025 (PENDING)_ **Bugfixes:** - Fixed an issue where Cypress would incorrectly navigate to `about:blank` when test isolation was disabled and the last test would fail and then retry. Fixes [#28527](https://github.com/cypress-io/cypress/issues/28527). +- Fixed an issue where the configuration setting `trashAssetsBeforeRuns=false` was ignored for assets in the `videosfolder` and these assets were incorrectly deleted before running tests with `cypress run`. Addresses [#8280](https://github.com/cypress-io/cypress/issues/8280). ## 14.0.0 @@ -91,7 +92,6 @@ _Released 12/17/2024_ **Bugfixes:** - Fixed an issue where targets may hang if `Network.enable` is not implemented for the target. Addresses [#29876](https://github.com/cypress-io/cypress/issues/29876). -- Fixed an issue where the configuration setting `trashAssetsBeforeRuns=false` was ignored for assets in the `videosfolder` and these assets were incorrectly deleted before running tests with `cypress run`. Addresses [#8280](https://github.com/cypress-io/cypress/issues/8280). - Updated Firefox `userChrome.css` to correctly hide the toolbox during headless mode. Addresses [#30721](https://github.com/cypress-io/cypress/issues/30721). - Fixed an issue loading the `cypress.config.ts` file with Node.js version `22.12.0` if it is loaded as an ESM. Addresses [#30715](https://github.com/cypress-io/cypress/issues/30715). From 166da24b39adb92fea7fc1c1402402b9ce4fbfce Mon Sep 17 00:00:00 2001 From: Jennifer Shehane Date: Thu, 23 Jan 2025 17:08:01 -0500 Subject: [PATCH 07/16] fix extension type --- packages/server/lib/util/fs.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/lib/util/fs.ts b/packages/server/lib/util/fs.ts index 367c90125420..9d46caed4af8 100644 --- a/packages/server/lib/util/fs.ts +++ b/packages/server/lib/util/fs.ts @@ -60,7 +60,7 @@ export interface Data { export const fs = Bluebird.promisifyAll(fsExtra) as PromisifiedFsExtra & typeof fsExtra -const ensureSafePath = async function (withoutExt: string, extension: string, overwrite: boolean, num: number = 0): Promise { +const ensureSafePath = async function (withoutExt: string, extension: string | null, overwrite: boolean, num: number = 0): Promise { const suffix = `${(num && !overwrite) ? ` (${num})` : ''}.${extension}` const maxSafePrefixBytes = maxSafeBytes - suffix.length @@ -101,7 +101,7 @@ const sanitizeToString = (title: any, idx: number, arr: Array) => { return sanitize(_.toString(title)) } -export const getPath = async function (data: Data, ext: string, screenshotsFolder: string, overwrite: boolean): Promise { +export const getPath = async function (data: Data, ext: string | null, screenshotsFolder: string, overwrite: boolean): Promise { let names const specNames = (data.specName || '') .split(pathSeparatorRe) From 8c11c073c8c0aaea46c2138da4ea0b74f21175ca Mon Sep 17 00:00:00 2001 From: Jennifer Shehane Date: Fri, 24 Jan 2025 10:14:01 -0500 Subject: [PATCH 08/16] more types fixes --- packages/server/lib/modes/run.ts | 5 +++-- packages/server/lib/screenshots.ts | 4 +--- packages/server/lib/util/fs.ts | 18 +++++++++++++----- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/packages/server/lib/modes/run.ts b/packages/server/lib/modes/run.ts index 2d39e7f176ad..b12d5e9b9c48 100644 --- a/packages/server/lib/modes/run.ts +++ b/packages/server/lib/modes/run.ts @@ -13,7 +13,7 @@ import Reporter from '../reporter' import browserUtils from '../browsers' import { openProject } from '../open_project' import * as videoCapture from '../video_capture' -import { fs, getPath } from '../util/fs' +import { fs, getPath, type Data } from '../util/fs' import runEvents from '../plugins/run_events' import env from '../util/env' import trash from '../util/trash' @@ -229,7 +229,8 @@ async function startVideoRecording (options: { previous?: VideoRecording, projec async function videoPath (suffix: string, ext: string) { const specPath = options.spec.relativeToCommonRoot + suffix - const data = { + // tslint:disable-next-line + const data: Data = { name: specPath, testFailure: false, testAttemptIndex: 0, diff --git a/packages/server/lib/screenshots.ts b/packages/server/lib/screenshots.ts index 8eb90b5a13c9..cee5d9a5f3e0 100644 --- a/packages/server/lib/screenshots.ts +++ b/packages/server/lib/screenshots.ts @@ -8,15 +8,13 @@ import sizeOf from 'image-size' import colorString from 'color-string' import * as plugins from './plugins' import { fs, getPath } from './util/fs' -import type { Data } from './util/fs' +import type { Data, ScreenshotsFolder } from './util/fs' let debug = Debug('cypress:server:screenshot') // internal id incrementor let __ID__: string | null = null -type ScreenshotsFolder = string | false | undefined - // TODO: This is likely not representative of the entire Type and should be updated interface Details { image: any diff --git a/packages/server/lib/util/fs.ts b/packages/server/lib/util/fs.ts index 9d46caed4af8..42a1e733894e 100644 --- a/packages/server/lib/util/fs.ts +++ b/packages/server/lib/util/fs.ts @@ -36,12 +36,14 @@ interface Clip { height: number } +export type ScreenshotsFolder = string | false | undefined + // TODO: This is likely not representative of the entire Type and should be updated export interface Data { - specName: string name: string startTime: Date - viewport: { + specName?: string + viewport?: { width: number height: number } @@ -60,7 +62,7 @@ export interface Data { export const fs = Bluebird.promisifyAll(fsExtra) as PromisifiedFsExtra & typeof fsExtra -const ensureSafePath = async function (withoutExt: string, extension: string | null, overwrite: boolean, num: number = 0): Promise { +const ensureSafePath = async function (withoutExt: string, extension: string | null, overwrite: boolean | undefined, num: number = 0): Promise { const suffix = `${(num && !overwrite) ? ` (${num})` : ''}.${extension}` const maxSafePrefixBytes = maxSafeBytes - suffix.length @@ -101,7 +103,7 @@ const sanitizeToString = (title: any, idx: number, arr: Array) => { return sanitize(_.toString(title)) } -export const getPath = async function (data: Data, ext: string | null, screenshotsFolder: string, overwrite: boolean): Promise { +export const getPath = async function (data: Data, ext: string | null, screenshotsFolder: ScreenshotsFolder, overwrite: boolean | undefined): Promise { let names const specNames = (data.specName || '') .split(pathSeparatorRe) @@ -128,7 +130,13 @@ export const getPath = async function (data: Data, ext: string | null, screensho names[index] = `${names[index]} (attempt ${data.testAttemptIndex + 1})` } - const withoutExt = path.join(screenshotsFolder, ...specNames, ...names) + let withoutExt + + if (screenshotsFolder) { + withoutExt = path.join(screenshotsFolder, ...specNames, ...names) + } else { + withoutExt = path.join(...specNames, ...names) + } return await ensureSafePath(withoutExt, ext, overwrite) } From 70daf1de0556c18d8e991405389194ae527cb7bf Mon Sep 17 00:00:00 2001 From: Yashodhan Joshi Date: Mon, 31 Mar 2025 20:35:57 +0530 Subject: [PATCH 09/16] fix: add required field in getPath call to satisfy ts Signed-off-by: Yashodhan Joshi --- packages/server/lib/modes/run.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/server/lib/modes/run.ts b/packages/server/lib/modes/run.ts index 2ca67dd881bd..561471bacd56 100644 --- a/packages/server/lib/modes/run.ts +++ b/packages/server/lib/modes/run.ts @@ -232,6 +232,7 @@ async function startVideoRecording (options: { previous?: VideoRecording, projec // tslint:disable-next-line const data: Data = { name: specPath, + startTime: new Date(), // needed for ts-lint testFailure: false, testAttemptIndex: 0, titles: [], From d65ce4c0ca9757c59aec80d600b5f67383eba981 Mon Sep 17 00:00:00 2001 From: Yashodhan Joshi Date: Mon, 31 Mar 2025 20:51:48 +0530 Subject: [PATCH 10/16] fix: sync Data interface from develop branch Signed-off-by: Yashodhan Joshi --- packages/server/lib/modes/run.ts | 7 ++++++- packages/server/lib/util/fs.ts | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/server/lib/modes/run.ts b/packages/server/lib/modes/run.ts index 561471bacd56..4893eadbb528 100644 --- a/packages/server/lib/modes/run.ts +++ b/packages/server/lib/modes/run.ts @@ -233,7 +233,12 @@ async function startVideoRecording (options: { previous?: VideoRecording, projec const data: Data = { name: specPath, startTime: new Date(), // needed for ts-lint - testFailure: false, + viewport: { + width: 0, + height: 0, + }, + specName: '', // this is optional, the getPath will pick up from specPath + testFailure: false, // this is only applicable for screenshot, not for video testAttemptIndex: 0, titles: [], } diff --git a/packages/server/lib/util/fs.ts b/packages/server/lib/util/fs.ts index 42a1e733894e..533803dbedc5 100644 --- a/packages/server/lib/util/fs.ts +++ b/packages/server/lib/util/fs.ts @@ -40,10 +40,10 @@ export type ScreenshotsFolder = string | false | undefined // TODO: This is likely not representative of the entire Type and should be updated export interface Data { + specName: string name: string startTime: Date - specName?: string - viewport?: { + viewport: { width: number height: number } From a3af41acd84d1d156821c42cfca947fde2dc0e7a Mon Sep 17 00:00:00 2001 From: Yashodhan Joshi Date: Mon, 31 Mar 2025 21:08:36 +0530 Subject: [PATCH 11/16] fix: update SavedDetails type to better definition Signed-off-by: Yashodhan Joshi --- packages/server/lib/screenshots.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/server/lib/screenshots.ts b/packages/server/lib/screenshots.ts index cee5d9a5f3e0..ff624162eb12 100644 --- a/packages/server/lib/screenshots.ts +++ b/packages/server/lib/screenshots.ts @@ -27,7 +27,10 @@ interface Details { interface SavedDetails { size?: string takenAt?: Date - dimensions?: string + dimensions?: { + width: number + height: number + } multipart?: any pixelRatio?: number name?: any From 1ed0dc85fd373bb3d6ad97117e8bbaab2a91a145 Mon Sep 17 00:00:00 2001 From: AtofStryker Date: Fri, 18 Apr 2025 12:11:42 -0400 Subject: [PATCH 12/16] update changelog --- cli/CHANGELOG.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/cli/CHANGELOG.md b/cli/CHANGELOG.md index bc1816a6ca67..ee353089528c 100644 --- a/cli/CHANGELOG.md +++ b/cli/CHANGELOG.md @@ -1,4 +1,11 @@ + +## 14.3.2 + +**Bugfixes** + +- Fixed an issue where the configuration setting `trashAssetsBeforeRuns=false` was ignored for assets in the `videosfolder` and these assets were incorrectly deleted before running tests with `cypress run`. Addresses [#8280](https://github.com/cypress-io/cypress/issues/8280). + ## 14.3.1 _Released 4/17/2025_ @@ -32,9 +39,6 @@ _Released 4/8/2025_ - Fixed an issue where Firefox BiDi was prematurely removing prerequests on pending requests. Fixes [#31376](https://github.com/cypress-io/cypress/issues/31376). - Fixed an [issue](https://github.com/electron/electron/issues/45398) with Electron causing slow animations and increased test times by starting a CDP screencast with a noop configuration. Fixes [#30980](https://github.com/cypress-io/cypress/issues/30980). -**Bugfixes** -- Fixed an issue where the configuration setting `trashAssetsBeforeRuns=false` was ignored for assets in the `videosfolder` and these assets were incorrectly deleted before running tests with `cypress run`. Addresses [#8280](https://github.com/cypress-io/cypress/issues/8280). - **Misc:** - Added an automation command for dispatching key press events to CDP and BiDi automated browsers. Addressed in [#31366](https://github.com/cypress-io/cypress/pull/31366). From 7f513cdd0d36a996b7d4f75e6023822b9c344540 Mon Sep 17 00:00:00 2001 From: AtofStryker Date: Fri, 18 Apr 2025 12:16:59 -0400 Subject: [PATCH 13/16] break out type import into unique line to allow mksnapshot to work --- packages/server/lib/modes/run.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/server/lib/modes/run.ts b/packages/server/lib/modes/run.ts index 4893eadbb528..7b02b01f135f 100644 --- a/packages/server/lib/modes/run.ts +++ b/packages/server/lib/modes/run.ts @@ -13,7 +13,7 @@ import Reporter from '../reporter' import browserUtils from '../browsers' import { openProject } from '../open_project' import * as videoCapture from '../video_capture' -import { fs, getPath, type Data } from '../util/fs' +import { fs, getPath } from '../util/fs' import runEvents from '../plugins/run_events' import env from '../util/env' import trash from '../util/trash' @@ -23,6 +23,7 @@ import chromePolicyCheck from '../util/chrome_policy_check' import type { SpecWithRelativeRoot, SpecFile, TestingType, OpenProjectLaunchOpts, FoundBrowser, BrowserVideoController, VideoRecording, ProcessOptions, ProtocolManagerShape, AutomationCommands } from '@packages/types' import type { Cfg, ProjectBase } from '../project-base' import type { Browser } from '../browsers/types' +import type { Data } from '../util/fs' import * as printResults from '../util/print-run' import { telemetry } from '@packages/telemetry' import { CypressRunResult, createPublicBrowser, createPublicConfig, createPublicRunResults, createPublicSpec, createPublicSpecResults } from './results' From 64385d972dcbfd9413884a0a5266ef19e89052a7 Mon Sep 17 00:00:00 2001 From: Yashodhan Joshi Date: Mon, 28 Apr 2025 10:19:56 +0530 Subject: [PATCH 14/16] fix: minor comment fixes Signed-off-by: Yashodhan Joshi --- packages/server/lib/modes/run.ts | 3 +-- .../projects/issue-8280-retain-video/cypress/e2e/spec1.cy.js | 2 +- .../projects/issue-8280-retain-video/cypress/e2e/spec2.cy.js | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/server/lib/modes/run.ts b/packages/server/lib/modes/run.ts index 7b02b01f135f..d6c6002efbdc 100644 --- a/packages/server/lib/modes/run.ts +++ b/packages/server/lib/modes/run.ts @@ -244,8 +244,7 @@ async function startVideoRecording (options: { previous?: VideoRecording, projec titles: [], } - // getPath returns a Promise!!! - return await getPath(data, ext, options.videosFolder, options.overwrite) + return getPath(data, ext, options.videosFolder, options.overwrite) } const videoName = await videoPath('', 'mp4') diff --git a/system-tests/projects/issue-8280-retain-video/cypress/e2e/spec1.cy.js b/system-tests/projects/issue-8280-retain-video/cypress/e2e/spec1.cy.js index 8908e841df82..d46b2c9742a8 100644 --- a/system-tests/projects/issue-8280-retain-video/cypress/e2e/spec1.cy.js +++ b/system-tests/projects/issue-8280-retain-video/cypress/e2e/spec1.cy.js @@ -1,4 +1,4 @@ -// here the delays are just so there is something in the screenshots and recordings. +// here the delays are just so there is something in the screenshots and recordings. describe('spec1', () => { it('testCase1', () => { diff --git a/system-tests/projects/issue-8280-retain-video/cypress/e2e/spec2.cy.js b/system-tests/projects/issue-8280-retain-video/cypress/e2e/spec2.cy.js index 4a7edb96217c..1e54de87ca0f 100644 --- a/system-tests/projects/issue-8280-retain-video/cypress/e2e/spec2.cy.js +++ b/system-tests/projects/issue-8280-retain-video/cypress/e2e/spec2.cy.js @@ -1,4 +1,4 @@ -// here the delays are just so there is something in the screenshots and recordings. +// here the delays are just so there is something in the screenshots and recordings. describe('spec2', () => { it('testCase1', () => { From 07577ed25df157e9f3d8af55df4e44c028ecad54 Mon Sep 17 00:00:00 2001 From: Yashodhan Joshi Date: Mon, 28 Apr 2025 10:27:55 +0530 Subject: [PATCH 15/16] fix: change videoPath fn signature as per comment Signed-off-by: Yashodhan Joshi --- packages/server/lib/modes/run.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/server/lib/modes/run.ts b/packages/server/lib/modes/run.ts index d6c6002efbdc..b8af4ec865c2 100644 --- a/packages/server/lib/modes/run.ts +++ b/packages/server/lib/modes/run.ts @@ -228,7 +228,7 @@ async function trashAssets (config: Cfg) { async function startVideoRecording (options: { previous?: VideoRecording, project: Project, spec: SpecWithRelativeRoot, videosFolder: string, overwrite: boolean }): Promise { if (!options.videosFolder) throw new Error('Missing videoFolder for recording') - async function videoPath (suffix: string, ext: string) { + async function videoPath (ext: string, suffix: string = '') { const specPath = options.spec.relativeToCommonRoot + suffix // tslint:disable-next-line const data: Data = { @@ -247,8 +247,8 @@ async function startVideoRecording (options: { previous?: VideoRecording, projec return getPath(data, ext, options.videosFolder, options.overwrite) } - const videoName = await videoPath('', 'mp4') - const compressedVideoName = await videoPath('-compressed', 'mp4') + const videoName = await videoPath('mp4') + const compressedVideoName = await videoPath('mp4', '-compressed') const outputDir = path.dirname(videoName) From 95f04f01e51eb62d510488cf129b0f1e21df0cc8 Mon Sep 17 00:00:00 2001 From: Yashodhan Joshi Date: Mon, 28 Apr 2025 10:37:34 +0530 Subject: [PATCH 16/16] fix: convert the test to async/await Signed-off-by: Yashodhan Joshi --- system-tests/test/issue_8280_spec.js | 119 ++++++++++++++------------- 1 file changed, 60 insertions(+), 59 deletions(-) diff --git a/system-tests/test/issue_8280_spec.js b/system-tests/test/issue_8280_spec.js index a113ee843c44..edbccbfa768d 100644 --- a/system-tests/test/issue_8280_spec.js +++ b/system-tests/test/issue_8280_spec.js @@ -9,8 +9,9 @@ describe('e2e issue 8280', () => { // https://github.com/cypress-io/cypress/issues/8280 - it('should retain the videos from previous runs if trashAssetsBeforeRuns=false', function () { - return systemTests.exec(this, { + it('should retain the videos from previous runs if trashAssetsBeforeRuns=false', async function () { + // first run + await systemTests.exec(this, { project: PROJECT_NAME, snapshot: false, expectedExitCode: 2, @@ -18,42 +19,42 @@ describe('e2e issue 8280', () => { 'CYPRESS_trashAssetsBeforeRuns': 'false', }, }) - .then(() => { - return systemTests.exec(this, { - project: PROJECT_NAME, - snapshot: false, - expectedExitCode: 2, - processEnv: { - 'CYPRESS_trashAssetsBeforeRuns': 'false', - }, - }) - }).then(() => { - return fs.readdir(Fixtures.projectPath(`${PROJECT_NAME}/cypress/screenshots/spec1.cy.js`)) - }).then((screenshots) => { - expect(screenshots.length).to.eq(2) - expect(screenshots).to.include('spec1 -- testCase1 (failed).png') - expect(screenshots).to.include('spec1 -- testCase1 (failed) (1).png') - }).then(() => { - return fs.readdir(Fixtures.projectPath(`${PROJECT_NAME}/cypress/screenshots/spec2.cy.js`)) - }).then((screenshots) => { - expect(screenshots.length).to.eq(2) - expect(screenshots).to.include('spec2 -- testCase1 (failed).png') - expect(screenshots).to.include('spec2 -- testCase1 (failed) (1).png') - }) - .then(() => { - return fs.readdir(Fixtures.projectPath(`${PROJECT_NAME}/cypress/videos`)) - }).then((videos) => { - expect(videos.length).to.eq(4) - expect(videos).to.include('spec1.cy.js.mp4') - expect(videos).to.include('spec1.cy.js (1).mp4') - expect(videos).to.include('spec2.cy.js.mp4') - expect(videos).to.include('spec2.cy.js (1).mp4') + + // second run + await systemTests.exec(this, { + project: PROJECT_NAME, + snapshot: false, + expectedExitCode: 2, + processEnv: { + 'CYPRESS_trashAssetsBeforeRuns': 'false', + }, }) + + const spec1Screenshots = await fs.readdir(Fixtures.projectPath(`${PROJECT_NAME}/cypress/screenshots/spec1.cy.js`)) + + expect(spec1Screenshots.length).to.eq(2) + expect(spec1Screenshots).to.include('spec1 -- testCase1 (failed).png') + expect(spec1Screenshots).to.include('spec1 -- testCase1 (failed) (1).png') + + const spec2Screenshots = await fs.readdir(Fixtures.projectPath(`${PROJECT_NAME}/cypress/screenshots/spec2.cy.js`)) + + expect(spec2Screenshots.length).to.eq(2) + expect(spec2Screenshots).to.include('spec2 -- testCase1 (failed).png') + expect(spec2Screenshots).to.include('spec2 -- testCase1 (failed) (1).png') + + const videos = await fs.readdir(Fixtures.projectPath(`${PROJECT_NAME}/cypress/videos`)) + + expect(videos.length).to.eq(4) + expect(videos).to.include('spec1.cy.js.mp4') + expect(videos).to.include('spec1.cy.js (1).mp4') + expect(videos).to.include('spec2.cy.js.mp4') + expect(videos).to.include('spec2.cy.js (1).mp4') }) // if trash assets = true, then there will be no retention of screenshots or videos - it('should not retain the videos from previous runs if trashAssetsBeforeRuns=true', function () { - return systemTests.exec(this, { + it('should not retain the videos from previous runs if trashAssetsBeforeRuns=true', async function () { + // first run + await systemTests.exec(this, { project: PROJECT_NAME, snapshot: false, expectedExitCode: 2, @@ -61,31 +62,31 @@ describe('e2e issue 8280', () => { 'CYPRESS_trashAssetsBeforeRuns': 'true', }, }) - .then(() => { - return systemTests.exec(this, { - project: PROJECT_NAME, - snapshot: false, - expectedExitCode: 2, - processEnv: { - 'CYPRESS_trashAssetsBeforeRuns': 'true', - }, - }) - }).then(() => { - return fs.readdir(Fixtures.projectPath(`${PROJECT_NAME}/cypress/screenshots/spec1.cy.js`)) - }).then((screenshots) => { - expect(screenshots.length).to.eq(1) - expect(screenshots).to.include('spec1 -- testCase1 (failed).png') - }).then(() => { - return fs.readdir(Fixtures.projectPath(`${PROJECT_NAME}/cypress/screenshots/spec2.cy.js`)) - }).then((screenshots) => { - expect(screenshots.length).to.eq(1) - expect(screenshots).to.include('spec2 -- testCase1 (failed).png') - }).then(() => { - return fs.readdir(Fixtures.projectPath(`${PROJECT_NAME}/cypress/videos`)) - }).then((videos) => { - expect(videos.length).to.eq(2) - expect(videos).to.include('spec1.cy.js.mp4') - expect(videos).to.include('spec2.cy.js.mp4') + + // second run + await systemTests.exec(this, { + project: PROJECT_NAME, + snapshot: false, + expectedExitCode: 2, + processEnv: { + 'CYPRESS_trashAssetsBeforeRuns': 'true', + }, }) + + const spec1Screenshots = await fs.readdir(Fixtures.projectPath(`${PROJECT_NAME}/cypress/screenshots/spec1.cy.js`)) + + expect(spec1Screenshots.length).to.eq(1) + expect(spec1Screenshots).to.include('spec1 -- testCase1 (failed).png') + + const spec2Screenshots = await fs.readdir(Fixtures.projectPath(`${PROJECT_NAME}/cypress/screenshots/spec2.cy.js`)) + + expect(spec2Screenshots.length).to.eq(1) + expect(spec2Screenshots).to.include('spec2 -- testCase1 (failed).png') + + const videos = await fs.readdir(Fixtures.projectPath(`${PROJECT_NAME}/cypress/videos`)) + + expect(videos.length).to.eq(2) + expect(videos).to.include('spec1.cy.js.mp4') + expect(videos).to.include('spec2.cy.js.mp4') }) })