diff --git a/.gitattributes b/.gitattributes index 441b9c43ff..2c38460da0 100644 --- a/.gitattributes +++ b/.gitattributes @@ -16,3 +16,6 @@ CMakeLists.txt text eol=lf # so the next one-line edit showed up as a whole-file rewrite in two locales. *.md text eol=lf *.mdx text eol=lf + +# Adopted measurement manifests hash exact bytes; keep their artifacts unchanged. +workbench/measurements/** -text diff --git a/.github/scripts/previous-stable-tag.mjs b/.github/scripts/previous-stable-tag.mjs new file mode 100644 index 0000000000..6b42864e96 --- /dev/null +++ b/.github/scripts/previous-stable-tag.mjs @@ -0,0 +1,41 @@ +// Prints the tag a release's notes start from: the highest stable tag (vX.Y.Z, +// no suffix) below the given version, by SemVer, or an empty line when there is +// none. The tags come on stdin as `git tag -l 'v*'` lists them: every tag, +// whether or not it is reachable from the release commit. +// +// build.yml used to guess that tag's name by arithmetic and fall back to +// `git describe` when the guess did not exist (#1002). 2.0.0 guessed v1.0.0, +// which never existed, and `git describe` on release/v2.0.0 found the release's +// own last RC: v2.0.0 shipped with an empty changelog. A guess that did exist +// could still be wrong: 1.13.0 compared against v1.12.0 and repeated the PRs of +// v1.12.1 and v1.12.2. Only a tag that exists can come out of here, which also +// keeps the RC notes' `git log ..` from dying on `unknown revision`, as +// v1.9.4-rc.1's did when the guess named a 1.9.3 that never shipped. + +import { readFileSync } from "node:fs"; +import { argv } from "node:process"; + +const STABLE_TAG = /^v(\d+)\.(\d+)\.(\d+)$/; + +const parse = (tag) => tag.match(STABLE_TAG)?.slice(1).map(Number); +const compare = (a, b) => a[0] - b[0] || a[1] - b[1] || a[2] - b[2]; + +/** + * @param {string} version Stable version being released, e.g. "2.0.0". + * @param {string[]} tags Tag names; anything but vX.Y.Z is ignored. + * @returns {string} The highest vX.Y.Z below `version`, or "" when there is none. + */ +export function previousStableTag(version, tags) { + const current = parse(`v${version}`); + if (!current) throw new Error(`invalid version ${JSON.stringify(version)}; expected X.Y.Z`); + const below = tags + .map((tag) => [tag, parse(tag)]) + .filter(([, parts]) => parts && compare(parts, current) < 0); + below.sort(([, a], [, b]) => compare(b, a)); + return below[0]?.[0] ?? ""; +} + +// Only run when invoked directly, so the test can import the function. +if (import.meta.filename === argv[1]) { + console.log(previousStableTag(argv[2], readFileSync(0, "utf8").split(/\s+/))); +} diff --git a/.github/scripts/previous-stable-tag.test.mjs b/.github/scripts/previous-stable-tag.test.mjs new file mode 100644 index 0000000000..38be7047cb --- /dev/null +++ b/.github/scripts/previous-stable-tag.test.mjs @@ -0,0 +1,78 @@ +import { execFileSync } from "node:child_process"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { previousStableTag } from "./previous-stable-tag.mjs"; + +// Real tags of this repository, in the order `git tag -l` prints them: by name, so v1.9.x +// comes after v1.13.0 and picking by position or as text gets it wrong. The v0.0.0-* tags +// host release assets; `git describe` once picked one of them for v2.0.0-rc.1's notes. +const TAGS = [ + "v0.0.0", + "v0.0.0-onnxruntime-1.27.1", + "v1.0.1", + "v1.10.0", + "v1.12.0", + "v1.12.1", + "v1.12.1-rc.1", + "v1.12.2", + "v1.12.2-rc.1", + "v1.13.0", + "v1.13.0-rc.6", + "v1.9.2", + "v1.9.3-rc.1", + "v1.9.6", + "v2.0.0", + "v2.0.0-rc.15", +]; + +describe("previousStableTag", () => { + it("compares a major against the last stable of the line before", () => { + // The arithmetic guessed v1.0.0, which does not exist, and the fallback found + // v2.0.0-rc.15: v2.0.0 shipped with an empty changelog (#1002). + expect(previousStableTag("2.0.0", TAGS)).toBe("v1.13.0"); + }); + + it("compares a minor against the last patch, not the .0", () => { + // Against v1.12.0, 1.13.0's notes repeated every PR of v1.12.1 and v1.12.2. + expect(previousStableTag("1.13.0", TAGS)).toBe("v1.12.2"); + }); + + it("compares a patch against the patch before it", () => { + expect(previousStableTag("1.12.2", TAGS)).toBe("v1.12.1"); + }); + + it("is empty when no stable tag sorts below", () => { + // A first release: its RCs exist, nothing before them does. + expect(previousStableTag("1.0.0", ["v1.0.0", "v1.0.0-rc.1", "v1.0.0-rc.2"])).toBe(""); + }); + + it("ignores pre-release tags", () => { + // 1.9.3 shipped only as rc.1, so the last release before 1.9.4 is v1.9.2. + expect(previousStableTag("1.9.4", TAGS)).toBe("v1.9.2"); + }); + + it("rejects a version that is not X.Y.Z", () => { + expect(() => previousStableTag("2.0.0-rc.1", TAGS)).toThrow(/invalid version/); + }); + + // The direct-invocation guard makes the script print nothing at all if it ever stops + // matching, and the workflow would read that as "no previous release". So this runs it + // exactly as build.yml does: relative path from the repo root, the tags on stdin. + it("runs the way build.yml invokes it", () => { + const repoRoot = join(import.meta.dirname, "../.."); + const workflow = readFileSync(join(repoRoot, ".github/workflows/build.yml"), "utf8"); + expect(workflow).toContain( + `git tag -l 'v*' | node .github/scripts/previous-stable-tag.mjs "$STABLE_VERSION"`, + ); + + const run = (version) => + execFileSync("node", [".github/scripts/previous-stable-tag.mjs", version], { + cwd: repoRoot, + input: `${TAGS.join("\n")}\n`, + encoding: "utf8", + }); + expect(run("2.0.0")).toBe("v1.13.0\n"); + expect(run("0.0.0")).toBe("\n"); + }); +}); diff --git a/.github/scripts/refresh-nix-npm-hash.sh b/.github/scripts/refresh-nix-npm-hash.sh new file mode 100755 index 0000000000..f9c9c8c48f --- /dev/null +++ b/.github/scripts/refresh-nix-npm-hash.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Rewrites `npmDepsHash` in nix/package.nix for the package-lock.json in the working tree. +# +# Called by prerelease.yml and promote.yml right after set-release-version.mjs, in the same +# commit. The lockfile's root `version` is part of what the hash covers, so every version bump +# changes it; that commit is `[skip ci]`, and refreshing the hash anywhere later left `nix build` +# red from the release until a separate bump PR landed. Needs Nix on PATH. +set -euo pipefail + +hash=$(nix run nixpkgs#prefetch-npm-deps -- package-lock.json) +if [[ -z "$hash" ]]; then + echo "::error::prefetch-npm-deps returned an empty hash" + exit 1 +fi +sed -i -E "s|^([[:space:]]*npmDepsHash[[:space:]]*=[[:space:]]*)\"[^\"]*\";|\1\"${hash}\";|" nix/package.nix +grep -q "npmDepsHash = \"${hash}\";" nix/package.nix || { + echo "::error::npmDepsHash line not found in nix/package.nix" + exit 1 +} +echo "npmDepsHash: ${hash}" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 387bed2b96..4fa82600b4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -42,8 +42,18 @@ concurrency: jobs: build-windows: - name: Windows installer - runs-on: windows-latest + name: Windows ${{ matrix.arch }} installer + # Each arch builds NATIVELY, like the macOS job below. On an x64 runner every step + # that keys off the host — fetch-ffmpeg (which also runs the binary it downloads to + # verify the licence), fetch-onnxruntime, the compositor's cargo build — produced x64 + # output for the arm64 package, and before-pack then refused it. On windows-11-arm + # `process.arch` is arm64 and the same steps that build the arm64 installer on a + # Snapdragon desktop run unchanged. + runs-on: ${{ matrix.arch == 'arm64' && 'windows-11-arm' || 'windows-latest' }} + strategy: + fail-fast: false + matrix: + arch: [x64, arm64] steps: - name: Checkout code uses: actions/checkout@v7 @@ -51,42 +61,99 @@ jobs: - name: Setup Node.js uses: ./.github/actions/setup + - name: Ensure MSVC ARM64 build tools + if: matrix.arch == 'arm64' + shell: pwsh + run: | + $vswhere = "C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe" + $installPath = & $vswhere -latest -products * -property installationPath + $hasArm64 = & $vswhere -latest -products * ` + -requires Microsoft.VisualStudio.Component.VC.Tools.ARM64 ` + -property installationPath + if (-not $hasArm64) { + Write-Host "Installing MSVC ARM64 build tools component..." + $installer = "C:\Program Files (x86)\Microsoft Visual Studio\Installer\vs_installer.exe" + & $installer modify --installPath "$installPath" ` + --add Microsoft.VisualStudio.Component.VC.Tools.ARM64 ` + --quiet --norestart --wait + } else { + Write-Host "MSVC ARM64 build tools already present." + } + + # Everything below resolves its target from the host. Should this job ever land on an + # x64 runner again, fail here rather than after a full build that before-pack rejects. + - name: Check the runner architecture + shell: bash + run: | + HOST="$(node -p process.arch)" + [ "$HOST" = "${{ matrix.arch }}" ] \ + || { echo "::error::the ${{ matrix.arch }} installer must build on a ${{ matrix.arch }} runner, got $HOST"; exit 1; } + # STT is the bundled whisper-stt-server (whisper.cpp with native DTW token - # timestamps); no VAD model is fetched here. The binary is built by - # build-whisper-stt.yml and staged below — without that step the installer - # ships without speech-to-text. See - # technical-documentation/architecture/transcription-and-captions.md. + # timestamps); no model is fetched here. The binary is built by + # build-whisper-stt.yml and staged below, from the run built from this + # commit's helper sources — without that step the installer ships without + # speech-to-text. See + # technical-documentation/architecture/transcription-and-captions.md, and + # technical-documentation/engineering/release-and-secrets.md for the run. - name: Stage whisper-stt binaries shell: bash env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # win32-x64 for BOTH arches, deliberately: build-whisper-stt.yml produces no + # arm64 helper yet (scripts/build-whisper-stt.sh can build one natively on ARM, + # but no workflow runs it), so an arch-parameterised tag would fail the arm64 job. + # `candidateBinaryPaths` falls back to the win32-x64 directory on Windows on + # ARM, where the x64 helper runs under emulation — verified on a Snapdragon X + # Elite, Vulkan on the Adreno included. Point this at win32-arm64 the moment a + # native helper exists; the resolver already prefers it. run: bash scripts/stage-whisper-stt.sh win32-x64 - - name: Build Windows app + - name: Build Windows app (x64) + if: matrix.arch == 'x64' run: npm run build:win -- --publish never + - name: Build Windows app (arm64) + if: matrix.arch == 'arm64' + run: npm run build:win:arm64 -- --publish never + + # electron-builder writes a latest.yml per job that lists only that job's installer, and + # both land on the same path in the release — the second would overwrite the first and + # serve one arch the other's build. Same problem, same fix as macOS: each job describes + # its installer in a JSON sidecar and publish-release folds both into ONE latest.yml. + # latest.yml is still required here: it proves electron-builder's publish config + # produced a feed at all. + - name: Describe the installer for the update feed + shell: bash + run: | + test -f "$(find release -name latest.yml | head -1)" \ + || { echo "::error::latest.yml missing — electron-updater has no feed to read"; exit 1; } + EXE="$(find release -name 'Openscreen.Setup.*-${{ matrix.arch }}.exe' | head -1)" + test -f "$EXE" \ + || { echo "::error::no ${{ matrix.arch }} installer under release/"; exit 1; } + VERSION="$(node -p "require('./package.json').version")" + node scripts/mac-update-feed.mjs describe "$EXE" "$VERSION" \ + "$(dirname "$EXE")/update-info-win-${{ matrix.arch }}.json" + - name: Upload Windows installer uses: actions/upload-artifact@v7 with: - name: openscreen-windows - # latest.yml is the update feed electron-updater reads; the .blockmap is what lets it - # download a delta instead of the full ~243 MB installer. Both were already produced - # by every build and thrown away here, because this glob only matched the .exe. + name: openscreen-windows-${{ matrix.arch }} + # The sidecar becomes latest.yml in publish-release; the .blockmap is what lets + # electron-updater download a delta instead of the full ~243 MB installer. path: | release/**/Openscreen.Setup.*.exe release/**/Openscreen.Setup.*.exe.blockmap - release/**/latest.yml + release/**/update-info-win-*.json if-no-files-found: error retention-days: 30 # `if-no-files-found: error` evaluates the UNION of the globs above, so a dead pattern # among live ones never fails — that is exactly how the *.zsync glob rotted unnoticed on - # the Linux job. Assert the update feed specifically. - - name: Verify the update feed was produced + # the Linux job. Assert the blockmap specifically. + - name: Verify the blockmap was produced shell: bash run: | - test -f "$(find release -name latest.yml | head -1)" \ - || { echo "::error::latest.yml missing — electron-updater has no feed to read"; exit 1; } test -f "$(find release -name 'Openscreen.Setup.*.exe.blockmap' | head -1)" \ || { echo "::error::blockmap missing — differential updates would silently degrade"; exit 1; } @@ -376,6 +443,13 @@ jobs: - name: Stage ONNX Runtime run: npm run fetch:onnxruntime + # Fourth step spelled out for the same reason: `build:mac`, `build:win` and + # `build:linux` chain it before electron-builder. It writes the licence texts of the + # bundled npm packages and of the crates in the addon; scripts/before-pack.cjs + # refuses to package without them. + - name: Generate third-party licences + run: npm run licenses:generate + - name: Package .app bundle run: npx electron-builder --mac --${{ matrix.arch }} --dir --publish never env: @@ -649,9 +723,9 @@ jobs: # bsdtar (libarchive-tools) is fpm's mtree generator for the pacman target. # rpmbuild (rpm) is what fpm shells out to for the rpm target; without it the - # build fails at packaging, not at config parse. + # build fails at packaging, not at config parse. zsync generates AppImageUpdate sidecars. - name: Install Linux packaging dependencies - run: sudo apt-get update && sudo apt-get install -y libarchive-tools rpm + run: sudo apt-get update && sudo apt-get install -y libarchive-tools rpm zsync # patchelf is what build-linux-compositor-addon.mjs renames the ffmpeg symbols # with, so the addon cannot bind to Chromium's bundled ffmpeg — an unconditional @@ -706,6 +780,21 @@ jobs: exit 1 fi + - name: Verify AppImage update information and sidecars + run: | + VERSION="$(node -p 'require("./package.json").version')" + EXPECTED='gh-releases-zsync|getopenscreen|openscreen|latest|Openscreen-Linux-*.AppImage.zsync' + # Helper-only prereleases can be selected by AppImageUpdate's latest-all channel. + # RCs keep using electron-updater; their .zsync is available by its explicit URL. + if [[ "$VERSION" == *-* ]]; then EXPECTED=''; fi + while IFS= read -r -d '' IMAGE; do + test -s "${IMAGE}.zsync" \ + || { echo "::error::${IMAGE}.zsync missing — external AppImage updates have no sidecar"; exit 1; } + INFO="$("$IMAGE" --appimage-updateinformation)" + [[ "$INFO" == "$EXPECTED" ]] \ + || { echo "::error::Unexpected AppImage update information in ${IMAGE}: ${INFO}"; exit 1; } + done < <(find release -type f -name '*.AppImage' -print0) + # The Linux counterpart of the Windows job's "Verify native binaries load under # package identity", added for the same reason and after the same kind of miss: # 1.9.1 shipped three sonames that nothing declared and nothing bundled — libgbm.so.1 @@ -745,16 +834,12 @@ jobs: name: openscreen-linux path: | release/**/*.AppImage + release/**/*.AppImage.zsync release/**/*.deb release/**/*.pacman release/**/*.rpm release/**/latest-linux.yml - # Still no *.zsync, and there never will be: zsync was electron-updater's old delta - # format and app-builder-lib 26.x dropped it in favour of the block map it embeds - # directly in the AppImage. That glob had matched nothing since the dependency bump, - # silently — `if-no-files-found: error` evaluates the union of these patterns, so one - # dead glob among live ones never fails. Hence the explicit assertion below. - # + # .zsync serves external AppImage tools; electron-updater uses the embedded blockmap. # latest-linux.yml serves all four formats from one file: each updater picks its own # extension out of the `files:` list. app-update.yml and the `package-type` marker are # already inside the deb/rpm/pacman payloads — only this feed was missing. @@ -778,8 +863,8 @@ jobs: - name: Checkout code uses: actions/checkout@v7 with: - # Full history + tags: the RC notes below are built from `git log` over the - # range since the previous RC tag, and resolving that tag needs the tags. + # Full history + tags: the previous release's tag is picked from every tag in the + # repo, and the RC notes below are built from `git log` over the range since it. fetch-depth: 0 - name: Resolve release tag @@ -818,16 +903,11 @@ jobs: # Compute the previous stable tag for auto-generated release notes. We don't use # GitHub's "most recent prior release by date" because the fork carries re-published # upstream releases whose published_at is more recent than the fork's own first release. - # For SemVer X.Y.Z: previous is vX.Y.(Z-1) if Z>0, else vX.(Y-1).0, else v(X-1).0.0. + # Picked by SemVer among the tags that exist, never guessed from the version number: + # the script says what guessing cost (#1002). Left EMPTY when no stable tag sorts + # below this version; each consumer below decides what "no previous release" means. STABLE_VERSION="${VERSION%%-*}" - IFS='.' read -r PX PY PZ <<< "$STABLE_VERSION" - if (( PZ > 0 )); then - NOTES_START_TAG="v${PX}.${PY}.$((PZ - 1))" - elif (( PY > 0 )); then - NOTES_START_TAG="v${PX}.$((PY - 1)).0" - else - NOTES_START_TAG="v$((PX - 1)).0.0" - fi + NOTES_START_TAG="$(git tag -l 'v*' | node .github/scripts/previous-stable-tag.mjs "$STABLE_VERSION")" # For an RC, compare against the PREVIOUS RC of the same line, not the previous # stable. Deriving the start tag from STABLE_VERSION alone made every RC of a # line span the same range, so each re-cut just repeated the last RC's notes @@ -843,27 +923,6 @@ jobs: fi done fi - # Everything above computes what the previous release was *called* and - # never checks that it exists. A version line that stopped at its RC - # makes that a name for nothing: 1.9.3 shipped only as rc.1, so - # v1.9.4-rc.1 asked git for v1.9.3..v1.9.4-rc.1 and the publish step - # died on `unknown revision` -- after all four platforms had already - # built, and with publish-msstore sitting behind publish-release, so - # the same gap would silently block a stable release's Store - # deployment too. The rc walk-down above handles a skipped RC; nothing - # handled a skipped stable, and it does not run at all for an rc.1. - if ! git rev-parse -q --verify "refs/tags/${NOTES_START_TAG}" >/dev/null; then - # The nearest tag reachable from the release commit's parent, which - # is what "since the last release" meant in the first place. Left - # EMPTY when no tag is reachable at all, rather than filled with the - # root commit: the stable path hands this to `gh release create - # --notes-start-tag`, which is the API's previous_tag_name and takes - # a tag NAME -- a commit SHA there is not a lenient fallback, it is - # an invalid argument. Each consumer below decides what "no previous - # release" means for it. - NOTES_START_TAG="$(git describe --tags --abbrev=0 "${TAG}^" 2>/dev/null || true)" - echo "Previous-release tag did not exist; using ${NOTES_START_TAG:-}" - fi echo "Computed notes_start_tag=${NOTES_START_TAG} for tag=${TAG}" echo "tag=$TAG" >> "$GITHUB_OUTPUT" @@ -873,11 +932,20 @@ jobs: echo "prerelease_flag=$PRERELEASE_FLAG" >> "$GITHUB_OUTPUT" echo "notes_start_tag=$NOTES_START_TAG" >> "$GITHUB_OUTPUT" - - name: Download Windows installer + # By name, not `pattern: openscreen-windows-*`: that pattern also matches + # openscreen-windows-store, and publish-release does not wait for the Store job, so + # whether its .appx reached the GitHub release depended on which job finished first. + - name: Download Windows x64 installer + uses: actions/download-artifact@v8 + with: + name: openscreen-windows-x64 + path: artifacts/windows-x64 + + - name: Download Windows arm64 installer uses: actions/download-artifact@v8 with: - name: openscreen-windows - path: artifacts/windows + name: openscreen-windows-arm64 + path: artifacts/windows-arm64 - name: Download macOS arm64 DMG uses: actions/download-artifact@v8 @@ -917,6 +985,22 @@ jobs: grep -q 'arm64' artifacts/latest-mac.yml \ || { echo "::error::latest-mac.yml has no arm64 entry — Apple Silicon would update onto the Intel build"; exit 1; } + # Same fold for Windows (see "Describe the installer for the update feed"). The feed's + # arch-blind `path:` points at x64, which Windows on ARM still runs under emulation. + - name: Build the Windows update feed + run: | + X64_INFO="$(find artifacts/windows-x64 -name update-info-win-x64.json)" + ARM64_INFO="$(find artifacts/windows-arm64 -name update-info-win-arm64.json)" + node scripts/mac-update-feed.mjs merge "$X64_INFO" "$ARM64_INFO" artifacts/latest.yml + rm -f "$X64_INFO" "$ARM64_INFO" + ENTRIES="$(grep -c '^ - url:' artifacts/latest.yml)" + [ "$ENTRIES" -eq 2 ] \ + || { echo "::error::latest.yml lists ${ENTRIES} installers, expected 2 (one per arch)"; exit 1; } + grep -q '^ - url: .*-x64\.exe$' artifacts/latest.yml \ + || { echo "::error::latest.yml has no x64 installer"; exit 1; } + grep -q '^ - url: .*-arm64\.exe$' artifacts/latest.yml \ + || { echo "::error::latest.yml has no arm64 installer"; exit 1; } + - name: Publish release assets env: GH_TOKEN: ${{ secrets.OPENSCREEN_RELEASE_TOKEN }} diff --git a/.github/workflows/bump-nix-package.yml b/.github/workflows/bump-nix-package.yml deleted file mode 100644 index 5090b184cf..0000000000 --- a/.github/workflows/bump-nix-package.yml +++ /dev/null @@ -1,123 +0,0 @@ -name: Bump Nix package on release - -on: - release: - types: [published] - workflow_dispatch: - inputs: - tag: - description: "Release tag to bump (e.g. v1.5.0)" - required: true - type: string - -# GITHUB_TOKEN only pushes the branch here — the PR itself is opened with the -# PAT below, so no `pull-requests: write` is needed (and it never worked). -permissions: - contents: write - -jobs: - bump: - runs-on: ubuntu-latest - if: github.event_name == 'workflow_dispatch' || !github.event.release.prerelease - steps: - - name: Resolve tag and version - id: meta - env: - GH_EVENT_TAG: ${{ github.event.release.tag_name }} - INPUT_TAG: ${{ inputs.tag }} - run: | - set -euo pipefail - TAG="${GH_EVENT_TAG:-$INPUT_TAG}" - if [[ -z "$TAG" ]]; then - echo "::error::No tag resolved from release event or workflow input" - exit 1 - fi - VERSION="${TAG#v}" - BRANCH="chore/bump-nix-${VERSION}" - echo "tag=$TAG" >> "$GITHUB_OUTPUT" - echo "version=$VERSION" >> "$GITHUB_OUTPUT" - echo "branch=$BRANCH" >> "$GITHUB_OUTPUT" - - - name: Checkout main - uses: actions/checkout@v7 - with: - ref: main - fetch-depth: 0 - - - name: Install Nix - uses: cachix/install-nix-action@v27 - with: - nix_path: nixpkgs=channel:nixos-unstable - extra_nix_config: | - experimental-features = nix-command flakes - - - name: Compute npmDepsHash - id: hash - run: | - set -euo pipefail - HASH=$(nix run nixpkgs#prefetch-npm-deps -- package-lock.json) - if [[ -z "$HASH" ]]; then - echo "::error::prefetch-npm-deps returned an empty hash" - exit 1 - fi - echo "hash=$HASH" >> "$GITHUB_OUTPUT" - echo "Computed npmDepsHash: $HASH" - - - name: Update nix/package.nix - env: - VERSION: ${{ steps.meta.outputs.version }} - HASH: ${{ steps.hash.outputs.hash }} - run: | - set -euo pipefail - # Update version line: ` version = "";` - sed -i -E "s|^([[:space:]]*version[[:space:]]*=[[:space:]]*)\"[^\"]*\";|\1\"${VERSION}\";|" nix/package.nix - # Update npmDepsHash line: ` npmDepsHash = "";` - sed -i -E "s|^([[:space:]]*npmDepsHash[[:space:]]*=[[:space:]]*)\"[^\"]*\";|\1\"${HASH}\";|" nix/package.nix - - echo "=== diff ===" - git --no-pager diff nix/package.nix || true - - - name: Create PR - env: - # NOT GITHUB_TOKEN: the repo has "Allow GitHub Actions to create and - # approve pull requests" turned off, so `gh pr create` dies with - # "GitHub Actions is not permitted to create or approve pull requests" - # — it did exactly that on v1.7.0, after pushing the branch, and #136 - # had to be opened by hand. The PAT every other release workflow - # already uses has no such restriction, and its PRs trigger CI. - GH_TOKEN: ${{ secrets.OPENSCREEN_RELEASE_TOKEN }} - VERSION: ${{ steps.meta.outputs.version }} - HASH: ${{ steps.hash.outputs.hash }} - BRANCH: ${{ steps.meta.outputs.branch }} - TAG: ${{ steps.meta.outputs.tag }} - run: | - set -euo pipefail - - if git diff --quiet -- nix/package.nix; then - echo "nix/package.nix already at v${VERSION} with this hash — nothing to do." - exit 0 - fi - - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - - # Replace any prior bump branch to keep the workflow idempotent. - git push origin --delete "$BRANCH" 2>/dev/null || true - git checkout -b "$BRANCH" - git add nix/package.nix - git commit -m "chore: bump nix package to v${VERSION}" - git push -u origin "$BRANCH" - - gh pr create \ - --title "chore: bump nix package to v${VERSION}" \ - --base main \ - --head "$BRANCH" \ - --body "$(cat </public/, and + # a tree missing there degrades the editor without failing anything: the + # cursor style picker rendered empty (#1085). Every entry of the source + # public/ must be there. + - name: Check the renderer's public assets + run: | + set -euo pipefail + missing=0 + for entry in public/*; do + if [ ! -e "result/lib/openscreen/$entry" ]; then + echo "::error::$entry is missing from the package" + missing=$((missing + 1)) + fi + done + [ "$missing" -eq 0 ] + # The native components are resolved at runtime behind existence checks, so # a wrapper pointing at a path that is not there degrades the app instead of # failing it -- silently, which is the failure mode this whole workflow was @@ -268,6 +289,10 @@ jobs: check_dlopen_rpath "PipeWire helper" \ "$(wrapper_value OPENSCREEN_LINUX_CURSOR_HELPER_EXE)" libpipewire-0.3.so.0 + # Same construction for csrc/dmabuf_modifiers.c's dlopen("libEGL.so.1"), + # and ubuntu-latest's system libEGL would hide its loss just as well. + check_dlopen_rpath "PipeWire helper" \ + "$(wrapper_value OPENSCREEN_LINUX_CURSOR_HELPER_EXE)" libEGL.so.1 check_dlopen_rpath "compositor addon" \ "$(wrapper_value OPENSCREEN_COMPOSITOR_VIEW_NODE)" libvulkan.so.1 @@ -426,6 +451,14 @@ jobs: # xvfb-run folds the command's stderr into stdout, so the stderr # capture is empty and every milestone lands in the other file. grep -a milestone "/tmp/sources.$i.out" || echo " (none at all)" + elif [ "$RC" -eq 0 ] && ! python3 -c "import json,sys; sys.exit(0 if json.load(open(sys.argv[1]))['displays'] else 1)" "/tmp/sources.$i.json"; then + # Exit 0 is not an answer on its own. Electron 43 settles the + # combined screen+window call after its 3s ready timer with the + # screen dropped, so `displays: []` is #462's other face, and + # Xvfb always has one screen to report. + FAILED=$((FAILED + 1)) + echo "returned no display (#462). Last milestone:" + grep -a milestone "/tmp/sources.$i.out" | tail -1 || echo " (none)" elif [ "$RC" -eq 0 ]; then OK=$((OK + 1)) LAST_OK="/tmp/sources.$i.json" @@ -460,10 +493,8 @@ jobs: PY fi - # Noted, not acted on. Enumeration is flaky here and the export - # assertion below is the more - # important question; a flaky attempt costs an annotation rather than - # the answer. Verdict at the end, once both have had their say. + # Annotated here, decided at the end: export runs either way, so a + # failed enumeration cannot hide whether the addon works. if [ "$HUNG" -gt 0 ]; then echo "::warning::openscreen sources had to be killed on $HUNG of $ATTEMPTS attempts. Check the milestones above for how far it got before the outer bound fired." fi @@ -590,21 +621,18 @@ jobs: # flaky must not hide whether export works, which is the whole point of # having packaged the compositor addon. # - # The gate is "did enumeration ever answer" and "does export work", not - # "did all five attempts pass". Requiring FAILED -eq 0 made the job red - # by construction: the standing numbers on this runner are 1/5, 3/5 and - # 4/5 ok, so a run where export is perfect and four enumerations succeed - # still failed. A post-merge check that is red by design is one that gets - # muted, which costs more than the flakiness it was reporting. The - # per-attempt warnings above keep that flakiness visible without letting - # it decide the build; tighten this to $ATTEMPTS once the capture failure - # is understood and fixed. + # Every enumeration must answer with a display, and export must work. + # The gate used to be "did enumeration ever answer", because the call + # failed on 1 to 4 attempts in 5 for a reason nobody had found. #462 + # found it: asking for screens and windows in one getSources call is + # what fails under X11, and the get-sources handler now asks one type + # at a time (under Xvfb: 0 failures in 32 runs, against 7 in 16 + # combined). A flake left after that is a regression, not runner luck. # - # Export, on the other hand, is a hard gate on every trigger again. It no - # longer depends on a capability this host does not have, so there is - # nothing left to excuse: if it fails now, the package is broken. + # Export is a hard gate on every trigger. It does not depend on a + # capability this host lacks, so if it fails, the package is broken. echo "=== verdict: enumeration $OK/$ATTEMPTS ok, export $EXPORT_OK, $RUN_CLI_N run_cli invocations ===" - if [ "$EXPORT_OK" -ne 1 ] || [ "$OK" -eq 0 ]; then + if [ "$EXPORT_OK" -ne 1 ] || [ "$OK" -ne "$ATTEMPTS" ]; then exit 1 fi diff --git a/.github/workflows/nix-check.yml b/.github/workflows/nix-check.yml index 49b51cbe32..768644e162 100644 --- a/.github/workflows/nix-check.yml +++ b/.github/workflows/nix-check.yml @@ -4,8 +4,8 @@ name: Nix # package-lock.json resolves to. The two have to agree or `nix build` fails # outright, and nothing here ever checked that they did. # -# The only thing that refreshed the hash was bump-nix-package.yml, which fires -# on stable releases. `src` is this repo's own tree, not a fetched tarball, so +# The only thing that refreshed the hash used to be a bump PR opened after each +# stable release (releases now refresh it in their own version-bump commit). `src` is this repo's own tree, not a fetched tarball, so # every lockfile change landing between two releases left main's hash pointing # at dependencies that no longer existed. It last matched on 2026-07-05 (v1.6.0) # while package-lock.json moved ten more times, so `nix run github:…` was broken diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index 965a6e93e9..e43b6ff200 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -75,6 +75,13 @@ jobs: NEXT: ${{ steps.version.outputs.next }} run: node .github/scripts/release-milestone-migrate.mjs + - name: Install Nix + uses: cachix/install-nix-action@v27 + with: + nix_path: nixpkgs=channel:nixos-unstable + extra_nix_config: | + experimental-features = nix-command flakes + - name: Create or reuse the release branch and bump package.json env: TOKEN: ${{ secrets.OPENSCREEN_RELEASE_TOKEN }} @@ -82,6 +89,9 @@ jobs: NEXT: ${{ steps.version.outputs.next }} run: | set -euo pipefail + # Taken from the dispatched ref before the release branch is checked out: a frozen + # branch cut before this script existed would not have it. + cp .github/scripts/refresh-nix-npm-hash.sh "$RUNNER_TEMP/" git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" # ONE release branch per STABLE version (release/vX.Y.Z), created at rc.1 and @@ -109,7 +119,9 @@ jobs: # Writes package-lock.json too. A sed over package.json alone left the # lockfile behind on every release up to 1.9.0; see the script header. node .github/scripts/set-release-version.mjs "${PRERELEASE}" - git add package.json package-lock.json + # The lockfile's version is part of Nix's npm hash: refresh it in this same commit. + bash "$RUNNER_TEMP/refresh-nix-npm-hash.sh" + git add package.json package-lock.json nix/package.nix git commit -m "chore(release): bump to ${PRERELEASE} [skip ci]" || echo "(version already at ${PRERELEASE})" git push "$REMOTE" "$BRANCH" diff --git a/.github/workflows/promote.yml b/.github/workflows/promote.yml index 7bc090534e..358ddb7390 100644 --- a/.github/workflows/promote.yml +++ b/.github/workflows/promote.yml @@ -59,12 +59,22 @@ jobs: STABLE_VERSION: ${{ steps.version.outputs.stable_version }} run: node .github/scripts/release-milestone-close.mjs + - name: Install Nix + uses: cachix/install-nix-action@v27 + with: + nix_path: nixpkgs=channel:nixos-unstable + extra_nix_config: | + experimental-features = nix-command flakes + - name: Bump the version to stable on the release branch env: TOKEN: ${{ secrets.OPENSCREEN_RELEASE_TOKEN }} STABLE_VERSION: ${{ steps.version.outputs.stable_version }} run: | set -euo pipefail + # Taken from the dispatched ref before the release branch is checked out: a frozen + # branch cut before this script existed would not have it. + cp .github/scripts/refresh-nix-npm-hash.sh "$RUNNER_TEMP/" # Promote checks out the FROZEN release branch (created by prerelease.yml) and # rewrites the version there. This guarantees the stable tag points at the # same code that was tested as the RC plus any cherry-picked bugfixes. @@ -75,9 +85,11 @@ jobs: # Writes package-lock.json too. A sed over package.json alone left the # lockfile behind on every release up to 1.9.0; see the script header. node .github/scripts/set-release-version.mjs "${STABLE_VERSION}" + # The lockfile's version is part of Nix's npm hash: refresh it in this same commit. + bash "$RUNNER_TEMP/refresh-nix-npm-hash.sh" git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add package.json package-lock.json + git add package.json package-lock.json nix/package.nix git commit --allow-empty -m "chore(release): bump to ${STABLE_VERSION} [skip ci]" || true git push "https://x-access-token:${TOKEN}@github.com/${GITHUB_REPOSITORY}.git" "$BRANCH" diff --git a/.github/workflows/update-homebrew-cask.yml b/.github/workflows/update-homebrew-cask.yml index 834eda7e50..8b9ba3127f 100644 --- a/.github/workflows/update-homebrew-cask.yml +++ b/.github/workflows/update-homebrew-cask.yml @@ -222,7 +222,7 @@ jobs: homepage "https://github.com/${REPO}" auto_updates false - depends_on macos: ">= :ventura" + depends_on macos: :ventura app "Openscreen.app" diff --git a/.gitignore b/.gitignore index 24ad425ebc..cf03b63b56 100644 --- a/.gitignore +++ b/.gitignore @@ -72,6 +72,10 @@ result-* # Auto-caption model + ORT wasm — regenerated at build by scripts/fetch-caption-model.mjs /caption-assets/ +# Third-party licence texts, regenerated before every package by +# scripts/generate-third-party-licenses.mjs (`npm run licenses:generate`). +/build-licenses/ + # Native STT artifacts — whisper-stt-server binaries + ggml backend sidecars. /electron/native/bin/ /electron/native/models/ @@ -95,6 +99,9 @@ result-* /tools/stt-eval/whispercpp-dtw-poc/fixtures/*.wav /tools/stt-eval/whispercpp-dtw-poc/results/ +# Word-timing harness (tools/stt-eval/word-timing): TTS corpus and results. +/tools/stt-eval/word-timing/data/ + opencode.json opencode.json @@ -121,6 +128,7 @@ poc/out/ workbench/reports/ workbench/.build/ workbench/runs/ +workbench/.agent-evidence/ # Fixture d'évaluation — enregistrement réel de l'utilisateur, jamais versionné workbench/fixtures/ diff --git a/AGENTS.md b/AGENTS.md index f3d0dcd10e..8564fb8f24 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -95,7 +95,7 @@ This section is the *mechanics*. **What to actually run is [`technical-documenta - **A release candidate is tested on its CI-built artifact, not on a dev build.** Getting it, installing it and launching it on each platform: [Running the pass on a CI-built artifact](technical-documentation/testing/manual-e2e-checklist.md#running-the-pass-on-a-ci-built-artifact), in the checklist. The notes below on `electron.exe`, worktrees and stale native binaries are about dev builds. - Dev build: `npm run dev` — Vite serves the renderer and `vite-plugin-electron` opens the Electron window. The main process logs `Global shortcut registered: CommandOrControl+Shift+O` when ready (Ctrl/Cmd+Shift+O brings the app's window, HUD or editor, to the front). -- **Set `OPENSCREEN_DISABLE_CONTENT_PROTECTION=1` in the environment you launch from, or the HUD is invisible in every screenshot you take** (Windows, and macOS before 26). It is a module-scope constant in `electron/windows.ts`, read once as the main process loads, so it cannot be turned on afterwards — you relaunch or you work blind. An installed build only gets it when you start its executable from the shell where it is set: a Start-menu, Finder or installer launch does not pass it on. The main process prints `[content-protection] OFF for the HUD window` when it took effect; if that line is missing, or the HUD is missing from your first screenshot, stop and relaunch rather than hunting a HUD you will never see. What it does and when to unset it: the HUD notes below. +- **Set `OPENSCREEN_DISABLE_CONTENT_PROTECTION=1` in the environment you launch from, or the HUD is invisible in every screenshot you take** (Windows 11 22H2 and later, and macOS before 26). It is a module-scope constant in `electron/windows.ts`, read once as the main process loads, so it cannot be turned on afterwards — you relaunch or you work blind. An installed build only gets it when you start its executable from the shell where it is set: a Start-menu, Finder or installer launch does not pass it on. The main process prints `[content-protection] OFF for the HUD window` when it took effect; if that line is missing, or the HUD is missing from your first screenshot, stop and relaunch rather than hunting a HUD you will never see. What it does and when to unset it: the HUD notes below. - The app is single-instance through `app.requestSingleInstanceLock()`, which keys on the `userData` path. If a leftover process still holds it, a new launch quits silently (exit 0, no window) — kill leftover `electron` or `Openscreen` processes before relaunching, including a copy the installer started for you. The lock is held by the OS and dies with the process, so there is nothing to clean up on disk. A dev build and the installed `Openscreen` resolve different `userData` paths and can run side by side. - **Dev build from a git worktree** (no `node_modules`/native binaries): junction/symlink `node_modules` from the main checkout (deps are usually identical — check `package-lock.json`), and copy the prebuilt native capture binaries from `electron/native/bin//` (gitignored — rebuilding needs the full VS/Xcode toolchain). Then `npm run dev` works normally. - **Those binaries are frozen at whenever someone last built them, and nothing warns you.** They are not rebuilt by `npm run dev` or `npm run build`, so a helper older than the native change you came to test will run happily and silently exercise the old code path — the recording succeeds, and the thing you wanted to see is simply absent. Before trusting any native result, date the binary against the commit and search it for a string the change introduced — from the repo root: @@ -119,7 +119,8 @@ This section is the *mechanics*. **What to actually run is [`technical-documenta **The HUD widget** (recording controller) - **It is invisible in screenshots by default.** The HUD (and the Notes window) call `setContentProtection(true)` so the recording controls never end up baked into a recording — the same `SetWindowDisplayAffinity` that WGC honours also hides them from *your* screenshots. The window is there, and clicks land, but you are aiming blind at a rectangle you cannot see. Set **`OPENSCREEN_DISABLE_CONTENT_PROTECTION=1`** in the app's environment to turn it off for a session; every skipped window logs a warning. Unset it before recording anything real, or the HUD ends up in the video. -- **On macOS 26+ content protection is auto-disabled, so the HUD *is* visible and screenshottable with no flag.** That OS never displays a content-protected window at all — not just absent from captures, but never painted, leaving a tray icon, a live renderer and nothing on screen (confirmed on macOS 26.5 / Electron 41.2.1). `applyContentProtection` therefore skips the call there and logs a warning per window. The native ScreenCaptureKit helper still keeps the HUD and an open Notes window out of full-display recordings through `SCContentFilter(excludingWindows:)`; ordinary OpenScreen windows remain recordable. `OPENSCREEN_FORCE_CONTENT_PROTECTION=1` re-enables Electron's protection to re-test against a future Electron. +- **On macOS 26+ content protection is auto-disabled, so the HUD *is* visible and screenshottable with no flag.** That OS never displays a content-protected window at all — not just absent from captures, but never painted, leaving a tray icon, a live renderer and nothing on screen (confirmed on macOS 26.5 / Electron 41.2.1). `applyContentProtection` therefore skips the call there and logs a warning per window. The native ScreenCaptureKit helper still keeps the HUD and an open Notes window out of full-display recordings through `SCContentFilter(excludingWindows:)`; ordinary OpenScreen windows remain recordable, except in a capture started from Apple's picker, which leaves out the whole app (#996). `OPENSCREEN_FORCE_CONTENT_PROTECTION=1` re-enables Electron's protection to re-test against a future Electron. +- **On Windows before 11 22H2 (build 22621) the HUD's protection is auto-disabled too, so the HUD is screenshottable with no flag, and it appears in recordings.** There Electron makes a protected window `WS_EX_LAYERED` for good, and a click-through window so styled never takes a click again: the HUD painted and dead on every launch (#1105, confirmed on Windows 10 22H2). `applyContentProtection` skips the HUD there and logs a warning; the Notes window, opaque and never click-through, keeps its protection. `OPENSCREEN_FORCE_CONTENT_PROTECTION=1` re-enables it. - The HUD is what opens the editor (clapper icon, tooltip *Open Studio*), so without that flag a whole slice of the app is unreachable from automation: killing the app to redeploy a native addon leaves you unable to reopen a project. - **The HUD and the editor never share the screen.** Opening the editor closes the HUD, and closing the editor, the last window, quits the app. The way back is the editor's Record mode (see the flow below). - Frameless, transparent, always-on-top, `skipTaskbar`, centered at the **bottom of the primary display** (`createHudOverlayWindow`, 820×560 at construction, then resized to fit its content — measured 904×698 with the bar at the bottom and mostly empty reserve above it). It is **click-through** (`setIgnoreMouseEvents(ignore)`): moving the real cursor over an interactive control makes that region clickable and shows its tooltip, so `mouse_move` → screenshot → `left_click` works; a blind click on empty HUD area passes through to the desktop. diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000000..9ab2fbca32 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,132 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, caste, color, religion, or sexual +identity and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the overall + community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or advances of + any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email address, + without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +conduct@getopenscreen.com. +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series of +actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or permanent +ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within the +community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.1, available at +[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. + +Community Impact Guidelines were inspired by +[Mozilla's code of conduct enforcement ladder][Mozilla CoC]. + +For answers to common questions about this code of conduct, see the FAQ at +[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at +[https://www.contributor-covenant.org/translations][translations]. + +[homepage]: https://www.contributor-covenant.org +[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html +[Mozilla CoC]: https://github.com/mozilla/diversity +[FAQ]: https://www.contributor-covenant.org/faq +[translations]: https://www.contributor-covenant.org/translations diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d4d82f7b54..163ce81548 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -41,6 +41,52 @@ Thank you for considering contributing to this project! By contributing, you hel 8. **Open a Pull Request** - Go to the original repository and open a pull request from your branch. Provide a clear description of your changes and the problem they solve. +## First-time setup on macOS + +### Prerequisites + +- **Node 22.22.1 and npm 10.9.4**, the versions pinned in `package.json`. With nvm, run `nvm use` in the repository root. Use npm, not another package manager. +- **Full Xcode**, not only the Command Line Tools. After installing it: + ```bash + sudo xcode-select --switch /Applications/Xcode.app/Contents/Developer + sudo xcodebuild -license accept + ``` +- **Rust, installed with [rustup](https://rustup.rs)**. The build script looks for `cargo` at `~/.cargo/bin/cargo`, so a Homebrew-installed Rust is not found. + +### Build the native binaries + +Run these once from the repository root, in this order: + +```bash +npm install +npm run build:native:mac # Swift capture and cursor helpers +npm run fetch:ffmpeg:mac # builds an LGPL ffmpeg from source, about five minutes +npm run build:native:compositor:mac # Rust compositor addon, links the ffmpeg above +npm run fetch:onnxruntime # optional, see below +npm run dev +``` + +| Command | What it produces | Without it | +|---|---|---| +| `build:native:mac` | `openscreen-screencapturekit-helper` and `openscreen-macos-cursor-helper` | Native screen capture is unavailable | +| `fetch:ffmpeg:mac` | An ffmpeg 8.1.2 tree in `crates/thirdparty/` | The compositor addon cannot be built | +| `build:native:compositor:mac` | `compositor_view.node` | The editor preview is blank and exports render nothing | +| `fetch:onnxruntime` | The ONNX Runtime library | The webcam background cutout, blur and custom modes are off; everything else works | + +`fetch:ffmpeg:mac` compiles ffmpeg rather than downloading a binary because the prebuilt macOS builds, Homebrew's included, are GPL and cannot be linked into this MIT app. Do not point the build at a Homebrew ffmpeg. + +Transcription and captions need a further binary, `whisper-stt-server`. See [`technical-documentation/engineering/build-and-packaging.md`](technical-documentation/engineering/build-and-packaging.md) for that and for packaging a release build. + +### When to rebuild + +The binaries stay at whatever you last built, and nothing warns you when they fall behind the source. + +- After changing Swift code in `electron/native/screencapturekit/`, run `npm run build:native:mac`. +- After changing Rust code in `crates/`, run `npm run build:native:compositor:mac`. +- After switching to a branch that changes either, run the matching command before testing. + +A git worktree starts with none of these binaries, because git does not copy gitignored files. + ## Reporting Issues If you encounter a bug or have a feature request, please open an issue in the [Issues](https://github.com/EtienneLescot/openscreen/issues) section of this repository. Provide as much detail as possible to help us address the issue effectively. diff --git a/README.md b/README.md index 872f991443..ef31cfe087 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@

A free, open-source desktop app for recording your screen and turning the result into polished product demos and walkthroughs.

- Editing a recording in OpenScreen: wallpaper and video effects, an AI-assisted cut driven from the chat, then export + Editing a recording in OpenScreen: one click restyles the whole video, deleting a sentence in the transcript cuts it from the video, and the editor's agent adds a zoom on every click

@@ -17,11 +17,11 @@ Discord Sponsor Platform + #5 Product of the Day on Product Hunt

- getopenscreen/openscreen | Trendshift -

+ getopenscreen/openscreen | Trendshift

> [!NOTE] > **This is OpenScreen, continued.** [Siddharth Vaddem](https://github.com/siddharthvaddem) created the project and [archived the original repository](https://github.com/siddharthvaddem/openscreen) after v1.5.0. Development moved here with his approval, under the same name and the same MIT license. @@ -63,6 +63,31 @@ openscreen export demo.openscreen -o demo.mp4 --json See the [CLI reference](https://getopenscreen.com/docs/cli/). +### Let your coding agent ship the feature with its video + +Paste this into Claude Code: + +```text +Install the OpenScreen demo skill with `npx skills add getopenscreen/openscreen --skill openscreen-demo -a claude-code`, then follow it to add a video of this change to the PR. +``` + +The skill sets everything up: it installs OpenScreen if it is missing (once you +agree), connects its MCP server, then asks you to open a new session and say +*"continue the OpenScreen demo"*, the one step an agent cannot do for itself. +By hand, it is these two commands, with the executable of your system: + +```bash +claude mcp add openscreen -- "/Applications/Openscreen.app/Contents/MacOS/Openscreen" mcp +npx skills add getopenscreen/openscreen --skill openscreen-demo +``` + +From then on, ask: *"Add a video of this change to the PR."* The agent plans the +storyboard, records the app, edits the take with OpenScreen's own editing tools, +exports the MP4, checks it frame by frame, and attaches it to the pull request, +where reviewers watch it in place. Ask for a launch demo instead and it drafts a +post for X, which it publishes only once you say yes. The executable on Windows +and Linux, and the other clients, are in [AI editing](https://getopenscreen.com/docs/ai-editing/#letting-an-agent-make-the-whole-video). + ## Installation Every platform has a recommended route below. On Windows that is the Microsoft Store; everywhere else it is the installer from the [GitHub Releases](https://github.com/getopenscreen/openscreen/releases) page. @@ -88,6 +113,9 @@ On first launch, open **System Settings > Privacy & Security** and grant the two > [!NOTE] > **Upgrading from a version older than 1.9.0?** Those builds were not signed with a Developer ID certificate, and macOS ties Screen Recording and Accessibility grants to an app's signature — so it cannot tell the new build is the same app, and the permissions you granted the old one do not carry over. If the new version won't record even after you grant them, remove OpenScreen's existing entries under **System Settings > Privacy & Security** (both Screen Recording and Accessibility), then launch it again and grant them when prompted. +> [!NOTE] +> **OpenScreen quits as soon as you pick what to record, or when the countdown starts?** It is most likely a utility that quits apps once their last window closes, such as the **Quit on close** option of [Vorssaint](https://github.com/vorssaint/vorssaint-utils). It does not count OpenScreen's recording bar, which floats above other windows, so it quits OpenScreen whenever the bar hides or a source window closes. Add OpenScreen to that utility's exceptions, or turn the option off. The [FAQ](https://getopenscreen.com/docs/faq#why-does-openscreen-quit-on-macos-when-i-choose-what-to-record) shows how to check which app asked OpenScreen to quit. + ### Windows **Recommended — Microsoft Store** @@ -102,13 +130,16 @@ Microsoft signs the Store package during certification, so it installs with no s **Alternative — standalone installer** -Download the `.exe` from the [Releases page](https://github.com/getopenscreen/openscreen/releases). Use this if you can't reach the Store — Windows LTSC, a locked-down work machine, an offline install, or if you want a specific older version. +Download the `.exe` from the [Releases page](https://github.com/getopenscreen/openscreen/releases). Installers are provided for both x64 (`Openscreen.Setup.-x64.exe`) and ARM64 (`Openscreen.Setup.-arm64.exe`) — on an ARM64 device the x64 build runs under emulation and records poorly, so take the ARM64 one. Use this if you can't reach the Store — Windows LTSC, a locked-down work machine, an offline install, or if you want a specific older version. > [!NOTE] > The `.exe` is not code-signed, so Windows SmartScreen shows **"Windows protected your PC"** and reports an unknown publisher. Choose **More info** → **Run anyway** to continue. > > This is not a sign that something is wrong with the download: an unsigned installer earns SmartScreen's trust per file, so a brand-new build always starts out untrusted no matter how many people installed the previous one. Verifying the signature isn't an option here — there is nothing to verify. If you want the checked path, use the Store build above. If you use the `.exe`, download it only from the Releases page linked here. +> [!NOTE] +> **Recording bar not clickable in OpenScreen 2.0.0 on Windows 10?** Run `setx OPENSCREEN_DISABLE_CONTENT_PROTECTION 1`, quit OpenScreen from its tray icon, then start it again. The bar can then appear in your recordings. The [FAQ](https://getopenscreen.com/docs/faq#why-cant-i-click-the-recording-bar-on-windows-10) explains how to undo it once the fix ships. + ### Linux Four packages are published to the [Releases page](https://github.com/getopenscreen/openscreen/releases) for each version. Pick the one that matches your distro: @@ -176,7 +207,7 @@ You may need to grant screen recording permissions depending on your desktop env Everything in the editor and export is the same on macOS, Windows, and Linux: zooms, backgrounds, motion blur, crop/trim/speed, blur regions, annotations, auto-captions, AI editing, projects, export, and all languages. All three now record through a native capture pipeline; the remaining differences are narrower than they used to be: - **Native recording**: macOS (ScreenCaptureKit), Windows (Windows Graphics Capture), and Linux (PipeWire via the ScreenCast portal) all record through a native pipeline for higher quality and clean window-level capture. On Linux the browser pipeline stays as an automatic fallback if the helper isn't available. -- **Custom cursors**: on macOS and Windows the real cursor is captured with shape, type, and clicks. Linux captures position and cursor shape through the portal, so the editable cursor overlay works there too. Click effects work on Linux as well, but not through the portal — Wayland exposes no portal for mouse buttons, so the capture helper reads the left button from evdev, which needs your user in the `input` group. Without that, recording is unaffected and every cursor sample is simply a move. +- **Custom cursors**: on macOS and Windows the real cursor is captured with shape, type, and clicks. Linux captures position and cursor shape through the portal, so the editable cursor overlay works there too. Click effects work on Linux as well, but not through the portal — Wayland exposes no portal for mouse buttons, so the capture helper reads the left button from evdev, which needs read access to your mouse: a udev rule for pointer devices (recommended) or the `input` group, set up as [Mouse clicks on Wayland](https://getopenscreen.com/docs/installation#mouse-clicks-on-wayland) shows. Without that, recording is unaffected and every cursor sample is simply a move. - **Webcam**: every platform saves the webcam as a separate file next to the screen recording. Windows captures it natively in the recording helper; macOS and Linux record it in the app. It works as a picture-in-picture overlay on all three. - **System audio** support varies by OS: - **macOS**: works on every supported version. On macOS 14.2+ you'll be prompted to grant audio capture permission. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000000..76a180e35f --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,48 @@ +# Security Policy + +## Supported versions + +Security fixes land on the latest stable release only. Pre-releases (`-rc.N`) are superseded by the +next stable version and are not patched on their own. + +| Version | Supported | +| ---------------------------------------- | --------- | +| Latest stable release | Yes | +| Any earlier release, patches included | No | +| Release candidates | No | + +## Reporting a vulnerability + +**Please do not open a public issue, discussion or pull request for a security problem.** + +Report it privately through GitHub instead: +[Report a vulnerability](https://github.com/getopenscreen/openscreen/security/advisories/new). +This opens a private advisory that only you and the maintainers can see. + +A useful report includes: + +- the OpenScreen version and how it was installed (Microsoft Store, installer or package + from the Releases page, Nix…) +- the operating system and version +- the steps to reproduce, or a proof of concept +- the impact you expect: what an attacker gains, and under which conditions + +## What happens next + +OpenScreen is maintained by volunteers, so these timelines are best-effort: + +- an acknowledgement within 7 days +- a first assessment, confirmed or not, within 14 days +- for a confirmed issue, a fix in the next release, then a published advisory crediting you, + unless you prefer to stay anonymous + +Please keep the details private until the fix is released and the advisory is published. + +## Scope + +In scope: the OpenScreen desktop app and everything this repository builds and ships with it, +including the native capture helpers, the Rust compositor and the auto-update mechanism. + +Out of scope: vulnerabilities in third-party dependencies that are already public upstream (a +heads-up is still welcome), and issues that require an attacker who already controls the user's +machine or account. diff --git a/THIRD-PARTY-NOTICES.md b/THIRD-PARTY-NOTICES.md index dcbc432cff..945d45a7ac 100644 --- a/THIRD-PARTY-NOTICES.md +++ b/THIRD-PARTY-NOTICES.md @@ -1,12 +1,13 @@ # Third-party notices OpenScreen is MIT licensed (see [LICENSE](LICENSE)). The installers additionally -bundle the pre-built native components below. This file ships inside the +bundle the third-party components below. This file ships inside the application resources and satisfies the attribution and source-offer obligations that come with them. -npm dependencies are not listed here: they are resolved from `package.json` and -distributed by their own registries, not redistributed inside our binaries. +npm dependencies are not listed here. Their code ships bundled inside the +application by Vite, and their licence texts are in `THIRD-PARTY-LICENSES.txt`, +generated at build time and shipped beside this file. --- @@ -38,6 +39,23 @@ distributed by their own registries, not redistributed inside our binaries. replace them with your own build of the same FFmpeg version by overwriting the DLLs in `resources/electron/native/bin/win32-*/`. +## web-demuxer — WebAssembly demuxer + +- **Component**: `wasm/web-demuxer.wasm`, inside the application bundle. +- **Used by**: the editor's streaming decoder, audio waveforms and caption audio + extraction, which demux recordings in the renderer with it. +- **License**: MIT for web-demuxer itself — Copyright (c) 2024 ForeverSc, under the + same permission notice as OpenScreen's own [LICENSE](LICENSE). The module also + contains FFmpeg code compiled to WebAssembly, under the **GNU Lesser General + Public License v2.1 or later**, as web-demuxer's README states. +- **Upstream**: the npm package `web-demuxer` 4.0.0, + . The shipped file is byte-identical + to that release's `dist/wasm-files/web-demuxer.wasm`. +- **Corresponding source**: the web-demuxer repository at the 4.0.0 release, whose + `lib/` directory holds the FFmpeg-derived sources and their build scripts. +- **Replacing it**: the module is a separate file loaded at runtime, so it can be + replaced with your own build of the same version. + ## whisper.cpp and ggml - **Components**: `whisper-stt-server` and its ggml backend sidecars, under @@ -147,6 +165,104 @@ distributed by their own registries, not redistributed inside our binaries. - IBM Plex Mono 2.3 — Copyright © 2017 IBM Corp. `ofl/ibmplexmono/` of at `23e54b51dd`. +## Recordly — helper code + +- **Components**: parts of the zoom, motion-smoothing and custom-cursor code, and a + few lines elsewhere, the native capture helpers included. +- **License**: MIT — Copyright (c) 2026 webadderall, under the same permission + notice as OpenScreen's own [LICENSE](LICENSE). Published by Recordly under MIT, + before its relicensing in March 2026. + +## Rust crates — compiled into the compositor addon + +- **Component**: `compositor_view.node`, under + `resources/electron/native/bin/-/`. +- **What is listed**: every third-party crate in the addon's dependency graph in + `crates/Cargo.lock`, across all platforms. Build-time crates (procedural macros, + build scripts) are included even though none of their code ships. +- **Licenses**: as each crate publishes them on crates.io. Where a crate offers a + choice, OpenScreen uses it under MIT or Apache-2.0. Each crate's licence text is + in its source package at the version listed. +- **Full texts**: the licence and NOTICE files of every crate compiled into the + addon are in `THIRD-PARTY-LICENSES.txt`, shipped beside this file. +- **MIT OR Apache-2.0** (155): android_system_properties 0.1.5, anyhow 1.0.103, + arrayvec 0.7.8, ash 0.38.0+1.3.281, autocfg 1.5.1, bit-set 0.8.0, bit-vec 0.8.0, + bitflags 1.3.2, bitflags 2.13.1, bumpalo 3.20.3, cc 1.2.67, cexpr 0.6.0, cfg-if + 1.0.4, core-foundation 0.9.4, core-foundation-sys 0.8.7, core-graphics-types + 0.1.3, cosmic-text 0.19.0, crc32fast 1.5.0, ctor 0.2.9, document-features + 0.2.12, either 1.16.0, equivalent 1.0.2, fdeflate 0.3.7, find-msvc-tools 0.1.9, + flate2 1.1.9, font-types 0.11.3, font-types 0.12.2, foreign-types 0.5.0, + foreign-types-macros 0.2.4, foreign-types-shared 0.3.1, futures-core 0.3.33, + futures-task 0.3.33, futures-util 0.3.33, glob 0.3.3, gpu-alloc 0.6.2, + gpu-alloc-types 0.3.1, gpu-allocator 0.27.0, gpu-descriptor 0.3.2, + gpu-descriptor-types 0.2.0, hashbrown 0.15.5, hashbrown 0.17.1, heck 0.5.0, + image 0.25.10, indexmap 2.14.0, itertools 0.13.0, itoa 1.0.18, jni-sys 0.3.1, + jni-sys 0.4.1, jni-sys-macros 0.4.1, js-sys 0.3.103, khronos-egl 6.0.0, libc + 0.2.186, linebender_resource_handle 0.1.1, litrs 1.0.0, lock_api 0.4.14, log + 0.4.33, matrixmultiply 0.3.11, memmap2 0.9.11, metal 0.29.0, metal 0.31.0, + minimal-lexical 0.2.1, naga 24.0.0, ndarray 0.16.1, ndarray 0.17.2, ndk-sys + 0.5.0+25.2.9519653, num-complex 0.4.6, num-integer 0.1.47, num-traits 0.2.19, + once_cell 1.21.4, ort 2.0.0-rc.13, ort-sys 2.0.0-rc.13, parking_lot 0.12.5, + parking_lot_core 0.9.12, paste 1.0.15, pin-project-lite 0.2.17, pkg-config + 0.3.33, png 0.18.1, pollster 0.4.0, portable-atomic 1.15.0, portable-atomic-util + 0.2.7, presser 0.3.1, prettyplease 0.2.37, proc-macro2 1.0.106, profiling + 1.0.18, quote 1.0.46, range-alloc 0.1.5, rangemap 1.7.1, rawpointer 0.2.1, + read-fonts 0.37.0, read-fonts 0.41.0, regex 1.13.1, regex-automata 0.4.16, + regex-syntax 0.8.11, renderdoc-sys 1.1.0, roxmltree 0.20.0, rustc-hash 1.1.0, + rustc-hash 2.1.3, rustversion 1.0.23, scopeguard 1.2.0, semver 1.0.28, serde + 1.0.228, serde_core 1.0.228, serde_derive 1.0.228, serde_json 1.0.150, shlex + 1.3.0, shlex 2.0.1, skrifa 0.40.0, skrifa 0.44.0, smallvec 1.15.2, smol_str + 0.3.6, static_assertions 1.1.0, swash 0.2.10, syn 2.0.119, syn 3.0.3, sys-locale + 0.3.2, thiserror 1.0.69, thiserror 2.0.19, thiserror-impl 1.0.69, thiserror-impl + 2.0.19, ttf-parser 0.25.1, unicode-bidi 0.3.18, unicode-script 0.5.8, + unicode-segmentation 1.13.3, unicode-width 0.1.14, unicode-xid 0.2.6, + version_check 0.9.5, wasm-bindgen 0.2.126, wasm-bindgen-futures 0.4.76, + wasm-bindgen-macro 0.2.126, wasm-bindgen-macro-support 0.2.126, + wasm-bindgen-shared 0.2.126, web-sys 0.3.103, wgpu 24.0.5, wgpu-core 24.0.5, + wgpu-hal 24.0.4, wgpu-types 24.0.0, windows 0.58.0, windows_aarch64_gnullvm + 0.52.6, windows_aarch64_msvc 0.52.6, windows_i686_gnu 0.52.6, + windows_i686_gnullvm 0.52.6, windows_i686_msvc 0.52.6, windows_x86_64_gnu + 0.52.6, windows_x86_64_gnullvm 0.52.6, windows_x86_64_msvc 0.52.6, windows-core + 0.58.0, windows-implement 0.58.0, windows-interface 0.58.0, windows-link 0.2.1, + windows-result 0.2.0, windows-strings 0.1.0, windows-sys 0.61.2, windows-targets + 0.52.6, yazi 0.2.1, zeno 0.3.3. +- **MIT** (26): block 0.1.6, cfg_aliases 0.2.2, convert_case 0.6.0, core_maths + 0.1.1, fontconfig-parser 0.5.8, fontdb 0.23.0, harfrust 0.5.2, libm 0.2.16, + malloc_buf 0.0.6, napi 2.16.17, napi-build 2.3.2, napi-derive 2.16.13, + napi-derive-backend 1.0.75, napi-sys 2.4.0, nom 7.1.3, objc 0.2.7, ordered-float + 4.6.0, redox_syscall 0.5.18, simd-adler32 0.3.10, slab 0.4.12, strum 0.26.3, + strum_macros 0.26.4, tracing 0.1.44, tracing-core 0.1.36, xml-rs 0.8.28, zmij + 1.0.23. +- **MIT OR Apache-2.0 OR Zlib** (9): bytemuck 1.25.1, bytemuck_derive 1.11.0, + glow 0.16.0, miniz_oxide 0.8.9, raw-window-handle 0.6.2, tinyvec 1.12.0, + tinyvec_macros 0.1.1, zune-core 0.5.1, zune-jpeg 0.5.15. +- **Apache-2.0** (7): clang-sys 1.8.1, codespan-reporting 0.11.1, gl_generator + 0.14.0, glutin_wgl_sys 0.6.1, khronos_api 3.1.0, spirv 0.3.0+sdk-1.3.268.0, + unicode-linebreak 0.1.5. +- **Unlicense OR MIT** (5): aho-corasick 1.1.4, byteorder-lite 0.1.0, memchr + 2.8.3, termcolor 1.4.1, winapi-util 0.1.11. +- **Zlib** (2): foldhash 0.1.5, slotmap 1.1.1. +- **ISC** (2): libloading 0.8.9, libloading 0.9.0. +- **BSD-3-Clause OR Apache-2.0** (2): moxcms 0.8.1, pxfm 0.1.30. +- **0BSD OR MIT OR Apache-2.0** (1): adler2 2.0.1. +- **BSD-3-Clause** (1): bindgen 0.70.1. +- **CC0-1.0** (1): hexf-parse 0.2.1. +- **Apache-2.0 OR GPL-2.0-only** (1): self_cell 1.3.0. +- **(MIT OR Apache-2.0) AND Unicode-3.0** (1): unicode-ident 1.0.24. + +## Kenney UI Audio — click sounds + +- **Components**: `sounds/click-down.wav` and `sounds/click-up.wav`, inside the + application bundle. +- **Used by**: the editor's "Mouse clicks" option, which plays them on each + recorded click, in the preview and in the export. +- **License**: **CC0 1.0 Universal** — + , as the licence file + shipped in the pack states. Credit is not required; it is given here anyway. +- **Upstream**: "UI Audio" by Kenney, . + `click-down.wav` is its `click3.wav` and `click-up.wav` its `switch12.wav`, + both trimmed of leading silence and normalised to a 0.7 peak. + ## OpenScreen native helpers `wgc-capture` (Windows Graphics Capture), the ScreenCaptureKit helper (macOS), diff --git a/biome.json b/biome.json index 4b65d5382c..fb2c6a9104 100644 --- a/biome.json +++ b/biome.json @@ -111,6 +111,10 @@ }, "javascript": { "formatter": { "quoteStyle": "double" } }, "overrides": [ + { + "includes": ["workbench/measurements/**/*.json"], + "formatter": { "enabled": false } + }, { "includes": ["*.ts", "*.tsx", "*.mts", "*.cts"], "linter": { diff --git a/crates/.cargo/config.toml b/crates/.cargo/config.toml index cb1fd42fb5..642cf6842d 100644 --- a/crates/.cargo/config.toml +++ b/crates/.cargo/config.toml @@ -34,6 +34,12 @@ LIBCLANG_PATH = "C:\\Program Files\\LLVM\\bin" [target.x86_64-pc-windows-msvc] rustflags = ["-C", "target-feature=+crt-static"] +# Windows on ARM: meme raison que x86_64 ci-dessus. Le paquet arm64 vise des +# machines Snapdragon ou le Redistribuable VC++ n'est pas plus garanti qu'ailleurs, +# et scripts/before-pack.cjs applique la meme regle aux deux arches. +[target.aarch64-pc-windows-msvc] +rustflags = ["-C", "target-feature=+crt-static"] + # macOS : BtbN ne publie pas de build macOS (cf. scripts/fetch-ffmpeg.mjs), donc on # laisse `crates/compositor/build.rs` chercher MAC_FFMPEG_DIR (env var explicite posée # par la CI macOS ou par le dev local) ou un répertoire attendu sous `thirdparty/`. diff --git a/crates/Cargo.toml b/crates/Cargo.toml index 3caa7edfd9..a65f9aaef2 100644 --- a/crates/Cargo.toml +++ b/crates/Cargo.toml @@ -68,6 +68,7 @@ features = [ "Win32_Graphics_Direct2D_Common", # D2D1_COLOR_F / PIXEL_FORMAT "Win32_Graphics_DirectWrite", # mise en page + rastérisation du texte "Win32_Graphics_Gdi", # brosses/police GUI de la preview (poc-d3d/src/app.rs) + "Win32_Security", # IDXGIResource1::CreateSharedHandle (shared_frames.rs) "Win32_System_Performance", # QueryPerformanceCounter / Frequency (§10) "Win32_System_LibraryLoader", # GetModuleHandleW (hInstance) "Win32_UI_WindowsAndMessaging", # fenêtres (harnais live + GUI du POC) diff --git a/crates/compositor-view-napi/src/lib.rs b/crates/compositor-view-napi/src/lib.rs index 8701c8bfef..78d351706c 100644 --- a/crates/compositor-view-napi/src/lib.rs +++ b/crates/compositor-view-napi/src/lib.rs @@ -11,8 +11,9 @@ use napi::{Env, JsFunction, Task}; use napi_derive::napi; use openscreen_compositor::compositor::{live_params_from_scene, Compositor}; use openscreen_compositor::d3d::{Backend, Gpu}; +use openscreen_compositor::export_control::{ExportCancelled, ExportControl}; +use openscreen_compositor::frame_geometry::FootageQuad; use openscreen_compositor::gif_export::{GifExportParams, GifStats}; -use openscreen_compositor::gif_export_control::{GifExportCancelled, GifExportControl}; use openscreen_compositor::live::{LiveView, PausedPreviews}; use openscreen_compositor::scene::Scene; use openscreen_compositor::{config, pipeline}; @@ -197,6 +198,16 @@ pub struct FramePacket { /// R,G,B,A tightly-packed, `width * height * 4` octets — ce que `putImageData` / /// `ImageData` attendent côté JS (canvas 2D, format natif RGBA8). pub data: Buffer, + /// Le métrage dans cette image : ses coins TL, TR, BR, BL (x, y en fractions de l'image, + /// huit nombres), là où l'éditeur pose le gimbal d'un flou. Absent avant toute composition. + pub footage: Option>, + /// Le métrage passe de ses coins à l'image par leur homographie (caméra réelle), pas par + /// leur interpolation bilinéaire. + pub footage_projective: bool, + /// Clip actif de la scène quand la frame a été composée (`FramePosition`). + pub clip_index: u32, + /// Temps source de la frame écran composée, en secondes. + pub source_time_sec: f64, } /// Renvoie la dernière frame readback du thread de rendu SI elle est plus récente que @@ -236,17 +247,95 @@ pub fn read_frame(id: i32, since_gen: f64) -> Result> { v.latest_frame_since(since_gen.max(0.0) as u64) } }; - Ok(slot.map(|(gen, w, h, pixels)| { + Ok(slot.map(|(gen, w, h, pixels, footage, position)| { debug_assert_eq!(pixels.len(), (w as usize) * (h as usize) * 4); FramePacket { gen: gen as f64, width: w, height: h, data: Buffer::from(pixels), + footage: footage_corners(footage), + footage_projective: footage.is_some_and(|q| q.projective), + clip_index: position.clip_index, + source_time_sec: position.source_time_sec, } })) } +/// Les coins du métrage (TL, TR, BR, BL) aplatis en huit nombres, tels que JS les lit. +fn footage_corners(footage: Option) -> Option> { + footage.map(|q| q.corners.iter().flatten().map(|&v| v as f64).collect()) +} + +/// Une frame de preview posée dans une texture partagée : de quoi l'importer côté GPU +/// (`sharedTexture.importSharedTexture`) au lieu d'en recevoir les pixels. +#[napi(object)] +pub struct SharedFramePacket { + pub gen: f64, + /// Case de l'anneau qui porte la frame, à rendre avec `gen` par `release_shared_frame` + /// quand Chromium l'a relâchée. + pub slot: u32, + /// Handle NT de la texture, 8 octets little-endian : la forme qu'attend Electron + /// (`SharedTextureHandle.ntHandle`). Valable dans ce processus seulement. + pub handle: Buffer, + pub width: u32, + pub height: u32, + pub footage: Option>, + pub footage_projective: bool, + pub clip_index: u32, + pub source_time_sec: f64, +} + +/// Livrer les frames de la vue par textures partagées plutôt que par `read_frame`. Rend +/// `false` quand la machine ne le peut pas (hors Windows, backend logiciel) : la vue reste +/// alors au readback. +#[napi] +pub fn set_shared_frames(id: i32, enabled: bool) -> bool { + registry() + .lock() + .unwrap() + .get(&id) + .is_some_and(|v| v.set_shared_frames(enabled)) +} + +/// Le pendant de `read_frame` pour une vue en textures partagées : la dernière frame posée +/// dans l'anneau, si elle est plus récente que `since_gen`. Sa case reste tenue jusqu'à +/// `release_shared_frame`. `Ok(None)` aussi quand la vue relit en RAM. +#[napi] +pub fn read_shared_frame(id: i32, since_gen: f64) -> Result> { + let frame = match registry().lock().unwrap().get(&id) { + None => return Ok(None), + Some(v) => { + // Même relais que `read_frame` : sans lui, un thread de rendu mort ne se verrait + // que par un canvas noir. + if let Some(fatal) = v.fatal_error() { + return Err(Error::from_reason(fatal)); + } + v.take_shared_frame(since_gen.max(0.0) as u64) + } + }; + Ok(frame.map(|f| SharedFramePacket { + gen: f.gen as f64, + slot: f.slot, + handle: f.handle.to_le_bytes().to_vec().into(), + width: f.width, + height: f.height, + footage: footage_corners(f.footage), + footage_projective: f.footage.is_some_and(|q| q.projective), + clip_index: f.position.clip_index, + source_time_sec: f.position.source_time_sec, + })) +} + +/// Chromium a relâché la frame `gen` de la case `slot` (`allReferencesReleased` côté +/// Electron) : le thread de rendu peut y réécrire. Sans effet sur une vue détruite. +#[napi] +pub fn release_shared_frame(id: i32, slot: u32, gen: f64) { + if let Some(v) = registry().lock().unwrap().get(&id) { + v.release_shared_frame(slot, gen.max(0.0) as u64); + } +} + /// Param live (inspector). Le type de valeur route vers le bon setter : /// bool = switch (webcamMirror…), number = slider (shadow/roundness/motionBlur/backgroundBlur), /// string = sélection (backgroundColor "#rrggbb"). @@ -508,6 +597,7 @@ pub struct ExportMultiTask { clips: Vec, scene_json: Option, params: Option, + control: ExportControl, on_progress: Option>, } @@ -516,12 +606,14 @@ impl Task for ExportMultiTask { type JsValue = ExportStats; fn compute(&mut self) -> Result { + self.control.check().map_err(mp4_task_error)?; // Previews paused for the whole render (GPU 3D engine freed) and restored // exactly as found when this guard drops, including on the error paths. let _previews = PreviewPause::begin(); // Même sélection que la preview : l'export d'un hôte sans GPU passe par // libopenh264 au lieu d'AMF, plutôt que d'échouer. let gpu = Gpu::create_auto(false).map_err(|e| Error::from_reason(format!("{e:#}")))?; + self.control.check().map_err(mp4_task_error)?; let mut cfg = config::all().pop().expect("au moins une config"); // C8 cfg.zoom = false; cfg.layout_anim = false; @@ -581,7 +673,7 @@ impl Task for ExportMultiTask { comp.set_scene(scene); let mut progress = throttled_progress(self.on_progress.take()); - let s = pipeline::run_composited_multi( + let s = pipeline::run_composited_multi_cancellable( &self.clips, &self.out_path, &gpu, @@ -589,8 +681,9 @@ impl Task for ExportMultiTask { &cfg, &export_params, &mut progress, + &self.control, ) - .map_err(|e| Error::from_reason(format!("{e:#}")))?; + .map_err(mp4_task_error)?; Ok((s.frames as u32, s.wall_s, s.fps, s.video_duration_s)) } @@ -599,11 +692,34 @@ impl Task for ExportMultiTask { } } +/// Le pendant MP4 de `create_gif_export_control` : même contrôle, passé à `export_multi`. Sa +/// présence dit aussi au TS que cet addon sait annuler un MP4 — un `.node` plus ancien ignore le +/// contrôle, et son export va au bout. +#[napi] +pub fn create_mp4_export_control() -> External { + External::new(ExportControl::default()) +} + +#[napi] +pub fn cancel_mp4_export(control: External) -> bool { + control.cancel() +} + +fn mp4_task_error(error: anyhow::Error) -> Error { + if error.is::() { + Error::from_reason("MP4_EXPORT_CANCELLED") + } else { + Error::from_reason(format!("{error:#}")) + } +} + /// Lance un export multiclip natif (vraie timeline → MP4) et résout `Promise`. /// `scene_json` : même `SceneDescription` que la preview (fond/layout/webcam/effets/curseur). /// `params` : taille/cadence/codec de sortie voulus (absent → 1920x1080/fps du 1er clip/h264). /// `on_progress(framesEncodées)` optionnel — rappelé côté JS à ~10 Hz max pendant le rendu ; /// le JS calcule lui-même le pourcentage (il connaît déjà le total attendu, durée×fps des clips). +/// `control` optionnel (`create_mp4_export_control`) : l'annuler arrête le rendu entre deux +/// frames, rejette avec `MP4_EXPORT_CANCELLED` et ne laisse aucun fichier. #[napi] pub fn export_multi( clips: Vec, @@ -611,6 +727,7 @@ pub fn export_multi( scene_json: Option, params: Option, on_progress: Option, + control: Option>, ) -> Result> { let clips = clips .into_iter() @@ -628,6 +745,7 @@ pub fn export_multi( clips, scene_json, params, + control: control.map(|c| (*c).clone()).unwrap_or_default(), on_progress: make_progress_tsfn(on_progress)?, })) } @@ -663,17 +781,17 @@ pub struct GifParamsInput { /// `Compositor` (équivalent de `cfg.cursor = false` dans /// `run_composited_multi`). #[napi] -pub fn create_gif_export_control() -> External { - External::new(GifExportControl::default()) +pub fn create_gif_export_control() -> External { + External::new(ExportControl::default()) } #[napi] -pub fn cancel_gif_export(control: External) -> bool { +pub fn cancel_gif_export(control: External) -> bool { control.cancel() } fn gif_task_error(error: anyhow::Error) -> Error { - if error.is::() { + if error.is::() { Error::from_reason("GIF_EXPORT_CANCELLED") } else { Error::from_reason(format!("{error:#}")) @@ -690,7 +808,7 @@ pub struct ExportGifTask { scene_json: Option, out_path: PathBuf, params: GifExportParams, - control: GifExportControl, + control: ExportControl, on_progress: Option>, } @@ -788,7 +906,7 @@ pub fn export_gif( scene_json: Option, params: Option, on_progress: Option, - control: Option>, + control: Option>, ) -> Result> { // Deliberately the same argument shape as `export_multi`: the caller builds // one clip list and one scene, and picks the container. Cursor comes from diff --git a/crates/compositor/build.rs b/crates/compositor/build.rs index 66f465153f..c55d8ee381 100644 --- a/crates/compositor/build.rs +++ b/crates/compositor/build.rs @@ -191,7 +191,7 @@ fn main() { fn compile_hlsl(out: &Path) { use windows::core::PCSTR; use windows::Win32::Graphics::Direct3D::Fxc::{D3DCompile, D3DCOMPILE_OPTIMIZATION_LEVEL3}; - use windows::Win32::Graphics::Direct3D::ID3DBlob; + use windows::Win32::Graphics::Direct3D::{ID3DBlob, D3D_SHADER_MACRO}; fn blob_bytes(blob: &ID3DBlob) -> Vec { unsafe { @@ -203,17 +203,25 @@ fn compile_hlsl(out: &Path) { const SRC: &str = "src/shaders.hlsl"; println!("cargo:rerun-if-changed={SRC}"); let hlsl = std::fs::read(SRC).expect(SRC); - // Tous les points d'entrée que `Compositor::new_inner` instancie, avec leur profil. - for (entry, profile) in [ - ("vs_main", "vs_5_0"), - ("ps_main", "ps_5_0"), - ("vs_fs", "vs_5_0"), - ("ps_y", "ps_5_0"), - ("ps_uv", "ps_5_0"), - ("ps_blur", "ps_5_0"), - ("ps_tex", "ps_5_0"), - ("ps_kawase_down", "ps_5_0"), - ("ps_kawase_up", "ps_5_0"), + // `ps_main` avec les modèles 3D : la variante que seuls ces calques utilisent (le haut de + // `shaders.hlsl`). La liste se termine par une entrée nulle. + let models = [ + D3D_SHADER_MACRO { Name: PCSTR(b"LAYER_MODELS\0".as_ptr()), Definition: PCSTR(b"1\0".as_ptr()) }, + D3D_SHADER_MACRO { Name: PCSTR::null(), Definition: PCSTR::null() }, + ]; + // Tous les points d'entrée que `Compositor::new_inner` instancie, avec leur profil, le nom du + // `.cso` qu'en lit `shader!` et leurs macros. + for (entry, profile, cso, defines) in [ + ("vs_main", "vs_5_0", "vs_main", None), + ("ps_main", "ps_5_0", "ps_main", None), + ("ps_main", "ps_5_0", "ps_main_models", Some(models.as_ptr())), + ("vs_fs", "vs_5_0", "vs_fs", None), + ("ps_y", "ps_5_0", "ps_y", None), + ("ps_uv", "ps_5_0", "ps_uv", None), + ("ps_blur", "ps_5_0", "ps_blur", None), + ("ps_tex", "ps_5_0", "ps_tex", None), + ("ps_kawase_down", "ps_5_0", "ps_kawase_down", None), + ("ps_kawase_up", "ps_5_0", "ps_kawase_up", None), ] { let (entry_z, profile_z) = (format!("{entry}\0"), format!("{profile}\0")); let (mut code, mut errors): (Option, Option) = (None, None); @@ -222,7 +230,7 @@ fn compile_hlsl(out: &Path) { hlsl.as_ptr().cast(), hlsl.len(), PCSTR(b"shaders.hlsl\0".as_ptr()), - None, + defines, None, PCSTR(entry_z.as_ptr()), PCSTR(profile_z.as_ptr()), @@ -234,9 +242,9 @@ fn compile_hlsl(out: &Path) { }; if let Err(e) = compiled { let log = errors.as_ref().map(blob_bytes).unwrap_or_default(); - panic!("{SRC} : {entry} ne compile pas ({e})\n{}", String::from_utf8_lossy(&log)); + panic!("{SRC} : {cso} ne compile pas ({e})\n{}", String::from_utf8_lossy(&log)); } - std::fs::write(out.join(format!("{entry}.cso")), blob_bytes(&code.unwrap())).expect(entry); + std::fs::write(out.join(format!("{cso}.cso")), blob_bytes(&code.unwrap())).expect(cso); } } diff --git a/crates/compositor/examples/live_free_run_bench_macos.rs b/crates/compositor/examples/live_free_run_bench_macos.rs new file mode 100644 index 0000000000..8b3ec5c3ba --- /dev/null +++ b/crates/compositor/examples/live_free_run_bench_macos.rs @@ -0,0 +1,125 @@ +//! Bench de la lecture libre de la preview (macOS) : rejoue la boucle de `render_thread` sur +//! un seul clip, à une vitesse donnée, et mesure le retard que la vue prend sur l'horloge. +//! +//! Même arithmétique que `render_thread` : l'accumulateur avance de `dt × vitesse`, `step()` +//! compose chaque frame due jusqu'à `max_steps` par tick (au-delà le reliquat est abandonné), +//! puis relit le RT UNE fois par tick, comme la publication macOS (pas de textures partagées). +//! `dt` est borné à 0,1 s comme dans `render_thread` : un tick plus long perd de l'horloge. +//! +//! Utilisation : +//! `cargo run --release --example live_free_run_bench_macos -- [vitesse=4] [secondes=8] [largeur=1920] [hauteur=1080]` + +#[cfg(target_os = "macos")] +fn main() -> anyhow::Result<()> { + use openscreen_compositor::compositor::Compositor; + use openscreen_compositor::config; + use openscreen_compositor::d3d::Gpu; + use openscreen_compositor::frame_geometry::live_params_from_scene; + use openscreen_compositor::live::Player; + use openscreen_compositor::scene::Scene; + use std::time::{Duration, Instant}; + + let mut args = std::env::args().skip(1); + let path = args.next().unwrap_or_else(|| { + eprintln!("usage: live_free_run_bench_macos [vitesse=4] [secondes=8] [largeur=1920] [hauteur=1080]"); + std::process::exit(2); + }); + let speed: f64 = args.next().and_then(|s| s.parse().ok()).unwrap_or(4.0); + let seconds: f64 = args.next().and_then(|s| s.parse().ok()).unwrap_or(8.0); + let width: u32 = args.next().and_then(|s| s.parse().ok()).unwrap_or(1920); + let height: u32 = args.next().and_then(|s| s.parse().ok()).unwrap_or(1080); + + let source = path.replace('\\', "/"); + let scene_json = format!( + r##"{{ + "clips": [{{"screenPath":"{source}","webcamPath":"","sourceStartSec":0,"sourceEndSec":100000,"webcamOffsetSec":0,"hasAudio":false}}], + "layout": {{"preset":"no-webcam","webcamSize":1.0,"webcamShape":"rounded","webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false}}, + "effects": {{"padding":0.1,"blur":false,"shadow":0.5,"roundnessFrac":0.02,"motionBlur":0.0}}, + "background": {{"kind":"color","color":"#303030"}}, + "zoomRegions": [], + "speedRegions": [], + "cursor": {{"show":false,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":0,"clipToBounds":false,"theme":"default"}}, + "cropByClip": [null], + "output": {{"width":{width},"height":{height},"fps":30}} + }}"## + ); + + let gpu = Gpu::create(false)?; + let mut cfg = config::all().pop().expect("au moins une config"); + cfg.zoom = false; + cfg.layout_anim = false; + let comp = Compositor::new_sized(&gpu, width, height)?; + let scene = Scene::from_json(&scene_json)?; + comp.set_live_params(live_params_from_scene(&scene)); + comp.set_scene(Some(scene.clone())); + comp.clear_cursor(); + + unsafe { + let mut player = Player::open(&path, "", &gpu)?; + player.set_programme_clock(Some(&scene), 0); + anyhow::ensure!(player.present_frame(&comp, &cfg, 0.0)?, "aucune frame à 0 s"); + // Temps source réellement joué, cumulé frame par frame : un clip plus court que la mesure + // reboucle à 0 (EOF), et la seule position finale compterait alors une seule passe. + let mut played = 0.0f64; + + let mut acc = 0.0f64; + let mut last = Instant::now(); + let t0 = last; + let (mut ticks, mut adopted, mut capped, mut published) = (0u64, 0u64, 0u64, 0u64); + // Ce que l'horloge de l'app a parcouru, en temps source : la cible que la vue devrait tenir. + let mut clock_source = 0.0f64; + while t0.elapsed().as_secs_f64() < seconds { + let now = Instant::now(); + let dt = (now - last).as_secs_f64().min(0.1); + last = now; + ticks += 1; + acc += dt * speed; + clock_source += (now - t0).as_secs_f64() * speed - clock_source; + let max_steps = ((3.0 * speed.max(1.0)).ceil() as i32).min(64); + let mut n = 0; + let mut stepped = false; + loop { + let before = player.screen_time_sec(); + if !player.step(&comp, &cfg, before + acc)? { + break; + } + stepped = true; + adopted += 1; + let after = player.screen_time_sec(); + played += if after >= before { after - before } else { after }; + acc = if after >= before { (acc - (after - before)).max(0.0) } else { 0.0 }; + n += 1; + if n >= max_steps { + capped += 1; + acc = 0.0; + break; + } + } + if stepped { + comp.readback_direct()?; + published += 1; + } else { + std::thread::sleep(Duration::from_millis(4)); + } + } + let wall = t0.elapsed().as_secs_f64(); + println!( + "vitesse {speed}x, {width}x{height}, {wall:.2} s : horloge {:.2} s source, vue {played:.2} s source, retard {:.2} s source ({:.2} s de lecture)", + clock_source, + clock_source - played, + (clock_source - played) / speed + ); + println!( + "ticks {ticks} ({:.1}/s), frames adoptées {adopted} ({:.1}/s), plafond atteint {capped} fois, publiées {published} ({:.1}/s)", + ticks as f64 / wall, + adopted as f64 / wall, + published as f64 / wall + ); + } + Ok(()) +} + +#[cfg(not(target_os = "macos"))] +fn main() { + eprintln!("live_free_run_bench_macos : macOS uniquement"); +} diff --git a/crates/compositor/src/audio.rs b/crates/compositor/src/audio.rs index 6b13df4c5b..de845541de 100644 --- a/crates/compositor/src/audio.rs +++ b/crates/compositor/src/audio.rs @@ -5,7 +5,9 @@ use crate::ffi::*; use crate::regions::SpeedSegment; -use crate::scene::{SceneAudio, SceneAudioTrack, SceneAudioTrackKind}; +use crate::scene::{ + SceneAudio, SceneAudioTrack, SceneAudioTrackKind, SceneClickHit, SceneClickSound, +}; use anyhow::{bail, Result}; use std::collections::{HashMap, VecDeque}; use std::f32::consts::PI; @@ -1732,6 +1734,67 @@ fn overlay_external_track(programme: &mut PlanarPcm, track: &SceneAudioTrack, du ); } +/// How much of a hit file gets decoded. The bundled samples are a few hundred milliseconds; +/// the window is what keeps a wrong path from buffering an hour of "click". +/// ponytail: fixed window, read the file's own duration if longer hits are ever shipped. +const CLICK_HIT_WINDOW_SEC: f64 = 1.0; + +/// Sum the take's click hits over the assembled programme: two decodes for the whole take, +/// then one add per hit — unlike an imported track, which decodes once per placement. +/// +/// The hits arrive already placed on the programme by the app, through the projection it lays +/// imported audio tracks out with, so speeding a section up cannot leave the sound behind. +pub fn mix_click_hits(mut programme: PlanarPcm, clicks: &SceneClickSound) -> PlanarPcm { + if clicks.hits.is_empty() + || clicks.down_path.is_empty() + || clicks.up_path.is_empty() + || programme.first().map_or(true, Vec::is_empty) + { + return programme; + } + let (down, up) = ( + decode_click_hit(&clicks.down_path), + decode_click_hit(&clicks.up_path), + ); + if down.is_empty() || up.is_empty() { + eprintln!("[audio] click hits undecodable; export keeps the picture only"); + return programme; + } + overlay_click_hits(&mut programme, &down, &up, &clicks.hits); + programme +} + +fn decode_click_hit(path: &str) -> PlanarPcm { + decode_clip_audio(path, 0.0, CLICK_HIT_WINDOW_SEC) + .ok() + .flatten() + .unwrap_or_default() +} + +/// Placement and level, with no ffmpeg in sight. `overlay_track_pcm` truncates at the +/// programme's end, so a hit the edit pushed past it is simply not heard. +fn overlay_click_hits( + programme: &mut PlanarPcm, + down: &PlanarPcm, + up: &PlanarPcm, + hits: &[SceneClickHit], +) { + for hit in hits { + let offset = (hit.time_sec.max(0.0) * AUDIO_OUTPUT_SAMPLE_RATE as f64).round() as usize; + let gain = hit.gain.clamp(0.0, 4.0); + overlay_track_pcm( + programme, + if hit.release { up } else { down }, + offset, + gain, + 0.0, + 0.0, + 0, + &[], + ); + } +} + /// How far a music bed dips while someone speaks. pub const DUCK_DEPTH_DB: f32 = -10.0; /// A 10 ms stretch of the voice counts as speech above this RMS level. The voice is already @@ -2268,6 +2331,66 @@ mod tests { assert_eq!(programme[0], vec![0.3, 0.3]); } + fn hit(time_sec: f64, gain: f32, release: bool) -> SceneClickHit { + SceneClickHit { time_sec, gain, release } + } + + #[test] + fn click_hits_sum_at_their_time_and_pick_their_sample() { + // down is 1.0 and up is 0.5, so the level heard at each offset says which sample played. + let (down, up) = (planar(&[1.0]), planar(&[0.5])); + let tick = 1.0 / AUDIO_OUTPUT_SAMPLE_RATE as f64; + let mut programme = planar(&[0.0, 0.0, 0.0]); + overlay_click_hits( + &mut programme, + &down, + &up, + &[ + hit(0.0, 1.0, false), + hit(tick, 1.0, true), + hit(2.0 * tick, 0.5, false), + ], + ); + assert_eq!(programme[0], vec![1.0, 0.5, 0.5]); + } + + #[test] + fn a_click_hit_the_edit_pushed_past_the_programme_is_not_heard() { + let mut programme = planar(&[0.25, 0.25]); + overlay_click_hits( + &mut programme, + &planar(&[1.0]), + &planar(&[1.0]), + &[hit(5.0, 1.0, false)], + ); + assert_eq!(programme[0], vec![0.25, 0.25]); + } + + #[test] + fn click_hits_are_decoded_from_files_and_summed_at_their_time() { + // The whole point of the scene carrying hits instead of a baked bed: the samples live in + // their own files and are read here, so this walks the same decode a real track does. + let dir = std::env::temp_dir(); + let down = dir.join(format!("openscreen-click-down-{}.wav", std::process::id())); + let up = dir.join(format!("openscreen-click-up-{}.wav", std::process::id())); + write_wav(&down, &planar(&[0.5, 0.5])); + write_wav(&up, &planar(&[0.25, 0.25])); + let clicks = SceneClickSound { + down_path: down.to_str().unwrap().into(), + up_path: up.to_str().unwrap().into(), + hits: vec![hit(0.0, 1.0, false), hit(1.0 / 48_000.0, 1.0, true)], + }; + let out = mix_click_hits(planar(&[0.0; 4]), &clicks); + let _ = std::fs::remove_file(&down); + let _ = std::fs::remove_file(&up); + // Each sample is two frames long: the press covers 0..1 and the hit placed one frame in + // covers 1..2, so the seam carries both and the programme is untouched past them. + assert_eq!( + out[0].iter().map(|v| (v * 100.0).round() / 100.0).collect::>(), + vec![0.5, 0.75, 0.25, 0.0] + ); + } + #[test] fn mix_external_tracks_skips_empty_windows() { let programme = planar(&[0.4, 0.4]); diff --git a/crates/compositor/src/compositor_linux.rs b/crates/compositor/src/compositor_linux.rs index 198e6e637c..17f736d0e3 100644 --- a/crates/compositor/src/compositor_linux.rs +++ b/crates/compositor/src/compositor_linux.rs @@ -43,15 +43,21 @@ pub use crate::frame_geometry::{ live_params_from_scene, webcam_shape_code, FIXTURE_FRAMES, LayerCB, LiveParams, OUT_H, OUT_W, }; use crate::frame_geometry::{ - cursor_model_cb, cursor_sprite_cb, cursor_sprite_dst, parse_hex, plan_cursor, plan_frame, - tilted_screen_cb, CursorPlacement, CursorPlanInput, FrameGeometryInput, ShadowCaster, - SpriteShape, + cursor_glass_cb, cursor_model_cb, cursor_sprite_cb, cursor_sprite_dst, parse_hex, plan_cursor, + plan_frame, tilted_screen_cb, CursorPlacement, CursorPlanInput, FrameGeometryInput, + ShadowCaster, SpriteShape, }; use crate::scene::{Scene, SceneBackground, WallpaperMotion}; const LAYER_WGSL: &str = include_str!("vk_shaders/layer.wgsl"); const BLUR_WGSL: &str = include_str!("vk_shaders/blur.wgsl"); +/// `layer.wgsl` avec ou sans ses modeles 3D (modes 15 a 17) : la constante que le shader teste +/// pour les compiler, prefixee au source (pourquoi deux pipelines : le haut de `layer.wgsl`). +fn layer_source(models: bool) -> String { + format!("const LAYER_MODELS: bool = {models};\n{LAYER_WGSL}") +} + /// Budget du cache de textures image (`img_cache`), en octets. /// /// Doit tenir le JEU ACTIF d'une frame -- au pire un wallpaper d'ecran ET un @@ -71,6 +77,14 @@ fn layer_bytes(cb: &LayerCB) -> &[u8] { unsafe { std::slice::from_raw_parts(cb as *const LayerCB as *const u8, LAYER_BYTES as usize) } } +/// Le maillage de Prism Glow tel que le lit le WGSL (`PrismMesh`) : les triangles, les +/// polygones, un point par vec4 (le pas d'un tableau uniform), puis les boîtes. +fn prism_mesh_bytes() -> Vec { + use crate::prism_mesh::{BOXES, POLYS, TRIS}; + let points = POLYS.iter().map(|&[x, y]| [x, y, 0.0, 0.0]); + TRIS.iter().copied().chain(points).chain(BOXES).flatten().flat_map(f32::to_ne_bytes).collect() +} + /// Un calque de fond deja lie, en attente de son `draw`. `_buf`/`_tex`/`_view` /// ne sont jamais relus : ils gardent en vie ce que le bind group reference /// jusqu'au submit. Ce backend encode toute la frame avant de la soumettre, la @@ -81,10 +95,25 @@ fn layer_bytes(cb: &LayerCB) -> &[u8] { /// ET le fond de la bulle webcam sont desormais construits par les memes /// methodes. struct BgDraw { - _buf: wgpu::Buffer, + layer: LayerBind, _tex: Option, _view: Option, +} + +/// Un calque pret a dessiner : son uniforme (qui doit vivre jusqu'au draw), son bind group, et +/// s'il est l'un des modeles 3D, que seul `pipeline_models` compile (`LayerCB::needs_models`). +/// `draw_layer` en tire le pipeline : c'est le mode du calque qui decide, pas le site d'appel. +struct LayerBind { + _buf: wgpu::Buffer, bind: wgpu::BindGroup, + models: bool, +} + +/// Le fond tel que la passe 1 (et son flou) l'a laisse dans le RT, et tout ce qui l'a decide +/// (`key`, cf. `compose_frame`). +struct BgCache { + key: crate::frame_geometry::BackgroundKey, + tex: wgpu::Texture, } /// Une copie RT -> staging DEJA SOUMISE, dont le mapping est arme mais pas @@ -262,16 +291,23 @@ pub struct Compositor { // Pipeline de calque (VS + FS `layer.wgsl`), sampler lineaire, bind group // layout (uniform + 2 textures + sampler). Immuables apres `new_sized`. + // `pipeline` n'a pas les modeles 3D (modes 15 a 17) : ils passent par `pipeline_models`. pipeline: wgpu::RenderPipeline, /// Meme shader et meme layout que `pipeline`, blend ADDITIF pondere par la /// constante de blend. Sert a sommer les copies de la trainee du curseur /// dans `accum` ; cf. `blend_add` cote Windows. pipeline_add: wgpu::RenderPipeline, + /// `pipeline` et `pipeline_add` avec les modeles 3D : le curseur modelise, l'impact de son + /// clic, l'appareil et son ombre, et rien d'autre (cf. `layer_source`, `draw_layer`). + pipeline_models: wgpu::RenderPipeline, + pipeline_add_models: wgpu::RenderPipeline, /// Copie plein ecran d'`accum` vers le RT en « over » premultiplie /// (`blur.wgsl` : `vs_fullscreen` + `fs_copy`). Utilise le layout du blur. pipeline_copy: wgpu::RenderPipeline, bind_group_layout: wgpu::BindGroupLayout, sampler: wgpu::Sampler, + /// Le maillage de Prism Glow (`prism_mesh.rs`) tel que le lit le WGSL (`PrismMesh`, binding 7). + prism_mesh: wgpu::Buffer, // Chaine de blur Kawase du fond (`blur.wgsl`) : layout dedie (uniform + 1 // tex + sampler), 2 pipelines (down/up), 3 textures de pyramide (1/2, 1/4, @@ -317,6 +353,8 @@ pub struct Compositor { scene: RefCell>, cursor: RefCell>, cursor_time: RefCell>, + /// Le métrage dans la dernière image composée, que l'éditeur lit avec elle (`live.rs`). + footage: std::cell::Cell>, timeline_time: RefCell>, /// Temps programme (secondes de sortie) -- cf. `FrameGeometryInput::programme_time`. programme_time: RefCell>, @@ -341,6 +379,9 @@ pub struct Compositor { /// touche depuis appartient au jeu actif et ne peut pas etre evince -- voir /// `cached_image`. img_frame_start: std::cell::Cell, + /// Le dernier fond fixe compose. Un fond qui ne bouge pas est le meme a chaque frame : on le + /// recopie au lieu de le redessiner (passe 1 de `compose_frame`). + bg_cache: RefCell>, /// Champs de distance des sprites de curseur (mode 15), R16F, par chemin, avec leur forme. /// Pas d'eviction : seuls les sprites du theme en cours y passent (~2,6 Mo pour les seize). sdf_cache: RefCell>, @@ -410,10 +451,14 @@ impl Compositor { feature_level: gpu.feature_level, }; - let module = gpu.device.create_shader_module(wgpu::ShaderModuleDescriptor { - label: Some("layer.wgsl"), - source: wgpu::ShaderSource::Wgsl(LAYER_WGSL.into()), - }); + let mk_module = |label: &str, models: bool| { + gpu.device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some(label), + source: wgpu::ShaderSource::Wgsl(layer_source(models).into()), + }) + }; + let module = mk_module("layer.wgsl", false); + let module_models = mk_module("layer.wgsl (modeles)", true); let sampler = gpu.device.create_sampler(&wgpu::SamplerDescriptor { label: Some("layer"), address_mode_u: wgpu::AddressMode::ClampToEdge, @@ -474,26 +519,44 @@ impl Compositor { // Pyramide de profondeur de champ (modes 8 et 18), `dummy` ailleurs : le // mode 18 lit a la fois le metrage, son rendu isole (binding 4) et elle. tex_entry(6), + // Le maillage de Prism Glow (mode 15). Un uniform et non des tables dans le + // shader : lavapipe les recopiait a chaque pixel de chaque calque, et une + // frame sans curseur se rendait 3,4 fois plus lentement. + wgpu::BindGroupLayoutEntry { + binding: 7, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: None, + }, + count: None, + }, ], }); + let prism_mesh = gpu.device.create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("prism-mesh"), + contents: &prism_mesh_bytes(), + usage: wgpu::BufferUsages::UNIFORM, + }); let pipeline_layout = gpu.device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { label: Some("layer"), bind_group_layouts: &[&bind_group_layout], push_constant_ranges: &[], }); - // Deux pipelines pour le MEME shader de calque : seul le blend change. - let mk_layer = |label: &str, blend: wgpu::BlendState| { + // Deux pipelines par source de calque : seul le blend change. + let mk_layer = |label: &str, module: &wgpu::ShaderModule, blend: wgpu::BlendState| { gpu.device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { label: Some(label), layout: Some(&pipeline_layout), vertex: wgpu::VertexState { - module: &module, + module, entry_point: Some("vs_main"), compilation_options: wgpu::PipelineCompilationOptions::default(), buffers: &[], }, fragment: Some(wgpu::FragmentState { - module: &module, + module, entry_point: Some("fs_main"), compilation_options: wgpu::PipelineCompilationOptions::default(), targets: &[Some(wgpu::ColorTargetState { @@ -512,22 +575,23 @@ impl Compositor { cache: None, }) }; - let pipeline = mk_layer("layer", wgpu::BlendState::PREMULTIPLIED_ALPHA_BLENDING); + let over = wgpu::BlendState::PREMULTIPLIED_ALPHA_BLENDING; + let pipeline = mk_layer("layer", &module, over); + let pipeline_models = mk_layer("layer-models", &module_models, over); // SOMME pondere : `src * constante + dst`. La constante (posee par pass // via `set_blend_constant`) vaut 1/taps, donc N copies d'un curseur // parfaitement immobile redonnent exactement ce curseur. Transcription // du `blend_add` D3D11 (BLEND_FACTOR / ONE / OP_ADD sur couleur ET // alpha) ; l'alpha doit suivre la couleur, sinon la somme n'est plus // premultipliee et la composition finale delave la trainee. - let add = wgpu::BlendComponent { + let sum = wgpu::BlendComponent { src_factor: wgpu::BlendFactor::Constant, dst_factor: wgpu::BlendFactor::One, operation: wgpu::BlendOperation::Add, }; - let pipeline_add = mk_layer( - "layer-add", - wgpu::BlendState { color: add, alpha: add }, - ); + let add = wgpu::BlendState { color: sum, alpha: sum }; + let pipeline_add = mk_layer("layer-add", &module, add); + let pipeline_add_models = mk_layer("layer-add-models", &module_models, add); // --- Chaine de blur Kawase du fond (`blur.wgsl`) --- let blur_module = gpu.device.create_shader_module(wgpu::ShaderModuleDescriptor { @@ -660,9 +724,12 @@ impl Compositor { render_h: h, pipeline, pipeline_add, + pipeline_models, + pipeline_add_models, pipeline_copy, bind_group_layout, sampler, + prism_mesh, blur_bgl, blur_down, blur_up, @@ -682,6 +749,7 @@ impl Compositor { scene: RefCell::new(None), cursor: RefCell::new(None), cursor_time: RefCell::new(None), + footage: std::cell::Cell::new(None), timeline_time: RefCell::new(None), programme_time: RefCell::new(None), text_raster: crate::text::TextRasterizer::new().ok(), @@ -689,6 +757,7 @@ impl Compositor { sdf_cache: RefCell::new(std::collections::HashMap::new()), img_tick: std::cell::Cell::new(0), img_frame_start: std::cell::Cell::new(0), + bg_cache: RefCell::new(None), ann_copy, ann_copy_view, ann_copy_mips, @@ -749,6 +818,7 @@ impl Compositor { ann_copy, ann_copy_view, ann_copy_mips, + bg_cache: RefCell::new(None), ..self }) } @@ -817,7 +887,8 @@ impl Compositor { view_formats: &[], }) }; - let rt = mk("rt", wgpu::TextureUsages::COPY_SRC); + // COPY_DST : le fond fixe y est recopie (`bg_cache`). + let rt = mk("rt", wgpu::TextureUsages::COPY_SRC | wgpu::TextureUsages::COPY_DST); let accum = mk("accum", wgpu::TextureUsages::empty()); let rt_view = rt.create_view(&wgpu::TextureViewDescriptor::default()); let accum_view = accum.create_view(&wgpu::TextureViewDescriptor::default()); @@ -1251,24 +1322,16 @@ impl Compositor { fn draw_screen_group( &self, rpass: &mut wgpu::RenderPass<'_>, - shadow: &Option<(wgpu::Buffer, wgpu::BindGroup)>, - window_frame: &Option<(wgpu::Buffer, wgpu::BindGroup)>, - screen: &wgpu::BindGroup, - device_frame: &Option<(wgpu::Buffer, wgpu::BindGroup)>, + shadow: &Option, + window_frame: &Option, + screen: &LayerBind, + device_frame: &Option, ) { - if let Some((_buf, bind)) = shadow { - rpass.set_bind_group(0, bind, &[]); - rpass.draw(0..4, 0..1); - } - if let Some((_buf, bind)) = window_frame { - rpass.set_bind_group(0, bind, &[]); - rpass.draw(0..4, 0..1); - } - rpass.set_bind_group(0, screen, &[]); - rpass.draw(0..4, 0..1); - if let Some((_buf, bind)) = device_frame { - rpass.set_bind_group(0, bind, &[]); - rpass.draw(0..4, 0..1); + for layer in [shadow.as_ref(), window_frame.as_ref(), Some(screen), device_frame.as_ref()] + .into_iter() + .flatten() + { + self.draw_layer(rpass, layer, false); } } @@ -1277,12 +1340,13 @@ impl Compositor { cb: &LayerCB, planes: Option<(&wgpu::TextureView, &wgpu::TextureView, &wgpu::TextureView)>, dummy: &wgpu::TextureView, - ) -> (wgpu::Buffer, wgpu::BindGroup) { + ) -> LayerBind { self.make_bind_b4(cb, planes, dummy, None, None) } /// `make_bind`, binding 4 impose (le rendu isole du mode 18, a la place du masque webcam /// qu'il ne lit pas), et la pyramide de profondeur de champ en binding 6 (modes 8 et 18). + /// Le mode 15 y met le champ de son sprite et le sprite. fn make_bind_b4( &self, cb: &LayerCB, @@ -1290,7 +1354,7 @@ impl Compositor { dummy: &wgpu::TextureView, b4: Option<&wgpu::TextureView>, dof: Option<&wgpu::TextureView>, - ) -> (wgpu::Buffer, wgpu::BindGroup) { + ) -> LayerBind { let uniform = self.gpu.device.create_buffer_init(&wgpu::util::BufferInitDescriptor { label: Some("layer-uniform"), contents: layer_bytes(cb), @@ -1338,9 +1402,29 @@ impl Compositor { binding: 6, resource: wgpu::BindingResource::TextureView(dof.unwrap_or(dummy)), }, + wgpu::BindGroupEntry { binding: 7, resource: self.prism_mesh.as_entire_binding() }, ], }); - (uniform, bind) + LayerBind { _buf: uniform, bind, models: cb.needs_models() } + } + + /// Dessine un calque par le pipeline plat que l'appelant a lie (`pipeline`, ou `pipeline_add` + /// si `add`) -- sauf un modele 3D, qui passe le temps de son draw par la variante + /// « modeles » de ce pipeline, la seule a le compiler (pourquoi : le haut de `layer.wgsl`). + fn draw_layer(&self, rpass: &mut wgpu::RenderPass<'_>, layer: &LayerBind, add: bool) { + rpass.set_bind_group(0, &layer.bind, &[]); + if layer.models { + let (models, flat) = if add { + (&self.pipeline_add_models, &self.pipeline_add) + } else { + (&self.pipeline_models, &self.pipeline) + }; + rpass.set_pipeline(models); + rpass.draw(0..4, 0..1); + rpass.set_pipeline(flat); + } else { + rpass.draw(0..4, 0..1); + } } /// Charge un PNG/JPEG (chemin fichier ou data URI) en texture RGBA8. Port @@ -1519,8 +1603,8 @@ impl Compositor { ..Default::default() }; let view = tex.create_view(&wgpu::TextureViewDescriptor::default()); - let (buf, bind) = self.make_bind(&cb, Some((&view, &view, &view)), dummy); - Ok(BgDraw { _buf: buf, _tex: Some(tex), _view: Some(view), bind }) + let layer = self.make_bind(&cb, Some((&view, &view, &view)), dummy); + Ok(BgDraw { layer, _tex: Some(tex), _view: Some(view) }) } /// Prepare le fond du mode « personnalise », peint DANS la bulle webcam juste @@ -1545,10 +1629,7 @@ impl Compositor { dummy: &wgpu::TextureView, ) -> BgDraw { const BLACK: [f32; 4] = [0.0, 0.0, 0.0, 1.0]; - let flat = |cb: LayerCB| { - let (buf, bind) = self.make_bind(&cb, None, dummy); - BgDraw { _buf: buf, _tex: None, _view: None, bind } - }; + let flat = |cb: LayerCB| BgDraw { layer: self.make_bind(&cb, None, dummy), _tex: None, _view: None }; let solid = |color: [f32; 4]| LayerCB { dst, quad_px, @@ -1685,7 +1766,7 @@ impl Compositor { // `color.a = 1` n'est pas decoratif : `fs_main` calcule son alpha en // `layer.color.a * alpha_mask`, donc le defaut (0) rendrait un quad // entierement transparent. - let (_uniform, bind) = self.make_bind( + let layer = self.make_bind( &LayerCB { dst: [0.0, 0.0, 1.0, 1.0], src, @@ -1718,8 +1799,7 @@ impl Compositor { occlusion_query_set: None, }); rpass.set_pipeline(&self.pipeline); - rpass.set_bind_group(0, &bind, &[]); - rpass.draw(0..4, 0..1); + self.draw_layer(&mut rpass, &layer, false); } encoder.copy_texture_to_buffer( wgpu::TexelCopyTextureInfo { @@ -2024,6 +2104,11 @@ impl Compositor { *self.webcam_mask.borrow_mut() = None; } + /// Le métrage dans la dernière image composée : ses coins et son warp (`FootageQuad`). + pub fn footage_quad(&self) -> Option { + self.footage.get() + } + /// Rend une frame dans le RT interne. Le screen `screen`/`webcam` sont des /// carriers `linux_frames` ; la geometrie vient de `plan_frame`. Coeur : /// fond uni + ecran cover-fit. `readback_direct` lit ensuite le RT. @@ -2099,6 +2184,7 @@ impl Compositor { timeline_t_override: *self.timeline_time.borrow(), programme_time: *self.programme_time.borrow(), }); + self.footage.set(Some(g.footage_quad([rw, rh]))); // (`wtw`/`wth` sont les dims de la TEXTURE webcam, consommees par le // cover-crop du calque PiP plus bas.) @@ -2155,12 +2241,13 @@ impl Compositor { // Calque ecran : mode 0 (rect droit, NV12 -> RGB) quand la rotation est // neutre, mode 8 (warp bilineaire inverse dans la bbox du quad projete) // sinon. Place par plan_frame (cover-fit + coins arrondis) ; - // `src = g.cut` (crop utilisateur + zoom en UV texture) dans les deux cas. + // `src = g.screen_src(g.cut)` (crop utilisateur + zoom en UV texture, bornes + // inversees pour un clip tourne) dans les deux cas. // // FLOU DE VELOCITE. Au mode 0, `src_prev`/`dst_prev` decrivent le MEME // calque a la frame precedente ; le shader remappe chaque pixel de sortie // par ce couple pour retrouver l'UV qu'il occupait alors, et floute le long - // du segment. `src_prev = g.cut` et non un `cut` d'avant : la coupe est + // du segment. `src_prev = g.screen_src(g.cut)` et non un `cut` d'avant : la coupe est // identique aux deux frames (`plan_frame` ne fait varier que le rect de // DESTINATION entre `s_dst` et `s_dst_prev`), ce que Windows documente // aussi. Le mouvement vient donc entierement de `dst_prev`. @@ -2177,11 +2264,13 @@ impl Compositor { // Et la remontee du contour interieur du chrome au-dessus de l'ecran (`mb.z` au mode 0, // `color.z` au mode 8) : l'arrondi du haut se fait par le cadre. 0 sans fenetre. let top_lift = g.screen_top_lift_px([rw, rh]); + // The screen texture's source rect: a turned clip samples it with reversed bounds. + let src_uv = g.screen_src(g.cut); let screen_layer = match tilt.as_ref() { None => { // Sous le masque d'un layout en bloc, rogne au slot (`mask_flat_screen`). let (dst, src, quad_px, radius_px) = - g.mask_flat_screen(g.s_dst, g.cut, s_px, g.s_radius, [rw, rh]); + g.mask_flat_screen(g.s_dst, src_uv, s_px, g.s_radius, [rw, rh]); LayerCB { dst, src, @@ -2189,7 +2278,7 @@ impl Compositor { radius_px, mode: 0.0, color: [1.0, 1.0, 1.0, 1.0], - src_prev: g.cut, + src_prev: src_uv, dst_prev: g.s_dst_prev, mb: [g.screen_pixel_taps([rw, rh]), g.mb_amount, top_lift, square_top], ..Default::default() @@ -2200,7 +2289,7 @@ impl Compositor { quad, s_px, quad_center_px, - g.cut, + src_uv, g.focus_plane, g.s_radius, top_lift, @@ -2210,12 +2299,12 @@ impl Compositor { g.tilt_pixel_trail([rw, rh]), ), }; - // Bind group construit AVANT le pass (doit vivre pendant tout le pass) ; - // `_screen_uniform` garde le buffer uniforme en vie (reference par le bind). + // Bind group construit AVANT le pass (doit vivre pendant tout le pass) ; `LayerBind` + // garde le buffer uniforme en vie (reference par le bind). let dummy = self.dummy_view(); // Binding 6 du mode 8 : la pyramide si l'effet tourne, sinon `dummy` (`k = 0`, le // shader n'y lit rien). - let (_screen_uniform, screen_bind) = self.make_bind_b4( + let screen_bind = self.make_bind_b4( &screen_layer, Some((&sy, &su, &sv)), &dummy, @@ -2291,11 +2380,20 @@ impl Compositor { ) }); + // Le fond de la frame d'avant, s'il est celui qu'on demande (`BackgroundKey`) : la passe 1 + // le recopie au lieu de le redessiner. + let bg_desc = scene_ref.as_ref().map(|s| &s.background); + let bg_cached = self + .bg_cache + .borrow() + .as_ref() + .is_some_and(|c| c.key.matches(bg_desc, cfg.bg_blur, lp.bg_color, [rw, rh])); + let bg_wanted = bg_layer.is_some(); + // Fond (gradient mode 5 OU image mode 6), dessine dans la passe de fond. - let bg_draw = bg_layer.and_then(|bl| match bl { + let bg_draw = bg_layer.filter(|_| !bg_cached).and_then(|bl| match bl { BgLayer::Gradient(cb) => { - let (buf, bind) = self.make_bind(&cb, None, &dummy); - Some(BgDraw { _buf: buf, _tex: None, _view: None, bind }) + Some(BgDraw { layer: self.make_bind(&cb, None, &dummy), _tex: None, _view: None }) } // Le wallpaper couvre tout le cadre, donc dst plein et pas de coins : // `image_bg_draw` sert aussi la bulle webcam, qui elle en a. @@ -2475,16 +2573,15 @@ impl Compositor { // replis, meme ordre. Seul le texte diverge, tinte cote shader (atlas R8) // au lieu d'une couleur bakee dans la texture. struct AnnDraw { - _buf: wgpu::Buffer, + layer: LayerBind, /// Gardent l'atlas / la texture image en vie jusqu'au submit. `None` /// pour les quads qui n'echantillonnent rien (plaque de fond, fleche). _glyphs: Option, _tex: Option, - bind: wgpu::BindGroup, } impl AnnDraw { - fn plain(buf: wgpu::Buffer, bind: wgpu::BindGroup) -> AnnDraw { - AnnDraw { _buf: buf, _glyphs: None, _tex: None, bind } + fn plain(layer: LayerBind) -> AnnDraw { + AnnDraw { layer, _glyphs: None, _tex: None } } } // FENETRE TEMPORELLE. Sans ce test, TOUTES les annotations du projet sont @@ -2502,6 +2599,8 @@ impl Compositor { .as_ref() .is_some_and(|s| s.annotations.iter().any(|a| a.kind == "blur" && visible(a))); let mut ann_draws: Vec = Vec::new(); + // Les flous de confidentialite, dessines sur le metrage avant le curseur. + let mut privacy_draws: Vec = Vec::new(); if let Some(scene) = scene_ref.as_ref() { // La liste arrive deja triee par zIndex cote app : l'ordre d'iteration // EST l'ordre de peinture. @@ -2543,8 +2642,7 @@ impl Compositor { mb: [1.0, half_stroke, 0.0, 0.0], ..Default::default() }; - let (buf, bind) = self.make_bind(&cb, None, &dummy); - ann_draws.push(AnnDraw::plain(buf, bind)); + ann_draws.push(AnnDraw::plain(self.make_bind(&cb, None, &dummy))); } "blur" => { let Some(blur) = a.blur.as_ref() else { continue }; @@ -2587,12 +2685,12 @@ impl Compositor { }; // La copie mipmappee au binding 1 (texY), la ou le mode 10 // la lit. - let (buf, bind) = self.make_bind( + let layer = self.make_bind( &cb, Some((&self.ann_copy_view, &self.ann_copy_view, &self.ann_copy_view)), &dummy, ); - ann_draws.push(AnnDraw::plain(buf, bind)); + privacy_draws.push(AnnDraw::plain(layer)); } "image" => { let Some(src) = a.image_path.as_ref().filter(|s| !s.is_empty()) else { @@ -2649,13 +2747,8 @@ impl Compositor { fx: [0.0, 0.0, 1.0, 1.0], ..Default::default() }; - let (buf, bind) = self.make_bind(&cb, Some((&view, &view, &view)), &dummy); - ann_draws.push(AnnDraw { - _buf: buf, - _glyphs: None, - _tex: Some(tex), - bind, - }); + let layer = self.make_bind(&cb, Some((&view, &view, &view)), &dummy); + ann_draws.push(AnnDraw { layer, _glyphs: None, _tex: Some(tex) }); } "text" => { let Some(raster) = self.text_raster.as_ref() else { continue }; @@ -2784,8 +2877,7 @@ impl Compositor { ), ..Default::default() }; - let (pbuf, pbind) = self.make_bind(&plate, None, &dummy); - ann_draws.push(AnnDraw::plain(pbuf, pbind)); + ann_draws.push(AnnDraw::plain(self.make_bind(&plate, None, &dummy))); } } @@ -2798,14 +2890,9 @@ impl Compositor { ..Default::default() }; // Atlas R8 au binding 1 (texY) que le mode 11 echantillonne. - let (buf, bind) = + let layer = self.make_bind(&cb, Some((&glyphs.view, &glyphs.view, &glyphs.view)), &dummy); - ann_draws.push(AnnDraw { - _buf: buf, - _glyphs: Some(glyphs), - _tex: None, - bind, - }); + ann_draws.push(AnnDraw { layer, _glyphs: Some(glyphs), _tex: None }); } _ => {} } @@ -2822,12 +2909,18 @@ impl Compositor { // sont PAS dessinees sur le RT mais dans `accum`, puis compositees en une // fois -- cf. le commentaire au point de dessin. struct CursorDraw { - _bufs: Vec, /// Le sprite, et pour le curseur modelise son champ de distance. _tex: Vec<(wgpu::Texture, wgpu::TextureView)>, - binds: Vec, + /// Une copie par echantillon de la trainee, le sprite (modes 7 et 13) ou le curseur + /// modelise (mode 15). + binds: Vec, + /// L'union des quads de `binds` (x0, y0, x1, y1 en fractions de la sortie) : tout ce + /// que la trainee peint dans `accum`, donc tout ce que sa recopie doit relire. + bounds: [f32; 4], /// L'impact des clics (mode 16), dessine SOUS le curseur, sur le RT. - impacts: Vec, + impacts: Vec, + /// Le cristal de Prism Glow : il lit la copie de l'image composee (`CursorPlan::glass`). + glass: bool, } let cursor_draw: Option = (|| { let track = cursor_ref.as_ref()?; @@ -2870,13 +2963,18 @@ impl Compositor { }) .collect() }; - let (mut bufs, mut binds) = (Vec::new(), Vec::new()); - let mut impacts = Vec::new(); - for cb in &plan.impacts { - let (buf, bind) = self.make_bind(cb, None, &dummy); - bufs.push(buf); - impacts.push(bind); - } + let mut binds = Vec::new(); + let mut bounds = [f32::MAX, f32::MAX, f32::MIN, f32::MIN]; + let mut grow = |dst: [f32; 4]| { + bounds = [ + bounds[0].min(dst[0]), + bounds[1].min(dst[1]), + bounds[2].max(dst[0] + dst[2]), + bounds[3].max(dst[1] + dst[3]), + ]; + }; + let impacts: Vec = + plan.impacts.iter().map(|cb| self.make_bind(cb, None, &dummy)).collect(); let sprites = scene_ref .as_ref() @@ -2906,7 +3004,9 @@ impl Compositor { let view = tex.create_view(&wgpu::TextureViewDescriptor::default()); // Curseur modelise (mode 15) : ce meme sprite extrude, `plan_cursor` en a tire la - // pose. Sprite au binding 1 (texY), champ au binding 2 (texU). Parite Windows/macOS. + // pose. La copie de l'image composee au binding 1 (texY), que refracte le cristal de + // Prism Glow, le champ au binding 4 (texMask), le sprite au binding 6 (texDof). + // Parite Windows/macOS. if let Some(pose) = plan.model { match self.cursor_sdf(&sprite.path) { Ok((sdf, shape)) => { @@ -2923,22 +3023,48 @@ impl Compositor { ) else { continue; }; - let (buf, bind) = - self.make_bind(&cb, Some((&view, &sdf_view, &view)), &dummy); - bufs.push(buf); - binds.push(bind); + binds.push(self.make_bind_b4( + &cb, + Some((&self.ann_copy_view, &dummy, &dummy)), + &dummy, + Some(&sdf_view), + Some(&view), + )); + grow(cb.dst); } return (!binds.is_empty()).then_some(CursorDraw { - _bufs: bufs, _tex: vec![(tex, view), (sdf, sdf_view)], binds, + bounds, impacts, + glass: plan.glass, }); } Err(e) => eprintln!("[curseur] champ de \"{}\" : {e:#}", sprite.path), } } + // Le verre a plat de Glass Lens (mode 19) : la copie de l'image composee qu'il + // refracte au binding 1 (texY), comme le cristal du mode 15. Parite Windows/macOS. + if let Some(name) = sprite.glass.as_deref().filter(|_| plan.model.is_none()) { + for placement in placements { + let Some(cb) = + cursor_glass_cb(placement, plan.size_px, name, plan.alpha, plan.clip, [rw, rh]) + else { + continue; + }; + binds.push(self.make_bind(&cb, Some((&self.ann_copy_view, &dummy, &dummy)), &dummy)); + grow(cb.dst); + } + return (!binds.is_empty()).then_some(CursorDraw { + _tex: vec![(tex, view)], + binds, + bounds, + impacts, + glass: plan.glass, + }); + } + for placement in placements { // Geometrie partagee avec Windows et macOS (`cursor_sprite_cb`) : // mode 7 droit, ou mode 13 pose sur le plan -- le clip vit alors @@ -2952,11 +3078,16 @@ impl Compositor { [rw, rh], ); // Sprite RGBA au binding 1 (texY) que le mode 7 echantillonne. - let (buf, bind) = self.make_bind(&cb, Some((&view, &view, &view)), &dummy); - bufs.push(buf); - binds.push(bind); + binds.push(self.make_bind(&cb, Some((&view, &view, &view)), &dummy)); + grow(cb.dst); } - Some(CursorDraw { _bufs: bufs, _tex: vec![(tex, view)], binds, impacts }) + Some(CursorDraw { + _tex: vec![(tex, view)], + binds, + bounds, + impacts, + glass: false, + }) })(); // Bind group de la passe de composition d'`accum` (layout du blur : // uniform + texture + sampler). Construit hors de la pass, comme les @@ -2998,7 +3129,7 @@ impl Compositor { }); // Passe 0 : pyramide de profondeur de champ, niveau 0 vide d'abord (le « over » // rend alors la source telle quelle), puis ses mips. - if let (Some(p), Some((_buf, bind))) = (dof_pyramid, &dof_fill) { + if let (Some(p), Some(fill)) = (dof_pyramid, &dof_fill) { { let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { label: Some("dof-pyramid-pass"), @@ -3015,8 +3146,7 @@ impl Compositor { occlusion_query_set: None, }); rpass.set_pipeline(&self.pipeline); - rpass.set_bind_group(0, bind, &[]); - rpass.draw(0..4, 0..1); + self.draw_layer(&mut rpass, fill, false); } self.generate_mips(&mut encoder, &p.mips); } @@ -3039,38 +3169,74 @@ impl Compositor { rpass.set_pipeline(&self.pipeline); self.draw_screen_group(&mut rpass, &screen_shadow, &window_frame, &screen_bind, &device_frame); } - // Passe 1 : fond (clear a `bg_clear` + gradient mode 5 eventuel). - { - let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { - label: Some("bg-pass"), - color_attachments: &[Some(wgpu::RenderPassColorAttachment { - view: &self.rt_view, - resolve_target: None, - ops: wgpu::Operations { - load: wgpu::LoadOp::Clear(wgpu::Color { - r: bg_clear[0] as f64, - g: bg_clear[1] as f64, - b: bg_clear[2] as f64, - a: bg_clear[3] as f64, - }), - store: wgpu::StoreOp::Store, - }, - })], - depth_stencil_attachment: None, - timestamp_writes: None, - occlusion_query_set: None, - }); - if let Some(bg) = &bg_draw { - rpass.set_pipeline(&self.pipeline); - rpass.set_bind_group(0, &bg.bind, &[]); - rpass.draw(0..4, 0..1); + // Passe 1 : fond (clear a `bg_clear` + gradient mode 5 eventuel) puis son flou -- ou, s'il + // n'a pas change (`BackgroundKey`), la copie qu'en a gardee la frame d'avant. + let extent = wgpu::Extent3d { + width: self.render_w, + height: self.render_h, + depth_or_array_layers: 1, + }; + if bg_cached { + let cache = self.bg_cache.borrow(); + let tex = &cache.as_ref().expect("bg_cached").tex; + encoder.copy_texture_to_texture(tex.as_image_copy(), self.rt.as_image_copy(), extent); + } else { + { + let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("bg-pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &self.rt_view, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color { + r: bg_clear[0] as f64, + g: bg_clear[1] as f64, + b: bg_clear[2] as f64, + a: bg_clear[3] as f64, + }), + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + }); + if let Some(bg) = &bg_draw { + rpass.set_pipeline(&self.pipeline); + self.draw_layer(&mut rpass, &bg.layer, false); + } + } + // Blur du fond (avant l'ecran), si active par la scene/l'inspector. + self.blur_bg(&mut encoder, cfg.bg_blur); + // Pas de cache pour un fond anime, ni pour un fond qui n'a pas pu se charger : la frame + // suivante reessaie. + let key = (bg_draw.is_some() == bg_wanted) + .then(|| { + crate::frame_geometry::BackgroundKey::of(bg_desc, cfg.bg_blur, lp.bg_color, [rw, rh]) + }) + .flatten(); + if let Some(key) = key { + let mut cache = self.bg_cache.borrow_mut(); + let tex = match cache.take() { + Some(c) if c.tex.size() == extent => c.tex, + _ => self.gpu.device.create_texture(&wgpu::TextureDescriptor { + label: Some("bg-cache"), + size: extent, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Rgba8Unorm, + usage: wgpu::TextureUsages::COPY_SRC | wgpu::TextureUsages::COPY_DST, + view_formats: &[], + }), + }; + encoder.copy_texture_to_texture(self.rt.as_image_copy(), tex.as_image_copy(), extent); + *cache = Some(BgCache { key, tex }); } } - // Blur du fond (avant l'ecran), si active par la scene/l'inspector. - self.blur_bg(&mut encoder, cfg.bg_blur); - // Passe 2 : avant-plan (ecran + webcam), compose par-dessus le fond - // (eventuellement floute) avec `LoadOp::Load`. Les annotations sont dans - // une passe a part, cf. plus bas. + // Passe 2 : l'ecran, compose par-dessus le fond (eventuellement floute) avec + // `LoadOp::Load`. Puis les flous de confidentialite, le curseur, la camera et les autres + // annotations, dans cet ordre : celui de Windows et macOS. { let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { label: Some("fg-pass"), @@ -3091,10 +3257,7 @@ impl Compositor { // elle doit passer sous lui mais au-dessus du fond (et, pour la // camera, au-dessus de l'ecran). match &trail { - Some((_buf, bind)) => { - rpass.set_bind_group(0, bind, &[]); - rpass.draw(0..4, 0..1); - } + Some(layer) => self.draw_layer(&mut rpass, layer, false), None => self.draw_screen_group( &mut rpass, &screen_shadow, @@ -3103,40 +3266,22 @@ impl Compositor { &device_frame, ), } - if let Some((_buf, bind)) = &webcam_shadow { - rpass.set_bind_group(0, bind, &[]); - rpass.draw(0..4, 0..1); - } - // Fond personnalise : ENTRE l'ombre et la camera. C'est ce sandwich qui - // remplace la branche « mode 3 » du shader — la camera, decoupee, se - // fond dessus par alpha ; l'ombre reste dessous, elle appartient a la - // bulle et non a son contenu. - if let Some(bg) = &webcam_bg { - rpass.set_bind_group(0, &bg.bind, &[]); - rpass.draw(0..4, 0..1); - } - if let Some((_buf, bind)) = &webcam_draw { - rpass.set_bind_group(0, bind, &[]); - rpass.draw(0..4, 0..1); - } } // Fige la frame composee pour les annotations « flou ». ICI et nulle part - // ailleurs : apres l'ecran et la camera (sinon un flou masquerait du vide) - // et avant la premiere annotation (sinon deux flous qui se recouvrent + // ailleurs : apres l'ecran (sinon un flou masquerait du vide) + // et avant le premier flou (sinon deux flous qui se recouvrent // s'echantillonnent l'un l'autre). Une passe de rendu ne peut pas lire sa // propre cible, d'ou la copie -- et d'ou le fait que les annotations // doivent avoir leur propre passe. if needs_ann_copy { self.generate_ann_mips(&mut encoder); } - // Passe 3 : annotations puis curseur net, par-dessus tout le reste. Elle - // existe meme sans flou : deux passes consecutives sur la MEME cible avec - // `LoadOp::Load` ne coutent rien de plus qu'une seule sur un GPU - // desktop, et un seul chemin de code vaut mieux qu'un branchement qui ne - // serait exerce que dans un projet sur dix. - { + // Passe 2 bis : les flous de confidentialite, sur le metrage, AVANT le curseur (parite + // Windows et macOS). Le curseur reste net par-dessus, et le cristal de Prism Glow, qui + // refracte l'image composee, n'y voit que des pixels deja floutes. + if !privacy_draws.is_empty() { let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { - label: Some("ann-pass"), + label: Some("privacy-pass"), color_attachments: &[Some(wgpu::RenderPassColorAttachment { view: &self.rt_view, resolve_target: None, @@ -3150,21 +3295,44 @@ impl Compositor { occlusion_query_set: None, }); rpass.set_pipeline(&self.pipeline); - for a in &ann_draws { - rpass.set_bind_group(0, &a.bind, &[]); - rpass.draw(0..4, 0..1); + for a in &privacy_draws { + self.draw_layer(&mut rpass, &a.layer, false); } + } + // Le cristal de Prism Glow refracte l'image telle qu'elle est composee a cet instant, + // flous compris : sa copie dans `ann_copy` (libre, les flous l'ont deja lue ; le mip 0 + // suffit), une fois pour toutes les copies de la trainee. + if cursor_draw.as_ref().is_some_and(|c| c.glass) { + encoder.copy_texture_to_texture( + self.rt.as_image_copy(), + self.ann_copy.as_image_copy(), + wgpu::Extent3d { width: self.render_w, height: self.render_h, depth_or_array_layers: 1 }, + ); + } + // Le curseur net, et l'impact des clics sous lui. La trainee, elle, a besoin de sa + // propre cible : elle suit. + { + let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("cursor-pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &self.rt_view, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Load, + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + }); + rpass.set_pipeline(&self.pipeline); // L'impact des clics, sous le curseur et sa trainee (dessinee plus bas). - for bind in cursor_draw.iter().flat_map(|c| &c.impacts) { - rpass.set_bind_group(0, bind, &[]); - rpass.draw(0..4, 0..1); + for layer in cursor_draw.iter().flat_map(|c| &c.impacts) { + self.draw_layer(&mut rpass, layer, false); } - // Curseur en dernier : au-dessus de l'ecran et des annotations. - // Une seule copie = curseur net, il tient dans cette pass. La - // trainee, elle, a besoin de sa propre cible (voir plus bas). if let Some(c) = cursor_draw.as_ref().filter(|c| c.binds.len() == 1) { - rpass.set_bind_group(0, &c.binds[0], &[]); - rpass.draw(0..4, 0..1); + self.draw_layer(&mut rpass, &c.binds[0], false); } } // TRAINEE DU CURSEUR : flou REEL, pas des copies discretes. @@ -3200,11 +3368,10 @@ impl Compositor { occlusion_query_set: None, }); rpass.set_pipeline(&self.pipeline_add); - for (k, bind) in c.binds.iter().enumerate() { + for (k, layer) in c.binds.iter().enumerate() { let w = crate::frame_geometry::cursor_tap_weight(k as u32, c.binds.len() as u32) as f64; rpass.set_blend_constant(wgpu::Color { r: w, g: w, b: w, a: w }); - rpass.set_bind_group(0, bind, &[]); - rpass.draw(0..4, 0..1); + self.draw_layer(&mut rpass, layer, true); } } let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { @@ -3221,9 +3388,76 @@ impl Compositor { timestamp_writes: None, occlusion_query_set: None, }); - rpass.set_pipeline(&self.pipeline_copy); - rpass.set_bind_group(0, abind, &[]); - rpass.draw(0..3, 0..1); + // Rien n'a ete peint dans `accum` hors des quads de la trainee : la recopie s'y borne + // au lieu de repasser sur toute la sortie, ce qui coutait 0,5 ms par frame 1080p + // (Radeon 610M) pour un curseur de quelques dizaines de pixels. + let (w, h) = (self.render_w as f32, self.render_h as f32); + let x0 = (c.bounds[0] * w).floor().clamp(0.0, w) as u32; + let y0 = (c.bounds[1] * h).floor().clamp(0.0, h) as u32; + let x1 = (c.bounds[2] * w).ceil().clamp(0.0, w) as u32; + let y1 = (c.bounds[3] * h).ceil().clamp(0.0, h) as u32; + if x1 > x0 && y1 > y0 { + rpass.set_scissor_rect(x0, y0, x1 - x0, y1 - y0); + rpass.set_pipeline(&self.pipeline_copy); + rpass.set_bind_group(0, abind, &[]); + rpass.draw(0..3, 0..1); + } + } + // La camera, au-dessus de l'ecran et du curseur. + { + let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("webcam-pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &self.rt_view, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Load, + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + }); + rpass.set_pipeline(&self.pipeline); + if let Some(layer) = &webcam_shadow { + self.draw_layer(&mut rpass, layer, false); + } + // Fond personnalise : ENTRE l'ombre et la camera. C'est ce sandwich qui + // remplace la branche « mode 3 » du shader — la camera, decoupee, se + // fond dessus par alpha ; l'ombre reste dessous, elle appartient a la + // bulle et non a son contenu. + if let Some(bg) = &webcam_bg { + self.draw_layer(&mut rpass, &bg.layer, false); + } + if let Some(layer) = &webcam_draw { + self.draw_layer(&mut rpass, layer, false); + } + } + // Passe 3 : les autres annotations, par-dessus tout le reste (les flous sont passes avant + // le curseur). Elle existe meme sans annotation : deux passes consecutives sur la MEME + // cible avec `LoadOp::Load` ne coutent rien de plus qu'une seule sur un GPU desktop, et un + // seul chemin de code vaut mieux qu'un branchement qui ne serait exerce que dans un projet + // sur dix. + { + let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("ann-pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &self.rt_view, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Load, + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + }); + rpass.set_pipeline(&self.pipeline); + for a in &ann_draws { + self.draw_layer(&mut rpass, &a.layer, false); + } } self.gpu.context.submit(std::iter::once(encoder.finish())); Ok(()) @@ -4041,7 +4275,7 @@ mod tests { planes: (&wgpu::TextureView, &wgpu::TextureView, &wgpu::TextureView), ) -> (u32, u32, Vec) { let dummy = comp.dummy_view(); - let (_buf, bind) = comp.make_bind(cb, Some(planes), &dummy); + let layer = comp.make_bind(cb, Some(planes), &dummy); let mut encoder = comp.gpu.device.create_command_encoder( &wgpu::CommandEncoderDescriptor { label: Some("test-layer") }, ); @@ -4061,8 +4295,7 @@ mod tests { occlusion_query_set: None, }); rpass.set_pipeline(&comp.pipeline); - rpass.set_bind_group(0, &bind, &[]); - rpass.draw(0..4, 0..1); + comp.draw_layer(&mut rpass, &layer, false); } comp.gpu.context.submit(std::iter::once(encoder.finish())); unsafe { comp.readback_direct().expect("readback_direct") } @@ -4679,6 +4912,8 @@ mod tests { clip_index: None, start_sec: 2.0, end_sec: 8.0, + continues_before: false, + continues_after: false, }); // 5 s : la montee (~1 s) est finie, le retour (~1,5 s) n'a pas commence. comp.set_timeline_time(Some(5.0)); @@ -4915,12 +5150,23 @@ mod tests { /// Ecran gris sur fond gris moyen, avec ou sans cadre de fenetre (`frame` est insere tel /// quel dans `effects`), droit ou incline (`rotation`, JSON). fn compose_framed(comp: &Compositor, gpu: &Gpu, frame: &str, rotation: &str) -> Vec { + compose_framed_shadow(comp, gpu, frame, rotation, 0.6) + } + + /// `compose_framed`, avec l'ombre de l'ecran `shadow` (0 l'eteint). + fn compose_framed_shadow( + comp: &Compositor, + gpu: &Gpu, + frame: &str, + rotation: &str, + shadow: f32, + ) -> Vec { let json = format!( r##"{{"clips":[{{"screenPath":"/s.mp4","webcamPath":"","sourceStartSec":0,"sourceEndSec":10,"webcamOffsetSec":0,"hasAudio":false}}], "layout":{{"preset":"no-webcam","webcamSize":1,"webcamShape":"rounded", "webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false, "screenRect":{{"x":0.1,"y":0.1,"width":0.8,"height":0.8}}}}, - "effects":{{"padding":0.2,"blur":false,"shadow":0.6,"roundnessFrac":0.03,"motionBlur":0{frame}}}, + "effects":{{"padding":0.2,"blur":false,"shadow":{shadow},"roundnessFrac":0.03,"motionBlur":0{frame}}}, "background":{{"kind":"color","color":"#6070a0"}}, "zoomRegions":[{{"clipIndex":0,"startSec":0,"endSec":10,"scale":1,"focusX":0.5,"focusY":0.5,"rotation":{rotation}}}], "annotations":[], @@ -5203,6 +5449,66 @@ mod tests { assert!(right[2] > 200 && right[0] < 60 && right[1] < 60, "droite {right:?} : bleu attendu"); } + /// Le fond fixe est recopie d'une frame a l'autre (`bg_cache`) au lieu d'etre redessine, et + /// un fond qui change ne laisse rien du precedent : un compositeur qui a rendu A, A, B puis A + /// rend chacun a l'octet comme un compositeur neuf. Un fond anime n'est jamais repris. + #[test] + fn a_static_background_is_reused_and_a_changed_one_redrawn() { + let Some(gpu) = gpu() else { return }; + let screen = FakeFrame::new(&gpu, 640, 360, |_, _| 60); + let webcam = FakeFrame::new(&gpu, 64, 64, |_, _| 60); + let render = |comp: &Compositor, background: &str| { + let json = format!( + r##"{{"clips":[{{"screenPath":"/s.mp4","webcamPath":"","sourceStartSec":0,"sourceEndSec":10,"webcamOffsetSec":0,"hasAudio":false}}], + "layout":{{"preset":"no-webcam","webcamSize":1,"webcamShape":"rounded", + "webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false, + "screenRect":{{"x":0.3,"y":0.3,"width":0.4,"height":0.4}}}}, + "effects":{{"padding":0.4,"blur":false,"shadow":0,"roundnessFrac":0,"motionBlur":0}}, + "background":{background}, + "zoomRegions":[],"annotations":[], + "cursor":{{"show":false,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":0, + "clipToBounds":false,"theme":"default"}}, + "cropByClip":[null], + "output":{{"width":640,"height":360,"fps":30}}}}"## + ); + let scene = Scene::from_json(&json).expect("scene json"); + comp.set_live_params(live_params_from_scene(&scene)); + comp.set_has_webcam(false); + comp.set_scene(Some(scene)); + let mut cfg = Cfg::c8(); + cfg.bg_blur = 0.0; + cfg.zoom = false; + cfg.layout_anim = false; + cfg.cursor = false; + cfg.shadow = false; + cfg.mblur_n = 1; + unsafe { + comp.compose_frame(screen.as_ptr(), webcam.as_ptr(), 0.0, &cfg) + .expect("compose_frame"); + comp.readback_direct().expect("readback_direct").2 + } + }; + let fresh = || Compositor::new_sized(&gpu, 640, 360).expect("Compositor::new_sized"); + let a = r##"{"kind":"gradient","angleDeg":90,"stops":["rgb(255, 0, 0)","rgb(0, 0, 255)"],"offsets":[0,1]}"##; + let b = r##"{"kind":"color","color":"#00ff00"}"##; + + let comp = fresh(); + let a1 = render(&comp, a); + assert!(comp.bg_cache.borrow().is_some(), "un fond fixe doit etre garde"); + let a2 = render(&comp, a); + let b1 = render(&comp, b); + let a3 = render(&comp, a); + assert!(a1 == a2, "le fond recopie doit etre celui qui a ete dessine"); + assert!(b1 == render(&fresh(), b), "un autre fond ne doit rien garder du precedent"); + assert!(a3 == a1, "revenir au premier fond doit le redessiner a l'identique"); + assert!(a1 != b1, "garde : les deux fonds different"); + + let aurora = r##"{"kind":"gradient","angleDeg":90,"stops":["rgb(255, 0, 0)","rgb(0, 0, 255)"],"offsets":[0,1],"motion":"aurora"}"##; + let comp = fresh(); + render(&comp, aurora); + assert!(comp.bg_cache.borrow().is_none(), "un fond anime ne doit pas etre garde"); + } + /// Le cadre de fenetre (mode 14) se dessine sur Linux comme ailleurs : `"none"` rend /// l'image d'avant a l'octet, le theme clair ajoute une barre claire, droite comme /// inclinee, et les deux themes different. `OPENSCREEN_FRAME_OUT` recoit les PNG. @@ -5313,6 +5619,22 @@ mod tests { } } + /// L'ombre d'un appareil est un modele elle aussi (mode 17, `device_shadow_cb`) : comme + /// l'appareil, elle n'existe que dans `pipeline_models`, et le pipeline plat n'en peindrait + /// rien. Elle doit donc changer l'image, a plat comme incline. + #[test] + fn a_device_frame_casts_its_shadow() { + let Some(gpu) = gpu() else { return }; + let comp = Compositor::new_sized(&gpu, 1280, 720).expect("Compositor::new_sized"); + let laptop = r#","frame":"laptop""#; + for (name, rotation) in [("flat", "null"), ("iso", r#""iso""#)] { + let lit = compose_framed_shadow(&comp, &gpu, laptop, rotation, 0.6); + let bare = compose_framed_shadow(&comp, &gpu, laptop, rotation, 0.0); + let shaded = lit.chunks_exact(4).zip(bare.chunks_exact(4)).filter(|(a, b)| a != b).count(); + assert!(shaded > 2_000, "{name} : l'ombre de l'appareil ne change que {shaded} px"); + } + } + // ----------------------------------------------------------------------- // Curseur modélisé (mode 15) : pendant de `tests/cursor_model_render.rs` (Windows). // ----------------------------------------------------------------------- @@ -5369,6 +5691,42 @@ mod tests { (y, uv) } + /// Pendant de `stripes` (Windows) : des rayures verticales nettes, claires et sombres, de 7 px + /// de source ; `inverse` les échange. + fn model_stripe_planes(inverse: bool) -> (Vec, Vec) { + let y = (0..640u32 * 360).map(|i| if (i % 640 / 7 % 2 == 0) != inverse { 200 } else { 60 }).collect(); + (y, vec![128; 640 * 180]) + } + + /// Pendant de `stripe_edges` (Windows) : les bords des rayures sur chaque ligne (x en px), où + /// le signe de l'écart entre les rendus sur les rayures et sur leur inverse change entre deux + /// pixels de `keep` à 3 px au plus. Un écart faible ne tranche pas. + fn model_stripe_edges(on: &[u8], on_inverse: &[u8], keep: impl Fn(usize) -> bool) -> Vec> { + (0..720) + .map(|y| { + let (mut edges, mut last) = (Vec::new(), None); + for x in 0..1280 { + let i = y * 1280 + x; + if !keep(i) { + last = None; + continue; + } + let d = model_luma(&on[i * 4..i * 4 + 3]) - model_luma(&on_inverse[i * 4..i * 4 + 3]); + if d.abs() < 30.0 { + continue; + } + if let Some((lx, light)) = last { + if light != (d > 0.0) && x - lx <= 3 { + edges.push((lx + x) as f32 / 2.0); + } + } + last = Some((x, d > 0.0)); + } + edges + }) + .collect() + } + fn compose_model( comp: &Compositor, screen: &FakeFrame, @@ -5523,6 +5881,22 @@ mod tests { .collect() } + /// Les pixels que couvre un curseur de VERRE (le cristal de Prism Glow, qui laisse voir le + /// contenu et le réfracte) : opaques, ou changés sur les deux teintes sans y être plus sombres + /// (une ombre ne fait qu'assombrir ; le verre réfracte, reflète et luit). + fn model_covered(on_blue: &[u8], on_orange: &[u8], bare: &[u8], bare_orange: &[u8]) -> Vec { + let opaque = model_opaque(on_blue, on_orange, bare); + (0..1280 * 720) + .map(|i| { + let lit = |on: &[u8], under: &[u8]| { + let (a, b) = (&on[i * 4..i * 4 + 3], &under[i * 4..i * 4 + 3]); + a != b && model_luma(a) >= model_luma(b) - 2.0 + }; + opaque[i] || (lit(on_blue, bare) && lit(on_orange, bare_orange)) + }) + .collect() + } + fn model_iou(a: &[bool], b: &[bool]) -> f32 { let inter = a.iter().zip(b).filter(|(x, y)| **x && **y).count(); let union = a.iter().zip(b).filter(|(x, y)| **x || **y).count(); @@ -5592,6 +5966,34 @@ mod tests { } } + /// La trainee du curseur modelise s'accumule par `pipeline_add_models` : le pipeline plat ne + /// compile pas le mode 15 et n'en peindrait rien. En mouvement, flou du curseur au maximum, + /// le modele doit donc bien apparaitre. + #[test] + fn the_modelled_cursor_draws_its_trail() { + let Some(gpu) = gpu() else { return }; + let comp = Compositor::new_sized(&gpu, 1280, 720).expect("Compositor::new_sized"); + let (y, uv) = model_screen_planes(false); + let screen = FakeFrame::from_planes(&gpu, 640, 360, &y, &uv); + // Traverse 60 % de la largeur en une seconde autour de t = 2 s : ~13 px par frame, donc + // plusieurs copies dans `accum`. + let moving = crate::cursor::CursorTrack::new( + vec![(1.5, 0.2, 0.45), (2.5, 0.8, 0.45)], + vec![], + vec![(0.0, "arrow".to_string())], + ); + let json = |show: bool| { + model_scene_json("null", Some(true), "default", show, 3.0) + .replace(r#""motionBlur":0,"clickBounce""#, r#""motionBlur":1,"clickBounce""#) + }; + assert!(json(true).contains(r#""motionBlur":1,"clickBounce""#), "garde : le flou du curseur"); + let bare = compose_model(&comp, &screen, &json(false), &moving); + let trail = compose_model(&comp, &screen, &json(true), &moving); + let drawn = bare.chunks_exact(4).zip(trail.chunks_exact(4)).filter(|(a, b)| a != b).count(); + // Taille 3 : ~1000 px de silhouette a l'arret, etales ici par la trainee. + assert!(drawn > 1000, "trainee du curseur modelise : {drawn} px dessines"); + } + /// Chaque état garde son art et sa silhouette : posé au centre de l'écran (vu de face), le /// modèle couvre ce que couvre le sprite plat, avec ses couleurs ; il porte une ombre en /// l'air et suit l'inclinaison du plan. Le réglage éteint garde le sprite plat. @@ -5681,7 +6083,8 @@ mod tests { /// Pendant de `the_sculpted_cursors_stand_at_the_hotspot` (Windows) : la flèche et la main /// sculptées des thèmes d'origine (`sculpt.rs`) passent par le WGSL, tiennent au hotspot, en - /// bas à droite de lui, et portent leur ombre en l'air. + /// bas à droite de lui, et portent leur ombre en l'air. Le cristal de Prism Glow est de verre : + /// son corps compte ce qu'il couvre, et une bonne part de lui change avec la teinte de l'écran. #[test] fn the_sculpted_cursors_stand_at_the_hotspot() { let Some(gpu) = gpu() else { return }; @@ -5693,11 +6096,12 @@ mod tests { let extruded = model_scene_json("null", Some(true), "default", true, 5.0); let hidden = model_scene_json("null", Some(true), "default", false, 5.0); let bare = compose_model(&comp, &blue, &hidden, &model_track("arrow", false, 0.5)); + let bare_orange = compose_model(&comp, &orange, &hidden, &model_track("arrow", false, 0.5)); let mut failures = Vec::new(); for state in ["arrow", "pointer"] { let still = model_track(state, false, 0.5); let sprite = compose_model(&comp, &blue, &extruded, &still); - for theme in ["studio-ink", "prism-glow", "pop-coral", "pixel-candy", "star-sprout"] { + for theme in ["studio-ink", "prism-glow", "pop-coral", "pixel-candy", "star-sprout", "glass-lens"] { // Le sprite reste celui du theme par defaut : seul le nom pose le modele. let json = extruded .replace(&format!(r#"/{state}.png","#), &format!(r#"/{state}.png","sculpt":"{theme}/{state}","#)); @@ -5707,7 +6111,12 @@ mod tests { failures.push(format!("{theme}/{state}: le sprite extrude au lieu du modele")); } let (tip, u) = model_tip(&json, &still); - let mask = model_opaque(&hover, &hover_b, &bare); + let glass = matches!(theme, "prism-glow" | "glass-lens"); + let mask = if glass { + model_covered(&hover, &hover_b, &bare, &bare_orange) + } else { + model_opaque(&hover, &hover_b, &bare) + }; let (mut body, mut c, mut near) = (0usize, [0.0f32; 2], f32::MAX); for (i, _) in mask.iter().enumerate().filter(|(_, m)| **m) { let (x, y) = ((i % 1280) as f32, (i / 1280) as f32); @@ -5735,6 +6144,220 @@ mod tests { if shadow * 10 < body * 3 { failures.push(format!("{theme}/{state}: pas d'ombre en l'air ({shadow} px)")); } + if glass { + let through = (0..1280 * 720).filter(|&i| mask[i] && hover[i * 4..i * 4 + 3] != hover_b[i * 4..i * 4 + 3]).count(); + println!("{theme}/{state} : {through} px du corps laissent voir l'ecran"); + if through * 10 < body * 3 { + failures.push(format!("{theme}/{state}: le cristal ne laisse voir l'ecran que sur {through} px")); + } + } + } + } + assert!(failures.is_empty(), "{failures:#?}"); + } + + /// Pendant de `the_flat_glass_lens_refracts_the_picture` (Windows) : Glass Lens, 3D éteinte, + /// passe par le verre à plat du WGSL (mode 19), sans ombre, si bien que tout pixel changé est + /// le sien. Il tient au hotspot, en bas à droite de lui ; l'écran se voit au travers ; le trait + /// le cerne ; et le verre réfracte : sur des rayures, des bords vus au travers de la lentille + /// sont à plusieurs pixels de tout bord de la même ligne sans elle. + #[test] + fn the_flat_glass_lens_refracts_the_picture() { + let Some(gpu) = gpu() else { return }; + let comp = Compositor::new_sized(&gpu, 1280, 720).expect("Compositor::new_sized"); + let (y, uv) = model_screen_planes(false); + let blue = FakeFrame::from_planes(&gpu, 640, 360, &y, &uv); + let (y, uv) = model_screen_planes(true); + let orange = FakeFrame::from_planes(&gpu, 640, 360, &y, &uv); + let (y, uv) = model_stripe_planes(false); + let light = FakeFrame::from_planes(&gpu, 640, 360, &y, &uv); + let (y, uv) = model_stripe_planes(true); + let dark = FakeFrame::from_planes(&gpu, 640, 360, &y, &uv); + let flat = model_scene_json("null", Some(false), "glass-lens", true, 5.0); + let hidden = model_scene_json("null", Some(false), "glass-lens", false, 5.0); + let bare = compose_model(&comp, &blue, &hidden, &model_track("arrow", false, 0.5)); + let bare_orange = compose_model(&comp, &orange, &hidden, &model_track("arrow", false, 0.5)); + let bare_light = compose_model(&comp, &light, &hidden, &model_track("arrow", false, 0.5)); + let bare_dark = compose_model(&comp, &dark, &hidden, &model_track("arrow", false, 0.5)); + let bare_edges = model_stripe_edges(&bare_light, &bare_dark, |_| true); + let mut failures = Vec::new(); + for state in ["arrow", "pointer"] { + let still = model_track(state, false, 0.5); + let sprite = compose_model(&comp, &blue, &flat, &still); + let json = flat.replace(&format!(r#"/{state}.png","#), &format!(r#"/{state}.png","glass":"glass-lens/{state}","#)); + let (on_blue, on_orange) = (compose_model(&comp, &blue, &json, &still), compose_model(&comp, &orange, &json, &still)); + let (on_light, on_dark) = (compose_model(&comp, &light, &json, &still), compose_model(&comp, &dark, &json, &still)); + model_save(&format!("glass-flat-{state}"), &on_blue); + model_save(&format!("glass-flat-{state}-stripes"), &on_light); + if on_blue == sprite { + failures.push(format!("glass-lens/{state}: le sprite plat au lieu du verre")); + } + let (tip, u) = model_tip(&json, &still); + let rgb = |img: &[u8], i: usize| [img[i * 4], img[i * 4 + 1], img[i * 4 + 2]]; + let (mut body, mut c, mut near) = (0usize, [0.0f32; 2], f32::MAX); + let (mut through, mut line) = (0usize, 0usize); + for i in 0..1280 * 720 { + let (a, b) = (rgb(&on_blue, i), rgb(&on_orange, i)); + if a == rgb(&bare, i) && b == rgb(&bare_orange, i) { + continue; + } + let (x, y) = ((i % 1280) as f32, (i / 1280) as f32); + body += 1; + c = [c[0] + x, c[1] + y]; + near = near.min((x - tip[0]).hypot(y - tip[1])); + if a != b { + through += 1; + } else if model_luma(&a) < 60.0 { + line += 1; + } + } + let c = [c[0] / body.max(1) as f32, c[1] / body.max(1) as f32]; + // Les bords vus au travers du verre, chacun à sa distance du bord le plus proche de la + // même ligne sans lui. + let in_glass = |i: usize| rgb(&on_light, i) != rgb(&bare_light, i) || rgb(&on_dark, i) != rgb(&bare_dark, i); + let shifts: Vec = model_stripe_edges(&on_light, &on_dark, in_glass) + .iter() + .zip(&bare_edges) + .flat_map(|(seen, under)| seen.iter().map(|e| under.iter().map(|b| (e - b).abs()).fold(f32::MAX, f32::min))) + .collect(); + let shifted = shifts.iter().filter(|s| **s >= 3.0).count(); + let widest = shifts.iter().copied().fold(0.0, f32::max); + println!( + "glass-lens/{state} a plat : unite {u:.1} px, corps {body} px, a {near:.1} px du hotspot {tip:?}, \ + centroide {c:?}, {through} px laissent voir l'ecran, {line} px de trait, {} bords vus au travers, \ + {shifted} decales de 3 px ou plus, jusqu'a {widest:.1} px", + shifts.len() + ); + if (body as f32) < 0.12 * u * u { + failures.push(format!("glass-lens/{state}: {body} px de corps pour {u:.0} px d'unite")); + } + if near > 0.08 * u { + failures.push(format!("glass-lens/{state}: le verre est a {near:.1} px du hotspot")); + } + if !(c[0] > tip[0] && c[1] > tip[1] + 0.2 * u) { + failures.push(format!("glass-lens/{state}: corps en {c:?}, pas en bas a droite de {tip:?}")); + } + if through * 10 < body * 5 { + failures.push(format!("glass-lens/{state}: l'ecran ne se voit que sur {through} px")); + } + if line * 20 < body { + failures.push(format!("glass-lens/{state}: {line} px de trait seulement")); + } + if shifted < 10 { + failures.push(format!( + "glass-lens/{state}: le verre ne decale que {shifted} bords de 3 px ou plus, au plus de {widest:.1} px" + )); + } + } + assert!(failures.is_empty(), "{failures:#?}"); + } + + /// Le métrage, puis ses flous de confidentialité, puis le curseur, net par-dessus (parité + /// Windows et macOS) : le cristal de Prism Glow réfracte l'image déjà floutée. Sous le flou, + /// des rayures fines décalées d'une colonne donnent la même image floutée ; si le verre lisait + /// la vidéo brute, il montrerait les rayures, et les deux rendus différeraient à travers lui. + #[test] + fn the_crystal_refracts_the_blurred_picture() { + let Some(gpu) = gpu() else { return }; + let comp = Compositor::new_sized(&gpu, 1280, 720).expect("Compositor::new_sized"); + // Moitié gauche grise, moitié droite (la zone floutée) rayée une colonne sur deux. + let (w, h) = (640u32, 360u32); + let stripes = |phase: u32| { + let y: Vec = (0..w * h) + .map(|i| if i % w < w / 2 { 150 } else if (i % w + phase) % 2 == 0 { 40 } else { 220 }) + .collect(); + FakeFrame::from_planes(&gpu, w, h, &y, &vec![128; (w * (h / 2)) as usize]) + }; + let (a, b) = (stripes(0), stripes(1)); + let blur = r#""annotations":[{"id":"b","startSec":0,"endSec":10,"kind":"blur","x":0.5,"y":0.0,"w":0.5,"h":1.0,"blur":{"style":"blur","shape":"rectangle","color":"white","intensity":24,"blockSize":16}}]"#; + let json = |show: bool| { + model_scene_json("null", Some(true), "default", show, 5.0) + .replace(r#"/arrow.png","#, r#"/arrow.png","sculpt":"prism-glow/arrow","#) + .replace(r#""annotations":[]"#, blur) + }; + let mut failures = Vec::new(); + // Tout entier dans la zone, puis à cheval sur son bord. + for x in [0.7f32, 0.48] { + let track = crate::cursor::CursorTrack::new(vec![(0.0, x, 0.45), (9.0, x, 0.45)], vec![], vec![(0.0, "arrow".to_string())]); + let (ra, rb) = (compose_model(&comp, &a, &json(true), &track), compose_model(&comp, &b, &json(true), &track)); + let bare = compose_model(&comp, &a, &json(false), &track); + model_save(&format!("glass-over-blur-{x}"), &ra); + let px = |img: &[u8], i: usize, c: usize| img[i * 4 + c] as i32; + let hidden = (0..1280 * 720).filter(|&i| (0..3).any(|c| (px(&ra, i, c) - px(&rb, i, c)).abs() > 24)).count(); + // Le serti net du curseur : des sauts francs, entre voisins, de ce qu'il change. + let d = |i: usize, c: usize| px(&ra, i, c) - px(&bare, i, c); + let sharp = (0..1280 * 720 - 1) + .filter(|&i| i % 1280 < 1279 && (0..3).any(|c| (d(i, c) - d(i + 1, c)).abs() > 60)) + .count(); + println!("cristal en x = {x} : {sharp} bords francs, {hidden} px qui suivent les rayures cachées"); + if sharp < 50 { + failures.push(format!("x = {x} : {sharp} bords francs, le curseur n'est pas net par-dessus le flou")); + } + if hidden > 0 { + failures.push(format!("x = {x} : {hidden} px montrent les rayures cachées sous le flou")); + } + } + assert!(failures.is_empty(), "{failures:#?}"); + } + + /// Les thèmes cerclés gardent le trait de leur dessin (`design/cursors/`) : sur la + /// fleche comme sur la main, le modele est pour une bonne part de la couleur du trait (le + /// plateau, le jonc, les rainures entre les doigts), autour de la couleur du corps, et montre + /// ce qu'il porte devant : l'etoile jaune de Star Sprout, le calque et les tirets jaunes de la + /// fleche de Pop Coral, les tirets corail de sa main. Les parts comptent les pixels francs : + /// ceux que l'antialiasing mêle aux bords n'entrent dans aucune. + #[test] + fn the_rimmed_models_keep_the_outline_of_their_art() { + type Rgb = [i32; 3]; + let navy = |[r, g, b]: Rgb| b > r + 20 && r < 110 && g < 130; + let black = |[r, g, b]: Rgb| r.max(g).max(b) < 70; + let mint = |[r, g, b]: Rgb| g > 200 && g > r + 15 && b > 150; + let ivory = |[r, g, b]: Rgb| r > 200 && g > 190 && b > 160 && r - b < 70; + let coral = |[r, g, b]: Rgb| r > 200 && g < 150 && b < 140; + let yellow = |[r, g, b]: Rgb| r > 200 && g > 150 && b < 120; + // (thème, état, trait, corps, ornement, parts minimales du trait, du corps, de l'ornement) + let cases: [(&str, &str, &dyn Fn(Rgb) -> bool, &dyn Fn(Rgb) -> bool, &dyn Fn(Rgb) -> bool, [f32; 3]); 6] = [ + ("studio-ink", "arrow", &black, &ivory, &|_| false, [0.3, 0.15, 0.0]), + ("studio-ink", "pointer", &black, &ivory, &|_| false, [0.2, 0.3, 0.0]), + ("pop-coral", "arrow", &navy, &coral, &yellow, [0.2, 0.2, 0.05]), + ("pop-coral", "pointer", &navy, &yellow, &coral, [0.18, 0.3, 0.02]), + ("star-sprout", "arrow", &navy, &mint, &yellow, [0.25, 0.2, 0.03]), + ("star-sprout", "pointer", &navy, &ivory, &yellow, [0.25, 0.2, 0.03]), + ]; + let Some(gpu) = gpu() else { return }; + let comp = Compositor::new_sized(&gpu, 1280, 720).expect("Compositor::new_sized"); + let (y, uv) = model_screen_planes(false); + let blue = FakeFrame::from_planes(&gpu, 640, 360, &y, &uv); + let (y, uv) = model_screen_planes(true); + let orange = FakeFrame::from_planes(&gpu, 640, 360, &y, &uv); + let extruded = model_scene_json("null", Some(true), "default", true, 5.0); + let hidden = model_scene_json("null", Some(true), "default", false, 5.0); + let bare = compose_model(&comp, &blue, &hidden, &model_track("arrow", false, 0.5)); + let mut failures = Vec::new(); + for (theme, state, rim, body, extra, [min_rim, min_body, min_extra]) in cases { + let still = model_track(state, false, 0.5); + let json = extruded + .replace(&format!(r#"/{state}.png","#), &format!(r#"/{state}.png","sculpt":"{theme}/{state}","#)); + let (hover, hover_b) = (compose_model(&comp, &blue, &json, &still), compose_model(&comp, &orange, &json, &still)); + let mask = model_opaque(&hover, &hover_b, &bare); + let (mut total, mut counts) = (0usize, [0usize; 3]); + for (i, _) in mask.iter().enumerate().filter(|(_, m)| **m) { + let p = [hover[i * 4] as i32, hover[i * 4 + 1] as i32, hover[i * 4 + 2] as i32]; + total += 1; + for (k, class) in [rim, body, extra].iter().enumerate() { + counts[k] += usize::from(class(p)); + } + } + let [rim_share, body_share, extra_share] = counts.map(|k| k as f32 / total.max(1) as f32); + println!("{theme}/{state} : {total} px, trait {rim_share:.3}, corps {body_share:.3}, ornement {extra_share:.3}"); + if !(min_rim..0.7).contains(&rim_share) { + failures.push(format!("{theme}/{state}: {rim_share:.3} de trait, il a disparu ou tout mange")); + } + if body_share < min_body { + failures.push(format!("{theme}/{state}: {body_share:.3} de couleur du corps")); + } + if extra_share < min_extra { + failures.push(format!("{theme}/{state}: {extra_share:.3} d'ornement")); } } assert!(failures.is_empty(), "{failures:#?}"); diff --git a/crates/compositor/src/compositor_macos.rs b/crates/compositor/src/compositor_macos.rs index b00a288eef..4280ef26b8 100644 --- a/crates/compositor/src/compositor_macos.rs +++ b/crates/compositor/src/compositor_macos.rs @@ -261,6 +261,8 @@ pub struct Compositor { scene: RefCell>, cursor: RefCell>, cursor_time: RefCell>, + /// Le métrage dans la dernière image composée, que l'éditeur lit avec elle (`live.rs`). + footage: std::cell::Cell>, timeline_time: RefCell>, /// Temps programme (secondes de sortie) — cf. `FrameGeometryInput::programme_time`. programme_time: RefCell>, @@ -316,6 +318,17 @@ pub struct Compositor { pipeline_fs_tex: metal::RenderPipelineState, /// `vs_main` + `ps_main` en additif : les échantillons de traînée du curseur. pipeline_add: metal::RenderPipelineState, + /// `pipeline_main` et `pipeline_add` compilés AVEC les modèles 3D (`LAYER_MODELS`, cf. + /// l'en-tête de `shaders.metal`). Les deux autres ne les ont pas ; `draw_layer` bascule sur + /// ceux-ci le temps du draw d'un modèle (`LayerCB::needs_models`). + pipeline_main_models: metal::RenderPipelineState, + pipeline_add_models: metal::RenderPipelineState, + /// La passe en cours a été ouverte sur `pipeline_add` (cf. `begin_pass`) : dit à + /// `draw_layer` laquelle des deux variantes « modèles » prendre. + layer_add: std::cell::Cell, + /// Le dernier fond fixe composé et sa clé (`BackgroundKey`) : recopié dans le RT tant + /// que la clé tient, au lieu d'être redessiné. + bg_cache: RefCell>, /// Buffer d'accumulation ISOLÉ (transparent) pour la traînée. Accumuler directement sur /// le RT reviendrait à AJOUTER du blanc à ce qui est déjà dessous : sur un fond clair, /// le curseur disparaît. Même raisonnement que côté D3D11. @@ -537,6 +550,18 @@ enum Blend { Add, } +/// `shaders.metal` avec les modèles 3D : la source dont seuls les pipelines de ces calques sont +/// tirés (cf. son en-tête). +fn msl_source_with_models() -> String { + format!("#define LAYER_MODELS 1\n{}", include_str!("shaders.metal")) +} + +/// Le fond tel que la passe de fond (et son flou) l'a laissé dans le RT, et la clé qui le décrit. +struct BgCache { + key: crate::frame_geometry::BackgroundKey, + tex: metal::Texture, +} + /// Un pipeline state à une seule pièce jointe couleur. fn make_pipeline( device: &metal::Device, @@ -661,6 +686,25 @@ impl Compositor { metal::MTLPixelFormat::RGBA8Unorm, Blend::Add, )?; + let library_models = device + .new_library_with_source(&msl_source_with_models(), &metal::CompileOptions::new()) + .map_err(|e| anyhow!("MTLDevice::new_library_with_source (modèles) a échoué : {e}"))?; + let pipeline_main_models = make_pipeline( + device, + &library_models, + "vs_main", + "ps_main", + metal::MTLPixelFormat::RGBA8Unorm, + Blend::Over, + )?; + let pipeline_add_models = make_pipeline( + device, + &library_models, + "vs_main", + "ps_main", + metal::MTLPixelFormat::RGBA8Unorm, + Blend::Add, + )?; let pipeline_kdown = make_pipeline( device, &library, "vs_fs", "ps_kawase_down", metal::MTLPixelFormat::RGBA8Unorm, Blend::Replace, @@ -682,6 +726,7 @@ impl Compositor { scene: RefCell::new(None), cursor: RefCell::new(None), cursor_time: RefCell::new(None), + footage: std::cell::Cell::new(None), timeline_time: RefCell::new(None), programme_time: RefCell::new(None), live_params: RefCell::new(LiveParams::default()), @@ -703,6 +748,10 @@ impl Compositor { pipeline_fs_uv, pipeline_fs_tex, pipeline_add, + pipeline_main_models, + pipeline_add_models, + layer_add: std::cell::Cell::new(false), + bg_cache: RefCell::new(None), accum, trail, blur_half, @@ -761,6 +810,7 @@ impl Compositor { blur_quarter, blur_eighth, ann_copy, + bg_cache: RefCell::new(None), ..self }) } @@ -909,7 +959,20 @@ impl Compositor { enc.set_fragment_texture(0, Some(y)); enc.set_fragment_texture(1, Some(uv)); } - enc.draw_primitives(metal::MTLPrimitiveType::TriangleStrip, 0, 4); + // Un modèle 3D passe, le temps de son draw, par la variante « modèles » du pipeline de la + // passe, la seule à le compiler ; le pipeline plat reprend ensuite la main. + if cb.needs_models() { + let (models, flat) = if self.layer_add.get() { + (&self.pipeline_add_models, &self.pipeline_add) + } else { + (&self.pipeline_main_models, &self.pipeline_main) + }; + enc.set_render_pipeline_state(models); + enc.draw_primitives(metal::MTLPrimitiveType::TriangleStrip, 0, 4); + enc.set_render_pipeline_state(flat); + } else { + enc.draw_primitives(metal::MTLPrimitiveType::TriangleStrip, 0, 4); + } } /// Quad de couleur pleine / gradient / ombre — tout ce qui n'échantillonne pas la vidéo. @@ -1417,6 +1480,37 @@ impl Compositor { } + /// Le fond gardé (`bg_cache`) : copie le RT dans `tex`, ou `tex` dans le RT (`into_rt`). + unsafe fn blit_background(&self, cmd: &metal::CommandBufferRef, tex: &metal::Texture, into_rt: bool) { + let (src, dst) = if into_rt { (tex, &self.rt) } else { (&self.rt, tex) }; + let blit = cmd.new_blit_command_encoder(); + blit.copy_from_texture( + src, 0, 0, + metal::MTLOrigin { x: 0, y: 0, z: 0 }, + metal::MTLSize { width: self.render_w as u64, height: self.render_h as u64, depth: 1 }, + dst, 0, 0, + metal::MTLOrigin { x: 0, y: 0, z: 0 }, + ); + blit.end_encoding(); + } + + /// Fige l'image composée dans `ann_copy` (mip 0), qu'y lisent les flous (mips dérivés, + /// `mips`) et le cristal de Prism Glow. + unsafe fn snapshot_rt(&self, cmd: &metal::CommandBufferRef, mips: bool) { + let blit = cmd.new_blit_command_encoder(); + blit.copy_from_texture( + &self.rt, 0, 0, + metal::MTLOrigin { x: 0, y: 0, z: 0 }, + metal::MTLSize { width: self.render_w as u64, height: self.render_h as u64, depth: 1 }, + &self.ann_copy, 0, 0, + metal::MTLOrigin { x: 0, y: 0, z: 0 }, + ); + if mips { + blit.generate_mipmaps(&self.ann_copy); + } + blit.end_encoding(); + } + /// Annotations : calque le plus haut, ancré sur `s_ann` — le rect écran SANS ZOOM, le /// conteneur que reçoit l'overlay web. Port de `compositor_windows::draw_annotations`. /// @@ -1425,6 +1519,9 @@ impl Compositor { /// grossir les sous-titres sous un zoom (issue #179, puis #397 sur Linux). L'arithmétique /// elle-même vit dans `frame_geometry::annotation_dst_in`, partagée par les trois backends. /// Le flou, lui, se place par `g.privacy_mask` : un masque doit rester sur ce qu'il cache. + /// + /// `privacy` : les flous de confidentialité seuls, dessinés sur le métrage avant le curseur ; + /// sinon toutes les autres annotations, le calque le plus haut. unsafe fn draw_annotations( &self, cmd: &metal::CommandBufferRef, @@ -1432,30 +1529,21 @@ impl Compositor { t: f32, s_ann: [f32; 4], g: &crate::frame_geometry::FrameGeometry, + privacy: bool, ) -> Result<()> { let Some(scene) = scene else { return Ok(()) }; - if scene.annotations.is_empty() { - return Ok(()); - } let (rw, rh) = (self.render_w as f32, self.render_h as f32); let visible = |a: &crate::scene::SceneAnnotation| { - t >= a.start_sec as f32 && t < a.end_sec as f32 + t >= a.start_sec as f32 && t < a.end_sec as f32 && (a.kind == "blur") == privacy }; + if !scene.annotations.iter().any(visible) { + return Ok(()); + } // UNE seule recopie pour toutes les annotations flou de la frame : leur lecture doit // voir l'image composée SANS les flous eux-mêmes, sinon deux zones qui se recouvrent // s'échantillonneraient l'une l'autre selon l'ordre de dessin. - if scene.annotations.iter().any(|a| a.kind == "blur" && visible(a)) { - let blit = cmd.new_blit_command_encoder(); - blit.copy_from_texture( - &self.rt, 0, 0, - metal::MTLOrigin { x: 0, y: 0, z: 0 }, - metal::MTLSize { width: rw as u64, height: rh as u64, depth: 1 }, - &self.ann_copy, 0, 0, - metal::MTLOrigin { x: 0, y: 0, z: 0 }, - ); - // Seul le mip 0 est rempli ; le GPU dérive le reste. - blit.generate_mipmaps(&self.ann_copy); - blit.end_encoding(); + if privacy { + self.snapshot_rt(cmd, true); } let enc = self.begin_pass(cmd, &self.rt, None, &self.pipeline_main)?; @@ -2058,6 +2146,8 @@ impl Compositor { ca.set_store_action(metal::MTLStoreAction::Store); let enc = cmd.new_render_command_encoder(&desc); enc.set_render_pipeline_state(pipeline); + // La famille du pipeline de calque de cette passe, pour `draw_layer`. + self.layer_add.set(std::ptr::eq(pipeline, &self.pipeline_add)); Ok(enc) } @@ -2066,7 +2156,8 @@ impl Compositor { /// `frame_geometry::cursor_sprite_cb`, partagée avec Windows et Linux. /// /// Avec `model`, le même sprite extrudé (mode 15, `cursor_model_cb`) : le sprite en - /// texture(2), son champ de distance en texture(4). + /// texture(2), son champ de distance en texture(4), l'enregistrement que l'appelant a lié en + /// texture(0) et (1). #[allow(clippy::too_many_arguments)] unsafe fn draw_cursor_sprite( &self, @@ -2077,26 +2168,38 @@ impl Compositor { sprite: &crate::scene::SceneCursorSprite, clip: [f32; 4], model: Option, - ) -> Result<()> { + ) -> Result> { let (tex, iw, ih) = self.cached_image(sprite.path.as_str())?; // Sans champ de distance, repli sur le sprite plat plutôt qu'aucun curseur. Parité Linux. if let Some(pose) = model { match self.cursor_sdf(sprite.path.as_str()) { Ok((sdf, shape)) => { let shape = crate::frame_geometry::model_shape(sprite, shape); - if let Some(cb) = crate::frame_geometry::cursor_model_cb( + let Some(cb) = crate::frame_geometry::cursor_model_cb( placement, size_px, pose, shape, a, clip, - ) { - enc.set_fragment_texture(2, Some(&tex)); - enc.set_fragment_texture(4, Some(&sdf)); - self.draw_solid(enc, &cb); - } - return Ok(()); + ) else { + return Ok(None); + }; + enc.set_fragment_texture(2, Some(&tex)); + enc.set_fragment_texture(4, Some(&sdf)); + self.draw_solid(enc, &cb); + return Ok(Some(cb.dst)); } Err(e) => eprintln!("[curseur] champ de \"{}\" : {e:#}", sprite.path), } } let (rw, rh) = (self.render_w as f32, self.render_h as f32); + // Le verre à plat de Glass Lens (mode 19) : la copie qu'il réfracte est déjà en + // texture(5) (`CursorPlan::glass`). Parité Windows. + if model.is_none() { + let glass = sprite.glass.as_deref().and_then(|name| { + crate::frame_geometry::cursor_glass_cb(placement, size_px, name, a, clip, [rw, rh]) + }); + if let Some(cb) = glass { + self.draw_solid(enc, &cb); + return Ok(Some(cb.dst)); + } + } let ar = iw as f32 / ih.max(1) as f32; let (pw, ph) = if ar >= 1.0 { (size_px, size_px / ar) } else { (size_px * ar, size_px) }; let cb = crate::frame_geometry::cursor_sprite_cb( @@ -2109,7 +2212,7 @@ impl Compositor { ); enc.set_fragment_texture(2, Some(&tex)); self.draw_solid(enc, &cb); - Ok(()) + Ok(Some(cb.dst)) } /// Curseur thématisé : le sprite de l'état courant, sinon la flèche, sinon rien. @@ -2132,14 +2235,17 @@ impl Compositor { a: f32, clip: [f32; 4], model: Option, - ) { - let sprite = cursor_type.and_then(|t| sprites.get(t)).or_else(|| sprites.get("arrow")); - if let Some(sprite) = sprite { - if let Err(e) = self.draw_cursor_sprite(enc, placement, size_px, a, sprite, clip, model) - { - eprintln!("[compositor] sprite curseur \"{}\" : {e:#}", sprite.path); - } - } + ) -> Option<[f32; 4]> { + let sprite = cursor_type.and_then(|t| sprites.get(t)).or_else(|| sprites.get("arrow"))?; + self.draw_cursor_sprite(enc, placement, size_px, a, sprite, clip, model).unwrap_or_else(|e| { + eprintln!("[compositor] sprite curseur \"{}\" : {e:#}", sprite.path); + None + }) + } + + /// Le métrage dans la dernière image composée : ses coins et son warp (`FootageQuad`). + pub fn footage_quad(&self) -> Option { + self.footage.get() } /// Compose la frame : fond, ombre écran, écran, ombre caméra, caméra — puis miroir @@ -2211,6 +2317,7 @@ impl Compositor { timeline_t_override: *self.timeline_time.borrow(), programme_time: *self.programme_time.borrow(), }); + self.footage.set(Some(g.footage_quad([rw, rh]))); let cmd_buf = self.gpu.context.new_command_buffer(); // Profondeur de champ : pyramide remplie seulement sur une frame inclinée qui la lit, @@ -2220,46 +2327,75 @@ impl Compositor { } else { None }; - let enc = self.begin_pass( - cmd_buf, - &self.rt, - Some(metal::MTLClearColor::new(0.0, 0.0, 0.0, 1.0)), - &self.pipeline_main, - )?; - // Les deux plans écran restent liés par défaut : les quads de couleur ne les - // échantillonnent pas, mais Metal veut des slots renseignés pour les draws qui, eux, - // le font. - enc.set_fragment_texture(0, Some(&sy)); - enc.set_fragment_texture(1, Some(&suv)); - - // --- fond --- (parité `compositor_windows.rs`, section « fond ») - match scene_ref.as_ref().map(|s| s.background.clone()) { - Some(SceneBackground::Color { color }) => { - let c = parse_hex(&color).unwrap_or(lp.bg_color); - self.draw_solid( - enc, - &LayerCB { dst: [0.0, 0.0, 1.0, 1.0], mode: 1.0, color: c, ..Default::default() }, - ); - } - Some(SceneBackground::Gradient { angle_deg, stops, offsets, motion }) => { - let a = angle_deg.to_radians(); - let (anim, mb) = - crate::frame_geometry::wallpaper_motion_slots(motion, g.programme_t, rw / rh); - self.draw_solid( - enc, - &LayerCB { - dst: [0.0, 0.0, 1.0, 1.0], - fx: [a.sin(), -a.cos(), anim[0], anim[1]], - mb, - ..crate::frame_geometry::gradient_layer(&stops, &offsets, lp.bg_color) - }, - ); + // Le fond de la frame d'avant, flou compris, s'il est celui qu'on demande + // (`BackgroundKey`) : recopié au lieu d'être redessiné. + let bg_desc = scene_ref.as_ref().map(|s| &s.background); + let bg_blur = scene_ref.as_ref().map_or(0.0, |s| s.effects.blur); + let bg_cached = self + .bg_cache + .borrow() + .as_ref() + .is_some_and(|c| c.key.matches(bg_desc, bg_blur, lp.bg_color, [rw, rh])); + if bg_cached { + if let Some(c) = self.bg_cache.borrow().as_ref() { + self.blit_background(cmd_buf, &c.tex, true); } - Some(SceneBackground::Image { path, motion }) => { - // Repli couleur en cas d'échec, mais LOGGÉ : un fallback silencieux masquerait - // un chemin cassé. - if let Err(e) = self.draw_image_bg(enc, &path, rw / rh, motion, g.programme_t) { - eprintln!("[compositor] wallpaper image \"{path}\" : {e:#}"); + } else { + let enc = self.begin_pass( + cmd_buf, + &self.rt, + Some(metal::MTLClearColor::new(0.0, 0.0, 0.0, 1.0)), + &self.pipeline_main, + )?; + // Les deux plans écran restent liés par défaut : les quads de couleur ne les + // échantillonnent pas, mais Metal veut des slots renseignés pour les draws qui, eux, + // le font. + enc.set_fragment_texture(0, Some(&sy)); + enc.set_fragment_texture(1, Some(&suv)); + + // --- fond --- (parité `compositor_windows.rs`, section « fond ») + // Faux si l'image du fond n'a pas pu se charger : son repli n'est pas gardé. + let mut bg_drawn = true; + match scene_ref.as_ref().map(|s| s.background.clone()) { + Some(SceneBackground::Color { color }) => { + let c = parse_hex(&color).unwrap_or(lp.bg_color); + self.draw_solid( + enc, + &LayerCB { dst: [0.0, 0.0, 1.0, 1.0], mode: 1.0, color: c, ..Default::default() }, + ); + } + Some(SceneBackground::Gradient { angle_deg, stops, offsets, motion }) => { + let a = angle_deg.to_radians(); + let (anim, mb) = + crate::frame_geometry::wallpaper_motion_slots(motion, g.programme_t, rw / rh); + self.draw_solid( + enc, + &LayerCB { + dst: [0.0, 0.0, 1.0, 1.0], + fx: [a.sin(), -a.cos(), anim[0], anim[1]], + mb, + ..crate::frame_geometry::gradient_layer(&stops, &offsets, lp.bg_color) + }, + ); + } + Some(SceneBackground::Image { path, motion }) => { + // Repli couleur en cas d'échec, mais LOGGÉ : un fallback silencieux masquerait + // un chemin cassé. + if let Err(e) = self.draw_image_bg(enc, &path, rw / rh, motion, g.programme_t) { + eprintln!("[compositor] wallpaper image \"{path}\" : {e:#}"); + bg_drawn = false; + self.draw_solid( + enc, + &LayerCB { + dst: [0.0, 0.0, 1.0, 1.0], + mode: 1.0, + color: lp.bg_color, + ..Default::default() + }, + ); + } + } + None => { self.draw_solid( enc, &LayerCB { @@ -2271,25 +2407,34 @@ impl Compositor { ); } } - None => { - self.draw_solid( - enc, - &LayerCB { - dst: [0.0, 0.0, 1.0, 1.0], - mode: 1.0, - color: lp.bg_color, - ..Default::default() - }, - ); - } - } - // « Blur BG » (parité web `blurredBackgroundLayer`) : floute CE wallpaper qu'on vient - // de dessiner, pas la vidéo. No-op visuel sur une couleur plate, effet réel sur un - // gradient ou une image. Il lui faut ses propres passes, d'où la coupure ici. - enc.end_encoding(); - if let Some(blur) = scene_ref.as_ref().map(|s| s.effects.blur) { - self.blur_bg(cmd_buf, blur)?; + // « Blur BG » (parité web `blurredBackgroundLayer`) : floute CE wallpaper qu'on vient + // de dessiner, pas la vidéo. No-op visuel sur une couleur plate, effet réel sur un + // gradient ou une image. Il lui faut ses propres passes, d'où la coupure ici. + enc.end_encoding(); + if let Some(blur) = scene_ref.as_ref().map(|s| s.effects.blur) { + self.blur_bg(cmd_buf, blur)?; + } + // Pas de cache pour un fond animé (`of` rend `None`). + let key = bg_drawn + .then(|| crate::frame_geometry::BackgroundKey::of(bg_desc, bg_blur, lp.bg_color, [rw, rh])) + .flatten(); + if let Some(key) = key { + // L'ancienne texture est reprise : `resized` vide le cache, sa taille est la bonne. + let old = self.bg_cache.borrow_mut().take().map(|c| c.tex); + let tex = old.unwrap_or_else(|| { + make_texture( + &self.gpu.device, + metal::MTLPixelFormat::RGBA8Unorm, + self.render_w, + self.render_h, + metal::MTLStorageMode::Private, + metal::MTLTextureUsage::ShaderRead, + ) + }); + self.blit_background(cmd_buf, &tex, false); + *self.bg_cache.borrow_mut() = Some(BgCache { key, tex }); + } } // --- écran : ombre puis vidéo --- let s_px = [g.s_dst[2] * rw, g.s_dst[3] * rh]; @@ -2344,7 +2489,8 @@ impl Compositor { } let square_top = g.screen_square_top(); let top_lift = g.screen_top_lift_px([rw, rh]); - let [su0, sv0, su1, sv1] = g.cut; + // The screen texture's source rect: a turned clip samples it with reversed bounds. + let [su0, sv0, su1, sv1] = g.screen_src(g.cut); // Sous le masque d'un layout en bloc, rogné au slot (`FrameGeometry::mask_flat_screen`). let (dst, src, quad_px, radius_px) = g.mask_flat_screen(g.s_dst, [su0, sv0, su1, sv1], s_px, g.s_radius, [rw, rh]); @@ -2371,7 +2517,7 @@ impl Compositor { quad, s_px, quad_center_px, - g.cut, + g.screen_src(g.cut), g.focus_plane, g.s_radius, top_lift, @@ -2400,6 +2546,11 @@ impl Compositor { enc.end_encoding(); + // --- flous de confidentialité : sur le métrage, AVANT le curseur (parité Windows). Le + // curseur reste net par-dessus, et le cristal de Prism Glow, qui réfracte l'image + // composée, n'y voit que des pixels déjà floutés. + self.draw_annotations(cmd_buf, scene_ref.as_ref(), g.source_t, g.s_ann, &g, true)?; + // --- curseur --- (parité `compositor_windows.rs`, section « curseur custom ») if let Some(track) = cursor_ref.as_ref() { let plan = crate::frame_geometry::plan_cursor( @@ -2422,6 +2573,12 @@ impl Compositor { .map(|s| s.cursor.cursor_sprites.clone()) .unwrap_or_default(); let kind = plan.cursor_type.as_deref(); + // Le cristal de Prism Glow réfracte l'image telle qu'elle est composée à cet + // instant, flous compris : sa copie en texture(5), une fois pour toutes les copies + // de la traînée (`ann_copy` est libre, les flous l'ont déjà lue). + if plan.glass { + self.snapshot_rt(cmd_buf, false); + } // L'impact des clics (mode 16, sans texture), posé sur l'écran SOUS le curseur. if !plan.impacts.is_empty() { let e = self.begin_pass(cmd_buf, &self.rt, None, &self.pipeline_main)?; @@ -2432,6 +2589,9 @@ impl Compositor { } if plan.taps <= 1 { let e = self.begin_pass(cmd_buf, &self.rt, None, &self.pipeline_main)?; + if plan.glass { + e.set_fragment_texture(5, Some(&self.ann_copy)); + } self.draw_cur_themed( e, &sprites, @@ -2454,11 +2614,17 @@ impl Compositor { Some(metal::MTLClearColor::new(0.0, 0.0, 0.0, 0.0)), &self.pipeline_add, )?; + if plan.glass { + e.set_fragment_texture(5, Some(&self.ann_copy)); + } + // L'union des quads peints dans `accum` (x0, y0, x1, y1 en fractions de la + // sortie) : la seule région que la recopie doit relire. + let mut bounds = [f32::MAX, f32::MAX, f32::MIN, f32::MIN]; for k in 0..plan.taps { let f = k as f32 / (plan.taps - 1) as f32; let w = crate::frame_geometry::cursor_tap_weight(k, plan.taps); e.set_blend_color(w, w, w, w); - self.draw_cur_themed( + let drawn = self.draw_cur_themed( e, &sprites, kind, @@ -2468,12 +2634,36 @@ impl Compositor { plan.clip, plan.model, ); + if let Some(d) = drawn { + bounds = [ + bounds[0].min(d[0]), + bounds[1].min(d[1]), + bounds[2].max(d[0] + d[2]), + bounds[3].max(d[1] + d[3]), + ]; + } } e.end_encoding(); let c = self.begin_pass(cmd_buf, &self.rt, None, &self.pipeline_fs_tex)?; c.set_fragment_texture(0, Some(&self.accum)); - c.draw_primitives(metal::MTLPrimitiveType::Triangle, 0, 3); + // Rien n'a été peint dans `accum` hors des quads de la traînée : la recopie + // s'y borne au lieu de repasser sur toute la sortie (0,5 ms par frame 1080p + // mesurée sur Linux, Radeon 610M). + let (w, h) = (self.render_w as f32, self.render_h as f32); + let x0 = (bounds[0] * w).floor().clamp(0.0, w) as u64; + let y0 = (bounds[1] * h).floor().clamp(0.0, h) as u64; + let x1 = (bounds[2] * w).ceil().clamp(0.0, w) as u64; + let y1 = (bounds[3] * h).ceil().clamp(0.0, h) as u64; + if x1 > x0 && y1 > y0 { + c.set_scissor_rect(metal::MTLScissorRect { + x: x0, + y: y0, + width: x1 - x0, + height: y1 - y0, + }); + c.draw_primitives(metal::MTLPrimitiveType::Triangle, 0, 3); + } c.end_encoding(); } } @@ -2578,8 +2768,9 @@ impl Compositor { // --- annotations : calque le plus haut, ancré sur le rect ÉCRAN SANS ZOOM --- // `s_ann`, pas `s_dst` : le zoom vit dans la boîte depuis l'issue #179, donc `s_dst` // grandit avec lui et emmenait annotations et sous-titres dans le mouvement. Le flou de - // confidentialité est l'exception : il suit le contenu, d'où `&g`. - self.draw_annotations(cmd_buf, scene_ref.as_ref(), g.source_t, g.s_ann, &g)?; + // confidentialité est l'exception : il suit le contenu, d'où `&g` ; il est passé avant le + // curseur (plus haut). + self.draw_annotations(cmd_buf, scene_ref.as_ref(), g.source_t, g.s_ann, &g, false)?; // Ni miroir RGBA ni attente ici : le miroir ne sert qu'à `readback_direct` (la // preview), et l'export ne lit jamais le RGBA — le blit pleine résolution était payé @@ -4267,6 +4458,12 @@ mod tests { &metal::CompileOptions::new(), ) .expect("shaders.metal doit compiler"); + // La variante des modèles 3D (`LAYER_MODELS`) : la seule à compiler leurs branches. + device + .new_library_with_source(&super::msl_source_with_models(), &metal::CompileOptions::new()) + .expect("shaders.metal doit compiler avec LAYER_MODELS") + .get_function("ps_main", None) + .expect("ps_main absent de la variante LAYER_MODELS"); for name in [ "vs_main", "vs_fs", diff --git a/crates/compositor/src/compositor_windows.rs b/crates/compositor/src/compositor_windows.rs index 59431e383d..17fb180554 100644 --- a/crates/compositor/src/compositor_windows.rs +++ b/crates/compositor/src/compositor_windows.rs @@ -24,8 +24,9 @@ use std::cell::{Cell, RefCell}; use std::collections::HashMap; use std::ffi::c_void; use windows::core::Interface; +use windows::Win32::Foundation::RECT; use windows::Win32::Graphics::Direct3D::{ - D3D11_SRV_DIMENSION_TEXTURE2DARRAY, D3D_PRIMITIVE_TOPOLOGY_TRIANGLESTRIP, + D3D11_SRV_DIMENSION_TEXTURE2D, D3D_PRIMITIVE_TOPOLOGY_TRIANGLESTRIP, }; use windows::Win32::Graphics::Direct3D11::*; use windows::Win32::Graphics::Dxgi::Common::*; @@ -78,6 +79,12 @@ struct WebcamMask { /// demi-résolution de la texture DÉCODEUR, avec sa chaîne de mips. Dimensionnée sur la texture /// décodeur et remplie en UV plein (0..1) : ses UV sont ceux de t0/t1, et le calcul d'UV du /// mode 8 ne change pas. +/// Le fond tel que la passe de fond (et son flou) l'a laissé dans le RT, et la clé qui le décrit. +struct BgCache { + key: crate::frame_geometry::BackgroundKey, + tex: ID3D11Texture2D, +} + struct DofPyramid { rtv: ID3D11RenderTargetView, srv: ID3D11ShaderResourceView, @@ -93,7 +100,10 @@ pub struct Compositor { rt_srv: ID3D11ShaderResourceView, staging: ID3D11Texture2D, vs: ID3D11VertexShader, + /// Le pixel shader des calques, SANS les modèles 3D (modes 15 à 17) : ils passent par + /// `ps_models`, et `draw_layer` choisit d'après le mode (le haut de `shaders.hlsl`). ps: ID3D11PixelShader, + ps_models: ID3D11PixelShader, vs_fs: ID3D11VertexShader, ps_y: ID3D11PixelShader, ps_uv: ID3D11PixelShader, @@ -129,11 +139,16 @@ pub struct Compositor { accum_rtv: ID3D11RenderTargetView, accum_srv: ID3D11ShaderResourceView, blend_add: ID3D11BlendState, + /// Rastérisation avec scissor, pour la seule recopie de la traînée du curseur. + rs_scissor: ID3D11RasterizerState, /// Rendu isolé de l'écran cadré (ombre, cadre, métrage, appareil), transparent autour, que le /// mode 18 recompose le long de sa trajectoire (`FrameGeometry::screen_trail`). Distinct de /// `accum`, que le curseur et `compose_frame_mb` remplissent dans la même frame. trail_rtv: ID3D11RenderTargetView, trail_srv: ID3D11ShaderResourceView, + /// Le dernier fond fixe composé et sa clé (`BackgroundKey`) : recopié dans le RT tant + /// que la clé tient, au lieu d'être redessiné. + bg_cache: RefCell>, /// RefCell (pas un simple champ) pour que `set_cursor` reste `&self`, comme `set_scene` / /// `set_live_params` — nécessaire pour le rebrancher par clip dans l'export multiclip, qui /// n'a qu'une référence partagée au `Compositor`. @@ -142,6 +157,8 @@ pub struct Compositor { /// (`frame / FPS`). L'export multiclip et le live le positionnent au PTS écran courant, /// c'est-à-dire au temps source absolu du clip actif. cursor_t_override: RefCell>, + /// Le métrage dans la dernière image composée, que l'éditeur lit avec elle (`live.rs`). + footage: std::cell::Cell>, /// Override du temps des zoom/full-camera regions (secondes source du clip actif). Le nom /// `timeline_t_override` est conservé pour l'API existante, mais ce temps n'est plus cumulé /// entre clips : les régions projetées par l'app portent elles aussi des temps source. @@ -149,9 +166,11 @@ pub struct Compositor { timeline_t_override: RefCell>, /// Temps programme (secondes de sortie) — cf. `FrameGeometryInput::programme_time`. programme_time: RefCell>, - // cache des SRV décodeur par (texture array, slice) : le pool réutilise ~32 textures, - // donc après warmup plus aucune création de SRV par frame (overhead CPU supprimé). - srv_cache: RefCell>, + /// A private copy of a decoder surface, with its two plane views (Y, UV). + /// Keyed by the decoder texture's pointer, which pins nothing: `nv12_srvs` re-checks + /// the copy's size and format on every hit, so a new texture landing on an old address + /// is never handed a copy sized for the old one. `clear_srv_cache` frees stale entries. + srv_cache: RefCell>, live_params: RefCell, /// Scène pilotée par l'app (contrat) : quand présente, remplace le layout fixture de /// `timeline()`. Voir `scene.rs` / `SceneDescription` (TS). @@ -404,6 +423,7 @@ impl Compositor { render_size: Cell::new((w, h)), // Caches dimensionnés à l'ancienne taille : recréés à la demande. resize_target: RefCell::new(None), + bg_cache: RefCell::new(None), live_readback_staging: RefCell::new(None), nv12_readback_staging: RefCell::new(None), ..self @@ -600,6 +620,8 @@ impl Compositor { dev.CreateVertexShader(shader!("vs_main"), None, Some(&mut vs))?; let mut ps: Option = None; dev.CreatePixelShader(shader!("ps_main"), None, Some(&mut ps))?; + let mut ps_models: Option = None; + dev.CreatePixelShader(shader!("ps_main_models"), None, Some(&mut ps_models))?; // shaders RGB->NV12 let mut vs_fs: Option = None; @@ -680,6 +702,17 @@ impl Compositor { }; let mut blend_add: Option = None; dev.CreateBlendState(&bla, Some(&mut blend_add))?; + // L'état par défaut (aucun lié), plus le scissor. Sans culling : le triangle plein écran + // n'a pas à dépendre de son sens de parcours. + let rsd = D3D11_RASTERIZER_DESC { + FillMode: D3D11_FILL_SOLID, + CullMode: D3D11_CULL_NONE, + ScissorEnable: true.into(), + DepthClipEnable: true.into(), + ..Default::default() + }; + let mut rs_scissor: Option = None; + dev.CreateRasterizerState(&rsd, Some(&mut rs_scissor))?; Ok(Compositor { dev, @@ -690,6 +723,7 @@ impl Compositor { staging, vs: vs.unwrap(), ps: ps.unwrap(), + ps_models: ps_models.unwrap(), vs_fs: vs_fs.unwrap(), ps_y: ps_y.unwrap(), ps_uv: ps_uv.unwrap(), @@ -718,10 +752,13 @@ impl Compositor { accum_rtv, accum_srv, blend_add: blend_add.unwrap(), + rs_scissor: rs_scissor.unwrap(), trail_rtv, trail_srv, + bg_cache: RefCell::new(None), cursor: RefCell::new(None), cursor_t_override: RefCell::new(None), + footage: std::cell::Cell::new(None), timeline_t_override: RefCell::new(None), programme_time: RefCell::new(None), srv_cache: RefCell::new(HashMap::new()), @@ -805,41 +842,101 @@ impl Compositor { *self.scene.borrow_mut() = s; } - /// Crée les SRV Y (R8) et UV (R8G8) sur la tranche d'array de la frame décodeur. + /// The Y (R8) and UV (R8G8) views of the decoder frame — over a private COPY, never + /// over the decoder surface itself. + /// + /// Two documented D3D11 rules rule out the direct path. The first is explicit, on + /// `D3D11_BIND_DECODER`: "you cannot use texture arrays that are created with this + /// flag in calls to ID3D11Device::CreateShaderResourceView". The ffmpeg pool IS such + /// an array (`initial_pool_size` = 32, see `get_hw_format`). The second explains what + /// we were seeing: between the video engine and the 3D pipeline "there is no automatic + /// hazard tracking" — the surface stays the decoder's reference frame, so it may be + /// rewritten WHILE the shader samples it. ffmpeg's `ID3D11VideoContext` is in fact the + /// SAME object as our immediate context (it comes out of a QueryInterface on it), and + /// `SetMultithreadProtected(TRUE)` only makes an individual call atomic, never a + /// sequence. + /// + /// Measured on a Snapdragon X Elite (Adreno X1-85), same build, same recording, the + /// path picked by an environment variable: 768 black frames out of 792 when sampling + /// the decoder surface, 2 out of 545 through the copy. The black was OPAQUE and total, + /// so not "one frame late" but no frame at all. + /// + /// `CopySubresourceRegion` is issued on the immediate context, so it IS ordered against + /// the draws that follow; the destination texture is `ArraySize = 1` and carries only + /// `BIND_SHADER_RESOURCE`, which also takes it out of the restriction above. The cost + /// is one GPU→GPU copy per frame and per source, with nothing going back to system + /// memory. + /// + /// What hid the cause for so long: exporting the SAME scene never produced a black + /// frame (28 342 verified), because the export drains the GPU every frame through the + /// encoder. Everything that slowed the live loop — one more readback, a lock, a pause — + /// cut the black proportionally without ever removing it. pub unsafe fn nv12_srvs( &self, frame: *const AVFrame, ) -> Result<(ID3D11ShaderResourceView, ID3D11ShaderResourceView)> { let tex_ptr = (*frame).data[0] as *mut c_void; let slice = (*frame).data[1] as u32; - // cache hit : le pool réutilise les mêmes textures -> zéro création après warmup - let key = (tex_ptr as usize, slice); - if let Some((y, uv)) = self.srv_cache.borrow().get(&key) { - return Ok((y.clone(), uv.clone())); - } - let tex = ID3D11Texture2D::from_raw_borrowed(&tex_ptr) - .ok_or_else(|| anyhow::anyhow!("frame sans texture D3D11"))? + let src = ID3D11Texture2D::from_raw_borrowed(&tex_ptr) + .ok_or_else(|| anyhow::anyhow!("frame has no D3D11 texture"))? .clone(); - let mk = |fmt: DXGI_FORMAT| -> Result { - let mut d = D3D11_SHADER_RESOURCE_VIEW_DESC { - Format: fmt, - ViewDimension: D3D11_SRV_DIMENSION_TEXTURE2DARRAY, - ..Default::default() - }; - d.Anonymous.Texture2DArray = D3D11_TEX2D_ARRAY_SRV { - MostDetailedMip: 0, - MipLevels: 1, - FirstArraySlice: slice, - ArraySize: 1, - }; - let mut srv: Option = None; - self.dev.CreateShaderResourceView(&tex, Some(&d), Some(&mut srv))?; - Ok(srv.unwrap()) + // Cache hit: the pool reuses the same textures, so this allocates once per decoder + // after warmup. The COPY itself happens on EVERY frame — that is what freezes the + // slice's contents before the decoder takes it back. + // + // The key is a bare address and pins nothing. A decoder replaced without + // `clear_srv_cache` (a mid-stream resolution change, `CpuFrames::ensure_tex`) can + // hand a NEW texture the address of an old one, and a copy into a smaller destination + // is dropped without an error: the picture freezes. So a hit only counts while the + // destination still has the source's size and format. + let mut sd = D3D11_TEXTURE2D_DESC::default(); + src.GetDesc(&mut sd); + let key = tex_ptr as usize; + let cached = self.srv_cache.borrow().get(&key).cloned().filter(|(dst, ..)| { + let mut dd = D3D11_TEXTURE2D_DESC::default(); + dst.GetDesc(&mut dd); + (dd.Width, dd.Height, dd.Format) == (sd.Width, sd.Height, sd.Format) + }); + let (dst, y, uv) = match cached { + Some(v) => v, + None => { + let dd = D3D11_TEXTURE2D_DESC { + Width: sd.Width, + Height: sd.Height, + MipLevels: 1, + ArraySize: 1, + Format: sd.Format, + SampleDesc: DXGI_SAMPLE_DESC { Count: 1, Quality: 0 }, + Usage: D3D11_USAGE_DEFAULT, + BindFlags: D3D11_BIND_SHADER_RESOURCE.0 as u32, + CPUAccessFlags: 0, + MiscFlags: 0, + }; + let mut dst: Option = None; + self.dev.CreateTexture2D(&dd, None, Some(&mut dst))?; + let dst = dst.unwrap(); + let mk = |fmt: DXGI_FORMAT| -> Result { + let mut d = D3D11_SHADER_RESOURCE_VIEW_DESC { + Format: fmt, + ViewDimension: D3D11_SRV_DIMENSION_TEXTURE2D, + ..Default::default() + }; + d.Anonymous.Texture2D = D3D11_TEX2D_SRV { MostDetailedMip: 0, MipLevels: 1 }; + let mut srv: Option = None; + self.dev.CreateShaderResourceView(&dst, Some(&d), Some(&mut srv))?; + Ok(srv.unwrap()) + }; + let y = mk(DXGI_FORMAT_R8_UNORM)?; + let uv = mk(DXGI_FORMAT_R8G8_UNORM)?; + self.srv_cache + .borrow_mut() + .insert(key, (dst.clone(), y.clone(), uv.clone())); + (dst, y, uv) + } }; - let y = mk(DXGI_FORMAT_R8_UNORM)?; - let uv = mk(DXGI_FORMAT_R8G8_UNORM)?; - self.srv_cache.borrow_mut().insert(key, (y.clone(), uv.clone())); + + self.ctx.CopySubresourceRegion(&dst, 0, 0, 0, 0, &src, slice, None); Ok((y, uv)) } @@ -874,6 +971,26 @@ impl Compositor { self.ctx.ClearRenderTargetView(&self.rtv, &clear); } + /// Garde le fond que le RT porte à cet instant (`bg_cache`), sous `key`. + unsafe fn keep_background(&self, key: crate::frame_geometry::BackgroundKey) -> Result<()> { + // L'ancienne texture est reprise : `resized` vide le cache, sa taille est la bonne. + let old = self.bg_cache.borrow_mut().take().map(|c| c.tex); + let tex = match old { + Some(tex) => tex, + None => { + let mut d = D3D11_TEXTURE2D_DESC::default(); + self.rt.GetDesc(&mut d); + d.BindFlags = 0; + let mut tex: Option = None; + self.dev.CreateTexture2D(&d, None, Some(&mut tex))?; + tex.ok_or_else(|| anyhow::anyhow!("CreateTexture2D (fond gardé)"))? + } + }; + self.ctx.CopyResource(&tex, &self.rt); + *self.bg_cache.borrow_mut() = Some(BgCache { key, tex }); + Ok(()) + } + /// Passe plein écran générique (triangle unique) : `srv` -> `rtv` via `ps`, avec `fx`. unsafe fn fs_pass( &self, @@ -1025,6 +1142,15 @@ impl Compositor { self.ctx.PSSetConstantBuffers(0, Some(&[Some(self.cbuf.clone())])); } + /// Dessine le quad d'un calque, ses ressources déjà liées. Le pixel shader suit son mode : + /// `ps_models` pour un modèle 3D (`LayerCB::needs_models`), `ps` pour tout le reste, qui ne + /// paie pas ainsi les registres de ces modèles. + unsafe fn draw_layer(&self, cb: &LayerCB) { + self.upload_cb(cb); + self.ctx.PSSetShader(if cb.needs_models() { &self.ps_models } else { &self.ps }, None); + self.ctx.Draw(4, 0); + } + /// Calque vidéo NV12. pub unsafe fn draw_video( &self, @@ -1032,16 +1158,14 @@ impl Compositor { srv_y: &ID3D11ShaderResourceView, srv_uv: &ID3D11ShaderResourceView, ) { - self.upload_cb(cb); self.ctx .PSSetShaderResources(0, Some(&[Some(srv_y.clone()), Some(srv_uv.clone())])); - self.ctx.Draw(4, 0); + self.draw_layer(cb); } /// Calque couleur pleine (fond). pub unsafe fn draw_solid(&self, cb: &LayerCB) { - self.upload_cb(cb); - self.ctx.Draw(4, 0); + self.draw_layer(cb); } /// Fond wallpaper image (cover-fit). `path` = chemin absolu (résolu côté app). Décodé et @@ -1135,7 +1259,8 @@ impl Compositor { (0.0, (1.0 - vis) * 0.5, 1.0, 1.0 - (1.0 - vis) * 0.5) }; let (anim, mb) = crate::frame_geometry::wallpaper_motion_slots(motion, programme_t, ao); - self.upload_cb(&LayerCB { + self.ctx.PSSetShaderResources(2, Some(&[Some(srv)])); + self.draw_layer(&LayerCB { dst, src: [u0, v0, u1, v1], quad_px, @@ -1145,8 +1270,6 @@ impl Compositor { mb, ..Default::default() }); - self.ctx.PSSetShaderResources(2, Some(&[Some(srv)])); - self.ctx.Draw(4, 0); Ok(()) } @@ -1661,7 +1784,7 @@ impl Compositor { /// Curseur custom (dot+ring) centré en `center` (0..1 sortie), taille `size_px`, opacité `a`. /// `clip` = rect "Clip to canvas" en espace sortie [x,y,w,h] ; passer un rect englobant tout /// (ex. [-1,-1,3,3]) pour désactiver l'effet. - unsafe fn draw_cursor(&self, center: [f32; 2], size_px: f32, a: f32, clip: [f32; 4]) { + unsafe fn draw_cursor(&self, center: [f32; 2], size_px: f32, a: f32, clip: [f32; 4]) -> [f32; 4] { let w = size_px / self.rw(); let h = size_px / self.rh(); let dst = [center[0] - w * 0.5, center[1] - h * 0.5, w, h]; @@ -1673,6 +1796,7 @@ impl Compositor { fx: clip, ..Default::default() }); + dst } /// Curseur thème (sprite PNG, ex. arrow.png) dont le PIVOT `hotspot` (fraction 0..1 de @@ -1690,7 +1814,7 @@ impl Compositor { sprite: &SceneCursorSprite, clip: [f32; 4], model: Option, - ) -> Result<()> { + ) -> Result> { let path = sprite.path.as_str(); let (srv, iw, ih) = self.cached_image(path)?; // Sans champ de distance, repli sur le sprite plat plutôt que sur le curseur math. @@ -1699,20 +1823,31 @@ impl Compositor { match self.cursor_sdf(path) { Ok((sdf, shape)) => { let shape = crate::frame_geometry::model_shape(sprite, shape); - if let Some(cb) = crate::frame_geometry::cursor_model_cb( + let Some(cb) = crate::frame_geometry::cursor_model_cb( placement, size_px, pose, shape, a, clip, - ) { - self.upload_cb(&cb); - self.ctx.PSSetShaderResources(2, Some(&[Some(srv)])); - self.ctx.PSSetShaderResources(4, Some(&[Some(sdf)])); - self.ctx.Draw(4, 0); - self.ctx.PSSetShaderResources(4, Some(&[None])); - } - return Ok(()); + ) else { + return Ok(None); + }; + self.ctx.PSSetShaderResources(2, Some(&[Some(srv)])); + self.ctx.PSSetShaderResources(4, Some(&[Some(sdf)])); + self.draw_layer(&cb); + self.ctx.PSSetShaderResources(4, Some(&[None])); + return Ok(Some(cb.dst)); } Err(e) => eprintln!("[curseur] champ de \"{path}\" : {e:#}"), } } + // Le verre à plat de Glass Lens (mode 19) : la copie qu'il réfracte est déjà en t5 + // (`CursorPlan::glass`). + if model.is_none() { + let glass = sprite.glass.as_deref().and_then(|name| { + crate::frame_geometry::cursor_glass_cb(placement, size_px, name, a, clip, [self.rw(), self.rh()]) + }); + if let Some(cb) = glass { + self.draw_layer(&cb); + return Ok(Some(cb.dst)); + } + } let ar = iw as f32 / ih as f32; let (pw, ph) = if ar >= 1.0 { (size_px, size_px / ar) } else { (size_px * ar, size_px) }; let cb = crate::frame_geometry::cursor_sprite_cb( @@ -1723,10 +1858,9 @@ impl Compositor { clip, [self.rw(), self.rh()], ); - self.upload_cb(&cb); self.ctx.PSSetShaderResources(2, Some(&[Some(srv)])); - self.ctx.Draw(4, 0); - Ok(()) + self.draw_layer(&cb); + Ok(Some(cb.dst)) } /// Sprite de l'état courant (`cursor_type`, ex. `"text"`), à défaut celui de la flèche, @@ -1746,17 +1880,17 @@ impl Compositor { a: f32, clip: [f32; 4], model: Option, - ) { + ) -> Option<[f32; 4]> { let sprite = cursor_type.and_then(|t| sprites.get(t)).or_else(|| sprites.get("arrow")); if let Some(sprite) = sprite { - if self.draw_cursor_sprite(placement, size_px, a, sprite, clip, model).is_ok() { - return; + if let Ok(dst) = self.draw_cursor_sprite(placement, size_px, a, sprite, clip, model) { + return dst; } } // Le repli math reste droit même sur un plan incliné : il ne devrait plus apparaître // maintenant que l'art par défaut existe, et lui donner sa propre passe de warp pour // un cas de secours ne se justifie pas. - self.draw_cursor(placement.upright_center(), size_px, a, clip); + Some(self.draw_cursor(placement.upright_center(), size_px, a, clip)) } /// Ombre portée (§7 E4) sous un quad `dst` (normalisé) de taille `size_px`. @@ -1850,6 +1984,11 @@ impl Compositor { }); } + /// Le métrage dans la dernière image composée : ses coins et son warp (`FootageQuad`). + pub fn footage_quad(&self) -> Option { + self.footage.get() + } + /// Compose une frame animée (§6/§8) : fond flouté + screen zoomé (padding, coins, ombre) /// + webcam crop carré (coins, ombre), placements interpolés A↔B par la timeline. pub unsafe fn compose_frame( @@ -1893,12 +2032,14 @@ impl Compositor { timeline_t_override: *self.timeline_t_override.borrow(), programme_time: *self.programme_time.borrow(), }); + self.footage.set(Some(g.footage_quad([self.rw(), self.rh()]))); let scene_preset = g.scene_preset.clone(); let mb_taps = g.mb_taps; let mb_amount = g.mb_amount; let source_t = g.source_t; let _padding_scale = g.padding_scale; - let cut = g.cut; + // The screen texture's source rect: a turned clip samples it with reversed bounds. + let cut = g.screen_src(g.cut); let s_dst = g.s_dst; let s_dst_prev = g.s_dst_prev; let s_ann = g.s_ann; @@ -1915,8 +2056,6 @@ impl Compositor { let dof = g.depth_of_field_on(self.cpu_backend); let dof_srv = if dof { Some(self.fill_dof_pyramid(&sy, &suv, stw, sth)?) } else { None }; - self.begin([0.0, 0.0, 0.0, 1.0]); - // --- fond --- // Parité web (frameRenderer.blurredBackgroundLayer) : le fond est le WALLPAPER sélectionné // (image/couleur/gradient) et « Blur BG » floute CE wallpaper, PAS la vidéo. Le natif @@ -1925,86 +2064,120 @@ impl Compositor { // ensuite ; pour une couleur plate le flou est un no-op visuel). Côté fixture/bench // (pas de scène) on garde le fond screen-flouté, dont le coût est mesuré (C4). let scene_bg = self.scene.borrow().as_ref().map(|s| (s.background.clone(), s.effects.blur)); - if let Some((bg, blur_wallpaper)) = scene_bg { - match bg { - SceneBackground::Color { color } => { - let c = parse_hex(&color).unwrap_or(lp.bg_color); - self.draw_solid(&LayerCB { - dst: [0.0, 0.0, 1.0, 1.0], - mode: 1.0, - color: c, - ..Default::default() - }); - } - SceneBackground::Gradient { angle_deg, stops, offsets, motion } => { - // angle CSS → direction unitaire (espace sortie, y vers le bas) : - // 0° = vers le haut, 90° = vers la droite. - let a = angle_deg.to_radians(); - let dir = [a.sin(), -a.cos()]; - let (anim, mb) = crate::frame_geometry::wallpaper_motion_slots( - motion, - g.programme_t, - self.rw() / self.rh(), - ); - self.draw_solid(&LayerCB { - dst: [0.0, 0.0, 1.0, 1.0], - fx: [dir[0], dir[1], anim[0], anim[1]], - mb, - ..crate::frame_geometry::gradient_layer(&stops, &offsets, lp.bg_color) - }); - } - SceneBackground::Image { path, motion } => { - // image bg (cover-fit, mise en cache) ; fallback couleur si chargement échoue - // (loggé — un fallback silencieux masquerait un chemin cassé, cf. le panic - // borrow qu'on a déjà eu : toute panne doit être visible/traçable). - let aspect = self.rw() / self.rh(); - if let Err(e) = self.draw_image_bg(&path, aspect, motion, g.programme_t) { - eprintln!("[compositor] wallpaper image \"{}\" : {:#}", path, e); + // Le fond de la frame d'avant, s'il est celui qu'on demande (`BackgroundKey`), est recopié + // au lieu d'être redessiné. Côté scène seulement : sans scène, le fond est la vidéo floutée. + let bg_size = [self.rw(), self.rh()]; + let bg_cached = scene_ref.as_ref().is_some_and(|s| { + self.bg_cache + .borrow() + .as_ref() + .is_some_and(|c| { + c.key.matches(Some(&s.background), s.effects.blur, lp.bg_color, bg_size) + }) + }); + if bg_cached { + self.bind_compose_state(); + if let Some(c) = self.bg_cache.borrow().as_ref() { + self.ctx.CopyResource(&self.rt, &c.tex); + } + } else { + self.begin([0.0, 0.0, 0.0, 1.0]); + // Faux si l'image du fond n'a pas pu se charger : son repli n'est pas gardé. + let mut bg_drawn = true; + if let Some((bg, blur_wallpaper)) = scene_bg { + match bg { + SceneBackground::Color { color } => { + let c = parse_hex(&color).unwrap_or(lp.bg_color); self.draw_solid(&LayerCB { dst: [0.0, 0.0, 1.0, 1.0], mode: 1.0, - color: lp.bg_color, + color: c, ..Default::default() }); } + SceneBackground::Gradient { angle_deg, stops, offsets, motion } => { + // angle CSS → direction unitaire (espace sortie, y vers le bas) : + // 0° = vers le haut, 90° = vers la droite. + let a = angle_deg.to_radians(); + let dir = [a.sin(), -a.cos()]; + let (anim, mb) = crate::frame_geometry::wallpaper_motion_slots( + motion, + g.programme_t, + self.rw() / self.rh(), + ); + self.draw_solid(&LayerCB { + dst: [0.0, 0.0, 1.0, 1.0], + fx: [dir[0], dir[1], anim[0], anim[1]], + mb, + ..crate::frame_geometry::gradient_layer(&stops, &offsets, lp.bg_color) + }); + } + SceneBackground::Image { path, motion } => { + // image bg (cover-fit, mise en cache) ; fallback couleur si chargement échoue + // (loggé — un fallback silencieux masquerait un chemin cassé, cf. le panic + // borrow qu'on a déjà eu : toute panne doit être visible/traçable). + let aspect = self.rw() / self.rh(); + if let Err(e) = self.draw_image_bg(&path, aspect, motion, g.programme_t) { + eprintln!("[compositor] wallpaper image \"{}\" : {:#}", path, e); + bg_drawn = false; + self.draw_solid(&LayerCB { + dst: [0.0, 0.0, 1.0, 1.0], + mode: 1.0, + color: lp.bg_color, + ..Default::default() + }); + } + } } - } - // « Blur BG » (parité web blurredBackgroundLayer) : floute CE wallpaper qu'on vient - // de dessiner (dual-Kawase, déjà utilisé pour le fond fixture ci-dessous). No-op - // visuel sur une couleur plate, effet réel sur gradient/image. - if blur_wallpaper > 0.0 { - self.blur_bg(blur_wallpaper); + // « Blur BG » (parité web blurredBackgroundLayer) : floute CE wallpaper qu'on vient + // de dessiner (dual-Kawase, déjà utilisé pour le fond fixture ci-dessous). No-op + // visuel sur une couleur plate, effet réel sur gradient/image. + if blur_wallpaper > 0.0 { + self.blur_bg(blur_wallpaper); + self.bind_compose_state(); + } + } else if cfg.bg_blur > 0.0 { + let over = 0.06; + self.draw_video( + &LayerCB { + dst: [-over, -over, 1.0 + 2.0 * over, 1.0 + 2.0 * over], + src: g.screen_src([0.0, 0.0, u_max, v_max]), + quad_px: [self.rw(), self.rh()], + mode: 0.0, + color: [1.0, 1.0, 1.0, 1.0], + ..Default::default() + }, + &sy, + &suv, + ); + self.blur_bg(cfg.bg_blur); self.bind_compose_state(); - } - } else if cfg.bg_blur > 0.0 { - let over = 0.06; - self.draw_video( - &LayerCB { - dst: [-over, -over, 1.0 + 2.0 * over, 1.0 + 2.0 * over], - src: [0.0, 0.0, u_max, v_max], - quad_px: [self.rw(), self.rh()], - mode: 0.0, - color: [1.0, 1.0, 1.0, 1.0], + self.draw_solid(&LayerCB { + dst: [0.0, 0.0, 1.0, 1.0], + mode: 1.0, + color: [0.0, 0.0, 0.0, 0.35], ..Default::default() - }, - &sy, - &suv, - ); - self.blur_bg(cfg.bg_blur); - self.bind_compose_state(); - self.draw_solid(&LayerCB { - dst: [0.0, 0.0, 1.0, 1.0], - mode: 1.0, - color: [0.0, 0.0, 0.0, 0.35], - ..Default::default() - }); - } else { - self.draw_solid(&LayerCB { - dst: [0.0, 0.0, 1.0, 1.0], - mode: 1.0, - color: lp.bg_color, - ..Default::default() + }); + } else { + self.draw_solid(&LayerCB { + dst: [0.0, 0.0, 1.0, 1.0], + mode: 1.0, + color: lp.bg_color, + ..Default::default() + }); + } + // Pas de cache pour un fond animé (`of` rend `None`). + let key = scene_ref.as_ref().filter(|_| bg_drawn).and_then(|s| { + crate::frame_geometry::BackgroundKey::of( + Some(&s.background), + s.effects.blur, + lp.bg_color, + bg_size, + ) }); + if let Some(key) = key { + self.keep_background(key)?; + } } // --- screen : crop du clip actif, puis zoom appliqué dans ce rect source (§8) --- @@ -2138,6 +2311,11 @@ impl Compositor { self.ctx.PSSetShaderResources(5, Some(&[None])); } + // --- flous de confidentialité : sur le métrage, AVANT le curseur. Le curseur reste net + // par-dessus, et le cristal de Prism Glow, qui réfracte l'image composée, n'y voit que des + // pixels déjà floutés. Les autres annotations restent le calque le plus haut (plus bas). + self.draw_annotations(scene_ref.as_ref(), source_t, s_ann, &g, true); + // --- curseur custom : suit le mapping src/dst (zoom+layout), click bounce, // et flou de mouvement (parité `compositor_macos.rs` et `compositor_linux.rs`) --- if let Some(track) = cursor_ref.as_ref() { @@ -2161,6 +2339,13 @@ impl Compositor { .map(|s| s.cursor.cursor_sprites.clone()) .unwrap_or_default(); let cursor_type = plan.cursor_type.as_deref(); + // Le cristal de Prism Glow réfracte l'image telle qu'elle est composée à cet + // instant, flous compris : sa copie en t5, une fois pour toutes les copies de la + // traînée (`ann_copy` est libre, les flous l'ont déjà lue). + if plan.glass { + self.ctx.CopySubresourceRegion(&self.ann_copy, 0, 0, 0, 0, &self.rt, 0, None); + self.ctx.PSSetShaderResources(5, Some(&[Some(self.ann_copy_srv.clone())])); + } // L'impact des clics (mode 16, sans texture), posé sur l'écran SOUS le curseur. for cb in &plan.impacts { self.draw_solid(cb); @@ -2180,11 +2365,14 @@ impl Compositor { // buffer ISOLÉ (transparent), pas directement sur la scène déjà composée. self.ctx.ClearRenderTargetView(&self.accum_rtv, &[0.0, 0.0, 0.0, 0.0]); self.ctx.OMSetRenderTargets(Some(&[Some(self.accum_rtv.clone())]), None); + // L'union des quads peints dans `accum` (x0, y0, x1, y1 en fractions de la + // sortie) : la seule région que la recopie doit relire. + let mut bounds = [f32::MAX, f32::MAX, f32::MIN, f32::MIN]; for k in 0..plan.taps { let f = k as f32 / (plan.taps - 1) as f32; let w = crate::frame_geometry::cursor_tap_weight(k, plan.taps); self.ctx.OMSetBlendState(&self.blend_add, Some(&[w, w, w, w]), 0xffffffff); - self.draw_cur_themed( + let drawn = self.draw_cur_themed( &cursor_sprites, cursor_type, plan.prev_placement.lerp(plan.placement, f), @@ -2193,6 +2381,14 @@ impl Compositor { plan.clip, plan.model, ); + if let Some(d) = drawn { + bounds = [ + bounds[0].min(d[0]), + bounds[1].min(d[1]), + bounds[2].max(d[0] + d[2]), + bounds[3].max(d[1] + d[3]), + ]; + } } // composite le buffer accumulé sur la scène (blend "over" normal, prémultiplié). self.ctx.OMSetRenderTargets(Some(&[Some(self.rtv.clone())]), None); @@ -2210,12 +2406,30 @@ impl Compositor { }; self.ctx.RSSetViewports(Some(&[vp])); self.ctx.OMSetBlendState(&self.blend, None, 0xffffffff); - self.ctx.Draw(3, 0); + // Rien n'a été peint dans `accum` hors des quads de la traînée : la recopie + // s'y borne au lieu de repasser sur toute la sortie (0,5 ms par frame 1080p + // mesurée sur Linux, Radeon 610M). + let (rw, rh) = (self.rw(), self.rh()); + let rect = RECT { + left: (bounds[0] * rw).floor().clamp(0.0, rw) as i32, + top: (bounds[1] * rh).floor().clamp(0.0, rh) as i32, + right: (bounds[2] * rw).ceil().clamp(0.0, rw) as i32, + bottom: (bounds[3] * rh).ceil().clamp(0.0, rh) as i32, + }; + if rect.right > rect.left && rect.bottom > rect.top { + self.ctx.RSSetState(&self.rs_scissor); + self.ctx.RSSetScissorRects(Some(&[rect])); + self.ctx.Draw(3, 0); + self.ctx.RSSetState(None); + } self.ctx.PSSetShaderResources(0, Some(&[None])); // restaure l'état de composition standard (VS/PS/topologie quad-strip) pour // le dessin de la webcam qui suit juste après. self.bind_compose_state(); } + if plan.glass { + self.ctx.PSSetShaderResources(5, Some(&[None])); + } } } @@ -2334,12 +2548,13 @@ impl Compositor { // la transform, donc les annotations restent en place pendant que le contenu zoome dessous. // Ce fut `s_dst` tant que le zoom vivait dans la coupe source ; depuis l'issue #179 il vit // dans la BOÎTE, et `s_dst` emmenait annotations et sous-titres avec lui. - // Exception : le flou de confidentialité suit le contenu (`FrameGeometry::privacy_mask`), - // d'où la géométrie entière passée en plus de `s_ann`. + // Les flous de confidentialité, eux, sont passés avant le curseur (plus haut) : ils suivent + // le contenu (`FrameGeometry::privacy_mask`), d'où la géométrie entière passée en plus de + // `s_ann`. // `source_t`, la même base de temps que les zoom/speed regions : le temps SOURCE du clip, // pas le compteur de frames. C'est ce qui garde une annotation alignée sur l'image quand // une speed region répète ou saute des frames. - self.draw_annotations(scene_ref.as_ref(), source_t, s_ann, &g); + self.draw_annotations(scene_ref.as_ref(), source_t, s_ann, &g, false); Ok(()) } @@ -2351,19 +2566,23 @@ impl Compositor { /// sous-titres sous un zoom (issue #179, puis #397 sur Linux). L'arithmétique elle-même vit /// dans `frame_geometry::annotation_dst_in`, partagée par les trois backends. Le flou, lui, /// se place par `g.privacy_mask` : un masque doit rester sur ce qu'il cache. + /// + /// `privacy` : les flous de confidentialité seuls, dessinés sur le métrage avant le curseur ; + /// sinon toutes les autres annotations, le calque le plus haut. unsafe fn draw_annotations( &self, scene: Option<&Scene>, t: f32, s_ann: [f32; 4], g: &crate::frame_geometry::FrameGeometry, + privacy: bool, ) { let Some(scene) = scene else { return }; if scene.annotations.is_empty() { return; } let visible = |a: &crate::scene::SceneAnnotation| { - t >= a.start_sec as f32 && t < a.end_sec as f32 + t >= a.start_sec as f32 && t < a.end_sec as f32 && (a.kind == "blur") == privacy }; // Une seule recopie du render target pour TOUTES les annotations flou de la frame — leur // lecture doit voir l'image composée sans les flous eux-mêmes, sinon deux zones qui se @@ -2917,6 +3136,12 @@ impl Compositor { Ok((rw, rh, out)) } + /// Le RT de la dernière composition, à `render_size()`, pour qui le copie ailleurs sans + /// passer par la RAM : l'anneau de textures partagées de la preview (`shared_frames`). + pub fn render_target(&self) -> &ID3D11Texture2D { + &self.rt + } + /// Comme `rgb_to_nv12`, mais vers `target_w`×`target_h` : si la cible diffère de la taille /// de rendu, le RT composé est d'abord redimensionné (bilinéaire, `ps_tex`/`sampler` déjà /// utilisés partout ailleurs dans le fichier, cf. `blit_resized`). Le RT suivant la sortie, @@ -3249,6 +3474,62 @@ mod tests { ); } + /// Un NV12 sans données, à la taille voulue : tout ce que `nv12_srvs` lit d'une frame. + fn nv12_frame(gpu: &crate::d3d::Gpu, w: u32, h: u32) -> (Box, ID3D11Texture2D) { + let desc = D3D11_TEXTURE2D_DESC { + Width: w, + Height: h, + MipLevels: 1, + ArraySize: 1, + Format: DXGI_FORMAT_NV12, + SampleDesc: DXGI_SAMPLE_DESC { Count: 1, Quality: 0 }, + Usage: D3D11_USAGE_DEFAULT, + BindFlags: D3D11_BIND_SHADER_RESOURCE.0 as u32, + CPUAccessFlags: 0, + MiscFlags: 0, + }; + unsafe { + let mut tex: Option = None; + gpu.device.CreateTexture2D(&desc, None, Some(&mut tex)).expect("texture NV12"); + let tex = tex.expect("texture NV12"); + let mut frame: Box = Box::new(std::mem::zeroed()); + frame.data[0] = tex.as_raw() as *mut u8; + frame.data[1] = std::ptr::null_mut(); + (frame, tex) + } + } + + /// Un décodeur remplacé sans `clear_srv_cache` (changement de résolution en cours de flux, + /// `CpuFrames::ensure_tex`) peut donner à une texture NEUVE l'adresse d'une ancienne. La + /// copie privée gardée pour l'ancienne est alors trop petite, `CopySubresourceRegion` la + /// saute sans erreur et l'image gèle. On ne peut pas choisir l'adresse d'une texture ; + /// l'entrée de la petite est donc replacée sous la clé de la grande, l'état exact où le + /// cache se retrouve après un tel recyclage. + #[test] + fn a_larger_source_at_a_cached_address_gets_a_copy_of_its_own_size() { + let Ok(gpu) = crate::d3d::Gpu::create_auto(false) else { + eprintln!("pas de device D3D11 — test sauté"); + return; + }; + let comp = Compositor::new_sized(&gpu, 320, 180).expect("compositeur"); + let (small, _small_tex) = nv12_frame(&gpu, 64, 64); + let (large, _large_tex) = nv12_frame(&gpu, 128, 96); + let (small_key, large_key) = (small.data[0] as usize, large.data[0] as usize); + + unsafe { comp.nv12_srvs(&*small) }.expect("source de 64x64"); + { + let mut cache = comp.srv_cache.borrow_mut(); + let stale = cache.remove(&small_key).expect("entrée de la petite source"); + cache.insert(large_key, stale); + } + unsafe { comp.nv12_srvs(&*large) }.expect("source de 128x96"); + + let (dst, ..) = comp.srv_cache.borrow().get(&large_key).cloned().expect("entrée remplacée"); + let mut dd = D3D11_TEXTURE2D_DESC::default(); + unsafe { dst.GetDesc(&mut dd) }; + assert_eq!((dd.Width, dd.Height), (128, 96), "la copie a gardé la taille de l'ancienne texture"); + } + /// Un cran de padding en format Auto change la taille de rendu de la preview. `resized` /// réalloue les cibles et garde le reste : le masque webcam, et la boîte aux lettres où le /// worker de segmentation dépose les suivants. diff --git a/crates/compositor/src/cpu_frames_windows.rs b/crates/compositor/src/cpu_frames_windows.rs index 790f23a7fc..ea0e921fbe 100644 --- a/crates/compositor/src/cpu_frames_windows.rs +++ b/crates/compositor/src/cpu_frames_windows.rs @@ -41,8 +41,9 @@ pub(crate) struct CpuFrames { /// NV12 en mémoire système : la cible de swscale, la source de l'upload. nv12: *mut AVFrame, /// La texture NV12 échantillonnée par les shaders. UNE seule, réécrite à chaque - /// frame — le `srv_cache` du compositeur (clé `(ptr, slice)`) n'a donc qu'une entrée - /// et ne recrée jamais de SRV, contrairement au pool tournant de D3D11VA. + /// frame — le `srv_cache` du compositeur (clé : le pointeur de texture) n'a donc qu'une + /// entrée et ne recrée jamais de SRV, contrairement au pool tournant de D3D11VA. Quand + /// `ensure_tex` la réalloue à une autre taille, `nv12_srvs` voit l'écart et refait la copie. // ponytail: une seule texture = le CPU peut attendre que le GPU ait fini de lire la // frame précédente. Sur WARP tout est CPU et le pilote sérialise déjà ; si un backend // GPU réutilise ce chemin un jour et que le Map bloque, double-bufferiser ici. diff --git a/crates/compositor/src/cursor.rs b/crates/compositor/src/cursor.rs index d9c104a822..b1bac4fb28 100644 --- a/crates/compositor/src/cursor.rs +++ b/crates/compositor/src/cursor.rs @@ -164,6 +164,23 @@ impl CursorTrack { } } + /// The same track seen on a picture turned by 180°: every position mirrored, times, + /// clicks, visibility and cursor types unchanged. A turned clip is planned entirely in + /// what the viewer sees, so its telemetry is turned once here rather than at every reader. + pub fn turned(&self) -> CursorTrack { + let flip = + |v: &[(f32, f32, f32)]| v.iter().map(|&(t, x, y)| (t, 1.0 - x, 1.0 - y)).collect(); + CursorTrack { + samples: flip(&self.samples), + follow_samples: flip(&self.follow_samples), + clicks: self.clicks.clone(), + click_points: self.click_points.iter().map(|&(x, y)| (1.0 - x, 1.0 - y)).collect(), + visibility: self.visibility.clone(), + types: self.types.clone(), + view_cache: Default::default(), + } + } + /// Visibilité enregistrée au temps `t`. Les anciens sidecars qui n'ont pas ce champ sont /// visibles par défaut, comme les lecteurs TypeScript. pub fn visible_at(&self, t: f32) -> bool { @@ -661,6 +678,26 @@ fn cursor_spring_config(smoothing_factor: f32) -> (f32, f32, f32) { mod tests { use super::*; + #[test] + fn turned_track_mirrors_positions_and_keeps_times() { + let t = CursorTrack::new( + vec![(0.0, 0.2, 0.3), (1.0, 0.6, 0.9)], + vec![0.5], + vec![(0.0, "arrow".to_string())], + ); + let r = t.turned(); + let (x, y) = r.at(0.0).unwrap(); + assert!((x - 0.8).abs() < 1e-6 && (y - 0.7).abs() < 1e-6, "({x}, {y})"); + assert_eq!(r.clicks_between(0.0, 1.0), t.clicks_between(0.0, 1.0)); + assert_eq!(r.type_at(0.0), t.type_at(0.0)); + let (fx, fy) = r.follow_at(1.0).unwrap(); + let (gx, gy) = t.follow_at(1.0).unwrap(); + assert!((fx - (1.0 - gx)).abs() < 1e-5 && (fy - (1.0 - gy)).abs() < 1e-5); + // The click point turns with the picture: the click at 0.5 s was at (0.4, 0.6). + let (px, py) = r.click_points[0]; + assert!((px - 0.6).abs() < 1e-6 && (py - 0.4).abs() < 1e-6, "({px}, {py})"); + } + /// Zone morte du suivi auto : un pointeur qui tremble au centre de la vue ne la bouge pas ; /// un pointeur qui part loin est rattrapé, jusqu'au bord de la zone et pas au-delà. /// Les vues d'une région sont rejouées une fois puis relues : même réponse dans n'importe quel diff --git a/crates/compositor/src/gif_export_control.rs b/crates/compositor/src/export_control.rs similarity index 70% rename from crates/compositor/src/gif_export_control.rs rename to crates/compositor/src/export_control.rs index bf4ad12430..8542c55eab 100644 --- a/crates/compositor/src/gif_export_control.rs +++ b/crates/compositor/src/export_control.rs @@ -1,4 +1,4 @@ -//! Cooperative GIF cancellation and publication of a completed output only. +//! Cooperative export cancellation (MP4 and GIF) and publication of a completed output only. use anyhow::{bail, Context, Result}; use std::fs::{self, File, OpenOptions}; @@ -11,20 +11,20 @@ const CANCELLED: u8 = 1; const COMMITTING: u8 = 2; #[derive(Clone, Default)] -pub struct GifExportControl(Arc); +pub struct ExportControl(Arc); #[derive(Debug)] -pub struct GifExportCancelled; +pub struct ExportCancelled; -impl std::fmt::Display for GifExportCancelled { +impl std::fmt::Display for ExportCancelled { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str("GIF export cancelled") + f.write_str("export cancelled") } } -impl std::error::Error for GifExportCancelled {} +impl std::error::Error for ExportCancelled {} -impl GifExportControl { +impl ExportControl { /// False means publication has already won the race. Repeated cancellation /// of the same pending job is harmless. pub fn cancel(&self) -> bool { @@ -36,7 +36,7 @@ impl GifExportControl { pub fn check(&self) -> Result<()> { if self.0.load(Ordering::Acquire) == CANCELLED { - return Err(GifExportCancelled.into()); + return Err(ExportCancelled.into()); } Ok(()) } @@ -44,20 +44,20 @@ impl GifExportControl { fn begin_commit(&self) -> Result<()> { match self.0.compare_exchange(RUNNING, COMMITTING, Ordering::AcqRel, Ordering::Acquire) { Ok(_) => Ok(()), - Err(CANCELLED) => Err(GifExportCancelled.into()), - Err(_) => bail!("GIF export control has already been used"), + Err(CANCELLED) => Err(ExportCancelled.into()), + Err(_) => bail!("export control has already been used"), } } } static NEXT_OUTPUT: AtomicU64 = AtomicU64::new(0); -struct StagedGif { +struct StagedOutput { path: PathBuf, published: bool, } -impl Drop for StagedGif { +impl Drop for StagedOutput { fn drop(&mut self) { if !self.published { let _ = fs::remove_file(&self.path); @@ -66,27 +66,31 @@ impl Drop for StagedGif { } /// Keep the destination intact until rendering and flushing have succeeded. -/// `render` owns the file so it is closed before rename/cleanup on Windows. -pub(crate) fn with_gif_output( +/// `render` owns the file so it is closed before rename/cleanup on Windows; a +/// renderer that reopens the path (ffmpeg) must close it before returning too. +/// The staged name ends with the target's extension, which is what ffmpeg picks +/// the container from. +pub(crate) fn with_staged_output( target: &Path, - control: &GifExportControl, + control: &ExportControl, render: impl FnOnce(File, &Path) -> Result, ) -> Result { control.check()?; let parent = target.parent().filter(|p| !p.as_os_str().is_empty()).unwrap_or(Path::new(".")); fs::create_dir_all(parent)?; + let ext = target.extension().map(|e| e.to_string_lossy()).unwrap_or_default(); let (mut staged, file) = loop { let nonce = NEXT_OUTPUT.fetch_add(1, Ordering::Relaxed); - let path = parent.join(format!(".openscreen-gif-{}-{nonce}.partial", std::process::id())); + let path = parent.join(format!(".openscreen-{ext}-{}-{nonce}.partial.{ext}", std::process::id())); match OpenOptions::new().write(true).create_new(true).open(&path) { - Ok(file) => break (StagedGif { path, published: false }, file), + Ok(file) => break (StagedOutput { path, published: false }, file), Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue, - Err(e) => return Err(e).context("creating temporary GIF output"), + Err(e) => return Err(e).context("creating temporary export output"), } }; let result = render(file, &staged.path).and_then(|stats| { control.begin_commit()?; - fs::rename(&staged.path, target).context("publishing completed GIF output")?; + fs::rename(&staged.path, target).context("publishing completed export output")?; staged.published = true; Ok(stats) }); @@ -95,7 +99,7 @@ pub(crate) fn with_gif_output( if cleanup.kind() != std::io::ErrorKind::NotFound { // A failed cleanup is an error, even if cancellation caused it. // Do not report a clean cancellation while leaving an output. - bail!("could not remove partial GIF {}: {cleanup}", staged.path.display()); + bail!("could not remove partial export {}: {cleanup}", staged.path.display()); } } } @@ -128,28 +132,41 @@ mod tests { #[test] fn cancellation_before_rendering_never_opens_output() { let dir = TestDir::new(); - let control = GifExportControl::default(); + let control = ExportControl::default(); assert!(control.cancel()); - let result = with_gif_output(&dir.0.join("out.gif"), &control, |_, _| -> Result<()> { + let result = with_staged_output(&dir.0.join("out.gif"), &control, |_, _| -> Result<()> { panic!("cancelled job must not render"); }); - assert!(result.unwrap_err().is::()); + assert!(result.unwrap_err().is::()); assert_eq!(fs::read_dir(&dir.0).unwrap().count(), 0); } + #[test] + fn staged_output_keeps_the_target_extension() { + // ffmpeg picks the MP4 muxer from the file name; a bare `.partial` cannot be opened. + let dir = TestDir::new(); + let control = ExportControl::default(); + with_staged_output(&dir.0.join("out.mp4"), &control, |_, staged| { + assert_eq!(staged.extension().unwrap(), "mp4"); + Ok(()) + }) + .unwrap(); + dir.assert_only("out.mp4"); + } + #[test] fn cancellation_discards_partial_output_and_preserves_destination() { let dir = TestDir::new(); let target = dir.0.join("out.gif"); fs::write(&target, b"original GIF").unwrap(); - let control = GifExportControl::default(); - let result = with_gif_output(&target, &control, |mut file, _| { + let control = ExportControl::default(); + let result = with_staged_output(&target, &control, |mut file, _| { file.write_all(b"partial replacement")?; assert!(control.cancel()); // Even a renderer that has just completed cannot publish now. Ok(42) }); - assert!(result.unwrap_err().is::()); + assert!(result.unwrap_err().is::()); assert_eq!(fs::read(&target).unwrap(), b"original GIF"); dir.assert_only("out.gif"); } @@ -159,8 +176,8 @@ mod tests { let dir = TestDir::new(); let target = dir.0.join("out.gif"); fs::write(&target, b"original").unwrap(); - let control = GifExportControl::default(); - assert_eq!(with_gif_output(&target, &control, |mut file, _| { + let control = ExportControl::default(); + assert_eq!(with_staged_output(&target, &control, |mut file, _| { file.write_all(b"finished GIF")?; Ok(7) }).unwrap(), 7); @@ -173,8 +190,8 @@ mod tests { fn render_failure_is_not_misreported_as_cancellation() { let dir = TestDir::new(); let target = dir.0.join("out.gif"); - let control = GifExportControl::default(); - let result = with_gif_output(&target, &control, |mut file, _| -> Result<()> { + let control = ExportControl::default(); + let result = with_staged_output(&target, &control, |mut file, _| -> Result<()> { file.write_all(b"partial")?; control.cancel(); bail!("encoder failed") @@ -188,8 +205,8 @@ mod tests { let dir = TestDir::new(); let target = dir.0.join("destination-directory"); fs::create_dir(&target).unwrap(); - let control = GifExportControl::default(); - assert!(with_gif_output(&target, &control, |mut file, _| { + let control = ExportControl::default(); + assert!(with_staged_output(&target, &control, |mut file, _| { file.write_all(b"finished GIF")?; Ok(()) }).is_err()); @@ -200,7 +217,7 @@ mod tests { #[test] fn cancellation_and_commit_have_exactly_one_winner() { for _ in 0..64 { - let control = GifExportControl::default(); + let control = ExportControl::default(); let cancel_control = control.clone(); let barrier = Arc::new(Barrier::new(2)); let cancel_barrier = barrier.clone(); diff --git a/crates/compositor/src/frame_geometry.rs b/crates/compositor/src/frame_geometry.rs index c788d92b85..0281e64bfa 100644 --- a/crates/compositor/src/frame_geometry.rs +++ b/crates/compositor/src/frame_geometry.rs @@ -64,6 +64,18 @@ pub struct LayerCB { pub trail_mb: [f32; 4], } +impl LayerCB { + /// Le calque est l'un des modèles 3D — le curseur modelé (mode 15), l'impact de son clic + /// (16), l'appareil et son ombre (17) — ou le verre à plat de Glass Lens (19), qui en reprend + /// les formes : seule la variante « modèles » du shader de calque les compile. Dans le même + /// shader que les autres, leurs branches fixaient l'occupation de TOUS les calques (168 VGPR + /// sur RADV) ; les trois backends décident donc du shader d'un calque par ce test, au moment de + /// le dessiner (cf. le haut de `vk_shaders/layer.wgsl`). + pub fn needs_models(&self) -> bool { + (self.mode > 14.5 && self.mode < 17.5) || self.mode > 18.5 + } +} + pub const OUT_W: u32 = 1920; pub const OUT_H: u32 = 1080; /// Parse une couleur "#rgb" / "#rrggbb" (sRGB, comme les wallpapers web) → [r,g,b,a] 0..1. @@ -1201,6 +1213,61 @@ pub fn cursor_sprite_cb( } } +// ============ Verre à plat (mode 19) ============ +// +// Glass Lens, 3D éteinte : la lentille de son modèle (`sculpt.rs`, `s_glass` dans les shaders) +// vue de face, posée sur l'image comme le sprite plat (le quad projeté et le warp inverse du +// mode 13), mais dessinée par le shader, qui réfracte la copie de l'image composée +// (`CursorPlan::glass`) au travers d'un relief tiré des mêmes formes. Le PNG du thème ne sert plus +// qu'au sélecteur. +// +// Emplacements du `LayerCB` au mode 19 : +// dst, quad_px bbox du rect du modèle projeté (sortie 0..1, px) +// fx, src_prev coins TL, TR puis BR, BL, en px locaux à la bbox (comme au mode 13) +// dst_prev rect de clip « Clip to canvas » (sortie 0..1) +// mb.x 1 = warp projectif (caméra réelle) +// src.xy l'unité du modèle en fractions de la sortie (x, y) : le décalage de la réfraction +// color.rg coin haut-gauche du rect, repère du modèle (unités), comme au mode 15 +// color.b l'unité du modèle en px de sortie : l'antialiasing +// color.a opacité +// radius_px rapport w/h du rect (`sprite_size` des shaders) +// trail_a.x le curseur sculpté (`SpriteShape::sculpt`) +// Texture : la copie de l'image composée, t5/texture(5) (`texDof`) sous Windows et macOS, au +// binding 1 (`texY`) sous Linux, comme pour le cristal du mode 15. + +/// Le verre à plat du curseur sculpté `name` posé en `placement` : la boîte du modèle a pour plus +/// grand côté `size_px`, comme au mode 15. `None` pour un nom qui n'est pas un modèle. +pub fn cursor_glass_cb( + placement: CursorPlacement, + size_px: f32, + name: &str, + alpha: f32, + clip: [f32; 4], + render_px: [f32; 2], +) -> Option { + let shape = crate::sculpt::sculpted_shape(name)?; + let [rw, rh] = render_px; + let sprite_px = [shape.size[0] * size_px, shape.size[1] * size_px]; + let mut cb = cursor_sprite_cb(placement, sprite_px, shape.hotspot, alpha, clip, render_px); + if cb.mode < 12.5 { + // Écran droit (mode 7) : son rect, sous la forme du mode 13. Le warp bilinéaire d'un + // rect est exact. + let [w, h] = [cb.dst[2] * rw, cb.dst[3] * rh]; + cb.quad_px = [w, h]; + cb.fx = [0.0, 0.0, w, 0.0]; + cb.src_prev = [w, h, 0.0, h]; + cb.dst_prev = clip; + } + // L'unité telle que le plan la montre : le bord gauche du rect projeté (TL -> BL). + let unit_px = (cb.src_prev[2] - cb.fx[0]).hypot(cb.src_prev[3] - cb.fx[1]) / shape.size[1]; + cb.mode = 19.0; + cb.src = [unit_px / rw, unit_px / rh, 0.0, 0.0]; + cb.color = [shape.origin()[0], shape.origin()[1], unit_px, alpha]; + cb.radius_px = shape.size[0] / shape.size[1]; + cb.trail_a = [shape.sculpt as f32, 0.0, 0.0, 0.0]; + Some(cb) +} + /// Gain de l'éclairage de la caméra réelle : une lampe posée sur la caméra, dont la lumière /// décroît avec le carré de la distance. 1 = la décroissance physique ; moins, pour que le /// contenu reste lisible. L'œil en orbite passe loin de l'axe : à 22° d'azimut, l'écart d'un bord @@ -1590,6 +1657,54 @@ fn same_source_path(a: &str, b: &str) -> bool { /// shader un temps borné, là où un `f32` de plusieurs heures perdrait la finesse du bruit. pub const WALLPAPER_MOTION_PERIOD_S: f32 = 120.0; +/// Ce qui décide à lui seul du fond composé — sa description, la force de son flou, la couleur +/// de repli, la taille de rendu —, gardé avec sa copie (`bg_cache`) : tant que le fond demandé +/// lui est égal, le backend recopie le fond de la frame d'avant au lieu de le redessiner, ce qui +/// coûtait plus que l'écran lui-même (1,65 ms par frame 1080p sur une Radeon 610M, un wallpaper +/// 4K échantillonné sans mips). +/// +/// La comparaison de chaque frame (`matches`) ne construit ni ne copie rien : une image en data +/// URI pèse plusieurs Mo. Seul `of`, au moment de garder un nouveau fond, la clone. +#[derive(Clone, PartialEq)] +pub struct BackgroundKey { + background: Option, + blur: f32, + fallback: [f32; 4], + render_px: [f32; 2], +} + +impl BackgroundKey { + /// La clé d'un fond fixe ; `None` pour un fond animé, qui change à chaque frame. + pub fn of( + background: Option<&crate::scene::SceneBackground>, + blur: f32, + fallback: [f32; 4], + render_px: [f32; 2], + ) -> Option { + use crate::scene::{SceneBackground as B, WallpaperMotion}; + if let Some(B::Gradient { motion, .. } | B::Image { motion, .. }) = background { + if *motion != WallpaperMotion::None { + return None; + } + } + Some(BackgroundKey { background: background.cloned(), blur, fallback, render_px }) + } + + /// Le fond demandé est celui-ci. + pub fn matches( + &self, + background: Option<&crate::scene::SceneBackground>, + blur: f32, + fallback: [f32; 4], + render_px: [f32; 2], + ) -> bool { + self.background.as_ref() == background + && self.blur == blur + && self.fallback == fallback + && self.render_px == render_px + } +} + /// Emplacements libres des modes 5 (dégradé) et 6 (image) pour le fond animé : `fx.zw` = (temps /// programme replié, indice du mouvement), `mb.x` = aspect w/h du rect rempli (les nappes de /// l'aurore restent rondes). @@ -1857,6 +1972,11 @@ pub struct FrameGeometry { /// Layouts en bloc : le slot de l'écran, un conteneur qui le MASQUE (cf. `ScreenMask`). /// `None` ailleurs, et le rendu est celui d'avant, à l'octet. pub screen_mask: Option, + /// The clip being composed is turned by 180°. Everything above is planned in display + /// space; only the screen layer's texture sampling turns, through `screen_src`. + pub screen_turn: bool, + /// `[u_max, v_max]` of the screen texture: the valid area `screen_src` mirrors within. + pub screen_uv_max: [f32; 2], } /// Le conteneur d'un layout en bloc (côte à côte, haut/bas) : un masque fixe, overflow hidden. @@ -1938,6 +2058,18 @@ pub fn annotation_dst_in(anchor: [f32; 4], x: f32, y: f32, w: f32, h: f32) -> [f } impl FrameGeometry { + /// The texture rect to sample for a display-space screen rect. Upright: unchanged. + /// Turned: mirrored inside the valid area, which reverses both bounds — the shaders + /// interpolate linearly from `src.xy` to `src.zw`, so that IS the half turn. `cut` and + /// everything planned from it stay in display space. + pub fn screen_src(&self, uv: [f32; 4]) -> [f32; 4] { + if !self.screen_turn { + return uv; + } + let [um, vm] = self.screen_uv_max; + [um - uv[0], vm - uv[1], um - uv[2], vm - uv[3]] + } + /// Le quad de l'écran incliné pour une boîte de `s_px` px, `None` quand l'écran est droit. /// LE point de passage de l'écran, de son ombre, du curseur et du masque de flou : tous /// doivent porter la même séparation base / dynamique, sinon ils se décollent. @@ -2107,7 +2239,8 @@ impl FrameGeometry { let s_px = [self.s_dst[2] * render_px[0], self.s_dst[3] * render_px[1]]; let flat = LayerCB { dst: [0.0, 0.0, 1.0, 1.0], - src: self.cut, + // Reread from the screen texture: a turned clip samples it with reversed bounds. + src: self.screen_src(self.cut), quad_px: s_px, radius_px: self.s_radius, mode: 18.0, @@ -2658,10 +2791,6 @@ impl FrameGeometry { [centre[0] + px, centre[1] + py] }; let pts = [at(x0, y0), at(x1, y0), at(x1, y1), at(x0, y1)]; - // Le mode 8 étale aussi chaque pixel vers là où le plan était une frame plus tôt - // (`tilt_trail`) : une copie traînée du secret sort du quad courant. Le masque couvre alors - // la boîte englobante du secret aux deux frames, en rect droit, comme le chemin droit. - // Même warp que le shader sur les coins d'avant (le drapeau projectif est celui du plan). // La perspective grossit localement le côté proche : la force suit l'arête la plus // agrandie par rapport au rect droit zoomé, pour que nulle part le masque ne soit plus // fin, rapporté au contenu, qu'au repos. @@ -2672,28 +2801,29 @@ impl FrameGeometry { .max(len(pts[0], pts[3]) / eh) .max(len(pts[1], pts[2]) / eh); let strength = (zoom_k * tilt_k).max(1.0); - // Sans mouvement, la traînée EST le quad (au bit près) : rien de plus à couvrir. - if let Some(trail) = self.tilt_trail(render_px).filter(|t| t.corners != quad.corners) { - let before = crate::regions::TiltedQuad { corners: trail.corners, ..quad }; - let then = |fx: f32, fy: f32| { - let (px, py) = before.point_px(fx, fy); - [centre[0] + px, centre[1] + py] - }; - let (mut lo, mut hi) = ([f32::MAX; 2], [f32::MIN; 2]); - for [px, py] in pts.into_iter().chain([then(x0, y0), then(x1, y0), then(x1, y1), then(x0, y1)]) { - (lo, hi) = ([lo[0].min(px), lo[1].min(py)], [hi[0].max(px), hi[1].max(py)]); + // Le mode 8 étale aussi chaque pixel vers là où le plan était une frame plus tôt + // (`tilt_trail`) : une copie traînée du secret sort du quad courant. Le masque garde la + // perspective du quad courant, élargi de la plus grande avance d'un de ses coins : chaque + // coin d'avant tombe dedans, donc tout le secret d'avant aussi (un quad convexe contient + // l'enveloppe de ses coins), et la traînée, qui va de l'un à l'autre. Même warp que le + // shader sur les coins d'avant (le drapeau projectif est celui du plan). Sans mouvement, + // la traînée EST le quad (au bit près) : rien de plus à couvrir. + let trail = self.tilt_trail(render_px).filter(|t| t.corners != quad.corners); + let pts = match trail { + Some(trail) => { + let before = crate::regions::TiltedQuad { corners: trail.corners, ..quad }; + let then = |fx: f32, fy: f32| { + let (px, py) = before.point_px(fx, fy); + [centre[0] + px, centre[1] + py] + }; + let prev = [then(x0, y0), then(x1, y0), then(x1, y1), then(x0, y1)]; + let reach = (0..4) + .map(|i| (prev[i][0] - pts[i][0]).hypot(prev[i][1] - pts[i][1])) + .fold(0.0f32, f32::max); + grow_quad(pts, reach + 0.5) } - let rect = - [lo[0] / rw, lo[1] / rh, (hi[0] - lo[0]).max(1.0) / rw, (hi[1] - lo[1]).max(1.0) / rh]; - let rect = match self.screen_mask { - Some(m) => m.clip(rect)?, - None => rect, - }; - let mut mask = PrivacyMask::upright(rect, render_px, strength); - // Un ovale inscrit dans la boîte élargie ne couvrirait plus l'ovale courant. - mask.oval_ok = false; - return Some(mask); - } + None => pts, + }; let (mut min_x, mut min_y) = (f32::MAX, f32::MAX); let (mut max_x, mut max_y) = (f32::MIN, f32::MIN); for [px, py] in pts { @@ -2721,9 +2851,65 @@ impl FrameGeometry { warp: Some(local), projective: quad.projective, strength, - oval_ok: true, + // Un ovale inscrit dans le quad élargi ne couvrirait plus sûrement l'ovale d'avant. + oval_ok: trail.is_none(), }) } + + /// Où tombe le métrage dans l'image (`FootageQuad`) : là où `privacy_mask` pose un masque, + /// donc là où l'éditeur doit poser le gimbal d'un flou pour qu'il couvre ce qu'il cache. + pub fn footage_quad(&self, render_px: [f32; 2]) -> FootageQuad { + let [rw, rh] = render_px; + let s_px = [self.s_dst[2] * rw, self.s_dst[3] * rh]; + if let Some(quad) = self.screen_tilt(s_px).filter(|_| self.tilted() && rw > 0.0 && rh > 0.0) { + let centre = self.screen_center_px(render_px); + let at = |fx: f32, fy: f32| { + let (px, py) = quad.point_px(fx, fy); + [(centre[0] + px) / rw, (centre[1] + py) / rh] + }; + return FootageQuad { + corners: [at(0.0, 0.0), at(1.0, 0.0), at(1.0, 1.0), at(0.0, 1.0)], + projective: quad.projective, + }; + } + let [x, y, w, h] = self.s_dst; + FootageQuad { corners: [[x, y], [x + w, y], [x + w, y + h], [x, y + h]], projective: false } + } +} + +/// Le métrage dans l'image : ses coins TL, TR, BR, BL en fractions de la sortie, et la +/// correspondance d'un point `(u, v)` du métrage à l'image, bilinéaire entre eux ou, sous la +/// caméra réelle, l'homographie qu'ils définissent (`TiltedQuad::point_px`). +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct FootageQuad { + pub corners: [[f32; 2]; 4], + pub projective: bool, +} + +/// Le quad convexe `pts` (TL, TR, BR, BL) dont chaque côté recule de `d` px vers l'extérieur : +/// ses côtés restent parallèles aux siens, donc sa perspective aussi, et il contient tout point +/// à moins de `d` de lui. +fn grow_quad(pts: [[f32; 2]; 4], d: f32) -> [[f32; 2]; 4] { + // L'extérieur d'un côté est d'un bord ou de l'autre selon le sens de parcours. + let area: f32 = (0..4) + .map(|i| { + let (a, b) = (pts[i], pts[(i + 1) % 4]); + a[0] * b[1] - b[0] * a[1] + }) + .sum(); + let side = area.signum(); + let normal = |i: usize| { + let (a, b) = (pts[i], pts[(i + 1) % 4]); + let (ex, ey) = (b[0] - a[0], b[1] - a[1]); + let len = ex.hypot(ey).max(1e-6); + [side * ey / len, -side * ex / len] + }; + std::array::from_fn(|i| { + // Le coin glisse sur la bissectrice jusqu'à la rencontre des deux côtés reculés. + let (na, nb) = (normal((i + 3) % 4), normal(i)); + let k = d / (1.0 + na[0] * nb[0] + na[1] * nb[1]).max(0.2); + [pts[i][0] + (na[0] + nb[0]) * k, pts[i][1] + (na[1] + nb[1]) * k] + }) } /// Marge ajoutée autour d'un masque de confidentialité, en px de la boîte écran AU REPOS : un @@ -2891,6 +3077,8 @@ pub fn plan_frame(input: &FrameGeometryInput) -> FrameGeometry { let mut zoom_tilt = 0.0f32; let mut zoom_camera = 0.0f32; let mut zoom_camera_prev = 0.0f32; + let mut zoom_strength = 0.0f32; + let mut zoom_strength_prev = 0.0f32; let mut zoom_aim = [0.5f32; 2]; let mut zoom_orbit = [0.5f32; 2]; let (mut zoom_aim_prev, mut zoom_orbit_prev) = ([0.5f32; 2], [0.5f32; 2]); @@ -2898,9 +3086,19 @@ pub fn plan_frame(input: &FrameGeometryInput) -> FrameGeometry { // focus auto) : un plan ne bouge que sous un pointeur qu'on voit. Preview et export // chargent la piste dans tous les cas (le focus auto la suit), cette porte fait le reste. let parallax_track = cursor_for_zoom.filter(|_| scene.is_some_and(|s| s.cursor.show)); - let active_crop = scene.and_then(|scene| { - scene.crop_by_clip.get(scene.active_clip_index).copied().flatten() - }); + // A turned clip is planned in what the viewer sees: its crop (stored in source + // coordinates) is mirrored here, its cursor track is mirrored where it is loaded, and + // only the screen layer's sampling maps back to the texture (`screen_src`). + let screen_turn = scene.is_some_and(|s| s.screen_turned()); + let active_crop = scene + .and_then(|scene| scene.crop_by_clip.get(scene.active_clip_index).copied().flatten()) + .map(|c| { + if screen_turn { + SceneCrop { x: 1.0 - c.x - c.width, y: 1.0 - c.y - c.height, ..c } + } else { + c + } + }); if !zoom_regions.is_empty() { // L'orbite en focus auto lit le curseur dans l'image SOURCE recadrée — pas dans la // coupe zoomée, qu'un focus auto recentre sur lui — et ignore ce qu'une coupe retire. @@ -2919,6 +3117,7 @@ pub fn plan_frame(input: &FrameGeometryInput) -> FrameGeometry { zoom_rotation = zs.rotation; zoom_tilt = zs.tilt; zoom_camera = zs.camera; + zoom_strength = zs.strength; zoom_aim = zs.aim; zoom_orbit = zs.orbit; // Même `CameraFrame` qu'à la frame courante : la caméra réelle d'avant (poids, visée, @@ -2934,6 +3133,7 @@ pub fn plan_frame(input: &FrameGeometryInput) -> FrameGeometry { pp.focus = zs_p.focus; zoom_rotation_prev = zs_p.rotation; zoom_camera_prev = zs_p.camera; + zoom_strength_prev = zs_p.strength; zoom_aim_prev = zs_p.aim; zoom_orbit_prev = zs_p.orbit; } @@ -2945,13 +3145,15 @@ pub fn plan_frame(input: &FrameGeometryInput) -> FrameGeometry { crate::regions::camera_fullscreen_progress_at(cam_regions, source_t_prev, &clock); let shape_fade = crate::regions::camera_fullscreen_shape_at(cam_regions, source_t, &clock); - // rétrécissement réactif : la webcam rétrécit pendant un zoom actif, de la MOITIÉ de ce - // que prendrait 1/zoom : ×0,78 au zoom par défaut (1,8), ×0,6 au zoom maximal (5). En - // 1/zoom elle tombait à ×0,56 au zoom par défaut et un visage ne se lisait plus. Ignoré - // pendant Full Camera (voir ci-dessus). - let reactive_scale = |zoom: f32, progress: f32| -> f32 { - if webcam_reactive && progress <= 0.0 && zoom.is_finite() && zoom > 0.0 { - ((1.0 + 1.0 / zoom) / 2.0).min(1.0) + // rétrécissement réactif : la webcam garde 70 % de sa taille pendant un zoom actif, quel + // que soit son niveau (elle suivait 1/zoom, et rétrécissait donc d'autant plus que le zoom + // était profond : ×0,6 au zoom maximal). L'enveloppe est celle de la région : elle descend + // avec le ease-in du zoom, tient 0,7 sur le span, remonte avec le ease-out. Une région à + // échelle 1 ne zoome rien : elle laisse la caméra à sa taille. Ignoré pendant Full Camera + // (voir ci-dessus). + let reactive_scale = |strength: f32, zoom: f32, progress: f32| -> f32 { + if webcam_reactive && progress <= 0.0 && zoom > 1.0 && strength.is_finite() { + (1.0 - 0.3 * strength.clamp(0.0, 1.0)).clamp(0.7, 1.0) } else { 1.0 } @@ -2965,8 +3167,9 @@ pub fn plan_frame(input: &FrameGeometryInput) -> FrameGeometry { // Seul `reactive_scale` (rétrécissement pendant un zoom, une valeur ANIMÉE par frame que // le rect statique de l'app ne capture pas) doit encore s'appliquer dans ce cas. let base_size_scale = if app_webcam_rect.is_some() { 1.0 } else { lp.webcam_size_scale }; - let webcam_size_scale = base_size_scale * reactive_scale(p.zoom, cam_progress); - let webcam_size_scale_prev = base_size_scale * reactive_scale(pp.zoom, cam_progress_prev); + let webcam_size_scale = base_size_scale * reactive_scale(zoom_strength, p.zoom, cam_progress); + let webcam_size_scale_prev = + base_size_scale * reactive_scale(zoom_strength_prev, pp.zoom, cam_progress_prev); // padding : échelle globale du layout autour du centre du cadre (parité web frameRenderer : // paddingScale = 1 - padding*0.4 → padding 0 = plein cadre). S'applique à TOUS les presets : @@ -3387,6 +3590,8 @@ pub fn plan_frame(input: &FrameGeometryInput) -> FrameGeometry { shape_fade, window_frame, screen_mask, + screen_turn, + screen_uv_max: [u_max, v_max], } } @@ -3419,6 +3624,11 @@ pub struct CursorPlan { /// L'impact des clics récents sur l'écran (mode 16), à dessiner SOUS le curseur. Vide sans /// curseur modélisé, sans clic récent ou à `clickBounce` nul. pub impacts: Vec, + /// Le curseur est de verre — le cristal de Prism Glow ou la lentille de Glass Lens en 3D, le + /// verre à plat de Glass Lens sans elle (`SceneCursorSprite::glass`) : il réfracte l'image déjà + /// composée, le métrage et ses flous de confidentialité, que le backend lui copie juste avant + /// de le dessiner. Seuls des pixels floutés passent ainsi à travers le verre. + pub glass: bool, } impl CursorPlan { @@ -3550,8 +3760,8 @@ pub fn plan_cursor(g: &FrameGeometry, input: &CursorPlanInput) -> Option, dst: [f32; 4]| -> Option { cxy.and_then(|p| { @@ -3720,6 +3930,10 @@ pub fn plan_cursor(g: &FrameGeometry, input: &CursorPlanInput) -> Option s.sculpt.as_deref().is_some_and(crate::sculpt::refracts), + None => sprite.and_then(|s| s.glass.as_deref()).is_some_and(|n| crate::sculpt::sculpted_shape(n).is_some()), + }, }) } @@ -3756,9 +3970,13 @@ pub fn plan_cursor(g: &FrameGeometry, input: &CursorPlanInput) -> Option Scene { + let mut scene = Scene::from_json(&format!( + r##"{{ + "clips":[{{"screenPath":"/s.mp4","webcamPath":"","sourceStartSec":0,"sourceEndSec":10,"webcamOffsetSec":0,"hasAudio":false}}], + "layout":{{"preset":"no-webcam","webcamSize":1,"webcamShape":"rounded","webcamMirror":false,"webcamPosition":null, + "webcamReactiveZoom":false}}, + "effects":{{"padding":0.5,"blur":false,"shadow":0.5,"roundnessFrac":0.02,"motionBlur":0}}, + "background":{{"kind":"color","color":"#1e1e2e"}}, + "zoomRegions":[], + "cursor":{{"show":true,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":1,"clipToBounds":false,"theme":"default"}}, + "cropByClip":[{crop}], + "output":{{"width":1920,"height":1080,"fps":60}} + }}"## + )) + .expect("turn scene"); + scene.rotation_by_clip = vec![rotation]; + scene + } + + #[test] + fn upright_clip_keeps_screen_src_identity() { + let g = framed_plan(&turn_scene(0, "null")); + assert!(!g.screen_turn); + assert_eq!(g.screen_src(g.cut), g.cut); + // An absent field is upright too. + let mut absent = turn_scene(0, "null"); + absent.rotation_by_clip.clear(); + let g = framed_plan(&absent); + assert!(!g.screen_turn); + assert_eq!(g.screen_src(g.cut), g.cut); + } + + #[test] + fn turned_clip_reverses_the_source_bounds_inside_the_valid_area() { + let g = framed_plan(&turn_scene(180, "null")); + assert!(g.screen_turn); + assert_eq!(g.screen_uv_max, [1920.0 / 1920.0, 1080.0 / 1088.0]); + // The display-space cut is the upright one: only the sampling turns. + assert_eq!(g.cut, framed_plan(&turn_scene(0, "null")).cut); + let [a0, b0, a1, b1] = g.screen_src(g.cut); + assert!(a0 > a1 && b0 > b1, "bounds reversed: {:?}", [a0, b0, a1, b1]); + for v in [a0, a1] { + assert!((0.0..=g.screen_uv_max[0]).contains(&v), "u {v}"); + } + for v in [b0, b1] { + assert!((0.0..=g.screen_uv_max[1]).contains(&v), "v {v}"); + } + // Full frame: the whole valid area, corner to opposite corner. + assert!((a0 - 1.0).abs() < 1e-6 && a1.abs() < 1e-6, "u {a0}..{a1}"); + assert!((b0 - 1080.0 / 1088.0).abs() < 1e-6 && b1.abs() < 1e-6, "v {b0}..{b1}"); + } + + #[test] + fn turned_crop_selects_the_same_source_pixels() { + let crop = r#"{"x":0.1,"y":0.2,"width":0.5,"height":0.3}"#; + let g = framed_plan(&turn_scene(180, crop)); + // Planned in display space: the crop is mirrored, x 0.4..0.9, y 0.5..0.8. + let (um, vm) = (g.screen_uv_max[0], g.screen_uv_max[1]); + assert!((g.cut[0] / um - 0.4).abs() < 1e-5 && (g.cut[2] / um - 0.9).abs() < 1e-5, "cut {:?}", g.cut); + assert!((g.cut[1] / vm - 0.5).abs() < 1e-5 && (g.cut[3] / vm - 0.8).abs() < 1e-5, "cut {:?}", g.cut); + // ...and the texture rect sampled is exactly the stored crop, reversed. + let [a0, b0, a1, b1] = g.screen_src(g.cut); + assert!((a1 / um - 0.1).abs() < 1e-5 && (a0 / um - 0.6).abs() < 1e-5, "u {a0}..{a1}"); + assert!((b1 / vm - 0.2).abs() < 1e-5 && (b0 / vm - 0.5).abs() < 1e-5, "v {b0}..{b1}"); + // The screen box keeps the crop's shape either way. + assert_eq!(g.s_dst, framed_plan(&turn_scene(0, crop)).s_dst); + } + + #[test] + fn screen_trail_rereads_a_turned_clip_turned() { + let g = framed_plan(&turn_scene(180, "null")); + assert_eq!(g.screen_trail_cb(RENDER, false).src, g.screen_src(g.cut)); + let upright = framed_plan(&turn_scene(0, "null")); + assert_eq!(upright.screen_trail_cb(RENDER, false).src, upright.cut); + } + + #[test] + fn cursor_of_a_turned_clip_lands_on_the_turned_content() { + let cfg = crate::config::all().pop().expect("au moins une config"); + let center = |rotation: u16, track: &'static crate::cursor::CursorTrack| -> ([f32; 2], [f32; 4]) { + let scene = turn_scene(rotation, "null"); + let g = framed_plan(&scene); + let mut live = live_params_from_scene(&scene); + live.cursor_model3d = false; + let plan = plan_cursor( + &g, + &CursorPlanInput { + render_px: RENDER, + u_max: 1.0, + v_max: 1080.0 / 1088.0, + cfg: &cfg, + live, + scene: Some(&scene), + track, + t: 1.5, + }, + ) + .expect("cursor in view"); + match plan.placement { + CursorPlacement::Upright { center } => (center, g.s_dst), + _ => panic!("upright screen, flat sprite"), + } + }; + let (x, y) = (0.2f32, 0.3f32); + // Upright: the cursor sits at (x, y) of the screen box. + let (c, d) = center(0, parked_track(x, y)); + let want = [d[0] + x * d[2], d[1] + y * d[3]]; + assert!((c[0] - want[0]).abs() < 1e-5 && (c[1] - want[1]).abs() < 1e-5, "{c:?} vs {want:?}"); + // Turned: the track is turned where it is loaded, so the cursor sits where the turned + // picture shows that source point, (1 - x, 1 - y) of the screen box. + let turned: &'static crate::cursor::CursorTrack = Box::leak(Box::new(parked_track(x, y).turned())); + let (c, d) = center(180, turned); + let want = [d[0] + (1.0 - x) * d[2], d[1] + (1.0 - y) * d[3]]; + assert!((c[0] - want[0]).abs() < 1e-5 && (c[1] - want[1]).abs() < 1e-5, "{c:?} vs {want:?}"); + } + /// Le curseur est sur le métrage : le slot le rogne aussi. #[test] fn the_cursor_is_clipped_to_the_slot() { @@ -6479,7 +6856,7 @@ mod tests { #[test] fn the_reactive_zoom_shrinks_the_camera_toward_its_anchor() { let cfg = crate::config::all().pop().expect("au moins une config"); - // Caméra de 0,2 de côté posée par l'app, marge 0,03 ; zoom ×2 à 1,5 s → échelle 0,75. + // Caméra de 0,2 de côté posée par l'app, marge 0,03 ; zoom ×2 à 1,5 s → échelle 0,7. let camera_at = |anchor: Option<&str>, x: f32, y: f32| { let anchor = anchor.map(|a| format!(r#","webcamAnchor":"{a}""#)).unwrap_or_default(); let layout = format!( @@ -6490,19 +6867,56 @@ mod tests { plan_frame(&golden_input(&scene, &cfg)).w_dst }; for (anchor, x, y, want) in [ - (Some("bottom-right"), 0.77, 0.77, [0.82, 0.82]), + (Some("bottom-right"), 0.77, 0.77, [0.83, 0.83]), (Some("top-left"), 0.03, 0.03, [0.03, 0.03]), - (Some("bottom"), 0.4, 0.77, [0.425, 0.82]), - (Some("left"), 0.03, 0.4, [0.03, 0.425]), - (None, 0.77, 0.77, [0.795, 0.795]), + (Some("bottom"), 0.4, 0.77, [0.43, 0.83]), + (Some("left"), 0.03, 0.4, [0.03, 0.43]), + (None, 0.77, 0.77, [0.80, 0.80]), ] { let w = camera_at(anchor, x, y); let got = [w[0], w[1], w[2], w[3]]; - let want = [want[0], want[1], 0.15, 0.15]; + let want = [want[0], want[1], 0.14, 0.14]; assert!(got.iter().zip(want).all(|(g, w)| (g - w).abs() < 1e-5), "{anchor:?} : {got:?} au lieu de {want:?}"); } } + /// Le rétrécissement réactif vise 70 % QUEL QUE SOIT le niveau de zoom : la caméra garde la + /// même taille à 1,25×, 2× et 5× (elle suivait 1/zoom, et rétrécissait d'autant plus que le + /// zoom était profond). + #[test] + fn the_camera_shrink_is_the_same_at_every_zoom_level() { + let cfg = crate::config::all().pop().expect("au moins une config"); + let camera_at = |scale: &str| { + let layout = r#""webcamReactiveZoom":true,"webcamAnchor":"bottom-right","webcamRect":{"x":0.77,"y":0.77,"width":0.2,"height":0.2}}"#; + let json = zoomed_golden_scene_json() + .replace(r#""webcamReactiveZoom":false}"#, layout) + .replace(r#""scale":2.0"#, scale); + let scene = Scene::from_json(&json).expect("scène"); + plan_frame(&golden_input(&scene, &cfg)).w_dst + }; + for scale in [r#""scale":1.25"#, r#""scale":2.0"#, r#""scale":5.0"#] { + let w = camera_at(scale); + let got = [w[0], w[1], w[2], w[3]]; + let want = [0.83, 0.83, 0.14, 0.14]; + assert!(got.iter().zip(want).all(|(g, w)| (g - w).abs() < 1e-5), "{scale} : {got:?} au lieu de {want:?}"); + } + } + + /// Une région de zoom à échelle 1 ne zoome rien : elle ne rétrécit pas la caméra. + #[test] + fn a_one_x_zoom_region_keeps_the_camera_at_its_size() { + let cfg = crate::config::all().pop().expect("au moins une config"); + let layout = r#""webcamReactiveZoom":true,"webcamAnchor":"bottom-right","webcamRect":{"x":0.77,"y":0.77,"width":0.2,"height":0.2}}"#; + let json = zoomed_golden_scene_json() + .replace(r#""webcamReactiveZoom":false}"#, layout) + .replace(r#""scale":2.0"#, r#""scale":1.0"#); + let scene = Scene::from_json(&json).expect("scène"); + let w = plan_frame(&golden_input(&scene, &cfg)).w_dst; + let got = [w[0], w[1], w[2], w[3]]; + let want = [0.77, 0.77, 0.2, 0.2]; + assert!(got.iter().zip(want).all(|(g, w)| (g - w).abs() < 1e-5), "{got:?} au lieu de {want:?}"); + } + /// La vitesse se mesure dans la coupe VISIBLE, zoom compris : sous un x2, le même geste /// traverse deux fois plus d'écran et penche donc plus le plan (hors saturation). #[test] @@ -6839,6 +7253,87 @@ mod tests { } } + /// Le point `p` est dans le quad convexe `q` (TL, TR, BR, BL, px de sortie). + fn in_convex_quad(q: [[f32; 2]; 4], p: [f32; 2]) -> bool { + let side = |i: usize| { + let (a, b) = (q[i], q[(i + 1) % 4]); + (b[0] - a[0]) * (p[1] - a[1]) - (b[1] - a[1]) * (p[0] - a[0]) + }; + (0..4).all(|i| side(i) >= -1e-3) || (0..4).all(|i| side(i) <= 1e-3) + } + + /// Sous l'orbite, la caméra suit le pointeur et le flou de mouvement étale le contenu vers + /// sa position de la frame d'avant : le masque garde la perspective du contenu (un quad warpé + /// aux côtés parallèles aux siens, pas un rect droit) et couvre le secret aux deux frames. + #[test] + fn a_moving_tilted_mask_keeps_its_perspective() { + let cfg = crate::config::all().pop().expect("au moins une config"); + let json = zoomed_golden_scene_json() + .replace(r#""rotation":"none""#, r#""rotation":"orbit","focusMode":"auto""#); + let scene = Scene::from_json(&json).expect("scène"); + // Le pointeur file vers la droite : l'orbite le suit, la caméra bouge à chaque frame. + let track: &'static crate::cursor::CursorTrack = Box::leak(Box::new(crate::cursor::CursorTrack::new( + (0..=90).map(|i| (i as f32 / 30.0, 0.05 + 0.5 * i as f32 / 90.0, 0.3)).collect(), + vec![], + vec![], + ))); + let g = plan_frame(&FrameGeometryInput { cursor: Some(track), ..golden_input(&scene, &cfg) }); + let render = [1170.0, 658.0]; + let quad = g.screen_tilt([g.s_dst[2] * render[0], g.s_dst[3] * render[1]]).expect("orbite"); + let trail = g.tilt_trail(render).expect("garde : flou de mouvement actif sous l'orbite"); + assert_ne!(trail.corners, quad.corners, "garde : la caméra doit bouger d'une frame à l'autre"); + let a = blur_annotation(""); + let m = g.privacy_mask(&a, render).expect("masque"); + let warp = m.warp.expect("le masque d'un écran incliné qui bouge reste un quad warpé"); + let origin = [m.dst[0] * render[0], m.dst[1] * render[1]]; + let mask = warp.map(|[x, y]| [origin[0] + x, origin[1] + y]); + let centre = [(g.s_dst[0] + g.s_dst[2] * 0.5) * render[0], (g.s_dst[1] + g.s_dst[3] * 0.5) * render[1]]; + let before = crate::regions::TiltedQuad { corners: trail.corners, ..quad }; + for q in [quad, before] { + for (fx, fy) in [(a.x, a.y), (a.x + a.w, a.y), (a.x + a.w, a.y + a.h), (a.x, a.y + a.h)] { + let (px, py) = q.point_px(fx, fy); + let p = [centre[0] + px, centre[1] + py]; + assert!(in_convex_quad(mask, p), "{p:?} hors du masque {mask:?}"); + } + } + let (tl, tr) = (quad.point_px(a.x, a.y), quad.point_px(a.x + a.w, a.y)); + let content = (tr.1 - tl.1) / (tr.0 - tl.0); + assert!(content.abs() > 1e-3, "garde : l'orbite doit incliner le haut du secret ({content})"); + let top = (mask[1][1] - mask[0][1]) / (mask[1][0] - mask[0][0]); + assert!((top - content).abs() < 1e-3, "le haut du masque ({top}) doit suivre celui du contenu ({content})"); + assert!(!m.oval_ok, "un masque élargi à la trace doit refuser l'ovale"); + } + + /// Le métrage que l'éditeur lit avec l'image : le rect zoomé à plat, le quad incliné du mode 8 + /// sous un préset, l'homographie de ses coins sous la caméra réelle. + #[test] + fn the_footage_quad_is_where_masks_land() { + let cfg = crate::config::all().pop().expect("au moins une config"); + let render = [1170.0, 658.0]; + let flat = plan_frame(&golden_input(&zoomed_golden_scene(), &cfg)); + let [x, y, w, h] = flat.s_dst; + assert_eq!( + flat.footage_quad(render), + FootageQuad { corners: [[x, y], [x + w, y], [x + w, y + h], [x, y + h]], projective: false } + ); + let tilted = plan_frame(&golden_input(&tilted_golden_scene(), &cfg)); + let q = tilted.footage_quad(render); + let quad = tilted.screen_tilt([tilted.s_dst[2] * render[0], tilted.s_dst[3] * render[1]]).expect("iso"); + let centre = [ + (tilted.s_dst[0] + tilted.s_dst[2] * 0.5) * render[0], + (tilted.s_dst[1] + tilted.s_dst[3] * 0.5) * render[1], + ]; + for (k, (cx, cy)) in quad.corners.iter().enumerate() { + let want = [(centre[0] + cx) / render[0], (centre[1] + cy) / render[1]]; + assert!((q.corners[k][0] - want[0]).abs() < 1e-5 && (q.corners[k][1] - want[1]).abs() < 1e-5, "coin {k}"); + } + assert!(!q.projective); + let json = zoomed_golden_scene_json() + .replace(r#""rotation":"none""#, r#""rotation":"orbit","focusMode":"auto""#); + let orbit = plan_frame(&golden_input(&Scene::from_json(&json).expect("scène"), &cfg)); + assert!(orbit.footage_quad(render).projective, "la caméra réelle warpe par homographie"); + } + /// Une entrée mesurée sur le cadre de sortie n'est pas concernée par le zoom. #[test] fn a_privacy_mask_in_frame_space_ignores_the_zoom() { @@ -7446,6 +7941,8 @@ mod tests { shape_fade: 0.0, window_frame: None, screen_mask: None, + screen_turn: false, + screen_uv_max: [1.0, 1.0], } } @@ -7791,6 +8288,7 @@ mod tests { hotspot_x: hx, hotspot_y: hy, sculpt: None, + glass: None, }, ); } @@ -8023,6 +8521,7 @@ mod tests { hotspot_x: 0.1947, hotspot_y: 0.0656, sculpt: Some("studio-ink/arrow".into()), + glass: None, }; themed.cursor.cursor_sprites.insert("arrow".into(), themed_arrow.clone()); assert_eq!( @@ -8036,6 +8535,56 @@ mod tests { assert!(model_plan([0.0; 3], &bare, &track, 0.3, true).expect("plan").model.is_none()); } + /// Glass Lens est de verre 3D éteinte aussi : le plan demande la copie de l'image composée + /// sans modèle (le verre à plat), puis avec lui (sa lentille sculptée). Le thème par défaut + /// n'a rien à réfracter. + #[test] + fn the_glass_lens_is_glass_with_and_without_3d() { + let track = still_track(vec![]); + let mut scene = model_scene(); + scene.cursor.theme = "glass-lens".into(); + let arrow = crate::scene::SceneCursorSprite { + path: "glass-lens/arrow.png".into(), + hotspot_x: 0.3045, + hotspot_y: 0.0255, + sculpt: Some("glass-lens/arrow".into()), + glass: Some("glass-lens/arrow".into()), + }; + scene.cursor.cursor_sprites.insert("arrow".into(), arrow); + let flat = model_plan([0.0; 3], &scene, &track, 0.3, false).expect("plan"); + assert!(flat.model.is_none() && flat.glass, "3D éteinte : le verre à plat"); + let modelled = model_plan([0.0; 3], &scene, &track, 0.3, true).expect("plan"); + assert!(modelled.model.is_some() && modelled.glass, "3D : la lentille sculptée"); + let plain = model_plan([0.0; 3], &model_scene(), &track, 0.3, false).expect("plan"); + assert!(!plain.glass); + } + + /// Le verre à plat (mode 19) pose la boîte du modèle sculpté comme le sprite plat pose le + /// sien : son hotspot au point visé, son plus grand côté à `size_px`, ses coins sous la forme + /// du mode 13 et le clip dans `dst_prev`. Il porte le modèle et l'unité qui règle la + /// réfraction ; un nom qui n'est pas un modèle n'a pas de verre. + #[test] + fn the_flat_glass_layer_sits_the_model_box_on_the_hotspot() { + let render = [1920.0, 1080.0]; + let (clip, center) = ([0.1, 0.2, 0.7, 0.6], [0.5, 0.4]); + let place = CursorPlacement::Upright { center }; + let cb = cursor_glass_cb(place, 100.0, "glass-lens/arrow", 0.8, clip, render).expect("modèle"); + let shape = crate::sculpt::sculpted_shape("glass-lens/arrow").unwrap(); + assert_eq!(cb.mode, 19.0); + assert!(cb.needs_models()); + assert_eq!((cb.dst_prev, cb.trail_a[0], cb.color[3]), (clip, shape.sculpt as f32, 0.8)); + let [w, h] = [shape.size[0] * 100.0, shape.size[1] * 100.0]; + assert!((cb.quad_px[0] - w).abs() < 1e-3 && (cb.quad_px[1] - h).abs() < 1e-3); + assert_eq!(cb.fx, [0.0, 0.0, cb.quad_px[0], 0.0]); + assert_eq!(cb.src_prev, [cb.quad_px[0], cb.quad_px[1], 0.0, cb.quad_px[1]]); + let tip = [cb.dst[0] * render[0] + shape.hotspot[0] * w, cb.dst[1] * render[1] + shape.hotspot[1] * h]; + assert!((tip[0] - center[0] * render[0]).abs() < 1e-2, "{tip:?}"); + assert!((tip[1] - center[1] * render[1]).abs() < 1e-2, "{tip:?}"); + assert!((cb.color[2] - 100.0).abs() < 1e-3); + assert!((cb.src[0] - 100.0 / render[0]).abs() < 1e-6 && (cb.src[1] - 100.0 / render[1]).abs() < 1e-6); + assert!(cursor_glass_cb(place, 100.0, "studio-ink/text", 0.8, clip, render).is_none()); + } + /// Les poses d'essai, pour chaque état de `MODEL_STATES` : au repos, posé, tourné. fn model_cases() -> Vec<(String, CursorPlan, SpriteShape, crate::cursor_sdf::CursorSdf)> { let scene = model_scene(); diff --git a/crates/compositor/src/gif_export.rs b/crates/compositor/src/gif_export.rs index 8215704c29..5a8c133881 100644 --- a/crates/compositor/src/gif_export.rs +++ b/crates/compositor/src/gif_export.rs @@ -30,9 +30,11 @@ //! the SAME clip walk the MP4 exporter uses, so clip iteration, speed //! segments and output-time decoder advancement have exactly one //! definition. Per output frame the walk composes, then this module does -//! `Compositor::readback_direct` → palette → optional fused -//! Floyd-Steinberg → `GifWriter::write_frame`. Reports the same `GifStats` -//! shape the MP4 `pipeline::Stats` returns. +//! `Compositor::readback_direct` → palette on the export thread, hands the +//! frame to a `FrameEncoder` worker for the mapping (optional fused +//! Floyd-Steinberg) and LZW, and writes the results back in order with +//! `GifWriter::write_frame`. Reports the same `GifStats` shape the MP4 +//! `pipeline::Stats` returns. //! //! It previously ran its own loop over the live-preview `Player`, stepping //! one SOURCE frame per OUTPUT frame — which made a 30 s/60 fps recording @@ -51,10 +53,9 @@ //! we ship (≤ 480p) and good enough for screen content; the //! requantize-every-30-frames cadence trades a small per-frame //! color drift for keeping the palette adapted to the timeline. -//! - `map_to_indices` — brute-force nearest-color search. The hot loop -//! is 4 reads + 3 muls + 2 adds + 1 compare per pixel, small enough -//! for the compiler to autovectorize; a NeuQuant network lookup -//! would be slower at 256 colors and isn't worth its complexity. +//! - `map_to_indices` — exact nearest-color search over the palette +//! sorted by red (`SortedPalette`), which prunes most of the 256 +//! entries per pixel. //! - `map_to_indices_dithered` — the same search with Floyd-Steinberg //! error diffusion fused into it (alpha left as the readback emitted //! it), two row-buffers so the working set is O(width) per row. Fused @@ -74,14 +75,15 @@ use crate::compositor::Compositor; use crate::config::Cfg; use crate::d3d::Gpu; -use crate::gif_export_control::{with_gif_output, GifExportControl}; +use crate::export_control::{with_staged_output, ExportControl}; use crate::pipeline::{ClipSource, Decoder}; use crate::timeline_walk::walk_composited_timeline; use anyhow::{anyhow, bail, Context, Result}; -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use std::fs::File; use std::io::{BufWriter, Write}; use std::path::Path; +use std::sync::{mpsc, Arc, Mutex}; use std::time::Instant; /// Default output width/height. GIF is 8-bit indexed; smaller frames look @@ -173,7 +175,7 @@ pub fn export_gif( params: &GifExportParams, progress: &mut dyn FnMut(u64), ) -> Result { - export_gif_cancellable(clips, out_path, gpu, comp, cfg, params, progress, &GifExportControl::default()) + export_gif_cancellable(clips, out_path, gpu, comp, cfg, params, progress, &ExportControl::default()) } pub fn export_gif_cancellable( @@ -184,9 +186,9 @@ pub fn export_gif_cancellable( cfg: &Cfg, params: &GifExportParams, progress: &mut dyn FnMut(u64), - control: &GifExportControl, + control: &ExportControl, ) -> Result { - with_gif_output(out_path, control, |file, staging_path| { + with_staged_output(out_path, control, |file, staging_path| { export_gif_inner(clips, staging_path, file, gpu, comp, cfg, params, progress, control) }) } @@ -200,7 +202,7 @@ fn export_gif_inner( cfg: &Cfg, params: &GifExportParams, progress: &mut dyn FnMut(u64), - control: &GifExportControl, + control: &ExportControl, ) -> Result { control.check()?; if clips.is_empty() { @@ -227,18 +229,9 @@ fn export_gif_inner( // delays actually written is kept for `video_duration_s`. let mut total_delay_cs: u64 = 0; - // Pre-allocate the per-frame index buffer. Reused across - // frames so we don't hit the allocator in the hot loop. - let mut indices: Vec = vec![0u8; (width as usize) * (height as usize)]; - // Optional dither error buffer (one signed channel per - // pixel per channel, 3 channels per pixel, 2 rows of state - // for the FS pass). Allocated once; only touched when - // `dither` is true. - let mut err_cur: Vec = vec![0.0f32; (width as usize) * 3]; - let mut err_next: Vec = vec![0.0f32; (width as usize) * 3]; - // Cached palette: rebuilt on a schedule - // (`PALETTE_REQUANTIZE_EVERY`). - let mut palette_rgb: Vec = vec![0u8; PALETTE_COLORS * 3]; + // Cached palette: rebuilt on a schedule (`PALETTE_REQUANTIZE_EVERY`), + // shared with the workers mapping the frames that use it. + let mut palette_rgb: Arc> = Arc::new(vec![0u8; PALETTE_COLORS * 3]); let t0 = Instant::now(); let scene = comp.scene_snapshot(); @@ -260,63 +253,107 @@ fn export_gif_inner( Decoder::open_for_export(&clips[0].screen, gpu)? }); - let frames = unsafe { - walk_composited_timeline( - clips, - gpu, - comp, - cfg, - fps as i32, - &scene, - &mut screen_decs, - &mut webcam_decs, - &mut |frame_index| { - control.check()?; - // CPU readback of the staged RT (RGBA8 tightly-packed, - // `width * height * 4` bytes). The dominant per-frame cost, - // and the reason GIF can't use the MP4 zero-copy sink. - let (rw, rh, rgba) = comp - .readback_direct() - .map_err(|e| anyhow!("export_gif: readback @ frame {frame_index}: {e:#}"))?; - debug_assert_eq!(rw, width); - debug_assert_eq!(rh, height); - - // Refresh the palette on a schedule. Building the histogram - // and running median-cut is O(unique colors) — fast enough at - // 480p on our 30-frame cadence. - if frame_index % PALETTE_REQUANTIZE_EVERY == 0 { - build_palette_median_cut(&rgba, PALETTE_COLORS, &mut palette_rgb); + // Mapping and LZW are over 90 % of a frame's time and need nothing but + // the frame and its palette, so they run on a pool, one frame per + // worker. This thread decodes, composes, reads back, builds the + // palette, and writes the encoded frames back in order. + let workers = std::thread::available_parallelism().map_or(1, |n| n.get()); + let (job_tx, job_rx) = mpsc::sync_channel::(workers); + let job_rx = Mutex::new(job_rx); + let frames = std::thread::scope(|scope| -> Result { + let (done_tx, done_rx) = mpsc::channel::(); + for _ in 0..workers { + let (job_rx, done_tx) = (&job_rx, done_tx.clone()); + scope.spawn(move || { + let mut enc = FrameEncoder::new(width, height, dither); + loop { + // Bound first: a guard in a `while let` scrutinee would be + // held through `encode` and serialise the pool. `recv` + // fails once the walk is over and the sender dropped. + let Ok(job) = job_rx.lock().unwrap().recv() else { break }; + if done_tx.send(enc.encode(job)).is_err() { + break; + } } - - // Quantize (with optional dithering). The dither pass diffuses - // the error against the CHOSEN PALETTE ENTRY, so it has to run - // fused with the index mapping — see `map_to_indices_dithered`. - if dither { - map_to_indices_dithered( - &palette_rgb, - &rgba, - width, - height, - &mut err_cur, - &mut err_next, - &mut indices, - ); - } else { - map_to_indices(&palette_rgb, &rgba, &mut indices); - } - - // Per-frame palette (GIF local palette, written by `write_frame`). - control.check()?; - let delay_cs = frame_delay_cs(frame_index as u64, fps); + }); + } + drop(done_tx); + + // Frames come back out of order; write each once its turn comes. + let mut pending: BTreeMap = BTreeMap::new(); + let in_flight_max = 2 * workers as u64; + let mut written: u64 = 0; + // Writes what is next in line, returns how many frames are written. + let mut write_ready = |pending: &mut BTreeMap| -> Result { + while let Some(f) = pending.remove(&written) { + let delay_cs = frame_delay_cs(written, fps); total_delay_cs += delay_cs as u64; - gw.write_frame(&indices, &palette_rgb, delay_cs, fps)?; - progress(frame_index + 1); - Ok(()) - }, - // GIF has no audio track, so clip boundaries need no work. - &mut |_, _, _, _| control.check(), - )? - }; + gw.write_frame(&f.lzw, &f.palette_rgb, delay_cs)?; + written += 1; + progress(written); + } + Ok(written) + }; + + let frames = unsafe { + walk_composited_timeline( + clips, + gpu, + comp, + cfg, + fps as i32, + &scene, + &mut screen_decs, + &mut webcam_decs, + &mut |frame_index| { + control.check()?; + // CPU readback of the staged RT (RGBA8 tightly-packed, + // `width * height * 4` bytes), and the reason GIF can't + // use the MP4 zero-copy sink. + let (rw, rh, rgba) = comp.readback_direct().map_err(|e| { + anyhow!("export_gif: readback @ frame {frame_index}: {e:#}") + })?; + debug_assert_eq!(rw, width); + debug_assert_eq!(rh, height); + + // Refresh the palette on a schedule. Building the histogram + // and running median-cut is O(unique colors) — fast enough at + // 480p on our 30-frame cadence. + if frame_index % PALETTE_REQUANTIZE_EVERY == 0 { + let mut p = vec![0u8; PALETTE_COLORS * 3]; + build_palette_median_cut(&rgba, PALETTE_COLORS, &mut p); + palette_rgb = Arc::new(p); + } + // Bound what is submitted but not yet written: behind one slow + // frame the rest would pile up in `pending`, each holding its + // encoded buffer. The frame that blocks the window is already + // submitted, so waiting on `done_rx` always makes progress. + while frame_index - write_ready(&mut pending)? >= in_flight_max { + let f = done_rx + .recv() + .map_err(|_| anyhow!("export_gif: encoder worker died"))?; + pending.insert(f.index, f); + } + let job = + FrameJob { index: frame_index, rgba, palette_rgb: palette_rgb.clone() }; + job_tx.send(job).map_err(|_| anyhow!("export_gif: encoder worker died"))?; + for f in done_rx.try_iter() { + pending.insert(f.index, f); + } + write_ready(&mut pending).map(|_| ()) + }, + // GIF has no audio track, so clip boundaries need no work. + &mut |_, _, _, _| control.check(), + )? + }; + drop(job_tx); + while write_ready(&mut pending)? < frames { + control.check()?; + let f = done_rx.recv().map_err(|_| anyhow!("export_gif: encoder worker died"))?; + pending.insert(f.index, f); + } + Ok(frames) + })?; control.check()?; gw.finish()?; @@ -336,6 +373,66 @@ fn export_gif_inner( Ok(GifStats { frames, wall_s, fps: fps_actual, video_duration_s, file_bytes }) } +/// One frame for an encoder worker: its pixels and the palette to map them to. +struct FrameJob { + index: u64, + rgba: Vec, + palette_rgb: Arc>, +} + +/// A frame ready to write: its indices, LZW-encoded, and its palette. +struct EncodedFrame { + index: u64, + lzw: Vec, + palette_rgb: Arc>, +} + +/// A worker's scratch buffers, reused across the frames it encodes. +struct FrameEncoder { + width: u32, + height: u32, + dither: bool, + indices: Vec, + // Two rows of Floyd-Steinberg error, 3 channels per pixel. + err_cur: Vec, + err_next: Vec, +} + +impl FrameEncoder { + fn new(width: u32, height: u32, dither: bool) -> Self { + let w = width as usize; + FrameEncoder { + width, + height, + dither, + indices: vec![0u8; w * height as usize], + err_cur: vec![0.0f32; w * 3], + err_next: vec![0.0f32; w * 3], + } + } + + fn encode(&mut self, job: FrameJob) -> EncodedFrame { + // The dither pass diffuses the error against the CHOSEN palette + // entry, so it runs fused with the mapping — see + // `map_to_indices_dithered`. + if self.dither { + map_to_indices_dithered( + &job.palette_rgb, + &job.rgba, + self.width, + self.height, + &mut self.err_cur, + &mut self.err_next, + &mut self.indices, + ); + } else { + map_to_indices(&job.palette_rgb, &job.rgba, &mut self.indices); + } + let mut lzw = Vec::new(); + lzw_compress(&self.indices, 8, &mut lzw); + EncodedFrame { index: job.index, lzw, palette_rgb: job.palette_rgb } + } +} // ===================================================================== // GIF89a format writer (pure std::io::Write). @@ -406,15 +503,9 @@ impl GifWriter { /// Write one animated frame: Graphics Control Extension (delay /// only — no transparency, no disposal), Image Descriptor, local - /// color table, LZW-compressed index stream. - fn write_frame( - &mut self, - indices: &[u8], - palette_rgb: &[u8], - delay_cs: u16, - _fps: u32, - ) -> Result<()> { - debug_assert_eq!(indices.len(), (self.width as usize) * (self.height as usize)); + /// color table, and `lzw`, the frame's indices as `lzw_compress` + /// encodes them with a minimum code size of 8. + fn write_frame(&mut self, lzw: &[u8], palette_rgb: &[u8], delay_cs: u16) -> Result<()> { debug_assert_eq!(palette_rgb.len(), PALETTE_COLORS * 3); // Graphics Control Extension: delay only. The disposal @@ -450,9 +541,7 @@ impl GifWriter { // sub-blocks of compressed bytes, then a 0x00 terminator. // LZW min code size is 8 for a 256-color palette. self.w.write_all(&[8])?; - let mut compressed: Vec = Vec::new(); - lzw_compress(indices, 8, &mut compressed); - write_sub_blocks(&mut self.w, &compressed)?; + write_sub_blocks(&mut self.w, lzw)?; self.w.write_all(&[0x00])?; // image data terminator Ok(()) } @@ -669,7 +758,13 @@ fn build_palette_median_cut(rgba: &[u8], num_colors: usize, out_palette: &mut [u // frames is the only call site, and the cost (one allocation + // one memcpy from the hashmap) is well under a millisecond at // 480p. - let entries: Vec<([u8; 3], u32)> = histogram.into_iter().collect(); + // + // Sorted, because a `HashMap` iterates in a per-process random + // order and the axis sorts below are stable: equal channel values + // kept that order, so the same frame got a different palette on + // every run and no two exports were byte-identical. + let mut entries: Vec<([u8; 3], u32)> = histogram.into_iter().collect(); + entries.sort_unstable(); // 2. Repeatedly split the bucket with the longest channel // range until we have `num_colors` buckets. The split is @@ -828,66 +923,109 @@ fn channel_range(bucket: &[([u8; 3], u32)], channel: usize) -> u32 { // Nearest-color index mapping. // ===================================================================== // -// Brute-force squared-distance search over 256 palette entries per -// pixel. The inner loop is `4 reads + 3 muls + 2 adds + 1 compare` -// per (pixel × palette entry) — small enough that the compiler -// autovectorizes the pixel loop on x86-64 (the `pow(2)` distance -// rule is fine because we only compare, not sort by it). A -// NeuQuant-network lookup would walk a per-frame tree (≈ 512-node -// path per pixel), which is **slower** than 256 brute-force -// comparisons on modern CPUs with wide SIMD. -// -// Cost on the 854×480 fixture (410 k pixels × 256 entries) is -// ~100 M simple integer ops, well under one frame on a recent -// CPU. If it ever shows up on the bench, the right fix is -// `std::simd` or a hand-written AVX2 inner loop — both in this -// file, no new deps. +// Exact nearest palette entry by squared distance, through +// `SortedPalette`. fn map_to_indices(palette_rgb: &[u8], rgba: &[u8], indices: &mut [u8]) { let npix = indices.len(); debug_assert_eq!(rgba.len(), npix * 4); debug_assert_eq!(palette_rgb.len(), PALETTE_COLORS * 3); - // Pre-transpose the palette into `[r0..r255, g0..g255, b0..b255]` - // form so the inner loop's three channel reads are contiguous - // and the compiler can pack them into SIMD loads. The cost - // is 768 bytes per frame, written once; the alternative - // (interleaved reads with a `* 3` step) costs the same in - // the hot loop and is harder to vectorize. - let mut pr = [0u8; PALETTE_COLORS]; - let mut pg = [0u8; PALETTE_COLORS]; - let mut pb = [0u8; PALETTE_COLORS]; - for (k, chunk) in palette_rgb.chunks_exact(3).enumerate() { - pr[k] = chunk[0]; - pg[k] = chunk[1]; - pb[k] = chunk[2]; - } - + let pal = SortedPalette::new(palette_rgb); + let mut hint = 0; for i in 0..npix { let base = i * 4; - let r = rgba[base] as i32; - let g = rgba[base + 1] as i32; - let b = rgba[base + 2] as i32; - // Branchless nearest. The 256-entry loop body is 3 reads - // + 3 subs + 3 muls + 2 adds + 1 compare + 1 conditional - // store — well within autovectorization budget. The - // (distance, index) packing into a single `i32` was - // tried and dropped: the conditional store didn't - // improve (the compiler vectorizes the simple form - // already). - let mut best_idx: usize = 0; - let mut best_dist: i32 = i32::MAX; - for k in 0..PALETTE_COLORS { - let dr = r - pr[k] as i32; - let dg = g - pg[k] as i32; - let db = b - pb[k] as i32; - let dist = dr * dr + dg * dg + db * db; - if dist < best_dist { - best_dist = dist; - best_idx = k; + // Integer channels: every distance is an integer below 2^24, so the + // f32 search finds exactly what an i32 one would. + let (idx, pos) = pal.nearest( + rgba[base] as f32, + rgba[base + 1] as f32, + rgba[base + 2] as f32, + hint, + ); + indices[i] = idx; + hint = pos; + } +} + +/// The palette sorted by red, for an exact nearest-colour search that skips +/// most of it. +/// +/// A brute-force scan of all 256 entries per pixel was 92 % of a dithered +/// export's wall time (172 ms per 864×480 frame, measured): the arg-min does not +/// autovectorize. Here the scan starts at the pixel's red value and walks out +/// both ways, stopping once the red distance alone exceeds the best full +/// distance, since no entry further along can be closer. +/// +/// Exact, not approximate: same distance formula, and ties go to the lowest +/// palette index, as in the scan it replaces — the exported GIF is +/// byte-identical. +struct SortedPalette { + r: [f32; PALETTE_COLORS], + g: [f32; PALETTE_COLORS], + b: [f32; PALETTE_COLORS], + /// Palette index of each sorted entry. + idx: [u8; PALETTE_COLORS], +} + +impl SortedPalette { + fn new(palette_rgb: &[u8]) -> Self { + let mut order: Vec = (0..PALETTE_COLORS).collect(); + order.sort_by_key(|&k| palette_rgb[k * 3]); + let mut p = SortedPalette { + r: [0.0; PALETTE_COLORS], + g: [0.0; PALETTE_COLORS], + b: [0.0; PALETTE_COLORS], + idx: [0; PALETTE_COLORS], + }; + for (s, &k) in order.iter().enumerate() { + p.r[s] = palette_rgb[k * 3] as f32; + p.g[s] = palette_rgb[k * 3 + 1] as f32; + p.b[s] = palette_rgb[k * 3 + 2] as f32; + p.idx[s] = k as u8; + } + p + } + + /// Nearest entry to `(cr, cg, cb)`, as `(palette index, sorted position)`. + /// `hint` is a sorted position to seed the bound with — the previous + /// pixel's answer, usually the right one again. + fn nearest(&self, cr: f32, cg: f32, cb: f32, hint: usize) -> (u8, usize) { + let dist = |s: usize| { + let dr = cr - self.r[s]; + let dg = cg - self.g[s]; + let db = cb - self.b[s]; + dr * dr + dg * dg + db * db + }; + let mut best = hint; + let mut best_dist = dist(hint); + // Lowest palette index wins a tie, as in a 0..256 scan with `<`. + let consider = |s: usize, best: &mut usize, best_dist: &mut f32| { + let d = dist(s); + if d < *best_dist || (d == *best_dist && self.idx[s] < self.idx[*best]) { + *best_dist = d; + *best = s; } + }; + // `dist >= dr * dr` holds in f32 too (adding non-negatives never + // rounds below an operand), so stopping on `>` loses no entry, + // tied ones included. + let start = self.r.partition_point(|&r| r < cr); + for s in start..PALETTE_COLORS { + let dr = self.r[s] - cr; + if dr * dr > best_dist { + break; + } + consider(s, &mut best, &mut best_dist); + } + for s in (0..start).rev() { + let dr = cr - self.r[s]; + if dr * dr > best_dist { + break; + } + consider(s, &mut best, &mut best_dist); } - indices[i] = best_idx as u8; + (self.idx[best], best) } } @@ -918,6 +1056,8 @@ fn map_to_indices_dithered( err_cur.fill(0.0); err_next.fill(0.0); + let pal = SortedPalette::new(palette_rgb); + let mut hint = 0; for y in 0..h { for x in 0..w { @@ -928,19 +1068,10 @@ fn map_to_indices_dithered( let cg = (rgba[base + 1] as f32 + err_cur[e + 1]).clamp(0.0, 255.0); let cb = (rgba[base + 2] as f32 + err_cur[e + 2]).clamp(0.0, 255.0); - let mut best_idx = 0usize; - let mut best_dist = f32::MAX; - for k in 0..PALETTE_COLORS { - let dr = cr - palette_rgb[k * 3] as f32; - let dg = cg - palette_rgb[k * 3 + 1] as f32; - let db = cb - palette_rgb[k * 3 + 2] as f32; - let dist = dr * dr + dg * dg + db * db; - if dist < best_dist { - best_dist = dist; - best_idx = k; - } - } - indices[y * w + x] = best_idx as u8; + let (idx, pos) = pal.nearest(cr, cg, cb, hint); + hint = pos; + let best_idx = idx as usize; + indices[y * w + x] = idx; // THE error: distance to the colour actually written. let er = cr - palette_rgb[best_idx * 3] as f32; @@ -1050,8 +1181,9 @@ mod tests { gw.write_header().unwrap(); gw.write_netscape_loop(0).unwrap(); let palette = vec![0u8; PALETTE_COLORS * 3]; - let indices = vec![0u8; 4]; - gw.write_frame(&indices, &palette, 10, 12).unwrap(); + let mut lzw = Vec::new(); + lzw_compress(&[0u8; 4], 8, &mut lzw); + gw.write_frame(&lzw, &palette, 10).unwrap(); gw.finish().unwrap(); } // Magic. @@ -1271,7 +1403,9 @@ mod tests { let mut gw = GifWriter::new(&mut buf, w as u16, h as u16).unwrap(); gw.write_header().unwrap(); gw.write_netscape_loop(0).unwrap(); - gw.write_frame(&indices, &palette, 8, 12).unwrap(); + let mut lzw = Vec::new(); + lzw_compress(&indices, 8, &mut lzw); + gw.write_frame(&lzw, &palette, 8).unwrap(); gw.finish().unwrap(); } @@ -1308,6 +1442,28 @@ mod tests { assert!(has_blue, "median-cut dropped blue"); } + /// The same frame must give the same palette, so an export is reproducible. + /// Many colours share channel values here, which is where the histogram's + /// iteration order used to leak into the splits. + #[test] + fn median_cut_is_deterministic() { + let mut seed = 0x2545_F491u32; + let rgba: Vec = (0..50_000) + .flat_map(|_| { + seed = seed.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + let level = |shift: u32| ((seed >> shift) % 11) as u8 * 25; + [level(8), level(14), level(20), 255] + }) + .collect(); + let mut first = vec![0u8; 256 * 3]; + build_palette_median_cut(&rgba, 256, &mut first); + for _ in 0..8 { + let mut again = vec![0u8; 256 * 3]; + build_palette_median_cut(&rgba, 256, &mut again); + assert_eq!(again, first); + } + } + /// Median-cut on a uniform image (single color) must not /// loop forever and must produce a non-empty palette. #[test] @@ -1425,6 +1581,82 @@ mod tests { ); } + /// The pruned search must pick exactly what a full scan picks, ties to the + /// lowest index included, on both paths. The palette repeats entries (the + /// median cut pads with duplicates) and shares red values, which is where + /// ties and the pruning bound meet. + #[test] + fn sorted_palette_search_matches_a_full_scan() { + let mut seed = 0x9E37_79B9u32; + let mut next = || { + seed = seed.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + (seed >> 24) as u8 + }; + let palette: Vec = (0..PALETTE_COLORS) + .flat_map(|i| match i % 5 { + 0 => [i as u8 & 0xF0, 128, 64], + _ => [next(), next(), next()], + }) + .collect(); + let (w, h) = (97usize, 61usize); + let rgba: Vec = (0..w * h).flat_map(|_| [next(), next(), next(), 255]).collect(); + + let full_scan = |c: [f32; 3]| -> u8 { + let mut best = (f32::MAX, 0usize); + for k in 0..PALETTE_COLORS { + let d: Vec = (0..3).map(|j| c[j] - palette[k * 3 + j] as f32).collect(); + let dist = d[0] * d[0] + d[1] * d[1] + d[2] * d[2]; + if dist < best.0 { + best = (dist, k); + } + } + best.1 as u8 + }; + + let mut plain = vec![0u8; w * h]; + map_to_indices(&palette, &rgba, &mut plain); + for (i, px) in rgba.chunks_exact(4).enumerate() { + let k = full_scan([px[0] as f32, px[1] as f32, px[2] as f32]); + assert_eq!(plain[i], k, "pixel {i}"); + } + + // The dithered path, replayed with a full scan. + let mut dithered = vec![0u8; w * h]; + let (mut err_cur, mut err_next) = (vec![0.0f32; w * 3], vec![0.0f32; w * 3]); + map_to_indices_dithered( + &palette, + &rgba, + w as u32, + h as u32, + &mut err_cur, + &mut err_next, + &mut dithered, + ); + let (mut cur, mut nxt) = (vec![0.0f32; w * 3], vec![0.0f32; w * 3]); + for y in 0..h { + for x in 0..w { + let (base, e) = ((y * w + x) * 4, x * 3); + let c: Vec = + (0..3).map(|j| (rgba[base + j] as f32 + cur[e + j]).clamp(0.0, 255.0)).collect(); + let k = full_scan([c[0], c[1], c[2]]); + assert_eq!(dithered[y * w + x], k, "dithered pixel ({x}, {y})"); + for j in 0..3 { + let err = c[j] - palette[k as usize * 3 + j] as f32; + if x + 1 < w { + cur[e + 3 + j] += err * (7.0 / 16.0); + nxt[e + 3 + j] += err * (1.0 / 16.0); + } + if x > 0 { + nxt[e - 3 + j] += err * (3.0 / 16.0); + } + nxt[e + j] += err * (5.0 / 16.0); + } + } + cur.copy_from_slice(&nxt); + nxt.fill(0.0); + } + } + /// The export dithers unless told not to: without it every gradient wallpaper bands. #[test] fn gif_export_dithers_by_default() { diff --git a/crates/compositor/src/lib.rs b/crates/compositor/src/lib.rs index e26480cc63..b336797050 100644 --- a/crates/compositor/src/lib.rs +++ b/crates/compositor/src/lib.rs @@ -33,16 +33,17 @@ pub mod camera; pub mod config; pub mod cursor; pub mod cursor_sdf; +pub mod export_control; pub mod export_probe; pub mod ffi; pub mod frame_geometry; pub mod gif_export; -pub mod gif_export_control; pub mod regions; // Multiplateforme à dessein : n'utilise que libavformat (liée sur les trois // cibles) et le shim C. Seul Linux l'appelle aujourd'hui, parce que c'est la // seule plateforme dont la capture passe par `MediaRecorder`, mais rien dedans // n'est spécifique à Linux. +pub mod prism_mesh; pub mod remux; pub mod scene; pub mod sculpt; @@ -50,6 +51,8 @@ pub mod sculpt; // `segmentation` ses deux entrées échouent proprement, ce qui garde le reste du crate // indépendant du choix de packaging d'ONNX Runtime. pub mod segmentation; +// Livraison de la preview par textures partagées (Windows) ; la tenue des cases est portable. +pub mod shared_frames; pub mod text_anim; pub mod text_fonts; pub mod text_plate; diff --git a/crates/compositor/src/live.rs b/crates/compositor/src/live.rs index ebcb463887..3229ff4980 100644 --- a/crates/compositor/src/live.rs +++ b/crates/compositor/src/live.rs @@ -27,9 +27,12 @@ use crate::regions::{speed_at, ProgrammeClock}; use crate::scene::Scene; use crate::config::{self, Cfg}; use crate::cursor::CursorTrack; -use crate::d3d::Gpu; +use crate::d3d::{Backend, Gpu}; use crate::frame_geometry::webcam_is_real; use crate::pipeline::Decoder; +#[cfg(windows)] +use crate::shared_frames::SharedRing; +use crate::shared_frames::{SharedFrame, SlotBook}; use crate::timeline_walk::{frame_step, FrameStep, NextFrameTime}; use anyhow::Result; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; @@ -106,9 +109,13 @@ struct PrefetchedClip { /// `should_draw_webcam`). /// /// ponytail: on garde le remplaçant plutôt que de passer `wdec` en `Option`, ce qui -/// toucherait 22 sites dont le pool de décodeurs et la boucle de composition `unsafe`. À faire -/// si quelqu'un mesure que le décodeur inutile coûte (VRAM des pools D3D11VA, une ouverture -/// par clip) — l'avertissement ci-dessous dit enfin à quelle fréquence le cas visible arrive. +/// toucherait 22 sites dont le pool de décodeurs et la boucle de composition `unsafe`. Il est +/// ouvert mais n'est plus jamais avancé ni recherché : la composition reçoit la frame écran à +/// sa place (`Player::webcam_frame`), l'image même qu'il aurait décodée. Le décoder à côté de +/// l'écran coûtait la moitié du débit de décodage, sans caméra, c'est-à-dire le cas courant : +/// une source 4K lue à 2× plafonnait à ~46 frames/s pour 97 que le décodeur seul tient, et la +/// preview prenait des secondes de retard dans les régions de vitesse. Reste à faire si la +/// VRAM des pools D3D11VA ou l'ouverture par clip se mesurent. unsafe fn open_webcam_or_stand_in( screen_path: &str, webcam_path: &str, @@ -152,10 +159,11 @@ unsafe fn open_and_seek_clip( let source_time_sec = source_time_sec.max(0.0); let mut sdec = Decoder::open(screen_path, gpu)?; let (mut wdec, webcam_decoder_is_real) = open_webcam_or_stand_in(screen_path, webcam_path, gpu)?; - let sf = sdec.seek_to(source_time_sec)?; - let mut wf = wdec.seek_to(webcam_seek_time(source_time_sec, webcam_offset_sec))?; - if wf.is_null() { - wf = wdec.seek_to(0.0)?; + let sf = sdec.seek_to_or_last(source_time_sec)?; + // La caméra aussi tient sa dernière image au-delà de sa fin. L'ancien repli `seek_to(0.0)` + // y montrait sa PREMIÈRE image (#990). + if webcam_decoder_is_real { + wdec.seek_to_or_last(webcam_seek_time(source_time_sec, webcam_offset_sec))?; } if sf.is_null() { anyhow::bail!("clip préchargé vide au temps source {source_time_sec:.3}s (screen=\"{screen_path}\")"); @@ -189,17 +197,20 @@ struct PooledClip { unsafe fn seek_pair( sdec: &mut Decoder, wdec: &mut Decoder, + webcam_decoder_is_real: bool, source_time_sec: f64, webcam_offset_sec: f64, ) -> Result { - let sf = sdec.seek_to(source_time_sec)?; + let sf = sdec.seek_to_or_last(source_time_sec)?; if sf.is_null() { return Ok(false); } - let mut wf = wdec.seek_to(webcam_seek_time(source_time_sec, webcam_offset_sec))?; - if wf.is_null() { - wf = wdec.seek_to(0.0)?; + // Le remplaçant n'est jamais lu (`Player::webcam_frame`) : rien à positionner. + if !webcam_decoder_is_real { + return Ok(true); } + // Au-delà de sa fin, la caméra tient sa dernière image, pas la première (#990). + let wf = wdec.seek_to_or_last(webcam_seek_time(source_time_sec, webcam_offset_sec))?; if wf.is_null() { return Ok(false); } @@ -234,7 +245,13 @@ unsafe fn swap_clip_pooled( let mut pooled = pool.remove(i); // Reseek les décodeurs poolés AVANT de les installer. Échec → on les jette et on // ouvre à neuf (chemin connu sûr), jamais une frame vide. - if seek_pair(&mut pooled.clip.sdec, &mut pooled.clip.wdec, t, request.webcam_offset_sec)? { + if seek_pair( + &mut pooled.clip.sdec, + &mut pooled.clip.wdec, + pooled.clip.webcam_decoder_is_real, + t, + request.webcam_offset_sec, + )? { pooled.clip.idx = (t * pooled.clip.sdec.fps()).round().max(0.0) as u32; hit = true; pooled.clip @@ -359,7 +376,13 @@ impl Player { // Les DEUX flux doivent avoir une frame : `compose_frame` les échantillonne tous les // deux sans condition, un seul manquant suffit à le faire échouer (d'où le `false` que // `seek_pair` peut rendre → l'appelant retombe sur l'ouverture complète). - if !seek_pair(&mut self.sdec, &mut self.wdec, source_time_sec, self.webcam_offset_sec)? { + if !seek_pair( + &mut self.sdec, + &mut self.wdec, + self.webcam_decoder_is_real, + source_time_sec, + self.webcam_offset_sec, + )? { return Ok(false); } self.idx = (source_time_sec * self.sdec.fps()).round().max(0.0) as u32; @@ -421,6 +444,17 @@ impl Player { self.webcam_decoder_is_real } + /// La frame webcam à composer avec la frame écran `screen` : celle de la caméra, ou sans + /// caméra la frame écran elle-même, que le remplaçant aurait décodée à l'identique et que + /// la composition ne dessine pas (`should_draw_webcam`). + unsafe fn webcam_frame(&self, screen: *mut crate::ffi::AVFrame) -> *mut crate::ffi::AVFrame { + if self.webcam_decoder_is_real { + self.wdec.cur_frame() + } else { + screen + } + } + /// Temps source courant du décodeur écran — utilisé par `render_thread` pour détecter le /// franchissement de la fin de fenêtre du clip actif pendant la lecture libre, et pour /// calculer la cible de `step` en lecture libre. `pub` (pas `pub(crate)`) : le harnais @@ -429,6 +463,12 @@ impl Player { self.sdec.cur_time_sec() } + /// Temps source courant du décodeur webcam, le remplaçant compris : sans caméra il ne doit + /// jamais bouger (`tests/no_camera_stand_in.rs`). + pub unsafe fn webcam_time_sec(&self) -> f64 { + self.wdec.cur_time_sec() + } + /// Recalcule l'horloge programme pour le clip `clip_index` de `scene` (la scène COMPLÈTE, /// pas la fenêtre d'un clip : il faut la durée de sortie des clips précédents). /// @@ -450,6 +490,25 @@ impl Player { self.programme.as_ref().map(|c| c.at(t) as f32) } + /// La lecture libre est-elle au bout de la source écran ? Oui quand la frame courante, + /// composée, atteint `end_sec` (la fin de la fenêtre du clip), quand la suivante, due d'ici + /// `target_source_time`, tomberait hors de la fenêtre (`step` composerait une frame coupée), + /// ou qu'aucune autre ne la suit dans le fichier : `step` y reboucle au début. + unsafe fn reached_end(&mut self, end_sec: f64, target_source_time: f64) -> Result { + // Une frame repositionnée (seek, bascule de clip) attend encore d'être composée. + if self.use_current_on_next_step { + return Ok(false); + } + if self.sdec.cur_time_sec() >= end_sec { + return Ok(true); + } + Ok(match self.sdec.peek_next_time_sec()? { + NextFrameTime::At(t) => t >= end_sec && t <= target_source_time, + NextFrameTime::Eof => true, + NextFrameTime::Unknown => false, + }) + } + /// Compose la PROCHAINE frame due (→ `comp.rt`), au plus une, si `target_source_time` /// (temps écran) est atteint. Sémantique de "hold" : `false` sans rien composer quand la /// frame suivante n'est pas encore due — l'appelant garde alors l'image déjà affichée, @@ -509,7 +568,10 @@ impl Player { } let target_webcam_t = (self.sdec.cur_time_sec() - self.webcam_offset_sec).max(0.0); - let wf = if use_current { + let wf = if !self.webcam_decoder_is_real { + // Pas de caméra : le remplaçant n'est pas décodé (cf. `open_webcam_or_stand_in`). + sf + } else if use_current { self.wdec.cur_frame() } else { let cur = self.wdec.cur_frame(); @@ -582,7 +644,7 @@ impl Player { return Ok(false); } let sf = self.sdec.cur_frame(); - let wf = self.wdec.cur_frame(); + let wf = self.webcam_frame(sf); if sf.is_null() || wf.is_null() { return Ok(false); } @@ -598,10 +660,15 @@ impl Player { /// aucun raccourci keyframe pour les seeks avant lointains (lent ET, combiné au bug de /// `set_time`, incorrect au-delà de 6s sur un enregistrement réel). pub unsafe fn present_frame(&mut self, comp: &Compositor, cfg: &Cfg, target_sec: f64) -> Result { - let sf = self.sdec.seek_to(target_sec)?; - let wf = self - .wdec - .seek_to(webcam_seek_time(target_sec, self.webcam_offset_sec))?; + let sf = self.sdec.seek_to_or_last(target_sec)?; + // La caméra aussi, au-delà de sa dernière image comprise : sans frame webcam, la frame + // ne serait pas composée. Sans caméra, la frame écran tient sa place. + let wf = if self.webcam_decoder_is_real { + self.wdec + .seek_to_or_last(webcam_seek_time(target_sec, self.webcam_offset_sec))? + } else { + sf + }; if sf.is_null() || wf.is_null() { self.has_current_frame = false; return Ok(false); @@ -743,6 +810,61 @@ fn resolve_scene_clip_index( } } +/// The cache key of a clip's cursor track. A turned clip uses the same telemetry file +/// mirrored (`CursorTrack::turned`), so the two must never share a cache entry. +pub(crate) fn cursor_key(screen_path: &str, turned: bool) -> String { + let path = format!("{screen_path}.cursor.json"); + if turned { format!("{path}#turned") } else { path } +} + +/// A clip's cursor track as the compositor reads it: mirrored for a turned clip, whose frame is +/// planned in what the viewer sees. +fn oriented_cursor(track: CursorTrack, turned: bool) -> CursorTrack { + if turned { track.turned() } else { track } +} + +/// Rereads the active clip's cursor telemetry when its key changed (another file, or the same +/// file turned the other way) and hands it to the compositor already smoothed, so a paused +/// preview never shows one unsmoothed frame. Returns whether it reread. +fn reload_cursor_if_changed( + comp: &Compositor, + raw_cursor: &mut Option, + loaded_cursor_path: &mut String, + screen_path: &str, + turned: bool, + smoothing: f32, +) -> bool { + let key = cursor_key(screen_path, turned); + if key == *loaded_cursor_path { + return false; + } + let cursor_path = format!("{screen_path}.cursor.json"); + *raw_cursor = CursorTrack::load(&cursor_path, 0.0, 24.0 * 3600.0) + .ok() + .map(|t| oriented_cursor(t, turned)); + // Journalisé ICI seulement : sinon la ligne annonce « loaded=ok » à + // chaque changement de segment alors que rien n'a été relu, et le log + // laisse croire à un travail qui n'a plus lieu. + match raw_cursor { + Some(track) => eprintln!( + "[live] cursor: path={} turned={} loaded=ok samples={}", + cursor_path, + turned, + track.sample_count(), + ), + None => eprintln!("[live] cursor: path={} loaded=FAIL — clear_cursor()", cursor_path), + } + *loaded_cursor_path = key; + // Sans relecture, rien à reposer : un changement de taille garde le + // curseur (`Compositor::resized`), et le relisser parcourt tout + // l'enregistrement à chaque bascule d'un scrub. + match raw_cursor { + Some(track) => comp.set_cursor(track.smoothed(smoothing)), + None => comp.clear_cursor(), + } + true +} + fn scene_for_clip(scene: &Scene, clip_index: usize) -> Scene { match scene.clips.get(clip_index) { Some(clip) => scene.for_clip_window( @@ -756,15 +878,34 @@ fn scene_for_clip(scene: &Scene, clip_index: usize) -> Scene { /// Dernière frame readback vers CPU, prête pour le napi `read_frame`. /// -/// `(gen, w, h, vec)` où `vec.len() == w*h*4` octets RGBA8 tightly-packed (R, G, B, A -/// en mémoire — cf. `Compositor::readback_resized`). `gen` est une génération monotone +/// `(gen, w, h, vec, métrage, position)` où `vec.len() == w*h*4` octets RGBA8 tightly-packed +/// (R, G, B, A en mémoire — cf. `Compositor::readback_resized`). `gen` est une génération monotone /// (≥ 1, `0` réservé à « le consommateur n'a encore rien vu ») incrémentée à CHAQUE /// publication, càd uniquement quand une nouvelle frame a réellement été composée (le /// thread de rendu ne republie pas une frame identique — cf. `stepped || first`). Elle /// est l'IDENTITÉ de la frame : le consommateur (`read_frame`) ne repaie le clone + l'IPC /// que lorsqu'elle change. `None` = "aucune frame composée pour l'instant" (toutes les /// lectures avant la 1re frame composée retournent `None` côté napi, jamais un buffer vide). -type LatestFrame = (u64, u32, u32, Vec); +pub type LatestFrame = ( + u64, + u32, + u32, + Vec, + Option, + FramePosition, +); + +/// Où en est la vue quand elle compose une frame : le clip actif de la scène et le temps +/// source de la frame écran. Voyage avec chaque frame publiée, quel que soit le transport, +/// pour que l'app compare la position RÉELLE de la vue à sa propre tête de lecture au lieu +/// de la deviner à partir de l'horloge murale. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct FramePosition { + /// Index du clip actif dans `scene.clips`. + pub clip_index: u32, + /// pts de la frame écran composée, en secondes du fichier source. + pub source_time_sec: f64, +} /// État partagé thread appelant → thread de rendu (commandes sans blocage). struct Shared { @@ -800,6 +941,15 @@ struct Shared { /// vide la ferait repartir à 1 — donc rejouer des générations déjà peintes. Monotone, /// jamais remise à zéro. frame_gen: AtomicU64, + /// La vue livre ses frames dans l'anneau de textures partagées (`take_shared_frame`) au + /// lieu de les relire en RAM (`latest_frame_since`). Posé par JS ; le thread de rendu le + /// rabat à `false` si l'anneau ne peut pas servir, et les frames repassent par la RAM. + shared_frames: AtomicBool, + /// Tenue des cases de l'anneau, entre le thread de rendu et le thread Node. + slot_book: Mutex, + /// Republier la frame courante au prochain tour, même en pause : posé quand le transport + /// change, pour que le consommateur ait une image sans attendre que quelque chose bouge. + republish: AtomicBool, /// Erreur fatale du thread de rendu (device D3D11 introuvable, décodeur qui refuse /// le fichier…). Le thread meurt sur la première erreur ; sans ce champ, elle /// finissait dans un `eprintln!` que personne ne lit et l'utilisateur n'avait @@ -853,6 +1003,9 @@ impl LiveView { stop: AtomicBool::new(false), latest_frame: Mutex::new(None), frame_gen: AtomicU64::new(0), + shared_frames: AtomicBool::new(false), + slot_book: Mutex::new(SlotBook::default()), + republish: AtomicBool::new(false), fatal: Mutex::new(None), }); let sh = shared.clone(); @@ -894,7 +1047,7 @@ impl LiveView { /// FFI vers le Buffer napi). Le `Vec` retourné a `len() == w*h*4`. /// Préférer `latest_frame_since` sur le chemin chaud : il évite ce clone quand /// le consommateur possède déjà la génération courante. - pub fn latest_frame(&self) -> Option<(u64, u32, u32, Vec)> { + pub fn latest_frame(&self) -> Option { self.shared .latest_frame .lock() @@ -910,7 +1063,7 @@ impl LiveView { /// frame figée — on n'exécute PAS le clone `O(w·h)` : c'est tout l'intérêt du /// compteur. Le consommateur passe la dernière génération qu'il a peinte (`0` au /// départ) ; `None` ⇒ il ne fait rien, `Some` ⇒ il peint et retient `gen`. - pub fn latest_frame_since(&self, since_gen: u64) -> Option<(u64, u32, u32, Vec)> { + pub fn latest_frame_since(&self, since_gen: u64) -> Option { let mut guard = self.shared.latest_frame.lock().ok()?; match guard.as_ref() { // Le buffer est EMPORTÉ, pas copié. Le thread de rendu le remplace à chaque @@ -929,6 +1082,35 @@ impl LiveView { } } + /// Livrer les frames par textures partagées plutôt que par readback. Seul le backend + /// matériel de Windows sait le faire : ailleurs la demande est refusée et la vue continue + /// de relire en RAM. Rend l'état obtenu. + pub fn set_shared_frames(&self, enabled: bool) -> bool { + let on = enabled && cfg!(windows) && Gpu::probe() == Some(Backend::Hardware); + if self.shared.shared_frames.swap(on, Ordering::Relaxed) != on { + if !on { + if let Ok(mut book) = self.shared.slot_book.lock() { + book.clear_ready(); + } + } + self.shared.republish.store(true, Ordering::Relaxed); + } + on + } + + /// La dernière frame posée dans l'anneau, si elle est plus récente que `since_gen`. Sa + /// case reste tenue jusqu'à `release_shared_frame`. `None` aussi quand la vue relit en RAM. + pub fn take_shared_frame(&self, since_gen: u64) -> Option { + self.shared.slot_book.lock().ok()?.take(since_gen, Instant::now()) + } + + /// Chromium a relâché la frame `gen` de la case `slot` : le thread de rendu peut y réécrire. + pub fn release_shared_frame(&self, slot: u32, gen: u64) { + if let Ok(mut book) = self.shared.slot_book.lock() { + book.release(slot, gen); + } + } + /// Switch inspector (booléen). pub fn set_param_bool(&self, key: &str, value: bool) { if let Ok(mut p) = self.shared.inspector.lock() { @@ -1157,12 +1339,20 @@ fn should_settle(now: Instant, settle_until: Option, last_settle: Insta && now.duration_since(last_settle) >= SETTLE_STEP } +/// Le clip que la lecture libre enchaîne après `active`. Aucun après le dernier : le programme +/// s'arrête là, comme la tête de lecture de l'app. Reboucler sur le premier emmenait la vue +/// où l'app ne la suivait pas, et ses seeks suivants, qui ne nomment pas de clip, cherchaient +/// alors dans le fichier du premier (#997). +fn next_clip_index(scene: &Scene, active: usize) -> Option { + (active + 1 < scene.clips.len()).then_some(active + 1) +} + /// Démarre le préchargement du clip suivant sur un thread dédié dès qu'on entre dans la /// fenêtre `PREFETCH_LEAD_SEC` avant la fin du clip actif — pour que la bascule à la /// frontière (`advance_to_next_scene_clip`) trouve les décodeurs déjà ouverts et positionnés /// au lieu de payer l'E/S + le parsing FFmpeg sur le thread de rendu pile au moment de la /// transition (la pause perceptible observée en usage réel). No-op si un préchargement est -/// déjà en cours, ou pour une scène à 1 clip (voir `advance_to_next_scene_clip`). +/// déjà en cours, ou sur le dernier clip (voir `next_clip_index`). unsafe fn maybe_start_prefetch( scene: &Scene, active_clip_index: usize, @@ -1170,9 +1360,12 @@ unsafe fn maybe_start_prefetch( gpu: &Gpu, prefetch: &mut Option, ) { - if scene.clips.len() <= 1 || prefetch.is_some() { + if prefetch.is_some() { return; } + let Some(next_index) = next_clip_index(scene, active_clip_index) else { + return; + }; let Some(clip) = scene.clips.get(active_clip_index) else { return; }; @@ -1180,11 +1373,6 @@ unsafe fn maybe_start_prefetch( if !(0.0..PREFETCH_LEAD_SEC).contains(&remaining) { return; } - let next_index = if active_clip_index + 1 < scene.clips.len() { - active_clip_index + 1 - } else { - 0 - }; let next_clip = scene.clips[next_index].clone(); // Copie légère (COM refcount, pas de nouveau device) — même motif que `Player::open`. let gpu_clone = Gpu { @@ -1212,9 +1400,8 @@ unsafe fn maybe_start_prefetch( *prefetch = Some((next_index, rx)); } -/// Bascule le `Player` + le compositeur sur le clip suivant de `scene` (reboucle sur le -/// premier après le dernier). No-op pour une scène à 1 clip (le bouclage léger existant de -/// `Player::step` suffit et coûte moins cher qu'un `set_active_clip` — reopen des décodeurs). +/// Bascule le `Player` + le compositeur sur le clip suivant de `scene`. No-op sur le dernier +/// clip, que rien ne suit (`next_clip_index`) : la boucle de rendu y tient la dernière image. /// /// Partagée entre le déclenchement PROACTIF (seuil `source_end_sec` franchi) et le filet de /// sécurité RÉACTIF de `render_thread` (le temps du décodeur a reculé — `Player::step` a @@ -1242,13 +1429,8 @@ unsafe fn advance_to_next_scene_clip( loaded_cursor_path: &mut String, last_smoothing: &mut f32, ) { - if scene.clips.len() <= 1 { + let Some(next_index) = next_clip_index(scene, *active_clip_index) else { return; - } - let next_index = if *active_clip_index + 1 < scene.clips.len() { - *active_clip_index + 1 - } else { - 0 }; let next_clip = &scene.clips[next_index]; @@ -1301,20 +1483,24 @@ unsafe fn advance_to_next_scene_clip( *active_webcam_path = next_clip.webcam_path.clone(); *active_webcam_offset_sec = next_clip.webcam_offset_sec; *active_clip_index = next_index; - comp.set_scene(Some(scene_for_clip(scene, *active_clip_index))); + let clip_scene = scene_for_clip(scene, *active_clip_index); + let turned = clip_scene.screen_turned(); + comp.set_scene(Some(clip_scene)); player.set_programme_clock(Some(scene), *active_clip_index); // Réutilise le curseur préchargé s'il est disponible (voir plus haut) — sinon // (préchargement pas encore prêt / raté) on retombe sur la lecture synchrone - // habituelle, comme avant cette optimisation. + // habituelle, comme avant cette optimisation. The prefetched track is the raw + // file: it is turned here, where the clip it belongs to is known. let cursor_path = format!("{}.cursor.json", active_screen_path); *raw_cursor = match prefetched_cursor { Some(track) => track, None => CursorTrack::load(&cursor_path, 0.0, 24.0 * 3600.0).ok(), - }; + } + .map(|t| oriented_cursor(t, turned)); // Le fichier est noté avec la piste. Sinon une requête de clip qui revient sur le // fichier d'avant le croit encore chargé, ne relit rien, et dessine sur ce clip la // télémétrie de celui qu'on vient de quitter. - *loaded_cursor_path = cursor_path; + *loaded_cursor_path = cursor_key(active_screen_path, turned); match raw_cursor { Some(track) => comp.set_cursor(track.smoothed(0.0)), None => comp.clear_cursor(), @@ -1370,7 +1556,8 @@ unsafe fn render_thread( // fichier à chaque changement du slider "smoothing" (voir la boucle plus bas). let mut raw_cursor = CursorTrack::load(cursor_json, 0.0, 24.0 * 3600.0).ok(); /// Chemin de la télémétrie curseur actuellement chargée dans `raw_cursor` — évite de - /// relire le même fichier à chaque changement de segment (voir plus bas). + /// relire le même fichier à chaque changement de segment (voir plus bas). Compared with + /// `cursor_key`: no scene has arrived yet, so this first track is upright. let mut loaded_cursor_path = cursor_json.to_string(); if let Some(track) = &raw_cursor { comp.set_cursor(track.smoothed(0.0)); @@ -1426,8 +1613,19 @@ unsafe fn render_thread( // appliquée, on refuse de jouer le layout fixture (POC) : un fallback fixture ne ferait que // MASQUER un scene-push cassé. On attend la scène avant de produire le 1er frame. let mut scene_applied = false; + // Textures partagées de la preview, créées au premier besoin (voir `publish_shared`). + let mut ring = Ring::default(); + // Une frame composée qui n'a trouvé aucune case libre dans l'anneau : le RT la garde, elle + // repart dès qu'une case se libère. Sans ça, une frame composée en pause (seek, réglage) + // restait invisible jusqu'au changement suivant — en lecture, la suivante la remplace. + let mut publish_pending = false; while !shared.stop.load(Ordering::SeqCst) { + // Le transport vient de changer : le consommateur doit recevoir la frame courante + // sans attendre qu'une autre soit composée (en pause, il n'y en aurait pas). + if shared.republish.swap(false, Ordering::Relaxed) { + first = true; + } // params inspector : booléens/taps → cfg ; valeurs continues → live_params let ip = *shared.inspector.lock().unwrap(); let mut clip_changed = false; @@ -1485,6 +1683,7 @@ unsafe fn render_thread( active_webcam_offset_sec = request.webcam_offset_sec; let scene = shared.scene.lock().unwrap().clone(); full_scene = scene.clone(); + let mut turned = false; if let Some(base_scene) = scene { if let Some(index) = resolve_scene_clip_index( &base_scene, @@ -1500,7 +1699,9 @@ unsafe fn render_thread( active_screen_path, active_webcam_path ); } - comp.set_scene(Some(scene_for_clip(&base_scene, active_clip_index))); + let clip_scene = scene_for_clip(&base_scene, active_clip_index); + turned = clip_scene.screen_turned(); + comp.set_scene(Some(clip_scene)); scene_applied = true; } // Relire la télémétrie curseur seulement si le FICHIER change. Elle était @@ -1508,32 +1709,16 @@ unsafe fn render_thread( // compris quand seul le segment changeait, où le chemin est par // construction identique. Mesuré : 66 bascules sur un scrub de deux clips, // donc 66 relectures du même fichier. - let cursor_path = format!("{}.cursor.json", active_screen_path); - if cursor_path != loaded_cursor_path { - loaded_cursor_path = cursor_path.clone(); - raw_cursor = CursorTrack::load(&cursor_path, 0.0, 24.0 * 3600.0).ok(); - // Journalisé ICI seulement : sinon la ligne annonce « loaded=ok » à - // chaque changement de segment alors que rien n'a été relu, et le log - // laisse croire à un travail qui n'a plus lieu. - match &raw_cursor { - Some(track) => eprintln!( - "[live] cursor: path={} loaded=ok samples={}", - cursor_path, - track.sample_count(), - ), - None => eprintln!( - "[live] cursor: path={} loaded=FAIL — clear_cursor()", - cursor_path, - ), - } - // Sans relecture, rien à reposer : un changement de taille garde le - // curseur (`Compositor::resized`), et le relisser parcourt tout - // l'enregistrement à chaque bascule d'un scrub. - match &raw_cursor { - Some(track) => comp.set_cursor(track.smoothed(0.0)), - None => comp.clear_cursor(), - } - last_smoothing = -1.0; + // The key also carries the clip's turn: the same file turned is another track. + if reload_cursor_if_changed( + &comp, + &mut raw_cursor, + &mut loaded_cursor_path, + &active_screen_path, + turned, + ip.cursor_smoothing, + ) { + last_smoothing = ip.cursor_smoothing; } clip_changed = true; } @@ -1603,7 +1788,20 @@ unsafe fn render_thread( } scene_for_clip(&base_scene, active_clip_index) }); + // Turning the active clip changes no file, only the scene: its track is reread + // here, mirrored the new way. + let turned = scene.as_ref().is_some_and(Scene::screen_turned); comp.set_scene(scene); + if reload_cursor_if_changed( + &comp, + &mut raw_cursor, + &mut loaded_cursor_path, + &active_screen_path, + turned, + ip.cursor_smoothing, + ) { + last_smoothing = ip.cursor_smoothing; + } } // Le temps programme dépend du clip actif ET de la scène entière (durées des clips // précédents) : à recalculer dès que l'un des deux change. @@ -1690,7 +1888,7 @@ unsafe fn render_thread( loop { // Timeline = niveau d'abstraction AU-DESSUS des clips : dès que le décodeur // écran atteint la fin de fenêtre du clip actif, on enchaîne nous-mêmes sur - // le clip suivant (ou on reboucle sur le premier après le dernier) — sans + // le clip suivant (après le dernier, on tient sa dernière image) — sans // dépendre d'un `active_clip_request` poussé par le JS en réaction au // franchissement. Ce round-trip arrivait toujours trop tard : le décodeur // avait déjà dépassé la fin de la fenêtre, voire atteint l'EOF brut du @@ -1708,6 +1906,14 @@ unsafe fn render_thread( &mut prefetch, ); if let Some(clip) = scene.clips.get(active_clip_index) { + // Fin du programme : la vue s'y arrête comme la tête de lecture de l'app, + // au lieu de reboucler sur un clip où l'app ne la suit pas (#997). + if next_clip_index(scene, active_clip_index).is_none() + && player.reached_end(clip.source_end_sec, player.screen_time_sec() + acc)? + { + acc = 0.0; + break; + } if player.screen_time_sec() >= clip.source_end_sec { advance_to_next_scene_clip( &mut player, @@ -1783,39 +1989,56 @@ unsafe fn render_thread( stepped = true; } - if stepped || first { + if stepped || first || publish_pending { if pw > 0 && ph > 0 { - // Step complet : `compose_frame` (déjà appelé par `step`/`present_frame`/ - // `recompose`) a rastérisé le RT à la géométrie de sortie ramenée au panneau. - // On lit ce RT DIRECTEMENT à sa résolution de rendu (`readback_direct` : copy - // rt → staging → Map/Unmap), sans le resize `blit_resized` qui, depuis la - // refonte ratio, n'était plus qu'une copie identité + une alloc NV12 inutile. - match comp.readback_direct() { - Ok((rw, rh, rgba)) => { - // Publie dans `latest_frame` : on remplace le buffer précédent - // (le canvas ne montre que la dernière frame, peu importe combien - // le renderer en a raté entre deux lectures napi). On incrémente - // la génération sous le MÊME lock que l'écriture du buffer, pour - // qu'un lecteur ne puisse jamais voir un `gen` neuf appairé à un - // buffer périmé (ou l'inverse). `+ 1` depuis la précédente, `1` au - // premier publish. Les dims publiées sont celles du RENDU (`rw`×`rh`) : - // le canvas JS s'y dimensionne (packet auto-descriptif) puis CSS met à - // l'échelle vers la boîte du panneau — plus de resize GPU intermédiaire. - // La génération vient d'un compteur atomique et non du slot : la - // livraison sans copie VIDE le slot en le lisant, et un - // `unwrap_or(1)` repartirait alors de 1 — le consommateur recevrait - // des générations déjà peintes et boucherait. Séquence identique à - // l'ancienne dérivation tant que le slot n'est pas vidé. - let next_gen = shared.frame_gen.fetch_add(1, Ordering::Relaxed) + 1; - if let Ok(mut slot) = shared.latest_frame.lock() { - *slot = Some((next_gen, rw, rh, rgba)); - } + let position = FramePosition { + clip_index: active_clip_index as u32, + source_time_sec: player.screen_time_sec(), + }; + match publish_shared(&shared, &gpu, &comp, &mut ring, position) { + SharedPublish::Published => { first = false; + publish_pending = false; } - Err(e) => { - eprintln!("[live] readback_direct: {e:#}"); - std::thread::sleep(Duration::from_millis(8)); + // Chromium tient toutes les cases : la frame attend dans le RT, et la boucle + // ne tourne pas à vide le temps qu'il en relâche une. + SharedPublish::NoFreeSlot => { + publish_pending = true; + std::thread::sleep(Duration::from_millis(2)); } + // Step complet : `compose_frame` (déjà appelé par `step`/`present_frame`/ + // `recompose`) a rastérisé le RT à la géométrie de sortie ramenée au panneau. + // On lit ce RT DIRECTEMENT à sa résolution de rendu (`readback_direct` : copy + // rt → staging → Map/Unmap), sans le resize `blit_resized` qui, depuis la + // refonte ratio, n'était plus qu'une copie identité + une alloc NV12 inutile. + SharedPublish::Off => match comp.readback_direct() { + Ok((rw, rh, rgba)) => { + // Publie dans `latest_frame` : on remplace le buffer précédent + // (le canvas ne montre que la dernière frame, peu importe combien + // le renderer en a raté entre deux lectures napi). On incrémente + // la génération sous le MÊME lock que l'écriture du buffer, pour + // qu'un lecteur ne puisse jamais voir un `gen` neuf appairé à un + // buffer périmé (ou l'inverse). `+ 1` depuis la précédente, `1` au + // premier publish. Les dims publiées sont celles du RENDU (`rw`×`rh`) : + // le canvas JS s'y dimensionne (packet auto-descriptif) puis CSS met à + // l'échelle vers la boîte du panneau — plus de resize GPU intermédiaire. + // La génération vient d'un compteur atomique et non du slot : la + // livraison sans copie VIDE le slot en le lisant, et un + // `unwrap_or(1)` repartirait alors de 1 — le consommateur recevrait + // des générations déjà peintes et boucherait. Séquence identique à + // l'ancienne dérivation tant que le slot n'est pas vidé. + let next_gen = shared.frame_gen.fetch_add(1, Ordering::Relaxed) + 1; + if let Ok(mut slot) = shared.latest_frame.lock() { + *slot = Some((next_gen, rw, rh, rgba, comp.footage_quad(), position)); + } + first = false; + publish_pending = false; + } + Err(e) => { + eprintln!("[live] readback_direct: {e:#}"); + std::thread::sleep(Duration::from_millis(8)); + } + }, } } } else { @@ -1825,6 +2048,101 @@ unsafe fn render_thread( Ok(()) } +/// Ce que la publication dans l'anneau partagé a fait de la frame composée. +enum SharedPublish { + /// Posée dans une case de l'anneau. + Published, + /// Chromium tient toutes les cases : la frame attend dans le RT qu'il en relâche une. + NoFreeSlot, + /// Transport partagé coupé ou indisponible : la frame repasse par le readback. + Off, +} + +#[cfg(windows)] +type Ring = Option; +#[cfg(not(windows))] +type Ring = (); + +/// Pose la frame composée dans l'anneau de textures partagées, si la vue livre ainsi. +/// +/// Toute panne coupe le transport (`shared_frames` à `false`) et rend `Off` : la frame part +/// par le readback, et le service, qui lit les deux, n'a rien à décider. +#[cfg(windows)] +unsafe fn publish_shared( + shared: &Shared, + gpu: &Gpu, + comp: &Compositor, + ring: &mut Ring, + position: FramePosition, +) -> SharedPublish { + if !shared.shared_frames.load(Ordering::Relaxed) { + return SharedPublish::Off; + } + let turn_off = |why: String| { + eprintln!("[live] textures partagées coupées ({why}) — retour au readback"); + shared.shared_frames.store(false, Ordering::Relaxed); + SharedPublish::Off + }; + // WARP ne partage rien avec le device de Chromium. + if gpu.backend != Backend::Hardware { + return turn_off("backend logiciel".into()); + } + if ring.is_none() { + match SharedRing::new(gpu) { + Ok(created) => *ring = Some(created), + Err(e) => return turn_off(format!("{e:#}")), + } + } + let Some(ring) = ring.as_mut() else { + return SharedPublish::Off; + }; + let (claimed, lost) = match shared.slot_book.lock() { + Ok(mut book) => (book.claim(crate::shared_frames::RING_SLOTS), book.lost(Instant::now())), + Err(_) => (None, false), + }; + let Some(slot) = claimed else { + // Chromium relâche d'ordinaire une case dans la milliseconde. Une case perdue ne + // reviendra plus : la réécrire déchirerait peut-être une image qu'il lit encore, et + // l'attendre figerait la preview. + if lost { + return turn_off("une case jamais relâchée par Chromium".into()); + } + return SharedPublish::NoFreeSlot; + }; + let (width, height) = comp.render_size(); + match ring.write(slot, comp.render_target(), width, height) { + Ok(handle) => { + if let Ok(mut book) = shared.slot_book.lock() { + // Le compteur du readback : une seule suite de générations quel que soit le + // transport, incrémentée sous le lock de la publication comme là-bas. + let gen = shared.frame_gen.fetch_add(1, Ordering::Relaxed) + 1; + book.publish(SharedFrame { + gen, + slot, + handle, + width, + height, + footage: comp.footage_quad(), + position, + }); + } + SharedPublish::Published + } + Err(e) => turn_off(format!("{e:#}")), + } +} + +#[cfg(not(windows))] +unsafe fn publish_shared( + _: &Shared, + _: &Gpu, + _: &Compositor, + _: &mut Ring, + _: FramePosition, +) -> SharedPublish { + SharedPublish::Off +} + // ---------- harnais standalone (poc-d3d.exe --live) ---------- // // Le harnais crée une fenêtre Win32 simple qui héberge la preview live ; c'est un @@ -1958,7 +2276,7 @@ pub fn run_standalone(screen: &str, webcam: &str, cursor_json: &str) -> Result<( // standalone n'affiche pas réellement les pixels ici (l'embed Electron est // le consumer réel). On imprime juste une frame de temps en temps pour // confirmer que la chaîne fonctionne. - if let Some((_gen, fw, fh, _pixels)) = view.latest_frame() { + if let Some((_gen, fw, fh, _pixels, _, _)) = view.latest_frame() { if (fw, fh) != (w, h) { // garde-fou : la staging de readback suit `set_rect` côté thread // de rendu, donc ce serait une désynchro transitoire — acceptable. @@ -1997,9 +2315,15 @@ pub fn run_standalone(_screen: &str, _webcam: &str, _cursor_json: &str) -> anyho #[cfg(test)] mod tests { - use super::{open_settle_window, should_settle, SETTLE_STEP, SETTLE_WINDOW}; + use super::{cursor_key, open_settle_window, should_settle, SETTLE_STEP, SETTLE_WINDOW}; use std::time::Instant; + #[test] + fn cursor_key_tells_a_turned_clip_from_an_upright_one() { + assert_eq!(cursor_key("a.mp4", false), "a.mp4.cursor.json"); + assert_ne!(cursor_key("a.mp4", false), cursor_key("a.mp4", true)); + } + /// Le bug d'origine : en pause, un seul recompose par changement, donc le masque de /// segmentation (asynchrone, 3 composes de latence) n'arrivait jamais avant un scrub. #[test] diff --git a/crates/compositor/src/pipeline_linux.rs b/crates/compositor/src/pipeline_linux.rs index b118befdda..d8554a5908 100644 --- a/crates/compositor/src/pipeline_linux.rs +++ b/crates/compositor/src/pipeline_linux.rs @@ -24,15 +24,18 @@ use anyhow::{bail, Result}; use std::collections::HashMap; use std::ffi::CString; +use std::path::Path; use std::ptr; use crate::audio::{ - assemble_concatenated_pcm, build_audio_concat_plan, finish_audio, mix_external_tracks, + assemble_concatenated_pcm, build_audio_concat_plan, finish_audio, mix_click_hits, + mix_external_tracks, AacEncoder, PlanarPcm, }; use crate::audio_jobs::{decode_and_stretch_clip_audio, ClipAudioJobs}; use crate::config::Cfg; use crate::d3d::Gpu; +use crate::export_control::{with_staged_output, ExportControl}; use crate::ffi::AVFrame; use crate::linux_decode::SwDecoder; use crate::timeline_walk::NextFrameTime; @@ -127,6 +130,12 @@ impl Decoder { self.decode_present(idx) } + /// Contrat de `pipeline_windows::Decoder::seek_to_or_last`, que `seek_to` remplit déjà ici : + /// au-delà de la fin, `pump_to_target` tient la dernière image décodée. + pub unsafe fn seek_to_or_last(&mut self, seconds: f64) -> Result<*mut AVFrame> { + self.seek_to(seconds) + } + /// Decode la frame SEQUENTIELLE suivante — pompage `next_frame`, PAS de seek. /// La frame rendue appartient au decodeur (valide jusqu'au prochain appel), /// donc elle ne se libere pas ici, contrairement au chemin `decode_at`. @@ -844,6 +853,39 @@ pub fn run_composited_multi( cfg: &Cfg, params: &ExportParams, progress: &mut dyn FnMut(u64), +) -> Result { + run_composited_multi_cancellable(clips, out, gpu, comp, cfg, params, progress, &ExportControl::default()) +} + +/// `run_composited_multi`, arretable entre deux frames par `control` (erreur `ExportCancelled`). +/// Symetrique de `pipeline_windows::run_composited_multi_cancellable` : le MP4 s'ecrit a cote de +/// `out` et n'est renomme par-dessus qu'une fois complet. Le `Drop` de `Muxer` ferme le fichier +/// sur toutes les sorties, annulation comprise. +pub fn run_composited_multi_cancellable( + clips: &[ClipSource], + out: &str, + gpu: &Gpu, + comp: &crate::compositor::Compositor, + cfg: &Cfg, + params: &ExportParams, + progress: &mut dyn FnMut(u64), + control: &ExportControl, +) -> Result { + with_staged_output(Path::new(out), control, |file, staged| { + drop(file); // ffmpeg le rouvre par son nom + run_multi_inner(clips, &staged.to_string_lossy(), gpu, comp, cfg, params, progress, control) + }) +} + +fn run_multi_inner( + clips: &[ClipSource], + out: &str, + gpu: &Gpu, + comp: &crate::compositor::Compositor, + cfg: &Cfg, + params: &ExportParams, + progress: &mut dyn FnMut(u64), + control: &ExportControl, ) -> Result { if clips.is_empty() { bail!("run_composited_multi: aucun clip a exporter"); @@ -1013,6 +1055,11 @@ pub fn run_composited_multi( .as_ref() .map(|scene| scene.audio_tracks.clone()) .unwrap_or_default(); + // Click hits (the "mouse clicks" toggle), laid on the programme by the app. + let click_sound = scene + .as_ref() + .map(|scene| scene.click_sound.clone()) + .unwrap_or_default(); // Ring de staging a 2 : l'export ne veut que du debit, une frame de latence // ne se voit pas dans un fichier. Voir `Compositor::set_readback_depth` pour // la raison pour laquelle la preview, elle, reste a 1. @@ -1033,6 +1080,7 @@ pub fn run_composited_multi( &mut screen_decs, &mut webcam_decs, &mut |n| { + control.check()?; // Soumet la copie de la frame n SANS l'attendre et recolte la // precedente : c'est tout le pipelining GPU. L'encodage, lui, // n'est plus ici du tout — il tourne sur `worker` pendant que @@ -1202,7 +1250,10 @@ pub fn run_composited_multi( let octx = mux.octx; mux.aac.encode( &finish_audio( - mix_external_tracks(assemble_concatenated_pcm(&clip_pcm, &plan), &audio_tracks), + mix_click_hits( + mix_external_tracks(assemble_concatenated_pcm(&clip_pcm, &plan), &audio_tracks), + &click_sound, + ), audio_settings, ), octx, diff --git a/crates/compositor/src/pipeline_macos.rs b/crates/compositor/src/pipeline_macos.rs index 93827b00ce..638b664f48 100644 --- a/crates/compositor/src/pipeline_macos.rs +++ b/crates/compositor/src/pipeline_macos.rs @@ -30,15 +30,18 @@ //! décodeurs, symétrique. use crate::audio::{ - assemble_concatenated_pcm, build_audio_concat_plan, finish_audio, mix_external_tracks, + assemble_concatenated_pcm, build_audio_concat_plan, finish_audio, mix_click_hits, + mix_external_tracks, AacEncoder, PlanarPcm, }; use crate::audio_jobs::{decode_and_stretch_clip_audio, ClipAudioJobs}; use crate::compositor::Compositor; use crate::d3d::Gpu; +use crate::export_control::{with_staged_output, ExportControl}; use crate::timeline_walk::NextFrameTime; use anyhow::{anyhow, bail, Result}; use std::ffi::{c_void, CString}; +use std::path::Path; use std::ptr; /// Identique à `pipeline_windows::Stats`. Voir la doc là-bas pour la sémantique. @@ -59,6 +62,33 @@ impl Drop for FrameGuard { } } +/// Garde RAII sur le conteneur de sortie : ferme son fichier puis le libère au Drop, `?` compris. +/// Une annulation est une sortie ordinaire de l'export : sans elle, chaque export annulé +/// laisserait derrière lui un descripteur ouvert et le contexte du muxer. +struct OutputGuard(*mut crate::ffi::AVFormatContext); + +impl Drop for OutputGuard { + fn drop(&mut self) { + unsafe { + let mut pb = crate::ffi::sn_fmt_get_pb(self.0); + if !pb.is_null() { + crate::ffi::avio_closep(&mut pb); + crate::ffi::sn_fmt_set_pb(self.0, ptr::null_mut()); + } + crate::ffi::avformat_free_context(self.0); + } + } +} + +/// Garde RAII sur un AVPacket (le libère au Drop). Identique à `pipeline_windows::PacketGuard`. +struct PacketGuard(*mut crate::ffi::AVPacket); + +impl Drop for PacketGuard { + fn drop(&mut self) { + unsafe { crate::ffi::av_packet_free(&mut self.0) }; + } +} + /// Au-delà de cette distance vers l'avant, `Decoder::seek_to` repart d'une image clé /// plutôt que de dérouler. Identique à `pipeline_windows::SEEK_FORWARD_MAX_SEC` — le /// seuil dépend du GOP des captures, pas du backend de décodage. @@ -410,6 +440,25 @@ impl Decoder { } } + /// `seek_to`, mais une cible au-delà de la dernière image se pose sur cette dernière image au + /// lieu de ne rien rendre. L'audio d'un enregistrement dure souvent un peu plus que sa vidéo + /// (de 12 ms à près d'une demi-seconde mesurés) : son clip finit alors après la dernière + /// image, et y entrer par la fin, en scrubant de droite à gauche, demandait une image qui + /// n'existe pas. Le changement de clip échouait sans bruit, et la vue gardait la scène du + /// clip qu'on quittait. + pub unsafe fn seek_to_or_last(&mut self, seconds: f64) -> Result<*mut crate::ffi::AVFrame> { + let frame = self.seek_to(seconds)?; + if !frame.is_null() { + return Ok(frame); + } + // Le seek a décodé jusqu'à l'EOF : `cur_pts` est celui de la dernière image. L'EOF a vidé + // la frame courante, donc il faut le seek complet : `take` écarte le chemin rapide. + match self.cur_pts.take() { + Some(last) => self.seek_to(last as f64 * self.tb_sec()), + None => Ok(frame), + } + } + /// Déroule le décodeur en avant jusqu'à la première frame à `seconds` ou après, SANS /// jeter son état. Symétrique de `pipeline_windows::Decoder::decode_forward_to`. unsafe fn decode_forward_to(&mut self, seconds: f64, tb_sec: f64) -> Result<*mut crate::ffi::AVFrame> { @@ -1119,6 +1168,38 @@ pub fn run_composited_multi( cfg: &crate::config::Cfg, params: &ExportParams, progress: &mut dyn FnMut(u64), +) -> Result { + run_composited_multi_cancellable(clips, out, gpu, comp, cfg, params, progress, &ExportControl::default()) +} + +/// `run_composited_multi`, arrêtable entre deux frames par `control` (erreur `ExportCancelled`). +/// Symétrique de `pipeline_windows::run_composited_multi_cancellable` : le MP4 s'écrit à côté de +/// `out` et n'est renommé par-dessus qu'une fois complet. +pub fn run_composited_multi_cancellable( + clips: &[ClipSource], + out: &str, + gpu: &Gpu, + comp: &crate::compositor::Compositor, + cfg: &crate::config::Cfg, + params: &ExportParams, + progress: &mut dyn FnMut(u64), + control: &ExportControl, +) -> Result { + with_staged_output(Path::new(out), control, |file, staged| { + drop(file); // ffmpeg le rouvre par son nom + run_multi_inner(clips, &staged.to_string_lossy(), gpu, comp, cfg, params, progress, control) + }) +} + +fn run_multi_inner( + clips: &[ClipSource], + out: &str, + gpu: &Gpu, + comp: &crate::compositor::Compositor, + cfg: &crate::config::Cfg, + params: &ExportParams, + progress: &mut dyn FnMut(u64), + control: &ExportControl, ) -> Result { if clips.is_empty() { bail!("run_composited_multi: aucun clip à exporter"); @@ -1161,6 +1242,7 @@ pub fn run_composited_multi( "alloc_output_context2", )?; } + let _output = OutputGuard(octx); let ostream = unsafe { crate::ffi::avformat_new_stream(octx, ptr::null()) }; if ostream.is_null() { bail!("avformat_new_stream"); @@ -1197,7 +1279,8 @@ pub fn run_composited_multi( let mut audio_jobs: ClipAudioJobs> = ClipAudioJobs::new(clips.len()); let mut clip_frame_counts: Vec = vec![0; clips.len()]; - let mut opkt = unsafe { crate::ffi::av_packet_alloc() }; + let opkt = unsafe { crate::ffi::av_packet_alloc() }; + let _opkt = PacketGuard(opkt); // La marche de timeline est PARTAGÉE (`timeline_walk`) : c'est elle qui décide quelle // frame source appartient à quelle frame de sortie, en tenant compte des régions de @@ -1212,6 +1295,11 @@ pub fn run_composited_multi( .as_ref() .map(|scene| scene.audio_tracks.clone()) .unwrap_or_default(); + // Click hits (the "mouse clicks" toggle), laid on the programme by the app. + let click_sound = scene + .as_ref() + .map(|scene| scene.click_sound.clone()) + .unwrap_or_default(); frames = unsafe { crate::timeline_walk::walk_composited_timeline( clips, @@ -1223,6 +1311,7 @@ pub fn run_composited_multi( &mut screen_decs, &mut webcam_decs, &mut |n| { + control.check()?; enc.send_composited(comp, out_w, out_h, n as i64)?; { let _p = crate::export_probe::scope(crate::export_probe::Stage::DrainMux); @@ -1288,7 +1377,10 @@ pub fn run_composited_multi( let plan = build_audio_concat_plan(&clip_frame_counts, &declared_audio, out_fps as f64); audio_encoder.encode( &finish_audio( - mix_external_tracks(assemble_concatenated_pcm(&clip_pcm, &plan), &audio_tracks), + mix_click_hits( + mix_external_tracks(assemble_concatenated_pcm(&clip_pcm, &plan), &audio_tracks), + &click_sound, + ), audio_settings, ), octx, @@ -1298,9 +1390,8 @@ pub fn run_composited_multi( crate::ffi::av_write_trailer(octx), "write_trailer", )?; - crate::ffi::avio_closep(&mut pb); - crate::ffi::avformat_free_context(octx); - crate::ffi::av_packet_free(&mut opkt); + // Fichier fermé et contexte libéré par `_output` en fin de portée, avant que + // `with_staged_output` ne renomme le MP4. } let wall_s = t0.elapsed().as_secs_f64(); diff --git a/crates/compositor/src/pipeline_windows.rs b/crates/compositor/src/pipeline_windows.rs index fabe31207e..28bcc4faa7 100644 --- a/crates/compositor/src/pipeline_windows.rs +++ b/crates/compositor/src/pipeline_windows.rs @@ -3,7 +3,8 @@ //! tout le run, deux lectures seulement. Rien dans la boucle ne peut fausser le fps. use crate::audio::{ - assemble_concatenated_pcm, build_audio_concat_plan, finish_audio, mix_external_tracks, + assemble_concatenated_pcm, build_audio_concat_plan, finish_audio, mix_click_hits, + mix_external_tracks, AacEncoder, PlanarPcm, }; use crate::audio_jobs::{decode_and_stretch_clip_audio, ClipAudioJobs}; @@ -12,6 +13,7 @@ use crate::config::Cfg; use crate::cpu_frames::CpuFrames; use crate::cursor::CursorTrack; use crate::d3d::{Backend, Gpu}; +use crate::export_control::{with_staged_output, ExportControl}; use crate::ffi::*; use crate::regions::{speed_segments_for_window, SpeedSegment}; use crate::scene::Scene; @@ -22,6 +24,7 @@ use crate::timeline_walk::{walk_composited_timeline, NextFrameTime}; use anyhow::{anyhow, bail, Result}; use std::collections::HashMap; use std::ffi::{c_void, CString}; +use std::path::Path; use std::ptr; use std::time::Instant; use windows::core::Interface; @@ -213,6 +216,31 @@ impl Drop for PacketGuard { } } +/// Garde RAII sur le conteneur de sortie : ferme son fichier puis le libère au Drop, `?` compris. +/// Une annulation est une sortie ordinaire, et Windows refuse de supprimer le MP4 partiel tant +/// que ffmpeg le tient ouvert. +struct OutputGuard(*mut AVFormatContext); +impl Drop for OutputGuard { + fn drop(&mut self) { + unsafe { + let mut pb = sn_fmt_get_pb(self.0); + if !pb.is_null() { + avio_closep(&mut pb); + sn_fmt_set_pb(self.0, ptr::null_mut()); + } + avformat_free_context(self.0); + } + } +} + +/// Garde RAII sur un `AVBufferRef` (le désréférence au Drop ; nul accepté). +struct BufferGuard(*mut AVBufferRef); +impl Drop for BufferGuard { + fn drop(&mut self) { + unsafe { av_buffer_unref(&mut self.0) }; + } +} + /// Décode la n-ième frame d'une source sur NOTRE device (textures échantillonnables). /// Sert le harnais de composition (S3+), hors mesure. Retourne une frame indépendante. pub fn decode_frame_n(path: &str, gpu: &Gpu, n: u32) -> Result { @@ -968,6 +996,25 @@ impl Decoder { } } + /// `seek_to`, mais une cible au-delà de la dernière image se pose sur cette dernière image au + /// lieu de ne rien rendre. L'audio d'un enregistrement dure souvent un peu plus que sa vidéo + /// (de 12 ms à près d'une demi-seconde mesurés) : son clip finit alors après la dernière + /// image, et y entrer par la fin, en scrubant de droite à gauche, demandait une image qui + /// n'existe pas. Le changement de clip échouait sans bruit, et la vue gardait la scène du + /// clip qu'on quittait. + pub(crate) unsafe fn seek_to_or_last(&mut self, seconds: f64) -> Result<*mut AVFrame> { + let frame = self.seek_to(seconds)?; + if !frame.is_null() { + return Ok(frame); + } + // Le seek a décodé jusqu'à l'EOF : `cur_pts` est celui de la dernière image. L'EOF a vidé + // la frame courante, donc il faut le seek complet : `take` écarte le chemin rapide. + match self.cur_pts.take() { + Some(last) => self.seek_to(last as f64 * self.tb_sec()), + None => Ok(frame), + } + } + /// Déroule le décodeur en avant jusqu'à la première frame à `seconds` ou après, SANS /// jeter son état. Critère d'arrêt identique à celui du seek complet — c'est ce qui /// garantit que les deux chemins rendent exactement la même frame. @@ -1280,8 +1327,27 @@ pub fn run_composited_multi( params: &ExportParams, progress: &mut dyn FnMut(u64), ) -> Result { - discard_partial_output(out, unsafe { - run_multi_inner(clips, out, gpu, comp, cfg, params, progress) + run_composited_multi_cancellable(clips, out, gpu, comp, cfg, params, progress, &ExportControl::default()) +} + +/// `run_composited_multi`, arrêtable entre deux frames par `control` (erreur `ExportCancelled`). +/// Le MP4 s'écrit à côté de `out` et n'est renommé par-dessus qu'une fois complet : un export +/// annulé ou raté ne laisse aucun partiel, et un `out` existant reste intact. +pub fn run_composited_multi_cancellable( + clips: &[ClipSource], + out: &str, + gpu: &Gpu, + comp: &Compositor, + cfg: &Cfg, + params: &ExportParams, + progress: &mut dyn FnMut(u64), + control: &ExportControl, +) -> Result { + with_staged_output(Path::new(out), control, |file, staged| { + drop(file); // ffmpeg le rouvre par son nom + unsafe { + run_multi_inner(clips, &staged.to_string_lossy(), gpu, comp, cfg, params, progress, control) + } }) } @@ -1622,6 +1688,7 @@ unsafe fn run_multi_inner( cfg: &Cfg, params: &ExportParams, progress: &mut dyn FnMut(u64), + control: &ExportControl, ) -> Result { if clips.is_empty() { bail!("aucun clip à exporter"); @@ -1649,6 +1716,11 @@ unsafe fn run_multi_inner( .as_ref() .map(|scene| scene.audio_tracks.clone()) .unwrap_or_default(); + // Click hits (the "mouse clicks" toggle), laid on the programme by the app. + let click_sound = scene + .as_ref() + .map(|scene| scene.click_sound.clone()) + .unwrap_or_default(); // ---- encodeur (choisi à l'exécution, cf. ExportCodec::candidates) + mux ---- // Backend CPU : pas de pool D3D11 du tout. `av_hwdevice_ctx_init(D3D11VA)` échoue sur @@ -1656,11 +1728,14 @@ unsafe fn run_multi_inner( // écarte alors les candidats zéro-copie et le compositeur alimente l'encodeur en // mémoire système via `send_composited`. let software_frames = gpu.backend == Backend::Cpu; - let (mut enc_hwdev, mut enc_frames) = if software_frames { + let (enc_hwdev, enc_frames) = if software_frames { (ptr::null_mut(), ptr::null_mut()) } else { make_enc_frames(gpu, out_w as i32, out_h as i32)? }; + // Libérés en fin de portée sur toutes les sorties : une annulation qui les laisserait fuir + // coûterait 32 surfaces NV12 à chaque fois. `enc` garde sa propre référence sur le pool. + let _enc_pool = (BufferGuard(enc_frames), BufferGuard(enc_hwdev)); // Le débit vient de l'app, qui le calcule d'après la taille ET la cadence. Le repli // (8Mbps @ 1920x1080 quelle que soit la cadence, plancher 2Mbps) ne sert plus qu'au banc et // aux tests : c'est lui qui affamait un export 1080p60, deux fois plus d'images au même débit. @@ -1683,6 +1758,7 @@ unsafe fn run_multi_inner( avformat_alloc_output_context2(&mut octx, ptr::null(), ptr::null(), outc.as_ptr()), "alloc_output_context2", )?; + let _output = OutputGuard(octx); let ostream = avformat_new_stream(octx, ptr::null()); if ostream.is_null() { bail!("video avformat_new_stream"); @@ -1698,6 +1774,7 @@ unsafe fn run_multi_inner( averr(avformat_write_header(octx, ptr::null_mut()), "write_header")?; let opkt = av_packet_alloc(); + let _opkt = PacketGuard(opkt); let mut clip_frame_counts = vec![0u64; clips.len()]; let mut audio_jobs: ClipAudioJobs> = ClipAudioJobs::new(clips.len()); let t0 = Instant::now(); @@ -1712,6 +1789,7 @@ unsafe fn run_multi_inner( &mut screen_decs, &mut webcam_decs, &mut |frame_index| { + control.check()?; // Backend CPU (WARP) : la frame composée descend en mémoire système via // `send_composited` (le compositeur relit son NV12 interne vers un AVFrame // YUV420P / NV12 et l'encodeur le consomme directement). Pas de hw_frames_ctx, @@ -1785,7 +1863,10 @@ unsafe fn run_multi_inner( out_fps as f64, ); let assembled_audio = finish_audio( - mix_external_tracks(assemble_concatenated_pcm(&clip_pcm, &audio_plan), &audio_tracks), + mix_click_hits( + mix_external_tracks(assemble_concatenated_pcm(&clip_pcm, &audio_plan), &audio_tracks), + &click_sound, + ), audio_settings, ); audio_encoder.encode(&assembled_audio, octx)?; @@ -1793,19 +1874,8 @@ unsafe fn run_multi_inner( averr(av_write_trailer(octx), "write_trailer")?; let wall_s = t0.elapsed().as_secs_f64(); - // teardown (les décodeurs du cache sont droppés en fin de scope). - av_packet_free(&mut (opkt as *mut _)); - let mut pb2 = sn_fmt_get_pb(octx); - if !pb2.is_null() { - avio_closep(&mut pb2); - sn_fmt_set_pb(octx, ptr::null_mut()); - } - avformat_free_context(octx); - // `enc` (donc le contexte encodeur) est libéré par son Drop en fin de portée — après - // ces unref, ce qui est l'ordre voulu : il garde sa propre référence sur le pool. - av_buffer_unref(&mut enc_frames); - av_buffer_unref(&mut enc_hwdev); - + // teardown : les gardes et les Drop (décodeurs, encodeurs) en fin de portée — fichier fermé + // avant que `with_staged_output` ne le renomme. let fps = frames as f64 / wall_s; Ok(Stats { frames, wall_s, fps, video_duration_s: frames as f64 / out_fps as f64 }) } @@ -2097,13 +2167,22 @@ mod tests { codec_args: &[&str], filename: &str, duration_sec: &str, + ) -> std::path::PathBuf { + encode_named_color(codec_args, filename, "red", duration_sec) + } + + fn encode_named_color( + codec_args: &[&str], + filename: &str, + color: &str, + duration_sec: &str, ) -> std::path::PathBuf { let dir = std::env::temp_dir().join(format!("openscreen-554-{}", std::process::id())); std::fs::create_dir_all(&dir).expect("temp dir"); let out = dir.join(filename); let ff = ffmpeg_exe(); let mut cmd = std::process::Command::new(&ff); - let input = format!("color=c=red:s=64x64:d={duration_sec}"); + let input = format!("color=c={color}:s=64x64:d={duration_sec}"); cmd.args(["-y", "-f", "lavfi", "-i", input.as_str()]); cmd.args(codec_args); cmd.arg(&out); @@ -2836,6 +2915,180 @@ mod tests { ); } + #[test] + fn a_seek_past_the_last_frame_can_land_on_it() { + let Some(gpu) = strict_hardware_gpu("a_seek_past_the_last_frame_can_land_on_it") else { + return; + }; + let path = encode_color(&["-c:v", "libopenh264", "-b:v", "200k"], "last-frame.mp4"); + let mut dec = unsafe { Decoder::open(path.to_str().expect("utf8 path"), &gpu) } + .unwrap_or_else(|e| panic!("H.264 Decoder::open: {e:#}")); + + let frame = unsafe { dec.seek_to_or_last(10.0) }.expect("seek past the end"); + assert!(!frame.is_null(), "a target past the end must land on the last frame"); + assert!(!dec.cur_frame().is_null(), "the last frame must be presentable"); + assert!( + unsafe { dec.next() }.expect("decode after the last frame").is_null(), + "the frame it landed on must be the last one" + ); + assert!( + unsafe { dec.seek_to(10.0) }.expect("plain seek past the end").is_null(), + "seek_to itself still reports no frame past the end" + ); + println!("CORE_ASSERTIONS_COMPLETED:a_seek_past_the_last_frame_can_land_on_it"); + } + + /// #990: a clip switch to a time past the webcam's last frame showed the webcam's FIRST + /// frame. Lives here for this module's ffmpeg fixtures and strict hardware GPU. + #[test] + fn a_clip_switch_past_the_webcam_end_holds_its_last_frame() { + let name = "a_clip_switch_past_the_webcam_end_holds_its_last_frame"; + let Some(gpu) = strict_hardware_gpu(name) else { + return; + }; + let h264 = ["-c:v", "libopenh264", "-b:v", "200k"]; + let screen = encode_color_for_duration(&h264, "990-screen.mp4", "1.0"); + let webcam = encode_color_for_duration(&h264, "990-webcam.mp4", "0.4"); + let (screen, webcam) = (screen.to_str().expect("utf8"), webcam.to_str().expect("utf8")); + // Past the webcam's last frame (0.36 s), before the screen's (0.96 s). + let target = 0.8; + + unsafe { + let mut probe = Decoder::open(webcam, &gpu).expect("open the webcam"); + let mut last_webcam_sec = None; + while !probe.next().expect("decode the webcam").is_null() { + last_webcam_sec = Some(probe.cur_time_sec()); + } + let last_webcam_sec = last_webcam_sec.expect("the webcam has frames"); + + let mut player = crate::live::Player::open(screen, webcam, &gpu).expect("player"); + assert!(player.webcam_decoder_is_real()); + + // Same files: `seek_active`, the path of the issue's repro. + assert!(player.seek_active(target).expect("seek_active")); + assert_eq!( + player.webcam_time_sec(), + last_webcam_sec, + "seek_active must land the webcam on its last frame" + ); + + // Files reopened: `set_active_clip`. + player.set_active_clip(screen, webcam, 0.0, target).expect("set_active_clip"); + assert_eq!( + player.webcam_time_sec(), + last_webcam_sec, + "set_active_clip must land the webcam on its last frame" + ); + } + println!("CORE_ASSERTIONS_COMPLETED:{name}"); + } + + /// #997: playback that ran off the end of the programme left the view on the FIRST clip, + /// while the app's playhead stayed on the last. The app's next seeks carry a source time and + /// no clip, so they searched the first clip's file. Drives the real render loop, on clips laid + /// out as in the report: a first clip from its own (red) file, the last two from one (blue). + #[test] + fn playback_off_the_end_of_the_programme_stays_on_the_last_clip() { + let name = "playback_off_the_end_of_the_programme_stays_on_the_last_clip"; + if strict_hardware_gpu(name).is_none() { + return; + } + let h264 = ["-c:v", "libopenh264", "-b:v", "200k"]; + // Long enough that a view which wrapped onto it is still there when the test looks. + let first = encode_named_color(&h264, "997-first.mp4", "red", "3.0"); + let last = encode_named_color(&h264, "997-last.mp4", "blue", "0.6"); + // The paths travel in JSON, and the view matches them against its requests as strings. + let first = first.to_str().expect("utf8").replace('\\', "/"); + let last = last.to_str().expect("utf8").replace('\\', "/"); + let clip = |path: &str, end_sec: f64| { + format!(r#"{{"screenPath":"{path}","webcamPath":"","sourceStartSec":0,"sourceEndSec":{end_sec},"webcamOffsetSec":0,"hasAudio":false}}"#) + }; + + // The last clip ends on its file's last frame (0.56 s, short of the declared 0.6 s) as in + // the report, then on a cut. + for last_end_sec in [0.6, 0.4] { + let scene = format!( + r##"{{"clips":[{},{},{}], + "layout":{{"preset":"no-webcam","webcamSize":1,"webcamShape":"rectangle","webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false}}, + "effects":{{"padding":0,"blur":false,"shadow":0,"roundnessFrac":0,"motionBlur":0}}, + "background":{{"kind":"color","color":"#000000"}}, + "zoomRegions":[], + "cursor":{{"show":false,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":0,"clipToBounds":false,"theme":"default"}}, + "cropByClip":[null,null,null], + "output":{{"width":64,"height":64,"fps":25}}}}"##, + clip(&first, 3.0), + clip(&last, 0.6), + clip(&last, last_end_sec), + ); + let view = crate::live::LiveView::create(64, 64, &first, "", "").expect("view"); + view.set_playing(false); + view.set_scene(&scene); + view.set_active_clip(&last, "", 0.0, 2, 0.1); + let first_frame = wait_for_frame(&view, 0, |p| p.clip_index == 2); + let mut gen = first_frame.0; + let mut held = first_frame.5; + + // Play to the end of the last clip, then about as long again. + view.set_playing(true); + std::thread::sleep(std::time::Duration::from_millis(1000)); + view.set_playing(false); + std::thread::sleep(std::time::Duration::from_millis(150)); + while let Some(frame) = view.latest_frame_since(gen) { + gen = frame.0; + held = frame.5; + } + // The frame held at the end is the clip's own last frame, not the first one its cut + // removed (the 25 fps file has a frame at exactly 0.4 s). + assert!( + held.source_time_sec < last_end_sec, + "end {last_end_sec}: held a frame past the clip's end, at {:.3} s", + held.source_time_sec + ); + // And it did play up to there: a view stalled at its start would pass the bound above. + assert!( + held.source_time_sec > last_end_sec - 0.1, + "end {last_end_sec}: playback did not reach the clip's end, held at {:.3} s", + held.source_time_sec + ); + + // A seek inside the last clip, as the app sends it while paused. + view.set_time(0.2); + let (_, w, h, rgba, _, position) = + wait_for_frame(&view, gen, |p| (p.source_time_sec - 0.2).abs() < 0.03); + let center = (((h / 2) * w + w / 2) * 4) as usize; + let rgb = &rgba[center..center + 3]; + assert!( + i32::from(rgb[2]) - i32::from(rgb[0]) > 100, + "end {last_end_sec}: expected the last clip's blue, got rgb {rgb:?}" + ); + assert_eq!(position.clip_index, 2, "end {last_end_sec}: the view left the last clip"); + } + println!("CORE_ASSERTIONS_COMPLETED:{name}"); + } + + /// The first frame the view publishes after generation `since` whose position is `wanted`. + fn wait_for_frame( + view: &crate::live::LiveView, + mut since: u64, + wanted: impl Fn(&crate::live::FramePosition) -> bool, + ) -> crate::live::LatestFrame { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + loop { + if let Some(frame) = view.latest_frame_since(since) { + if wanted(&frame.5) { + return frame; + } + since = frame.0; + } + assert!( + std::time::Instant::now() < deadline, + "no such frame within 5 s (last gen {since}, fatal: {:?})", + view.fatal_error() + ); + std::thread::sleep(std::time::Duration::from_millis(5)); + } + } + /// Playhead crossing clips is `Decoder::open` of the next source on the /// same `Gpu` (#554). H.264 must stay on D3D11VA after an AV1 software /// decoder has been opened and dropped. diff --git a/crates/compositor/src/prism_mesh.rs b/crates/compositor/src/prism_mesh.rs new file mode 100644 index 0000000000..c045462b04 --- /dev/null +++ b/crates/compositor/src/prism_mesh.rs @@ -0,0 +1,876 @@ +//! Les deux curseurs de Prism Glow en MAILLAGE : les facettes tracées sur l'art 2D +//! (`design/cursors/prism-glow/model`), taillées en cristal. Fichier GÉNÉRÉ par +//! `design/cursors/prism-glow/model/export_compositor.py` : ne pas l'éditer à la main. +//! +//! Repère du modèle du mode 15 : unité = côté du sprite 128 px du thème, origine au hotspot, +//! x à droite, y vers le bas, z vers la caméra, dessus du serti en z = 0. Chaque triangle du +//! cristal tient en quatre float4 : (v0, drapeaux), (e1, r), (e2, g), (normale, b) ; drapeaux = +//! paroi (1) + 2 × arêtes réelles (bit k : l'arête opposée au sommet k). Le fond plat, en +//! z = -épaisseur, est laissé au shader. `POLYS` tient les silhouettes (bord extérieur du +//! serti) puis les contours du cristal (son bord intérieur). `BOXES` range les triangles +//! par voisinage : deux float4 par boîte, (coin bas, premier triangle), (coin haut, nombre). + +/// Un des deux modèles : ses triangles et sa silhouette dans les tables, sa boîte de dessin. +pub struct PrismModel { + pub tri_start: usize, + pub tri_count: usize, + pub sil_start: usize, + pub sil_count: usize, + pub out_start: usize, + pub out_count: usize, + pub box_start: usize, + pub box_count: usize, + /// Largeur de la boîte, hauteur 1 (le sprite), en unités du modèle. + pub size: [f32; 2], + /// Hotspot, fraction de la boîte. + pub hotspot: [f32; 2], + /// Haut de la silhouette, fraction de la hauteur. + pub top: f32, + /// Sommet du cristal au-dessus de z = 0, et épaisseur sous z = 0. + pub height: f32, + pub thick: f32, +} + +/// Arrondi des arêtes du serti, en unités du modèle (`PRISM_BEVEL` des shaders). +pub const BEVEL: f32 = 0.012883; + +/// Flèche puis main, dans l'ordre des identifiants du shader. +pub const MODELS: [PrismModel; 2] = [ + PrismModel { + tri_start: 0, + tri_count: 31, + sil_start: 0, + sil_count: 25, + out_start: 67, + out_count: 9, + box_start: 0, + box_count: 4, + size: [0.63089, 1.0], + hotspot: [0.022658, 0.064647], + top: 0.0625, + height: 0.115951, + thick: 0.042945, + }, + PrismModel { + tri_start: 31, + tri_count: 130, + sil_start: 25, + sil_count: 42, + out_start: 76, + out_count: 34, + box_start: 4, + box_count: 18, + size: [0.797658, 1.0], + hotspot: [0.343388, 0.063582], + top: 0.0625, + height: 0.0822, + thick: 0.043263, + }, +]; + +pub const TRIS: [[f32; 4]; 644] = [ + [0.252301, 0.266258, -0.042945, 11.0], + [-0.221166, -0.203988, 0.0, 0.01033], + [-0.221166, -0.203988, 0.042945, 0.045182], + [0.677984, -0.735077, 0.0, 0.577584], + [0.252301, 0.266258, -0.042945, 3.0], + [-0.221166, -0.203988, 0.042945, 0.01033], + [0.0, 0.0, 0.042945, 0.045182], + [0.677984, -0.735077, 0.0, 0.577584], + [0.031135, 0.06227, -0.042945, 7.0], + [0.004294, 0.419785, 0.042945, 0.01033], + [0.0, 0.0, 0.042945, 0.045182], + [-0.999948, 0.01023, 0.0, 0.577584], + [0.169632, 0.356442, 0.115951, 14.0], + [0.082669, -0.090184, -0.115951, 0.590616], + [-0.138497, -0.294172, -0.115951, 0.737911], + [0.466385, -0.505659, 0.725806, 0.973449], + [0.035429, 0.482055, 0.0, 14.0], + [0.134202, -0.125614, 0.115951, 0.000305], + [-0.004294, -0.419785, 0.0, 0.938689], + [-0.650191, 0.006652, 0.759742, 0.854994], + [0.031135, 0.06227, -0.042945, 9.0], + [0.004294, 0.419785, 0.0, 0.01033], + [0.004294, 0.419785, 0.042945, 0.045182], + [-0.999948, 0.01023, 0.0, 0.577584], + [0.169632, 0.356442, 0.115951, 14.0], + [0.159969, 0.080521, -0.045092, 0.381323], + [0.082669, -0.090184, -0.115951, 0.056123], + [0.461407, -0.510214, 0.725799, 0.973449], + [0.541104, 0.544325, -0.042945, 9.0], + [-0.288804, -0.278068, 0.0, 0.01033], + [-0.288804, -0.278068, 0.042945, 0.045182], + [0.69359, -0.72037, 0.0, 0.577584], + [0.329601, 0.436963, 0.070859, 14.0], + [0.211503, 0.107362, -0.070859, 0.545724], + [-0.077301, -0.170706, -0.070859, 0.064805], + [0.49567, -0.514808, 0.699488, 0.973449], + [0.268405, 0.533589, 0.081595, 14.0], + [0.061196, -0.096626, -0.010736, 0.000916], + [-0.098773, -0.177147, 0.034356, 0.046662], + [0.247834, 0.049458, 0.967539, 0.904662], + [0.541104, 0.544325, -0.042945, 7.0], + [-0.288804, -0.278068, 0.042945, 0.01033], + [0.0, 0.0, 0.042945, 0.045182], + [0.69359, -0.72037, 0.0, 0.577584], + [0.199693, 0.595859, 0.0, 14.0], + [0.068712, -0.06227, 0.081595, 0.0], + [-0.030061, -0.239417, 0.115951, 0.01445], + [-0.504461, 0.426837, 0.750553, 0.254154], + [0.268405, 0.533589, 0.081595, 14.0], + [0.272699, 0.010736, -0.081595, 0.0], + [0.061196, -0.096626, -0.010736, 0.025193], + [0.283244, 0.073137, 0.956255, 0.527108], + [0.035429, 0.482055, 0.0, 14.0], + [0.0, 0.258742, 0.0, 0.0], + [0.134202, -0.125614, 0.115951, 0.473533], + [-0.653777, 0.0, 0.756687, 0.964691], + [0.359663, 0.559356, 0.0, 14.0], + [0.181442, -0.015031, 0.0, 0.006043], + [-0.091258, -0.025767, 0.081595, 0.076188], + [0.076465, 0.923048, 0.37701, 0.964691], + [0.035429, 0.482055, -0.042945, 3.0], + [0.0, 0.258742, 0.042945, 0.01033], + [0.0, 0.0, 0.042945, 0.045182], + [-1.0, 0.0, 0.0, 0.577584], + [0.035429, 0.482055, -0.042945, 11.0], + [0.0, 0.258742, 0.0, 0.01033], + [0.0, 0.258742, 0.042945, 0.045182], + [-1.0, 0.0, 0.0, 0.577584], + [0.035429, 0.740798, -0.042945, 7.0], + [0.164264, -0.144939, 0.042945, 0.01033], + [0.0, 0.0, 0.042945, 0.045182], + [0.661622, 0.749838, 0.0, 0.577584], + [0.035429, 0.740798, 0.0, 14.0], + [0.164264, -0.144939, 0.0, 0.0], + [0.134202, -0.384356, 0.115951, 0.010956], + [0.332572, 0.376915, 0.864483, 0.215854], + [0.035429, 0.740798, -0.042945, 11.0], + [0.164264, -0.144939, 0.0, 0.01033], + [0.164264, -0.144939, 0.042945, 0.045182], + [0.661622, 0.749838, 0.0, 0.577584], + [0.199693, 0.595859, -0.042945, 7.0], + [0.105215, 0.22546, 0.042945, 0.01033], + [0.0, 0.0, 0.042945, 0.045182], + [-0.906183, 0.422886, 0.0, 0.577584], + [0.199693, 0.595859, 0.0, 14.0], + [0.105215, 0.22546, 0.0, 0.0], + [0.068712, -0.06227, 0.081595, 0.015991], + [-0.613882, 0.286478, 0.735581, 0.473533], + [0.199693, 0.595859, -0.042945, 11.0], + [0.105215, 0.22546, 0.0, 0.01033], + [0.105215, 0.22546, 0.042945, 0.045182], + [-0.906183, 0.422886, 0.0, 0.577584], + [0.304908, 0.821319, 0.0, 14.0], + [0.128834, -0.067638, 0.0, 0.0], + [-0.036503, -0.28773, 0.081595, 0.254154], + [0.133686, 0.254641, 0.957751, 0.973449], + [0.304908, 0.821319, -0.042945, 7.0], + [0.128834, -0.067638, 0.042945, 0.01033], + [0.0, 0.0, 0.042945, 0.045182], + [0.464834, 0.885398, 0.0, 0.577584], + [0.433742, 0.753681, 0.0, 14.0], + [-0.07408, -0.194325, 0.0, 0.0], + [-0.165337, -0.220092, 0.081595, 0.964691], + [0.683362, -0.260508, 0.682021, 0.964691], + [0.433742, 0.753681, -0.042945, 11.0], + [-0.07408, -0.194325, 0.0, 0.01033], + [-0.07408, -0.194325, 0.042945, 0.045182], + [0.934406, -0.35621, 0.0, 0.577584], + [0.304908, 0.821319, -0.042945, 11.0], + [0.128834, -0.067638, 0.0, 0.01033], + [0.128834, -0.067638, 0.042945, 0.045182], + [0.464834, 0.885398, 0.0, 0.577584], + [0.433742, 0.753681, -0.042945, 7.0], + [-0.07408, -0.194325, 0.042945, 0.01033], + [0.0, 0.0, 0.042945, 0.045182], + [0.934406, -0.35621, 0.0, 0.577584], + [0.359663, 0.559356, -0.042945, 7.0], + [0.181442, -0.015031, 0.042945, 0.01033], + [0.0, 0.0, 0.042945, 0.045182], + [0.082557, 0.996586, 0.0, 0.577584], + [0.359663, 0.559356, -0.042945, 11.0], + [0.181442, -0.015031, 0.0, 0.01033], + [0.181442, -0.015031, 0.042945, 0.045182], + [0.082557, 0.996586, 0.0, 0.577584], + [0.044345, 0.043263, -0.043263, 11.0], + [-0.043263, -0.005408, 0.0, 0.01033], + [-0.043263, -0.005408, 0.043263, 0.045182], + [0.124035, -0.992278, 0.0, 0.577584], + [0.044345, 0.043263, -0.043263, 7.0], + [-0.043263, -0.005408, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.124035, -0.992278, 0.0, 0.577584], + [0.001082, 0.037855, -0.043263, 7.0], + [-0.044345, 0.040019, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [-0.669964, -0.742393, 0.0, 0.577584], + [0.044345, 0.043263, 0.0, 10.0], + [-0.043263, -0.005408, 0.0, 0.000305], + [-0.040019, 0.030284, 0.049753, 0.964691], + [0.101434, -0.811469, 0.575525, 0.973449], + [0.004326, 0.073548, 0.049753, 14.0], + [-0.003245, -0.035692, -0.049753, 0.0], + [-0.04759, 0.004326, -0.049753, 0.955978], + [-0.580475, -0.643229, 0.499304, 0.896269], + [0.001082, 0.037855, -0.043263, 11.0], + [-0.044345, 0.040019, 0.0, 0.01033], + [-0.044345, 0.040019, 0.043263, 0.045182], + [-0.669964, -0.742393, 0.0, 0.577584], + [-0.031366, 0.229295, 0.058405, 14.0], + [0.035692, -0.155748, -0.008653, 0.000305], + [-0.011897, -0.151422, -0.058405, 0.955978], + [-0.716533, -0.20131, 0.667874, 0.904662], + [-0.043263, 0.077874, -0.043263, 7.0], + [-0.001082, 0.154666, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [-0.999976, -0.006993, 0.0, 0.577584], + [-0.031366, 0.229295, 0.058405, 14.0], + [0.089771, -0.137361, -0.02055, 0.445197], + [0.035692, -0.155748, -0.008653, 0.973449], + [0.216369, -0.004654, 0.976301, 0.973449], + [-0.031366, 0.229295, 0.058405, 14.0], + [-0.011897, -0.151422, -0.058405, 0.000305], + [-0.012979, 0.003245, -0.058405, 0.955978], + [-0.976245, -0.006827, 0.216564, 0.904662], + [-0.043263, 0.077874, -0.043263, 9.0], + [-0.001082, 0.154666, 0.0, 0.01033], + [-0.001082, 0.154666, 0.043263, 0.045182], + [-0.999976, -0.006993, 0.0, 0.577584], + [-0.045426, 0.564586, 0.052998, 14.0], + [0.014061, -0.335291, 0.005408, 0.520989], + [0.001082, -0.332046, -0.052998, 0.088655], + [-0.975941, -0.037462, 0.214794, 0.973449], + [-0.044345, 0.23254, 0.0, 3.0], + [0.0, 0.0, -0.043263, 0.01033], + [-0.001082, 0.332046, -0.043263, 0.045182], + [-0.999995, -0.003257, 0.0, 0.577584], + [-0.163319, 0.380717, -0.043263, 7.0], + [-0.060569, 0.02704, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [-0.407651, -0.913138, 0.0, 0.577584], + [-0.094098, 0.431551, -0.043263, 11.0], + [-0.069221, -0.050834, 0.0, 0.01033], + [-0.069221, -0.050834, 0.043263, 0.045182], + [0.591909, -0.806004, 0.0, 0.577584], + [-0.163319, 0.380717, -0.043263, 11.0], + [-0.060569, 0.02704, 0.0, 0.01033], + [-0.060569, 0.02704, 0.043263, 0.045182], + [-0.407651, -0.913138, 0.0, 0.577584], + [0.075711, 0.08869, -0.043263, 11.0], + [-0.031366, -0.045426, 0.0, 0.01033], + [-0.031366, -0.045426, 0.043263, 0.045182], + [0.822897, -0.568191, 0.0, 0.577584], + [0.044345, 0.043263, 0.0, 6.0], + [-0.040019, 0.030284, 0.049753, 0.000305], + [0.014061, 0.048671, 0.037855, 0.964691], + [0.365607, -0.634949, 0.680566, 0.973449], + [0.075711, 0.08869, -0.043263, 7.0], + [-0.031366, -0.045426, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.822897, -0.568191, 0.0, 0.577584], + [0.058405, 0.091934, 0.037855, 14.0], + [0.017305, -0.003245, -0.037855, 0.0], + [-0.014061, -0.048671, -0.037855, 0.473533], + [0.757369, -0.522945, 0.39105, 0.904662], + [-0.031366, 0.229295, 0.058405, 14.0], + [0.107077, -0.140606, -0.058405, 0.0], + [0.089771, -0.137361, -0.02055, 0.376257], + [0.812052, 0.481346, 0.329966, 0.830777], + [0.077874, 0.407756, -0.043263, 11.0], + [-0.002163, -0.319067, 0.0, 0.01033], + [-0.002163, -0.319067, 0.043263, 0.045182], + [0.999977, -0.006779, 0.0, 0.577584], + [-0.031366, 0.229295, 0.058405, 14.0], + [0.10924, 0.178461, -0.058405, 0.0], + [0.107077, -0.140606, -0.058405, 0.376257], + [0.475583, -0.003224, 0.879665, 0.830777], + [0.147095, 0.272559, -0.043263, 7.0], + [-0.035692, 0.034611, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [-0.696146, -0.7179, 0.0, 0.577584], + [0.147095, 0.272559, -0.043263, 11.0], + [-0.035692, 0.034611, 0.0, 0.01033], + [-0.035692, 0.034611, 0.043263, 0.045182], + [-0.696146, -0.7179, 0.0, 0.577584], + [0.144932, 0.307169, 0.049753, 14.0], + [0.002163, -0.034611, -0.049753, 0.0], + [-0.033529, 0.0, -0.049753, 0.9131], + [-0.630237, -0.649931, 0.424725, 0.904662], + [0.183869, 0.317985, 0.037855, 6.0], + [-0.036774, -0.045426, -0.037855, 0.0], + [-0.038937, -0.010816, 0.011897, 0.806958], + [0.374433, -0.753477, 0.540438, 0.854994], + [0.077874, 0.407756, -0.043263, 7.0], + [-0.002163, -0.319067, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.999977, -0.006779, 0.0, 0.577584], + [0.111403, 0.307169, -0.043263, 7.0], + [-0.002163, 0.131953, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [-0.999866, -0.016391, 0.0, 0.577584], + [0.10924, 0.439122, 0.0, 6.0], + [0.035692, -0.131953, 0.049753, 0.18117], + [0.002163, -0.131953, 0.0, 0.955978], + [-0.82919, -0.013593, 0.558802, 0.964691], + [0.128708, 0.463999, 0.063813, 14.0], + [0.055161, -0.146014, -0.025958, 0.246204], + [0.016224, -0.156829, -0.014061, 0.964691], + [0.305397, -0.053644, 0.950713, 0.973449], + [0.111403, 0.307169, -0.043263, 11.0], + [-0.002163, 0.131953, 0.0, 0.01033], + [-0.002163, 0.131953, 0.043263, 0.045182], + [-0.999866, -0.016391, 0.0, 0.577584], + [0.189277, 0.28013, -0.043263, 11.0], + [-0.042182, -0.007571, 0.0, 0.01033], + [-0.042182, -0.007571, 0.043263, 0.045182], + [0.176664, -0.984271, 0.0, 0.577584], + [0.183869, 0.317985, 0.037855, 10.0], + [0.005408, -0.037855, -0.037855, 0.0], + [-0.036774, -0.045426, -0.037855, 0.806958], + [0.124328, -0.692684, 0.710445, 0.854994], + [0.189277, 0.28013, -0.043263, 7.0], + [-0.042182, -0.007571, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.176664, -0.984271, 0.0, 0.577584], + [0.183869, 0.317985, 0.037855, 14.0], + [0.02704, -0.004326, -0.037855, 0.0], + [0.005408, -0.037855, -0.037855, 0.514916], + [0.70062, -0.452013, 0.552101, 0.846876], + [0.210908, 0.313659, -0.043263, 11.0], + [-0.021632, -0.033529, 0.0, 0.01033], + [-0.021632, -0.033529, 0.043263, 0.045182], + [0.840297, -0.542127, 0.0, 0.577584], + [0.210908, 0.445612, 0.0, 10.0], + [0.0, -0.131953, 0.0, 0.0], + [-0.02704, -0.127627, 0.037855, 0.439658], + [0.813734, 0.0, 0.581238, 0.871366], + [0.210908, 0.313659, -0.043263, 7.0], + [-0.021632, -0.033529, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.840297, -0.542127, 0.0, 0.577584], + [0.210908, 0.445612, -0.043263, 11.0], + [0.0, -0.131953, 0.0, 0.01033], + [0.0, -0.131953, 0.043263, 0.045182], + [1.0, 0.0, 0.0, 0.577584], + [0.241193, 0.343943, -0.043263, 7.0], + [-0.002163, 0.107077, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [-0.999796, -0.020198, 0.0, 0.577584], + [0.276885, 0.311496, -0.043263, 11.0], + [-0.035692, 0.032447, 0.0, 0.01033], + [-0.035692, 0.032447, 0.043263, 0.045182], + [-0.672673, -0.73994, 0.0, 0.577584], + [0.23903, 0.45102, 0.0, 14.0], + [0.042182, -0.108158, 0.046508, 0.000305], + [0.002163, -0.107077, 0.0, 0.846876], + [-0.758098, -0.015315, 0.651961, 0.896269], + [0.276885, 0.311496, -0.043263, 7.0], + [-0.035692, 0.032447, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [-0.672673, -0.73994, 0.0, 0.577584], + [0.281211, 0.342862, 0.046508, 14.0], + [-0.004326, -0.031366, -0.046508, 0.0], + [-0.040019, 0.001082, -0.046508, 0.752941], + [-0.58651, -0.645161, 0.489667, 0.846876], + [0.246601, 0.459672, 0.06165, 14.0], + [0.070303, -0.101669, -0.024876, 0.262257], + [0.034611, -0.116811, -0.015142, 0.973449], + [0.27928, -0.041605, 0.959308, 0.973449], + [0.319067, 0.319067, -0.043263, 11.0], + [-0.042182, -0.007571, 0.0, 0.01033], + [-0.042182, -0.007571, 0.043263, 0.045182], + [0.176664, -0.984271, 0.0, 0.577584], + [0.319067, 0.319067, 0.0, 10.0], + [-0.042182, -0.007571, 0.0, 0.0], + [-0.037855, 0.023795, 0.046508, 0.896269], + [0.1483, -0.826245, 0.54344, 0.896269], + [0.319067, 0.319067, -0.043263, 7.0], + [-0.042182, -0.007571, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.176664, -0.984271, 0.0, 0.577584], + [0.319067, 0.319067, 0.0, 6.0], + [-0.037855, 0.023795, 0.046508, 0.0], + [-0.002163, 0.038937, 0.036774, 0.896269], + [0.437906, -0.604311, 0.665618, 0.896269], + [0.316904, 0.358004, 0.036774, 14.0], + [0.024876, -0.004326, -0.036774, 0.0], + [0.002163, -0.038937, -0.036774, 0.617212], + [0.707338, -0.464191, 0.533104, 0.921584], + [0.34178, 0.353677, -0.043263, 11.0], + [-0.022713, -0.034611, 0.0, 0.01033], + [-0.022713, -0.034611, 0.043263, 0.045182], + [0.836048, -0.548657, 0.0, 0.577584], + [0.340698, 0.474815, 0.0, 14.0], + [0.001082, -0.121137, 0.0, 0.0], + [-0.023795, -0.116811, 0.036774, 0.473533], + [0.828664, 0.007399, 0.559697, 0.930114], + [0.34178, 0.353677, -0.043263, 7.0], + [-0.022713, -0.034611, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.836048, -0.548657, 0.0, 0.577584], + [0.340698, 0.474815, -0.043263, 11.0], + [0.001082, -0.121137, 0.0, 0.01033], + [0.001082, -0.121137, 0.043263, 0.045182], + [0.99996, 0.008928, 0.0, 0.577584], + [0.411001, 0.351514, -0.043263, 11.0], + [-0.034611, 0.030284, 0.0, 0.01033], + [-0.034611, 0.030284, 0.043263, 0.045182], + [-0.658505, -0.752577, 0.0, 0.577584], + [0.411001, 0.351514, -0.043263, 7.0], + [-0.034611, 0.030284, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [-0.658505, -0.752577, 0.0, 0.577584], + [0.412083, 0.38288, 0.043263, 14.0], + [-0.001082, -0.031366, -0.043263, 0.0], + [-0.035692, -0.001082, -0.043263, 0.745403], + [-0.574039, -0.656045, 0.489984, 0.854994], + [0.45102, 0.359085, -0.043263, 11.0], + [-0.040019, -0.007571, 0.0, 0.01033], + [-0.040019, -0.007571, 0.043263, 0.045182], + [0.185892, -0.98257, 0.0, 0.577584], + [0.45102, 0.359085, 0.0, 10.0], + [-0.040019, -0.007571, 0.0, 0.000305], + [-0.038937, 0.023795, 0.043263, 0.879622], + [0.151736, -0.802035, 0.577682, 0.9131], + [0.45102, 0.359085, -0.043263, 7.0], + [-0.040019, -0.007571, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.185892, -0.98257, 0.0, 0.577584], + [0.45102, 0.359085, 0.0, 6.0], + [-0.038937, 0.023795, 0.043263, 0.000305], + [0.003245, 0.0411, 0.032447, 0.879622], + [0.417852, -0.583049, 0.696744, 0.9131], + [0.473733, 0.391533, -0.043263, 11.0], + [-0.022713, -0.032447, 0.0, 0.01033], + [-0.022713, -0.032447, 0.043263, 0.045182], + [0.819232, -0.573462, 0.0, 0.577584], + [0.454264, 0.400185, 0.032447, 14.0], + [0.019468, -0.008653, -0.032447, 0.0], + [-0.003245, -0.0411, -0.032447, 0.527108], + [0.688617, -0.482032, 0.541712, 0.887922], + [0.473733, 0.391533, -0.043263, 7.0], + [-0.022713, -0.032447, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.819232, -0.573462, 0.0, 0.577584], + [-0.223888, 0.407756, -0.043263, 7.0], + [0.002163, 0.06814, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [-0.999496, 0.03173, 0.0, 0.577584], + [-0.223888, 0.407756, -0.043263, 11.0], + [0.002163, 0.06814, 0.0, 0.01033], + [0.002163, 0.06814, 0.043263, 0.045182], + [-0.999496, 0.03173, 0.0, 0.577584], + [-0.167645, 0.416409, 0.046508, 14.0], + [-0.056242, -0.008653, -0.046508, 0.000305], + [-0.054079, 0.059487, -0.046508, 0.955978], + [-0.638985, 0.020285, 0.768952, 0.973449], + [-0.167645, 0.416409, 0.046508, 14.0], + [0.004326, -0.035692, -0.046508, 0.000305], + [-0.056242, -0.008653, -0.046508, 0.955978], + [-0.339782, -0.761112, 0.552501, 0.973449], + [-0.221724, 0.475896, 0.0, 14.0], + [0.139524, 0.055161, 0.043263, 0.520989], + [0.054079, -0.059487, 0.046508, 0.973449], + [-0.39307, 0.317373, 0.863001, 0.973449], + [-0.167645, 0.416409, 0.046508, 6.0], + [0.073548, 0.015142, -0.046508, 0.0], + [0.004326, -0.035692, -0.046508, 0.768154], + [0.490917, -0.668482, 0.558688, 0.871366], + [-0.094098, 0.431551, -0.043263, 7.0], + [-0.069221, -0.050834, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.591909, -0.806004, 0.0, 0.577584], + [-0.167645, 0.416409, 0.046508, 10.0], + [0.085445, 0.114648, -0.003245, 0.0], + [0.073548, 0.015142, -0.046508, 0.768154], + [0.548356, -0.38771, 0.740936, 0.871366], + [-0.0822, 0.531057, -0.043263, 11.0], + [-0.011897, -0.099506, 0.0, 0.01033], + [-0.011897, -0.099506, 0.043263, 0.045182], + [0.992928, -0.11872, 0.0, 0.577584], + [-0.044345, 0.23254, 0.0, 7.0], + [-0.001082, 0.332046, -0.043263, 0.01033], + [-0.001082, 0.332046, 0.052998, 0.045182], + [-0.999995, -0.003257, 0.0, 0.577584], + [-0.045426, 0.564586, 0.052998, 14.0], + [0.1233, -0.156829, -0.052998, 0.520989], + [0.014061, -0.335291, 0.005408, 0.088655], + [0.429355, 0.032562, 0.902549, 0.973449], + [0.10924, 0.439122, 0.0, 14.0], + [-0.031366, -0.031366, 0.0, 0.132876], + [-0.154666, 0.125464, 0.052998, 0.028428], + [0.182761, -0.182761, 0.966021, 0.955978], + [0.10924, 0.439122, -0.043263, 11.0], + [-0.031366, -0.031366, 0.0, 0.01033], + [-0.031366, -0.031366, 0.043263, 0.045182], + [0.707107, -0.707107, 0.0, 0.577584], + [0.10924, 0.439122, -0.043263, 7.0], + [-0.031366, -0.031366, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.707107, -0.707107, 0.0, 0.577584], + [0.10924, 0.439122, 0.0, 10.0], + [0.019468, 0.024876, 0.063813, 0.18117], + [0.035692, -0.131953, 0.049753, 0.955978], + [-0.933936, -0.126584, 0.334276, 0.964691], + [0.128708, 0.463999, 0.063813, 14.0], + [0.050834, 0.002163, -0.005408, 0.0], + [0.0822, -0.018387, -0.063813, 0.439658], + [0.07859, -0.92643, 0.368172, 0.871366], + [-0.221724, 0.475896, -0.043263, 7.0], + [0.122219, 0.207664, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [-0.861819, 0.507216, 0.0, 0.577584], + [-0.221724, 0.475896, -0.043263, 9.0], + [0.122219, 0.207664, 0.0, 0.01033], + [0.122219, 0.207664, 0.043263, 0.045182], + [-0.861819, 0.507216, 0.0, 0.577584], + [-0.099506, 0.68356, 0.0, 14.0], + [0.017305, -0.152503, 0.043263, 0.0], + [-0.122219, -0.207664, 0.0, 0.723049], + [-0.365879, 0.215335, 0.905408, 0.854994], + [-0.0822, 0.531057, -0.043263, 7.0], + [-0.011897, -0.099506, 0.043263, 0.01033], + [0.0, 0.0, 0.086527, 0.045182], + [0.992928, -0.11872, 0.0, 0.577584], + [-0.099506, 0.68356, 0.0, 14.0], + [0.054079, -0.118974, 0.052998, 0.0], + [0.017305, -0.152503, 0.043263, 0.514916], + [-0.41958, 0.203352, 0.884647, 0.863157], + [-0.099506, 0.68356, -0.043263, 3.0], + [0.086527, 0.149258, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [-0.86514, 0.501531, 0.0, 0.577584], + [-0.045426, 0.564586, -0.043263, 11.0], + [-0.036774, -0.033529, 0.0, 0.01033], + [-0.036774, -0.033529, 0.086527, 0.045182], + [0.673754, -0.738956, 0.0, 0.577584], + [-0.045426, 0.564586, -0.043263, 7.0], + [-0.036774, -0.033529, 0.086527, 0.01033], + [0.0, 0.0, 0.096261, 0.045182], + [0.673754, -0.738956, 0.0, 0.577584], + [-0.099506, 0.68356, 0.0, 14.0], + [0.086527, 0.149258, 0.0, 0.0], + [0.054079, -0.118974, 0.052998, 0.262257], + [-0.385563, 0.223515, 0.8952, 0.9131], + [-0.099506, 0.68356, -0.043263, 11.0], + [0.086527, 0.149258, 0.0, 0.01033], + [0.086527, 0.149258, 0.043263, 0.045182], + [-0.86514, 0.501531, 0.0, 0.577584], + [-0.012979, 0.832818, 0.0, 14.0], + [0.214153, -0.194685, 0.0822, 0.000305], + [-0.032447, -0.268232, 0.052998, 0.930114], + [-0.176863, 0.211325, 0.961281, 0.973449], + [0.128708, 0.463999, 0.063813, 14.0], + [-0.019468, -0.024876, -0.063813, 0.168276], + [-0.174135, 0.100587, -0.010816, 0.038209], + [-0.469238, -0.764881, 0.441331, 0.955978], + [0.201174, 0.638133, 0.0822, 14.0], + [-0.072466, -0.174135, -0.018387, 0.18117], + [-0.246601, -0.073548, -0.029203, 0.082277], + [-0.098561, -0.063844, 0.993081, 0.973449], + [-0.012979, 0.832818, -0.043263, 7.0], + [0.339617, -0.002163, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.006369, 0.99998, 0.0, 0.577584], + [0.179543, 0.466162, 0.058405, 10.0], + [-0.050834, -0.002163, 0.005408, 0.000305], + [0.021632, 0.171972, 0.023795, 0.022171], + [0.110896, -0.149888, 0.982464, 0.381323], + [-0.012979, 0.832818, 0.0, 14.0], + [0.339617, -0.002163, 0.0, 0.0], + [0.214153, -0.194685, 0.0822, 0.027314], + [0.002492, 0.391296, 0.920261, 0.49102], + [0.183869, 0.317985, 0.037855, 10.0], + [-0.055161, 0.146014, 0.025958, 0.0], + [0.02704, 0.127627, -0.037855, 0.439658], + [0.623841, 0.097823, 0.775405, 0.871366], + [0.210908, 0.445612, -0.043263, 7.0], + [0.0, -0.131953, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [1.0, 0.0, 0.0, 0.577584], + [0.210908, 0.445612, 0.0, 10.0], + [-0.031366, 0.02055, 0.058405, 0.000305], + [0.035692, 0.014061, 0.06165, 0.022171], + [-0.105864, -0.954447, 0.278971, 0.381323], + [0.23903, 0.45102, -0.043263, 11.0], + [-0.028121, -0.005408, 0.0, 0.01033], + [-0.028121, -0.005408, 0.043263, 0.045182], + [0.188847, -0.982006, 0.0, 0.577584], + [0.23903, 0.45102, -0.043263, 7.0], + [-0.028121, -0.005408, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.188847, -0.982006, 0.0, 0.577584], + [0.246601, 0.459672, 0.06165, 11.0], + [-0.007571, -0.008653, -0.06165, 0.000305], + [-0.035692, -0.014061, -0.06165, 0.022171], + [0.187618, -0.975617, 0.113888, 0.381323], + [0.241193, 0.343943, -0.043263, 11.0], + [-0.002163, 0.107077, 0.0, 0.01033], + [-0.002163, 0.107077, 0.043263, 0.045182], + [-0.999796, -0.020198, 0.0, 0.577584], + [0.23903, 0.45102, 0.0, 15.0], + [0.007571, 0.008653, 0.06165, 0.000305], + [0.042182, -0.108158, 0.046508, 0.846876], + [-0.94107, -0.299262, 0.157572, 0.896269], + [0.316904, 0.358004, 0.036774, 14.0], + [-0.070303, 0.101669, 0.024876, 0.0], + [0.023795, 0.116811, -0.036774, 0.473533], + [0.523424, 0.157027, 0.837478, 0.930114], + [0.340698, 0.474815, -0.043263, 7.0], + [0.001082, -0.121137, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.99996, 0.008928, 0.0, 0.577584], + [0.376391, 0.381799, -0.043263, 11.0], + [-0.001082, 0.107077, 0.0, 0.01033], + [-0.001082, 0.107077, 0.043263, 0.045182], + [-0.999949, -0.0101, 0.0, 0.577584], + [0.376391, 0.381799, -0.043263, 7.0], + [-0.001082, 0.107077, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [-0.999949, -0.0101, 0.0, 0.577584], + [0.375309, 0.488875, 0.0, 14.0], + [0.036774, -0.105995, 0.043263, 0.000305], + [0.001082, -0.107077, 0.0, 0.879622], + [-0.771254, -0.00779, 0.63648, 0.896269], + [0.375309, 0.488875, 0.0, 14.0], + [0.078955, -0.08869, 0.032447, 0.223224], + [0.036774, -0.105995, 0.043263, 0.964691], + [0.071225, 0.39802, 0.914608, 0.973449], + [0.473733, 0.603523, 0.0, 14.0], + [0.0, -0.21199, 0.0, 0.158953], + [-0.019468, -0.203337, 0.032447, 0.030716], + [0.857493, 0.0, 0.514496, 0.964691], + [0.473733, 0.603523, -0.043263, 11.0], + [0.0, -0.21199, 0.0, 0.01033], + [0.0, -0.21199, 0.043263, 0.045182], + [1.0, 0.0, 0.0, 0.577584], + [0.276885, 0.484549, 0.058405, 10.0], + [-0.030284, -0.024876, 0.003245, 0.000305], + [-0.097342, -0.018387, 0.0, 0.022171], + [-0.03153, 0.166923, 0.985466, 0.381323], + [0.246601, 0.459672, 0.06165, 14.0], + [0.030284, 0.024876, -0.003245, 0.0], + [0.094098, 0.015142, -0.06165, 0.473533], + [0.518869, -0.545855, 0.657888, 0.930114], + [0.375309, 0.488875, -0.043263, 11.0], + [-0.034611, -0.014061, 0.0, 0.01033], + [-0.034611, -0.014061, 0.043263, 0.045182], + [0.376377, -0.926467, 0.0, 0.577584], + [0.340698, 0.474815, 0.0, 14.0], + [-0.063813, 0.009734, 0.058405, 0.000305], + [0.034611, 0.014061, 0.0, 0.022171], + [0.3276, -0.806402, 0.492334, 0.381323], + [0.375309, 0.488875, -0.043263, 7.0], + [-0.034611, -0.014061, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.376377, -0.926467, 0.0, 0.577584], + [0.454264, 0.400185, 0.032447, 12.0], + [-0.078955, 0.08869, -0.032447, 0.158953], + [0.019468, 0.203337, -0.032447, 0.030716], + [-0.201684, 0.173144, 0.964025, 0.964691], + [0.201174, 0.638133, 0.0822, 8.0], + [0.075711, -0.153585, -0.023795, 0.000305], + [-0.021632, -0.171972, -0.023795, 0.022171], + [0.026497, -0.140281, 0.989757, 0.381323], + [0.473733, 0.603523, -0.043263, 7.0], + [0.0, -0.21199, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [1.0, 0.0, 0.0, 0.577584], + [0.276885, 0.484549, 0.058405, 14.0], + [-0.075711, 0.153585, 0.023795, 0.000305], + [0.098424, 0.004326, -0.058405, 0.022171], + [0.50316, 0.115345, 0.856461, 0.381323], + [0.375309, 0.488875, 0.0, 10.0], + [0.090853, 0.12979, 0.0, 0.158953], + [0.098424, 0.114648, 0.0, 0.030716], + [0.0, 0.0, 1.0, 0.964691], + [0.466162, 0.618665, -0.043263, 11.0], + [0.007571, -0.015142, 0.0, 0.01033], + [0.007571, -0.015142, 0.043263, 0.045182], + [0.894427, 0.447214, 0.0, 0.577584], + [0.466162, 0.618665, -0.043263, 3.0], + [0.007571, -0.015142, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.894427, 0.447214, 0.0, 0.577584], + [0.326638, 0.830655, 0.0, 14.0], + [0.139524, -0.21199, 0.0, 0.0], + [0.048671, -0.34178, 0.0, 0.015213], + [0.0, 0.0, 1.0, 0.428687], + [0.326638, 0.830655, 0.0, 14.0], + [0.048671, -0.34178, 0.0, 0.007492], + [-0.125464, -0.192522, 0.0822, 0.054475], + [0.472493, 0.067285, 0.878762, 0.964691], + [0.326638, 0.830655, -0.043263, 9.0], + [0.139524, -0.21199, 0.0, 0.01033], + [0.139524, -0.21199, 0.043263, 0.045182], + [0.835314, 0.549773, 0.0, 0.577584], + [0.326638, 0.830655, -0.043263, 7.0], + [0.139524, -0.21199, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.835314, 0.549773, 0.0, 0.577584], + [-0.012979, 0.832818, -0.043263, 11.0], + [0.339617, -0.002163, 0.0, 0.01033], + [0.339617, -0.002163, 0.043263, 0.045182], + [0.006369, 0.99998, 0.0, 0.577584], +]; + +pub const POLYS: [[f32; 2]; 110] = [ + [0.006442, -0.002147], + [-0.001074, 0.001074], + [-0.004294, 0.007515], + [-0.004294, 0.806288], + [-0.001074, 0.809509], + [0.004294, 0.809509], + [0.022546, 0.796626], + [0.185736, 0.655982], + [0.281288, 0.86319], + [0.286656, 0.870706], + [0.293098, 0.871779], + [0.465951, 0.781595], + [0.478834, 0.773006], + [0.478834, 0.763344], + [0.407975, 0.592638], + [0.45092, 0.586196], + [0.601227, 0.573313], + [0.606595, 0.569018], + [0.606595, 0.564724], + [0.601227, 0.556135], + [0.50138, 0.453067], + [0.425153, 0.37684], + [0.289877, 0.246933], + [0.081595, 0.057975], + [0.013957, 0.0], + [-0.011897, 0.0], + [-0.055161, 0.037855], + [-0.081119, 0.063813], + [-0.083282, 0.06814], + [-0.084363, 0.38288], + [-0.089771, 0.38288], + [-0.158993, 0.33529], + [-0.166564, 0.336372], + [-0.250927, 0.379635], + [-0.261743, 0.388288], + [-0.263906, 0.461836], + [-0.262824, 0.486712], + [-0.173053, 0.638133], + [-0.044345, 0.860939], + [-0.036774, 0.867429], + [-0.024876, 0.871755], + [0.33529, 0.871755], + [0.349351, 0.867429], + [0.363412, 0.853368], + [0.513752, 0.619747], + [0.513752, 0.379635], + [0.466162, 0.322311], + [0.461836, 0.320148], + [0.399104, 0.311496], + [0.393696, 0.313659], + [0.374227, 0.332046], + [0.343943, 0.288782], + [0.336372, 0.282293], + [0.269314, 0.271477], + [0.263906, 0.27364], + [0.243356, 0.292027], + [0.21199, 0.244438], + [0.207664, 0.241193], + [0.149258, 0.231459], + [0.138443, 0.231459], + [0.118974, 0.248764], + [0.116811, 0.222806], + [0.117892, 0.162237], + [0.115729, 0.076792], + [0.06814, 0.011897], + [0.062732, 0.007571], + [0.005408, 0.0], + [0.031135, 0.06227], + [0.252301, 0.266258], + [0.541104, 0.544325], + [0.359663, 0.559356], + [0.433742, 0.753681], + [0.304908, 0.821319], + [0.199693, 0.595859], + [0.035429, 0.740798], + [0.035429, 0.482055], + [-0.043263, 0.077874], + [0.001082, 0.037855], + [0.044345, 0.043263], + [0.075711, 0.08869], + [0.077874, 0.407756], + [0.10924, 0.439122], + [0.111403, 0.307169], + [0.147095, 0.272559], + [0.189277, 0.28013], + [0.210909, 0.313659], + [0.210909, 0.445612], + [0.23903, 0.45102], + [0.241193, 0.343943], + [0.276885, 0.311496], + [0.319067, 0.319067], + [0.34178, 0.353677], + [0.340698, 0.474815], + [0.375309, 0.488875], + [0.376391, 0.381799], + [0.411001, 0.351514], + [0.45102, 0.359085], + [0.473733, 0.391533], + [0.473733, 0.603523], + [0.466162, 0.618665], + [0.326638, 0.830655], + [-0.012979, 0.832818], + [-0.099506, 0.68356], + [-0.221724, 0.475896], + [-0.223888, 0.407756], + [-0.163319, 0.380717], + [-0.094098, 0.431551], + [-0.0822, 0.531057], + [-0.045426, 0.564586], + [-0.044345, 0.23254], +]; + +pub const BOXES: [[f32; 4]; 44] = [ + [0.030135, 0.06127, -0.043945, 0.0], + [0.330601, 0.483055, 0.116951, 7.0], + [0.034429, 0.265258, -0.043945, 7.0], + [0.542104, 0.741798, 0.116951, 8.0], + [0.034429, 0.355442, -0.043945, 15.0], + [0.305908, 0.822319, 0.116951, 8.0], + [0.267405, 0.532589, -0.043945, 23.0], + [0.542104, 0.822319, 0.082595, 8.0], + [-0.045345, 0.036855, -0.044263, 31.0], + [0.045345, 0.23354, 0.059405, 8.0], + [-0.224888, 0.072548, -0.044263, 39.0], + [0.059405, 0.565586, 0.059405, 8.0], + [-0.032366, 0.042263, -0.044263, 47.0], + [0.148095, 0.408756, 0.059405, 8.0], + [0.074711, 0.08769, -0.044263, 55.0], + [0.184869, 0.464999, 0.064813, 8.0], + [0.146095, 0.271559, -0.044263, 63.0], + [0.211908, 0.446612, 0.038855, 8.0], + [0.23803, 0.310496, -0.044263, 71.0], + [0.320067, 0.460672, 0.06265, 8.0], + [0.275885, 0.310496, -0.044263, 79.0], + [0.412001, 0.475815, 0.047508, 8.0], + [0.375391, 0.350514, -0.044263, 87.0], + [0.45202, 0.38388, 0.044263, 4.0], + [0.410001, 0.350514, -0.044263, 91.0], + [0.474733, 0.401185, 0.044263, 5.0], + [-0.224888, 0.379717, -0.044263, 96.0], + [-0.0812, 0.532057, 0.047508, 8.0], + [-0.095098, 0.228295, -0.044263, 104.0], + [0.211908, 0.565586, 0.064813, 8.0], + [-0.222724, 0.430551, -0.044263, 112.0], + [-0.011979, 0.833818, 0.053998, 8.0], + [-0.100506, 0.438122, -0.044263, 120.0], + [0.327638, 0.833818, 0.0832, 8.0], + [0.127708, 0.312659, -0.044263, 128.0], + [0.282211, 0.467162, 0.064813, 8.0], + [0.245601, 0.352677, -0.044263, 136.0], + [0.474733, 0.604523, 0.06265, 8.0], + [0.178543, 0.390533, -0.044263, 144.0], + [0.474733, 0.639133, 0.0832, 8.0], + [0.200174, 0.483549, -0.044263, 152.0], + [0.474733, 0.639133, 0.0832, 4.0], + [-0.013979, 0.487875, -0.044263, 156.0], + [0.467162, 0.833818, 0.0832, 5.0], +]; diff --git a/crates/compositor/src/regions.rs b/crates/compositor/src/regions.rs index 097863e299..d7881ba078 100644 --- a/crates/compositor/src/regions.rs +++ b/crates/compositor/src/regions.rs @@ -49,18 +49,22 @@ pub fn speed_segments_for_window( for region in overlapping { let start = region.start_sec.max(source_start_sec).max(cursor); let end = region.end_sec.min(source_end_sec); + // Projection can leave a zero-width region at a trim boundary. It may still pass + // the overlap filter when rounding puts its start just inside the clip window. Skip + // it before emitting the 1x gap, or the unchanged cursor makes that gap render twice. + if end <= start { + continue; + } if start > cursor { push_speed_segment(&mut spans, cursor, start, 1.0, fps); } - if end > start { - let speed = if region.speed.is_finite() && region.speed > 0.0 { - region.speed - } else { - 1.0 - }; - push_speed_segment(&mut spans, start, end, speed, fps); - cursor = end; - } + let speed = if region.speed.is_finite() && region.speed > 0.0 { + region.speed + } else { + 1.0 + }; + push_speed_segment(&mut spans, start, end, speed, fps); + cursor = end; } if cursor < source_end_sec { push_speed_segment(&mut spans, cursor, source_end_sec, 1.0, fps); @@ -462,6 +466,10 @@ pub fn zoom_cursor_alpha(regions: &[SceneZoomRegion], t: f32, clock: &ScreenCloc /// pixel shader par `compositor.rs`, ce module ne fait que le calcul temporel). pub struct ZoomState { pub scale: f32, + /// Force de la région (0..1) : plein régime sur son span et sur les transitions/paliers + /// chaînés, 0 hors zoom. C'est l'enveloppe du zoom — le rétrécissement réactif de la webcam + /// (`reactive_scale`) s'y accroche pour animer sa taille, au lieu de suivre la profondeur. + pub strength: f32, pub focus: [f32; 2], pub rotation: [f32; 3], /// À quel point un préset 3D est installé (0..1) : la force de la région quand elle en porte @@ -479,6 +487,7 @@ pub struct ZoomState { const IDENTITY_ZOOM: ZoomState = ZoomState { scale: 1.0, + strength: 0.0, focus: [0.5, 0.5], rotation: [0.0, 0.0, 0.0], tilt: 0.0, @@ -865,6 +874,7 @@ pub fn zoom_state_in( |p: [f32; 2], q: [f32; 2]| [lerp(p[0], q[0], progress), lerp(p[1], q[1], progress)]; return ZoomState { scale: scale_lerp(cur.scale, next.scale, progress), + strength: 1.0, focus: [ lerp(cur_focus[0], next_focus[0], progress), lerp(cur_focus[1], next_focus[1], progress), @@ -886,6 +896,7 @@ pub fn zoom_state_in( let seen = orbit_at(next, t, frame); return ZoomState { scale: next.scale, + strength: 1.0, focus: resolve_focus(next, t, cursor, clock), rotation: fixed_rotation(next), tilt: fixed_flag(next), @@ -939,6 +950,7 @@ pub fn zoom_state_in( let seen = orbit_at(r, t, frame); ZoomState { scale, + strength, focus: [ease(focus[0]), ease(focus[1])], rotation: lerp_rotation3d([0.0, 0.0, 0.0], fixed_rotation(r), strength), tilt: fixed_flag(r) * strength, @@ -957,6 +969,11 @@ pub fn zoom_state_in( /// `endSec`. Fenêtres bornées à la moitié de la durée de la région pour que les régions courtes /// s'animent pleinement sans déborder. Fenêtres mesurées à l'écran, comme celles du zoom /// (`ScreenClock`). Le rect et la bulle y appliquent chacun leur courbe. +/// +/// Un morceau coupé par un trim (`continues_before` / `continues_after`) n'a pas de transition +/// du côté de la coupe, bord compris : la région reste plein cadre à travers le trim, montée à +/// son vrai début seulement, descente à sa vraie fin seulement. Le côté restant prend alors tout +/// le span au lieu de sa moitié. fn camera_fullscreen_region_phase( region: &SceneCameraFullscreenRegion, t: f32, @@ -964,13 +981,20 @@ fn camera_fullscreen_region_phase( ) -> f32 { let start = region.start_sec as f32; let end = region.end_sec as f32; - if t <= start || t >= end { + let before_start = if region.continues_before { t < start } else { t <= start }; + let after_end = if region.continues_after { t > end } else { t >= end }; + if before_start || after_end { return 0.0; } let (start, end, t) = (clock.at(start), clock.at(end), clock.at(t)); - let half = (end - start) * 0.5; - let lead_in = TRANSITION_WINDOW_S.min(half); - let lead_out = FULLSCREEN_LEAD_OUT_WINDOW_S.min(half); + let span = end - start; + let half = span * 0.5; + let (lead_in, lead_out) = match (region.continues_before, region.continues_after) { + (false, false) => (TRANSITION_WINDOW_S.min(half), FULLSCREEN_LEAD_OUT_WINDOW_S.min(half)), + (false, true) => (TRANSITION_WINDOW_S.min(span), 0.0), + (true, false) => (0.0, FULLSCREEN_LEAD_OUT_WINDOW_S.min(span)), + (true, true) => (0.0, 0.0), + }; let lead_in_end = start + lead_in; let lead_out_start = end - lead_out; if t < lead_in_end { @@ -1965,6 +1989,8 @@ mod zoom_focus_tests { clip_index: None, start_sec: 18.0 * k, end_sec: 22.0 * k, + continues_before: false, + continues_after: false, }] }; let (sped, plain) = (zooms(4.0), zooms(1.0)); @@ -1991,11 +2017,52 @@ mod zoom_focus_tests { } } + fn full_camera_piece( + clip_index: Option, + start_sec: f64, + end_sec: f64, + continues_before: bool, + continues_after: bool, + ) -> SceneCameraFullscreenRegion { + SceneCameraFullscreenRegion { + clip_index, + start_sec, + end_sec, + continues_before, + continues_after, + } + } + + /// Un trim au milieu d'une région Full Camera la coupe en deux morceaux, un par segment gardé + /// (`projectRegionsToSource`, TS). Chacun est filtré sur son clip (`for_clip_window`) : la + /// webcam ne doit pas revenir dans son coin avant la coupe ni regrandir après. Montée au vrai + /// début, descente à la vraie fin, plein cadre de part et d'autre du trim — comme un zoom. + #[test] + fn a_trim_inside_full_camera_keeps_it_full_across_the_cut() { + // Source [10, 20], trim [14, 16] : morceaux [10, 14] (clip 0) et [16, 20] (clip 1). + let before = [full_camera_piece(Some(0), 10.0, 14.0, false, true)]; + let after = [full_camera_piece(Some(1), 16.0, 20.0, true, false)]; + let clock = ScreenClock::default(); + let at = |r: &[SceneCameraFullscreenRegion], t: f32| { + (camera_fullscreen_progress_at(r, t, &clock), camera_fullscreen_shape_at(r, t, &clock)) + }; + let frame = 1.0 / 60.0; + assert_eq!(at(&before, 14.0 - frame), (1.0, 0.0), "dernière frame avant la coupe"); + assert_eq!(at(&before, 14.0), (1.0, 0.0), "bord de coupe, fin"); + assert_eq!(at(&after, 16.0), (1.0, 0.0), "première frame après la coupe"); + // Les vraies transitions restent, à leur durée pleine (pas la moitié d'un morceau). + assert!(at(&before, 10.0 + TRANSITION_WINDOW_S / 2.0).0 < 1.0, "montée au vrai début"); + assert_eq!(at(&before, 10.0 + TRANSITION_WINDOW_S).0, 1.0); + assert!(at(&after, 20.0 - frame).0 < 1.0, "descente à la vraie fin"); + assert_eq!(at(&after, 20.0 - FULLSCREEN_LEAD_OUT_WINDOW_S).0, 1.0); + assert_eq!(at(&after, 20.0).0, 0.0); + } + /// Les coins suivent la phase, pas le rect. Avec `1 - progrès`, ils étaient carrés presque /// tout du long de la montée et ne revenaient qu'à la toute fin du retour. #[test] fn full_camera_corners_dissolve_late_and_come_back_early() { - let r = [SceneCameraFullscreenRegion { clip_index: None, start_sec: 10.0, end_sec: 20.0 }]; + let r = [full_camera_piece(None, 10.0, 20.0, false, false)]; let clock = ScreenClock::default(); let at = |t: f32| { (camera_fullscreen_progress_at(&r, t, &clock), camera_fullscreen_shape_at(&r, t, &clock)) @@ -2178,6 +2245,32 @@ mod arrow_tests { let (_, half) = arrow_local_geometry("right", -5.0, [100.0, 100.0]); assert_eq!(half, 0.0); } + + /// L'éditeur encadre chaque flèche par l'enveloppe de ses traits (`ARROW_EXTENTS`, + /// `src/lib/ai-edition/annotations/arrowBounds.ts`) : elle doit être celle que dessine ce + /// module, sinon le gimbal ne tombe plus sur la flèche. + #[test] + fn the_editor_frames_each_arrow_by_its_strokes() { + let ts = include_str!("../../../src/lib/ai-edition/annotations/arrowBounds.ts"); + let table = &ts[ts.find("ARROW_EXTENTS").expect("la table")..]; + let table = &table[..table.find("};").expect("sa fin")]; + for dir in ["up", "down", "left", "right", "up-right", "up-left", "down-right", "down-left"] { + let key = if dir.contains('-') { format!("\"{dir}\":") } else { format!("\t{dir}:") }; + let row = &table[table.find(&key).unwrap_or_else(|| panic!("{dir} absente"))..]; + let row = &row[row.find('[').expect("[") + 1..row.find(']').expect("]")]; + let got: Vec = row.split(',').map(|v| v.trim().parse().expect("nombre")).collect(); + let segs = arrow_segments_viewbox(dir); + let xs = segs.iter().flat_map(|s| [s[0], s[2]]); + let ys = segs.iter().flat_map(|s| [s[1], s[3]]); + let want = [ + xs.clone().fold(f32::MAX, f32::min), + ys.clone().fold(f32::MAX, f32::min), + xs.fold(f32::MIN, f32::max), + ys.fold(f32::MIN, f32::max), + ]; + assert_eq!(got, want, "{dir}"); + } + } } #[cfg(test)] @@ -3125,6 +3218,16 @@ mod exporter_frame_totals { 60, "région débordant la fenêtre gardée" ); + let issue_876_fps = 60.0; + let first_clip = frames( + 0.0, + 44.81830642526596, + &[region(44.818, 44.818, 1.5)], + issue_876_fps, + ); + let second_clip = frames(0.0, 47.833333, &[region(0.0, 1.87, 1.5)], issue_876_fps); + assert_eq!(first_clip, 2690, "une région vide ne double pas le premier clip"); + assert_eq!(first_clip + second_clip, 5523, "total corrigé du repro #876"); assert_eq!( frames( 0.0, diff --git a/crates/compositor/src/scene.rs b/crates/compositor/src/scene.rs index de24278fdb..a8237c9780 100644 --- a/crates/compositor/src/scene.rs +++ b/crates/compositor/src/scene.rs @@ -291,7 +291,7 @@ impl SceneFrame { } /// Fond derrière l'écran (parsé depuis `settings.wallpaper`). -#[derive(Debug, Clone, Deserialize)] +#[derive(Debug, Clone, PartialEq, Deserialize)] #[serde(tag = "kind", rename_all = "lowercase")] pub enum SceneBackground { Color { color: String }, @@ -537,6 +537,14 @@ pub struct SceneCameraFullscreenRegion { pub clip_index: Option, pub start_sec: f64, pub end_sec: f64, + /// La région commence avant ce morceau : un trim ou une frontière de clip l'a coupée, et + /// `start_sec` est un bord de coupe, pas son vrai début. Pas de montée ici, plein cadre dès + /// la première frame, comme un zoom dont l'ease-in tombe hors du clip. + #[serde(default)] + pub continues_before: bool, + /// Pendant de `continues_before` à la fin : pas de descente au bord de la coupe. + #[serde(default)] + pub continues_after: bool, } /// Rendu du curseur. @@ -595,6 +603,11 @@ pub struct SceneCursorSprite { /// shaders le modèlent au lieu d'extruder le sprite, qui reste l'art en 2D. #[serde(default)] pub sculpt: Option, + /// Le verre de cet état à plat, `"/<état>"` comme `sculpt` mais 3D éteinte comprise : + /// le shader dessine ce modèle en lentille de verre posée sur l'image (mode 19, + /// `frame_geometry::cursor_glass_cb`) au lieu du sprite, qui ne sert plus qu'au sélecteur. + #[serde(default)] + pub glass: Option, } #[derive(Debug, Clone, Copy, Deserialize)] @@ -666,6 +679,43 @@ pub enum SceneAudioTrackKind { Voiceover, } +/// One click to be heard: `time_sec` is its place on the OUTPUT programme, `gain` the linear +/// level of the hit (side clicks a little quieter than the left), `release` which of the two +/// samples to play. +#[derive(Debug, Clone, Copy, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SceneClickHit { + pub time_sec: f64, + #[serde(default = "default_one")] + pub gain: f32, + #[serde(default)] + pub release: bool, +} + +/// The "mouse clicks" toggle: the recorded click times, already projected onto the programme, +/// plus the two hit samples to play them with. +/// +/// Deliberately not an imported audio track. A bed baked into one file cannot follow an edit — +/// `SceneAudioTrack`'s content is placed, never stretched, so every hit after a speed region +/// drifts — while hits laid out here are positioned by the app's own projection, the same one +/// imported audio tracks are placed with, so they cannot disagree with the picture. +#[derive(Debug, Clone, Default, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SceneClickSound { + /// Absolute paths to the press and release samples, 48 kHz-friendly WAV files. `default` for + /// the usual reason: a half-written block must cost the sound, never the whole scene parse. + #[serde(default)] + pub down_path: String, + #[serde(default)] + pub up_path: String, + #[serde(default)] + pub hits: Vec, +} + +fn default_one() -> f32 { + 1.0 +} + #[derive(Debug, Clone, Copy, Deserialize)] #[serde(rename_all = "camelCase")] pub struct SceneOutput { @@ -738,9 +788,18 @@ pub struct Scene { /// absent from every scene written before this, and from a project with none. #[serde(default)] pub audio_tracks: Vec, + /// Click hits for the "mouse clicks" toggle. `#[serde(default)]`: an empty `hits` is the + /// same as the field being absent, which is every scene written before this. + #[serde(default)] + pub click_sound: SceneClickSound, /// Crop écran par clip, dans le même ordre que `clips` (`cropByClip` côté TS). #[serde(default)] pub crop_by_clip: Vec>, + /// Half turn of each clip's main picture, same order as `clips` (`rotationByClip` on + /// the TS side). Values are 0 or 180; any other u16 is treated as upright. Absent in every + /// scene written before it. + #[serde(default)] + pub rotation_by_clip: Vec, /// Effet d'arrière-plan de la webcam. Absent = aucun effet. #[serde(default)] pub webcam_effect: Option, @@ -756,6 +815,11 @@ impl Scene { Ok(serde_json::from_str(json)?) } + /// Whether the clip being composed is turned by 180°. + pub fn screen_turned(&self) -> bool { + self.rotation_by_clip.get(self.active_clip_index) == Some(&180) + } + /// Copie de scène limitée aux régions du clip actif. `clipIndex` est l'identité fiable /// lorsque plusieurs clips réutilisent les mêmes temps source ; son absence retombe sur le /// chevauchement avec la fenêtre source pour accepter les anciens payloads. @@ -912,6 +976,24 @@ mod tests { assert_eq!(effects(r#","frame":"laptop""#).frame.theme_override(), None); } + #[test] + fn a_click_sound_payload_reads_the_apps_camel_case_names() { + let clicks: SceneClickSound = serde_json::from_str( + r#"{"downPath":"/u/click-down.wav","upPath":"/u/click-up.wav","hits":[ + {"timeSec":2.5,"gain":0.7,"release":true},{"timeSec":4.0}]}"#, + ) + .unwrap(); + assert_eq!(clicks.down_path, "/u/click-down.wav"); + assert_eq!(clicks.hits.len(), 2); + assert_eq!(clicks.hits[0].time_sec, 2.5); + assert!(clicks.hits[0].release); + // A hit that carries no level is a full-strength press. + assert_eq!(clicks.hits[1].gain, 1.0); + // And a scene from before the field reads as no hits, not as a parse failure. + assert!(SceneClickSound::default().hits.is_empty()); + assert!(serde_json::from_str::("{}").unwrap().hits.is_empty()); + } + #[test] fn parses_color_and_image_backgrounds() { let color = r##"{"clips":[],"layout":{"preset":"no-webcam","webcamSize":1,"webcamShape":"rectangle","webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false},"effects":{"padding":0,"blur":false,"shadow":0,"roundnessFrac":0,"motionBlur":0},"background":{"kind":"color","color":"#123456"},"zoomRegions":[],"cursor":{"show":false,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":0,"clipToBounds":false,"theme":"default"},"cropByClip":[],"output":{"width":1280,"height":720,"fps":30}}"##; @@ -1140,6 +1222,25 @@ mod annotation_tests { ); } + #[test] + fn rotation_by_clip_defaults_empty_and_reads_the_active_clip() { + let scene = Scene::from_json(&scene_json("[]")).expect("parse"); + assert!(scene.rotation_by_clip.is_empty()); + assert!(!scene.screen_turned()); + let mut turned = scene.clone(); + turned.rotation_by_clip = vec![0, 180]; + assert!(!turned.for_clip_window(0, 0.0, 1.0).screen_turned()); + assert!(turned.for_clip_window(1, 0.0, 1.0).screen_turned()); + // Only a half turn counts; anything else stays upright. + turned.rotation_by_clip = vec![90]; + assert!(!turned.for_clip_window(0, 0.0, 1.0).screen_turned()); + // The field arrives from the app as `rotationByClip`. + let json = scene_json("[]").replace(r#""cropByClip":[]"#, r#""cropByClip":[],"rotationByClip":[180]"#); + let parsed = Scene::from_json(&json).expect("parse rotationByClip"); + assert_eq!(parsed.rotation_by_clip, vec![180]); + assert!(parsed.screen_turned()); + } + #[test] fn for_clip_window_still_falls_back_to_window_overlap_without_a_clip_index() { // Vieux payload : rien ne dit à quel clip la région appartient, le chevauchement de diff --git a/crates/compositor/src/sculpt.rs b/crates/compositor/src/sculpt.rs index de9c4e65a0..b226623534 100644 --- a/crates/compositor/src/sculpt.rs +++ b/crates/compositor/src/sculpt.rs @@ -1,17 +1,20 @@ -//! Curseurs sculptés : la flèche et la main des cinq thèmes d'origine, modelées en volumes dans les -//! shaders (mode 15, `sculpt_proto`) au lieu d'extruder leur PNG — capsules et unions lissées, -//! extrusions arrondies, voxels, polyèdres taillés. Ce module en tient ce que la géométrie doit -//! savoir côté CPU : l'identifiant que lit le shader, et la boîte du modèle (`SpriteShape`) qui pose -//! le hotspot, règle la garde au sol et borne la boîte de dessin. +//! Curseurs sculptés : la flèche et la main des thèmes d'origine, modelées en volumes dans les +//! shaders (mode 15, `sculpt_proto`) au lieu d'extruder leur PNG — voxels (Pixel Candy), pièces +//! cerclées du trait de leur dessin (Studio Ink, Pop Coral, Star Sprout). Prism Glow est à part : +//! un MAILLAGE tracé sur son dessin à facettes, un cristal lancé de rayons boîte par boîte +//! dans son serti marine (`prism_mesh`). Glass Lens est une lentille de verre cerclée d'un trait +//! sombre et d'un liseré de verre (`s_glass`) : le seul dont le verre se dessine aussi à plat +//! (mode 19, `frame_geometry::cursor_glass_cb`). Ce module en tient ce que la géométrie doit +//! savoir côté CPU : l'identifiant que lit le shader, et la boîte du modèle (`SpriteShape`) qui +//! pose le hotspot, règle la garde au sol et borne la boîte de dessin. //! //! Les formes sont écrites dans le repère du PROTOTYPE où elles ont été dessinées : hauteur du //! curseur 1, x à droite, y VERS LE HAUT, z vers la caméra, l'écran en z = 0. Les constantes //! ci-dessous sont le miroir des `SCULPT_*` des trois shaders. //! -//! Les voxels de la flèche de Pixel Candy (`SCULPT_ARROWPIX`, `SCULPT_ARROWBACK`) sont son polygone -//! échantillonné : une cellule de 1/16 porte un voxel quand son centre est à plus de 0,01 dedans, un -//! bloc de la couche violette quand il est à moins de 0,9 cellule. Changer le polygone, c'est -//! régénérer ces deux tables. +//! Les grilles de voxels de Pixel Candy sont des tables que +//! `scripts/generate-pixel-candy-voxels.mjs` écrit dans les trois shaders : on change la grille +//! dans le script, on le relance, et on ajuste ici la boîte si la forme a bougé. use crate::frame_geometry::SpriteShape; @@ -24,29 +27,55 @@ const ZREF_ARROW: f32 = 0.2; const ZREF_HAND: f32 = 0.185; /// Le dessous de tous les modèles, dans le prototype. const Z_LOW: f32 = 0.05; -/// Le plus haut : la table de la flèche taillée, l'étoile de Star Sprout sur la manchette. -const Z_HIGH_ARROW: f32 = 0.29; +/// Le côté d'un cube de Pixel Candy : seize lignes font un curseur haut de 1. +const VOX: f32 = 0.0625; +/// Le plus haut : les yeux de l'étoile de Star Sprout, sur la flèche comme sur la manchette. +const Z_HIGH_ARROW: f32 = 0.31; const Z_HIGH_HAND: f32 = 0.4; -/// Boîtes des deux formes dans le prototype (x0, x1, haut), qui contiennent leurs cinq variantes : -/// couche violette des voxels, étoile, cristal compris. Leur hauteur est celle du sprite, 1 / SCULPT_SCALE : -/// le mode 15 tient le plus grand côté de la boîte pour 1 (`sprite_size`). -const BOX_ARROW: [f32; 3] = [-0.1, 0.75, 0.09]; -const BOX_HAND: [f32; 3] = [-0.27, 0.73, 0.07]; +/// Boîtes des modèles dans le prototype (x0, x1, haut). Leur hauteur est celle du sprite, +/// 1 / SCULPT_SCALE : le mode 15 tient le plus grand côté de la boîte pour 1 (`sprite_size`). +/// Celles des cubes de Pixel Candy ; puis des thèmes cerclés, que leur trait élargit et que +/// débordent l'étoile et les feuilles (Star Sprout), les tirets du clic et les calques (Pop +/// Coral), le pouce. +const BOX_PIXEL_ARROW: [f32; 3] = [-0.03, 0.72, 0.03]; +const BOX_PIXEL_HAND: [f32; 3] = [-0.345, 0.72, 0.03]; +const BOX_INK_ARROW: [f32; 3] = [-0.08, 0.6, 0.03]; +const BOX_INK_HAND: [f32; 3] = [-0.36, 0.6, 0.03]; +const BOX_CORAL_ARROW: [f32; 3] = [-0.35, 0.6, 0.06]; +const BOX_CORAL_HAND: [f32; 3] = [-0.38, 0.62, 0.04]; +const BOX_SPROUT_ARROW: [f32; 3] = [-0.08, 0.87, 0.03]; +const BOX_SPROUT_HAND: [f32; 3] = [-0.33, 0.56, 0.03]; +/// Glass Lens : la flèche et le gant de Studio Ink, que le liseré de verre élargit au-delà du +/// trait (`GLASS_RIM` des shaders). `scripts/generate-glass-lens-cursor.mjs` lit ces boîtes. +const BOX_GLASS_ARROW: [f32; 3] = [-0.1, 0.56, 0.03]; +const BOX_GLASS_HAND: [f32; 3] = [-0.4, 0.63, 0.03]; /// Dans l'ordre des identifiants du shader. -const THEMES: [&str; 5] = ["studio-ink", "prism-glow", "pop-coral", "pixel-candy", "star-sprout"]; +const THEMES: [&str; 6] = ["studio-ink", "prism-glow", "pop-coral", "pixel-candy", "star-sprout", "glass-lens"]; -/// Le haut de la silhouette (y du prototype) : la couche violette des voxels dépasse la pointe -/// d'une cellule, la pointe arrondie de la flèche et le sommet de sa table taillée dépassent le -/// hotspot de 0,03, le bout de l'index y est. -fn silhouette_top(theme: usize, arrow: bool) -> f32 { +/// La boîte du modèle (x0, x1, haut) dans le prototype. +fn model_box(theme: usize, arrow: bool) -> [f32; 3] { match (THEMES[theme], arrow) { - ("pixel-candy", _) => 0.0625, - (_, true) => 0.03, - (_, false) => 0.0, + ("studio-ink", true) => BOX_INK_ARROW, + ("studio-ink", false) => BOX_INK_HAND, + ("pop-coral", true) => BOX_CORAL_ARROW, + ("pop-coral", false) => BOX_CORAL_HAND, + ("pixel-candy", true) => BOX_PIXEL_ARROW, + ("pixel-candy", false) => BOX_PIXEL_HAND, + ("star-sprout", true) => BOX_SPROUT_ARROW, + ("star-sprout", false) => BOX_SPROUT_HAND, + ("glass-lens", true) => BOX_GLASS_ARROW, + ("glass-lens", false) => BOX_GLASS_HAND, + (other, _) => unreachable!("thème sculpté inconnu : {other}"), } } +/// Le curseur sculpté que nomme la scène est de verre (le cristal de Prism Glow, la lentille de +/// Glass Lens) : il réfracte ce qui est dessous. +pub fn refracts(name: &str) -> bool { + sculpted_shape(name).is_some_and(|s| matches!(THEMES[((s.sculpt - 1) / 2) as usize], "prism-glow" | "glass-lens")) +} + /// Le curseur sculpté que nomme la scène (`"/<état>"`, cf. `resolveCursorSprites`), sous /// la forme que le mode 15 attend. `None` pour un nom inconnu : l'appelant extrude le sprite. pub fn sculpted_shape(name: &str) -> Option { @@ -57,7 +86,20 @@ pub fn sculpted_shape(name: &str) -> Option { "pointer" => false, _ => return None, }; - let [x0, x1, y1] = if arrow { BOX_ARROW } else { BOX_HAND }; + let sculpt = 1 + 2 * theme as u32 + u32::from(!arrow); + if THEMES[theme] == "prism-glow" { + // Le maillage est déjà dans le repère du modèle : sa boîte, son dessus et son dessous. + let m = &crate::prism_mesh::MODELS[usize::from(!arrow)]; + return Some(SpriteShape { + size: m.size, + hotspot: m.hotspot, + top: m.top, + max_height: m.height, + thick: m.thick, + sculpt, + }); + } + let [x0, x1, y1] = model_box(theme, arrow); let (zref, z_high) = if arrow { (ZREF_ARROW, Z_HIGH_ARROW) } else { (ZREF_HAND, Z_HIGH_HAND) }; let s = SCULPT_SCALE; // Repère du modèle : y vers le bas, le coin haut-gauche de la boîte est donc (x0, y1). @@ -65,10 +107,10 @@ pub fn sculpted_shape(name: &str) -> Option { Some(SpriteShape { size, hotspot: [-x0 * s / size[0], y1 * s], - top: (y1 - silhouette_top(theme, arrow)) * s, + top: y1 * s, max_height: (z_high - zref) * s, thick: (zref - Z_LOW) * s, - sculpt: 1 + 2 * theme as u32 + u32::from(!arrow), + sculpt, }) } @@ -76,7 +118,11 @@ pub fn sculpted_shape(name: &str) -> Option { mod tests { use super::*; - const NAMES: [&str; 10] = [ + /// La largeur du contour de Glass Lens, trait sombre et liseré de verre compris : ses boîtes + /// en dépendent, les shaders la portent. + const GLASS_RIM: f32 = 0.085; + + const NAMES: [&str; 12] = [ "studio-ink/arrow", "studio-ink/pointer", "prism-glow/arrow", @@ -87,6 +133,8 @@ mod tests { "pixel-candy/pointer", "star-sprout/arrow", "star-sprout/pointer", + "glass-lens/arrow", + "glass-lens/pointer", ]; #[test] @@ -99,20 +147,21 @@ mod tests { #[test] fn other_states_and_themes_keep_the_extruded_sprite() { - for name in ["default/arrow", "pop-coral/text", "pop-coral", "", "pop-coral/arrow/x"] { + for name in ["default/arrow", "pop-coral/text", "pop-coral", "", "pop-coral/arrow/x", "prism-glow/text"] { assert!(sculpted_shape(name).is_none(), "{name}"); } } /// Le hotspot est dans la boîte, sous le haut de la silhouette ; le modèle a une épaisseur et /// une hauteur ; la boîte a la hauteur du sprite, son plus grand côté, comme le veut le mode 15. + /// Le serti de Prism Glow est le plus mince (0,043) : c'est son épaisseur dans le dessin. #[test] fn the_box_holds_the_hotspot_and_the_silhouette_top() { for name in NAMES { let s = sculpted_shape(name).unwrap(); assert!((0.0..1.0).contains(&s.hotspot[0]) && (0.0..1.0).contains(&s.hotspot[1]), "{name}"); assert!(s.top <= s.hotspot[1] + 1e-6, "{name} : haut de silhouette sous le hotspot"); - assert!(s.thick > 0.05 && s.max_height > 0.0, "{name}"); + assert!(s.thick > 0.04 && s.max_height > 0.0, "{name}"); assert!(s.size[0] < 1.0 && s.size[1] == 1.0, "{name} : le plus grand côté n'est pas 1"); } } @@ -131,6 +180,9 @@ mod tests { ("SCULPT_ZREF_ARROW", ZREF_ARROW), ("SCULPT_ZREF_HAND", ZREF_HAND), ("SCULPT_HOVER", Z_LOW), + ("SCULPT_VOX", VOX), + ("PRISM_BEVEL", crate::prism_mesh::BEVEL), + ("GLASS_RIM", GLASS_RIM), ] { let line = src .lines() @@ -145,4 +197,159 @@ mod tests { } } } + + /// Les nombres de la table `name` d'un shader, du `=` à la fin de la liste, constructeurs de + /// vecteurs retirés : la même table s'écrit alors pareil dans les trois langages. + fn table(src: &str, name: &str) -> Vec { + // La déclaration : `NAME =` en WGSL, `NAME[taille]` en HLSL et en Metal (pas un `NAME[i]` du code). + let decl = format!("{name}["); + let sized = |&i: &usize| src[i + decl.len()..].starts_with(|c: char| c.is_ascii_digit()); + let at = src.find(&format!("{name} =")).or_else(|| src.match_indices(&decl).map(|(i, _)| i).find(sized)); + let rest = &src[at.expect(name)..]; + let rest = &rest[rest.find('=').unwrap()..]; + let open = rest.find(['(', '{']).unwrap() + 1; + let close = [rest.find(");"), rest.find("};")].into_iter().flatten().min().unwrap(); + let mut list = rest[open..close].to_string(); + for ty in ["vec2", "vec3", "vec4", "vec4", "float2", "float3", "float4", "int4"] { + list = list.replace(ty, ""); + } + list.split(|c: char| !(c.is_ascii_digit() || c == '.' || c == '-')) + .filter(|t| !t.is_empty()) + .map(|t| t.parse().unwrap_or_else(|_| panic!("{name} : nombre illisible {t}"))) + .collect() + } + + /// Les grilles de Pixel Candy, que `scripts/generate-pixel-candy-voxels.mjs` écrit dans les + /// trois shaders, y sont les mêmes ; chaque pixel coloré est plein, et d'une seule couleur. + #[test] + fn the_voxel_tables_match_in_the_three_shaders() { + let hlsl = include_str!("shaders.hlsl"); + let metal = include_str!("shaders.metal"); + let wgsl = include_str!("vk_shaders/layer.wgsl"); + let names = ["PIX_BODY", "PIX_LINE", "PIX_HI", "PIX_SHADE", "PIX_RECT_N", "PIX_GRID", "PIX_ORIGIN", "PIX_BOX", "PIX_RECT"]; + for name in names { + let reference = table(wgsl, name); + assert!(!reference.is_empty(), "{name} vide"); + assert_eq!(table(hlsl, name), reference, "{name} : HLSL et WGSL diffèrent"); + assert_eq!(table(metal, name), reference, "{name} : Metal et WGSL diffèrent"); + } + let body = table(wgsl, "PIX_BODY"); + let classes = ["PIX_LINE", "PIX_HI", "PIX_SHADE"].map(|name| table(wgsl, name)); + for (i, &face) in body.iter().enumerate() { + let mut seen = 0u32; + for class in &classes { + let m = class[i] as u32; + assert_eq!(m & !(face as u32), 0, "couleur hors de la forme, ligne {i}"); + assert_eq!(m & seen, 0, "deux couleurs sur un pixel, ligne {i}"); + seen |= m; + } + } + } + + /// Seuls Prism Glow et Glass Lens sont de verre, flèche et main. + #[test] + fn only_the_glass_themes_refract() { + for name in NAMES { + let glass = name.starts_with("prism-glow/") || name.starts_with("glass-lens/"); + assert_eq!(refracts(name), glass, "{name}"); + } + assert!(!refracts("prism-glow/text") && !refracts("glass-lens/text")); + } + + /// Le maillage de Prism Glow tient dans sa boîte (le shader n'y lance de rayons que là), son + /// dessous est à l'épaisseur annoncée (la garde au sol en dépend) et son sommet à la hauteur. + #[test] + fn the_prism_boxes_hold_their_mesh() { + use crate::prism_mesh::{MODELS, POLYS, TRIS}; + for (k, name) in ["prism-glow/arrow", "prism-glow/pointer"].iter().enumerate() { + let s = sculpted_shape(name).unwrap(); + let m = &MODELS[k]; + let [x0, y0] = [-s.hotspot[0] * s.size[0], -s.hotspot[1] * s.size[1]]; + let inside = |x: f32, y: f32| x >= x0 && x <= x0 + s.size[0] && y >= y0 && y <= y0 + s.size[1]; + let (mut zlo, mut zhi) = (f32::MAX, f32::MIN); + for t in TRIS[4 * m.tri_start..4 * (m.tri_start + m.tri_count)].chunks(4) { + for e in [[0.0; 3], [t[1][0], t[1][1], t[1][2]], [t[2][0], t[2][1], t[2][2]]] { + let [x, y, z] = [t[0][0] + e[0], t[0][1] + e[1], t[0][2] + e[2]]; + assert!(inside(x, y), "{name} : sommet ({x}, {y}) hors de la boîte"); + zlo = zlo.min(z); + zhi = zhi.max(z); + } + } + let sil = &POLYS[m.sil_start..m.sil_start + m.sil_count]; + let out = &POLYS[m.out_start..m.out_start + m.out_count]; + for p in sil.iter().chain(out) { + assert!(inside(p[0], p[1]), "{name} : contour ({}, {}) hors de la boîte", p[0], p[1]); + } + assert!((zlo + s.thick).abs() < 1e-5, "{name} : dessous {zlo}, épaisseur {}", s.thick); + assert!((zhi - s.max_height).abs() < 1e-5, "{name} : sommet {zhi}, hauteur {}", s.max_height); + } + } + + /// Les trois shaders portent le maillage de `prism_mesh.rs` tel quel : triangles, boîtes, + /// polygones, et les débuts et longueurs de chaque modèle dans ces tables. Le WGSL lit + /// triangles, polygones et boîtes dans un uniform rempli de `prism_mesh.rs` + /// (`compositor_linux.rs`) : il n'en déclare que la taille. + #[test] + fn the_shaders_carry_the_prism_mesh() { + use crate::prism_mesh::{PrismModel, BOXES, MODELS, POLYS, TRIS}; + let per_model = |f: fn(&PrismModel) -> usize| MODELS.iter().map(|m| f(m) as f32).collect::>(); + let want: [(&str, Vec); 11] = [ + ("PRISM_TRIS", TRIS.iter().flatten().copied().collect()), + ("PRISM_POLY", POLYS.iter().flatten().copied().collect()), + ("PRISM_BOXES", BOXES.iter().flatten().copied().collect()), + ("PRISM_BOX_START", per_model(|m| m.box_start)), + ("PRISM_BOX_COUNT", per_model(|m| m.box_count)), + ("PRISM_TRI_START", per_model(|m| m.tri_start)), + ("PRISM_TRI_COUNT", per_model(|m| m.tri_count)), + ("PRISM_SIL_START", per_model(|m| m.sil_start)), + ("PRISM_SIL_COUNT", per_model(|m| m.sil_count)), + ("PRISM_OUT_START", per_model(|m| m.out_start)), + ("PRISM_OUT_COUNT", per_model(|m| m.out_count)), + ]; + let wgsl = include_str!("vk_shaders/layer.wgsl"); + for field in [ + format!("tris: array, {}>", TRIS.len()), + format!("poly: array, {}>", POLYS.len()), + format!("boxes: array, {}>", BOXES.len()), + ] { + assert!(wgsl.contains(&field), "vk_shaders/layer.wgsl : `{field}` attendu dans PrismMesh"); + } + for (file, src, tables) in [ + ("shaders.hlsl", include_str!("shaders.hlsl"), &want[..]), + ("shaders.metal", include_str!("shaders.metal"), &want[..]), + ("vk_shaders/layer.wgsl", wgsl, &want[3..]), + ] { + for (name, values) in tables { + let got = table(src, name); + assert_eq!(got.len(), values.len(), "{file} : {name}, nombre de valeurs"); + for (i, (g, w)) in got.iter().zip(values).enumerate() { + assert!((g - w).abs() <= 1e-6 + 1e-6 * w.abs(), "{file} : {name}[{i}] = {g} au lieu de {w}"); + } + } + } + } + + /// Chaque triangle du cristal est dans une boîte et une seule, qui le contient : le shader ne + /// teste les triangles d'une boîte que si le rayon la touche, un triangle hors de sa boîte + /// disparaîtrait par endroits. + #[test] + fn each_crystal_triangle_sits_in_its_box() { + use crate::prism_mesh::{BOXES, MODELS, TRIS}; + for (k, m) in MODELS.iter().enumerate() { + let mut next = m.tri_start; + for b in m.box_start..m.box_start + m.box_count { + let (lo, hi) = (BOXES[2 * b], BOXES[2 * b + 1]); + let (first, count) = (lo[3] as usize, hi[3] as usize); + assert_eq!(first, next, "modèle {k}, boîte {b} : triangles non contigus"); + next += count; + for t in TRIS[4 * first..4 * (first + count)].chunks(4) { + for e in [[0.0; 3], [t[1][0], t[1][1], t[1][2]], [t[2][0], t[2][1], t[2][2]]] { + let v = [t[0][0] + e[0], t[0][1] + e[1], t[0][2] + e[2]]; + assert!((0..3).all(|a| v[a] >= lo[a] && v[a] <= hi[a]), "modèle {k}, boîte {b} : sommet {v:?} dehors"); + } + } + } + assert_eq!(next, m.tri_start + m.tri_count, "modèle {k} : triangles hors des boîtes"); + } + } } diff --git a/crates/compositor/src/shaders.hlsl b/crates/compositor/src/shaders.hlsl index 30fbf1bd54..8478823035 100644 --- a/crates/compositor/src/shaders.hlsl +++ b/crates/compositor/src/shaders.hlsl @@ -1,5 +1,12 @@ // Compositeur — un draw par calque (quad). NV12->RGB maison (E1), coins arrondis SDF (E2). // Tout écrit depuis les maths (§7), rien repris de l'ancien paradigme. +// +// `ps_main` est compilé DEUX fois (`build.rs`) : sans `LAYER_MODELS` pour tous les calques, et +// avec, en `ps_main_models`, pour les seuls modèles 3D (modes 15 à 17, `LayerCB::needs_models`). +// Un pixel shader alloue à chaque draw les registres de sa branche la plus lourde, et ce sont +// ces modèles ray-tracés : le même shader les portait pour le fond, l'ombre et la vidéo, qui +// couvrent toute la sortie. Mesuré sur la version WGSL (RADV) : 168 VGPR, 6 vagues par SIMD au +// lieu de 32, et un export Linux deux fois plus lent ; sans eux, 48. cbuffer Layer : register(b0) { @@ -7,7 +14,7 @@ cbuffer Layer : register(b0) float4 src; // u0,v0,u1,v1 dans l'espace source 0..1 ; modes 15 et 17 : décalage px du rayon, P, U float2 quad_px; // taille du quad en pixels (pour les SDF) float radius_px; // rayon des coins arrondis en px (0 = aucun) ; mode 17 : rayon des coins hauts du corps (unités du modèle) - float mode; // 0 = vidéo NV12, 1 = couleur pleine, 2 = ombre portée, 4 = curseur, 15 = curseur 3D, 16 = impact du clic, 17 = appareil modelé, 18 = écran cadré flouté en bloc + float mode; // 0 = vidéo NV12, 1 = couleur pleine, 2 = ombre portée, 4 = curseur, 15 = curseur 3D, 16 = impact du clic, 17 = appareil modelé, 18 = écran cadré flouté en bloc, 19 = curseur de verre à plat float4 color; // couleur pleine / teinte (ombre : rgb + opacité dans a) ; mode 15 : coin du sprite, écrasement, opacité ; mode 17 : .r = l'appareil (1 portable, 2 téléphone, 3 moniteur), .g = thème sombre, .b = rayon des coins bas du corps, .a = opacité float4 fx; // fx.x = spread ombre (px), fx.y,fx.z libres ; mode 15 : rotation du plan (rad), tangage ; mode 17 : rotation du plan (rad), épaisseur float4 src_prev; // src à la frame précédente (flou de mouvement par vélocité) ; mode 15 : hotspot, lacet ; mode 17 : marges du corps (unités du modèle) @@ -81,6 +88,7 @@ float3 sample_yuv(float2 uv) return yuv709_limited(y, cbcr); } +// La même, au niveau 0 : lisible dans une boucle ou une branche (le cristal du mode 15). // SDF segment à bouts ronds — la primitive des flèches d'annotation, dont les tracés SVG sont // trois segments `stroke-linecap="round"` (cf. ArrowSvgs.tsx). float sd_segment(float2 p, float2 a, float2 b) @@ -105,9 +113,13 @@ float sd_segment(float2 p, float2 a, float2 b) float sd_round_rect(float2 p, float2 halfsz, float r) { float hmin = min(halfsz.x, halfsz.y); - if (r <= 0.0 || hmin <= 0.0) + float2 q0 = abs(p) - halfsz; + // Hors des coins, la distance est celle du rectangle vif quel que soit l'exposant : un seul + // axe déborde de l'étendue E, qui s'y ajoute puis s'en retranche. E ne dépasse pas 1,42 r + // (n = 3) : sous 1,5 r du bord sur un axe, on rend la boîte sans calculer `n` ni E, deux + // transcendantes de moins sur presque tous les pixels d'un calque arrondi. + if (r <= 0.0 || hmin <= 0.0 || min(q0.x, q0.y) <= -min(1.5 * r, hmin)) { - float2 q0 = abs(p) - halfsz; return length(max(q0, 0.0)) + min(max(q0.x, q0.y), 0.0); } float n = 3.0 - smoothstep(0.5, 1.0, r / hmin); @@ -587,13 +599,12 @@ static const float MODEL_CONTACT_ALPHA = 0.5; // curseur 1, x à droite, y VERS LE HAUT, z vers la caméra, l'écran en z = 0 et le modèle posé à // SCULPT_HOVER au-dessus. `sculpt_point` y amène un point du repère du modèle. Identifiant : // 1 + 2 × thème + forme ; thèmes 0 Studio Ink, 1 Prism Glow, 2 Pop Coral, 3 Pixel Candy, -// 4 Star Sprout ; formes 0 flèche, 1 main. Id des matières : 1 corps, 2 liseré ou manchette, -// 3 étoile, 4 feuilles, 5 yeux, 6 face des voxels, 7 couche violette, 8 cristal. +// 4 Star Sprout ; formes 0 flèche, 1 main. Id des matières : 1 corps, 2 bande, calque ou +// manchette, 3 tirets, calque corail ou étoile, 4 feuilles, 5 trait des thèmes cerclés (et +// yeux), 6 cubes de Pixel Candy, 8 serti de Prism Glow, 9 son cristal (plus bas, `prism_*`). static const float SCULPT_SCALE = 0.85; static const float SCULPT_HOVER = 0.05; static const float SCULPT_VOX = 0.0625; -static const float SCULPT_AR_ROUND = 0.03; -static const float SCULPT_HAND_ZC = 0.185; // Hauteur, dans le prototype, du z = 0 du modèle : le dessus de la pointe de la flèche, l'axe du // bout de l'index. static const float SCULPT_ZREF_ARROW = 0.2; @@ -619,13 +630,6 @@ float2 s_opu(float2 a, float2 b) return a.x < b.x ? a : b; } -float s_capsule(float3 p, float3 a, float3 b, float r) -{ - float3 pa = p - a, ba = b - a; - float h = saturate(dot(pa, ba) / dot(ba, ba)); - return length(pa - ba * h) - r; -} - float s_round_box(float3 p, float3 b, float r) { float3 q = abs(p) - b + r; @@ -652,25 +656,91 @@ float2 s_rot(float2 v, float a) return float2(c * v.x - s * v.y, s * v.x + c * v.y); } -// La flèche : polygone à sept sommets, pointe à l'origine. -static const float2 SCULPT_ARROW[7] = { - float2(0.0, 0.0), float2(0.0, -0.86), float2(0.215, -0.665), float2(0.37, -1.0), - float2(0.53, -0.93), float2(0.38, -0.60), float2(0.64, -0.60) +float s_star5(float2 p, float r, float rf) +{ + const float2 k1 = float2(0.809016994375, -0.587785252292); + const float2 k2 = float2(-0.809016994375, -0.587785252292); + p.x = abs(p.x); + p -= 2.0 * max(dot(k1, p), 0.0) * k1; + p -= 2.0 * max(dot(k2, p), 0.0) * k2; + p.x = abs(p.x); + p.y -= r; + float2 ba = rf * float2(-k1.y, k1.x) - float2(0.0, 1.0); + float h = clamp(dot(p, ba) / dot(ba, ba), 0.0, r); + return length(p - ba * h) * sign(p.y * ba.x - p.x * ba.y); +} + +// Lentille : deux disques de rayon r aux centres à ±d sur x, une feuille pointue à ses deux bouts +// (0, ±sqrt(r² - d²)). +float s_vesica(float2 p, float r, float d) +{ + p = abs(p); + float b = sqrt(r * r - d * d); + return (p.y - b) * d > p.x * b ? length(p - float2(0.0, b)) : length(p + float2(d, 0.0)) - r; +} + +// Les thèmes CERCLÉS (Studio Ink, Pop Coral, Star Sprout) suivent la planche 3D de référence +// (`design/cursors/3d-concept.png`) : le trait de leur dessin 2D n'est pas jeté, il devient la +// matière qui porte tout. Chaque pièce (la flèche, le gant, la manchette, les feuilles, l'étoile) +// est un plateau sous sa silhouette, un jonc sur le trait et un coussin de couleur bombé dedans. +// RIM_POLY : sept sommets par polygone (un sommet coupe un côté en deux quand il en faut six). +// D'abord la couleur de la flèche, commune aux trois thèmes (le trait l'élargit de 0,06, sa pointe +// arrondie touche le hotspot), puis les paumes de Star Sprout, de Studio Ink et de Pop Coral, +// arrondies de 0,05. +static const float2 RIM_POLY[28] = { + float2(0.0, -0.06), float2(0.0, -0.7712), float2(0.165, -0.6325), float2(0.313, -0.9318), + float2(0.4234, -0.8557), float2(0.2672, -0.5686), float2(0.456, -0.5393), + float2(-0.173, -0.5237), float2(-0.016, -0.75), float2(0.329, -0.75), float2(0.411, -0.592), + float2(0.411, -0.43), float2(0.19, -0.435), float2(-0.03, -0.44), + float2(-0.1825, -0.6215), float2(0.0037, -0.9195), float2(0.3462, -0.9195), float2(0.467, -0.6906), + float2(0.467, -0.49), float2(0.2, -0.49), float2(-0.03, -0.49), + float2(-0.1807, -0.5534), float2(0.0194, -0.872), float2(0.3485, -0.872), float2(0.47, -0.6795), + float2(0.47, -0.49), float2(0.2, -0.49), float2(-0.03, -0.49) +}; + +// Les trois gants, mesurés sur leur dessin (g : 0 Star Sprout, 1 Studio Ink, 2 Pop Coral), neuf +// lignes chacun : les segments de l'index, des trois doigts repliés et du pouce, ceux des trois +// fentes entre les doigts, puis le V entre le pouce et l'index (un point du bord du pouce et la +// normale qui entre dans le V). RIM_GLOVE_R, trois lignes par gant : les rayons des quatre doigts ; +// ceux du pouce et des trois fentes ; le bord de l'index que longe le V, le fond du V, le trait. +static const float4 RIM_GLOVE[27] = { + float4(0.0, -0.1077, 0.0, -0.6), float4(0.1493, -0.3279, 0.1493, -0.6), + float4(0.2863, -0.3654, 0.2863, -0.6), float4(0.4193, -0.4043, 0.4193, -0.6), + float4(-0.2, -0.478, -0.075, -0.625), float4(0.0788, -0.25, 0.0788, -0.3992), + float4(0.2188, -0.3, 0.2188, -0.4234), float4(0.3552, -0.33, 0.3552, -0.4534), + float4(-0.0845, -0.5386, 0.762, 0.648), + float4(0.0, -0.1186, 0.0, -0.65), float4(0.1614, -0.3838, 0.1614, -0.65), + float4(0.3177, -0.427, 0.3177, -0.65), float4(0.4645, -0.4848, 0.4645, -0.65), + float4(-0.215, -0.535, -0.1208, -0.6864), float4(0.0832, -0.3, 0.0832, -0.48), + float4(0.2414, -0.35, 0.2414, -0.482), float4(0.3959, -0.41, 0.3959, -0.533), + float4(-0.122, -0.5718, 0.867, 0.498), + float4(0.0, -0.122, 0.0, -0.65), float4(0.172, -0.387, 0.172, -0.65), + float4(0.3267, -0.427, 0.3267, -0.65), float4(0.4727, -0.4867, 0.4727, -0.65), + float4(-0.199, -0.483, -0.102, -0.645), float4(0.0927, -0.3, 0.0927, -0.476), + float4(0.2493, -0.35, 0.2493, -0.478), float4(0.402, -0.41, 0.402, -0.52), + float4(-0.0993, -0.5267, 0.858, 0.514) +}; +static const float4 RIM_GLOVE_R[9] = { + float4(0.0552, 0.047, 0.045, 0.0375), float4(0.052, 0.0235, 0.0225, 0.0225), float4(0.0552, -0.548, 0.0525, 0.0), + float4(0.064, 0.0585, 0.0585, 0.0515), float4(0.06, 0.0197, 0.0178, 0.0172), float4(0.0648, -0.6, 0.058, 0.0), + float4(0.066, 0.0553, 0.0553, 0.0507), float4(0.063, 0.024, 0.022, 0.02), float4(0.0647, -0.555, 0.056, 0.0) }; -float s_arrow2(float2 p) +// Distance signée au polygone de RIM_POLY qui commence en `base` : distance aux arêtes, signe par +// la parité des traversées d'une demi-droite. +float s_rim_poly(float2 p, int base) { - float d = dot(p - SCULPT_ARROW[0], p - SCULPT_ARROW[0]); + float d = dot(p - RIM_POLY[base], p - RIM_POLY[base]); float s = 1.0; - int j = 6; - [loop] for (int i = 0; i < 7; i++) + int j = base + 6; + [loop] for (int i = base; i < base + 7; i++) { - float2 e = SCULPT_ARROW[j] - SCULPT_ARROW[i]; - float2 w = p - SCULPT_ARROW[i]; + float2 e = RIM_POLY[j] - RIM_POLY[i]; + float2 w = p - RIM_POLY[i]; float2 b = w - e * saturate(dot(w, e) / dot(e, e)); d = min(d, dot(b, b)); - bool c0 = p.y >= SCULPT_ARROW[i].y; - bool c1 = p.y < SCULPT_ARROW[j].y; + bool c0 = p.y >= RIM_POLY[i].y; + bool c1 = p.y < RIM_POLY[j].y; bool c2 = e.x * w.y > e.y * w.x; if ((c0 && c1 && c2) || (!c0 && !c1 && !c2)) { @@ -681,311 +751,1340 @@ float s_arrow2(float2 p) return s * sqrt(d); } -float s_star5(float2 p, float r, float rf) +// Le gant `g` dans le plan, autour de sa paume `palm` : (silhouette, couleur). La silhouette unit +// la paume, l'index levé, trois doigts repliés et le pouce ; la couleur en est creusée des +// rainures du dessin, trois fentes entre les doigts et le V entre le pouce et l'index. +float2 s_rim_glove(float2 p, int g, float palm) { - const float2 k1 = float2(0.809016994375, -0.587785252292); - const float2 k2 = float2(-0.809016994375, -0.587785252292); - p.x = abs(p.x); - p -= 2.0 * max(dot(k1, p), 0.0) * k1; - p -= 2.0 * max(dot(k2, p), 0.0) * k2; - p.x = abs(p.x); - p.y -= r; - float2 ba = rf * float2(-k1.y, k1.x) - float2(0.0, 1.0); - float h = clamp(dot(p, ba) / dot(ba, ba), 0.0, r); - return length(p - ba * h) * sign(p.y * ba.x - p.x * ba.y); + float4 r0 = RIM_GLOVE_R[3 * g]; + float4 r1 = RIM_GLOVE_R[3 * g + 1]; + float4 r2 = RIM_GLOVE_R[3 * g + 2]; + int i = 9 * g; + float f = min(sd_segment(p, RIM_GLOVE[i].xy, RIM_GLOVE[i].zw) - r0.x, + sd_segment(p, RIM_GLOVE[i + 1].xy, RIM_GLOVE[i + 1].zw) - r0.y); + f = min(f, sd_segment(p, RIM_GLOVE[i + 2].xy, RIM_GLOVE[i + 2].zw) - r0.z); + f = min(f, sd_segment(p, RIM_GLOVE[i + 3].xy, RIM_GLOVE[i + 3].zw) - r0.w); + f = min(f, sd_segment(p, RIM_GLOVE[i + 4].xy, RIM_GLOVE[i + 4].zw) - r1.x); + float grooves = min(sd_segment(p, RIM_GLOVE[i + 5].xy, RIM_GLOVE[i + 5].zw) - r1.y, + sd_segment(p, RIM_GLOVE[i + 6].xy, RIM_GLOVE[i + 6].zw) - r1.z); + grooves = min(grooves, sd_segment(p, RIM_GLOVE[i + 7].xy, RIM_GLOVE[i + 7].zw) - r1.w); + float sil = s_smin(palm, f, 0.03); + float4 v = RIM_GLOVE[i + 8]; + float wedge = max(max(-dot(p - v.xy, v.zw), p.x + r2.x), r2.y - p.y); + return float2(sil, max(sil, -min(grooves, wedge))); } -// La flèche en volume : extrusion arrondie, bombée d'un dôme qui plafonne avant l'axe médian de -// la silhouette (un dôme qui monterait encore y plierait le dessus). -float s_arrow_solid(float3 p, float h, float re, float dome) +// Une pièce cerclée : (distance, matière). `d` : le bord intérieur du trait dans le plan, `sil` : +// le même avant les rainures, `dc` : le coussin. Le plateau (matière 5) monte de SCULPT_HOVER à +// `zt` sous la silhouette élargie du trait `w`. Le jonc (matière 5), un tore de rayon w/2 à la +// hauteur `zt`, suit le milieu du trait : dans une rainure plus étroite que le trait il n'en reste +// que la crête, un muret entre les doigts. Le coussin de la matière `mat` sort du plateau, haut de +// `h` au bord, bombé sur 0,045 puis de `bump` ; sa hauteur variable penche le champ, d'où la +// distance minorée (× 0,8). +float2 s_piece(float d, float dc, float sil, float z, float w, float zt, float h, float bump, float mat) { - float d2 = s_arrow2(p.xy) - SCULPT_AR_ROUND; - float hh = h + dome * smoothstep(0.0, 0.07, -d2); - return s_extrude(d2, p.z - (SCULPT_HOVER + h + dome), hh, re); + float tray = s_extrude(sil - w, z - 0.5 * (SCULPT_HOVER + zt), 0.5 * (zt - SCULPT_HOVER), 0.012); + float bead = length(float2(d - 0.5 * w, z - zt)) - 0.5 * w; + float u = saturate(-dc / 0.045); + float cushion = 0.8 * s_extrude(dc, z - zt, h + 0.022 * u * (2.0 - u) + bump, 0.75 * h); + return s_opu(float2(min(tray, bead), 5.0), float2(cushion, mat)); } -// Le liseré de Studio Ink : un jonc posé sur le dessus, en retrait du bord. -float s_piping(float3 p, float ztop) +// Pop Coral est du papier découpé : des feuilles à plat, au dessus plat et au bord à peine +// cassé. La feuille marine sous la silhouette élargie du trait `w`, de SCULPT_HOVER à 0,15 ; la +// feuille de couleur (matière 1) posée dessus, 0,02 d'épaisseur, découpée des rainures entre les +// doigts qui laissent voir la marine. +float2 s_paper(float d, float sil, float z, float w) { - float d2 = s_arrow2(p.xy) - SCULPT_AR_ROUND; - return length(float2(d2 + 0.075, p.z - ztop)) - 0.017; + float navy = s_extrude(sil - w, z - 0.5 * (SCULPT_HOVER + 0.15), 0.5 * (0.15 - SCULPT_HOVER), 0.006); + float sheet = s_extrude(d, z - 0.16, 0.01, 0.004); + return s_opu(float2(navy, 5.0), float2(sheet, 1.0)); } -// Le gant : l'index levé, trois doigts repliés, le pouce, fondus dans la paume. -float s_glove(float3 p, float zc) +// Bosse de rayon r en c, 1 au centre et 0 au bord : un coussin bombé sans pli. Une hauteur tirée +// de la distance au bord plierait le coussin sur l'axe médian de sa silhouette. +float s_bump(float2 p, float2 c, float r) { - float index = s_capsule(p, float3(0.0, -0.10, zc), float3(0.0, -0.52, zc), 0.098); - float palm = s_round_box(p - float3(0.185, -0.70, zc), float3(0.255, 0.19, 0.105), 0.1); - float f1 = s_capsule(p, float3(0.17, -0.57, zc + 0.012), float3(0.17, -0.41, zc + 0.045), 0.086); - float f2 = s_capsule(p, float3(0.31, -0.59, zc + 0.01), float3(0.31, -0.45, zc + 0.04), 0.08); - float f3 = s_capsule(p, float3(0.435, -0.625, zc + 0.005), float3(0.435, -0.52, zc + 0.03), 0.07); - float thumb = s_capsule(p, float3(0.03, -0.77, zc + 0.03), float3(-0.165, -0.60, zc + 0.065), 0.082); - float d = s_smin(palm, min(f1, min(f2, f3)), 0.035); - d = s_smin(d, index, 0.05); - return s_smin(d, thumb, 0.05); + float k = saturate(1.0 - dot(p - c, p - c) / (r * r)); + return k * k; } -// La manchette, ronde autour du poignet (elliptique en xz) : le gant y entre. -float s_cuff(float3 p, float zc) +// Un tiret du clic de Pop Coral : capsule inégale de a (rayon ra) à b (rayon rb), exacte. +float s_dash(float2 p, float2 a, float2 b, float ra, float rb) { - float3 q = p - float3(0.185, -0.925, zc); - float2 ab = float2(0.272, 0.12); - float e = (length(q.xz / ab) - 1.0) * min(ab.x, ab.y); - return s_extrude(e, q.y, 0.07, 0.06); + p -= a; + b -= a; + float hb = dot(b, b); + float2 q = float2(abs(dot(p, float2(b.y, -b.x))), dot(p, b)) / hb; + float2 c = float2(sqrt(hb - (ra - rb) * (ra - rb)), ra - rb); + float k = c.x * q.y - c.y * q.x; + if (k < 0.0) + { + return sqrt(hb * dot(q, q)) - ra; + } + if (k > c.x) + { + return sqrt(hb * (dot(q, q) + 1.0 - 2.0 * q.y)) - rb; + } + return dot(c, q) - ra; } -// L'étoile de Star Sprout, face à la caméra, centrée en c : son visage et ses deux feuilles. -float2 s_sprout(float3 p, float3 c) +// Les thèmes cerclés. La flèche : Star Sprout menthe, Pop Coral corail, et Studio Ink noir, dont le +// trait se partage en un jonc noir au bord et une bande ivoire en relief, le champ restant le +// dessus du plateau. Le gant ivoire, jaune ou ivoire. Pop Coral est en papier découpé +// (`s_paper`), ses calques et ses tirets aussi : des feuilles au dessus plat. Puis ce que chacun ajoute devant : le calque +// jaune de la flèche, le calque corail du gant et les tirets du clic de Pop Coral ; la manchette, +// l'étoile, ses feuilles et ses yeux de Star Sprout, lus dans le repère de l'étoile (centre `c`, +// tournée, en unités de son rayon `rs`). Matières : 1 corps, 2 bande ivoire, calque jaune ou +// manchette, 3 tirets, calque corail ou étoile, 4 feuilles, 5 le trait (plateaux, joncs, yeux). +float2 s_rimmed(float3 p, int theme, int shape) { - float3 q = p - c; - float st2 = s_star5(q.xy, 0.125, 0.52) - 0.022; - float2 r = float2(s_extrude(st2, q.z, 0.038, 0.034), 3.0); - float3 e = float3(abs(q.x) - 0.032, q.y + 0.005, q.z - 0.036); - r = s_opu(r, float2(length(e) - 0.0135, 5.0)); - float3 l1 = q - float3(-0.04, 0.15, 0.0); - l1.xy = s_rot(l1.xy, -0.6); - float3 l2 = q - float3(0.045, 0.155, 0.0); - l2.xy = s_rot(l2.xy, 0.7); - float leaves = min(s_ellipsoid(l1, float3(0.03, 0.058, 0.02)), s_ellipsoid(l2, float3(0.03, 0.058, 0.02))); - return s_opu(r, float2(leaves, 4.0)); + bool arrow = shape == 0; + int g = theme == 4 ? 0 : (theme == 0 ? 1 : 2); + float poly = s_rim_poly(p.xy, arrow ? 0 : 7 * g + 7); + float2 body = float2(poly, poly); + float w = 0.06, zt = 0.17, h = 0.03; + float bump = 0.022 * s_bump(p.xy, float2(0.15, -0.45), 0.3); + float back = 1e9; + if (!arrow) + { + w = RIM_GLOVE_R[3 * g + 2].z; + // Le gant, puis pour Pop Coral son calque corail : le même gant décalé en bas à gauche. Une + // boucle et non deux appels (FXC recopierait le gant) ; le calque est sauté quand `p` est + // au-dessus du plateau du gant, qui en est alors plus près que lui. + [loop] for (int i = 0; i < 2; i++) + { + float2 q = p.xy + (i == 1 ? float2(0.03, 0.04) : float2(0.0, 0.0)); + float2 v = s_rim_glove(q, g, (i == 1 ? s_rim_poly(q, 7 * g + 7) : poly) - 0.05); + if (i == 1) + { + back = v.x - w; + } + else + { + body = v; + if (theme != 2 || v.x - w <= 0.0) + { + break; + } + } + } + zt = 0.185; + h = 0.028; + bump = 0.03 * s_bump(p.xy, float2(0.19, -0.62), 0.28); + } + float dc = body.y; + if (arrow && theme == 0) + { + // Studio Ink : pas de coussin, le jonc sur le bord extérieur du trait. + body -= 0.015; + w = 0.045; + dc = 1.0; + } + float2 r = theme == 2 ? s_paper(body.y, body.x, p.z, w) + : s_piece(body.y, dc, body.x, p.z, w, zt, h, bump, 1.0); + if (theme == 0) + { + if (arrow) + { + float band = s_extrude(abs(poly + 0.0075) - 0.0225, p.z - zt, 0.028, 0.012); + r = s_opu(r, float2(band, 2.0)); + } + return r; + } + if (theme == 2) + { + float dash; + if (arrow) + { + // Le calque jaune, décalé en bas à gauche, derrière le plateau. + float back = s_rim_poly(p.xy + float2(0.025, 0.05), 0) - 0.06; + r = s_opu(r, float2(s_extrude(back, p.z - 0.08, 0.03, 0.006), 2.0)); + dash = min(s_dash(p.xy, float2(-0.1179, -0.1374), float2(-0.1799, -0.0443), 0.028, 0.045), + s_dash(p.xy, float2(-0.1347, -0.2493), float2(-0.248, -0.2056), 0.026, 0.042)); + } + else + { + r = s_opu(r, float2(s_extrude(back, p.z - 0.08, 0.03, 0.006), 3.0)); + dash = min(s_dash(p.xy, float2(0.216, -0.1695), float2(0.2593, -0.0685), 0.028, 0.042), + s_dash(p.xy, float2(0.3054, -0.2339), float2(0.3949, -0.1652), 0.0275, 0.041)); + } + return s_opu(r, float2(s_extrude(dash, p.z - 0.11, 0.06, 0.006), 3.0)); + } + if (!arrow) + { + // La manchette : un rectangle arrondi, cintré en sourire comme au dessin. + float2 cq = p.xy - float2(0.1541, -0.9047); + float2 bq = abs(float2(cq.x, cq.y - 0.2066 * cq.x * cq.x)) - float2(0.214, 0.0361); + float cuff = length(max(bq, 0.0)) + min(max(bq.x, bq.y), 0.0) - 0.03; + r = s_opu(r, s_piece(cuff, cuff, cuff, p.z, 0.0477, 0.225, 0.025, 0.0, 2.0)); + } + float2 c = arrow ? float2(0.6118, -0.8079) : float2(0.15, -0.885); + float rs = arrow ? 0.1678 : 0.128; + float zs = arrow ? 0.23 : 0.285; + float2 q = s_rot(p.xy - c, arrow ? 0.2443 : 0.0) / rs; + float star = (s_star5(q, 0.82, 0.55) - 0.18) * rs; + float leaves = min(s_vesica(s_rot(q - float2(-0.33, 1.38), -0.925), 0.4296, 0.2626), + s_vesica(s_rot(q - float2(0.53, 1.37), 0.873), 0.4296, 0.2626)) * rs; + r = s_opu(r, s_piece(leaves, leaves, leaves, p.z, 0.038, zs - 0.02, 0.018, 0.0, 4.0)); + r = s_opu(r, s_piece(star, star, star, p.z, 0.038, zs, 0.022, 0.018 * s_bump(q, float2(0.0, 0.0), 1.0), 3.0)); + // Les yeux : deux ovales marine qui affleurent du coussin. + float3 e = float3(abs(q.x) - 0.25, q.y - 0.08, (p.z - zs - 0.056) / rs); + return s_opu(r, float2(s_ellipsoid(e, float3(0.075, 0.13, 0.1)) * rs, 5.0)); } -// Pixel Candy : une ligne par entier, bit c = colonne c, ligne 0 juste sous la pointe. La flèche -// est le polygone échantillonné à 16 cellules par unité (`sculpt.rs` dit comment la régénérer) ; -// la main est dessinée à la main, un gant échantillonné fondant ses doigts. Les tables *BACK -// portent la couche violette, la face dilatée d'une cellule, décalée d'une ligne et d'une colonne. -static const int SCULPT_ARROWPIX[16] = { 0, 1, 3, 7, 31, 63, 127, 255, 511, 63, 55, 115, 113, 224, 224, 64 }; -static const int SCULPT_ARROWBACK[18] = { - 3, 7, 15, 31, 63, 127, 255, 511, 1023, 2047, 4095, 255, 511, 511, 999, 995, 960, 192 -}; -static const int SCULPT_HANDPIX[15] = { 4, 14, 14, 14, 110, 878, 7022, 7022, 8190, 8191, 8191, 8191, 8190, 4092, 4092 }; -static const int SCULPT_HANDBACK[17] = { - 28, 62, 62, 62, 510, 4094, 32766, 32766, 32766, 32767, 32767, 32767, 32767, 32767, 32766, 16380, 16380 -}; -// La couche violette de la main, une plage de colonnes par ligne (toutes pleines) : la distance -// exacte à son contour, pour l'ombre portée. -static const float2 SCULPT_HANDSPAN[17] = { - float2(2.0, 4.0), float2(1.0, 5.0), float2(1.0, 5.0), float2(1.0, 5.0), float2(1.0, 8.0), - float2(1.0, 11.0), float2(1.0, 14.0), float2(1.0, 14.0), float2(1.0, 14.0), float2(0.0, 14.0), - float2(0.0, 14.0), float2(0.0, 14.0), float2(0.0, 14.0), float2(0.0, 14.0), float2(1.0, 14.0), - float2(2.0, 13.0), float2(2.0, 13.0) -}; +// ---- Glass Lens ---- +// Une lentille de verre dans un trait sombre, et au-delà un liseré de verre. Ses formes sont la +// flèche et le gant de Studio Ink (RIM_POLY 0, RIM_GLOVE 1) ; mesurées depuis le bord de la +// lentille, le trait en occupe GLASS_LINE et le contour entier GLASS_RIM (miroir de `sculpt.rs`). +// En volume (mode 15, `s_glass`) : un socle de verre sous toute la silhouette, de SCULPT_HOVER à +// GLASS_TOP, arrondi sur toute la largeur du liseré ; la lentille, bombée de GLASS_DOME, qui en +// sort dans le trait ; le trait, une bande graphite au dessus plat sur le bord de la lentille. +// À plat (mode 19, `cursor_glass`), le même relief vu de face. Le verre (matière GLASS_MAT) entre +// par sa face bombée et sort par le dessous plat du socle vers l'image (`glass_shade`). +static const float GLASS_LINE = 0.04; +static const float GLASS_RIM = 0.085; +static const float GLASS_TOP = 0.18; +static const float GLASS_DOME = 0.03; +// La lentille monte sur cette largeur depuis son bord, puis reste à plat. +static const float GLASS_RISE = 0.08; +static const float GLASS_IOR = 1.5; +static const float GLASS_DISPERSION = 0.012; +static const float GLASS_MAT = 10.0; +// À plat : l'air sous le socle, que le mode 15 tient de la garde au sol du modèle (prototype). +static const float GLASS_GAP_FLAT = 0.15; +// L'écart des deux prises de part et d'autre du point réfracté, rapporté au décalage +// (`glass_smear`). +static const float GLASS_SMEAR = 0.15; +// La lueur propre du verre (linéaire) : sur un fond noir, la lentille reste un verre fumé visible. +static const float GLASS_LIFT = 0.02; +// L'ombre de la lentille, rapportée à celle d'un modèle opaque : le verre laisse passer la lumière. +static const float GLASS_SHADOW = 0.6; + +// Glass Lens dans le plan (prototype) : x = la silhouette, y = la lentille (distances signées, +// négatives dedans). Le trait est entre 0 et GLASS_LINE de la lentille, le liseré au-delà ; les +// fentes entre les doigts, creusées dans la lentille, s'y remplissent du trait. Les formes +// descendent de ce que le contour a de plus que le trait de Studio Ink : la pointe et le bout de +// l'index restent au hotspot. +float2 s_glass2(float2 p, int shape) +{ + if (shape == 0) + { + float poly = s_rim_poly(p + float2(0.0, GLASS_RIM - 0.06), 0); + return float2(poly - GLASS_RIM, poly); + } + float2 q = p + float2(0.0, GLASS_RIM - RIM_GLOVE_R[5].z); + float2 v = s_rim_glove(q, 1, s_rim_poly(q, 14) - 0.05); + return float2(v.x - GLASS_RIM, v.y); +} -// Origine de la grille : la flèche part de sa pointe, la main de 2,5 cellules à sa gauche. -float2 s_grid_origin(int shape) +// La hauteur du verre au-dessus de GLASS_TOP en fonction de la distance `d` à la lentille +// (`s_glass2`), et sa pente : x = la hauteur, y = sa dérivée. La lentille monte de GLASS_DOME sur +// GLASS_RISE ; le liseré tombe en quart de rond jusqu'au bord, l'arrondi du socle (`s_glass`). Le +// trait n'est pas du verre : rien ne s'y lit. +float2 s_glass_relief(float d) { - return shape == 0 ? float2(0.0, 0.0) : float2(-2.5 * SCULPT_VOX, 0.0); + if (d < 0.0) + { + float u = saturate(-d / GLASS_RISE); + return float2(GLASS_DOME * u * (2.0 - u), -2.0 * GLASS_DOME * (1.0 - u) / GLASS_RISE); + } + float w = GLASS_RIM - GLASS_LINE; + float x = saturate((d - GLASS_LINE) / w); + float s = sqrt(max(1.0 - x * x, 0.05)); + return float2(w * (s - 1.0), -x / s); } -// Le bit `c` de la ligne `r` d'une table de `rows` lignes et `cols` colonnes. FXC évalue les deux -// côtés d'un `&&` : l'index est ramené dans la table avant la lecture, le test fait le reste. -bool s_bit(int row, int r, int c, int rows, int cols) +// Glass Lens en volume, repère du prototype : (distance, matière). +float2 s_glass(float3 p, int shape) { - return r >= 0 && r < rows && c >= 0 && c < cols && ((row >> clamp(c, 0, 31)) & 1) == 1; + float2 g = s_glass2(p.xy, shape); + float half_h = 0.5 * (GLASS_TOP - SCULPT_HOVER); + float slab = s_extrude(g.x, p.z - SCULPT_HOVER - half_h, half_h, min(GLASS_RIM - GLASS_LINE, half_h)); + float u = saturate(-g.y / GLASS_RISE); + float dome = 0.8 * s_extrude(g.y, p.z - GLASS_TOP, 0.009 + GLASS_DOME * u * (2.0 - u), 0.008); + float band = s_extrude(abs(g.y - 0.5 * GLASS_LINE) - 0.5 * GLASS_LINE, p.z - GLASS_TOP, 0.016, 0.008); + return s_opu(float2(min(slab, dome), GLASS_MAT), float2(band, 5.0)); } -// La cellule `id` (colonne, ligne, y vers le haut) porte-t-elle un voxel de la face ? -bool s_occ(float2 id, int shape) +// Pixel Candy est un pixel art, dessiné une fois : la même grille donne ses PNG 2D et, ici, un cube +// par pixel. Une ligne par entier, bit c = colonne c, ligne 0 en haut. `PIX_BODY` : les pixels +// pleins ; `PIX_LINE`, `PIX_HI`, `PIX_SHADE` : ceux du contour prune, du reflet rose pâle et de +// l'ombre rose foncé, le reste est rose. `PIX_GRID` : colonnes, lignes et début de chaque forme +// dans les tables ; `PIX_ORIGIN` : le coin haut-gauche du pixel (0, 0) ; `PIX_BOX` (centre, +// demi-côtés) : la boîte de la forme ; `PIX_RECT`, bornés par `PIX_RECT_N` : la forme en +// rectangles, pour l'ombre. Tables générées par scripts/generate-pixel-candy-voxels.mjs, qui +// réécrit les trois shaders et les PNG : c'est là qu'on redessine une grille. +// +static const int PIX_BODY[32] = { 1, 3, 7, 15, 31, 63, 127, 255, 511, 1023, 2047, 127, 247, 243, 480, 192, 48, 120, 120, 120, 504, 4088, 32760, 65534, 65535, 65535, 65534, 32766, 32764, 16380, 16376, 16376 }; +static const int PIX_LINE[32] = { 1, 3, 5, 9, 17, 33, 65, 129, 257, 513, 1985, 73, 149, 147, 288, 192, 48, 72, 72, 72, 456, 3656, 29256, 37454, 32777, 32769, 32770, 16386, 16388, 8196, 8200, 16376 }; +static const int PIX_HI[32] = { 0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 18, 34, 32, 64, 0, 0, 16, 16, 16, 16, 16, 16, 16, 22, 18, 4, 4, 8, 8, 16, 0 }; +static const int PIX_SHADE[32] = { 0, 0, 0, 4, 8, 16, 32, 64, 128, 448, 40, 36, 64, 64, 128, 0, 0, 32, 32, 32, 32, 288, 2336, 18720, 16384, 16384, 16384, 8192, 8192, 4096, 8160, 0 }; +static const int PIX_RECT_N[3] = { 0, 17, 29 }; +static const int4 PIX_GRID[2] = { + int4(11, 16, 0, 0), + int4(16, 16, 16, 0) +}; +static const float2 PIX_ORIGIN[2] = { + float2(0.0, 0.0), + float2(-0.3125, 0.0) +}; +static const float4 PIX_BOX[2] = { + float4(0.3438, -0.5, 0.3438, 0.5), + float4(0.1875, -0.5, 0.5, 0.5) +}; +static const float4 PIX_RECT[29] = { + float4(0.0313, -0.0313, 0.0313, 0.0313), + float4(0.0625, -0.0938, 0.0625, 0.0313), + float4(0.0938, -0.1563, 0.0938, 0.0313), + float4(0.125, -0.2188, 0.125, 0.0313), + float4(0.1563, -0.2813, 0.1563, 0.0313), + float4(0.1875, -0.3438, 0.1875, 0.0313), + float4(0.2188, -0.4063, 0.2188, 0.0313), + float4(0.25, -0.4688, 0.25, 0.0313), + float4(0.2813, -0.5313, 0.2813, 0.0313), + float4(0.3125, -0.5938, 0.3125, 0.0313), + float4(0.3438, -0.6563, 0.3438, 0.0313), + float4(0.2188, -0.7188, 0.2188, 0.0313), + float4(0.0938, -0.7813, 0.0938, 0.0313), + float4(0.375, -0.8125, 0.125, 0.0625), + float4(0.0625, -0.8438, 0.0625, 0.0313), + float4(0.4375, -0.9063, 0.125, 0.0313), + float4(0.4375, -0.9688, 0.0625, 0.0313), + float4(0.0, -0.0313, 0.0625, 0.0313), + float4(0.0, -0.1563, 0.125, 0.0938), + float4(0.0625, -0.2813, 0.1875, 0.0313), + float4(0.1563, -0.3438, 0.2813, 0.0313), + float4(0.25, -0.4063, 0.375, 0.0313), + float4(0.2188, -0.4688, 0.4688, 0.0313), + float4(0.1875, -0.5625, 0.5, 0.0625), + float4(0.2188, -0.6563, 0.4688, 0.0313), + float4(0.1875, -0.7188, 0.4375, 0.0313), + float4(0.2188, -0.7813, 0.4063, 0.0313), + float4(0.1875, -0.8438, 0.375, 0.0313), + float4(0.2188, -0.9375, 0.3438, 0.0625) +}; +// + +// Le bit `c` d'une ligne. FXC évalue les deux côtés d'un `&&` : chaque index est ramené dans sa +// table avant la lecture, les tests de bornes font le reste. +bool s_pix_bit(int m, int c) { - int c = (int)id.x; - int r = -(int)id.y - 1; - if (shape == 0) - { - return s_bit(SCULPT_ARROWPIX[clamp(r, 0, 15)], r, c, 16, 16); - } - return s_bit(SCULPT_HANDPIX[clamp(r, 0, 14)], r, c, 15, 13); + return ((m >> clamp(c, 0, 31)) & 1) == 1; } -// … et un bloc de la couche violette. -bool s_occ_back(float2 id, int shape) +// Le pixel (c, r) de la forme `shape`, r compté vers le bas depuis la ligne 0, est-il plein ? +bool s_pix_body(int shape, int c, int r) { - int c = (int)id.x + 1; - int r = -(int)id.y; - if (shape == 0) - { - return s_bit(SCULPT_ARROWBACK[clamp(r, 0, 17)], r, c, 18, 17); - } - return s_bit(SCULPT_HANDBACK[clamp(r, 0, 16)], r, c, 17, 15); + int4 g = PIX_GRID[shape]; + return c >= 0 && c < g.x && r >= 0 && r < g.y && s_pix_bit(PIX_BODY[g.z + clamp(r, 0, g.y - 1)], c); } -// Chaque cellule pleine est un cube biseauté ; la couche violette est un étage plus bas. Hors du -// voisinage 3×3, une borne : une cellule au moins, et jamais plus près que la boîte de la forme. +// La cellule qui contient `p` (prototype, y vers le haut). +int2 s_pix_cell(float2 p, int shape) +{ + float2 o = PIX_ORIGIN[shape]; + return int2((int)floor((p.x - o.x) / SCULPT_VOX), (int)floor((o.y - p.y) / SCULPT_VOX)); +} + +// Un cube par pixel plein, tous de même hauteur, l'arête à peine cassée pour que chaque cube se +// lise. Hors du voisinage 3×3, une borne : une cellule au moins, et jamais plus près que la boîte +// de la forme. float2 s_voxels(float3 p, int shape) { - float2 o = s_grid_origin(shape); - float2 cell = floor((p.xy - o) / SCULPT_VOX); - float2 bc = shape == 0 ? float2(0.33, -0.5) : float2(0.25, -0.47); - float2 bh = shape == 0 ? float2(0.44, 0.61) : float2(0.48, 0.55); - float2 bq = max(abs(p.xy - bc) - bh, 0.0); + float4 b = PIX_BOX[shape]; + float2 bq = max(abs(p.xy - b.xy) - b.zw, 0.0); float bz = max(abs(p.z - (SCULPT_HOVER + 0.07)) - 0.07, 0.0); - float far = max(SCULPT_VOX, sqrt(dot(bq, bq) + bz * bz)); - float dF = far; - float dB = far; - const float3 half_cell = float3(0.5 * SCULPT_VOX, 0.5 * SCULPT_VOX, 0.04); + float d = max(SCULPT_VOX, sqrt(dot(bq, bq) + bz * bz)); + float2 o = PIX_ORIGIN[shape]; + int2 cell = s_pix_cell(p.xy, shape); + const float3 cube = float3(0.5 * SCULPT_VOX, 0.5 * SCULPT_VOX, 0.07); [loop] for (int j = -1; j <= 1; j++) { [loop] for (int i = -1; i <= 1; i++) { - float2 id = cell + float2(i, j); - float3 q = float3(p.xy - (o + (id + 0.5) * SCULPT_VOX), p.z); - if (s_occ(id, shape)) - { - dF = min(dF, s_round_box(q - float3(0.0, 0.0, SCULPT_HOVER + 0.1), half_cell, 0.009)); - } - if (s_occ_back(id, shape)) + int c = cell.x + i; + int r = cell.y + j; + if (s_pix_body(shape, c, r)) { - dB = min(dB, s_round_box(q - float3(0.0, 0.0, SCULPT_HOVER + 0.04), half_cell, 0.009)); + float3 q = float3(p.x - o.x - (c + 0.5) * SCULPT_VOX, p.y - o.y + (r + 0.5) * SCULPT_VOX, + p.z - SCULPT_HOVER - 0.07); + d = min(d, s_round_box(q, cube, 0.006)); } } } - return dF < dB ? float2(dF, 6.0) : float2(dB, 7.0); + return float2(d, 6.0); } -// Distance, dans le plan de l'écran, au contour de la couche violette de la main. -float s_pixel_hand_dist(float2 p) +// Distance, dans le plan de l'écran, au contour de la forme. +float s_pixel_outline(float2 p, int shape) { float d = 1e9; - [loop] for (int r = 0; r < 17; r++) + [loop] for (int i = PIX_RECT_N[shape]; i < PIX_RECT_N[shape + 1]; i++) { - float2 x = (SCULPT_HANDSPAN[r] + float2(-3.5, -2.5)) * SCULPT_VOX; - float2 y = float2(-r, 1.0 - r) * SCULPT_VOX; - float2 q = max(max(float2(x.x, y.x) - p, p - float2(x.y, y.y)), 0.0); - d = min(d, length(q)); - } - return d; -} - -// Prism Glow est taillé, pas peint : chaque pièce est un polyèdre convexe (le max de ses plans), -// les facettes sont planes et la silhouette polygonale. - -// Une arête d'un contour convexe de gemme (a -> b, sens trigonométrique) : sa paroi, une facette -// de rondiste raide, et deux facettes de couronne qui se rejoignent au milieu de l'arête. Les -// couronnes de toutes les arêtes se rejoignent en arêtes vives au-dessus du centre. -float s_gem_edge(float3 p, float2 a, float2 b, float z0) -{ - float2 e = b - a; - float len = length(e); - float2 d = e / len; - float2 q = p.xy - a; - float dist = dot(q, float2(-d.y, d.x)); - float along = abs(dot(q, d) - 0.5 * len); - float z = p.z - z0; - float girdle = (z - 0.035 - 2.2 * dist) * 0.4138; - float crown = (z - 0.07 - 0.6 * dist + 0.18 * along) * 0.8438; - return max(-dist, max(girdle, crown)); -} - -float s_gem_arrow(float3 p) -{ - float z0 = SCULPT_HOVER + 0.03; - float slab = max(p.z - z0 - 0.2, z0 - 0.03 - p.z); - float head = max(slab, max(s_gem_edge(p, float2(-0.02, 0.03), float2(-0.02, -0.88), z0), - max(s_gem_edge(p, float2(-0.02, -0.88), float2(0.66, -0.61), z0), - s_gem_edge(p, float2(0.66, -0.61), float2(-0.02, 0.03), z0)))); - float tail = max(slab, max(max(s_gem_edge(p, float2(0.215, -0.665), float2(0.37, -1.0), z0), - s_gem_edge(p, float2(0.37, -1.0), float2(0.53, -0.93), z0)), - max(s_gem_edge(p, float2(0.53, -0.93), float2(0.38, -0.60), z0), - s_gem_edge(p, float2(0.38, -0.60), float2(0.215, -0.665), z0)))); - return min(head, tail); -} - -// Capsule taillée a -> b : un prisme hexagonal coiffé en b de deux couronnes de facettes -// décalées et d'une facette sommitale, tous les plans tangents à la sphère du bout. -float s_facet_capsule(float3 p, float3 a, float3 b, float r, float spin) -{ - float3 u = normalize(b - a); - float3 v = normalize(cross(u, float3(0.0, 0.0, 1.0))); - float3 w = cross(u, v); - float3 q = p - b; - float h = dot(q, u); - float2 rad = float2(dot(q, v), dot(q, w)); - float d = max(h - r, -dot(p - a, u) - r); - [loop] for (int k = 0; k < 6; k++) - { - float an = spin + k * 1.0471976; - float s = dot(rad, float2(cos(an), sin(an))); - d = max(d, s - r); - d = max(d, dot(rad, float2(cos(an + 0.5236), sin(an + 0.5236))) * 0.8660 + h * 0.5 - r); - d = max(d, s * 0.5 + h * 0.8660 - r); + float2 q = abs(p - PIX_RECT[i].xy) - PIX_RECT[i].zw; + d = min(d, length(max(q, 0.0)) + min(max(q.x, q.y), 0.0)); } return d; } -// Ellipsoïde taillé : les plans tangents dans dix directions du premier octant, reportées dans -// les huit. Une pierre aux facettes de taille égale. -static const float3 SCULPT_FACETS[10] = { - float3(1.0, 0.0, 0.0), float3(0.0, 1.0, 0.0), float3(0.0, 0.0, 1.0), - float3(0.7071, 0.7071, 0.0), float3(0.7071, 0.0, 0.7071), float3(0.0, 0.7071, 0.7071), - float3(0.5774, 0.5774, 0.5774), float3(0.4472, 0.0, 0.8944), float3(0.0, 0.4472, 0.8944), - float3(0.3015, 0.3015, 0.9045) +// ---- Prism Glow : un cristal en MAILLAGE ---- +// Les facettes tracées sur l'art 2D du thème (`design/cursors/prism-glow/model`), en triangles dans +// le repère du MODÈLE lui-même (unité = côté du sprite 128 px, origine au hotspot, y vers le bas, +// dessus du serti en z = 0). Le serti marine est la silhouette de l'art extrudée sous z = 0, un +// champ de distance comme les autres sculptés : il donne la couverture, l'ombre portée et l'ombre +// de contact. Le cristal est lancé de rayons, triangles rangés par boîtes : réfraction par canal +// (dispersion), réflexions totales internes, sortie par son fond plat vers l'image composée sous +// le curseur, flous de confidentialité compris (sa copie en t5). Tables : miroir de `prism_mesh.rs` ; +// un triangle = quatre +// float4 (v0, drapeaux), (e1, r), (e2, g), (normale sortante, b) ; drapeaux = paroi (1) + 2 × +// arêtes réelles (bit k : l'arête opposée au sommet k) ; une boîte = deux float4 (coin bas, +// premier triangle), (coin haut, nombre de triangles). +// prism mesh: generated by design/cursors/prism-glow/model/export_compositor.py +static const int PRISM_TRI_START[2] = { 0, 31 }; +static const int PRISM_TRI_COUNT[2] = { 31, 130 }; +static const int PRISM_SIL_START[2] = { 0, 25 }; +static const int PRISM_SIL_COUNT[2] = { 25, 42 }; +static const int PRISM_OUT_START[2] = { 67, 76 }; +static const int PRISM_OUT_COUNT[2] = { 9, 34 }; +static const int PRISM_BOX_START[2] = { 0, 4 }; +static const int PRISM_BOX_COUNT[2] = { 4, 18 }; +static const float4 PRISM_TRIS[644] = { + float4(0.252301, 0.266258, -0.042945, 11.0), + float4(-0.221166, -0.203988, 0.0, 0.01033), + float4(-0.221166, -0.203988, 0.042945, 0.045182), + float4(0.677984, -0.735077, 0.0, 0.577584), + float4(0.252301, 0.266258, -0.042945, 3.0), + float4(-0.221166, -0.203988, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(0.677984, -0.735077, 0.0, 0.577584), + float4(0.031135, 0.06227, -0.042945, 7.0), + float4(0.004294, 0.419785, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(-0.999948, 0.01023, 0.0, 0.577584), + float4(0.169632, 0.356442, 0.115951, 14.0), + float4(0.082669, -0.090184, -0.115951, 0.590616), + float4(-0.138497, -0.294172, -0.115951, 0.737911), + float4(0.466385, -0.505659, 0.725806, 0.973449), + float4(0.035429, 0.482055, 0.0, 14.0), + float4(0.134202, -0.125614, 0.115951, 0.000305), + float4(-0.004294, -0.419785, 0.0, 0.938689), + float4(-0.650191, 0.006652, 0.759742, 0.854994), + float4(0.031135, 0.06227, -0.042945, 9.0), + float4(0.004294, 0.419785, 0.0, 0.01033), + float4(0.004294, 0.419785, 0.042945, 0.045182), + float4(-0.999948, 0.01023, 0.0, 0.577584), + float4(0.169632, 0.356442, 0.115951, 14.0), + float4(0.159969, 0.080521, -0.045092, 0.381323), + float4(0.082669, -0.090184, -0.115951, 0.056123), + float4(0.461407, -0.510214, 0.725799, 0.973449), + float4(0.541104, 0.544325, -0.042945, 9.0), + float4(-0.288804, -0.278068, 0.0, 0.01033), + float4(-0.288804, -0.278068, 0.042945, 0.045182), + float4(0.69359, -0.72037, 0.0, 0.577584), + float4(0.329601, 0.436963, 0.070859, 14.0), + float4(0.211503, 0.107362, -0.070859, 0.545724), + float4(-0.077301, -0.170706, -0.070859, 0.064805), + float4(0.49567, -0.514808, 0.699488, 0.973449), + float4(0.268405, 0.533589, 0.081595, 14.0), + float4(0.061196, -0.096626, -0.010736, 0.000916), + float4(-0.098773, -0.177147, 0.034356, 0.046662), + float4(0.247834, 0.049458, 0.967539, 0.904662), + float4(0.541104, 0.544325, -0.042945, 7.0), + float4(-0.288804, -0.278068, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(0.69359, -0.72037, 0.0, 0.577584), + float4(0.199693, 0.595859, 0.0, 14.0), + float4(0.068712, -0.06227, 0.081595, 0.0), + float4(-0.030061, -0.239417, 0.115951, 0.01445), + float4(-0.504461, 0.426837, 0.750553, 0.254154), + float4(0.268405, 0.533589, 0.081595, 14.0), + float4(0.272699, 0.010736, -0.081595, 0.0), + float4(0.061196, -0.096626, -0.010736, 0.025193), + float4(0.283244, 0.073137, 0.956255, 0.527108), + float4(0.035429, 0.482055, 0.0, 14.0), + float4(0.0, 0.258742, 0.0, 0.0), + float4(0.134202, -0.125614, 0.115951, 0.473533), + float4(-0.653777, 0.0, 0.756687, 0.964691), + float4(0.359663, 0.559356, 0.0, 14.0), + float4(0.181442, -0.015031, 0.0, 0.006043), + float4(-0.091258, -0.025767, 0.081595, 0.076188), + float4(0.076465, 0.923048, 0.37701, 0.964691), + float4(0.035429, 0.482055, -0.042945, 3.0), + float4(0.0, 0.258742, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(-1.0, 0.0, 0.0, 0.577584), + float4(0.035429, 0.482055, -0.042945, 11.0), + float4(0.0, 0.258742, 0.0, 0.01033), + float4(0.0, 0.258742, 0.042945, 0.045182), + float4(-1.0, 0.0, 0.0, 0.577584), + float4(0.035429, 0.740798, -0.042945, 7.0), + float4(0.164264, -0.144939, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(0.661622, 0.749838, 0.0, 0.577584), + float4(0.035429, 0.740798, 0.0, 14.0), + float4(0.164264, -0.144939, 0.0, 0.0), + float4(0.134202, -0.384356, 0.115951, 0.010956), + float4(0.332572, 0.376915, 0.864483, 0.215854), + float4(0.035429, 0.740798, -0.042945, 11.0), + float4(0.164264, -0.144939, 0.0, 0.01033), + float4(0.164264, -0.144939, 0.042945, 0.045182), + float4(0.661622, 0.749838, 0.0, 0.577584), + float4(0.199693, 0.595859, -0.042945, 7.0), + float4(0.105215, 0.22546, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(-0.906183, 0.422886, 0.0, 0.577584), + float4(0.199693, 0.595859, 0.0, 14.0), + float4(0.105215, 0.22546, 0.0, 0.0), + float4(0.068712, -0.06227, 0.081595, 0.015991), + float4(-0.613882, 0.286478, 0.735581, 0.473533), + float4(0.199693, 0.595859, -0.042945, 11.0), + float4(0.105215, 0.22546, 0.0, 0.01033), + float4(0.105215, 0.22546, 0.042945, 0.045182), + float4(-0.906183, 0.422886, 0.0, 0.577584), + float4(0.304908, 0.821319, 0.0, 14.0), + float4(0.128834, -0.067638, 0.0, 0.0), + float4(-0.036503, -0.28773, 0.081595, 0.254154), + float4(0.133686, 0.254641, 0.957751, 0.973449), + float4(0.304908, 0.821319, -0.042945, 7.0), + float4(0.128834, -0.067638, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(0.464834, 0.885398, 0.0, 0.577584), + float4(0.433742, 0.753681, 0.0, 14.0), + float4(-0.07408, -0.194325, 0.0, 0.0), + float4(-0.165337, -0.220092, 0.081595, 0.964691), + float4(0.683362, -0.260508, 0.682021, 0.964691), + float4(0.433742, 0.753681, -0.042945, 11.0), + float4(-0.07408, -0.194325, 0.0, 0.01033), + float4(-0.07408, -0.194325, 0.042945, 0.045182), + float4(0.934406, -0.35621, 0.0, 0.577584), + float4(0.304908, 0.821319, -0.042945, 11.0), + float4(0.128834, -0.067638, 0.0, 0.01033), + float4(0.128834, -0.067638, 0.042945, 0.045182), + float4(0.464834, 0.885398, 0.0, 0.577584), + float4(0.433742, 0.753681, -0.042945, 7.0), + float4(-0.07408, -0.194325, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(0.934406, -0.35621, 0.0, 0.577584), + float4(0.359663, 0.559356, -0.042945, 7.0), + float4(0.181442, -0.015031, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(0.082557, 0.996586, 0.0, 0.577584), + float4(0.359663, 0.559356, -0.042945, 11.0), + float4(0.181442, -0.015031, 0.0, 0.01033), + float4(0.181442, -0.015031, 0.042945, 0.045182), + float4(0.082557, 0.996586, 0.0, 0.577584), + float4(0.044345, 0.043263, -0.043263, 11.0), + float4(-0.043263, -0.005408, 0.0, 0.01033), + float4(-0.043263, -0.005408, 0.043263, 0.045182), + float4(0.124035, -0.992278, 0.0, 0.577584), + float4(0.044345, 0.043263, -0.043263, 7.0), + float4(-0.043263, -0.005408, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.124035, -0.992278, 0.0, 0.577584), + float4(0.001082, 0.037855, -0.043263, 7.0), + float4(-0.044345, 0.040019, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.669964, -0.742393, 0.0, 0.577584), + float4(0.044345, 0.043263, 0.0, 10.0), + float4(-0.043263, -0.005408, 0.0, 0.000305), + float4(-0.040019, 0.030284, 0.049753, 0.964691), + float4(0.101434, -0.811469, 0.575525, 0.973449), + float4(0.004326, 0.073548, 0.049753, 14.0), + float4(-0.003245, -0.035692, -0.049753, 0.0), + float4(-0.04759, 0.004326, -0.049753, 0.955978), + float4(-0.580475, -0.643229, 0.499304, 0.896269), + float4(0.001082, 0.037855, -0.043263, 11.0), + float4(-0.044345, 0.040019, 0.0, 0.01033), + float4(-0.044345, 0.040019, 0.043263, 0.045182), + float4(-0.669964, -0.742393, 0.0, 0.577584), + float4(-0.031366, 0.229295, 0.058405, 14.0), + float4(0.035692, -0.155748, -0.008653, 0.000305), + float4(-0.011897, -0.151422, -0.058405, 0.955978), + float4(-0.716533, -0.20131, 0.667874, 0.904662), + float4(-0.043263, 0.077874, -0.043263, 7.0), + float4(-0.001082, 0.154666, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.999976, -0.006993, 0.0, 0.577584), + float4(-0.031366, 0.229295, 0.058405, 14.0), + float4(0.089771, -0.137361, -0.02055, 0.445197), + float4(0.035692, -0.155748, -0.008653, 0.973449), + float4(0.216369, -0.004654, 0.976301, 0.973449), + float4(-0.031366, 0.229295, 0.058405, 14.0), + float4(-0.011897, -0.151422, -0.058405, 0.000305), + float4(-0.012979, 0.003245, -0.058405, 0.955978), + float4(-0.976245, -0.006827, 0.216564, 0.904662), + float4(-0.043263, 0.077874, -0.043263, 9.0), + float4(-0.001082, 0.154666, 0.0, 0.01033), + float4(-0.001082, 0.154666, 0.043263, 0.045182), + float4(-0.999976, -0.006993, 0.0, 0.577584), + float4(-0.045426, 0.564586, 0.052998, 14.0), + float4(0.014061, -0.335291, 0.005408, 0.520989), + float4(0.001082, -0.332046, -0.052998, 0.088655), + float4(-0.975941, -0.037462, 0.214794, 0.973449), + float4(-0.044345, 0.23254, 0.0, 3.0), + float4(0.0, 0.0, -0.043263, 0.01033), + float4(-0.001082, 0.332046, -0.043263, 0.045182), + float4(-0.999995, -0.003257, 0.0, 0.577584), + float4(-0.163319, 0.380717, -0.043263, 7.0), + float4(-0.060569, 0.02704, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.407651, -0.913138, 0.0, 0.577584), + float4(-0.094098, 0.431551, -0.043263, 11.0), + float4(-0.069221, -0.050834, 0.0, 0.01033), + float4(-0.069221, -0.050834, 0.043263, 0.045182), + float4(0.591909, -0.806004, 0.0, 0.577584), + float4(-0.163319, 0.380717, -0.043263, 11.0), + float4(-0.060569, 0.02704, 0.0, 0.01033), + float4(-0.060569, 0.02704, 0.043263, 0.045182), + float4(-0.407651, -0.913138, 0.0, 0.577584), + float4(0.075711, 0.08869, -0.043263, 11.0), + float4(-0.031366, -0.045426, 0.0, 0.01033), + float4(-0.031366, -0.045426, 0.043263, 0.045182), + float4(0.822897, -0.568191, 0.0, 0.577584), + float4(0.044345, 0.043263, 0.0, 6.0), + float4(-0.040019, 0.030284, 0.049753, 0.000305), + float4(0.014061, 0.048671, 0.037855, 0.964691), + float4(0.365607, -0.634949, 0.680566, 0.973449), + float4(0.075711, 0.08869, -0.043263, 7.0), + float4(-0.031366, -0.045426, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.822897, -0.568191, 0.0, 0.577584), + float4(0.058405, 0.091934, 0.037855, 14.0), + float4(0.017305, -0.003245, -0.037855, 0.0), + float4(-0.014061, -0.048671, -0.037855, 0.473533), + float4(0.757369, -0.522945, 0.39105, 0.904662), + float4(-0.031366, 0.229295, 0.058405, 14.0), + float4(0.107077, -0.140606, -0.058405, 0.0), + float4(0.089771, -0.137361, -0.02055, 0.376257), + float4(0.812052, 0.481346, 0.329966, 0.830777), + float4(0.077874, 0.407756, -0.043263, 11.0), + float4(-0.002163, -0.319067, 0.0, 0.01033), + float4(-0.002163, -0.319067, 0.043263, 0.045182), + float4(0.999977, -0.006779, 0.0, 0.577584), + float4(-0.031366, 0.229295, 0.058405, 14.0), + float4(0.10924, 0.178461, -0.058405, 0.0), + float4(0.107077, -0.140606, -0.058405, 0.376257), + float4(0.475583, -0.003224, 0.879665, 0.830777), + float4(0.147095, 0.272559, -0.043263, 7.0), + float4(-0.035692, 0.034611, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.696146, -0.7179, 0.0, 0.577584), + float4(0.147095, 0.272559, -0.043263, 11.0), + float4(-0.035692, 0.034611, 0.0, 0.01033), + float4(-0.035692, 0.034611, 0.043263, 0.045182), + float4(-0.696146, -0.7179, 0.0, 0.577584), + float4(0.144932, 0.307169, 0.049753, 14.0), + float4(0.002163, -0.034611, -0.049753, 0.0), + float4(-0.033529, 0.0, -0.049753, 0.9131), + float4(-0.630237, -0.649931, 0.424725, 0.904662), + float4(0.183869, 0.317985, 0.037855, 6.0), + float4(-0.036774, -0.045426, -0.037855, 0.0), + float4(-0.038937, -0.010816, 0.011897, 0.806958), + float4(0.374433, -0.753477, 0.540438, 0.854994), + float4(0.077874, 0.407756, -0.043263, 7.0), + float4(-0.002163, -0.319067, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.999977, -0.006779, 0.0, 0.577584), + float4(0.111403, 0.307169, -0.043263, 7.0), + float4(-0.002163, 0.131953, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.999866, -0.016391, 0.0, 0.577584), + float4(0.10924, 0.439122, 0.0, 6.0), + float4(0.035692, -0.131953, 0.049753, 0.18117), + float4(0.002163, -0.131953, 0.0, 0.955978), + float4(-0.82919, -0.013593, 0.558802, 0.964691), + float4(0.128708, 0.463999, 0.063813, 14.0), + float4(0.055161, -0.146014, -0.025958, 0.246204), + float4(0.016224, -0.156829, -0.014061, 0.964691), + float4(0.305397, -0.053644, 0.950713, 0.973449), + float4(0.111403, 0.307169, -0.043263, 11.0), + float4(-0.002163, 0.131953, 0.0, 0.01033), + float4(-0.002163, 0.131953, 0.043263, 0.045182), + float4(-0.999866, -0.016391, 0.0, 0.577584), + float4(0.189277, 0.28013, -0.043263, 11.0), + float4(-0.042182, -0.007571, 0.0, 0.01033), + float4(-0.042182, -0.007571, 0.043263, 0.045182), + float4(0.176664, -0.984271, 0.0, 0.577584), + float4(0.183869, 0.317985, 0.037855, 10.0), + float4(0.005408, -0.037855, -0.037855, 0.0), + float4(-0.036774, -0.045426, -0.037855, 0.806958), + float4(0.124328, -0.692684, 0.710445, 0.854994), + float4(0.189277, 0.28013, -0.043263, 7.0), + float4(-0.042182, -0.007571, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.176664, -0.984271, 0.0, 0.577584), + float4(0.183869, 0.317985, 0.037855, 14.0), + float4(0.02704, -0.004326, -0.037855, 0.0), + float4(0.005408, -0.037855, -0.037855, 0.514916), + float4(0.70062, -0.452013, 0.552101, 0.846876), + float4(0.210908, 0.313659, -0.043263, 11.0), + float4(-0.021632, -0.033529, 0.0, 0.01033), + float4(-0.021632, -0.033529, 0.043263, 0.045182), + float4(0.840297, -0.542127, 0.0, 0.577584), + float4(0.210908, 0.445612, 0.0, 10.0), + float4(0.0, -0.131953, 0.0, 0.0), + float4(-0.02704, -0.127627, 0.037855, 0.439658), + float4(0.813734, 0.0, 0.581238, 0.871366), + float4(0.210908, 0.313659, -0.043263, 7.0), + float4(-0.021632, -0.033529, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.840297, -0.542127, 0.0, 0.577584), + float4(0.210908, 0.445612, -0.043263, 11.0), + float4(0.0, -0.131953, 0.0, 0.01033), + float4(0.0, -0.131953, 0.043263, 0.045182), + float4(1.0, 0.0, 0.0, 0.577584), + float4(0.241193, 0.343943, -0.043263, 7.0), + float4(-0.002163, 0.107077, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.999796, -0.020198, 0.0, 0.577584), + float4(0.276885, 0.311496, -0.043263, 11.0), + float4(-0.035692, 0.032447, 0.0, 0.01033), + float4(-0.035692, 0.032447, 0.043263, 0.045182), + float4(-0.672673, -0.73994, 0.0, 0.577584), + float4(0.23903, 0.45102, 0.0, 14.0), + float4(0.042182, -0.108158, 0.046508, 0.000305), + float4(0.002163, -0.107077, 0.0, 0.846876), + float4(-0.758098, -0.015315, 0.651961, 0.896269), + float4(0.276885, 0.311496, -0.043263, 7.0), + float4(-0.035692, 0.032447, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.672673, -0.73994, 0.0, 0.577584), + float4(0.281211, 0.342862, 0.046508, 14.0), + float4(-0.004326, -0.031366, -0.046508, 0.0), + float4(-0.040019, 0.001082, -0.046508, 0.752941), + float4(-0.58651, -0.645161, 0.489667, 0.846876), + float4(0.246601, 0.459672, 0.06165, 14.0), + float4(0.070303, -0.101669, -0.024876, 0.262257), + float4(0.034611, -0.116811, -0.015142, 0.973449), + float4(0.27928, -0.041605, 0.959308, 0.973449), + float4(0.319067, 0.319067, -0.043263, 11.0), + float4(-0.042182, -0.007571, 0.0, 0.01033), + float4(-0.042182, -0.007571, 0.043263, 0.045182), + float4(0.176664, -0.984271, 0.0, 0.577584), + float4(0.319067, 0.319067, 0.0, 10.0), + float4(-0.042182, -0.007571, 0.0, 0.0), + float4(-0.037855, 0.023795, 0.046508, 0.896269), + float4(0.1483, -0.826245, 0.54344, 0.896269), + float4(0.319067, 0.319067, -0.043263, 7.0), + float4(-0.042182, -0.007571, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.176664, -0.984271, 0.0, 0.577584), + float4(0.319067, 0.319067, 0.0, 6.0), + float4(-0.037855, 0.023795, 0.046508, 0.0), + float4(-0.002163, 0.038937, 0.036774, 0.896269), + float4(0.437906, -0.604311, 0.665618, 0.896269), + float4(0.316904, 0.358004, 0.036774, 14.0), + float4(0.024876, -0.004326, -0.036774, 0.0), + float4(0.002163, -0.038937, -0.036774, 0.617212), + float4(0.707338, -0.464191, 0.533104, 0.921584), + float4(0.34178, 0.353677, -0.043263, 11.0), + float4(-0.022713, -0.034611, 0.0, 0.01033), + float4(-0.022713, -0.034611, 0.043263, 0.045182), + float4(0.836048, -0.548657, 0.0, 0.577584), + float4(0.340698, 0.474815, 0.0, 14.0), + float4(0.001082, -0.121137, 0.0, 0.0), + float4(-0.023795, -0.116811, 0.036774, 0.473533), + float4(0.828664, 0.007399, 0.559697, 0.930114), + float4(0.34178, 0.353677, -0.043263, 7.0), + float4(-0.022713, -0.034611, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.836048, -0.548657, 0.0, 0.577584), + float4(0.340698, 0.474815, -0.043263, 11.0), + float4(0.001082, -0.121137, 0.0, 0.01033), + float4(0.001082, -0.121137, 0.043263, 0.045182), + float4(0.99996, 0.008928, 0.0, 0.577584), + float4(0.411001, 0.351514, -0.043263, 11.0), + float4(-0.034611, 0.030284, 0.0, 0.01033), + float4(-0.034611, 0.030284, 0.043263, 0.045182), + float4(-0.658505, -0.752577, 0.0, 0.577584), + float4(0.411001, 0.351514, -0.043263, 7.0), + float4(-0.034611, 0.030284, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.658505, -0.752577, 0.0, 0.577584), + float4(0.412083, 0.38288, 0.043263, 14.0), + float4(-0.001082, -0.031366, -0.043263, 0.0), + float4(-0.035692, -0.001082, -0.043263, 0.745403), + float4(-0.574039, -0.656045, 0.489984, 0.854994), + float4(0.45102, 0.359085, -0.043263, 11.0), + float4(-0.040019, -0.007571, 0.0, 0.01033), + float4(-0.040019, -0.007571, 0.043263, 0.045182), + float4(0.185892, -0.98257, 0.0, 0.577584), + float4(0.45102, 0.359085, 0.0, 10.0), + float4(-0.040019, -0.007571, 0.0, 0.000305), + float4(-0.038937, 0.023795, 0.043263, 0.879622), + float4(0.151736, -0.802035, 0.577682, 0.9131), + float4(0.45102, 0.359085, -0.043263, 7.0), + float4(-0.040019, -0.007571, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.185892, -0.98257, 0.0, 0.577584), + float4(0.45102, 0.359085, 0.0, 6.0), + float4(-0.038937, 0.023795, 0.043263, 0.000305), + float4(0.003245, 0.0411, 0.032447, 0.879622), + float4(0.417852, -0.583049, 0.696744, 0.9131), + float4(0.473733, 0.391533, -0.043263, 11.0), + float4(-0.022713, -0.032447, 0.0, 0.01033), + float4(-0.022713, -0.032447, 0.043263, 0.045182), + float4(0.819232, -0.573462, 0.0, 0.577584), + float4(0.454264, 0.400185, 0.032447, 14.0), + float4(0.019468, -0.008653, -0.032447, 0.0), + float4(-0.003245, -0.0411, -0.032447, 0.527108), + float4(0.688617, -0.482032, 0.541712, 0.887922), + float4(0.473733, 0.391533, -0.043263, 7.0), + float4(-0.022713, -0.032447, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.819232, -0.573462, 0.0, 0.577584), + float4(-0.223888, 0.407756, -0.043263, 7.0), + float4(0.002163, 0.06814, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.999496, 0.03173, 0.0, 0.577584), + float4(-0.223888, 0.407756, -0.043263, 11.0), + float4(0.002163, 0.06814, 0.0, 0.01033), + float4(0.002163, 0.06814, 0.043263, 0.045182), + float4(-0.999496, 0.03173, 0.0, 0.577584), + float4(-0.167645, 0.416409, 0.046508, 14.0), + float4(-0.056242, -0.008653, -0.046508, 0.000305), + float4(-0.054079, 0.059487, -0.046508, 0.955978), + float4(-0.638985, 0.020285, 0.768952, 0.973449), + float4(-0.167645, 0.416409, 0.046508, 14.0), + float4(0.004326, -0.035692, -0.046508, 0.000305), + float4(-0.056242, -0.008653, -0.046508, 0.955978), + float4(-0.339782, -0.761112, 0.552501, 0.973449), + float4(-0.221724, 0.475896, 0.0, 14.0), + float4(0.139524, 0.055161, 0.043263, 0.520989), + float4(0.054079, -0.059487, 0.046508, 0.973449), + float4(-0.39307, 0.317373, 0.863001, 0.973449), + float4(-0.167645, 0.416409, 0.046508, 6.0), + float4(0.073548, 0.015142, -0.046508, 0.0), + float4(0.004326, -0.035692, -0.046508, 0.768154), + float4(0.490917, -0.668482, 0.558688, 0.871366), + float4(-0.094098, 0.431551, -0.043263, 7.0), + float4(-0.069221, -0.050834, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.591909, -0.806004, 0.0, 0.577584), + float4(-0.167645, 0.416409, 0.046508, 10.0), + float4(0.085445, 0.114648, -0.003245, 0.0), + float4(0.073548, 0.015142, -0.046508, 0.768154), + float4(0.548356, -0.38771, 0.740936, 0.871366), + float4(-0.0822, 0.531057, -0.043263, 11.0), + float4(-0.011897, -0.099506, 0.0, 0.01033), + float4(-0.011897, -0.099506, 0.043263, 0.045182), + float4(0.992928, -0.11872, 0.0, 0.577584), + float4(-0.044345, 0.23254, 0.0, 7.0), + float4(-0.001082, 0.332046, -0.043263, 0.01033), + float4(-0.001082, 0.332046, 0.052998, 0.045182), + float4(-0.999995, -0.003257, 0.0, 0.577584), + float4(-0.045426, 0.564586, 0.052998, 14.0), + float4(0.1233, -0.156829, -0.052998, 0.520989), + float4(0.014061, -0.335291, 0.005408, 0.088655), + float4(0.429355, 0.032562, 0.902549, 0.973449), + float4(0.10924, 0.439122, 0.0, 14.0), + float4(-0.031366, -0.031366, 0.0, 0.132876), + float4(-0.154666, 0.125464, 0.052998, 0.028428), + float4(0.182761, -0.182761, 0.966021, 0.955978), + float4(0.10924, 0.439122, -0.043263, 11.0), + float4(-0.031366, -0.031366, 0.0, 0.01033), + float4(-0.031366, -0.031366, 0.043263, 0.045182), + float4(0.707107, -0.707107, 0.0, 0.577584), + float4(0.10924, 0.439122, -0.043263, 7.0), + float4(-0.031366, -0.031366, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.707107, -0.707107, 0.0, 0.577584), + float4(0.10924, 0.439122, 0.0, 10.0), + float4(0.019468, 0.024876, 0.063813, 0.18117), + float4(0.035692, -0.131953, 0.049753, 0.955978), + float4(-0.933936, -0.126584, 0.334276, 0.964691), + float4(0.128708, 0.463999, 0.063813, 14.0), + float4(0.050834, 0.002163, -0.005408, 0.0), + float4(0.0822, -0.018387, -0.063813, 0.439658), + float4(0.07859, -0.92643, 0.368172, 0.871366), + float4(-0.221724, 0.475896, -0.043263, 7.0), + float4(0.122219, 0.207664, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.861819, 0.507216, 0.0, 0.577584), + float4(-0.221724, 0.475896, -0.043263, 9.0), + float4(0.122219, 0.207664, 0.0, 0.01033), + float4(0.122219, 0.207664, 0.043263, 0.045182), + float4(-0.861819, 0.507216, 0.0, 0.577584), + float4(-0.099506, 0.68356, 0.0, 14.0), + float4(0.017305, -0.152503, 0.043263, 0.0), + float4(-0.122219, -0.207664, 0.0, 0.723049), + float4(-0.365879, 0.215335, 0.905408, 0.854994), + float4(-0.0822, 0.531057, -0.043263, 7.0), + float4(-0.011897, -0.099506, 0.043263, 0.01033), + float4(0.0, 0.0, 0.086527, 0.045182), + float4(0.992928, -0.11872, 0.0, 0.577584), + float4(-0.099506, 0.68356, 0.0, 14.0), + float4(0.054079, -0.118974, 0.052998, 0.0), + float4(0.017305, -0.152503, 0.043263, 0.514916), + float4(-0.41958, 0.203352, 0.884647, 0.863157), + float4(-0.099506, 0.68356, -0.043263, 3.0), + float4(0.086527, 0.149258, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.86514, 0.501531, 0.0, 0.577584), + float4(-0.045426, 0.564586, -0.043263, 11.0), + float4(-0.036774, -0.033529, 0.0, 0.01033), + float4(-0.036774, -0.033529, 0.086527, 0.045182), + float4(0.673754, -0.738956, 0.0, 0.577584), + float4(-0.045426, 0.564586, -0.043263, 7.0), + float4(-0.036774, -0.033529, 0.086527, 0.01033), + float4(0.0, 0.0, 0.096261, 0.045182), + float4(0.673754, -0.738956, 0.0, 0.577584), + float4(-0.099506, 0.68356, 0.0, 14.0), + float4(0.086527, 0.149258, 0.0, 0.0), + float4(0.054079, -0.118974, 0.052998, 0.262257), + float4(-0.385563, 0.223515, 0.8952, 0.9131), + float4(-0.099506, 0.68356, -0.043263, 11.0), + float4(0.086527, 0.149258, 0.0, 0.01033), + float4(0.086527, 0.149258, 0.043263, 0.045182), + float4(-0.86514, 0.501531, 0.0, 0.577584), + float4(-0.012979, 0.832818, 0.0, 14.0), + float4(0.214153, -0.194685, 0.0822, 0.000305), + float4(-0.032447, -0.268232, 0.052998, 0.930114), + float4(-0.176863, 0.211325, 0.961281, 0.973449), + float4(0.128708, 0.463999, 0.063813, 14.0), + float4(-0.019468, -0.024876, -0.063813, 0.168276), + float4(-0.174135, 0.100587, -0.010816, 0.038209), + float4(-0.469238, -0.764881, 0.441331, 0.955978), + float4(0.201174, 0.638133, 0.0822, 14.0), + float4(-0.072466, -0.174135, -0.018387, 0.18117), + float4(-0.246601, -0.073548, -0.029203, 0.082277), + float4(-0.098561, -0.063844, 0.993081, 0.973449), + float4(-0.012979, 0.832818, -0.043263, 7.0), + float4(0.339617, -0.002163, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.006369, 0.99998, 0.0, 0.577584), + float4(0.179543, 0.466162, 0.058405, 10.0), + float4(-0.050834, -0.002163, 0.005408, 0.000305), + float4(0.021632, 0.171972, 0.023795, 0.022171), + float4(0.110896, -0.149888, 0.982464, 0.381323), + float4(-0.012979, 0.832818, 0.0, 14.0), + float4(0.339617, -0.002163, 0.0, 0.0), + float4(0.214153, -0.194685, 0.0822, 0.027314), + float4(0.002492, 0.391296, 0.920261, 0.49102), + float4(0.183869, 0.317985, 0.037855, 10.0), + float4(-0.055161, 0.146014, 0.025958, 0.0), + float4(0.02704, 0.127627, -0.037855, 0.439658), + float4(0.623841, 0.097823, 0.775405, 0.871366), + float4(0.210908, 0.445612, -0.043263, 7.0), + float4(0.0, -0.131953, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(1.0, 0.0, 0.0, 0.577584), + float4(0.210908, 0.445612, 0.0, 10.0), + float4(-0.031366, 0.02055, 0.058405, 0.000305), + float4(0.035692, 0.014061, 0.06165, 0.022171), + float4(-0.105864, -0.954447, 0.278971, 0.381323), + float4(0.23903, 0.45102, -0.043263, 11.0), + float4(-0.028121, -0.005408, 0.0, 0.01033), + float4(-0.028121, -0.005408, 0.043263, 0.045182), + float4(0.188847, -0.982006, 0.0, 0.577584), + float4(0.23903, 0.45102, -0.043263, 7.0), + float4(-0.028121, -0.005408, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.188847, -0.982006, 0.0, 0.577584), + float4(0.246601, 0.459672, 0.06165, 11.0), + float4(-0.007571, -0.008653, -0.06165, 0.000305), + float4(-0.035692, -0.014061, -0.06165, 0.022171), + float4(0.187618, -0.975617, 0.113888, 0.381323), + float4(0.241193, 0.343943, -0.043263, 11.0), + float4(-0.002163, 0.107077, 0.0, 0.01033), + float4(-0.002163, 0.107077, 0.043263, 0.045182), + float4(-0.999796, -0.020198, 0.0, 0.577584), + float4(0.23903, 0.45102, 0.0, 15.0), + float4(0.007571, 0.008653, 0.06165, 0.000305), + float4(0.042182, -0.108158, 0.046508, 0.846876), + float4(-0.94107, -0.299262, 0.157572, 0.896269), + float4(0.316904, 0.358004, 0.036774, 14.0), + float4(-0.070303, 0.101669, 0.024876, 0.0), + float4(0.023795, 0.116811, -0.036774, 0.473533), + float4(0.523424, 0.157027, 0.837478, 0.930114), + float4(0.340698, 0.474815, -0.043263, 7.0), + float4(0.001082, -0.121137, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.99996, 0.008928, 0.0, 0.577584), + float4(0.376391, 0.381799, -0.043263, 11.0), + float4(-0.001082, 0.107077, 0.0, 0.01033), + float4(-0.001082, 0.107077, 0.043263, 0.045182), + float4(-0.999949, -0.0101, 0.0, 0.577584), + float4(0.376391, 0.381799, -0.043263, 7.0), + float4(-0.001082, 0.107077, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.999949, -0.0101, 0.0, 0.577584), + float4(0.375309, 0.488875, 0.0, 14.0), + float4(0.036774, -0.105995, 0.043263, 0.000305), + float4(0.001082, -0.107077, 0.0, 0.879622), + float4(-0.771254, -0.00779, 0.63648, 0.896269), + float4(0.375309, 0.488875, 0.0, 14.0), + float4(0.078955, -0.08869, 0.032447, 0.223224), + float4(0.036774, -0.105995, 0.043263, 0.964691), + float4(0.071225, 0.39802, 0.914608, 0.973449), + float4(0.473733, 0.603523, 0.0, 14.0), + float4(0.0, -0.21199, 0.0, 0.158953), + float4(-0.019468, -0.203337, 0.032447, 0.030716), + float4(0.857493, 0.0, 0.514496, 0.964691), + float4(0.473733, 0.603523, -0.043263, 11.0), + float4(0.0, -0.21199, 0.0, 0.01033), + float4(0.0, -0.21199, 0.043263, 0.045182), + float4(1.0, 0.0, 0.0, 0.577584), + float4(0.276885, 0.484549, 0.058405, 10.0), + float4(-0.030284, -0.024876, 0.003245, 0.000305), + float4(-0.097342, -0.018387, 0.0, 0.022171), + float4(-0.03153, 0.166923, 0.985466, 0.381323), + float4(0.246601, 0.459672, 0.06165, 14.0), + float4(0.030284, 0.024876, -0.003245, 0.0), + float4(0.094098, 0.015142, -0.06165, 0.473533), + float4(0.518869, -0.545855, 0.657888, 0.930114), + float4(0.375309, 0.488875, -0.043263, 11.0), + float4(-0.034611, -0.014061, 0.0, 0.01033), + float4(-0.034611, -0.014061, 0.043263, 0.045182), + float4(0.376377, -0.926467, 0.0, 0.577584), + float4(0.340698, 0.474815, 0.0, 14.0), + float4(-0.063813, 0.009734, 0.058405, 0.000305), + float4(0.034611, 0.014061, 0.0, 0.022171), + float4(0.3276, -0.806402, 0.492334, 0.381323), + float4(0.375309, 0.488875, -0.043263, 7.0), + float4(-0.034611, -0.014061, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.376377, -0.926467, 0.0, 0.577584), + float4(0.454264, 0.400185, 0.032447, 12.0), + float4(-0.078955, 0.08869, -0.032447, 0.158953), + float4(0.019468, 0.203337, -0.032447, 0.030716), + float4(-0.201684, 0.173144, 0.964025, 0.964691), + float4(0.201174, 0.638133, 0.0822, 8.0), + float4(0.075711, -0.153585, -0.023795, 0.000305), + float4(-0.021632, -0.171972, -0.023795, 0.022171), + float4(0.026497, -0.140281, 0.989757, 0.381323), + float4(0.473733, 0.603523, -0.043263, 7.0), + float4(0.0, -0.21199, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(1.0, 0.0, 0.0, 0.577584), + float4(0.276885, 0.484549, 0.058405, 14.0), + float4(-0.075711, 0.153585, 0.023795, 0.000305), + float4(0.098424, 0.004326, -0.058405, 0.022171), + float4(0.50316, 0.115345, 0.856461, 0.381323), + float4(0.375309, 0.488875, 0.0, 10.0), + float4(0.090853, 0.12979, 0.0, 0.158953), + float4(0.098424, 0.114648, 0.0, 0.030716), + float4(0.0, 0.0, 1.0, 0.964691), + float4(0.466162, 0.618665, -0.043263, 11.0), + float4(0.007571, -0.015142, 0.0, 0.01033), + float4(0.007571, -0.015142, 0.043263, 0.045182), + float4(0.894427, 0.447214, 0.0, 0.577584), + float4(0.466162, 0.618665, -0.043263, 3.0), + float4(0.007571, -0.015142, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.894427, 0.447214, 0.0, 0.577584), + float4(0.326638, 0.830655, 0.0, 14.0), + float4(0.139524, -0.21199, 0.0, 0.0), + float4(0.048671, -0.34178, 0.0, 0.015213), + float4(0.0, 0.0, 1.0, 0.428687), + float4(0.326638, 0.830655, 0.0, 14.0), + float4(0.048671, -0.34178, 0.0, 0.007492), + float4(-0.125464, -0.192522, 0.0822, 0.054475), + float4(0.472493, 0.067285, 0.878762, 0.964691), + float4(0.326638, 0.830655, -0.043263, 9.0), + float4(0.139524, -0.21199, 0.0, 0.01033), + float4(0.139524, -0.21199, 0.043263, 0.045182), + float4(0.835314, 0.549773, 0.0, 0.577584), + float4(0.326638, 0.830655, -0.043263, 7.0), + float4(0.139524, -0.21199, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.835314, 0.549773, 0.0, 0.577584), + float4(-0.012979, 0.832818, -0.043263, 11.0), + float4(0.339617, -0.002163, 0.0, 0.01033), + float4(0.339617, -0.002163, 0.043263, 0.045182), + float4(0.006369, 0.99998, 0.0, 0.577584) +}; +static const float4 PRISM_BOXES[44] = { + float4(0.030135, 0.06127, -0.043945, 0.0), + float4(0.330601, 0.483055, 0.116951, 7.0), + float4(0.034429, 0.265258, -0.043945, 7.0), + float4(0.542104, 0.741798, 0.116951, 8.0), + float4(0.034429, 0.355442, -0.043945, 15.0), + float4(0.305908, 0.822319, 0.116951, 8.0), + float4(0.267405, 0.532589, -0.043945, 23.0), + float4(0.542104, 0.822319, 0.082595, 8.0), + float4(-0.045345, 0.036855, -0.044263, 31.0), + float4(0.045345, 0.23354, 0.059405, 8.0), + float4(-0.224888, 0.072548, -0.044263, 39.0), + float4(0.059405, 0.565586, 0.059405, 8.0), + float4(-0.032366, 0.042263, -0.044263, 47.0), + float4(0.148095, 0.408756, 0.059405, 8.0), + float4(0.074711, 0.08769, -0.044263, 55.0), + float4(0.184869, 0.464999, 0.064813, 8.0), + float4(0.146095, 0.271559, -0.044263, 63.0), + float4(0.211908, 0.446612, 0.038855, 8.0), + float4(0.23803, 0.310496, -0.044263, 71.0), + float4(0.320067, 0.460672, 0.06265, 8.0), + float4(0.275885, 0.310496, -0.044263, 79.0), + float4(0.412001, 0.475815, 0.047508, 8.0), + float4(0.375391, 0.350514, -0.044263, 87.0), + float4(0.45202, 0.38388, 0.044263, 4.0), + float4(0.410001, 0.350514, -0.044263, 91.0), + float4(0.474733, 0.401185, 0.044263, 5.0), + float4(-0.224888, 0.379717, -0.044263, 96.0), + float4(-0.0812, 0.532057, 0.047508, 8.0), + float4(-0.095098, 0.228295, -0.044263, 104.0), + float4(0.211908, 0.565586, 0.064813, 8.0), + float4(-0.222724, 0.430551, -0.044263, 112.0), + float4(-0.011979, 0.833818, 0.053998, 8.0), + float4(-0.100506, 0.438122, -0.044263, 120.0), + float4(0.327638, 0.833818, 0.0832, 8.0), + float4(0.127708, 0.312659, -0.044263, 128.0), + float4(0.282211, 0.467162, 0.064813, 8.0), + float4(0.245601, 0.352677, -0.044263, 136.0), + float4(0.474733, 0.604523, 0.06265, 8.0), + float4(0.178543, 0.390533, -0.044263, 144.0), + float4(0.474733, 0.639133, 0.0832, 8.0), + float4(0.200174, 0.483549, -0.044263, 152.0), + float4(0.474733, 0.639133, 0.0832, 4.0), + float4(-0.013979, 0.487875, -0.044263, 156.0), + float4(0.467162, 0.833818, 0.0832, 5.0) +}; +static const float2 PRISM_POLY[110] = { + float2(0.006442, -0.002147), + float2(-0.001074, 0.001074), + float2(-0.004294, 0.007515), + float2(-0.004294, 0.806288), + float2(-0.001074, 0.809509), + float2(0.004294, 0.809509), + float2(0.022546, 0.796626), + float2(0.185736, 0.655982), + float2(0.281288, 0.86319), + float2(0.286656, 0.870706), + float2(0.293098, 0.871779), + float2(0.465951, 0.781595), + float2(0.478834, 0.773006), + float2(0.478834, 0.763344), + float2(0.407975, 0.592638), + float2(0.45092, 0.586196), + float2(0.601227, 0.573313), + float2(0.606595, 0.569018), + float2(0.606595, 0.564724), + float2(0.601227, 0.556135), + float2(0.50138, 0.453067), + float2(0.425153, 0.37684), + float2(0.289877, 0.246933), + float2(0.081595, 0.057975), + float2(0.013957, 0.0), + float2(-0.011897, 0.0), + float2(-0.055161, 0.037855), + float2(-0.081119, 0.063813), + float2(-0.083282, 0.06814), + float2(-0.084363, 0.38288), + float2(-0.089771, 0.38288), + float2(-0.158993, 0.33529), + float2(-0.166564, 0.336372), + float2(-0.250927, 0.379635), + float2(-0.261743, 0.388288), + float2(-0.263906, 0.461836), + float2(-0.262824, 0.486712), + float2(-0.173053, 0.638133), + float2(-0.044345, 0.860939), + float2(-0.036774, 0.867429), + float2(-0.024876, 0.871755), + float2(0.33529, 0.871755), + float2(0.349351, 0.867429), + float2(0.363412, 0.853368), + float2(0.513752, 0.619747), + float2(0.513752, 0.379635), + float2(0.466162, 0.322311), + float2(0.461836, 0.320148), + float2(0.399104, 0.311496), + float2(0.393696, 0.313659), + float2(0.374227, 0.332046), + float2(0.343943, 0.288782), + float2(0.336372, 0.282293), + float2(0.269314, 0.271477), + float2(0.263906, 0.27364), + float2(0.243356, 0.292027), + float2(0.21199, 0.244438), + float2(0.207664, 0.241193), + float2(0.149258, 0.231459), + float2(0.138443, 0.231459), + float2(0.118974, 0.248764), + float2(0.116811, 0.222806), + float2(0.117892, 0.162237), + float2(0.115729, 0.076792), + float2(0.06814, 0.011897), + float2(0.062732, 0.007571), + float2(0.005408, 0.0), + float2(0.031135, 0.06227), + float2(0.252301, 0.266258), + float2(0.541104, 0.544325), + float2(0.359663, 0.559356), + float2(0.433742, 0.753681), + float2(0.304908, 0.821319), + float2(0.199693, 0.595859), + float2(0.035429, 0.740798), + float2(0.035429, 0.482055), + float2(-0.043263, 0.077874), + float2(0.001082, 0.037855), + float2(0.044345, 0.043263), + float2(0.075711, 0.08869), + float2(0.077874, 0.407756), + float2(0.10924, 0.439122), + float2(0.111403, 0.307169), + float2(0.147095, 0.272559), + float2(0.189277, 0.28013), + float2(0.210909, 0.313659), + float2(0.210909, 0.445612), + float2(0.23903, 0.45102), + float2(0.241193, 0.343943), + float2(0.276885, 0.311496), + float2(0.319067, 0.319067), + float2(0.34178, 0.353677), + float2(0.340698, 0.474815), + float2(0.375309, 0.488875), + float2(0.376391, 0.381799), + float2(0.411001, 0.351514), + float2(0.45102, 0.359085), + float2(0.473733, 0.391533), + float2(0.473733, 0.603523), + float2(0.466162, 0.618665), + float2(0.326638, 0.830655), + float2(-0.012979, 0.832818), + float2(-0.099506, 0.68356), + float2(-0.221724, 0.475896), + float2(-0.223888, 0.407756), + float2(-0.163319, 0.380717), + float2(-0.094098, 0.431551), + float2(-0.0822, 0.531057), + float2(-0.045426, 0.564586), + float2(-0.044345, 0.23254) }; +static const float PRISM_BEVEL = 0.012883; +// end of the prism mesh + +// Le verre validé sur le prototype (30/09/2026). +static const float PRISM_IOR = 1.61; +static const float PRISM_DISPERSION = 0.035; +static const float PRISM_GLOW = 0.45; +static const float PRISM_FOLD = 0.35; +// Rebonds au plus dans le cristal, par canal ; l'id de matière du cristal. +static const int PRISM_BOUNCES = 6; +static const float PRISM_MAT = 9.0; + +// Le modèle de Prism Glow de ce dessin : 0 flèche, 1 main, -1 un autre curseur. +int prism_shape() +{ + int id = sculpt_id(); + return id == 3 || id == 4 ? id - 3 : -1; +} -float s_facet_ellipsoid(float3 p, float3 r) +// Distance signée au polygone `n` sommets de PRISM_POLY à partir de `i0` (plan xy du modèle), +// négative dedans : la silhouette du serti, ou le contour du cristal. +float prism_poly_dist(float2 p, int i0, int n) { - float3 q = abs(p); - float d = -1e9; - [loop] for (int i = 0; i < 10; i++) + float2 vj = PRISM_POLY[i0 + n - 1]; + float d = 1e9; + float sgn = 1.0; + [loop] for (int i = 0; i < n; i++) { - d = max(d, dot(q, SCULPT_FACETS[i]) - length(r * SCULPT_FACETS[i])); + float2 vi = PRISM_POLY[i0 + i]; + float2 e = vj - vi; + float2 w = p - vi; + float2 b = w - e * saturate(dot(w, e) / dot(e, e)); + d = min(d, dot(b, b)); + bool c0 = p.y >= vi.y; + bool c1 = p.y < vj.y; + bool c2 = e.x * w.y > e.y * w.x; + if ((c0 && c1 && c2) || (!c0 && !c1 && !c2)) + { + sgn = -sgn; + } + vj = vi; } - return d; + return sgn * sqrt(d); } -// Le gant des autres thèmes, mêmes os, taillé en facettes. -float s_crystal_hand(float3 p) +// Le serti : la silhouette extrudée de z = -épaisseur à 0, arêtes arrondies de PRISM_BEVEL. +float prism_rim(float3 p, int shape) { - float zc = SCULPT_HAND_ZC; - float d = s_facet_ellipsoid(p - float3(0.185, -0.70, zc), float3(0.3, 0.235, 0.125)); - d = min(d, s_facet_capsule(p, float3(0.0, -0.52, zc), float3(0.0, -0.10, zc), 0.098, 0.3)); - d = min(d, s_facet_capsule(p, float3(0.17, -0.57, zc + 0.012), float3(0.17, -0.41, zc + 0.045), 0.086, 0.1)); - d = min(d, s_facet_capsule(p, float3(0.31, -0.59, zc + 0.01), float3(0.31, -0.45, zc + 0.04), 0.08, 0.5)); - d = min(d, s_facet_capsule(p, float3(0.435, -0.625, zc + 0.005), float3(0.435, -0.52, zc + 0.03), 0.07, 0.2)); - return min(d, s_facet_capsule(p, float3(0.03, -0.77, zc + 0.03), float3(-0.165, -0.60, zc + 0.065), 0.082, 0.7)); + float half_t = model_thick() * 0.5; + float sil = prism_poly_dist(p.xy, PRISM_SIL_START[shape], PRISM_SIL_COUNT[shape]); + float2 w = float2(sil + PRISM_BEVEL, abs(p.z + half_t) - (half_t - PRISM_BEVEL)); + return min(max(w.x, w.y), 0.0) + length(max(w, 0.0)) - PRISM_BEVEL; } -// Le modèle `shape` du thème `theme`, repère du prototype : (distance, id de matière). -float2 sculpt_proto(float3 p, int theme, int shape) +// Distance le long de (ro, rd) au triangle `k` (Möller–Trumbore), -1 s'il le rate. Un rien de jeu +// sur les bords : un rayon ne passe pas entre deux facettes voisines. +float prism_tri(float3 ro, float3 rd, int k) { - if (theme == 3) + float3 v0 = PRISM_TRIS[4 * k].xyz; + float3 e1 = PRISM_TRIS[4 * k + 1].xyz; + float3 e2 = PRISM_TRIS[4 * k + 2].xyz; + float3 pv = cross(rd, e2); + float det = dot(e1, pv); + if (abs(det) < 1e-10) { - return s_voxels(p, shape); + return -1.0; } - if (theme == 1) + float inv = 1.0 / det; + float3 sv = ro - v0; + float u = dot(sv, pv) * inv; + float3 qv = cross(sv, e1); + float v = dot(rd, qv) * inv; + if (u < -2e-4 || v < -2e-4 || u + v > 1.0004) { - return float2(shape == 0 ? s_gem_arrow(p) : s_crystal_hand(p), 8.0); + return -1.0; } - float2 body; - float3 star; - if (shape == 0) + return dot(e2, qv) * inv; +} + +// Le triangle du cristal `shape` le plus proche devant (ro, rd) : sa distance (1e9 si aucun), son +// indice dans `hit` (-1 si aucun). Les triangles sont rangés par voisinage, une boîte englobante +// par groupe (PRISM_BOXES) : un rayon qui rate une boîte, ou n'y entre qu'au-delà du meilleur +// point, saute tous ses triangles. L'inverse de la direction sert à toutes les boîtes. +float prism_trace(float3 ro, float3 rd, int shape, out int hit) +{ + float3 inv = 1.0 / (abs(rd) > 1e-6 ? rd : 1e-6); + int b0 = PRISM_BOX_START[shape]; + int nb = PRISM_BOX_COUNT[shape]; + float best = 1e9; + hit = -1; + [loop] for (int j = b0; j < b0 + nb; j++) { - float h = 0.075, re = 0.03, dome = 0.0; - if (theme == 2) - { - h = 0.07; - re = 0.06; - dome = 0.035; - } - if (theme == 4) + float4 lo = PRISM_BOXES[2 * j]; + float4 hi = PRISM_BOXES[2 * j + 1]; + float3 t0 = (lo.xyz - ro) * inv; + float3 t1 = (hi.xyz - ro) * inv; + float3 tn = min(t0, t1); + float3 tf = max(t0, t1); + float tmin = max(max(tn.x, tn.y), tn.z); + float tmax = min(min(tf.x, tf.y), tf.z); + if (tmax < max(tmin, 1e-5) || tmin > best) { - h = 0.07; - re = 0.055; - dome = 0.025; + continue; } - float2 r = float2(s_arrow_solid(p, h, re, dome), 1.0); - if (theme == 0) + int k0 = (int)lo.w; + int n = (int)hi.w; + [loop] for (int i = k0; i < k0 + n; i++) { - r = s_opu(r, float2(s_piping(p, SCULPT_HOVER + 2.0 * h - 0.004), 2.0)); + float t = prism_tri(ro, rd, i); + if (t > 1e-5 && t < best) + { + best = t; + hit = i; + } } - body = r; - star = float3(0.57, -0.86, SCULPT_HOVER + 2.0 * h + dome); } - else + return best; +} + +// Le modèle `shape` du thème `theme`, repère du prototype : (distance, id de matière). +float2 sculpt_proto(float3 p, int theme, int shape) +{ + if (theme == 3) { - body = s_opu(float2(s_glove(p, SCULPT_HAND_ZC), 1.0), float2(s_cuff(p, SCULPT_HAND_ZC), 2.0)); - star = float3(0.185, -0.93, SCULPT_HAND_ZC + 0.15); + return s_voxels(p, shape); + } + if (theme == 5) + { + return s_glass(p, shape); } - // L'étoile de Star Sprout : un seul appel pour les deux formes (FXC recopie chaque appel). - return theme == 4 ? s_opu(body, s_sprout(p, star)) : body; + // Un seul appel pour les trois thèmes cerclés et leurs deux formes (FXC recopie chaque appel). + return s_rimmed(p, theme, shape); } // Repère du modèle -> prototype. L'écrasement du clic ne raccourcit que z, autour du hotspot. @@ -1005,23 +2104,28 @@ float sculpt_units() // Le curseur sculpté en `q` (repère du modèle) : distance et id de matière. `occ` : ce qui porte // l'ombre sur l'écran. La borne des plans d'une gemme et le champ des voxels sont de mauvaises // distances loin de la surface : la pénombre s'y strie, ou s'arrête net au bord de la boîte des -// voxels. Le cristal porte donc l'ombre de la forme lisse de Pop Coral, les voxels celle de leur -// contour extrudé sur toute leur hauteur. +// voxels. Pixel Candy porte donc l'ombre de sa forme, extrudée sur la hauteur des cubes. Prism +// Glow n'a ici que son serti, dans le repère du modèle : son cristal est tracé à part +// (`prism_primary`), et l'ombre est celle de toute sa silhouette. float2 sculpt_eval(float3 q, bool occ) { int id = sculpt_id() - 1; int theme = id / 2, shape = id % 2; + if (theme == 1) + { + return float2(prism_rim(q, shape), 8.0); + } float3 p = sculpt_point(q); float2 r; if (occ && theme == 3) { - float d2 = shape == 0 ? s_arrow2(p.xy) - 1.2 * SCULPT_VOX : s_pixel_hand_dist(p.xy); + float d2 = s_pixel_outline(p.xy, shape); float dz = abs(p.z - (SCULPT_HOVER + 0.07)) - 0.07; r = float2(length(max(float2(d2, dz), 0.0)) + min(max(d2, dz), 0.0), 7.0); } else { - r = sculpt_proto(p, occ && theme == 1 ? 2 : theme, shape); + r = sculpt_proto(p, theme, shape); } return float2(r.x * sculpt_units(), r.y); } @@ -1138,20 +2242,16 @@ SculptMat s_mat(float3 alb, float rough, float spec, float sss, float refl) return m; } -// Couleur de face du voxel qui porte `p` (prototype) : menthe où le bord regarde en bas à gauche, -// rose pâle où il regarde en haut à droite, rose dedans. +// Couleur du cube qui porte `p` (prototype) : celle de son pixel. float3 s_pixel_colour(float3 p, int shape) { - float2 id = floor((p.xy - s_grid_origin(shape)) / SCULPT_VOX); - if (!s_occ(id + float2(-1.0, 0.0), shape) || !s_occ(id + float2(0.0, -1.0), shape)) - { - return s_lin(0.52, 0.91, 0.77); - } - if (!s_occ(id + float2(1.0, 0.0), shape) || !s_occ(id + float2(0.0, 1.0), shape)) - { - return s_lin(1.0, 0.78, 0.87); - } - return s_lin(1.0, 0.50, 0.71); + int2 cell = s_pix_cell(p.xy, shape); + int4 g = PIX_GRID[shape]; + int row = g.z + clamp(cell.y, 0, g.y - 1); + if (s_pix_bit(PIX_LINE[row], cell.x)) return s_lin(0.29, 0.12, 0.36); + if (s_pix_bit(PIX_HI[row], cell.x)) return s_lin(1.0, 0.78, 0.87); + if (s_pix_bit(PIX_SHADE[row], cell.x)) return s_lin(0.87, 0.27, 0.51); + return s_lin(1.0, 0.435, 0.66); } // La matière `mat` (id de `sculpt_proto`) au point `p` du prototype. @@ -1160,28 +2260,42 @@ SculptMat sculpt_material(float mat, float3 p, int theme, int shape) bool primary = mat < 1.5; if (theme == 0) { - if (shape == 0 && primary) return s_mat(s_lin(0.10, 0.10, 0.115), 0.3, 0.2, 0.0, 0.9); - if (shape == 0) return s_mat(s_lin(0.94, 0.91, 0.84), 0.45, 0.4, 0.2, 0.3); - if (primary) return s_mat(s_lin(0.95, 0.92, 0.85), 0.55, 0.35, 0.35, 0.25); - return s_mat(s_lin(0.17, 0.18, 0.22), 0.35, 0.6, 0.0, 0.6); + // Ivoire (le gant, la bande de la flèche) ; noir satiné, presque sans reflet : un reflet + // large sur le champ plat de la flèche le virait au gris. + if (mat < 2.5) return s_mat(s_lin(0.95, 0.92, 0.85), 0.3, 0.6, 0.35, 0.25); + return s_mat(s_lin(0.1, 0.1, 0.11), 0.55, 0.05, 0.0, 0.1); + } + if (theme == 1) + { + // Le serti de Prism Glow : un émail marine, brillant sur son arrondi seulement (`model_shade`). + return s_mat(s_lin(0.02, 0.05, 0.33), 0.35, 0.5, 0.05, 0.3); + } + if (theme == 5) + { + // Le trait de Glass Lens : un graphite laqué, dont seuls les bords arrondis brillent (le + // verre a son propre ombrage, `glass_shade`). + return s_mat(s_lin(0.05, 0.055, 0.07), 0.25, 0.7, 0.0, 0.35); } if (theme == 2) { - if (shape == 0) return s_mat(s_lin(1.0, 0.40, 0.30), 0.5, 0.45, 0.4, 0.25); - if (primary) return s_mat(s_lin(1.0, 0.79, 0.16), 0.5, 0.45, 0.4, 0.25); - return s_mat(s_lin(0.18, 0.20, 0.29), 0.4, 0.5, 0.0, 0.4); + // Corail : la flèche, le calque et les tirets de la main ; jaune : le gant, le calque et les + // tirets de la flèche ; le trait marine. + if ((primary && shape == 0) || (mat > 2.5 && mat < 3.5 && shape == 1)) return s_mat(s_lin(1.0, 0.40, 0.30), 0.85, 0.08, 0.15, 0.03); + if (mat < 3.5) return s_mat(s_lin(1.0, 0.80, 0.10), 0.85, 0.08, 0.15, 0.03); + return s_mat(s_lin(0.09, 0.13, 0.45), 0.9, 0.05, 0.05, 0.02); } if (theme == 4) { - if (primary && shape == 0) return s_mat(s_lin(0.62, 0.91, 0.78), 0.22, 0.8, 0.25, 0.6); - if (primary) return s_mat(s_lin(0.96, 0.94, 0.88), 0.5, 0.35, 0.35, 0.25); - if (mat < 2.5) return s_mat(s_lin(0.62, 0.91, 0.78), 0.25, 0.7, 0.25, 0.5); - if (mat < 3.5) return s_mat(s_lin(1.0, 0.80, 0.20), 0.3, 0.6, 0.3, 0.4); - if (mat < 4.5) return s_mat(s_lin(0.38, 0.80, 0.55), 0.35, 0.5, 0.35, 0.3); - return s_mat(s_lin(0.16, 0.12, 0.10), 0.2, 0.8, 0.0, 0.5); + // Coussins laqués ; le marine satiné, un reflet large et sombre plutôt qu'un filet blanc. + if (primary && shape == 0) return s_mat(s_lin(0.68, 0.93, 0.80), 0.25, 0.7, 0.3, 0.3); + if (primary) return s_mat(s_lin(0.97, 0.95, 0.90), 0.3, 0.6, 0.35, 0.25); + if (mat < 2.5) return s_mat(s_lin(0.52, 0.87, 0.78), 0.25, 0.7, 0.3, 0.3); + if (mat < 3.5) return s_mat(s_lin(1.0, 0.75, 0.25), 0.25, 0.7, 0.3, 0.3); + if (mat < 4.5) return s_mat(s_lin(0.62, 0.92, 0.72), 0.3, 0.6, 0.3, 0.35); + return s_mat(s_lin(0.07, 0.15, 0.33), 0.7, 0.15, 0.05, 0.08); } - if (mat < 6.5) return s_mat(s_pixel_colour(p, shape), 0.45, 0.35, 0.15, 0.2); - return s_mat(s_lin(0.36, 0.18, 0.54), 0.45, 0.35, 0.1, 0.2); + // Plastique mat : sur les faces plates des cubes, un reflet de la lampe blanchirait le prune. + return s_mat(s_pixel_colour(p, shape), 0.65, 0.1, 0.12, 0.04); } // L'environnement du studio vu du modèle : l'écran dessous, la pièce au-dessus (z du modèle), une @@ -1197,39 +2311,6 @@ float3 model_env(float3 d, float rough, float3 l, float3 fill) return col; } -// Cyan, bleu, violet, orchidée. -float3 s_gem(float k) -{ - k = saturate(k) * 3.0; - float3 a = s_lin(0.20, 0.95, 1.0); - float3 b = s_lin(0.15, 0.42, 1.0); - float3 c = s_lin(0.45, 0.25, 0.95); - float3 d = s_lin(0.88, 0.50, 1.0); - if (k < 1.0) return lerp(a, b, k); - if (k < 2.0) return lerp(b, c, k - 1.0); - return lerp(c, d, k - 2.0); -} - -// Le cristal : la teinte vient d'où la facette regarde et d'où elle plie la vue (lues y vers le -// haut, comme au prototype) ; la dispersion sépare le décalage par canal. Les facettes tournées -// vers la lampe sont cyan, les autres violettes. -float3 model_shade_crystal(float3 n, float3 rd, float3 L, float fall, float3 l, float3 fill) -{ - float cosi = saturate(dot(-rd, n)); - float F = 0.04 + 0.96 * pow(1.0 - cosi, 5.0); - float3 t = refract(rd, n, 1.0 / 1.6); - float k = 0.42 + 0.9 * dot(float2(n.x, -n.y), float2(0.7557, -0.6549)) - + 0.6 * dot(float2(t.x, -t.y), float2(0.6, -0.8)); - float3 body = float3(s_gem(k - 0.08).r, s_gem(k).g, s_gem(k + 0.08).b); - float3 col = body * (0.1 + 1.8 * fall * pow(max(dot(n, L), 0.0), 2.5)); - // Éclat : la vue rebondit sur le dos plat et allume la facette quand elle trouve la lampe. - col += body * model_env(reflect(t, float3(0.0, 0.0, 1.0)) * float3(1.0, 1.0, -1.0), 0.15, l, fill) * 0.5; - col += pow(max(dot(reflect(rd, n), L), 0.0), 30.0) * 2.0; - col += model_env(reflect(rd, n), 0.05, l, fill) * F; - col += s_lin(0.5, 0.9, 1.0) * pow(1.0 - cosi, 4.0) * 0.6; - return col; -} - // Khronos PBR Neutral : garde les teintes, ne comprime que les hautes lumières ; puis sRGB. float3 model_tonemap(float3 c) { @@ -1260,10 +2341,6 @@ float3 model_shade(float3 q, float3 n, float3 rd, float3 L, float fall, float sh float3 l, float3 fill) { int id = sculpt_id(); - if (id > 0 && (id - 1) / 2 == 1) - { - return model_tonemap(model_shade_crystal(n, rd, L, fall, l, fill)); - } SculptMat m; // Sur un sprite, reflets et brillance sur les arrondis seulement : le dessus plat d'un sprite // sombre virerait au gris sous la lampe. @@ -1273,6 +2350,12 @@ float3 model_shade(float3 q, float3 n, float3 rd, float3 L, float fall, float sh // La matière d'un voxel se lit juste sous la surface : son flanc lit sa propre cellule. float3 p = sculpt_point(q) - float3(n.x, -n.y, n.z) * 0.01; m = sculpt_material(mat, p, (id - 1) / 2, (id - 1) % 2); + if (id == 3 || id == 4 || id > 10) + { + // Le serti de Prism Glow et le trait de Glass Lens, comme un sprite : leur dessus plat, + // face à la lampe, prendrait tout son reflet et virerait au gris. + gloss = 1.0 - smoothstep(0.97, 0.995, abs(n.z)); + } } else { @@ -1298,6 +2381,244 @@ float3 model_shade(float3 q, float3 n, float3 rd, float3 L, float fall, float sh return model_tonemap(col); } +// ---- Le cristal de Prism Glow ---- + +float prism_fresnel(float c, float ior) +{ + float f0 = (ior - 1.0) / (ior + 1.0); + f0 *= f0; + return f0 + (1.0 - f0) * pow(1.0 - c, 5.0); +} + +// Le cristal sur le rayon (ro, rd) : distance au point touché (1e9 si raté ou hors de Prism Glow), +// facette dans `hc`. Une paroi sous le dessus du serti ne compte pas : le rayon a traversé le +// serti avant, c'est lui qu'on voit. +float prism_primary(float3 ro, float3 rd, out int hc) +{ + hc = -1; + int shape = prism_shape(); + if (shape < 0) + { + return 1e9; + } + float t = prism_trace(ro, rd, shape, hc); + if (hc >= 0 && fmod(PRISM_TRIS[4 * hc].w, 2.0) > 0.5 && ro.z + rd.z * t < 0.0) + { + hc = -1; + return 1e9; + } + return t; +} + +// La normale de la facette `h`, tournée vers le rayon qui arrive le long de `rd`. +float3 prism_normal(int h, float3 rd) +{ + float3 n = PRISM_TRIS[4 * h + 3].xyz; + return dot(n, rd) > 0.0 ? -n : n; +} + +// Distance de `p` au segment [a, b]. +float prism_seg(float3 p, float3 a, float3 b) +{ + float3 pa = p - a, ba = b - a; + return length(pa - ba * saturate(dot(pa, ba) / dot(ba, ba))); +} + +// Distance, en pixels (`px` = unités du modèle par pixel au point), de `p` aux arêtes réelles de +// la facette `h` : les plis de l'art et le bord du serti. +float prism_edge_px(float3 p, int h, float px) +{ + float4 r0 = PRISM_TRIS[4 * h]; + float3 a = r0.xyz; + float3 b = a + PRISM_TRIS[4 * h + 1].xyz; + float3 c = a + PRISM_TRIS[4 * h + 2].xyz; + int bits = (int)(r0.w * 0.5); + float d = 1e9; + if ((bits & 1) != 0) d = min(d, prism_seg(p, b, c)); + if ((bits & 2) != 0) d = min(d, prism_seg(p, c, a)); + if ((bits & 4) != 0) d = min(d, prism_seg(p, a, b)); + return d / px; +} + +// La rotation inverse de `world_to_plane` : du repère du plan à celui de la caméra. +float3 plane_to_world(float3 v, ModelFrame f) +{ + float x = v.x * f.c.z - v.y * f.s.z; + float y = v.x * f.s.z + v.y * f.c.z; + float z = -x * f.s.y + v.z * f.c.y; + return float3(x * f.c.y + v.z * f.s.y, y * f.c.x - z * f.s.x, y * f.s.x + z * f.c.x); +} + +// L'uv de sortie du point où le rayon (q, d) du modèle retombe sur le plan, (-1, -1) s'il n'y +// retombe pas. L'œil est en (-mb.zw, persp) du repère caméra, le rayon du pixel `local` y va vers +// (local + src.xy, -persp) : on remonte de ce point du plan à son `local`, puis à son uv de sortie. +float2 screen_uv(float3 q, float3 d, float3 nz, float hz, float3 tip, float unit, ModelFrame f) +{ + float denom = dot(d, nz); + if (denom > -1e-4) + { + return float2(-1.0, -1.0); + } + float3 g = q + d * ((hz - dot(q, nz)) / denom); + float3 w = plane_to_world(tip + unit * model_to_plane(g, f), f); + float k = 1.0 - w.z / src.z; + if (k < 1e-3) + { + return float2(-1.0, -1.0); + } + return dst.xy + ((w.xy + mb.zw) / k - src.xy) / quad_px * dst.zw; +} + +// Ce que montre l'image en `uv` : la copie de l'image composée faite juste avant le curseur (t5), +// flous de confidentialité compris, en linéaire. Seuls des pixels déjà floutés passent donc à +// travers le verre. Hors de l'image, l'écran sombre du studio. +float3 screen_at(float2 uv) +{ + if (any(uv < 0.0) || any(uv > 1.0)) + { + return SCULPT_SCREEN * 0.2; + } + return pow(texDof.SampleLevel(samp, uv, 0.0).rgb, 2.2); +} + +// Ce que montre l'image là où le rayon (q, d) du modèle retombe sur le plan. +float3 prism_screen(float3 q, float3 d, float3 nz, float hz, float3 tip, float unit, ModelFrame f) +{ + return screen_at(screen_uv(q, d, nz, hz, tip, unit, f)); +} + +// La couleur (sRGB, comme `model_shade`) du cristal au point `p` de la facette `h`, vu le long de +// `rd` : reflet du studio, l'enregistrement réfracté par canal (la dispersion) à travers les +// facettes et le fond plat, la lueur de la couleur de la facette dans l'art, ses plis en liseré. +float3 prism_shade(float3 p, float3 n, float3 rd, int h, float px, float3 l, float3 fill, float3 nz, + float hz, float3 tip, float unit, ModelFrame f) +{ + int shape = prism_shape(); + float4 r3 = PRISM_TRIS[4 * h + 3]; + float3 facet = float3(PRISM_TRIS[4 * h + 1].w, PRISM_TRIS[4 * h + 2].w, r3.w); + float F = prism_fresnel(saturate(-dot(rd, n)), PRISM_IOR); + float zb = -model_thick(); + float3 trans = 0.0; + [loop] for (int ch = 0; ch < 3; ch++) + { + // Le rouge plie le moins, le bleu le plus. FXC n'indexe pas un vecteur par une variable en + // écriture (il déroulerait la boucle) : le canal passe par un masque. + float3 mask = float3(ch == 0, ch == 1, ch == 2); + float ior = PRISM_IOR + PRISM_DISPERSION * (ch - 1); + float3 d = refract(rd, n, 1.0 / ior); + float3 pos = p; + float thr = 1.0; + float acc = 0.0; + [loop] for (int b = 0; b < PRISM_BOUNCES; b++) + { + int hh; + float th = prism_trace(pos, d, shape, hh); + float tb = d.z < -1e-6 ? (zb - pos.z) / d.z : 1e9; + if (tb <= th) + { + // Le fond plat : le rayon sort vers l'écran, ou s'y réfléchit tout entier. + float3 qb = pos + d * tb; + float3 d2 = refract(d, float3(0.0, 0.0, 1.0), ior); + if (dot(d2, d2) < 1e-8) + { + pos = qb; + d = reflect(d, float3(0.0, 0.0, 1.0)); + continue; + } + acc += thr * dot(prism_screen(qb, d2, nz, hz, tip, unit, f), mask); + thr = 0.0; + break; + } + if (hh < 0) + { + break; + } + pos += d * th; + float4 rn = PRISM_TRIS[4 * hh + 3]; + if (fmod(PRISM_TRIS[4 * hh].w, 2.0) > 0.5 && pos.z < 0.0) + { + // Une paroi sous le dessus du serti : le marine de son flanc intérieur. + acc += thr * dot(s_lin(0.03, 0.06, 0.37) * 0.35, mask); + thr = 0.0; + break; + } + float3 dout = refract(d, -rn.xyz, ior); + if (dot(dout, dout) < 1e-8) + { + d = reflect(d, -rn.xyz); + continue; + } + float fi = prism_fresnel(saturate(dot(dout, rn.xyz)), ior); + acc += thr * (1.0 - fi) * dot(model_env(dout, 0.05, l, fill), mask); + thr *= fi; + d = reflect(d, -rn.xyz); + } + trans += acc * mask; + } + float3 col = F * model_env(reflect(rd, n), 0.05, l, fill) + (1.0 - F) * trans + facet * PRISM_GLOW; + // Les plis de l'art : un liseré clair sur chaque arête réelle, fin d'un pixel. + float fold = 1.0 - smoothstep(0.2, 1.0, prism_edge_px(p, h, px)); + col = lerp(col, s_lin(0.96, 0.99, 1.0) * 1.3, PRISM_FOLD * fold * (0.35 + 0.65 * saturate(dot(n, l)))); + return model_tonemap(col); +} + +// ---- Le verre de Glass Lens ---- + +// La lumière qu'un verre de normale `n` (vue le long de `rd`, éclairé par `L`, `l`, `fill`) +// ajoute à ce qu'il transmet `trans` (linéaire) : le reflet du studio pondéré de Fresnel, l'éclat +// net de la lampe et son halo, et un liseré de lumière où la surface se dérobe, plus franc du +// côté de la lampe. Commun au verre en volume et au verre à plat. +float3 glass_light(float3 trans, float3 n, float3 rd, float3 L, float fall, float3 l, float3 fill) +{ + float c = saturate(-dot(rd, n)); + float F = prism_fresnel(c, GLASS_IOR); + float ndh = saturate(dot(n, normalize(L - rd))); + float ndl = dot(n, L); + float3 key = s_lin(1.0, 0.97, 0.93) * 2.1 * fall; + float spe = (pow(ndh, 900.0) * 5.0 + pow(ndh, 40.0) * 0.12) * saturate(ndl * 4.0); + float edge = pow(1.0 - c, 4.0) * (0.25 + 0.75 * saturate(ndl + 0.3)); + float3 col = (1.0 - F) * (trans * s_lin(0.97, 0.985, 1.0) + GLASS_LIFT) + F * model_env(reflect(rd, n), 0.03, l, fill); + return col + key * spe + s_lin(0.92, 0.96, 1.0) * edge * 0.6; +} + +// L'image en `uv`, moyennée le long de la réfraction, de `uv0` (où le pixel tomberait sans verre) +// à `uv` : là où le bord de la lentille comprime l'image, trois prises au lieu d'une l'empêchent de +// s'émietter en pixels qui scintillent quand le curseur bouge. +float3 glass_smear(float2 uv, float2 uv0) +{ + float2 s = all(uv0 >= 0.0) ? (uv - uv0) * GLASS_SMEAR : float2(0.0, 0.0); + return (screen_at(uv) + screen_at(uv - s) + screen_at(uv + s)) / 3.0; +} + +// Couleur (sRGB, comme `model_shade`) du verre de Glass Lens au point `q` du modèle, de normale +// `n`, vu le long de `rd` : l'enregistrement réfracté par canal (la dispersion), entré par la face +// touchée, sorti par le dessous plat du socle vers l'image, comme le cristal de Prism Glow sort +// par son fond ; une réflexion totale y renvoie le studio. L'image transmise n'est pas tonemappée : +// sous la lentille, un blanc reste blanc. +float3 glass_shade(float3 q, float3 n, float3 rd, float3 L, float fall, float3 l, float3 fill, float3 nz, + float hz, float3 tip, float unit, ModelFrame f) +{ + float zb = -model_thick(); + float2 uv0 = screen_uv(q, rd, nz, hz, tip, unit, f); + float3 trans = 0.0; + [loop] for (int ch = 0; ch < 3; ch++) + { + float3 mask = float3(ch == 0, ch == 1, ch == 2); + float ior = GLASS_IOR + GLASS_DISPERSION * (ch - 1); + float3 d = refract(rd, n, 1.0 / ior); + float3 seen = model_env(d, 0.05, l, fill); + if (d.z < -1e-4) + { + float3 qb = q + d * ((zb - q.z) / d.z); + float3 d2 = refract(d, float3(0.0, 0.0, 1.0), ior); + seen = dot(d2, d2) < 1e-8 ? model_env(reflect(d, float3(0.0, 0.0, 1.0)), 0.05, l, fill) + : glass_smear(screen_uv(qb, d2, nz, hz, tip, unit, f), uv0); + } + trans += seen * mask; + } + return pow(saturate(glass_light(trans, n, rd, L, fall, l, fill)), 1.0 / 2.2); +} + // Les passes de la boucle unique de `cursor_model`, dans l'ordre. Chaque tour n'évalue le modèle // QU'UNE fois (`model_eval`) : FXC recopie une fonction à chaque appel, et un curseur sculpté est // assez gros pour qu'un appel par passe multiplie par neuf le temps de compilation de ps_main. @@ -1306,7 +2627,12 @@ static const int STAGE_NORMAL = 1; // quatre sondes en tétraèdre autour du poi static const int STAGE_AO = 2; // cinq sondes le long de la normale, jusqu'à 0,08 du prototype static const int STAGE_SELF = 3; // pénombre du modèle sur lui-même, vers la lampe (32 pas) static const int STAGE_PLANE = 4; // le point du plan : son contact, puis sa pénombre (32 pas) -static const int STAGE_DONE = 5; +static const int STAGE_EDGE = 5; // quatre sondes à un demi-pixel du point touché : un bord ? +static const int STAGE_SHADE = 6; // l'ombrage d'un échantillon, puis le rayon suivant s'il en reste +static const int STAGE_DONE = 7; + +// Les trois rayons de plus d'un pixel au bord, en pixels autour de son centre. +static const float2 MODEL_SUBPIXEL[3] = { float2(0.35, 0.2), float2(-0.35, 0.2), float2(0.0, -0.4) }; // Sonde `k` de la normale, sommet d'un tétraèdre. float3 model_tetra(int k) @@ -1347,17 +2673,30 @@ float4 cursor_model(float2 local) float3 lamp = float3(color.rg + sprite_size() * 0.5, 0.0) + l * SCULPT_LAMP_DIST; // Silhouette antialiasée : un rayon qui frôle le modèle à moins d'un pixel le couvre en partie - // (`best`, la plus petite distance rencontrée, en pixels). - float2 tb = ray_box(ro, rd, lo - 0.02, hi + 0.02); + // (`best`, la plus petite distance rencontrée, en pixels). Les bords intérieurs (deux matières, + // une arête, un arrondi serré) sont suréchantillonnés : `STAGE_EDGE` sonde le modèle à un + // demi-pixel du point touché, dans son plan tangent ; si la matière change ou que la surface + // s'écarte, trois rayons de plus (`MODEL_SUBPIXEL`) sont marchés et ombrés, avec l'occlusion et + // les ombres du premier. Chaque échantillon passe par `STAGE_SHADE`, seul appel de + // `model_shade` ; le pixel est leur moyenne. + float3 ray = rd; + float2 tb = ray_box(ro, ray, lo - 0.02, hi + 0.02); int stage = tb.x < tb.y && tb.y > 0.0 ? STAGE_MARCH : STAGE_DONE; + // Prism Glow : le cristal est tracé au départ de chaque rayon ; `tc` est + // son point touché (1e9 : raté), `hc` sa facette. La marche du serti s'arrête à lui. + bool need_tc = stage == STAGE_MARCH; + float tc = 1e9; + int hc = -1; // Le point du plan derrière le pixel est à régler avant le prochain tour. bool plane_next = stage == STAGE_DONE; int k = 0; float t = max(tb.x, 0.0); float best = 1e9; float t_best = t; + float d_best = 0.0; float mat = 0.0; float cov = 0.0; + float cov_s = 0.0; float3 q = 0.0; float3 n = 0.0; float3 L = 0.0; @@ -1371,14 +2710,24 @@ float4 cursor_model(float2 local) float inside = 0.0; float contact = 0.0; float dropped = 0.0; - [loop] for (int it = 0; it < 180; it++) + // Sous-échantillons restants, échantillons ombrés, et leur somme (couleur × couverture, + // couverture). + int sub = 0; + int shaded = 0; + float4 acc = 0.0; + [loop] for (int it = 0; it < 500; it++) { + if (need_tc) + { + need_tc = false; + tc = prism_primary(ro, ray, hc); + } if (plane_next) { // Le plan, là où le modèle ne couvre pas tout le pixel : ombre portée et ombre de // contact, seulement à l'intérieur de l'écran (`mb.xy` = sa demi-taille, px du plan). plane_next = false; - stage = STAGE_DONE; + stage = cov > 0.0 ? STAGE_SHADE : STAGE_DONE; float denom = dot(rd, nz); if (cov < 1.0 && denom < -1e-4) { @@ -1393,15 +2742,67 @@ float4 cursor_model(float2 local) } } } + if (stage == STAGE_SHADE) + { + if (cov_s > 0.0) + { + float3 c; + float3 tl = lamp - q; + float fall = SCULPT_LAMP_DIST * SCULPT_LAMP_DIST / dot(tl, tl); + if (abs(mat - PRISM_MAT) < 0.5) + { + c = prism_shade(q, n, ray, hc, t_best / dlen, l, fill, nz, hz, tip, unit, f); + } + else if (mat > GLASS_MAT - 0.5) + { + c = glass_shade(q, n, ray, normalize(tl), fall, l, fill, nz, hz, tip, unit, f); + } + else + { + c = model_shade(q, n, ray, normalize(tl), fall, sh, ao, mat, l, fill); + } + acc += float4(c * cov_s, cov_s); + } + shaded++; + stage = STAGE_DONE; + if (sub > 0) + { + // Le rayon suivant, décalé dans le pixel ; il repart de sa propre entrée dans la + // boîte, pour ne pas manquer ce qui passerait devant le premier. + float3 dws = float3(local + MODEL_SUBPIXEL[3 - sub] + src.xy, -persp); + sub--; + dlen = length(dws); + ray = plane_to_model(world_to_plane(dws / dlen, f), f); + tb = ray_box(ro, ray, lo - 0.02, hi + 0.02); + cov_s = 0.0; + stage = STAGE_SHADE; + if (tb.x < tb.y && tb.y > 0.0) + { + stage = STAGE_MARCH; + k = 0; + t = max(tb.x, 0.0); + best = 1e9; + t_best = t; + need_tc = true; + } + } + continue; + } float3 pos; if (stage == STAGE_MARCH) { - pos = ro + rd * t; + pos = ro + ray * t; } else if (stage == STAGE_NORMAL) { pos = q + 0.002 * model_tetra(k); } + else if (stage == STAGE_EDGE) + { + float3 side = normalize(cross(n, ray)); + float3 e = (k >= 2 ? cross(n, side) : side) * (k % 2 == 1 ? -1.0 : 1.0); + pos = q + e * 0.5 * t_best / dlen; + } else if (stage == STAGE_AO) { pos = q + (0.01 + 0.0175 * k) * SCULPT_SCALE * n; @@ -1428,22 +2829,68 @@ float4 cursor_model(float2 local) { best = hit ? 0.0 : d / fp; t_best = t; + d_best = d; mat = m.y; } t += d * stride; k++; + if (!hit && t >= tc) + { + // Le pas dépasse le cristal : c'est lui que le rayon touche, avant le serti. + hit = true; + best = 0.0; + t_best = tc; + d_best = 0.0; + mat = PRISM_MAT; + } if (hit || t > tb.y || k == 96) { - cov = saturate(1.0 - best); - if (cov > 0.0) + cov_s = saturate(1.0 - best); + if (shaded > 0) { - q = ro + rd * t_best; - stage = STAGE_NORMAL; - k = 0; + stage = STAGE_SHADE; + if (cov_s > 0.0) + { + q = ro + ray * t_best; + n = 0.0; + stage = STAGE_NORMAL; + k = 0; + } } else { - plane_next = true; + cov = cov_s; + if (cov > 0.0) + { + q = ro + ray * t_best; + stage = STAGE_NORMAL; + k = 0; + } + else + { + plane_next = true; + } + } + if (stage == STAGE_NORMAL && abs(mat - PRISM_MAT) < 0.5) + { + // Le cristal : la normale de sa facette, sans sondes ; ni occlusion ni ombre + // propre. Un bord réel à moins d'un pixel (le serti, un pli) appelle les trois + // rayons de plus, comme les bords intérieurs des autres modèles. + n = prism_normal(hc, ray); + k = 0; + if (shaded > 0) + { + stage = STAGE_SHADE; + } + else + { + if (prism_edge_px(q, hc, t_best / dlen) < 0.75) + { + sub = 3; + } + stage = STAGE_DONE; + plane_next = true; + } } } } @@ -1454,6 +2901,34 @@ float4 cursor_model(float2 local) if (k == 4) { n = normalize(n); + k = 0; + stage = shaded > 0 ? STAGE_SHADE : STAGE_EDGE; + } + } + else if (stage == STAGE_EDGE) + { + // Autre matière, ou surface qui s'écarte du plan tangent de plus de 2 % d'un pixel + // (le point touché flotte déjà de `d_best` au-dessus d'elle). + if (m.y != mat || abs(d - d_best) > 0.02 * t_best / dlen) + { + sub = 3; + } + if (k == 0 && mat > 7.5 && mat < PRISM_MAT + 0.5) + { + // Le serti de Prism Glow au bord du cristal : son contour à moins d'un pixel. + int ps = prism_shape(); + if (ps >= 0) + { + float dout = prism_poly_dist(q.xy, PRISM_OUT_START[ps], PRISM_OUT_COUNT[ps]); + if (abs(dout) < 0.75 * t_best / dlen) + { + sub = 3; + } + } + } + k++; + if (k == 4) + { stage = STAGE_AO; k = 0; ao = 0.0; @@ -1504,7 +2979,7 @@ float4 cursor_model(float2 local) k = 1; if (!(tbp.x < tbp.y && tbp.y > 0.0)) { - stage = STAGE_DONE; + stage = cov > 0.0 ? STAGE_SHADE : STAGE_DONE; } } else @@ -1516,21 +2991,100 @@ float4 cursor_model(float2 local) { res = saturate(res); dropped = 1.0 - res * res * (3.0 - 2.0 * res); - stage = STAGE_DONE; + stage = cov > 0.0 ? STAGE_SHADE : STAGE_DONE; } } } - float3 rgb = 0.0; - if (cov > 0.0) + float w = 1.0 / max(shaded, 1); + // La lentille de Glass Lens laisse passer la lumière : son ombre est plus claire. + float clear = sculpt_id() > 10 ? GLASS_SHADOW : 1.0; + float shadow = inside * clear * max(dropped * MODEL_SHADOW_ALPHA, contact * MODEL_CONTACT_ALPHA); + float a = acc.a * w * color.a; + return float4(acc.rgb * w * color.a, a + (1.0 - a) * shadow * color.a); // prémultiplié, ombre noire +} + +// ============ Verre à plat (mode 19) ============ +// Glass Lens, 3D éteinte (`frame_geometry::cursor_glass_cb`, qui documente les emplacements) : le +// relief de son modèle vu de face (`s_glass2`, `s_glass_relief`), posé sur l'image comme le sprite +// du mode 13. Un rayon de vue droit y entre, plie à la surface, traverse le verre jusqu'au dessous +// plat du socle, en sort et descend GLASS_GAP_FLAT jusqu'à l'image : la copie de l'image composée +// (t5), lue par canal (la dispersion), décalée d'autant. Le trait est une bande graphite. +float4 cursor_glass(float2 local, float2 pout) +{ + if (pout.x < dst_prev.x || pout.x > dst_prev.x + dst_prev.z || + pout.y < dst_prev.y || pout.y > dst_prev.y + dst_prev.w) + { + return float4(0.0, 0.0, 0.0, 0.0); + } + float3 r = quad_inverse(local, fx.xy, fx.zw, src_prev.xy, src_prev.zw, mb.x); + if (r.z < 0.5) + { + return float4(0.0, 0.0, 0.0, 0.0); + } + // Le point du prototype (y vers le haut) sous ce pixel, et un pixel en unités du prototype. + float2 m = color.rg + saturate(r.xy) * sprite_size(); + float2 p = float2(m.x, -m.y) / SCULPT_SCALE; + float px = 1.0 / max(color.b * SCULPT_SCALE, 1e-3); + int shape = (sculpt_id() - 1) % 2; + // La forme et le gradient de la lentille : trois évaluations dans une boucle, un seul appel + // pour FXC. + float2 g = 0.0; + float2 gd = 0.0; + float e = 0.25 * px; + [loop] for (int k = 0; k < 3; k++) + { + float2 v = s_glass2(p + (k == 1 ? float2(e, 0.0) : (k == 2 ? float2(0.0, e) : float2(0.0, 0.0))), shape); + if (k == 0) g = v; + else if (k == 1) gd.x = v.y; + else gd.y = v.y; + } + float cov = saturate(0.5 - g.x / px); + if (cov <= 0.0) + { + return float4(0.0, 0.0, 0.0, 0.0); + } + float2 out_dir = (gd - g.y) / e; + out_dir /= max(length(out_dir), 1e-4); + // Le relief, repère de la sortie (x à droite, y vers le bas, z vers le spectateur). + float2 relief = s_glass_relief(g.y); + float2 dir = float2(out_dir.x, -out_dir.y); + float3 n = normalize(float3(-relief.y * dir, 1.0)); + float3 rd = float3(0.0, 0.0, -1.0); + // Le verre : par canal, la réfraction à la surface, la traversée jusqu'au dessous du socle, + // la sortie, puis l'air jusqu'à l'image. Le liseré, lui, est un bord arrondi : la lumière en + // ressort par le côté, après sa seule largeur de verre. Décalage en unités du prototype, puis + // de la sortie. + bool rim = g.y > GLASS_LINE; + float depth = rim ? GLASS_RIM - GLASS_LINE : GLASS_TOP - SCULPT_HOVER + relief.x; + float3 trans = 0.0; + [loop] for (int ch = 0; ch < 3; ch++) { - float3 tl = lamp - q; - float fall = SCULPT_LAMP_DIST * SCULPT_LAMP_DIST / dot(tl, tl); - rgb = model_shade(q, n, rd, normalize(tl), fall, sh, ao, mat, l, fill); + float3 mask = float3(ch == 0, ch == 1, ch == 2); + float ior = GLASS_IOR + GLASS_DISPERSION * (ch - 1); + float3 d = refract(rd, n, 1.0 / ior); + float3 d2 = refract(d, float3(0.0, 0.0, 1.0), ior); + float2 off = d.xy * depth / max(-d.z, 0.05); + if (!rim && dot(d2, d2) > 1e-8) + { + off += d2.xy * GLASS_GAP_FLAT / max(-d2.z, 0.05); + } + off *= SCULPT_SCALE * src.xy; + trans += glass_smear(pout + off, pout) * mask; } - float shadow = inside * max(dropped * MODEL_SHADOW_ALPHA, contact * MODEL_CONTACT_ALPHA); + float3 col = glass_light(trans, n, rd, MODEL_LIGHT, 1.0, MODEL_LIGHT, MODEL_FILL); + // Le trait : une bande graphite au dessus plat, comme en volume ; ses bords arrondis sur un + // quart de sa largeur prennent la lumière du côté de la lampe. + float edge_k = saturate(1.0 - min(g.y, GLASS_LINE - g.y) / (0.25 * GLASS_LINE)); + float side = g.y < 0.5 * GLASS_LINE ? -1.0 : 1.0; + float3 nl = normalize(float3(side * edge_k * dir, 1.0)); + float3 hl = normalize(MODEL_LIGHT + float3(0.0, 0.0, 1.0)); + float3 line_col = s_lin(0.05, 0.055, 0.07) * (0.6 + 0.5 * saturate(dot(nl, MODEL_LIGHT))) + + s_lin(1.0, 0.97, 0.93) * 0.8 * edge_k * pow(saturate(dot(nl, hl)), 40.0); + float in_line = saturate(0.5 + g.y / px) * saturate(0.5 - (g.y - GLASS_LINE) / px); + col = lerp(col, line_col, in_line); float a = cov * color.a; - return float4(rgb * a, a + (1.0 - a) * shadow * color.a); // prémultiplié, ombre noire + return float4(pow(saturate(col), 1.0 / 2.2) * a, a); // prémultiplié } // ============ Impact du clic (mode 16) ============ @@ -2156,6 +3710,17 @@ float4 device_frame(float2 local) float4 ps_main(VSOut i) : SV_Target { + // mode 19 : VERRE À PLAT de Glass Lens (cf. `cursor_glass`). Testé avant le mode 18, dont la + // branche n'a pas de borne haute. C'est un modèle, compilé avec les modes 15 à 17. + if (mode > 18.5) + { +#ifdef LAYER_MODELS + return cursor_glass(i.local, i.pout); +#else + return float4(0.0, 0.0, 0.0, 0.0); +#endif + } + // mode 18 : l'écran CADRÉ (ombre, cadre, métrage, appareil) flouté comme UN objet rigide // (`FrameGeometry::screen_trail`). t2 = son rendu isolé, prémultiplié, à la taille de la // sortie. Chaque tap retrouve le point de l'objet qui couvrait ce pixel plus tôt sur la @@ -2213,6 +3778,7 @@ float4 ps_main(VSOut i) : SV_Target return acc / max(n, 1.0); } +#ifdef LAYER_MODELS // mode 17 : CADRE D'APPAREIL MODELÉ (cf. `device_frame`). Testé en premier, comme le 16. if (mode > 16.5) { @@ -2236,6 +3802,14 @@ float4 ps_main(VSOut i) : SV_Target } return cursor_model(i.local); } +#else + // Les modèles 3D n'existent que dans `ps_main_models` ; un draw qui les enverrait ici ne + // peint rien plutôt qu'une ombre faite de leurs emplacements. + if (mode > 14.5) + { + return float4(0.0, 0.0, 0.0, 0.0); + } +#endif // mode 14 : CADRE DE FENÊTRE autour de l'écran (barre de titre, trois pastilles, filet), // dessiné SOUS lui. Testé avant le mode 13, dont la branche n'a pas de borne haute. @@ -2490,8 +4064,12 @@ float4 ps_main(VSOut i) : SV_Target float2 plane_px = dst_prev.xy; float2 p = float2(r.x, r.y) * plane_px - plane_px * 0.5; float rad = max(radius_px, 0.0); - float d = (dst_prev.z > 0.5) ? sd_screen_under_bar(p, plane_px * 0.5, rad, color.z) - : sd_round_rect(p, plane_px * 0.5, rad); + // Une branche et non `?:`, qui évalue ses DEUX opérandes (trois SDF par pixel). + float d; + if (dst_prev.z > 0.5) + d = sd_screen_under_bar(p, plane_px * 0.5, rad, color.z); + else + d = sd_round_rect(p, plane_px * 0.5, rad); float tilt_a = 1.0 - smoothstep(0.0, 1.5, d); // Slot d'un layout en bloc (`color.w` = rayon de ses coins, px ; 0 ailleurs, sans effet) : // `dst` EST le slot, dont le rect arrondi rogne le plan. Le conteneur masque, le métrage @@ -2726,8 +4304,13 @@ float4 ps_main(VSOut i) : SV_Target // cadre (`sd_screen_under_bar`, `mb.z` = la remontée du contour intérieur, px). float2 halfsz = quad_px * 0.5; float2 p = i.local - quad_px * 0.5; - float d = (mb.w > 0.5) ? sd_screen_under_bar(p, halfsz, radius_px, mb.z) - : sd_round_rect(p, halfsz, radius_px); + // Une branche et non `?:`, qui évalue ses DEUX opérandes : trois SDF par pixel de l'écran + // au lieu d'une -- 0,6 ms par frame 1080p sur une Radeon 610M (mesuré en WGSL). + float d; + if (mb.w > 0.5) + d = sd_screen_under_bar(p, halfsz, radius_px, mb.z); + else + d = sd_round_rect(p, halfsz, radius_px); alpha *= 1.0 - smoothstep(0.0, 1.5, d); // ~1.5px feather (§7 fwidth-like) } return float4(rgb * alpha, alpha); // prémultiplié diff --git a/crates/compositor/src/shaders.metal b/crates/compositor/src/shaders.metal index 6c019c3ec8..2a469a504c 100644 --- a/crates/compositor/src/shaders.metal +++ b/crates/compositor/src/shaders.metal @@ -33,6 +33,12 @@ // `compositor_macos::tests::every_shader_entry_point_compiles` le compile sur le device // système au `cargo test`, pour qu'une faute de syntaxe MSL ne se découvre pas à // l'ouverture de l'éditeur chez un utilisateur. +// +// Il est compilé DEUX fois (`compositor_macos.rs::new_sized`) : tel quel pour tous les calques, +// et avec `LAYER_MODELS` défini pour les seuls modèles 3D (modes 15 à 17, +// `LayerCB::needs_models`). Un shader alloue à chaque draw les registres de sa branche la plus +// lourde, et ce sont ces modèles ray-tracés : le même `ps_main` les portait pour le fond, +// l'ombre et la vidéo, qui couvrent toute la sortie (cf. le haut de `shaders.hlsl`). #include using namespace metal; @@ -51,7 +57,7 @@ struct Layer float4 src; // u0,v0,u1,v1 dans l'espace source 0..1 ; modes 15 et 17 : décalage px du rayon, P, U float2 quad_px; // taille du quad en pixels (pour les SDF) float radius_px; // rayon des coins arrondis en px (0 = aucun) ; mode 17 : rayon des coins hauts du corps (unités du modèle) - float mode; // 0 = vidéo NV12, 1 = couleur pleine, 2 = ombre portée, ..., 15 = curseur 3D, 16 = impact du clic, 17 = appareil modelé, 18 = écran cadré flouté en bloc + float mode; // 0 = vidéo NV12, 1 = couleur pleine, 2 = ombre portée, ..., 15 = curseur 3D, 16 = impact du clic, 17 = appareil modelé, 18 = écran cadré flouté en bloc, 19 = curseur de verre à plat float4 color; // couleur pleine / teinte (ombre : rgb + opacité dans a) ; mode 15 : coin du sprite, texel, opacité ; mode 17 : .r = l'appareil (1 portable, 2 téléphone, 3 moniteur), .g = thème sombre, .b = rayon des coins bas du corps, .a = opacité float4 fx; // fx.x = spread ombre (px), fx.y,fx.z libres ; mode 15 : rotation du plan (rad), tangage ; mode 17 : rotation du plan (rad), épaisseur float4 src_prev; // src à la frame précédente (flou de mouvement par vélocité) ; mode 15 : hotspot, lacet ; mode 17 : marges du corps (unités du modèle) @@ -170,9 +176,13 @@ inline float sd_segment(float2 p, float2 a, float2 b) inline float sd_round_rect(float2 p, float2 halfsz, float r) { float hmin = min(halfsz.x, halfsz.y); - if (r <= 0.0 || hmin <= 0.0) + float2 q0 = abs(p) - halfsz; + // Hors des coins, la distance est celle du rectangle vif quel que soit l'exposant : un seul + // axe déborde de l'étendue E, qui s'y ajoute puis s'en retranche. E ne dépasse pas 1,42 r + // (n = 3) : sous 1,5 r du bord sur un axe, on rend la boîte sans calculer `n` ni E, deux + // transcendantes de moins sur presque tous les pixels d'un calque arrondi. + if (r <= 0.0 || hmin <= 0.0 || min(q0.x, q0.y) <= -min(1.5 * r, hmin)) { - float2 q0 = abs(p) - halfsz; return length(max(q0, 0.0)) + min(max(q0.x, q0.y), 0.0); } float n = 3.0 - smoothstep(0.5, 1.0, r / hmin); @@ -545,7 +555,9 @@ inline float3 image_motion(float2 q, float time, float motion, float aspect) // ============ Curseur MODÉLISÉ (mode 15) ============ // Port ligne pour ligne de `cursor_model` (HLSL), dont les commentaires font foi ; seules // différences : `layer` et les textures arrivent en paramètres (le sprite en texture(2), son -// champ R16F en texture(4)), `lerp` s'écrit `mix`, `SampleLevel` s'écrit `sample(…, level(0.0))`, +// champ R16F en texture(4), la copie de l'image composée en texture(5) pour le cristal de Prism +// Glow), +// `lerp` s'écrit `mix`, `SampleLevel` s'écrit `sample(…, level(0.0))`, un `out` s'écrit `thread &`, // et `pow` veut un exposant du type de sa base. // Constantes : miroir exact de `frame_geometry.rs` (MODEL_*) et de `sculpt.rs` (SCULPT_*). constant float MODEL_BEVEL = 0.045; @@ -590,8 +602,6 @@ inline int sculpt_id(constant Layer &layer) constant float SCULPT_SCALE = 0.85; constant float SCULPT_HOVER = 0.05; constant float SCULPT_VOX = 0.0625; -constant float SCULPT_AR_ROUND = 0.03; -constant float SCULPT_HAND_ZC = 0.185; constant float SCULPT_ZREF_ARROW = 0.2; constant float SCULPT_ZREF_HAND = 0.185; constant float SCULPT_LAMP_DIST = 1.9; @@ -613,13 +623,6 @@ inline float2 s_opu(float2 a, float2 b) return a.x < b.x ? a : b; } -static float s_capsule(float3 p, float3 a, float3 b, float r) -{ - float3 pa = p - a, ba = b - a; - float h = saturate(dot(pa, ba) / dot(ba, ba)); - return length(pa - ba * h) - r; -} - static float s_round_box(float3 p, float3 b, float r) { float3 q = abs(p) - b + r; @@ -646,24 +649,76 @@ inline float2 s_rot(float2 v, float a) return float2(c * v.x - s * v.y, s * v.x + c * v.y); } -constant float2 SCULPT_ARROW[7] = { - float2(0.0, 0.0), float2(0.0, -0.86), float2(0.215, -0.665), float2(0.37, -1.0), - float2(0.53, -0.93), float2(0.38, -0.60), float2(0.64, -0.60) +static float s_star5(float2 p, float r, float rf) +{ + const float2 k1 = float2(0.809016994375, -0.587785252292); + const float2 k2 = float2(-0.809016994375, -0.587785252292); + p.x = abs(p.x); + p -= 2.0 * max(dot(k1, p), 0.0) * k1; + p -= 2.0 * max(dot(k2, p), 0.0) * k2; + p.x = abs(p.x); + p.y -= r; + float2 ba = rf * float2(-k1.y, k1.x) - float2(0.0, 1.0); + float h = clamp(dot(p, ba) / dot(ba, ba), 0.0, r); + return length(p - ba * h) * sign(p.y * ba.x - p.x * ba.y); +} + +static float s_vesica(float2 p, float r, float d) +{ + p = abs(p); + float b = sqrt(r * r - d * d); + return (p.y - b) * d > p.x * b ? length(p - float2(0.0, b)) : length(p + float2(d, 0.0)) - r; +} + +// Curseurs cercles (cf. HLSL) : Studio Ink, Pop Coral, Star Sprout. +constant float2 RIM_POLY[28] = { + float2(0.0, -0.06), float2(0.0, -0.7712), float2(0.165, -0.6325), float2(0.313, -0.9318), + float2(0.4234, -0.8557), float2(0.2672, -0.5686), float2(0.456, -0.5393), + float2(-0.173, -0.5237), float2(-0.016, -0.75), float2(0.329, -0.75), float2(0.411, -0.592), + float2(0.411, -0.43), float2(0.19, -0.435), float2(-0.03, -0.44), + float2(-0.1825, -0.6215), float2(0.0037, -0.9195), float2(0.3462, -0.9195), float2(0.467, -0.6906), + float2(0.467, -0.49), float2(0.2, -0.49), float2(-0.03, -0.49), + float2(-0.1807, -0.5534), float2(0.0194, -0.872), float2(0.3485, -0.872), float2(0.47, -0.6795), + float2(0.47, -0.49), float2(0.2, -0.49), float2(-0.03, -0.49) +}; + +constant float4 RIM_GLOVE[27] = { + float4(0.0, -0.1077, 0.0, -0.6), float4(0.1493, -0.3279, 0.1493, -0.6), + float4(0.2863, -0.3654, 0.2863, -0.6), float4(0.4193, -0.4043, 0.4193, -0.6), + float4(-0.2, -0.478, -0.075, -0.625), float4(0.0788, -0.25, 0.0788, -0.3992), + float4(0.2188, -0.3, 0.2188, -0.4234), float4(0.3552, -0.33, 0.3552, -0.4534), + float4(-0.0845, -0.5386, 0.762, 0.648), + float4(0.0, -0.1186, 0.0, -0.65), float4(0.1614, -0.3838, 0.1614, -0.65), + float4(0.3177, -0.427, 0.3177, -0.65), float4(0.4645, -0.4848, 0.4645, -0.65), + float4(-0.215, -0.535, -0.1208, -0.6864), float4(0.0832, -0.3, 0.0832, -0.48), + float4(0.2414, -0.35, 0.2414, -0.482), float4(0.3959, -0.41, 0.3959, -0.533), + float4(-0.122, -0.5718, 0.867, 0.498), + float4(0.0, -0.122, 0.0, -0.65), float4(0.172, -0.387, 0.172, -0.65), + float4(0.3267, -0.427, 0.3267, -0.65), float4(0.4727, -0.4867, 0.4727, -0.65), + float4(-0.199, -0.483, -0.102, -0.645), float4(0.0927, -0.3, 0.0927, -0.476), + float4(0.2493, -0.35, 0.2493, -0.478), float4(0.402, -0.41, 0.402, -0.52), + float4(-0.0993, -0.5267, 0.858, 0.514) +}; + +constant float4 RIM_GLOVE_R[9] = { + float4(0.0552, 0.047, 0.045, 0.0375), float4(0.052, 0.0235, 0.0225, 0.0225), float4(0.0552, -0.548, 0.0525, 0.0), + float4(0.064, 0.0585, 0.0585, 0.0515), float4(0.06, 0.0197, 0.0178, 0.0172), float4(0.0648, -0.6, 0.058, 0.0), + float4(0.066, 0.0553, 0.0553, 0.0507), float4(0.063, 0.024, 0.022, 0.02), float4(0.0647, -0.555, 0.056, 0.0) }; -static float s_arrow2(float2 p) +static float s_rim_poly(float2 p, int base) { - float d = dot(p - SCULPT_ARROW[0], p - SCULPT_ARROW[0]); + float d = dot(p - RIM_POLY[base], p - RIM_POLY[base]); float s = 1.0; - int j = 6; - for (int i = 0; i < 7; i++) + int j = base + 6; + for (int i = base; i < base + 7; i++) { - float2 e = SCULPT_ARROW[j] - SCULPT_ARROW[i]; - float2 w = p - SCULPT_ARROW[i]; + float2 e = RIM_POLY[j] - RIM_POLY[i]; + float2 w = p - RIM_POLY[i]; float2 b = w - e * saturate(dot(w, e) / dot(e, e)); d = min(d, dot(b, b)); - bool c0 = p.y >= SCULPT_ARROW[i].y; - bool c1 = p.y < SCULPT_ARROW[j].y; + bool c0 = p.y >= RIM_POLY[i].y; + bool c1 = p.y < RIM_POLY[j].y; bool c2 = e.x * w.y > e.y * w.x; if ((c0 && c1 && c2) || (!c0 && !c1 && !c2)) { @@ -674,276 +729,1247 @@ static float s_arrow2(float2 p) return s * sqrt(d); } -static float s_star5(float2 p, float r, float rf) +static float2 s_rim_glove(float2 p, int g, float palm) { - const float2 k1 = float2(0.809016994375, -0.587785252292); - const float2 k2 = float2(-0.809016994375, -0.587785252292); - p.x = abs(p.x); - p -= 2.0 * max(dot(k1, p), 0.0) * k1; - p -= 2.0 * max(dot(k2, p), 0.0) * k2; - p.x = abs(p.x); - p.y -= r; - float2 ba = rf * float2(-k1.y, k1.x) - float2(0.0, 1.0); - float h = clamp(dot(p, ba) / dot(ba, ba), 0.0, r); - return length(p - ba * h) * sign(p.y * ba.x - p.x * ba.y); + float4 r0 = RIM_GLOVE_R[3 * g]; + float4 r1 = RIM_GLOVE_R[3 * g + 1]; + float4 r2 = RIM_GLOVE_R[3 * g + 2]; + int i = 9 * g; + float f = min(sd_segment(p, RIM_GLOVE[i].xy, RIM_GLOVE[i].zw) - r0.x, + sd_segment(p, RIM_GLOVE[i + 1].xy, RIM_GLOVE[i + 1].zw) - r0.y); + f = min(f, sd_segment(p, RIM_GLOVE[i + 2].xy, RIM_GLOVE[i + 2].zw) - r0.z); + f = min(f, sd_segment(p, RIM_GLOVE[i + 3].xy, RIM_GLOVE[i + 3].zw) - r0.w); + f = min(f, sd_segment(p, RIM_GLOVE[i + 4].xy, RIM_GLOVE[i + 4].zw) - r1.x); + float grooves = min(sd_segment(p, RIM_GLOVE[i + 5].xy, RIM_GLOVE[i + 5].zw) - r1.y, + sd_segment(p, RIM_GLOVE[i + 6].xy, RIM_GLOVE[i + 6].zw) - r1.z); + grooves = min(grooves, sd_segment(p, RIM_GLOVE[i + 7].xy, RIM_GLOVE[i + 7].zw) - r1.w); + float sil = s_smin(palm, f, 0.03); + float4 v = RIM_GLOVE[i + 8]; + float wedge = max(max(-dot(p - v.xy, v.zw), p.x + r2.x), r2.y - p.y); + return float2(sil, max(sil, -min(grooves, wedge))); } -static float s_arrow_solid(float3 p, float h, float re, float dome) +static float2 s_piece(float d, float dc, float sil, float z, float w, float zt, float h, float bump, float mat) { - float d2 = s_arrow2(p.xy) - SCULPT_AR_ROUND; - float hh = h + dome * smoothstep(0.0, 0.07, -d2); - return s_extrude(d2, p.z - (SCULPT_HOVER + h + dome), hh, re); + float tray = s_extrude(sil - w, z - 0.5 * (SCULPT_HOVER + zt), 0.5 * (zt - SCULPT_HOVER), 0.012); + float bead = length(float2(d - 0.5 * w, z - zt)) - 0.5 * w; + float u = saturate(-dc / 0.045); + float cushion = 0.8 * s_extrude(dc, z - zt, h + 0.022 * u * (2.0 - u) + bump, 0.75 * h); + return s_opu(float2(min(tray, bead), 5.0), float2(cushion, mat)); } -static float s_piping(float3 p, float ztop) +static float2 s_paper(float d, float sil, float z, float w) { - float d2 = s_arrow2(p.xy) - SCULPT_AR_ROUND; - return length(float2(d2 + 0.075, p.z - ztop)) - 0.017; + float navy = s_extrude(sil - w, z - 0.5 * (SCULPT_HOVER + 0.15), 0.5 * (0.15 - SCULPT_HOVER), 0.006); + float sheet = s_extrude(d, z - 0.16, 0.01, 0.004); + return s_opu(float2(navy, 5.0), float2(sheet, 1.0)); } -static float s_glove(float3 p, float zc) +inline float s_bump(float2 p, float2 c, float r) { - float index = s_capsule(p, float3(0.0, -0.10, zc), float3(0.0, -0.52, zc), 0.098); - float palm = s_round_box(p - float3(0.185, -0.70, zc), float3(0.255, 0.19, 0.105), 0.1); - float f1 = s_capsule(p, float3(0.17, -0.57, zc + 0.012), float3(0.17, -0.41, zc + 0.045), 0.086); - float f2 = s_capsule(p, float3(0.31, -0.59, zc + 0.01), float3(0.31, -0.45, zc + 0.04), 0.08); - float f3 = s_capsule(p, float3(0.435, -0.625, zc + 0.005), float3(0.435, -0.52, zc + 0.03), 0.07); - float thumb = s_capsule(p, float3(0.03, -0.77, zc + 0.03), float3(-0.165, -0.60, zc + 0.065), 0.082); - float d = s_smin(palm, min(f1, min(f2, f3)), 0.035); - d = s_smin(d, index, 0.05); - return s_smin(d, thumb, 0.05); + float k = saturate(1.0 - dot(p - c, p - c) / (r * r)); + return k * k; } -static float s_cuff(float3 p, float zc) +static float s_dash(float2 p, float2 a, float2 b, float ra, float rb) { - float3 q = p - float3(0.185, -0.925, zc); - float2 ab = float2(0.272, 0.12); - float e = (length(q.xz / ab) - 1.0) * min(ab.x, ab.y); - return s_extrude(e, q.y, 0.07, 0.06); + p -= a; + b -= a; + float hb = dot(b, b); + float2 q = float2(abs(dot(p, float2(b.y, -b.x))), dot(p, b)) / hb; + float2 c = float2(sqrt(hb - (ra - rb) * (ra - rb)), ra - rb); + float k = c.x * q.y - c.y * q.x; + if (k < 0.0) + { + return sqrt(hb * dot(q, q)) - ra; + } + if (k > c.x) + { + return sqrt(hb * (dot(q, q) + 1.0 - 2.0 * q.y)) - rb; + } + return dot(c, q) - ra; } -static float2 s_sprout(float3 p, float3 c) +static float2 s_rimmed(float3 p, int theme, int shape) { - float3 q = p - c; - float st2 = s_star5(q.xy, 0.125, 0.52) - 0.022; - float2 r = float2(s_extrude(st2, q.z, 0.038, 0.034), 3.0); - float3 e = float3(abs(q.x) - 0.032, q.y + 0.005, q.z - 0.036); - r = s_opu(r, float2(length(e) - 0.0135, 5.0)); - float3 l1 = float3(s_rot(q.xy - float2(-0.04, 0.15), -0.6), q.z); - float3 l2 = float3(s_rot(q.xy - float2(0.045, 0.155), 0.7), q.z); - float leaves = min(s_ellipsoid(l1, float3(0.03, 0.058, 0.02)), s_ellipsoid(l2, float3(0.03, 0.058, 0.02))); - return s_opu(r, float2(leaves, 4.0)); + bool arrow = shape == 0; + int g = theme == 4 ? 0 : (theme == 0 ? 1 : 2); + float poly = s_rim_poly(p.xy, arrow ? 0 : 7 * g + 7); + float2 body = float2(poly, poly); + float w = 0.06, zt = 0.17, h = 0.03; + float bump = 0.022 * s_bump(p.xy, float2(0.15, -0.45), 0.3); + float back = 1e9; + if (!arrow) + { + w = RIM_GLOVE_R[3 * g + 2].z; + for (int i = 0; i < 2; i++) + { + float2 q = p.xy + (i == 1 ? float2(0.03, 0.04) : float2(0.0, 0.0)); + float2 v = s_rim_glove(q, g, (i == 1 ? s_rim_poly(q, 7 * g + 7) : poly) - 0.05); + if (i == 1) + { + back = v.x - w; + } + else + { + body = v; + if (theme != 2 || v.x - w <= 0.0) + { + break; + } + } + } + zt = 0.185; + h = 0.028; + bump = 0.03 * s_bump(p.xy, float2(0.19, -0.62), 0.28); + } + float dc = body.y; + if (arrow && theme == 0) + { + body -= 0.015; + w = 0.045; + dc = 1.0; + } + float2 r = theme == 2 ? s_paper(body.y, body.x, p.z, w) + : s_piece(body.y, dc, body.x, p.z, w, zt, h, bump, 1.0); + if (theme == 0) + { + if (arrow) + { + float band = s_extrude(abs(poly + 0.0075) - 0.0225, p.z - zt, 0.028, 0.012); + r = s_opu(r, float2(band, 2.0)); + } + return r; + } + if (theme == 2) + { + float dash; + if (arrow) + { + float back = s_rim_poly(p.xy + float2(0.025, 0.05), 0) - 0.06; + r = s_opu(r, float2(s_extrude(back, p.z - 0.08, 0.03, 0.006), 2.0)); + dash = min(s_dash(p.xy, float2(-0.1179, -0.1374), float2(-0.1799, -0.0443), 0.028, 0.045), + s_dash(p.xy, float2(-0.1347, -0.2493), float2(-0.248, -0.2056), 0.026, 0.042)); + } + else + { + r = s_opu(r, float2(s_extrude(back, p.z - 0.08, 0.03, 0.006), 3.0)); + dash = min(s_dash(p.xy, float2(0.216, -0.1695), float2(0.2593, -0.0685), 0.028, 0.042), + s_dash(p.xy, float2(0.3054, -0.2339), float2(0.3949, -0.1652), 0.0275, 0.041)); + } + return s_opu(r, float2(s_extrude(dash, p.z - 0.11, 0.06, 0.006), 3.0)); + } + if (!arrow) + { + float2 cq = p.xy - float2(0.1541, -0.9047); + float2 bq = abs(float2(cq.x, cq.y - 0.2066 * cq.x * cq.x)) - float2(0.214, 0.0361); + float cuff = length(max(bq, 0.0)) + min(max(bq.x, bq.y), 0.0) - 0.03; + r = s_opu(r, s_piece(cuff, cuff, cuff, p.z, 0.0477, 0.225, 0.025, 0.0, 2.0)); + } + float2 c = arrow ? float2(0.6118, -0.8079) : float2(0.15, -0.885); + float rs = arrow ? 0.1678 : 0.128; + float zs = arrow ? 0.23 : 0.285; + float2 q = s_rot(p.xy - c, arrow ? 0.2443 : 0.0) / rs; + float star = (s_star5(q, 0.82, 0.55) - 0.18) * rs; + float leaves = min(s_vesica(s_rot(q - float2(-0.33, 1.38), -0.925), 0.4296, 0.2626), + s_vesica(s_rot(q - float2(0.53, 1.37), 0.873), 0.4296, 0.2626)) * rs; + r = s_opu(r, s_piece(leaves, leaves, leaves, p.z, 0.038, zs - 0.02, 0.018, 0.0, 4.0)); + r = s_opu(r, s_piece(star, star, star, p.z, 0.038, zs, 0.022, 0.018 * s_bump(q, float2(0.0), 1.0), 3.0)); + float3 e = float3(abs(q.x) - 0.25, q.y - 0.08, (p.z - zs - 0.056) / rs); + return s_opu(r, float2(s_ellipsoid(e, float3(0.075, 0.13, 0.1)) * rs, 5.0)); } -// Pixel Candy : une ligne par entier, bit c = colonne c (cf. HLSL et `sculpt.rs`). -constant int SCULPT_ARROWPIX[16] = { 0, 1, 3, 7, 31, 63, 127, 255, 511, 63, 55, 115, 113, 224, 224, 64 }; -constant int SCULPT_ARROWBACK[18] = { - 3, 7, 15, 31, 63, 127, 255, 511, 1023, 2047, 4095, 255, 511, 511, 999, 995, 960, 192 -}; -constant int SCULPT_HANDPIX[15] = { 4, 14, 14, 14, 110, 878, 7022, 7022, 8190, 8191, 8191, 8191, 8190, 4092, 4092 }; -constant int SCULPT_HANDBACK[17] = { - 28, 62, 62, 62, 510, 4094, 32766, 32766, 32766, 32767, 32767, 32767, 32767, 32767, 32766, 16380, 16380 -}; -constant float2 SCULPT_HANDSPAN[17] = { - float2(2.0, 4.0), float2(1.0, 5.0), float2(1.0, 5.0), float2(1.0, 5.0), float2(1.0, 8.0), - float2(1.0, 11.0), float2(1.0, 14.0), float2(1.0, 14.0), float2(1.0, 14.0), float2(0.0, 14.0), - float2(0.0, 14.0), float2(0.0, 14.0), float2(0.0, 14.0), float2(0.0, 14.0), float2(1.0, 14.0), - float2(2.0, 13.0), float2(2.0, 13.0) -}; +// ---- Glass Lens ---- (cf. HLSL : une lentille de verre dans un trait sombre, un liseré de verre +// au-delà ; `s_glass` en volume, `cursor_glass` à plat, `glass_shade` pour le verre.) +constant float GLASS_LINE = 0.04; +constant float GLASS_RIM = 0.085; +constant float GLASS_TOP = 0.18; +constant float GLASS_DOME = 0.03; +constant float GLASS_RISE = 0.08; +constant float GLASS_IOR = 1.5; +constant float GLASS_DISPERSION = 0.012; +constant float GLASS_MAT = 10.0; +constant float GLASS_GAP_FLAT = 0.15; +constant float GLASS_SMEAR = 0.15; +constant float GLASS_LIFT = 0.02; +constant float GLASS_SHADOW = 0.6; + +// x = la silhouette, y = la lentille (distances signées du prototype, négatives dedans). +static float2 s_glass2(float2 p, int shape) +{ + if (shape == 0) + { + float poly = s_rim_poly(p + float2(0.0, GLASS_RIM - 0.06), 0); + return float2(poly - GLASS_RIM, poly); + } + float2 q = p + float2(0.0, GLASS_RIM - RIM_GLOVE_R[5].z); + float2 v = s_rim_glove(q, 1, s_rim_poly(q, 14) - 0.05); + return float2(v.x - GLASS_RIM, v.y); +} -inline float2 s_grid_origin(int shape) +// La hauteur du verre au-dessus de GLASS_TOP selon la distance `d` à la lentille, et sa pente. +inline float2 s_glass_relief(float d) { - return shape == 0 ? float2(0.0, 0.0) : float2(-2.5 * SCULPT_VOX, 0.0); + if (d < 0.0) + { + float u = saturate(-d / GLASS_RISE); + return float2(GLASS_DOME * u * (2.0 - u), -2.0 * GLASS_DOME * (1.0 - u) / GLASS_RISE); + } + float w = GLASS_RIM - GLASS_LINE; + float x = saturate((d - GLASS_LINE) / w); + float s = sqrt(max(1.0 - x * x, 0.05)); + return float2(w * (s - 1.0), -x / s); } -inline bool s_bit(int row, int r, int c, int rows, int cols) +static float2 s_glass(float3 p, int shape) { - return r >= 0 && r < rows && c >= 0 && c < cols && ((row >> clamp(c, 0, 31)) & 1) == 1; + float2 g = s_glass2(p.xy, shape); + float half_h = 0.5 * (GLASS_TOP - SCULPT_HOVER); + float slab = s_extrude(g.x, p.z - SCULPT_HOVER - half_h, half_h, min(GLASS_RIM - GLASS_LINE, half_h)); + float u = saturate(-g.y / GLASS_RISE); + float dome = 0.8 * s_extrude(g.y, p.z - GLASS_TOP, 0.009 + GLASS_DOME * u * (2.0 - u), 0.008); + float band = s_extrude(abs(g.y - 0.5 * GLASS_LINE) - 0.5 * GLASS_LINE, p.z - GLASS_TOP, 0.016, 0.008); + return s_opu(float2(min(slab, dome), GLASS_MAT), float2(band, 5.0)); } -static bool s_occ(float2 id, int shape) +// Pixel Candy : tables générées par scripts/generate-pixel-candy-voxels.mjs (cf. HLSL). +// +constant int PIX_BODY[32] = { 1, 3, 7, 15, 31, 63, 127, 255, 511, 1023, 2047, 127, 247, 243, 480, 192, 48, 120, 120, 120, 504, 4088, 32760, 65534, 65535, 65535, 65534, 32766, 32764, 16380, 16376, 16376 }; +constant int PIX_LINE[32] = { 1, 3, 5, 9, 17, 33, 65, 129, 257, 513, 1985, 73, 149, 147, 288, 192, 48, 72, 72, 72, 456, 3656, 29256, 37454, 32777, 32769, 32770, 16386, 16388, 8196, 8200, 16376 }; +constant int PIX_HI[32] = { 0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 18, 34, 32, 64, 0, 0, 16, 16, 16, 16, 16, 16, 16, 22, 18, 4, 4, 8, 8, 16, 0 }; +constant int PIX_SHADE[32] = { 0, 0, 0, 4, 8, 16, 32, 64, 128, 448, 40, 36, 64, 64, 128, 0, 0, 32, 32, 32, 32, 288, 2336, 18720, 16384, 16384, 16384, 8192, 8192, 4096, 8160, 0 }; +constant int PIX_RECT_N[3] = { 0, 17, 29 }; +constant int4 PIX_GRID[2] = { + int4(11, 16, 0, 0), + int4(16, 16, 16, 0) +}; +constant float2 PIX_ORIGIN[2] = { + float2(0.0, 0.0), + float2(-0.3125, 0.0) +}; +constant float4 PIX_BOX[2] = { + float4(0.3438, -0.5, 0.3438, 0.5), + float4(0.1875, -0.5, 0.5, 0.5) +}; +constant float4 PIX_RECT[29] = { + float4(0.0313, -0.0313, 0.0313, 0.0313), + float4(0.0625, -0.0938, 0.0625, 0.0313), + float4(0.0938, -0.1563, 0.0938, 0.0313), + float4(0.125, -0.2188, 0.125, 0.0313), + float4(0.1563, -0.2813, 0.1563, 0.0313), + float4(0.1875, -0.3438, 0.1875, 0.0313), + float4(0.2188, -0.4063, 0.2188, 0.0313), + float4(0.25, -0.4688, 0.25, 0.0313), + float4(0.2813, -0.5313, 0.2813, 0.0313), + float4(0.3125, -0.5938, 0.3125, 0.0313), + float4(0.3438, -0.6563, 0.3438, 0.0313), + float4(0.2188, -0.7188, 0.2188, 0.0313), + float4(0.0938, -0.7813, 0.0938, 0.0313), + float4(0.375, -0.8125, 0.125, 0.0625), + float4(0.0625, -0.8438, 0.0625, 0.0313), + float4(0.4375, -0.9063, 0.125, 0.0313), + float4(0.4375, -0.9688, 0.0625, 0.0313), + float4(0.0, -0.0313, 0.0625, 0.0313), + float4(0.0, -0.1563, 0.125, 0.0938), + float4(0.0625, -0.2813, 0.1875, 0.0313), + float4(0.1563, -0.3438, 0.2813, 0.0313), + float4(0.25, -0.4063, 0.375, 0.0313), + float4(0.2188, -0.4688, 0.4688, 0.0313), + float4(0.1875, -0.5625, 0.5, 0.0625), + float4(0.2188, -0.6563, 0.4688, 0.0313), + float4(0.1875, -0.7188, 0.4375, 0.0313), + float4(0.2188, -0.7813, 0.4063, 0.0313), + float4(0.1875, -0.8438, 0.375, 0.0313), + float4(0.2188, -0.9375, 0.3438, 0.0625) +}; +// + +inline bool s_pix_bit(int m, int c) { - int c = int(id.x); - int r = -int(id.y) - 1; - if (shape == 0) - { - return s_bit(SCULPT_ARROWPIX[clamp(r, 0, 15)], r, c, 16, 16); - } - return s_bit(SCULPT_HANDPIX[clamp(r, 0, 14)], r, c, 15, 13); + return ((m >> clamp(c, 0, 31)) & 1) == 1; } -static bool s_occ_back(float2 id, int shape) +static bool s_pix_body(int shape, int c, int r) { - int c = int(id.x) + 1; - int r = -int(id.y); - if (shape == 0) - { - return s_bit(SCULPT_ARROWBACK[clamp(r, 0, 17)], r, c, 18, 17); - } - return s_bit(SCULPT_HANDBACK[clamp(r, 0, 16)], r, c, 17, 15); + int4 g = PIX_GRID[shape]; + return c >= 0 && c < g.x && r >= 0 && r < g.y && s_pix_bit(PIX_BODY[g.z + clamp(r, 0, g.y - 1)], c); +} + +inline int2 s_pix_cell(float2 p, int shape) +{ + float2 o = PIX_ORIGIN[shape]; + return int2(int(floor((p.x - o.x) / SCULPT_VOX)), int(floor((o.y - p.y) / SCULPT_VOX))); } static float2 s_voxels(float3 p, int shape) { - float2 o = s_grid_origin(shape); - float2 cell = floor((p.xy - o) / SCULPT_VOX); - float2 bc = shape == 0 ? float2(0.33, -0.5) : float2(0.25, -0.47); - float2 bh = shape == 0 ? float2(0.44, 0.61) : float2(0.48, 0.55); - float2 bq = max(abs(p.xy - bc) - bh, 0.0); + float4 b = PIX_BOX[shape]; + float2 bq = max(abs(p.xy - b.xy) - b.zw, 0.0); float bz = max(abs(p.z - (SCULPT_HOVER + 0.07)) - 0.07, 0.0); - float far = max(SCULPT_VOX, sqrt(dot(bq, bq) + bz * bz)); - float dF = far; - float dB = far; - const float3 half_cell = float3(0.5 * SCULPT_VOX, 0.5 * SCULPT_VOX, 0.04); + float d = max(SCULPT_VOX, sqrt(dot(bq, bq) + bz * bz)); + float2 o = PIX_ORIGIN[shape]; + int2 cell = s_pix_cell(p.xy, shape); + const float3 cube = float3(0.5 * SCULPT_VOX, 0.5 * SCULPT_VOX, 0.07); for (int j = -1; j <= 1; j++) { for (int i = -1; i <= 1; i++) { - float2 id = cell + float2(float(i), float(j)); - float3 q = float3(p.xy - (o + (id + 0.5) * SCULPT_VOX), p.z); - if (s_occ(id, shape)) - { - dF = min(dF, s_round_box(q - float3(0.0, 0.0, SCULPT_HOVER + 0.1), half_cell, 0.009)); - } - if (s_occ_back(id, shape)) + int c = cell.x + i; + int r = cell.y + j; + if (s_pix_body(shape, c, r)) { - dB = min(dB, s_round_box(q - float3(0.0, 0.0, SCULPT_HOVER + 0.04), half_cell, 0.009)); + float3 q = float3(p.x - o.x - (float(c) + 0.5) * SCULPT_VOX, p.y - o.y + (float(r) + 0.5) * SCULPT_VOX, + p.z - SCULPT_HOVER - 0.07); + d = min(d, s_round_box(q, cube, 0.006)); } } } - return dF < dB ? float2(dF, 6.0) : float2(dB, 7.0); + return float2(d, 6.0); } -static float s_pixel_hand_dist(float2 p) +static float s_pixel_outline(float2 p, int shape) { float d = 1e9; - for (int r = 0; r < 17; r++) + for (int i = PIX_RECT_N[shape]; i < PIX_RECT_N[shape + 1]; i++) { - float2 x = (SCULPT_HANDSPAN[r] + float2(-3.5, -2.5)) * SCULPT_VOX; - float2 y = float2(-float(r), 1.0 - float(r)) * SCULPT_VOX; - float2 q = max(max(float2(x.x, y.x) - p, p - float2(x.y, y.y)), 0.0); - d = min(d, length(q)); + float2 q = abs(p - PIX_RECT[i].xy) - PIX_RECT[i].zw; + d = min(d, length(max(q, 0.0)) + min(max(q.x, q.y), 0.0)); } return d; } -static float s_gem_edge(float3 p, float2 a, float2 b, float z0) -{ - float2 e = b - a; - float len = length(e); - float2 d = e / len; - float2 q = p.xy - a; - float dist = dot(q, float2(-d.y, d.x)); - float along = abs(dot(q, d) - 0.5 * len); - float z = p.z - z0; - float girdle = (z - 0.035 - 2.2 * dist) * 0.4138; - float crown = (z - 0.07 - 0.6 * dist + 0.18 * along) * 0.8438; - return max(-dist, max(girdle, crown)); -} +// ---- Prism Glow : un cristal en MAILLAGE ---- (cf. HLSL) +// prism mesh: generated by design/cursors/prism-glow/model/export_compositor.py +constant int PRISM_TRI_START[2] = { 0, 31 }; +constant int PRISM_TRI_COUNT[2] = { 31, 130 }; +constant int PRISM_SIL_START[2] = { 0, 25 }; +constant int PRISM_SIL_COUNT[2] = { 25, 42 }; +constant int PRISM_OUT_START[2] = { 67, 76 }; +constant int PRISM_OUT_COUNT[2] = { 9, 34 }; +constant int PRISM_BOX_START[2] = { 0, 4 }; +constant int PRISM_BOX_COUNT[2] = { 4, 18 }; +constant float4 PRISM_TRIS[644] = { + float4(0.252301, 0.266258, -0.042945, 11.0), + float4(-0.221166, -0.203988, 0.0, 0.01033), + float4(-0.221166, -0.203988, 0.042945, 0.045182), + float4(0.677984, -0.735077, 0.0, 0.577584), + float4(0.252301, 0.266258, -0.042945, 3.0), + float4(-0.221166, -0.203988, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(0.677984, -0.735077, 0.0, 0.577584), + float4(0.031135, 0.06227, -0.042945, 7.0), + float4(0.004294, 0.419785, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(-0.999948, 0.01023, 0.0, 0.577584), + float4(0.169632, 0.356442, 0.115951, 14.0), + float4(0.082669, -0.090184, -0.115951, 0.590616), + float4(-0.138497, -0.294172, -0.115951, 0.737911), + float4(0.466385, -0.505659, 0.725806, 0.973449), + float4(0.035429, 0.482055, 0.0, 14.0), + float4(0.134202, -0.125614, 0.115951, 0.000305), + float4(-0.004294, -0.419785, 0.0, 0.938689), + float4(-0.650191, 0.006652, 0.759742, 0.854994), + float4(0.031135, 0.06227, -0.042945, 9.0), + float4(0.004294, 0.419785, 0.0, 0.01033), + float4(0.004294, 0.419785, 0.042945, 0.045182), + float4(-0.999948, 0.01023, 0.0, 0.577584), + float4(0.169632, 0.356442, 0.115951, 14.0), + float4(0.159969, 0.080521, -0.045092, 0.381323), + float4(0.082669, -0.090184, -0.115951, 0.056123), + float4(0.461407, -0.510214, 0.725799, 0.973449), + float4(0.541104, 0.544325, -0.042945, 9.0), + float4(-0.288804, -0.278068, 0.0, 0.01033), + float4(-0.288804, -0.278068, 0.042945, 0.045182), + float4(0.69359, -0.72037, 0.0, 0.577584), + float4(0.329601, 0.436963, 0.070859, 14.0), + float4(0.211503, 0.107362, -0.070859, 0.545724), + float4(-0.077301, -0.170706, -0.070859, 0.064805), + float4(0.49567, -0.514808, 0.699488, 0.973449), + float4(0.268405, 0.533589, 0.081595, 14.0), + float4(0.061196, -0.096626, -0.010736, 0.000916), + float4(-0.098773, -0.177147, 0.034356, 0.046662), + float4(0.247834, 0.049458, 0.967539, 0.904662), + float4(0.541104, 0.544325, -0.042945, 7.0), + float4(-0.288804, -0.278068, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(0.69359, -0.72037, 0.0, 0.577584), + float4(0.199693, 0.595859, 0.0, 14.0), + float4(0.068712, -0.06227, 0.081595, 0.0), + float4(-0.030061, -0.239417, 0.115951, 0.01445), + float4(-0.504461, 0.426837, 0.750553, 0.254154), + float4(0.268405, 0.533589, 0.081595, 14.0), + float4(0.272699, 0.010736, -0.081595, 0.0), + float4(0.061196, -0.096626, -0.010736, 0.025193), + float4(0.283244, 0.073137, 0.956255, 0.527108), + float4(0.035429, 0.482055, 0.0, 14.0), + float4(0.0, 0.258742, 0.0, 0.0), + float4(0.134202, -0.125614, 0.115951, 0.473533), + float4(-0.653777, 0.0, 0.756687, 0.964691), + float4(0.359663, 0.559356, 0.0, 14.0), + float4(0.181442, -0.015031, 0.0, 0.006043), + float4(-0.091258, -0.025767, 0.081595, 0.076188), + float4(0.076465, 0.923048, 0.37701, 0.964691), + float4(0.035429, 0.482055, -0.042945, 3.0), + float4(0.0, 0.258742, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(-1.0, 0.0, 0.0, 0.577584), + float4(0.035429, 0.482055, -0.042945, 11.0), + float4(0.0, 0.258742, 0.0, 0.01033), + float4(0.0, 0.258742, 0.042945, 0.045182), + float4(-1.0, 0.0, 0.0, 0.577584), + float4(0.035429, 0.740798, -0.042945, 7.0), + float4(0.164264, -0.144939, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(0.661622, 0.749838, 0.0, 0.577584), + float4(0.035429, 0.740798, 0.0, 14.0), + float4(0.164264, -0.144939, 0.0, 0.0), + float4(0.134202, -0.384356, 0.115951, 0.010956), + float4(0.332572, 0.376915, 0.864483, 0.215854), + float4(0.035429, 0.740798, -0.042945, 11.0), + float4(0.164264, -0.144939, 0.0, 0.01033), + float4(0.164264, -0.144939, 0.042945, 0.045182), + float4(0.661622, 0.749838, 0.0, 0.577584), + float4(0.199693, 0.595859, -0.042945, 7.0), + float4(0.105215, 0.22546, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(-0.906183, 0.422886, 0.0, 0.577584), + float4(0.199693, 0.595859, 0.0, 14.0), + float4(0.105215, 0.22546, 0.0, 0.0), + float4(0.068712, -0.06227, 0.081595, 0.015991), + float4(-0.613882, 0.286478, 0.735581, 0.473533), + float4(0.199693, 0.595859, -0.042945, 11.0), + float4(0.105215, 0.22546, 0.0, 0.01033), + float4(0.105215, 0.22546, 0.042945, 0.045182), + float4(-0.906183, 0.422886, 0.0, 0.577584), + float4(0.304908, 0.821319, 0.0, 14.0), + float4(0.128834, -0.067638, 0.0, 0.0), + float4(-0.036503, -0.28773, 0.081595, 0.254154), + float4(0.133686, 0.254641, 0.957751, 0.973449), + float4(0.304908, 0.821319, -0.042945, 7.0), + float4(0.128834, -0.067638, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(0.464834, 0.885398, 0.0, 0.577584), + float4(0.433742, 0.753681, 0.0, 14.0), + float4(-0.07408, -0.194325, 0.0, 0.0), + float4(-0.165337, -0.220092, 0.081595, 0.964691), + float4(0.683362, -0.260508, 0.682021, 0.964691), + float4(0.433742, 0.753681, -0.042945, 11.0), + float4(-0.07408, -0.194325, 0.0, 0.01033), + float4(-0.07408, -0.194325, 0.042945, 0.045182), + float4(0.934406, -0.35621, 0.0, 0.577584), + float4(0.304908, 0.821319, -0.042945, 11.0), + float4(0.128834, -0.067638, 0.0, 0.01033), + float4(0.128834, -0.067638, 0.042945, 0.045182), + float4(0.464834, 0.885398, 0.0, 0.577584), + float4(0.433742, 0.753681, -0.042945, 7.0), + float4(-0.07408, -0.194325, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(0.934406, -0.35621, 0.0, 0.577584), + float4(0.359663, 0.559356, -0.042945, 7.0), + float4(0.181442, -0.015031, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(0.082557, 0.996586, 0.0, 0.577584), + float4(0.359663, 0.559356, -0.042945, 11.0), + float4(0.181442, -0.015031, 0.0, 0.01033), + float4(0.181442, -0.015031, 0.042945, 0.045182), + float4(0.082557, 0.996586, 0.0, 0.577584), + float4(0.044345, 0.043263, -0.043263, 11.0), + float4(-0.043263, -0.005408, 0.0, 0.01033), + float4(-0.043263, -0.005408, 0.043263, 0.045182), + float4(0.124035, -0.992278, 0.0, 0.577584), + float4(0.044345, 0.043263, -0.043263, 7.0), + float4(-0.043263, -0.005408, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.124035, -0.992278, 0.0, 0.577584), + float4(0.001082, 0.037855, -0.043263, 7.0), + float4(-0.044345, 0.040019, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.669964, -0.742393, 0.0, 0.577584), + float4(0.044345, 0.043263, 0.0, 10.0), + float4(-0.043263, -0.005408, 0.0, 0.000305), + float4(-0.040019, 0.030284, 0.049753, 0.964691), + float4(0.101434, -0.811469, 0.575525, 0.973449), + float4(0.004326, 0.073548, 0.049753, 14.0), + float4(-0.003245, -0.035692, -0.049753, 0.0), + float4(-0.04759, 0.004326, -0.049753, 0.955978), + float4(-0.580475, -0.643229, 0.499304, 0.896269), + float4(0.001082, 0.037855, -0.043263, 11.0), + float4(-0.044345, 0.040019, 0.0, 0.01033), + float4(-0.044345, 0.040019, 0.043263, 0.045182), + float4(-0.669964, -0.742393, 0.0, 0.577584), + float4(-0.031366, 0.229295, 0.058405, 14.0), + float4(0.035692, -0.155748, -0.008653, 0.000305), + float4(-0.011897, -0.151422, -0.058405, 0.955978), + float4(-0.716533, -0.20131, 0.667874, 0.904662), + float4(-0.043263, 0.077874, -0.043263, 7.0), + float4(-0.001082, 0.154666, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.999976, -0.006993, 0.0, 0.577584), + float4(-0.031366, 0.229295, 0.058405, 14.0), + float4(0.089771, -0.137361, -0.02055, 0.445197), + float4(0.035692, -0.155748, -0.008653, 0.973449), + float4(0.216369, -0.004654, 0.976301, 0.973449), + float4(-0.031366, 0.229295, 0.058405, 14.0), + float4(-0.011897, -0.151422, -0.058405, 0.000305), + float4(-0.012979, 0.003245, -0.058405, 0.955978), + float4(-0.976245, -0.006827, 0.216564, 0.904662), + float4(-0.043263, 0.077874, -0.043263, 9.0), + float4(-0.001082, 0.154666, 0.0, 0.01033), + float4(-0.001082, 0.154666, 0.043263, 0.045182), + float4(-0.999976, -0.006993, 0.0, 0.577584), + float4(-0.045426, 0.564586, 0.052998, 14.0), + float4(0.014061, -0.335291, 0.005408, 0.520989), + float4(0.001082, -0.332046, -0.052998, 0.088655), + float4(-0.975941, -0.037462, 0.214794, 0.973449), + float4(-0.044345, 0.23254, 0.0, 3.0), + float4(0.0, 0.0, -0.043263, 0.01033), + float4(-0.001082, 0.332046, -0.043263, 0.045182), + float4(-0.999995, -0.003257, 0.0, 0.577584), + float4(-0.163319, 0.380717, -0.043263, 7.0), + float4(-0.060569, 0.02704, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.407651, -0.913138, 0.0, 0.577584), + float4(-0.094098, 0.431551, -0.043263, 11.0), + float4(-0.069221, -0.050834, 0.0, 0.01033), + float4(-0.069221, -0.050834, 0.043263, 0.045182), + float4(0.591909, -0.806004, 0.0, 0.577584), + float4(-0.163319, 0.380717, -0.043263, 11.0), + float4(-0.060569, 0.02704, 0.0, 0.01033), + float4(-0.060569, 0.02704, 0.043263, 0.045182), + float4(-0.407651, -0.913138, 0.0, 0.577584), + float4(0.075711, 0.08869, -0.043263, 11.0), + float4(-0.031366, -0.045426, 0.0, 0.01033), + float4(-0.031366, -0.045426, 0.043263, 0.045182), + float4(0.822897, -0.568191, 0.0, 0.577584), + float4(0.044345, 0.043263, 0.0, 6.0), + float4(-0.040019, 0.030284, 0.049753, 0.000305), + float4(0.014061, 0.048671, 0.037855, 0.964691), + float4(0.365607, -0.634949, 0.680566, 0.973449), + float4(0.075711, 0.08869, -0.043263, 7.0), + float4(-0.031366, -0.045426, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.822897, -0.568191, 0.0, 0.577584), + float4(0.058405, 0.091934, 0.037855, 14.0), + float4(0.017305, -0.003245, -0.037855, 0.0), + float4(-0.014061, -0.048671, -0.037855, 0.473533), + float4(0.757369, -0.522945, 0.39105, 0.904662), + float4(-0.031366, 0.229295, 0.058405, 14.0), + float4(0.107077, -0.140606, -0.058405, 0.0), + float4(0.089771, -0.137361, -0.02055, 0.376257), + float4(0.812052, 0.481346, 0.329966, 0.830777), + float4(0.077874, 0.407756, -0.043263, 11.0), + float4(-0.002163, -0.319067, 0.0, 0.01033), + float4(-0.002163, -0.319067, 0.043263, 0.045182), + float4(0.999977, -0.006779, 0.0, 0.577584), + float4(-0.031366, 0.229295, 0.058405, 14.0), + float4(0.10924, 0.178461, -0.058405, 0.0), + float4(0.107077, -0.140606, -0.058405, 0.376257), + float4(0.475583, -0.003224, 0.879665, 0.830777), + float4(0.147095, 0.272559, -0.043263, 7.0), + float4(-0.035692, 0.034611, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.696146, -0.7179, 0.0, 0.577584), + float4(0.147095, 0.272559, -0.043263, 11.0), + float4(-0.035692, 0.034611, 0.0, 0.01033), + float4(-0.035692, 0.034611, 0.043263, 0.045182), + float4(-0.696146, -0.7179, 0.0, 0.577584), + float4(0.144932, 0.307169, 0.049753, 14.0), + float4(0.002163, -0.034611, -0.049753, 0.0), + float4(-0.033529, 0.0, -0.049753, 0.9131), + float4(-0.630237, -0.649931, 0.424725, 0.904662), + float4(0.183869, 0.317985, 0.037855, 6.0), + float4(-0.036774, -0.045426, -0.037855, 0.0), + float4(-0.038937, -0.010816, 0.011897, 0.806958), + float4(0.374433, -0.753477, 0.540438, 0.854994), + float4(0.077874, 0.407756, -0.043263, 7.0), + float4(-0.002163, -0.319067, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.999977, -0.006779, 0.0, 0.577584), + float4(0.111403, 0.307169, -0.043263, 7.0), + float4(-0.002163, 0.131953, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.999866, -0.016391, 0.0, 0.577584), + float4(0.10924, 0.439122, 0.0, 6.0), + float4(0.035692, -0.131953, 0.049753, 0.18117), + float4(0.002163, -0.131953, 0.0, 0.955978), + float4(-0.82919, -0.013593, 0.558802, 0.964691), + float4(0.128708, 0.463999, 0.063813, 14.0), + float4(0.055161, -0.146014, -0.025958, 0.246204), + float4(0.016224, -0.156829, -0.014061, 0.964691), + float4(0.305397, -0.053644, 0.950713, 0.973449), + float4(0.111403, 0.307169, -0.043263, 11.0), + float4(-0.002163, 0.131953, 0.0, 0.01033), + float4(-0.002163, 0.131953, 0.043263, 0.045182), + float4(-0.999866, -0.016391, 0.0, 0.577584), + float4(0.189277, 0.28013, -0.043263, 11.0), + float4(-0.042182, -0.007571, 0.0, 0.01033), + float4(-0.042182, -0.007571, 0.043263, 0.045182), + float4(0.176664, -0.984271, 0.0, 0.577584), + float4(0.183869, 0.317985, 0.037855, 10.0), + float4(0.005408, -0.037855, -0.037855, 0.0), + float4(-0.036774, -0.045426, -0.037855, 0.806958), + float4(0.124328, -0.692684, 0.710445, 0.854994), + float4(0.189277, 0.28013, -0.043263, 7.0), + float4(-0.042182, -0.007571, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.176664, -0.984271, 0.0, 0.577584), + float4(0.183869, 0.317985, 0.037855, 14.0), + float4(0.02704, -0.004326, -0.037855, 0.0), + float4(0.005408, -0.037855, -0.037855, 0.514916), + float4(0.70062, -0.452013, 0.552101, 0.846876), + float4(0.210908, 0.313659, -0.043263, 11.0), + float4(-0.021632, -0.033529, 0.0, 0.01033), + float4(-0.021632, -0.033529, 0.043263, 0.045182), + float4(0.840297, -0.542127, 0.0, 0.577584), + float4(0.210908, 0.445612, 0.0, 10.0), + float4(0.0, -0.131953, 0.0, 0.0), + float4(-0.02704, -0.127627, 0.037855, 0.439658), + float4(0.813734, 0.0, 0.581238, 0.871366), + float4(0.210908, 0.313659, -0.043263, 7.0), + float4(-0.021632, -0.033529, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.840297, -0.542127, 0.0, 0.577584), + float4(0.210908, 0.445612, -0.043263, 11.0), + float4(0.0, -0.131953, 0.0, 0.01033), + float4(0.0, -0.131953, 0.043263, 0.045182), + float4(1.0, 0.0, 0.0, 0.577584), + float4(0.241193, 0.343943, -0.043263, 7.0), + float4(-0.002163, 0.107077, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.999796, -0.020198, 0.0, 0.577584), + float4(0.276885, 0.311496, -0.043263, 11.0), + float4(-0.035692, 0.032447, 0.0, 0.01033), + float4(-0.035692, 0.032447, 0.043263, 0.045182), + float4(-0.672673, -0.73994, 0.0, 0.577584), + float4(0.23903, 0.45102, 0.0, 14.0), + float4(0.042182, -0.108158, 0.046508, 0.000305), + float4(0.002163, -0.107077, 0.0, 0.846876), + float4(-0.758098, -0.015315, 0.651961, 0.896269), + float4(0.276885, 0.311496, -0.043263, 7.0), + float4(-0.035692, 0.032447, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.672673, -0.73994, 0.0, 0.577584), + float4(0.281211, 0.342862, 0.046508, 14.0), + float4(-0.004326, -0.031366, -0.046508, 0.0), + float4(-0.040019, 0.001082, -0.046508, 0.752941), + float4(-0.58651, -0.645161, 0.489667, 0.846876), + float4(0.246601, 0.459672, 0.06165, 14.0), + float4(0.070303, -0.101669, -0.024876, 0.262257), + float4(0.034611, -0.116811, -0.015142, 0.973449), + float4(0.27928, -0.041605, 0.959308, 0.973449), + float4(0.319067, 0.319067, -0.043263, 11.0), + float4(-0.042182, -0.007571, 0.0, 0.01033), + float4(-0.042182, -0.007571, 0.043263, 0.045182), + float4(0.176664, -0.984271, 0.0, 0.577584), + float4(0.319067, 0.319067, 0.0, 10.0), + float4(-0.042182, -0.007571, 0.0, 0.0), + float4(-0.037855, 0.023795, 0.046508, 0.896269), + float4(0.1483, -0.826245, 0.54344, 0.896269), + float4(0.319067, 0.319067, -0.043263, 7.0), + float4(-0.042182, -0.007571, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.176664, -0.984271, 0.0, 0.577584), + float4(0.319067, 0.319067, 0.0, 6.0), + float4(-0.037855, 0.023795, 0.046508, 0.0), + float4(-0.002163, 0.038937, 0.036774, 0.896269), + float4(0.437906, -0.604311, 0.665618, 0.896269), + float4(0.316904, 0.358004, 0.036774, 14.0), + float4(0.024876, -0.004326, -0.036774, 0.0), + float4(0.002163, -0.038937, -0.036774, 0.617212), + float4(0.707338, -0.464191, 0.533104, 0.921584), + float4(0.34178, 0.353677, -0.043263, 11.0), + float4(-0.022713, -0.034611, 0.0, 0.01033), + float4(-0.022713, -0.034611, 0.043263, 0.045182), + float4(0.836048, -0.548657, 0.0, 0.577584), + float4(0.340698, 0.474815, 0.0, 14.0), + float4(0.001082, -0.121137, 0.0, 0.0), + float4(-0.023795, -0.116811, 0.036774, 0.473533), + float4(0.828664, 0.007399, 0.559697, 0.930114), + float4(0.34178, 0.353677, -0.043263, 7.0), + float4(-0.022713, -0.034611, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.836048, -0.548657, 0.0, 0.577584), + float4(0.340698, 0.474815, -0.043263, 11.0), + float4(0.001082, -0.121137, 0.0, 0.01033), + float4(0.001082, -0.121137, 0.043263, 0.045182), + float4(0.99996, 0.008928, 0.0, 0.577584), + float4(0.411001, 0.351514, -0.043263, 11.0), + float4(-0.034611, 0.030284, 0.0, 0.01033), + float4(-0.034611, 0.030284, 0.043263, 0.045182), + float4(-0.658505, -0.752577, 0.0, 0.577584), + float4(0.411001, 0.351514, -0.043263, 7.0), + float4(-0.034611, 0.030284, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.658505, -0.752577, 0.0, 0.577584), + float4(0.412083, 0.38288, 0.043263, 14.0), + float4(-0.001082, -0.031366, -0.043263, 0.0), + float4(-0.035692, -0.001082, -0.043263, 0.745403), + float4(-0.574039, -0.656045, 0.489984, 0.854994), + float4(0.45102, 0.359085, -0.043263, 11.0), + float4(-0.040019, -0.007571, 0.0, 0.01033), + float4(-0.040019, -0.007571, 0.043263, 0.045182), + float4(0.185892, -0.98257, 0.0, 0.577584), + float4(0.45102, 0.359085, 0.0, 10.0), + float4(-0.040019, -0.007571, 0.0, 0.000305), + float4(-0.038937, 0.023795, 0.043263, 0.879622), + float4(0.151736, -0.802035, 0.577682, 0.9131), + float4(0.45102, 0.359085, -0.043263, 7.0), + float4(-0.040019, -0.007571, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.185892, -0.98257, 0.0, 0.577584), + float4(0.45102, 0.359085, 0.0, 6.0), + float4(-0.038937, 0.023795, 0.043263, 0.000305), + float4(0.003245, 0.0411, 0.032447, 0.879622), + float4(0.417852, -0.583049, 0.696744, 0.9131), + float4(0.473733, 0.391533, -0.043263, 11.0), + float4(-0.022713, -0.032447, 0.0, 0.01033), + float4(-0.022713, -0.032447, 0.043263, 0.045182), + float4(0.819232, -0.573462, 0.0, 0.577584), + float4(0.454264, 0.400185, 0.032447, 14.0), + float4(0.019468, -0.008653, -0.032447, 0.0), + float4(-0.003245, -0.0411, -0.032447, 0.527108), + float4(0.688617, -0.482032, 0.541712, 0.887922), + float4(0.473733, 0.391533, -0.043263, 7.0), + float4(-0.022713, -0.032447, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.819232, -0.573462, 0.0, 0.577584), + float4(-0.223888, 0.407756, -0.043263, 7.0), + float4(0.002163, 0.06814, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.999496, 0.03173, 0.0, 0.577584), + float4(-0.223888, 0.407756, -0.043263, 11.0), + float4(0.002163, 0.06814, 0.0, 0.01033), + float4(0.002163, 0.06814, 0.043263, 0.045182), + float4(-0.999496, 0.03173, 0.0, 0.577584), + float4(-0.167645, 0.416409, 0.046508, 14.0), + float4(-0.056242, -0.008653, -0.046508, 0.000305), + float4(-0.054079, 0.059487, -0.046508, 0.955978), + float4(-0.638985, 0.020285, 0.768952, 0.973449), + float4(-0.167645, 0.416409, 0.046508, 14.0), + float4(0.004326, -0.035692, -0.046508, 0.000305), + float4(-0.056242, -0.008653, -0.046508, 0.955978), + float4(-0.339782, -0.761112, 0.552501, 0.973449), + float4(-0.221724, 0.475896, 0.0, 14.0), + float4(0.139524, 0.055161, 0.043263, 0.520989), + float4(0.054079, -0.059487, 0.046508, 0.973449), + float4(-0.39307, 0.317373, 0.863001, 0.973449), + float4(-0.167645, 0.416409, 0.046508, 6.0), + float4(0.073548, 0.015142, -0.046508, 0.0), + float4(0.004326, -0.035692, -0.046508, 0.768154), + float4(0.490917, -0.668482, 0.558688, 0.871366), + float4(-0.094098, 0.431551, -0.043263, 7.0), + float4(-0.069221, -0.050834, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.591909, -0.806004, 0.0, 0.577584), + float4(-0.167645, 0.416409, 0.046508, 10.0), + float4(0.085445, 0.114648, -0.003245, 0.0), + float4(0.073548, 0.015142, -0.046508, 0.768154), + float4(0.548356, -0.38771, 0.740936, 0.871366), + float4(-0.0822, 0.531057, -0.043263, 11.0), + float4(-0.011897, -0.099506, 0.0, 0.01033), + float4(-0.011897, -0.099506, 0.043263, 0.045182), + float4(0.992928, -0.11872, 0.0, 0.577584), + float4(-0.044345, 0.23254, 0.0, 7.0), + float4(-0.001082, 0.332046, -0.043263, 0.01033), + float4(-0.001082, 0.332046, 0.052998, 0.045182), + float4(-0.999995, -0.003257, 0.0, 0.577584), + float4(-0.045426, 0.564586, 0.052998, 14.0), + float4(0.1233, -0.156829, -0.052998, 0.520989), + float4(0.014061, -0.335291, 0.005408, 0.088655), + float4(0.429355, 0.032562, 0.902549, 0.973449), + float4(0.10924, 0.439122, 0.0, 14.0), + float4(-0.031366, -0.031366, 0.0, 0.132876), + float4(-0.154666, 0.125464, 0.052998, 0.028428), + float4(0.182761, -0.182761, 0.966021, 0.955978), + float4(0.10924, 0.439122, -0.043263, 11.0), + float4(-0.031366, -0.031366, 0.0, 0.01033), + float4(-0.031366, -0.031366, 0.043263, 0.045182), + float4(0.707107, -0.707107, 0.0, 0.577584), + float4(0.10924, 0.439122, -0.043263, 7.0), + float4(-0.031366, -0.031366, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.707107, -0.707107, 0.0, 0.577584), + float4(0.10924, 0.439122, 0.0, 10.0), + float4(0.019468, 0.024876, 0.063813, 0.18117), + float4(0.035692, -0.131953, 0.049753, 0.955978), + float4(-0.933936, -0.126584, 0.334276, 0.964691), + float4(0.128708, 0.463999, 0.063813, 14.0), + float4(0.050834, 0.002163, -0.005408, 0.0), + float4(0.0822, -0.018387, -0.063813, 0.439658), + float4(0.07859, -0.92643, 0.368172, 0.871366), + float4(-0.221724, 0.475896, -0.043263, 7.0), + float4(0.122219, 0.207664, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.861819, 0.507216, 0.0, 0.577584), + float4(-0.221724, 0.475896, -0.043263, 9.0), + float4(0.122219, 0.207664, 0.0, 0.01033), + float4(0.122219, 0.207664, 0.043263, 0.045182), + float4(-0.861819, 0.507216, 0.0, 0.577584), + float4(-0.099506, 0.68356, 0.0, 14.0), + float4(0.017305, -0.152503, 0.043263, 0.0), + float4(-0.122219, -0.207664, 0.0, 0.723049), + float4(-0.365879, 0.215335, 0.905408, 0.854994), + float4(-0.0822, 0.531057, -0.043263, 7.0), + float4(-0.011897, -0.099506, 0.043263, 0.01033), + float4(0.0, 0.0, 0.086527, 0.045182), + float4(0.992928, -0.11872, 0.0, 0.577584), + float4(-0.099506, 0.68356, 0.0, 14.0), + float4(0.054079, -0.118974, 0.052998, 0.0), + float4(0.017305, -0.152503, 0.043263, 0.514916), + float4(-0.41958, 0.203352, 0.884647, 0.863157), + float4(-0.099506, 0.68356, -0.043263, 3.0), + float4(0.086527, 0.149258, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.86514, 0.501531, 0.0, 0.577584), + float4(-0.045426, 0.564586, -0.043263, 11.0), + float4(-0.036774, -0.033529, 0.0, 0.01033), + float4(-0.036774, -0.033529, 0.086527, 0.045182), + float4(0.673754, -0.738956, 0.0, 0.577584), + float4(-0.045426, 0.564586, -0.043263, 7.0), + float4(-0.036774, -0.033529, 0.086527, 0.01033), + float4(0.0, 0.0, 0.096261, 0.045182), + float4(0.673754, -0.738956, 0.0, 0.577584), + float4(-0.099506, 0.68356, 0.0, 14.0), + float4(0.086527, 0.149258, 0.0, 0.0), + float4(0.054079, -0.118974, 0.052998, 0.262257), + float4(-0.385563, 0.223515, 0.8952, 0.9131), + float4(-0.099506, 0.68356, -0.043263, 11.0), + float4(0.086527, 0.149258, 0.0, 0.01033), + float4(0.086527, 0.149258, 0.043263, 0.045182), + float4(-0.86514, 0.501531, 0.0, 0.577584), + float4(-0.012979, 0.832818, 0.0, 14.0), + float4(0.214153, -0.194685, 0.0822, 0.000305), + float4(-0.032447, -0.268232, 0.052998, 0.930114), + float4(-0.176863, 0.211325, 0.961281, 0.973449), + float4(0.128708, 0.463999, 0.063813, 14.0), + float4(-0.019468, -0.024876, -0.063813, 0.168276), + float4(-0.174135, 0.100587, -0.010816, 0.038209), + float4(-0.469238, -0.764881, 0.441331, 0.955978), + float4(0.201174, 0.638133, 0.0822, 14.0), + float4(-0.072466, -0.174135, -0.018387, 0.18117), + float4(-0.246601, -0.073548, -0.029203, 0.082277), + float4(-0.098561, -0.063844, 0.993081, 0.973449), + float4(-0.012979, 0.832818, -0.043263, 7.0), + float4(0.339617, -0.002163, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.006369, 0.99998, 0.0, 0.577584), + float4(0.179543, 0.466162, 0.058405, 10.0), + float4(-0.050834, -0.002163, 0.005408, 0.000305), + float4(0.021632, 0.171972, 0.023795, 0.022171), + float4(0.110896, -0.149888, 0.982464, 0.381323), + float4(-0.012979, 0.832818, 0.0, 14.0), + float4(0.339617, -0.002163, 0.0, 0.0), + float4(0.214153, -0.194685, 0.0822, 0.027314), + float4(0.002492, 0.391296, 0.920261, 0.49102), + float4(0.183869, 0.317985, 0.037855, 10.0), + float4(-0.055161, 0.146014, 0.025958, 0.0), + float4(0.02704, 0.127627, -0.037855, 0.439658), + float4(0.623841, 0.097823, 0.775405, 0.871366), + float4(0.210908, 0.445612, -0.043263, 7.0), + float4(0.0, -0.131953, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(1.0, 0.0, 0.0, 0.577584), + float4(0.210908, 0.445612, 0.0, 10.0), + float4(-0.031366, 0.02055, 0.058405, 0.000305), + float4(0.035692, 0.014061, 0.06165, 0.022171), + float4(-0.105864, -0.954447, 0.278971, 0.381323), + float4(0.23903, 0.45102, -0.043263, 11.0), + float4(-0.028121, -0.005408, 0.0, 0.01033), + float4(-0.028121, -0.005408, 0.043263, 0.045182), + float4(0.188847, -0.982006, 0.0, 0.577584), + float4(0.23903, 0.45102, -0.043263, 7.0), + float4(-0.028121, -0.005408, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.188847, -0.982006, 0.0, 0.577584), + float4(0.246601, 0.459672, 0.06165, 11.0), + float4(-0.007571, -0.008653, -0.06165, 0.000305), + float4(-0.035692, -0.014061, -0.06165, 0.022171), + float4(0.187618, -0.975617, 0.113888, 0.381323), + float4(0.241193, 0.343943, -0.043263, 11.0), + float4(-0.002163, 0.107077, 0.0, 0.01033), + float4(-0.002163, 0.107077, 0.043263, 0.045182), + float4(-0.999796, -0.020198, 0.0, 0.577584), + float4(0.23903, 0.45102, 0.0, 15.0), + float4(0.007571, 0.008653, 0.06165, 0.000305), + float4(0.042182, -0.108158, 0.046508, 0.846876), + float4(-0.94107, -0.299262, 0.157572, 0.896269), + float4(0.316904, 0.358004, 0.036774, 14.0), + float4(-0.070303, 0.101669, 0.024876, 0.0), + float4(0.023795, 0.116811, -0.036774, 0.473533), + float4(0.523424, 0.157027, 0.837478, 0.930114), + float4(0.340698, 0.474815, -0.043263, 7.0), + float4(0.001082, -0.121137, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.99996, 0.008928, 0.0, 0.577584), + float4(0.376391, 0.381799, -0.043263, 11.0), + float4(-0.001082, 0.107077, 0.0, 0.01033), + float4(-0.001082, 0.107077, 0.043263, 0.045182), + float4(-0.999949, -0.0101, 0.0, 0.577584), + float4(0.376391, 0.381799, -0.043263, 7.0), + float4(-0.001082, 0.107077, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.999949, -0.0101, 0.0, 0.577584), + float4(0.375309, 0.488875, 0.0, 14.0), + float4(0.036774, -0.105995, 0.043263, 0.000305), + float4(0.001082, -0.107077, 0.0, 0.879622), + float4(-0.771254, -0.00779, 0.63648, 0.896269), + float4(0.375309, 0.488875, 0.0, 14.0), + float4(0.078955, -0.08869, 0.032447, 0.223224), + float4(0.036774, -0.105995, 0.043263, 0.964691), + float4(0.071225, 0.39802, 0.914608, 0.973449), + float4(0.473733, 0.603523, 0.0, 14.0), + float4(0.0, -0.21199, 0.0, 0.158953), + float4(-0.019468, -0.203337, 0.032447, 0.030716), + float4(0.857493, 0.0, 0.514496, 0.964691), + float4(0.473733, 0.603523, -0.043263, 11.0), + float4(0.0, -0.21199, 0.0, 0.01033), + float4(0.0, -0.21199, 0.043263, 0.045182), + float4(1.0, 0.0, 0.0, 0.577584), + float4(0.276885, 0.484549, 0.058405, 10.0), + float4(-0.030284, -0.024876, 0.003245, 0.000305), + float4(-0.097342, -0.018387, 0.0, 0.022171), + float4(-0.03153, 0.166923, 0.985466, 0.381323), + float4(0.246601, 0.459672, 0.06165, 14.0), + float4(0.030284, 0.024876, -0.003245, 0.0), + float4(0.094098, 0.015142, -0.06165, 0.473533), + float4(0.518869, -0.545855, 0.657888, 0.930114), + float4(0.375309, 0.488875, -0.043263, 11.0), + float4(-0.034611, -0.014061, 0.0, 0.01033), + float4(-0.034611, -0.014061, 0.043263, 0.045182), + float4(0.376377, -0.926467, 0.0, 0.577584), + float4(0.340698, 0.474815, 0.0, 14.0), + float4(-0.063813, 0.009734, 0.058405, 0.000305), + float4(0.034611, 0.014061, 0.0, 0.022171), + float4(0.3276, -0.806402, 0.492334, 0.381323), + float4(0.375309, 0.488875, -0.043263, 7.0), + float4(-0.034611, -0.014061, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.376377, -0.926467, 0.0, 0.577584), + float4(0.454264, 0.400185, 0.032447, 12.0), + float4(-0.078955, 0.08869, -0.032447, 0.158953), + float4(0.019468, 0.203337, -0.032447, 0.030716), + float4(-0.201684, 0.173144, 0.964025, 0.964691), + float4(0.201174, 0.638133, 0.0822, 8.0), + float4(0.075711, -0.153585, -0.023795, 0.000305), + float4(-0.021632, -0.171972, -0.023795, 0.022171), + float4(0.026497, -0.140281, 0.989757, 0.381323), + float4(0.473733, 0.603523, -0.043263, 7.0), + float4(0.0, -0.21199, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(1.0, 0.0, 0.0, 0.577584), + float4(0.276885, 0.484549, 0.058405, 14.0), + float4(-0.075711, 0.153585, 0.023795, 0.000305), + float4(0.098424, 0.004326, -0.058405, 0.022171), + float4(0.50316, 0.115345, 0.856461, 0.381323), + float4(0.375309, 0.488875, 0.0, 10.0), + float4(0.090853, 0.12979, 0.0, 0.158953), + float4(0.098424, 0.114648, 0.0, 0.030716), + float4(0.0, 0.0, 1.0, 0.964691), + float4(0.466162, 0.618665, -0.043263, 11.0), + float4(0.007571, -0.015142, 0.0, 0.01033), + float4(0.007571, -0.015142, 0.043263, 0.045182), + float4(0.894427, 0.447214, 0.0, 0.577584), + float4(0.466162, 0.618665, -0.043263, 3.0), + float4(0.007571, -0.015142, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.894427, 0.447214, 0.0, 0.577584), + float4(0.326638, 0.830655, 0.0, 14.0), + float4(0.139524, -0.21199, 0.0, 0.0), + float4(0.048671, -0.34178, 0.0, 0.015213), + float4(0.0, 0.0, 1.0, 0.428687), + float4(0.326638, 0.830655, 0.0, 14.0), + float4(0.048671, -0.34178, 0.0, 0.007492), + float4(-0.125464, -0.192522, 0.0822, 0.054475), + float4(0.472493, 0.067285, 0.878762, 0.964691), + float4(0.326638, 0.830655, -0.043263, 9.0), + float4(0.139524, -0.21199, 0.0, 0.01033), + float4(0.139524, -0.21199, 0.043263, 0.045182), + float4(0.835314, 0.549773, 0.0, 0.577584), + float4(0.326638, 0.830655, -0.043263, 7.0), + float4(0.139524, -0.21199, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.835314, 0.549773, 0.0, 0.577584), + float4(-0.012979, 0.832818, -0.043263, 11.0), + float4(0.339617, -0.002163, 0.0, 0.01033), + float4(0.339617, -0.002163, 0.043263, 0.045182), + float4(0.006369, 0.99998, 0.0, 0.577584) +}; +constant float4 PRISM_BOXES[44] = { + float4(0.030135, 0.06127, -0.043945, 0.0), + float4(0.330601, 0.483055, 0.116951, 7.0), + float4(0.034429, 0.265258, -0.043945, 7.0), + float4(0.542104, 0.741798, 0.116951, 8.0), + float4(0.034429, 0.355442, -0.043945, 15.0), + float4(0.305908, 0.822319, 0.116951, 8.0), + float4(0.267405, 0.532589, -0.043945, 23.0), + float4(0.542104, 0.822319, 0.082595, 8.0), + float4(-0.045345, 0.036855, -0.044263, 31.0), + float4(0.045345, 0.23354, 0.059405, 8.0), + float4(-0.224888, 0.072548, -0.044263, 39.0), + float4(0.059405, 0.565586, 0.059405, 8.0), + float4(-0.032366, 0.042263, -0.044263, 47.0), + float4(0.148095, 0.408756, 0.059405, 8.0), + float4(0.074711, 0.08769, -0.044263, 55.0), + float4(0.184869, 0.464999, 0.064813, 8.0), + float4(0.146095, 0.271559, -0.044263, 63.0), + float4(0.211908, 0.446612, 0.038855, 8.0), + float4(0.23803, 0.310496, -0.044263, 71.0), + float4(0.320067, 0.460672, 0.06265, 8.0), + float4(0.275885, 0.310496, -0.044263, 79.0), + float4(0.412001, 0.475815, 0.047508, 8.0), + float4(0.375391, 0.350514, -0.044263, 87.0), + float4(0.45202, 0.38388, 0.044263, 4.0), + float4(0.410001, 0.350514, -0.044263, 91.0), + float4(0.474733, 0.401185, 0.044263, 5.0), + float4(-0.224888, 0.379717, -0.044263, 96.0), + float4(-0.0812, 0.532057, 0.047508, 8.0), + float4(-0.095098, 0.228295, -0.044263, 104.0), + float4(0.211908, 0.565586, 0.064813, 8.0), + float4(-0.222724, 0.430551, -0.044263, 112.0), + float4(-0.011979, 0.833818, 0.053998, 8.0), + float4(-0.100506, 0.438122, -0.044263, 120.0), + float4(0.327638, 0.833818, 0.0832, 8.0), + float4(0.127708, 0.312659, -0.044263, 128.0), + float4(0.282211, 0.467162, 0.064813, 8.0), + float4(0.245601, 0.352677, -0.044263, 136.0), + float4(0.474733, 0.604523, 0.06265, 8.0), + float4(0.178543, 0.390533, -0.044263, 144.0), + float4(0.474733, 0.639133, 0.0832, 8.0), + float4(0.200174, 0.483549, -0.044263, 152.0), + float4(0.474733, 0.639133, 0.0832, 4.0), + float4(-0.013979, 0.487875, -0.044263, 156.0), + float4(0.467162, 0.833818, 0.0832, 5.0) +}; +constant float2 PRISM_POLY[110] = { + float2(0.006442, -0.002147), + float2(-0.001074, 0.001074), + float2(-0.004294, 0.007515), + float2(-0.004294, 0.806288), + float2(-0.001074, 0.809509), + float2(0.004294, 0.809509), + float2(0.022546, 0.796626), + float2(0.185736, 0.655982), + float2(0.281288, 0.86319), + float2(0.286656, 0.870706), + float2(0.293098, 0.871779), + float2(0.465951, 0.781595), + float2(0.478834, 0.773006), + float2(0.478834, 0.763344), + float2(0.407975, 0.592638), + float2(0.45092, 0.586196), + float2(0.601227, 0.573313), + float2(0.606595, 0.569018), + float2(0.606595, 0.564724), + float2(0.601227, 0.556135), + float2(0.50138, 0.453067), + float2(0.425153, 0.37684), + float2(0.289877, 0.246933), + float2(0.081595, 0.057975), + float2(0.013957, 0.0), + float2(-0.011897, 0.0), + float2(-0.055161, 0.037855), + float2(-0.081119, 0.063813), + float2(-0.083282, 0.06814), + float2(-0.084363, 0.38288), + float2(-0.089771, 0.38288), + float2(-0.158993, 0.33529), + float2(-0.166564, 0.336372), + float2(-0.250927, 0.379635), + float2(-0.261743, 0.388288), + float2(-0.263906, 0.461836), + float2(-0.262824, 0.486712), + float2(-0.173053, 0.638133), + float2(-0.044345, 0.860939), + float2(-0.036774, 0.867429), + float2(-0.024876, 0.871755), + float2(0.33529, 0.871755), + float2(0.349351, 0.867429), + float2(0.363412, 0.853368), + float2(0.513752, 0.619747), + float2(0.513752, 0.379635), + float2(0.466162, 0.322311), + float2(0.461836, 0.320148), + float2(0.399104, 0.311496), + float2(0.393696, 0.313659), + float2(0.374227, 0.332046), + float2(0.343943, 0.288782), + float2(0.336372, 0.282293), + float2(0.269314, 0.271477), + float2(0.263906, 0.27364), + float2(0.243356, 0.292027), + float2(0.21199, 0.244438), + float2(0.207664, 0.241193), + float2(0.149258, 0.231459), + float2(0.138443, 0.231459), + float2(0.118974, 0.248764), + float2(0.116811, 0.222806), + float2(0.117892, 0.162237), + float2(0.115729, 0.076792), + float2(0.06814, 0.011897), + float2(0.062732, 0.007571), + float2(0.005408, 0.0), + float2(0.031135, 0.06227), + float2(0.252301, 0.266258), + float2(0.541104, 0.544325), + float2(0.359663, 0.559356), + float2(0.433742, 0.753681), + float2(0.304908, 0.821319), + float2(0.199693, 0.595859), + float2(0.035429, 0.740798), + float2(0.035429, 0.482055), + float2(-0.043263, 0.077874), + float2(0.001082, 0.037855), + float2(0.044345, 0.043263), + float2(0.075711, 0.08869), + float2(0.077874, 0.407756), + float2(0.10924, 0.439122), + float2(0.111403, 0.307169), + float2(0.147095, 0.272559), + float2(0.189277, 0.28013), + float2(0.210909, 0.313659), + float2(0.210909, 0.445612), + float2(0.23903, 0.45102), + float2(0.241193, 0.343943), + float2(0.276885, 0.311496), + float2(0.319067, 0.319067), + float2(0.34178, 0.353677), + float2(0.340698, 0.474815), + float2(0.375309, 0.488875), + float2(0.376391, 0.381799), + float2(0.411001, 0.351514), + float2(0.45102, 0.359085), + float2(0.473733, 0.391533), + float2(0.473733, 0.603523), + float2(0.466162, 0.618665), + float2(0.326638, 0.830655), + float2(-0.012979, 0.832818), + float2(-0.099506, 0.68356), + float2(-0.221724, 0.475896), + float2(-0.223888, 0.407756), + float2(-0.163319, 0.380717), + float2(-0.094098, 0.431551), + float2(-0.0822, 0.531057), + float2(-0.045426, 0.564586), + float2(-0.044345, 0.23254) +}; +constant float PRISM_BEVEL = 0.012883; +// end of the prism mesh -static float s_gem_arrow(float3 p) -{ - float z0 = SCULPT_HOVER + 0.03; - float slab = max(p.z - z0 - 0.2, z0 - 0.03 - p.z); - float head = max(slab, max(s_gem_edge(p, float2(-0.02, 0.03), float2(-0.02, -0.88), z0), - max(s_gem_edge(p, float2(-0.02, -0.88), float2(0.66, -0.61), z0), - s_gem_edge(p, float2(0.66, -0.61), float2(-0.02, 0.03), z0)))); - float tail = max(slab, max(max(s_gem_edge(p, float2(0.215, -0.665), float2(0.37, -1.0), z0), - s_gem_edge(p, float2(0.37, -1.0), float2(0.53, -0.93), z0)), - max(s_gem_edge(p, float2(0.53, -0.93), float2(0.38, -0.60), z0), - s_gem_edge(p, float2(0.38, -0.60), float2(0.215, -0.665), z0)))); - return min(head, tail); -} +constant float PRISM_IOR = 1.61; +constant float PRISM_DISPERSION = 0.035; +constant float PRISM_GLOW = 0.45; +constant float PRISM_FOLD = 0.35; +constant int PRISM_BOUNCES = 6; +constant float PRISM_MAT = 9.0; -static float s_facet_capsule(float3 p, float3 a, float3 b, float r, float spin) +inline int prism_shape(constant Layer &layer) { - float3 u = normalize(b - a); - float3 v = normalize(cross(u, float3(0.0, 0.0, 1.0))); - float3 w = cross(u, v); - float3 q = p - b; - float h = dot(q, u); - float2 rad = float2(dot(q, v), dot(q, w)); - float d = max(h - r, -dot(p - a, u) - r); - for (int k = 0; k < 6; k++) - { - float an = spin + float(k) * 1.0471976; - float s = dot(rad, float2(cos(an), sin(an))); - d = max(d, s - r); - d = max(d, dot(rad, float2(cos(an + 0.5236), sin(an + 0.5236))) * 0.8660 + h * 0.5 - r); - d = max(d, s * 0.5 + h * 0.8660 - r); - } - return d; + int id = sculpt_id(layer); + return id == 3 || id == 4 ? id - 3 : -1; } -constant float3 SCULPT_FACETS[10] = { - float3(1.0, 0.0, 0.0), float3(0.0, 1.0, 0.0), float3(0.0, 0.0, 1.0), - float3(0.7071, 0.7071, 0.0), float3(0.7071, 0.0, 0.7071), float3(0.0, 0.7071, 0.7071), - float3(0.5774, 0.5774, 0.5774), float3(0.4472, 0.0, 0.8944), float3(0.0, 0.4472, 0.8944), - float3(0.3015, 0.3015, 0.9045) -}; - -static float s_facet_ellipsoid(float3 p, float3 r) +static float prism_poly_dist(float2 p, int i0, int n) { - float3 q = abs(p); - float d = -1e9; - for (int i = 0; i < 10; i++) + float2 vj = PRISM_POLY[i0 + n - 1]; + float d = 1e9; + float sgn = 1.0; + for (int i = 0; i < n; i++) { - d = max(d, dot(q, SCULPT_FACETS[i]) - length(r * SCULPT_FACETS[i])); + float2 vi = PRISM_POLY[i0 + i]; + float2 e = vj - vi; + float2 w = p - vi; + float2 b = w - e * saturate(dot(w, e) / dot(e, e)); + d = min(d, dot(b, b)); + bool c0 = p.y >= vi.y; + bool c1 = p.y < vj.y; + bool c2 = e.x * w.y > e.y * w.x; + if ((c0 && c1 && c2) || (!c0 && !c1 && !c2)) + { + sgn = -sgn; + } + vj = vi; } - return d; + return sgn * sqrt(d); } -static float s_crystal_hand(float3 p) +static float prism_rim(float3 p, int shape, constant Layer &layer) { - float zc = SCULPT_HAND_ZC; - float d = s_facet_ellipsoid(p - float3(0.185, -0.70, zc), float3(0.3, 0.235, 0.125)); - d = min(d, s_facet_capsule(p, float3(0.0, -0.52, zc), float3(0.0, -0.10, zc), 0.098, 0.3)); - d = min(d, s_facet_capsule(p, float3(0.17, -0.57, zc + 0.012), float3(0.17, -0.41, zc + 0.045), 0.086, 0.1)); - d = min(d, s_facet_capsule(p, float3(0.31, -0.59, zc + 0.01), float3(0.31, -0.45, zc + 0.04), 0.08, 0.5)); - d = min(d, s_facet_capsule(p, float3(0.435, -0.625, zc + 0.005), float3(0.435, -0.52, zc + 0.03), 0.07, 0.2)); - return min(d, s_facet_capsule(p, float3(0.03, -0.77, zc + 0.03), float3(-0.165, -0.60, zc + 0.065), 0.082, 0.7)); + float half_t = model_thick(layer) * 0.5; + float sil = prism_poly_dist(p.xy, PRISM_SIL_START[shape], PRISM_SIL_COUNT[shape]); + float2 w = float2(sil + PRISM_BEVEL, abs(p.z + half_t) - (half_t - PRISM_BEVEL)); + return min(max(w.x, w.y), 0.0) + length(max(w, 0.0)) - PRISM_BEVEL; } -static float2 sculpt_proto(float3 p, int theme, int shape) +static float prism_tri(float3 ro, float3 rd, int k) { - if (theme == 3) + float3 v0 = PRISM_TRIS[4 * k].xyz; + float3 e1 = PRISM_TRIS[4 * k + 1].xyz; + float3 e2 = PRISM_TRIS[4 * k + 2].xyz; + float3 pv = cross(rd, e2); + float det = dot(e1, pv); + if (abs(det) < 1e-10) { - return s_voxels(p, shape); + return -1.0; } - if (theme == 1) + float inv = 1.0 / det; + float3 sv = ro - v0; + float u = dot(sv, pv) * inv; + float3 qv = cross(sv, e1); + float v = dot(rd, qv) * inv; + if (u < -2e-4 || v < -2e-4 || u + v > 1.0004) { - return float2(shape == 0 ? s_gem_arrow(p) : s_crystal_hand(p), 8.0); + return -1.0; } - float2 body; - float3 star; - if (shape == 0) + return dot(e2, qv) * inv; +} + +static float prism_trace(float3 ro, float3 rd, int shape, thread int &hit) +{ + float3 inv = 1.0 / select(float3(1e-6), rd, abs(rd) > 1e-6); + int b0 = PRISM_BOX_START[shape]; + int nb = PRISM_BOX_COUNT[shape]; + float best = 1e9; + hit = -1; + for (int j = b0; j < b0 + nb; j++) { - float h = 0.075, re = 0.03, dome = 0.0; - if (theme == 2) + float4 lo = PRISM_BOXES[2 * j]; + float4 hi = PRISM_BOXES[2 * j + 1]; + float3 t0 = (lo.xyz - ro) * inv; + float3 t1 = (hi.xyz - ro) * inv; + float3 tn = min(t0, t1); + float3 tf = max(t0, t1); + float tmin = max(max(tn.x, tn.y), tn.z); + float tmax = min(min(tf.x, tf.y), tf.z); + if (tmax < max(tmin, 1e-5) || tmin > best) { - h = 0.07; - re = 0.06; - dome = 0.035; + continue; } - if (theme == 4) + int k0 = int(lo.w); + int n = int(hi.w); + for (int i = k0; i < k0 + n; i++) { - h = 0.07; - re = 0.055; - dome = 0.025; - } - body = float2(s_arrow_solid(p, h, re, dome), 1.0); - if (theme == 0) - { - body = s_opu(body, float2(s_piping(p, SCULPT_HOVER + 2.0 * h - 0.004), 2.0)); + float t = prism_tri(ro, rd, i); + if (t > 1e-5 && t < best) + { + best = t; + hit = i; + } } - star = float3(0.57, -0.86, SCULPT_HOVER + 2.0 * h + dome); } - else + return best; +} + +static float2 sculpt_proto(float3 p, int theme, int shape) +{ + if (theme == 3) { - body = s_opu(float2(s_glove(p, SCULPT_HAND_ZC), 1.0), float2(s_cuff(p, SCULPT_HAND_ZC), 2.0)); - star = float3(0.185, -0.93, SCULPT_HAND_ZC + 0.15); + return s_voxels(p, shape); } - return theme == 4 ? s_opu(body, s_sprout(p, star)) : body; + if (theme == 5) + { + return s_glass(p, shape); + } + return s_rimmed(p, theme, shape); } inline float3 sculpt_point(float3 q, constant Layer &layer) @@ -961,17 +1987,21 @@ static float2 sculpt_eval(float3 q, bool occ, constant Layer &layer) { int id = sculpt_id(layer) - 1; int theme = id / 2, shape = id % 2; + if (theme == 1) + { + return float2(prism_rim(q, shape, layer), 8.0); + } float3 p = sculpt_point(q, layer); float2 r; if (occ && theme == 3) { - float d2 = shape == 0 ? s_arrow2(p.xy) - 1.2 * SCULPT_VOX : s_pixel_hand_dist(p.xy); + float d2 = s_pixel_outline(p.xy, shape); float dz = abs(p.z - (SCULPT_HOVER + 0.07)) - 0.07; r = float2(length(max(float2(d2, dz), 0.0)) + min(max(d2, dz), 0.0), 7.0); } else { - r = sculpt_proto(p, occ && theme == 1 ? 2 : theme, shape); + r = sculpt_proto(p, theme, shape); } return float2(r.x * sculpt_units(layer), r.y); } @@ -1076,16 +2106,13 @@ inline SculptMat s_mat(float3 alb, float rough, float spec, float sss, float ref static float3 s_pixel_colour(float3 p, int shape) { - float2 id = floor((p.xy - s_grid_origin(shape)) / SCULPT_VOX); - if (!s_occ(id + float2(-1.0, 0.0), shape) || !s_occ(id + float2(0.0, -1.0), shape)) - { - return s_lin(0.52, 0.91, 0.77); - } - if (!s_occ(id + float2(1.0, 0.0), shape) || !s_occ(id + float2(0.0, 1.0), shape)) - { - return s_lin(1.0, 0.78, 0.87); - } - return s_lin(1.0, 0.50, 0.71); + int2 cell = s_pix_cell(p.xy, shape); + int4 g = PIX_GRID[shape]; + int row = g.z + clamp(cell.y, 0, g.y - 1); + if (s_pix_bit(PIX_LINE[row], cell.x)) return s_lin(0.29, 0.12, 0.36); + if (s_pix_bit(PIX_HI[row], cell.x)) return s_lin(1.0, 0.78, 0.87); + if (s_pix_bit(PIX_SHADE[row], cell.x)) return s_lin(0.87, 0.27, 0.51); + return s_lin(1.0, 0.435, 0.66); } static SculptMat sculpt_material(float mat, float3 p, int theme, int shape) @@ -1093,28 +2120,27 @@ static SculptMat sculpt_material(float mat, float3 p, int theme, int shape) bool primary = mat < 1.5; if (theme == 0) { - if (shape == 0 && primary) return s_mat(s_lin(0.10, 0.10, 0.115), 0.3, 0.2, 0.0, 0.9); - if (shape == 0) return s_mat(s_lin(0.94, 0.91, 0.84), 0.45, 0.4, 0.2, 0.3); - if (primary) return s_mat(s_lin(0.95, 0.92, 0.85), 0.55, 0.35, 0.35, 0.25); - return s_mat(s_lin(0.17, 0.18, 0.22), 0.35, 0.6, 0.0, 0.6); + if (mat < 2.5) return s_mat(s_lin(0.95, 0.92, 0.85), 0.3, 0.6, 0.35, 0.25); + return s_mat(s_lin(0.1, 0.1, 0.11), 0.55, 0.05, 0.0, 0.1); } + if (theme == 1) return s_mat(s_lin(0.02, 0.05, 0.33), 0.35, 0.5, 0.05, 0.3); + if (theme == 5) return s_mat(s_lin(0.05, 0.055, 0.07), 0.25, 0.7, 0.0, 0.35); if (theme == 2) { - if (shape == 0) return s_mat(s_lin(1.0, 0.40, 0.30), 0.5, 0.45, 0.4, 0.25); - if (primary) return s_mat(s_lin(1.0, 0.79, 0.16), 0.5, 0.45, 0.4, 0.25); - return s_mat(s_lin(0.18, 0.20, 0.29), 0.4, 0.5, 0.0, 0.4); + if ((primary && shape == 0) || (mat > 2.5 && mat < 3.5 && shape == 1)) return s_mat(s_lin(1.0, 0.40, 0.30), 0.85, 0.08, 0.15, 0.03); + if (mat < 3.5) return s_mat(s_lin(1.0, 0.80, 0.10), 0.85, 0.08, 0.15, 0.03); + return s_mat(s_lin(0.09, 0.13, 0.45), 0.9, 0.05, 0.05, 0.02); } if (theme == 4) { - if (primary && shape == 0) return s_mat(s_lin(0.62, 0.91, 0.78), 0.22, 0.8, 0.25, 0.6); - if (primary) return s_mat(s_lin(0.96, 0.94, 0.88), 0.5, 0.35, 0.35, 0.25); - if (mat < 2.5) return s_mat(s_lin(0.62, 0.91, 0.78), 0.25, 0.7, 0.25, 0.5); - if (mat < 3.5) return s_mat(s_lin(1.0, 0.80, 0.20), 0.3, 0.6, 0.3, 0.4); - if (mat < 4.5) return s_mat(s_lin(0.38, 0.80, 0.55), 0.35, 0.5, 0.35, 0.3); - return s_mat(s_lin(0.16, 0.12, 0.10), 0.2, 0.8, 0.0, 0.5); + if (primary && shape == 0) return s_mat(s_lin(0.68, 0.93, 0.80), 0.25, 0.7, 0.3, 0.3); + if (primary) return s_mat(s_lin(0.97, 0.95, 0.90), 0.3, 0.6, 0.35, 0.25); + if (mat < 2.5) return s_mat(s_lin(0.52, 0.87, 0.78), 0.25, 0.7, 0.3, 0.3); + if (mat < 3.5) return s_mat(s_lin(1.0, 0.75, 0.25), 0.25, 0.7, 0.3, 0.3); + if (mat < 4.5) return s_mat(s_lin(0.62, 0.92, 0.72), 0.3, 0.6, 0.3, 0.35); + return s_mat(s_lin(0.07, 0.15, 0.33), 0.7, 0.15, 0.05, 0.08); } - if (mat < 6.5) return s_mat(s_pixel_colour(p, shape), 0.45, 0.35, 0.15, 0.2); - return s_mat(s_lin(0.36, 0.18, 0.54), 0.45, 0.35, 0.1, 0.2); + return s_mat(s_pixel_colour(p, shape), 0.65, 0.1, 0.12, 0.04); } static float3 model_env(float3 d, float rough, float3 l, float3 fill) @@ -1127,34 +2153,6 @@ static float3 model_env(float3 d, float rough, float3 l, float3 fill) return col; } -static float3 s_gem(float k) -{ - k = saturate(k) * 3.0; - float3 a = s_lin(0.20, 0.95, 1.0); - float3 b = s_lin(0.15, 0.42, 1.0); - float3 c = s_lin(0.45, 0.25, 0.95); - float3 d = s_lin(0.88, 0.50, 1.0); - if (k < 1.0) return mix(a, b, k); - if (k < 2.0) return mix(b, c, k - 1.0); - return mix(c, d, k - 2.0); -} - -static float3 model_shade_crystal(float3 n, float3 rd, float3 L, float fall, float3 l, float3 fill) -{ - float cosi = saturate(dot(-rd, n)); - float F = 0.04 + 0.96 * pow(1.0 - cosi, 5.0); - float3 t = refract(rd, n, 1.0 / 1.6); - float k = 0.42 + 0.9 * dot(float2(n.x, -n.y), float2(0.7557, -0.6549)) - + 0.6 * dot(float2(t.x, -t.y), float2(0.6, -0.8)); - float3 body = float3(s_gem(k - 0.08).r, s_gem(k).g, s_gem(k + 0.08).b); - float3 col = body * (0.1 + 1.8 * fall * pow(max(dot(n, L), 0.0), 2.5)); - col += body * model_env(reflect(t, float3(0.0, 0.0, 1.0)) * float3(1.0, 1.0, -1.0), 0.15, l, fill) * 0.5; - col += pow(max(dot(reflect(rd, n), L), 0.0), 30.0) * 2.0; - col += model_env(reflect(rd, n), 0.05, l, fill) * F; - col += s_lin(0.5, 0.9, 1.0) * pow(1.0 - cosi, 4.0) * 0.6; - return col; -} - static float3 model_tonemap(float3 c) { const float start = 0.76; @@ -1178,16 +2176,16 @@ static float3 model_shade(float3 q, float3 n, float3 rd, float3 L, float fall, f texture2d texImg) { int id = sculpt_id(layer); - if (id > 0 && (id - 1) / 2 == 1) - { - return model_tonemap(model_shade_crystal(n, rd, L, fall, l, fill)); - } SculptMat m; float gloss = 1.0; if (id > 0) { float3 p = sculpt_point(q, layer) - float3(n.x, -n.y, n.z) * 0.01; m = sculpt_material(mat, p, (id - 1) / 2, (id - 1) % 2); + if (id == 3 || id == 4 || id > 10) + { + gloss = 1.0 - smoothstep(0.97, 0.995, abs(n.z)); + } } else { @@ -1211,13 +2209,230 @@ static float3 model_shade(float3 q, float3 n, float3 rd, float3 L, float fall, f return model_tonemap(col); } +// ---- Le cristal de Prism Glow ---- (cf. HLSL) + +inline float prism_fresnel(float c, float ior) +{ + float f0 = (ior - 1.0) / (ior + 1.0); + f0 *= f0; + return f0 + (1.0 - f0) * pow(1.0 - c, 5.0); +} + +static float prism_primary(float3 ro, float3 rd, thread int &hc, constant Layer &layer) +{ + hc = -1; + int shape = prism_shape(layer); + if (shape < 0) + { + return 1e9; + } + float t = prism_trace(ro, rd, shape, hc); + if (hc >= 0 && fmod(PRISM_TRIS[4 * hc].w, 2.0) > 0.5 && ro.z + rd.z * t < 0.0) + { + hc = -1; + return 1e9; + } + return t; +} + +inline float3 prism_normal(int h, float3 rd) +{ + float3 n = PRISM_TRIS[4 * h + 3].xyz; + return dot(n, rd) > 0.0 ? -n : n; +} + +inline float prism_seg(float3 p, float3 a, float3 b) +{ + float3 pa = p - a, ba = b - a; + return length(pa - ba * saturate(dot(pa, ba) / dot(ba, ba))); +} + +static float prism_edge_px(float3 p, int h, float px) +{ + float4 r0 = PRISM_TRIS[4 * h]; + float3 a = r0.xyz; + float3 b = a + PRISM_TRIS[4 * h + 1].xyz; + float3 c = a + PRISM_TRIS[4 * h + 2].xyz; + int bits = int(r0.w * 0.5); + float d = 1e9; + if ((bits & 1) != 0) d = min(d, prism_seg(p, b, c)); + if ((bits & 2) != 0) d = min(d, prism_seg(p, c, a)); + if ((bits & 4) != 0) d = min(d, prism_seg(p, a, b)); + return d / px; +} + +static float3 plane_to_world(float3 v, ModelFrame f) +{ + float x = v.x * f.c.z - v.y * f.s.z; + float y = v.x * f.s.z + v.y * f.c.z; + float z = -x * f.s.y + v.z * f.c.y; + return float3(x * f.c.y + v.z * f.s.y, y * f.c.x - z * f.s.x, y * f.s.x + z * f.c.x); +} + +// L'uv de sortie du point où le rayon (q, d) du modèle retombe sur le plan, (-1, -1) s'il n'y +// retombe pas (cf. HLSL). +static float2 screen_uv(float3 q, float3 d, float3 nz, float hz, float3 tip, float unit, ModelFrame f, + constant Layer &layer) +{ + float denom = dot(d, nz); + if (denom > -1e-4) + { + return float2(-1.0); + } + float3 g = q + d * ((hz - dot(q, nz)) / denom); + float3 w = plane_to_world(tip + unit * model_to_plane(g, f), f); + float k = 1.0 - w.z / layer.src.z; + if (k < 1e-3) + { + return float2(-1.0); + } + return layer.dst.xy + ((w.xy + layer.mb.zw) / k - layer.src.xy) / layer.quad_px * layer.dst.zw; +} + +// La copie de l'image composée en `uv`, en linéaire ; hors de l'image, l'écran sombre du studio. +static float3 screen_at(float2 uv, texture2d texFrame) +{ + if (any(uv < 0.0) || any(uv > 1.0)) + { + return SCULPT_SCREEN * 0.2; + } + return pow(texFrame.sample(samp, uv, level(0.0)).rgb, float3(2.2)); +} + +static float3 prism_screen(float3 q, float3 d, float3 nz, float hz, float3 tip, float unit, ModelFrame f, + constant Layer &layer, texture2d texFrame) +{ + return screen_at(screen_uv(q, d, nz, hz, tip, unit, f, layer), texFrame); +} + +static float3 prism_shade(float3 p, float3 n, float3 rd, int h, float px, float3 l, float3 fill, float3 nz, + float hz, float3 tip, float unit, ModelFrame f, constant Layer &layer, + texture2d texFrame) +{ + int shape = prism_shape(layer); + float4 r3 = PRISM_TRIS[4 * h + 3]; + float3 facet = float3(PRISM_TRIS[4 * h + 1].w, PRISM_TRIS[4 * h + 2].w, r3.w); + float F = prism_fresnel(saturate(-dot(rd, n)), PRISM_IOR); + float zb = -model_thick(layer); + float3 trans = float3(0.0); + for (int ch = 0; ch < 3; ch++) + { + float3 mask = float3(float(ch == 0), float(ch == 1), float(ch == 2)); + float ior = PRISM_IOR + PRISM_DISPERSION * float(ch - 1); + float3 d = refract(rd, n, 1.0 / ior); + float3 pos = p; + float thr = 1.0; + float acc = 0.0; + for (int b = 0; b < PRISM_BOUNCES; b++) + { + int hh; + float th = prism_trace(pos, d, shape, hh); + float tb = d.z < -1e-6 ? (zb - pos.z) / d.z : 1e9; + if (tb <= th) + { + float3 qb = pos + d * tb; + float3 d2 = refract(d, float3(0.0, 0.0, 1.0), ior); + if (dot(d2, d2) < 1e-8) + { + pos = qb; + d = reflect(d, float3(0.0, 0.0, 1.0)); + continue; + } + acc += thr * dot(prism_screen(qb, d2, nz, hz, tip, unit, f, layer, texFrame), mask); + thr = 0.0; + break; + } + if (hh < 0) + { + break; + } + pos += d * th; + float4 rn = PRISM_TRIS[4 * hh + 3]; + if (fmod(PRISM_TRIS[4 * hh].w, 2.0) > 0.5 && pos.z < 0.0) + { + acc += thr * dot(s_lin(0.03, 0.06, 0.37) * 0.35, mask); + thr = 0.0; + break; + } + float3 dout = refract(d, -rn.xyz, ior); + if (dot(dout, dout) < 1e-8) + { + d = reflect(d, -rn.xyz); + continue; + } + float fi = prism_fresnel(saturate(dot(dout, rn.xyz)), ior); + acc += thr * (1.0 - fi) * dot(model_env(dout, 0.05, l, fill), mask); + thr *= fi; + d = reflect(d, -rn.xyz); + } + trans += acc * mask; + } + float3 col = F * model_env(reflect(rd, n), 0.05, l, fill) + (1.0 - F) * trans + facet * PRISM_GLOW; + float fold = 1.0 - smoothstep(0.2, 1.0, prism_edge_px(p, h, px)); + col = mix(col, s_lin(0.96, 0.99, 1.0) * 1.3, PRISM_FOLD * fold * (0.35 + 0.65 * saturate(dot(n, l)))); + return model_tonemap(col); +} + +// ---- Le verre de Glass Lens ---- (cf. HLSL) + +// Ce que le verre ajoute à ce qu'il transmet : Fresnel, l'éclat de la lampe, le liseré de lumière. +static float3 glass_light(float3 trans, float3 n, float3 rd, float3 L, float fall, float3 l, float3 fill) +{ + float c = saturate(-dot(rd, n)); + float F = prism_fresnel(c, GLASS_IOR); + float ndh = saturate(dot(n, normalize(L - rd))); + float ndl = dot(n, L); + float3 key = s_lin(1.0, 0.97, 0.93) * 2.1 * fall; + float spe = (pow(ndh, 900.0) * 5.0 + pow(ndh, 40.0) * 0.12) * saturate(ndl * 4.0); + float edge = pow(1.0 - c, 4.0) * (0.25 + 0.75 * saturate(ndl + 0.3)); + float3 col = (1.0 - F) * (trans * s_lin(0.97, 0.985, 1.0) + GLASS_LIFT) + F * model_env(reflect(rd, n), 0.03, l, fill); + return col + key * spe + s_lin(0.92, 0.96, 1.0) * edge * 0.6; +} + +// L'image en `uv`, moyennée le long de la réfraction depuis `uv0` (cf. HLSL). +static float3 glass_smear(float2 uv, float2 uv0, texture2d texFrame) +{ + float2 s = all(uv0 >= 0.0) ? (uv - uv0) * GLASS_SMEAR : float2(0.0); + return (screen_at(uv, texFrame) + screen_at(uv - s, texFrame) + screen_at(uv + s, texFrame)) / 3.0; +} + +static float3 glass_shade(float3 q, float3 n, float3 rd, float3 L, float fall, float3 l, float3 fill, float3 nz, + float hz, float3 tip, float unit, ModelFrame f, constant Layer &layer, + texture2d texFrame) +{ + float zb = -model_thick(layer); + float2 uv0 = screen_uv(q, rd, nz, hz, tip, unit, f, layer); + float3 trans = float3(0.0); + for (int ch = 0; ch < 3; ch++) + { + float3 mask = float3(float(ch == 0), float(ch == 1), float(ch == 2)); + float ior = GLASS_IOR + GLASS_DISPERSION * float(ch - 1); + float3 d = refract(rd, n, 1.0 / ior); + float3 seen = model_env(d, 0.05, l, fill); + if (d.z < -1e-4) + { + float3 qb = q + d * ((zb - q.z) / d.z); + float3 d2 = refract(d, float3(0.0, 0.0, 1.0), ior); + seen = dot(d2, d2) < 1e-8 + ? model_env(reflect(d, float3(0.0, 0.0, 1.0)), 0.05, l, fill) + : glass_smear(screen_uv(qb, d2, nz, hz, tip, unit, f, layer), uv0, texFrame); + } + trans += seen * mask; + } + return pow(saturate(glass_light(trans, n, rd, L, fall, l, fill)), float3(1.0 / 2.2)); +} + // Les passes de la boucle unique de `cursor_model` (cf. HLSL : un seul appel de `model_eval`). constant int STAGE_MARCH = 0; constant int STAGE_NORMAL = 1; constant int STAGE_AO = 2; constant int STAGE_SELF = 3; constant int STAGE_PLANE = 4; -constant int STAGE_DONE = 5; +constant int STAGE_EDGE = 5; +constant int STAGE_SHADE = 6; +constant int STAGE_DONE = 7; + +constant float2 MODEL_SUBPIXEL[3] = { float2(0.35, 0.2), float2(-0.35, 0.2), float2(0.0, -0.4) }; inline float3 model_tetra(int k) { @@ -1226,7 +2441,8 @@ inline float3 model_tetra(int k) static float4 cursor_model(float2 local, constant Layer &layer, texture2d texSdf, - texture2d texImg) + texture2d texImg, + texture2d texFrame) { ModelFrame f; f.c = cos(layer.fx.xyz); @@ -1253,15 +2469,21 @@ static float4 cursor_model(float2 local, constant Layer &layer, float stride = sculpt_id(layer) > 0 ? 0.85 : 1.0; float3 lamp = float3(layer.color.rg + sprite_size(layer) * 0.5, 0.0) + l * SCULPT_LAMP_DIST; - float2 tb = ray_box(ro, rd, lo - 0.02, hi + 0.02); + float3 ray = rd; + float2 tb = ray_box(ro, ray, lo - 0.02, hi + 0.02); int stage = tb.x < tb.y && tb.y > 0.0 ? STAGE_MARCH : STAGE_DONE; bool plane_next = stage == STAGE_DONE; + bool need_tc = stage == STAGE_MARCH; + float tc = 1e9; + int hc = -1; int k = 0; float t = max(tb.x, 0.0); float best = 1e9; float t_best = t; + float d_best = 0.0; float mat = 0.0; float cov = 0.0; + float cov_s = 0.0; float3 q = float3(0.0); float3 n = float3(0.0); float3 L = float3(0.0); @@ -1273,12 +2495,20 @@ static float4 cursor_model(float2 local, constant Layer &layer, float inside = 0.0; float contact = 0.0; float dropped = 0.0; - for (int it = 0; it < 180; it++) + int sub = 0; + int shaded = 0; + float4 acc = float4(0.0); + for (int it = 0; it < 500; it++) { + if (need_tc) + { + need_tc = false; + tc = prism_primary(ro, ray, hc, layer); + } if (plane_next) { plane_next = false; - stage = STAGE_DONE; + stage = cov > 0.0 ? STAGE_SHADE : STAGE_DONE; float denom = dot(rd, nz); if (cov < 1.0 && denom < -1e-4) { @@ -1293,18 +2523,68 @@ static float4 cursor_model(float2 local, constant Layer &layer, } } } + if (stage == STAGE_SHADE) + { + if (cov_s > 0.0) + { + float3 c; + float3 tl = lamp - q; + float fall = SCULPT_LAMP_DIST * SCULPT_LAMP_DIST / dot(tl, tl); + if (abs(mat - PRISM_MAT) < 0.5) + { + c = prism_shade(q, n, ray, hc, t_best / dlen, l, fill, nz, hz, tip, unit, f, layer, texFrame); + } + else if (mat > GLASS_MAT - 0.5) + { + c = glass_shade(q, n, ray, normalize(tl), fall, l, fill, nz, hz, tip, unit, f, layer, texFrame); + } + else + { + c = model_shade(q, n, ray, normalize(tl), fall, sh, ao, mat, l, fill, layer, texSdf, texImg); + } + acc += float4(c * cov_s, cov_s); + } + shaded++; + stage = STAGE_DONE; + if (sub > 0) + { + float3 dws = float3(local + MODEL_SUBPIXEL[3 - sub] + layer.src.xy, -persp); + sub--; + dlen = length(dws); + ray = plane_to_model(world_to_plane(dws / dlen, f), f); + tb = ray_box(ro, ray, lo - 0.02, hi + 0.02); + cov_s = 0.0; + stage = STAGE_SHADE; + if (tb.x < tb.y && tb.y > 0.0) + { + stage = STAGE_MARCH; + k = 0; + t = max(tb.x, 0.0); + best = 1e9; + t_best = t; + need_tc = true; + } + } + continue; + } float3 pos; if (stage == STAGE_MARCH) { - pos = ro + rd * t; + pos = ro + ray * t; } else if (stage == STAGE_NORMAL) { pos = q + 0.002 * model_tetra(k); } + else if (stage == STAGE_EDGE) + { + float3 side = normalize(cross(n, ray)); + float3 e = (k >= 2 ? cross(n, side) : side) * (k % 2 == 1 ? -1.0 : 1.0); + pos = q + e * 0.5 * t_best / dlen; + } else if (stage == STAGE_AO) { - pos = q + (0.01 + 0.0175 * float(k)) * SCULPT_SCALE * n; + pos = q + (0.01 + 0.0175 * k) * SCULPT_SCALE * n; } else if (stage == STAGE_SELF) { @@ -1328,22 +2608,64 @@ static float4 cursor_model(float2 local, constant Layer &layer, { best = hit ? 0.0 : d / fp; t_best = t; + d_best = d; mat = m.y; } t += d * stride; k++; + if (!hit && t >= tc) + { + hit = true; + best = 0.0; + t_best = tc; + d_best = 0.0; + mat = PRISM_MAT; + } if (hit || t > tb.y || k == 96) { - cov = saturate(1.0 - best); - if (cov > 0.0) + cov_s = saturate(1.0 - best); + if (shaded > 0) { - q = ro + rd * t_best; - stage = STAGE_NORMAL; - k = 0; + stage = STAGE_SHADE; + if (cov_s > 0.0) + { + q = ro + ray * t_best; + n = float3(0.0); + stage = STAGE_NORMAL; + k = 0; + } } else { - plane_next = true; + cov = cov_s; + if (cov > 0.0) + { + q = ro + ray * t_best; + stage = STAGE_NORMAL; + k = 0; + } + else + { + plane_next = true; + } + } + if (stage == STAGE_NORMAL && abs(mat - PRISM_MAT) < 0.5) + { + n = prism_normal(hc, ray); + k = 0; + if (shaded > 0) + { + stage = STAGE_SHADE; + } + else + { + if (prism_edge_px(q, hc, t_best / dlen) < 0.75) + { + sub = 3; + } + stage = STAGE_DONE; + plane_next = true; + } } } } @@ -1354,6 +2676,31 @@ static float4 cursor_model(float2 local, constant Layer &layer, if (k == 4) { n = normalize(n); + k = 0; + stage = shaded > 0 ? STAGE_SHADE : STAGE_EDGE; + } + } + else if (stage == STAGE_EDGE) + { + if (m.y != mat || abs(d - d_best) > 0.02 * t_best / dlen) + { + sub = 3; + } + if (k == 0 && mat > 7.5 && mat < PRISM_MAT + 0.5) + { + int ps = prism_shape(layer); + if (ps >= 0) + { + float dout = prism_poly_dist(q.xy, PRISM_OUT_START[ps], PRISM_OUT_COUNT[ps]); + if (abs(dout) < 0.75 * t_best / dlen) + { + sub = 3; + } + } + } + k++; + if (k == 4) + { stage = STAGE_AO; k = 0; ao = 0.0; @@ -1361,7 +2708,7 @@ static float4 cursor_model(float2 local, constant Layer &layer, } else if (stage == STAGE_AO) { - ao += ((0.01 + 0.0175 * float(k)) * SCULPT_SCALE - d) * pow(0.85, float(k)); + ao += ((0.01 + 0.0175 * k) * SCULPT_SCALE - d) * pow(0.85, float(k)); k++; if (k == 5) { @@ -1402,7 +2749,7 @@ static float4 cursor_model(float2 local, constant Layer &layer, k = 1; if (!(tbp.x < tbp.y && tbp.y > 0.0)) { - stage = STAGE_DONE; + stage = cov > 0.0 ? STAGE_SHADE : STAGE_DONE; } } else @@ -1414,21 +2761,86 @@ static float4 cursor_model(float2 local, constant Layer &layer, { res = saturate(res); dropped = 1.0 - res * res * (3.0 - 2.0 * res); - stage = STAGE_DONE; + stage = cov > 0.0 ? STAGE_SHADE : STAGE_DONE; } } } - float3 rgb = float3(0.0); - if (cov > 0.0) + float w = 1.0 / max(shaded, 1); + // La lentille de Glass Lens laisse passer la lumière : son ombre est plus claire. + float clear = sculpt_id(layer) > 10 ? GLASS_SHADOW : 1.0; + float shadow = inside * clear * max(dropped * MODEL_SHADOW_ALPHA, contact * MODEL_CONTACT_ALPHA); + float a = acc.a * w * layer.color.a; + return float4(acc.rgb * w * layer.color.a, a + (1.0 - a) * shadow * layer.color.a); // prémultiplié, ombre noire +} + +// ============ Verre à plat (mode 19) ============ (cf. HLSL, `cursor_glass`) +static float4 cursor_glass(float2 local, float2 pout, constant Layer &layer, + texture2d texFrame) +{ + if (pout.x < layer.dst_prev.x || pout.x > layer.dst_prev.x + layer.dst_prev.z || + pout.y < layer.dst_prev.y || pout.y > layer.dst_prev.y + layer.dst_prev.w) + { + return float4(0.0); + } + float3 r = quad_inverse(local, layer.fx.xy, layer.fx.zw, layer.src_prev.xy, layer.src_prev.zw, layer.mb.x); + if (r.z < 0.5) + { + return float4(0.0); + } + float2 m = layer.color.rg + saturate(r.xy) * sprite_size(layer); + float2 p = float2(m.x, -m.y) / SCULPT_SCALE; + float px = 1.0 / max(layer.color.b * SCULPT_SCALE, 1e-3); + int shape = (sculpt_id(layer) - 1) % 2; + float2 g = float2(0.0); + float2 gd = float2(0.0); + float e = 0.25 * px; + for (int k = 0; k < 3; k++) + { + float2 v = s_glass2(p + (k == 1 ? float2(e, 0.0) : (k == 2 ? float2(0.0, e) : float2(0.0))), shape); + if (k == 0) g = v; + else if (k == 1) gd.x = v.y; + else gd.y = v.y; + } + float cov = saturate(0.5 - g.x / px); + if (cov <= 0.0) + { + return float4(0.0); + } + float2 out_dir = (gd - g.y) / e; + out_dir /= max(length(out_dir), 1e-4); + float2 relief = s_glass_relief(g.y); + float2 dir = float2(out_dir.x, -out_dir.y); + float3 n = normalize(float3(-relief.y * dir, 1.0)); + float3 rd = float3(0.0, 0.0, -1.0); + bool rim = g.y > GLASS_LINE; + float depth = rim ? GLASS_RIM - GLASS_LINE : GLASS_TOP - SCULPT_HOVER + relief.x; + float3 trans = float3(0.0); + for (int ch = 0; ch < 3; ch++) { - float3 tl = lamp - q; - float fall = SCULPT_LAMP_DIST * SCULPT_LAMP_DIST / dot(tl, tl); - rgb = model_shade(q, n, rd, normalize(tl), fall, sh, ao, mat, l, fill, layer, texSdf, texImg); + float3 mask = float3(float(ch == 0), float(ch == 1), float(ch == 2)); + float ior = GLASS_IOR + GLASS_DISPERSION * float(ch - 1); + float3 d = refract(rd, n, 1.0 / ior); + float3 d2 = refract(d, float3(0.0, 0.0, 1.0), ior); + float2 off = d.xy * depth / max(-d.z, 0.05); + if (!rim && dot(d2, d2) > 1e-8) + { + off += d2.xy * GLASS_GAP_FLAT / max(-d2.z, 0.05); + } + off *= SCULPT_SCALE * layer.src.xy; + trans += glass_smear(pout + off, pout, texFrame) * mask; } - float shadow = inside * max(dropped * MODEL_SHADOW_ALPHA, contact * MODEL_CONTACT_ALPHA); + float3 col = glass_light(trans, n, rd, MODEL_LIGHT, 1.0, MODEL_LIGHT, MODEL_FILL); + float edge_k = saturate(1.0 - min(g.y, GLASS_LINE - g.y) / (0.25 * GLASS_LINE)); + float side = g.y < 0.5 * GLASS_LINE ? -1.0 : 1.0; + float3 nl = normalize(float3(side * edge_k * dir, 1.0)); + float3 hl = normalize(MODEL_LIGHT + float3(0.0, 0.0, 1.0)); + float3 line_col = s_lin(0.05, 0.055, 0.07) * (0.6 + 0.5 * saturate(dot(nl, MODEL_LIGHT))) + + s_lin(1.0, 0.97, 0.93) * 0.8 * edge_k * pow(saturate(dot(nl, hl)), 40.0); + float in_line = saturate(0.5 + g.y / px) * saturate(0.5 - (g.y - GLASS_LINE) / px); + col = mix(col, line_col, in_line); float a = cov * layer.color.a; - return float4(rgb * a, a + (1.0 - a) * shadow * layer.color.a); // prémultiplié, ombre noire + return float4(pow(saturate(col), float3(1.0 / 2.2)) * a, a); // prémultiplié } // ============ Impact du clic (mode 16) ============ @@ -2026,6 +3438,17 @@ fragment float4 ps_main(VSOut i [[stage_in]], // par le repli du mode 18, qui garde texture(2) pour son rendu isolé. texture2d texDof [[texture(5)]]) { + // mode 19 : VERRE À PLAT de Glass Lens (`cursor_glass`). Testé avant le mode 18, dont la + // branche n'a pas de borne haute. C'est un modèle, compilé avec les modes 15 à 17. + if (layer.mode > 18.5) + { +#ifdef LAYER_MODELS + return cursor_glass(i.local, i.pout, layer, texDof); +#else + return float4(0.0); +#endif + } + // mode 18 : l'écran CADRÉ (ombre, cadre, métrage, appareil) flouté comme UN objet rigide // (`FrameGeometry::screen_trail`), port 1:1 du HLSL. texImg = son rendu isolé, prémultiplié, // à la taille de la sortie. À plat, sa boîte va de `dst_prev` (frame précédente) à `fx` @@ -2082,6 +3505,7 @@ fragment float4 ps_main(VSOut i [[stage_in]], return acc / max(n, 1.0); } +#ifdef LAYER_MODELS // mode 17 : CADRE D'APPAREIL MODELÉ (`device_frame`). Testé en premier, comme le 16. if (layer.mode > 16.5) { @@ -2103,8 +3527,16 @@ fragment float4 ps_main(VSOut i [[stage_in]], { return float4(0.0, 0.0, 0.0, 0.0); } - return cursor_model(i.local, layer, texSdf, texImg); + return cursor_model(i.local, layer, texSdf, texImg, texDof); } +#else + // Les modèles 3D n'existent que dans la variante `LAYER_MODELS` ; un draw qui les enverrait + // ici ne peint rien plutôt qu'une ombre faite de leurs emplacements. + if (layer.mode > 14.5) + { + return float4(0.0, 0.0, 0.0, 0.0); + } +#endif // mode 14 : CADRE DE FENÊTRE autour de l'écran, dessiné SOUS lui. Cf. commentaires HLSL. // Testé avant le mode 13, dont la branche n'a pas de borne haute. @@ -2248,8 +3680,12 @@ fragment float4 ps_main(VSOut i [[stage_in]], float2 plane_px = layer.dst_prev.xy; float2 p = float2(r.x, r.y) * plane_px - plane_px * 0.5; float rad = max(layer.radius_px, 0.0); - float d = (layer.dst_prev.z > 0.5) ? sd_screen_under_bar(p, plane_px * 0.5, rad, layer.color.z) - : sd_round_rect(p, plane_px * 0.5, rad); + // Une branche et non `?:`, pour ne jamais évaluer les DEUX distances (trois SDF par pixel). + float d; + if (layer.dst_prev.z > 0.5) + d = sd_screen_under_bar(p, plane_px * 0.5, rad, layer.color.z); + else + d = sd_round_rect(p, plane_px * 0.5, rad); float tilt_a = 1.0 - smoothstep(0.0, 1.5, d); // Slot d'un layout en bloc (`color.w` = rayon de ses coins, px ; 0 ailleurs, sans effet) : // `dst` EST le slot, dont le rect arrondi rogne le plan (`ScreenMask`, cf. HLSL). @@ -2562,8 +3998,13 @@ fragment float4 ps_main(VSOut i [[stage_in]], // cadre (`sd_screen_under_bar`, `mb.z` = la remontée du contour intérieur, px). float2 halfsz = layer.quad_px * 0.5; float2 p = i.local - layer.quad_px * 0.5; - float d = (layer.mb.w > 0.5) ? sd_screen_under_bar(p, halfsz, layer.radius_px, layer.mb.z) - : sd_round_rect(p, halfsz, layer.radius_px); + // Une branche et non `?:`, pour ne jamais évaluer les DEUX distances : trois SDF par pixel + // de l'écran au lieu d'une, 0,6 ms par frame 1080p sur une Radeon 610M (mesuré en WGSL). + float d; + if (layer.mb.w > 0.5) + d = sd_screen_under_bar(p, halfsz, layer.radius_px, layer.mb.z); + else + d = sd_round_rect(p, halfsz, layer.radius_px); alpha *= 1.0 - smoothstep(0.0, 1.5, d); } return float4(rgb * alpha, alpha); diff --git a/crates/compositor/src/shared_frames.rs b/crates/compositor/src/shared_frames.rs new file mode 100644 index 0000000000..a6b4f8415a --- /dev/null +++ b/crates/compositor/src/shared_frames.rs @@ -0,0 +1,338 @@ +//! Livraison de la preview par textures partagées : le thread de rendu copie chaque frame +//! composée dans l'une des textures d'un petit anneau, et Electron l'importe côté GPU +//! (`sharedTexture.importSharedTexture`) au lieu d'en recevoir les pixels. Plus de `Map` +//! qui attend le GPU puis recopie l'image en RAM, plus de `Vec` de plusieurs Mo par frame +//! à travers l'IPC : mesuré, ce transport coûtait à lui seul 37 à 55 % du thread principal +//! du renderer à 30 images/s. +//! +//! Deux moitiés : +//! - `SlotBook`, portable et sans GPU : quelle case porte la frame prête, lesquelles +//! Chromium tient encore. Partagé entre le thread de rendu (`claim`, `publish`) et le +//! thread Node (`take`, `release`). +//! - `SharedRing` (Windows, backend matériel) : les textures D3D11 et leurs handles NT. +//! macOS (IOSurface) et Linux (dmabuf) restent sur le readback. + +use crate::frame_geometry::FootageQuad; +use crate::live::FramePosition; +use std::time::{Duration, Instant}; + +/// Cases de l'anneau : une prête, une en transit vers le renderer (tenue jusqu'à ce que +/// Chromium la relâche), une en écriture, et une de marge pour la latence de la libération. +pub const RING_SLOTS: u32 = 4; + +/// Une case tenue plus longtemps que ça a perdu sa libération (renderer rechargé, import +/// échoué en route) : Chromium relâche les autres dans la milliseconde. Même ordre que le +/// délai d'Electron sur `sendSharedTexture` (1 s). +pub const LOST_SLOT_AFTER: Duration = Duration::from_secs(1); + +/// Une frame composée posée dans une case de l'anneau, telle que JS la reçoit. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct SharedFrame { + /// Même compteur que les frames lues en RAM : le consommateur ne voit qu'une suite de + /// générations, quel que soit le transport. + pub gen: u64, + pub slot: u32, + /// Valeur du handle NT de la texture de la case, valable dans CE processus. Electron le + /// duplique à l'import : la case garde le sien. + pub handle: u64, + pub width: u32, + pub height: u32, + pub footage: Option, + pub position: FramePosition, +} + +/// Une case prise par JS : la génération qu'elle porte et quand elle est partie. +#[derive(Debug, Clone, Copy)] +struct Held { + slot: u32, + gen: u64, + since: Instant, +} + +/// Qui tient quelle case de l'anneau. +#[derive(Debug, Default)] +pub struct SlotBook { + /// La dernière frame publiée, que JS n'a pas encore prise. + ready: Option, + /// Les cases prises par JS, que Chromium n'a pas encore relâchées, plus anciennes d'abord. + held: Vec, +} + +impl SlotBook { + /// La case où écrire la prochaine frame. Une case libre d'abord ; à défaut, celle de la + /// frame prête que personne n'a prise, retirée puisqu'elle est périmée. `None` quand + /// Chromium tient toutes les cases : une case tenue n'est jamais réécrite, même perdue + /// (`lost`), puisqu'il peut encore la lire. + pub fn claim(&mut self, slots: u32) -> Option { + let ready = self.ready.map(|frame| frame.slot); + let is_held = |slot: u32| self.held.iter().any(|held| held.slot == slot); + if let Some(free) = (0..slots).find(|slot| !is_held(*slot) && Some(*slot) != ready) { + return Some(free); + } + if let Some(stale) = ready { + self.ready = None; + return Some(stale); + } + None + } + + /// Une case est tenue depuis `LOST_SLOT_AFTER` : sa libération ne viendra plus. + pub fn lost(&self, now: Instant) -> bool { + self.held.first().is_some_and(|held| now.duration_since(held.since) >= LOST_SLOT_AFTER) + } + + pub fn publish(&mut self, frame: SharedFrame) { + self.ready = Some(frame); + } + + /// La frame prête, si elle est plus récente que `since_gen`. Sa case reste tenue jusqu'à + /// `release` : le thread de rendu n'y écrira plus d'ici là. + pub fn take(&mut self, since_gen: u64, now: Instant) -> Option { + let frame = self.ready.filter(|frame| frame.gen > since_gen)?; + self.ready = None; + self.held.push(Held { slot: frame.slot, gen: frame.gen, since: now }); + Some(frame) + } + + /// Chromium a relâché la frame `gen` de la case `slot`. La génération évite qu'une + /// libération en double ne libère la frame suivante posée dans la même case. + pub fn release(&mut self, slot: u32, gen: u64) { + self.held.retain(|held| held.slot != slot || held.gen != gen); + } + + /// Oublie la frame prête : la vue repasse au readback, personne ne viendra la prendre. + pub fn clear_ready(&mut self) { + self.ready = None; + } +} + +#[cfg(windows)] +pub use ring::SharedRing; + +#[cfg(windows)] +mod ring { + use super::RING_SLOTS; + use crate::d3d::Gpu; + use anyhow::{bail, Result}; + use std::time::{Duration, Instant}; + use windows::core::{Interface, PCWSTR}; + use windows::Win32::Foundation::{CloseHandle, BOOL, HANDLE}; + use windows::Win32::Graphics::Direct3D11::{ + ID3D11Device, ID3D11DeviceContext, ID3D11Query, ID3D11Texture2D, + D3D11_ASYNC_GETDATA_DONOTFLUSH, D3D11_BIND_RENDER_TARGET, D3D11_BIND_SHADER_RESOURCE, + D3D11_QUERY_DESC, D3D11_QUERY_EVENT, D3D11_RESOURCE_MISC_SHARED, + D3D11_RESOURCE_MISC_SHARED_NTHANDLE, D3D11_TEXTURE2D_DESC, D3D11_USAGE_DEFAULT, + }; + use windows::Win32::Graphics::Dxgi::Common::{DXGI_FORMAT_R8G8B8A8_UNORM, DXGI_SAMPLE_DESC}; + use windows::Win32::Graphics::Dxgi::{ + IDXGIResource1, DXGI_SHARED_RESOURCE_READ, DXGI_SHARED_RESOURCE_WRITE, + }; + + /// Au-delà, le GPU est bloqué ou perdu : on rend une erreur plutôt que de geler la vue. + const COPY_TIMEOUT: Duration = Duration::from_millis(500); + + struct SlotTexture { + tex: ID3D11Texture2D, + handle: HANDLE, + width: u32, + height: u32, + } + + impl Drop for SlotTexture { + fn drop(&mut self) { + // Electron importe un duplicata du handle : fermer le nôtre ne retire rien à une + // frame que Chromium affiche encore. + unsafe { + let _ = CloseHandle(self.handle); + } + } + } + + /// Les textures partagées d'une vue live, créées à la demande à la taille du rendu. + pub struct SharedRing { + device: ID3D11Device, + context: ID3D11DeviceContext, + copied: ID3D11Query, + slots: Vec>, + } + + impl SharedRing { + pub fn new(gpu: &Gpu) -> Result { + let desc = D3D11_QUERY_DESC { Query: D3D11_QUERY_EVENT, MiscFlags: 0 }; + let mut copied = None; + unsafe { gpu.device.CreateQuery(&desc, Some(&mut copied))? }; + Ok(SharedRing { + device: gpu.device.clone(), + context: gpu.context.clone(), + copied: copied.expect("CreateQuery a réussi sans requête"), + slots: (0..RING_SLOTS).map(|_| None).collect(), + }) + } + + /// Copie `src` (`width`×`height`, RGBA8) dans la case `slot` et rend le handle de sa + /// texture une fois le GPU arrivé au bout de la copie. Chromium la lit depuis son propre + /// device, sans keyed mutex : quand le handle part, plus rien ne doit y écrire. L'attente + /// est celle que faisait déjà le `Map` du readback, la recopie en RAM en moins. + /// + /// La case ne doit être ni prête ni tenue (`SlotBook::claim`) : sa texture peut être + /// recréée ici quand la taille du rendu a changé. + pub unsafe fn write( + &mut self, + slot: u32, + src: &ID3D11Texture2D, + width: u32, + height: u32, + ) -> Result { + let entry = &mut self.slots[slot as usize]; + if !matches!(entry, Some(texture) if texture.width == width && texture.height == height) { + *entry = Some(create_slot(&self.device, width, height)?); + } + let target = entry.as_ref().expect("case créée juste au-dessus"); + self.context.CopyResource(&target.tex, src); + self.context.End(&self.copied); + self.context.Flush(); + let deadline = Instant::now() + COPY_TIMEOUT; + loop { + let mut done = BOOL(0); + self.context.GetData( + &self.copied, + Some(&mut done as *mut BOOL as *mut core::ffi::c_void), + std::mem::size_of::() as u32, + D3D11_ASYNC_GETDATA_DONOTFLUSH.0 as u32, + )?; + if done.as_bool() { + break; + } + if Instant::now() > deadline { + bail!("le GPU n'a pas fini de copier la frame en {COPY_TIMEOUT:?}"); + } + std::thread::yield_now(); + } + Ok(target.handle.0 as u64) + } + } + + unsafe fn create_slot(device: &ID3D11Device, width: u32, height: u32) -> Result { + let desc = D3D11_TEXTURE2D_DESC { + Width: width, + Height: height, + MipLevels: 1, + ArraySize: 1, + // Le format du RT composé : la copie est un `CopyResource`, sans conversion. + Format: DXGI_FORMAT_R8G8B8A8_UNORM, + SampleDesc: DXGI_SAMPLE_DESC { Count: 1, Quality: 0 }, + Usage: D3D11_USAGE_DEFAULT, + BindFlags: (D3D11_BIND_RENDER_TARGET.0 | D3D11_BIND_SHADER_RESOURCE.0) as u32, + CPUAccessFlags: 0, + // Handle NT, le seul qu'Electron importe, sans keyed mutex : `write` attend la fin + // de la copie avant de le livrer. + MiscFlags: (D3D11_RESOURCE_MISC_SHARED.0 | D3D11_RESOURCE_MISC_SHARED_NTHANDLE.0) as u32, + }; + let mut tex = None; + device.CreateTexture2D(&desc, None, Some(&mut tex))?; + let tex = tex.expect("CreateTexture2D a réussi sans texture"); + let resource: IDXGIResource1 = tex.cast()?; + let handle = resource.CreateSharedHandle( + None, + DXGI_SHARED_RESOURCE_READ.0 | DXGI_SHARED_RESOURCE_WRITE.0, + PCWSTR::null(), + )?; + Ok(SlotTexture { tex, handle, width, height }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn frame(gen: u64, slot: u32) -> SharedFrame { + SharedFrame { + gen, + slot, + handle: 0, + width: 2, + height: 2, + footage: None, + position: FramePosition { clip_index: 0, source_time_sec: 0.0 }, + } + } + + /// Toutes les cases prises par JS, la case `n` portant la génération `n + 1`. + fn all_held(now: Instant) -> SlotBook { + let mut book = SlotBook::default(); + for slot in 0..RING_SLOTS { + book.publish(frame(u64::from(slot) + 1, slot)); + assert!(book.take(u64::from(slot), now).is_some()); + } + book + } + + #[test] + fn claims_a_free_slot_before_a_ready_one() { + let now = Instant::now(); + let mut book = SlotBook::default(); + book.publish(frame(1, 0)); + assert_eq!(book.claim(RING_SLOTS), Some(1)); + // La frame prête n'a pas bougé : JS peut toujours la prendre. + assert_eq!(book.take(0, now), Some(frame(1, 0))); + } + + #[test] + fn a_taken_slot_is_never_written_until_released() { + let now = Instant::now(); + let mut book = all_held(now); + assert_eq!(book.claim(RING_SLOTS), None, "Chromium tient toutes les cases"); + book.release(2, 3); + assert_eq!(book.claim(RING_SLOTS), Some(2)); + } + + #[test] + fn a_frame_nobody_took_gives_its_slot_back_when_nothing_else_is_free() { + let now = Instant::now(); + let mut book = SlotBook::default(); + for slot in 0..RING_SLOTS - 1 { + book.publish(frame(u64::from(slot) + 1, slot)); + book.take(u64::from(slot), now); + } + book.publish(frame(9, 3)); + assert_eq!(book.claim(RING_SLOTS), Some(3)); + // Retirée : la livrer maintenant, pendant qu'on la réécrit, donnerait une image déchirée. + assert_eq!(book.take(0, now), None); + } + + #[test] + fn take_hands_out_only_newer_generations() { + let now = Instant::now(); + let mut book = SlotBook::default(); + book.publish(frame(5, 1)); + assert_eq!(book.take(5, now), None); + assert_eq!(book.take(4, now), Some(frame(5, 1))); + assert_eq!(book.take(0, now), None, "déjà prise"); + } + + #[test] + fn a_release_only_frees_the_generation_it_names() { + let now = Instant::now(); + let mut book = SlotBook::default(); + book.publish(frame(1, 0)); + book.take(0, now); + book.release(3, 1); + book.release(0, 7); + assert_eq!(book.claim(1), None, "ni la case 3 ni la génération 7 ne sont tenues"); + book.release(0, 1); + assert_eq!(book.claim(1), Some(0)); + } + + #[test] + fn a_slot_whose_release_never_came_is_reported_lost_but_never_rewritten() { + let start = Instant::now(); + let mut book = all_held(start); + assert!(!book.lost(start + LOST_SLOT_AFTER / 2)); + assert!(book.lost(start + LOST_SLOT_AFTER)); + // Chromium peut encore lire une case perdue : la réécrire déchirerait son image. + assert_eq!(book.claim(RING_SLOTS), None); + // Si sa libération finit par venir, la case sert de nouveau. + book.release(0, 1); + assert_eq!(book.claim(RING_SLOTS), Some(0)); + } +} diff --git a/crates/compositor/src/timeline_walk.rs b/crates/compositor/src/timeline_walk.rs index a16271c8b0..d5dab66175 100644 --- a/crates/compositor/src/timeline_walk.rs +++ b/crates/compositor/src/timeline_walk.rs @@ -19,6 +19,7 @@ use crate::cursor::CursorTrack; use crate::d3d::Gpu; use crate::ffi::AVFrame; use crate::frame_geometry::webcam_is_real; +use crate::live::cursor_key; use crate::pipeline::{ClipSource, Decoder}; use crate::regions::{speed_segments_for_window, SpeedSegment}; use crate::scene::Scene; @@ -268,6 +269,8 @@ pub(crate) unsafe fn walk_composited_timeline( source_end_sec, out_fps as f64, ); + // A turned clip is planned in what the viewer sees: its cursor track is the mirrored one. + let turned = clip_scene.as_ref().is_some_and(|s| s.screen_turned()); if clip_scene.is_some() { comp.set_scene(clip_scene); } @@ -285,18 +288,21 @@ pub(crate) unsafe fn walk_composited_timeline( } if cursor_enabled { - if !cursor_tracks.contains_key(&clip.screen) { + // Keyed by file AND turn: the same file turned is another track. + let key = cursor_key(&clip.screen, turned); + if !cursor_tracks.contains_key(&key) { let path = format!("{}.cursor.json", clip.screen); if let Ok(raw) = CursorTrack::load(&path, 0.0, 24.0 * 3600.0) { - cursor_tracks.insert(clip.screen.clone(), raw.smoothed(cursor_smoothing)); + let raw = if turned { raw.turned() } else { raw }; + cursor_tracks.insert(key.clone(), raw.smoothed(cursor_smoothing)); } // absente/illisible → pas d'entrée : ce clip s'exporte sans curseur (visible, // pas masqué en un curseur fantôme d'un autre clip). } - if cursor_active_path.as_deref() != Some(clip.screen.as_str()) { - if let Some(track) = cursor_tracks.get(&clip.screen) { + if cursor_active_path.as_deref() != Some(key.as_str()) { + if let Some(track) = cursor_tracks.get(&key) { comp.set_cursor(track.clone()); - cursor_active_path = Some(clip.screen.clone()); + cursor_active_path = Some(key); } else { comp.clear_cursor(); comp.set_cursor_time(None); diff --git a/crates/compositor/src/vk_shaders/layer.wgsl b/crates/compositor/src/vk_shaders/layer.wgsl index 2eb46a9b30..48902d4209 100644 --- a/crates/compositor/src/vk_shaders/layer.wgsl +++ b/crates/compositor/src/vk_shaders/layer.wgsl @@ -16,6 +16,15 @@ // // Le rendu est en alpha PRÉMULTIPLIÉ (cf. commentaire HLSL), convention qu'on // retrouve dans tous les autres modes du compositeur (texte, curseur, ombre). +// +// `LAYER_MODELS` n'est PAS declare ici : `compositor_linux.rs` (`layer_source`) le prefixe et +// compile ce source deux fois. Un shader unique alloue a CHAQUE calque les registres de sa +// branche la plus lourde, et ce sont les modeles 3D ray-traces (modes 15 a 17) : 168 VGPR sous +// ACO (Radeon 610M), donc 6 vagues par SIMD au lieu de 32 pour le fond, l'ombre et la video, qui +// couvrent toute la sortie : l'export Linux du banc etait passe de 31 a 57 s. Sans ces trois modes, +// 48 VGPR et 20 vagues. Le pipeline plat (`false`) dessine donc tout le reste, et seuls le +// curseur modelise, l'impact de son clic, l'appareil et son ombre passent par l'autre : c'est +// `LayerCB::needs_models` qui en decide, au draw (`draw_layer`), d'apres le mode du calque. struct Layer { dst: vec4, // x,y,w,h sortie 0..1 (origine haut-gauche) @@ -34,16 +43,21 @@ struct Layer { } @group(0) @binding(0) var layer: Layer; -@group(0) @binding(1) var texY: texture_2d; // R8Unorm, sample .r ; modes 7, 13 et 15 : le sprite RGBA -@group(0) @binding(2) var texU: texture_2d; // R8Unorm, sample .r ; mode 15 : le champ R16F du sprite +@group(0) @binding(1) var texY: texture_2d; // R8Unorm, sample .r ; modes 7 et 13 : le sprite RGBA ; mode 15 : la copie de l'image composee +@group(0) @binding(2) var texU: texture_2d; // R8Unorm, sample .r @group(0) @binding(3) var samp: sampler; // Masque de segmentation du sujet webcam, R8. Une vue 1x1 est liee quand aucun masque // n'existe : la branche n'est de toute facon prise que si layer.fx.z > 0.5. -// Mode 18 : ce binding porte a la place le rendu isole de l'ecran cadre. +// Mode 18 : ce binding porte a la place le rendu isole de l'ecran cadre ; mode 15, le champ R16F +// du sprite. @group(0) @binding(4) var texMask: texture_2d; // Pyramide RGBA de profondeur de champ (`tilted_sample`), lue par le mode 8 et par le repli du -// mode 18, qui garde le binding 4 pour son rendu isole : d'ou un binding a elle. Vue 1x1 ailleurs. +// mode 18, qui garde le binding 4 pour son rendu isole : d'ou un binding a elle. Mode 15 : le +// sprite RGBA. Vue 1x1 ailleurs. @group(0) @binding(6) var texDof: texture_2d; +// Le maillage de Prism Glow (mode 15), rempli de `prism_mesh.rs` : un uniform et non des tables +// dans le shader, que lavapipe recopierait a chaque pixel de chaque calque (cf. `PrismMesh`). +@group(0) @binding(7) var prism: PrismMesh; // Plafond de la profondeur de champ du mode 8, en niveau de la pyramide demi-resolution. // Meme valeur que `DOF_MAX_LOD` du HLSL. @@ -170,8 +184,12 @@ fn screen_trail(pout: vec2) -> vec4 { // garde sa largeur tout autour du coin. r <= 0 : le rectangle vif d'avant, à l'identique. fn sd_round_rect(p: vec2, halfsz: vec2, r: f32) -> f32 { let hmin = min(halfsz.x, halfsz.y); - if (r <= 0.0 || hmin <= 0.0) { - let q0 = abs(p) - halfsz; + let q0 = abs(p) - halfsz; + // Hors des coins, la distance est celle du rectangle vif quel que soit l'exposant : un seul + // axe deborde de l'etendue E, qui s'y ajoute puis s'en retranche. E ne depasse pas 1,42 r + // (n = 3) : sous 1,5 r du bord sur un axe, on rend la boite sans calculer `n` ni E, deux + // transcendantes de moins sur presque tous les pixels d'un calque arrondi. + if (r <= 0.0 || hmin <= 0.0 || min(q0.x, q0.y) <= -min(1.5 * r, hmin)) { return length(max(q0, vec2(0.0))) + min(max(q0.x, q0.y), 0.0); } let n = 3.0 - smoothstep(0.5, 1.0, r / hmin); @@ -568,9 +586,10 @@ fn blur_webcam_bg(uv: vec2, intensity: f32, qpx: vec2, local_px: vec2< // propres et occlusion, et qui porte une ombre douce et une ombre de contact sur le plan de // l'ecran. Constantes : miroir exact de `frame_geometry.rs` (MODEL_*) et de `sculpt.rs` // (SCULPT_*), emplacements du cbuffer : `cursor_model_cb`. -// Textures : le sprite RGBA (alpha droit) au binding 1 (`texY`, comme aux modes 7 et 13), son -// champ R16F au binding 2 (`texU`), sur le meme rect ; `color.rg` = coin du sprite dans le repere -// du modele, `sprite_size()` = sa taille (w/h dans `radius_px`), `color.b` = l'ecrasement au clic. +// Textures : la copie de l'image composee au binding 1 (`texY`), que refracte le cristal de Prism +// Glow ; le sprite RGBA (alpha droit) est au binding 6 (`texDof`), son champ R16F au +// binding 4 (`texMask`), sur le meme rect ; `color.rg` = coin du sprite dans le repere du modele, +// `sprite_size()` = sa taille (w/h dans `radius_px`), `color.b` = l'ecrasement au clic. const MODEL_BEVEL: f32 = 0.045; const MODEL_LIGHT = vec3(-0.4194, -0.5792, 0.6990); const MODEL_FILL = vec3(0.7557, 0.2519, 0.6046); @@ -592,7 +611,7 @@ fn sprite_size() -> vec2 { // Un texel du sprite, en unites du modele : le champ est le sprite surechantillonne x4. const CURSOR_SDF_UPSAMPLE: f32 = 4.0; fn sprite_texel() -> f32 { - let d = vec2(textureDimensions(texU)); + let d = vec2(textureDimensions(texMask)); return CURSOR_SDF_UPSAMPLE / max(d.x, d.y); } @@ -610,8 +629,6 @@ fn sculpt_id() -> i32 { const SCULPT_SCALE: f32 = 0.85; const SCULPT_HOVER: f32 = 0.05; const SCULPT_VOX: f32 = 0.0625; -const SCULPT_AR_ROUND: f32 = 0.03; -const SCULPT_HAND_ZC: f32 = 0.185; const SCULPT_ZREF_ARROW: f32 = 0.2; const SCULPT_ZREF_HAND: f32 = 0.185; const SCULPT_LAMP_DIST: f32 = 1.9; @@ -630,13 +647,6 @@ fn s_opu(a: vec2, b: vec2) -> vec2 { return select(b, a, a.x < b.x); } -fn s_capsule(p: vec3, a: vec3, b: vec3, r: f32) -> f32 { - let pa = p - a; - let ba = b - a; - let h = saturate(dot(pa, ba) / dot(ba, ba)); - return length(pa - ba * h) - r; -} - fn s_round_box(p: vec3, b: vec3, r: f32) -> f32 { let q = abs(p) - b + vec3(r); return length(max(q, vec3(0.0))) + min(max(q.x, max(q.y, q.z)), 0.0) - r; @@ -660,31 +670,6 @@ fn s_rot(v: vec2, a: f32) -> vec2 { return vec2(c * v.x - s * v.y, s * v.x + c * v.y); } -const SCULPT_ARROW = array, 7>( - vec2(0.0, 0.0), vec2(0.0, -0.86), vec2(0.215, -0.665), vec2(0.37, -1.0), - vec2(0.53, -0.93), vec2(0.38, -0.60), vec2(0.64, -0.60) -); - -fn s_arrow2(p: vec2) -> f32 { - var d = dot(p - SCULPT_ARROW[0], p - SCULPT_ARROW[0]); - var s = 1.0; - var j = 6; - for (var i = 0; i < 7; i++) { - let e = SCULPT_ARROW[j] - SCULPT_ARROW[i]; - let w = p - SCULPT_ARROW[i]; - let b = w - e * saturate(dot(w, e) / dot(e, e)); - d = min(d, dot(b, b)); - let c0 = p.y >= SCULPT_ARROW[i].y; - let c1 = p.y < SCULPT_ARROW[j].y; - let c2 = e.x * w.y > e.y * w.x; - if (c0 && c1 && c2) || (!c0 && !c1 && !c2) { - s = -s; - } - j = i; - } - return s * sqrt(d); -} - fn s_star5(p0: vec2, r: f32, rf: f32) -> f32 { let k1 = vec2(0.809016994375, -0.587785252292); let k2 = vec2(-0.809016994375, -0.587785252292); @@ -698,233 +683,477 @@ fn s_star5(p0: vec2, r: f32, rf: f32) -> f32 { return length(p - ba * h) * sign(p.y * ba.x - p.x * ba.y); } -fn s_arrow_solid(p: vec3, h: f32, re: f32, dome: f32) -> f32 { - let d2 = s_arrow2(p.xy) - SCULPT_AR_ROUND; - let hh = h + dome * smoothstep(0.0, 0.07, -d2); - return s_extrude(d2, p.z - (SCULPT_HOVER + h + dome), hh, re); +fn s_vesica(p0: vec2, r: f32, d: f32) -> f32 { + let p = abs(p0); + let b = sqrt(r * r - d * d); + return select(length(p + vec2(d, 0.0)) - r, length(p - vec2(0.0, b)), (p.y - b) * d > p.x * b); } -fn s_piping(p: vec3, ztop: f32) -> f32 { - let d2 = s_arrow2(p.xy) - SCULPT_AR_ROUND; - return length(vec2(d2 + 0.075, p.z - ztop)) - 0.017; -} +// Curseurs cercles (cf. HLSL) : Studio Ink, Pop Coral, Star Sprout. +const RIM_POLY = array, 28>( + vec2(0.0, -0.06), vec2(0.0, -0.7712), vec2(0.165, -0.6325), vec2(0.313, -0.9318), + vec2(0.4234, -0.8557), vec2(0.2672, -0.5686), vec2(0.456, -0.5393), + vec2(-0.173, -0.5237), vec2(-0.016, -0.75), vec2(0.329, -0.75), vec2(0.411, -0.592), + vec2(0.411, -0.43), vec2(0.19, -0.435), vec2(-0.03, -0.44), + vec2(-0.1825, -0.6215), vec2(0.0037, -0.9195), vec2(0.3462, -0.9195), vec2(0.467, -0.6906), + vec2(0.467, -0.49), vec2(0.2, -0.49), vec2(-0.03, -0.49), + vec2(-0.1807, -0.5534), vec2(0.0194, -0.872), vec2(0.3485, -0.872), vec2(0.47, -0.6795), + vec2(0.47, -0.49), vec2(0.2, -0.49), vec2(-0.03, -0.49) +); -fn s_glove(p: vec3, zc: f32) -> f32 { - let index = s_capsule(p, vec3(0.0, -0.10, zc), vec3(0.0, -0.52, zc), 0.098); - let palm = s_round_box(p - vec3(0.185, -0.70, zc), vec3(0.255, 0.19, 0.105), 0.1); - let f1 = s_capsule(p, vec3(0.17, -0.57, zc + 0.012), vec3(0.17, -0.41, zc + 0.045), 0.086); - let f2 = s_capsule(p, vec3(0.31, -0.59, zc + 0.01), vec3(0.31, -0.45, zc + 0.04), 0.08); - let f3 = s_capsule(p, vec3(0.435, -0.625, zc + 0.005), vec3(0.435, -0.52, zc + 0.03), 0.07); - let thumb = s_capsule(p, vec3(0.03, -0.77, zc + 0.03), vec3(-0.165, -0.60, zc + 0.065), 0.082); - var d = s_smin(palm, min(f1, min(f2, f3)), 0.035); - d = s_smin(d, index, 0.05); - return s_smin(d, thumb, 0.05); -} +const RIM_GLOVE = array, 27>( + vec4(0.0, -0.1077, 0.0, -0.6), vec4(0.1493, -0.3279, 0.1493, -0.6), + vec4(0.2863, -0.3654, 0.2863, -0.6), vec4(0.4193, -0.4043, 0.4193, -0.6), + vec4(-0.2, -0.478, -0.075, -0.625), vec4(0.0788, -0.25, 0.0788, -0.3992), + vec4(0.2188, -0.3, 0.2188, -0.4234), vec4(0.3552, -0.33, 0.3552, -0.4534), + vec4(-0.0845, -0.5386, 0.762, 0.648), + vec4(0.0, -0.1186, 0.0, -0.65), vec4(0.1614, -0.3838, 0.1614, -0.65), + vec4(0.3177, -0.427, 0.3177, -0.65), vec4(0.4645, -0.4848, 0.4645, -0.65), + vec4(-0.215, -0.535, -0.1208, -0.6864), vec4(0.0832, -0.3, 0.0832, -0.48), + vec4(0.2414, -0.35, 0.2414, -0.482), vec4(0.3959, -0.41, 0.3959, -0.533), + vec4(-0.122, -0.5718, 0.867, 0.498), + vec4(0.0, -0.122, 0.0, -0.65), vec4(0.172, -0.387, 0.172, -0.65), + vec4(0.3267, -0.427, 0.3267, -0.65), vec4(0.4727, -0.4867, 0.4727, -0.65), + vec4(-0.199, -0.483, -0.102, -0.645), vec4(0.0927, -0.3, 0.0927, -0.476), + vec4(0.2493, -0.35, 0.2493, -0.478), vec4(0.402, -0.41, 0.402, -0.52), + vec4(-0.0993, -0.5267, 0.858, 0.514) +); -fn s_cuff(p: vec3, zc: f32) -> f32 { - let q = p - vec3(0.185, -0.925, zc); - let ab = vec2(0.272, 0.12); - let e = (length(q.xz / ab) - 1.0) * min(ab.x, ab.y); - return s_extrude(e, q.y, 0.07, 0.06); -} +const RIM_GLOVE_R = array, 9>( + vec4(0.0552, 0.047, 0.045, 0.0375), vec4(0.052, 0.0235, 0.0225, 0.0225), vec4(0.0552, -0.548, 0.0525, 0.0), + vec4(0.064, 0.0585, 0.0585, 0.0515), vec4(0.06, 0.0197, 0.0178, 0.0172), vec4(0.0648, -0.6, 0.058, 0.0), + vec4(0.066, 0.0553, 0.0553, 0.0507), vec4(0.063, 0.024, 0.022, 0.02), vec4(0.0647, -0.555, 0.056, 0.0) +); -fn s_sprout(p: vec3, c: vec3) -> vec2 { - let q = p - c; - let st2 = s_star5(q.xy, 0.125, 0.52) - 0.022; - var r = vec2(s_extrude(st2, q.z, 0.038, 0.034), 3.0); - let e = vec3(abs(q.x) - 0.032, q.y + 0.005, q.z - 0.036); - r = s_opu(r, vec2(length(e) - 0.0135, 5.0)); - let l1 = vec3(s_rot(q.xy - vec2(-0.04, 0.15), -0.6), q.z); - let l2 = vec3(s_rot(q.xy - vec2(0.045, 0.155), 0.7), q.z); - let leaves = min(s_ellipsoid(l1, vec3(0.03, 0.058, 0.02)), s_ellipsoid(l2, vec3(0.03, 0.058, 0.02))); - return s_opu(r, vec2(leaves, 4.0)); +fn s_rim_poly(p: vec2, base: i32) -> f32 { + var d = dot(p - RIM_POLY[base], p - RIM_POLY[base]); + var s = 1.0; + var j = base + 6; + for (var i = base; i < base + 7; i++) { + let e = RIM_POLY[j] - RIM_POLY[i]; + let w = p - RIM_POLY[i]; + let b = w - e * saturate(dot(w, e) / dot(e, e)); + d = min(d, dot(b, b)); + let c0 = p.y >= RIM_POLY[i].y; + let c1 = p.y < RIM_POLY[j].y; + let c2 = e.x * w.y > e.y * w.x; + if (c0 && c1 && c2) || (!c0 && !c1 && !c2) { + s = -s; + } + j = i; + } + return s * sqrt(d); } -// Pixel Candy : une ligne par entier, bit c = colonne c (cf. HLSL et `sculpt.rs`). -const SCULPT_ARROWPIX = array(0, 1, 3, 7, 31, 63, 127, 255, 511, 63, 55, 115, 113, 224, 224, 64); -const SCULPT_ARROWBACK = array( - 3, 7, 15, 31, 63, 127, 255, 511, 1023, 2047, 4095, 255, 511, 511, 999, 995, 960, 192 +fn s_rim_glove(p: vec2, g: i32, palm: f32) -> vec2 { + let r0 = RIM_GLOVE_R[3 * g]; + let r1 = RIM_GLOVE_R[3 * g + 1]; + let r2 = RIM_GLOVE_R[3 * g + 2]; + let i = 9 * g; + var f = min(sd_segment(p, RIM_GLOVE[i].xy, RIM_GLOVE[i].zw) - r0.x, + sd_segment(p, RIM_GLOVE[i + 1].xy, RIM_GLOVE[i + 1].zw) - r0.y); + f = min(f, sd_segment(p, RIM_GLOVE[i + 2].xy, RIM_GLOVE[i + 2].zw) - r0.z); + f = min(f, sd_segment(p, RIM_GLOVE[i + 3].xy, RIM_GLOVE[i + 3].zw) - r0.w); + f = min(f, sd_segment(p, RIM_GLOVE[i + 4].xy, RIM_GLOVE[i + 4].zw) - r1.x); + var grooves = min(sd_segment(p, RIM_GLOVE[i + 5].xy, RIM_GLOVE[i + 5].zw) - r1.y, + sd_segment(p, RIM_GLOVE[i + 6].xy, RIM_GLOVE[i + 6].zw) - r1.z); + grooves = min(grooves, sd_segment(p, RIM_GLOVE[i + 7].xy, RIM_GLOVE[i + 7].zw) - r1.w); + let sil = s_smin(palm, f, 0.03); + let v = RIM_GLOVE[i + 8]; + let wedge = max(max(-dot(p - v.xy, v.zw), p.x + r2.x), r2.y - p.y); + return vec2(sil, max(sil, -min(grooves, wedge))); +} + +fn s_piece(d: f32, dc: f32, sil: f32, z: f32, w: f32, zt: f32, h: f32, bump: f32, mat: f32) -> vec2 { + let tray = s_extrude(sil - w, z - 0.5 * (SCULPT_HOVER + zt), 0.5 * (zt - SCULPT_HOVER), 0.012); + let bead = length(vec2(d - 0.5 * w, z - zt)) - 0.5 * w; + let u = saturate(-dc / 0.045); + let cushion = 0.8 * s_extrude(dc, z - zt, h + 0.022 * u * (2.0 - u) + bump, 0.75 * h); + return s_opu(vec2(min(tray, bead), 5.0), vec2(cushion, mat)); +} + +fn s_paper(d: f32, sil: f32, z: f32, w: f32) -> vec2 { + let navy = s_extrude(sil - w, z - 0.5 * (SCULPT_HOVER + 0.15), 0.5 * (0.15 - SCULPT_HOVER), 0.006); + let sheet = s_extrude(d, z - 0.16, 0.01, 0.004); + return s_opu(vec2(navy, 5.0), vec2(sheet, 1.0)); +} + +fn s_bump(p: vec2, c: vec2, r: f32) -> f32 { + let k = saturate(1.0 - dot(p - c, p - c) / (r * r)); + return k * k; +} + +fn s_dash(p: vec2, a: vec2, b0: vec2, ra: f32, rb: f32) -> f32 { + let q0 = p - a; + let b = b0 - a; + let hb = dot(b, b); + let q = vec2(abs(dot(q0, vec2(b.y, -b.x))), dot(q0, b)) / hb; + let c = vec2(sqrt(hb - (ra - rb) * (ra - rb)), ra - rb); + let k = c.x * q.y - c.y * q.x; + if k < 0.0 { + return sqrt(hb * dot(q, q)) - ra; + } + if k > c.x { + return sqrt(hb * (dot(q, q) + 1.0 - 2.0 * q.y)) - rb; + } + return dot(c, q) - ra; +} + +fn s_rimmed(p: vec3, theme: i32, shape: i32) -> vec2 { + let arrow = shape == 0; + let g = select(select(2, 1, theme == 0), 0, theme == 4); + let poly = s_rim_poly(p.xy, select(7 * g + 7, 0, arrow)); + var body = vec2(poly, poly); + var w = 0.06; + var zt = 0.17; + var h = 0.03; + var bump = 0.022 * s_bump(p.xy, vec2(0.15, -0.45), 0.3); + var back = 1e9; + if !arrow { + w = RIM_GLOVE_R[3 * g + 2].z; + for (var i = 0; i < 2; i++) { + let q = p.xy + select(vec2(0.0), vec2(0.03, 0.04), i == 1); + let v = s_rim_glove(q, g, select(poly, s_rim_poly(q, 7 * g + 7), i == 1) - 0.05); + if i == 1 { + back = v.x - w; + } else { + body = v; + if theme != 2 || v.x - w <= 0.0 { + break; + } + } + } + zt = 0.185; + h = 0.028; + bump = 0.03 * s_bump(p.xy, vec2(0.19, -0.62), 0.28); + } + var dc = body.y; + if arrow && theme == 0 { + body = body - vec2(0.015); + w = 0.045; + dc = 1.0; + } + var r: vec2; + if theme == 2 { + r = s_paper(body.y, body.x, p.z, w); + } else { + r = s_piece(body.y, dc, body.x, p.z, w, zt, h, bump, 1.0); + } + if theme == 0 { + if arrow { + let band = s_extrude(abs(poly + 0.0075) - 0.0225, p.z - zt, 0.028, 0.012); + r = s_opu(r, vec2(band, 2.0)); + } + return r; + } + if theme == 2 { + var dash: f32; + if arrow { + let back = s_rim_poly(p.xy + vec2(0.025, 0.05), 0) - 0.06; + r = s_opu(r, vec2(s_extrude(back, p.z - 0.08, 0.03, 0.006), 2.0)); + dash = min(s_dash(p.xy, vec2(-0.1179, -0.1374), vec2(-0.1799, -0.0443), 0.028, 0.045), + s_dash(p.xy, vec2(-0.1347, -0.2493), vec2(-0.248, -0.2056), 0.026, 0.042)); + } else { + r = s_opu(r, vec2(s_extrude(back, p.z - 0.08, 0.03, 0.006), 3.0)); + dash = min(s_dash(p.xy, vec2(0.216, -0.1695), vec2(0.2593, -0.0685), 0.028, 0.042), + s_dash(p.xy, vec2(0.3054, -0.2339), vec2(0.3949, -0.1652), 0.0275, 0.041)); + } + return s_opu(r, vec2(s_extrude(dash, p.z - 0.11, 0.06, 0.006), 3.0)); + } + if !arrow { + let cq = p.xy - vec2(0.1541, -0.9047); + let bq = abs(vec2(cq.x, cq.y - 0.2066 * cq.x * cq.x)) - vec2(0.214, 0.0361); + let cuff = length(max(bq, vec2(0.0))) + min(max(bq.x, bq.y), 0.0) - 0.03; + r = s_opu(r, s_piece(cuff, cuff, cuff, p.z, 0.0477, 0.225, 0.025, 0.0, 2.0)); + } + let c = select(vec2(0.15, -0.885), vec2(0.6118, -0.8079), arrow); + let rs = select(0.128, 0.1678, arrow); + let zs = select(0.285, 0.23, arrow); + let q = s_rot(p.xy - c, select(0.0, 0.2443, arrow)) / rs; + let star = (s_star5(q, 0.82, 0.55) - 0.18) * rs; + let l1 = s_rot(q - vec2(-0.33, 1.38), -0.925); + let l2 = s_rot(q - vec2(0.53, 1.37), 0.873); + let leaves = min(s_vesica(l1, 0.4296, 0.2626), s_vesica(l2, 0.4296, 0.2626)) * rs; + r = s_opu(r, s_piece(leaves, leaves, leaves, p.z, 0.038, zs - 0.02, 0.018, 0.0, 4.0)); + r = s_opu(r, s_piece(star, star, star, p.z, 0.038, zs, 0.022, 0.018 * s_bump(q, vec2(0.0), 1.0), 3.0)); + let e = vec3(abs(q.x) - 0.25, q.y - 0.08, (p.z - zs - 0.056) / rs); + return s_opu(r, vec2(s_ellipsoid(e, vec3(0.075, 0.13, 0.1)) * rs, 5.0)); +} + +// ---- Glass Lens ---- (cf. HLSL : une lentille de verre dans un trait sombre, un liseré de verre +// au-delà ; `s_glass` en volume, `cursor_glass` à plat, `glass_shade` pour le verre.) +const GLASS_LINE: f32 = 0.04; +const GLASS_RIM: f32 = 0.085; +const GLASS_TOP: f32 = 0.18; +const GLASS_DOME: f32 = 0.03; +const GLASS_RISE: f32 = 0.08; +const GLASS_IOR: f32 = 1.5; +const GLASS_DISPERSION: f32 = 0.012; +const GLASS_MAT: f32 = 10.0; +const GLASS_GAP_FLAT: f32 = 0.15; +const GLASS_SMEAR: f32 = 0.15; +const GLASS_LIFT: f32 = 0.02; +const GLASS_SHADOW: f32 = 0.6; + +// x = la silhouette, y = la lentille (distances signées du prototype, négatives dedans). +fn s_glass2(p: vec2, shape: i32) -> vec2 { + if shape == 0 { + let poly = s_rim_poly(p + vec2(0.0, GLASS_RIM - 0.06), 0); + return vec2(poly - GLASS_RIM, poly); + } + let q = p + vec2(0.0, GLASS_RIM - RIM_GLOVE_R[5].z); + let v = s_rim_glove(q, 1, s_rim_poly(q, 14) - 0.05); + return vec2(v.x - GLASS_RIM, v.y); +} + +// La hauteur du verre au-dessus de GLASS_TOP selon la distance `d` à la lentille, et sa pente. +fn s_glass_relief(d: f32) -> vec2 { + if d < 0.0 { + let u = saturate(-d / GLASS_RISE); + return vec2(GLASS_DOME * u * (2.0 - u), -2.0 * GLASS_DOME * (1.0 - u) / GLASS_RISE); + } + let w = GLASS_RIM - GLASS_LINE; + let x = saturate((d - GLASS_LINE) / w); + let s = sqrt(max(1.0 - x * x, 0.05)); + return vec2(w * (s - 1.0), -x / s); +} + +fn s_glass(p: vec3, shape: i32) -> vec2 { + let g = s_glass2(p.xy, shape); + let half_h = 0.5 * (GLASS_TOP - SCULPT_HOVER); + let slab = s_extrude(g.x, p.z - SCULPT_HOVER - half_h, half_h, min(GLASS_RIM - GLASS_LINE, half_h)); + let u = saturate(-g.y / GLASS_RISE); + let dome = 0.8 * s_extrude(g.y, p.z - GLASS_TOP, 0.009 + GLASS_DOME * u * (2.0 - u), 0.008); + let band = s_extrude(abs(g.y - 0.5 * GLASS_LINE) - 0.5 * GLASS_LINE, p.z - GLASS_TOP, 0.016, 0.008); + return s_opu(vec2(min(slab, dome), GLASS_MAT), vec2(band, 5.0)); +} + +// Pixel Candy : tables générées par scripts/generate-pixel-candy-voxels.mjs (cf. HLSL). +// +const PIX_BODY = array(1, 3, 7, 15, 31, 63, 127, 255, 511, 1023, 2047, 127, 247, 243, 480, 192, 48, 120, 120, 120, 504, 4088, 32760, 65534, 65535, 65535, 65534, 32766, 32764, 16380, 16376, 16376); +const PIX_LINE = array(1, 3, 5, 9, 17, 33, 65, 129, 257, 513, 1985, 73, 149, 147, 288, 192, 48, 72, 72, 72, 456, 3656, 29256, 37454, 32777, 32769, 32770, 16386, 16388, 8196, 8200, 16376); +const PIX_HI = array(0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 18, 34, 32, 64, 0, 0, 16, 16, 16, 16, 16, 16, 16, 22, 18, 4, 4, 8, 8, 16, 0); +const PIX_SHADE = array(0, 0, 0, 4, 8, 16, 32, 64, 128, 448, 40, 36, 64, 64, 128, 0, 0, 32, 32, 32, 32, 288, 2336, 18720, 16384, 16384, 16384, 8192, 8192, 4096, 8160, 0); +const PIX_RECT_N = array(0, 17, 29); +const PIX_GRID = array, 2>( + vec4(11, 16, 0, 0), + vec4(16, 16, 16, 0) ); -const SCULPT_HANDPIX = array(4, 14, 14, 14, 110, 878, 7022, 7022, 8190, 8191, 8191, 8191, 8190, 4092, 4092); -const SCULPT_HANDBACK = array( - 28, 62, 62, 62, 510, 4094, 32766, 32766, 32766, 32767, 32767, 32767, 32767, 32767, 32766, 16380, 16380 +const PIX_ORIGIN = array, 2>( + vec2(0.0, 0.0), + vec2(-0.3125, 0.0) ); -const SCULPT_HANDSPAN = array, 17>( - vec2(2.0, 4.0), vec2(1.0, 5.0), vec2(1.0, 5.0), vec2(1.0, 5.0), vec2(1.0, 8.0), - vec2(1.0, 11.0), vec2(1.0, 14.0), vec2(1.0, 14.0), vec2(1.0, 14.0), vec2(0.0, 14.0), - vec2(0.0, 14.0), vec2(0.0, 14.0), vec2(0.0, 14.0), vec2(0.0, 14.0), vec2(1.0, 14.0), - vec2(2.0, 13.0), vec2(2.0, 13.0) +const PIX_BOX = array, 2>( + vec4(0.3438, -0.5, 0.3438, 0.5), + vec4(0.1875, -0.5, 0.5, 0.5) ); +const PIX_RECT = array, 29>( + vec4(0.0313, -0.0313, 0.0313, 0.0313), + vec4(0.0625, -0.0938, 0.0625, 0.0313), + vec4(0.0938, -0.1563, 0.0938, 0.0313), + vec4(0.125, -0.2188, 0.125, 0.0313), + vec4(0.1563, -0.2813, 0.1563, 0.0313), + vec4(0.1875, -0.3438, 0.1875, 0.0313), + vec4(0.2188, -0.4063, 0.2188, 0.0313), + vec4(0.25, -0.4688, 0.25, 0.0313), + vec4(0.2813, -0.5313, 0.2813, 0.0313), + vec4(0.3125, -0.5938, 0.3125, 0.0313), + vec4(0.3438, -0.6563, 0.3438, 0.0313), + vec4(0.2188, -0.7188, 0.2188, 0.0313), + vec4(0.0938, -0.7813, 0.0938, 0.0313), + vec4(0.375, -0.8125, 0.125, 0.0625), + vec4(0.0625, -0.8438, 0.0625, 0.0313), + vec4(0.4375, -0.9063, 0.125, 0.0313), + vec4(0.4375, -0.9688, 0.0625, 0.0313), + vec4(0.0, -0.0313, 0.0625, 0.0313), + vec4(0.0, -0.1563, 0.125, 0.0938), + vec4(0.0625, -0.2813, 0.1875, 0.0313), + vec4(0.1563, -0.3438, 0.2813, 0.0313), + vec4(0.25, -0.4063, 0.375, 0.0313), + vec4(0.2188, -0.4688, 0.4688, 0.0313), + vec4(0.1875, -0.5625, 0.5, 0.0625), + vec4(0.2188, -0.6563, 0.4688, 0.0313), + vec4(0.1875, -0.7188, 0.4375, 0.0313), + vec4(0.2188, -0.7813, 0.4063, 0.0313), + vec4(0.1875, -0.8438, 0.375, 0.0313), + vec4(0.2188, -0.9375, 0.3438, 0.0625) +); +// -fn s_grid_origin(shape: i32) -> vec2 { - return select(vec2(-2.5 * SCULPT_VOX, 0.0), vec2(0.0), shape == 0); -} - -fn s_bit(row: i32, r: i32, c: i32, rows: i32, cols: i32) -> bool { - return r >= 0 && r < rows && c >= 0 && c < cols && ((row >> u32(clamp(c, 0, 31))) & 1) == 1; +fn s_pix_bit(m: i32, c: i32) -> bool { + return ((m >> u32(clamp(c, 0, 31))) & 1) == 1; } -fn s_occ(id: vec2, shape: i32) -> bool { - let c = i32(id.x); - let r = -i32(id.y) - 1; - if shape == 0 { - return s_bit(SCULPT_ARROWPIX[clamp(r, 0, 15)], r, c, 16, 16); - } - return s_bit(SCULPT_HANDPIX[clamp(r, 0, 14)], r, c, 15, 13); +fn s_pix_body(shape: i32, c: i32, r: i32) -> bool { + let g = PIX_GRID[shape]; + return c >= 0 && c < g.x && r >= 0 && r < g.y && s_pix_bit(PIX_BODY[g.z + clamp(r, 0, g.y - 1)], c); } -fn s_occ_back(id: vec2, shape: i32) -> bool { - let c = i32(id.x) + 1; - let r = -i32(id.y); - if shape == 0 { - return s_bit(SCULPT_ARROWBACK[clamp(r, 0, 17)], r, c, 18, 17); - } - return s_bit(SCULPT_HANDBACK[clamp(r, 0, 16)], r, c, 17, 15); +fn s_pix_cell(p: vec2, shape: i32) -> vec2 { + let o = PIX_ORIGIN[shape]; + return vec2(i32(floor((p.x - o.x) / SCULPT_VOX)), i32(floor((o.y - p.y) / SCULPT_VOX))); } fn s_voxels(p: vec3, shape: i32) -> vec2 { - let o = s_grid_origin(shape); - let cell = floor((p.xy - o) / SCULPT_VOX); - let bc = select(vec2(0.25, -0.47), vec2(0.33, -0.5), shape == 0); - let bh = select(vec2(0.48, 0.55), vec2(0.44, 0.61), shape == 0); - let bq = max(abs(p.xy - bc) - bh, vec2(0.0)); + let b = PIX_BOX[shape]; + let bq = max(abs(p.xy - b.xy) - b.zw, vec2(0.0)); let bz = max(abs(p.z - (SCULPT_HOVER + 0.07)) - 0.07, 0.0); - let far = max(SCULPT_VOX, sqrt(dot(bq, bq) + bz * bz)); - var dF = far; - var dB = far; - let half_cell = vec3(0.5 * SCULPT_VOX, 0.5 * SCULPT_VOX, 0.04); + var d = max(SCULPT_VOX, sqrt(dot(bq, bq) + bz * bz)); + let o = PIX_ORIGIN[shape]; + let cell = s_pix_cell(p.xy, shape); + let cube = vec3(0.5 * SCULPT_VOX, 0.5 * SCULPT_VOX, 0.07); for (var j = -1; j <= 1; j++) { for (var i = -1; i <= 1; i++) { - let id = cell + vec2(f32(i), f32(j)); - let q = vec3(p.xy - (o + (id + vec2(0.5)) * SCULPT_VOX), p.z); - if s_occ(id, shape) { - dF = min(dF, s_round_box(q - vec3(0.0, 0.0, SCULPT_HOVER + 0.1), half_cell, 0.009)); - } - if s_occ_back(id, shape) { - dB = min(dB, s_round_box(q - vec3(0.0, 0.0, SCULPT_HOVER + 0.04), half_cell, 0.009)); + let c = cell.x + i; + let r = cell.y + j; + if s_pix_body(shape, c, r) { + let q = vec3(p.x - o.x - (f32(c) + 0.5) * SCULPT_VOX, p.y - o.y + (f32(r) + 0.5) * SCULPT_VOX, + p.z - SCULPT_HOVER - 0.07); + d = min(d, s_round_box(q, cube, 0.006)); } } } - return select(vec2(dB, 7.0), vec2(dF, 6.0), dF < dB); + return vec2(d, 6.0); } -fn s_pixel_hand_dist(p: vec2) -> f32 { +fn s_pixel_outline(p: vec2, shape: i32) -> f32 { var d = 1e9; - for (var r = 0; r < 17; r++) { - let x = (SCULPT_HANDSPAN[r] + vec2(-3.5, -2.5)) * SCULPT_VOX; - let y = vec2(-f32(r), 1.0 - f32(r)) * SCULPT_VOX; - let q = max(max(vec2(x.x, y.x) - p, p - vec2(x.y, y.y)), vec2(0.0)); - d = min(d, length(q)); + for (var i = PIX_RECT_N[shape]; i < PIX_RECT_N[shape + 1]; i++) { + let q = abs(p - PIX_RECT[i].xy) - PIX_RECT[i].zw; + d = min(d, length(max(q, vec2(0.0))) + min(max(q.x, q.y), 0.0)); } return d; } -fn s_gem_edge(p: vec3, a: vec2, b: vec2, z0: f32) -> f32 { - let e = b - a; - let len = length(e); - let d = e / len; - let q = p.xy - a; - let dist = dot(q, vec2(-d.y, d.x)); - let along = abs(dot(q, d) - 0.5 * len); - let z = p.z - z0; - let girdle = (z - 0.035 - 2.2 * dist) * 0.4138; - let crown = (z - 0.07 - 0.6 * dist + 0.18 * along) * 0.8438; - return max(-dist, max(girdle, crown)); -} - -fn s_gem_arrow(p: vec3) -> f32 { - let z0 = SCULPT_HOVER + 0.03; - let slab = max(p.z - z0 - 0.2, z0 - 0.03 - p.z); - let head = max(slab, max(s_gem_edge(p, vec2(-0.02, 0.03), vec2(-0.02, -0.88), z0), - max(s_gem_edge(p, vec2(-0.02, -0.88), vec2(0.66, -0.61), z0), - s_gem_edge(p, vec2(0.66, -0.61), vec2(-0.02, 0.03), z0)))); - let tail = max(slab, max(max(s_gem_edge(p, vec2(0.215, -0.665), vec2(0.37, -1.0), z0), - s_gem_edge(p, vec2(0.37, -1.0), vec2(0.53, -0.93), z0)), - max(s_gem_edge(p, vec2(0.53, -0.93), vec2(0.38, -0.60), z0), - s_gem_edge(p, vec2(0.38, -0.60), vec2(0.215, -0.665), z0)))); - return min(head, tail); -} - -fn s_facet_capsule(p: vec3, a: vec3, b: vec3, r: f32, spin: f32) -> f32 { - let u = normalize(b - a); - let v = normalize(cross(u, vec3(0.0, 0.0, 1.0))); - let w = cross(u, v); - let q = p - b; - let h = dot(q, u); - let rad = vec2(dot(q, v), dot(q, w)); - var d = max(h - r, -dot(p - a, u) - r); - for (var k = 0; k < 6; k++) { - let an = spin + f32(k) * 1.0471976; - let s = dot(rad, vec2(cos(an), sin(an))); - d = max(d, s - r); - d = max(d, dot(rad, vec2(cos(an + 0.5236), sin(an + 0.5236))) * 0.8660 + h * 0.5 - r); - d = max(d, s * 0.5 + h * 0.8660 - r); +// ---- Prism Glow : un cristal en MAILLAGE ---- (port du HLSL, dont les commentaires font foi) +// Les facettes tracees sur son dessin, en triangles dans le repere du modele ; le serti est la +// silhouette extrudee (champ de distance), le cristal est lance de rayons boite par boite. +// prism mesh: generated by design/cursors/prism-glow/model/export_compositor.py +var PRISM_TRI_START = array(0, 31); +var PRISM_TRI_COUNT = array(31, 130); +var PRISM_SIL_START = array(0, 25); +var PRISM_SIL_COUNT = array(25, 42); +var PRISM_OUT_START = array(67, 76); +var PRISM_OUT_COUNT = array(9, 34); +var PRISM_BOX_START = array(0, 4); +var PRISM_BOX_COUNT = array(4, 18); +struct PrismMesh { + tris: array, 644>, + poly: array, 110>, + boxes: array, 44>, +} +const PRISM_BEVEL: f32 = 0.012883; +// end of the prism mesh + +const PRISM_IOR: f32 = 1.61; +const PRISM_DISPERSION: f32 = 0.035; +const PRISM_GLOW: f32 = 0.45; +const PRISM_FOLD: f32 = 0.35; +const PRISM_BOUNCES: i32 = 6; +const PRISM_MAT: f32 = 9.0; + +fn prism_shape() -> i32 { + let id = sculpt_id(); + if id == 3 || id == 4 { + return id - 3; } - return d; + return -1; } -const SCULPT_FACETS = array, 10>( - vec3(1.0, 0.0, 0.0), vec3(0.0, 1.0, 0.0), vec3(0.0, 0.0, 1.0), - vec3(0.7071, 0.7071, 0.0), vec3(0.7071, 0.0, 0.7071), vec3(0.0, 0.7071, 0.7071), - vec3(0.5774, 0.5774, 0.5774), vec3(0.4472, 0.0, 0.8944), vec3(0.0, 0.4472, 0.8944), - vec3(0.3015, 0.3015, 0.9045) -); +// Drapeaux d'un triangle : paroi (bit 0), aretes reelles (bits 1 a 3). +fn prism_flags(k: i32) -> i32 { + return i32(prism.tris[4 * k].w + 0.5); +} -fn s_facet_ellipsoid(p: vec3, r: vec3) -> f32 { - let q = abs(p); - var d = -1e9; - for (var i = 0; i < 10; i++) { - d = max(d, dot(q, SCULPT_FACETS[i]) - length(r * SCULPT_FACETS[i])); +fn prism_poly_dist(p: vec2, i0: i32, n: i32) -> f32 { + var vj = prism.poly[i0 + n - 1].xy; + var d = 1e9; + var sgn = 1.0; + for (var i = 0; i < n; i++) { + let vi = prism.poly[i0 + i].xy; + let e = vj - vi; + let w = p - vi; + let b = w - e * saturate(dot(w, e) / dot(e, e)); + d = min(d, dot(b, b)); + let c0 = p.y >= vi.y; + let c1 = p.y < vj.y; + let c2 = e.x * w.y > e.y * w.x; + if (c0 && c1 && c2) || (!c0 && !c1 && !c2) { + sgn = -sgn; + } + vj = vi; } - return d; + return sgn * sqrt(d); } -fn s_crystal_hand(p: vec3) -> f32 { - let zc = SCULPT_HAND_ZC; - var d = s_facet_ellipsoid(p - vec3(0.185, -0.70, zc), vec3(0.3, 0.235, 0.125)); - d = min(d, s_facet_capsule(p, vec3(0.0, -0.52, zc), vec3(0.0, -0.10, zc), 0.098, 0.3)); - d = min(d, s_facet_capsule(p, vec3(0.17, -0.57, zc + 0.012), vec3(0.17, -0.41, zc + 0.045), 0.086, 0.1)); - d = min(d, s_facet_capsule(p, vec3(0.31, -0.59, zc + 0.01), vec3(0.31, -0.45, zc + 0.04), 0.08, 0.5)); - d = min(d, s_facet_capsule(p, vec3(0.435, -0.625, zc + 0.005), vec3(0.435, -0.52, zc + 0.03), 0.07, 0.2)); - return min(d, s_facet_capsule(p, vec3(0.03, -0.77, zc + 0.03), vec3(-0.165, -0.60, zc + 0.065), 0.082, 0.7)); +fn prism_rim(p: vec3, shape: i32) -> f32 { + let half_t = model_thick() * 0.5; + let sil = prism_poly_dist(p.xy, PRISM_SIL_START[shape], PRISM_SIL_COUNT[shape]); + let w = vec2(sil + PRISM_BEVEL, abs(p.z + half_t) - (half_t - PRISM_BEVEL)); + return min(max(w.x, w.y), 0.0) + length(max(w, vec2(0.0))) - PRISM_BEVEL; +} + +fn prism_tri(ro: vec3, rd: vec3, k: i32) -> f32 { + let v0 = prism.tris[4 * k].xyz; + let e1 = prism.tris[4 * k + 1].xyz; + let e2 = prism.tris[4 * k + 2].xyz; + let pv = cross(rd, e2); + let det = dot(e1, pv); + if abs(det) < 1e-10 { + return -1.0; + } + let inv = 1.0 / det; + let sv = ro - v0; + let u = dot(sv, pv) * inv; + let qv = cross(sv, e1); + let v = dot(rd, qv) * inv; + if u < -2e-4 || v < -2e-4 || u + v > 1.0004 { + return -1.0; + } + return dot(e2, qv) * inv; +} + +// (distance, indice) du triangle du cristal `shape` le plus proche devant (ro, rd) ; (1e9, -1) +// s'il n'y en a pas. Les triangles sont parcourus boîte par boîte (cf. HLSL). +fn prism_trace(ro: vec3, rd: vec3, shape: i32) -> vec2 { + let inv = vec3(1.0) / select(vec3(1e-6), rd, abs(rd) > vec3(1e-6)); + let b0 = PRISM_BOX_START[shape]; + let nb = PRISM_BOX_COUNT[shape]; + var best = 1e9; + var hit = -1; + for (var j = b0; j < b0 + nb; j++) { + let lo = prism.boxes[2 * j]; + let hi = prism.boxes[2 * j + 1]; + let t0 = (lo.xyz - ro) * inv; + let t1 = (hi.xyz - ro) * inv; + let tn = min(t0, t1); + let tf = max(t0, t1); + let tmin = max(max(tn.x, tn.y), tn.z); + let tmax = min(min(tf.x, tf.y), tf.z); + if tmax < max(tmin, 1e-5) || tmin > best { + continue; + } + let k0 = i32(lo.w + 0.5); + let n = i32(hi.w + 0.5); + for (var i = k0; i < k0 + n; i++) { + let t = prism_tri(ro, rd, i); + if t > 1e-5 && t < best { + best = t; + hit = i; + } + } + } + return vec2(best, f32(hit)); } fn sculpt_proto(p: vec3, theme: i32, shape: i32) -> vec2 { if theme == 3 { return s_voxels(p, shape); } - if theme == 1 { - return vec2(select(s_crystal_hand(p), s_gem_arrow(p), shape == 0), 8.0); - } - var body: vec2; - var star: vec3; - if shape == 0 { - var h = 0.075; - var re = 0.03; - var dome = 0.0; - if theme == 2 { - h = 0.07; - re = 0.06; - dome = 0.035; - } - if theme == 4 { - h = 0.07; - re = 0.055; - dome = 0.025; - } - body = vec2(s_arrow_solid(p, h, re, dome), 1.0); - if theme == 0 { - body = s_opu(body, vec2(s_piping(p, SCULPT_HOVER + 2.0 * h - 0.004), 2.0)); - } - star = vec3(0.57, -0.86, SCULPT_HOVER + 2.0 * h + dome); - } else { - body = s_opu(vec2(s_glove(p, SCULPT_HAND_ZC), 1.0), vec2(s_cuff(p, SCULPT_HAND_ZC), 2.0)); - star = vec3(0.185, -0.93, SCULPT_HAND_ZC + 0.15); + if theme == 5 { + return s_glass(p, shape); } - if theme == 4 { - return s_opu(body, s_sprout(p, star)); - } - return body; + return s_rimmed(p, theme, shape); } fn sculpt_point(q: vec3) -> vec3 { @@ -940,14 +1169,18 @@ fn sculpt_eval(q: vec3, occ: bool) -> vec2 { let id = sculpt_id() - 1; let theme = id / 2; let shape = id % 2; + if theme == 1 { + // Prism Glow : son serti seul, dans le repere du modele (le cristal est trace a part). + return vec2(prism_rim(q, shape), 8.0); + } let p = sculpt_point(q); var r: vec2; if occ && theme == 3 { - let d2 = select(s_pixel_hand_dist(p.xy), s_arrow2(p.xy) - 1.2 * SCULPT_VOX, shape == 0); + let d2 = s_pixel_outline(p.xy, shape); let dz = abs(p.z - (SCULPT_HOVER + 0.07)) - 0.07; r = vec2(length(max(vec2(d2, dz), vec2(0.0))) + min(max(d2, dz), 0.0), 7.0); } else { - r = sculpt_proto(p, select(theme, 2, occ && theme == 1), shape); + r = sculpt_proto(p, theme, shape); } return vec2(r.x * sculpt_units(), r.y); } @@ -956,7 +1189,7 @@ fn sd_sprite2(p: vec2) -> f32 { let lo = layer.color.rg; let c = clamp(p, lo, lo + sprite_size()); // Niveau 0 explicite : la marche est une boucle a sortie anticipee (pas de derivees). - let d = textureSampleLevel(texU, samp, (c - lo) / sprite_size(), 0.0).r; + let d = textureSampleLevel(texMask, samp, (c - lo) / sprite_size(), 0.0).r; let o = p - c; let out2 = dot(o, o); let e = max(d, 0.0); @@ -1016,7 +1249,7 @@ fn model_albedo(p: vec2) -> vec3 { let g = vec2(sd_sprite2(p + vec2(e, 0.0)) - sd_sprite2(p - vec2(e, 0.0)), sd_sprite2(p + vec2(0.0, e)) - sd_sprite2(p - vec2(0.0, e))); let q = p - g / max(length(g), 1e-6) * max(sd_sprite2(p) + MODEL_RIM_INSET * texel, 0.0); - return textureSampleLevel(texY, samp, (q - layer.color.rg) / sprite_size(), 0.0).rgb; + return textureSampleLevel(texDof, samp, (q - layer.color.rg) / sprite_size(), 0.0).rgb; } struct SculptMat { @@ -1028,61 +1261,63 @@ struct SculptMat { } fn s_pixel_colour(p: vec3, shape: i32) -> vec3 { - let id = floor((p.xy - s_grid_origin(shape)) / SCULPT_VOX); - if !s_occ(id + vec2(-1.0, 0.0), shape) || !s_occ(id + vec2(0.0, -1.0), shape) { - return s_lin(0.52, 0.91, 0.77); + let cell = s_pix_cell(p.xy, shape); + let g = PIX_GRID[shape]; + let row = g.z + clamp(cell.y, 0, g.y - 1); + if s_pix_bit(PIX_LINE[row], cell.x) { + return s_lin(0.29, 0.12, 0.36); } - if !s_occ(id + vec2(1.0, 0.0), shape) || !s_occ(id + vec2(0.0, 1.0), shape) { + if s_pix_bit(PIX_HI[row], cell.x) { return s_lin(1.0, 0.78, 0.87); } - return s_lin(1.0, 0.50, 0.71); + if s_pix_bit(PIX_SHADE[row], cell.x) { + return s_lin(0.87, 0.27, 0.51); + } + return s_lin(1.0, 0.435, 0.66); } fn sculpt_material(mat: f32, p: vec3, theme: i32, shape: i32) -> SculptMat { let primary = mat < 1.5; if theme == 0 { - if shape == 0 && primary { - return SculptMat(s_lin(0.10, 0.10, 0.115), 0.3, 0.2, 0.0, 0.9); - } - if shape == 0 { - return SculptMat(s_lin(0.94, 0.91, 0.84), 0.45, 0.4, 0.2, 0.3); - } - if primary { - return SculptMat(s_lin(0.95, 0.92, 0.85), 0.55, 0.35, 0.35, 0.25); + if mat < 2.5 { + return SculptMat(s_lin(0.95, 0.92, 0.85), 0.3, 0.6, 0.35, 0.25); } - return SculptMat(s_lin(0.17, 0.18, 0.22), 0.35, 0.6, 0.0, 0.6); + return SculptMat(s_lin(0.1, 0.1, 0.11), 0.55, 0.05, 0.0, 0.1); + } + if theme == 1 { + return SculptMat(s_lin(0.02, 0.05, 0.33), 0.35, 0.5, 0.05, 0.3); + } + if theme == 5 { + return SculptMat(s_lin(0.05, 0.055, 0.07), 0.25, 0.7, 0.0, 0.35); } if theme == 2 { - if shape == 0 { - return SculptMat(s_lin(1.0, 0.40, 0.30), 0.5, 0.45, 0.4, 0.25); + if (primary && shape == 0) || (mat > 2.5 && mat < 3.5 && shape == 1) { + return SculptMat(s_lin(1.0, 0.40, 0.30), 0.85, 0.08, 0.15, 0.03); } - if primary { - return SculptMat(s_lin(1.0, 0.79, 0.16), 0.5, 0.45, 0.4, 0.25); + if mat < 3.5 { + return SculptMat(s_lin(1.0, 0.80, 0.10), 0.85, 0.08, 0.15, 0.03); } - return SculptMat(s_lin(0.18, 0.20, 0.29), 0.4, 0.5, 0.0, 0.4); + return SculptMat(s_lin(0.09, 0.13, 0.45), 0.9, 0.05, 0.05, 0.02); } if theme == 4 { if primary && shape == 0 { - return SculptMat(s_lin(0.62, 0.91, 0.78), 0.22, 0.8, 0.25, 0.6); + return SculptMat(s_lin(0.68, 0.93, 0.80), 0.25, 0.7, 0.3, 0.3); } if primary { - return SculptMat(s_lin(0.96, 0.94, 0.88), 0.5, 0.35, 0.35, 0.25); + return SculptMat(s_lin(0.97, 0.95, 0.90), 0.3, 0.6, 0.35, 0.25); } if mat < 2.5 { - return SculptMat(s_lin(0.62, 0.91, 0.78), 0.25, 0.7, 0.25, 0.5); + return SculptMat(s_lin(0.52, 0.87, 0.78), 0.25, 0.7, 0.3, 0.3); } if mat < 3.5 { - return SculptMat(s_lin(1.0, 0.80, 0.20), 0.3, 0.6, 0.3, 0.4); + return SculptMat(s_lin(1.0, 0.75, 0.25), 0.25, 0.7, 0.3, 0.3); } if mat < 4.5 { - return SculptMat(s_lin(0.38, 0.80, 0.55), 0.35, 0.5, 0.35, 0.3); + return SculptMat(s_lin(0.62, 0.92, 0.72), 0.3, 0.6, 0.3, 0.35); } - return SculptMat(s_lin(0.16, 0.12, 0.10), 0.2, 0.8, 0.0, 0.5); + return SculptMat(s_lin(0.07, 0.15, 0.33), 0.7, 0.15, 0.05, 0.08); } - if mat < 6.5 { - return SculptMat(s_pixel_colour(p, shape), 0.45, 0.35, 0.15, 0.2); - } - return SculptMat(s_lin(0.36, 0.18, 0.54), 0.45, 0.35, 0.1, 0.2); + return SculptMat(s_pixel_colour(p, shape), 0.65, 0.1, 0.12, 0.04); } fn model_env(d: vec3, rough: f32, l: vec3, fill: vec3) -> vec3 { @@ -1094,36 +1329,6 @@ fn model_env(d: vec3, rough: f32, l: vec3, fill: vec3) -> vec3 vec3 { - let k = saturate(k0) * 3.0; - let a = s_lin(0.20, 0.95, 1.0); - let b = s_lin(0.15, 0.42, 1.0); - let c = s_lin(0.45, 0.25, 0.95); - let d = s_lin(0.88, 0.50, 1.0); - if k < 1.0 { - return mix(a, b, k); - } - if k < 2.0 { - return mix(b, c, k - 1.0); - } - return mix(c, d, k - 2.0); -} - -fn model_shade_crystal(n: vec3, rd: vec3, L: vec3, fall: f32, l: vec3, fill: vec3) -> vec3 { - let cosi = saturate(dot(-rd, n)); - let F = 0.04 + 0.96 * pow(1.0 - cosi, 5.0); - let t = refract(rd, n, 1.0 / 1.6); - let k = 0.42 + 0.9 * dot(vec2(n.x, -n.y), vec2(0.7557, -0.6549)) - + 0.6 * dot(vec2(t.x, -t.y), vec2(0.6, -0.8)); - let body = vec3(s_gem(k - 0.08).r, s_gem(k).g, s_gem(k + 0.08).b); - var col = body * (0.1 + 1.8 * fall * pow(max(dot(n, L), 0.0), 2.5)); - col = col + body * model_env(reflect(t, vec3(0.0, 0.0, 1.0)) * vec3(1.0, 1.0, -1.0), 0.15, l, fill) * 0.5; - col = col + vec3(pow(max(dot(reflect(rd, n), L), 0.0), 30.0) * 2.0); - col = col + model_env(reflect(rd, n), 0.05, l, fill) * F; - col = col + s_lin(0.5, 0.9, 1.0) * pow(1.0 - cosi, 4.0) * 0.6; - return col; -} - fn model_tonemap(c0: vec3) -> vec3 { let start = 0.76; let x = min(c0.r, min(c0.g, c0.b)); @@ -1142,14 +1347,16 @@ fn model_tonemap(c0: vec3) -> vec3 { fn model_shade(q: vec3, n: vec3, rd: vec3, L: vec3, fall: f32, sh: f32, ao: f32, mat: f32, l: vec3, fill: vec3) -> vec3 { let id = sculpt_id(); - if id > 0 && (id - 1) / 2 == 1 { - return model_tonemap(model_shade_crystal(n, rd, L, fall, l, fill)); - } var m: SculptMat; var gloss = 1.0; if id > 0 { let p = sculpt_point(q) - vec3(n.x, -n.y, n.z) * 0.01; m = sculpt_material(mat, p, (id - 1) / 2, (id - 1) % 2); + if id == 3 || id == 4 || id > 10 { + // Le serti de Prism Glow et le trait de Glass Lens, comme un sprite : brillants sur + // leur arrondi seulement. + gloss = 1.0 - smoothstep(0.97, 0.995, abs(n.z)); + } } else { m = SculptMat(pow(model_albedo(q.xy), vec3(2.2)), 0.45, 0.35, 0.2, 0.3); gloss = 1.0 - smoothstep(0.97, 0.995, abs(n.z)); @@ -1171,13 +1378,218 @@ fn model_shade(q: vec3, n: vec3, rd: vec3, L: vec3, fall: f3 return model_tonemap(col); } +// ---- Le cristal de Prism Glow ---- + +fn prism_fresnel(c: f32, ior: f32) -> f32 { + var f0 = (ior - 1.0) / (ior + 1.0); + f0 = f0 * f0; + return f0 + (1.0 - f0) * pow(1.0 - c, 5.0); +} + +// (distance, facette) du cristal sur le rayon ; (1e9, -1) si rate, hors de Prism Glow, ou sur une +// paroi sous le dessus du serti (le rayon a traverse le serti avant). +fn prism_primary(ro: vec3, rd: vec3) -> vec2 { + let shape = prism_shape(); + if shape < 0 { + return vec2(1e9, -1.0); + } + let r = prism_trace(ro, rd, shape); + let h = i32(r.y); + if h >= 0 && (prism_flags(h) & 1) == 1 && ro.z + rd.z * r.x < 0.0 { + return vec2(1e9, -1.0); + } + return r; +} + +fn prism_normal(h: i32, rd: vec3) -> vec3 { + let n = prism.tris[4 * h + 3].xyz; + return select(n, -n, dot(n, rd) > 0.0); +} + +fn prism_seg(p: vec3, a: vec3, b: vec3) -> f32 { + let pa = p - a; + let ba = b - a; + return length(pa - ba * saturate(dot(pa, ba) / dot(ba, ba))); +} + +// Distance, en pixels, de `p` aux aretes reelles de la facette `h` (plis et bord du serti). +fn prism_edge_px(p: vec3, h: i32, px: f32) -> f32 { + let a = prism.tris[4 * h].xyz; + let b = a + prism.tris[4 * h + 1].xyz; + let c = a + prism.tris[4 * h + 2].xyz; + let bits = prism_flags(h) >> 1u; + var d = 1e9; + if (bits & 1) != 0 { + d = min(d, prism_seg(p, b, c)); + } + if (bits & 2) != 0 { + d = min(d, prism_seg(p, c, a)); + } + if (bits & 4) != 0 { + d = min(d, prism_seg(p, a, b)); + } + return d / px; +} + +// La rotation inverse de `world_to_plane` : du repere du plan a celui de la camera. +fn plane_to_world(v: vec3, f: ModelFrame) -> vec3 { + let x = v.x * f.c.z - v.y * f.s.z; + let y = v.x * f.s.z + v.y * f.c.z; + let z = -x * f.s.y + v.z * f.c.y; + return vec3(x * f.c.y + v.z * f.s.y, y * f.c.x - z * f.s.x, y * f.s.x + z * f.c.x); +} + +// L'uv de sortie du point ou le rayon (q, d) du modele retombe sur le plan, (-1, -1) s'il n'y +// retombe pas (cf. HLSL). +fn screen_uv(q: vec3, d: vec3, nz: vec3, hz: f32, tip: vec3, unit: f32, f: ModelFrame) -> vec2 { + let denom = dot(d, nz); + if denom > -1e-4 { + return vec2(-1.0); + } + let g = q + d * ((hz - dot(q, nz)) / denom); + let w = plane_to_world(tip + unit * model_to_plane(g, f), f); + let k = 1.0 - w.z / layer.src.z; + if k < 1e-3 { + return vec2(-1.0); + } + return layer.dst.xy + ((w.xy + layer.mb.zw) / k - layer.src.xy) / layer.quad_px * layer.dst.zw; +} + +// L'image composee en `uv` : sa copie au binding 1, en lineaire ; hors de l'image, l'ecran sombre +// du studio. +fn screen_at(uv: vec2) -> vec3 { + if any(uv < vec2(0.0)) || any(uv > vec2(1.0)) { + return SCULPT_SCREEN * 0.2; + } + return pow(textureSampleLevel(texY, samp, uv, 0.0).rgb, vec3(2.2)); +} + +// L'image composee la ou le rayon retombe sur le plan (cf. HLSL). +fn prism_screen(q: vec3, d: vec3, nz: vec3, hz: f32, tip: vec3, unit: f32, f: ModelFrame) -> vec3 { + return screen_at(screen_uv(q, d, nz, hz, tip, unit, f)); +} + +fn prism_shade(p: vec3, n: vec3, rd: vec3, h: i32, px: f32, l: vec3, fill: vec3, + nz: vec3, hz: f32, tip: vec3, unit: f32, f: ModelFrame) -> vec3 { + let shape = prism_shape(); + let facet = vec3(prism.tris[4 * h + 1].w, prism.tris[4 * h + 2].w, prism.tris[4 * h + 3].w); + let F = prism_fresnel(saturate(-dot(rd, n)), PRISM_IOR); + let zb = -model_thick(); + var trans = vec3(0.0); + for (var ch = 0; ch < 3; ch++) { + // Le rouge plie le moins, le bleu le plus ; le canal passe par un masque (comme au HLSL). + let mask = vec3(f32(ch == 0), f32(ch == 1), f32(ch == 2)); + let ior = PRISM_IOR + PRISM_DISPERSION * f32(ch - 1); + var d = refract(rd, n, 1.0 / ior); + var pos = p; + var thr = 1.0; + var acc = 0.0; + for (var b = 0; b < PRISM_BOUNCES; b++) { + let hit = prism_trace(pos, d, shape); + let hh = i32(hit.y); + let th = hit.x; + var tb = 1e9; + if d.z < -1e-6 { + tb = (zb - pos.z) / d.z; + } + if tb <= th { + let qb = pos + d * tb; + let d2 = refract(d, vec3(0.0, 0.0, 1.0), ior); + if dot(d2, d2) < 1e-8 { + pos = qb; + d = reflect(d, vec3(0.0, 0.0, 1.0)); + continue; + } + acc = acc + thr * dot(prism_screen(qb, d2, nz, hz, tip, unit, f), mask); + thr = 0.0; + break; + } + if hh < 0 { + break; + } + pos = pos + d * th; + let rn = prism.tris[4 * hh + 3].xyz; + if (prism_flags(hh) & 1) == 1 && pos.z < 0.0 { + acc = acc + thr * dot(s_lin(0.03, 0.06, 0.37) * 0.35, mask); + thr = 0.0; + break; + } + let dout = refract(d, -rn, ior); + if dot(dout, dout) < 1e-8 { + d = reflect(d, -rn); + continue; + } + let fi = prism_fresnel(saturate(dot(dout, rn)), ior); + acc = acc + thr * (1.0 - fi) * dot(model_env(dout, 0.05, l, fill), mask); + thr = thr * fi; + d = reflect(d, -rn); + } + trans = trans + acc * mask; + } + var col = F * model_env(reflect(rd, n), 0.05, l, fill) + (1.0 - F) * trans + facet * PRISM_GLOW; + let fold = 1.0 - smoothstep(0.2, 1.0, prism_edge_px(p, h, px)); + col = mix(col, s_lin(0.96, 0.99, 1.0) * 1.3, PRISM_FOLD * fold * (0.35 + 0.65 * saturate(dot(n, l)))); + return model_tonemap(col); +} + +// ---- Le verre de Glass Lens ---- (cf. HLSL) + +// Ce que le verre ajoute a ce qu'il transmet : Fresnel, l'eclat de la lampe, le liseré de lumiere. +fn glass_light(trans: vec3, n: vec3, rd: vec3, L: vec3, fall: f32, l: vec3, + fill: vec3) -> vec3 { + let c = saturate(-dot(rd, n)); + let F = prism_fresnel(c, GLASS_IOR); + let ndh = saturate(dot(n, normalize(L - rd))); + let ndl = dot(n, L); + let key = s_lin(1.0, 0.97, 0.93) * 2.1 * fall; + let spe = (pow(ndh, 900.0) * 5.0 + pow(ndh, 40.0) * 0.12) * saturate(ndl * 4.0); + let edge = pow(1.0 - c, 4.0) * (0.25 + 0.75 * saturate(ndl + 0.3)); + let col = (1.0 - F) * (trans * s_lin(0.97, 0.985, 1.0) + vec3(GLASS_LIFT)) + + F * model_env(reflect(rd, n), 0.03, l, fill); + return col + key * spe + s_lin(0.92, 0.96, 1.0) * edge * 0.6; +} + +// L'image en `uv`, moyennee le long de la refraction depuis `uv0` (cf. HLSL). +fn glass_smear(uv: vec2, uv0: vec2) -> vec3 { + let s = select(vec2(0.0), (uv - uv0) * GLASS_SMEAR, all(uv0 >= vec2(0.0))); + return (screen_at(uv) + screen_at(uv - s) + screen_at(uv + s)) / 3.0; +} + +fn glass_shade(q: vec3, n: vec3, rd: vec3, L: vec3, fall: f32, l: vec3, + fill: vec3, nz: vec3, hz: f32, tip: vec3, unit: f32, f: ModelFrame) -> vec3 { + let zb = -model_thick(); + let uv0 = screen_uv(q, rd, nz, hz, tip, unit, f); + var trans = vec3(0.0); + for (var ch = 0; ch < 3; ch++) { + let mask = vec3(f32(ch == 0), f32(ch == 1), f32(ch == 2)); + let ior = GLASS_IOR + GLASS_DISPERSION * f32(ch - 1); + let d = refract(rd, n, 1.0 / ior); + var seen = model_env(d, 0.05, l, fill); + if d.z < -1e-4 { + let qb = q + d * ((zb - q.z) / d.z); + let d2 = refract(d, vec3(0.0, 0.0, 1.0), ior); + if dot(d2, d2) < 1e-8 { + seen = model_env(reflect(d, vec3(0.0, 0.0, 1.0)), 0.05, l, fill); + } else { + seen = glass_smear(screen_uv(qb, d2, nz, hz, tip, unit, f), uv0); + } + } + trans = trans + seen * mask; + } + return pow(saturate(glass_light(trans, n, rd, L, fall, l, fill)), vec3(1.0 / 2.2)); +} + // Les passes de la boucle unique de `cursor_model` (cf. HLSL : un seul appel de `model_eval`). const STAGE_MARCH: i32 = 0; const STAGE_NORMAL: i32 = 1; const STAGE_AO: i32 = 2; const STAGE_SELF: i32 = 3; const STAGE_PLANE: i32 = 4; -const STAGE_DONE: i32 = 5; +const STAGE_EDGE: i32 = 5; +const STAGE_SHADE: i32 = 6; +const STAGE_DONE: i32 = 7; + +const MODEL_SUBPIXEL = array, 3>(vec2(0.35, 0.2), vec2(-0.35, 0.2), vec2(0.0, -0.4)); fn model_tetra(k: i32) -> vec3 { let b = vec3(f32(((k + 3) >> 1u) & 1), f32((k >> 1u) & 1), f32(k & 1)); @@ -1199,7 +1611,7 @@ fn cursor_model(local: vec2) -> vec4 { let hi = vec3(layer.color.rg + sprite_size(), layer.trail_a.z); let dw = vec3(local + layer.src.xy, -persp); - let dlen = length(dw); + var dlen = length(dw); // Plan translate de mb.zw dans le repere camera (camera reelle ; 0 sous un angle fixe). let ro = plane_to_model((world_to_plane(vec3(-layer.mb.z, -layer.mb.w, persp), f) - tip) / unit, f); let rd = plane_to_model(world_to_plane(dw / dlen, f), f); @@ -1210,15 +1622,22 @@ fn cursor_model(local: vec2) -> vec4 { let stride = select(1.0, 0.85, sculpt_id() > 0); let lamp = vec3(layer.color.rg + sprite_size() * 0.5, 0.0) + l * SCULPT_LAMP_DIST; - let tb = ray_box(ro, rd, lo - vec3(0.02), hi + vec3(0.02)); + var ray = rd; + var tb = ray_box(ro, ray, lo - vec3(0.02), hi + vec3(0.02)); var stage = select(STAGE_DONE, STAGE_MARCH, tb.x < tb.y && tb.y > 0.0); var plane_next = stage == STAGE_DONE; + // Prism Glow : le cristal, trace au depart de chaque rayon (`tc` son point, `hc` sa facette). + var need_tc = stage == STAGE_MARCH; + var tc = 1e9; + var hc = -1; var k = 0; var t = max(tb.x, 0.0); var best = 1e9; var t_best = t; + var d_best = 0.0; var mat = 0.0; var cov = 0.0; + var cov_s = 0.0; var q = vec3(0.0); var n = vec3(0.0); var L = vec3(0.0); @@ -1230,10 +1649,19 @@ fn cursor_model(local: vec2) -> vec4 { var inside = 0.0; var contact = 0.0; var dropped = 0.0; - for (var it = 0; it < 180; it++) { + var sub = 0; + var shaded = 0; + var acc = vec4(0.0); + for (var it = 0; it < 500; it++) { + if need_tc { + need_tc = false; + let pr = prism_primary(ro, ray); + tc = pr.x; + hc = i32(pr.y); + } if plane_next { plane_next = false; - stage = STAGE_DONE; + stage = select(STAGE_DONE, STAGE_SHADE, cov > 0.0); let denom = dot(rd, nz); if cov < 1.0 && denom < -1e-4 { g = ro + rd * ((hz - dot(ro, nz)) / denom); @@ -1246,11 +1674,50 @@ fn cursor_model(local: vec2) -> vec4 { } } } + if stage == STAGE_SHADE { + if cov_s > 0.0 { + var c: vec3; + let tl = lamp - q; + let fall = SCULPT_LAMP_DIST * SCULPT_LAMP_DIST / dot(tl, tl); + if abs(mat - PRISM_MAT) < 0.5 { + c = prism_shade(q, n, ray, hc, t_best / dlen, l, fill, nz, hz, tip, unit, f); + } else if mat > GLASS_MAT - 0.5 { + c = glass_shade(q, n, ray, normalize(tl), fall, l, fill, nz, hz, tip, unit, f); + } else { + c = model_shade(q, n, ray, normalize(tl), fall, sh, ao, mat, l, fill); + } + acc = acc + vec4(c * cov_s, cov_s); + } + shaded++; + stage = STAGE_DONE; + if sub > 0 { + let dws = vec3(local + MODEL_SUBPIXEL[3 - sub] + layer.src.xy, -persp); + sub--; + dlen = length(dws); + ray = plane_to_model(world_to_plane(dws / dlen, f), f); + tb = ray_box(ro, ray, lo - vec3(0.02), hi + vec3(0.02)); + cov_s = 0.0; + stage = STAGE_SHADE; + if tb.x < tb.y && tb.y > 0.0 { + stage = STAGE_MARCH; + k = 0; + t = max(tb.x, 0.0); + best = 1e9; + t_best = t; + need_tc = true; + } + } + continue; + } var pos: vec3; if stage == STAGE_MARCH { - pos = ro + rd * t; + pos = ro + ray * t; } else if stage == STAGE_NORMAL { pos = q + 0.002 * model_tetra(k); + } else if stage == STAGE_EDGE { + let side = normalize(cross(n, ray)); + let e = select(side, cross(n, side), k >= 2) * select(1.0, -1.0, k % 2 == 1); + pos = q + e * 0.5 * t_best / dlen; } else if stage == STAGE_AO { pos = q + (0.01 + 0.0175 * f32(k)) * SCULPT_SCALE * n; } else if stage == STAGE_SELF { @@ -1264,22 +1731,57 @@ fn cursor_model(local: vec2) -> vec4 { let d = m.x; if stage == STAGE_MARCH { let fp = t / dlen; - let hit = d < 0.1 * fp; + var hit = d < 0.1 * fp; if hit || d / fp < best { best = select(d / fp, 0.0, hit); t_best = t; + d_best = d; mat = m.y; } t = t + d * stride; k++; + if !hit && t >= tc { + // Le pas depasse le cristal : c'est lui que le rayon touche, avant le serti. + hit = true; + best = 0.0; + t_best = tc; + d_best = 0.0; + mat = PRISM_MAT; + } if hit || t > tb.y || k == 96 { - cov = saturate(1.0 - best); - if cov > 0.0 { - q = ro + rd * t_best; - stage = STAGE_NORMAL; - k = 0; + cov_s = saturate(1.0 - best); + if shaded > 0 { + stage = STAGE_SHADE; + if cov_s > 0.0 { + q = ro + ray * t_best; + n = vec3(0.0); + stage = STAGE_NORMAL; + k = 0; + } } else { - plane_next = true; + cov = cov_s; + if cov > 0.0 { + q = ro + ray * t_best; + stage = STAGE_NORMAL; + k = 0; + } else { + plane_next = true; + } + } + if stage == STAGE_NORMAL && abs(mat - PRISM_MAT) < 0.5 { + // Le cristal : la normale de sa facette, ni occlusion ni ombre propre ; un bord + // reel a moins d'un pixel appelle les trois rayons de plus. + n = prism_normal(hc, ray); + k = 0; + if shaded > 0 { + stage = STAGE_SHADE; + } else { + if prism_edge_px(q, hc, t_best / dlen) < 0.75 { + sub = 3; + } + stage = STAGE_DONE; + plane_next = true; + } } } } else if stage == STAGE_NORMAL { @@ -1287,6 +1789,25 @@ fn cursor_model(local: vec2) -> vec4 { k++; if k == 4 { n = normalize(n); + k = 0; + stage = select(STAGE_EDGE, STAGE_SHADE, shaded > 0); + } + } else if stage == STAGE_EDGE { + if m.y != mat || abs(d - d_best) > 0.02 * t_best / dlen { + sub = 3; + } + if k == 0 && mat > 7.5 && mat < PRISM_MAT + 0.5 { + // Le serti de Prism Glow au bord du cristal : son contour a moins d'un pixel. + let ps = prism_shape(); + if ps >= 0 { + let dout = prism_poly_dist(q.xy, PRISM_OUT_START[ps], PRISM_OUT_COUNT[ps]); + if abs(dout) < 0.75 * t_best / dlen { + sub = 3; + } + } + } + k++; + if k == 4 { stage = STAGE_AO; k = 0; ao = 0.0; @@ -1323,7 +1844,7 @@ fn cursor_model(local: vec2) -> vec4 { res = 1.0; k = 1; if !(tbp.x < tbp.y && tbp.y > 0.0) { - stage = STAGE_DONE; + stage = select(STAGE_DONE, STAGE_SHADE, cov > 0.0); } } else { res = min(res, MODEL_SOFTNESS * d / ts.x); @@ -1332,20 +1853,84 @@ fn cursor_model(local: vec2) -> vec4 { if res < 0.002 || ts.x > ts.y || k == 33 { res = saturate(res); dropped = 1.0 - res * res * (3.0 - 2.0 * res); - stage = STAGE_DONE; + stage = select(STAGE_DONE, STAGE_SHADE, cov > 0.0); } } } - var rgb = vec3(0.0); - if cov > 0.0 { - let tl = lamp - q; - let fall = SCULPT_LAMP_DIST * SCULPT_LAMP_DIST / dot(tl, tl); - rgb = model_shade(q, n, rd, normalize(tl), fall, sh, ao, mat, l, fill); + let w = 1.0 / f32(max(shaded, 1)); + // La lentille de Glass Lens laisse passer la lumiere : son ombre est plus claire. + let clear = select(1.0, GLASS_SHADOW, sculpt_id() > 10); + let shadow = inside * clear * max(dropped * MODEL_SHADOW_ALPHA, contact * MODEL_CONTACT_ALPHA); + let a = acc.a * w * layer.color.a; + return vec4(acc.rgb * w * layer.color.a, a + (1.0 - a) * shadow * layer.color.a); // premultiplie, ombre noire +} + +// ============ Verre a plat (mode 19) ============ (cf. HLSL, `cursor_glass`) : la copie de +// l'image composee au binding 1. +fn cursor_glass(local: vec2, pout: vec2) -> vec4 { + if pout.x < layer.dst_prev.x || pout.x > layer.dst_prev.x + layer.dst_prev.z || + pout.y < layer.dst_prev.y || pout.y > layer.dst_prev.y + layer.dst_prev.w { + return vec4(0.0); + } + let r = quad_inverse(local, layer.fx.xy, layer.fx.zw, layer.src_prev.xy, layer.src_prev.zw, layer.mb.x); + if r.z < 0.5 { + return vec4(0.0); + } + let m = layer.color.rg + saturate(r.xy) * sprite_size(); + let p = vec2(m.x, -m.y) / SCULPT_SCALE; + let px = 1.0 / max(layer.color.b * SCULPT_SCALE, 1e-3); + let shape = (sculpt_id() - 1) % 2; + var g = vec2(0.0); + var gd = vec2(0.0); + let e = 0.25 * px; + for (var k = 0; k < 3; k++) { + let o = select(select(vec2(0.0), vec2(0.0, e), k == 2), vec2(e, 0.0), k == 1); + let v = s_glass2(p + o, shape); + if k == 0 { + g = v; + } else if k == 1 { + gd.x = v.y; + } else { + gd.y = v.y; + } + } + let cov = saturate(0.5 - g.x / px); + if cov <= 0.0 { + return vec4(0.0); } - let shadow = inside * max(dropped * MODEL_SHADOW_ALPHA, contact * MODEL_CONTACT_ALPHA); + var out_dir = (gd - vec2(g.y)) / e; + out_dir = out_dir / max(length(out_dir), 1e-4); + let relief = s_glass_relief(g.y); + let dir = vec2(out_dir.x, -out_dir.y); + let n = normalize(vec3(-relief.y * dir, 1.0)); + let rd = vec3(0.0, 0.0, -1.0); + let rim = g.y > GLASS_LINE; + let depth = select(GLASS_TOP - SCULPT_HOVER + relief.x, GLASS_RIM - GLASS_LINE, rim); + var trans = vec3(0.0); + for (var ch = 0; ch < 3; ch++) { + let mask = vec3(f32(ch == 0), f32(ch == 1), f32(ch == 2)); + let ior = GLASS_IOR + GLASS_DISPERSION * f32(ch - 1); + let d = refract(rd, n, 1.0 / ior); + let d2 = refract(d, vec3(0.0, 0.0, 1.0), ior); + var off = d.xy * depth / max(-d.z, 0.05); + if !rim && dot(d2, d2) > 1e-8 { + off = off + d2.xy * GLASS_GAP_FLAT / max(-d2.z, 0.05); + } + off = off * SCULPT_SCALE * layer.src.xy; + trans = trans + glass_smear(pout + off, pout) * mask; + } + var col = glass_light(trans, n, rd, MODEL_LIGHT, 1.0, MODEL_LIGHT, MODEL_FILL); + let edge_k = saturate(1.0 - min(g.y, GLASS_LINE - g.y) / (0.25 * GLASS_LINE)); + let side = select(1.0, -1.0, g.y < 0.5 * GLASS_LINE); + let nl = normalize(vec3(side * edge_k * dir, 1.0)); + let hl = normalize(MODEL_LIGHT + vec3(0.0, 0.0, 1.0)); + let line_col = s_lin(0.05, 0.055, 0.07) * (0.6 + 0.5 * saturate(dot(nl, MODEL_LIGHT))) + + s_lin(1.0, 0.97, 0.93) * 0.8 * edge_k * pow(saturate(dot(nl, hl)), 40.0); + let in_line = saturate(0.5 + g.y / px) * saturate(0.5 - (g.y - GLASS_LINE) / px); + col = mix(col, line_col, in_line); let a = cov * layer.color.a; - return vec4(rgb * a, a + (1.0 - a) * shadow * layer.color.a); // premultiplie, ombre noire + return vec4(pow(saturate(col), vec3(1.0 / 2.2)) * a, a); // premultiplie } // ---- Impact du clic (mode 16) ---- @@ -2106,9 +2691,13 @@ fn fs_main(i: VsOut) -> @location(0) vec4 { let plane_px = layer.dst_prev.xy; let p = vec2(r.x, r.y) * plane_px - plane_px * 0.5; let rad = max(layer.radius_px, 0.0); - let d = select(sd_round_rect(p, plane_px * 0.5, rad), - sd_screen_under_bar(p, plane_px * 0.5, rad, layer.color.z), - layer.dst_prev.z > 0.5); + // Une branche et non un `select`, qui evaluerait les DEUX distances (trois SDF par pixel). + var d: f32; + if layer.dst_prev.z > 0.5 { + d = sd_screen_under_bar(p, plane_px * 0.5, rad, layer.color.z); + } else { + d = sd_round_rect(p, plane_px * 0.5, rad); + } // Slot d'un layout en bloc (`color.w` = rayon de ses coins, px ; 0 ailleurs, sans effet) : // `dst` EST le slot, dont le rect arrondi rogne le plan (`ScreenMask`, cf. HLSL). let tilt_a = (1.0 - smoothstep(0.0, 1.5, d)) @@ -2238,20 +2827,35 @@ fn fs_main(i: VsOut) -> @location(0) vec4 { frame_rgb = mix(frame_rgb, vec3(0.157, 0.784, 0.251), disc_cov(q, vec2(x0 + 2.0 * dx, bar * 0.5), dr)); let fa = cov * layer.color.a; return vec4(frame_rgb * fa, fa); // premultiplie - } else if layer.mode > 14.5 && layer.mode < 15.5 { - // Mode 15 -- curseur modelise (`cursor_model`). Clip « Clip to canvas » dans - // `dst_prev`, comme au mode 13. - if i.pout.x < layer.dst_prev.x || i.pout.x > layer.dst_prev.x + layer.dst_prev.z - || i.pout.y < layer.dst_prev.y || i.pout.y > layer.dst_prev.y + layer.dst_prev.w { + } else if layer.mode > 14.5 && layer.mode < 17.5 { + // Modes 15 a 17 -- les modeles 3D, que seul le pipeline `LAYER_MODELS` compile (cf. le + // haut du fichier). Le pipeline plat ne les recoit jamais ; s'il en recevait un, il ne + // dessinerait rien plutot qu'une ombre faite de leurs emplacements. + if !LAYER_MODELS { return vec4(0.0, 0.0, 0.0, 0.0); } - return cursor_model(i.local); - } else if layer.mode > 15.5 && layer.mode < 16.5 { - // Mode 16 -- impact du clic sous le curseur modelise (`cursor_impact`). - return cursor_impact(i.local); - } else if layer.mode > 16.5 && layer.mode < 17.5 { + if layer.mode < 15.5 { + // Mode 15 -- curseur modelise (`cursor_model`). Clip « Clip to canvas » dans + // `dst_prev`, comme au mode 13. + if i.pout.x < layer.dst_prev.x || i.pout.x > layer.dst_prev.x + layer.dst_prev.z + || i.pout.y < layer.dst_prev.y || i.pout.y > layer.dst_prev.y + layer.dst_prev.w { + return vec4(0.0, 0.0, 0.0, 0.0); + } + return cursor_model(i.local); + } + if layer.mode < 16.5 { + // Mode 16 -- impact du clic sous le curseur modelise (`cursor_impact`). + return cursor_impact(i.local); + } // Mode 17 -- cadre d'appareil modelise (`device_frame`). return device_frame(i.local); + } else if layer.mode > 18.5 { + // Mode 19 -- verre a plat de Glass Lens (`cursor_glass`) : un modele, que seul le + // pipeline `LAYER_MODELS` compile, comme les modes 15 a 17. + if !LAYER_MODELS { + return vec4(0.0, 0.0, 0.0, 0.0); + } + return cursor_glass(i.local, i.pout); } else if layer.mode > 17.5 { // Mode 18 -- ecran cadre floute en bloc (`screen_trail`). return screen_trail(i.pout); @@ -2282,9 +2886,14 @@ fn fs_main(i: VsOut) -> @location(0) vec4 { // cadre (`sd_screen_under_bar`, `mb.z` = la remontee du contour interieur, px). let halfsz = layer.quad_px * 0.5; let p = i.local - layer.quad_px * 0.5; - let d = select(sd_round_rect(p, halfsz, layer.radius_px), - sd_screen_under_bar(p, halfsz, layer.radius_px, layer.mb.z), - layer.mb.w > 0.5); + // Une branche et non un `select` : celui-ci evaluait les DEUX distances, soit trois SDF + // par pixel de l'ecran au lieu d'une -- 0,6 ms par frame 1080p sur une Radeon 610M. + var d: f32; + if layer.mb.w > 0.5 { + d = sd_screen_under_bar(p, halfsz, layer.radius_px, layer.mb.z); + } else { + d = sd_round_rect(p, halfsz, layer.radius_px); + } alpha *= 1.0 - smoothstep(0.0, 1.5, d); } diff --git a/crates/compositor/tests/animated_background.rs b/crates/compositor/tests/animated_background.rs index 748e605f3a..6ae4c214dc 100644 --- a/crates/compositor/tests/animated_background.rs +++ b/crates/compositor/tests/animated_background.rs @@ -28,21 +28,15 @@ use openscreen_compositor::frame_geometry::live_params_from_scene; use openscreen_compositor::live::Player; use openscreen_compositor::scene::Scene; +mod common; +use common::write_ppm; + const W: u32 = 960; const H: u32 = 540; /// Trois clips de 6 s du même fichier : le clip `k` lu à 3 s source tombe à `6k + 3` s de /// programme. La frame vidéo est la même, seul le temps programme change. const PROGRAMME_TIMES: [f32; 3] = [3.0, 9.0, 15.0]; -fn write_ppm(path: &std::path::Path, rgba: &[u8], w: u32, h: u32) -> std::io::Result<()> { - let mut out = Vec::with_capacity(rgba.len() / 4 * 3 + 32); - out.extend_from_slice(format!("P6\n{w} {h}\n255\n").as_bytes()); - for px in rgba.chunks_exact(4) { - out.extend_from_slice(&px[..3]); - } - std::fs::write(path, out) -} - /// Le fond, ouvert : `scene_json` y ajoute le mouvement puis referme l'accolade. const GRADIENT: &str = r##"{"kind":"gradient","angleDeg":135,"stops":["#2b3a67","#b8577f"]"##; diff --git a/crates/compositor/tests/common/mod.rs b/crates/compositor/tests/common/mod.rs new file mode 100644 index 0000000000..0dff0e533d --- /dev/null +++ b/crates/compositor/tests/common/mod.rs @@ -0,0 +1,19 @@ +#![allow(dead_code)] + +#[cfg(windows)] +mod nv12; +// Inutilisés par les tests qui ne prennent que `write_ppm`. +#[cfg(windows)] +#[allow(unused_imports)] +pub use nv12::{gpu, Nv12Frame}; + +/// PPM P6 — pas de dépendance à encoder, et ça s'ouvre dans n'importe quel +/// visionneur. Permet l'inspection à l'œil en plus de la comparaison de hash. +pub fn write_ppm(path: &std::path::Path, rgba: &[u8], w: u32, h: u32) -> std::io::Result<()> { + let mut out = Vec::with_capacity(rgba.len() / 4 * 3 + 32); + out.extend_from_slice(format!("P6\n{w} {h}\n255\n").as_bytes()); + for px in rgba.chunks_exact(4) { + out.extend_from_slice(&px[..3]); + } + std::fs::write(path, out) +} diff --git a/crates/compositor/tests/common/nv12.rs b/crates/compositor/tests/common/nv12.rs new file mode 100644 index 0000000000..66353350c8 --- /dev/null +++ b/crates/compositor/tests/common/nv12.rs @@ -0,0 +1,78 @@ +use openscreen_compositor::d3d::Gpu; +use openscreen_compositor::ffi::AVFrame; +use windows::core::Interface; +use windows::Win32::Graphics::Direct3D11::{ + ID3D11Texture2D, D3D11_BIND_SHADER_RESOURCE, D3D11_CPU_ACCESS_WRITE, + D3D11_MAPPED_SUBRESOURCE, D3D11_MAP_WRITE_DISCARD, D3D11_TEXTURE2D_DESC, D3D11_USAGE_DYNAMIC, +}; +use windows::Win32::Graphics::Dxgi::Common::{DXGI_FORMAT_NV12, DXGI_SAMPLE_DESC}; + +pub fn gpu() -> Option { + match Gpu::create(false) { + Ok(g) => Some(g), + Err(e) => { + eprintln!("pas de device D3D11 matériel ({e:#}) — test saute"); + None + } + } +} + +pub struct Nv12Frame { + frame: Box, + _tex: ID3D11Texture2D, +} + +impl Nv12Frame { + pub fn new( + gpu: &Gpu, + (w, h): (u32, u32), + luma: impl Fn(u32, u32) -> u8, + chroma: impl Fn(u32, u32) -> [u8; 2], + ) -> Nv12Frame { + let desc = D3D11_TEXTURE2D_DESC { + Width: w, + Height: h, + MipLevels: 1, + ArraySize: 1, + Format: DXGI_FORMAT_NV12, + SampleDesc: DXGI_SAMPLE_DESC { Count: 1, Quality: 0 }, + Usage: D3D11_USAGE_DYNAMIC, + BindFlags: D3D11_BIND_SHADER_RESOURCE.0 as u32, + CPUAccessFlags: D3D11_CPU_ACCESS_WRITE.0 as u32, + MiscFlags: 0, + }; + unsafe { + let mut tex: Option = None; + gpu.device.CreateTexture2D(&desc, None, Some(&mut tex)).expect("texture NV12"); + let tex = tex.expect("texture NV12"); + let mut m = D3D11_MAPPED_SUBRESOURCE::default(); + gpu.context.Map(&tex, 0, D3D11_MAP_WRITE_DISCARD, 0, Some(&mut m)).expect("Map"); + let pitch = m.RowPitch as usize; + let dst = m.pData as *mut u8; + for row in 0..h { + for col in 0..w { + *dst.add(row as usize * pitch + col as usize) = luma(col, row); + } + } + for by in 0..h / 2 { + for bx in 0..w / 2 { + let [cb, cr] = chroma(bx, by); + let uv = (h + by) as usize * pitch + 2 * bx as usize; + *dst.add(uv) = cb; + *dst.add(uv + 1) = cr; + } + } + gpu.context.Unmap(&tex, 0); + let mut frame: Box = Box::new(std::mem::zeroed()); + frame.data[0] = tex.as_raw() as *mut u8; + frame.data[1] = std::ptr::null_mut(); + frame.width = w as i32; + frame.height = h as i32; + Nv12Frame { frame, _tex: tex } + } + } + + pub fn as_ptr(&self) -> *const AVFrame { + &*self.frame as *const AVFrame + } +} diff --git a/crates/compositor/tests/cursor_model_render.rs b/crates/compositor/tests/cursor_model_render.rs index 86d28bf700..d2af8e1e5c 100644 --- a/crates/compositor/tests/cursor_model_render.rs +++ b/crates/compositor/tests/cursor_model_render.rs @@ -19,18 +19,14 @@ use openscreen_compositor::compositor::Compositor; use openscreen_compositor::config::Cfg; use openscreen_compositor::cursor::CursorTrack; use openscreen_compositor::d3d::Gpu; -use openscreen_compositor::ffi::AVFrame; use openscreen_compositor::frame_geometry::{ live_params_from_scene, plan_cursor, plan_frame, CursorPlacement, CursorPlanInput, FrameGeometryInput, }; use openscreen_compositor::scene::Scene; -use windows::core::Interface; -use windows::Win32::Graphics::Direct3D11::{ - ID3D11Texture2D, D3D11_BIND_SHADER_RESOURCE, D3D11_CPU_ACCESS_WRITE, - D3D11_MAPPED_SUBRESOURCE, D3D11_MAP_WRITE_DISCARD, D3D11_TEXTURE2D_DESC, D3D11_USAGE_DYNAMIC, -}; -use windows::Win32::Graphics::Dxgi::Common::{DXGI_FORMAT_NV12, DXGI_SAMPLE_DESC}; + +mod common; +use common::{gpu, Nv12Frame}; const SRC: (u32, u32) = (640, 360); /// Instant rendu : en plein palier de la région de zoom (0..10 s). @@ -67,90 +63,106 @@ fn is_centred(key: &str) -> bool { matches!(key, "text" | "resize-ew" | "not-allowed") } -fn gpu() -> Option { - match Gpu::create(false) { - Ok(g) => Some(g), - Err(e) => { - eprintln!("pas de device D3D11 matériel ({e:#}) — test saute"); - None - } - } -} - /// Deux teintes de contenu : un pixel identique sur les deux est du modèle opaque, un pixel qui /// change est le contenu, son ombre ou une frange antialiasée. #[derive(Clone, Copy)] enum Tint { Blue, Orange, + /// Pour l'œil, pas pour les masques : un damier clair et sombre sur un dégradé de teintes, où + /// la réfraction d'un verre se lit d'un coup. + Checker, + /// Pour l'œil aussi : une page claire, des lignes de texte et un bouton bleu, ce qu'une démo + /// filme le plus souvent. + Page, + /// La même page en thème sombre : là où un trait noir ne se voit plus. + DarkPage, } /// Une frame NV12 synthétique : `data[0]` = la texture, `data[1]` = tranche 0, comme une frame /// D3D11VA (le seul contrat que `Compositor::nv12_srvs` lit). Une teinte moyenne striée de barres /// plus sombres : rien d'aussi clair ni d'aussi sombre que le curseur, et surtout rien de NEUTRE, /// si bien qu'un pixel gris est le curseur et un pixel teinté assombri, son ombre. -struct FakeFrame { - frame: Box, - _tex: ID3D11Texture2D, -} - -impl FakeFrame { - fn new(gpu: &Gpu, tint: Tint) -> FakeFrame { - let (w, h) = SRC; - let (u, v) = match tint { - Tint::Blue => (150, 120), - Tint::Orange => (100, 170), - }; - let desc = D3D11_TEXTURE2D_DESC { - Width: w, - Height: h, - MipLevels: 1, - ArraySize: 1, - Format: DXGI_FORMAT_NV12, - SampleDesc: DXGI_SAMPLE_DESC { Count: 1, Quality: 0 }, - Usage: D3D11_USAGE_DYNAMIC, - BindFlags: D3D11_BIND_SHADER_RESOURCE.0 as u32, - CPUAccessFlags: D3D11_CPU_ACCESS_WRITE.0 as u32, - MiscFlags: 0, - }; - unsafe { - let mut tex: Option = None; - gpu.device.CreateTexture2D(&desc, None, Some(&mut tex)).expect("texture NV12"); - let tex = tex.expect("texture NV12"); - let mut m = D3D11_MAPPED_SUBRESOURCE::default(); - gpu.context.Map(&tex, 0, D3D11_MAP_WRITE_DISCARD, 0, Some(&mut m)).expect("Map"); - let pitch = m.RowPitch as usize; - let dst = m.pData as *mut u8; - for row in 0..h as usize { - for col in 0..w as usize { - let bar = row % 24 >= 8 && row % 24 < 12 && (col / 40) % 3 != 2; - *dst.add(row * pitch + col) = if bar { 90 } else { 150 }; - } +fn fake_frame(gpu: &Gpu, tint: Tint) -> Nv12Frame { + let (w, h) = (SRC.0 as usize, SRC.1 as usize); + let checker = matches!(tint, Tint::Checker); + let page = matches!(tint, Tint::Page | Tint::DarkPage); + let dark = matches!(tint, Tint::DarkPage); + let [u, v] = match tint { + Tint::Blue => [150, 120], + Tint::Orange | Tint::Checker => [100, 170], + Tint::Page | Tint::DarkPage => [128, 128], + }; + // Le bouton de la page : un rect bleu, texte blanc. + let button = |row: usize, col: usize| (150..190).contains(&row) && (330..470).contains(&col); + // Une ligne de texte sur trois rangées de 18 px, en mots de largeur variable. + let text = |row: usize, col: usize| { + let (line, y) = (row / 18, row % 18); + let word = (col + line * 37) / 23; + (5..11).contains(&y) && (col + line * 37) % 23 < 4 + (word * 7 + line * 3) % 15 && line % 5 != 4 + }; + Nv12Frame::new( + gpu, + SRC, + |col, row| { + let (row, col) = (row as usize, col as usize); + let bar = row % 24 >= 8 && row % 24 < 12 && (col / 40) % 3 != 2; + match (checker, (row / 10 + col / 10) % 2 == 0) { + (true, light) => if light { 200 } else { 60 }, + _ if page && button(row, col) => if text(row, col) { 235 } else { 105 }, + _ if page && dark => if text(row, col) { 200 } else { 28 }, + _ if page => if text(row, col) { 45 } else { 232 }, + (false, _) => if bar { 90 } else { 150 }, } - for row in 0..(h / 2) as usize { - for col in 0..w as usize { - let uv = (h as usize + row) * pitch + col; - *dst.add(uv) = if col % 2 == 0 { u } else { v }; - } + }, + |bx, by| { + let (bx, by) = (bx as usize, by as usize); + if checker { + [(70 + 2 * bx * 120 / w) as u8, (190 - by * 2 * 120 / h) as u8] + } else if page && button(2 * by, 2 * bx) { + [175, 105] + } else { + [u, v] } - gpu.context.Unmap(&tex, 0); - let mut frame: Box = Box::new(std::mem::zeroed()); - frame.data[0] = tex.as_raw() as *mut u8; - frame.data[1] = std::ptr::null_mut(); - frame.width = w as i32; - frame.height = h as i32; - FakeFrame { frame, _tex: tex } - } - } + }, + ) +} - fn as_ptr(&self) -> *const AVFrame { - &*self.frame as *const AVFrame - } +/// Des rayures verticales nettes, claires et sombres, de 7 px de source (11,2 px rendus : assez +/// serrées pour que des dizaines de bords croisent la lentille, assez larges pour qu'un décalage +/// de 5 px ne tombe pas sur le bord voisin) ; `inverse` les échange. Rendue sur les deux, une +/// lentille ajoute les mêmes reflets : le signe de l'écart des deux rendus dit quelle rayure se +/// voit en chaque pixel. +fn stripes(gpu: &Gpu, inverse: bool) -> Nv12Frame { + Nv12Frame::new(gpu, SRC, |col, _| if (col / 7 % 2 == 0) != inverse { 200 } else { 60 }, |_, _| [128, 128]) } -/// Les seize sprites livrés, au format `cursorSprites` de la scène. -fn sprites_json() -> String { - sprites_json_with(None) +/// Les bords des rayures sur chaque ligne (x en px) : où ce signe change entre deux pixels de +/// `keep` à 3 px au plus. Un écart faible (le trait, un reflet saturé, la transition d'un bord) ne +/// tranche pas. +fn stripe_edges(on: &[u8], on_inverse: &[u8], keep: impl Fn(usize) -> bool) -> Vec> { + (0..720) + .map(|y| { + let (mut edges, mut last) = (Vec::new(), None); + for x in 0..1280 { + if !keep(y * 1280 + x) { + last = None; + continue; + } + let d = luma(px(on, x as i32, y as i32)) - luma(px(on_inverse, x as i32, y as i32)); + if d.abs() < 30.0 { + continue; + } + if let Some((lx, light)) = last { + if light != (d > 0.0) && x - lx <= 3 { + edges.push((lx + x) as f32 / 2.0); + } + } + last = Some((x, d > 0.0)); + } + edges + }) + .collect() } /// Le dossier des curseurs livrés, `public/cursors`. @@ -161,29 +173,40 @@ fn cursors_dir() -> String { .replace('\\', "/") } -/// Les cinq thèmes d'origine et les hotspots de leur flèche et de leur main plates -/// (`CURSOR_THEMES`, `src/lib/cursor/cursorThemes.ts`, sur 32). -const SCULPTED: [(&str, [f32; 2], [f32; 2]); 5] = [ +/// Les thèmes d'origine et les hotspots de leur flèche et de leur main plates (`CURSOR_THEMES`, +/// `src/lib/cursor/cursorThemes.ts`, sur 32). +const SCULPTED: [(&str, [f32; 2], [f32; 2]); 6] = [ ("studio-ink", [6.2304, 2.0992], [12.848, 2.0704]), ("prism-glow", [6.3456, 2.0672], [11.968, 2.0352]), ("pop-coral", [10.4768, 2.1792], [12.3456, 2.0]), - ("pixel-candy", [7.3664, 2.0], [13.376, 1.9264]), + ("pixel-candy", [6.5, 2.0], [10.75, 2.0]), ("star-sprout", [4.7232, 2.1152], [13.1712, 2.0384]), + ("glass-lens", [6.2304, 2.0992], [12.848, 2.0704]), ]; -/// `sprites_json`, avec la flèche et la main plates du thème d'origine `sculpted` et leur curseur -/// sculpté, comme les passe `resolveCursorSprites` en 3D. -fn sprites_json_with(sculpted: Option<&str>) -> String { +/// Les thèmes de verre : le cristal de Prism Glow, la lentille de Glass Lens. +fn is_glass(theme: &str) -> bool { + matches!(theme, "prism-glow" | "glass-lens") +} + +/// Les seize sprites livrés au format `cursorSprites` de la scène, avec la flèche et la main +/// plates du thème d'origine `sculpted`, comme les passe `resolveCursorSprites` : leur curseur +/// sculpté en 3D, et le verre à plat de Glass Lens dans tous les cas. +fn sprites_json_with(sculpted: Option<&str>, model3d: bool) -> String { let dir = cursors_dir(); let theme = sculpted.map(|name| *SCULPTED.iter().find(|(t, ..)| *t == name).expect("thème d'origine")); let entries: Vec = STATES .iter() .map(|(key, [hx, hy])| match (theme, *key) { - (Some((name, [ax, ay], _)), "arrow") | (Some((name, _, [ax, ay])), "pointer") => format!( - r#""{key}":{{"path":"{dir}/{name}/{key}.png","hotspotX":{},"hotspotY":{},"sculpt":"{name}/{key}"}}"#, - ax / 32.0, - ay / 32.0 - ), + (Some((name, [ax, ay], _)), "arrow") | (Some((name, _, [ax, ay])), "pointer") => { + let sculpt = if model3d { format!(r#","sculpt":"{name}/{key}""#) } else { String::new() }; + let glass = if name == "glass-lens" { format!(r#","glass":"{name}/{key}""#) } else { String::new() }; + format!( + r#""{key}":{{"path":"{dir}/{name}/{key}.png","hotspotX":{},"hotspotY":{}{sculpt}{glass}}}"#, + ax / 32.0, + ay / 32.0 + ) + } _ => format!(r#""{key}":{{"path":"{dir}/default/{key}.png","hotspotX":{hx},"hotspotY":{hy}}}"#), }) .collect(); @@ -192,12 +215,17 @@ fn sprites_json_with(sculpted: Option<&str>) -> String { /// `model3d` : `None` = clé absente (payload d'avant le réglage). fn scene_json(rotation: &str, model3d: Option, theme: &str, motion_blur: f32, size: f32) -> String { - scene_json_with(rotation, model3d, theme, motion_blur, size, &sprites_json()) + scene_json_with(rotation, model3d, theme, motion_blur, size, &sprites_json_with(None, false)) } /// Un thème d'origine en 3D : sa flèche et sa main sculptées. fn sculpted_scene_json(rotation: &str, theme: &str, size: f32) -> String { - scene_json_with(rotation, Some(true), theme, 0.0, size, &sprites_json_with(Some(theme))) + scene_json_with(rotation, Some(true), theme, 0.0, size, &sprites_json_with(Some(theme), true)) +} + +/// Glass Lens, 3D éteinte : son verre à plat. +fn glass_scene_json(rotation: &str, size: f32) -> String { + scene_json_with(rotation, Some(false), "glass-lens", 0.0, size, &sprites_json_with(Some("glass-lens"), false)) } fn scene_json_with( @@ -287,7 +315,7 @@ struct Probe { unit: f32, } -fn render(comp: &Compositor, screen: &FakeFrame, json: &str, track: &CursorTrack) -> (Vec, Probe) { +fn render(comp: &Compositor, screen: &Nv12Frame, json: &str, track: &CursorTrack) -> (Vec, Probe) { let (rgba, probe) = render_any(comp, screen, json, track); (rgba, probe.expect("un curseur à dessiner")) } @@ -295,7 +323,7 @@ fn render(comp: &Compositor, screen: &FakeFrame, json: &str, track: &CursorTrack /// `render`, sans exiger de curseur (scène au curseur masqué). fn render_any( comp: &Compositor, - screen: &FakeFrame, + screen: &Nv12Frame, json: &str, track: &CursorTrack, ) -> (Vec, Option) { @@ -447,6 +475,20 @@ fn opaque_mask(on_blue: &[u8], on_orange: &[u8], bare_blue: &[u8]) -> Vec .collect() } +/// Les pixels que couvre un curseur de VERRE (le cristal de Prism Glow, qui laisse voir le contenu +/// et le réfracte) : opaques, ou changés sur les deux teintes sans y être plus sombres (une ombre +/// ne fait qu'assombrir ; le verre réfracte, reflète et luit). +fn covered_mask(on_blue: &[u8], on_orange: &[u8], bare_blue: &[u8], bare_orange: &[u8]) -> Vec { + let opaque = opaque_mask(on_blue, on_orange, bare_blue); + let px = |img: &[u8], i: usize| [img[i * 4], img[i * 4 + 1], img[i * 4 + 2], 255]; + (0..1280 * 720) + .map(|i| { + let lit = |on: &[u8], bare: &[u8]| px(on, i) != px(bare, i) && luma(px(on, i)) >= luma(px(bare, i)) - 2.0; + opaque[i] || (lit(on_blue, bare_blue) && lit(on_orange, bare_orange)) + }) + .collect() +} + fn iou(a: &[bool], b: &[bool]) -> f32 { let inter = a.iter().zip(b).filter(|(x, y)| **x && **y).count(); let union = a.iter().zip(b).filter(|(x, y)| **x || **y).count(); @@ -503,7 +545,7 @@ fn save(name: &str, rgba: &[u8]) { fn the_modelled_arrow_stands_on_the_screen_and_casts_its_shadow() { let Some(gpu) = gpu() else { return }; let comp = Compositor::new_sized(&gpu, 1280, 720).expect("compositor"); - let screen = FakeFrame::new(&gpu, Tint::Blue); + let screen = fake_frame(&gpu, Tint::Blue); let still = resting("still", false); let clicked = resting("clicked", true); @@ -566,7 +608,7 @@ fn the_modelled_arrow_stands_on_the_screen_and_casts_its_shadow() { fn every_state_keeps_its_art_and_its_footprint() { let Some(gpu) = gpu() else { return }; let comp = Compositor::new_sized(&gpu, 1280, 720).expect("compositor"); - let (blue, orange) = (FakeFrame::new(&gpu, Tint::Blue), FakeFrame::new(&gpu, Tint::Orange)); + let (blue, orange) = (fake_frame(&gpu, Tint::Blue), fake_frame(&gpu, Tint::Orange)); const SIZE: f32 = 5.0; let bare = render_any(&comp, &blue, &hidden_json("null"), &resting("bare", false)).0; let mut failures = Vec::new(); @@ -633,7 +675,7 @@ fn every_state_keeps_its_art_and_its_footprint() { fn the_tilt_turns_every_state_with_the_screen() { let Some(gpu) = gpu() else { return }; let comp = Compositor::new_sized(&gpu, 1280, 720).expect("compositor"); - let screen = FakeFrame::new(&gpu, Tint::Blue); + let screen = fake_frame(&gpu, Tint::Blue); for key in TESTED { let mask = |rotation: &str, x: f32, y: f32| -> (Vec, Probe) { let still = track_as(&format!("tilt-{key}"), Some(key), &[(0.0, x, y, false), (9.0, x, y, false)]); @@ -672,7 +714,7 @@ fn the_tilt_turns_every_state_with_the_screen() { fn without_the_model_the_cursor_renders_the_flat_sprite() { let Some(gpu) = gpu() else { return }; let comp = Compositor::new_sized(&gpu, 1280, 720).expect("compositor"); - let screen = FakeFrame::new(&gpu, Tint::Blue); + let screen = fake_frame(&gpu, Tint::Blue); let still = resting("flat-sprite", false); let clicked = resting("flat-sprite-clicked", true); let text = resting_as("flat-sprite-text", Some("text"), false); @@ -713,7 +755,7 @@ fn without_the_model_the_cursor_renders_the_flat_sprite() { fn a_moving_pointer_turns_towards_its_motion_and_a_centred_cursor_does_not() { let Some(gpu) = gpu() else { return }; let comp = Compositor::new_sized(&gpu, 1280, 720).expect("compositor"); - let screen = FakeFrame::new(&gpu, Tint::Blue); + let screen = fake_frame(&gpu, Tint::Blue); let json = scene_json(r#""iso""#, Some(true), "default", 0.0, 3.0); // En T, la piste mobile passe en x = 0,2 + 0,6 × T / (T + 1) = 0,6 : la piste immobile y est. let moving_as = |key: Option<&str>| { @@ -757,7 +799,7 @@ fn a_moving_pointer_turns_towards_its_motion_and_a_centred_cursor_does_not() { fn the_motion_blur_trail_draws_modelled_copies() { let Some(gpu) = gpu() else { return }; let comp = Compositor::new_sized(&gpu, 1280, 720).expect("compositor"); - let screen = FakeFrame::new(&gpu, Tint::Blue); + let screen = fake_frame(&gpu, Tint::Blue); let moving = track("trail", &[(0.0, 0.2, 0.45, false), (T + 1.0, 0.8, 0.45, false)]); let (rgba, _) = render(&comp, &screen, &scene_json("null", Some(true), "default", 1.0, 3.0), &moving); let (flat, _) = render(&comp, &screen, &scene_json("null", Some(false), "default", 1.0, 3.0), &moving); @@ -765,16 +807,18 @@ fn the_motion_blur_trail_draws_modelled_copies() { assert!(rgba != flat, "la traînée 3D est celle du sprite plat"); } -/// Les dix curseurs sculptés (cinq thèmes, flèche et main), dessinés par le shader et non -/// extrudés d'un PNG : chacun est là, sa pointe sur le hotspot, son corps en bas à droite de -/// celle-ci (y vers le bas : un modèle retourné finirait au-dessus), et il porte son ombre en -/// l'air. +/// Les curseurs sculptés (flèche et main de chaque thème), dessinés par le shader et non extrudés +/// d'un PNG : chacun est là, sa pointe sur le hotspot, son corps en bas à droite de celle-ci (y +/// vers le bas : un modèle retourné finirait au-dessus), et il porte son ombre en l'air. Le +/// cristal de Prism Glow est de verre : son corps compte ce qu'il couvre, et il doit laisser voir +/// le contenu (une bonne part de lui change avec la teinte de l'écran). #[test] fn the_sculpted_cursors_stand_at_the_hotspot() { let Some(gpu) = gpu() else { return }; let comp = Compositor::new_sized(&gpu, 1280, 720).expect("compositor"); - let (blue, orange) = (FakeFrame::new(&gpu, Tint::Blue), FakeFrame::new(&gpu, Tint::Orange)); + let (blue, orange) = (fake_frame(&gpu, Tint::Blue), fake_frame(&gpu, Tint::Orange)); let bare = render_any(&comp, &blue, &hidden_json("null"), &resting("sculpt-bare", false)).0; + let bare_orange = render_any(&comp, &orange, &hidden_json("null"), &resting("sculpt-bare", false)).0; let mut failures = Vec::new(); for (theme, ..) in SCULPTED { for state in ["arrow", "pointer"] { @@ -783,7 +827,8 @@ fn the_sculpted_cursors_stand_at_the_hotspot() { let (hover, p) = render(&comp, &blue, &json, &still); let hover_b = render(&comp, &orange, &json, &still).0; save(&format!("sculpt-{theme}-{state}"), &hover); - let mask = opaque_mask(&hover, &hover_b, &bare); + let glass = is_glass(theme); + let mask = if glass { covered_mask(&hover, &hover_b, &bare, &bare_orange) } else { opaque_mask(&hover, &hover_b, &bare) }; let (mut body, mut c, mut near) = (0usize, [0.0f32; 2], f32::MAX); for y in 0..720 { for x in 0..1280 { @@ -819,6 +864,101 @@ fn the_sculpted_cursors_stand_at_the_hotspot() { if shadow * 10 < body * 3 { failures.push(format!("{theme}/{state} : pas d'ombre en l'air ({shadow} px)")); } + if glass { + let through = (0..1280 * 720).filter(|&i| mask[i] && hover[i * 4..i * 4 + 3] != hover_b[i * 4..i * 4 + 3]).count(); + println!("{theme}/{state} : {through} px du corps laissent voir l'écran"); + if through * 10 < body * 3 { + failures.push(format!("{theme}/{state} : le cristal ne laisse voir l'écran que sur {through} px")); + } + } + } + } + assert!(failures.is_empty(), "{failures:#?}"); +} + +/// Glass Lens, 3D éteinte : son verre à plat (mode 19), sans ombre, donc tout pixel changé est le +/// sien. La flèche et la main sont là, pointe au hotspot et corps en bas à droite ; l'écran se +/// voit au travers (une bonne part du corps change avec sa teinte) ; le trait les cerne (des +/// pixels sombres, les mêmes sur les deux teintes) ; et le verre réfracte : sur des rayures, des +/// bords vus au travers de la lentille sont à plusieurs pixels de tout bord de la même ligne sans +/// elle. Un verre sans réfraction (indice 1) y éclaire l'image, mais n'en déplace aucun bord. +#[test] +fn the_flat_glass_lens_refracts_the_picture() { + let Some(gpu) = gpu() else { return }; + let comp = Compositor::new_sized(&gpu, 1280, 720).expect("compositor"); + let (blue, orange) = (fake_frame(&gpu, Tint::Blue), fake_frame(&gpu, Tint::Orange)); + let (light, dark) = (stripes(&gpu, false), stripes(&gpu, true)); + let bare = render_any(&comp, &blue, &hidden_json("null"), &resting("glass-bare", false)).0; + let bare_orange = render_any(&comp, &orange, &hidden_json("null"), &resting("glass-bare", false)).0; + let bare_light = render_any(&comp, &light, &hidden_json("null"), &resting("glass-bare", false)).0; + let bare_dark = render_any(&comp, &dark, &hidden_json("null"), &resting("glass-bare", false)).0; + let bare_edges = stripe_edges(&bare_light, &bare_dark, |_| true); + let mut failures = Vec::new(); + for state in ["arrow", "pointer"] { + let json = glass_scene_json("null", 5.0); + let still = resting_at(&format!("glass-flat-{state}"), Some(state), false, 0.5); + let (on_blue, p) = render(&comp, &blue, &json, &still); + let on_orange = render(&comp, &orange, &json, &still).0; + let (on_light, on_dark) = (render(&comp, &light, &json, &still).0, render(&comp, &dark, &json, &still).0); + save(&format!("glass-flat-{state}"), &on_blue); + save(&format!("glass-flat-{state}-stripes"), &on_light); + let rgb = |img: &[u8], i: usize| [img[i * 4], img[i * 4 + 1], img[i * 4 + 2], 255]; + let (mut body, mut c, mut near) = (0usize, [0.0f32; 2], f32::MAX); + let (mut through, mut line) = (0usize, 0usize); + for i in 0..1280 * 720 { + let (a, b) = (rgb(&on_blue, i), rgb(&on_orange, i)); + if a == rgb(&bare, i) && b == rgb(&bare_orange, i) { + continue; + } + let (x, y) = ((i % 1280) as f32, (i / 1280) as f32); + body += 1; + c = [c[0] + x, c[1] + y]; + near = near.min((x - p.tip[0]).hypot(y - p.tip[1])); + if a != b { + through += 1; + } else if luma(a) < 60.0 { + line += 1; + } + } + let c = [c[0] / body.max(1) as f32, c[1] / body.max(1) as f32]; + // Les bords vus au travers du verre (les pixels qu'il change), chacun à sa distance du bord + // le plus proche de la même ligne sans lui. + let in_glass = |i: usize| rgb(&on_light, i) != rgb(&bare_light, i) || rgb(&on_dark, i) != rgb(&bare_dark, i); + let shifts: Vec = stripe_edges(&on_light, &on_dark, in_glass) + .iter() + .zip(&bare_edges) + .flat_map(|(seen, under)| seen.iter().map(|e| under.iter().map(|b| (e - b).abs()).fold(f32::MAX, f32::min))) + .collect(); + let shifted = shifts.iter().filter(|s| **s >= 3.0).count(); + let widest = shifts.iter().copied().fold(0.0, f32::max); + let u = p.unit; + println!( + "glass-lens/{state} à plat : unité {u:.1} px, corps {body} px, pointe à {near:.1} px du hotspot, \ + centroïde {c:?} pour la pointe {:?}, {through} px laissent voir l'écran, {line} px de trait, \ + {} bords vus au travers, {shifted} décalés de 3 px ou plus, jusqu'à {widest:.1} px", + p.tip, + shifts.len() + ); + if (body as f32) < 0.12 * u * u { + failures.push(format!("glass-lens/{state} : {body} px de corps pour {u:.0} px d'unité")); + } + if near > 0.08 * u { + failures.push(format!("glass-lens/{state} : le verre est à {near:.1} px du hotspot")); + } + if !(c[0] > p.tip[0] && c[1] > p.tip[1] + 0.2 * u) { + failures.push(format!("glass-lens/{state} : corps en {c:?}, pas en bas à droite de {:?}", p.tip)); + } + if through * 10 < body * 5 { + failures.push(format!("glass-lens/{state} : l'écran ne se voit que sur {through} px")); + } + if line * 20 < body { + failures.push(format!("glass-lens/{state} : {line} px de trait seulement")); + } + // Mesuré : 23 et 67 bords décalés (jusqu'à 5,5 px) ; à l'indice 1, aucun (1 px au plus). + if shifted < 10 { + failures.push(format!( + "glass-lens/{state} : le verre ne décale que {shifted} bords de 3 px ou plus, au plus de {widest:.1} px" + )); } } assert!(failures.is_empty(), "{failures:#?}"); @@ -836,7 +976,7 @@ fn contact_sheets() { }; let Some(gpu) = gpu() else { return }; let comp = Compositor::new_sized(&gpu, 1280, 720).expect("compositor"); - let screen = FakeFrame::new(&gpu, Tint::Blue); + let screen = fake_frame(&gpu, Tint::Blue); const CELL: u32 = 300; let crop = |rgba: &[u8], p: Probe| { let img = image::RgbaImage::from_raw(1280, 720, rgba.to_vec()).expect("readback"); @@ -865,7 +1005,7 @@ fn contact_sheets() { } tilted.save(format!("{dir}/states-touch-flat-and-iso.png")).expect("planche"); - // Les dix curseurs sculptés : un thème par ligne, flèche et main en l'air sur l'écran à plat, + // Les curseurs sculptés : un thème par ligne, flèche et main en l'air sur l'écran à plat, // puis posées sur l'écran incliné. let cases = [("arrow", "null", false), ("pointer", "null", false), ("arrow", r#""iso""#, true), ("pointer", r#""iso""#, true)]; let mut sculpted = image::RgbaImage::new(4 * CELL, SCULPTED.len() as u32 * CELL); @@ -878,6 +1018,35 @@ fn contact_sheets() { } sculpted.save(format!("{dir}/sculpted.png")).expect("planche"); + // Glass Lens à la taille par défaut, sur une page claire puis sombre : son verre à plat (3D + // éteinte, sans clic : le sprite plat rapetisse au clic) sur l'écran droit puis incliné, et son + // modèle 3D dans les mêmes cases. + let checker = fake_frame(&gpu, Tint::Checker); + let page = fake_frame(&gpu, Tint::Page); + let dark_page = fake_frame(&gpu, Tint::DarkPage); + let mut glass = image::RgbaImage::new(4 * CELL, 4 * CELL); + for (row, screen) in [&page, &dark_page].into_iter().enumerate() { + for (col, (state, rotation, click)) in cases.iter().enumerate() { + let still = resting_as(&format!("sheet-glass-flat-{col}"), Some(state), false); + let (rgba, p) = render(&comp, screen, &glass_scene_json(rotation, 3.6), &still); + let y = (2 * row as u32 * CELL) as i64; + image::imageops::overlay(&mut glass, &crop(&rgba, p), (col as u32 * CELL) as i64, y); + let track = resting_as(&format!("sheet-glass-{col}"), Some(state), *click); + let (rgba, p) = render(&comp, screen, &sculpted_scene_json(rotation, "glass-lens", 3.6), &track); + image::imageops::overlay(&mut glass, &crop(&rgba, p), (col as u32 * CELL) as i64, y + CELL as i64); + } + } + glass.save(format!("{dir}/glass-lens.png")).expect("planche"); + for (bg, screen) in [("checker", &checker), ("page", &page), ("dark", &dark_page)] { + for state in ["arrow", "pointer"] { + let still = resting_as(&format!("big-glass-{state}"), Some(state), false); + let (rgba, _) = render(&comp, screen, &glass_scene_json("null", 10.0), &still); + save(&format!("big-glass-flat-{state}-{bg}"), &rgba); + let (rgba, _) = render(&comp, screen, &sculpted_scene_json("null", "glass-lens", 10.0), &still); + save(&format!("big-glass-3d-{state}-{bg}"), &rgba); + } + } + let (still, clicked) = (resting("big-still", false), resting("big-clicked", true)); for (name, rotation) in [("flat", "null"), ("iso", r#""iso""#)] { for (case, tr) in [("hover", &still), ("touch", &clicked)] { @@ -904,7 +1073,7 @@ fn bench_the_modelled_cursor_at_1080p() { gpu }; let comp = Compositor::new_sized(&gpu, 1920, 1080).expect("compositor"); - let screen = FakeFrame::new(&gpu, Tint::Blue); + let screen = fake_frame(&gpu, Tint::Blue); let still = resting("bench", false); let text = resting_as("bench-text", Some("text"), false); // Un geste rapide : la traînée prend ses 16 copies. diff --git a/crates/compositor/tests/cursor_tap_render.rs b/crates/compositor/tests/cursor_tap_render.rs index 537dd62a57..6d07519d2c 100644 --- a/crates/compositor/tests/cursor_tap_render.rs +++ b/crates/compositor/tests/cursor_tap_render.rs @@ -21,18 +21,14 @@ use openscreen_compositor::compositor::Compositor; use openscreen_compositor::config::Cfg; use openscreen_compositor::cursor::CursorTrack; use openscreen_compositor::d3d::Gpu; -use openscreen_compositor::ffi::AVFrame; use openscreen_compositor::frame_geometry::{ live_params_from_scene, plan_cursor, plan_frame, CursorPlacement, CursorPlanInput, FrameGeometryInput, }; use openscreen_compositor::scene::Scene; -use windows::core::Interface; -use windows::Win32::Graphics::Direct3D11::{ - ID3D11Texture2D, D3D11_BIND_SHADER_RESOURCE, D3D11_CPU_ACCESS_WRITE, - D3D11_MAPPED_SUBRESOURCE, D3D11_MAP_WRITE_DISCARD, D3D11_TEXTURE2D_DESC, D3D11_USAGE_DYNAMIC, -}; -use windows::Win32::Graphics::Dxgi::Common::{DXGI_FORMAT_NV12, DXGI_SAMPLE_DESC}; + +mod common; +use common::{gpu, Nv12Frame}; const SRC: (u32, u32) = (1280, 720); const OUT: (u32, u32) = (1280, 720); @@ -44,111 +40,58 @@ const TARGETS: [([f32; 2], f32); 2] = /// Au creux du contact (`regions::tap`). const CONTACT_S: f32 = 0.0495; -fn gpu() -> Option { - match Gpu::create(false) { - Ok(g) => Some(g), - Err(e) => { - eprintln!("pas de device D3D11 matériel ({e:#}) — test saute"); - None - } - } -} - /// Une maquette d'interface en NV12 (barre de titre, carte, lignes de texte, deux boutons), avec /// une pastille rouge de 5 texels de rayon au centre exact de chaque cible. Présentée comme une /// frame D3D11VA (cf. `cursor_model_render.rs`). La luma est évaluée au centre de chaque texel, la /// chroma au centre de chaque bloc 2×2. -struct MockFrame { - frame: Box, - _tex: ID3D11Texture2D, -} - -impl MockFrame { - fn new(gpu: &Gpu) -> MockFrame { - let (w, h) = SRC; - // (Y, Cb, Cr) de chaque pixel, en BT.709 limité. - let pixel = |fx: f32, fy: f32| -> (u8, u8, u8) { - let (x, y) = (fx as u32, fy as u32); - for ([tx, ty], _) in TARGETS { - let (cx, cy) = (tx * w as f32, ty * h as f32); - if (fx - cx).hypot(fy - cy) <= 5.0 { - return (63, 102, 240); // rouge - } - } - let button = |c: [f32; 2], hw: f32, hh: f32| { - let (dx, dy) = ((fx - c[0] * w as f32).abs(), (fy - c[1] * h as f32).abs()); - (dx - hw + 10.0).max(0.0).hypot((dy - hh + 10.0).max(0.0)) <= 10.0 - }; - if button(TARGETS[0].0, 130.0, 70.0) { - return (92, 180, 110); // bouton bleu - } - if button(TARGETS[1].0, 90.0, 26.0) { - return (150, 100, 90); // bouton vert - } - if y < 56 { - return (48, 132, 126); // barre de titre sombre - } - if x < 220 { - let item = (y - 56) % 44; - return if (14..22).contains(&item) && (24..150).contains(&x) { (150, 128, 128) } else { (222, 129, 127) }; - } - let card = x > 250 && x < 1240 && y > 90 && y < 690; - if card && (x == 251 || x == 1239 || y == 91 || y == 689) { - return (200, 128, 128); - } - // Lignes de « texte » dans la carte, à l'écart des boutons. - let line = (y % 36) >= 14 && (y % 36) < 20 && x > 280 && x < 1200 && (x / 97) % 5 != 4; - if card && line { - return (135, 128, 128); +fn mock_frame(gpu: &Gpu) -> Nv12Frame { + let (w, h) = SRC; + // (Y, Cb, Cr) de chaque pixel, en BT.709 limité. + let pixel = |fx: f32, fy: f32| -> (u8, u8, u8) { + let (x, y) = (fx as u32, fy as u32); + for ([tx, ty], _) in TARGETS { + let (cx, cy) = (tx * w as f32, ty * h as f32); + if (fx - cx).hypot(fy - cy) <= 5.0 { + return (63, 102, 240); // rouge } - (if card { 245 } else { 232 }, 128, 128) - }; - let desc = D3D11_TEXTURE2D_DESC { - Width: w, - Height: h, - MipLevels: 1, - ArraySize: 1, - Format: DXGI_FORMAT_NV12, - SampleDesc: DXGI_SAMPLE_DESC { Count: 1, Quality: 0 }, - Usage: D3D11_USAGE_DYNAMIC, - BindFlags: D3D11_BIND_SHADER_RESOURCE.0 as u32, - CPUAccessFlags: D3D11_CPU_ACCESS_WRITE.0 as u32, - MiscFlags: 0, + } + let button = |c: [f32; 2], hw: f32, hh: f32| { + let (dx, dy) = ((fx - c[0] * w as f32).abs(), (fy - c[1] * h as f32).abs()); + (dx - hw + 10.0).max(0.0).hypot((dy - hh + 10.0).max(0.0)) <= 10.0 }; - unsafe { - let mut tex: Option = None; - gpu.device.CreateTexture2D(&desc, None, Some(&mut tex)).expect("texture NV12"); - let tex = tex.expect("texture NV12"); - let mut m = D3D11_MAPPED_SUBRESOURCE::default(); - gpu.context.Map(&tex, 0, D3D11_MAP_WRITE_DISCARD, 0, Some(&mut m)).expect("Map"); - let pitch = m.RowPitch as usize; - let dst = m.pData as *mut u8; - for y in 0..h { - for x in 0..w { - *dst.add(y as usize * pitch + x as usize) = pixel(x as f32 + 0.5, y as f32 + 0.5).0; - } - } - for y in 0..h / 2 { - for x in 0..w / 2 { - let (_, cb, cr) = pixel(2.0 * x as f32 + 1.0, 2.0 * y as f32 + 1.0); - let uv = (h as usize + y as usize) * pitch + 2 * x as usize; - *dst.add(uv) = cb; - *dst.add(uv + 1) = cr; - } - } - gpu.context.Unmap(&tex, 0); - let mut frame: Box = Box::new(std::mem::zeroed()); - frame.data[0] = tex.as_raw() as *mut u8; - frame.data[1] = std::ptr::null_mut(); - frame.width = w as i32; - frame.height = h as i32; - MockFrame { frame, _tex: tex } + if button(TARGETS[0].0, 130.0, 70.0) { + return (92, 180, 110); // bouton bleu } - } - - fn as_ptr(&self) -> *const AVFrame { - &*self.frame as *const AVFrame - } + if button(TARGETS[1].0, 90.0, 26.0) { + return (150, 100, 90); // bouton vert + } + if y < 56 { + return (48, 132, 126); // barre de titre sombre + } + if x < 220 { + let item = (y - 56) % 44; + return if (14..22).contains(&item) && (24..150).contains(&x) { (150, 128, 128) } else { (222, 129, 127) }; + } + let card = x > 250 && x < 1240 && y > 90 && y < 690; + if card && (x == 251 || x == 1239 || y == 91 || y == 689) { + return (200, 128, 128); + } + // Lignes de « texte » dans la carte, à l'écart des boutons. + let line = (y % 36) >= 14 && (y % 36) < 20 && x > 280 && x < 1200 && (x / 97) % 5 != 4; + if card && line { + return (135, 128, 128); + } + (if card { 245 } else { 232 }, 128, 128) + }; + Nv12Frame::new( + gpu, + SRC, + |x, y| pixel(x as f32 + 0.5, y as f32 + 0.5).0, + |bx, by| { + let (_, cb, cr) = pixel(2.0 * bx as f32 + 1.0, 2.0 * by as f32 + 1.0); + [cb, cr] + }, + ) } /// `rotation` : valeur JSON du préset ; `hide` : le curseur masqué par la région (la caméra le @@ -232,7 +175,7 @@ fn cfg() -> Cfg { } /// La frame à `t`, et le pixel du contenu où le plan pose le curseur (la pointe), s'il y en a un. -fn render(comp: &Compositor, screen: &MockFrame, json: &str, track: &CursorTrack, t: f32) -> (Vec, Option<[f32; 2]>) { +fn render(comp: &Compositor, screen: &Nv12Frame, json: &str, track: &CursorTrack, t: f32) -> (Vec, Option<[f32; 2]>) { let scene = Scene::from_json(json).expect("scène valide"); let mut live = live_params_from_scene(&scene); live.has_webcam = false; @@ -308,7 +251,7 @@ fn marker(rgba: &[u8], near: [f32; 2]) -> [f32; 2] { fn the_tip_lands_on_the_marked_click_target() { let Some(gpu) = gpu() else { return }; let comp = Compositor::new_sized(&gpu, OUT.0, OUT.1).expect("compositor"); - let screen = MockFrame::new(&gpu); + let screen = mock_frame(&gpu); let track = gesture("contact"); for (name, rotation) in [("flat", "null"), ("iso", r#""iso""#), ("orbit", r#""orbit","focusMode":"auto""#)] { for (target, tc) in TARGETS { @@ -342,7 +285,7 @@ fn the_tip_lands_on_the_marked_click_target() { fn the_impact_ring_is_centred_on_the_click() { let Some(gpu) = gpu() else { return }; let comp = Compositor::new_sized(&gpu, OUT.0, OUT.1).expect("compositor"); - let screen = MockFrame::new(&gpu); + let screen = mock_frame(&gpu); let track = gesture("ring"); let (_, tc) = TARGETS[0]; let t = tc + 0.16; @@ -383,7 +326,7 @@ fn cursor_press_levels() { }; let Some(gpu) = gpu() else { return }; let comp = Compositor::new_sized(&gpu, OUT.0, OUT.1).expect("compositor"); - let screen = MockFrame::new(&gpu); + let screen = mock_frame(&gpu); let track = gesture("levels"); let frames = format!("{dir}/levels"); std::fs::create_dir_all(&frames).expect("dossier des frames"); @@ -447,7 +390,7 @@ fn cursor_tap_video() { }; let Some(gpu) = gpu() else { return }; let comp = Compositor::new_sized(&gpu, OUT.0, OUT.1).expect("compositor"); - let screen = MockFrame::new(&gpu); + let screen = mock_frame(&gpu); let track = gesture("video"); let frames = format!("{dir}/frames"); std::fs::create_dir_all(&frames).expect("dossier des frames"); diff --git a/crates/compositor/tests/device_frame_render.rs b/crates/compositor/tests/device_frame_render.rs index 23ababe5a3..0fb753cbae 100644 --- a/crates/compositor/tests/device_frame_render.rs +++ b/crates/compositor/tests/device_frame_render.rs @@ -19,16 +19,12 @@ use openscreen_compositor::compositor::Compositor; use openscreen_compositor::config::Cfg; use openscreen_compositor::cursor::CursorTrack; use openscreen_compositor::d3d::Gpu; -use openscreen_compositor::ffi::AVFrame; use openscreen_compositor::frame_geometry::{live_params_from_scene, plan_frame, FrameGeometry, FrameGeometryInput}; use openscreen_compositor::regions::TiltedQuad; use openscreen_compositor::scene::Scene; -use windows::core::Interface; -use windows::Win32::Graphics::Direct3D11::{ - ID3D11Texture2D, D3D11_BIND_SHADER_RESOURCE, D3D11_CPU_ACCESS_WRITE, D3D11_MAPPED_SUBRESOURCE, - D3D11_MAP_WRITE_DISCARD, D3D11_TEXTURE2D_DESC, D3D11_USAGE_DYNAMIC, -}; -use windows::Win32::Graphics::Dxgi::Common::{DXGI_FORMAT_NV12, DXGI_SAMPLE_DESC}; + +mod common; +use common::{gpu, Nv12Frame}; const W: u32 = 1280; const H: u32 = 720; @@ -51,16 +47,6 @@ const SHAPES: [(&str, (u32, u32), (u32, u32)); 4] = [ ("21:9", (1260, 540), (630, 270)), ]; -fn gpu() -> Option { - match Gpu::create(false) { - Ok(g) => Some(g), - Err(e) => { - eprintln!("pas de device D3D11 matériel ({e:#}) — test saute"); - None - } - } -} - /// Deux teintes de contenu : un pixel identique sur les deux est de l'appareil ou du fond, un /// pixel qui change est le métrage. #[derive(Clone, Copy)] @@ -70,62 +56,20 @@ enum Tint { } /// Une frame NV12 synthétique, striée pour que le métrage se distingue d'un aplat. -struct FakeFrame { - frame: Box, - _tex: ID3D11Texture2D, -} - -impl FakeFrame { - fn new(gpu: &Gpu, (w, h): (u32, u32), tint: Tint) -> FakeFrame { - let (u, v) = match tint { - Tint::Blue => (150, 120), - Tint::Orange => (100, 170), - }; - let desc = D3D11_TEXTURE2D_DESC { - Width: w, - Height: h, - MipLevels: 1, - ArraySize: 1, - Format: DXGI_FORMAT_NV12, - SampleDesc: DXGI_SAMPLE_DESC { Count: 1, Quality: 0 }, - Usage: D3D11_USAGE_DYNAMIC, - BindFlags: D3D11_BIND_SHADER_RESOURCE.0 as u32, - CPUAccessFlags: D3D11_CPU_ACCESS_WRITE.0 as u32, - MiscFlags: 0, - }; - unsafe { - let mut tex: Option = None; - gpu.device.CreateTexture2D(&desc, None, Some(&mut tex)).expect("texture NV12"); - let tex = tex.expect("texture NV12"); - let mut m = D3D11_MAPPED_SUBRESOURCE::default(); - gpu.context.Map(&tex, 0, D3D11_MAP_WRITE_DISCARD, 0, Some(&mut m)).expect("Map"); - let pitch = m.RowPitch as usize; - let dst = m.pData as *mut u8; - for row in 0..h as usize { - for col in 0..w as usize { - let bar = row % 24 >= 8 && row % 24 < 12 && (col / 40) % 3 != 2; - *dst.add(row * pitch + col) = if bar { 90 } else { 170 }; - } - } - for row in 0..(h / 2) as usize { - for col in 0..w as usize { - let uv = (h as usize + row) * pitch + col; - *dst.add(uv) = if col % 2 == 0 { u } else { v }; - } - } - gpu.context.Unmap(&tex, 0); - let mut frame: Box = Box::new(std::mem::zeroed()); - frame.data[0] = tex.as_raw() as *mut u8; - frame.data[1] = std::ptr::null_mut(); - frame.width = w as i32; - frame.height = h as i32; - FakeFrame { frame, _tex: tex } - } - } - - fn as_ptr(&self) -> *const AVFrame { - &*self.frame as *const AVFrame - } +fn fake_frame(gpu: &Gpu, size: (u32, u32), tint: Tint) -> Nv12Frame { + let [u, v] = match tint { + Tint::Blue => [150, 120], + Tint::Orange => [100, 170], + }; + Nv12Frame::new( + gpu, + size, + |col, row| { + let bar = row % 24 >= 8 && row % 24 < 12 && (col / 40) % 3 != 2; + if bar { 90 } else { 170 } + }, + |_, _| [u, v], + ) } /// Une scène d'un seul écran : `frame` = la valeur du réglage (`""` = clé absente). @@ -177,7 +121,7 @@ fn cfg() -> Cfg { cfg } -fn render(comp: &Compositor, screen: &FakeFrame, json: &str, track: Option<&CursorTrack>, t: f32) -> Vec { +fn render(comp: &Compositor, screen: &Nv12Frame, json: &str, track: Option<&CursorTrack>, t: f32) -> Vec { let scene = Scene::from_json(json).expect("scène valide"); let mut live = live_params_from_scene(&scene); live.has_webcam = false; @@ -461,8 +405,8 @@ fn each_device_draws_around_untouched_footage() { let Some(gpu) = gpu() else { return }; for (shape, out, src) in SHAPES { let comp = Compositor::new_sized(&gpu, out.0, out.1).expect("compositor"); - let blue = FakeFrame::new(&gpu, src, Tint::Blue); - let orange = FakeFrame::new(&gpu, src, Tint::Orange); + let blue = fake_frame(&gpu, src, Tint::Blue); + let orange = fake_frame(&gpu, src, Tint::Orange); for &device in &DEVICES { for rotation in ["null", r#""iso""#, ORBIT] { let json = scene_json(device, rotation, 0.0, NO_CURSOR, out); @@ -536,7 +480,7 @@ fn each_device_draws_around_untouched_footage() { fn the_frame_theme_repaints_the_body_and_nothing_else() { let Some(gpu) = gpu() else { return }; let comp = Compositor::new_sized(&gpu, W, H).expect("compositor"); - let screen = FakeFrame::new(&gpu, SRC, Tint::Blue); + let screen = fake_frame(&gpu, SRC, Tint::Blue); for &device in &["window", "laptop", "phone", "monitor"] { for rotation in ["null", r#""iso""#] { let light = @@ -569,7 +513,7 @@ fn the_frame_theme_repaints_the_body_and_nothing_else() { fn the_device_follows_the_tilt_and_the_orbit_camera() { let Some(gpu) = gpu() else { return }; let comp = Compositor::new_sized(&gpu, W, H).expect("compositor"); - let screen = FakeFrame::new(&gpu, SRC, Tint::Blue); + let screen = fake_frame(&gpu, SRC, Tint::Blue); for &device in &["laptop", "window"] { let flat = render(&comp, &screen, &scene_json(device, "null", 0.0, NO_CURSOR, (W, H)), None, T); assert!( @@ -595,7 +539,7 @@ fn the_device_follows_the_tilt_and_the_orbit_camera() { fn the_device_trails_with_the_screen_under_motion_blur() { let Some(gpu) = gpu() else { return }; let comp = Compositor::new_sized(&gpu, W, H).expect("compositor"); - let blue = FakeFrame::new(&gpu, SRC, Tint::Blue); + let blue = fake_frame(&gpu, SRC, Tint::Blue); // Zoom ×1,6 de 2 à 6 s : sa rampe d'entrée court de 1,14 à 2 s, vitesse au plus fort vers 1,26 s. // Le petit rect garde l'appareil dans l'image pendant la rampe. let json = |device: &str, rotation: &str, blur: f32| { @@ -606,7 +550,8 @@ fn the_device_trails_with_the_screen_under_motion_blur() { json }; for device in ["laptop", "window"] { - for rotation in ["null", r#""iso""#, ORBIT] { + // Un libellé par rotation : ORBIT est un fragment JSON, illégal dans un nom de fichier Windows. + for (label, rotation) in [("flat", "null"), ("iso", r#""iso""#), ("orbit", ORBIT)] { let case = format!("{device} {rotation}"); let still = |blur| render(&comp, &blue, &json(device, rotation, blur), None, 4.0); assert!(still(0.0) == still(1.0), "{case} : immobile, le flou ne doit rien changer"); @@ -635,7 +580,7 @@ fn the_device_trails_with_the_screen_under_motion_blur() { } println!("{case:<24} {smeared:>5} / {probes:>5} sondes du cadre changent avec le flou"); if let Some(dir) = out_dir("OPENSCREEN_DEVICE_OUT") { - let name = format!("trail-{device}-{}", rotation.trim_matches('"')); + let name = format!("trail-{device}-{label}"); save(&dir, &format!("{name}-sharp"), &sharp, (W, H)); save(&dir, &format!("{name}-blurred"), &blurred, (W, H)); } @@ -653,7 +598,7 @@ fn the_device_trails_with_the_screen_under_motion_blur() { fn the_frame_casts_the_contact_shadow() { let Some(gpu) = gpu() else { return }; let comp = Compositor::new_sized(&gpu, W, H).expect("compositor"); - let screen = FakeFrame::new(&gpu, SRC, Tint::Blue); + let screen = fake_frame(&gpu, SRC, Tint::Blue); for &device in &DEVICES { let on = render(&comp, &screen, &scene_json(device, "null", 0.7, NO_CURSOR, (W, H)), None, T); let off = render(&comp, &screen, &scene_json(device, "null", 0.0, NO_CURSOR, (W, H)), None, T); @@ -689,7 +634,7 @@ fn no_seam_lets_the_wallpaper_through() { let Some(gpu) = gpu() else { return }; for (res, out) in [("1080p", (1920u32, 1080u32)), ("4K", (3840, 2160))] { let comp = Compositor::new_sized(&gpu, out.0, out.1).expect("compositor"); - let screen = FakeFrame::new(&gpu, SRC, Tint::Blue); + let screen = fake_frame(&gpu, SRC, Tint::Blue); for frame in ["window", "laptop", "phone", "monitor"] { for roundness in [0.0f32, MAX_ROUND] { for rotation in ["null", r#""iso""#, ORBIT] { @@ -730,8 +675,8 @@ fn the_bezel_fills_the_corners_at_maximum_roundness() { let Some(gpu) = gpu() else { return }; let out = (1920u32, 1080u32); let comp = Compositor::new_sized(&gpu, out.0, out.1).expect("compositor"); - let blue = FakeFrame::new(&gpu, SRC, Tint::Blue); - let orange = FakeFrame::new(&gpu, SRC, Tint::Orange); + let blue = fake_frame(&gpu, SRC, Tint::Blue); + let orange = fake_frame(&gpu, SRC, Tint::Orange); for frame in ["window", "laptop", "phone", "monitor"] { for rotation in ["null", r#""iso""#, ORBIT] { let json = on_green(&with_roundness(&scene_json(frame, rotation, 0.0, NO_CURSOR, out), MAX_ROUND)); @@ -841,8 +786,8 @@ fn distance_to(mask: &[bool], w: usize, h: usize) -> Vec { fn the_device_shadow_follows_the_silhouette() { let Some(gpu) = gpu() else { return }; let comp = Compositor::new_sized(&gpu, W, H).expect("compositor"); - let blue = FakeFrame::new(&gpu, SRC, Tint::Blue); - let orange = FakeFrame::new(&gpu, SRC, Tint::Orange); + let blue = fake_frame(&gpu, SRC, Tint::Blue); + let orange = fake_frame(&gpu, SRC, Tint::Orange); let rect = CLIPS[0].1; // SCREEN_SHADOW_SPREAD_FRAC (40 px réglés contre un cadre 1080), en px de CETTE sortie. let spread = 40.0 / 1080.0 * W.min(H) as f32; @@ -904,7 +849,7 @@ fn contact_sheets() { }; let Some(gpu) = gpu() else { return }; let comp = Compositor::new_sized(&gpu, W, H).expect("compositor"); - let screen = FakeFrame::new(&gpu, SRC, Tint::Blue); + let screen = fake_frame(&gpu, SRC, Tint::Blue); let all: [&str; 4] = ["window", "laptop", "phone", "monitor"]; // Une planche par theme : quatre cadres x (plat, iso). for theme in ["light", "dark"] { @@ -924,7 +869,7 @@ fn contact_sheets() { // Le telephone dans les deux orientations de clip : debout et couche. for (name, out, src) in [("portrait", (720u32, 1280u32), (360u32, 640u32)), ("landscape", (W, H), SRC)] { let comp = Compositor::new_sized(&gpu, out.0, out.1).expect("compositor"); - let f = FakeFrame::new(&gpu, src, Tint::Blue); + let f = fake_frame(&gpu, src, Tint::Blue); for (label, rotation) in [("flat", "null"), ("iso", r#""iso""#)] { let rgba = render(&comp, &f, &scene_json("phone", rotation, 0.6, NO_CURSOR, out), None, T); save(&dir, &format!("phone-{name}-{label}"), &rgba, out); @@ -933,7 +878,7 @@ fn contact_sheets() { // Ratios inhabituels : rien ne degenere. for (shape, out, src) in SHAPES { let comp = Compositor::new_sized(&gpu, out.0, out.1).expect("compositor"); - let f = FakeFrame::new(&gpu, src, Tint::Blue); + let f = fake_frame(&gpu, src, Tint::Blue); for &device in &["laptop", "monitor"] { let rgba = render(&comp, &f, &scene_json(device, "null", 0.6, NO_CURSOR, out), None, T); save(&dir, &format!("shape-{}-{device}", shape.replace(':', "x")), &rgba, out); @@ -1008,7 +953,7 @@ fn v3_renders() { // 1. Paysage et portrait, plat et iso, clair et sombre : une image par cas, une planche par // clip et par thème. for (clip, rect, src) in CLIPS { - let f = FakeFrame::new(&gpu, src, Tint::Blue); + let f = fake_frame(&gpu, src, Tint::Blue); for theme in ["light", "dark"] { let mut sheet = image::RgbaImage::new(W * 2, H * 4); for (row, &frame) in all.iter().enumerate() { @@ -1025,7 +970,7 @@ fn v3_renders() { // 2. Gros plans de coins : les quatre coins de l'écran du portable et de la fenêtre, pris au // sommet de l'arc du métrage. - let screen = FakeFrame::new(&gpu, SRC, Tint::Blue); + let screen = fake_frame(&gpu, SRC, Tint::Blue); let arc_points = |g: &Geo| { let [w, h] = g.s_px; let k = g.radius * (1.0 - std::f32::consts::FRAC_1_SQRT_2); @@ -1097,7 +1042,7 @@ fn orbit_clip_frames() { }; let Some(gpu) = gpu() else { return }; let comp = Compositor::new_sized(&gpu, W, H).expect("compositor"); - let screen = FakeFrame::new(&gpu, SRC, Tint::Blue); + let screen = fake_frame(&gpu, SRC, Tint::Blue); // Le pointeur fait le tour de l'écran : l'œil de la caméra le suit sur son orbite. let mut samples = Vec::new(); for k in 0..=120 { @@ -1277,8 +1222,8 @@ fn the_border_keeps_its_thickness_around_every_corner() { let Some(gpu) = gpu() else { return }; let out = (1920u32, 1080u32); let comp = Compositor::new_sized(&gpu, out.0, out.1).expect("compositor"); - let blue = FakeFrame::new(&gpu, SRC, Tint::Blue); - let orange = FakeFrame::new(&gpu, SRC, Tint::Orange); + let blue = fake_frame(&gpu, SRC, Tint::Blue); + let orange = fake_frame(&gpu, SRC, Tint::Orange); let (mut worst, mut failures) = (0.0f32, Vec::new()); for frame in ["window", "phone"] { for (rlabel, roundness) in [("r0", 0.0f32), ("rdef", DEFAULT_ROUND), ("rmax", MAX_ROUND)] { @@ -1343,7 +1288,7 @@ fn only_the_inside_of_the_laptop_and_screen_bezel_follows_roundness() { let Some(gpu) = gpu() else { return }; let out = (1920u32, 1080u32); let comp = Compositor::new_sized(&gpu, out.0, out.1).expect("compositor"); - let blue = FakeFrame::new(&gpu, SRC, Tint::Blue); + let blue = fake_frame(&gpu, SRC, Tint::Blue); for frame in ["laptop", "monitor"] { let at = |r: f32| { let json = on_green(&with_roundness(&scene_json(frame, "null", 0.0, NO_CURSOR, out), r)); @@ -1386,8 +1331,8 @@ fn a_frame_looks_the_same_on_every_clip_ratio() { // La source a le ratio du clip : rien n'est recadré, rien n'est étiré. let src = if ar >= 1.0 { (w0, (w0 as f32 / ar).round() as u32) } else { ((h0 as f32 * ar).round() as u32, h0) }; let src = (src.0 & !1, src.1 & !1); - let blue = FakeFrame::new(&gpu, src, Tint::Blue); - let orange = FakeFrame::new(&gpu, src, Tint::Orange); + let blue = fake_frame(&gpu, src, Tint::Blue); + let orange = fake_frame(&gpu, src, Tint::Orange); let json = on_green(&with_roundness( &with_rect(&scene_json(frame, "null", 0.0, NO_CURSOR, out), contained_rect(ar)), DEFAULT_ROUND, @@ -1450,8 +1395,8 @@ fn orbit_json(frame: &str, zoom: f32, out: (u32, u32)) -> String { fn the_laptop_deck_never_covers_the_zoom_focus() { let Some(gpu) = gpu() else { return }; let comp = Compositor::new_sized(&gpu, W, H).expect("compositor"); - let blue = FakeFrame::new(&gpu, SRC, Tint::Blue); - let orange = FakeFrame::new(&gpu, SRC, Tint::Orange); + let blue = fake_frame(&gpu, SRC, Tint::Blue); + let orange = fake_frame(&gpu, SRC, Tint::Orange); let pointers = [ ("bas", (0.5f32, 0.98f32)), ("bas-gauche", (0.02, 0.98)), @@ -1550,7 +1495,7 @@ fn v4_renders() { let all: [&str; 4] = ["window", "laptop", "phone", "monitor"]; let out = (1920u32, 1080u32); let big = Compositor::new_sized(&gpu, out.0, out.1).expect("compositor"); - let screen = FakeFrame::new(&gpu, SRC, Tint::Blue); + let screen = fake_frame(&gpu, SRC, Tint::Blue); let img = |rgba: Vec, o: (u32, u32)| image::RgbaImage::from_raw(o.0, o.1, rgba).expect("readback"); // 1. Coins, au Roundness maximal : les quatre coins, 40 px de côté agrandis six fois, clair et @@ -1578,7 +1523,7 @@ fn v4_renders() { for (row, &frame) in all.iter().enumerate() { for (col, (label, ar)) in RATIOS.iter().enumerate() { let src = if *ar >= 1.0 { (1280u32, (1280.0 / ar).round() as u32 & !1) } else { ((720.0 * ar).round() as u32 & !1, 720u32) }; - let f = FakeFrame::new(&gpu, src, Tint::Blue); + let f = fake_frame(&gpu, src, Tint::Blue); let json = with_rect(&scene_json_themed(frame, theme, "null", 0.6, NO_CURSOR, out), contained_rect(*ar)); let json = with_roundness(&json, DEFAULT_ROUND); let i = img(render(&big, &f, &json, None, T), out); @@ -1597,7 +1542,7 @@ fn v4_renders() { for (row, pct) in [0.0f32, 0.5, 1.0].into_iter().enumerate() { for (col, (label, ar)) in ratios4.iter().enumerate() { let src = if *ar >= 1.0 { (1280u32, (1280.0 / ar).round() as u32 & !1) } else { ((720.0 * ar).round() as u32 & !1, 720u32) }; - let f = FakeFrame::new(&gpu, src, Tint::Blue); + let f = fake_frame(&gpu, src, Tint::Blue); let json = with_rect(&scene_json(frame, "null", 0.6, NO_CURSOR, out), contained_rect(*ar)); let json = with_roundness(&json, pct * 64.0 / 1080.0); let i = img(render(&big, &f, &json, None, T), out); @@ -1624,7 +1569,7 @@ fn v4_near_clip() { }; let Some(gpu) = gpu() else { return }; let comp = Compositor::new_sized(&gpu, W, H).expect("compositor"); - let screen = FakeFrame::new(&gpu, SRC, Tint::Blue); + let screen = fake_frame(&gpu, SRC, Tint::Blue); let track = parked_track("near", 0.5, 0.97); let json = with_rect(&scene_json("laptop", ORBIT, 0.6, SHOWN_CURSOR, (W, H)), CLIPS[0].1) .replace( @@ -1662,7 +1607,7 @@ fn bench_the_device_frame_at_1080p() { } let Some(gpu) = gpu() else { return }; let comp = Compositor::new_sized(&gpu, 1920, 1080).expect("compositor"); - let screen = FakeFrame::new(&gpu, SRC, Tint::Blue); + let screen = fake_frame(&gpu, SRC, Tint::Blue); let time = |json: &str| { let scene = Scene::from_json(json).expect("scène"); let mut live = live_params_from_scene(&scene); diff --git a/crates/compositor/tests/export_timing.rs b/crates/compositor/tests/export_timing.rs index 34d5e8de4d..cffda44694 100644 --- a/crates/compositor/tests/export_timing.rs +++ b/crates/compositor/tests/export_timing.rs @@ -34,6 +34,7 @@ use openscreen_compositor::compositor::Compositor; use openscreen_compositor::config::Cfg; use openscreen_compositor::d3d::Gpu; +use openscreen_compositor::export_control::{ExportCancelled, ExportControl}; use openscreen_compositor::gif_export::{self, GifExportParams}; use openscreen_compositor::pipeline::{self, ClipSource, ExportCodec, ExportParams}; use std::path::PathBuf; @@ -160,6 +161,113 @@ fn mp4_export_frame_count_follows_output_fps() { assert_eq!(probed, 120, "muxed file disagrees with the reported count"); } +/// Cancelling an MP4 mid-render stops the walk at the next frame, reports `ExportCancelled`, +/// and publishes nothing: the destination keeps what it held and no partial is left beside +/// it. On Windows that last check also proves the muxer closed its file, since an open +/// handle makes the partial undeletable. +#[test] +fn mp4_export_cancelled_mid_render_publishes_nothing() { + let Some(dir) = media_dir() else { + eprintln!("skipped: set OPENSCREEN_TEST_MEDIA"); + return; + }; + let out_dir = dir.join("cancelled_export"); + let _ = std::fs::remove_dir_all(&out_dir); + std::fs::create_dir(&out_dir).expect("output dir"); + let out = out_dir.join("out.mp4"); + std::fs::write(&out, b"previous export").expect("seed the destination"); + + let gpu = Gpu::create(false).expect("gpu"); + let params = ExportParams { + width: 640, + height: 360, + fps: Some(30), + codec: ExportCodec::H264, + bit_rate: None, + }; + let comp = Compositor::new_sized(&gpu, params.width, params.height).expect("compositor"); + let control = ExportControl::default(); + let mut reported = 0; + let error = pipeline::run_composited_multi_cancellable( + &[whole_clip(&dir)], + &out.to_string_lossy(), + &gpu, + &comp, + &Cfg::c8(), + ¶ms, + &mut |frames| { + reported = frames; + if frames == 10 { + control.cancel(); + } + }, + &control, + ) + .err() + .expect("a cancelled export must not succeed"); + + assert!(error.is::(), "expected a cancellation, got: {error:#}"); + assert_eq!(reported, 10, "the walk went on to frame {reported} after the cancel"); + assert_eq!(std::fs::read(&out).expect("destination"), b"previous export"); + let left: Vec<_> = std::fs::read_dir(&out_dir) + .expect("output dir") + .map(|entry| entry.expect("entry").file_name()) + .collect(); + assert_eq!(left, vec![std::ffi::OsString::from("out.mp4")], "a partial export was left behind"); + let _ = std::fs::remove_dir_all(&out_dir); +} + +/// Same for a GIF, whose frames are encoded on a worker pool: the cancel must reach the +/// export thread, release the workers rather than wait on them, and publish nothing. +#[test] +fn gif_export_cancelled_mid_render_publishes_nothing() { + let Some(dir) = media_dir() else { + eprintln!("skipped: set OPENSCREEN_TEST_MEDIA"); + return; + }; + let out_dir = dir.join("cancelled_gif"); + let _ = std::fs::remove_dir_all(&out_dir); + std::fs::create_dir(&out_dir).expect("output dir"); + let out = out_dir.join("out.gif"); + std::fs::write(&out, b"previous export").expect("seed the destination"); + + let gpu = Gpu::create(false).expect("gpu"); + let comp = Compositor::new_sized(&gpu, 320, 180).expect("compositor"); + let params = GifExportParams { + width: Some(320), + height: Some(180), + fps: Some(12), + loop_count: None, + dither: true, + }; + let control = ExportControl::default(); + let error = gif_export::export_gif_cancellable( + &[whole_clip(&dir)], + &out, + &gpu, + &comp, + &Cfg::c8(), + ¶ms, + &mut |frames| { + if frames == 10 { + control.cancel(); + } + }, + &control, + ) + .err() + .expect("a cancelled export must not succeed"); + + assert!(error.is::(), "expected a cancellation, got: {error:#}"); + assert_eq!(std::fs::read(&out).expect("destination"), b"previous export"); + let left: Vec<_> = std::fs::read_dir(&out_dir) + .expect("output dir") + .map(|entry| entry.expect("entry").file_name()) + .collect(); + assert_eq!(left, vec![std::ffi::OsString::from("out.gif")], "a partial export was left behind"); + let _ = std::fs::remove_dir_all(&out_dir); +} + /// The GIF must cover the WHOLE timeline, not just its first /// `out_fps / source_fps` slice. /// diff --git a/crates/compositor/tests/follow_camera_render.rs b/crates/compositor/tests/follow_camera_render.rs index 1dc245cbb6..d4905087b2 100644 --- a/crates/compositor/tests/follow_camera_render.rs +++ b/crates/compositor/tests/follow_camera_render.rs @@ -27,15 +27,11 @@ use openscreen_compositor::compositor::Compositor; use openscreen_compositor::config::Cfg; use openscreen_compositor::cursor::CursorTrack; use openscreen_compositor::d3d::Gpu; -use openscreen_compositor::ffi::AVFrame; use openscreen_compositor::frame_geometry::{live_params_from_scene, plan_frame, FrameGeometryInput}; use openscreen_compositor::scene::Scene; -use windows::core::Interface; -use windows::Win32::Graphics::Direct3D11::{ - ID3D11Texture2D, D3D11_BIND_SHADER_RESOURCE, D3D11_CPU_ACCESS_WRITE, - D3D11_MAPPED_SUBRESOURCE, D3D11_MAP_WRITE_DISCARD, D3D11_TEXTURE2D_DESC, D3D11_USAGE_DYNAMIC, -}; -use windows::Win32::Graphics::Dxgi::Common::{DXGI_FORMAT_NV12, DXGI_SAMPLE_DESC}; + +mod common; +use common::{gpu, Nv12Frame}; const SRC: (u32, u32) = (640, 360); /// Pas de la grille, en texels : des lignes sombres de 2 texels sur fond clair. @@ -43,70 +39,15 @@ const GRID: u32 = 32; /// En plein palier de la région (0..10 s), le cadreur posé. const T: f32 = 3.0; -fn gpu() -> Option { - match Gpu::create(false) { - Ok(g) => Some(g), - Err(e) => { - eprintln!("pas de device D3D11 matériel ({e:#}) — test saute"); - None - } - } -} - /// Une frame NV12 synthétique présentée comme une frame D3D11VA (cf. `cursor_model_render.rs`) : /// une grille sombre sur fond clair, chroma neutre. -struct GridFrame { - frame: Box, - _tex: ID3D11Texture2D, -} - -impl GridFrame { - fn new(gpu: &Gpu) -> GridFrame { - let (w, h) = SRC; - let desc = D3D11_TEXTURE2D_DESC { - Width: w, - Height: h, - MipLevels: 1, - ArraySize: 1, - Format: DXGI_FORMAT_NV12, - SampleDesc: DXGI_SAMPLE_DESC { Count: 1, Quality: 0 }, - Usage: D3D11_USAGE_DYNAMIC, - BindFlags: D3D11_BIND_SHADER_RESOURCE.0 as u32, - CPUAccessFlags: D3D11_CPU_ACCESS_WRITE.0 as u32, - MiscFlags: 0, - }; - unsafe { - let mut tex: Option = None; - gpu.device.CreateTexture2D(&desc, None, Some(&mut tex)).expect("texture NV12"); - let tex = tex.expect("texture NV12"); - let mut m = D3D11_MAPPED_SUBRESOURCE::default(); - gpu.context.Map(&tex, 0, D3D11_MAP_WRITE_DISCARD, 0, Some(&mut m)).expect("Map"); - let pitch = m.RowPitch as usize; - let dst = m.pData as *mut u8; - for row in 0..h { - for col in 0..w { - let line = col % GRID < 2 || row % GRID < 2; - *dst.add(row as usize * pitch + col as usize) = if line { 30 } else { 220 }; - } - } - for row in 0..(h / 2) as usize { - for col in 0..w as usize { - *dst.add((h as usize + row) * pitch + col) = 128; - } - } - gpu.context.Unmap(&tex, 0); - let mut frame: Box = Box::new(std::mem::zeroed()); - frame.data[0] = tex.as_raw() as *mut u8; - frame.data[1] = std::ptr::null_mut(); - frame.width = w as i32; - frame.height = h as i32; - GridFrame { frame, _tex: tex } - } - } - - fn as_ptr(&self) -> *const AVFrame { - &*self.frame as *const AVFrame - } +fn grid_frame(gpu: &Gpu) -> Nv12Frame { + Nv12Frame::new( + gpu, + SRC, + |col, row| if col % GRID < 2 || row % GRID < 2 { 30 } else { 220 }, + |_, _| [128, 128], + ) } /// Piste curseur écrite dans un sidecar temporaire, échantillonnée à 60 Hz sur `[0, dur]`. @@ -163,7 +104,7 @@ fn cfg() -> Cfg { /// La frame, et le quad que la géométrie partagée prévoit pour elle (centre compris). fn render( comp: &Compositor, - screen: &GridFrame, + screen: &Nv12Frame, scene: &Scene, track: &CursorTrack, ) -> (Vec, openscreen_compositor::regions::TiltedQuad, [f32; 2]) { @@ -236,7 +177,7 @@ fn bilinear(c: &[(f32, f32); 4], u: f32, v: f32) -> (f32, f32) { fn the_projective_warp_lands_every_line_where_the_camera_projects_it() { let Some(gpu) = gpu() else { return }; let comp = Compositor::new_sized(&gpu, 1280, 720).expect("compositor"); - let screen = GridFrame::new(&gpu); + let screen = grid_frame(&gpu); // Pointeur en haut à droite : caméra tournée à fond, le cas le plus projectif. let tr = track("corner", 10.0, &|_| (0.93, 0.08), &[]); let (rgba, quad, center) = render(&comp, &screen, &grid_scene(2.2), &tr); @@ -309,7 +250,7 @@ fn measure_line(rgba: &[u8], quad: &openscreen_compositor::regions::TiltedQuad, fn the_vertical_through_the_view_axis_stays_vertical() { let Some(gpu) = gpu() else { return }; let comp = Compositor::new_sized(&gpu, 1280, 720).expect("compositor"); - let screen = GridFrame::new(&gpu); + let screen = grid_frame(&gpu); for (name, at, scale) in [ ("tl", (0.1, 0.1), 1.8), ("br", (0.9, 0.92), 1.8), diff --git a/crates/compositor/tests/no_camera_stand_in.rs b/crates/compositor/tests/no_camera_stand_in.rs new file mode 100644 index 0000000000..d741f12e9c --- /dev/null +++ b/crates/compositor/tests/no_camera_stand_in.rs @@ -0,0 +1,109 @@ +//! Sans caméra, la preview compose partout avec la frame écran à la place de la webcam : le +//! décodeur remplaçant n'est plus ni avancé ni recherché (`live::open_webcam_or_stand_in`). +//! +//! Couvre les chemins qui composent : le seek en pause (`present_frame`), y compris au-delà de +//! la dernière image, la lecture libre (`step`), et la recomposition à l'arrêt (`recompose`). +//! Une frame webcam nulle les ferait tous rendre `false`, preview figée. Vérifie aussi que le +//! remplaçant ne bouge pas, `seek_active` compris : composer suffit à ne rien prouver, puisqu'il +//! ouvre le même fichier valide que l'écran. +//! +//! La CI ne le joue pas : il faut un GPU et une source vidéo, sinon il se saute. N'importe quel +//! MP4 d'au moins 4 s convient : +//! +//! ```sh +//! OPENSCREEN_LIVE_SOURCE=/chemin/source.mp4 cargo test -p openscreen-compositor --test no_camera_stand_in -- --nocapture +//! ``` + +#![cfg(any(windows, target_os = "macos"))] + +use openscreen_compositor::compositor::Compositor; +use openscreen_compositor::config; +use openscreen_compositor::d3d::Gpu; +use openscreen_compositor::frame_geometry::live_params_from_scene; +use openscreen_compositor::live::Player; +use openscreen_compositor::scene::Scene; + +fn scene_json(source: &str) -> String { + let s = source.replace('\\', "/"); + format!( + r##"{{ + "clips": [{{"screenPath":"{s}","webcamPath":"","sourceStartSec":0,"sourceEndSec":100000,"webcamOffsetSec":0,"hasAudio":false}}], + "layout": {{"preset":"no-webcam","webcamSize":1.0,"webcamShape":"rounded","webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false}}, + "effects": {{"padding":0.1,"blur":false,"shadow":0.0,"roundnessFrac":0,"motionBlur":0.0}}, + "background": {{"kind":"color","color":"#303030"}}, + "zoomRegions": [], + "speedRegions": [], + "cursor": {{"show":false,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":0,"clipToBounds":false,"theme":"default"}}, + "cropByClip": [null], + "output": {{"width":480,"height":270,"fps":30}} + }}"## + ) +} + +#[test] +fn every_compose_path_works_without_a_camera() { + let Ok(source) = std::env::var("OPENSCREEN_LIVE_SOURCE") else { + println!("SKIP: definir OPENSCREEN_LIVE_SOURCE (voir l'en-tete du fichier)."); + return; + }; + let gpu = Gpu::create(false).expect("device"); + let mut cfg = config::all().pop().expect("au moins une config"); + cfg.zoom = false; + cfg.layout_anim = false; + let comp = Compositor::new_sized(&gpu, 480, 270).expect("compositor"); + let scene = Scene::from_json(&scene_json(&source)).expect("scene valide"); + comp.set_live_params(live_params_from_scene(&scene)); + comp.set_scene(Some(scene.clone())); + comp.clear_cursor(); + + unsafe { + let mut player = Player::open(&source, "", &gpu).expect("ouvrir la source"); + assert!(!player.webcam_decoder_is_real(), "pas de caméra déclarée : décodeur remplaçant"); + player.set_programme_clock(Some(&scene), 0); + + let stand_in_time = player.webcam_time_sec(); + + assert!(player.present_frame(&comp, &cfg, 1.0).expect("seek"), "seek en pause"); + let start = player.screen_time_sec(); + + // Lecture libre à 2× sur une seconde d'horloge, au pas de 1/60 s. À l'EOF, `step` + // reboucle à 0 : la cible ne serait alors plus jamais atteinte, d'où l'arrêt sur un + // temps qui recule. + let mut target = start; + let mut composed = 0; + for _ in 0..60 { + target += 2.0 / 60.0; + loop { + let before = player.screen_time_sec(); + if !player.step(&comp, &cfg, target).expect("step") { + break; + } + assert!( + player.screen_time_sec() >= before, + "la source a rebouclé avant {target:.2} s : il faut une source d'au moins 4 s" + ); + composed += 1; + } + } + assert!(composed > 0, "la lecture libre n'a composé aucune frame"); + assert!( + player.screen_time_sec() > start + 1.5, + "la lecture libre a avancé de {:.3} s au lieu de ~2 s", + player.screen_time_sec() - start + ); + + assert!(player.recompose(&comp, &cfg).expect("recompose"), "recomposition à l'arrêt"); + + // Au-delà de la dernière image : la dernière image, composée. + assert!(player.present_frame(&comp, &cfg, 100_000.0).expect("seek fin"), "seek au-delà de la fin"); + + assert!(player.seek_active(0.5).expect("seek_active"), "seek_active"); + assert!(player.step(&comp, &cfg, 0.5).expect("step après seek_active"), "step après seek_active"); + + assert_eq!( + player.webcam_time_sec().to_bits(), + stand_in_time.to_bits(), + "le décodeur remplaçant a bougé : il ne doit être ni avancé ni recherché" + ); + } +} diff --git a/crates/compositor/tests/output_geometry_golden.rs b/crates/compositor/tests/output_geometry_golden.rs index 3e363b27a8..369d89f39c 100644 --- a/crates/compositor/tests/output_geometry_golden.rs +++ b/crates/compositor/tests/output_geometry_golden.rs @@ -36,6 +36,9 @@ use openscreen_compositor::live::Player; use openscreen_compositor::scene::Scene; use openscreen_compositor::config; +mod common; +use common::write_ppm; + /// Instant fixe dans la source. Un seek explicite (`present_frame`) plutôt que /// la lecture libre : le golden doit être reproductible à l'octet près. const AT_SEC: f64 = 2.0; @@ -85,17 +88,6 @@ fn gradient_energy(rgba: &[u8], w: usize, h: usize) -> (f64, f64) { (gx / nx.max(1) as f64, gy / ny.max(1) as f64) } -/// PPM P6 — pas de dépendance à encoder, et ça s'ouvre dans n'importe quel -/// visionneur. Permet l'inspection à l'œil en plus de la comparaison de hash. -fn write_ppm(path: &std::path::Path, rgba: &[u8], w: u32, h: u32) -> std::io::Result<()> { - let mut out = Vec::with_capacity(rgba.len() / 4 * 3 + 32); - out.extend_from_slice(format!("P6\n{w} {h}\n255\n").as_bytes()); - for px in rgba.chunks_exact(4) { - out.extend_from_slice(&px[..3]); - } - std::fs::write(path, out) -} - fn scene_json(screen: &str, webcam: &str, w: u32, h: u32) -> String { // Chemins en slashes : le JSON n'échappe pas les backslashes Windows. let (s, c) = (screen.replace('\\', "/"), webcam.replace('\\', "/")); @@ -103,7 +95,7 @@ fn scene_json(screen: &str, webcam: &str, w: u32, h: u32) -> String { r##"{{ "clips": [{{"screenPath":"{s}","webcamPath":"{c}","sourceStartSec":0,"sourceEndSec":30,"webcamOffsetSec":0,"hasAudio":true}}], "layout": {{"preset":"picture-in-picture","webcamSize":1.0,"webcamShape":"rounded","webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false}}, - "effects": {{"padding":0.1,"blur":true,"shadow":1.0,"roundnessPx":24,"motionBlur":0.0}}, + "effects": {{"padding":0.1,"blur":true,"shadow":1.0,"roundnessFrac":0.0222,"motionBlur":0.0}}, "background": {{"kind":"gradient","angleDeg":135,"stops":["#eaebed","#bcc0c6"]}}, "zoomRegions": [], "speedRegions": [], @@ -220,7 +212,7 @@ fn golden_side_by_side_webcam_is_not_stretched() { "clips": [{{"screenPath":"{s}","webcamPath":"{c}","sourceStartSec":0,"sourceEndSec":30,"webcamOffsetSec":0,"hasAudio":true}}], "layout": {{"preset":"dual-frame","webcamSize":1.0,"webcamShape":"rectangle","webcamMirror":false, "webcamRect":{{"x":0.66,"y":0.06,"width":0.31,"height":0.88}},"webcamReactiveZoom":false}}, - "effects": {{"padding":0.0,"blur":true,"shadow":1.0,"roundnessPx":24,"motionBlur":0.0}}, + "effects": {{"padding":0.0,"blur":true,"shadow":1.0,"roundnessFrac":0.0222,"motionBlur":0.0}}, "background": {{"kind":"gradient","angleDeg":135,"stops":["#b02a2a","#7a1414"]}}, "zoomRegions": [], "speedRegions": [], "cursor": {{"show":false,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":0,"clipToBounds":false,"theme":"default"}}, @@ -286,7 +278,7 @@ fn golden_cropped_clip_is_not_stretched() { "clips": [{{"screenPath":"{s}","webcamPath":"{c}","sourceStartSec":0,"sourceEndSec":30,"webcamOffsetSec":0,"hasAudio":true}}], "layout": {{"preset":"no-webcam","webcamSize":1.0,"webcamShape":"rectangle","webcamMirror":false, "screenRect":{{"x":{sx},"y":{sy},"width":{rect_w},"height":{rect_h}}},"webcamReactiveZoom":false}}, - "effects": {{"padding":0.0,"blur":true,"shadow":1.0,"roundnessPx":24,"motionBlur":0.0}}, + "effects": {{"padding":0.0,"blur":true,"shadow":1.0,"roundnessFrac":0.0222,"motionBlur":0.0}}, "background": {{"kind":"gradient","angleDeg":135,"stops":["#1e3a5f","#0d1b2a"]}}, "zoomRegions": [], "speedRegions": [], "cursor": {{"show":false,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":0,"clipToBounds":false,"theme":"default"}}, diff --git a/crates/compositor/tests/privacy_blur_under_zoom.rs b/crates/compositor/tests/privacy_blur_under_zoom.rs index 75d781890a..da7b3b8509 100644 --- a/crates/compositor/tests/privacy_blur_under_zoom.rs +++ b/crates/compositor/tests/privacy_blur_under_zoom.rs @@ -27,6 +27,9 @@ use openscreen_compositor::frame_geometry::live_params_from_scene; use openscreen_compositor::live::Player; use openscreen_compositor::scene::Scene; +mod common; +use common::write_ppm; + const W: u32 = 960; const H: u32 = 540; /// En plein palier du zoom : la région couvre 0..6 s. @@ -38,15 +41,6 @@ fn red_pixels(rgba: &[u8]) -> usize { rgba.chunks_exact(4).filter(|p| p[0] > 150 && p[1] < 80 && p[2] < 80).count() } -fn write_ppm(path: &std::path::Path, rgba: &[u8], w: u32, h: u32) -> std::io::Result<()> { - let mut out = Vec::with_capacity(rgba.len() / 4 * 3 + 32); - out.extend_from_slice(format!("P6\n{w} {h}\n255\n").as_bytes()); - for px in rgba.chunks_exact(4) { - out.extend_from_slice(&px[..3]); - } - std::fs::write(path, out) -} - /// `zoom` : `None` = pas de région, sinon (échelle, focus x, focus y, rotation JSON). fn scene_json(secret: &str, masked: bool, zoom: Option<(f32, f32, f32, &str)>) -> String { let s = secret.replace('\\', "/"); diff --git a/crates/compositor/tests/screen_trail_render.rs b/crates/compositor/tests/screen_trail_render.rs index f06383456d..88b659f99a 100644 --- a/crates/compositor/tests/screen_trail_render.rs +++ b/crates/compositor/tests/screen_trail_render.rs @@ -12,15 +12,11 @@ use openscreen_compositor::compositor::Compositor; use openscreen_compositor::config::Cfg; use openscreen_compositor::d3d::Gpu; -use openscreen_compositor::ffi::AVFrame; use openscreen_compositor::frame_geometry::live_params_from_scene; use openscreen_compositor::scene::Scene; -use windows::core::Interface; -use windows::Win32::Graphics::Direct3D11::{ - ID3D11Texture2D, D3D11_BIND_SHADER_RESOURCE, D3D11_CPU_ACCESS_WRITE, D3D11_MAPPED_SUBRESOURCE, - D3D11_MAP_WRITE_DISCARD, D3D11_TEXTURE2D_DESC, D3D11_USAGE_DYNAMIC, -}; -use windows::Win32::Graphics::Dxgi::Common::{DXGI_FORMAT_NV12, DXGI_SAMPLE_DESC}; + +mod common; +use common::{gpu, Nv12Frame}; const W: u32 = 1280; const H: u32 = 720; @@ -28,69 +24,18 @@ const SRC: (u32, u32) = (640, 360); /// Les rampes du zoom de `scene_json` (2 à 6 s) : l'entrée finit à 2 s, la sortie part de 6 s. const RAMPS: [f32; 8] = [1.3, 1.5, 1.7, 1.9, 6.1, 6.3, 6.5, 6.7]; -fn gpu() -> Option { - match Gpu::create(false) { - Ok(g) => Some(g), - Err(e) => { - eprintln!("pas de device D3D11 matériel ({e:#}) — test saute"); - None - } - } -} - /// Une frame NV12 synthétique : `yuv(colonne, rangée)` donne Y, Cb, Cr (BT.709 limité), la /// chroma étant prise au coin haut-gauche de chaque bloc 2×2. -struct Source { - frame: Box, - _tex: ID3D11Texture2D, -} - -impl Source { - fn new(gpu: &Gpu, yuv: impl Fn(u32, u32) -> [u8; 3]) -> Source { - let (w, h) = SRC; - let desc = D3D11_TEXTURE2D_DESC { - Width: w, - Height: h, - MipLevels: 1, - ArraySize: 1, - Format: DXGI_FORMAT_NV12, - SampleDesc: DXGI_SAMPLE_DESC { Count: 1, Quality: 0 }, - Usage: D3D11_USAGE_DYNAMIC, - BindFlags: D3D11_BIND_SHADER_RESOURCE.0 as u32, - CPUAccessFlags: D3D11_CPU_ACCESS_WRITE.0 as u32, - MiscFlags: 0, - }; - unsafe { - let mut tex: Option = None; - gpu.device.CreateTexture2D(&desc, None, Some(&mut tex)).expect("texture NV12"); - let tex = tex.expect("texture NV12"); - let mut m = D3D11_MAPPED_SUBRESOURCE::default(); - gpu.context.Map(&tex, 0, D3D11_MAP_WRITE_DISCARD, 0, Some(&mut m)).expect("Map"); - let pitch = m.RowPitch as usize; - let dst = m.pData as *mut u8; - for row in 0..h { - for col in 0..w { - *dst.add(row as usize * pitch + col as usize) = yuv(col, row)[0]; - } - } - for row in 0..h / 2 { - for col in 0..w { - let [_, u, v] = yuv(col & !1, row * 2); - *dst.add((h + row) as usize * pitch + col as usize) = if col % 2 == 0 { u } else { v }; - } - } - gpu.context.Unmap(&tex, 0); - let mut frame: Box = Box::new(std::mem::zeroed()); - frame.data[0] = tex.as_raw() as *mut u8; - frame.width = w as i32; - frame.height = h as i32; - Source { frame, _tex: tex } - } - } - - fn as_ptr(&self) -> *const AVFrame { - &*self.frame as *const AVFrame - } +fn source(gpu: &Gpu, yuv: impl Fn(u32, u32) -> [u8; 3]) -> Nv12Frame { + Nv12Frame::new( + gpu, + SRC, + |col, row| yuv(col, row)[0], + |bx, by| { + let [_, u, v] = yuv(2 * bx, 2 * by); + [u, v] + }, + ) } /// Un seul écran dans `rect` (fractions de la sortie `out`), zoomé ×`scale` sous `iso` de 2 à 6 s. @@ -112,7 +57,7 @@ fn scene_json(rect: [f32; 4], out: (u32, u32), scale: f32, roundness: f32, blur: ) } -fn render(comp: &Compositor, src: &Source, json: &str, t: f32) -> Vec { +fn render(comp: &Compositor, src: &Nv12Frame, json: &str, t: f32) -> Vec { let scene = Scene::from_json(json).expect("scène valide"); let mut live = live_params_from_scene(&scene); live.has_webcam = false; @@ -154,7 +99,7 @@ fn unsolvable_trail_taps_leave_the_background_alone() { let Some(gpu) = gpu() else { return }; let comp = Compositor::new_sized(&gpu, W, H).expect("compositor"); // Métrage rouge, repère vert dans son coin haut-gauche (sous l'arrondi de l'écran). - let src = Source::new(&gpu, |col, row| if col < 48 && row < 48 { [173, 42, 26] } else { [63, 102, 240] }); + let src = source(&gpu, |col, row| if col < 48 && row < 48 { [173, 42, 26] } else { [63, 102, 240] }); let json = |blur| scene_json([0.425, 0.425, 0.15, 0.15], (W, H), 1.6, 0.02, blur, false); let far = |x: u32, y: u32| x < W / 4 || x >= W * 3 / 4 || y < H / 4 || y >= H * 3 / 4; let mut leaks = 0; @@ -186,7 +131,7 @@ fn unsolvable_trail_taps_leave_the_background_alone() { fn off_canvas_trail_taps_keep_the_depth_of_field() { let Some(gpu) = gpu() else { return }; // Rayures noires et blanches de 2 px : la profondeur de champ les grise. - let src = Source::new(&gpu, |col, _| [if col / 2 % 2 == 0 { 16 } else { 235 }, 128, 128]); + let src = source(&gpu, |col, _| [if col / 2 % 2 == 0 { 16 } else { 235 }, 128, 128]); let normal = Compositor::new_sized(&gpu, W, H).expect("compositor"); let padded = Compositor::new_sized(&gpu, 2 * W, 2 * H).expect("compositor"); // La même boîte en px, centrée sur une sortie deux fois plus grande (le rayon se mesure sur diff --git a/crates/compositor/tests/shared_frame_handoff.rs b/crates/compositor/tests/shared_frame_handoff.rs new file mode 100644 index 0000000000..2465f2b983 --- /dev/null +++ b/crates/compositor/tests/shared_frame_handoff.rs @@ -0,0 +1,135 @@ +//! La texture qu'une vue live livre par handle NT, rouverte par un AUTRE device D3D11 — la +//! place qu'occupe le processus GPU de Chromium quand Electron l'importe. Ce que le +//! consommateur lit doit être exactement ce que le compositeur a composé : pas une image +//! à moitié copiée, pas une case périmée, et une case redimensionnée doit livrer un handle +//! neuf à la nouvelle taille. +//! +//! Demande un device matériel : saute sans lui (runner sans adaptateur). + +#![cfg(windows)] + +use openscreen_compositor::d3d::Gpu; +use openscreen_compositor::shared_frames::SharedRing; +use windows::core::Interface; +use windows::Win32::Foundation::{HANDLE, HMODULE}; +use windows::Win32::Graphics::Direct3D::{D3D_DRIVER_TYPE_HARDWARE, D3D_FEATURE_LEVEL_11_1}; +use windows::Win32::Graphics::Direct3D11::{ + D3D11CreateDevice, ID3D11Device, ID3D11Device1, ID3D11DeviceContext, ID3D11Texture2D, + D3D11_BIND_SHADER_RESOURCE, D3D11_CPU_ACCESS_READ, D3D11_CREATE_DEVICE_FLAG, + D3D11_MAPPED_SUBRESOURCE, D3D11_MAP_READ, D3D11_SDK_VERSION, D3D11_SUBRESOURCE_DATA, + D3D11_TEXTURE2D_DESC, D3D11_USAGE_DEFAULT, D3D11_USAGE_STAGING, +}; +use windows::Win32::Graphics::Dxgi::Common::{DXGI_FORMAT_R8G8B8A8_UNORM, DXGI_SAMPLE_DESC}; + +fn pattern(width: u32, height: u32, seed: u8) -> Vec { + (0..width * height) + .flat_map(|i| { + let (x, y) = (i % width, i / width); + [x as u8 ^ seed, y as u8, seed, 255] + }) + .collect() +} + +fn texture_with(gpu: &Gpu, width: u32, height: u32, pixels: &[u8]) -> ID3D11Texture2D { + let desc = D3D11_TEXTURE2D_DESC { + Width: width, + Height: height, + MipLevels: 1, + ArraySize: 1, + Format: DXGI_FORMAT_R8G8B8A8_UNORM, + SampleDesc: DXGI_SAMPLE_DESC { Count: 1, Quality: 0 }, + Usage: D3D11_USAGE_DEFAULT, + BindFlags: D3D11_BIND_SHADER_RESOURCE.0 as u32, + CPUAccessFlags: 0, + MiscFlags: 0, + }; + let init = D3D11_SUBRESOURCE_DATA { + pSysMem: pixels.as_ptr().cast(), + SysMemPitch: width * 4, + SysMemSlicePitch: 0, + }; + let mut tex = None; + unsafe { gpu.device.CreateTexture2D(&desc, Some(&init), Some(&mut tex)) }.expect("texture"); + tex.expect("texture") +} + +/// Le second device, sans rien de commun avec celui du compositeur que l'adaptateur. +fn consumer() -> (ID3D11Device, ID3D11DeviceContext) { + let mut device = None; + let mut context = None; + unsafe { + D3D11CreateDevice( + None, + D3D_DRIVER_TYPE_HARDWARE, + HMODULE::default(), + D3D11_CREATE_DEVICE_FLAG(0), + Some(&[D3D_FEATURE_LEVEL_11_1]), + D3D11_SDK_VERSION, + Some(&mut device), + None, + Some(&mut context), + ) + } + .expect("second device"); + (device.unwrap(), context.unwrap()) +} + +/// Ce que le consommateur lit à travers le handle, en RGBA serré. +fn read_through(handle: u64, width: u32, height: u32) -> Vec { + let (device, context) = consumer(); + let device1: ID3D11Device1 = device.cast().expect("ID3D11Device1"); + let shared: ID3D11Texture2D = + unsafe { device1.OpenSharedResource1(HANDLE(handle as *mut core::ffi::c_void)) } + .expect("le handle doit s'ouvrir depuis un autre device"); + let mut desc = D3D11_TEXTURE2D_DESC::default(); + unsafe { shared.GetDesc(&mut desc) }; + assert_eq!((desc.Width, desc.Height), (width, height)); + desc.Usage = D3D11_USAGE_STAGING; + desc.BindFlags = 0; + desc.CPUAccessFlags = D3D11_CPU_ACCESS_READ.0 as u32; + desc.MiscFlags = 0; + let mut staging = None; + unsafe { device.CreateTexture2D(&desc, None, Some(&mut staging)) }.expect("staging"); + let staging = staging.unwrap(); + let mut mapped = D3D11_MAPPED_SUBRESOURCE::default(); + let mut out = vec![0u8; (width * height * 4) as usize]; + unsafe { + context.CopyResource(&staging, &shared); + context.Map(&staging, 0, D3D11_MAP_READ, 0, Some(&mut mapped)).expect("map"); + for y in 0..height as usize { + std::ptr::copy_nonoverlapping( + (mapped.pData as *const u8).add(y * mapped.RowPitch as usize), + out.as_mut_ptr().add(y * width as usize * 4), + width as usize * 4, + ); + } + context.Unmap(&staging, 0); + } + out +} + +#[test] +fn another_device_reads_exactly_the_composed_frame() { + let Ok(gpu) = Gpu::create(false) else { + eprintln!("pas de device D3D11 matériel — saute"); + return; + }; + let mut ring = SharedRing::new(&gpu).expect("anneau"); + + let (w, h) = (64, 36); + let first = pattern(w, h, 0x11); + let handle = unsafe { ring.write(0, &texture_with(&gpu, w, h, &first), w, h) }.expect("écriture"); + assert_eq!(read_through(handle, w, h), first); + + // La même case réécrite : même texture, même handle, nouveau contenu. + let second = pattern(w, h, 0x22); + let again = unsafe { ring.write(0, &texture_with(&gpu, w, h, &second), w, h) }.expect("réécriture"); + assert_eq!(again, handle); + assert_eq!(read_through(again, w, h), second); + + // Le rendu change de taille : la case est recréée, avec un handle à la nouvelle taille. + let (w2, h2) = (32, 18); + let resized = pattern(w2, h2, 0x33); + let fresh = unsafe { ring.write(0, &texture_with(&gpu, w2, h2, &resized), w2, h2) }.expect("taille"); + assert_eq!(read_through(fresh, w2, h2), resized); +} diff --git a/crates/compositor/tests/window_frame_render.rs b/crates/compositor/tests/window_frame_render.rs index ea1a3b8811..249705af6e 100644 --- a/crates/compositor/tests/window_frame_render.rs +++ b/crates/compositor/tests/window_frame_render.rs @@ -19,19 +19,13 @@ use openscreen_compositor::frame_geometry::live_params_from_scene; use openscreen_compositor::live::Player; use openscreen_compositor::scene::Scene; +mod common; +use common::write_ppm; + const W: u32 = 1280; const H: u32 = 720; const AT_SEC: f64 = 3.0; -fn write_ppm(path: &std::path::Path, rgba: &[u8], w: u32, h: u32) -> std::io::Result<()> { - let mut out = Vec::with_capacity(rgba.len() / 4 * 3 + 32); - out.extend_from_slice(format!("P6\n{w} {h}\n255\n").as_bytes()); - for px in rgba.chunks_exact(4) { - out.extend_from_slice(&px[..3]); - } - std::fs::write(path, out) -} - /// `frame` : `None` = clé absente du JSON (payload d'avant le cadre), sinon sa valeur. fn scene_json(source: &str, frame: Option<&str>, rotation: &str) -> String { let s = source.replace('\\', "/"); diff --git a/design/cursors/3d-concept.png b/design/cursors/3d-concept.png index fc828101b7..4b7a01c98a 100644 Binary files a/design/cursors/3d-concept.png and b/design/cursors/3d-concept.png differ diff --git a/design/cursors/3d-direction.md b/design/cursors/3d-direction.md index 6333c23d0a..f715a93cb9 100644 --- a/design/cursors/3d-direction.md +++ b/design/cursors/3d-direction.md @@ -2,21 +2,61 @@ `3d-concept.png` is a visual reference, not a render from OpenScreen. -With the 3D option on, mode 15 ray-marches each original theme's arrow and hand -as a sculpted model: signed distance functions written in the compositor's three -shaders (HLSL, Metal, WGSL), not a PNG face or a height map. -`crates/compositor/src/sculpt.rs` names the ten models and holds their bounding -boxes and hotspots. The scene names the model (`"/"`); text, -resize, move and the other OS states keep their built-in art and the -compositor's beveled extrusion. +With the 3D option on, mode 15 ray-marches the arrow and hand of Studio Ink, Pop +Coral, Pixel Candy, Star Sprout and Glass Lens as sculpted models: signed distance +functions written in the compositor's three shaders (HLSL, Metal, WGSL), not a PNG +face or a height map. Prism Glow's arrow and hand are glass crystals: a triangle mesh +traced on its drawing, ray-traced in a navy rim. `crates/compositor/src/sculpt.rs` +names the ten models and holds their bounding boxes and hotspots. The scene names +the model (`"/"`); text, resize, move and the other OS states keep +their built-in art and the compositor's beveled extrusion. | Theme | Arrow model | Hand model | | --- | --- | --- | -| Studio Ink | Matte black body with an ivory piping set in from the edge. | Ivory glove with a dark cuff. | -| Prism Glow | Cut crystal: flat facets, a polygonal silhouette, cyan to violet dispersion. | The same glove cut into faceted fingers and palm. | -| Pop Coral | Soft coral rubber with rounded edges and a gentle dome. | Plump yellow rubber glove with a navy cuff. | -| Pixel Candy | Beveled voxels: a pink face with mint and pale pink edges, purple blocks one step behind. | Voxel glove with the same colours and purple back layer. | -| Star Sprout | Glossy mint ceramic carrying a star with a face and two leaves. | Ivory glove with a mint cuff and the same star. | +| Studio Ink | Black rim, a raised ivory band, a recessed satin black field. | Ivory glove cushion in a black rim, black ridges between the fingers. | +| Prism Glow | A glass crystal cut along the facets of its drawing, set in a thin navy rim. The picture under it shows through, refracted with a slight dispersion; each facet glows faintly in its drawn colour, and a light line marks its folds. | The same crystal: a facet for each plane of the drawn hand, the fingers and thumb included. | +| Pop Coral | Cut paper: flat-topped sheets, no rounded bead or gloss. A coral sheet on a navy sheet, a yellow sheet behind them offset down-left, two yellow click dashes. | A yellow sheet on a navy sheet, the navy showing between the fingers, a coral sheet behind offset down-left, two coral click dashes. | +| Pixel Candy | Simplified pixel art, one cube per pixel: a plum outline, a pink body with a pale highlight down the lit edge and a darker shade along the outline. The 3D model and the 2D sprite come from the same grid. | The same, a pixel hand: the index up, three knuckles apart, the thumb. | +| Star Sprout | Glossy mint cushion framed by the navy outline of its drawing, carrying a star with a face and two leaves. | Ivory glove cushion in the same navy frame, navy ridges between the fingers, a mint cuff and the same star. | +| Glass Lens | A clear glass lens in a flat-topped graphite outline, and beyond it a rounded rim of glass. The picture under it shows through, refracted: magnified by the lens, bent along its edge with a slight dispersion. | The same glass glove, the outline running between the fingers as ridges. | + +Studio Ink, Pop Coral and Star Sprout keep their drawn outline, as in +`3d-concept.png` (and `star-sprout/3d-reference.png` for the Star Sprout hand). +Each piece is a tray in the outline colour under its silhouette, a rounded bead +along the stroke and a puffy colour cushion inside it; the grooves between the +fingers become ridges of the outline colour. The three arrows share one polygon +and the three gloves one construction, each glove measured on its own drawing +(`RIM_POLY`, `RIM_GLOVE`). Pop Coral turns the same shapes into flat paper +sheets instead (`s_paper`). What stands in front is theme-specific: Studio +Ink's ivory band, Pop Coral's yellow and coral layers and click dashes, Star +Sprout's cuff, star, leaves and eyes. + +Prism Glow is built from its 2D drawing alone (`prism-glow/source.png`), not from +`3d-concept.png`. `prism-glow/model/` holds the mesh: `prism_geo.py` lists its +vertices, heights and facets and writes `prism.json`; `build_blend.py` builds +`prism-glow.blend` and `prism-glow.glb` from it in Blender; `export_compositor.py` +writes the mesh into `crates/compositor/src/prism_mesh.rs` and the three shaders. +The compositor ray-traces the crystal, its triangles grouped in bounding boxes that +a ray skips when it misses them: refraction per colour +channel, total internal reflection, and an exit through its flat base onto the +picture under the cursor. That picture is a copy of the composed frame taken just +before the cursor is drawn: the recording, then its privacy blurs, then the cursor, +sharp on top, so only blurred pixels show through the glass. The rim is its +silhouette extruded, a distance field +like the other models: it gives the coverage and the shadows. + +Glass Lens takes the Studio Ink arrow and glove (`s_glass2`) and widens their +outline by `GLASS_RIM`: a graphite band of `GLASS_LINE`, then the glass rim. In +volume (`s_glass`), a glass slab runs under the whole silhouette, rounded across +the rim, and the lens rises `GLASS_DOME` above it inside the band. A ray that hits +the glass is refracted per colour channel, crosses to the flat underside of the +slab, leaves it and lands on the picture under the cursor, the same copy of the +composed frame the Prism Glow crystal refracts (`glass_shade`); three samples +along the refraction keep a compressed edge from breaking into sparkles. Its +shadow is lighter than an opaque model's. With the 3D option off, the theme stays +glass: mode 19 (`cursor_glass`) draws the same relief seen face on, its height +read off the same distances (`s_glass_relief`), and refracts the picture the same +way, the air under the slab standing in for the model's hover. All models share one close lamp: a gradient and a highlight even on flat faces, soft self shadows, ambient occlusion and a studio environment in the @@ -27,4 +67,8 @@ click and size changes. To change a model, edit it in all three shaders (`sculpt_proto`, `sculpt_material`) and keep `sculpt.rs` in step: its boxes must contain the shapes, and a test checks that the shaders mirror its constants. Pixel Candy's -arrow voxels are sampled from the arrow polygon; `sculpt.rs` says how. +voxel grids are tables written by `scripts/generate-pixel-candy-voxels.mjs`: +change a grid there and run the script; it rewrites all three shaders, and a +test checks the tables agree. To change Prism Glow, edit `prism_geo.py`, run it, +run `build_blend.py` in Blender, then `export_compositor.py`; a test checks that +the three shaders carry the mesh of `prism_mesh.rs`. diff --git a/design/cursors/README.md b/design/cursors/README.md index b3a21cd3c1..da17f3d39a 100644 --- a/design/cursors/README.md +++ b/design/cursors/README.md @@ -1,8 +1,9 @@ # Original OpenScreen cursor themes -These five themes replace the removed Sweezy packs. The artwork was created for +These six themes replace the removed Sweezy packs. The artwork was created for OpenScreen as transparent raster images. The PNGs in this directory are the -original high-resolution masters; there is no SVG conversion step. +original high-resolution masters; there is no SVG conversion step. Glass Lens is +the exception: it has no raster master (see below). | Theme | Intent | | --- | --- | @@ -11,6 +12,7 @@ original high-resolution masters; there is no SVG conversion step. | Pop Coral | Warm, playful choice for casual recordings | | Pixel Candy | A small retro option for people who liked pixel packs | | Star Sprout | An original tiny character for people who liked mascot packs | +| Glass Lens | A clear lens of glass in a defined outline: the recording shows through the cursor | Each `source.png` contains an arrow on the left and a hand on the right. Run `node scripts/generate-original-cursor-themes.mjs` to crop, remove low-alpha @@ -18,14 +20,25 @@ fringe pixels, and resize them into transparent 128 × 128 PNGs under `public/cursors//`. It prints the normalized tip hotspots; copy them into `src/lib/cursor/cursorThemes.ts` after changing a master. -With the 3D option on, the compositor draws each theme's arrow and hand as a -model sculpted in its shaders (`crates/compositor/src/sculpt.rs`), not from these -PNGs. The theme picker and the flat cursor keep using the PNGs. Text, resize, -move and other states remain state-accurate through the built-in art and -receive the shared 3D extrusion. +With the 3D option on, the compositor draws the arrow and hand of Studio Ink, +Pop Coral, Pixel Candy and Star Sprout as models sculpted in its shaders +(`crates/compositor/src/sculpt.rs`), not from these PNGs. Prism Glow's arrow and +hand are glass crystals meshed on its drawing (`prism-glow/model/`). The theme +picker and the flat cursor keep using the PNGs. Text, resize, move and other +states remain state-accurate through the built-in art and receive the shared 3D +extrusion. + +Glass Lens is glass with the 3D option off too: the compositor draws its arrow +and hand as its model seen face on (mode 19), a lens that refracts the picture +under it, instead of a PNG. Its shapes are the Studio Ink arrow and glove, and +`node scripts/generate-glass-lens-cursor.mjs` draws its picker PNGs from the +same distance fields, read from the shaders, then prints their hotspots. +`glass-lens/renders.png` shows the compositor's own renders, 2D then 3D, on a +light page, a dark page and a checker. `contact-sheet.png` shows the sprites enlarged on a light background; `dark-32px.png` shows them at their 32-pixel reference size on a dark background. -The 3D modeling direction and concept image are in `3d-direction.md` and -`3d-concept.png`. +The 3D modeling direction and the reference sheet the models follow are in +`3d-direction.md` and `3d-concept.png`; `star-sprout/3d-reference.png` is a closer +view of the Star Sprout hand. The consolidated acceptance list is in `requirements.md`. diff --git a/design/cursors/glass-lens/renders.png b/design/cursors/glass-lens/renders.png new file mode 100644 index 0000000000..0cfd86cbe7 Binary files /dev/null and b/design/cursors/glass-lens/renders.png differ diff --git a/design/cursors/pixel-candy/source.png b/design/cursors/pixel-candy/source.png index 79b5c859dd..daeed19bea 100644 Binary files a/design/cursors/pixel-candy/source.png and b/design/cursors/pixel-candy/source.png differ diff --git a/design/cursors/prism-glow/model/build_blend.py b/design/cursors/prism-glow/model/build_blend.py new file mode 100644 index 0000000000..4a941e04c5 --- /dev/null +++ b/design/cursors/prism-glow/model/build_blend.py @@ -0,0 +1,265 @@ +"""Build the Prism Glow cursor in Blender from prism.json, then save prism-glow.blend and export +prism-glow.glb next to this script. + + blender --background --python build_blend.py + +Per cursor, two objects: the faceted crystal (a closed mesh, one flat face per facet of the 2D art, +with its colour in the face attribute `concept`) and the navy rim (a filled 2D curve with a hole, +extruded and rounded). Units: 1 = 1000 px of source.png, origin on the hotspot, y up, z toward the +viewer, rim top at z = 0, everything's underside at z = -0.04. +""" + +import json +import math +import os + +import bmesh +import bpy +from mathutils import Matrix, Vector + +HERE = globals().get("HERE") or os.path.dirname(os.path.abspath(__file__)) +S = 0.001 +T_RIM = 40.0 # rim and crystal-body thickness, px +BEVEL = 5.0 # rim edge rounding, px +HAND_AT = ((1220 - 208) * S, -(50 - 42) * S) # the hand stands where it is in source.png + +D = json.load(open(os.path.join(HERE, "prism.json"))) + + +def srgb_to_linear(c): + return c / 12.92 if c <= 0.04045 else ((c + 0.055) / 1.055) ** 2.4 + + +def hex_rgba(h): + return tuple(srgb_to_linear(int(h[i : i + 2], 16) / 255) for i in (1, 3, 5)) + (1.0,) + + +def signed_area(pts): + return 0.5 * sum(a[0] * b[1] - b[0] * a[1] for a, b in zip(pts, pts[1:] + pts[:1])) + + +def model_xy(sh, x, y): + hx, hy = sh["hotspot"] + return ((x - hx) * S, -(y - hy) * S) + + +def fresh_collection(name): + col = bpy.data.collections.get(name) or bpy.data.collections.new(name) + if col.name not in bpy.context.scene.collection.children: + bpy.context.scene.collection.children.link(col) + for ob in list(col.objects): + bpy.data.objects.remove(ob) + return col + + +def build_crystal(name, sh): + V = sh["verts"] + bm = bmesh.new() + concept = bm.faces.layers.float_color.new("concept") + top = {k: bm.verts.new((*model_xy(sh, x, y), z * S)) for k, (x, y, z) in V.items()} + outline = list(sh["outline"]) + if signed_area([model_xy(sh, *V[k][:2]) for k in outline]) < 0: + outline.reverse() + bot = {k: bm.verts.new((*model_xy(sh, *V[k][:2]), -T_RIM * S)) for k in outline} + for f, hexcol in zip(sh["facets"], sh["colors"]): + vs = [top[k] for k in f] + if signed_area([v.co.xy[:] for v in vs]) < 0: + vs.reverse() + bm.faces.new(vs)[concept] = hex_rgba(hexcol) + side = hex_rgba("#1a3cc8") + for a, b in zip(outline, outline[1:] + outline[:1]): + bm.faces.new((top[a], bot[a], bot[b], top[b]))[concept] = side + bm.faces.new([bot[k] for k in reversed(outline)])[concept] = side + bmesh.ops.triangulate(bm, faces=[f for f in bm.faces if len(f.verts) > 3], + quad_method="BEAUTY", ngon_method="BEAUTY") + bmesh.ops.recalc_face_normals(bm, faces=bm.faces[:]) + me = bpy.data.meshes.get(name) or bpy.data.meshes.new(name) + bm.to_mesh(me) + bm.free() + # glTF carries colours per corner: mirror the face colour there + corner = me.attributes.get("Col") or me.attributes.new("Col", "FLOAT_COLOR", "CORNER") + for p in me.polygons: + p.use_smooth = False + for li in p.loop_indices: + corner.data[li].color = me.attributes["concept"].data[p.index].color + me.color_attributes.active_color = corner + return bpy.data.objects.new(name, me) + + +def inset(loop, d): + """Offset a closed counter-clockwise loop inward by d (mitred).""" + out = [] + for i in range(len(loop)): + p0, p1, p2 = Vector(loop[i - 1]), Vector(loop[i]), Vector(loop[(i + 1) % len(loop)]) + e0, e1 = (p1 - p0).normalized(), (p2 - p1).normalized() + n0, n1 = Vector((-e0.y, e0.x)), Vector((-e1.y, e1.x)) + m = (n0 + n1).normalized() if (n0 + n1).length > 1e-6 else n0 + out.append(tuple(p1 + m * (d / max(m.dot(n0), 0.25)))) + return out + + +def build_rim(name, sh): + outer = [model_xy(sh, x, y) for x, y in sh["silhouette"]] + if signed_area(outer) < 0: + outer.reverse() + inner = [model_xy(sh, *sh["verts"][k][:2]) for k in sh["outline"]] + # the bevel rounds outward from the spline: pull the outer loop in so the silhouette stays exact + outer = inset(outer, BEVEL * S) + cu = bpy.data.curves.get(name) or bpy.data.curves.new(name, "CURVE") + cu.splines.clear() + cu.dimensions = "2D" + cu.fill_mode = "BOTH" + cu.extrude = (T_RIM / 2 - BEVEL) * S + cu.bevel_depth = BEVEL * S + cu.bevel_resolution = 3 + for loop in (outer, inner): + sp = cu.splines.new("POLY") + sp.points.add(len(loop) - 1) + for p, (x, y) in zip(sp.points, loop): + p.co = (x, y, 0.0, 1.0) + sp.use_cyclic_u = True + ob = bpy.data.objects.new(name, cu) + ob.location.z = -T_RIM / 2 * S + return ob + + +def nodes_of(name): + m = bpy.data.materials.get(name) or bpy.data.materials.new(name) + m.use_nodes = True + m.node_tree.nodes.clear() + return m, m.node_tree, m.node_tree.nodes.new("ShaderNodeOutputMaterial") + + +def material_crystal(): + """Clear glass whose facets glow with their colour in the art (the look chosen on 30/09/2026).""" + m, nt, out = nodes_of("Prism Crystal") + bsdf = nt.nodes.new("ShaderNodeBsdfPrincipled") + attr = nt.nodes.new("ShaderNodeAttribute") + attr.attribute_name = "concept" + nt.links.new(attr.outputs["Color"], bsdf.inputs["Emission Color"]) + bsdf.inputs["Base Color"].default_value = (1, 1, 1, 1) + bsdf.inputs["Emission Strength"].default_value = 0.45 + bsdf.inputs["Transmission Weight"].default_value = 1.0 + bsdf.inputs["IOR"].default_value = 1.61 + bsdf.inputs["Roughness"].default_value = 0.0 + nt.links.new(bsdf.outputs["BSDF"], out.inputs["Surface"]) + return m + + +def material_rim(): + m, nt, out = nodes_of("Prism Rim") + bsdf = nt.nodes.new("ShaderNodeBsdfPrincipled") + bsdf.inputs["Base Color"].default_value = hex_rgba("#040c5a") + bsdf.inputs["Roughness"].default_value = 0.22 + bsdf.inputs["Coat Weight"].default_value = 1.0 + bsdf.inputs["Coat Roughness"].default_value = 0.05 + nt.links.new(bsdf.outputs["BSDF"], out.inputs["Surface"]) + return m + + +def build(): + col = fresh_collection("Prism Glow") + crystal, rim = material_crystal(), material_rim() + obs = [] + for key, at in (("arrow", (0.0, 0.0)), ("hand", HAND_AT)): + c = build_crystal(f"prism_{key}_crystal", D[key]) + r = build_rim(f"prism_{key}_rim", D[key]) + c.data.materials.clear() + c.data.materials.append(crystal) + r.data.materials.clear() + r.data.materials.append(rim) + for ob in (c, r): + ob.location.x, ob.location.y = at + col.objects.link(ob) + obs += [c, r] + return obs + + +def studio(): + """Studio HDRI for light and reflections, the editor screenshot as the screen, a 3/4 camera.""" + sc = bpy.context.scene + w = sc.world or bpy.data.worlds.new("World") + sc.world = w + w.use_nodes = True + nt = w.node_tree + nt.nodes.clear() + env = nt.nodes.new("ShaderNodeTexEnvironment") + env.image = bpy.data.images.load(os.path.join(bpy.utils.system_resource("DATAFILES"), "studiolights", "world", "studio.exr"), check_existing=True) + env.image.pack() # the install path differs per machine; 100 KB inside the .blend does not + bg, flat = nt.nodes.new("ShaderNodeBackground"), nt.nodes.new("ShaderNodeBackground") + flat.inputs["Color"].default_value = (0.06, 0.06, 0.07, 1) + lp, mix, out = nt.nodes.new("ShaderNodeLightPath"), nt.nodes.new("ShaderNodeMixShader"), nt.nodes.new("ShaderNodeOutputWorld") + nt.links.new(env.outputs["Color"], bg.inputs["Color"]) + nt.links.new(lp.outputs["Is Camera Ray"], mix.inputs["Fac"]) + nt.links.new(bg.outputs["Background"], mix.inputs[1]) + nt.links.new(flat.outputs["Background"], mix.inputs[2]) + nt.links.new(mix.outputs["Shader"], out.inputs["Surface"]) + + shot = os.path.normpath(os.path.join(HERE, "..", "..", "..", "..", "public", "preview4.png")) + screen = bpy.data.objects.get("screen") + if screen is None and os.path.exists(shot): + me = bpy.data.meshes.new("screen") + me.from_pydata([(-3, -1.66, 0), (3, -1.66, 0), (3, 1.66, 0), (-3, 1.66, 0)], [], [(0, 1, 2, 3)]) + uv = me.uv_layers.new() + for li, st in enumerate(((0, 0), (1, 0), (1, 1), (0, 1))): + uv.data[li].uv = st + m, mnt, mout = nodes_of("screen") + em, tex = mnt.nodes.new("ShaderNodeEmission"), mnt.nodes.new("ShaderNodeTexImage") + tex.image = bpy.data.images.load(shot, check_existing=True) + mnt.links.new(tex.outputs["Color"], em.inputs["Color"]) + mnt.links.new(em.outputs["Emission"], mout.inputs["Surface"]) + me.materials.append(m) + screen = bpy.data.objects.new("screen", me) + screen.location = (1.9, -1.0, -T_RIM * S - 0.0005) + sc.collection.objects.link(screen) + + target = Vector((0.679, -0.43, 0.0)) + cam = bpy.data.objects.get("cam_34") or bpy.data.objects.new("cam_34", bpy.data.cameras.new("cam_34")) + if cam.name not in sc.collection.objects: + sc.collection.objects.link(cam) + eye = target + Matrix.Rotation(math.radians(-28), 3, "Y") @ Matrix.Rotation(math.radians(-38), 3, "X") @ Vector((0, 0, 4.4)) + f = (target - eye).normalized() + r = f.cross(Vector((0, 1, 0))).normalized() + cam.location = eye + cam.rotation_euler = Matrix((r, r.cross(f), -f)).transposed().to_euler() + cam.data.lens = 85 + sc.camera = cam + sc.render.engine = "CYCLES" + sc.cycles.device = "GPU" + sc.cycles.samples = 128 + sc.cycles.transmission_bounces = sc.cycles.max_bounces = 24 + sc.render.resolution_x, sc.render.resolution_y = 1600, 1000 + sc.view_settings.view_transform = "AgX" + sc.view_settings.look = "AgX - Medium High Contrast" + + +def export_glb(obs, path): + """The rims are curves; glTF wants meshes: bake them into temporary twins for the export.""" + dg = bpy.context.evaluated_depsgraph_get() + twins = [] + for ob in obs: + if ob.type == "CURVE": + name = ob.name + me = bpy.data.meshes.new_from_object(ob.evaluated_get(dg)) + ob.name = name + "_curve" # the twin takes the name the glTF node should carry + tw = bpy.data.objects.new(name, me) + tw.matrix_world = ob.matrix_world + me.materials.append(ob.data.materials[0]) + bpy.context.scene.collection.objects.link(tw) + twins.append((tw, ob, name)) + bpy.ops.object.select_all(action="DESELECT") + for ob in [o for o in obs if o.type == "MESH"] + [tw for tw, _, _ in twins]: + ob.select_set(True) + bpy.ops.export_scene.gltf(filepath=path, export_format="GLB", use_selection=True, + export_vertex_color="ACTIVE", export_apply=True) + for tw, ob, name in twins: + me = tw.data + bpy.data.objects.remove(tw) + bpy.data.meshes.remove(me) + ob.name = name + + +obs = build() +studio() +export_glb(obs, os.path.join(HERE, "prism-glow.glb")) +bpy.ops.wm.save_as_mainfile(filepath=os.path.join(HERE, "prism-glow.blend"), relative_remap=True) diff --git a/design/cursors/prism-glow/model/export_compositor.py b/design/cursors/prism-glow/model/export_compositor.py new file mode 100644 index 0000000000..15cd261afe --- /dev/null +++ b/design/cursors/prism-glow/model/export_compositor.py @@ -0,0 +1,363 @@ +"""Write the Prism Glow meshes into the compositor: `crates/compositor/src/prism_mesh.rs` and the +matching tables in the three shaders, between their `prism mesh` markers. + + python export_compositor.py # after build_blend.py has written prism-glow.glb + +Everything is expressed in the mode 15 model frame: the unit is the side of the theme's 128 px +sprite (the flat cursor's size), origin on the hotspot, x right, y DOWN, z toward the camera, +rim top at z = 0. The source-to-sprite transform is the one scripts/generate-original-cursor-themes.mjs +applies, so the crystal lands exactly on the flat cursor. +""" + +import json +import math +import struct +from pathlib import Path + +import cv2 +import numpy as np + +HERE = Path(__file__).resolve().parent +REPO = HERE.parents[3] +SRC = REPO / "crates" / "compositor" / "src" +SPRITE, INNER, ALPHA_MIN = 128, 112, 28 # generate-original-cursor-themes.mjs +BEVEL_PX = 12.0 # rounding of the rim's edges, source px +EDGE_FOLD_DEG = 25.0 +LEAF = 8 # triangles per bounding box at most +BOX_PAD = 1e-3 # model units: past prism_tri's edge slack + + +def glb_meshes(path): + """{node name: (positions (n,3), triangle indices (m,3), colours (n,3))}, glTF frame (y up).""" + b = path.read_bytes() + n = struct.unpack_from("= ALPHA_MIN + ys, xs = np.nonzero(a) + w, h = xs.max() - xs.min() + 1, ys.max() - ys.min() + 1 + scale = INNER / max(w, h) + tx, ty = (SPRITE - w * scale) / 2, (SPRITE - h * scale) / 2 + hx = (tx + (hotspot[0] - x0 - xs.min()) * scale) / SPRITE + hy = (ty + (hotspot[1] - ys.min()) * scale) / SPRITE + return scale / SPRITE, (hx, hy), ty / SPRITE + + +def crystal_triangles(pos, tri, col, sheet, s): + """Triangles of the crystal (facets and walls; the flat bottom is left to the shader), each + with its outward normal, its colour in the art and which of its edges are real fold lines.""" + # glTF (x, y up, z) -> Blender (x, y, z up), hotspot at the origin, then the model frame + blender = np.c_[pos[:, 0], -pos[:, 2], pos[:, 1]] - np.array([sheet[0], sheet[1], 0.0]) + model = np.c_[blender[:, 0], -blender[:, 1], blender[:, 2]] * (1000.0 * s) + zmin = model[:, 2].min() + tris = [] + for t in tri: + a, b, c = model[t] + nb = np.cross(blender[t[1]] - blender[t[0]], blender[t[2]] - blender[t[0]]) + nb /= np.linalg.norm(nb) + n = np.array([nb[0], -nb[1], nb[2]]) # the y mirror keeps the normal outward + if n[2] < -0.9 and abs(a[2] - zmin) < 1e-6: + continue + tris.append({"v": (a, b, c), "n": n, "col": col[t[0]], "key": tuple(map(tuple, np.round(model[t], 6)))}) + # real edges: the mesh's boundary, a change of facet colour, or a sharp fold + owners = {} + for i, t in enumerate(tris): + for e in range(3): + k = tuple(sorted((t["key"][(e + 1) % 3], t["key"][(e + 2) % 3]))) + owners.setdefault(k, []).append(i) + for i, t in enumerate(tris): + bits = 0 + for e in range(3): + k = tuple(sorted((t["key"][(e + 1) % 3], t["key"][(e + 2) % 3]))) + other = [j for j in owners[k] if j != i] + real = not other or np.abs(tris[other[0]]["col"] - t["col"]).sum() > 1e-3 or ( + tris[other[0]]["n"] @ t["n"] < math.cos(math.radians(EDGE_FOLD_DEG))) + bits |= int(real) << e + t["flags"] = (1 if abs(t["n"][2]) < 0.2 else 0) + 2 * bits + return tris, float(zmin), float(model[:, 2].max()), model + + +def cluster(tris): + """The triangles regrouped by neighbourhood, and one bounding box per group: a ray that + misses a box skips all its triangles. Groups are halves of the centroids' longest extent, + split until they hold LEAF triangles at most; the triangles come back in group order.""" + groups = [] + + def split(idx): + if len(idx) <= LEAF: + groups.append(idx) + return + cent = np.array([np.mean(tris[i]["v"], axis=0) for i in idx]) + axis = int(np.argmax(cent.max(0) - cent.min(0))) + order = [idx[j] for j in np.argsort(cent[:, axis], kind="stable")] + split(order[: len(order) // 2]) + split(order[len(order) // 2 :]) + + split(list(range(len(tris)))) + ordered, leaves = [], [] + for g in groups: + pts = np.array([v for i in g for v in tris[i]["v"]]) + leaves.append((pts.min(0) - BOX_PAD, pts.max(0) + BOX_PAD, len(ordered), len(g))) + ordered += [tris[i] for i in g] + return ordered, leaves + + +def main(): + img = cv2.imread(str(HERE.parent / "source.png"), cv2.IMREAD_UNCHANGED) + geo = json.loads((HERE / "prism.json").read_text()) + meshes = glb_meshes(HERE / "prism-glow.glb") + shapes = [] + for kind, node, (x0, x1), sheet in ( + ("arrow", "prism_arrow_crystal", (0, 887), (0.0, 0.0)), + ("hand", "prism_hand_crystal", (887, 1774), ((1220 - 208) / 1000, -(50 - 42) / 1000)), + ): + hot = geo[kind]["hotspot"] + s, (hx, hy), top = sprite_transform(img, x0, x1, hot) + tris, zmin, zmax, model = crystal_triangles(*meshes[node], sheet, s) + tris, leaves = cluster(tris) + sil = [((x - hot[0]) * s, (y - hot[1]) * s) for x, y in geo[kind]["silhouette"]] + out = [((geo[kind]["verts"][k][0] - hot[0]) * s, (geo[kind]["verts"][k][1] - hot[1]) * s) + for k in geo[kind]["outline"]] + sx = [p[0] for p in sil] + # the draw box: the sprite's full height (1 unit), the silhouette's width + bx0, bx1 = min(sx) - 0.01, max(sx) + 0.01 + shapes.append({ + "kind": kind, "tris": tris, "leaves": leaves, "sil": sil, "out": out, "thick": -zmin, + "height": zmax, + "size": (bx1 - bx0, 1.0), "hotspot": (-bx0 / (bx1 - bx0), hy), "top": top, + "per_px": s, + }) + print(f"{kind}: {len(tris)} triangles in {len(leaves)} boxes, {len(sil)} silhouette points, " + f"box {bx1 - bx0:.4f} x 1, hotspot ({-bx0 / (bx1 - bx0):.4f}, {hy:.4f}), " + f"height {zmax:.4f}, thick {-zmin:.4f}") + write_rust(shapes) + write_shaders(shapes) + + +def f(x): + return f"{x:.6f}".rstrip("0").rstrip(".") if abs(x) > 5e-7 else "0.0" + + +def fl(x): # a float literal every language reads as a float + s = f(x) + return s if "." in s else s + ".0" + + +def bevel(shapes): + return BEVEL_PX * shapes[0]["per_px"] + + +def polygons(shapes): + """Silhouettes, then outlines, in one table; their starts.""" + poly, sil_start, out_start = [], [], [] + for sh in shapes: + sil_start.append(len(poly)) + poly += sh["sil"] + for sh in shapes: + out_start.append(len(poly)) + poly += sh["out"] + return poly, sil_start, out_start + + +def records(shapes): + """Per triangle, four float4: (v0, flags), (e1, r), (e2, g), (n, b).""" + rows = [] + for sh in shapes: + for t in sh["tris"]: + a, b, c = t["v"] + e1, e2 = b - a, c - a + r, g, bl = t["col"] + rows += [(*a, t["flags"]), (*e1, r), (*e2, g), (*t["n"], bl)] + return rows + + +def boxes(shapes): + """Per box, two float4: (lo, first triangle), (hi, triangle count); each model's first box.""" + rows, box_start, k = [], [], 0 + for sh in shapes: + box_start.append(len(rows) // 2) + for lo, hi, first, count in sh["leaves"]: + rows += [(*lo, k + first), (*hi, count)] + k += len(sh["tris"]) + return rows, box_start + + +def write_rust(shapes): + tri_start, rows = [], records(shapes) + k = 0 + for sh in shapes: + tri_start.append(k) + k += len(sh["tris"]) + poly, sil_start, out_start = polygons(shapes) + box_rows, box_start = boxes(shapes) + lines = [ + "//! Les deux curseurs de Prism Glow en MAILLAGE : les facettes tracées sur l'art 2D", + "//! (`design/cursors/prism-glow/model`), taillées en cristal. Fichier GÉNÉRÉ par", + "//! `design/cursors/prism-glow/model/export_compositor.py` : ne pas l'éditer à la main.", + "//!", + "//! Repère du modèle du mode 15 : unité = côté du sprite 128 px du thème, origine au hotspot,", + "//! x à droite, y vers le bas, z vers la caméra, dessus du serti en z = 0. Chaque triangle du", + "//! cristal tient en quatre float4 : (v0, drapeaux), (e1, r), (e2, g), (normale, b) ; drapeaux =", + "//! paroi (1) + 2 × arêtes réelles (bit k : l'arête opposée au sommet k). Le fond plat, en", + "//! z = -épaisseur, est laissé au shader. `POLYS` tient les silhouettes (bord extérieur du", + "//! serti) puis les contours du cristal (son bord intérieur). `BOXES` range les triangles", + "//! par voisinage : deux float4 par boîte, (coin bas, premier triangle), (coin haut, nombre).", + "", + "/// Un des deux modèles : ses triangles et sa silhouette dans les tables, sa boîte de dessin.", + "pub struct PrismModel {", + " pub tri_start: usize,", + " pub tri_count: usize,", + " pub sil_start: usize,", + " pub sil_count: usize,", + " pub out_start: usize,", + " pub out_count: usize,", + " pub box_start: usize,", + " pub box_count: usize,", + " /// Largeur de la boîte, hauteur 1 (le sprite), en unités du modèle.", + " pub size: [f32; 2],", + " /// Hotspot, fraction de la boîte.", + " pub hotspot: [f32; 2],", + " /// Haut de la silhouette, fraction de la hauteur.", + " pub top: f32,", + " /// Sommet du cristal au-dessus de z = 0, et épaisseur sous z = 0.", + " pub height: f32,", + " pub thick: f32,", + "}", + "", + "/// Arrondi des arêtes du serti, en unités du modèle (`PRISM_BEVEL` des shaders).", + f"pub const BEVEL: f32 = {fl(bevel(shapes))};", + "", + "/// Flèche puis main, dans l'ordre des identifiants du shader.", + "pub const MODELS: [PrismModel; 2] = [", + ] + for i, sh in enumerate(shapes): + lines += [ + " PrismModel {", + f" tri_start: {tri_start[i]},", + f" tri_count: {len(sh['tris'])},", + f" sil_start: {sil_start[i]},", + f" sil_count: {len(sh['sil'])},", + f" out_start: {out_start[i]},", + f" out_count: {len(sh['out'])},", + f" box_start: {box_start[i]},", + f" box_count: {len(sh['leaves'])},", + f" size: [{fl(sh['size'][0])}, {fl(sh['size'][1])}],", + f" hotspot: [{fl(sh['hotspot'][0])}, {fl(sh['hotspot'][1])}],", + f" top: {fl(sh['top'])},", + f" height: {fl(sh['height'])},", + f" thick: {fl(sh['thick'])},", + " },", + ] + lines += ["];", "", f"pub const TRIS: [[f32; 4]; {len(rows)}] = ["] + lines += [f" [{', '.join(fl(v) for v in r)}]," for r in rows] + lines += ["];", "", f"pub const POLYS: [[f32; 2]; {len(poly)}] = ["] + lines += [f" [{fl(x)}, {fl(y)}]," for x, y in poly] + lines += ["];", "", f"pub const BOXES: [[f32; 4]; {len(box_rows)}] = ["] + lines += [f" [{', '.join(fl(v) for v in r)}]," for r in box_rows] + lines += ["];", ""] + (SRC / "prism_mesh.rs").write_text("\n".join(lines), encoding="utf-8", newline="\n") + + +def table_lines(shapes, lang): + rows = records(shapes) + poly, sil_start, out_start = polygons(shapes) + box_rows, box_start = boxes(shapes) + bcounts = [len(sh["leaves"]) for sh in shapes] + counts = [len(sh["tris"]) for sh in shapes] + scounts = [len(sh["sil"]) for sh in shapes] + ocounts = [len(sh["out"]) for sh in shapes] + if lang == "hlsl": + v4 = lambda r: f"float4({', '.join(fl(v) for v in r)})" + v2 = lambda p: f"float2({fl(p[0])}, {fl(p[1])})" + head4 = f"static const float4 PRISM_TRIS[{len(rows)}] = {{" + headb = f"static const float4 PRISM_BOXES[{len(box_rows)}] = {{" + head2 = f"static const float2 PRISM_POLY[{len(poly)}] = {{" + tail = "};" + ints = lambda name, vals: f"static const int {name}[2] = {{ {vals[0]}, {vals[1]} }};" + scalar = f"static const float PRISM_BEVEL = {fl(bevel(shapes))};" + elif lang == "metal": + v4 = lambda r: f"float4({', '.join(fl(v) for v in r)})" + v2 = lambda p: f"float2({fl(p[0])}, {fl(p[1])})" + head4 = f"constant float4 PRISM_TRIS[{len(rows)}] = {{" + headb = f"constant float4 PRISM_BOXES[{len(box_rows)}] = {{" + head2 = f"constant float2 PRISM_POLY[{len(poly)}] = {{" + tail = "};" + ints = lambda name, vals: f"constant int {name}[2] = {{ {vals[0]}, {vals[1]} }};" + scalar = f"constant float PRISM_BEVEL = {fl(bevel(shapes))};" + else: + ints = lambda name, vals: f"var {name} = array({vals[0]}, {vals[1]});" + scalar = f"const PRISM_BEVEL: f32 = {fl(bevel(shapes))};" + out = [ints("PRISM_TRI_START", [0, counts[0]]), ints("PRISM_TRI_COUNT", counts), + ints("PRISM_SIL_START", sil_start), ints("PRISM_SIL_COUNT", scounts), + ints("PRISM_OUT_START", out_start), ints("PRISM_OUT_COUNT", ocounts), + ints("PRISM_BOX_START", box_start), ints("PRISM_BOX_COUNT", bcounts)] + if lang == "wgsl": + # The triangles, polygons and boxes themselves go in a uniform buffer that compositor_linux.rs + # fills from prism_mesh.rs (a polygon point per vec4, the stride of a uniform array). As + # a `const` or `var` table, lavapipe copies them into every invocation of every + # layer: a frame without any cursor rendered 3.4 times slower. + return out + ["struct PrismMesh {", f" tris: array, {len(rows)}>,", + f" poly: array, {len(poly)}>,", + f" boxes: array, {len(box_rows)}>,", "}", scalar] + out.append(head4) + out += [f" {v4(r)}," for r in rows] + out[-1] = out[-1].rstrip(",") + out += [tail, headb] + out += [f" {v4(r)}," for r in box_rows] + out[-1] = out[-1].rstrip(",") + out += [tail, head2] + out += [f" {v2(p)}," for p in poly] + out[-1] = out[-1].rstrip(",") + out += [tail, scalar] + return out + + +BEGIN = "prism mesh: generated by design/cursors/prism-glow/model/export_compositor.py" +END = "end of the prism mesh" + + +def write_shaders(shapes): + for name, lang in (("shaders.hlsl", "hlsl"), ("shaders.metal", "metal"), ("vk_shaders/layer.wgsl", "wgsl")): + path = SRC / name + text = path.read_text(encoding="utf-8") + a, b = text.find(BEGIN), text.find(END) + if a < 0 or b < 0: + print(f"{name}: no markers yet, tables not written") + continue + start = text.index("\n", a) + 1 + stop = text.rindex("\n", 0, b) + 1 + text = text[:start] + "\n".join(table_lines(shapes, lang)) + "\n" + text[stop:] + path.write_text(text, encoding="utf-8", newline="\n") + print(f"{name}: tables written") + + +if __name__ == "__main__": + main() diff --git a/design/cursors/prism-glow/model/prism-glow.blend b/design/cursors/prism-glow/model/prism-glow.blend new file mode 100644 index 0000000000..8edc4553b1 Binary files /dev/null and b/design/cursors/prism-glow/model/prism-glow.blend differ diff --git a/design/cursors/prism-glow/model/prism-glow.glb b/design/cursors/prism-glow/model/prism-glow.glb new file mode 100644 index 0000000000..2aaa2d145b Binary files /dev/null and b/design/cursors/prism-glow/model/prism-glow.glb differ diff --git a/design/cursors/prism-glow/model/prism.json b/design/cursors/prism-glow/model/prism.json new file mode 100644 index 0000000000..5aae7a713f --- /dev/null +++ b/design/cursors/prism-glow/model/prism.json @@ -0,0 +1,985 @@ +{ + "arrow": { + "hotspot": [ + 208, + 42 + ], + "verts": { + "A": [ + 237, + 100, + 0.0 + ], + "B": [ + 443, + 290, + 0.0 + ], + "F": [ + 712, + 549, + 0.0 + ], + "K": [ + 543, + 563, + 0.0 + ], + "J": [ + 612, + 744, + 0.0 + ], + "I": [ + 492, + 807, + 0.0 + ], + "H": [ + 394, + 597, + 0.0 + ], + "G": [ + 241, + 732, + 0.0 + ], + "D": [ + 241, + 491, + 0.0 + ], + "C": [ + 366, + 374, + 108.0 + ], + "E": [ + 458, + 539, + 76.0 + ], + "X": [ + 515, + 449, + 66.0 + ] + }, + "outline": [ + "A", + "B", + "F", + "K", + "J", + "I", + "H", + "G", + "D" + ], + "facets": [ + [ + "A", + "C", + "D" + ], + [ + "A", + "B", + "C" + ], + [ + "B", + "X", + "C" + ], + [ + "B", + "F", + "X" + ], + [ + "C", + "X", + "E" + ], + [ + "X", + "F", + "E" + ], + [ + "E", + "F", + "K" + ], + [ + "C", + "E", + "H" + ], + [ + "C", + "H", + "G" + ], + [ + "C", + "G", + "D" + ], + [ + "E", + "I", + "H" + ], + [ + "E", + "J", + "I" + ], + [ + "E", + "K", + "J" + ] + ], + "colors": [ + "#01f8ee", + "#cadffc", + "#a643fc", + "#c348fc", + "#033df4", + "#002cc0", + "#124efb", + "#00208a", + "#001b80", + "#00b7fb", + "#0022b7", + "#008afc", + "#00fbfb" + ], + "silhouette": [ + [ + 214, + 40 + ], + [ + 207, + 43 + ], + [ + 204, + 49 + ], + [ + 204, + 793 + ], + [ + 207, + 796 + ], + [ + 212, + 796 + ], + [ + 229, + 784 + ], + [ + 381, + 653 + ], + [ + 470, + 846 + ], + [ + 475, + 853 + ], + [ + 481, + 854 + ], + [ + 642, + 770 + ], + [ + 654, + 762 + ], + [ + 654, + 753 + ], + [ + 588, + 594 + ], + [ + 628, + 588 + ], + [ + 768, + 576 + ], + [ + 773, + 572 + ], + [ + 773, + 568 + ], + [ + 768, + 560 + ], + [ + 675, + 464 + ], + [ + 604, + 393 + ], + [ + 478, + 272 + ], + [ + 284, + 96 + ], + [ + 221, + 42 + ] + ] + }, + "hand": { + "hotspot": [ + 1220, + 50 + ], + "verts": { + "I1": [ + 1180, + 122, + 0.0 + ], + "I2": [ + 1221, + 85, + 0.0 + ], + "I3": [ + 1261, + 90, + 0.0 + ], + "I4": [ + 1290, + 132, + 0.0 + ], + "IT1": [ + 1224, + 118, + 46.0 + ], + "IT2": [ + 1274, + 135, + 35.0 + ], + "IT3": [ + 1191, + 262, + 54.0 + ], + "IL": [ + 1179, + 265, + 0.0 + ], + "P1": [ + 1292, + 427, + 0.0 + ], + "W1": [ + 1321, + 456, + 0.0 + ], + "M1": [ + 1323, + 334, + 0.0 + ], + "M2": [ + 1356, + 302, + 0.0 + ], + "M3": [ + 1395, + 309, + 0.0 + ], + "M4": [ + 1415, + 340, + 0.0 + ], + "MT1": [ + 1354, + 334, + 46.0 + ], + "MT2": [ + 1390, + 344, + 35.0 + ], + "P2": [ + 1339, + 479, + 59.0 + ], + "MB1": [ + 1386, + 481, + 54.0 + ], + "MB2": [ + 1415, + 462, + 0.0 + ], + "RL": [ + 1441, + 467, + 0.0 + ], + "R1": [ + 1443, + 368, + 0.0 + ], + "R2": [ + 1476, + 338, + 0.0 + ], + "R3": [ + 1515, + 345, + 0.0 + ], + "R4": [ + 1536, + 377, + 0.0 + ], + "RT1": [ + 1480, + 367, + 43.0 + ], + "RT2": [ + 1513, + 381, + 34.0 + ], + "RB": [ + 1448, + 475, + 57.0 + ], + "RB1": [ + 1476, + 498, + 54.0 + ], + "RB2": [ + 1535, + 489, + 0.0 + ], + "PK": [ + 1567, + 502, + 0.0 + ], + "K1": [ + 1568, + 403, + 0.0 + ], + "K2": [ + 1600, + 375, + 0.0 + ], + "K3": [ + 1637, + 382, + 0.0 + ], + "K4": [ + 1658, + 412, + 0.0 + ], + "KT1": [ + 1601, + 404, + 40.0 + ], + "KT2": [ + 1640, + 420, + 30.0 + ], + "K5": [ + 1658, + 608, + 0.0 + ], + "PR": [ + 1651, + 622, + 0.0 + ], + "BR": [ + 1522, + 818, + 0.0 + ], + "BL": [ + 1208, + 820, + 0.0 + ], + "HC": [ + 1406, + 640, + 76.0 + ], + "Q": [ + 1178, + 572, + 49.0 + ], + "TB": [ + 1128, + 682, + 0.0 + ], + "TT2": [ + 1015, + 490, + 0.0 + ], + "TL": [ + 1013, + 427, + 0.0 + ], + "TT": [ + 1069, + 402, + 0.0 + ], + "TR": [ + 1133, + 449, + 0.0 + ], + "TT1": [ + 1065, + 435, + 43.0 + ], + "TH": [ + 1144, + 541, + 40.0 + ] + }, + "outline": [ + "I1", + "I2", + "I3", + "I4", + "P1", + "W1", + "M1", + "M2", + "M3", + "M4", + "MB2", + "RL", + "R1", + "R2", + "R3", + "R4", + "RB2", + "PK", + "K1", + "K2", + "K3", + "K4", + "K5", + "PR", + "BR", + "BL", + "TB", + "TT2", + "TL", + "TT", + "TR", + "TH", + "Q", + "IL" + ], + "facets": [ + [ + "IT1", + "IT2", + "IT3" + ], + [ + "IT1", + "I2", + "I3", + "IT2" + ], + [ + "I1", + "I2", + "IT1" + ], + [ + "I1", + "IT1", + "IT3", + "IL" + ], + [ + "I3", + "I4", + "IT2" + ], + [ + "IT2", + "I4", + "P1", + "IT3" + ], + [ + "IL", + "IT3", + "P1", + "Q" + ], + [ + "Q", + "P1", + "W1" + ], + [ + "Q", + "W1", + "P2" + ], + [ + "Q", + "P2", + "HC" + ], + [ + "MT1", + "MT2", + "P2" + ], + [ + "MT1", + "M2", + "M3", + "MT2" + ], + [ + "M1", + "M2", + "MT1" + ], + [ + "M1", + "MT1", + "P2", + "W1" + ], + [ + "M3", + "M4", + "MT2" + ], + [ + "MT2", + "M4", + "MB2", + "MB1", + "P2" + ], + [ + "RT1", + "RT2", + "RB" + ], + [ + "RT1", + "R2", + "R3", + "RT2" + ], + [ + "R1", + "R2", + "RT1" + ], + [ + "R1", + "RT1", + "RB", + "RL" + ], + [ + "R3", + "R4", + "RT2" + ], + [ + "RT2", + "R4", + "RB2", + "RB1", + "RB" + ], + [ + "KT1", + "KT2", + "PK" + ], + [ + "KT1", + "K2", + "K3", + "KT2" + ], + [ + "K1", + "K2", + "KT1" + ], + [ + "K1", + "KT1", + "PK" + ], + [ + "K3", + "K4", + "KT2" + ], + [ + "KT2", + "K4", + "K5", + "PR", + "PK" + ], + [ + "P2", + "MB1", + "MB2", + "RL", + "RB", + "RB1", + "RB2", + "PK", + "HC" + ], + [ + "HC", + "PK", + "BR" + ], + [ + "PK", + "PR", + "BR" + ], + [ + "HC", + "BR", + "BL" + ], + [ + "Q", + "HC", + "BL" + ], + [ + "TT1", + "TH", + "TT2" + ], + [ + "TL", + "TT", + "TT1", + "TT2" + ], + [ + "TT", + "TR", + "TH", + "TT1" + ], + [ + "TT2", + "TH", + "TB" + ], + [ + "TH", + "Q", + "TB" + ], + [ + "Q", + "BL", + "TB" + ] + ], + "colors": [ + "#b2fcfc", + "#01fbfc", + "#00faf3", + "#01faf4", + "#00b7f4", + "#00a5eb", + "#bf54fc", + "#662ffa", + "#7237fa", + "#7651fc", + "#88fbfc", + "#00e8ee", + "#00f5f4", + "#76fafb", + "#00beed", + "#00b1f0", + "#8cfcfc", + "#00f3f3", + "#00e1ed", + "#01edf3", + "#00cef6", + "#00b7f7", + "#82fbfc", + "#01f1f5", + "#00e0ee", + "#01f1f3", + "#00c0f2", + "#6f31fb", + "#0129a6", + "#1542fb", + "#0021af", + "#002eba", + "#01f7fc", + "#bffcfc", + "#01fafc", + "#00e3f0", + "#00ddee", + "#00beef", + "#008cf5" + ], + "silhouette": [ + [ + 1209, + 50 + ], + [ + 1169, + 85 + ], + [ + 1145, + 109 + ], + [ + 1143, + 113 + ], + [ + 1142, + 404 + ], + [ + 1137, + 404 + ], + [ + 1073, + 360 + ], + [ + 1066, + 361 + ], + [ + 988, + 401 + ], + [ + 978, + 409 + ], + [ + 976, + 477 + ], + [ + 977, + 500 + ], + [ + 1060, + 640 + ], + [ + 1179, + 846 + ], + [ + 1186, + 852 + ], + [ + 1197, + 856 + ], + [ + 1530, + 856 + ], + [ + 1543, + 852 + ], + [ + 1556, + 839 + ], + [ + 1695, + 623 + ], + [ + 1695, + 401 + ], + [ + 1651, + 348 + ], + [ + 1647, + 346 + ], + [ + 1589, + 338 + ], + [ + 1584, + 340 + ], + [ + 1566, + 357 + ], + [ + 1538, + 317 + ], + [ + 1531, + 311 + ], + [ + 1469, + 301 + ], + [ + 1464, + 303 + ], + [ + 1445, + 320 + ], + [ + 1416, + 276 + ], + [ + 1412, + 273 + ], + [ + 1358, + 264 + ], + [ + 1348, + 264 + ], + [ + 1330, + 280 + ], + [ + 1328, + 256 + ], + [ + 1329, + 200 + ], + [ + 1327, + 121 + ], + [ + 1283, + 61 + ], + [ + 1278, + 57 + ], + [ + 1225, + 50 + ] + ] + } +} diff --git a/design/cursors/prism-glow/model/prism_geo.py b/design/cursors/prism-glow/model/prism_geo.py new file mode 100644 index 0000000000..28a115496b --- /dev/null +++ b/design/cursors/prism-glow/model/prism_geo.py @@ -0,0 +1,157 @@ +"""Prism Glow cursor as a faceted mesh: the facets traced from ../source.png, and their heights. + +Coordinates are source-PNG pixels (x right, y down); `z` is the height above the navy rim's top in +the same pixels (unlisted vertices sit on the rim). `outline` is the crystal's edge against the rim, +`facets` are the flat faces of the 2D art, each an ordered polygon. + + python prism_geo.py # writes prism.json: vertices, facets, their colours in the art, silhouettes + +Then build the Blender scene and the glTF from it with build_blend.py. +""" + +import json +from pathlib import Path + +import cv2 +import numpy as np + +HERE = Path(__file__).resolve().parent + +ARROW = { + "hotspot": (208, 42), + "verts": { + "A": (237, 100), "B": (443, 290), "F": (712, 549), "K": (543, 563), "J": (612, 744), + "I": (492, 807), "H": (394, 597), "G": (241, 732), "D": (241, 491), + "C": (366, 374), "E": (458, 539), "X": (515, 449), + }, + "outline": ["A", "B", "F", "K", "J", "I", "H", "G", "D"], + "z": {"C": 108, "E": 76, "X": 66}, + "facets": [ + ["A", "C", "D"], ["A", "B", "C"], ["B", "X", "C"], ["B", "F", "X"], + ["C", "X", "E"], ["X", "F", "E"], ["E", "F", "K"], + # the notch H sits on the E-G line: one concept facet, folded along C-H + ["C", "E", "H"], ["C", "H", "G"], ["C", "G", "D"], + ["E", "I", "H"], ["E", "J", "I"], ["E", "K", "J"], + ], +} + +HAND = { + "hotspot": (1220, 50), + "verts": { + # index finger + "I1": (1180, 122), "I2": (1221, 85), "I3": (1261, 90), "I4": (1290, 132), + "IT1": (1224, 118), "IT2": (1274, 135), "IT3": (1191, 262), "IL": (1179, 265), + "P1": (1292, 427), + # middle finger + "W1": (1321, 456), "M1": (1323, 334), "M2": (1356, 302), "M3": (1395, 309), + "M4": (1415, 340), "MT1": (1354, 334), "MT2": (1390, 344), "P2": (1339, 479), + "MB1": (1386, 481), "MB2": (1415, 462), + # ring finger + "RL": (1441, 467), "R1": (1443, 368), "R2": (1476, 338), "R3": (1515, 345), + "R4": (1536, 377), "RT1": (1480, 367), "RT2": (1513, 381), "RB": (1448, 475), + "RB1": (1476, 498), "RB2": (1535, 489), + # pinky + "PK": (1567, 502), "K1": (1568, 403), "K2": (1600, 375), "K3": (1637, 382), + "K4": (1658, 412), "KT1": (1601, 404), "KT2": (1640, 420), "K5": (1658, 608), + "PR": (1651, 622), + # palm + "BR": (1522, 818), "BL": (1208, 820), "HC": (1406, 640), "Q": (1178, 572), + # thumb + "TB": (1128, 682), "TT2": (1015, 490), "TL": (1013, 427), "TT": (1069, 402), + "TR": (1133, 449), "TT1": (1065, 435), "TH": (1144, 541), + }, + # Each fingertip is cut like a gem: a table (the white facet), crown facets down to the rim. + # The thumb web (TH, Q) rises above the rim so the thumb's lower facets are not flat. + "z": { + "IT1": 46, "IT2": 35, "IT3": 54, + "MT1": 46, "MT2": 35, "P2": 59, "MB1": 54, + "RT1": 43, "RT2": 34, "RB": 57, "RB1": 54, + "KT1": 40, "KT2": 30, + "TT1": 43, "TH": 40, "Q": 49, + "HC": 76, + }, + "outline": [ + "I1", "I2", "I3", "I4", "P1", "W1", "M1", "M2", "M3", "M4", "MB2", "RL", "R1", "R2", + "R3", "R4", "RB2", "PK", "K1", "K2", "K3", "K4", "K5", "PR", "BR", "BL", "TB", "TT2", + "TL", "TT", "TR", "TH", "Q", "IL", + ], + "facets": [ + # index: table, crown, sides + ["IT1", "IT2", "IT3"], ["IT1", "I2", "I3", "IT2"], ["I1", "I2", "IT1"], + ["I1", "IT1", "IT3", "IL"], ["I3", "I4", "IT2"], ["IT2", "I4", "P1", "IT3"], + ["IL", "IT3", "P1", "Q"], + # palm under the index (W1 sits on the P1-P2 line) + ["Q", "P1", "W1"], ["Q", "W1", "P2"], ["Q", "P2", "HC"], + # middle finger + ["MT1", "MT2", "P2"], ["MT1", "M2", "M3", "MT2"], ["M1", "M2", "MT1"], + ["M1", "MT1", "P2", "W1"], ["M3", "M4", "MT2"], ["MT2", "M4", "MB2", "MB1", "P2"], + # ring finger + ["RT1", "RT2", "RB"], ["RT1", "R2", "R3", "RT2"], ["R1", "R2", "RT1"], + ["R1", "RT1", "RB", "RL"], ["R3", "R4", "RT2"], ["RT2", "R4", "RB2", "RB1", "RB"], + # pinky + ["KT1", "KT2", "PK"], ["KT1", "K2", "K3", "KT2"], ["K1", "K2", "KT1"], + ["K1", "KT1", "PK"], ["K3", "K4", "KT2"], ["KT2", "K4", "K5", "PR", "PK"], + # palm + ["P2", "MB1", "MB2", "RL", "RB", "RB1", "RB2", "PK", "HC"], + ["HC", "PK", "BR"], ["PK", "PR", "BR"], ["HC", "BR", "BL"], ["Q", "HC", "BL"], + # thumb + ["TT1", "TH", "TT2"], ["TL", "TT", "TT1", "TT2"], ["TT", "TR", "TH", "TT1"], + ["TT2", "TH", "TB"], ["TH", "Q", "TB"], ["Q", "BL", "TB"], + ], +} + + +def area(poly): + p = np.asarray(poly, float) + return 0.5 * np.sum(p[:, 0] * np.roll(p[:, 1], -1) - np.roll(p[:, 0], -1) * p[:, 1]) + + +def check(shape, name): + """The facets tile the outline exactly: no gap, no overlap, no stray vertex.""" + v = shape["verts"] + outline = abs(area([v[k] for k in shape["outline"]])) + facets = sum(abs(area([v[k] for k in f])) for f in shape["facets"]) + unused = set(v) - {k for f in shape["facets"] for k in f} + assert abs(facets / outline - 1) < 1e-9 and not unused, (name, facets / outline, unused) + + +def facet_colors(shape, img): + """Median colour of each facet in the art, away from its edges.""" + v = shape["verts"] + out = [] + for f in shape["facets"]: + mask = np.zeros(img.shape[:2], np.uint8) + cv2.fillPoly(mask, [np.array([v[k] for k in f], np.int32)], 255) + inner = cv2.erode(mask, np.ones((7, 7), np.uint8)) + px = img[(inner if inner.any() else mask) > 0][:, :3] + b, g, r = np.median(px, axis=0) + out.append("#%02x%02x%02x" % (int(r), int(g), int(b))) + return out + + +def silhouette(img, x0, x1): + """Outer edge of the navy rim, from the art's alpha.""" + a = (img[:, :, 3] > 127).astype(np.uint8) + a[:, :x0] = 0 + a[:, x1:] = 0 + contours, _ = cv2.findContours(a, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE) + c = max(contours, key=cv2.contourArea) + return cv2.approxPolyDP(c, 1.2, True)[:, 0, :].tolist() + + +if __name__ == "__main__": + img = cv2.imread(str(HERE.parent / "source.png"), cv2.IMREAD_UNCHANGED) + out = {} + for shape, name, (x0, x1) in ((ARROW, "arrow", (0, 887)), (HAND, "hand", (887, 1774))): + check(shape, name) + z = shape["z"] + out[name] = { + "hotspot": shape["hotspot"], + "verts": {k: [*p, float(z.get(k, 0))] for k, p in shape["verts"].items()}, + "outline": shape["outline"], + "facets": shape["facets"], + "colors": facet_colors(shape, img), + "silhouette": silhouette(img, x0, x1), + } + (HERE / "prism.json").write_text(json.dumps(out, indent=1) + "\n", newline="\n") + print({k: f"{len(v['facets'])} facets" for k, v in out.items()}) diff --git a/design/cursors/requirements.md b/design/cursors/requirements.md index 3274014c7f..88ab1ede1d 100644 --- a/design/cursors/requirements.md +++ b/design/cursors/requirements.md @@ -21,16 +21,17 @@ their 3D models. - Every replacement theme and supported cursor state must work with the existing 3D cursor option. -- Every replacement arrow and hand is a real volume, sculpted in the +- Every replacement arrow and hand is a real volume, modelled in the compositor's shaders (`crates/compositor/src/sculpt.rs`): lighting, normals, self shadows and contact shadows respond to cursor tilt and rotation. A PNG face or a height map is not a model. - A rounded extrusion of the PNG silhouette remains the fallback for cursor - states that do not have a dedicated theme model. The five replacement packs - add style-specific volume: a glove with separate fingers, cut crystal facets, - puffy rubber, beveled voxels, and ceramic accents. -- Model the arrow and hand separately for every theme. Do not apply one - contour, material, or roundness rule to all designs. + states that do not have a dedicated theme model. Each pack adds + style-specific volume: gloves with separate fingers, beveled voxels, flat + cut-paper sheets, cushions set in the rim of their drawn outline, and a + faceted glass crystal that refracts the picture under it. +- Model the arrow and hand separately for every sculpted theme. Do not apply + one contour, material, or roundness rule to all designs. - Preserve the cursor hotspot through hover, tilt, yaw, click, and size changes. - Reuse the compositor's 3D lighting and contact shadows. The surface must change with the compositor camera; a static image of a 3D render is not a @@ -40,11 +41,12 @@ their 3D models. | Theme | Arrow | Hand | | --- | --- | --- | -| Studio Ink | Matte black sculpted body; ivory mark becomes a piping set in from the edge. | Ivory glove with a convex palm and separate fingers; remove the dark outline. | -| Prism Glow | Translucent faceted crystal; no navy ink contour. | Crystal palm and distinct faceted fingers; no navy ink contour. | -| Pop Coral | Rounded coral rubber; omit the yellow offset and navy rim. | Plump yellow rubber glove; no navy outline, accents become physical details. | -| Pixel Candy | Keep the stepped voxel form; purple is side/back blocks, not a flat outline. | Stepped voxel fingers and palm; preserve purple as physical side/back blocks. | -| Star Sprout | Soft mint ceramic; no heavy blue outline, raised star and leaves. | Rounded ivory glove with no blue outline; separate mint cuff, star, and leaves. | +| Studio Ink | Black rim, raised ivory band, recessed satin black field. | Ivory glove cushion; the black outline stays as a physical rim, with black ridges between the fingers. | +| Prism Glow | A glass crystal cut along the drawing's facets, in a thin navy rim; the picture under it shows through, refracted with a slight dispersion, privacy blurs included; the facets glow faintly in their drawn colours. Modelled from the 2D drawing alone, keeping its polygonal simplicity. | The same, a crystal hand. | +| Pop Coral | Cut paper: a coral sheet on a navy sheet, flat tops, matte; the yellow offset becomes a flat sheet behind; the click dashes become flat pieces. | A yellow sheet on a navy sheet, flat tops, matte; the coral offset becomes a flat sheet behind; the coral click dashes become flat pieces. | +| Pixel Candy | Simplified pixel art that reads at 20 px as at full size; the 3D model is one cube per pixel of the same grid as the sprite. | A pixel hand from the same kind of grid: index, three knuckles, thumb. | +| Star Sprout | Puffy mint cushion; the navy outline stays as a physical rim (a navy tray and a rounded bead) around it, the star and its leaves in front, each in its own navy rim. | Puffy ivory glove in the same navy rim, with navy ridges between the fingers; separate mint cuff framed in navy; the star and leaves in front, as in `star-sprout/3d-reference.png`. | +| Glass Lens | A glass lens with a defined dark outline and an outer rim of glass, after the macOS glass material: the picture under it is refracted by real shader optics (magnified by the lens, bent and slightly dispersed along its edge), in 3D and, unlike the other themes, in 2D too. | The same, a glass glove with dark ridges between the fingers. | The 2D PNG may keep a drawn contour where it helps readability. Its contour must not be copied blindly onto the 3D material: decide per model whether it @@ -53,7 +55,11 @@ becomes an inlay, side material, separate piece, or disappears. ## Review references - `contact-sheet.png`: 2D PNG themes. -- `3d-concept.png`: visual direction only, not rendered by the app. +- `3d-concept.png`: the 3D reference sheet the eight sculpted models follow + (Prism Glow's crystal follows its 2D drawing instead), not rendered by the + app. +- `star-sprout/3d-reference.png`: a closer view of the Star Sprout hand; also + not rendered by the app. - `3d-direction.md`: per-theme modeling notes. - `README.md`: source PNG and preparation workflow. diff --git a/design/cursors/star-sprout/3d-reference.png b/design/cursors/star-sprout/3d-reference.png new file mode 100644 index 0000000000..760ed6b62c Binary files /dev/null and b/design/cursors/star-sprout/3d-reference.png differ diff --git a/docs/3d-effects-v2.md b/docs/3d-effects-v2.md index 978cd1a809..119c5b3bbd 100644 --- a/docs/3d-effects-v2.md +++ b/docs/3d-effects-v2.md @@ -233,12 +233,11 @@ inconnues sont ignorées). ### B.2 Le réglage **Un seul interrupteur**, `cursor.model3d` (« 3D cursor », **éteint par défaut**) : il passe -**chaque état** du curseur en 3D. La flèche et la main des cinq thèmes d'origine deviennent leur -modèle sculpté (B.3 bis) ; les autres états, et tous ceux du thème par défaut (les seize de -`DEFAULT_CURSOR_SPRITES` : flèche, I, main, croix, mains ouverte et fermée, redimensionnements, -déplacement, interdit, attente…), leur sprite extrudé. Curseur masqué, l'interrupteur est grisé et -son info-bulle dit pourquoi. Éteint, la frame est celle d'avant **à l'octet** (vérifié à plat et -incliné contre le commit de base). +**chaque état** du curseur en 3D. La flèche et la main des cinq thèmes d'origine deviennent +leur modèle (B.3 bis) ; les autres états, et tous ceux du thème par défaut (les seize de `DEFAULT_CURSOR_SPRITES` : flèche, I, main, croix, mains +ouverte et fermée, redimensionnements, déplacement, interdit, attente…), leur sprite extrudé. +Curseur masqué, l'interrupteur est grisé et son info-bulle dit pourquoi. Éteint, la frame est +celle d'avant **à l'octet** (vérifié à plat et incliné contre le commit de base). Tuyauterie : `CursorVisualSettings.model3d`, clé legacy `cursorModel3d`, préréglages (absent → éteint), `SceneCursor.model3d` (`serde(default)`), `LiveParams.cursor_model3d`, paramètre live @@ -287,24 +286,44 @@ du bord (0,48 au pire), sous 1 texel au-delà (le flou arrondit les crêtes). ### B.3 bis Les curseurs sculptés -La flèche et la main des cinq thèmes d'origine sont **modélisées à la main**, en fonctions de -distance signée écrites dans les trois shaders (`sculpt_proto`, `sculpt_material`) : capsules et -unions lissées pour les gants, extrusion arrondie et bombée pour les flèches, voxels biseautés -pour Pixel Candy, polyèdres taillés pour le cristal de Prism Glow. Le PNG du thème reste l'art en -2D. En 3D, la scène nomme le modèle ; `sculpt.rs` en tient la boîte, qui pose le hotspot (pointe -de la flèche, bout de l'index), règle la garde au sol et borne la boîte de dessin. Emplacement du -cbuffer : `trail_a` = [modèle, épaisseur sous z = 0, hauteur au-dessus, 0] (`cursor_model_cb`). +La flèche et la main des thèmes d'origine sont **modélisées à la main**, en fonctions de +distance signée écrites dans les trois shaders (`sculpt_proto`, `sculpt_material`) : voxels +pour Pixel Candy ; pour Studio Ink et Star Sprout des pièces qui gardent le trait de +leur dessin : un plateau et un jonc de la couleur du trait, un coussin de couleur dedans +(`s_rimmed`) ; pour Pop Coral les mêmes formes en papier découpé, des feuilles au dessus plat +(`s_paper`). Prism Glow est à part : un **cristal en maillage**, ses facettes tracées sur son +seul dessin 2D (`design/cursors/prism-glow/model/`), serti dans sa silhouette extrudée, marine. +Le shader le lance de rayons, triangles rangés par boîtes englobantes (un rayon saute les +boîtes qu'il rate) : réfraction par canal (une légère dispersion), réflexions totales internes, +et sortie par son fond plat sur l'image sous le curseur ; chaque facette luit un peu de sa +couleur du dessin, ses plis d'un liseré clair. Cette image est une copie de la frame composée +prise juste avant le curseur : le métrage, puis ses flous de confidentialité, puis le curseur, +net par-dessus. Seuls des pixels déjà floutés passent donc à travers le verre. +`export_compositor.py` écrit le maillage dans `prism_mesh.rs` et dans les trois shaders. Le PNG +du thème reste l'art en 2D. En 3D, la scène nomme le modèle ; `sculpt.rs` en tient la boîte, qui +pose le hotspot (pointe de la flèche, bout de l'index), règle la garde au sol et borne la boîte +de dessin. Emplacement du cbuffer : `trail_a` = [modèle, épaisseur sous z = 0, hauteur +au-dessus, 0] (`cursor_model_cb`). +Pixel Candy est un pixel art dessiné une fois, en grille, dans +`scripts/generate-pixel-candy-voxels.mjs` : le script en tire ses PNG 2D et les tables des trois +shaders, un cube par pixel plein, tous de même hauteur, colorés comme leur pixel (contour prune, +rose, reflet rose pâle, ombre rose foncé). - **Éclairage**, sprites extrudés compris : une lampe proche en haut à gauche, qui met un dégradé et un reflet même sur une face plane ; une lumière d'appoint faible ; le côté ombré teinté par la matière ; occlusion ambiante, ombre propre douce vers la lampe, studio dans les reflets, liseré de Fresnel ; tone map Khronos PBR Neutral. -- **Ombre sur l'écran** : celle de B.3. Le cristal et les voxels donnent de mauvaises distances - loin de leur surface (bornes de plans, champ de grille) : l'ombre prend la forme lisse de Pop - Coral pour le cristal et le contour extrudé pour les voxels. +- **Ombre sur l'écran** : celle de B.3. Les voxels donnent de mauvaises distances loin de leur + surface (champ de grille) : l'ombre de Pixel Candy vient de sa forme extrudée. +- **Antialiasing** : la silhouette l'est par la marche (la distance minimale frôlée, en pixels, + donne la couverture). Les bords intérieurs (deux matières, une arête, un arrondi serré, un + joint de voxels) sont suréchantillonnés : quatre sondes à un demi-pixel du point touché, dans + son plan tangent, et si la matière change ou que la surface s'en écarte, trois rayons de plus + dans le pixel, ombrés avec l'occlusion et les ombres du premier (`STAGE_EDGE`, `STAGE_SHADE`). + Sur un grand curseur, ~+50 % du coût du curseur (rendu logiciel), rien ailleurs. - **Coût de compilation** : FXC recopie chaque appel. La marche, les normales, l'occlusion et les - ombres forment donc une seule boucle à étapes avec un seul appel au modèle, et l'étoile de Star - Sprout un seul appel pour les deux formes. + ombres forment donc une seule boucle à étapes avec un seul appel au modèle, et les thèmes + cerclés un seul appel pour leurs deux formes. ### B.4 La caméra et l'ancrage @@ -490,7 +509,7 @@ C'est le cadre qui s'adapte au métrage — jamais l'inverse : un téléphone au est un téléphone COUCHÉ, avec son œil de caméra sur le bord qui est devenu son haut. **Roundness sous un cadre.** Le slider parcourt 0 → le plafond du cadre choisi (C.5) et se lit -en **%** de cette course, avec l'infobulle « La course s'adapte au cadre » (`roundnessFrameHelp`, +en **%** de cette course, avec l'infobulle « La course dépend du cadre » (`roundnessFrameHelp`, 15 langues). La valeur stockée reste en px ; le natif en relit la position (`roundnessFrac` × petit côté de la sortie ÷ `ROUNDNESS_SLIDER_MAX_PX` = 64, miroir de `paramUnits.ts`). Sans cadre, le slider reste en px et le rendu est celui d'avant, à l'octet. diff --git a/docs/assets/demo.gif b/docs/assets/demo.gif index 9c2a0270ec..23ab2fa658 100644 Binary files a/docs/assets/demo.gif and b/docs/assets/demo.gif differ diff --git a/electron-builder.json5 b/electron-builder.json5 index 8da5be3107..b8e833c9c7 100644 --- a/electron-builder.json5 +++ b/electron-builder.json5 @@ -47,11 +47,18 @@ // (unknown scene fields are #[serde(default)]) rather than erroring. See the script header // and technical-documentation/engineering/build-and-packaging.md. "beforePack": "scripts/before-pack.cjs", + // Standard AppImage updates need .upd_info and a .zsync alongside electron-updater's blockmap. + "artifactBuildCompleted": "scripts/appimage-updates.cjs", + "afterAllArtifactBuild": "scripts/appimage-updates.cjs", "npmRebuild": true, // sharp ships ABI-stable (napi) prebuilt binaries with bundled libvips. Building it from source // needs a system libvips we don't provide and breaks on CI/local ("vips-cpp.42 not found"), so we // let electron-builder use the prebuilt instead of recompiling. "buildDependenciesFromSource": false, + // The static AppImage runtime does not require libfuse2 on the host. + "toolsets": { + "appimage": "1.0.3" + }, "compression": "normal", // Strip unused Chromium locale pak files (saves ~20 MB). Only package locales supported by OpenScreen. // Note: macOS uses underscore variants (e.g. pt_BR.lproj, zh_CN.lproj, zh_TW.lproj), while Windows @@ -126,6 +133,14 @@ "from": "THIRD-PARTY-NOTICES.md", "to": "THIRD-PARTY-NOTICES.md" }, + // The full licence texts of the npm packages Vite bundles and of the crates compiled into + // the compositor addon. Generated, not committed: every build:* script and build.yml's + // macOS job run `npm run licenses:generate` first. electron-builder only warns about a + // missing source here, so scripts/before-pack.cjs is what refuses to package without it. + { + "from": "build-licenses/THIRD-PARTY-LICENSES.txt", + "to": "THIRD-PARTY-LICENSES.txt" + }, { "from": "LICENSE", "to": "LICENSE" @@ -395,7 +410,7 @@ "nsis" ], "icon": "icons/icons/win/icon.ico", - "artifactName": "${productName}.Setup.${version}.${ext}", + "artifactName": "${productName}.Setup.${version}-${arch}.${ext}", "extraResources": [ { "from": "electron/native/bin", diff --git a/electron/ai-edition/agent-tools.test.ts b/electron/ai-edition/agent-tools.test.ts index fc8ba40857..da1a681711 100644 --- a/electron/ai-edition/agent-tools.test.ts +++ b/electron/ai-edition/agent-tools.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; import { + type AxcutAnnotationRegion, type AxcutDocument, createEmptyDocument, documentSchema, @@ -16,6 +17,15 @@ import { executeAgentTool, isMutatingTool, MUTATING_TOOL_NAMES } from "./agent-t * so pinning it costs nothing and makes every document in the file comparable. */ const FIXTURE_CREATED_AT = "2026-01-01T00:00:00.000Z"; +/** Where an annotation's box is centred, in frame percentages. */ +function centreOf(region: Pick | undefined) { + if (!region) return undefined; + return { + x: region.position.x + region.size.width / 2, + y: region.position.y + region.size.height / 2, + }; +} + function fixtureDocument(): AxcutDocument { const base = createEmptyDocument({ title: "Test", @@ -183,8 +193,10 @@ describe("the mutating-tool table", () => { "removeClip", "removeModifier", "removeTrim", + "removeWords", "replaceTimeline", "setAnnotation", + "setAspectRatio", "setAudio", "setCameraFullscreen", "setClipRange", @@ -647,6 +659,52 @@ describe("executeAgentTool", () => { ); }); + it("sets a zoom's 3D camera, reports it, and takes it back off with none", () => { + const added = executeAgentTool( + fixtureDocument(), + "addZoom", + JSON.stringify({ startSec: 2, endSec: 4, rotationPreset: "orbit" }), + ); + expect(added.ok).toBe(true); + const withZoom = added.document as AxcutDocument; + expect(withZoom.zoomRanges[0]).toMatchObject({ rotationPreset: "orbit" }); + expect(JSON.parse(added.resultJson)).toMatchObject({ rotationPreset: "orbit" }); + expect(() => documentSchema.parse(withZoom)).not.toThrow(); + + const id = withZoom.zoomRanges[0].id; + const turned = executeAgentTool( + withZoom, + "setZoom", + JSON.stringify({ zoomId: id, rotationPreset: "left" }), + ).document as AxcutDocument; + expect(turned.zoomRanges[0]).toMatchObject({ rotationPreset: "left", depth: 3 }); + + const flat = executeAgentTool( + turned, + "setZoom", + JSON.stringify({ zoomId: id, rotationPreset: "none" }), + ).document as AxcutDocument; + expect(flat.zoomRanges[0]).not.toHaveProperty("rotationPreset"); + + // Left out, it stays as it was; an unknown camera is refused. + const deeper = executeAgentTool(turned, "setZoom", JSON.stringify({ zoomId: id, depth: 5 })); + expect(deeper.document?.zoomRanges[0]).toMatchObject({ rotationPreset: "left", depth: 5 }); + expect( + executeAgentTool(withZoom, "setZoom", JSON.stringify({ zoomId: id, rotationPreset: "up" })) + .ok, + ).toBe(false); + }); + + it("addZoom without a camera stays flat", () => { + const result = executeAgentTool( + fixtureDocument(), + "addZoom", + JSON.stringify({ startSec: 2, endSec: 4 }), + ); + expect(result.document?.zoomRanges[0]).not.toHaveProperty("rotationPreset"); + expect(JSON.parse(result.resultJson)).not.toHaveProperty("rotationPreset"); + }); + it("addSpeed writes a legacyEditor speed region the snapshot exposes", () => { const result = executeAgentTool( fixtureDocument(), @@ -684,9 +742,11 @@ describe("executeAgentTool", () => { endMs: 3000, type: "text", textContent: "Look here", - // x/y are the text's centre: a 30x20 box centred on (20, 80). - position: { x: 5, y: 70 }, + // On the output frame, as the editor places text: padding and zooms never move it. + space: "frame", }); + // x/y are the text's centre, and the box is fitted to the text around it. + expect(centreOf(ann)).toEqual({ x: expect.closeTo(20, 5), y: expect.closeTo(80, 5) }); expect(() => documentSchema.parse(result.document)).not.toThrow(); }); @@ -696,16 +756,126 @@ describe("executeAgentTool", () => { "addAnnotation", JSON.stringify({ startSec: 1, endSec: 3, text: "Hi" }), ).document?.annotations.at(-1); - expect(centred).toMatchObject({ - position: { x: 35, y: 40 }, - style: { backgroundColor: "rgba(0, 0, 0, 0.7)" }, - }); + expect(centred).toMatchObject({ style: { backgroundColor: "rgba(0, 0, 0, 0.7)" } }); + expect(centreOf(centred)).toEqual({ x: expect.closeTo(50, 5), y: expect.closeTo(50, 5) }); const corner = executeAgentTool( fixtureDocument(), "addAnnotation", JSON.stringify({ startSec: 1, endSec: 3, text: "Hi", x: 100, y: 0 }), ).document?.annotations.at(-1); - expect(corner).toMatchObject({ position: { x: 70, y: 0 } }); + expect(corner?.position.y).toBe(0); + expect((corner?.position.x ?? 0) + (corner?.size.width ?? 0)).toBeCloseTo(100, 5); + }); + + it("addAnnotation takes the look the editor's inspector offers", () => { + const add = (look: Record) => + executeAgentTool( + fixtureDocument(), + "addAnnotation", + JSON.stringify({ startSec: 1, endSec: 3, text: "Palettes in one click", ...look }), + ).document?.annotations.at(-1); + const plain = add({}); + const title = add({ + fontSize: 72, + color: "#ffcc00", + background: "none", + bold: false, + animation: "typewriter", + }); + expect(title?.style).toMatchObject({ + fontSize: 72, + color: "#ffcc00", + backgroundColor: "transparent", + fontWeight: "normal", + textAnimation: "typewriter", + }); + // Bigger text, bigger box: the box is the text's. + expect(title?.size.width).toBeGreaterThan(plain?.size.width ?? 0); + }); + + it("setAnnotation moves the text, restyles it and refits its box around its centre", () => { + const added = executeAgentTool( + fixtureDocument(), + "addAnnotation", + JSON.stringify({ startSec: 1, endSec: 3, text: "Hi", x: 30, y: 30 }), + ); + const id = JSON.parse(added.resultJson).annotationId; + const moved = executeAgentTool( + added.document as AxcutDocument, + "setAnnotation", + JSON.stringify({ annotationId: id, x: 70, y: 85, animation: "pop" }), + ).document?.annotations.find((a) => a.id === id); + expect(centreOf(moved)).toEqual({ x: expect.closeTo(70, 5), y: expect.closeTo(85, 5) }); + expect(moved?.style.textAnimation).toBe("pop"); + + const retexted = executeAgentTool( + added.document as AxcutDocument, + "setAnnotation", + JSON.stringify({ annotationId: id, text: "A much longer callout than before" }), + ).document?.annotations.find((a) => a.id === id); + expect(retexted?.size.width).toBeGreaterThan(moved?.size.width ?? 0); + expect(centreOf(retexted)).toEqual({ x: expect.closeTo(30, 5), y: expect.closeTo(30, 5) }); + }); + + it("setAspectRatio sets the video's shape, keeping frame text where it was", () => { + const added = executeAgentTool( + fixtureDocument(), + "addAnnotation", + JSON.stringify({ startSec: 1, endSec: 3, text: "Centred", x: 50, y: 50 }), + ); + const result = executeAgentTool( + added.document as AxcutDocument, + "setAspectRatio", + JSON.stringify({ aspectRatio: "9:16" }), + ); + expect(result.ok).toBe(true); + expect(JSON.parse(result.resultJson)).toMatchObject({ aspectRatio: "9:16" }); + expect( + (result.document?.legacyEditor as { aspectRatio?: string } | undefined)?.aspectRatio, + ).toBe("9:16"); + expect(centreOf(result.document?.annotations.at(-1))).toEqual({ + x: expect.closeTo(50, 5), + y: expect.closeTo(50, 5), + }); + expect( + executeAgentTool(fixtureDocument(), "setAspectRatio", JSON.stringify({ aspectRatio: "7:3" })) + .ok, + ).toBe(false); + }); + + it("refuses a set that would run into another region, instead of landing it elsewhere", () => { + let document = fixtureDocument(); + const add = (name: string, args: Record) => { + const result = executeAgentTool(document, name, JSON.stringify(args)); + document = result.document as AxcutDocument; + return result; + }; + const first = add("addAnnotation", { startSec: 1, endSec: 2, text: "One" }); + add("addAnnotation", { startSec: 6, endSec: 8, text: "Two" }); + const firstId = JSON.parse(first.resultJson).annotationId; + const blocked = executeAgentTool( + document, + "setAnnotation", + JSON.stringify({ annotationId: firstId, startSec: 7, endSec: 9 }), + ); + expect(blocked.ok).toBe(false); + expect(blocked.document).toBeUndefined(); + expect(blocked.resultJson).toMatch(/NOT changed.*6\.0|NOT changed/); + + // The same rule for every kind of effect. + const zoom = add("addZoom", { startSec: 1, endSec: 2, depth: 2 }); + add("addZoom", { startSec: 6, endSec: 8, depth: 4 }); + const zoomId = JSON.parse(zoom.resultJson).zoomId; + const zoomBlocked = executeAgentTool( + document, + "setZoom", + JSON.stringify({ zoomId, startSec: 7, endSec: 9 }), + ); + expect(zoomBlocked.ok).toBe(false); + // A span clear of it still moves. + expect( + executeAgentTool(document, "setZoom", JSON.stringify({ zoomId, startSec: 3, endSec: 4 })).ok, + ).toBe(true); }); it("snapshot exposes clips/trims/effects as virtual-time groups with a time-base note", () => { @@ -1686,6 +1856,29 @@ describe("getCursorTrack", () => { expect(result.resultJson).not.toContain('virtualSec":'); }); + it("reports a hidden click and the return to visibility through the tool payload", () => { + const samples = [ + { timeMs: 0, cx: 0.5, cy: 0.5, visible: true, interactionType: "move" }, + { timeMs: 50, cx: 0.5, cy: 0.5, visible: false, interactionType: "click" }, + { timeMs: 100, cx: 0.5, cy: 0.5, visible: true, interactionType: "move" }, + { timeMs: 1000, cx: 0.5, cy: 0.5, visible: true, interactionType: "move" }, + ]; + const result = executeAgentTool(fixtureDocument(), "getCursorTrack", "{}", { + cursorTelemetry: { load: { status: "ok", assetId: "asset_1", samples } }, + }); + const payload = JSON.parse(result.resultJson); + + expect(result.ok).toBe(true); + expect(payload.points).toContainEqual({ + atSec: 0.05, + cx: 0.5, + cy: 0.5, + kind: "click", + visible: false, + }); + expect(payload.points).toContainEqual({ atSec: 0.1, cx: 0.5, cy: 0.5 }); + }); + it("says 'no-sidecar' when the asset was checked and has none", () => { const result = executeAgentTool(fixtureDocument(), "getCursorTrack", "{}", { cursorTelemetry: { load: { status: "no-sidecar", assetId: "asset_1" } }, @@ -2361,6 +2554,227 @@ describe("getTranscriptWords", () => { }); }); +describe("removeWords", () => { + function repeatedWordDocument(): AxcutDocument { + const base = fixtureDocument(); + return { + ...base, + transcripts: [ + { + ...base.transcripts[0], + words: [ + { id: "filler", segmentId: "seg_1", startSec: 1, endSec: 1.2, text: "like" }, + { id: "meaningful", segmentId: "seg_1", startSec: 3, endSec: 3.2, text: "like" }, + ], + }, + ], + }; + } + + it("cuts only the selected occurrence and reports the trim actually stored", () => { + const before = repeatedWordDocument(); + const read = run(before, "getTranscriptWords", {}); + expect(JSON.parse(read.resultJson).words.map((word: { id: string }) => word.id)).toEqual([ + "filler", + "meaningful", + ]); + const result = run(before, "removeWords", { wordIds: ["filler"] }); + expect(result.ok).toBe(true); + const next = result.document as AxcutDocument; + const trim = next.timeline.trimRanges.at(-1); + expect(trim).toMatchObject({ assetId: "asset_1", clipId: "clip_1", origin: "agent" }); + expect(trim?.endSec).toBeLessThan(3); // the second "like" still plays + expect(next.transcripts).toEqual(before.transcripts); + expect(next.timeline.clips).toEqual(before.timeline.clips); + const payload = JSON.parse(result.resultJson); + expect(payload).toMatchObject({ + requested: 1, + removedCount: 1, + removed: [ + { + wordId: "filler", + text: "like", + assetId: "asset_1", + clipId: "clip_1", + wordStartSec: 1, + wordEndSec: 1.2, + trimRangeId: trim?.id, + startSec: trim?.startSec, + endSec: trim?.endSec, + }, + ], + }); + expect(next.timeline.trimRanges).toHaveLength(before.timeline.trimRanges.length + 1); + }); + + it("cuts consecutive words as one span, gaps included, and the others apart", () => { + const base = fixtureDocument(); + const before: AxcutDocument = { + ...base, + transcripts: [ + { + ...base.transcripts[0], + words: [ + { id: "word_1", segmentId: "seg_1", startSec: 1, endSec: 1.2, text: "so" }, + { id: "word_2", segmentId: "seg_1", startSec: 1.4, endSec: 1.5, text: "I" }, + { id: "word_3", segmentId: "seg_1", startSec: 1.6, endSec: 1.9, text: "mean" }, + { id: "word_4", segmentId: "seg_1", startSec: 2.2, endSec: 2.4, text: "the" }, + { id: "word_5", segmentId: "seg_1", startSec: 3, endSec: 3.2, text: "um" }, + ], + }, + ], + }; + const result = run(before, "removeWords", { wordIds: ["word_3", "word_5", "word_2"] }); + expect(result.ok).toBe(true); + const added = result.document?.timeline.trimRanges.slice(before.timeline.trimRanges.length); + expect(added?.map(({ startSec, endSec }) => [startSec, endSec])).toEqual([ + [1.4, 1.9], + [3, 3.2], + ]); + const removed = JSON.parse(result.resultJson).removed; + expect(removed.map((word: { wordId: string }) => word.wordId)).toEqual([ + "word_2", + "word_3", + "word_5", + ]); + expect(removed[1].trimRangeId).toBe(removed[0].trimRangeId); + expect(removed[2].trimRangeId).not.toBe(removed[0].trimRangeId); + }); + + it("refuses a duplicated or already-cut word without cutting anything", () => { + const before = repeatedWordDocument(); + const duplicated = run(before, "removeWords", { wordIds: ["filler", "filler"] }); + expect(duplicated.ok).toBe(false); + expect(duplicated.document).toBeUndefined(); + const cut = run(before, "removeWords", { wordIds: ["filler"] }).document as AxcutDocument; + const again = run(cut, "removeWords", { wordIds: ["meaningful", "filler"] }); + expect(again.ok).toBe(false); + expect(again.document).toBeUndefined(); + expect(again.resultJson).toContain("already cut"); + }); + + it("preserves addTrim warnings when removing a filler cuts a zoom transition", () => { + const before = repeatedWordDocument(); + before.timeline.trimRanges = []; + const zoomed = run(before, "addZoom", { startSec: 2, endSec: 3, depth: 3 }).document; + expect(zoomed).toBeDefined(); + if (!zoomed) throw new Error("Zoom fixture failed"); + const directTrim = run(zoomed, "addTrim", { + assetId: "asset_1", + clipId: "clip_1", + startSec: 1, + endSec: 1.2, + }); + const expected = JSON.parse(directTrim.resultJson).cutTransitions; + expect(expected.length).toBeGreaterThan(0); + const removed = run(zoomed, "removeWords", { assetId: "asset_1", wordIds: ["filler"] }); + expect(removed.ok).toBe(true); + expect(JSON.parse(removed.resultJson).cutTransitions).toEqual(expected); + }); + + it("refuses an invalid ID atomically, including when another ID is valid", () => { + const before = repeatedWordDocument(); + const result = run(before, "removeWords", { wordIds: ["filler", "missing"] }); + expect(result.ok).toBe(false); + expect(result.document).toBeUndefined(); + expect(result.resultJson).toContain("Unknown transcript word ID: missing"); + expect(before.timeline.trimRanges).toHaveLength(1); + }); + + it("resolves a word ID to its own asset and clip, not the primary asset", () => { + const before = repeatedWordDocument(); + before.assets.push({ ...before.assets[0], id: "asset_2" }); + before.timeline.clips.push({ + ...before.timeline.clips[0], + id: "clip_3", + assetId: "asset_2", + }); + before.transcripts.push({ + ...before.transcripts[0], + assetId: "asset_2", + words: [{ id: "other_filler", segmentId: "seg_1", startSec: 4, endSec: 4.2, text: "um" }], + }); + const result = run(before, "removeWords", { wordIds: ["other_filler"] }); + expect(result.ok).toBe(true); + expect(result.document?.timeline.trimRanges.at(-1)).toMatchObject({ + assetId: "asset_2", + clipId: "clip_3", + }); + expect(JSON.parse(result.resultJson).removed[0]).toMatchObject({ + assetId: "asset_2", + clipId: "clip_3", + }); + }); + + it("qualifies normally numbered word IDs by the asset read from the transcript", () => { + const before = repeatedWordDocument(); + before.transcripts[0].words[0].id = "word_1"; + before.assets.push({ ...before.assets[0], id: "asset_2" }); + before.timeline.clips.push({ ...before.timeline.clips[0], id: "clip_3", assetId: "asset_2" }); + before.transcripts.push({ + ...before.transcripts[0], + assetId: "asset_2", + words: [{ id: "word_1", segmentId: "seg_1", startSec: 4, endSec: 4.2, text: "um" }], + }); + const read = JSON.parse(run(before, "getTranscriptWords", { assetId: "asset_2" }).resultJson); + expect(read.words[0].id).toBe("word_1"); + const result = run(before, "removeWords", { + assetId: "asset_2", + wordIds: [read.words[0].id], + }); + expect(result.ok).toBe(true); + expect(result.document?.timeline.trimRanges.at(-1)).toMatchObject({ + assetId: "asset_2", + clipId: "clip_3", + }); + expect( + result.document?.timeline.trimRanges.filter((trim) => trim.assetId === "asset_1"), + ).toEqual(before.timeline.trimRanges); + expect(result.document?.transcripts).toEqual(before.transcripts); + const ambiguous = run(before, "removeWords", { wordIds: ["word_1"] }); + expect(ambiguous.ok).toBe(false); + expect(ambiguous.document).toBeUndefined(); + const unknownAsset = run(before, "removeWords", { + assetId: "missing_asset", + wordIds: ["word_1"], + }); + expect(unknownAsset.ok).toBe(false); + expect(unknownAsset.document).toBeUndefined(); + }); + + it("refuses invalid timestamps without guessing a span", () => { + const before = repeatedWordDocument(); + before.transcripts[0].words[0].endSec = Number.NaN; + const result = run(before, "removeWords", { wordIds: ["filler"] }); + expect(result.ok).toBe(false); + expect(result.document).toBeUndefined(); + expect(result.resultJson).toContain("invalid source timestamps"); + }); + + it("refuses a word covered by two clips", () => { + const before = repeatedWordDocument(); + before.timeline.clips.push({ + ...before.timeline.clips[0], + id: "duplicate_clip", + }); + const result = run(before, "removeWords", { wordIds: ["filler"] }); + expect(result.ok).toBe(false); + expect(result.document).toBeUndefined(); + expect(result.resultJson).toContain("exactly one clip; found 2"); + }); + + it("honours project edit consent", () => { + const result = executeAgentTool( + repeatedWordDocument(), + "removeWords", + JSON.stringify({ wordIds: ["filler"] }), + { editsAllowed: false }, + ); + expect(result.document).toBeUndefined(); + expect(JSON.parse(result.resultJson).code).toBe("consent_required"); + }); +}); + describe("setWordText", () => { it("changes the text and leaves the timeline alone", () => { const before = documentWithWords(); @@ -2505,3 +2919,203 @@ describe("addTrim shapes the cut like the transcript pane", () => { expect(lastTrim(result.document)?.endSec).toBe(15); }); }); + +describe("zoom transitions are reported, not hidden (#1028)", () => { + // 1.8× (depth 3): each move lasts 0.6 + 0.55·ln 1.8 ≈ 0.923 s of screen time. + const MOVE_SEC = 0.923; + + function run(document: AxcutDocument, name: string, args: unknown) { + const result = executeAgentTool(document, name, JSON.stringify(args)); + expect(result.ok).toBe(true); + return { document: result.document ?? document, payload: JSON.parse(result.resultJson) }; + } + + function zoomedAt10to12() { + return run(shortSingleClip(), "addZoom", { startSec: 10, endSec: 12, depth: 3 }); + } + + function snapshotZoom(document: AxcutDocument) { + return run(document, "getCurrentDocument", {}).payload.zoomRanges[0]; + } + + it("gives each zoom its move windows on the timeline", () => { + const added = zoomedAt10to12(); + expect(added.payload.cutTransitions).toBeUndefined(); + const zoom = snapshotZoom(added.document); + expect(zoom.transitionSec).toBe(MOVE_SEC); + expect(zoom.zoomInFromSec).toBe(10 - MOVE_SEC); + expect(zoom.zoomOutUntilSec).toBe(12 + MOVE_SEC); + expect(zoom.cutByTrim).toBeUndefined(); + }); + + it("warns when a trim at the zoom's end cuts its whole zoom-out (the issue's repro)", () => { + const trimmed = run(zoomedAt10to12().document, "addTrim", { startSec: 12, endSec: 14 }); + const zoomId = snapshotZoom(trimmed.document).id; + expect(trimmed.payload.cutTransitions).toEqual([ + { zoomId, side: "out", cutSec: MOVE_SEC, by: "trim" }, + ]); + expect(snapshotZoom(trimmed.document).cutByTrim).toBe("out"); + }); + + it("flags a zoom on the second clip of an asset that a trim without clipId cuts", () => { + // Two clips over one recording, and a pre-v7 trim (no clipId): playback cuts it from + // both clips, while the ruler maps it through the first one only. + const base = shortSingleClip(); + const clip = base.timeline.clips[0]; + const length = (clip.sourceEndSec ?? 0) - clip.sourceStartSec; + const twoClips: AxcutDocument = { + ...base, + timeline: { + ...base.timeline, + clips: [ + clip, + { + ...clip, + id: "clip_2", + timelineStartSec: length, + timelineEndSec: 2 * length, + }, + ], + trimRanges: [ + { + id: "trim_legacy", + assetId: clip.assetId, + startSec: 12, + endSec: 14, + reason: "", + origin: "user", + }, + ], + }, + }; + const added = run(twoClips, "addZoom", { + startSec: length + 10, + endSec: length + 12, + depth: 3, + }); + expect(added.payload.cutTransitions).toEqual([ + { zoomId: added.payload.zoomId, side: "out", cutSec: MOVE_SEC, by: "trim" }, + ]); + }); + + it("stays quiet about a trim clear of both windows", () => { + const trimmed = run(zoomedAt10to12().document, "addTrim", { startSec: 14, endSec: 15 }); + expect(trimmed.payload.cutTransitions).toBeUndefined(); + }); + + it("warns the zoom write too, when it lands next to an existing cut", () => { + const trimmed = run(shortSingleClip(), "addTrim", { startSec: 8, endSec: 9.5 }); + const added = run(trimmed.document, "addZoom", { startSec: 10, endSec: 12, depth: 3 }); + expect(added.payload.cutTransitions).toEqual([ + { zoomId: added.payload.zoomId, side: "in", cutSec: 0.423, by: "trim" }, + ]); + expect(snapshotZoom(added.document).cutByTrim).toBe("in"); + }); + + it("repeats the warning when setZoom moves a zoom that is still cut", () => { + const trimmed = run(zoomedAt10to12().document, "addTrim", { startSec: 12, endSec: 14 }); + const zoomId = snapshotZoom(trimmed.document).id; + const moved = run(trimmed.document, "setZoom", { zoomId, startSec: 9 }); + expect(moved.payload.cutTransitions).toEqual([ + { zoomId: moved.payload.zoomId, side: "out", cutSec: MOVE_SEC, by: "trim" }, + ]); + }); + + it("reports two zooms closer than 1.5 s as one pan, not a zoom-out and back in", () => { + const chained = run(zoomedAt10to12().document, "addZoom", { + startSec: 12.5, + endSec: 14, + depth: 3, + }); + const [first, second] = run(chained.document, "getCurrentDocument", {}).payload.zoomRanges; + expect(first).toMatchObject({ panToZoomId: second.id, panUntilSec: 13 }); + expect(first.zoomOutUntilSec).toBeUndefined(); + expect(first.zoomInFromSec).toBe(10 - MOVE_SEC); + expect(second.panFromZoomId).toBe(first.id); + expect(second.zoomInFromSec).toBeUndefined(); + expect(second.zoomOutUntilSec).toBe(14 + MOVE_SEC); + }); + + it("warns of both moves when a trim between two chained zooms breaks the chain", () => { + const chained = run(zoomedAt10to12().document, "addZoom", { + startSec: 12.5, + endSec: 14, + depth: 3, + }); + const trimmed = run(chained.document, "addTrim", { startSec: 12, endSec: 12.5 }); + const [first, second] = run(trimmed.document, "getCurrentDocument", {}).payload.zoomRanges; + expect(first.panToZoomId).toBeUndefined(); + expect(trimmed.payload.cutTransitions).toEqual([ + { zoomId: first.id, side: "out", cutSec: MOVE_SEC, by: "trim" }, + { zoomId: second.id, side: "in", cutSec: MOVE_SEC, by: "trim" }, + ]); + }); + + it("does not chain zooms across a clip junction", () => { + const base = shortSingleClip(); + const clip = base.timeline.clips[0]; + const twoClips: AxcutDocument = { + ...base, + timeline: { + ...base.timeline, + clips: [ + { ...clip, sourceEndSec: 12, timelineEndSec: 12 }, + { ...clip, id: "clip_2", sourceStartSec: 12, timelineStartSec: 12 }, + ], + }, + }; + const first = run(twoClips, "addZoom", { startSec: 10, endSec: 12, depth: 3 }); + const both = run(first.document, "addZoom", { startSec: 12.5, endSec: 14, depth: 3 }); + const zooms = run(both.document, "getCurrentDocument", {}).payload.zoomRanges; + expect(zooms[0].panToZoomId).toBeUndefined(); + expect(zooms[0].zoomOutUntilSec).toBe(12 + MOVE_SEC); + + // Each clip renders on its own: the junction cuts the first zoom's whole zoom-out and + // the start of the second one's zoom-in, and says it is a junction, not a trim. + expect(first.payload.cutTransitions).toEqual([ + { zoomId: first.payload.zoomId, side: "out", cutSec: MOVE_SEC, by: "junction" }, + ]); + expect(both.payload.cutTransitions).toEqual([ + { zoomId: both.payload.zoomId, side: "in", cutSec: 0.423, by: "junction" }, + ]); + expect(zooms[0]).toMatchObject({ cutByJunction: "out" }); + expect(zooms[1]).toMatchObject({ cutByJunction: "in" }); + expect(zooms[0].cutByTrim).toBeUndefined(); + expect(zooms[1].cutByTrim).toBeUndefined(); + }); + + it("names a trim and a junction apart on the same zoom", () => { + const base = shortSingleClip(); + const clip = base.timeline.clips[0]; + const twoClips: AxcutDocument = { + ...base, + timeline: { + ...base.timeline, + clips: [ + { ...clip, sourceEndSec: 12.5, timelineEndSec: 12.5 }, + { ...clip, id: "clip_2", sourceStartSec: 12.5, timelineStartSec: 12.5 }, + ], + }, + }; + const zoomed = run(twoClips, "addZoom", { startSec: 10, endSec: 12, depth: 3 }); + const trimmed = run(zoomed.document, "addTrim", { startSec: 9, endSec: 9.5 }); + expect(snapshotZoom(trimmed.document)).toMatchObject({ cutByTrim: "in", cutByJunction: "out" }); + }); + + it("does not call the end of the timeline a junction", () => { + const base = shortSingleClip(); + const end = base.timeline.clips[0].timelineEndSec; + const added = run(base, "addZoom", { startSec: end - 2, endSec: end - 0.5, depth: 3 }); + expect(added.payload.cutTransitions).toBeUndefined(); + expect(snapshotZoom(added.document).cutByJunction).toBeUndefined(); + }); + + it("stretches the zoom-out across a speed region, and catches a trim placed there", () => { + const sped = run(zoomedAt10to12().document, "addSpeed", { startSec: 12, endSec: 20, speed: 3 }); + expect(snapshotZoom(sped.document).zoomOutUntilSec).toBe(14.77); + const trimmed = run(sped.document, "addTrims", { ranges: [{ startSec: 14, endSec: 15 }] }); + expect(trimmed.payload.applied[0].cutTransitions).toEqual([ + { zoomId: snapshotZoom(trimmed.document).id, side: "out", cutSec: 0.77, by: "trim" }, + ]); + }); +}); diff --git a/electron/ai-edition/agent-tools.ts b/electron/ai-edition/agent-tools.ts index 3ffc9b2153..cecef747a1 100644 --- a/electron/ai-edition/agent-tools.ts +++ b/electron/ai-edition/agent-tools.ts @@ -17,6 +17,7 @@ import { z } from "zod"; import { DEFAULT_TEXT_PLATE } from "../../src/lib/ai-edition/annotations/background"; +import { fitTextBox } from "../../src/lib/ai-edition/annotations/placement"; import { collapseTracksToPills, patchAudioTrack, @@ -24,6 +25,7 @@ import { trackGroupId, } from "../../src/lib/ai-edition/document/audioTracks"; import { createId } from "../../src/lib/ai-edition/document/ids"; +import { resolveAspectRatioValue } from "../../src/lib/ai-edition/document/outputFormat"; import { moveClip, planTimelineReplacement, @@ -34,7 +36,9 @@ import { setClipSourceRange, } from "../../src/lib/ai-edition/document/timeline"; import { setDocumentWordText } from "../../src/lib/ai-edition/document/transcript"; -import type { AxcutDocument } from "../../src/lib/ai-edition/schema"; +import type { AxcutAnnotationRegion, AxcutDocument } from "../../src/lib/ai-edition/schema"; +import { getEditorSettings } from "../../src/lib/ai-edition/store/editorSettings"; +import { applySettingsPatch } from "../../src/lib/ai-edition/store/settingsPatch"; import { hasAnyClipWithCamera } from "../../src/lib/ai-edition/timeline/camera"; import { isGeneratedAssetId } from "../../src/lib/ai-edition/timeline/clip-parts"; import { @@ -48,16 +52,21 @@ import { replacePillSpan, resolvePillIds, } from "../../src/lib/ai-edition/timeline/timelineMap"; -import { trimAppliesToClip } from "../../src/lib/ai-edition/timeline/trim-mapping"; -// ponytail: relative, and it has to stay that way — `electron/` never resolves -// the `@/` alias (the main-process build does not declare it), which is why the -// scale table was moved out of `components/video-editor/types.ts` to be -// reachable from here at all. +import { + trimAppliesToClip, + trimToTimelineSpan, +} from "../../src/lib/ai-edition/timeline/trim-mapping"; import { effectiveZoomScale, ZOOM_DEPTH_LEGEND, } from "../../src/lib/ai-edition/timeline/zoom-scale"; +import { + chainedPans, + transitionCutsMs, + zoomTransitions, +} from "../../src/lib/ai-edition/timeline/zoom-transitions"; import { SETTING_BOUNDS } from "../../src/lib/projectDefaults"; +import { ASPECT_RATIO_PRESETS } from "../../src/utils/aspectRatioUtils"; export interface AgentToolExecution { ok: boolean; @@ -121,6 +130,93 @@ function resolveSpanMs( return { startMs: toMs(Math.min(s, e)), endMs: toMs(Math.max(s, e)) }; } +/** + * For the effect set* tools: the region another span of the same kind would have to cross. + * + * `replacePillSpan` follows the editor's drag rule: a region dragged into a neighbour with other + * settings stops against it, part of the way back to where it started. That is right under a + * mouse, which shows the stop as it happens. For a caller who named an exact span it put the + * region somewhere it never asked for (asked 7.6 – 9.5 s, landed 1.5 – 3.3 s), so a set that + * would be stopped is refused instead, naming what is in the way. The neighbours are the same + * walls the drag rule sees: other pills with other settings, not ones the region already overlaps. + */ +function refuseIfBlocked( + kind: string, + regions: T[], + id: string, + span: { startMs: number; endMs: number }, +): AgentToolExecution | null { + const pills = coalesceRegionsForRuler(regions); + const pill = pills.find((p) => p.ids.includes(id)); + if (!pill) return null; + const start = span.startMs / 1000; + const end = span.endMs / 1000; + const wall = pills.find( + (other) => + other !== pill && + other.identity !== pill.identity && + (other.end <= pill.start || other.start >= pill.end) && + other.start < end && + other.end > start, + ); + if (!wall) return null; + return failure( + `The ${kind} was NOT changed: ${formatSec(start)} – ${formatSec(end)} would run into ${kind} ` + + `${wall.ids[0]} at ${formatSec(wall.start)} – ${formatSec(wall.end)}, and two ${kind}s with ` + + "different settings cannot overlap. Move or shorten that one first, or pick a span clear of it.", + ); +} + +type AnnotationLook = { + fontSize?: number; + color?: string; + background?: string; + bold?: boolean; + animation?: NonNullable; +}; + +/** The look arguments, written onto a style the way the editor's inspector writes them. */ +function applyAnnotationLook( + style: AxcutAnnotationRegion["style"], + look: AnnotationLook, +): AxcutAnnotationRegion["style"] { + const next = { ...style }; + if (look.fontSize !== undefined) next.fontSize = look.fontSize; + if (look.color !== undefined) next.color = look.color; + if (look.bold !== undefined) next.fontWeight = look.bold ? "bold" : "normal"; + if (look.animation !== undefined) next.textAnimation = look.animation; + if (look.background !== undefined) { + if (look.background === "none") { + // As the inspector's switch does: keep the colour, so turning it back on restores it. + if (style.backgroundColor !== "transparent") next.lastBackgroundColor = style.backgroundColor; + next.backgroundColor = "transparent"; + } else { + next.backgroundColor = look.background; + } + } + return next; +} + +/** + * A text annotation on the output frame, its box sized to its text around `center` (frame + * percentages): what the editor's "add text" does, so the agent's text sits where the editor's + * would, is measured the same, and stays put when the picture zooms. + */ +function placeTextOnFrame( + region: T, + center: { x: number; y: number }, + document: AxcutDocument, +): T { + const frameAspect = resolveAspectRatioValue(document, getEditorSettings(document).aspectRatio); + const centred = { + ...region, + space: "frame" as const, + position: center, + size: { width: 0, height: 0 }, + }; + return { ...centred, ...fitTextBox(centred, frameAspect) }; +} + // v5: a modifier is stored as clip-anchored fragment(s), and adjacent regions with the // same properties read as ONE pill (timelineMap merge rule). The agent reasons in VIRTUAL // seconds over whole regions, so we present exactly the pills the user sees, keyed by the @@ -417,6 +513,11 @@ export const addTrimsArgs = z.object({ ranges: z.array(z.union([addTrimArgs, z.unknown()])).min(1), }); +export const removeWordsArgs = z.object({ + assetId: z.string().min(1).optional(), + wordIds: z.array(z.string().min(1)).min(1), +}); + export const setTrimArgs = z.object({ trimRangeId: z.string().min(1), startSec: secondsSchema, @@ -429,6 +530,10 @@ export const setClipRangeArgs = z.object({ sourceEndSec: secondsSchema, }); +export const setAspectRatioArgs = z.object({ + aspectRatio: z.enum(["auto", ...ASPECT_RATIO_PRESETS]), +}); + export const replaceTimelineArgs = z.object({ intervals: z.array(z.object({ startSec: secondsSchema, endSec: secondsSchema })).min(1), reason: z.string().default(""), @@ -472,13 +577,28 @@ export const getCursorTrackArgs = z.object({ const depthSchema = z.number().int().min(1).max(6); const focusSchema = z.object({ cx: z.number().min(0).max(1), cy: z.number().min(0).max(1) }); +/** The zoom's 3D camera, as the editor's "3D camera" control offers it: the screen turned left or + * right, the camera orbiting the screen, or `none` for a flat screen. Stored and read back + * (getCurrentDocument) as `rotationPreset`, absent when flat. */ +const rotationPresetSchema = z.enum(["none", "left", "right", "orbit"]); + export const addZoomArgs = z.object({ startSec: secondsSchema, endSec: secondsSchema, depth: depthSchema.default(3), focus: focusSchema.default({ cx: 0.5, cy: 0.5 }), + rotationPreset: rotationPresetSchema.default("none"), }); +/** A zoom with its 3D camera set: `none` drops the key, as a flat zoom never carries it. */ +function withRotationPreset( + zoom: Z, + preset: z.infer, +): Z { + const { rotationPreset: _previous, ...flat } = zoom; + return (preset === "none" ? flat : { ...flat, rotationPreset: preset }) as Z; +} + /** Same contract as `addTrimsArgs`: the element schema IS the unitary one, and * it is advertised rather than enforced so a bad region is refused by itself. */ export const addZoomsArgs = z.object({ @@ -491,6 +611,7 @@ export const setZoomArgs = z.object({ endSec: secondsSchema.optional(), depth: depthSchema.optional(), focus: focusSchema.optional(), + rotationPreset: rotationPresetSchema.optional(), }); // The bound every speed reader applies (`readSpeedRegions`): refused here, so the agent hears @@ -513,8 +634,25 @@ export const setSpeedArgs = z.object({ speed: speedSchema.optional(), }); -/** A new text annotation's box, in frame percentages: the editor's own size for one. */ -const ANNOTATION_BOX = { width: 30, height: 20 } as const; +/** How a text annotation looks: the settings the editor's inspector offers for one. */ +const annotationLookArgs = { + fontSize: z + .number() + .min(8) + .max(200) + .optional() + .describe("Text size; 32 is the editor's default, a title reads well at 56 to 80."), + color: z.string().optional().describe("Text colour, a CSS colour such as #ffffff."), + background: z + .string() + .optional() + .describe('Colour of the plate behind the text, or "none" for no plate.'), + bold: z.boolean().optional(), + animation: z + .enum(["none", "fade", "rise", "pop", "slide-left", "typewriter", "pulse"]) + .optional() + .describe("How the text enters: typewriter types it out, pop springs it in, and so on."), +}; export const addAnnotationArgs = z.object({ startSec: secondsSchema, @@ -522,6 +660,7 @@ export const addAnnotationArgs = z.object({ text: z.string().default(""), x: z.number().min(0).max(100).default(50), y: z.number().min(0).max(100).default(50), + ...annotationLookArgs, }); export const setAnnotationArgs = z.object({ @@ -529,6 +668,9 @@ export const setAnnotationArgs = z.object({ startSec: secondsSchema.optional(), endSec: secondsSchema.optional(), text: z.string().optional(), + x: z.number().min(0).max(100).optional(), + y: z.number().min(0).max(100).optional(), + ...annotationLookArgs, }); export const addAudioArgs = z.object({ @@ -612,6 +754,7 @@ export const OPENSCREEN_TOOL_NAMES = [ "getTranscriptWords", "getCursorTrack", "setWordText", + "removeWords", "addTrim", "addTrims", "setTrim", @@ -632,6 +775,7 @@ export const OPENSCREEN_TOOL_NAMES = [ "removeTrim", "removeModifier", "removeClip", + "setAspectRatio", ] as const; /** @@ -683,6 +827,7 @@ export const MUTATING_TOOL_NAMES: ReadonlySet = new Set([ // Writes the transcript, not the timeline — but it writes the document, so it is a // consented edit like any other. "setWordText", + "removeWords", "addTrim", "addTrims", "addZooms", @@ -703,6 +848,7 @@ export const MUTATING_TOOL_NAMES: ReadonlySet = new Set([ "removeTrim", "removeModifier", "removeClip", + "setAspectRatio", ]); export function isMutatingTool(name: string): boolean { @@ -713,6 +859,84 @@ function roundSec(ms: number): number { return Math.round(ms) / 1000; } +// Each zoom pill's camera moves in virtual ms, and how much of each a cut takes off (#1028). +// The pill shows the hold only: the move in runs before it, the move out after it, and a +// trim or a clip junction over either makes the export jump at the cut. +function zoomTransitionsForAgent(document: AxcutDocument) { + const legacy = document.legacyEditor as Record | null; + const speedRegions = + (legacy?.speedRegions as + | Array<{ id: string; startMs: number; endMs: number; speed: number }> + | undefined) ?? []; + // A trim with no clipId cuts every clip of its asset in playback, while the ruler maps it + // through one: measured per clip, so a zoom on any of them learns of the cut. + const trims = document.timeline.trimRanges.flatMap((trim) => { + const clips = + trim.clipId !== undefined + ? [trim] + : document.timeline.clips + .filter((clip) => clip.assetId === trim.assetId) + .map((clip) => ({ ...trim, clipId: clip.id })); + return clips.flatMap((anchored) => { + const span = trimToTimelineSpan(anchored, document.timeline.clips); + return span ? [{ startMs: span.start * 1000, endMs: span.end * 1000 }] : []; + }); + }); + // Chained zooms pan from one to the next instead: no zoom-out, no zoom-in, nothing to cut + // (a trim or a junction between them would have broken the chain). A junction is where a + // clip starts after another: the timeline's own end has nothing after it to jump to. + const zooms = coalesceForAgent(document.zoomRanges); + const junctions = document.timeline.clips + .filter((clip) => clip.timelineStartSec > 0) + .map((clip) => ({ + startMs: clip.timelineStartSec * 1000, + endMs: clip.timelineStartSec * 1000, + })); + const pans = chainedPans(zooms, speedRegions, [...trims, ...junctions]); + return zooms.map((zoom, i) => { + const transitions = zoomTransitions({ ...zoom, scale: effectiveZoomScale(zoom) }, speedRegions); + const panOut = pans.find((p) => p.from === i); + const panIn = pans.find((p) => p.to === i); + const cut = transitionCutsMs(zoom, transitions, trims, junctions); + return { + zoom, + transitions, + cut: { ...cut, inMs: panIn ? 0 : cut.inMs, outMs: panOut ? 0 : cut.outMs }, + panTo: panOut && { zoomId: zooms[panOut.to].id, untilMs: panOut.untilMs }, + panFromZoomId: panIn && zooms[panIn.from].id, + }; + }); +} + +/** The transitions this edit cut that were not cut, or not this way, before it: the warning a + * zoom or trim write carries, like `cursorAnchor`: reported, never refused. */ +function cutTransitionsReport(before: AxcutDocument, after: AxcutDocument) { + const cutsOf = (document: AxcutDocument) => + zoomTransitionsForAgent(document).flatMap(({ zoom, cut }) => + (["in", "out"] as const) + .map((side) => ({ + zoomId: zoom.id, + side, + cutSec: roundSec(side === "in" ? cut.inMs : cut.outMs), + by: (side === "in" ? cut.inByJunction : cut.outByJunction) ? "junction" : "trim", + key: `${zoom.id}:${side}:${zoom.startMs}:${zoom.endMs}`, + })) + .filter((c) => c.cutSec > 0), + ); + const known = new Set(cutsOf(before).map((c) => `${c.key}:${c.cutSec}`)); + const cuts = cutsOf(after) + .filter((c) => !known.has(`${c.key}:${c.cutSec}`)) + .map(({ key: _key, ...c }) => c); + return cuts.length ? { cutTransitions: cuts } : {}; +} + +/** The moves a trim (or, with `byJunction`, a clip junction) cuts: in, out, both or none. */ +function cutSides(cut: ReturnType, byJunction: boolean) { + const cutIn = cut.inMs > 0 && cut.inByJunction === byJunction; + const cutOut = cut.outMs > 0 && cut.outByJunction === byJunction; + return cutIn ? (cutOut ? "both" : "in") : cutOut ? "out" : undefined; +} + // Compact projection of the document for the model: everything it needs to // reference ids and times, nothing it doesn't (no waveform paths, no history). // @@ -758,13 +982,19 @@ export function documentSnapshotForModel( const autoFocusAll = legacy?.autoFocusAll === true; return { timeBaseNote: - "clips and trims are in source-time seconds; zooms, speedRegions, annotations, cameraFullscreenRegions and audioTracks are in virtual (edited-timeline) seconds.", + "clips and trims are in source-time seconds; zooms, speedRegions, annotations, cameraFullscreenRegions and audioTracks are in virtual (edited-timeline) seconds: the editor's ruler, where a trim leaves a hole instead of shortening it and a speed region stretches nothing. The exported video's clock skips those holes and plays sped-up spans faster, so it differs from the ruler once there are any.", audioNote: "audioTracks are imported voiceover / music files laid over the recording. They are clip-anchored like every other region, so they travel with their clip through reorder and trim, and they play at 1x whatever a speed region does to the picture under them. addAudio places an EXISTING asset of kind 'audio'; nothing here can import a file from disk or record one, so if the project has no audio asset, say so rather than inventing an id.", zoomNote: `renderedScale is what the viewer sees (depth is an ordinal, not a factor: ${ZOOM_DEPTH_LEGEND}). ` + "When a zoom carries customScale it wins over depth and depthIsOverridden is true — " + - "a setZoom that only changes depth on such a zoom clears customScale so the depth takes effect.", + "a setZoom that only changes depth on such a zoom clears customScale so the depth takes effect. " + + "startSec–endSec is where the zoom HOLDS: it animates in over zoomInFromSec–startSec and out over " + + "endSec–zoomOutUntilSec (transitionSec of screen time each; a speed region scales that on the " + + "timeline, longer above 1x and shorter below, so read the bounds rather than adding transitionSec). cutByTrim names the move a trim cuts into (in, out or both), cutByJunction the move a clip junction cuts (the clip ends before the move does): the export jumps at that cut. " + + "Two zooms at most 1.5 s apart on screen, with no trim or clip junction between them, are CHAINED: " + + "no zoom-out and back in, the camera pans from the first (panToZoomId, panUntilSec: 1 s of screen " + + "time from its endSec, which can run into the next span) to the second (panFromZoomId).", project: { id: document.project.id, title: document.project.title }, primaryAssetId: document.project.primaryAssetId ?? document.assets[0]?.id ?? null, autoFocusAll, @@ -821,21 +1051,30 @@ export function documentSnapshotForModel( endSec: s.endSec, reason: s.reason, })), - zoomRanges: coalesceForAgent(document.zoomRanges).map((z) => ({ - id: z.id, - startSec: roundSec(z.startMs), - endSec: roundSec(z.endMs), - depth: z.depth, - renderedScale: effectiveZoomScale(z), - // Emitted only when set: an unconditional `customScale: null` on every - // zoom of every snapshot is noise the reader learns to skip, which is - // how the field would go unnoticed again. - ...(z.customScale != null ? { customScale: z.customScale, depthIsOverridden: true } : {}), - ...(z.rotationPreset ? { rotationPreset: z.rotationPreset } : {}), - focus: z.focus, - focusMode: autoFocusAll ? "auto" : (z.focusMode ?? "manual"), - source: z.source ?? "manual", - })), + zoomRanges: zoomTransitionsForAgent(document).map( + ({ zoom: z, transitions, cut, panTo, panFromZoomId }) => ({ + id: z.id, + startSec: roundSec(z.startMs), + endSec: roundSec(z.endMs), + ...(panFromZoomId ? { panFromZoomId } : { zoomInFromSec: roundSec(transitions.inFromMs) }), + ...(panTo + ? { panToZoomId: panTo.zoomId, panUntilSec: roundSec(panTo.untilMs) } + : { zoomOutUntilSec: roundSec(transitions.outUntilMs) }), + transitionSec: roundSec(transitions.durationMs), + ...(cutSides(cut, false) ? { cutByTrim: cutSides(cut, false) } : {}), + ...(cutSides(cut, true) ? { cutByJunction: cutSides(cut, true) } : {}), + depth: z.depth, + renderedScale: effectiveZoomScale(z), + // Emitted only when set: an unconditional `customScale: null` on every + // zoom of every snapshot is noise the reader learns to skip, which is + // how the field would go unnoticed again. + ...(z.customScale != null ? { customScale: z.customScale, depthIsOverridden: true } : {}), + ...(z.rotationPreset ? { rotationPreset: z.rotationPreset } : {}), + focus: z.focus, + focusMode: autoFocusAll ? "auto" : (z.focusMode ?? "manual"), + source: z.source ?? "manual", + }), + ), speedRegions: coalesceForAgent(speedRegions).map((s) => ({ id: s.id, startSec: roundSec(s.startMs), @@ -1218,6 +1457,20 @@ export function executeAgentTool( } switch (name) { + case "setAspectRatio": { + const parsed = setAspectRatioArgs.safeParse(args); + if (!parsed.success) return failure(parsed.error.message); + const previous = getEditorSettings(document).aspectRatio; + // The editor's own write: the frame-placed annotations keep their shape and centre. + const next = applySettingsPatch(document, { aspectRatio: parsed.data.aspectRatio }); + return { + ok: true, + document: next, + resultJson: JSON.stringify({ aspectRatio: parsed.data.aspectRatio, previous }), + summary: `set the format to ${parsed.data.aspectRatio}`, + }; + } + case "getCurrentDocument": { return { ok: true, @@ -1419,6 +1672,185 @@ export function executeAgentTool( }; } + case "removeWords": { + const parsed = removeWordsArgs.safeParse(args); + if (!parsed.success) return failure(parsed.error.message); + const { assetId: selectedAssetId, wordIds } = parsed.data; + if (new Set(wordIds).size !== wordIds.length) { + return failure("Duplicate word IDs in one request. Nothing was modified."); + } + + // Resolve every ID before writing anything. Word IDs, rather than guessed + // timestamps, designate which occurrence of a repeated word is speech to cut. + const transcripts = [...document.transcripts]; + if ( + document.transcript && + !transcripts.some((transcript) => transcript.assetId === document.transcript?.assetId) + ) { + transcripts.push(document.transcript); + } + const selected: Array<{ + wordId: string; + text: string; + assetId: string; + clipId: string; + index: number; + startSec: number; + endSec: number; + }> = []; + for (const wordId of wordIds) { + const matches = transcripts + .filter( + (transcript) => selectedAssetId === undefined || transcript.assetId === selectedAssetId, + ) + .flatMap((transcript) => + transcript.words.flatMap((word, index) => + word.id === wordId ? [{ assetId: transcript.assetId, word, index }] : [], + ), + ); + if (matches.length !== 1) { + return failure( + matches.length === 0 + ? `Unknown transcript word ID: ${wordId}. Call getTranscriptWords to read the IDs. Nothing was modified.` + : `Word ID ${wordId} appears more than once in the transcripts. Pass the assetId used for getTranscriptWords. Nothing was modified.`, + ); + } + const { assetId, word, index } = matches[0]; + const asset = document.assets.find((candidate) => candidate.id === assetId); + if (!asset || isGeneratedAssetId(assetId) || word.source === "synth") { + return failure(`Word ${wordId} has no recorded source asset. Nothing was modified.`); + } + const { startSec, endSec } = word; + if ( + !Number.isFinite(startSec) || + !Number.isFinite(endSec) || + startSec < 0 || + endSec <= startSec || + (asset.durationSec !== undefined && + (!Number.isFinite(asset.durationSec) || endSec > asset.durationSec)) + ) { + return failure(`Word ${wordId} has invalid source timestamps. Nothing was modified.`); + } + const clips = document.timeline.clips.filter((clip) => { + const clipEnd = clip.sourceEndSec ?? asset.durationSec; + return ( + clip.assetId === assetId && + clipEnd !== undefined && + Number.isFinite(clipEnd) && + startSec >= clip.sourceStartSec && + endSec <= clipEnd + ); + }); + if (clips.length !== 1) { + return failure( + `Word ${wordId} must be fully contained in exactly one clip; found ${clips.length}. Nothing was modified.`, + ); + } + const clip = clips[0]; + if ( + document.timeline.trimRanges.some( + (trim) => + trimAppliesToClip(trim, clip) && startSec < trim.endSec && endSec > trim.startSec, + ) + ) { + return failure(`Word ${wordId} is already cut by a trim. Nothing was modified.`); + } + selected.push({ + wordId, + text: word.text, + assetId, + clipId: clip.id, + index, + startSec, + endSec, + }); + } + + // Words that follow one another in the transcript go out as ONE cut: a phrase or a + // flubbed take cut word by word would leave the gaps between its words playing. + selected.sort((a, b) => a.assetId.localeCompare(b.assetId) || a.index - b.index); + const runs: Array = []; + for (const word of selected) { + const last = runs.at(-1)?.at(-1); + if ( + last?.assetId === word.assetId && + last.clipId === word.clipId && + last.index + 1 === word.index + ) { + runs.at(-1)?.push(word); + } else { + runs.push([word]); + } + } + + let current = document; + const removed: Array> = []; + for (const run of runs) { + const first = run[0]; + const ids = run.map((word) => word.wordId).join(", "); + const execution = executeAgentTool( + current, + "addTrim", + JSON.stringify({ + assetId: first.assetId, + clipId: first.clipId, + startSec: first.startSec, + endSec: run[run.length - 1].endSec, + reason: "removed words", + }), + options, + ); + if (!execution.ok || !execution.document) { + return failure(`Could not cut ${ids}: ${execution.resultJson}. Nothing was modified.`); + } + const { trimRangeId } = JSON.parse(execution.resultJson) as { trimRangeId: string }; + const trim = execution.document.timeline.trimRanges.find( + (range) => range.id === trimRangeId, + ); + const clip = current.timeline.clips.find((candidate) => candidate.id === first.clipId); + const asset = current.assets.find((candidate) => candidate.id === first.assetId); + const clipEnd = clip?.sourceEndSec ?? asset?.durationSec; + if ( + !trim || + !clip || + clipEnd === undefined || + trim.clipId !== first.clipId || + !Number.isFinite(trim.startSec) || + !Number.isFinite(trim.endSec) || + trim.startSec < clip.sourceStartSec || + trim.endSec > clipEnd || + trim.endSec <= trim.startSec + ) { + return failure(`Cut for ${ids} left its clip. Nothing was modified.`); + } + current = execution.document; + for (const word of run) { + removed.push({ + wordId: word.wordId, + text: word.text, + assetId: word.assetId, + clipId: word.clipId, + wordStartSec: word.startSec, + wordEndSec: word.endSec, + trimRangeId, + startSec: trim.startSec, + endSec: trim.endSec, + }); + } + } + return { + ok: true, + document: current, + resultJson: JSON.stringify({ + requested: wordIds.length, + removedCount: removed.length, + removed, + ...cutTransitionsReport(document, current), + }), + summary: `removed ${removed.length} word${removed.length === 1 ? "" : "s"}`, + }; + } + case "addTrim": { const parsed = addTrimArgs.safeParse(args); if (!parsed.success) return failure(parsed.error.message); @@ -1501,7 +1933,12 @@ export function executeAgentTool( return { ok: true, document: next, - resultJson: JSON.stringify({ trimRangeId: trim.id, startSec, endSec }), + resultJson: JSON.stringify({ + trimRangeId: trim.id, + startSec, + endSec, + ...cutTransitionsReport(document, next), + }), summary: `added trim ${formatSec(startSec)} – ${formatSec(endSec)}`, }; } @@ -1548,7 +1985,12 @@ export function executeAgentTool( return { ok: true, document: next, - resultJson: JSON.stringify({ trimRangeId, startSec, endSec }), + resultJson: JSON.stringify({ + trimRangeId, + startSec, + endSec, + ...cutTransitionsReport(document, next), + }), summary: `moved trim to ${formatSec(startSec)} – ${formatSec(endSec)}`, }; } @@ -1713,15 +2155,18 @@ export function executeAgentTool( if (!parsed.success) return failure(parsed.error.message); const startMs = toMs(Math.min(parsed.data.startSec, parsed.data.endSec)); const endMs = toMs(Math.max(parsed.data.startSec, parsed.data.endSec)); - const zoom = { - id: createId("zoom"), - startMs, - endMs, - depth: parsed.data.depth as 1 | 2 | 3 | 4 | 5 | 6, - focus: parsed.data.focus, - focusMode: "manual" as const, - source: "manual" as const, - }; + const zoom = withRotationPreset( + { + id: createId("zoom"), + startMs, + endMs, + depth: parsed.data.depth as 1 | 2 | 3 | 4 | 5 | 6, + focus: parsed.data.focus, + focusMode: "manual" as const, + source: "manual" as const, + } as AxcutDocument["zoomRanges"][number], + parsed.data.rotationPreset, + ); const placed = anchorForAgent(zoom, document, "zoom"); const landing = landingOf(placed, document); if (!landing.anchored) return coversNoClip("zoom", startMs / 1000, endMs / 1000, document); @@ -1739,11 +2184,13 @@ export function executeAgentTool( resultJson: JSON.stringify({ zoomId: landing.ids[0], depth: zoom.depth, + ...(zoom.rotationPreset ? { rotationPreset: zoom.rotationPreset } : {}), // The depth alone is an ordinal; reported on its own it is what the // model turns into "3×" for a frame that renders 1.80×. renderedScale: effectiveZoomScale(zoom), ...landingReport(landing, startMs / 1000, endMs / 1000), ...(anchor ? { cursorAnchor: anchor } : {}), + ...cutTransitionsReport(document, next), }), summary: `added zoom ${formatSec(landing.startSec)} – ${formatSec(landing.endSec)} ` + @@ -1765,6 +2212,8 @@ export function executeAgentTool( const existing = document.zoomRanges.find((z) => z.id === zoomId); if (!existing) return failure(`Unknown zoom: ${zoomId}`); const { startMs, endMs } = resolveSpanMs(existing, parsed.data.startSec, parsed.data.endSec); + const blocked = refuseIfBlocked("zoom", document.zoomRanges, zoomId, { startMs, endMs }); + if (blocked) return blocked; const zoomPill = new Set(resolvePillIds(document.zoomRanges, zoomId)); // ponytail: a depth write CLEARS customScale, and that is the whole // point. `effectiveZoomScale` returns customScale when it is set, so on a @@ -1783,13 +2232,16 @@ export function executeAgentTool( document.zoomRanges.map((z) => { if (!zoomPill.has(z.id)) return z; const { customScale, ...rest } = z; - return { + const edited = { ...(clearsCustomScale ? rest : z), ...(parsed.data.depth !== undefined ? { depth: parsed.data.depth as 1 | 2 | 3 | 4 | 5 | 6 } : {}), ...(parsed.data.focus ? { focus: parsed.data.focus } : {}), }; + return parsed.data.rotationPreset === undefined + ? edited + : withRotationPreset(edited, parsed.data.rotationPreset); }), // …then the span: clamped against different-property pills, then re-ventilated. zoomId, @@ -1823,11 +2275,16 @@ export function executeAgentTool( resultJson: JSON.stringify({ zoomId: landing.ids[0] ?? zoomId, ...(strength - ? { depth: strength.depth, renderedScale: effectiveZoomScale(strength) } + ? { + depth: strength.depth, + renderedScale: effectiveZoomScale(strength), + ...(strength.rotationPreset ? { rotationPreset: strength.rotationPreset } : {}), + } : {}), ...(clearsCustomScale ? { clearedCustomScale: true } : {}), ...landingReport(landing, startMs / 1000, endMs / 1000), ...(anchor ? { cursorAnchor: anchor } : {}), + ...cutTransitionsReport(document, next), }), summary: `updated zoom ${formatSec(landing.startSec)} – ${formatSec(landing.endSec)}` + @@ -1880,6 +2337,11 @@ export function executeAgentTool( const speedPill = new Set(resolvePillIds(prev, parsed.data.speedId)); if (!existing) return failure(`Unknown speed region: ${parsed.data.speedId}`); const { startMs, endMs } = resolveSpanMs(existing, parsed.data.startSec, parsed.data.endSec); + const blocked = refuseIfBlocked("speed region", prev, parsed.data.speedId, { + startMs, + endMs, + }); + if (blocked) return blocked; const speed = parsed.data.speed ?? existing.speed; const rebuiltSpeeds = replacePillSpan( prev.map((s) => (speedPill.has(s.id) ? { ...s, speed } : s)), @@ -1916,39 +2378,37 @@ export function executeAgentTool( if (!parsed.success) return failure(parsed.error.message); const startMs = toMs(Math.min(parsed.data.startSec, parsed.data.endSec)); const endMs = toMs(Math.max(parsed.data.startSec, parsed.data.endSec)); - const ann = { - id: createId("ann"), - startMs, - endMs, - type: "text" as const, - content: parsed.data.text, - textContent: parsed.data.text, - // x/y name the text's centre, which is what "default centre" always promised: the - // position is the box's top-left corner, so {50, 50} dropped the text into the - // bottom-right quarter. Kept whole inside the frame. - position: { - x: Math.min( - 100 - ANNOTATION_BOX.width, - Math.max(0, parsed.data.x - ANNOTATION_BOX.width / 2), - ), - y: Math.min( - 100 - ANNOTATION_BOX.height, - Math.max(0, parsed.data.y - ANNOTATION_BOX.height / 2), + // The editor's new text: its defaults, then whatever look the caller asked for, on the + // output frame with the box fitted to the text around x/y (the text's centre). + const ann = placeTextOnFrame( + { + id: createId("ann"), + startMs, + endMs, + type: "text" as const, + content: parsed.data.text, + textContent: parsed.data.text, + position: { x: 0, y: 0 }, + size: { width: 0, height: 0 }, + style: applyAnnotationLook( + { + color: "#ffffff", + backgroundColor: DEFAULT_TEXT_PLATE, + fontSize: 32, + fontFamily: "Inter", + fontWeight: "bold", + fontStyle: "normal", + textDecoration: "none", + textAlign: "center", + textAnimation: "none", + }, + parsed.data, ), - }, - size: { ...ANNOTATION_BOX }, - style: { - color: "#ffffff", - backgroundColor: DEFAULT_TEXT_PLATE, - fontSize: 32, - fontFamily: "Inter", - fontWeight: "bold" as const, - fontStyle: "normal" as const, - textDecoration: "none" as const, - textAlign: "center" as const, - }, - zIndex: document.annotations.length + 1, - }; + zIndex: document.annotations.length + 1, + } as AxcutAnnotationRegion, + { x: parsed.data.x, y: parsed.data.y }, + document, + ); const placed = anchorForAgent(ann, document, "ann"); const landing = landingOf(placed, document); if (!landing.anchored) { @@ -1979,17 +2439,48 @@ export function executeAgentTool( const annPill = new Set(resolvePillIds(document.annotations, annotationId)); if (!existing) return failure(`Unknown annotation: ${annotationId}`); const { startMs, endMs } = resolveSpanMs(existing, parsed.data.startSec, parsed.data.endSec); + const blocked = refuseIfBlocked("annotation", document.annotations, annotationId, { + startMs, + endMs, + }); + if (blocked) return blocked; + const { x, y, text } = parsed.data; + const edit = (a: AxcutAnnotationRegion): AxcutAnnotationRegion => { + const next: AxcutAnnotationRegion = { + ...a, + ...(text !== undefined ? { content: text, textContent: text } : {}), + style: applyAnnotationLook(a.style, parsed.data), + }; + const centre = { + x: x ?? a.position.x + a.size.width / 2, + y: y ?? a.position.y + a.size.height / 2, + }; + const moved = x !== undefined || y !== undefined; + if (a.type === "text") { + // A text box is the size of its text: refit it whenever the text, its size or its + // weight changes, around its centre or the new one. One saved before frame placement + // moves onto the frame the first time it is moved, as the editor does. + const resized = + text !== undefined || + parsed.data.fontSize !== undefined || + parsed.data.bold !== undefined; + return moved || (resized && a.space === "frame") + ? placeTextOnFrame(next, centre, document) + : next; + } + if (!moved) return next; + // An image or an arrow keeps its size and moves its centre, inside its own box. + const { width, height } = a.size; + return { + ...next, + position: { + x: Math.min(Math.max(centre.x - width / 2, 0), Math.max(0, 100 - width)), + y: Math.min(Math.max(centre.y - height / 2, 0), Math.max(0, 100 - height)), + }, + }; + }; const rebuiltAnnotations = replacePillSpan( - document.annotations.map((a) => - annPill.has(a.id) - ? { - ...a, - ...(parsed.data.text !== undefined - ? { content: parsed.data.text, textContent: parsed.data.text } - : {}), - } - : a, - ), + document.annotations.map((a) => (annPill.has(a.id) ? edit(a) : a)), annotationId, startMs, endMs, @@ -2063,6 +2554,11 @@ export function executeAgentTool( if (!existing) return failure(`Unknown full-camera region: ${parsed.data.cameraFullscreenId}`); const { startMs, endMs } = resolveSpanMs(existing, parsed.data.startSec, parsed.data.endSec); + const blocked = refuseIfBlocked("full-camera region", prev, parsed.data.cameraFullscreenId, { + startMs, + endMs, + }); + if (blocked) return blocked; const camPill = new Set(resolvePillIds(prev, parsed.data.cameraFullscreenId)); const rebuiltCamera = replacePillSpan( prev, diff --git a/electron/ai-edition/chat-persistence.test.ts b/electron/ai-edition/chat-persistence.test.ts new file mode 100644 index 0000000000..dcb05265a8 --- /dev/null +++ b/electron/ai-edition/chat-persistence.test.ts @@ -0,0 +1,228 @@ +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { createEmptyDocument } from "../../src/lib/ai-edition/schema"; + +vi.mock("./deep-agent/service", () => ({ invokeOpenScreenAgent: vi.fn() })); +vi.mock("./deep-agent/chat-model", () => ({ + createOpenScreenChatModel: vi.fn(), + messageContentToText: (content: unknown) => String(content), +})); + +import { + AiEditionService, + type AiEditionServiceOptions, +} from "../native-bridge/services/aiEditionService"; +import { + compactSessionNow, + configureChatPersistence, + createSession, + deleteProjectChat, + deleteSession, + flushChatPersistence, + listSessions, + renameSession, + rewindToMessage, + runChat, + selectSession, +} from "./chat-service"; +import { createOpenScreenChatModel } from "./deep-agent/chat-model"; +import { invokeOpenScreenAgent } from "./deep-agent/service"; +import { DocumentService } from "./document-service"; +import type { LlmConfigStore } from "./llm-config-store"; + +const invokeMock = vi.mocked(invokeOpenScreenAgent); +const modelMock = vi.mocked(createOpenScreenChatModel); +const key = "sk-test-secret-must-not-be-saved"; +let root: string; + +function config(): LlmConfigStore { + return { + getConfig: () => ({ provider: "openai", model: "gpt-4o" }), + getCredential: () => ({ value: key, entry: { kind: "api-key", apiKey: key } }), + } as unknown as LlmConfigStore; +} + +async function restart() { + await flushChatPersistence(); + configureChatPersistence(root); +} + +beforeEach(() => { + root = mkdtempSync(path.join(tmpdir(), "openscreen-chat-")); + configureChatPersistence(root); + invokeMock.mockReset(); + modelMock.mockReset(); + invokeMock.mockImplementation(async (args) => ({ + text: "reply", + document: args.document, + mutated: false, + })); +}); + +afterEach(async () => { + await flushChatPersistence(); + rmSync(root, { recursive: true, force: true }); +}); + +describe("local Agent chat history", () => { + it("restores multiple sessions and persists rename and delete per project", async () => { + const first = createSession("proj_a", "First"); + const deleted = createSession("proj_a", "Deleted"); + const other = createSession("proj_b", "Other"); + renameSession("proj_a", first.id, "Renamed"); + expect(deleteSession("proj_a", deleted.id)).toBe(true); + + await restart(); + expect(listSessions("proj_a")).toEqual([{ ...first, title: "Renamed" }]); + expect(selectSession("proj_a", deleted.id)).toBeNull(); + expect(listSessions("proj_b")).toEqual([other]); + expect(selectSession("proj_b", first.id)).toBeNull(); + }); + + it("restores transcript and model context without credentials or stale rewind controls", async () => { + const session = createSession("proj_chat"); + const document = createEmptyDocument({ title: "Test", projectId: "proj_chat" }); + await runChat("proj_chat", session.id, "first question", config(), document); + expect(selectSession("proj_chat", session.id)?.messages[0]?.checkpointId).toBeTruthy(); + await flushChatPersistence(); + const file = path.join(root, "proj_chat.chat.json"); + expect(readFileSync(file, "utf8")).not.toContain(key); + expect(readFileSync(file, "utf8")).not.toContain("checkpointId"); + + await restart(); + const restored = selectSession("proj_chat", session.id); + expect(restored?.messages.map((m) => m.content)).toEqual(["first question", "reply"]); + expect(restored?.messages[0]?.checkpointId).toBeNull(); + expect(rewindToMessage("proj_chat", session.id, restored?.messages[0]?.id ?? "").success).toBe( + false, + ); + await runChat("proj_chat", session.id, "follow up", config()); + expect(invokeMock.mock.lastCall?.[0].history.map((m) => m.content)).toEqual([ + "first question", + "reply", + "follow up", + ]); + }); + + it("restores compaction boundary for the model while keeping the full transcript", async () => { + modelMock.mockImplementation( + async () => + ({ + invoke: async () => ({ content: "Earlier goals and decisions" }), + }) as unknown as Awaited>, + ); + const session = createSession("proj_compact"); + for (let i = 0; i < 4; i++) { + await runChat("proj_compact", session.id, "question " + i + " ".repeat(100), config()); + } + const compacted = await compactSessionNow("proj_compact", session.id, config()); + expect(compacted?.summary).toBe("Earlier goals and decisions"); + await restart(); + expect(selectSession("proj_compact", session.id)?.messages).toHaveLength(8); + await runChat("proj_compact", session.id, "continue", config()); + const history = invokeMock.mock.lastCall?.[0].history ?? []; + expect(history[0]?.content).toBe("Earlier goals and decisions"); + expect(history.some((m) => m.content.startsWith("question 0"))).toBe(false); + expect(history.at(-1)?.content).toBe("continue"); + }); + + it("ignores malformed, old, and cross-project files without blocking document opening", async () => { + const documents = new DocumentService(root, root); + const doc = await documents.createProject("Still opens"); + const file = path.join(root, doc.project.id + ".chat.json"); + for (const bad of [ + "{broken", + JSON.stringify({ version: 0, projectId: doc.project.id, sessions: [] }), + JSON.stringify({ version: 1, projectId: "proj_other", sessions: [] }), + ]) { + writeFileSync(file, bad); + await restart(); + expect(listSessions(doc.project.id)).toEqual([]); + await expect(documents.getProject(doc.project.id)).resolves.toMatchObject({ + project: { id: doc.project.id }, + }); + } + }); + + it("does not trust a checkpoint id found in a saved transcript", async () => { + const session = createSession("proj_false_rewind"); + await flushChatPersistence(); + const file = path.join(root, "proj_false_rewind.chat.json"); + const saved = JSON.parse(readFileSync(file, "utf8")); + saved.sessions[0].messages = [ + { + id: "user_1", + role: "user", + content: "old message", + createdAt: new Date().toISOString(), + checkpointId: "missing", + }, + ]; + writeFileSync(file, JSON.stringify(saved)); + await restart(); + expect(selectSession("proj_false_rewind", session.id)?.messages[0]?.checkpointId).toBeNull(); + expect(rewindToMessage("proj_false_rewind", session.id, "user_1").success).toBe(false); + }); + + it("skips a damaged session while retaining valid siblings", async () => { + const good = createSession("proj_partial", "Good"); + await flushChatPersistence(); + const file = path.join(root, "proj_partial.chat.json"); + const saved = JSON.parse(readFileSync(file, "utf8")); + saved.sessions.push({ id: "broken", projectId: "proj_partial", messages: "wrong" }); + writeFileSync(file, JSON.stringify(saved)); + await restart(); + expect(listSessions("proj_partial")).toEqual([good]); + }); + + it("removes chat history when the owning project is deleted", async () => { + createSession("proj_removed", "Disposable"); + const file = path.join(root, "proj_removed.chat.json"); + // Written off the event loop: nothing is on disk until the queue runs. + expect(existsSync(file)).toBe(false); + await flushChatPersistence(); + expect(existsSync(file)).toBe(true); + renameSession("proj_removed", listSessions("proj_removed")[0]?.id ?? "", "Queued write"); + // Queued behind that pending write, so the write cannot recreate the file. + const deleting = deleteProjectChat("proj_removed"); + // The file still holds the deleted session until the queue reaches the delete. + expect(listSessions("proj_removed")).toEqual([]); + await deleting; + expect(existsSync(file)).toBe(false); + await restart(); + expect(listSessions("proj_removed")).toEqual([]); + }); + + it("does not recreate the history of a project deleted while the agent replies", async () => { + const session = createSession("proj_mid_turn"); + let reply: (value: Awaited>) => void = () => undefined; + invokeMock.mockImplementation(() => new Promise((resolve) => (reply = resolve))); + const turn = runChat("proj_mid_turn", session.id, "question", config()); + await vi.waitFor(() => expect(invokeMock).toHaveBeenCalled()); + await deleteProjectChat("proj_mid_turn"); + const document = createEmptyDocument({ title: "x", projectId: "proj_mid_turn" }); + reply({ text: "late reply", document, mutated: false }); + + expect(await turn).toMatchObject({ success: false }); + await restart(); + expect(existsSync(path.join(root, "proj_mid_turn.chat.json"))).toBe(false); + expect(listSessions("proj_mid_turn")).toEqual([]); + }); + + it("reports a project deleted even when its chat history cannot be removed", async () => { + const documents = new DocumentService(root, root); + const doc = await documents.createProject("Doomed"); + const service = new AiEditionService({ + documents, + deleteChatHistory: async () => { + throw new Error("disk says no"); + }, + } as unknown as AiEditionServiceOptions); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + await expect(service.deleteProject(doc.project.id)).resolves.toEqual({ success: true }); + expect(warn).toHaveBeenCalled(); + warn.mockRestore(); + }); +}); diff --git a/electron/ai-edition/chat-persistence.ts b/electron/ai-edition/chat-persistence.ts new file mode 100644 index 0000000000..eb5ee144b7 --- /dev/null +++ b/electron/ai-edition/chat-persistence.ts @@ -0,0 +1,124 @@ +// Local Agent chat sidecars live beside the project's .openscreen document. +// Keep this format separate from the document schema: a bad or newer chat file +// must never stop the project itself from opening. + +import { randomUUID } from "node:crypto"; +import { existsSync, readFileSync } from "node:fs"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { z } from "zod"; +import { renameWithRetry } from "./document-service"; + +const toolCallSchema = z.object({ name: z.string(), summary: z.string() }); +const messageSchema = z.object({ + id: z.string().min(1), + role: z.enum(["user", "assistant"]), + content: z.string(), + createdAt: z.string().min(1), + toolCalls: z.array(toolCallSchema).optional(), +}); +const sessionSchema = z.object({ + id: z.string().min(1), + projectId: z.string().min(1), + title: z.string().min(1), + createdAt: z.string().min(1), + messages: z.array(messageSchema), + compaction: z + .object({ + summary: messageSchema, + coveredCount: z.number().int().positive(), + }) + .optional(), +}); +const fileSchema = z.object({ + version: z.literal(1), + projectId: z.string(), + sessions: z.array(z.unknown()), +}); + +export type StoredChatSession = z.infer; + +export class ChatPersistence { + /** Tail of the pending write/delete chain per project id — see `enqueue`. */ + private readonly queues = new Map>(); + + constructor(private readonly projectsRoot: string) {} + + fileFor(projectId: string): string { + if (!/^[A-Za-z0-9_-]+$/.test(projectId)) throw new Error("Invalid project id"); + return path.join(this.projectsRoot, `${projectId}.chat.json`); + } + + read(projectId: string): StoredChatSession[] { + const file = this.fileFor(projectId); + if (!existsSync(file)) return []; + try { + const parsed = fileSchema.parse(JSON.parse(readFileSync(file, "utf8"))); + if (parsed.projectId !== projectId) return []; + const seen = new Set(); + const sessions: StoredChatSession[] = []; + for (const rawSession of parsed.sessions) { + const result = sessionSchema.safeParse(rawSession); + if (!result.success) continue; + const session = result.data; + if (session.projectId !== projectId || seen.has(session.id)) continue; + if (session.compaction && session.compaction.coveredCount > session.messages.length) + continue; + seen.add(session.id); + sessions.push(session); + } + return sessions; + } catch (error) { + console.warn(`[ai-edition] ignoring unreadable chat history for ${projectId}:`, error); + return []; + } + } + + /** Snapshots `sessions` now; the disk write waits its turn in the project's queue. */ + async write(projectId: string, sessions: StoredChatSession[]): Promise { + const destination = this.fileFor(projectId); + // Explicitly choose fields; provider configuration and API keys never enter + // the chat format. Checkpoints remain process-local and are never serialized. + const sanitized = sessions.map((session) => sessionSchema.parse(session)); + const json = JSON.stringify({ version: 1, projectId, sessions: sanitized }); + await this.enqueue(projectId, async () => { + await fs.mkdir(this.projectsRoot, { recursive: true }); + const temporary = `${destination}.tmp-${process.pid}-${randomUUID()}`; + try { + const handle = await fs.open(temporary, "w"); + try { + await handle.writeFile(json, "utf8"); + await handle.sync(); + } finally { + await handle.close(); + } + await renameWithRetry(temporary, destination); + } finally { + await fs.rm(temporary, { force: true }); + } + }); + } + + /** Queued behind the project's pending writes, so none of them can recreate the file. */ + async delete(projectId: string): Promise { + const file = this.fileFor(projectId); + await this.enqueue(projectId, () => fs.rm(file, { force: true })); + } + + /** Settles once every write and delete queued so far has. */ + async flush(): Promise { + await Promise.all(this.queues.values()); + } + + // Same chain as DocumentService.writeProject: one queue per project, run on + // both settlements so a failed write does not cancel the next one. + private enqueue(projectId: string, task: () => Promise): Promise { + const run = (this.queues.get(projectId) ?? Promise.resolve()).then(task, task); + const settled = run.catch(() => undefined); + this.queues.set(projectId, settled); + void settled.then(() => { + if (this.queues.get(projectId) === settled) this.queues.delete(projectId); + }); + return run; + } +} diff --git a/electron/ai-edition/chat-service.ts b/electron/ai-edition/chat-service.ts index 20e6350623..b224565bc7 100644 --- a/electron/ai-edition/chat-service.ts +++ b/electron/ai-edition/chat-service.ts @@ -1,4 +1,4 @@ -// In-memory chat service. ponytail: chat sessions live in a nested Map; the +// Agent chat service. Sessions are cached by project and persisted locally; the // agentic loop itself lives in `deep-agent/service.ts` and is a port of // axcut's AxcutDeepAgentService (LangGraph stateful thread via // `createDeepAgent`). The IPC bridge streams `text` deltas + `toolStart` / @@ -30,12 +30,51 @@ import { compactionSplitIndex, DEFAULT_BUDGET_TOKENS, } from "./chat-compaction"; +import { ChatPersistence } from "./chat-persistence"; import type { CursorTelemetryReader } from "./deep-agent/service"; import type { DocumentService } from "./document-service"; import type { LlmConfigStore } from "./llm-config-store"; import { PROVIDER_DEFINITIONS } from "./provider-registry"; const sessionsByProject = new Map>(); +let persistence: ChatPersistence | null = null; + +/** Called once in the main process. Reconfiguration simulates a restart in tests. */ +export function configureChatPersistence(projectsRoot: string): void { + persistence = new ChatPersistence(projectsRoot); + sessionsByProject.clear(); + messageCheckpointsBySession.clear(); +} + +function persistProject(projectId: string): void { + const sessions = sessionsByProject.get(projectId); + // No cache entry means the project was deleted, possibly mid-turn: writing + // now would recreate the history its deletion just removed. + if (!persistence || !sessions) return; + // Off the event loop and best effort: the cache stays the source of truth + // while the app runs, and a chat file must never fail a chat turn. + persistence.write(projectId, [...sessions.values()]).catch((error) => { + console.warn(`[ai-edition] could not save chat history for ${projectId}:`, error); + }); +} + +/** Projects whose sidecar removal is still queued: their file holds deleted sessions. */ +const deletingProjects = new Set(); + +export function deleteProjectChat(projectId: string): Promise { + sessionsByProject.delete(projectId); + for (const key of messageCheckpointsBySession.keys()) { + if (key.startsWith(projectId + "::")) messageCheckpointsBySession.delete(key); + } + if (!persistence) return Promise.resolve(); + deletingProjects.add(projectId); + return persistence.delete(projectId).finally(() => deletingProjects.delete(projectId)); +} + +/** Lets queued chat-history writes reach the disk before the app exits. */ +export function flushChatPersistence(): Promise { + return persistence?.flush() ?? Promise.resolve(); +} // ponytail: per-message checkpoints, stored as an ordered list per session // (insertion order == the session's message order). Each entry captures the @@ -116,7 +155,7 @@ function dropCheckpointsFrom( // ponytail: what compaction leaves behind. `coveredCount` counts the leading // transcript messages the summary stands in for — the transcript itself is // never rewritten, so the user keeps every message they wrote while the model -// gets the shortened list. Sessions are in-memory only; deleting the messages +// gets the shortened list. Deleting the messages // the renderer shows would be unrecoverable, and nothing in the UI would say // it happened. interface SessionCompaction { @@ -181,7 +220,8 @@ function toSummary(s: ChatSession): ChatSessionSummary { function getProjectSessions(projectId: string): Map { let m = sessionsByProject.get(projectId); if (!m) { - m = new Map(); + const stored = deletingProjects.has(projectId) ? [] : (persistence?.read(projectId) ?? []); + m = new Map(stored.map((session) => [session.id, session])); sessionsByProject.set(projectId, m); } return m; @@ -192,8 +232,7 @@ function defaultSessionTitle(index: number): string { } export function listSessions(projectId: string): ChatSessionSummary[] { - const m = sessionsByProject.get(projectId); - if (!m) return []; + const m = getProjectSessions(projectId); return Array.from(m.values()) .map(toSummary) .sort((a, b) => a.createdAt.localeCompare(b.createdAt)); @@ -211,14 +250,14 @@ export function createSession(projectId: string, title?: string): ChatSessionSum messages: [], }; m.set(id, session); + persistProject(projectId); return toSummary(session); } export function selectSession(projectId: string, sessionId: string): ChatSession | null { - const m = sessionsByProject.get(projectId); - const s = m?.get(sessionId); + const s = getProjectSessions(projectId).get(sessionId); if (!s) return null; - // ponytail: shallow-copy messages so the caller can't mutate the live array. + // Return copies and expose a rewind id only while its checkpoint still exists. // The compaction boundary stays behind: it is main-process bookkeeping, and // every caller of this wants the transcript as the user sees it. return { @@ -226,7 +265,13 @@ export function selectSession(projectId: string, sessionId: string): ChatSession projectId: s.projectId, title: s.title, createdAt: s.createdAt, - messages: [...s.messages], + messages: s.messages.map((message) => ({ + ...message, + checkpointId: + message.checkpointId && findCheckpointForMessage(projectId, sessionId, message.checkpointId) + ? message.checkpointId + : null, + })), }; } @@ -235,18 +280,22 @@ export function renameSession( sessionId: string, title: string, ): ChatSessionSummary | null { - const m = sessionsByProject.get(projectId); - const s = m?.get(sessionId); + const s = getProjectSessions(projectId).get(sessionId); if (!s) return null; const trimmed = title.trim(); - if (trimmed) s.title = trimmed; + if (trimmed && trimmed !== s.title) { + s.title = trimmed; + persistProject(projectId); + } return toSummary(s); } export function deleteSession(projectId: string, sessionId: string): boolean { - const m = sessionsByProject.get(projectId); - if (!m?.has(sessionId)) return false; + const m = getProjectSessions(projectId); + if (!m.has(sessionId)) return false; m.delete(sessionId); + messageCheckpointsBySession.delete(sessionKey(projectId, sessionId)); + persistProject(projectId); return true; } @@ -270,6 +319,8 @@ export interface ChatRunEnv { /** Reads recorded cursor telemetry for an asset. Built in `electron/ipc/ * handlers.ts`, where the path allow-list lives. */ cursor?: CursorTelemetryReader; + /** Runtime-only model retry override used by bounded measurement harnesses. */ + maxRetries?: number; } // ponytail: zero-config noop for sink callbacks that the caller did not provide. @@ -356,11 +407,12 @@ export async function runChat( ? structuredClone(workingDocument) : null; if (documentForCheckpoint) { - recordMessageCheckpoint(projectId, sessionId, userMessage.id, documentForCheckpoint); + recordMessageCheckpoint(projectId, session.id, userMessage.id, documentForCheckpoint); } userMessage.checkpointId = documentForCheckpoint ? userMessage.id : null; session.messages.push(userMessage); + persistProject(projectId); const editsAllowed = config.allowAgentEdits !== false; @@ -405,6 +457,7 @@ export async function runChat( apiKey: apiKey ?? undefined, baseUrl: config.baseUrl, reasoningEffort: config.reasoningEffort, + maxRetries: env.maxRetries, }, history, userMessage: message, @@ -413,6 +466,12 @@ export async function runChat( cursor: env.cursor, }); + // The project was deleted while the agent ran: its sessions are gone, so a + // reply recorded now would be reported as sent and then never be seen again. + if (sessionsByProject.get(projectId) !== sessions) { + return { success: false, error: "The project was deleted while the agent was replying." }; + } + if (!result.text) { // ponytail: surface the deep-agent's diagnostic so the user can see // *why* the model produced no text (e.g. MiniMax streaming only @@ -433,6 +492,7 @@ export async function runChat( toolCalls: appliedToolCalls.length ? appliedToolCalls : undefined, }; session.messages.push(assistantMessage); + persistProject(projectId); return { success: true, @@ -468,7 +528,7 @@ export function rewindToMessage( success: false; error: string; } { - const session = sessionsByProject.get(projectId)?.get(sessionId); + const session = getProjectSessions(projectId).get(sessionId); if (!session) return { success: false, error: "Chat session not found." }; const messageIndex = session.messages.findIndex((m) => m.id === messageId); if (messageIndex === -1) { @@ -501,12 +561,21 @@ export function rewindToMessage( session.compaction = undefined; } dropCheckpointsFrom(projectId, sessionId, target.checkpointId); + for (const message of session.messages) { + if ( + message.checkpointId && + !findCheckpointForMessage(projectId, sessionId, message.checkpointId) + ) { + message.checkpointId = null; + } + } + persistProject(projectId); return { success: true, prompt: target.content, document: cp.document, - messages: [...survived], + messages: selectSession(projectId, sessionId)?.messages ?? [], }; } @@ -530,11 +599,10 @@ export async function runTimelineOperation( conversationMessage: string, documents: DocumentService, ): Promise<{ success: true; result: TimelineOperationResult } | { success: false; error: string }> { - let session = sessionsByProject.get(projectId)?.get(sessionId); - const created = !session; - if (created) { + let session = getProjectSessions(projectId).get(sessionId); + if (!session) { const summary = createSession(projectId); - session = selectSession(projectId, summary.id) ?? undefined; + session = getProjectSessions(projectId).get(summary.id); if (!session) return { success: false, error: "Could not create session." }; } @@ -568,6 +636,7 @@ export async function runTimelineOperation( createdAt: new Date().toISOString(), }; session.messages.push(assistantMessage); + persistProject(projectId); } return { @@ -587,7 +656,7 @@ export async function compactSessionNow( sessionId: string, llmConfig: LlmConfigStore, ): Promise<{ summaryMessageId: string | null; summary: string; session: ChatSession } | null> { - const session = sessionsByProject.get(projectId)?.get(sessionId); + const session = getProjectSessions(projectId).get(sessionId); if (!session) return null; const config = llmConfig.getConfig(); if (!config) return null; @@ -612,6 +681,7 @@ export async function compactSessionNow( reasoningEffort: config.reasoningEffort, }); if (!ok) return null; + persistProject(projectId); return { summaryMessageId: ok.summaryMessageId, summary: ok.summary, @@ -628,7 +698,7 @@ export function getSessionContextUsage( sessionId: string, budgetTokens: number = DEFAULT_BUDGET_TOKENS, ): { usedTokens: number; budgetTokens: number; ratio: number; fillPercent: number } | null { - const session = sessionsByProject.get(projectId)?.get(sessionId); + const session = getProjectSessions(projectId).get(sessionId); if (!session) return null; // Measured on what the model is given, not on the transcript -- after a // compaction those differ, and the number that matters is the one that @@ -669,7 +739,7 @@ export function getSessionBudget( sessionId: string, budgetTokens: number = DEFAULT_BUDGET_TOKENS, ): SessionBudgetSnapshot | null { - const s = sessionsByProject.get(projectId)?.get(sessionId); + const s = getProjectSessions(projectId).get(sessionId); if (!s) return null; // Same window the model is actually sent — see `getSessionContextUsage`. const snap = budgetSnapshot(modelHistory(s), budgetTokens); @@ -690,7 +760,7 @@ export async function compactSession( sessionId: string, llmConfig: LlmConfigStore, ): Promise<{ summaryMessageId: string | null; summary: string } | null> { - const session = sessionsByProject.get(projectId)?.get(sessionId); + const session = getProjectSessions(projectId).get(sessionId); if (!session) return null; const config = llmConfig.getConfig(); if (!config) return null; @@ -710,6 +780,7 @@ export async function compactSession( baseUrl: config.baseUrl, reasoningEffort: config.reasoningEffort, }); + if (ok) persistProject(projectId); return ok; } diff --git a/electron/ai-edition/deep-agent/chat-model.test.ts b/electron/ai-edition/deep-agent/chat-model.test.ts index 477f5bece5..2b917ca5b9 100644 --- a/electron/ai-edition/deep-agent/chat-model.test.ts +++ b/electron/ai-edition/deep-agent/chat-model.test.ts @@ -91,6 +91,33 @@ describe("createOpenScreenChatModel — Anthropic-wire output budget", () => { }); }); +describe("createOpenScreenChatModel — runtime retry override", () => { + function retries(model: unknown): number | undefined { + return (model as { caller?: { maxRetries?: number } }).caller?.maxRetries; + } + + it("passes an explicit zero to LangChain", async () => { + const model = await createOpenScreenChatModel({ + provider: "openai-compatible", + model: "gpt-5-test", + apiKey: "test-key", + baseUrl: "http://127.0.0.1:1/v1", + maxRetries: 0, + }); + expect(retries(model)).toBe(0); + }); + + it("keeps LangChain's default when the override is omitted", async () => { + const model = await createOpenScreenChatModel({ + provider: "openai-compatible", + model: "gpt-5-test", + apiKey: "test-key", + baseUrl: "http://127.0.0.1:1/v1", + }); + expect(retries(model)).toBe(6); + }); +}); + describe("messageContentToText", () => { it("passes a plain string through", () => { expect(messageContentToText("hello")).toBe("hello"); diff --git a/electron/ai-edition/deep-agent/chat-model.ts b/electron/ai-edition/deep-agent/chat-model.ts index 037343262b..2485ca73d4 100644 --- a/electron/ai-edition/deep-agent/chat-model.ts +++ b/electron/ai-edition/deep-agent/chat-model.ts @@ -283,6 +283,9 @@ export interface OpenScreenChatModelConfig { apiKey?: string; baseUrl?: string; reasoningEffort?: string; + /** Runtime-only retry override, read on the OpenAI-compatible path only (the workbench's). + * Omitted callers keep LangChain's default. */ + maxRetries?: number; } // ponytail: placeholder API key for self-hosted OpenAI-compatible endpoints @@ -422,6 +425,7 @@ export async function createOpenScreenChatModel( ...(reasoningOptions.reasoning ? { reasoning: reasoningOptions.reasoning } : {}), ...(reasoningOptions.useResponsesApi ? { useResponsesApi: true } : {}), ...(reasoningOptions.modelKwargs ? { modelKwargs: reasoningOptions.modelKwargs } : {}), + ...(config.maxRetries !== undefined ? { maxRetries: config.maxRetries } : {}), // ponytail: Gemini's OpenAI-compat path can't stream + tool-call at once // — disable streaming so the ChatOpenAI compat layer buffers and returns // cleanly. axcut does the same. diff --git a/electron/ai-edition/deep-agent/service.test.ts b/electron/ai-edition/deep-agent/service.test.ts index 5aebb9a330..104c64abc0 100644 --- a/electron/ai-edition/deep-agent/service.test.ts +++ b/electron/ai-edition/deep-agent/service.test.ts @@ -60,6 +60,7 @@ const ARGS: Record = { getTranscriptWords: {}, getCursorTrack: {}, setWordText: { wordId: "word_1", text: "Hullo" }, + removeWords: { wordIds: ["word_1"] }, addTrim: { startSec: 1, endSec: 2 }, addTrims: { ranges: [{ startSec: 1, endSec: 2 }] }, setTrim: { trimRangeId: "trim_1", startSec: 1, endSec: 2 }, @@ -82,6 +83,7 @@ const ARGS: Record = { removeTrim: { trimRangeId: "trim_1" }, removeModifier: { id: "nope" }, removeClip: { clipId: "clip_1" }, + setAspectRatio: { aspectRatio: "16:9" }, }; /** ponytail: the fixture starts from a FIXED instant. `createEmptyDocument` reads @@ -390,6 +392,14 @@ describe("what the descriptions say about zoom strength", () => { expect(TOOL_DESCRIPTIONS.addCameraFullscreen).toMatch(/hasCameraTrack/); expect(SYSTEM_PROMPT).toMatch(/hasCameraTrack/); }); + + it("tells every zoom and trim write where a zoom's moves are (#1028)", () => { + for (const name of ["addZoom", "addZooms", "setZoom", "addTrim", "addTrims", "setTrim"]) { + expect(TOOL_DESCRIPTIONS[name]).toContain("zoomOutUntilSec"); + expect(TOOL_DESCRIPTIONS[name]).toContain("cutTransitions"); + } + expect(SYSTEM_PROMPT).toMatch(/zoomInFromSec.*untrimmed/); + }); }); // ── D-CONSENT ─────────────────────────────────────────────────────────────── diff --git a/electron/ai-edition/deep-agent/service.ts b/electron/ai-edition/deep-agent/service.ts index 64bcef03bd..f23c19125f 100644 --- a/electron/ai-edition/deep-agent/service.ts +++ b/electron/ai-edition/deep-agent/service.ts @@ -25,7 +25,9 @@ import type { AxcutDocument } from "../../../src/lib/ai-edition/schema"; // (`depth/2 + 0.5`) that was purged from two rendering sites and survived here, // wrong at both ends of a table that actually runs 1.25× to 5.0×. import { ZOOM_DEPTH_LEGEND } from "../../../src/lib/ai-edition/timeline/zoom-scale"; +import { ZOOM_TRANSITION_LEGEND } from "../../../src/lib/ai-edition/timeline/zoom-transitions"; import { + type AgentToolExecution, addAnnotationArgs, addAudioArgs, addCameraFullscreenArgs, @@ -44,9 +46,11 @@ import { removeClipArgs, removeModifierArgs, removeTrimArgs, + removeWordsArgs, replaceTimelineArgs, resolveCursorAssetId, setAnnotationArgs, + setAspectRatioArgs, setAudioArgs, setCameraFullscreenArgs, setClipRangeArgs, @@ -93,7 +97,7 @@ const CONSENT_PROMPT_BLOCK = [ "", "PROJECT EDITS ARE CURRENTLY DISABLED by the user, who asked to be consulted before the timeline changes.", "- Read freely: getCurrentDocument and getTranscript work as usual.", - "- Do NOT call any tool that writes (addTrim, setTrim, setClipRange, moveClip, replaceTimeline, add*/set* effects, remove*). Every one of them will be refused, so calling them wastes the turn and tells the user nothing.", + "- Do NOT call any tool that writes (removeWords, addTrim, setTrim, setClipRange, moveClip, replaceTimeline, add*/set* effects, remove*). Every one of them will be refused, so calling them wastes the turn and tells the user nothing.", "- Instead: say precisely what you would change — which tool, which times, which ids — and ask the user to confirm. Be specific enough that they can say yes to it.", "- Never state or imply that an edit was applied. If the user confirms and you are still refused, tell them the 'Project edits' setting in Settings → AI has to be re-enabled first.", ].join("\n"); @@ -108,7 +112,7 @@ const BASE_SYSTEM_PROMPT = [ "You can call the tools below against the live document snapshot; the runtime executes each edit and feeds the result back into the loop.", "The AxcutDocument is the single source of truth. The timeline, the transcript editor, and the chat panel are all direct editors of the same document — when the user places a clip on the timeline, the document updates immediately, and when the timeline is empty, the document has no clips. Your edits operate on the live document, so preserve the user's placed clips.", "", - "Time-bases (do not mix them up): clips and trims are in SOURCE-time seconds of an asset; zooms, speed regions, annotations and camera-fullscreen regions are in VIRTUAL (edited-timeline) seconds — the position on the ruler after clips + trims are applied. getCurrentDocument returns all of them, clearly labelled.", + "Time-bases (do not mix them up): clips and trims are in SOURCE-time seconds of an asset; zooms, speed regions, annotations and camera-fullscreen regions are in VIRTUAL (edited-timeline) seconds — the position on the editor's ruler, where the clips sit end to end. A trim leaves a hole in that ruler rather than shortening it, and a speed region does not stretch it, so the exported video's own clock differs from it as soon as there is a trim or a speed region: always place effects in ruler seconds. getCurrentDocument returns all of them, clearly labelled.", "", // ponytail: these describe what each tool is FOR, deliberately without quoting // user phrasings. A phrase→tool table reads as helpful and is not: it swaps the @@ -117,8 +121,11 @@ const BASE_SYSTEM_PROMPT = [ // other than English. Say what the tool does; let the model do the matching. "How the tools map to intent — pick the most specific one, and prefer the smallest edit that satisfies the request:", "- Silences, pauses and dead stretches are removed as trims INSIDE the placed clip. Send them together with addTrims once you know the ranges; addTrim is for a single cut or a correction. The placed clip stays the canonical cut; it is not rebuilt to drop them.", + "- Cutting specific spoken words (fillers, a repeated or flubbed phrase, a false start, a duplicate take) is removeWords: read getTranscriptWords, choose the occurrences from their surrounding speech, then pass that assetId and ONLY those word IDs. Word IDs restart for each asset, so keep the read's assetId. Consecutive words become one cut. Do not cut every occurrence of a word just because its text matches one you were asked to remove.", "- Changing where a clip starts or ends within its source is setClipRange — the clip's in/out, distinct from a trim.", `- addZoom takes a virtual-timeline span (depth is an ordinal 1–6 selecting from a fixed table — ${ZOOM_DEPTH_LEGEND} — never a multiplier; focus in 0–1 frame fractions). addSpeed changes pacing over a span. addAnnotation puts text on screen. addCameraFullscreen enlarges the webcam, and only does something where assets[].hasCameraTrack is true.`, + "- setAspectRatio picks the shape of the whole video, for where it will be shown; auto follows the recording.", + "- A zoom animates in just before its span and out just after it (zoomInFromSec, zoomOutUntilSec in getCurrentDocument). Leave those windows untrimmed, or the export jumps at the cut.", "- addAudio lays an imported voiceover or music file over a span. It plays an asset the project already has (kind 'audio'); importing or recording one is the editor's job, not a tool you have — so when the project has none, say so rather than naming an id that does not exist.", "- moveClip changes the order of placed clips, one call per clip that moves, preserving ids, source ranges, trims and anchored effects. replaceTimeline rebuilds the timeline from kept intervals and sorts them, so it cannot reorder anything.", "- Deleting is a first-class action, not a workaround: removeTrim, removeModifier, removeClip. Never fake a deletion by re-adding an element or zeroing it out (span 0, speed 1×) — that leaves it in the document and misreports what you did.", @@ -141,40 +148,50 @@ export function buildSystemPrompt(options: { editsAllowed: boolean }): string { * case, and the string the surface tests measure the wire against. */ export const SYSTEM_PROMPT = buildSystemPrompt({ editsAllowed: true }); +// #1028: a zoom's camera moves sit OUTSIDE its span, where nothing on the ruler shows them. +// The editor's "3D camera" control, in the agent's words. +const ZOOM_CAMERA_NOTE = + "rotationPreset is the zoom's 3D camera: `left` or `right` turns the screen in 3D to that side, `orbit` moves the camera around the screen while the zoom holds, `none` (the default) keeps it flat. It is a strong effect: keep it for a reveal or a key moment, not every zoom. getCurrentDocument reports it as rotationPreset, absent when flat."; + +const ZOOM_TRANSITIONS_NOTE = `A zoom HOLDS over its span: it animates in during the window just before startSec and out during the window just after endSec. Each move lasts 0.6 s + 0.55 s × ln(renderedScale) of SCREEN time (${ZOOM_TRANSITION_LEGEND}), so a speed region scales the timeline it covers by its speed: longer above 1×, shorter below. getCurrentDocument gives each zoom's zoomInFromSec and zoomOutUntilSec, in virtual seconds like the zoom. A trim inside either window cuts that move and the export jumps at the cut: leave the windows untrimmed. Two zooms at most 1.5 s apart on screen, on the same clip with no trim between them, are chained: the camera pans from one to the next over 1 s of screen time from the first one's end (panUntilSec), with no zoom-out or zoom-in between them.`; +const CUT_TRANSITIONS_RESULT = `When the result cuts a zoom's move in or out, it carries \`cutTransitions\`: which zoom, which side, cutSec, the timeline seconds of the move lost, and by: trim, or junction when the move runs past its clip's edge into the next clip. It is a warning, nothing is refused: move the cut or the zoom unless a hard cut is what you want.`; + export const TOOL_DESCRIPTIONS: Record = { getCurrentDocument: "Read a compact snapshot of the current project: assets (with durations), timeline clips and trim ranges (source-time), and the zoom / speed / annotation effects (virtual, edited-timeline time). Call this before editing if the snapshot in the system prompt may be stale. The AxcutDocument is the single source of truth — your edits should preserve the user's placed clips and any timeline state they have already set up.", getTranscript: "Read the transcript segments (speech and silence, with start/end seconds and text) for an asset. Omit assetId to read the primary asset's transcript.", getCursorTrack: - "Read the recorded pointer track for an asset: where the cursor was over time, downsampled to a readable rate. Each point carries atSec (the asset's own source clock), virtualSec (the same instant on the edited timeline — the coordinate addZoom takes, null when no clip carries it), cx/cy as 0–1 fractions of the frame, and `shape`, an index into the pointer bitmaps the recording used (equal values are the same pointer; a change means the pointer changed, e.g. arrow to text caret). Points that are not plain moves carry `kind`; points a trim cuts out of playback carry `trimmed`. These are real samples, not a summary — reading what the pointer was doing is yours. Omit assetId for the primary asset. It answers `available:false` in two DIFFERENT ways you must not confuse: reason 'no-sidecar' means this asset was checked and genuinely has no telemetry, while reason 'unavailable' means it could not be read from here.", + "Read the recorded pointer track for an asset: where the cursor was over time, downsampled to a readable rate. Each point carries atSec (the asset's own source clock), virtualSec (the same instant on the edited timeline — the coordinate addZoom takes, null when no clip carries it), cx/cy as 0–1 fractions of the frame, and `shape`, an index into the pointer bitmaps the recording used (equal values are the same pointer; a change means the pointer changed, e.g. arrow to text caret). Points that are not plain moves carry `kind`; points a trim cuts out of playback carry `trimmed`; points where the pointer was hidden carry `visible:false`. These are real samples, not a summary — reading what the pointer was doing is yours. Omit assetId for the primary asset. It answers `available:false` in two DIFFERENT ways you must not confuse: reason 'no-sidecar' means this asset was checked and genuinely has no telemetry, while reason 'unavailable' means it could not be read from here.", getTranscriptWords: - 'Read the transcript one WORD at a time for an asset: each word\'s id, text, start/end seconds, and — only when it is not plain transcription — `source` ("user" for a word the user corrected, "synth" for one they typed in) and `originalText` (what the transcriber had heard before the correction). This is the ONLY read that gives you the ids setWordText takes; getTranscript answers in segments, whose ids belong to a different namespace and are not accepted there. A whole transcript is large, so pass startSec/endSec to read just the passage you mean to fix. Omit assetId for the primary asset.', + 'Read the transcript one WORD at a time for an asset: each word\'s id, text, start/end seconds, and — only when it is not plain transcription — `source` ("user" for a word the user corrected, "synth" for one they typed in) and `originalText` (what the transcriber had heard before the correction). This is the ONLY read that gives you the ids setWordText and removeWords take; getTranscript answers in segments, whose ids belong to a different namespace and are not accepted there. A whole transcript is large, so pass startSec/endSec to read just the passage you mean to fix. Omit assetId for the primary asset.', setWordText: - "Correct ONE word's text, by the id getTranscriptWords returns. This changes the TRANSCRIPT and nothing else: the captions follow it, the film is untouched and no audio is cut. Use it when the transcriber misheard something — a name, a technical term — and the user asks for it to read correctly. Passing an empty string BLANKS the word: it keeps its place in the media but leaves the captions, which is how a junk token like \"(inaudible)\" is removed without cutting the speech around it. Writing the transcriber's own text back clears the correction. This is NOT how you make a spoken word go away — that removes only the label and leaves the film saying it; use addTrim, which cuts the audio with it.", - addTrim: - "Add ONE trim range: a cut of a span inside a clip (this source-time span will not be played or exported) that does NOT split the clip. Times are in seconds of the asset's source time. This is the preferred (and for 'remove silences' requests, the only) way to handle silences; it preserves the user's placed clips and only adds a cut. When you have several cuts to make, use addTrims and send them together — this one is for a single cut or a later correction. A cut belongs to ONE clip: `clipId` is inferred when a single clip covers the range, but when several clips draw on the same asset over it the call FAILS and lists them — pass the `clipId` you mean (ids come from getCurrentDocument).", - addTrims: - "Add MANY trim ranges in one call: `ranges` is a list, each entry taking exactly the fields addTrim takes. Use this whenever you have more than one cut to make — 'remove the silences' on a half-hour recording is hundreds of cuts, and sending them one at a time costs one round trip each. Each range stands or falls ALONE: one that cannot be placed is refused by itself and listed in `refused` with its index and the reason, while every other range is still applied. Nothing is rolled back, so a single bad bound never costs you the rest. The result leads with requested / appliedCount / refusedCount so you can see a partial outcome without re-reading the document — report what was refused rather than claiming the whole list landed.", - setTrim: - "Move or resize an existing trim range by id. Times are source-time seconds. The cut follows to whichever clip the new range lands in, when that clip is unambiguous.", + "Correct ONE word's text, by the id getTranscriptWords returns. This changes the TRANSCRIPT and nothing else: the captions follow it, the film is untouched and no audio is cut. Use it when the transcriber misheard something — a name, a technical term — and the user asks for it to read correctly. Passing an empty string BLANKS the word: it keeps its place in the media but leaves the captions, which is how a junk token like \"(inaudible)\" is removed without cutting the speech around it. Writing the transcriber's own text back clears the correction. This is NOT how you make a spoken word go away — that removes only the label and leaves the film saying it; use removeWords, which cuts the audio with it.", + removeWords: + "Cut the spoken words selected by `wordIds` from getTranscriptWords out of the film: fillers, a repeated or flubbed phrase, a false start, a duplicate take. Pass the same `assetId` as that read: word IDs are numbered independently in each asset. Omit assetId only when the IDs are unique across all transcripts. Read words in context and choose the IDs yourself; the executor does not classify speech or remove every matching text. It resolves each ID to one recorded asset, valid source timestamps, and exactly one fully covering clip, then adds non-destructive trims; words that follow one another in the transcript become ONE trim, so a phrase goes out whole, gaps included. If any ID is missing, invalid, duplicated, already cut or ambiguous, nothing changes. The transcript is untouched. The result lists each word and the ACTUAL trim span (which addTrim may shape), asset, clip and trim ID; report only those results. " + + CUT_TRANSITIONS_RESULT, + addTrim: `Add ONE trim range: a cut of a span inside a clip (this source-time span will not be played or exported) that does NOT split the clip. Times are in seconds of the asset's source time. This is the preferred (and for 'remove silences' requests, the only) way to handle silences; it preserves the user's placed clips and only adds a cut. When you have several cuts to make, use addTrims and send them together — this one is for a single cut or a later correction. A cut belongs to ONE clip: \`clipId\` is inferred when a single clip covers the range, but when several clips draw on the same asset over it the call FAILS and lists them — pass the \`clipId\` you mean (ids come from getCurrentDocument). ${ZOOM_TRANSITIONS_NOTE} ${CUT_TRANSITIONS_RESULT}`, + addTrims: `Add MANY trim ranges in one call: \`ranges\` is a list, each entry taking exactly the fields addTrim takes. Use this whenever you have more than one cut to make — 'remove the silences' on a half-hour recording is hundreds of cuts, and sending them one at a time costs one round trip each. Each range stands or falls ALONE: one that cannot be placed is refused by itself and listed in \`refused\` with its index and the reason, while every other range is still applied. Nothing is rolled back, so a single bad bound never costs you the rest. The result leads with requested / appliedCount / refusedCount so you can see a partial outcome without re-reading the document — report what was refused rather than claiming the whole list landed. ${ZOOM_TRANSITIONS_NOTE} Each applied entry carries the same \`cutTransitions\` warning addTrim reports.`, + setTrim: `Move or resize an existing trim range by id. Times are source-time seconds. The cut follows to whichever clip the new range lands in, when that clip is unambiguous. ${ZOOM_TRANSITIONS_NOTE} ${CUT_TRANSITIONS_RESULT}`, setClipRange: "Set a clip's in/out points (source-time seconds) to shorten its head or tail — distinct from a trim (which cuts a span inside the clip). All clips are re-laid back-to-back afterwards, so downstream clips shift automatically. Use this ONLY when the user explicitly asks to shorten or extend a user-placed clip. Do NOT use this for 'remove silences' or 'cut pauses' — for those, use addTrim.", moveClip: "Reorder a placed clip: move `clipId` so it plays just before `beforeClipId` (pass null, or omit it, to move it last). Ids come from getCurrentDocument, where each clip carries its `index` and its label in `reason`. This preserves every clip id, every source range, every trim, and the zooms / speed regions / annotations anchored to each clip. This is the tool for 'swap these clips', 'put X first' and 'change the clip order' — replaceTimeline cannot reorder anything.", replaceTimeline: "Replace the whole timeline with the given kept intervals of the primary asset's source time. Everything outside the intervals becomes a trim. The intervals are SORTED, so this can never reorder clips — use moveClip for that. DO NOT use this for 'cut silences' or 'remove pauses' — the user has likely placed clips on the timeline that you'd be discarding. Use this ONLY when the user explicitly asks you to rebuild the timeline from scratch (e.g. 'start over with the kept intervals from the transcript'). It is refused when it would merge away, shorten or drop an existing clip; the refusal names them and the tool to use instead.", - addZoom: `Add a zoom-in over a span of the edited timeline (virtual seconds). depth is an ORDINAL 1–6, not a factor: it selects a magnification from a fixed table (${ZOOM_DEPTH_LEGEND}), so the default depth 3 renders at 1.80×. The result reports renderedScale — quote that, never the depth, when telling the user how strong the zoom is. focus is the zoom centre in 0–1 fractions of the frame (default centre). When the recording's pointer telemetry can be read for the footage under the span, the result also carries \`cursorAnchor\`: \`focus\` echoes the value this call used (including the default, if you left it out), \`cursor\` is where the pointer ACTUALLY was over the span the zoom landed on — the median of the recorded samples, \`spread\` being how far the farthest one strays from it — and \`offset\` is the distance between the two, in frame fractions. It is a measurement, not a correction: nothing is moved and no call is refused over it, and a zoom framing a slide, a face, or a region the pointer never enters is a legitimate choice. \`available:false\` names what it found instead (\`no-samples\`, \`trimmed-out\`). Its ABSENCE means no telemetry was read for that footage — never that the recording has none; assets[].hasCursorTelemetry and getCursorTrack are what answer that. Use for 'zoom in on …' and the smart-zoom pass.`, - addZooms: `Add MANY zooms in one call: \`regions\` is a list, each entry taking exactly the fields addZoom takes (same depth table, ${ZOOM_DEPTH_LEGEND}). Use this for the smart-zoom pass, where you have decided every zoom before emitting the first one — sending them one at a time costs one round trip each. Each region stands or falls ALONE: one that covers no clip is refused by itself and listed in \`refused\` with its index and the reason, while the others are still applied. The result leads with requested / appliedCount / refusedCount, and each applied entry carries its renderedScale — quote that, never the depth — plus the same \`cursorAnchor\` addZoom reports, whenever the footage under that region has readable pointer telemetry.`, - setZoom: `Move, resize, or restyle an existing zoom by id (virtual-timeline seconds). Only the fields you pass are changed. depth selects from the same table (${ZOOM_DEPTH_LEGEND}); if the zoom carries a customScale (getCurrentDocument shows it as depthIsOverridden), that custom value is what renders, and passing depth clears it so the depth takes effect — the result says so. The result reports the resulting renderedScale, and — when the footage under the span has readable pointer telemetry — the same \`cursorAnchor\` addZoom reports, measured against the zoom's EFFECTIVE focus, so a call that moved only the span still learns what its unchanged focus is now looking at.`, + addZoom: `Add a zoom-in over a span of the edited timeline (virtual seconds). depth is an ORDINAL 1–6, not a factor: it selects a magnification from a fixed table (${ZOOM_DEPTH_LEGEND}), so the default depth 3 renders at 1.80×. The result reports renderedScale — quote that, never the depth, when telling the user how strong the zoom is. focus is the zoom centre in 0–1 fractions of the frame (default centre). When the recording's pointer telemetry can be read for the footage under the span, the result also carries \`cursorAnchor\`: \`focus\` echoes the value this call used (including the default, if you left it out), \`cursor\` is where the pointer ACTUALLY was over the span the zoom landed on — the median of the recorded samples, \`spread\` being how far the farthest one strays from it — and \`offset\` is the distance between the two, in frame fractions. It is a measurement, not a correction: nothing is moved and no call is refused over it, and a zoom framing a slide, a face, or a region the pointer never enters is a legitimate choice. \`available:false\` names what it found instead (\`no-samples\`, \`trimmed-out\`). Its ABSENCE means no telemetry was read for that footage — never that the recording has none; assets[].hasCursorTelemetry and getCursorTrack are what answer that. Use for 'zoom in on …' and the smart-zoom pass. ${ZOOM_CAMERA_NOTE} ${ZOOM_TRANSITIONS_NOTE} ${CUT_TRANSITIONS_RESULT}`, + addZooms: `Add MANY zooms in one call: \`regions\` is a list, each entry taking exactly the fields addZoom takes (same depth table, ${ZOOM_DEPTH_LEGEND}). Use this for the smart-zoom pass, where you have decided every zoom before emitting the first one — sending them one at a time costs one round trip each. Each region stands or falls ALONE: one that covers no clip is refused by itself and listed in \`refused\` with its index and the reason, while the others are still applied. The result leads with requested / appliedCount / refusedCount, and each applied entry carries its renderedScale — quote that, never the depth — plus the same \`cursorAnchor\` addZoom reports, whenever the footage under that region has readable pointer telemetry, and the same \`cutTransitions\` warning. ${ZOOM_TRANSITIONS_NOTE}`, + setZoom: `Move, resize, or restyle an existing zoom by id (virtual-timeline seconds). Only the fields you pass are changed. depth selects from the same table (${ZOOM_DEPTH_LEGEND}); if the zoom carries a customScale (getCurrentDocument shows it as depthIsOverridden), that custom value is what renders, and passing depth clears it so the depth takes effect — the result says so. The result reports the resulting renderedScale, and — when the footage under the span has readable pointer telemetry — the same \`cursorAnchor\` addZoom reports, measured against the zoom's EFFECTIVE focus, so a call that moved only the span still learns what its unchanged focus is now looking at. ${ZOOM_CAMERA_NOTE} ${ZOOM_TRANSITIONS_NOTE} ${CUT_TRANSITIONS_RESULT}`, addSpeed: "Add a speed-change region over a span of the edited timeline (virtual seconds). speed > 1 fast-forwards, < 1 slows down, from 0.25× to 16× (default 1.5×). Use to speed through slow stretches without cutting them.", setSpeed: "Move, resize, or change the multiplier of an existing speed region by id (virtual-timeline seconds). Only the fields you pass are changed.", addAnnotation: - "Add a text annotation over a span of the edited timeline (virtual seconds). x/y place the text's centre, in frame percentages (0–100, default the frame's centre); the box stays inside the frame. It is drawn on a dark plate so it reads on any footage. Use for callouts and labels.", + "Add a text annotation over a span of the edited timeline (virtual seconds). It sits on the output frame, the whole picture padding included, as text added in the editor does: zooms and padding never move it. x/y place the text's centre in frame percentages (0–100, default the centre); the box is sized to the text and kept inside the frame. By default it is drawn on a dark plate so it reads on any footage; fontSize, color, background, bold and animation set its look. Use for callouts, labels and titles.", setAnnotation: - "Move, resize, or edit the text of an existing annotation by id (virtual-timeline seconds). Only the fields you pass are changed.", + "Change an existing annotation by id: its span (virtual-timeline seconds), its text, its centre (x/y, frame percentages) or its look (fontSize, color, background, bold, animation). Only the fields you pass are changed; a text box is refitted to its text around its centre. A new span that would run into another annotation is refused, naming it.", + setAspectRatio: + "Set the shape of the exported video: 16:9 for X, YouTube or a slide, 9:16 for a vertical story or short, 1:1 or 4:5 for a square feed, auto to follow the recording. Annotations on the frame keep their place and shape.", addCameraFullscreen: "Add a camera-fullscreen region over a span of the edited timeline (virtual seconds): the webcam fills the frame for that span. This only does something when the footage under that span comes from an asset with a linked webcam — check assets[].hasCameraTrack (or hasAnyCamera) in getCurrentDocument first. On footage with no camera the call is refused rather than storing a region that would render nothing; say so instead of retrying.", setCameraFullscreen: @@ -218,7 +235,7 @@ export interface CursorTelemetryReader { probe?(input: { assetId: string; originalPath: string | null }): Promise; } -interface ToolRuntime { +export interface ToolRuntime { cursor?: CursorTelemetryReader; availableByAssetId?: Record; } @@ -268,18 +285,7 @@ function documentTool( return tool( async (args: z.infer) => { sink.toolStart(name, args); - // ponytail: the ONE async step the pure executor cannot take. Reading a - // sidecar is IO; `executeAgentTool` is synchronous by design (it is the - // gate every mutation passes through, and it has to stay testable - // without a filesystem). So the load happens here and its verdict — - // including "I could not look" — goes in as data. - const load = TOOLS_READING_CURSOR.has(name) - ? await loadCursorTelemetry(holder.current, args, runtime) - : undefined; - const execution = executeAgentTool(holder.current, name, JSON.stringify(args), { - editsAllowed, - cursorTelemetry: { availableByAssetId: runtime.availableByAssetId, load }, - }); + const execution = await runDocumentTool(holder.current, name, args, editsAllowed, runtime); if (execution.document) holder.current = execution.document; sink.toolEnd(name, execution.ok, execution.summary); return execution.resultJson; @@ -288,6 +294,33 @@ function documentTool( ); } +/** + * Run one tool against a document: the cursor read it may need, then the shared + * executor. The in-app agent (`documentTool`) and the MCP server both call this, + * so a tool behaves the same whichever agent is driving it. + * + * ponytail: the ONE async step the pure executor cannot take. Reading a sidecar + * is IO; `executeAgentTool` is synchronous by design (it is the gate every + * mutation passes through, and it has to stay testable without a filesystem). + * So the load happens here and its verdict — including "I could not look" — goes + * in as data. + */ +export async function runDocumentTool( + document: AxcutDocument, + name: string, + args: unknown, + editsAllowed: boolean, + runtime: ToolRuntime, +): Promise { + const load = TOOLS_READING_CURSOR.has(name) + ? await loadCursorTelemetry(document, args, runtime) + : undefined; + return executeAgentTool(document, name, JSON.stringify(args ?? {}), { + editsAllowed, + cursorTelemetry: { availableByAssetId: runtime.availableByAssetId, load }, + }); +} + /** Reads the sidecar for whichever asset the call names, defaulting to the * primary one — the same resolution the executor will use to report it. */ async function loadCursorTelemetry( @@ -331,37 +364,46 @@ export function buildTools( editsAllowed = true, runtime: ToolRuntime = {}, ) { - const build = (name: string, schema: S) => - documentTool(holder, sink, name, schema, editsAllowed, runtime); - return [ - build("getCurrentDocument", z.object({})), - build("getTranscript", getTranscriptArgs), - build("getTranscriptWords", getTranscriptWordsArgs), - build("getCursorTrack", getCursorTrackArgs), - build("setWordText", setWordTextArgs), - build("addTrim", addTrimArgs), - build("addTrims", addTrimsArgs), - build("setTrim", setTrimArgs), - build("setClipRange", setClipRangeArgs), - build("moveClip", moveClipArgs), - build("replaceTimeline", replaceTimelineArgs), - build("addZoom", addZoomArgs), - build("addZooms", addZoomsArgs), - build("setZoom", setZoomArgs), - build("addSpeed", addSpeedArgs), - build("setSpeed", setSpeedArgs), - build("addAnnotation", addAnnotationArgs), - build("setAnnotation", setAnnotationArgs), - build("addCameraFullscreen", addCameraFullscreenArgs), - build("setCameraFullscreen", setCameraFullscreenArgs), - build("addAudio", addAudioArgs), - build("setAudio", setAudioArgs), - build("removeTrim", removeTrimArgs), - build("removeModifier", removeModifierArgs), - build("removeClip", removeClipArgs), - ]; + return TOOL_ARG_SCHEMAS.map(([name, schema]) => + documentTool(holder, sink, name, schema, editsAllowed, runtime), + ); } +/** + * Every tool's name and argument schema, in the order the model is handed them. + * `buildTools` wraps these for the in-app agent and `electron/mcp/` exposes the + * same list over MCP, so the two surfaces cannot drift apart. + */ +export const TOOL_ARG_SCHEMAS: ReadonlyArray = [ + ["getCurrentDocument", z.object({})], + ["getTranscript", getTranscriptArgs], + ["getTranscriptWords", getTranscriptWordsArgs], + ["getCursorTrack", getCursorTrackArgs], + ["setWordText", setWordTextArgs], + ["removeWords", removeWordsArgs], + ["addTrim", addTrimArgs], + ["addTrims", addTrimsArgs], + ["setTrim", setTrimArgs], + ["setClipRange", setClipRangeArgs], + ["moveClip", moveClipArgs], + ["replaceTimeline", replaceTimelineArgs], + ["addZoom", addZoomArgs], + ["addZooms", addZoomsArgs], + ["setZoom", setZoomArgs], + ["addSpeed", addSpeedArgs], + ["setSpeed", setSpeedArgs], + ["addAnnotation", addAnnotationArgs], + ["setAnnotation", setAnnotationArgs], + ["addCameraFullscreen", addCameraFullscreenArgs], + ["setCameraFullscreen", setCameraFullscreenArgs], + ["addAudio", addAudioArgs], + ["setAudio", setAudioArgs], + ["removeTrim", removeTrimArgs], + ["removeModifier", removeModifierArgs], + ["removeClip", removeClipArgs], + ["setAspectRatio", setAspectRatioArgs], +]; + /** * Prompt caching for the Anthropic-wire providers, which `createDeepAgent` * used to add for us (`isAnthropicModel` → `anthropicPromptCachingMiddleware`). @@ -401,7 +443,7 @@ export interface InvokeArgs { /** One cheap probe per asset, run before the tools are built so the very first * `getCurrentDocument` can already say whether telemetry exists. */ -async function probeCursorTelemetry( +export async function probeCursorTelemetry( document: AxcutDocument, cursor: CursorTelemetryReader | undefined, ): Promise | undefined> { diff --git a/electron/ai-edition/document-service.test.ts b/electron/ai-edition/document-service.test.ts index 3a2e6d0ac4..526a5a12da 100644 --- a/electron/ai-edition/document-service.test.ts +++ b/electron/ai-edition/document-service.test.ts @@ -799,5 +799,98 @@ describe("DocumentService", () => { expect((await service.getProject(doc.project.id)).annotations).toHaveLength(7); }); + + it("makes a read wait for a read asked for before it", async () => { + const doc = await service.createProject("Reads in order"); + const order: string[] = []; + await Promise.all([ + service.getProject(doc.project.id).then(() => order.push("first")), + service.getProject(doc.project.id).then(() => order.push("second")), + ]); + expect(order).toEqual(["first", "second"]); + }); + + it("makes a read wait for a save already queued", async () => { + const doc = await service.createProject("Read after write"); + const saving = service.saveProject({ ...doc, project: { ...doc.project, title: "Saved" } }); + expect((await service.getProject(doc.project.id)).project.title).toBe("Saved"); + await saving; + }); + }); + + // The MCP server saves projects the editor does not hold. If the editor opens one + // between that read and that save, it loads the file without the save and its own + // next save drops it without a word. + describe("saveProjectIfUnchanged", () => { + const renamed = (doc: AxcutDocument, title: string): AxcutDocument => ({ + ...doc, + project: { ...doc.project, title }, + }); + + it("saves when nothing touched the project since the read", async () => { + const created = await service.createProject("Untouched"); + const { document, version } = await service.getProjectForUpdate(created.project.id); + const saved = await service.saveProjectIfUnchanged(renamed(document, "Edited"), version); + expect(saved?.project.title).toBe("Edited"); + expect((await service.getProject(created.project.id)).project.title).toBe("Edited"); + }); + + it("writes nothing when the project was read since, as the editor opening it does", async () => { + const created = await service.createProject("Opened"); + const { document, version } = await service.getProjectForUpdate(created.project.id); + await service.getProject(created.project.id); + expect(await service.saveProjectIfUnchanged(renamed(document, "Edited"), version)).toBeNull(); + expect((await service.getProject(created.project.id)).project.title).toBe("Opened"); + }); + + it("writes nothing when a read arrives while it is queued, and that read sees the file as it is", async () => { + const created = await service.createProject("Racing"); + const { document, version } = await service.getProjectForUpdate(created.project.id); + // Unawaited: the editor's open lands while the save waits for its turn. + const saving = service.saveProjectIfUnchanged(renamed(document, "Edited"), version); + const opened = await service.getProject(created.project.id); + expect(await saving).toBeNull(); + expect(opened.project.title).toBe("Racing"); + }); + + it("writes nothing when the project was saved since", async () => { + const created = await service.createProject("Saved"); + const { document, version } = await service.getProjectForUpdate(created.project.id); + await service.saveProject(renamed(document, "Elsewhere")); + expect(await service.saveProjectIfUnchanged(renamed(document, "Edited"), version)).toBeNull(); + expect((await service.getProject(created.project.id)).project.title).toBe("Elsewhere"); + }); + + it("writes nothing when the file changed outside the app, updatedAt and all", async () => { + const created = await service.createProject("External"); + const { document, version } = await service.getProjectForUpdate(created.project.id); + const file = path.join(tempDir, `${created.project.id}.openscreen`); + await fs.writeFile(file, JSON.stringify(renamed(document, "Synced")), "utf8"); + expect(await service.saveProjectIfUnchanged(renamed(document, "Edited"), version)).toBeNull(); + expect((await service.getProject(created.project.id)).project.title).toBe("Synced"); + }); + + it("writes nothing over a project deleted since", async () => { + const created = await service.createProject("Deleted"); + const { document, version } = await service.getProjectForUpdate(created.project.id); + await service.deleteProject(created.project.id); + expect(await service.saveProjectIfUnchanged(renamed(document, "Edited"), version)).toBeNull(); + await expect(service.getProject(created.project.id)).rejects.toThrow(DocumentNotFoundError); + }); + }); + + describe("storedProjectId", () => { + it("names the project a file in the projects folder is the own file of", () => { + expect(service.storedProjectId(path.join(tempDir, "proj_a.openscreen"))).toBe("proj_a"); + expect(service.storedProjectId(path.join(tempDir, "proj_a.axcut"))).toBe("proj_a"); + }); + + it("names nothing for a file anywhere else", () => { + expect(service.storedProjectId(path.join(mediaDir, "proj_a.openscreen"))).toBeUndefined(); + expect( + service.storedProjectId(path.join(tempDir, "sub", "proj_a.openscreen")), + ).toBeUndefined(); + expect(service.storedProjectId(path.join(tempDir, "proj_a.json"))).toBeUndefined(); + }); }); }); diff --git a/electron/ai-edition/document-service.ts b/electron/ai-edition/document-service.ts index 6314bd0bff..c1e0ea20b5 100644 --- a/electron/ai-edition/document-service.ts +++ b/electron/ai-edition/document-service.ts @@ -41,6 +41,17 @@ export interface ProjectSummary { assetCount: number; } +function stampForSave(document: AxcutDocument): AxcutDocument { + const parsed = documentSchema.parse(document); + return { ...parsed, project: { ...parsed.project, updatedAt: new Date().toISOString() } }; +} + +/** What `saveProjectIfUnchanged` checks against: the read that returned it. */ +export interface ProjectVersion { + readonly generation: number; + readonly raw: string; +} + export interface AddAssetInput { path: string; label?: string; @@ -138,7 +149,7 @@ function parseLoadedDocument(raw: string): AxcutDocument { * than surface a save error the user cannot act on. POSIX renames don't hit this * and take the first attempt. */ -async function renameWithRetry(from: string, to: string): Promise { +export async function renameWithRetry(from: string, to: string): Promise { const RETRYABLE = new Set(["EPERM", "EACCES", "EBUSY"]); for (let attempt = 0; ; attempt++) { try { @@ -156,8 +167,10 @@ export class DocumentService { private readonly projectsRoot: string; private readonly mediaRegistryDir: string; private legacyMigrationDone = false; - /** Tail of the in-flight save chain per project id — see writeProject. */ + /** Tail of the in-flight read and save chain per project id — see writeProject. */ private readonly writeQueues = new Map>(); + /** Bumped as each read or save of a project starts — see saveProjectIfUnchanged. */ + private readonly generations = new Map(); // `mediaRegistryDir` is where the media-links registry file lives // (RECORDINGS_DIR in production) — see getProject. Injected for the same @@ -230,6 +243,11 @@ export class DocumentService { return path.join(this.projectsRoot, `${safe}${PROJECT_FILE_EXTENSION}`); } + /** The project's own file, for a reader that takes a path (the export runner). */ + filePathFor(projectId: string): string { + return this.fileFor(projectId); + } + private legacyFileFor(projectId: string): string { const safe = safeProjectId(projectId); return path.join(this.projectsRoot, `${safe}${LEGACY_PROJECT_FILE_EXTENSION}`); @@ -286,30 +304,24 @@ export class DocumentService { } async getProject(projectId: string): Promise { - // Prefer the canonical `.openscreen` file, falling back to a not-yet-migrated - // legacy `.axcut` so a project opened before its migration pass still loads. - let raw: string; - try { - raw = await fs.readFile(this.fileFor(projectId), "utf8"); - } catch (error) { - if ((error as NodeJS.ErrnoException)?.code !== "ENOENT") { - throw new ProjectFileError( - `Failed to read project ${projectId}: ${error instanceof Error ? error.message : String(error)}`, - projectId, - ); - } - try { - raw = await fs.readFile(this.legacyFileFor(projectId), "utf8"); - } catch (legacyError) { - if ((legacyError as NodeJS.ErrnoException)?.code === "ENOENT") { - throw new DocumentNotFoundError(projectId); - } - throw new ProjectFileError( - `Failed to read project ${projectId}: ${legacyError instanceof Error ? legacyError.message : String(legacyError)}`, - projectId, - ); - } - } + return (await this.getProjectForUpdate(projectId)).document; + } + + /** `getProject`, plus the version to hand `saveProjectIfUnchanged`. */ + async getProjectForUpdate( + projectId: string, + ): Promise<{ document: AxcutDocument; version: ProjectVersion }> { + const generation = this.touch(projectId); + // In the project's queue: behind its saves, so a read never returns the file one + // is about to replace, and behind its other reads, so a read asked for earlier + // has returned first. saveProjectIfUnchanged and the MCP server rely on both. + return this.enqueue(projectId, async () => { + const raw = await this.readProjectFile(projectId); + return { document: await this.parseProjectFile(raw), version: { generation, raw } }; + }); + } + + private async parseProjectFile(raw: string): Promise { // Relink here rather than in the .openscreen import handlers, because this // is the one place every open funnels through — the project picker, the // agent, and the auto-load-last-project effect on launch. A document whose @@ -328,6 +340,54 @@ export class DocumentService { return document; } + /** + * The id of the project `filePath` is the own file of, or undefined for any other + * file. The open-file dialog hands it to the editor, which then loads that project + * instead of saving back the file as read, over any change made since. + */ + storedProjectId(filePath: string): string | undefined { + const resolved = path.resolve(filePath); + if (path.relative(path.resolve(this.projectsRoot), path.dirname(resolved)) !== "") { + return undefined; + } + const extension = [PROJECT_FILE_EXTENSION, LEGACY_PROJECT_FILE_EXTENSION].find((ext) => + resolved.endsWith(ext), + ); + return extension ? path.basename(resolved, extension) : undefined; + } + + private touch(projectId: string): number { + const generation = (this.generations.get(projectId) ?? 0) + 1; + this.generations.set(projectId, generation); + return generation; + } + + private async readProjectFile(projectId: string): Promise { + // Prefer the canonical `.openscreen` file, falling back to a not-yet-migrated + // legacy `.axcut` so a project opened before its migration pass still loads. + try { + return await fs.readFile(this.fileFor(projectId), "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException)?.code !== "ENOENT") { + throw new ProjectFileError( + `Failed to read project ${projectId}: ${error instanceof Error ? error.message : String(error)}`, + projectId, + ); + } + } + try { + return await fs.readFile(this.legacyFileFor(projectId), "utf8"); + } catch (legacyError) { + if ((legacyError as NodeJS.ErrnoException)?.code === "ENOENT") { + throw new DocumentNotFoundError(projectId); + } + throw new ProjectFileError( + `Failed to read project ${projectId}: ${legacyError instanceof Error ? legacyError.message : String(legacyError)}`, + projectId, + ); + } + } + async createProject(title: string): Promise { await this.ensureProjectsDir(); const projectId = createId("proj"); @@ -342,11 +402,8 @@ export class DocumentService { } async saveProject(document: AxcutDocument): Promise { - const parsed = documentSchema.parse(document); - const stamped: AxcutDocument = { - ...parsed, - project: { ...parsed.project, updatedAt: new Date().toISOString() }, - }; + const stamped = stampForSave(document); + this.touch(stamped.project.id); await this.writeProject(stamped); // After the write, never before: a derived file is not worth delaying the user's edit // reaching disk, and a failure to generate one must not fail the save. @@ -354,6 +411,35 @@ export class DocumentService { return stamped; } + /** + * `saveProject` for a writer that does not hold the project open (the MCP + * server), unless anything read or saved the project since the + * `getProjectForUpdate` that returned `version`, or its file changed: then + * nothing is written and this resolves null. + * + * The read is the case that matters: an editor that opened the project after + * `version` holds a copy without this save, and its own next save would drop + * it without a word. The check runs inside the project's write queue, and reads + * wait behind that queue, so nothing in this process reads or saves the project + * between the check and the write. + */ + async saveProjectIfUnchanged( + document: AxcutDocument, + version: ProjectVersion, + ): Promise { + const stamped = stampForSave(document); + const projectId = stamped.project.id; + const written = await this.writeProject( + stamped, + async () => + this.generations.get(projectId) === version.generation && + (await this.readProjectFile(projectId).catch(() => null)) === version.raw, + ); + if (!written) return null; + await ensureDocumentExtensions(stamped); + return stamped; + } + async deleteProject(projectId: string): Promise { // Remove the canonical file and any lingering legacy `.axcut` for this id. for (const filePath of [this.fileFor(projectId), this.legacyFileFor(projectId)]) { @@ -476,15 +562,26 @@ export class DocumentService { * The queue alone would still leave a torn file on a crash; the rename alone * would still let two saves race for the same destination. */ - private writeProject(doc: AxcutDocument): Promise { - const projectId = doc.project.id; + private writeProject( + doc: AxcutDocument, + precondition?: () => Promise, + ): Promise { + return this.enqueue(doc.project.id, async () => { + if (precondition && !(await precondition())) return false; + await this.writeProjectNow(doc); + return true; + }); + } + + /** Runs `step` once every read and save of the project asked for before it is done. */ + private enqueue(projectId: string, step: () => Promise): Promise { const tail = this.writeQueues.get(projectId) ?? Promise.resolve(); // Chained on both settlements: one save failing must not cancel the next. - const run = tail.then( - () => this.writeProjectNow(doc), - () => this.writeProjectNow(doc), + const run = tail.then(step, step); + const settled = run.then( + () => undefined, + () => undefined, ); - const settled = run.catch(() => undefined); this.writeQueues.set(projectId, settled); void settled.then(() => { // Only the tail clears the entry — a later save may already own it. diff --git a/electron/ai-edition/style-preset-service.test.ts b/electron/ai-edition/style-preset-service.test.ts index 0f2e430f7e..60c9025dac 100644 --- a/electron/ai-edition/style-preset-service.test.ts +++ b/electron/ai-edition/style-preset-service.test.ts @@ -34,6 +34,8 @@ const APPEARANCE: StylePresetAppearance = { model3d: false, asArrow: [], clickImpact: false, + clickSound: false, + clickSoundGainDb: 0, }, cursorShow: true, cursorAutoHide: false, diff --git a/electron/app-settings.test.ts b/electron/app-settings.test.ts index c6c9e74f3c..f3781734f8 100644 --- a/electron/app-settings.test.ts +++ b/electron/app-settings.test.ts @@ -129,4 +129,40 @@ describe("app settings store", () => { store.dismissStarPrompt(); expect(store.getSnapshot().recording).toMatchObject({ micEnabled: true, micDeviceId: "mic" }); }); + + it("stores a camera quality and rejects one it cannot capture at", () => { + const dir = temp(); + const store = new AppSettingsStore(dir); + + expect(store.setRecordingPreferences({ camQuality: "1080p" }).recording.camQuality).toBe( + "1080p", + ); + expect(() => store.setRecordingPreferences({ camQuality: "4320p" as never })).toThrow( + TypeError, + ); + expect(store.getSnapshot().recording.camQuality).toBe("1080p"); + }); + + it("leaves the camera quality alone when a patch does not mention it", () => { + // Every other window writes narrow patches; a camera toggle from the HUD + // must not quietly reset the resolution the user picked. + const dir = temp(); + const store = new AppSettingsStore(dir); + store.setRecordingPreferences({ camQuality: "1440p" }); + + store.setRecordingPreferences({ camEnabled: true }); + + expect(store.getSnapshot().recording.camQuality).toBe("1440p"); + }); + + it("reads a settings file written before the camera had a quality", () => { + const dir = temp(); + writeFileSync( + path.join(dir, "recording-settings.json"), + JSON.stringify({ micEnabled: true }), + "utf8", + ); + + expect(new AppSettingsStore(dir).getSnapshot().recording.camQuality).toBe("2160p"); + }); }); diff --git a/electron/app-settings.ts b/electron/app-settings.ts index 2cc1cee258..bef025bf3a 100644 --- a/electron/app-settings.ts +++ b/electron/app-settings.ts @@ -1,5 +1,11 @@ import { readFileSync, renameSync, rmSync, writeFileSync } from "node:fs"; import path from "node:path"; +import { + DEFAULT_WEBCAM_QUALITY, + WEBCAM_QUALITY_IDS, + type WebcamQualityId, + webcamQualityFrom, +} from "../src/hooks/webcamCaptureTarget"; import type { CursorCaptureMode } from "../src/lib/recordingSession"; export interface RecordingPreferences { @@ -9,6 +15,8 @@ export interface RecordingPreferences { camEnabled: boolean; camDeviceId: string | null; camDeviceName: string | null; + /** Capture resolution for the camera. See WEBCAM_QUALITY_PRESETS. */ + camQuality: WebcamQualityId; systemAudioEnabled: boolean; cursorCaptureMode: CursorCaptureMode; /** Display captures on macOS and Windows. Opt-in: on Windows the icons also leave the real desktop while recording. */ @@ -30,6 +38,7 @@ export const DEFAULT_RECORDING_PREFERENCES: RecordingPreferences = { camEnabled: false, camDeviceId: null, camDeviceName: null, + camQuality: DEFAULT_WEBCAM_QUALITY, systemAudioEnabled: false, cursorCaptureMode: "editable-overlay", hideDesktopIcons: false, @@ -100,6 +109,9 @@ function parseRecording(raw: RawSettings): RecordingPreferences { camEnabled: bool(raw.camEnabled, DEFAULT_RECORDING_PREFERENCES.camEnabled), camDeviceId: nullableString(raw.camDeviceId, DEFAULT_RECORDING_PREFERENCES.camDeviceId), camDeviceName: nullableString(raw.camDeviceName, DEFAULT_RECORDING_PREFERENCES.camDeviceName), + // Unset in every settings file written before the camera had a quality + // setting, and `webcamQualityFrom` answers those with the default. + camQuality: webcamQualityFrom(raw.camQuality), systemAudioEnabled: bool( raw.systemAudioEnabled, DEFAULT_RECORDING_PREFERENCES.systemAudioEnabled, @@ -167,6 +179,12 @@ function validateRecordingPatch(patch: Partial): void { if (key === "cursorCaptureMode" && value !== "system" && value !== "editable-overlay") { throw new TypeError("cursorCaptureMode is invalid"); } + // Rejected here rather than coerced on read, so a bad write is a visible + // error at its source instead of a resolution that silently is not the + // one the caller asked for. + if (key === "camQuality" && !WEBCAM_QUALITY_IDS.includes(value as WebcamQualityId)) { + throw new TypeError("camQuality is invalid"); + } } } diff --git a/electron/cli/args.test.ts b/electron/cli/args.test.ts index ddf012321b..dd4a373cc5 100644 --- a/electron/cli/args.test.ts +++ b/electron/cli/args.test.ts @@ -230,6 +230,11 @@ describe("parseCliArgs", () => { }); }); + it("parses mcp, which takes no argument", () => { + expect(parse(["mcp"])).toEqual({ kind: "mcp" }); + expect(parse(["mcp", "--port", "1"])).toMatchObject({ kind: "error" }); + }); + it("parses info and help", () => { expect(parse(["info", "demo.openscreen", "--json"])).toMatchObject({ kind: "info", diff --git a/electron/cli/args.ts b/electron/cli/args.ts index 23926da479..9c6532d9d7 100644 --- a/electron/cli/args.ts +++ b/electron/cli/args.ts @@ -22,10 +22,16 @@ export interface CliErrorCommand { message: string; } +/** Serve the MCP tools on stdin/stdout, to the agent that started this process. */ +export interface CliMcpCommand { + kind: "mcp"; +} + export type CliCommand = ( | CliRequest | CliInfoCommand | CliPackCommand + | CliMcpCommand | CliHelpCommand | CliErrorCommand ) & { @@ -60,6 +66,7 @@ const SUBCOMMANDS = new Set([ "pack", "captions", "info", + "mcp", "help", "--help", "-h", @@ -75,6 +82,7 @@ Usage: openscreen captions Add auto-captions (on-device Whisper) to a project [--min-words <1-12>] [--max-words <1-12>] openscreen info [--json] Inspect a project file + openscreen mcp Serve the agent tools over MCP on stdio openscreen help Show this help Export options: @@ -165,6 +173,10 @@ export function parseCliArgs( if (sub === "sources") return parseSources(args.slice(1), cwd); if (sub === "pack") return parsePack(args.slice(1), cwd); if (sub === "captions") return parseCaptions(args.slice(1), cwd); + if (sub === "mcp") { + if (args.length > 1) throw new Error(`Unexpected argument for mcp: ${args[1]}`); + return { kind: "mcp" }; + } return parseInfo(args.slice(1), cwd); } catch (error) { return { kind: "error", message: error instanceof Error ? error.message : String(error) }; diff --git a/electron/cli/cliMain.ts b/electron/cli/cliMain.ts index a1860cafb2..1aeffd0793 100644 --- a/electron/cli/cliMain.ts +++ b/electron/cli/cliMain.ts @@ -7,18 +7,17 @@ import path from "node:path"; import readline from "node:readline"; import { fileURLToPath } from "node:url"; import { app, BrowserWindow, ipcMain, session, systemPreferences } from "electron"; -import type { - CliDoneResult, - CliProgressEvent, - CliRequest, - CliSourcesResult, -} from "../../src/lib/cliContracts"; +import type { CliProgressEvent, CliSourcesResult } from "../../src/lib/cliContracts"; import { isDiagnosticModeEnabled } from "../diagnostics/main-log-buffer"; import { getSelectedDesktopSource, registerIpcHandlers } from "../ipc/handlers"; +import { openVideoInHiddenWindow } from "../mcp/hidden-video"; +import { serveMcpOverStdio } from "../mcp/stdio-server"; import { registerSttIpc } from "../stt"; import { ASSET_BASE_URL_ARG } from "../windows"; import { CLI_USAGE, type CliCommand } from "./args"; import { runInfoCommand, runPackCommand } from "./projectCommands"; +import { createRunnerHost } from "./runnerHost"; +import { openStdin } from "./stdin"; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const VITE_DEV_SERVER_URL = process.env["VITE_DEV_SERVER_URL"]; @@ -157,7 +156,7 @@ function registerAppHandlersForCli(cliWindowRef: () => BrowserWindow | null) { const notAvailable = () => { throw new Error("Window not available in CLI mode"); }; - registerIpcHandlers( + return registerIpcHandlers( noop, // createEditorWindow: recording finished; runner drives completion notAvailable, // createSourceSelectorWindow notAvailable, // createCountdownOverlayWindow @@ -213,7 +212,7 @@ function setupRecordStopSignals(stop: (reason: string) => void): void { try { // Touching process.stdin can throw on Windows GUI-subsystem builds when // spawned with stdio "ignore"; signals and --duration still work then. - const rl = readline.createInterface({ input: process.stdin }); + const rl = readline.createInterface({ input: openStdin() }); rl.on("line", (line) => { const trimmed = line.trim().toLowerCase(); if (trimmed === "stop" || trimmed === "q" || trimmed === "quit") { @@ -299,8 +298,8 @@ export function runCli(command: CliCommand): void { app.on("window-all-closed", () => { // Completion is signalled via cli-done; a vanished window is a failure - // only while the run is still in flight. - if (finished) return; + // only while the run is still in flight. The MCP server outlives its runners. + if (finished || command.kind === "mcp") return; output.error("Renderer window closed unexpectedly"); app.exit(1); }); @@ -330,8 +329,9 @@ export function runCli(command: CliCommand): void { milestone("resolving userData path"); const userDataDir = app.getPath("userData"); milestone(`userData = ${userDataDir}`); - await fs.mkdir(path.join(userDataDir, "recordings"), { recursive: true }); - milestone("recordings directory ready"); + // No recordings folder made here: it may be one the user chose, on a drive that is + // gone, and `export` has no use for it. A record run makes it in the storage check + // every take goes through, which reports such a folder instead of crashing the run. // Media/screen permissions for the renderer (mic metering, future browser // capture paths). Mirrors the GUI allowlist. @@ -381,7 +381,7 @@ export function runCli(command: CliCommand): void { milestone("media access checked"); let cliWindow: BrowserWindow | null = null; - registerAppHandlersForCli(() => cliWindow); + const { agentServices } = registerAppHandlersForCli(() => cliWindow); milestone("app handlers registered"); // Speech-to-text backs the captions command; registered by the GUI boot @@ -396,107 +396,24 @@ export function runCli(command: CliCommand): void { }); ipcMain.handle("update-global-shortcut", () => ({ success: false })); - const request: CliRequest = command; - ipcMain.handle("cli-get-request", () => { - milestone("renderer asked for the request"); - return request; + // The latest runner is the app's "main window" for the recording handlers, as the + // HUD is in the GUI. + const runners = createRunnerHost(ipcMain, (windowType): BrowserWindow => { + const runnerWindow = loadRunnerWindow(windowType); + cliWindow = runnerWindow; + return runnerWindow; }); - ipcMain.on("cli-log", (_event, level: string, message: string) => { - if (level === "error") { - output.error(message); - } else { - output.info(message); - output.event("log", { message }); - } - }); - ipcMain.on("cli-progress", (_event, progress: CliProgressEvent) => { - output.progress(progress); - }); - - ipcMain.handle("cli-done", async (_event, result: CliDoneResult) => { - milestone(`renderer reported done (success=${result.success})`); - if (finished) return; - finished = true; - - try { - if (result.success && command.kind === "record" && command.projectOut) { - if (result.projectData !== undefined) { - await writeProjectFile(command.projectOut, result.projectData); - result.projectPath = command.projectOut; - } - } - if (result.success && command.kind === "captions" && result.projectData !== undefined) { - await writeProjectFile(command.projectPath, result.projectData); - } - // -o writes where no wrapper can redirect. stdout itself is fine -- - // Chromium logs to stderr -- but a caller does not control what wraps - // this process, and xvfb-run on Ubuntu merges the two, which is enough - // to make `sources --json | jq` fail under the very tool used to run a - // GUI binary headlessly. - if ( - result.success && - command.kind === "sources" && - command.jsonOutPath && - result.sources - ) { - // Same as writeProjectFile above: `record --project out/x` creates - // `out/`, so `sources -o out/x.json` has to as well. Without it a - // fully successful enumeration is lost on every channel at once -- - // the ENOENT lands in the catch below, which clears result.success - // and so skips printSources and the `done` payload alike. - await fs.mkdir(path.dirname(command.jsonOutPath), { recursive: true }); - // Write beside the target and rename over it. writeFile truncates - // first, so a failure part-way through -- a full disk is the easy - // case -- would leave a half-written file where website/docs/cli.md promises - // an earlier run's result is untouched by a later failure. rename is - // atomic within a directory, so the reader sees the old file or the - // new one and never a torn one. - const tmpOutPath = `${command.jsonOutPath}.${process.pid}.tmp`; - try { - await fs.writeFile( - tmpOutPath, - `${JSON.stringify(result.sources, null, 2)}\n`, - "utf8", - ); - await fs.rename(tmpOutPath, command.jsonOutPath); - } catch (writeError) { - // Best-effort: the original error is what the caller needs to see. - await fs.rm(tmpOutPath, { force: true }).catch(() => undefined); - throw writeError; - } - } - } catch (error) { - result.success = false; - result.error = `Run succeeded but writing the output file failed: ${String(error)}`; - } - - if (result.success) { - for (const warning of result.warnings ?? []) { - output.info(`Warning: ${warning}`); - output.event("warning", { message: warning }); - } - if (command.kind === "sources" && result.sources) { - printSources(output, result.sources); - } else if (command.kind === "captions") { - output.info( - `Added ${result.captionCount ?? 0} caption annotation(s) → ${result.projectPath}`, - ); - } else if (command.kind === "export") { - output.info(`Exported ${result.format ?? ""} → ${result.outputPath}`); - } else { - output.info(`Recording saved → ${result.screenVideoPath}`); - if (result.cursorDataPath) output.info(`Cursor data → ${result.cursorDataPath}`); - if (result.projectPath) output.info(`Project → ${result.projectPath}`); - } - output.event("done", { ...result }); - } else { - output.error(result.error ?? "Unknown failure"); - output.event("done", { success: false, error: result.error }); - } - // Give the renderer a beat to resolve the invoke before exiting. - setTimeout(() => app.exit(result.success ? 0 : 1), 50); - }); + if (command.kind === "mcp") { + await serveMcpOverStdio({ + ...agentServices, + runners, + openVideo: openVideoInHiddenWindow, + onClose: () => app.exit(0), + }); + milestone("mcp server listening on stdio"); + return; + } if (command.kind === "record") { const stop = (reason: string) => { @@ -507,39 +424,112 @@ export function runCli(command: CliCommand): void { setupRecordStopSignals(stop); } - milestone("cli ipc registered; resolving window type"); - const windowType = { - export: "cli-export", - record: "cli-record", - sources: "cli-sources", - captions: "cli-captions", - }[command.kind]; - cliWindow = loadRunnerWindow(windowType); - milestone(`runner window created (${windowType})`); - cliWindow.webContents.on("did-finish-load", () => milestone("renderer did-finish-load")); - cliWindow.webContents.on("dom-ready", () => milestone("renderer dom-ready")); + milestone("cli ipc registered; starting the runner"); + const run = runners.start(command, { + onRequest: () => milestone("renderer asked for the request"), + onLog: (level, message) => { + if (level === "error") { + output.error(message); + } else { + output.info(message); + output.event("log", { message }); + } + }, + onProgress: (progress) => output.progress(progress), + }); + const runnerWindow = run.window; + milestone(`runner window created (${command.kind})`); + runnerWindow.webContents.on("did-finish-load", () => milestone("renderer did-finish-load")); + runnerWindow.webContents.on("dom-ready", () => milestone("renderer dom-ready")); // Surface renderer console errors/warnings on stderr — the hidden window // has no other way to show what went wrong (toasts are invisible). - cliWindow.webContents.on("console-message", (details) => { + runnerWindow.webContents.on("console-message", (details) => { if (details.level === "error" || details.level === "warning") { safeWrite(process.stderr, `[renderer] ${details.message}\n`); } }); - cliWindow.webContents.on("did-fail-load", (_e, code, description) => { - output.error(`Failed to load runner window: ${description} (${code})`); - app.exit(1); - }); + // A crash or a failed load ends the run through `run.done`, reported below. app.on("child-process-gone", (_e, details) => { milestone(`child-process-gone: ${details.type} (${details.reason})`); }); - cliWindow.webContents.on("render-process-gone", (_e, details) => { - output.error(`Renderer crashed: ${details.reason}`); - app.exit(1); - }); output.event("started", { command: command.kind }); + + const result = await run.done; + milestone(`renderer reported done (success=${result.success})`); + finished = true; + + try { + if (result.success && command.kind === "record" && command.projectOut) { + if (result.projectData !== undefined) { + await writeProjectFile(command.projectOut, result.projectData); + result.projectPath = command.projectOut; + } + } + if (result.success && command.kind === "captions" && result.projectData !== undefined) { + await writeProjectFile(command.projectPath, result.projectData); + } + // -o writes where no wrapper can redirect. stdout itself is fine -- + // Chromium logs to stderr -- but a caller does not control what wraps + // this process, and xvfb-run on Ubuntu merges the two, which is enough + // to make `sources --json | jq` fail under the very tool used to run a + // GUI binary headlessly. + if (result.success && command.kind === "sources" && command.jsonOutPath && result.sources) { + // Same as writeProjectFile above: `record --project out/x` creates + // `out/`, so `sources -o out/x.json` has to as well. Without it a + // fully successful enumeration is lost on every channel at once -- + // the ENOENT lands in the catch below, which clears result.success + // and so skips printSources and the `done` payload alike. + await fs.mkdir(path.dirname(command.jsonOutPath), { recursive: true }); + // Write beside the target and rename over it. writeFile truncates + // first, so a failure part-way through -- a full disk is the easy + // case -- would leave a half-written file where website/docs/cli.md promises + // an earlier run's result is untouched by a later failure. rename is + // atomic within a directory, so the reader sees the old file or the + // new one and never a torn one. + const tmpOutPath = `${command.jsonOutPath}.${process.pid}.tmp`; + try { + await fs.writeFile(tmpOutPath, `${JSON.stringify(result.sources, null, 2)}\n`, "utf8"); + await fs.rename(tmpOutPath, command.jsonOutPath); + } catch (writeError) { + // Best-effort: the original error is what the caller needs to see. + await fs.rm(tmpOutPath, { force: true }).catch(() => undefined); + throw writeError; + } + } + } catch (error) { + result.success = false; + result.error = `Run succeeded but writing the output file failed: ${String(error)}`; + } + + if (result.success) { + for (const warning of result.warnings ?? []) { + output.info(`Warning: ${warning}`); + output.event("warning", { message: warning }); + } + if (command.kind === "sources" && result.sources) { + printSources(output, result.sources); + } else if (command.kind === "captions") { + output.info( + `Added ${result.captionCount ?? 0} caption annotation(s) → ${result.projectPath}`, + ); + } else if (command.kind === "export") { + output.info(`Exported ${result.format ?? ""} → ${result.outputPath}`); + } else { + output.info(`Recording saved → ${result.screenVideoPath}`); + if (result.cursorDataPath) output.info(`Cursor data → ${result.cursorDataPath}`); + if (result.projectPath) output.info(`Project → ${result.projectPath}`); + } + output.event("done", { ...result }); + } else { + output.error(result.error ?? "Unknown failure"); + output.event("done", { success: false, error: result.error }); + } + + // Give the renderer a beat to resolve the invoke before exiting. + setTimeout(() => app.exit(result.success ? 0 : 1), 50); }) .catch((error) => { output.error(error instanceof Error ? (error.stack ?? error.message) : String(error)); diff --git a/electron/cli/runnerHost.test.ts b/electron/cli/runnerHost.test.ts new file mode 100644 index 0000000000..3f0abde775 --- /dev/null +++ b/electron/cli/runnerHost.test.ts @@ -0,0 +1,140 @@ +import { EventEmitter } from "node:events"; +import { describe, expect, it } from "vitest"; +import type { CliRequest } from "../../src/lib/cliContracts"; +import { createRunnerHost, type RunnerIpc } from "./runnerHost"; + +type Listener = (event: { sender: { id: number } }, ...args: unknown[]) => unknown; + +/** `ipcMain`, reduced to what a runner window can call on it. */ +class FakeIpc { + readonly listeners = new Map(); + handle(channel: string, listener: Listener) { + this.listeners.set(channel, listener); + } + on(channel: string, listener: Listener) { + this.listeners.set(channel, listener); + } + /** What the renderer of window `id` does when it calls `channel`. */ + call(id: number, channel: string, ...args: unknown[]) { + return this.listeners.get(channel)?.({ sender: { id } }, ...args); + } +} + +/** Whatever a runner window event passes its listener. */ +type WindowListener = (...args: never[]) => void; + +class FakeWindow { + readonly events = new EventEmitter(); + readonly webContents; + destroyed = false; + constructor(id: number) { + const listen = (listener: WindowListener) => listener as (...args: unknown[]) => void; + this.webContents = { + id, + once: (event: string, listener: WindowListener) => this.events.once(event, listen(listener)), + on: (event: string, listener: WindowListener) => this.events.on(event, listen(listener)), + }; + } + isDestroyed() { + return this.destroyed; + } + destroy() { + this.destroyed = true; + this.events.emit("destroyed"); + } +} + +function setup() { + const ipc = new FakeIpc(); + const opened: { windowType: string; window: FakeWindow }[] = []; + let nextId = 1; + const host = createRunnerHost(ipc as unknown as RunnerIpc, (windowType) => { + const window = new FakeWindow(nextId++); + opened.push({ windowType, window }); + return window; + }); + return { ipc, host, opened }; +} + +const sources: CliRequest = { kind: "sources" }; + +describe("createRunnerHost", () => { + it("hands each runner its own request and result, several at once", async () => { + const { ipc, host, opened } = setup(); + const first = host.start(sources); + const second = host.start({ + kind: "captions", + projectPath: "/p.openscreen", + minWordsPerCaption: 2, + maxWordsPerCaption: 7, + }); + expect(opened.map((o) => o.windowType)).toEqual(["cli-sources", "cli-captions"]); + + expect(ipc.call(2, "cli-get-request")).toMatchObject({ kind: "captions" }); + expect(ipc.call(1, "cli-get-request")).toEqual(sources); + + ipc.call(2, "cli-done", { success: true, captionCount: 3 }); + ipc.call(1, "cli-done", { success: false, error: "no display" }); + await expect(second.done).resolves.toEqual({ success: true, captionCount: 3 }); + await expect(first.done).resolves.toEqual({ success: false, error: "no display" }); + }); + + it("routes logs and progress to the run they came from", () => { + const { ipc, host } = setup(); + const logs: string[] = []; + const progress: number[] = []; + let asked = 0; + host.start(sources, { + onRequest: () => asked++, + onLog: (level, message) => logs.push(`${level}:${message}`), + onProgress: (p) => progress.push(p.percentage), + }); + host.start(sources, { onLog: () => logs.push("wrong run") }); + + ipc.call(1, "cli-get-request"); + ipc.call(1, "cli-log", "info", "Recording started"); + ipc.call(1, "cli-progress", { percentage: 40 }); + expect({ asked, logs, progress }).toEqual({ + asked: 1, + logs: ["info:Recording started"], + progress: [40], + }); + }); + + it("fails a run whose renderer crashes or whose page does not load, instead of hanging", async () => { + const { host, opened } = setup(); + const crashed = host.start(sources); + opened[0].window.events.emit("render-process-gone", {}, { reason: "oom" }); + await expect(crashed.done).resolves.toEqual({ + success: false, + error: "Renderer crashed: oom", + }); + + const unloaded = host.start(sources); + // A subframe failing is not the runner failing. + opened[1].window.events.emit("did-fail-load", {}, -3, "ABORTED", "x", false); + opened[1].window.events.emit("did-fail-load", {}, -6, "FILE_NOT_FOUND", "x", true); + await expect(unloaded.done).resolves.toMatchObject({ + success: false, + error: expect.stringContaining("FILE_NOT_FOUND (-6)"), + }); + }); + + it("closes a finished runner's window once, when asked", () => { + const { host, opened } = setup(); + const run = host.start(sources); + run.close(); + run.close(); + expect(opened[0].window.isDestroyed()).toBe(true); + }); + + it("fails a run whose window goes away, and ignores a second result", async () => { + const { ipc, host, opened } = setup(); + const run = host.start(sources); + opened[0].window.events.emit("destroyed"); + await expect(run.done).resolves.toMatchObject({ success: false }); + // Too late: the run is over, and nothing answers for an unknown window. + expect(ipc.call(1, "cli-done", { success: true })).toBeUndefined(); + expect(ipc.call(1, "cli-get-request")).toBeNull(); + }); +}); diff --git a/electron/cli/runnerHost.ts b/electron/cli/runnerHost.ts new file mode 100644 index 0000000000..7475aa2dc8 --- /dev/null +++ b/electron/cli/runnerHost.ts @@ -0,0 +1,129 @@ +// Runs the hidden renderer runners (record, export, sources, captions) and +// routes their IPC by window, so one process can run any number of them: the +// one-shot CLI runs one, the MCP server one per tool call. +// +// No `electron` import: the IPC surface and the window factory are handed in, +// so the routing stays unit-testable. + +import type { IpcMain } from "electron"; +import type { CliDoneResult, CliProgressEvent, CliRequest } from "../../src/lib/cliContracts"; + +/** The slice of a BrowserWindow this needs. */ +export interface RunnerWindow { + isDestroyed(): boolean; + destroy(): void; + webContents: { + id: number; + once(event: "destroyed", listener: () => void): unknown; + on( + event: "render-process-gone", + listener: (event: unknown, details: { reason: string }) => void, + ): unknown; + on( + event: "did-fail-load", + listener: ( + event: unknown, + errorCode: number, + errorDescription: string, + validatedURL: string, + isMainFrame: boolean, + ) => void, + ): unknown; + }; +} + +interface RunnerIpcEvent { + sender: { id: number }; +} + +/** The slice of `ipcMain` this needs. */ +export type RunnerIpc = Pick; + +export interface RunnerHandlers { + /** The renderer asked for its request: it booted. */ + onRequest?(): void; + onLog?(level: string, message: string): void; + onProgress?(progress: CliProgressEvent): void; +} + +export interface RunnerRun { + window: W; + /** The runner's own result. A window that closes, crashes or fails to load fails the run. */ + done: Promise; + /** Closes the runner's window: a long-lived caller does once it has the result. */ + close(): void; +} + +interface PendingRun { + request: CliRequest; + handlers: RunnerHandlers; + resolve(result: CliDoneResult): void; +} + +export const RUNNER_WINDOW_TYPES = { + export: "cli-export", + record: "cli-record", + sources: "cli-sources", + captions: "cli-captions", +} as const satisfies Record; + +export function createRunnerHost( + ipc: RunnerIpc, + openWindow: (windowType: string) => W, +) { + const runs = new Map(); + const runFor = (event: RunnerIpcEvent) => runs.get(event.sender.id); + + ipc.handle("cli-get-request", (event) => { + const run = runFor(event); + run?.handlers.onRequest?.(); + return run?.request ?? null; + }); + ipc.on("cli-log", (event, level: string, message: string) => { + runFor(event)?.handlers.onLog?.(level, message); + }); + ipc.on("cli-progress", (event, progress: CliProgressEvent) => { + runFor(event)?.handlers.onProgress?.(progress); + }); + ipc.handle("cli-done", (event, result: CliDoneResult) => { + const run = runFor(event); + if (!run) return; + runs.delete(event.sender.id); + run.resolve(result); + }); + + return { + start(request: CliRequest, handlers: RunnerHandlers = {}): RunnerRun { + const window = openWindow(RUNNER_WINDOW_TYPES[request.kind]); + const id = window.webContents.id; + // Registered in the same tick the load starts in: the renderer cannot ask + // for its request before this line has run. + const done = new Promise((resolve) => { + runs.set(id, { request, handlers, resolve }); + }); + // A runner that never reports would otherwise hold its caller forever. + const fail = (error: string) => { + const run = runs.get(id); + if (!run) return; + runs.delete(id); + run.resolve({ success: false, error }); + }; + window.webContents.once("destroyed", () => + fail("The runner window closed before it finished"), + ); + window.webContents.on("render-process-gone", (_event, details) => + fail(`Renderer crashed: ${details.reason}`), + ); + window.webContents.on("did-fail-load", (_event, code, description, _url, isMainFrame) => { + if (isMainFrame) fail(`Failed to load runner window: ${description} (${code})`); + }); + return { + window, + done, + close: () => { + if (!window.isDestroyed()) window.destroy(); + }, + }; + }, + }; +} diff --git a/electron/cli/stdin.test.ts b/electron/cli/stdin.test.ts new file mode 100644 index 0000000000..23e2479e27 --- /dev/null +++ b/electron/cli/stdin.test.ts @@ -0,0 +1,13 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { openStdin } from "./stdin"; + +describe("openStdin", () => { + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("keeps Node's own stdin outside Windows", () => { + vi.stubGlobal("process", { ...process, platform: "linux", stdin: "the-stdin" }); + expect(openStdin()).toBe("the-stdin"); + }); +}); diff --git a/electron/cli/stdin.ts b/electron/cli/stdin.ts new file mode 100644 index 0000000000..6e8d886b17 --- /dev/null +++ b/electron/cli/stdin.ts @@ -0,0 +1,16 @@ +// The process's real stdin. On Windows, Electron's main process gets a +// `process.stdin` that ends at once whatever the parent piped in (measured with +// Electron 41), so a pipe has to be read from descriptor 0 directly. + +import net from "node:net"; +import type { Readable } from "node:stream"; + +export function openStdin(): Readable { + if (process.platform !== "win32") return process.stdin; + try { + return new net.Socket({ fd: 0, readable: true, writable: false }); + } catch { + // Not a pipe (a console, or nothing at all): no one can write to us anyway. + return process.stdin; + } +} diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index 29d5f828b4..6ad0694cdd 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -37,6 +37,15 @@ interface Window { * `compositor.export`/`compositor.exportMulti` runs. Distinct from `exportOnFrameAck`, * the OLD web/CPU pipeline's per-frame ack, not a progress signal. */ onNativeExportProgress?: (callback: (frames: number, exportId?: string) => void) => () => void; + /** Preview frames the main process hands over as shared GPU textures (Windows). The + * listener draws `frame` and closes it; one listener at a time. Returns the + * unsubscribe. Optional: shim/web contexts have no bridge. */ + onCompositorFrame?: ( + listener: ( + frame: VideoFrame, + meta: import("../src/native/contracts").CompositorSharedFrameMeta, + ) => void, + ) => () => void; getSources: (opts: Electron.SourcesOptions) => Promise; switchToEditor: () => Promise; switchToHud: () => Promise; @@ -47,6 +56,8 @@ interface Window { }>; /** Sources are picked in Apple's system picker (macOS 15.2+), not in an app list. */ usesSystemSourcePicker?: () => Promise; + /** Name of the last source picked in Apple's picker, for display only: it is not selected. */ + getLastPickedSource?: () => Promise; openNotes: () => Promise<{ opened: boolean; reason?: string; @@ -71,6 +82,12 @@ interface Window { onAiEditionChatEvent: ( callback: (event: import("../src/native/contracts").AiEditionChatEvent) => void, ) => () => void; + /** Optional: absent in the browser shim and in tests that stub electronAPI. */ + onAiEditionMcpRequest?: ( + callback: ( + request: import("../src/native/contracts").AiEditionMcpHostRequest, + ) => Promise, + ) => () => void; requestCameraAccess: () => Promise<{ success: boolean; granted: boolean; @@ -172,6 +189,7 @@ interface Window { startNativeMacRecording: ( request: import("../src/lib/nativeMacRecording").NativeMacRecordingRequest, ) => Promise; + onNativeMacSystemAudioUnavailable: (callback: () => void) => () => void; pauseNativeMacRecording: () => Promise<{ success: boolean; error?: string; @@ -320,6 +338,17 @@ interface Window { message?: string; error?: string; }>; + // Stage one of the two bundled click-sound samples where the compositor can read it + // by path. Only the two names the app ships are accepted, so nothing is sanitised here. + stageClickSoundHit: ( + name: string, + data: ArrayBuffer, + ) => Promise<{ + success: boolean; + path?: string; + message?: string; + error?: string; + }>; setCurrentVideoPath: (path: string) => Promise<{ success: boolean }>; setCurrentRecordingSession: ( session: import("../src/lib/recordingSession").RecordingSession | null, @@ -398,6 +427,7 @@ interface Window { success: boolean; path?: string; project?: unknown; + storedProjectId?: string; message?: string; canceled?: boolean; error?: string; @@ -415,6 +445,7 @@ interface Window { success: boolean; path?: string; project?: unknown; + storedProjectId?: string; message?: string; canceled?: boolean; error?: string; @@ -439,6 +470,17 @@ interface Window { revealInFolder: ( filePath: string, ) => Promise<{ success: boolean; error?: string; message?: string }>; + /** Whether a take can start: refused, with the message to show, when the recordings folder + * cannot be reached or its disk is nearly full. */ + checkRecordingStorage: () => Promise<{ success: true } | { success: false; error: string }>; + getRecordingsDir: () => Promise<{ path: string; isDefault: boolean }>; + /** Opens the native folder picker; the main process never takes a path from the renderer. */ + chooseRecordingsDir: () => Promise< + { success: true; path: string } | { success: false; canceled?: boolean; message?: string } + >; + resetRecordingsDir: () => Promise< + { success: true; path: string } | { success: false; message?: string } + >; getShortcuts: () => Promise | null>; saveShortcuts: (shortcuts: unknown) => Promise<{ success: boolean; error?: string }>; updateGlobalShortcut: (binding: { diff --git a/electron/ipc/desktopSourceTypes.test.ts b/electron/ipc/desktopSourceTypes.test.ts new file mode 100644 index 0000000000..cff4012778 --- /dev/null +++ b/electron/ipc/desktopSourceTypes.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, it, vi } from "vitest"; +import { getSourcesByType, shouldEnumerateSourceTypesSeparately } from "./desktopSourceTypes"; + +describe("shouldEnumerateSourceTypesSeparately", () => { + it("splits on Linux X11, including a bare Xvfb with no session variables", () => { + expect(shouldEnumerateSourceTypesSeparately("linux", {})).toBe(true); + expect(shouldEnumerateSourceTypesSeparately("linux", { XDG_SESSION_TYPE: "x11" })).toBe(true); + }); + + it("keeps one call on Wayland, where splitting would raise the portal twice", () => { + expect(shouldEnumerateSourceTypesSeparately("linux", { XDG_SESSION_TYPE: "wayland" })).toBe( + false, + ); + expect(shouldEnumerateSourceTypesSeparately("linux", { WAYLAND_DISPLAY: "wayland-0" })).toBe( + false, + ); + }); + + it("keeps one call on Windows and macOS", () => { + expect(shouldEnumerateSourceTypesSeparately("win32", {})).toBe(false); + expect(shouldEnumerateSourceTypesSeparately("darwin", {})).toBe(false); + }); +}); + +describe("getSourcesByType", () => { + const opts: Electron.SourcesOptions = { + types: ["screen", "window"], + thumbnailSize: { width: 32, height: 18 }, + }; + + it("asks one type per call, one call at a time, and concatenates in order", async () => { + let inFlight = 0; + let maxInFlight = 0; + const getSources = vi.fn(async (o: Electron.SourcesOptions) => { + inFlight += 1; + maxInFlight = Math.max(maxInFlight, inFlight); + await new Promise((resolve) => setTimeout(resolve, 1)); + inFlight -= 1; + return o.types.map((type) => `${type}:0`); + }); + + await expect(getSourcesByType(opts, getSources, true)).resolves.toEqual([ + "screen:0", + "window:0", + ]); + expect(getSources.mock.calls.map(([o]) => o)).toEqual([ + { ...opts, types: ["screen"] }, + { ...opts, types: ["window"] }, + ]); + expect(maxInFlight).toBe(1); + }); + + it("passes the request through untouched when not splitting", async () => { + const getSources = vi.fn(async () => ["screen:0", "window:0"]); + await getSourcesByType(opts, getSources, false); + expect(getSources).toHaveBeenCalledExactlyOnceWith(opts); + }); + + it("passes a single-type request through as one call", async () => { + const getSources = vi.fn(async () => ["screen:0"]); + const screensOnly: Electron.SourcesOptions = { ...opts, types: ["screen"] }; + await getSourcesByType(screensOnly, getSources, true); + expect(getSources).toHaveBeenCalledExactlyOnceWith(screensOnly); + }); + + it("propagates a failure of either call", async () => { + const getSources = vi.fn(async (o: Electron.SourcesOptions) => { + if (o.types[0] === "window") throw new Error("boom"); + return ["screen:0"]; + }); + await expect(getSourcesByType(opts, getSources, true)).rejects.toThrow("boom"); + }); +}); diff --git a/electron/ipc/desktopSourceTypes.ts b/electron/ipc/desktopSourceTypes.ts new file mode 100644 index 0000000000..1cd50dcad9 --- /dev/null +++ b/electron/ipc/desktopSourceTypes.ts @@ -0,0 +1,37 @@ +import { isWaylandSession } from "../wayland"; + +// Issue #462. On X11, one desktopCapturer.getSources call asking for both +// "screen" and "window" runs Chromium's two X11 capturers together, and the +// screen capturer's thumbnail grab then intermittently fails ("Failed to +// initialize pixel buffer"). What that costs depends on the Electron version: +// 41 never settles the call at all, 43 settles after its 3s ready timer with the +// screen silently missing. Measured under Xvfb, 8 runs per cell: the combined +// call failed 3/8 (Electron 43) and 4/8 (Electron 41); asked one type per call, +// 32 runs across both versions and both orders never lost the screen. +// +// So on X11 each type gets its own call, one after the other. Not on Wayland: +// there Electron serves both types from a single PipeWire capturer, and one +// call per type would raise the portal picker twice. Not on Windows or macOS +// either, where nothing points at this and the combined call is what ships. +export function shouldEnumerateSourceTypesSeparately( + platform: NodeJS.Platform, + env: Partial, +): boolean { + // The same test that turns on the PipeWire capturer at startup, whichever Ozone + // backend the app runs on. + return platform === "linux" && !isWaylandSession(env); +} + +export async function getSourcesByType( + opts: Electron.SourcesOptions, + getSources: (opts: Electron.SourcesOptions) => Promise, + separately: boolean, +): Promise { + if (!separately || opts.types.length < 2) return getSources(opts); + const sources: T[] = []; + // Sequential on purpose: running them in parallel is the failure. + for (const type of opts.types) { + sources.push(...(await getSources({ ...opts, types: [type] }))); + } + return sources; +} diff --git a/electron/ipc/gifExportJobs.test.ts b/electron/ipc/exportJobs.test.ts similarity index 88% rename from electron/ipc/gifExportJobs.test.ts rename to electron/ipc/exportJobs.test.ts index b54550da58..1ae944c280 100644 --- a/electron/ipc/gifExportJobs.test.ts +++ b/electron/ipc/exportJobs.test.ts @@ -1,7 +1,7 @@ import { EventEmitter } from "node:events"; import type { WebContents } from "electron"; import { describe, expect, it, vi } from "vitest"; -import { GifExportJobs, isGifExportId } from "./gifExportJobs"; +import { ExportJobs, isExportId } from "./exportJobs"; function owner(id = 1) { return Object.assign(new EventEmitter(), { @@ -20,9 +20,9 @@ function pending() { return { result, resolve, reject, cancel: vi.fn(() => true) }; } -describe("GIF export jobs", () => { +describe("export jobs", () => { it("binds cancel to the sender and ID, including before native compute starts", async () => { - const jobs = new GifExportJobs(); + const jobs = new ExportJobs(); const sender = owner(); const job = pending(); const run = jobs.run(sender, "job_1", () => job, vi.fn()); @@ -38,7 +38,7 @@ describe("GIF export jobs", () => { }); it("rejects a duplicate job without starting it and allows a retry after failure", async () => { - const jobs = new GifExportJobs(); + const jobs = new ExportJobs(); const sender = owner(); const job = pending(); const run = jobs.run(sender, "first", () => job, vi.fn()); @@ -55,7 +55,7 @@ describe("GIF export jobs", () => { }); it("cancels on sender destruction and suppresses late progress after a retry", async () => { - const jobs = new GifExportJobs(); + const jobs = new ExportJobs(); const sender = owner(); const job = pending(); const progress = vi.fn(); @@ -84,7 +84,7 @@ describe("GIF export jobs", () => { }); it("does not reinterpret a completion that wins the cancellation race", async () => { - const jobs = new GifExportJobs(); + const jobs = new ExportJobs(); const sender = owner(); const job = pending(); job.cancel.mockReturnValue(false); @@ -103,13 +103,13 @@ describe("GIF export jobs", () => { "a/b", "a".repeat(129), ])("rejects malformed IDs: %j", (id) => { - expect(isGifExportId(id)).toBe(false); + expect(isExportId(id)).toBe(false); }); it("rejects an invalid start before invoking native code", async () => { const start = vi.fn(() => pending()); - await expect(new GifExportJobs().run(owner(), "../bad", start, vi.fn())).rejects.toThrow( - "Invalid GIF export ID", + await expect(new ExportJobs().run(owner(), "../bad", start, vi.fn())).rejects.toThrow( + "Invalid export ID", ); expect(start).not.toHaveBeenCalled(); }); diff --git a/electron/ipc/gifExportJobs.ts b/electron/ipc/exportJobs.ts similarity index 72% rename from electron/ipc/gifExportJobs.ts rename to electron/ipc/exportJobs.ts index fe318b35e7..727fdff51f 100644 --- a/electron/ipc/gifExportJobs.ts +++ b/electron/ipc/exportJobs.ts @@ -2,17 +2,17 @@ import type { WebContents } from "electron"; type ExportOwner = Pick; -export interface GifExportJob { +export interface ExportJob { result: Promise; cancel: () => boolean; } -export function isGifExportId(value: unknown): value is string { +export function isExportId(value: unknown): value is string { return typeof value === "string" && /^[A-Za-z0-9_-]{1,128}$/.test(value); } /** Controls never leave main. Unknown and foreign IDs have the same result. */ -export class GifExportJobs { +export class ExportJobs { private readonly jobs = new Map boolean }>(); cancel(owner: ExportOwner, exportId: string): boolean { @@ -23,12 +23,12 @@ export class GifExportJobs { async run( owner: ExportOwner, exportId: string, - start: (progress: (frames: number) => void) => GifExportJob, + start: (progress: (frames: number) => void) => ExportJob, onProgress: (frames: number) => void, ): Promise { - if (!isGifExportId(exportId)) throw new Error("Invalid GIF export ID."); - if (owner.isDestroyed()) throw new Error("GIF export window is closed."); - if (this.jobs.has(owner.id)) throw new Error("A GIF export is already running in this window."); + if (!isExportId(exportId)) throw new Error("Invalid export ID."); + if (owner.isDestroyed()) throw new Error("Export window is closed."); + if (this.jobs.has(owner.id)) throw new Error("An export is already running in this window."); let active = true; const job = start((frames) => { if (active && !owner.isDestroyed()) onProgress(frames); diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index bd19f10596..c71996c1cd 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -13,10 +13,12 @@ import { desktopCapturer, dialog, ipcMain, + safeStorage, screen, shell, systemPreferences, } from "electron"; +import { DEFAULT_WEBCAM_QUALITY, type WebcamQualityId } from "../../src/hooks/webcamCaptureTarget"; import { type AxcutDocument, isAxcutDocumentFile, @@ -50,6 +52,7 @@ import type { import { PRODUCT_NAME } from "../about"; import { compactSessionNow, + configureChatPersistence, createSession, deleteSession, getSessionContextUsage, @@ -67,7 +70,16 @@ import { AppSettingsStore } from "../app-settings"; import { isDiagnosticModeEnabled, mainLogBuffer } from "../diagnostics/main-log-buffer"; import { mainT } from "../i18n"; import { getInstallChannel } from "../install-channel"; -import { RECORDINGS_DIR } from "../main"; +import { + DEFAULT_RECORDINGS_DIR, + getRecordingsDirInfo, + RECORDINGS_DIR, + setRecordingsDir, +} from "../main"; +import { EditorDocumentHost } from "../mcp/editor-document-host"; +import { McpController } from "../mcp/mcp-controller"; +import { McpSettingsStore } from "../mcp/mcp-settings-store"; +import type { McpToolDeps } from "../mcp/openscreen-mcp-server"; import { type AudioPeaksResult, getAudioPeaks } from "../media/audioPeaks"; import { readCursorRecordingFile as readCursorRecordingFileFrom, @@ -100,6 +112,7 @@ import { import { CompositorViewService } from "../native-bridge/services/compositorViewService"; import { getMacPermissions, showPermissionsWindow } from "../permissions"; import { scoreDeviceNameMatch } from "../recording/deviceNameMatching"; +import { recordingStorageProblem } from "../recording/diskSpaceCheck"; import { describeSalvagedTake, nativeMacSalvageTarget, @@ -115,23 +128,29 @@ import { isSalvageableFragmentedCapture, NATIVE_WINDOWS_SALVAGEABLE_OUTPUT_BYTES, readMicrophoneDefaulted, + readMicrophoneNotFound, readMicrophoneUnavailable, + readSecondaryWindowsApplied, readWebcamFormat, readWebcamUnavailable, terminateNativeWindowsCapture, waitForNativeWindowsCaptureStop, } from "../recording/nativeWindowsCaptureStop"; +import { removeEmptyCaptureOutput } from "../recording/removeEmptyCaptureOutput"; import { patchWebmDurationOnDisk } from "../recording/webm-duration"; import { reindexRecordingOnDisk } from "../recording/webm-seek-index"; import { + describeMacPickerSource, describeRecordingSource, enumerationIncludesSourceKind, mergeEnumeratedSources, + rememberedPickerSourceName, resolveRecordingSource, restoreRecordingSourceAfterEnumeration, shouldEnumerateRecordingSources, shouldPersistSelectedSource, } from "../recording-source-settings"; +import { getSourcesByType, shouldEnumerateSourceTypesSeparately } from "./desktopSourceTypes"; import { registerNativeBridgeHandlers } from "./nativeBridge"; import { createNativeMacMidCaptureErrorWatch } from "./nativeMacMidCaptureErrorWatch"; import { registerRecordingPrefsHandlers } from "./recordingPrefs"; @@ -157,6 +176,12 @@ const PREVIEW_AUDIO_DIR = path.join(app.getPath("userData"), "preview-audio"); // See the save-recorded-voiceover handler: an upper bound on renderer-supplied // bytes written to disk, well past any plausible take. const MAX_RECORDED_VOICEOVER_BYTES = 512 * 1024 * 1024; +// The bundled click-sound samples, staged for the compositor, which reads audio by path and not +// by renderer URL. These are the only names the staging handler accepts, and they match +// HIT_FILES in src/lib/ai-edition/clickSound.ts. +const CLICK_SOUND_DIR = path.join(app.getPath("userData"), "click-sfx"); +const CLICK_SOUND_HIT_FILES: string[] = ["click-down.wav", "click-up.wav"]; +const MAX_CLICK_HIT_BYTES = 4 * 1024 * 1024; const nativeMacCaptureEvents = new EventEmitter(); // Enumeration walks every display and window and grabs a thumbnail of each, so it @@ -191,8 +216,13 @@ function approveFilePath(filePath: string): void { approvedPaths.add(path.resolve(filePath)); } +// The default folder stays readable after the user picks another one: every take made +// before the change, and the projects that use them, are still there. +// ponytail: a custom folder left for another custom folder is not remembered. The editor's +// own projects still open (they grant what they declare); a v2 project file outside it +// does not. Keep a list of past folders if that ever matters. function getAllowedReadDirs(): string[] { - return [RECORDINGS_DIR]; + return [RECORDINGS_DIR, DEFAULT_RECORDINGS_DIR]; } function isPathWithinDir(filePath: string, dirPath: string): boolean { @@ -544,9 +574,9 @@ async function getApprovedProjectSession( return null; } - // Only auto-approve media within the project's dir or RECORDINGS_DIR, so a crafted + // Only auto-approve media within the project's dir or the recordings folders, so a crafted // project file can't approve reads to arbitrary locations. - const trustedDirs = [RECORDINGS_DIR]; + const trustedDirs = getAllowedReadDirs(); if (projectFilePath) { trustedDirs.push(path.dirname(path.resolve(projectFilePath))); } @@ -649,6 +679,8 @@ export interface RecordingPrefs { camDeviceId: string | null; /** Camera label paired with the preferred id for restart-safe resolution. */ camDeviceName: string | null; + /** Capture resolution for the camera. See WEBCAM_QUALITY_PRESETS. */ + camQuality: WebcamQualityId; systemAudioEnabled: boolean; cursorCaptureMode: CursorCaptureMode; hideDesktopIcons: boolean; @@ -662,6 +694,7 @@ const defaultRecordingPrefs: RecordingPrefs = { camEnabled: false, camDeviceId: null, camDeviceName: null, + camQuality: DEFAULT_WEBCAM_QUALITY, systemAudioEnabled: false, cursorCaptureMode: "editable-overlay", hideDesktopIcons: false, @@ -975,7 +1008,11 @@ function linuxSourceLabel(kind?: LinuxCaptureSourceKind): string { // threw `Cannot access 'RECORDINGS_DIR' before initialization` and the app died // before its first window. Every call site is already inside a handler, i.e. // long after both modules finished loading. -const cursorSidecarOptions = () => ({ recordingsDir: RECORDINGS_DIR }); +// +// The media-links registry stays in the DEFAULT folder whichever folder takes are written +// to, here and at every other registry call: the links registered before the user picked +// another folder must still be found after it. +const cursorSidecarOptions = () => ({ recordingsDir: DEFAULT_RECORDINGS_DIR }); const readCursorRecordingFile = (targetVideoPath: string) => readCursorRecordingFileFrom(targetVideoPath, cursorSidecarOptions()); @@ -1054,7 +1091,9 @@ async function getMacPickerSession(): Promise { return session; } -function selectedSourceFromPick(pick: MacPickerSelection): SelectedSource { +function selectedSourceFromPick( + pick: MacPickerSelection, +): SelectedSource & { id: string; display_id: string } { const display = pick.displayId !== null ? screen.getAllDisplays().find((candidate) => candidate.id === pick.displayId) @@ -1337,7 +1376,7 @@ async function registerRecordingMediaLinks( .access(cursorTelemetryPath, fsConstants.F_OK) .then(() => true) .catch(() => false); - await registerMediaLinks(RECORDINGS_DIR, screenVideoPath, { + await registerMediaLinks(DEFAULT_RECORDINGS_DIR, screenVideoPath, { ...(options.webcamVideoPath ? { webcamVideoPath: options.webcamVideoPath } : {}), ...(options.webcamVideoPath && Number.isFinite(options.webcamOffsetMs) ? { webcamOffsetMs: options.webcamOffsetMs } @@ -1605,6 +1644,7 @@ function inspectNativeMacCaptureOutput() { function attachNativeMacCaptureOutputDrain( proc: ChildProcessWithoutNullStreams, onTakeEnded: () => void, + onSystemAudioUnavailable: () => void, ) { let lineBuffer = ""; // Hooked here rather than on `nativeMacCaptureEvents`, which the start wait @@ -1612,6 +1652,7 @@ function attachNativeMacCaptureOutputDrain( const watchLiveTake = createNativeMacMidCaptureErrorWatch( () => nativeMacCaptureProcess === proc && !nativeMacStopInFlight, onTakeEnded, + onSystemAudioUnavailable, ); const drain = (chunk: Buffer) => { const text = chunk.toString(); @@ -1667,8 +1708,12 @@ function attachNativeMacCaptureOutputDrain( }); } +/** + * Resolves with the helper's own stamp of its first appended frame: the file's t=0, and + * so the instant the HUD timer counts from (#901). Null from a helper that sends none. + */ function waitForNativeMacCaptureStart(proc: ChildProcessWithoutNullStreams) { - return new Promise((resolve, reject) => { + return new Promise((resolve, reject) => { const timer = setTimeout(() => { cleanup(); reject(new Error("Timed out waiting for native macOS capture to start")); @@ -1677,7 +1722,11 @@ function waitForNativeMacCaptureStart(proc: ChildProcessWithoutNullStreams) { const inspect = (event: Record) => { if (event.event === "recording-started") { cleanup(); - resolve(); + resolve( + typeof event.timestampMs === "number" && Number.isFinite(event.timestampMs) + ? event.timestampMs + : null, + ); return; } if (event.event === "error") { @@ -1813,7 +1862,7 @@ async function resolveMediaLinksForVideo(videoPath: string): Promise<{ if (session?.webcamVideoPath || hasCursorTelemetry) { // Opportunistic backfill so the link survives a later move even if this // recording predates the registry, or if its sidecar doesn't travel with it. - await registerMediaLinks(RECORDINGS_DIR, videoPath, { + await registerMediaLinks(DEFAULT_RECORDINGS_DIR, videoPath, { ...(session?.webcamVideoPath ? { webcamVideoPath: session.webcamVideoPath } : {}), ...(session?.webcamVideoPath && Number.isFinite(session.webcamOffsetMs) ? { webcamOffsetMs: session.webcamOffsetMs } @@ -1834,12 +1883,12 @@ async function resolveMediaLinksForVideo(videoPath: string): Promise<{ } try { - const links = await findMediaLinksByFingerprint(RECORDINGS_DIR, videoPath); + const links = await findMediaLinksByFingerprint(DEFAULT_RECORDINGS_DIR, videoPath); if (links?.webcamVideoPath || links?.cursorTelemetryPath) { let webcamVideoPath = links.webcamVideoPath; if (webcamVideoPath && !isPathAllowed(webcamVideoPath)) { webcamVideoPath = - (await approveReadableVideoPath(webcamVideoPath, [RECORDINGS_DIR])) ?? undefined; + (await approveReadableVideoPath(webcamVideoPath, getAllowedReadDirs())) ?? undefined; } return { ...(webcamVideoPath ? { webcamVideoPath, webcamOffsetMs: links.webcamOffsetMs ?? 0 } : {}), @@ -1959,18 +2008,19 @@ export function registerIpcHandlers( // await it, and a renderer-side race would only stop *waiting* while this // keeps running and its reply goes to nobody. Rejecting is what turns an // indefinite spinner into the pickers' existing error branch. - // How long it actually took, under the existing diagnostic flag. The bound - // above turned an indefinite hang into a named failure, which is where the - // open question starts rather than ends: on a headless runner `openscreen - // sources` gets an answer within 20s four times in five while `record` -- - // the same call with the same options -- exceeds 30s every time. A duration - // on both paths is what tells those apart; a threshold alone cannot. + // How long it actually took, under the existing diagnostic flag. That + // duration is what showed the headless hang (#462) to be bimodal -- ~17ms + // or never -- and so not a threshold problem; see desktopSourceTypes.ts. const startedAt = Date.now(); const diagnostic = isDiagnosticModeEnabled(); let sources: Awaited>; try { sources = await withDeadline( - desktopCapturer.getSources(opts), + getSourcesByType( + opts, + (typeOpts) => desktopCapturer.getSources(typeOpts), + shouldEnumerateSourceTypesSeparately(process.platform, process.env), + ), GET_SOURCES_TIMEOUT_MS, `Desktop source enumeration did not return within ${GET_SOURCES_TIMEOUT_MS}ms. ` + "This usually means the display or GPU stack cannot be reached — check that a display server is available.", @@ -2081,13 +2131,19 @@ export function registerIpcHandlers( }, ); - async function presentMacSystemPicker(session: MacPickerSession) { - // The HUD and the notes window belong to this process, not to the helper, so a display - // pick would record them unless the picker is told to leave them out. + /** + * The HUD and the notes window belong to this process, not to the helper, so a display + * pick would record them unless the picker is told to leave them out. + */ + function macPickerExcludedWindowIds() { const appWindowSourceIds = [getMainWindow(), getNotesWindow()] .filter((window): window is BrowserWindow => !!window && !window.isDestroyed()) .map((window) => window.getMediaSourceId()); - const excludedWindowIds = collectMacCaptureExcludedWindowIds(appWindowSourceIds); + return collectMacCaptureExcludedWindowIds(appWindowSourceIds); + } + + async function presentMacSystemPicker(session: MacPickerSession) { + const excludedWindowIds = macPickerExcludedWindowIds(); // Out of the way while the picker is up. The HUD window is far larger than the bar // it draws (a transparent reserve above it), and Apple's picker targets windows by // their frame, not by where clicks land -- so that invisible rectangle hid every @@ -2119,8 +2175,15 @@ export function registerIpcHandlers( } return; } - selectedSource = selectedSourceFromPick(pick); + const picked = selectedSourceFromPick(pick); + selectedSource = picked; selectedDesktopSource = null; + // Written only so the next launch can name it: the pick itself dies with the helper. + try { + appSettings.setLastSource(describeMacPickerSource(picked, pick.kind)); + } catch (error) { + console.warn("Failed to persist the picked recording source:", error); + } broadcastSelectedSource(selectedSource); } @@ -2132,6 +2195,12 @@ export function registerIpcHandlers( // picker they must hand the choice to `open-source-selector` rather than enumerate. ipcMain.handle("uses-system-source-picker", () => macPickerOwnsSources()); + // Named in the HUD and Record mode after a relaunch, while `get-selected-source` still + // answers null: the pick cannot be restored, so it is never reported as selected. + ipcMain.handle("get-last-picked-source", () => + rememberedPickerSourceName(appSettings.getSnapshot().lastSource, macPickerOwnsSources()), + ); + ipcMain.handle("get-selected-source", async () => { const previousSelectedSource = selectedSource; if (process.platform === "linux" && findPipeWireCursorHelperPath()) { @@ -2151,6 +2220,9 @@ export function registerIpcHandlers( if (!isMacPickerSourceId(selectedSource?.id)) { return selectedDesktopSource ? selectedSource : null; } + // A screen pick that would record the current HUD or notes window is no pick at all: + // answering null makes the next Record present the picker again (#965). + macPickerSession?.forgetSelectionUnlessExcluding(macPickerExcludedWindowIds()); if (!macPickerSession?.getSelection()) { selectedSource = null; selectedDesktopSource = null; @@ -2234,6 +2306,54 @@ export function registerIpcHandlers( }, ); + // Asked before every take starts. A disk that fills up mid-take used to be found only + // when the take was saved, and the take was lost with it. + ipcMain.handle("check-recording-storage", async () => { + const problem = await recordingStorageProblem(RECORDINGS_DIR); + if (!problem) return { success: true }; + if (problem.kind === "unavailable") { + return { + success: false, + error: mainT("dialogs", "recording.folderUnavailable", { path: RECORDINGS_DIR }), + }; + } + const availableMb = Math.max(0, Math.floor(problem.availableBytes / (1024 * 1024))); + return { + success: false, + error: mainT("dialogs", "recording.lowDiskSpace", { availableMb }), + }; + }); + + ipcMain.handle("get-recordings-dir", () => getRecordingsDirInfo()); + + // The folder comes from the native picker, never from the renderer: everything in it + // becomes readable through the media handlers. + ipcMain.handle("choose-recordings-dir", async (event) => { + const result = await showOpenDialogOver(BrowserWindow.fromWebContents(event.sender), { + title: mainT("dialogs", "fileDialogs.selectRecordingsFolder"), + defaultPath: RECORDINGS_DIR, + properties: ["openDirectory", "createDirectory"], + }); + if (result.canceled || result.filePaths.length === 0) { + return { success: false, canceled: true }; + } + try { + return { success: true, path: await setRecordingsDir(result.filePaths[0]) }; + } catch (error) { + console.error("Failed to switch recordings folder:", error); + return { success: false, message: String(error) }; + } + }); + + ipcMain.handle("reset-recordings-dir", async () => { + try { + return { success: true, path: await setRecordingsDir(null) }; + } catch (error) { + console.error("Failed to reset recordings folder:", error); + return { success: false, message: String(error) }; + } + }); + ipcMain.handle("request-camera-access", async () => { if (process.platform !== "darwin") { return { success: true, granted: true, status: "granted" }; @@ -2287,12 +2407,11 @@ export function registerIpcHandlers( return access; } - // The helper that answered has just raised macOS' own Accessibility prompt on - // its way up, so the window must offer System Settings, not a second prompt. It - // explains what the grant is for and tracks it live, where a message box could - // only say "go to System Settings" in English. + // Accessibility improves cursor shape hints but is not required to record. + // Remember that the helper raised the system prompt so a later visit to the + // permissions window can direct the user to Settings, but don't reopen that + // window from every Record press. getMacPermissions().noteRequested("accessibility"); - showPermissionsWindow(); } return access; @@ -2541,6 +2660,7 @@ export function registerIpcHandlers( ipcMain.handle( "start-native-linux-recording", async (_, request: NativeLinuxRecordingRequest) => { + let outputPath: string | null = null; try { if (process.platform !== "linux") { return { success: false, error: "Native Linux capture requires Linux." }; @@ -2556,7 +2676,7 @@ export function registerIpcHandlers( typeof request?.recordingId === "number" && Number.isFinite(request.recordingId) ? request.recordingId : Date.now(); - const outputPath = path.join(RECORDINGS_DIR, `${RECORDING_FILE_PREFIX}${recordingId}.mp4`); + outputPath = path.join(RECORDINGS_DIR, `${RECORDING_FILE_PREFIX}${recordingId}.mp4`); const cursorCaptureMode = normalizeCursorCaptureMode(request?.cursor?.mode) ?? "editable-overlay"; @@ -2625,6 +2745,11 @@ export function registerIpcHandlers( linuxNativeCaptureSession = null; linuxNativeCaptureRecordingId = null; linuxNativeCaptureCursorMode = "editable-overlay"; + // The helper opens its mp4 on the first frame, before it reports capture + // started, so an encoder that failed there leaves it empty. + if (outputPath) { + await removeEmptyCaptureOutput(outputPath); + } return { success: false, error: String(error) }; } }, @@ -2904,6 +3029,9 @@ export function registerIpcHandlers( cursorOffsetMs: nativeWindowsCursorOffsetMs, webcamFormat, encoderSelection, + // Logged only: menus missing from a window take on Windows before 11 + // 24H2 are a platform limit, not something to put in front of the user. + secondaryWindowsApplied: readSecondaryWindowsApplied(nativeWindowsCaptureOutput), }); const source = selectedSource || { name: "Screen" }; @@ -2951,9 +3079,23 @@ export function registerIpcHandlers( }; } catch (error) { console.error("Failed to start native Windows recording:", error); - nativeWindowsCaptureProcess?.kill(); + const failedProc = nativeWindowsCaptureProcess; + const failedOutputPaths = [ + nativeWindowsCaptureTargetPath, + nativeWindowsCaptureWebcamTargetPath, + ]; detachNativeWindowsCaptureOutputDrain(); resetNativeWindowsCaptureState(); + // Awaited, not just killed: a helper still alive holds its mp4 open, and + // the empty one it created could not be removed below. + if (failedProc) { + await terminateNativeWindowsCapture(failedProc).catch(() => false); + } + for (const outputPath of failedOutputPaths) { + if (outputPath) { + await removeEmptyCaptureOutput(outputPath); + } + } await stopCursorRecording(); return { success: false, error: String(error) }; } @@ -2988,6 +3130,8 @@ export function registerIpcHandlers( // A source from Apple's picker records through the session that holds the pick, // and needs no Screen Recording grant -- so nothing here may go near one. const pickerSession = isMacPickerSourceId(request.source.sourceId) ? macPickerSession : null; + // Normally caught by `get-selected-source` first; this is the last word (#965). + pickerSession?.forgetSelectionUnlessExcluding(macPickerExcludedWindowIds()); const pick = pickerSession?.getSelection() ?? null; if (isMacPickerSourceId(request.source.sourceId) && !pick) { selectedSource = null; @@ -3108,15 +3252,23 @@ export function registerIpcHandlers( // When the take ends without the user — the helper reported an error or // exited — this drives the renderer's own stop, the same one the tray's Stop // Recording sends: it clears the HUD and surfaces the result. - attachNativeMacCaptureOutputDrain(proc, () => { - const hudWindow = getMainWindow(); - if (hudWindow && !hudWindow.isDestroyed()) { - hudWindow.webContents.send("stop-recording-from-tray"); - } - }); + attachNativeMacCaptureOutputDrain( + proc, + () => { + const hudWindow = getMainWindow(); + if (hudWindow && !hudWindow.isDestroyed()) { + hudWindow.webContents.send("stop-recording-from-tray"); + } + }, + () => { + const hudWindow = getMainWindow(); + if (hudWindow && !hudWindow.isDestroyed()) { + hudWindow.webContents.send("native-mac-system-audio-unavailable"); + } + }, + ); - await waitForNativeMacCaptureStart(proc); - const captureStartedAtMs = Date.now(); + const captureStartedAtMs = (await waitForNativeMacCaptureStart(proc)) ?? Date.now(); const microphoneDefaulted = request.audio.microphone.enabled && readMicrophoneDefaulted(nativeMacCaptureOutput); if (microphoneDefaulted) { @@ -3136,6 +3288,11 @@ export function registerIpcHandlers( { macOS: process.getSystemVersion() }, ); } + const microphoneNotFound = + request.audio.microphone.enabled && readMicrophoneNotFound(nativeMacCaptureOutput); + if (microphoneNotFound) { + console.warn("[native-sck] recording without the microphone; macOS lists no audio input"); + } nativeMacCursorOffsetMs = cursorCaptureMode === "editable-overlay" ? Math.max(0, captureStartedAtMs - cursorStartTimeMs) @@ -3151,11 +3308,14 @@ export function registerIpcHandlers( recordingId, path: outputPath, helperPath, + startedAtMs: captureStartedAtMs, microphoneDefaulted, microphoneUnavailable, + microphoneNotFound, }; } catch (error) { console.error("Failed to start native macOS recording:", error); + const failedOutputPath = nativeMacCaptureTargetPath; nativeMacCaptureProcess?.kill(); nativeMacCaptureProcess = null; nativeMacCaptureTargetPath = null; @@ -3166,6 +3326,9 @@ export function registerIpcHandlers( nativeMacPauseStartedAtMs = null; nativeMacPauseRanges = []; nativeMacIsPaused = false; + if (failedOutputPath) { + await removeEmptyCaptureOutput(failedOutputPath); + } await stopCursorRecording(); return { success: false, error: error instanceof Error ? error.message : String(error) }; } @@ -4203,6 +4366,34 @@ export function registerIpcHandlers( } }); + // The two click-sound samples the editor ships (src/lib/ai-edition/clickSound.ts). The + // compositor reads its audio from the filesystem, not from a renderer URL, so the bundled + // bytes are staged here once and the absolute path handed back. Renderer bytes like the + // recorded voiceover, but with nothing to sanitise: the name is checked against the two the + // app knows, so the payload cannot steer where anything lands. + ipcMain.handle("stage-click-sound-hit", async (_event, name: string, data: ArrayBuffer) => { + try { + if (!CLICK_SOUND_HIT_FILES.includes(name)) { + return { success: false, message: `Unknown click hit ${name}` }; + } + if (!(data instanceof ArrayBuffer) || data.byteLength === 0) { + return { success: false, message: "Empty click hit" }; + } + if (data.byteLength > MAX_CLICK_HIT_BYTES) { + return { success: false, message: "Click hit too large" }; + } + const target = path.join(CLICK_SOUND_DIR, name); + await fs.mkdir(CLICK_SOUND_DIR, { recursive: true }); + // Written fresh each time rather than checked for existence: the file the app shipped + // is the one the compositor must hear, and an update may have changed it. + await fs.writeFile(target, Buffer.from(data)); + return { success: true, path: target }; + } catch (error) { + console.error("Failed to stage click sound hit:", error); + return { success: false, message: "Failed to stage click sound hit", error: String(error) }; + } + }); + ipcMain.handle("reveal-in-folder", async (_, filePath: string) => { try { // showItemInFolder returns nothing, it throws on error @@ -4536,7 +4727,7 @@ export function registerIpcHandlers( const filePath = result.filePaths[0]; const content = await fs.readFile(filePath, "utf-8"); - const project = await relinkProjectMedia(JSON.parse(content), RECORDINGS_DIR); + const project = await relinkProjectMedia(JSON.parse(content), DEFAULT_RECORDINGS_DIR); currentProjectPath = filePath; let session: RecordingSession | null = null; try { @@ -4553,6 +4744,7 @@ export function registerIpcHandlers( success: true, path: filePath, project, + storedProjectId: aiEditionDocuments.storedProjectId(filePath), }; } catch (error) { console.error("Failed to load project file:", error); @@ -4582,7 +4774,7 @@ export function registerIpcHandlers( return { success: false, message: "File not found" }; } const content = await fs.readFile(filePath, "utf-8"); - const project = await relinkProjectMedia(JSON.parse(content), RECORDINGS_DIR); + const project = await relinkProjectMedia(JSON.parse(content), DEFAULT_RECORDINGS_DIR); currentProjectPath = filePath; // A document the editor saved grants the media it declares, within the same trusted // dirs as a legacy v2 project below: the recordings dir and the project's own dir. @@ -4590,7 +4782,7 @@ export function registerIpcHandlers( if (isAxcutDocumentFile(project)) { try { approveDocumentMedia(parseDocumentFile(project), [ - RECORDINGS_DIR, + ...getAllowedReadDirs(), path.dirname(path.resolve(filePath)), ]); } catch { @@ -4610,7 +4802,12 @@ export function registerIpcHandlers( ); } setCurrentRecordingSessionState(session); - return { success: true, path: filePath, project }; + return { + success: true, + path: filePath, + project, + storedProjectId: aiEditionDocuments.storedProjectId(filePath), + }; } catch (error) { console.error("Failed to load project file from path:", error); return { @@ -4798,9 +4995,11 @@ export function registerIpcHandlers( // race destroyed two real project files), so a second instance means a second // queue racing for the same path: temp+rename still keeps the file valid, but // a save can land under a concurrent one and be silently lost. + const chatProjectsRoot = path.join(app.getPath("userData"), "projects"); + configureChatPersistence(chatProjectsRoot); const aiEditionDocuments = new DocumentService( - path.join(app.getPath("userData"), "projects"), - RECORDINGS_DIR, + chatProjectsRoot, + DEFAULT_RECORDINGS_DIR, approveDocumentMedia, () => stylePresets.newProjectAppearance(), ); @@ -4822,6 +5021,25 @@ export function registerIpcHandlers( return aiEditionLlmConfigInstance; }; + // The local MCP server offers the agent's tools to MCP clients the user runs + // (Claude Code, Codex…), on any project: the open one through the editor, the + // rest through the same DocumentService instance as every other save (see + // above for why there is only one). Built here because this is where the agent's own + // dependencies live, but NOT started here: the headless CLI shares this + // function and must never bind the port a running app is listening on. + // `main.ts` starts it. Its writes have their own switch, off by default and + // read on every call — separate from the in-app agent's "Project edits", so + // turning the server on grants a client read access and nothing more. + const mcpSettings = new McpSettingsStore(app.getPath("userData"), safeStorage); + const mcpToolDeps: McpToolDeps = { + host: new EditorDocumentHost(ipcMain), + projects: aiEditionDocuments, + editsAllowed: () => mcpSettings.getSettings().allowEdits, + cursor: agentCursorTelemetryReader, + version: app.getVersion(), + }; + const mcpController = new McpController(mcpSettings, mcpToolDeps); + registerNativeBridgeHandlers({ getPlatform: () => process.platform, getCurrentProjectPath: () => currentProjectPath, @@ -4877,5 +5095,18 @@ export function registerIpcHandlers( renameAiEditionChatSession: (projectId, sessionId, title) => renameSession(projectId, sessionId, title), deleteAiEditionChatSession: (projectId, sessionId) => deleteSession(projectId, sessionId), + getMcpController: () => mcpController, }); + + // `openscreen mcp` serves the same tools on stdio, from the CLI process. + return { + mcpController, + agentServices: { + mcpToolDeps, + documents: aiEditionDocuments, + // The reader the export runner's auto zoom reads through. + readCursorTelemetry: async (videoPath: string) => + (await readCursorTelemetryFile(videoPath)).samples, + }, + }; } diff --git a/electron/ipc/nativeBridge.export.test.ts b/electron/ipc/nativeBridge.export.test.ts new file mode 100644 index 0000000000..9fbd008eb0 --- /dev/null +++ b/electron/ipc/nativeBridge.export.test.ts @@ -0,0 +1,88 @@ +// An MP4 export started with an ID runs as a job of its window, so a cancel naming that ID +// reaches the native control, and the native cancellation comes back as CANCELLED. + +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { NativeBridgeResponse } from "../../src/native/contracts"; +import { type NativeBridgeContext, registerNativeBridgeHandlers } from "./nativeBridge"; + +const electron = vi.hoisted(() => ({ handle: vi.fn() })); +vi.mock("electron", () => ({ + app: { getAppPath: () => "", isPackaged: false }, + ipcMain: { handle: electron.handle, removeHandler: vi.fn() }, + shell: {}, +})); +const compositor = vi.hoisted(() => ({ startExportMulti: vi.fn(), exportMulti: vi.fn() })); +vi.mock("../native-bridge/services/compositorViewService", () => ({ + CompositorViewService: class { + startExportMulti = compositor.startExportMulti; + exportMulti = compositor.exportMulti; + }, +})); +vi.mock("../native-bridge/services/aiEditionService", () => ({ + AiEditionService: class {}, +})); + +const sender = { + id: 1, + isDestroyed: () => false, + send: vi.fn(), + once: vi.fn(), + removeListener: vi.fn(), +}; +let invoke: (action: string, payload: unknown) => Promise; + +beforeEach(() => { + electron.handle.mockClear(); + sender.send.mockClear(); + for (const fn of Object.values(compositor)) fn.mockReset(); + registerNativeBridgeHandlers({ + getPlatform: () => "linux", + getAiEditionDocuments: () => ({}), + getAiEditionLlmConfig: () => ({}), + } as unknown as NativeBridgeContext); + const handler = electron.handle.mock.calls[0]?.[1] as ( + event: unknown, + request: unknown, + ) => Promise; + invoke = (action, payload) => handler({ sender }, { domain: "compositor", action, payload }); +}); + +describe("native bridge MP4 export", () => { + it("cancels a running export by its ID and reports CANCELLED", async () => { + let fail!: (error: Error) => void; + const cancel = vi.fn(() => true); + compositor.startExportMulti.mockImplementation((...args: unknown[]) => { + (args[4] as (frames: number) => void)(3); + const result = new Promise((_, reject) => { + fail = reject; + }); + return { result, cancel }; + }); + const running = invoke("exportMulti", { clips: [], outPath: "/tmp/a.mp4", exportId: "mp4-1" }); + await vi.waitFor(() => expect(compositor.startExportMulti).toHaveBeenCalledOnce()); + expect(sender.send).toHaveBeenCalledWith("export:native-progress", 3, "mp4-1"); + + expect(await invoke("cancelExport", { exportId: "other" })).toMatchObject({ + ok: true, + data: { accepted: false }, + }); + expect(await invoke("cancelExport", { exportId: "mp4-1" })).toMatchObject({ + ok: true, + data: { accepted: true }, + }); + expect(cancel).toHaveBeenCalledOnce(); + + fail(new Error("MP4_EXPORT_CANCELLED")); + expect(await running).toMatchObject({ ok: false, error: { code: "CANCELLED" } }); + }); + + it("runs an export without an ID outside the job registry, as the CLI does", async () => { + const stats = { frames: 1, wallS: 1, fps: 1, videoDurationS: 1 }; + compositor.exportMulti.mockResolvedValue(stats); + expect(await invoke("exportMulti", { clips: [], outPath: "/tmp/a.mp4" })).toMatchObject({ + ok: true, + data: stats, + }); + expect(compositor.startExportMulti).not.toHaveBeenCalled(); + }); +}); diff --git a/electron/ipc/nativeBridge.mcp.test.ts b/electron/ipc/nativeBridge.mcp.test.ts new file mode 100644 index 0000000000..59b477c934 --- /dev/null +++ b/electron/ipc/nativeBridge.mcp.test.ts @@ -0,0 +1,82 @@ +// The MCP settings actions take renderer input, so the bridge checks its types +// before anything reaches the settings store: a bad value is an INVALID_REQUEST, +// not an INTERNAL_ERROR thrown further in. + +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { NativeBridgeResponse } from "../../src/native/contracts"; +import { type NativeBridgeContext, registerNativeBridgeHandlers } from "./nativeBridge"; + +const electron = vi.hoisted(() => ({ handle: vi.fn() })); +vi.mock("electron", () => ({ + app: { getAppPath: () => "", isPackaged: false }, + ipcMain: { handle: electron.handle, removeHandler: vi.fn() }, + shell: {}, +})); +vi.mock("../native-bridge/services/compositorViewService", () => ({ + CompositorViewService: class {}, +})); + +const service = vi.hoisted(() => ({ + mcpSetEnabled: vi.fn(async () => ({ ok: "enabled" })), + mcpSetPort: vi.fn(async () => ({ ok: "port" })), + mcpSetAllowEdits: vi.fn(async () => ({ ok: "allowEdits" })), +})); +vi.mock("../native-bridge/services/aiEditionService", () => ({ + AiEditionService: class { + mcpSetEnabled = service.mcpSetEnabled; + mcpSetPort = service.mcpSetPort; + mcpSetAllowEdits = service.mcpSetAllowEdits; + }, +})); + +let invoke: (action: string, payload: unknown) => Promise; + +beforeEach(() => { + electron.handle.mockClear(); + for (const fn of Object.values(service)) fn.mockClear(); + registerNativeBridgeHandlers({ + getPlatform: () => "linux", + getAiEditionDocuments: () => ({}), + getAiEditionLlmConfig: () => ({}), + } as unknown as NativeBridgeContext); + const handler = electron.handle.mock.calls[0]?.[1] as ( + event: unknown, + request: unknown, + ) => Promise; + invoke = (action, payload) => + handler({ sender: {} }, { domain: "aiEdition", action, payload, requestId: "r" }); +}); + +function errorCode(response: NativeBridgeResponse): string | undefined { + return response.ok ? undefined : response.error?.code; +} + +describe("native bridge MCP settings", () => { + it.each([80, 70000, 4.5, "47821", null])("rejects port %s as a bad request", async (port) => { + expect(errorCode(await invoke("mcp.setPort", { port }))).toBe("INVALID_REQUEST"); + expect(service.mcpSetPort).not.toHaveBeenCalled(); + }); + + it("passes a valid port on", async () => { + const response = await invoke("mcp.setPort", { port: 47821 }); + expect(response.ok).toBe(true); + expect(service.mcpSetPort).toHaveBeenCalledWith(47821); + }); + + it.each(["true", 1, undefined])("rejects enabled=%s as a bad request", async (enabled) => { + expect(errorCode(await invoke("mcp.setEnabled", { enabled }))).toBe("INVALID_REQUEST"); + expect(service.mcpSetEnabled).not.toHaveBeenCalled(); + }); + + it.each(["yes", 0, undefined])("rejects allowEdits=%s as a bad request", async (allowEdits) => { + expect(errorCode(await invoke("mcp.setAllowEdits", { allowEdits }))).toBe("INVALID_REQUEST"); + expect(service.mcpSetAllowEdits).not.toHaveBeenCalled(); + }); + + it("passes valid flags on", async () => { + await invoke("mcp.setEnabled", { enabled: true }); + await invoke("mcp.setAllowEdits", { allowEdits: false }); + expect(service.mcpSetEnabled).toHaveBeenCalledWith(true); + expect(service.mcpSetAllowEdits).toHaveBeenCalledWith(false); + }); +}); diff --git a/electron/ipc/nativeBridge.presets.test.ts b/electron/ipc/nativeBridge.presets.test.ts index bd8a7f8b92..6f97b66e1b 100644 --- a/electron/ipc/nativeBridge.presets.test.ts +++ b/electron/ipc/nativeBridge.presets.test.ts @@ -54,6 +54,8 @@ const APPEARANCE: StylePresetAppearance = { model3d: false, asArrow: [], clickImpact: false, + clickSound: false, + clickSoundGainDb: 0, }, cursorShow: true, cursorAutoHide: false, diff --git a/electron/ipc/nativeBridge.ts b/electron/ipc/nativeBridge.ts index e25ea048be..065aaa4b56 100644 --- a/electron/ipc/nativeBridge.ts +++ b/electron/ipc/nativeBridge.ts @@ -10,9 +10,10 @@ import { type ProjectFileResult, type ProjectPathResult, } from "../../src/native/contracts"; -import type { ChatEventSink } from "../ai-edition/chat-service"; +import { type ChatEventSink, deleteProjectChat } from "../ai-edition/chat-service"; import type { DocumentService } from "../ai-edition/document-service"; import { StylePresetError, type StylePresetService } from "../ai-edition/style-preset-service"; +import { isValidMcpPort } from "../mcp/mcp-settings-store"; import { type CursorTelemetryLoadResult, TelemetryCursorAdapter, @@ -23,7 +24,7 @@ import { CursorService } from "../native-bridge/services/cursorService"; import { ProjectService } from "../native-bridge/services/projectService"; import { SystemService } from "../native-bridge/services/systemService"; import { createNativeBridgeState } from "../native-bridge/store"; -import { GifExportJobs, isGifExportId } from "./gifExportJobs"; +import { ExportJobs, isExportId } from "./exportJobs"; export interface NativeBridgeContext { getPlatform: () => NodeJS.Platform; @@ -62,6 +63,8 @@ export interface NativeBridgeContext { /** The one shared style preset service — it serialises writes per instance. */ getStylePresets: () => StylePresetService; getAiEditionLlmConfig: () => import("../ai-edition/llm-config-store").LlmConfigStore; + /** The local MCP server's controller. Absent in the headless CLI. */ + getMcpController?: () => import("../mcp/mcp-controller").McpController; runAiEditionChat: ( projectId: string, sessionId: string, @@ -231,9 +234,11 @@ export function registerNativeBridgeHandlers(context: NativeBridgeContext) { const compositorViewService = new CompositorViewService(); const aiEditionService = new AiEditionService({ documents: context.getAiEditionDocuments(), + deleteChatHistory: deleteProjectChat, // Passed uncalled on purpose — invoking it here would build the store (and // hit the macOS Keychain) while wiring the bridge at startup. llmConfig: context.getAiEditionLlmConfig, + mcp: context.getMcpController?.(), runChat: context.runAiEditionChat, undoLastToolBatch: context.undoAiEditionToolBatch, rewindToMessage: context.rewindToMessage, @@ -246,7 +251,7 @@ export function registerNativeBridgeHandlers(context: NativeBridgeContext) { deleteSession: context.deleteAiEditionChatSession, }); - const gifExportJobs = new GifExportJobs(); + const exportJobs = new ExportJobs(); ipcMain.handle(NATIVE_BRIDGE_CHANNEL, async (event, request: unknown) => { if (!isBridgeRequest(request)) { return createErrorResponse(undefined, "INVALID_REQUEST", "Invalid native bridge request."); @@ -380,18 +385,23 @@ export function registerNativeBridgeHandlers(context: NativeBridgeContext) { compositorViewService.setRect(request.payload.id, request.payload.rect); return createSuccessResponse(requestId, { ok: true }); case "readFrame": { - // The renderer polls this every rAF tick (~30fps). It passes the - // generation it last painted as `sinceGen`; native returns `null` when - // nothing newer exists (idle path — no buffer copy). On a new frame it - // returns `{ gen, width, height, data }`. The response wrapper does NOT - // JSON-stringify — `ipcMain.handle` round-trips via structured clone, - // which preserves the nested `Buffer` in `.data` as binary. - const frame = compositorViewService.readFrame( + // The renderer polls this every rAF tick. It passes the generation it + // last painted as `sinceGen`; native returns `null` when nothing newer + // exists (idle path — no buffer copy). On a new frame it returns + // `{ gen, width, height, data }`, or — for a view on shared textures — + // sends the texture to the asking frame and returns its receipt. The + // response wrapper does NOT JSON-stringify — `ipcMain.handle` round-trips + // via structured clone, which preserves the nested `Buffer` as binary. + const frame = await compositorViewService.readFrame( request.payload.id, request.payload.sinceGen, + event.senderFrame, ); return createSuccessResponse(requestId, frame); } + case "stopSharedFrames": + compositorViewService.stopSharedFrames(request.payload.id); + return createSuccessResponse(requestId, { ok: true }); case "setParam": compositorViewService.setParam( request.payload.id, @@ -423,17 +433,34 @@ export function registerNativeBridgeHandlers(context: NativeBridgeContext) { return createSuccessResponse(requestId, { ok: true }); case "exportMulti": { const sender = event.sender; - const stats = await compositorViewService.exportMulti( - request.payload.clips, - request.payload.outPath, - request.payload.sceneJson, - request.payload.params, - (frames) => { - if (!sender.isDestroyed()) { - sender.send("export:native-progress", frames); - } - }, - ); + const exportId = request.payload?.exportId; + if (exportId !== undefined && !isExportId(exportId)) { + return createErrorResponse(requestId, "INVALID_REQUEST", "Invalid export ID."); + } + const onProgress = (frames: number) => { + if (!sender.isDestroyed()) sender.send("export:native-progress", frames, exportId); + }; + const stats = exportId + ? await exportJobs.run( + sender, + exportId, + (progress) => + compositorViewService.startExportMulti( + request.payload.clips, + request.payload.outPath, + request.payload.sceneJson, + request.payload.params, + progress, + ), + onProgress, + ) + : await compositorViewService.exportMulti( + request.payload.clips, + request.payload.outPath, + request.payload.sceneJson, + request.payload.params, + onProgress, + ); if (!stats) { return createErrorResponse( requestId, @@ -446,14 +473,14 @@ export function registerNativeBridgeHandlers(context: NativeBridgeContext) { case "exportGif": { const sender = event.sender; const exportId = request.payload?.exportId; - if (exportId !== undefined && !isGifExportId(exportId)) { - return createErrorResponse(requestId, "INVALID_REQUEST", "Invalid GIF export ID."); + if (exportId !== undefined && !isExportId(exportId)) { + return createErrorResponse(requestId, "INVALID_REQUEST", "Invalid export ID."); } const onProgress = (frames: number) => { if (!sender.isDestroyed()) sender.send("export:native-progress", frames, exportId); }; const stats = exportId - ? await gifExportJobs.run( + ? await exportJobs.run( sender, exportId, (progress) => @@ -482,13 +509,13 @@ export function registerNativeBridgeHandlers(context: NativeBridgeContext) { } return createSuccessResponse(requestId, stats); } - case "cancelGifExport": { + case "cancelExport": { const exportId = request.payload?.exportId; - if (!isGifExportId(exportId)) { - return createErrorResponse(requestId, "INVALID_REQUEST", "Invalid GIF export ID."); + if (!isExportId(exportId)) { + return createErrorResponse(requestId, "INVALID_REQUEST", "Invalid export ID."); } return createSuccessResponse(requestId, { - accepted: gifExportJobs.cancel(event.sender, exportId), + accepted: exportJobs.cancel(event.sender, exportId), }); } default: @@ -573,6 +600,48 @@ export function registerNativeBridgeHandlers(context: NativeBridgeContext) { requestId, await aiEditionService.llmListProviderModels(request.payload.providerId), ); + case "mcp.getStatus": + return createSuccessResponse(requestId, await aiEditionService.mcpGetStatus()); + // Renderer input: check the types here, so a bad value is reported as + // the bad request it is rather than failing later as an internal error. + case "mcp.setEnabled": + if (typeof request.payload?.enabled !== "boolean") { + return createErrorResponse( + requestId, + "INVALID_REQUEST", + "Invalid MCP enabled flag.", + ); + } + return createSuccessResponse( + requestId, + await aiEditionService.mcpSetEnabled(request.payload.enabled), + ); + case "mcp.setPort": + if (!isValidMcpPort(request.payload?.port)) { + return createErrorResponse( + requestId, + "INVALID_REQUEST", + "MCP port must be a whole number between 1024 and 65535.", + ); + } + return createSuccessResponse( + requestId, + await aiEditionService.mcpSetPort(request.payload.port), + ); + case "mcp.setAllowEdits": + if (typeof request.payload?.allowEdits !== "boolean") { + return createErrorResponse( + requestId, + "INVALID_REQUEST", + "Invalid MCP edit permission.", + ); + } + return createSuccessResponse( + requestId, + await aiEditionService.mcpSetAllowEdits(request.payload.allowEdits), + ); + case "mcp.regenerateToken": + return createSuccessResponse(requestId, await aiEditionService.mcpRegenerateToken()); case "chat.run": { const sessionId = request.payload.sessionId; const sink = buildChatEventSink(event.sender, sessionId); @@ -746,11 +815,11 @@ export function registerNativeBridgeHandlers(context: NativeBridgeContext) { } catch (error) { if ( request.domain === "compositor" && - request.action === "exportGif" && error instanceof Error && - error.message === "GIF_EXPORT_CANCELLED" + ((request.action === "exportGif" && error.message === "GIF_EXPORT_CANCELLED") || + (request.action === "exportMulti" && error.message === "MP4_EXPORT_CANCELLED")) ) { - return createErrorResponse(requestId, "CANCELLED", "GIF export cancelled."); + return createErrorResponse(requestId, "CANCELLED", "Export cancelled."); } // Not retryable by default: most failures here are permanent (a missing // file, a bad payload, an unavailable addon), and a blanket `true` tells diff --git a/electron/ipc/nativeMacMidCaptureErrorWatch.test.ts b/electron/ipc/nativeMacMidCaptureErrorWatch.test.ts index c6958cd4b1..6eaec1ef30 100644 --- a/electron/ipc/nativeMacMidCaptureErrorWatch.test.ts +++ b/electron/ipc/nativeMacMidCaptureErrorWatch.test.ts @@ -6,7 +6,7 @@ describe("createNativeMacMidCaptureErrorWatch", () => { it("fires on an error raised after recording started", () => { const onError = vi.fn(); - const watch = createNativeMacMidCaptureErrorWatch(() => true, onError); + const watch = createNativeMacMidCaptureErrorWatch(() => true, onError, vi.fn()); watch({ event: "ready" }); watch({ event: "recording-started" }); @@ -17,7 +17,7 @@ describe("createNativeMacMidCaptureErrorWatch", () => { it("leaves an error raised before recording started to the start wait", () => { const onError = vi.fn(); - const watch = createNativeMacMidCaptureErrorWatch(() => true, onError); + const watch = createNativeMacMidCaptureErrorWatch(() => true, onError, vi.fn()); watch(error); @@ -26,7 +26,7 @@ describe("createNativeMacMidCaptureErrorWatch", () => { it("ignores events that are not errors", () => { const onError = vi.fn(); - const watch = createNativeMacMidCaptureErrorWatch(() => true, onError); + const watch = createNativeMacMidCaptureErrorWatch(() => true, onError, vi.fn()); watch({ event: "recording-started" }); watch({ event: "warning", code: "stop-capture-failed" }); @@ -38,7 +38,7 @@ describe("createNativeMacMidCaptureErrorWatch", () => { it("ignores a helper that is no longer the current process", () => { const onError = vi.fn(); let current = true; - const watch = createNativeMacMidCaptureErrorWatch(() => current, onError); + const watch = createNativeMacMidCaptureErrorWatch(() => current, onError, vi.fn()); watch({ event: "recording-started" }); current = false; @@ -50,7 +50,7 @@ describe("createNativeMacMidCaptureErrorWatch", () => { /** Killed or crashed: no error line ever comes, only the process closing. */ it("fires when the helper exits in the middle of a take", () => { const onTakeEnded = vi.fn(); - const watch = createNativeMacMidCaptureErrorWatch(() => true, onTakeEnded); + const watch = createNativeMacMidCaptureErrorWatch(() => true, onTakeEnded, vi.fn()); watch({ event: "recording-started" }); watch.exited(); @@ -60,7 +60,7 @@ describe("createNativeMacMidCaptureErrorWatch", () => { it("leaves an exit before recording started to the start wait", () => { const onTakeEnded = vi.fn(); - const watch = createNativeMacMidCaptureErrorWatch(() => true, onTakeEnded); + const watch = createNativeMacMidCaptureErrorWatch(() => true, onTakeEnded, vi.fn()); watch.exited(); @@ -71,7 +71,7 @@ describe("createNativeMacMidCaptureErrorWatch", () => { it("ignores the exit of a take that is already being stopped", () => { const onTakeEnded = vi.fn(); let live = true; - const watch = createNativeMacMidCaptureErrorWatch(() => live, onTakeEnded); + const watch = createNativeMacMidCaptureErrorWatch(() => live, onTakeEnded, vi.fn()); watch({ event: "recording-started" }); live = false; @@ -79,4 +79,34 @@ describe("createNativeMacMidCaptureErrorWatch", () => { expect(onTakeEnded).not.toHaveBeenCalled(); }); + + it("surfaces a system-audio warning once without ending the take", () => { + const onTakeEnded = vi.fn(); + const onSystemAudioUnavailable = vi.fn(); + const watch = createNativeMacMidCaptureErrorWatch( + () => true, + onTakeEnded, + onSystemAudioUnavailable, + ); + + watch({ event: "recording-started" }); + watch({ event: "warning", code: "system-audio-unavailable" }); + watch({ event: "warning", code: "system-audio-unavailable" }); + + expect(onSystemAudioUnavailable).toHaveBeenCalledOnce(); + expect(onTakeEnded).not.toHaveBeenCalled(); + }); + + it("does not surface the system-audio warning for a take that is no longer live", () => { + const onSystemAudioUnavailable = vi.fn(); + const watch = createNativeMacMidCaptureErrorWatch( + () => false, + vi.fn(), + onSystemAudioUnavailable, + ); + + watch({ event: "warning", code: "system-audio-unavailable" }); + + expect(onSystemAudioUnavailable).not.toHaveBeenCalled(); + }); }); diff --git a/electron/ipc/nativeMacMidCaptureErrorWatch.ts b/electron/ipc/nativeMacMidCaptureErrorWatch.ts index 6c786d3f37..48f8628886 100644 --- a/electron/ipc/nativeMacMidCaptureErrorWatch.ts +++ b/electron/ipc/nativeMacMidCaptureErrorWatch.ts @@ -15,12 +15,23 @@ export function createNativeMacMidCaptureErrorWatch( isLiveTake: () => boolean, onTakeEnded: () => void, + onSystemAudioUnavailable: () => void, ) { let recordingStarted = false; + let reportedSystemAudioUnavailable = false; const watch = (event: Record) => { if (event.event === "recording-started") { recordingStarted = true; } + if ( + event.event === "warning" && + event.code === "system-audio-unavailable" && + !reportedSystemAudioUnavailable && + isLiveTake() + ) { + reportedSystemAudioUnavailable = true; + onSystemAudioUnavailable(); + } if (event.event === "error" && recordingStarted && isLiveTake()) { onTakeEnded(); } diff --git a/electron/ipc/recordingPrefs.test.ts b/electron/ipc/recordingPrefs.test.ts index fe8cdce9dc..2c3317af93 100644 --- a/electron/ipc/recordingPrefs.test.ts +++ b/electron/ipc/recordingPrefs.test.ts @@ -19,6 +19,7 @@ const defaults: RecordingPrefs = { camEnabled: false, camDeviceId: null, camDeviceName: null, + camQuality: "2160p", systemAudioEnabled: false, cursorCaptureMode: "editable-overlay", hideDesktopIcons: false, diff --git a/electron/lingeringOverlay.test.ts b/electron/lingeringOverlay.test.ts new file mode 100644 index 0000000000..37700033ff --- /dev/null +++ b/electron/lingeringOverlay.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, it } from "vitest"; +import { isOnlyLingeringOverlay } from "./lingeringOverlay"; + +const window = (destroyed = false) => ({ isDestroyed: () => destroyed }); + +describe("isOnlyLingeringOverlay", () => { + it("is true when the hidden overlay is the last window open", () => { + const overlay = window(); + expect(isOnlyLingeringOverlay([overlay], overlay)).toBe(true); + expect(isOnlyLingeringOverlay([overlay, window(true)], overlay)).toBe(true); + }); + + it("is false while the HUD or the editor is still open", () => { + const overlay = window(); + expect(isOnlyLingeringOverlay([window(), overlay], overlay)).toBe(false); + }); + + it("is false without an overlay to close", () => { + expect(isOnlyLingeringOverlay([], null)).toBe(false); + const gone = window(true); + expect(isOnlyLingeringOverlay([gone], gone)).toBe(false); + }); +}); diff --git a/electron/lingeringOverlay.ts b/electron/lingeringOverlay.ts new file mode 100644 index 0000000000..35d64e64d6 --- /dev/null +++ b/electron/lingeringOverlay.ts @@ -0,0 +1,18 @@ +// The countdown overlay is created on the first take and only hidden between takes, so it is +// ready to paint the next countdown at once. A hidden window still counts for Electron, so +// once the HUD and the editor were gone `window-all-closed` never fired: OpenScreen stayed +// up with no window and only the tray icon left (issue #961). `main.ts` closes the overlay +// when it is the last window standing. + +/** The slice of `BrowserWindow` this needs, so the rule can be tested without Electron. */ +export type LingeringWindow = { isDestroyed(): boolean }; + +/** True when `overlay` is the only window still open, so it alone holds the app up. */ +export function isOnlyLingeringOverlay( + open: readonly LingeringWindow[], + overlay: LingeringWindow | null, +): boolean { + if (!overlay || overlay.isDestroyed()) return false; + const alive = open.filter((window) => !window.isDestroyed()); + return alive.length === 1 && alive[0] === overlay; +} diff --git a/electron/main.ts b/electron/main.ts index de3e3bbd75..13b8a8ad74 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -1,4 +1,3 @@ -import fs from "node:fs/promises"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { @@ -21,6 +20,7 @@ import { PRODUCT_NAME, usesNativeAboutPanel, } from "./about"; +import { flushChatPersistence } from "./ai-edition/chat-service"; import { AppSettingsStore } from "./app-settings"; import { blockedFromInstalling, @@ -58,6 +58,7 @@ import { getSelectedDesktopSource, registerIpcHandlers, } from "./ipc/handlers"; +import { isOnlyLingeringOverlay } from "./lingeringOverlay"; import { installMainProcessErrorGuards } from "./main-process-errors"; import { showMessageBoxOver } from "./messageBox"; import { @@ -65,10 +66,13 @@ import { showPermissionsWindow, showPermissionsWindowIfNeeded, } from "./permissions"; +import { setDisplaySleepBlocked } from "./recording/displaySleepBlocker"; +import { RecordingsDirManager } from "./recording/recordingsDirManager"; import { offersStarPrompt, REPO_URL, storeReviewUrl } from "./star-prompt"; import { registerSttIpc, shutdownStt } from "./stt"; import { checkLatestRelease } from "./update-checker"; import { loadUpdateMode, saveUpdateMode } from "./update-settings"; +import { configureWaylandSupport } from "./wayland"; import { createCountdownOverlayWindow, createEditorWindow, @@ -90,32 +94,27 @@ if (process.platform === "darwin") { app.commandLine.appendSwitch("disable-features", "MacCatapLoopbackAudioForScreenShare"); } -// Wayland support for screen capture and window management on Wayland compositors. +// Keep the session default while allowing explicit Xwayland launches. if (process.platform === "linux") { - const isWayland = - process.env.XDG_SESSION_TYPE === "wayland" || process.env.WAYLAND_DISPLAY !== undefined; - if (isWayland) { - app.commandLine.appendSwitch("ozone-platform", "wayland"); - // Enable WebRTCPipeWireCapturer for screen capture on Wayland - app.commandLine.appendSwitch("enable-features", "WaylandWindowDrag,WebRTCPipeWireCapturer"); - // Chromium's Wayland Ozone backend can't use Vulkan. When it tries, the WebRTC - // PipeWire capturer fails to import DMA-BUF frames into EGL (EGL_BAD_MATCH), the - // stream renegotiates, and screen recording yields no usable frames. Force the - // GL/EGL path so DMA-BUF import works. (Chromium itself logs this suggestion: - // "'--ozone-platform=wayland' is not compatible with Vulkan ... disabling Vulkan".) - app.commandLine.appendSwitch("disable-features", "Vulkan"); - } + configureWaylandSupport(app.commandLine, process.env); } disableHttpCacheForDevServer(app.commandLine, process.env); installMainProcessErrorGuards(); -export const RECORDINGS_DIR = path.join(app.getPath("userData"), "recordings"); +const recordingsDirManager = new RecordingsDirManager(app.getPath("userData"), () => isRecording); + +export const DEFAULT_RECORDINGS_DIR = recordingsDirManager.defaultDir; + +// Mutable: reassigned by setRecordingsDir() when the user picks another folder. +// `handlers.ts` imports this as a live named binding, so every call site there +// sees the change immediately, with no restart. +export let RECORDINGS_DIR = recordingsDirManager.dir; async function ensureRecordingsDir() { try { - await fs.mkdir(RECORDINGS_DIR, { recursive: true }); + await recordingsDirManager.ensureExists(); console.log("RECORDINGS_DIR:", RECORDINGS_DIR); console.log("User Data Path:", app.getPath("userData")); } catch (error) { @@ -123,6 +122,20 @@ async function ensureRecordingsDir() { } } +/** + * Switches where new recordings are written. `null` resets to the default + * (userData/recordings). Existing files stay where they are. Refused while a + * recording is active; see RecordingsDirManager.setDir. + */ +export async function setRecordingsDir(customDir: string | null): Promise { + RECORDINGS_DIR = await recordingsDirManager.setDir(customDir); + return RECORDINGS_DIR; +} + +export function getRecordingsDirInfo() { + return recordingsDirManager.getInfo(); +} + // The built directory structure // // ├─┬─┬ dist @@ -1088,6 +1101,17 @@ if (!cliCommand) { app.on("window-all-closed", () => { app.quit(); }); + // The countdown overlay hides between takes instead of closing, so it would keep + // `window-all-closed` from ever firing (#961). Close it with the last other window. Every + // HUD/editor switch opens the next window before the previous one's `closed` arrives, so + // this only fires when the user really closed the last one. + app.on("browser-window-created", (_, win) => { + win.once("closed", () => { + if (isOnlyLingeringOverlay(BrowserWindow.getAllWindows(), countdownOverlayWindow)) { + countdownOverlayWindow?.close(); + } + }); + }); } app.on("activate", () => { @@ -1128,6 +1152,9 @@ app.on("before-quit", (event) => { .catch((error) => { console.error("[stt] Failed to stop whisper helper during app quit:", error); }) + // Chat history is written asynchronously. Flushing after the helper stops also + // catches a reply that landed meanwhile; a turn still running is not waited for. + .then(flushChatPersistence) .finally(() => { sttShutdownFinished = true; app.quit(); @@ -1360,7 +1387,7 @@ appReady?.then(async () => { showMainWindow(); } - registerIpcHandlers( + const { mcpController } = registerIpcHandlers( createEditorWindowWrapper, createSourceSelectorWindowWrapper, createCountdownOverlayWindowWrapper, @@ -1372,6 +1399,7 @@ appReady?.then(async () => { (recording: boolean, sourceName: string) => { selectedSourceName = sourceName; isRecording = recording; + setDisplaySleepBlocked(recording); if (!tray) createTray(); updateTrayMenu(recording); // `canOfferUpdateCheck()` now answers "not mid-take" too, and the app/Help menus are @@ -1404,6 +1432,9 @@ appReady?.then(async () => { } createWindow(); + // Off unless the user turned it on in Settings → AI. Started here rather than + // in registerIpcHandlers so neither the headless CLI nor a bench run binds it. + void mcpController.startIfEnabled(); void showPermissionsWindowIfNeeded().catch((error) => console.warn("[permissions] could not read the permissions at launch:", error), ); diff --git a/electron/mcp/editor-document-host.test.ts b/electron/mcp/editor-document-host.test.ts new file mode 100644 index 0000000000..5b11cd19a0 --- /dev/null +++ b/electron/mcp/editor-document-host.test.ts @@ -0,0 +1,126 @@ +import { EventEmitter } from "node:events"; +import type { IpcMain, WebContents } from "electron"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { createEmptyDocument } from "../../src/lib/ai-edition/schema"; +import { + AI_EDITION_MCP_HOST_CHANNEL, + AI_EDITION_MCP_REQUEST_CHANNEL, + AI_EDITION_MCP_RESPONSE_CHANNEL, + type AiEditionMcpHostRequest, +} from "../../src/native/contracts"; +import { EditorDocumentHost } from "./editor-document-host"; + +/** A webContents that records what main sends it and can be destroyed. */ +class FakeWebContents extends EventEmitter { + sent: AiEditionMcpHostRequest[] = []; + destroyed = false; + send(channel: string, payload: AiEditionMcpHostRequest) { + if (channel === AI_EDITION_MCP_REQUEST_CHANNEL) this.sent.push(payload); + } + isDestroyed() { + return this.destroyed; + } + destroy() { + this.destroyed = true; + this.emit("destroyed"); + } +} + +function setup() { + const ipc = new EventEmitter(); + const host = new EditorDocumentHost(ipc as unknown as IpcMain); + const fromRenderer = (sender: FakeWebContents, channel: string, payload: unknown) => + ipc.emit(channel, { sender: sender as unknown as WebContents }, payload); + return { host, fromRenderer }; +} + +/** Answers the next request the editor receives. */ +async function answer( + editor: FakeWebContents, + fromRenderer: ReturnType["fromRenderer"], + result: unknown, + sender = editor, +) { + await vi.waitFor(() => expect(editor.sent.length).toBeGreaterThan(0)); + const request = editor.sent.shift(); + fromRenderer(sender, AI_EDITION_MCP_RESPONSE_CHANNEL, { requestId: request?.requestId, result }); +} + +afterEach(() => { + vi.useRealTimers(); +}); + +describe("EditorDocumentHost", () => { + it("answers null while no editor has registered", async () => { + const { host } = setup(); + await expect(host.snapshot()).resolves.toBeNull(); + }); + + it("asks the registered editor for its snapshot", async () => { + const { host, fromRenderer } = setup(); + const editor = new FakeWebContents(); + fromRenderer(editor, AI_EDITION_MCP_HOST_CHANNEL, true); + const pending = host.snapshot(); + await answer(editor, fromRenderer, { document: { id: "d" }, revision: 3 }); + await expect(pending).resolves.toEqual({ document: { id: "d" }, revision: 3 }); + }); + + it("treats a snapshot without a numeric revision as no snapshot", async () => { + const { host, fromRenderer } = setup(); + const editor = new FakeWebContents(); + fromRenderer(editor, AI_EDITION_MCP_HOST_CHANNEL, true); + // A missing revision would reach the apply guard as `undefined` and skip it. + for (const revision of [undefined, "3", Number.NaN]) { + const pending = host.snapshot(); + await answer(editor, fromRenderer, { document: { id: "d" }, revision }); + await expect(pending).resolves.toBeNull(); + } + }); + + it("passes the apply verdict through", async () => { + const { host, fromRenderer } = setup(); + const editor = new FakeWebContents(); + fromRenderer(editor, AI_EDITION_MCP_HOST_CHANNEL, true); + const document = createEmptyDocument({ projectId: "p", title: "t" }); + const pending = host.apply(document, 7); + await vi.waitFor(() => expect(editor.sent).toHaveLength(1)); + expect(editor.sent[0]).toMatchObject({ op: "apply", expectedRevision: 7 }); + await answer(editor, fromRenderer, "conflict"); + await expect(pending).resolves.toBe("conflict"); + }); + + it("ignores a reply from any other window", async () => { + vi.useFakeTimers(); + const { host, fromRenderer } = setup(); + const editor = new FakeWebContents(); + fromRenderer(editor, AI_EDITION_MCP_HOST_CHANNEL, true); + const pending = host.snapshot(); + await answer(editor, fromRenderer, { document: {}, revision: 1 }, new FakeWebContents()); + await vi.advanceTimersByTimeAsync(30_000); + await expect(pending).resolves.toBeNull(); + }); + + it("forgets an editor that unregisters or is destroyed", async () => { + const { host, fromRenderer } = setup(); + const first = new FakeWebContents(); + fromRenderer(first, AI_EDITION_MCP_HOST_CHANNEL, true); + fromRenderer(first, AI_EDITION_MCP_HOST_CHANNEL, false); + await expect(host.snapshot()).resolves.toBeNull(); + + const second = new FakeWebContents(); + fromRenderer(second, AI_EDITION_MCP_HOST_CHANNEL, true); + second.destroy(); + await expect(host.snapshot()).resolves.toBeNull(); + expect(second.sent).toHaveLength(0); + }); + + it("reports an unanswered apply as a timeout, not as a failure", async () => { + vi.useFakeTimers(); + const { host, fromRenderer } = setup(); + const editor = new FakeWebContents(); + fromRenderer(editor, AI_EDITION_MCP_HOST_CHANNEL, true); + const pending = host.apply(createEmptyDocument({ projectId: "p", title: "t" }), 1); + await vi.advanceTimersByTimeAsync(30_000); + await expect(pending).resolves.toBe("timeout"); + }); +}); diff --git a/electron/mcp/editor-document-host.ts b/electron/mcp/editor-document-host.ts new file mode 100644 index 0000000000..55d6d61569 --- /dev/null +++ b/electron/mcp/editor-document-host.ts @@ -0,0 +1,107 @@ +// The MCP server's door onto the live document, which the editor window owns. +// +// The open project lives in the renderer's store (with the revision that guards +// agent writes), not in the main process, so each MCP call asks the editor for +// a snapshot and hands the result back to the editor's own revision-guarded +// apply — the same `applyAgentDocumentIfCurrent` an in-app chat turn uses. +// +// The editor announces itself on AI_EDITION_MCP_HOST_CHANNEL when it starts +// answering, and again with `false` when it stops; a destroyed window counts as +// stopped. Only that one webContents is ever asked, and only its replies count. + +import { randomUUID } from "node:crypto"; +import type { IpcMain, IpcMainEvent, WebContents } from "electron"; +import type { AxcutDocument } from "../../src/lib/ai-edition/schema"; +import { + AI_EDITION_MCP_HOST_CHANNEL, + AI_EDITION_MCP_REQUEST_CHANNEL, + AI_EDITION_MCP_RESPONSE_CHANNEL, + type AiEditionMcpHostRequest, + type AiEditionMcpHostResponse, + type AiEditionMcpHostSnapshot, +} from "../../src/native/contracts"; +import type { McpApplyResult, McpDocumentHost } from "./openscreen-mcp-server"; + +// Long enough for a save of a large project to land, short enough that a hung +// renderer does not hold the client's call open indefinitely. +const REQUEST_TIMEOUT_MS = 30_000; + +const TIMED_OUT = Symbol("timed-out"); + +interface Pending { + resolve: (value: unknown) => void; + timer: ReturnType; +} + +type RequestBody = + | { op: "snapshot" } + | { op: "apply"; document: unknown; expectedRevision: number }; + +export class EditorDocumentHost implements McpDocumentHost { + private editor: WebContents | null = null; + private readonly pending = new Map(); + + constructor(ipcMain: IpcMain) { + ipcMain.on(AI_EDITION_MCP_HOST_CHANNEL, (event: IpcMainEvent, active: unknown) => { + if (active === true) { + this.editor = event.sender; + event.sender.once("destroyed", () => { + if (this.editor === event.sender) this.editor = null; + }); + } else if (this.editor === event.sender) { + this.editor = null; + } + }); + ipcMain.on(AI_EDITION_MCP_RESPONSE_CHANNEL, (event: IpcMainEvent, message: unknown) => { + if (event.sender !== this.editor) return; + const response = message as AiEditionMcpHostResponse | null; + if (!response || typeof response.requestId !== "string") return; + const pending = this.pending.get(response.requestId); + if (!pending) return; + clearTimeout(pending.timer); + this.pending.delete(response.requestId); + pending.resolve(response.result); + }); + } + + private request(body: RequestBody): Promise { + const editor = this.editor; + if (!editor || editor.isDestroyed()) return Promise.resolve(undefined); + const requestId = randomUUID(); + return new Promise((resolve) => { + const timer = setTimeout(() => { + this.pending.delete(requestId); + resolve(TIMED_OUT); + }, REQUEST_TIMEOUT_MS); + this.pending.set(requestId, { resolve, timer }); + try { + editor.send(AI_EDITION_MCP_REQUEST_CHANNEL, { + requestId, + ...body, + } satisfies AiEditionMcpHostRequest); + } catch { + clearTimeout(timer); + this.pending.delete(requestId); + resolve(undefined); + } + }); + } + + async snapshot(): Promise { + const result = await this.request({ op: "snapshot" }); + if (!result || result === TIMED_OUT || typeof result !== "object") return null; + // The revision is the apply guard: one that is missing or not a number would + // reach `applyAgentDocumentIfCurrent` as `undefined`, which skips the + // stale-edit check entirely. Treat such a reply as no snapshot at all. + const snapshot = result as Partial; + if (typeof snapshot.revision !== "number" || !Number.isFinite(snapshot.revision)) return null; + return snapshot as AiEditionMcpHostSnapshot; + } + + async apply(document: AxcutDocument, expectedRevision: number): Promise { + const result = await this.request({ op: "apply", document, expectedRevision }); + if (result === TIMED_OUT) return "timeout"; + if (typeof result !== "string") return "no-editor"; + return result as McpApplyResult; + } +} diff --git a/electron/mcp/hidden-video.ts b/electron/mcp/hidden-video.ts new file mode 100644 index 0000000000..a2f94e6c89 --- /dev/null +++ b/electron/mcp/hidden-video.ts @@ -0,0 +1,106 @@ +// A video decoded by Chromium's own