WIP: @remotion/whisper-webgpu:

package.json

@@ -14,7 +14,7 @@
 		"testlambda": "turbo run testlambda --concurrency=1 --no-update-notifier",
 		"ci": "turbo run make test --concurrency=1 --no-update-notifier",
 		"watch": "turbo watch make --concurrency=2 --experimental-write-cache --ui=tui",
-		"watchwhisperwasm": "turbo watch make --experimental-write-cache --filter='@remotion/whisper-wasm'",
+		"watchwhisperwebpu": "turbo watch make --experimental-write-cache --filter='@remotion/whisper-webgpu'",
 		"watchwebcodecs": "turbo watch make --experimental-write-cache --filter='@remotion/media-parser' --filter='@remotion/webcodecs'",
 		"watchcore": "turbo watch make --experimental-write-cache --filter='remotion'",
 		"watchstudio": "turbo watch make --experimental-write-cache --filter='@remotion/studio' --filter='@remotion/studio-server'",

packages/STATS.md

@@ -1,4 +1,5 @@
 # Download statistics
+
 Monthly downloads of Remotion packages  
 [![NPM Downloads](https://img.shields.io/npm/dm/@remotion/animated-emoji.svg?style=flat&color=black&label=@remotion/animated-emoji)](https://npmcharts.com/compare/@remotion/animated-emoji?minimal=true)  
 [![NPM Downloads](https://img.shields.io/npm/dm/@remotion/animation-utils.svg?style=flat&color=black&label=@remotion/animation-utils)](https://npmcharts.com/compare/@remotion/animation-utils?minimal=true)  
@@ -54,5 +55,5 @@ Monthly downloads of Remotion packages
 [![NPM Downloads](https://img.shields.io/npm/dm/@remotion/three.svg?style=flat&color=black&label=@remotion/three)](https://npmcharts.com/compare/@remotion/three?minimal=true)  
 [![NPM Downloads](https://img.shields.io/npm/dm/@remotion/transitions.svg?style=flat&color=black&label=@remotion/transitions)](https://npmcharts.com/compare/@remotion/transitions?minimal=true)  
 [![NPM Downloads](https://img.shields.io/npm/dm/@remotion/webcodecs.svg?style=flat&color=black&label=@remotion/webcodecs)](https://npmcharts.com/compare/@remotion/webcodecs?minimal=true)  
-[![NPM Downloads](https://img.shields.io/npm/dm/@remotion/whisper-wasm.svg?style=flat&color=black&label=@remotion/whisper-wasm)](https://npmcharts.com/compare/@remotion/whisper-wasm?minimal=true)  
-[![NPM Downloads](https://img.shields.io/npm/dm/@remotion/zod-types.svg?style=flat&color=black&label=@remotion/zod-types)](https://npmcharts.com/compare/@remotion/zod-types?minimal=true)  
+[![NPM Downloads](https://img.shields.io/npm/dm/@remotion/whisper-webgpu.svg?style=flat&color=black&label=@remotion/whisper-webgpu)](https://npmcharts.com/compare/@remotion/whisper-webgpu?minimal=true)  
+[![NPM Downloads](https://img.shields.io/npm/dm/@remotion/zod-types.svg?style=flat&color=black&label=@remotion/zod-types)](https://npmcharts.com/compare/@remotion/zod-types?minimal=true)

packages/ai-improvements/package.json

@@ -9,5 +9,5 @@
 		"@babel/types": "7.24.0"
 	},
 	"private": true,
-	"version": "4.0.300"
+	"version": "4.0.301"
 }

packages/animated-emoji/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@remotion/animated-emoji",
-	"version": "4.0.300",
+	"version": "4.0.301",
 	"main": "dist/cjs/index.js",
 	"types": "dist/cjs/index.d.ts",
 	"module": "dist/esm/index.mjs",

packages/animation-utils/package.json

@@ -7,7 +7,7 @@
 		"name": "Chetan Karwa",
 		"email": "[email protected]"
 	},
-	"version": "4.0.300",
+	"version": "4.0.301",
 	"description": "Helpers for animating CSS properties",
 	"main": "./dist/index.js",
 	"module": "./dist/index.mjs",

packages/astro-example/package.json

@@ -5,7 +5,7 @@
 	"name": "@remotion/astro-example",
 	"private": true,
 	"type": "module",
-	"version": "4.0.300",
+	"version": "4.0.301",
 	"scripts": {
 		"dev": "astro dev",
 		"start": "astro dev",

packages/babel-loader/package.json

@@ -3,7 +3,7 @@
 		"url": "https://github.com/remotion-dev/remotion/tree/main/packages/babel-loader"
 	},
 	"name": "@remotion/babel-loader",
-	"version": "4.0.300",
+	"version": "4.0.301",
 	"description": "Babel loader for Remotion",
 	"main": "dist/index.js",
 	"scripts": {

packages/bugs/package.json

@@ -4,5 +4,5 @@
 	},
 	"name": "bugs",
 	"private": true,
-	"version": "4.0.300"
+	"version": "4.0.301"
 }

packages/bundler/package.json

@@ -3,7 +3,7 @@
 		"url": "https://github.com/remotion-dev/remotion/tree/main/packages/bundler"
 	},
 	"name": "@remotion/bundler",
-	"version": "4.0.300",
+	"version": "4.0.301",
 	"description": "Bundle Remotion compositions using Webpack",
 	"main": "dist/index.js",
 	"sideEffects": false,

packages/captions/package.json

@@ -3,7 +3,7 @@
 		"url": "https://github.com/remotion-dev/remotion/tree/main/packages/captions"
 	},
 	"name": "@remotion/captions",
-	"version": "4.0.300",
+	"version": "4.0.301",
 	"description": "Primitives for dealing with captions",
 	"main": "dist/index.js",
 	"sideEffects": false,

packages/cli/package.json

@@ -3,7 +3,7 @@
 		"url": "https://github.com/remotion-dev/remotion/tree/main/packages/cli"
 	},
 	"name": "@remotion/cli",
-	"version": "4.0.300",
+	"version": "4.0.301",
 	"description": "Control Remotion features using the `npx remotion` command",
 	"main": "dist/index.js",
 	"sideEffects": false,

packages/cli/src/list-of-remotion-packages.ts

@@ -76,5 +76,5 @@ export const listOfRemotionPackages = [
 	'@remotion/openai-whisper',
 	'@remotion/compositor',
 	'@remotion/example-videos',
-	'@remotion/whisper-wasm',
+	'@remotion/whisper-webgpu',
 ];

packages/cloudrun/container/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "cloud-run-render",
-	"version": "4.0.300",
+	"version": "4.0.301",
 	"description": "Render media and stills on GCP Cloud Run",
 	"main": "dist/index.js",
 	"scripts": {

packages/cloudrun/package.json

@@ -3,7 +3,7 @@
 		"url": "https://github.com/remotion-dev/remotion/tree/main/packages/cloudrun"
 	},
 	"name": "@remotion/cloudrun",
-	"version": "4.0.300",
+	"version": "4.0.301",
 	"description": "Render Remotion videos on Google Cloud Run",
 	"main": "dist/index.js",
 	"sideEffects": false,

packages/compositor-darwin-arm64/package.json

@@ -2,7 +2,7 @@
 	"repository": {
 		"url": "https://github.com/remotion-dev/remotion/tree/main/packages/compositor-darwin-arm64"
 	},
-	"version": "4.0.300",
+	"version": "4.0.301",
 	"name": "@remotion/compositor-darwin-arm64",
 	"os": [
 		"darwin"

packages/compositor-darwin-x64/package.json

@@ -2,7 +2,7 @@
 	"repository": {
 		"url": "https://github.com/remotion-dev/remotion/tree/main/packages/compositor-darwin-x64"
 	},
-	"version": "4.0.300",
+	"version": "4.0.301",
 	"name": "@remotion/compositor-darwin-x64",
 	"os": [
 		"darwin"

packages/compositor-linux-arm64-gnu/package.json

@@ -2,7 +2,7 @@
 	"repository": {
 		"url": "https://github.com/remotion-dev/remotion/tree/main/packages/compositor-linux-arm64-gnu"
 	},
-	"version": "4.0.300",
+	"version": "4.0.301",
 	"name": "@remotion/compositor-linux-arm64-gnu",
 	"os": [
 		"linux"

packages/compositor-linux-arm64-musl/package.json

@@ -2,7 +2,7 @@
 	"repository": {
 		"url": "https://github.com/remotion-dev/remotion/tree/main/packages/compositor-linux-arm64-musl"
 	},
-	"version": "4.0.300",
+	"version": "4.0.301",
 	"name": "@remotion/compositor-linux-arm64-musl",
 	"os": [
 		"linux"

packages/compositor-linux-x64-gnu/package.json

@@ -2,7 +2,7 @@
 	"repository": {
 		"url": "https://github.com/remotion-dev/remotion/tree/main/packages/compositor-linux-x64-gnu"
 	},
-	"version": "4.0.300",
+	"version": "4.0.301",
 	"name": "@remotion/compositor-linux-x64-gnu",
 	"os": [
 		"linux"

packages/compositor-linux-x64-musl/package.json

@@ -2,7 +2,7 @@
 	"repository": {
 		"url": "https://github.com/remotion-dev/remotion/tree/main/packages/compositor-linux-x64-musl"
 	},
-	"version": "4.0.300",
+	"version": "4.0.301",
 	"name": "@remotion/compositor-linux-x64-musl",
 	"os": [
 		"linux"

packages/compositor-win32-x64-msvc/package.json

@@ -2,7 +2,7 @@
 	"repository": {
 		"url": "https://github.com/remotion-dev/remotion/tree/main/packages/compositor-win32-x64-msvc"
 	},
-	"version": "4.0.300",
+	"version": "4.0.301",
 	"name": "@remotion/compositor-win32-x64-msvc",
 	"os": [
 		"win32"

packages/compositor/package.json

@@ -3,7 +3,7 @@
 		"url": "https://github.com/remotion-dev/remotion/tree/main/packages/compositor"
 	},
 	"name": "@remotion/compositor",
-	"version": "4.0.300",
+	"version": "4.0.301",
 	"description": "Rust binary for Remotion",
 	"sideEffects": false,
 	"scripts": {

packages/convert/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@remotion/convert",
-	"version": "4.0.300",
+	"version": "4.0.301",
 	"private": true,
 	"sideEffects": false,
 	"type": "module",

packages/core/package.json

@@ -3,7 +3,7 @@
 		"url": "https://github.com/remotion-dev/remotion/tree/main/packages/core"
 	},
 	"name": "remotion",
-	"version": "4.0.300",
+	"version": "4.0.301",
 	"description": "Make videos programmatically",
 	"main": "dist/cjs/index.js",
 	"types": "dist/cjs/index.d.ts",

packages/core/src/buffer-until-first-frame.ts

@@ -51,7 +51,7 @@ export const useBufferUntilFirstFrame = ({
 			}
 
 			if (
-				current.readyState >= current.HAVE_ENOUGH_DATA &&
+				current.readyState >= current.HAVE_FUTURE_DATA &&
 				!isSafariWebkit() &&
 				// In Desktop Chrome, the video might switch to playing
 				// but does not play due to Bluetooth headphones

packages/core/src/media-tag-current-time-timestamp.ts

@@ -0,0 +1,23 @@
+import React from 'react';
+
+export const useCurrentTimeOfMediaTagWithUpdateTimeStamp = (
+	mediaRef: React.RefObject<HTMLVideoElement | HTMLAudioElement | null>,
+) => {
+	const lastUpdate: React.RefObject<{
+		time: number;
+		lastUpdate: number;
+	}> = React.useRef({
+		time: mediaRef.current?.currentTime ?? 0,
+		lastUpdate: performance.now(),
+	});
+
+	const nowCurrentTime = mediaRef.current?.currentTime ?? null;
+	if (nowCurrentTime !== null) {
+		if (lastUpdate.current.time !== nowCurrentTime) {
+			lastUpdate.current.time = nowCurrentTime;
+			lastUpdate.current.lastUpdate = performance.now();
+		}
+	}
+
+	return lastUpdate;
+};

packages/core/src/use-amplification.ts

@@ -80,6 +80,12 @@ export const useAmplification = ({
 			logLevel,
 			`Starting to amplify ${mediaRef.current?.src}. Gain = ${currentVolumeRef.current}`,
 		);
+
+		return () => {
+			gainNode.disconnect();
+			source.disconnect();
+			audioContext.close();
+		};
 	}, [logLevel, mediaRef, shouldAmplify]);
 
 	if (audioStuffRef.current) {

packages/core/src/use-media-playback.ts

@@ -5,6 +5,7 @@ import {useBufferUntilFirstFrame} from './buffer-until-first-frame.js';
 import {BufferingContextReact, useIsPlayerBuffering} from './buffering.js';
 import {useLogLevel, useMountTime} from './log-level-context.js';
 import {Log} from './log.js';
+import {useCurrentTimeOfMediaTagWithUpdateTimeStamp} from './media-tag-current-time-timestamp.js';
 import {playAndHandleNotAllowedError} from './play-and-handle-not-allowed-error.js';
 import {playbackLogging} from './playback-logging.js';
 import {seek} from './seek.js';
@@ -77,13 +78,16 @@ export const useMediaPlayback = ({
 		isVariableFpsVideoMap.current[src] = true;
 	}, [logLevel, src]);
 
-	const currentTime = useRequestVideoCallbackTime({
+	const rvcCurrentTime = useRequestVideoCallbackTime({
 		mediaRef,
 		mediaType,
 		lastSeek,
 		onVariableFpsVideoDetected,
 	});
 
+	const mediaTagCurrentTime =
+		useCurrentTimeOfMediaTagWithUpdateTimeStamp(mediaRef);
+
 	const desiredUnclampedTime = getMediaTime({
 		frame,
 		playbackRate: localPlaybackRate,
@@ -199,16 +203,23 @@ export const useMediaPlayback = ({
 				? Math.min(duration, desiredUnclampedTime)
 				: desiredUnclampedTime;
 
-		const mediaTagTime = mediaRef.current.currentTime;
-		const rvcTime = currentTime.current ?? null;
+		const mediaTagTime = mediaTagCurrentTime.current.time;
+		const rvcTime = rvcCurrentTime.current?.time ?? null;
 
 		const isVariableFpsVideo = isVariableFpsVideoMap.current[src];
 
 		const timeShiftMediaTag = Math.abs(shouldBeTime - mediaTagTime);
 		const timeShiftRvcTag = rvcTime ? Math.abs(shouldBeTime - rvcTime) : null;
+
+		const mostRecentTimeshift =
+			rvcCurrentTime.current?.lastUpdate &&
+			rvcCurrentTime.current.time > mediaTagCurrentTime.current.lastUpdate
+				? (timeShiftRvcTag as number)
+				: timeShiftMediaTag;
+
 		const timeShift =
 			timeShiftRvcTag && !isVariableFpsVideo
-				? timeShiftRvcTag
+				? mostRecentTimeshift
 				: timeShiftMediaTag;
 
 		if (
@@ -320,7 +331,7 @@ export const useMediaPlayback = ({
 		acceptableTimeShiftButLessThanDuration,
 		bufferUntilFirstFrame,
 		buffering.buffering,
-		currentTime,
+		rvcCurrentTime,
 		logLevel,
 		desiredUnclampedTime,
 		isBuffering,
@@ -335,5 +346,6 @@ export const useMediaPlayback = ({
 		isPremounting,
 		pauseWhenBuffering,
 		mountTime,
+		mediaTagCurrentTime,
 	]);
 };

packages/core/src/use-request-video-callback-time.ts

@@ -12,12 +12,18 @@ export const useRequestVideoCallbackTime = ({
 	lastSeek: React.MutableRefObject<number | null>;
 	onVariableFpsVideoDetected: () => void;
 }) => {
-	const currentTime = useRef<number | null>(null);
+	const currentTime = useRef<{
+		time: number;
+		lastUpdate: number;
+	} | null>(null);
 
 	useEffect(() => {
 		const {current} = mediaRef;
 		if (current) {
-			currentTime.current = current.currentTime;
+			currentTime.current = {
+				time: current.currentTime,
+				lastUpdate: performance.now(),
+			};
 		} else {
 			currentTime.current = null;
 			return;
@@ -43,7 +49,9 @@ export const useRequestVideoCallbackTime = ({
 
 			const cb = videoTag.requestVideoFrameCallback((_, info) => {
 				if (currentTime.current !== null) {
-					const difference = Math.abs(currentTime.current - info.mediaTime);
+					const difference = Math.abs(
+						currentTime.current.time - info.mediaTime,
+					);
 					const differenceToLastSeek = Math.abs(
 						lastSeek.current === null
 							? Infinity
@@ -56,13 +64,16 @@ export const useRequestVideoCallbackTime = ({
 					if (
 						difference > 0.5 &&
 						differenceToLastSeek > 0.5 &&
-						info.mediaTime > currentTime.current
+						info.mediaTime > currentTime.current.time
 					) {
 						onVariableFpsVideoDetected();
 					}
 				}
 
-				currentTime.current = info.mediaTime;
+				currentTime.current = {
+					time: info.mediaTime,
+					lastUpdate: performance.now(),
+				};
 				request();
 			});
 

packages/core/src/version.ts

@@ -5,4 +5,4 @@
  * @see [Documentation](https://remotion.dev/docs/version)
  * @returns {string} The current version of the remotion package
  */
-export const VERSION = '4.0.300';
+export const VERSION = '4.0.301';

packages/create-video/package.json

@@ -3,7 +3,7 @@
 		"url": "https://github.com/remotion-dev/remotion/tree/main/packages/create-video"
 	},
 	"name": "create-video",
-	"version": "4.0.300",
+	"version": "4.0.301",
 	"description": "Create a new Remotion project",
 	"main": "dist/index.js",
 	"bin": {

packages/create-video/src/list-of-remotion-packages.ts

@@ -76,5 +76,5 @@ export const listOfRemotionPackages = [
 	'@remotion/openai-whisper',
 	'@remotion/compositor',
 	'@remotion/example-videos',
-	'@remotion/whisper-wasm',
+	'@remotion/whisper-webgpu',
 ];