From e79d430009f53a6b252f37840785e3d4b6511f99 Mon Sep 17 00:00:00 2001 From: Oliver Sluke <22557015+oliversluke@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:32:37 +0200 Subject: [PATCH 1/3] Fix H.265 decoder crushing 10-bit output and discarding bitstream colorimetry The VideoToolbox decompression session pinned its output to 8-bit NV12, and overrideColorSpaceAttachments() unconditionally stamped BT.709/sRGB attachments on every frame. Together these break HEVC Main10/HDR10 reception: 10-bit content is bit-crushed and PQ/BT.2020 colorimetry from the bitstream VUI is replaced, so HDR streams render washed out. Request a 10-bit biplanar output format when the hvcC configuration record signals a luma bit depth above 8, and only apply the BT.709/sRGB fallback attachments when the format description carries no colour information from the bitstream. 8-bit streams without VUI colour info are byte-for-byte unaffected. Co-Authored-By: Claude Fable 5 --- .../video_codec/RTCVideoDecoderH265.mm | 47 +++++++++++++++++-- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/sdk/objc/components/video_codec/RTCVideoDecoderH265.mm b/sdk/objc/components/video_codec/RTCVideoDecoderH265.mm index 10e6761fee5..0b6d595e14d 100644 --- a/sdk/objc/components/video_codec/RTCVideoDecoderH265.mm +++ b/sdk/objc/components/video_codec/RTCVideoDecoderH265.mm @@ -42,6 +42,7 @@ @interface RTC_OBJC_TYPE (RTCVideoDecoderH265) () - (void)setError:(OSStatus)error; - (void)processFrame:(RTC_OBJC_TYPE(RTCVideoFrame) *)decodedFrame reorderSize:(uint64_t)reorderSize; +- (bool)bitstreamCarriesColorInfo; @end static void overrideColorSpaceAttachments(CVImageBufferRef imageBuffer) { @@ -183,7 +184,14 @@ void h265DecompressionOutputCallback(void *decoderRef, void *params, OSStatus st return; } - overrideColorSpaceAttachments(imageBuffer); + // Only guess colour attachments when the bitstream signalled none. Streams + // that carry VUI colour information (e.g. HDR10: PQ transfer + BT.2020 + // primaries) already have correct attachments propagated by VideoToolbox + // from the format description; overriding them mistags the frames as + // BT.709/sRGB and HDR content renders washed out. + if (![decoder bitstreamCarriesColorInfo]) { + overrideColorSpaceAttachments(imageBuffer); + } // TODO(tkchin): Handle CVO properly. RTC_OBJC_TYPE(RTCCVPixelBuffer) *frameBuffer = @@ -420,8 +428,13 @@ - (int)resetDecompressionSession { #endif kCVPixelBufferIOSurfacePropertiesKey, kCVPixelBufferPixelFormatTypeKey}; CFDictionaryRef ioSurfaceValue = CreateCFTypeDictionary(nullptr, nullptr, 0); - int64_t nv12type = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange; - CFNumberRef pixelFormat = CFNumberCreate(nullptr, kCFNumberLongType, &nv12type); + // Forcing NV12 would crush high bit depth output (e.g. HEVC Main10) to + // 8 bits. Request a 10-bit biplanar format for such streams; 8-bit streams + // keep NV12 so existing consumers are unaffected. + int64_t pixelFormatType = [self isHighBitDepthFormat] + ? kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange + : kCVPixelFormatType_420YpCbCr8BiPlanarFullRange; + CFNumberRef pixelFormat = CFNumberCreate(nullptr, kCFNumberLongType, &pixelFormatType); CFTypeRef values[attributesSize] = {kCFBooleanTrue, ioSurfaceValue, pixelFormat}; CFDictionaryRef attributes = CreateCFTypeDictionary(keys, values, attributesSize); if (ioSurfaceValue) { @@ -453,6 +466,34 @@ - (void)configureDecompressionSession { VTSessionSetProperty(_decompressionSession, kVTDecompressionPropertyKey_RealTime, kCFBooleanTrue); } +// Returns true when the active format description signals a luma bit depth +// above 8 (e.g. HEVC Main10), read from bit_depth_luma_minus8 in the hvcC +// decoder configuration record. +- (bool)isHighBitDepthFormat { + if (!_videoFormat) { + return false; + } + CFDictionaryRef atoms = (CFDictionaryRef)CMFormatDescriptionGetExtension( + _videoFormat, kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms); + if (!atoms) { + return false; + } + CFDataRef hvcc = (CFDataRef)CFDictionaryGetValue(atoms, (CFStringRef) @"hvcC"); + if (!hvcc || CFDataGetLength(hvcc) < 18) { + return false; + } + return (CFDataGetBytePtr(hvcc)[17] & 0x07) > 0; +} + +// Whether the active format description carries colour information parsed +// from the bitstream (VUI colour description). VideoToolbox propagates these +// extensions onto output pixel buffers, so no fallback tagging is needed. +- (bool)bitstreamCarriesColorInfo { + return _videoFormat && + CMFormatDescriptionGetExtension(_videoFormat, kCMFormatDescriptionExtension_ColorPrimaries) != + nullptr; +} + - (void)destroyDecompressionSession { if (_decompressionSession) { VTDecompressionSessionWaitForAsynchronousFrames(_decompressionSession); From d4af43d95f80b74ef36e36baa770abbac596d3b9 Mon Sep 17 00:00:00 2001 From: Oliver Sluke <22557015+oliversluke@users.noreply.github.com> Date: Fri, 10 Jul 2026 12:22:36 +0200 Subject: [PATCH 2/3] Capture bitstream colour info per frame instead of reading decoder state from the VideoToolbox callback The decompression output callback runs on VideoToolbox's async callback thread and read _videoFormat via bitstreamCarriesColorInfo. decodeData: releases _videoFormat (through setVideoFormat:) on a mid-stream format change before destroyDecompressionSession waits for in-flight frames, leaving a use-after-free window; frames from the old session could also be tagged against the new format's colour info. Capture the flag in RTCH265FrameDecodeParams on the decode thread at decode time, so the callback never touches decoder state and each frame is judged against the format it was actually decoded with. Co-Authored-By: Claude Fable 5 --- .../video_codec/RTCVideoDecoderH265.mm | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/sdk/objc/components/video_codec/RTCVideoDecoderH265.mm b/sdk/objc/components/video_codec/RTCVideoDecoderH265.mm index 0b6d595e14d..29ea0d69a1b 100644 --- a/sdk/objc/components/video_codec/RTCVideoDecoderH265.mm +++ b/sdk/objc/components/video_codec/RTCVideoDecoderH265.mm @@ -33,16 +33,22 @@ // Struct that we pass to the decoder per frame to decode. We receive it again // in the decoder callback. struct RTCH265FrameDecodeParams { - RTCH265FrameDecodeParams(int64_t ts, uint64_t reorderSize) - : timestamp(ts), reorderSize(reorderSize) {} + RTCH265FrameDecodeParams(int64_t ts, uint64_t reorderSize, bool bitstreamCarriesColorInfo) + : timestamp(ts), + reorderSize(reorderSize), + bitstreamCarriesColorInfo(bitstreamCarriesColorInfo) {} int64_t timestamp; uint64_t reorderSize{0}; + // Captured on the decode thread at decode time. The VideoToolbox output + // callback runs on a separate thread and must not read decoder state: + // _videoFormat may be released and replaced by a new format while frames + // decoded against the previous one are still in flight. + bool bitstreamCarriesColorInfo{false}; }; @interface RTC_OBJC_TYPE (RTCVideoDecoderH265) () - (void)setError:(OSStatus)error; - (void)processFrame:(RTC_OBJC_TYPE(RTCVideoFrame) *)decodedFrame reorderSize:(uint64_t)reorderSize; -- (bool)bitstreamCarriesColorInfo; @end static void overrideColorSpaceAttachments(CVImageBufferRef imageBuffer) { @@ -188,8 +194,10 @@ void h265DecompressionOutputCallback(void *decoderRef, void *params, OSStatus st // that carry VUI colour information (e.g. HDR10: PQ transfer + BT.2020 // primaries) already have correct attachments propagated by VideoToolbox // from the format description; overriding them mistags the frames as - // BT.709/sRGB and HDR content renders washed out. - if (![decoder bitstreamCarriesColorInfo]) { + // BT.709/sRGB and HDR content renders washed out. Read from the per-frame + // params (captured at decode time) rather than the decoder, whose + // _videoFormat may already belong to a newer format. + if (!decodeParams->bitstreamCarriesColorInfo) { overrideColorSpaceAttachments(imageBuffer); } @@ -320,14 +328,17 @@ - (NSInteger)decodeData:(const uint8_t *)data size:(size_t)size timeStamp:(int64 } RTC_DCHECK(sampleBuffer); VTDecodeFrameFlags decodeFlags = kVTDecodeFrame_EnableAsynchronousDecompression; + bool bitstreamCarriesColorInfo = [self bitstreamCarriesColorInfo]; std::unique_ptr frameDecodeParams; - frameDecodeParams.reset(new RTCH265FrameDecodeParams(timeStamp, _reorderQueue.reorderSize())); + frameDecodeParams.reset(new RTCH265FrameDecodeParams(timeStamp, _reorderQueue.reorderSize(), + bitstreamCarriesColorInfo)); OSStatus status = VTDecompressionSessionDecodeFrame( _decompressionSession, sampleBuffer, decodeFlags, frameDecodeParams.release(), nullptr); // Re-initialize the decoder if we have an invalid session while the app is // active and retry the decode request. if (status == kVTInvalidSessionErr && [self resetDecompressionSession] == WEBRTC_VIDEO_CODEC_OK) { - frameDecodeParams.reset(new RTCH265FrameDecodeParams(timeStamp, _reorderQueue.reorderSize())); + frameDecodeParams.reset(new RTCH265FrameDecodeParams(timeStamp, _reorderQueue.reorderSize(), + bitstreamCarriesColorInfo)); status = VTDecompressionSessionDecodeFrame(_decompressionSession, sampleBuffer, decodeFlags, frameDecodeParams.release(), nullptr); } From ff6af96320ae8e691d894eb1ce962b1534190de5 Mon Sep 17 00:00:00 2001 From: Oliver Sluke <22557015+oliversluke@users.noreply.github.com> Date: Fri, 10 Jul 2026 12:23:48 +0200 Subject: [PATCH 3/3] Gate 10-bit decode output behind an opt-in class property Emitting kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange breaks consumers that assume 8-bit NV12 output: RTCMTLVideoView's R8/RG8 shaders render black and RTCCVPixelBuffer's I420 conversion paths hit RTC_DCHECK_NOTREACHED. Keep the historical NV12 downconversion as the default and add RTCVideoDecoderH265.preferHighBitDepthOutput so apps whose render path handles 10-bit biplanar buffers can opt in. The colorimetry fix is unaffected and stays on for everyone. Co-Authored-By: Claude Fable 5 --- .../video_codec/RTCVideoDecoderH265.h | 14 ++++++++++++++ .../video_codec/RTCVideoDecoderH265.mm | 19 ++++++++++++++++--- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/sdk/objc/components/video_codec/RTCVideoDecoderH265.h b/sdk/objc/components/video_codec/RTCVideoDecoderH265.h index 6e77bdf40d7..cdb76f1e36d 100644 --- a/sdk/objc/components/video_codec/RTCVideoDecoderH265.h +++ b/sdk/objc/components/video_codec/RTCVideoDecoderH265.h @@ -15,6 +15,20 @@ RTC_OBJC_EXPORT @interface RTC_OBJC_TYPE (RTCVideoDecoderH265) : NSObject + +/** When YES, streams whose decoder configuration signals a luma bit depth + * above 8 (e.g. HEVC Main10) are decoded to a 10-bit biplanar pixel format + * (kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange) so bit depth and HDR + * colorimetry survive decode. Defaults to NO, which keeps the historical + * behaviour of downconverting every stream to 8-bit NV12. + * + * Only enable this when every consumer of the decoded frames handles + * 10-bit biplanar output: RTCMTLVideoView's shaders and RTCCVPixelBuffer's + * I420 conversion currently assume 8-bit formats. Set before decoding + * starts; it is read when a decompression session is created. + */ +@property(class, nonatomic, assign) BOOL preferHighBitDepthOutput; + - (NSInteger)setHVCCFormat:(const uint8_t *)data size:(size_t)size width:(uint16_t)width height:(uint16_t)height; - (NSInteger)decodeData:(const uint8_t *)data size:(size_t)size diff --git a/sdk/objc/components/video_codec/RTCVideoDecoderH265.mm b/sdk/objc/components/video_codec/RTCVideoDecoderH265.mm index 29ea0d69a1b..e46f94f8fcd 100644 --- a/sdk/objc/components/video_codec/RTCVideoDecoderH265.mm +++ b/sdk/objc/components/video_codec/RTCVideoDecoderH265.mm @@ -222,6 +222,16 @@ @implementation RTC_OBJC_TYPE (RTCVideoDecoderH265) { webrtc::RTCVideoFrameReorderQueue _reorderQueue; } +static BOOL gPreferHighBitDepthOutput = NO; + ++ (BOOL)preferHighBitDepthOutput { + return gPreferHighBitDepthOutput; +} + ++ (void)setPreferHighBitDepthOutput:(BOOL)preferHighBitDepthOutput { + gPreferHighBitDepthOutput = preferHighBitDepthOutput; +} + - (instancetype)init { self = [super init]; if (self) { @@ -440,9 +450,12 @@ - (int)resetDecompressionSession { kCVPixelBufferIOSurfacePropertiesKey, kCVPixelBufferPixelFormatTypeKey}; CFDictionaryRef ioSurfaceValue = CreateCFTypeDictionary(nullptr, nullptr, 0); // Forcing NV12 would crush high bit depth output (e.g. HEVC Main10) to - // 8 bits. Request a 10-bit biplanar format for such streams; 8-bit streams - // keep NV12 so existing consumers are unaffected. - int64_t pixelFormatType = [self isHighBitDepthFormat] + // 8 bits. Request a 10-bit biplanar format for such streams, but only when + // the app has opted in via preferHighBitDepthOutput — downstream consumers + // that assume 8-bit NV12 (RTCMTLVideoView, RTCCVPixelBuffer's I420 + // conversion) do not handle 10-bit output. 8-bit streams keep NV12 either + // way. + int64_t pixelFormatType = (gPreferHighBitDepthOutput && [self isHighBitDepthFormat]) ? kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange : kCVPixelFormatType_420YpCbCr8BiPlanarFullRange; CFNumberRef pixelFormat = CFNumberCreate(nullptr, kCFNumberLongType, &pixelFormatType);