From 8419c8981adb249e8fb159e83b8dc151022806c6 Mon Sep 17 00:00:00 2001 From: Daniel McAssey Date: Tue, 12 Dec 2023 15:27:38 +0000 Subject: [PATCH] feat(RTC): add support for creating non-standard tracks (#2409) * feat(RTC): add support for creating non-standard tracks * fix(RTC): add additional checks for creating tracks via mediastream * fix(RTC): simplify options to create track * fix(RTC): fix tests * fix(RTC): update sourceId documentation --- JitsiMeetJS.spec.ts | 39 +++++++++++++++++++++++++++++ JitsiMeetJS.ts | 33 ++++++++++++++++++++++++ JitsiTrackErrors.spec.ts | 8 +++++- JitsiTrackErrors.ts | 14 ++++++++++- modules/RTC/JitsiLocalTrack.js | 4 +-- types/hand-crafted/JitsiMeetJS.d.ts | 2 ++ 6 files changed, 96 insertions(+), 4 deletions(-) create mode 100644 JitsiMeetJS.spec.ts diff --git a/JitsiMeetJS.spec.ts b/JitsiMeetJS.spec.ts new file mode 100644 index 0000000000..cddce83409 --- /dev/null +++ b/JitsiMeetJS.spec.ts @@ -0,0 +1,39 @@ +import JitsiMeetJS from './JitsiMeetJS'; +import { VideoType } from './service/RTC/VideoType'; +import { MediaType } from './service/RTC/MediaType'; +import { JitsiTrackErrors } from './JitsiTrackErrors'; + +describe('JitsiMeetJS', () => { + describe('createLocalTracksFromMediaStreams', () => { + it('creates a local track from a media stream', () => { + const canvas = document.createElement('canvas'); + + const canvasStream = canvas.captureStream(5); + const trackInfo = { + stream: canvasStream, + sourceType: 'canvas', + mediaType: MediaType.VIDEO, + videoType: VideoType.DESKTOP + }; + const newTracks = JitsiMeetJS.createLocalTracksFromMediaStreams([ trackInfo ]); + + expect(newTracks).toBeDefined(); + expect(newTracks.length).toBe(1); + }); + + it('throws an error if track is not the correct media type', () => { + const canvas = document.createElement('canvas'); + + const canvasStream = canvas.captureStream(5); + const trackInfo = { + stream: canvasStream, + sourceType: 'canvas', + mediaType: MediaType.AUDIO, + videoType: VideoType.DESKTOP + }; + + expect(() => JitsiMeetJS.createLocalTracksFromMediaStreams([ trackInfo ])) + .toThrowError(JitsiTrackErrors.TRACK_NO_STREAM_TRACKS_FOUND); + }); + }); +}); diff --git a/JitsiMeetJS.ts b/JitsiMeetJS.ts index a6d6436dda..c55c5e70d6 100644 --- a/JitsiMeetJS.ts +++ b/JitsiMeetJS.ts @@ -36,6 +36,7 @@ import * as ConnectionQualityEvents import * as E2ePingEvents from './service/e2eping/E2ePingEvents'; import { createGetUserMediaEvent } from './service/statistics/AnalyticsEvents'; import * as RTCStatsEvents from './modules/RTCStats/RTCStatsEvents'; +import { VideoType } from './service/RTC/VideoType'; const logger = Logger.getLogger(__filename); @@ -90,6 +91,13 @@ interface IJitsiMeetJSOptions { } } +interface ICreateLocalTrackFromMediaStreamOptions { + stream: MediaStream, + sourceType: string, + mediaType: MediaType, + videoType?: VideoType +} + /** * The public API of the Jitsi Meet library (a.k.a. {@code JitsiMeetJS}). */ @@ -421,6 +429,31 @@ export default { }); }, + /** + * Manually create JitsiLocalTrack's from the provided track info, by exposing the RTC method + * + * @param {Array} tracksInfo - array of track information + * @returns {Array} - created local tracks + */ + createLocalTracksFromMediaStreams(tracksInfo) { + return RTC.createLocalTracks(tracksInfo.map((trackInfo) => { + const tracks = trackInfo.stream.getTracks() + .filter(track => track.kind === trackInfo.mediaType); + + if (!tracks || tracks.length === 0) { + throw new JitsiTrackError(JitsiTrackErrors.TRACK_NO_STREAM_TRACKS_FOUND, null, null); + } + + if (tracks.length > 1) { + throw new JitsiTrackError(JitsiTrackErrors.TRACK_TOO_MANY_TRACKS_IN_STREAM, null, null); + } + + trackInfo.track = tracks[0]; + + return trackInfo; + })); + }, + /** * Create a TrackVADEmitter service that connects an audio track to an VAD (voice activity detection) processor in * order to obtain VAD scores for individual PCM audio samples. diff --git a/JitsiTrackErrors.spec.ts b/JitsiTrackErrors.spec.ts index d651ac6f3a..ca37bfae7e 100644 --- a/JitsiTrackErrors.spec.ts +++ b/JitsiTrackErrors.spec.ts @@ -16,6 +16,8 @@ describe( "/JitsiTrackErrors members", () => { TRACK_IS_DISPOSED, TRACK_NO_STREAM_FOUND, UNSUPPORTED_RESOLUTION, + TRACK_TOO_MANY_TRACKS_IN_STREAM, + TRACK_NO_STREAM_TRACKS_FOUND, JitsiTrackErrors, ...others } = exported; @@ -33,6 +35,8 @@ describe( "/JitsiTrackErrors members", () => { expect( TRACK_IS_DISPOSED ).toBe( 'track.track_is_disposed' ); expect( TRACK_NO_STREAM_FOUND ).toBe( 'track.no_stream_found' ); expect( UNSUPPORTED_RESOLUTION ).toBe( 'gum.unsupported_resolution' ); + expect( TRACK_TOO_MANY_TRACKS_IN_STREAM ).toBe( 'track.too_many_tracks_in_stream' ); + expect( TRACK_NO_STREAM_TRACKS_FOUND ).toBe( 'track.no_stream_tracks_found' ); expect( JitsiTrackErrors ).toBeDefined(); @@ -48,10 +52,12 @@ describe( "/JitsiTrackErrors members", () => { expect( JitsiTrackErrors.TRACK_IS_DISPOSED ).toBe( 'track.track_is_disposed' ); expect( JitsiTrackErrors.TRACK_NO_STREAM_FOUND ).toBe( 'track.no_stream_found' ); expect( JitsiTrackErrors.UNSUPPORTED_RESOLUTION ).toBe( 'gum.unsupported_resolution' ); + expect( JitsiTrackErrors.TRACK_TOO_MANY_TRACKS_IN_STREAM ).toBe( 'track.too_many_tracks_in_stream' ); + expect( JitsiTrackErrors.TRACK_NO_STREAM_TRACKS_FOUND ).toBe( 'track.no_stream_tracks_found' ); } ); it( "unknown members", () => { const keys = Object.keys( others ); expect( keys ).withContext( `Extra members: ${ keys.join( ", " ) }` ).toEqual( [] ); } ); -} ); \ No newline at end of file +} ); diff --git a/JitsiTrackErrors.ts b/JitsiTrackErrors.ts index 01535292f8..914f61436b 100644 --- a/JitsiTrackErrors.ts +++ b/JitsiTrackErrors.ts @@ -68,7 +68,17 @@ export enum JitsiTrackErrors { * An error which indicates that requested video resolution is not supported * by a webcam. */ - UNSUPPORTED_RESOLUTION = 'gum.unsupported_resolution' + UNSUPPORTED_RESOLUTION = 'gum.unsupported_resolution', + + /** + * An error which indicates that there are too many tracks in the provided media stream + */ + TRACK_TOO_MANY_TRACKS_IN_STREAM = 'track.too_many_tracks_in_stream', + + /** + * An error which indicates that no tracks were found in the media stream + */ + TRACK_NO_STREAM_TRACKS_FOUND = 'track.no_stream_tracks_found', } // exported for backward compatibility @@ -84,3 +94,5 @@ export const TIMEOUT = JitsiTrackErrors.TIMEOUT; export const TRACK_IS_DISPOSED = JitsiTrackErrors.TRACK_IS_DISPOSED; export const TRACK_NO_STREAM_FOUND = JitsiTrackErrors.TRACK_NO_STREAM_FOUND; export const UNSUPPORTED_RESOLUTION = JitsiTrackErrors.UNSUPPORTED_RESOLUTION; +export const TRACK_TOO_MANY_TRACKS_IN_STREAM = JitsiTrackErrors.TRACK_TOO_MANY_TRACKS_IN_STREAM; +export const TRACK_NO_STREAM_TRACKS_FOUND = JitsiTrackErrors.TRACK_NO_STREAM_TRACKS_FOUND; diff --git a/modules/RTC/JitsiLocalTrack.js b/modules/RTC/JitsiLocalTrack.js index cc4caf705e..ecfa7b98fd 100644 --- a/modules/RTC/JitsiLocalTrack.js +++ b/modules/RTC/JitsiLocalTrack.js @@ -47,8 +47,8 @@ export default class JitsiLocalTrack extends JitsiTrack { * @param {number} trackInfo.resolution - The the video resolution if it's a video track * @param {string} trackInfo.deviceId - The ID of the local device for this track. * @param {string} trackInfo.facingMode - Thehe camera facing mode used in getUserMedia call (for mobile only). - * @param {sourceId} trackInfo.sourceId - The id of the desktop sharing source. NOTE: defined for desktop sharing - * tracks only. + * @param {sourceId} trackInfo.sourceId - The id of the desktop sharing source, which is the Chrome media source ID, + * returned by Desktop Picker on Electron. NOTE: defined for desktop sharing tracks only. */ constructor({ deviceId, diff --git a/types/hand-crafted/JitsiMeetJS.d.ts b/types/hand-crafted/JitsiMeetJS.d.ts index 54e25a3039..e777d51380 100644 --- a/types/hand-crafted/JitsiMeetJS.d.ts +++ b/types/hand-crafted/JitsiMeetJS.d.ts @@ -122,6 +122,8 @@ export type JitsiMeetJSType = { getActiveAudioDevice: () => Promise; // TODO: can we improve on object? + createLocalTracksFromMediaStreams: ( tracksInfo: unknown[] ) => JitsiLocalTrack[]; // TODO: + // isDeviceListAvailable: () => boolean; // obsosete // isDeviceChangeAvailable: ( deviceType: string ) => boolean; // obsosete