From dba969e6b040d43b76efae53391c35b1020908c0 Mon Sep 17 00:00:00 2001 From: Chris Owen Date: Tue, 7 Sep 2021 13:08:55 +0200 Subject: [PATCH] Import latest release-7.2 files from old repo --- ChangeLog.txt | 14 +++ Makefile | 13 +- README.md | 14 +++ docs/api/api_changes_7.2.txt | 44 +++++++ include/avs_ccall.h | 4 + include/avs_ecall.h | 8 +- include/avs_econn.h | 33 ++++- include/avs_icall.h | 10 ++ include/avs_peerflow.h | 2 + include/avs_wcall.h | 21 +++- iosx/src/flowmgr/AVSVideoViewOSX.m | 25 +++- src/audio_io/osx/audio_io_osx.cpp | 59 ++++++++- src/ccall/ccall.c | 82 +++++++++++++ src/ccall/ccall.h | 2 + src/ecall/ecall.c | 37 +++++- src/ecall/ecall.h | 1 + src/ecall/trace.c | 2 +- src/econn/econn.c | 19 +++ src/econn/econn.h | 1 + src/econn/econn_help.c | 2 + src/econn/msg.c | 5 + src/econn_fmt/msg.c | 120 ++++++++++++++++++ src/egcall/egcall.c | 1 + src/frame_enc/frame_decryptor.c | 5 + src/frame_enc/frame_encryptor.c | 13 +- src/icall/icall.c | 2 + src/jsflow/jsflow.c | 37 ++++-- src/peerflow/peerflow.cpp | 53 +++++--- src/peerflow/video_renderer.cpp | 40 +++++- src/peerflow/video_renderer.h | 1 + src/wcall/marshal.c | 74 ++++++++++++ src/wcall/wcall.c | 81 ++++++++++++- src/wcall/wcall.h | 3 + test/test_econn.cpp | 1 + tools/verifyd/tool.mk | 1 + tools/zcall/conv.c | 20 +++ tools/zcall/osx_view.m | 172 ++++++++++++++++++++------ tools/zcall/test_view.c | 12 +- tools/zcall/tool.mk | 2 + tools/zcall/view.c | 12 +- tools/zcall/view.h | 2 + tools/zcall/view_internal.h | 12 +- wasm/src/avs_pc.ts | 188 ++++++++++++++++++++++------- 43 files changed, 1093 insertions(+), 157 deletions(-) create mode 100644 docs/api/api_changes_7.2.txt diff --git a/ChangeLog.txt b/ChangeLog.txt index ba73619c3..e2304e13c 100644 --- a/ChangeLog.txt +++ b/ChangeLog.txt @@ -1,5 +1,19 @@ ChangeLog for AVS -- latest on top! +============================================================================== +AVS Release 7.2 + +Date: May 10, 2021 +GIT: https://github.com/wearezeta/avs/tree/release-7.2 +Android: Android NDK r14b +iOS: iOS SDK 12.2 +OSX: MacOS SDK 10.14 +Linux: Ubuntu x86_64 18.04 (LTS) + +Summary: - Selective forwarding of video + +Hotfixes: + ============================================================================== AVS Release 7.1 diff --git a/Makefile b/Makefile index 6e5c440bf..d94f17e02 100644 --- a/Makefile +++ b/Makefile @@ -24,8 +24,6 @@ # Master version number # -VER_MAJOR := 7 -VER_MINOR := 1 ifeq ($(BUILD_NUMBER),) VER_PATCH := local @@ -35,14 +33,15 @@ endif VER_BRANCH := $(shell git rev-parse --abbrev-ref HEAD || echo "") -ifeq ($(VER_BRANCH),master) +ifeq ($(word 1, $(subst -, , $(VER_BRANCH))), release) +AVS_PROJECT := avs +AVS_RELEASE := 1 +VER_MAJOR_MINOR := $(word 2, $(subst -, , $(VER_BRANCH))) +AVS_VERSION := $(VER_MAJOR_MINOR).$(VER_PATCH) +else AVS_PROJECT := avsmaster AVS_RELEASE := 0 AVS_VERSION := 0.0.$(VER_PATCH) -else -AVS_PROJECT := avs -AVS_RELEASE := 1 -AVS_VERSION := $(VER_MAJOR).$(VER_MINOR).$(VER_PATCH) endif MK_COMPONENTS := toolchain contrib avs tools test android iosx dist diff --git a/README.md b/README.md index 003e2713c..d150bd9c4 100644 --- a/README.md +++ b/README.md @@ -353,3 +353,17 @@ When reporting bugs against AVS please include the following: - Exact time when call was started/stopped - Name/OS of device - Adb logcat for Android + +Run-time libraries +------------------ +FROM ubuntu:16.04 +RUN apt-get install -qqy --no-install-recommends \ + libprotobuf-c-dev \ + libc6-dev-i386 \ + libreadline-dev \ + libx11-dev \ + libxcomposite-dev \ + libxdamage-dev \ + libxrender-dev \ + libc++-dev \ + libc++abi-dev diff --git a/docs/api/api_changes_7.2.txt b/docs/api/api_changes_7.2.txt new file mode 100644 index 000000000..112036129 --- /dev/null +++ b/docs/api/api_changes_7.2.txt @@ -0,0 +1,44 @@ + +Changes introduced in AVS 7.2 +============================= + +AVS 7.2 comes with selective video, allowing clients to request a subset of available videos to receive. An additional call and mode value are available: + +#define WCALL_VSTREAMS_LIST 0 + +int wcall_request_video_streams(WUSER_HANDLE wuser, + const char *convid, + int mode, + const char *json); + + +export enum VSTREAMS { + LIST = 0 +} + +requestVideoStreams( + wuser: number, + convid: string, + mode: number, + json: string +): number + +The mode parameter can only be WCALL_VSTREAMS_LIST at present. json is a string containing a JSON blob similar to wcall_set_clients_for_conv: + +{ + "convid":"01234567-89ab-cdef-0123-456789abcdef", + "clients":[ + {"userid":"01234567-89ab-cdef-0123-456789abcdef","clientid":"0123456789abcdef"} + ] +} + +The videoStreamHandler callback on web can now receive null for the streams to allow the user for the stream to be switched. When receiving videoStreamHandler with null streams, the video element used to render the video for that user should be disconnected and/or removed. The function now looks as follows: + +export type VideoStreamHandler = ( + convid: string, + remote_userid: string, + remote_clientid: string, + streams: readonly MediaStream[] | null +) => void; + + diff --git a/include/avs_ccall.h b/include/avs_ccall.h index 8696a02f4..c989a9206 100644 --- a/include/avs_ccall.h +++ b/include/avs_ccall.h @@ -68,6 +68,10 @@ int ccall_set_quality_interval(struct icall *icall, uint64_t interval); int ccall_update_mute_state(const struct icall* icall); +int ccall_request_video_streams(struct icall *icall, + struct list *clientl, + enum icall_stream_mode mode); + int ccall_msg_recv(struct icall* icall, uint32_t curr_time, uint32_t msg_time, diff --git a/include/avs_ecall.h b/include/avs_ecall.h index f92c6cce1..d3969abf1 100644 --- a/include/avs_ecall.h +++ b/include/avs_ecall.h @@ -120,10 +120,16 @@ typedef void (ecall_confpart_h)(struct ecall *ecall, const struct econn_message *msg, void *arg); - int ecall_set_confpart_handler(struct ecall *ecall, ecall_confpart_h confparth); +typedef void (ecall_confstreams_h)(struct ecall *ecall, + const struct econn_message *msg, + void *arg); + +int ecall_set_confstreams_handler(struct ecall *ecall, + ecall_confstreams_h confstreamsh); + int ecall_dce_send(struct ecall *ecall, struct mbuf *mb); int ecall_dce_sendmsg(struct ecall *ecall, struct econn_message *msg); diff --git a/include/avs_econn.h b/include/avs_econn.h index ec2460746..ec551965a 100644 --- a/include/avs_econn.h +++ b/include/avs_econn.h @@ -34,12 +34,13 @@ enum econn_msg { ECONN_GROUP_SETUP = 0x08, /* Conference call: */ - ECONN_CONF_CONN = 0x09, - ECONN_CONF_START = 0x0A, - ECONN_CONF_END = 0x0B, - ECONN_CONF_PART = 0x0C, - ECONN_CONF_KEY = 0x0D, - ECONN_CONF_CHECK = 0x0E, + ECONN_CONF_CONN = 0x09, + ECONN_CONF_START = 0x0A, + ECONN_CONF_END = 0x0B, + ECONN_CONF_PART = 0x0C, + ECONN_CONF_KEY = 0x0D, + ECONN_CONF_CHECK = 0x0E, + ECONN_CONF_STREAMS = 0x0F, ECONN_UPDATE = 0x10, ECONN_REJECT = 0x11, @@ -198,6 +199,11 @@ struct econn_message { struct confkey { struct list keyl; /* list of struct econn_key_info */ } confkey; + + struct confstreams { + struct list streaml; /* list of struct econn_stream_info */ + char *mode; + } confstreams; } u; }; @@ -222,6 +228,14 @@ struct econn_key_info { uint32_t dlen; struct le le; }; + +/* key data for CONF_STREAMS message */ +struct econn_stream_info { + char userid[ECONN_ID_LEN]; + uint32_t quality; + struct le le; +}; + /** * Indicates an incoming call on this ECONN. * Should only be called once per ECONN. @@ -268,6 +282,10 @@ typedef void (econn_confpart_h)(struct econn *econn, const struct econn_message *msg, void *arg); +typedef void (econn_confstreams_h)(struct econn *econn, + const struct econn_message *msg, + void *arg); + typedef void (econn_ping_h)(struct econn *econn, bool response, void *arg); /** @@ -312,6 +330,7 @@ int econn_alloc(struct econn **econnp, econn_update_resp_h *update_resph, econn_alert_h *alerth, econn_confpart_h *confparth, + econn_confstreams_h *confstreamsh, econn_ping_h *pingh, econn_close_h *closeh, void *arg); int econn_start(struct econn *conn, const char *sdp, @@ -398,6 +417,8 @@ struct econn_key_info *econn_key_info_alloc(size_t keysz); int econn_send_ping(struct econn *conn, bool response); +struct econn_stream_info *econn_stream_info_alloc(const char *userid, + uint32_t quality); struct vector { uint8_t *bytes; size_t len; diff --git a/include/avs_icall.h b/include/avs_icall.h index 15c8c2d13..24bc12eac 100644 --- a/include/avs_icall.h +++ b/include/avs_icall.h @@ -63,10 +63,15 @@ enum icall_audio_state { ICALL_AUDIO_STATE_NETWORK_PROBLEM = 2, }; +enum icall_stream_mode { + ICALL_STREAM_MODE_DEFAULT = 0 +}; + struct icall_client { struct le le; char *userid; char *clientid; + int quality; }; /* Used in place of uploss/downloss in the quality handler, @@ -109,6 +114,9 @@ typedef int (icall_set_quality_interval)(struct icall *icall, typedef int (icall_dce_send)(struct icall *icall, struct mbuf *mb); typedef void (icall_set_clients)(struct icall* icall, struct list *clientl); typedef int (icall_update_mute_state)(const struct icall* icall); +typedef int (icall_request_video_streams)(struct icall *icall, + struct list *clientl, + enum icall_stream_mode mode); typedef int (icall_debug)(struct re_printf *pf, const struct icall* icall); typedef int (icall_stats)(struct re_printf *pf, const struct icall* icall); @@ -206,6 +214,7 @@ struct icall { icall_dce_send *dce_send; icall_set_clients *set_clients; icall_update_mute_state *update_mute_state; + icall_request_video_streams *request_video_streams; icall_debug *debug; icall_stats *stats; @@ -256,6 +265,7 @@ void icall_set_functions(struct icall *icall, icall_dce_send *dce_send, icall_set_clients *set_clients, icall_update_mute_state *update_ute_state, + icall_request_video_streams *request_video_streams, icall_debug *debug, icall_stats *stats); diff --git a/include/avs_peerflow.h b/include/avs_peerflow.h index 778122ee2..21d6f6ddb 100644 --- a/include/avs_peerflow.h +++ b/include/avs_peerflow.h @@ -30,6 +30,8 @@ void capture_source_handle_frame(struct avs_vidframe *frame); int peerflow_get_userid_for_ssrc(struct peerflow* pf, uint32_t csrc, bool video, + char **userid_real, + char **clientid_real, char **userid_hash); #ifdef __cplusplus } diff --git a/include/avs_wcall.h b/include/avs_wcall.h index 4adcbb6e5..d37ce1130 100644 --- a/include/avs_wcall.h +++ b/include/avs_wcall.h @@ -477,6 +477,11 @@ struct mediamgr *wcall_mediamgr(WUSER_HANDLE wuser); void wcall_handle_frame(struct avs_vidframe *frame); +int wcall_set_proxy(const char *host, int port); + +/** + * Client list management (conference calls). + */ typedef void (wcall_req_clients_h)(WUSER_HANDLE wuser, const char *convid, void *arg); @@ -486,17 +491,25 @@ int wcall_set_clients_for_conv(WUSER_HANDLE wuser, const char *convid, const char *json); - +/** + * Active speaker (conference calls). + */ typedef void (wcall_active_speaker_h)(WUSER_HANDLE wuser, const char *convid, const char *json_levels, void *arg); void wcall_set_active_speaker_handler(WUSER_HANDLE wuser, wcall_active_speaker_h *activeh); - -int wcall_set_proxy(const char *host, int port); - +#define WCALL_VSTREAMS_LIST 0 +/** + * Video stream selection (conference calls). + */ +int wcall_request_video_streams(WUSER_HANDLE wuser, + const char *convid, + int mode, + const char *json); + /* * Netprobe */ diff --git a/iosx/src/flowmgr/AVSVideoViewOSX.m b/iosx/src/flowmgr/AVSVideoViewOSX.m index cead0b34c..ec752afd4 100644 --- a/iosx/src/flowmgr/AVSVideoViewOSX.m +++ b/iosx/src/flowmgr/AVSVideoViewOSX.m @@ -81,6 +81,8 @@ @implementation AVSVideoViewOSX NSLock *_lock; BOOL _newFrame; BOOL _firstFrame; + BOOL _drawBlank; + NSString *_userid; CVDisplayLinkRef _displayLink; } @@ -292,6 +294,7 @@ - (BOOL) handleFrame:(struct avs_vidframe*) frame frame->w, frame->h); _firstFrame = NO; } + _drawBlank = NO; [_context makeCurrentContext]; if (_texWidth != (GLsizei)frame->ys || _texHeight != frame->h) { @@ -382,8 +385,10 @@ - (CVReturn)getFrameForTime:(const CVTimeStamp*)outputTime [_context makeCurrentContext]; if (_forceRecalc) { + NSScreen *nsscreen = [[NSScreen screens] objectAtIndex: 0]; + GLfloat scale = nsscreen.backingScaleFactor; NSRect frameRect = self.frame; - glViewport(0, 0, frameRect.size.width, frameRect.size.height); + glViewport(0, 0, frameRect.size.width * scale, frameRect.size.height * scale); [self setupVertices]; _forceRecalc = NO; } @@ -391,7 +396,8 @@ - (CVReturn)getFrameForTime:(const CVTimeStamp*)outputTime glClearColor(0.0, 0.0, 0.0, 1.0); glClear(GL_COLOR_BUFFER_BIT); - glDrawElements(GL_TRIANGLE_STRIP, 4, GL_UNSIGNED_BYTE, indices); + if (!_drawBlank) + glDrawElements(GL_TRIANGLE_STRIP, 4, GL_UNSIGNED_BYTE, indices); glSwapAPPLE(); _newFrame = NO; @@ -451,6 +457,21 @@ - (void)viewDidMoveToWindow [super viewDidMoveToWindow]; } + +- (void) setUserid:(NSString*)userid +{ + [_lock lock]; + _userid = userid; + _drawBlank = YES; + _newFrame = YES; + [_lock unlock]; +} + +- (NSString*) userid +{ + return _userid; +} + - (void)dealloc { CVDisplayLinkStop(_displayLink); diff --git a/src/audio_io/osx/audio_io_osx.cpp b/src/audio_io/osx/audio_io_osx.cpp index 535910396..fbfd60e61 100644 --- a/src/audio_io/osx/audio_io_osx.cpp +++ b/src/audio_io/osx/audio_io_osx.cpp @@ -608,6 +608,26 @@ namespace webrtc { init_play_audio_unit(); +#if 0 + _useSoundLink = false; + if( _adbRecordSampFreq > 44000 && + _adbPlaySampFreq > 44000) { + int soundLinkMode = 0; + int ret; + if( (ret = Init_SoundLink( _soundLink, soundLinkMode, _adbPlaySampFreq, _adbRecordSampFreq, NULL)) ) { + WEBRTC_TRACE(kTraceWarning, kTraceAudioDevice, _id, + " Error SoundLink returned %d", ret); + return 0; + } + // Register sound link callback + if( (ret = RegisterDetectionCallback( _soundLink, this )) ) { + WEBRTC_TRACE(kTraceWarning, kTraceAudioDevice, _id, + " Error SoundLink Callback not registered, returned %d", ret); + return 0; + } + _useSoundLink = true; + } +#endif return 0; } @@ -636,7 +656,10 @@ namespace webrtc { } au_play_ = NULL; } - +#if 0 + // Unregister soundlink callback + UnregisterDetectionCallback( _soundLink ); +#endif return 0; } @@ -1108,6 +1131,12 @@ namespace webrtc { rec_latency_ms_ = tmp_rec_latency_ms; if (is_recording_) { + // SoundLink Processing +#if 0 + if(_useSoundLink){ + SoundLinkRecord( _soundLink, dataTmp, inNumberFrames ); + } +#endif const unsigned int noSamp10ms = rec_fs_hz_ / 100; unsigned int dataPos = 0; uint16_t bufPos = 0; @@ -1274,6 +1303,24 @@ namespace webrtc { // samples we shall quit loop anyway dataPos += noSamp10ms; } +#if 0 + if(_useSoundLink){ + int16_t dataTmpSL[dataSize]; + + SoundLinkPlay( _soundLink, dataTmpSL, dataSize ); + if( is_stereo){ + /* Mix into left channel */ + for( unsigned int i = 0 ; i < dataSize; i++){ + data[2*i] += dataTmpSL[i]; + } + }else{ + /* Mix with output */ + for(unsigned int i = 0 ; i < dataSize; i++){ + data[i] += dataTmpSL[i]; + } + } + } +#endif } //_numRenderCalls+=1; @@ -1413,4 +1460,14 @@ namespace webrtc { rec_delay_ += (rec_buffer_total_size_ - noSamp10ms) / (rec_fs_hz_ / 1000); } } + + void audio_io_osx::DetectedSoundLink(const std::vector &msg, + const struct tm timeLastDetected, + const int deviceRoundTripLatencyMs) + { + if(deviceRoundTripLatencyMs > SOUNDLINK_LATENCY_FOR_RESET_MS){ + warning(" High Device Round trip Latency = %d ms", deviceRoundTripLatencyMs); + rec_delay_warning_ = 1; + } + } } diff --git a/src/ccall/ccall.c b/src/ccall/ccall.c index 2f9a3a441..7358f84d5 100644 --- a/src/ccall/ccall.c +++ b/src/ccall/ccall.c @@ -1257,6 +1257,81 @@ static int send_confpart_response(struct ccall *ccall) return err; } +int ccall_request_video_streams(struct icall *icall, + struct list *clientl, + enum icall_stream_mode mode) +{ + struct ccall *ccall = (struct ccall*)icall; + struct econn_stream_info *sinfo; + struct econn_message *msg; + char *str = NULL; + struct mbuf mb; + struct le *le = NULL; + int err = 0; + + if (!ccall) + return EINVAL; + + err = alloc_message(&msg, ccall, ECONN_CONF_STREAMS, false, + ccall->self->userid_hash, ccall->self->clientid_hash, + "SFT", "SFT", false); + if (err) { + goto out; + } + + str_dup(&msg->u.confstreams.mode, "list"); + LIST_FOREACH(clientl, le) { + struct icall_client *cli = le->data; + struct userinfo *user; + + user = find_userinfo_by_real(ccall, cli->userid, cli->clientid); + if (user) { + sinfo = econn_stream_info_alloc(user->userid_hash, 0); + if (!sinfo) { + err = ENOMEM; + goto out; + } + + list_append(&msg->u.confstreams.streaml, &sinfo->le, sinfo); + } + } + + info("ccall(%p): request_video_streams mode: %u clients: %u matched: %u\n", + ccall, + mode, + list_count(clientl), + list_count(&msg->u.confstreams.streaml)); + + err = econn_message_encode(&str, msg); + if (err) { + warning("ccall(%p): request_video_streams: econn_message_encode" + " failed (%m)\n", err); + goto out; + } + + ecall_trace(ccall->ecall, msg, true, ECONN_TRANSP_DIRECT, + "DataChan %H\n", + econn_message_brief, msg); + + mb.pos = 0; + mb.size = str_len(str); + mb.end = mb.size; + mb.buf = (uint8_t *)str; + + err = ecall_dce_send(ccall->ecall, &mb); + if (err) { + warning("ccall(%p): request_video_streams: ecall_dce_send" + " failed (%m)\n", err); + goto out; + } + + out: + mem_deref(str); + mem_deref(msg); + + return err; +} + static int ecall_propsync_handler(struct ecall *ecall, struct econn_message *msg, void *arg) @@ -1739,6 +1814,9 @@ static int alloc_message(struct econn_message **msgp, else if (type == ECONN_CONF_END) { str_ncpy(msg->sessid_sender, ccall->convid_hash, ECONN_ID_LEN); } + else if (type == ECONN_CONF_STREAMS) { + str_ncpy(msg->sessid_sender, ccall->convid_hash, ECONN_ID_LEN); + } out: skipprops: if (err) { @@ -1847,6 +1925,9 @@ static int ccall_send_conf_conn(struct ccall *ccall, } msg->u.confconn.update = update; msg->u.confconn.selective_audio = true; + msg->u.confconn.selective_video = true; + msg->u.confconn.vstreams = CCALL_MAX_VSTREAMS; + err = ccall_send_msg_sft(ccall, sft_url, msg); if (err != 0) { goto out; @@ -2007,6 +2088,7 @@ int ccall_alloc(struct ccall **ccallp, NULL, // ccall_dce_send ccall_set_clients, ccall_update_mute_state, + ccall_request_video_streams, ccall_debug, ccall_stats); out: diff --git a/src/ccall/ccall.h b/src/ccall/ccall.h index e9a72b0d6..7e6028f69 100644 --- a/src/ccall/ccall.h +++ b/src/ccall/ccall.h @@ -34,6 +34,8 @@ #define CCALL_SECRET_LEN ( 16) #define CCALL_MAX_RECONNECT_ATTEMPTS ( 2) +#define CCALL_MAX_VSTREAMS ( 9) + struct userinfo { struct le le; char *userid_real; diff --git a/src/ecall/ecall.c b/src/ecall/ecall.c index 83a5d7093..a8fceae5f 100644 --- a/src/ecall/ecall.c +++ b/src/ecall/ecall.c @@ -537,6 +537,34 @@ int ecall_set_confpart_handler(struct ecall *ecall, } +static void econn_confstreams_handler(struct econn *econn, + const struct econn_message *msg, + void *arg) +{ + struct ecall *ecall = arg; + + assert(ECALL_MAGIC == ecall->magic); + + if (ecall->confstreamsh) { + info("ecall(%p): confstreams: streams: %u\n", + ecall, list_count(&msg->u.confstreams.streaml)); + ecall->confstreamsh(ecall, msg, ecall->icall.arg); + } +} + + +int ecall_set_confstreams_handler(struct ecall *ecall, + ecall_confstreams_h confstreamsh) +{ + if (!ecall) { + return EINVAL; + } + + ecall->confstreamsh = confstreamsh; + return 0; +} + + int ecall_set_propsync_handler(struct ecall *ecall, ecall_propsync_h propsynch) { @@ -1027,6 +1055,7 @@ int ecall_alloc(struct ecall **ecallp, struct list *ecalls, _icall_dce_send, _icall_set_clients, _icall_update_mute_state, + NULL, // _icall_request_video_streams _icall_debug, _icall_stats); @@ -1859,7 +1888,7 @@ static void acbr_detect_handler(struct iflow *iflow, remote_cbr = cr && (0 == strcmp(cr, "true")); cbr_enabled = (enabled != 0) && (remote_cbr && local_cbr); - if (enabled != ecall->audio.cbr_state) { + if (cbr_enabled != ecall->audio.cbr_state) { info("ecall(%p): acbrh(%p) enabled=%d " "lcbr=%s rcbr=%s cbr=%d\n", ecall, ecall->icall.acbr_changedh, @@ -2041,6 +2070,7 @@ int ecall_create_econn(struct ecall *ecall) econn_update_resp_handler, econn_alert_handler, econn_confpart_handler, + econn_confstreams_handler, econn_ping_handler, econn_close_handler, ecall); @@ -2060,8 +2090,7 @@ int ecall_start(struct ecall *ecall, enum icall_call_type call_type, { int err; - info("ecall(%p): start call_type=%d cbr=%d\n", - ecall, call_type, audio_cbr); + info("ecall(%p): start\n", ecall); if (!ecall) return EINVAL; @@ -2150,7 +2179,7 @@ int ecall_answer(struct ecall *ecall, enum icall_call_type call_type, #endif - info("ecall(%p): answer on pending econn %p call_type=%d cbr=%d\n", ecall, ecall->econn, call_type, audio_cbr); + info("ecall(%p): answer on pending econn %p call_type=%d\n", ecall, ecall->econn, call_type); if (!ecall->econn) { warning("ecall: answer: econn does not exist!\n"); diff --git a/src/ecall/ecall.h b/src/ecall/ecall.h index 7bf88f59e..d4144820c 100644 --- a/src/ecall/ecall.h +++ b/src/ecall/ecall.h @@ -199,6 +199,7 @@ struct ecall { bool turn_added; ecall_confpart_h *confparth; + ecall_confstreams_h *confstreamsh; struct sa *media_laddr; bool ifs_added; diff --git a/src/ecall/trace.c b/src/ecall/trace.c index db6736788..78e4ef128 100644 --- a/src/ecall/trace.c +++ b/src/ecall/trace.c @@ -96,7 +96,7 @@ int ecall_show_trace(struct re_printf *pf, const struct ecall *ecall) for (le = list_head(&ecall->tracel); le; le = le->next) { struct trace_entry *ent = le->data; - err = re_hprintf(pf, "* %.3fs %s via %7s %10s %-8s" + err = re_hprintf(pf, "* %.3fs %s via %7s %11s %-8s" , .001 * ent->ts, ent->tx ? "send --->" : "recv <---", diff --git a/src/econn/econn.c b/src/econn/econn.c index 8160b446d..228554014 100644 --- a/src/econn/econn.c +++ b/src/econn/econn.c @@ -663,6 +663,19 @@ static void recv_confpart(struct econn *econn, } +static void recv_confstreams(struct econn *econn, + const char *userid_sender, + const char *clientid_sender, + const struct econn_message *msg) +{ + if (econn && econn->confstreamsh && msg) { + econn->confstreamsh(econn, + msg, + econn->arg); + } +} + + static void recv_ping(struct econn *econn, const char *userid_sender, const char *clientid_sender, @@ -726,6 +739,10 @@ void econn_recv_message(struct econn *conn, recv_ping(conn, userid_sender, clientid_sender, msg); break; + case ECONN_CONF_STREAMS: + recv_confstreams(conn, userid_sender, clientid_sender, msg); + break; + default: warning("econn(%p): recv: message not supported (%s)\n", conn, econn_msg_name(msg->msg_type)); @@ -751,6 +768,7 @@ int econn_alloc(struct econn **connp, econn_update_resp_h *update_resph, econn_alert_h *alerth, econn_confpart_h *confparth, + econn_confstreams_h *confstreamsh, econn_ping_h *pingh, econn_close_h *closeh, void *arg) @@ -782,6 +800,7 @@ int econn_alloc(struct econn **connp, conn->update_resph = update_resph; conn->alerth = alerth; conn->confparth = confparth; + conn->confstreamsh = confstreamsh; conn->pingh = pingh; conn->closeh = closeh; conn->arg = arg; diff --git a/src/econn/econn.h b/src/econn/econn.h index 1fc0e8715..428a84735 100644 --- a/src/econn/econn.h +++ b/src/econn/econn.h @@ -47,6 +47,7 @@ struct econn { econn_update_resp_h *update_resph; econn_alert_h *alerth; econn_confpart_h *confparth; + econn_confstreams_h *confstreamsh; econn_ping_h *pingh; econn_close_h *closeh; void *arg; diff --git a/src/econn/econn_help.c b/src/econn/econn_help.c index 2c74fc980..d1eabac39 100644 --- a/src/econn/econn_help.c +++ b/src/econn/econn_help.c @@ -48,6 +48,7 @@ const char *econn_msg_name(enum econn_msg msg) case ECONN_CONF_PART: return "CONFPART"; case ECONN_CONF_KEY: return "CONFKEY"; case ECONN_CONF_CHECK: return "CONFCHECK"; + case ECONN_CONF_STREAMS: return "CONFSTREAMS"; case ECONN_PING: return "PING"; default: return "???"; } @@ -162,6 +163,7 @@ enum econn_transport econn_transp_resolve(enum econn_msg type) case ECONN_CONF_END: return ECONN_TRANSP_BACKEND; case ECONN_CONF_PART: return ECONN_TRANSP_DIRECT; case ECONN_CONF_CHECK: return ECONN_TRANSP_BACKEND; + case ECONN_CONF_STREAMS: return ECONN_TRANSP_DIRECT; case ECONN_PING: return ECONN_TRANSP_DIRECT; default: diff --git a/src/econn/msg.c b/src/econn/msg.c index e075433eb..d144f3523 100644 --- a/src/econn/msg.c +++ b/src/econn/msg.c @@ -120,6 +120,11 @@ void econn_message_reset(struct econn_message *msg) msg->u.confpart.entropy = mem_deref(msg->u.confpart.entropy); break; + case ECONN_CONF_STREAMS: + list_flush(&msg->u.confstreams.streaml); + msg->u.confstreams.mode = mem_deref(msg->u.confstreams.mode); + break; + case ECONN_PING: break; diff --git a/src/econn_fmt/msg.c b/src/econn_fmt/msg.c index f6e2c0b0f..bfa0ce948 100644 --- a/src/econn_fmt/msg.c +++ b/src/econn_fmt/msg.c @@ -308,6 +308,107 @@ static int econn_keys_decode(struct list *keyl, struct json_object *jobj) return 0; } +static int econn_streams_encode(struct json_object *jobj, + const struct list *streaml) +{ + struct le *le; + struct json_object *jstreams; + int err = 0; + + jstreams = jzon_alloc_array(); + if (!jstreams) + return ENOMEM; + + LIST_FOREACH(streaml, le) { + struct econn_stream_info *stream = le->data; + struct json_object *jstream; + + jstream = jzon_alloc_object(); + if (!jstream) { + err = ENOMEM; + goto out; + } + jzon_add_str(jstream, "userid", "%s", stream->userid); + jzon_add_int(jstream, "quality", stream->quality); + + json_object_array_add(jstreams, jstream); + } + + json_object_object_add(jobj, "streams", jstreams); + + out: + return err; +} + +static void stream_destructor(void *arg) +{ + struct econn_stream_info *stream = arg; + + list_unlink(&stream->le); +} + +struct econn_stream_info *econn_stream_info_alloc(const char *userid, + uint32_t quality) +{ + struct econn_stream_info *stream; + + stream = mem_zalloc(sizeof(*stream), stream_destructor); + if (!stream) { + warning("econn: stream_decode_handler: could not alloc stream\n"); + return NULL; + } + + str_ncpy(stream->userid, userid, sizeof(stream->userid)); + + return stream; +} + +static bool stream_decode_handler(const char *keystr, + struct json_object *jobj, + void *arg) +{ + struct econn_stream_info *stream; + struct list *streaml = arg; + const char *userid = NULL; + int32_t quality = 0; + int err; + + err = jzon_int(&quality, jobj, "quality"); + if (err) + goto out; + + userid = jzon_str(jobj, "userid"); + if (!userid) + goto out; + + stream = econn_stream_info_alloc(userid, quality); + if (!stream) { + warning("econn: stream_decode_handler: could not alloc stream\n"); + goto out; + } + + list_append(streaml, &stream->le, stream); + +out: + return false; +} + +static int econn_streams_decode(struct list *streaml, struct json_object *jobj) +{ + struct json_object *jstreams; + int err = 0; + + err = jzon_array(&jstreams, jobj, "streams"); + if (err) { + warning("econn: streams decode: no streams\n"); + return err; + } + + jzon_apply(jstreams, stream_decode_handler, streaml); + + return 0; +} + int econn_message_encode(char **strp, const struct econn_message *msg) { struct json_object *jobj = NULL; @@ -477,6 +578,11 @@ int econn_message_encode(char **strp, const struct econn_message *msg) econn_keys_encode(jobj, &msg->u.confkey.keyl); break; + case ECONN_CONF_STREAMS: + jzon_add_str(jobj, "mode", "%s", msg->u.confstreams.mode); + econn_streams_encode(jobj, &msg->u.confstreams.streaml); + break; + case ECONN_DEVPAIR_PUBLISH: err = zapi_iceservers_encode(jobj, msg->u.devpair_publish.turnv, @@ -923,6 +1029,20 @@ int econn_message_decode(struct econn_message **msgp, if (err) return err; } + else if (0 == str_casecmp(type, econn_msg_name(ECONN_CONF_STREAMS))) { + msg->msg_type = ECONN_CONF_STREAMS; + err = econn_streams_decode(&msg->u.confstreams.streaml, jobj); + if (err) + return err; + + err = jzon_strdup(&msg->u.confstreams.mode, + jobj, "mode"); + if (err) { + warning("econn: conf_streams: " + "could not find mode in message\n"); + goto out; + } + } else if (0 == str_casecmp(type, econn_msg_name(ECONN_DEVPAIR_PUBLISH))) { diff --git a/src/egcall/egcall.c b/src/egcall/egcall.c index 4833b0ecf..fa3643a8c 100644 --- a/src/egcall/egcall.c +++ b/src/egcall/egcall.c @@ -606,6 +606,7 @@ int egcall_alloc(struct egcall **egcallp, NULL, // egcall_dce_send egcall_set_clients, egcall_update_mute_state, + NULL, // egcall_request_video_streams egcall_debug, egcall_stats); out: diff --git a/src/frame_enc/frame_decryptor.c b/src/frame_enc/frame_decryptor.c index ddf9ab830..1846ae302 100644 --- a/src/frame_enc/frame_decryptor.c +++ b/src/frame_enc/frame_decryptor.c @@ -88,6 +88,9 @@ int frame_decryptor_set_uid(struct frame_decryptor *dec, if (!dec || !userid_hash) return EINVAL; + info("frame_dec(%p): set_uid: %s\n", + dec, + userid_hash); dec->userid_hash = mem_deref(dec->userid_hash); err = str_dup(&dec->userid_hash, userid_hash); @@ -164,6 +167,8 @@ int frame_decryptor_decrypt(struct frame_decryptor *dec, err = peerflow_get_userid_for_ssrc(dec->pf, csrc, dec->mtype == FRAME_MEDIA_VIDEO, + NULL, + NULL, &dec->userid_hash); if (err) goto out; diff --git a/src/frame_enc/frame_encryptor.c b/src/frame_enc/frame_encryptor.c index 19a46c885..d212157f0 100644 --- a/src/frame_enc/frame_encryptor.c +++ b/src/frame_enc/frame_encryptor.c @@ -178,11 +178,12 @@ int frame_encryptor_encrypt(struct frame_encryptor *enc, if (!enc->frame_recv) { info("frame_enc(%p): encrypt: first frame received " - "type: %s uid: %s fid: %u\n", + "type: %s uid: %s fid: %u ssrc: %u\n", enc, frame_type_name(enc->mtype), enc->userid_hash, - enc->frameid); + enc->frameid, + ssrc); enc->frame_recv = true; } @@ -215,12 +216,11 @@ int frame_encryptor_encrypt(struct frame_encryptor *enc, enc->updated_ts = updated_ts; } - // TODO: set SSRC once all clients handle it hlen = frame_hdr_write(dst, frame_encryptor_max_size(enc, srcsz), enc->frameid, kid, - 0); + ssrc); err = frame_encryptor_xor_iv(enc->iv, enc->frameid, kid, iv, IV_SIZE); if (err) { @@ -266,11 +266,12 @@ int frame_encryptor_encrypt(struct frame_encryptor *enc, if (!err && !enc->frame_enc) { info("frame_enc(%p): encrypt: first frame encrypted " - "type: %s uid: %s fid: %u\n", + "type: %s uid: %s fid: %u ssrc: %u\n", enc, frame_type_name(enc->mtype), enc->userid_hash, - enc->frameid); + enc->frameid, + ssrc); enc->frame_enc = true; } out: diff --git a/src/icall/icall.c b/src/icall/icall.c index b1b98559d..81622a232 100644 --- a/src/icall/icall.c +++ b/src/icall/icall.c @@ -38,6 +38,7 @@ void icall_set_functions(struct icall *icall, icall_dce_send *dce_send, icall_set_clients *set_clients, icall_update_mute_state *update_mute_state, + icall_request_video_streams *request_video_streams, icall_debug *debug, icall_stats *stats) { @@ -62,6 +63,7 @@ void icall_set_functions(struct icall *icall, icall->dce_send = dce_send; icall->set_clients = set_clients; icall->update_mute_state = update_mute_state; + icall->request_video_streams = request_video_streams; icall->debug = debug; icall->stats = stats; } diff --git a/src/jsflow/jsflow.c b/src/jsflow/jsflow.c index 74f06ac33..778438bd8 100644 --- a/src/jsflow/jsflow.c +++ b/src/jsflow/jsflow.c @@ -226,6 +226,7 @@ struct jsflow { struct tmr tmr_stats; char *remote_sdp; bool selective_audio; + bool selective_video; struct tmr tmr_cbr; @@ -808,7 +809,7 @@ static void jsflow_tool_handler(const char *tool, void *arg) { struct jsflow *jsflow = (struct jsflow*)arg; char *tc = NULL, *v = NULL, *t = NULL; - int major = 0; + int major = 0, minor = 0; int err = 0; if (!jsflow || !tool) @@ -824,11 +825,19 @@ static void jsflow_tool_handler(const char *tool, void *arg) v = strsep(&t, " "); if (v && t && strcmp(v, "sftd") == 0) { v = strsep(&t, "."); - major = v ? atoi(v) : -1; + major = v ? atoi(v) : 0; + if (v && t) { + v = strsep(&t, "."); + minor = v ? atoi(v) : 0; + } jsflow->selective_audio = major >= 2; - info("jsflow(%p): set_sft_options: selective_audio: %s\n", + jsflow->selective_video = major > 2 || (major == 2 && minor >= 1); + info("jsflow(%p): set_sft_options: ver: %d.%d" + " selective_audio: %s selective_video: %s\n", jsflow, - jsflow->selective_audio ? "YES" : "NO"); + major, minor, + jsflow->selective_audio ? "YES" : "NO", + jsflow->selective_video ? "YES" : "NO"); } mem_deref(tc); @@ -1138,21 +1147,23 @@ int jsflow_remove_decoders_for_user(struct iflow *iflow, int jsflow_sync_decoders(struct iflow *iflow) { struct jsflow *jf = (struct jsflow *)iflow; - int err; + int err = 0; info("jsflow(%p): sync_decoders\n", jf); if (!jf) return EINVAL; - err = bundle_update((struct iflow *)jf, - jf->conv_type, - !jf->selective_audio, - jf->remote_sdp, - &jf->cml, - jsflow_bundle_update); - if (err) - goto out; + if (!jf->selective_video) { + err = bundle_update((struct iflow *)jf, + jf->conv_type, + !jf->selective_audio, + jf->remote_sdp, + &jf->cml, + jsflow_bundle_update); + if (err) + goto out; + } out: return err; diff --git a/src/peerflow/peerflow.cpp b/src/peerflow/peerflow.cpp index 17d4e6fda..fe3caa15a 100644 --- a/src/peerflow/peerflow.cpp +++ b/src/peerflow/peerflow.cpp @@ -186,6 +186,7 @@ struct peerflow { rtc::scoped_refptr netStatsCb; std::string remoteSdp; bool selective_audio; + bool selective_video; }; @@ -1217,8 +1218,8 @@ class AvsPeerConnectionObserver : public webrtc::PeerConnectionObserver { dec_uid = cm->userid_hash; } else if (pf_->conv_type == ICALL_CONV_TYPE_CONFERENCE) { - userid = "user"; - clientid = "client"; + userid = ""; + clientid = ""; } else if (pf_->conv_type != ICALL_CONV_TYPE_CONFERENCE && pf_->userid_remote && pf_->clientid_remote) { @@ -2233,7 +2234,7 @@ static void pf_tool_handler(const char *tool, void *arg) { struct peerflow *pf = (struct peerflow*)arg; char *tc = NULL, *v = NULL, *t = NULL; - int major = 0; + int major = 0, minor = 0; int err = 0; if (!pf || !tool) @@ -2249,11 +2250,19 @@ static void pf_tool_handler(const char *tool, void *arg) v = strsep(&t, " "); if (v && t && strcmp(v, "sftd") == 0) { v = strsep(&t, "."); - major = v ? atoi(v) : -1; + major = v ? atoi(v) : 0; + if (v && t) { + v = strsep(&t, "."); + minor = v ? atoi(v) : 0; + } pf->selective_audio = major >= 2; - info("peerflow(%p): set_sft_options: selective_audio: %s\n", + pf->selective_video = major > 2 || (major == 2 && minor >= 1); + info("peerflow(%p): set_sft_options: ver: %d.%d" + " selective_audio: %s selective_video: %s\n", pf, - pf->selective_audio ? "YES" : "NO"); + major, minor, + pf->selective_audio ? "YES" : "NO", + pf->selective_video ? "YES" : "NO"); } mem_deref(tc); @@ -2521,17 +2530,18 @@ int peerflow_sync_decoders(struct iflow *iflow) if (!pf) return EINVAL; - - str_dup(&sdp, pf->remoteSdp.c_str()); - err = bundle_update((struct iflow *)pf, - pf->conv_type, - !pf->selective_audio, - sdp, - &pf->cml.list, - peerflow_bundle_update); - if (err) - goto out; + if (!pf->selective_video) { + str_dup(&sdp, pf->remoteSdp.c_str()); + err = bundle_update((struct iflow *)pf, + pf->conv_type, + !pf->selective_audio, + sdp, + &pf->cml.list, + peerflow_bundle_update); + if (err) + goto out; + } out: mem_deref(sdp); @@ -2727,12 +2737,14 @@ void peerflow_stop_media(struct iflow *iflow) int peerflow_get_userid_for_ssrc(struct peerflow* pf, uint32_t csrc, bool video, + char **userid_real, + char **clientid_real, char **userid_hash) { struct conf_member *cm; int err = 0; - if (!pf || !userid_hash) + if (!pf) return EINVAL; lock_write_get(pf->cml.lock); @@ -2746,7 +2758,12 @@ int peerflow_get_userid_for_ssrc(struct peerflow* pf, goto out; } - err = str_dup(userid_hash, cm->userid_hash); + if (userid_real) + err = str_dup(userid_real, cm->userid); + if (clientid_real) + err |= str_dup(clientid_real, cm->clientid); + if (userid_hash) + err |= str_dup(userid_hash, cm->userid_hash); if (err) goto out; diff --git a/src/peerflow/video_renderer.cpp b/src/peerflow/video_renderer.cpp index f029c55d9..b399c1d0f 100644 --- a/src/peerflow/video_renderer.cpp +++ b/src/peerflow/video_renderer.cpp @@ -18,6 +18,9 @@ #include "video_renderer.h" + +#define SWAP(a, b, t) {t = a; a = b; b = t;} + namespace wire { VideoRendererSink::VideoRendererSink(struct peerflow *pf, @@ -27,7 +30,8 @@ VideoRendererSink::VideoRendererSink(struct peerflow *pf, last_width_(0), last_height_(0), fps_count_(0), - frame_count_(0) + frame_count_(0), + ssrc_(0) { char uid_anon[ANON_ID_LEN]; char cid_anon[ANON_CLIENT_LEN]; @@ -60,6 +64,39 @@ void VideoRendererSink::OnFrame(const webrtc::VideoFrame& frame) uint64_t now = tmr_jiffies(); char userid_anon[ANON_ID_LEN]; char clientid_anon[ANON_CLIENT_LEN]; + webrtc::RtpPacketInfos pinfos = frame.packet_infos(); + + if (pinfos.size() > 0) { + uint32_t sid; + int err = 0; + + sid = pinfos[0].csrcs().size() > 0 ? pinfos[0].csrcs()[0] : pinfos[0].ssrc(); + + if (sid != ssrc_) { + char *uid = NULL, *cid = NULL, *t; + + err = peerflow_get_userid_for_ssrc(pf_, + sid, + true, + &uid, + &cid, + NULL); + if (!err) { + SWAP(uid, userid_remote_, t); + SWAP(cid, clientid_remote_, t); + + info("VideoRenderSync::OnFrame: switch user: %s.%s res: %dx%d\n", + anon_id(userid_anon, userid_remote_), + anon_client(clientid_anon, clientid_remote_), + fw, fh); + fps_count_ = 0; + ts_fps_ = now; + } + ssrc_ = sid; + mem_deref(uid); + mem_deref(cid); + } + } frame_count_++; fps_count_++; @@ -120,6 +157,7 @@ void VideoRendererSink::OnFrame(const webrtc::VideoFrame& frame) avsframe.ys = i420->StrideY(); avsframe.us = i420->StrideU(); avsframe.vs = i420->StrideV(); + iflow_render_frameh(&avsframe, userid_remote_, clientid_remote_); } diff --git a/src/peerflow/video_renderer.h b/src/peerflow/video_renderer.h index 148de5907..f9724164b 100644 --- a/src/peerflow/video_renderer.h +++ b/src/peerflow/video_renderer.h @@ -69,6 +69,7 @@ class VideoRendererSink : public rtc::VideoSinkInterface { uint64_t ts_fps_; uint32_t fps_count_; uint32_t frame_count_; + uint32_t ssrc_; }; } diff --git a/src/wcall/marshal.c b/src/wcall/marshal.c index 27eb29ad4..5f5243aca 100644 --- a/src/wcall/marshal.c +++ b/src/wcall/marshal.c @@ -51,6 +51,7 @@ enum mq_event { WCALL_MEV_DCE_SEND, WCALL_MEV_DESTROY, WCALL_MEV_SET_MUTE, + WCALL_MEV_REQ_VSTREAMS, }; @@ -138,6 +139,11 @@ struct mq_data { struct { int muted; } set_mute; + + struct { + int mode; + char *json; + } req_vstreams; } u; }; @@ -187,6 +193,10 @@ static void md_destructor(void *arg) mem_deref(md->u.set_clients.json); break; + case WCALL_MEV_REQ_VSTREAMS: + mem_deref(md->u.req_vstreams.json); + break; + default: break; } @@ -325,6 +335,12 @@ static void mqueue_handler(int id, void *data, void *arg) wcall_i_set_mute(md->u.set_mute.muted); break; + case WCALL_MEV_REQ_VSTREAMS: + wcall_i_request_video_streams(md->wcall, + md->u.req_vstreams.mode, + md->u.req_vstreams.json); + break; + default: warning("wcall: marshal: unknown event: %d\n", id); break; @@ -991,6 +1007,64 @@ void wcall_set_mute(WUSER_HANDLE wuser, int muted) mem_deref(md); } + +AVS_EXPORT +int wcall_request_video_streams(WUSER_HANDLE wuser, + const char *convid, + int mode, + const char *json) +{ + struct calling_instance *inst; + struct mq_data *md = NULL; + struct wcall *wcall; + int err = 0; + + if (!convid) { + warning("wcall: request_video_streams: no convid set\n"); + return EINVAL; + } + + inst = wuser2inst(wuser); + if (!inst) { + warning("wcall: request_video_streams: " + "invalid handle: 0x%08X\n", + wuser); + return EINVAL; + } + + if (!json) { + warning("wcall: request_video_streams: no json\n"); + return EINVAL; + } + + wcall = wcall_lookup(inst, convid); + if (!wcall) { + warning("wcall: request_video_streams: couldnt find conv\n"); + return EPROTO; + } + + md = md_new(inst, wcall, WCALL_MEV_REQ_VSTREAMS); + if (!md) + return ENOMEM; + + err = str_dup(&md->u.req_vstreams.json, json); + if (err) + goto out; + + md->u.req_vstreams.mode = mode; + + err = md_enqueue(md); + if (err) + goto out; + + out: + if (err) + mem_deref(md); + + return err; +} + + void wcall_marshal_destroy(struct calling_instance *inst) { struct mq_data *md = NULL; diff --git a/src/wcall/wcall.c b/src/wcall/wcall.c index 129901bd1..65d414802 100644 --- a/src/wcall/wcall.c +++ b/src/wcall/wcall.c @@ -2247,7 +2247,7 @@ static void instance_destructor(void *arg) static void msys_mute_handler(bool muted, void *arg) { - WUSER_HANDLE wuser = (WUSER_HANDLE)arg; + WUSER_HANDLE wuser = (WUSER_HANDLE)(unsigned long)arg; struct calling_instance *inst; struct le *le; uint64_t now = tmr_jiffies(); @@ -2552,8 +2552,8 @@ int wcall_i_answer(struct wcall *wcall, call_type = (call_type == WCALL_CALL_TYPE_FORCED_AUDIO) ? WCALL_CALL_TYPE_NORMAL : call_type; - info(APITAG "wcall(%p): answer calltype=%s cbr=%d\n", - wcall, wcall_call_type_name(call_type), audio_cbr); + info(APITAG "wcall(%p): answer calltype=%s\n", + wcall, wcall_call_type_name(call_type)); if (wcall->disable_audio) wcall->disable_audio = false; @@ -3660,12 +3660,12 @@ void wcall_i_set_clients_for_conv(struct wcall *wcall, const char *json) int err = 0; if (!wcall) { - warning("wcall; set_clients_for_conv: no wcall\n"); + warning("wcall: set_clients_for_conv: no wcall\n"); return; } if (!wcall->icall) { - warning("wcall; set_clients_for_conv: no icall\n"); + warning("wcall: set_clients_for_conv: no icall\n"); return; } @@ -3718,6 +3718,77 @@ void wcall_i_set_clients_for_conv(struct wcall *wcall, const char *json) return; } +void wcall_i_request_video_streams(struct wcall *wcall, + int mode, + const char *json) +{ + struct json_object *jobj, *jclients; + size_t nclients, i; + struct list clientl = LIST_INIT; + + size_t len; + int err = 0; + + if (!wcall) { + warning("wcall: request_video_streams: no wcall\n"); + return; + } + + if (!wcall->icall) { + warning("wcall: request_video_streams: no icall\n"); + return; + } + + info(APITAG "wcall(%p): request_video_streams\n", wcall); + + len = strlen(json); + err = jzon_decode(&jobj, json, len); + info(APITAG "wcall(%p): request_video_streams err=%d\n", wcall, err); + if (err) + return; + +#if 0 + jzon_dump(jobj); +#endif + + err = jzon_array(&jclients, jobj, "clients"); + if (err) + goto out; + + if (!jzon_is_array(jclients)) { + warning("wcall(%p): request_video_streams: json object is not an array\n"); + goto out; + } + + nclients = json_object_array_length(jclients); + for (i = 0; i < nclients; ++i) { + const char *uid, *cid; + struct json_object *jcli; + struct icall_client *cli; + + jcli = json_object_array_get_idx(jclients, i); + if (!jcli) { + goto out; + } + + uid = jzon_str(jcli, "userid"); + cid = jzon_str(jcli, "clientid"); + //q = jzon_int(jcli, "quality"); + if (uid && cid) { + cli = icall_client_alloc(uid, cid); + list_append(&clientl, &cli->le, cli); + } + } + + ICALL_CALL(wcall->icall, request_video_streams, + &clientl, ICALL_STREAM_MODE_DEFAULT); + +out: + mem_deref(jobj); + list_flush(&clientl); + return; +} + AVS_EXPORT void wcall_poll(void) { diff --git a/src/wcall/wcall.h b/src/wcall/wcall.h index af45c6487..6a04c44ad 100644 --- a/src/wcall/wcall.h +++ b/src/wcall/wcall.h @@ -74,5 +74,8 @@ void wcall_i_set_media_laddr(struct wcall *wcall,const char *laddr); void wcall_i_set_clients_for_conv(struct wcall *wcall, const char *json); void wcall_i_destroy(struct calling_instance *inst); void wcall_i_set_mute(int muted); +void wcall_i_request_video_streams(struct wcall *wcall, + int mode, + const char *json); void wcall_marshal_destroy(struct calling_instance *inst); diff --git a/test/test_econn.cpp b/test/test_econn.cpp index 52e7b26a7..094d52260 100644 --- a/test/test_econn.cpp +++ b/test/test_econn.cpp @@ -335,6 +335,7 @@ static void client_new_econn(struct client *cli) NULL, NULL, NULL, + NULL, econn_close_handler, cli); ASSERT_EQ(0, err); diff --git a/tools/verifyd/tool.mk b/tools/verifyd/tool.mk index 0090427df..05e122d8b 100644 --- a/tools/verifyd/tool.mk +++ b/tools/verifyd/tool.mk @@ -8,5 +8,6 @@ verifyd_CFLAGS := $(AVS_CFLAGS) $(AVS_CFLAGS) verifyd_LIBS := $(AVS_LIBS) $(MENG_LIBS) verifyd_DEPS := $(AVS_DEPS) $(MENG_DEPS) verifyd_LIB_FILES := $(AVS_STATIC) $(MENG_STATIC) +verifyd_LIBS += -lpthread include mk/tool.mk diff --git a/tools/zcall/conv.c b/tools/zcall/conv.c index 5c0788068..2c21c18dc 100644 --- a/tools/zcall/conv.c +++ b/tools/zcall/conv.c @@ -978,6 +978,25 @@ struct key_stroke interrupt_stroke = { }; +/*** 'n' ... Next video page + */ + +static bool vpage_key_handler(int ch) +{ + (void) ch; + + view_next_page(); + + return true; +} + +struct key_stroke vpage_stroke = { + .ch = 'n', + .h = vpage_key_handler, + .help = "next page of incoming videos" +}; + + /*** 'a' ... Accept first pending call */ @@ -1898,6 +1917,7 @@ int conv_init(void) register_key_stroke(&videop_stroke); register_key_stroke(&interrupt_stroke); register_key_stroke(&propsync_stroke); + register_key_stroke(&vpage_stroke); register_command(&switch_command); register_command(&log_command); diff --git a/tools/zcall/osx_view.m b/tools/zcall/osx_view.m index a2fc3a83a..d3cdfe55c 100644 --- a/tools/zcall/osx_view.m +++ b/tools/zcall/osx_view.m @@ -36,9 +36,31 @@ #define ICON_WH 64 #define ICON_BW 16 +#define VIDEO_PAGE_SIZE 3 + +@interface VideoClient : NSObject +@property (copy) NSString *userid; +@property (copy) NSString *clientid; +- (id) initWithUser:(NSString*)uid client:(NSString*)cid; +@end + + +@implementation VideoClient +- (id) initWithUser:(NSString*)uid client:(NSString*)cid +{ + self = [super init]; + if (self) { + self.userid = uid; + self.clientid = cid; + } + return self; +} +@end + static struct { NSWindow *win; NSMutableArray *views; + NSMutableArray *clients; NSView *preview; NSView *muteView; NSTimer *timer; @@ -51,13 +73,17 @@ NSString *local_clientid; struct tmr tmr; + uint32_t page; + bool muted; + char convid[ECONN_ID_LEN]; } vidloop; WUSER_HANDLE calling3_get_wuser(void); int osx_view_init(struct view** v); static void osx_arrange_views(void); +static void osx_view_next_page(void); @interface VideoDelegate : NSObject - (void)applicationDidFinishLaunching:(NSNotification *)aNotification; @@ -158,6 +184,7 @@ static void osx_arrange_views(void) uint32_t vcount, i, v; uint32_t rows, cols, w, h, vh; uint32_t xp, yp; + uint32_t cp; vcount = vidloop.views.count + (vidloop.preview_visible ? 1 : 0); @@ -176,6 +203,8 @@ static void osx_arrange_views(void) h = WIN_H / rows; vh = w * 3 / 4; + cp = vidloop.page; + for (v = 0; v < vidloop.views.count; v++) { xp = (v % cols) * w; yp = WIN_H - h - ((v / cols) * h) + (h - vh) / 2; @@ -183,6 +212,18 @@ static void osx_arrange_views(void) NSRect rect = NSMakeRect(xp, yp, w, vh); AVSVideoViewOSX *oview = [vidloop.views objectAtIndex: v]; oview.frame = rect; + + if (cp < vidloop.clients.count) { + VideoClient *client = [vidloop.clients objectAtIndex:cp]; + oview.userid = client.userid; + oview.clientid = client.clientid; + } + else { + oview.userid = @""; + oview.clientid = @""; + } + cp++; + } vh = w * 3 / 4; @@ -194,11 +235,16 @@ static void osx_arrange_views(void) vidloop.muteView.frame = rect; osx_view_show_mute(wcall_get_mute(calling3_get_wuser())); NSApplication *app = [NSApplication sharedApplication]; - [vidloop.win makeKeyAndOrderFront:app]; - [NSApp activateIgnoringOtherApps:YES]; [[vidloop.win contentView] addSubview:vidloop.preview]; [vidloop.preview display]; + + if (vidloop.preview_visible || vidloop.view_visible) { + [vidloop.win makeKeyAndOrderFront:app]; + [NSApp activateIgnoringOtherApps:YES]; + } + else + [vidloop.win orderOut:nil]; } static const char *video_state_name(int vstate) @@ -219,60 +265,89 @@ static void osx_arrange_views(void) } } -static void osx_vidstate_changed(const char *userid, const char *clientid, int state) +static void osx_request_streams(void) +{ + char *json_str = NULL; + struct json_object *jobj; + struct json_object *jcli; + struct json_object *jclients; + uint32_t cp = vidloop.page; + VideoClient *client; + + uint32_t ep = MIN(cp + VIDEO_PAGE_SIZE, vidloop.clients.count); + + jobj = jzon_alloc_object(); + + jclients = json_object_new_array(); + + while (cp < ep) { + client = [vidloop.clients objectAtIndex:cp]; + jcli = jzon_alloc_object(); + + jzon_add_str(jcli, "userid", "%s", [client.userid UTF8String]); + jzon_add_str(jcli, "clientid", "%s", [client.clientid UTF8String]); + json_object_array_add(jclients, jcli); + + cp++; + } + + jzon_add_str(jobj, "convid", "%s", vidloop.convid); + json_object_object_add(jobj, "clients", jclients); + + jzon_encode(&json_str, jobj); + + if (json_str) { + WUSER_HANDLE wuser = calling3_get_wuser(); + wcall_request_video_streams(wuser, + vidloop.convid, + 0, + json_str); + } + mem_deref(jobj); + mem_deref(json_str); +} + +static void osx_vidstate_changed(const char *convid, + const char *userid, + const char *clientid, + int state) { NSString *uid = [NSString stringWithUTF8String: userid]; NSString *cid = [NSString stringWithUTF8String: clientid]; + NSString *coid = [NSString stringWithUTF8String: convid]; info("osx_vidstate_changed for %s.%s -> %s\n", userid, clientid, video_state_name(state)); dispatch_async(dispatch_get_main_queue(), ^{ - bool found = false; + VideoClient *client; + str_ncpy(vidloop.convid, [coid UTF8String], ECONN_ID_LEN); if (![vidloop.local_userid isEqualToString: uid] || ![vidloop.local_clientid isEqualToString: cid]) { switch(state) { case WCALL_VIDEO_STATE_STARTED: case WCALL_VIDEO_STATE_SCREENSHARE: - for (unsigned int v = 0; v < vidloop.views.count; v++) { - AVSVideoViewOSX *view = [vidloop.views objectAtIndex: v]; - if ([view.userid isEqualToString: uid] && - [view.clientid isEqualToString: cid]) { - found = true; - break; - } - } + for (unsigned int c = 0; c < vidloop.clients.count; c++) { + client = [vidloop.clients objectAtIndex: c]; - if (!found) { - NSRect rect = NSMakeRect(0, 0, WIN_W, WIN_W * 3 / 4); - AVSVideoViewOSX *v = [[AVSVideoViewOSX alloc] initWithFrame:rect]; - v.userid = uid; - v.clientid = cid; - [vidloop.views addObject: v]; - [[vidloop.win contentView] addSubview:v]; - [v display]; - - info("osx_view adding renderer for %s now %u\n", - [uid UTF8String], vidloop.views.count); + if ([client.userid isEqualToString: uid] && + [client.clientid isEqualToString: cid]) { + return; + } } + client = [[VideoClient alloc] initWithUser: uid client: cid]; + [vidloop.clients addObject: client]; break; default: { + for (unsigned int c = 0; c < vidloop.clients.count; c++) { + client = [vidloop.clients objectAtIndex: c]; - info("osx_view removing renderer for %s\n", - [uid UTF8String]); - for (unsigned int v = 0; v < vidloop.views.count; v++) { - AVSVideoViewOSX *view = [vidloop.views objectAtIndex: v]; - - if ([view.userid isEqualToString: uid] && - [view.clientid isEqualToString: cid]) { - - [view removeFromSuperview]; - [vidloop.views removeObject: view]; - info("osx_view removing renderer for %s now %u\n", - userid, vidloop.views.count); + if ([client.userid isEqualToString: uid] && + [client.clientid isEqualToString: cid]) { + [vidloop.clients removeObject: client]; break; } } @@ -280,16 +355,16 @@ static void osx_vidstate_changed(const char *userid, const char *clientid, int s break; } } - vidloop.view_visible = [vidloop.views count] > 0; - + osx_request_streams(); + vidloop.view_visible = (vidloop.clients.count > 0); + osx_arrange_views(); if (vidloop.view_visible || vidloop.preview_visible) { view_show(); } else { view_hide(); } - osx_arrange_views(); }); } @@ -356,7 +431,8 @@ static void osx_view_set_local_user(const char *userid, const char *clientid) .render_frame = osx_render_frame, .preview_start = osx_preview_start, .preview_stop = osx_preview_stop, - .view_show_mute = osx_view_show_mute + .show_mute = osx_view_show_mute, + .next_page = osx_view_next_page }; @@ -377,6 +453,7 @@ int osx_view_init(struct view** v) [vidloop.win orderOut:nil]; vidloop.views = [[NSMutableArray alloc] init]; + vidloop.clients = [[NSMutableArray alloc] init]; NSRect previewRect = NSMakeRect(0, 0, WIN_W, WIN_H); vidloop.preview = [[NSView alloc] initWithFrame:previewRect]; @@ -397,11 +474,26 @@ int osx_view_init(struct view** v) osx_view_hide(); vidloop.capturer = [[AVSCapturer alloc] init]; - osx_arrange_views(); + for (int i = 0; i < VIDEO_PAGE_SIZE; i++) { + NSRect rect = NSMakeRect(0, 0, WIN_W, WIN_W * 3 / 4); + AVSVideoViewOSX *cv = [[AVSVideoViewOSX alloc] initWithFrame:rect]; + [vidloop.views addObject: cv]; + [[vidloop.win contentView] addSubview:cv]; + [cv display]; + } *v = &_view; return 0; } +static void osx_view_next_page(void) +{ + vidloop.page += VIDEO_PAGE_SIZE; + if (vidloop.page >= vidloop.clients.count) + vidloop.page = 0; + + osx_request_streams(); + osx_arrange_views(); +} @implementation VideoDelegate - (void)applicationDidFinishLaunching:(NSNotification *)aNotification diff --git a/tools/zcall/test_view.c b/tools/zcall/test_view.c index 5e82e72ce..e0dbd13cf 100644 --- a/tools/zcall/test_view.c +++ b/tools/zcall/test_view.c @@ -78,7 +78,10 @@ static void test_preview_stop(void) test_capturer_stop(); } -static void test_vidstate_changed(const char *userid, const char *clientid, int state) +static void test_vidstate_changed(const char *convid, + const char *userid, + const char *clientid, + int state) { (void)userid; (void)clientid; @@ -131,6 +134,10 @@ static void test_view_show_mute(bool muted) { } +static void test_view_next_page(void) +{ +} + static struct view _view = { .runloop_start = test_runloop_start, .runloop_stop = test_runloop_stop, @@ -142,7 +149,8 @@ static struct view _view = { .render_frame = test_render_frame, .preview_start = test_preview_start, .preview_stop = test_preview_stop, - .view_show_mute = test_view_show_mute + .show_mute = test_view_show_mute, + .next_page = test_view_next_page }; diff --git a/tools/zcall/tool.mk b/tools/zcall/tool.mk index 84cbbd71b..00c660b8f 100644 --- a/tools/zcall/tool.mk +++ b/tools/zcall/tool.mk @@ -51,6 +51,8 @@ zcall_CPPFLAGS += -DHAVE_READLINE=1 zcall_LIBS += -lreadline endif +zcall_LIBS += -lpthread + zcall_DEPS := $(AVS_DEPS) $(MENG_DEPS) zcall_LIB_FILES := $(AVS_STATIC) $(MENG_STATIC) diff --git a/tools/zcall/view.c b/tools/zcall/view.c index a3d44dc4d..a34e33522 100644 --- a/tools/zcall/view.c +++ b/tools/zcall/view.c @@ -23,6 +23,7 @@ #include "view_internal.h" static struct view *_view = NULL; +WUSER_HANDLE calling3_get_wuser(void); int osx_view_init(struct view** v); int test_view_init(struct view** v); @@ -101,6 +102,13 @@ void view_set_local_user(const char *userid, const char *clientid) } } +void view_next_page(void) +{ + if (_view) { + _view->next_page(); + } +} + void wcall_vidstate_handler(const char *convid, const char *userid, const char *clientid, @@ -109,7 +117,7 @@ void wcall_vidstate_handler(const char *convid, { if (_view) { - _view->vidstate_changed(userid, clientid, state); + _view->vidstate_changed(convid, userid, clientid, state); } } @@ -139,7 +147,7 @@ void size_handler(int w, void view_show_mute(bool muted) { if (_view) { - _view->view_show_mute(muted); + _view->show_mute(muted); } } diff --git a/tools/zcall/view.h b/tools/zcall/view.h index 36eb8073e..45729b4f8 100644 --- a/tools/zcall/view.h +++ b/tools/zcall/view.h @@ -38,6 +38,8 @@ void vidstate_handler(const char *userid, enum flowmgr_video_receive_state state void view_show_mute(bool muted); +void view_next_page(void); + int render_handler(struct avs_vidframe * frame, const char *userid, const char *clientid, diff --git a/tools/zcall/view_internal.h b/tools/zcall/view_internal.h index fc3bc7195..7fc59962e 100644 --- a/tools/zcall/view_internal.h +++ b/tools/zcall/view_internal.h @@ -29,7 +29,10 @@ typedef void (view_hidef)(void); typedef void (set_local_userf)(const char *userid, const char *clientid); -typedef void (vidstate_changedf)(const char *userid, const char *clientid, int state); +typedef void (vidstate_changedf)(const char *convid, + const char *userid, + const char *clientid, + int state); typedef int (render_framef)(struct avs_vidframe * frame, const char *userid, @@ -38,7 +41,9 @@ typedef int (render_framef)(struct avs_vidframe * frame, typedef void (preview_startf)(void); typedef void (preview_stopf)(void); -typedef void (view_show_mutef)(bool muted); +typedef void (show_mutef)(bool muted); + +typedef void (next_pagef)(void); struct view { runloop_startf *runloop_start; @@ -51,7 +56,8 @@ struct view { render_framef *render_frame; preview_startf *preview_start; preview_stopf *preview_stop; - view_show_mutef *view_show_mute; + show_mutef *show_mute; + next_pagef *next_page; }; #endif diff --git a/wasm/src/avs_pc.ts b/wasm/src/avs_pc.ts index df7dfc33a..1e7bf1b79 100644 --- a/wasm/src/avs_pc.ts +++ b/wasm/src/avs_pc.ts @@ -19,7 +19,7 @@ export type VideoStreamHandler = ( convid: string, remote_userid: string, remote_clientid: string, - streams: readonly MediaStream[] + streams: readonly MediaStream[] | null ) => void; type RelayCand = { @@ -49,8 +49,8 @@ type UserInfo = { audio_level: number; first_recv_audio: boolean; first_succ_audio: boolean; - first_recv_video: boolean; first_succ_video: boolean; + transp_ssrcv: string | null; }; interface PeerConnection { @@ -70,6 +70,7 @@ interface PeerConnection { users: any; iva: Uint8Array; ivv: Uint8Array; + streams: {[ssrc: string]: string}; insertable_legacy: boolean; insertable_streams: boolean; @@ -248,7 +249,7 @@ function frameHeaderDecode(buf) { } } -function frameHeaderEncode(fid, kid) +function frameHeaderEncode(fid, kid, csrc) { const ab = new ArrayBuffer(32); const buf = new Uint8Array(ab); @@ -257,7 +258,7 @@ function frameHeaderEncode(fid, kid) let klen = 0; let flen = 0; let p = 2; - let ext = 0; + let ext = (csrc > 0) ? 1 : 0; if (kid > 7) { x = 1; @@ -276,6 +277,11 @@ function frameHeaderEncode(fid, kid) } p += writeBytes(buf, p, flen, fid); + if (csrc > 0) { + buf[p] = 0x31; + p++; + p+= writeBytes(buf, p, 4, csrc); + } return buf.subarray(0, p); } @@ -324,15 +330,18 @@ function encryptFrame(coder, rtcFrame, controller) { const isVideo = t === '[object RTCEncodedVideoFrame]'; const userid = "self"; + const meta = rtcFrame.getMetadata(); + const ssrc = meta.synchronizationSource; + if (isVideo && !coder.video.first_recv) { doLog("frame_enc: encrypt: first frame received type: video uid: " + - userid + " fid: " + coder.frameId + " csrc: " + coder.video.ssrc); + userid + " fid: " + coder.frameId + " ssrc: " + ssrc); coder.video.first_recv = true; coder.video.first_succ = false; } if (!isVideo && !coder.audio.first_recv) { doLog("frame_enc: encrypt: first frame received type: audio uid: " + - userid + " fid: " + coder.frameId + " csrc: " + coder.audio.ssrc); + userid + " fid: " + coder.frameId + " ssrc: " + ssrc); coder.audio.first_recv = true; coder.audio.first_succ = false; } @@ -345,7 +354,7 @@ function encryptFrame(coder, rtcFrame, controller) { const baseiv = isVideo ? coder.video.iv : coder.audio.iv; const iv = xor_iv(baseiv, coder.frameId, coder.currentKey.id); - const hdr = frameHeaderEncode(coder.frameId, coder.currentKey.id); + const hdr = frameHeaderEncode(coder.frameId, coder.currentKey.id, ssrc); crypto.subtle.encrypt( { name: "AES-GCM", @@ -368,12 +377,12 @@ function encryptFrame(coder, rtcFrame, controller) { if (isVideo && !coder.video.first_succ) { doLog("frame_enc: encrypt: first frame encrypted type: video uid: " + - userid + " fid: " + coder.frameId + " csrc: " + coder.video.ssrc); + userid + " fid: " + coder.frameId + " csrc: " + ssrc); coder.video.first_succ = true; } if (!isVideo && !coder.audio.first_succ) { doLog("frame_enc: encrypt: first frame encrypted type: audio uid: " + - userid + " fid: " + coder.frameId + " csrc: " + coder.audio.ssrc); + userid + " fid: " + coder.frameId + " csrc: " + ssrc); coder.audio.first_succ = true; } @@ -388,44 +397,79 @@ function decryptFrame(coder, rtcFrame, controller) { const t = Object.prototype.toString.call(rtcFrame); const isVideo = t === '[object RTCEncodedVideoFrame]' + const frameHdr = frameHeaderDecode(data); + + if (frameHdr == null) { + doLog('decryptFrame: failed to decode frame header'); + return; + } + const meta = rtcFrame.getMetadata(); - let ssrc = null; - if (meta.contributingSources && meta.contributingSources.length > 0) { - ssrc = meta.contributingSources[0].toString(); + let ssrc = meta.synchronizationSource.toString(); + let csrc = null; + if (frameHdr.csrc != "0") { + csrc = frameHdr.csrc; + } + else if (meta.contributingSources && meta.contributingSources.length > 0) { + csrc = meta.contributingSources[0].toString(); } else { - ssrc = meta.synchronizationSource.toString(); + csrc = ssrc; + } + + if (!csrc) { + doLog('decryptFrame: no csrc for ssrc ' + ssrc); + return; } if (isVideo) { - uinfo = coder.video.users[ssrc]; + uinfo = coder.video.users[csrc]; } else { - uinfo = coder.audio.users[ssrc]; + uinfo = coder.audio.users[csrc]; } - if (!uinfo || !ssrc) { - doLog('decryptFrame: no userinfo for ssrc: ' + ssrc); + if (!uinfo) { + doLog('decryptFrame: no userinfo for csrc: ' + csrc); return; } - //console.log('decryptFrame: video ' + isVideo + ' ssrc ' + ssrc + ' user ' + uinfo.userid); - const frameHdr = frameHeaderDecode(data); - - if (frameHdr == null) { - doLog('decryptFrame: failed to decode frame header'); - return; + //console.log('frame_dec: video ' + isVideo + ' ssrc ' + ssrc + ' csrc ' + csrc + ' user ' + uinfo.userid); + + if (isVideo && uinfo.transp_ssrcv != ssrc) { + doLog("frame_dec: decrypt: first frame received type: video uid: " + + uinfo.userid + " fid: " + frameHdr.frameId + " csrc: " + csrc); + uinfo.first_succ_video = false; + + /* Only call videoStreamHandler for selective video */ + if (uinfo.ssrcv != ssrc) { + for (const [key, u] of Object.entries(coder.video.users)) { + if (u.transp_ssrcv == ssrc) { + postMessage({ + op: "setvstream", + self: coder.self, + userid: u.userid, + clientid: u.clientid, + ssrc: "0", + }); + u.transp_ssrcv = null; + } + } + + postMessage({ + op: "setvstream", + self: coder.self, + userid: uinfo.userid, + clientid: uinfo.clientid, + ssrc: ssrc, + }); + } + uinfo.transp_ssrcv = ssrc; } - if (isVideo && !uinfo.first_recv_video) { - doLog("frame_dec: decrypt: first frame received type: video uid: " + - uinfo.userid + " fid: " + frameHdr.frameId + " csrc: " + ssrc); - uinfo.first_recv_video = true; - uinfo.first_succ_video = false; - } if (!isVideo && !uinfo.first_recv_audio) { doLog("frame_dec: decrypt: first frame received type: audio uid: " + - uinfo.userid + " fid: " + frameHdr.frameId + " csrc: " + ssrc); + uinfo.userid + " fid: " + frameHdr.frameId + " csrc: " + csrc); uinfo.first_recv_audio = true; uinfo.first_succ_audio = false; } @@ -473,12 +517,12 @@ function decryptFrame(coder, rtcFrame, controller) { controller.enqueue(rtcFrame); if (isVideo && !uinfo.first_succ_video) { doLog("frame_dec: decrypt: first frame decrypted type: video uid: " + - uinfo.userid + " fid: " + frameHdr.frameId + " csrc: " + ssrc); + uinfo.userid + " fid: " + frameHdr.frameId + " csrc: " + csrc); uinfo.first_succ_video = true; } if (!isVideo && !uinfo.first_succ_audio) { doLog("frame_dec: decrypt: first frame decrypted type: audio uid: " + - uinfo.userid + " fid: " + frameHdr.frameId + " csrc: " + ssrc); + uinfo.userid + " fid: " + frameHdr.frameId + " csrc: " + csrc); uinfo.first_succ_audio = true; } }) @@ -511,13 +555,13 @@ onmessage = async (event) => { first_recv: false, first_succ: false, iv: iva, - users: [], + users: {}, }, video: { first_recv: false, first_succ: false, iv: ivv, - users: [], + users: {}, }, } coders[self] = coder; @@ -604,6 +648,37 @@ onmessage = async (event) => { }; ` +function callStreamHandler(pc: PeerConnection, + userid: string, + clientid: string, + ssrc: string) { + + if (ssrc == "0") { + if (videoStreamHandler) { + pc_log(LOG_LEVEL_INFO, `calling vsh(${pc.convid}, ${userid}, ${clientid}) to remove renderer`); + videoStreamHandler(pc.convid, + userid, + clientid, + null); + } + } + else if (pc.rtc) { + const label = pc.streams[ssrc]; + pc.rtc.getTransceivers().forEach(trans => { + if (trans.receiver.track.label === label) { + let stream = new MediaStream([trans.receiver.track]); + if (videoStreamHandler) { + pc_log(LOG_LEVEL_INFO, `calling vsh(${pc.convid}, ${userid}, ${clientid}) with 1 stream`); + videoStreamHandler(pc.convid, + userid, + clientid, + [stream]); + } + } + }); + } +} + function createWorker() { pc_log(LOG_LEVEL_INFO, "Creating AVS worker"); const blob = new Blob([workerContent], { type: 'text/javascript' }); @@ -614,16 +689,24 @@ function createWorker() { const {op, self} = event.data; if (op === 'getMediaKey') { - const {index} = event.data; + const {index} = event.data; - ccallGetMediaKey(self, index); + ccallGetMediaKey(self, index); } else if (op === 'getCurrentMediaKey') { - ccallGetCurrentMediaKey(self); + ccallGetCurrentMediaKey(self); } else if (op === 'log') { - const {level, logString} = event.data; - pc_log(level, logString); + const {level, logString} = event.data; + pc_log(level, logString); + } + else if (op === 'setvstream') { + const {userid, clientid, ssrc} = event.data; + let pcs = connectionsStore.getPeerConnectionBySelf(self); + + if (pcs.length == 1) { + callStreamHandler(pcs[0], userid, clientid, ssrc); + } } } @@ -702,6 +785,11 @@ const connectionsStore = (() => { return !!pc && pc.convid === convid; }) as PeerConnection[]; }, + getPeerConnectionBySelf: (self: number): PeerConnection[] => { + return peerConnections.filter(pc => { + return !!pc && pc.self == self; + }) as PeerConnection[]; + }, removePeerConnection: (index: number) => removeItem(peerConnections, index), storeDataChannel: (dataChannel: RTCDataChannel) => storeItem(dataChannels, dataChannel), @@ -1174,6 +1262,7 @@ function pc_New(self: number, convidPtr: number, sent_vpkts: 0, rtt: 0 }, + streams: {}, }; worker.postMessage({op: 'create', self: pc.self, iva: iva8, ivv: ivv8}); @@ -1716,8 +1805,8 @@ function pc_AddUserInfo(hnd: number, labelPtr: number, audio_level: 0, first_recv_audio: false, first_succ_audio: false, - first_recv_video: false, - first_succ_video: false + first_succ_video: false, + transp_ssrcv: null, }; pc_log(LOG_LEVEL_INFO, `pc_AddUserInfo: label=${label} ${userId}/${clientId} ssrc:${ssrca}/${ssrcv}`); @@ -1749,6 +1838,21 @@ function pc_RemoveUserInfo(hnd: number, labelPtr: number) { } } + +function extractSSRCs(pc: PeerConnection, + sdp: string) { + + sdp.split('\r\n').forEach(l => { + let m = l.match(/a=ssrc:(\d+) label:(.*)/) + if (m) { + let ssrc = m[1]; + let label = m[2]; + + pc.streams[ssrc] = label; + } + }); +} + function pc_SetRemoteDescription(hnd: number, typePtr: number, sdpPtr: number) { pc_log(LOG_LEVEL_INFO, `pc_SetRemoteDescription: hnd=${hnd}`); @@ -1784,6 +1888,8 @@ function pc_SetRemoteDescription(hnd: number, typePtr: number, sdpPtr: number) { .catch((err: any) => { pc_log(LOG_LEVEL_WARN, "setRemoteDescription failed: " + err, err); }); + + extractSSRCs(pc, sdp); } function pc_SetLocalDescription(hnd: number, typePtr: number, sdpPtr: number) {