From 8270a994b2cd15a7eda43906503db5492790fd37 Mon Sep 17 00:00:00 2001 From: Miroslav Pejic Date: Tue, 30 Apr 2024 11:32:59 +0200 Subject: [PATCH] [mirotalksfu] - add dominant speaker observer, fix, update dep --- app/src/Peer.js | 2 +- app/src/Room.js | 48 +++++++++++++++++++++++++++++++++++--- app/src/Server.js | 5 ++-- app/src/config.template.js | 2 ++ package.json | 4 ++-- public/js/Room.js | 2 +- public/js/RoomClient.js | 21 ++++++++++++++++- 7 files changed, 74 insertions(+), 10 deletions(-) diff --git a/app/src/Peer.js b/app/src/Peer.js index 532722bb..5ad9d0a7 100644 --- a/app/src/Peer.js +++ b/app/src/Peer.js @@ -116,7 +116,7 @@ module.exports = class Peer { this.transports.forEach((transport, transport_id) => { transport.close(); this.delTransport(transport_id); - log.debug('Close and delete peer transports', { + log.debug('Closed and deleted peer transport', { //transport_id: transport_id, transportInternal: transport.internal, transport_closed: transport.closed, diff --git a/app/src/Room.js b/app/src/Room.js index ac1a23a8..70d86806 100644 --- a/app/src/Room.js +++ b/app/src/Room.js @@ -4,6 +4,8 @@ const config = require('./config'); const Logger = require('./Logger'); const log = new Logger('Room'); +const { audioLevelObserverEnabled, activeSpeakerObserverEnabled } = config.mediasoup.router; + module.exports = class Room { constructor(room_id, worker, io) { this.id = room_id; @@ -12,8 +14,11 @@ module.exports = class Room { this.webRtcServerActive = config.mediasoup.webRtcServerActive; this.io = io; this.audioLevelObserver = null; - this.audioLevelObserverEnabled = true; + this.audioLevelObserverEnabled = audioLevelObserverEnabled !== undefined ? audioLevelObserverEnabled : true; this.audioLastUpdateTime = 0; + this.activeSpeakerObserverEnabled = + activeSpeakerObserverEnabled !== undefined ? activeSpeakerObserverEnabled : false; + this.activeSpeakerObserver = null; // ########################## this._isBroadcasting = false; // ########################## @@ -79,6 +84,9 @@ module.exports = class Room { if (this.audioLevelObserverEnabled) { this.startAudioLevelObservation(); } + if (this.activeSpeakerObserverEnabled) { + this.startActiveSpeakerObserver(); + } this.router.observer.on('close', () => { log.info('---------------> Router is now closed as the last peer has left the room', { room: this.id, @@ -87,6 +95,10 @@ module.exports = class Room { }); } + getRtpCapabilities() { + return this.router.rtpCapabilities; + } + closeRouter() { this.router.close(); log.debug('Close Room router', { @@ -157,8 +169,38 @@ module.exports = class Room { } } - getRtpCapabilities() { - return this.router.rtpCapabilities; + // #################################################### + // PRODUCER DOMINANT ACTIVE SPEAKER + // #################################################### + + async startActiveSpeakerObserver() { + this.activeSpeakerObserver = await this.router.createActiveSpeakerObserver(); + this.activeSpeakerObserver.on('dominantspeaker', (dominantSpeaker) => { + log.debug('activeSpeakerObserver "dominantspeaker" event', dominantSpeaker.producer.id); + this.peers.forEach((peer) => { + const { id, peer_audio, peer_name } = peer; + peer.producers.forEach((peerProducer) => { + if ( + peerProducer.id === dominantSpeaker.producer.id && + peerProducer.kind === 'audio' && + peer_audio + ) { + const data = { + peer_id: id, + peer_name: peer_name, + }; + // log.debug('Sending dominant speaker', data); + this.sendToAll('dominantSpeaker', data); + } + }); + }); + }); + } + + addProducerToActiveSpeakerObserver(producer) { + if (this.activeSpeakerObserverEnabled) { + this.activeSpeakerObserver.addProducer(producer); + } } // #################################################### diff --git a/app/src/Server.js b/app/src/Server.js index 48f33603..8dd68f9d 100644 --- a/app/src/Server.js +++ b/app/src/Server.js @@ -41,7 +41,7 @@ dependencies: { * @license For commercial or closed source, contact us at license.mirotalk@gmail.com or purchase directly via CodeCanyon * @license CodeCanyon: https://codecanyon.net/item/mirotalk-sfu-webrtc-realtime-video-conferences/40769970 * @author Miroslav Pejic - miroslav.pejic.85@gmail.com - * @version 1.4.25 + * @version 1.4.26 * */ @@ -1189,9 +1189,10 @@ function startServer() { producer_id: producer_id, }); - // add & monitor producer audio level + // add & monitor producer audio level and active speaker if (kind === 'audio') { room.addProducerToAudioLevelObserver({ producerId: producer_id }); + room.addProducerToActiveSpeakerObserver({ producerId: producer_id }); } //log.debug('Producer transport callback', { callback: producer_id }); diff --git a/app/src/config.template.js b/app/src/config.template.js index fd329a09..47d36e2d 100644 --- a/app/src/config.template.js +++ b/app/src/config.template.js @@ -359,6 +359,8 @@ module.exports = { }, // Router settings router: { + audioLevelObserverEnabled: true, + activeSpeakerObserverEnabled: false, mediaCodecs: [ { kind: 'audio', diff --git a/package.json b/package.json index 7ae9e3c6..8491f3e7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "mirotalksfu", - "version": "1.4.25", + "version": "1.4.26", "description": "WebRTC SFU browser-based video calls", "main": "Server.js", "scripts": { @@ -56,7 +56,7 @@ "mediasoup-client": "3.7.7", "ngrok": "^5.0.0-beta.2", "nodemailer": "^6.9.13", - "openai": "^4.38.5", + "openai": "^4.39.0", "qs": "6.12.1", "socket.io": "4.7.5", "swagger-ui-express": "5.0.0", diff --git a/public/js/Room.js b/public/js/Room.js index 629776b3..200515e7 100644 --- a/public/js/Room.js +++ b/public/js/Room.js @@ -11,7 +11,7 @@ if (location.href.substr(0, 5) !== 'https') location.href = 'https' + location.h * @license For commercial or closed source, contact us at license.mirotalk@gmail.com or purchase directly via CodeCanyon * @license CodeCanyon: https://codecanyon.net/item/mirotalk-sfu-webrtc-realtime-video-conferences/40769970 * @author Miroslav Pejic - miroslav.pejic.85@gmail.com - * @version 1.4.25 + * @version 1.4.26 * */ diff --git a/public/js/RoomClient.js b/public/js/RoomClient.js index 10f83223..1749246d 100644 --- a/public/js/RoomClient.js +++ b/public/js/RoomClient.js @@ -9,7 +9,7 @@ * @license For commercial or closed source, contact us at license.mirotalk@gmail.com or purchase directly via CodeCanyon * @license CodeCanyon: https://codecanyon.net/item/mirotalk-sfu-webrtc-realtime-video-conferences/40769970 * @author Miroslav Pejic - miroslav.pejic.85@gmail.com - * @version 1.4.25 + * @version 1.4.26 * */ @@ -796,6 +796,7 @@ class RoomClient { this.socket.on('wbCanvasToJson', this.handleWbCanvasToJson); this.socket.on('whiteboardAction', this.handleWhiteboardAction); this.socket.on('audioVolume', this.handleAudioVolumeData); + this.socket.on('dominantSpeaker', this.handleDominantSpeakerData); this.socket.on('updateRoomModerator', this.handleUpdateRoomModeratorData); this.socket.on('updateRoomModeratorALL', this.handleUpdateRoomModeratorALLData); this.socket.on('recordingAction', this.handleRecordingActionData); @@ -920,6 +921,10 @@ class RoomClient { this.handleAudioVolume(data); }; + handleDominantSpeakerData = (data) => { + this.handleDominantSpeaker(data); + }; + handleUpdateRoomModeratorData = (data) => { console.log('SocketOn Update room moderator', data); this.handleUpdateRoomModerator(data); @@ -5666,6 +5671,15 @@ class RoomClient { } } + // #################################################### + // HANDLE DOMINANT SPEAKER + // ################################################### + + handleDominantSpeaker(data) { + console.log('Dominant Speaker', data); + //... + } + // #################################################### // HANDLE BAN // ################################################### @@ -6236,9 +6250,11 @@ class RoomClient { case 'hide': let peerVideoButton = this.getId(data.peer_id + '___pVideo'); if (peerVideoButton) peerVideoButton.innerHTML = _PEER.videoOff; + break; case 'stop': let peerScreenButton = this.getId(data.peer_id + '___pScreen'); if (peerScreenButton) peerScreenButton.innerHTML = _PEER.screenOff; + break; default: break; } @@ -6466,6 +6482,7 @@ class RoomClient { this._moderator.video_cant_unhide = data.status; this._moderator.video_cant_unhide ? hide(tabVideoDevicesBtn) : show(tabVideoDevicesBtn); rc.roomMessage('video_cant_unhide', data.status); + break; case 'screen_cant_share': this._moderator.screen_cant_share = data.status; rc.roomMessage('screen_cant_share', data.status); @@ -6695,6 +6712,8 @@ class RoomClient { case error.UNKNOWN_ERROR: geoError = 'An unknown error occurred'; break; + default: + break; } rc.sendPeerGeoLocation(peer_id, 'geoLocationKO', `${rc.peer_name}: ${geoError}`); rc.userLog('warning', geoError, 'top-end', 5000);