From 1d53dc5670be651bc7e647440495168f12b4972a Mon Sep 17 00:00:00 2001
From: Miroslav Pejic <miroslav.pejic.85@gmail.com>
Date: Thu, 20 Feb 2025 21:57:43 +0100
Subject: [PATCH] [mirotalksfu] - #194 - Optimize video frame processing to
 reduce CPU usage

---
 app/src/Server.js              |  2 +-
 package.json                   |  2 +-
 public/js/Brand.js             |  2 +-
 public/js/Room.js              |  4 +--
 public/js/RoomClient.js        |  2 +-
 public/js/VirtualBackground.js | 48 +++++++++++++++++++++++++++++-----
 6 files changed, 47 insertions(+), 13 deletions(-)

diff --git a/app/src/Server.js b/app/src/Server.js
index 23522724..4fce6283 100644
--- a/app/src/Server.js
+++ b/app/src/Server.js
@@ -58,7 +58,7 @@ dev dependencies: {
  * @license For commercial or closed source, contact us at license.mirotalk@gmail.com or purchase directly via CodeCanyon
  * @license CodeCanyon: https://codecanyon.net/item/mirotalk-sfu-webrtc-realtime-video-conferences/40769970
  * @author  Miroslav Pejic - miroslav.pejic.85@gmail.com
- * @version 1.7.53
+ * @version 1.7.54
  *
  */
 
diff --git a/package.json b/package.json
index 7318edbf..a2572877 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
     "name": "mirotalksfu",
-    "version": "1.7.53",
+    "version": "1.7.54",
     "description": "WebRTC SFU browser-based video calls",
     "main": "Server.js",
     "scripts": {
diff --git a/public/js/Brand.js b/public/js/Brand.js
index 86d6a367..9e987d13 100644
--- a/public/js/Brand.js
+++ b/public/js/Brand.js
@@ -64,7 +64,7 @@ let BRAND = {
     },
     about: {
         imageUrl: '../images/mirotalk-logo.gif',
-        title: '<strong>WebRTC SFU v1.7.53</strong>',
+        title: '<strong>WebRTC SFU v1.7.54</strong>',
         html: `
             <button 
                 id="support-button" 
diff --git a/public/js/Room.js b/public/js/Room.js
index 740d2f79..d7ea5d71 100644
--- a/public/js/Room.js
+++ b/public/js/Room.js
@@ -11,7 +11,7 @@ if (location.href.substr(0, 5) !== 'https') location.href = 'https' + location.h
  * @license For commercial or closed source, contact us at license.mirotalk@gmail.com or purchase directly via CodeCanyon
  * @license CodeCanyon: https://codecanyon.net/item/mirotalk-sfu-webrtc-realtime-video-conferences/40769970
  * @author  Miroslav Pejic - miroslav.pejic.85@gmail.com
- * @version 1.7.53
+ * @version 1.7.54
  *
  */
 
@@ -5270,7 +5270,7 @@ function showAbout() {
         position: 'center',
         imageUrl: BRAND.about?.imageUrl && BRAND.about.imageUrl.trim() !== '' ? BRAND.about.imageUrl : image.about,
         customClass: { image: 'img-about' },
-        title: BRAND.about?.title && BRAND.about.title.trim() !== '' ? BRAND.about.title : 'WebRTC SFU v1.7.53',
+        title: BRAND.about?.title && BRAND.about.title.trim() !== '' ? BRAND.about.title : 'WebRTC SFU v1.7.54',
         html: `
             <br />
             <div id="about">
diff --git a/public/js/RoomClient.js b/public/js/RoomClient.js
index 6742bab9..8b6ab755 100644
--- a/public/js/RoomClient.js
+++ b/public/js/RoomClient.js
@@ -9,7 +9,7 @@
  * @license For commercial or closed source, contact us at license.mirotalk@gmail.com or purchase directly via CodeCanyon
  * @license CodeCanyon: https://codecanyon.net/item/mirotalk-sfu-webrtc-realtime-video-conferences/40769970
  * @author  Miroslav Pejic - miroslav.pejic.85@gmail.com
- * @version 1.7.53
+ * @version 1.7.54
  *
  */
 
diff --git a/public/js/VirtualBackground.js b/public/js/VirtualBackground.js
index 3ba8b11a..2c8433a4 100644
--- a/public/js/VirtualBackground.js
+++ b/public/js/VirtualBackground.js
@@ -23,6 +23,8 @@ class VirtualBackground {
         this.currentGifFrame = null;
         this.gifCanvas = null;
         this.gifContext = null;
+        this.frameCounter = 0;
+        this.lastSegmentationMask = null;
     }
 
     async initializeSegmentation() {
@@ -34,7 +36,12 @@ class VirtualBackground {
                 locateFile: (file) => `https://cdn.jsdelivr.net/npm/@mediapipe/selfie_segmentation/${file}`,
             });
 
-            this.segmentation.setOptions({ modelSelection: 1 });
+            this.segmentation.setOptions({
+                modelSelection: 1, // Higher accuracy
+                runningMode: 'video', // Smoother segmentation for streaming
+                smoothSegmentation: true, // Enables smoother edges
+            });
+
             this.segmentation.onResults(this.handleSegmentationResults.bind(this));
 
             await this.segmentation.initialize();
@@ -47,9 +54,12 @@ class VirtualBackground {
     }
 
     handleSegmentationResults(results) {
-        // Handle the segmentation results by processing the next frame
+        if (!results?.segmentationMask) return;
+
+        this.lastSegmentationMask = results.segmentationMask; // Store mask for skipped frames
+
         const pending = this.pendingFrames.shift();
-        if (!pending || !results?.segmentationMask) return;
+        if (!pending || !pending.shouldProcess) return;
 
         const { videoFrame, controller, imageBitmap, maskHandler } = pending;
         this.processFrame(videoFrame, controller, imageBitmap, maskHandler, results.segmentationMask);
@@ -112,11 +122,35 @@ class VirtualBackground {
                         return;
                     }
 
-                    // Queue the frame for segmentation processing
-                    this.pendingFrames.push({ videoFrame, controller, imageBitmap, maskHandler });
+                    if (this.frameCounter % 3 === 0) {
+                        // Process only every 3rd frame
+                        this.pendingFrames.push({
+                            videoFrame,
+                            controller,
+                            imageBitmap,
+                            maskHandler,
+                            shouldProcess: true, // Mark frame for processing
+                        });
 
-                    // Send the image to the segmentation model for processing
-                    await this.segmentation.send({ image: imageBitmap });
+                        // Send the image to the segmentation model
+                        await this.segmentation.send({ image: imageBitmap });
+                    } else {
+                        // Use last segmentation mask for skipped frames
+                        if (this.lastSegmentationMask) {
+                            this.processFrame(
+                                videoFrame,
+                                controller,
+                                imageBitmap,
+                                maskHandler,
+                                this.lastSegmentationMask,
+                            );
+                        } else {
+                            // If no previous mask, just enqueue the original frame
+                            controller.enqueue(videoFrame);
+                        }
+                    }
+
+                    this.frameCounter++; // Increment frame counter
                 } catch (error) {
                     console.error('❌ Frame transformation error:', error);
                 } finally {