[mirotalksfu] - #194 Refactoring

public/js/Brand.js

@@ -64,7 +64,7 @@ let BRAND = {
     },
     about: {
         imageUrl: '../images/mirotalk-logo.gif',
-        title: '<strong>WebRTC SFU v1.7.57</strong>',
+        title: '<strong>WebRTC SFU v1.7.58</strong>',
         html: `
             <button 
                 id="support-button" 

public/js/Room.js

@@ -11,7 +11,7 @@ if (location.href.substr(0, 5) !== 'https') location.href = 'https' + location.h
  * @license For commercial or closed source, contact us at license.mirotalk@gmail.com or purchase directly via CodeCanyon
  * @license CodeCanyon: https://codecanyon.net/item/mirotalk-sfu-webrtc-realtime-video-conferences/40769970
  * @author  Miroslav Pejic - miroslav.pejic.85@gmail.com
- * @version 1.7.57
+ * @version 1.7.58
  *
  */
 
@@ -5270,7 +5270,7 @@ function showAbout() {
         position: 'center',
         imageUrl: BRAND.about?.imageUrl && BRAND.about.imageUrl.trim() !== '' ? BRAND.about.imageUrl : image.about,
         customClass: { image: 'img-about' },
-        title: BRAND.about?.title && BRAND.about.title.trim() !== '' ? BRAND.about.title : 'WebRTC SFU v1.7.57',
+        title: BRAND.about?.title && BRAND.about.title.trim() !== '' ? BRAND.about.title : 'WebRTC SFU v1.7.58',
         html: `
             <br />
             <div id="about">

public/js/RoomClient.js

@@ -9,7 +9,7 @@
  * @license For commercial or closed source, contact us at license.mirotalk@gmail.com or purchase directly via CodeCanyon
  * @license CodeCanyon: https://codecanyon.net/item/mirotalk-sfu-webrtc-realtime-video-conferences/40769970
  * @author  Miroslav Pejic - miroslav.pejic.85@gmail.com
- * @version 1.7.57
+ * @version 1.7.58
  *
  */
 

public/js/VirtualBackground.js

@@ -22,13 +22,13 @@ class VirtualBackground {
         this.gifAnimation = null;
         this.gifCanvas = null;
         this.frameCounter = 0;
+        this.frameSkipRatio = 3;
         this.lastSegmentationMask = null;
     }
 
     async initializeSegmentation() {
         // Initialize the segmentation model if not already done
         if (this.initialized) return;
-
         try {
             this.segmentation = new SelfieSegmentation({
                 locateFile: (file) => `https://cdn.jsdelivr.net/npm/@mediapipe/selfie_segmentation/${file}`,
@@ -54,13 +54,19 @@ class VirtualBackground {
     handleSegmentationResults(results) {
         if (!results?.segmentationMask) return;
 
-        this.lastSegmentationMask = results.segmentationMask; // Store mask for skipped frames
+        this.lastSegmentationMask = results.segmentationMask;
 
-        const pending = this.pendingFrames.shift();
-        if (!pending || !pending.shouldProcess) return;
+        const pendingFrame = this.pendingFrames.shift();
 
-        const { videoFrame, controller, imageBitmap, maskHandler } = pending;
-        this.processFrame(videoFrame, controller, imageBitmap, maskHandler, results.segmentationMask);
+        if (!pendingFrame) return;
+
+        this.processFrame(
+            pendingFrame.videoFrame,
+            pendingFrame.controller,
+            pendingFrame.imageBitmap,
+            pendingFrame.maskHandler,
+            this.lastSegmentationMask,
+        );
     }
 
     processFrame(videoFrame, controller, imageBitmap, maskHandler, segmentationMask) {
@@ -115,9 +121,11 @@ class VirtualBackground {
                     return;
                 }
 
+                let imageBitmap = null;
+
                 try {
                     // Create image bitmap from video frame
-                    const imageBitmap = await createImageBitmap(videoFrame);
+                    imageBitmap = await createImageBitmap(videoFrame);
 
                     if (!imageBitmap) {
                         console.warn('⚠️ Failed to create imageBitmap, skipping frame.');
@@ -125,32 +133,23 @@ class VirtualBackground {
                         return;
                     }
 
-                    if (this.frameCounter % 3 === 0) {
-                        // Process only every 3rd frame
+                    if (this.frameCounter % this.frameSkipRatio === 0) {
+                        // Process only every 3rd frame (reduce CPU load)
                         this.pendingFrames.push({
                             videoFrame,
                             controller,
                             imageBitmap,
                             maskHandler,
-                            shouldProcess: true, // Mark frame for processing
                         });
 
                         // Send the image to the segmentation model
                         await this.segmentation.send({ image: imageBitmap });
-                    } else {
+                    } else if (this.lastSegmentationMask) {
                         // Use last segmentation mask for skipped frames
-                        if (this.lastSegmentationMask) {
-                            this.processFrame(
-                                videoFrame,
-                                controller,
-                                imageBitmap,
-                                maskHandler,
-                                this.lastSegmentationMask,
-                            );
-                        } else {
-                            // If no previous mask, just enqueue the original frame
-                            controller.enqueue(videoFrame);
-                        }
+                        this.processFrame(videoFrame, controller, imageBitmap, maskHandler, this.lastSegmentationMask);
+                    } else {
+                        // If no previous mask, just enqueue the original frame
+                        controller.enqueue(videoFrame);
                     }
 
                     this.frameCounter++; // Increment frame counter
@@ -158,7 +157,7 @@ class VirtualBackground {
                     console.error('❌ Frame transformation error:', error);
                 } finally {
                     // Close the video frame after processing
-                    this.closeFrames(videoFrame);
+                    this.closeFrames(videoFrame, imageBitmap);
                 }
             },
             flush: () => this.cleanPendingFrames(), // Clean up any pending frames when the stream ends
@@ -173,7 +172,7 @@ class VirtualBackground {
         processor.readable
             .pipeThrough(transformer)
             .pipeTo(generator.writable)
-            .catch(() => this.stopCurrentProcessor());
+            .catch(async () => await this.stopCurrentProcessor());
 
         return new MediaStream([generator]);
     }