From 1582ec07a2f8208eef00db3d1c854e3759fc79c9 Mon Sep 17 00:00:00 2001
From: Miroslav Pejic <miroslav.pejic.85@gmail.com>
Date: Sun, 16 Mar 2025 19:06:48 +0100
Subject: [PATCH] [mirotalksfu] - #194 Add Transparent Virtual background

---
 app/src/Server.js               |   2 +-
 package.json                    |   2 +-
 public/images/transparentBg.png | Bin 0 -> 477 bytes
 public/js/Brand.js              |   2 +-
 public/js/Room.js               |  36 ++++++++++++++++++++++--------
 public/js/RoomClient.js         |  31 +++++++++++++++++++++-----
 public/js/VirtualBackground.js  |  38 +++++++++++++++++++++++++++++++-
 7 files changed, 92 insertions(+), 19 deletions(-)
 create mode 100644 public/images/transparentBg.png

diff --git a/app/src/Server.js b/app/src/Server.js
index f6f346b1..96ce6356 100644
--- a/app/src/Server.js
+++ b/app/src/Server.js
@@ -58,7 +58,7 @@ dev dependencies: {
  * @license For commercial or closed source, contact us at license.mirotalk@gmail.com or purchase directly via CodeCanyon
  * @license CodeCanyon: https://codecanyon.net/item/mirotalk-sfu-webrtc-realtime-video-conferences/40769970
  * @author  Miroslav Pejic - miroslav.pejic.85@gmail.com
- * @version 1.7.80
+ * @version 1.7.81
  *
  */
 
diff --git a/package.json b/package.json
index 704698ac..97c664f3 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
     "name": "mirotalksfu",
-    "version": "1.7.80",
+    "version": "1.7.81",
     "description": "WebRTC SFU browser-based video calls",
     "main": "Server.js",
     "scripts": {
diff --git a/public/images/transparentBg.png b/public/images/transparentBg.png
new file mode 100644
index 0000000000000000000000000000000000000000..972f809b7cb6426b3ddd03dbe9163a156a889496
GIT binary patch
literal 477
zcmeAS@N?(olHy`uVBq!ia0vp^0YDtc!3-peCSSAyQkelhA+A9B-@ku<!Qj`gUq65T
z1Tz2p`2!IIGJpUm0_Or5KmZg3k`R?(tsnwf5TXl60>S_P|AFL^OA=K;%M?q3{DK)e
zjw^Dr&g6gjxJ+Whd`r!*XZKF5&3C`NP51Wd)R;gco0Tf65ey8BOr9=|ArXh)UiI#4
zHsEPU6!vKNz>)u=`ps|oqAN#3R)$2V-1B+*o?(BI>O@Bl#TJ34nder^{0O<eD}HLE
z$%hCLL8ZiWt$A^dE&eX1?uKtU<K5a;s()&7<=$A-D!1vQu=tB?g=y9N*Q|cEs8*eD
zSd&x1eNG_vTmH@xfwfY59<cDp+5R)MEU<{Rs?u66Q;~9dN3M7^lUr2tkG#lV#hbrf
zTP9^;VLe4wDX4tidd(wQzMemvo~yj8T%PRsgf-bdVfBaRTdd#1{<KWtzI$RvGWYB%
zm-s(w6(YY6Zsp>Ly2XFVR^f&W>*NW4J$IaEVYwQ(fs^y!dto+~uXAK}Gz&O&B=7H8
X?Ix6zzil-z=oma*{an^LB{Ts5LP`5G

literal 0
HcmV?d00001

diff --git a/public/js/Brand.js b/public/js/Brand.js
index 21e71276..fa6e89f9 100644
--- a/public/js/Brand.js
+++ b/public/js/Brand.js
@@ -64,7 +64,7 @@ let BRAND = {
     },
     about: {
         imageUrl: '../images/mirotalk-logo.gif',
-        title: '<strong>WebRTC SFU v1.7.80</strong>',
+        title: '<strong>WebRTC SFU v1.7.81</strong>',
         html: `
             <button 
                 id="support-button" 
diff --git a/public/js/Room.js b/public/js/Room.js
index 28afc6a3..c9805f9f 100644
--- a/public/js/Room.js
+++ b/public/js/Room.js
@@ -11,7 +11,7 @@ if (location.href.substr(0, 5) !== 'https') location.href = 'https' + location.h
  * @license For commercial or closed source, contact us at license.mirotalk@gmail.com or purchase directly via CodeCanyon
  * @license CodeCanyon: https://codecanyon.net/item/mirotalk-sfu-webrtc-realtime-video-conferences/40769970
  * @author  Miroslav Pejic - miroslav.pejic.85@gmail.com
- * @version 1.7.80
+ * @version 1.7.81
  *
  */
 
@@ -204,6 +204,7 @@ const isMediaStreamTrackAndTransformerSupported = virtualBackground.checkSupport
 
 let virtualBackgroundBlurLevel;
 let virtualBackgroundSelectedImage;
+let virtualBackgroundTransparent;
 
 let swalBackground = 'radial-gradient(#393939, #000000)'; //'rgba(0, 0, 0, 0.7)';
 
@@ -4971,13 +4972,14 @@ function showImageSelector() {
     }
 
     // Common function to handle virtual background changes
-    async function handleVirtualBackground(blurLevel = null, imgSrc = null) {
-        if (!blurLevel && !imgSrc) {
+    async function handleVirtualBackground(blurLevel = null, imgSrc = null, bgTransparent = null) {
+        if (!blurLevel && !imgSrc && !bgTransparent) {
             virtualBackgroundBlurLevel = null;
             virtualBackgroundSelectedImage = null;
+            virtualBackgroundTransparent = null;
             elemDisplay('imageGrid', false);
         }
-        await applyVirtualBackground(initVideo, initStream, blurLevel, imgSrc);
+        await applyVirtualBackground(initVideo, initStream, blurLevel, imgSrc, bgTransparent);
     }
 
     // Create clean virtual bg Image
@@ -4989,6 +4991,11 @@ function showImageSelector() {
     // Create Low Blur Image
     createImage('initLowBlurImg', image.blurLow, 'Low Blur', 'low', () => handleVirtualBackground(10));
 
+    // Create transparent virtual bg Image
+    createImage('initTransparentBg', image.transparentBg, 'Transparent Virtual background', 'transparentVb', () =>
+        handleVirtualBackground(null, null, true),
+    );
+
     // Handle file upload (common logic for file selection)
     function setupFileUploadButton(buttonId, sourceImg, tooltip, handler) {
         const imgButton = document.createElement('img');
@@ -5131,13 +5138,14 @@ function showImageSelector() {
 // VIRTUAL BACKGROUND HELPER
 // ####################################################
 
-async function applyVirtualBackground(videoElement, stream, blurLevel, backgroundImage) {
+async function applyVirtualBackground(videoElement, stream, blurLevel, backgroundImage, backgroundTransparent) {
     const videoTrack = stream.getVideoTracks()[0];
 
     if (blurLevel) {
         videoElement.srcObject = await virtualBackground.applyBlurToWebRTCStream(videoTrack, blurLevel);
         virtualBackgroundBlurLevel = blurLevel;
         virtualBackgroundSelectedImage = null;
+        virtualBackgroundTransparent = null;
     } else if (backgroundImage) {
         videoElement.srcObject = await virtualBackground.applyVirtualBackgroundToWebRTCStream(
             videoTrack,
@@ -5145,13 +5153,20 @@ async function applyVirtualBackground(videoElement, stream, blurLevel, backgroun
         );
         virtualBackgroundSelectedImage = backgroundImage;
         virtualBackgroundBlurLevel = null;
+        virtualBackgroundTransparent = null;
+    } else if (backgroundTransparent) {
+        videoElement.srcObject = await virtualBackground.applyTransparentVirtualBackgroundToWebRTCStream(videoTrack);
+        virtualBackgroundTransparent = true;
+        virtualBackgroundBlurLevel = null;
+        virtualBackgroundSelectedImage = null;
     } else {
         videoElement.srcObject = stream; // Default case, use original stream
         virtualBackgroundBlurLevel = null;
         virtualBackgroundSelectedImage = null;
+        virtualBackgroundTransparent = null;
     }
 
-    saveVirtualBackgroundSettings(blurLevel, backgroundImage);
+    saveVirtualBackgroundSettings(blurLevel, backgroundImage, backgroundTransparent);
 }
 
 function isValidImageURL(url) {
@@ -5210,10 +5225,11 @@ const indexedDBHelper = {
 // VIRTUAL BACKGROUND LOCAL STORAGE SETTINGS
 // ####################################################
 
-function saveVirtualBackgroundSettings(blurLevel, imageUrl) {
+function saveVirtualBackgroundSettings(blurLevel, imageUrl, transparent) {
     const settings = {
         blurLevel: blurLevel || null,
         imageUrl: imageUrl || null,
+        transparent: transparent || null,
     };
     localStorage.setItem('virtualBackgroundSettings', JSON.stringify(settings));
 }
@@ -5225,12 +5241,14 @@ async function loadVirtualBackgroundSettings() {
 
     if (!savedSettings) return;
 
-    const { blurLevel, imageUrl } = JSON.parse(savedSettings);
+    const { blurLevel, imageUrl, transparent } = JSON.parse(savedSettings);
 
     if (blurLevel) {
         await applyVirtualBackground(initVideo, initStream, blurLevel);
     } else if (imageUrl) {
         await applyVirtualBackground(initVideo, initStream, null, imageUrl);
+    } else if (transparent) {
+        await applyVirtualBackground(initVideo, initStream, null, null, true);
     }
 }
 
@@ -5272,7 +5290,7 @@ function showAbout() {
         position: 'center',
         imageUrl: BRAND.about?.imageUrl && BRAND.about.imageUrl.trim() !== '' ? BRAND.about.imageUrl : image.about,
         customClass: { image: 'img-about' },
-        title: BRAND.about?.title && BRAND.about.title.trim() !== '' ? BRAND.about.title : 'WebRTC SFU v1.7.80',
+        title: BRAND.about?.title && BRAND.about.title.trim() !== '' ? BRAND.about.title : 'WebRTC SFU v1.7.81',
         html: `
             <br />
             <div id="about">
diff --git a/public/js/RoomClient.js b/public/js/RoomClient.js
index 501c7177..b49f0fe9 100644
--- a/public/js/RoomClient.js
+++ b/public/js/RoomClient.js
@@ -9,7 +9,7 @@
  * @license For commercial or closed source, contact us at license.mirotalk@gmail.com or purchase directly via CodeCanyon
  * @license CodeCanyon: https://codecanyon.net/item/mirotalk-sfu-webrtc-realtime-video-conferences/40769970
  * @author  Miroslav Pejic - miroslav.pejic.85@gmail.com
- * @version 1.7.80
+ * @version 1.7.81
  *
  */
 
@@ -116,6 +116,7 @@ const image = {
     blur: '../images/blur.png',
     blurLow: '../images/blur-low.png',
     blurHigh: '../images/blur-high.png',
+    transparentBg: '../images/transparentBg.png',
     link: '../images/link.png',
     upload: '../images/upload.png',
     virtualBackground: {
@@ -1556,6 +1557,11 @@ class RoomClient {
                             virtualBackgroundSelectedImage,
                         );
                     }
+
+                    if (virtualBackgroundTransparent) {
+                        // Apply Transparent virtual background to WebRTC stream
+                        stream = await virtualBackground.applyTransparentVirtualBackgroundToWebRTCStream(videoTrack);
+                    }
                 }
             }
 
@@ -1746,12 +1752,13 @@ class RoomClient {
         }
 
         // Common function to handle virtual background changes
-        async function handleVirtualBackground(blurLevel = null, imgSrc = null) {
-            if (!blurLevel && !imgSrc) {
+        async function handleVirtualBackground(blurLevel = null, imgSrc = null, transparentBg = null) {
+            if (!blurLevel && !imgSrc && !transparentBg) {
                 virtualBackgroundBlurLevel = null;
                 virtualBackgroundSelectedImage = null;
+                virtualBackgroundTransparent = null;
             }
-            await rc.applyVirtualBackground(blurLevel, imgSrc);
+            await rc.applyVirtualBackground(blurLevel, imgSrc, transparentBg);
         }
 
         // Create clean virtual bg Image
@@ -1763,6 +1770,11 @@ class RoomClient {
         // Create Low Blur Image
         createImage('lowBlurImg', image.blurLow, 'Low Blur', 'low', () => handleVirtualBackground(10));
 
+        // Create transparent virtual bg Image
+        createImage('transparentBg', image.transparentBg, 'Transparent Virtual background', 'transparentVb', () =>
+            handleVirtualBackground(null, null, true),
+        );
+
         // Handle file upload (common logic for file selection)
         function setupFileUploadButton(buttonId, sourceImg, tooltip, handler) {
             const imgButton = document.createElement('img');
@@ -1905,20 +1917,27 @@ class RoomClient {
     // VIRTUAL BACKGROUND HELPER
     // ####################################################
 
-    async applyVirtualBackground(blurLevel, backgroundImage) {
+    async applyVirtualBackground(blurLevel, backgroundImage, backgroundTransparent) {
         if (blurLevel) {
             virtualBackgroundBlurLevel = blurLevel;
             virtualBackgroundSelectedImage = null;
+            virtualBackgroundTransparent = null;
         } else if (backgroundImage) {
             virtualBackgroundSelectedImage = backgroundImage;
+            virtualBackgroundTransparent = null;
+            virtualBackgroundBlurLevel = null;
+        } else if (backgroundTransparent) {
+            virtualBackgroundTransparent = true;
+            virtualBackgroundSelectedImage = null;
             virtualBackgroundBlurLevel = null;
         } else {
             virtualBackgroundSelectedImage = null;
             virtualBackgroundBlurLevel = null;
+            virtualBackgroundTransparent = null;
         }
 
         videoSelect.onchange();
-        saveVirtualBackgroundSettings(blurLevel, backgroundImage);
+        saveVirtualBackgroundSettings(blurLevel, backgroundImage, backgroundTransparent);
     }
 
     // ####################################################
diff --git a/public/js/VirtualBackground.js b/public/js/VirtualBackground.js
index 3e0734d5..441bf48f 100644
--- a/public/js/VirtualBackground.js
+++ b/public/js/VirtualBackground.js
@@ -98,7 +98,7 @@ class VirtualBackground {
             // Create new video frame with the processed content
             const processedFrame = new VideoFrame(canvas, {
                 timestamp: videoFrame.timestamp,
-                alpha: 'keep',
+                alpha: 'keep', // Ensure transparency is preserved
             });
 
             // Enqueue the processed frame to continue the stream
@@ -298,6 +298,42 @@ class VirtualBackground {
         return this.processStreamWithSegmentation(videoTrack, maskHandler);
     }
 
+    async applyTransparentVirtualBackgroundToWebRTCStream(videoTrack) {
+        // Check if the required APIs are supported
+        if (!this.isSupported) {
+            throw new Error(
+                'MediaStreamTrackProcessor, MediaStreamTrackGenerator, or TransformStream is not supported in this environment.',
+            );
+        }
+
+        // Handler for applying transparency by using only the mask
+        const maskHandler = (ctx, canvas, mask, imageBitmap) => {
+            // Clear the canvas (ensures transparency)
+            ctx.clearRect(0, 0, canvas.width, canvas.height);
+
+            // Draw the original frame (so we start with the full image)
+            ctx.drawImage(imageBitmap, 0, 0, canvas.width, canvas.height);
+
+            // Create an offscreen canvas for smooth masking
+            const maskCanvas = new OffscreenCanvas(canvas.width, canvas.height);
+            const maskCtx = maskCanvas.getContext('2d');
+
+            // Blur the mask slightly for softer edges
+            maskCtx.filter = 'blur(5px)';
+            maskCtx.drawImage(mask, 0, 0, canvas.width, canvas.height);
+
+            // Apply the mask to keep only the person
+            ctx.globalCompositeOperation = 'destination-in';
+            ctx.drawImage(maskCanvas, 0, 0, canvas.width, canvas.height);
+
+            // Reset blending mode to normal
+            ctx.globalCompositeOperation = 'source-over';
+        };
+
+        console.log('✅ Apply Transparent Background');
+        return this.processStreamWithSegmentation(videoTrack, maskHandler);
+    }
+
     async loadImage(src) {
         // Load an image from the provided source URL
         return new Promise((resolve, reject) => {