[mirotalksfu] - #194 Add Transparent Virtual background

2025-03-16 19:06:48 +01:00
--- a/app/src/Server.js
+++ b/app/src/Server.js
@@ -58,7 +58,7 @@ dev dependencies: {
 * @license For commercial or closed source, contact us at license.mirotalk@gmail.com or purchase directly via CodeCanyon
 * @license CodeCanyon: https://codecanyon.net/item/mirotalk-sfu-webrtc-realtime-video-conferences/40769970
 * @author  Miroslav Pejic - miroslav.pejic.85@gmail.com
- * @version 1.7.80
+ * @version 1.7.81
 *
 */

--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
    "name": "mirotalksfu",
-    "version": "1.7.80",
+    "version": "1.7.81",
    "description": "WebRTC SFU browser-based video calls",
    "main": "Server.js",
    "scripts": {
--- a/public/images/transparentBg.png
+++ b/public/images/transparentBg.png
--- a/public/js/Brand.js
+++ b/public/js/Brand.js
@@ -64,7 +64,7 @@ let BRAND = {
    },
    about: {
        imageUrl: '../images/mirotalk-logo.gif',
-        title: '<strong>WebRTC SFU v1.7.80</strong>',
+        title: '<strong>WebRTC SFU v1.7.81</strong>',
        html: `
            <button 
                id="support-button" 
--- a/public/js/Room.js
+++ b/public/js/Room.js
@@ -11,7 +11,7 @@ if (location.href.substr(0, 5) !== 'https') location.href = 'https' + location.h
 * @license For commercial or closed source, contact us at license.mirotalk@gmail.com or purchase directly via CodeCanyon
 * @license CodeCanyon: https://codecanyon.net/item/mirotalk-sfu-webrtc-realtime-video-conferences/40769970
 * @author  Miroslav Pejic - miroslav.pejic.85@gmail.com
- * @version 1.7.80
+ * @version 1.7.81
 *
 */

@@ -204,6 +204,7 @@ const isMediaStreamTrackAndTransformerSupported = virtualBackground.checkSupport

 let virtualBackgroundBlurLevel;
 let virtualBackgroundSelectedImage;
+let virtualBackgroundTransparent;

 let swalBackground = 'radial-gradient(#393939, #000000)'; //'rgba(0, 0, 0, 0.7)';

@@ -4971,13 +4972,14 @@ function showImageSelector() {
    }

    // Common function to handle virtual background changes
-    async function handleVirtualBackground(blurLevel = null, imgSrc = null) {
-        if (!blurLevel && !imgSrc) {
+    async function handleVirtualBackground(blurLevel = null, imgSrc = null, bgTransparent = null) {
+        if (!blurLevel && !imgSrc && !bgTransparent) {
            virtualBackgroundBlurLevel = null;
            virtualBackgroundSelectedImage = null;
+            virtualBackgroundTransparent = null;
            elemDisplay('imageGrid', false);
        }
-        await applyVirtualBackground(initVideo, initStream, blurLevel, imgSrc);
+        await applyVirtualBackground(initVideo, initStream, blurLevel, imgSrc, bgTransparent);
    }

    // Create clean virtual bg Image
@@ -4989,6 +4991,11 @@ function showImageSelector() {
    // Create Low Blur Image
    createImage('initLowBlurImg', image.blurLow, 'Low Blur', 'low', () => handleVirtualBackground(10));

+    // Create transparent virtual bg Image
+    createImage('initTransparentBg', image.transparentBg, 'Transparent Virtual background', 'transparentVb', () =>
+        handleVirtualBackground(null, null, true),
+    );
+
    // Handle file upload (common logic for file selection)
    function setupFileUploadButton(buttonId, sourceImg, tooltip, handler) {
        const imgButton = document.createElement('img');
@@ -5131,13 +5138,14 @@ function showImageSelector() {
 // VIRTUAL BACKGROUND HELPER
 // ####################################################

-async function applyVirtualBackground(videoElement, stream, blurLevel, backgroundImage) {
+async function applyVirtualBackground(videoElement, stream, blurLevel, backgroundImage, backgroundTransparent) {
    const videoTrack = stream.getVideoTracks()[0];

    if (blurLevel) {
        videoElement.srcObject = await virtualBackground.applyBlurToWebRTCStream(videoTrack, blurLevel);
        virtualBackgroundBlurLevel = blurLevel;
        virtualBackgroundSelectedImage = null;
+        virtualBackgroundTransparent = null;
    } else if (backgroundImage) {
        videoElement.srcObject = await virtualBackground.applyVirtualBackgroundToWebRTCStream(
            videoTrack,
@@ -5145,13 +5153,20 @@ async function applyVirtualBackground(videoElement, stream, blurLevel, backgroun
        );
        virtualBackgroundSelectedImage = backgroundImage;
        virtualBackgroundBlurLevel = null;
+        virtualBackgroundTransparent = null;
+    } else if (backgroundTransparent) {
+        videoElement.srcObject = await virtualBackground.applyTransparentVirtualBackgroundToWebRTCStream(videoTrack);
+        virtualBackgroundTransparent = true;
+        virtualBackgroundBlurLevel = null;
+        virtualBackgroundSelectedImage = null;
    } else {
        videoElement.srcObject = stream; // Default case, use original stream
        virtualBackgroundBlurLevel = null;
        virtualBackgroundSelectedImage = null;
+        virtualBackgroundTransparent = null;
    }

-    saveVirtualBackgroundSettings(blurLevel, backgroundImage);
+    saveVirtualBackgroundSettings(blurLevel, backgroundImage, backgroundTransparent);
 }

 function isValidImageURL(url) {
@@ -5210,10 +5225,11 @@ const indexedDBHelper = {
 // VIRTUAL BACKGROUND LOCAL STORAGE SETTINGS
 // ####################################################

-function saveVirtualBackgroundSettings(blurLevel, imageUrl) {
+function saveVirtualBackgroundSettings(blurLevel, imageUrl, transparent) {
    const settings = {
        blurLevel: blurLevel || null,
        imageUrl: imageUrl || null,
+        transparent: transparent || null,
    };
    localStorage.setItem('virtualBackgroundSettings', JSON.stringify(settings));
 }
@@ -5225,12 +5241,14 @@ async function loadVirtualBackgroundSettings() {

    if (!savedSettings) return;

-    const { blurLevel, imageUrl } = JSON.parse(savedSettings);
+    const { blurLevel, imageUrl, transparent } = JSON.parse(savedSettings);

    if (blurLevel) {
        await applyVirtualBackground(initVideo, initStream, blurLevel);
    } else if (imageUrl) {
        await applyVirtualBackground(initVideo, initStream, null, imageUrl);
+    } else if (transparent) {
+        await applyVirtualBackground(initVideo, initStream, null, null, true);
    }
 }

@@ -5272,7 +5290,7 @@ function showAbout() {
        position: 'center',
        imageUrl: BRAND.about?.imageUrl && BRAND.about.imageUrl.trim() !== '' ? BRAND.about.imageUrl : image.about,
        customClass: { image: 'img-about' },
-        title: BRAND.about?.title && BRAND.about.title.trim() !== '' ? BRAND.about.title : 'WebRTC SFU v1.7.80',
+        title: BRAND.about?.title && BRAND.about.title.trim() !== '' ? BRAND.about.title : 'WebRTC SFU v1.7.81',
        html: `
            <br />
            <div id="about">
--- a/public/js/RoomClient.js
+++ b/public/js/RoomClient.js
@@ -9,7 +9,7 @@
 * @license For commercial or closed source, contact us at license.mirotalk@gmail.com or purchase directly via CodeCanyon
 * @license CodeCanyon: https://codecanyon.net/item/mirotalk-sfu-webrtc-realtime-video-conferences/40769970
 * @author  Miroslav Pejic - miroslav.pejic.85@gmail.com
- * @version 1.7.80
+ * @version 1.7.81
 *
 */

@@ -116,6 +116,7 @@ const image = {
    blur: '../images/blur.png',
    blurLow: '../images/blur-low.png',
    blurHigh: '../images/blur-high.png',
+    transparentBg: '../images/transparentBg.png',
    link: '../images/link.png',
    upload: '../images/upload.png',
    virtualBackground: {
@@ -1556,6 +1557,11 @@ class RoomClient {
                            virtualBackgroundSelectedImage,
                        );
                    }
+
+                    if (virtualBackgroundTransparent) {
+                        // Apply Transparent virtual background to WebRTC stream
+                        stream = await virtualBackground.applyTransparentVirtualBackgroundToWebRTCStream(videoTrack);
+                    }
                }
            }

@@ -1746,12 +1752,13 @@ class RoomClient {
        }

        // Common function to handle virtual background changes
-        async function handleVirtualBackground(blurLevel = null, imgSrc = null) {
-            if (!blurLevel && !imgSrc) {
+        async function handleVirtualBackground(blurLevel = null, imgSrc = null, transparentBg = null) {
+            if (!blurLevel && !imgSrc && !transparentBg) {
                virtualBackgroundBlurLevel = null;
                virtualBackgroundSelectedImage = null;
+                virtualBackgroundTransparent = null;
            }
-            await rc.applyVirtualBackground(blurLevel, imgSrc);
+            await rc.applyVirtualBackground(blurLevel, imgSrc, transparentBg);
        }

        // Create clean virtual bg Image
@@ -1763,6 +1770,11 @@ class RoomClient {
        // Create Low Blur Image
        createImage('lowBlurImg', image.blurLow, 'Low Blur', 'low', () => handleVirtualBackground(10));

+        // Create transparent virtual bg Image
+        createImage('transparentBg', image.transparentBg, 'Transparent Virtual background', 'transparentVb', () =>
+            handleVirtualBackground(null, null, true),
+        );
+
        // Handle file upload (common logic for file selection)
        function setupFileUploadButton(buttonId, sourceImg, tooltip, handler) {
            const imgButton = document.createElement('img');
@@ -1905,20 +1917,27 @@ class RoomClient {
    // VIRTUAL BACKGROUND HELPER
    // ####################################################

-    async applyVirtualBackground(blurLevel, backgroundImage) {
+    async applyVirtualBackground(blurLevel, backgroundImage, backgroundTransparent) {
        if (blurLevel) {
            virtualBackgroundBlurLevel = blurLevel;
            virtualBackgroundSelectedImage = null;
+            virtualBackgroundTransparent = null;
        } else if (backgroundImage) {
            virtualBackgroundSelectedImage = backgroundImage;
+            virtualBackgroundTransparent = null;
+            virtualBackgroundBlurLevel = null;
+        } else if (backgroundTransparent) {
+            virtualBackgroundTransparent = true;
+            virtualBackgroundSelectedImage = null;
            virtualBackgroundBlurLevel = null;
        } else {
            virtualBackgroundSelectedImage = null;
            virtualBackgroundBlurLevel = null;
+            virtualBackgroundTransparent = null;
        }

        videoSelect.onchange();
-        saveVirtualBackgroundSettings(blurLevel, backgroundImage);
+        saveVirtualBackgroundSettings(blurLevel, backgroundImage, backgroundTransparent);
    }

    // ####################################################
--- a/public/js/VirtualBackground.js
+++ b/public/js/VirtualBackground.js
@@ -98,7 +98,7 @@ class VirtualBackground {
            // Create new video frame with the processed content
            const processedFrame = new VideoFrame(canvas, {
                timestamp: videoFrame.timestamp,
-                alpha: 'keep',
+                alpha: 'keep', // Ensure transparency is preserved
            });

            // Enqueue the processed frame to continue the stream
@@ -298,6 +298,42 @@ class VirtualBackground {
        return this.processStreamWithSegmentation(videoTrack, maskHandler);
    }

+    async applyTransparentVirtualBackgroundToWebRTCStream(videoTrack) {
+        // Check if the required APIs are supported
+        if (!this.isSupported) {
+            throw new Error(
+                'MediaStreamTrackProcessor, MediaStreamTrackGenerator, or TransformStream is not supported in this environment.',
+            );
+        }
+
+        // Handler for applying transparency by using only the mask
+        const maskHandler = (ctx, canvas, mask, imageBitmap) => {
+            // Clear the canvas (ensures transparency)
+            ctx.clearRect(0, 0, canvas.width, canvas.height);
+
+            // Draw the original frame (so we start with the full image)
+            ctx.drawImage(imageBitmap, 0, 0, canvas.width, canvas.height);
+
+            // Create an offscreen canvas for smooth masking
+            const maskCanvas = new OffscreenCanvas(canvas.width, canvas.height);
+            const maskCtx = maskCanvas.getContext('2d');
+
+            // Blur the mask slightly for softer edges
+            maskCtx.filter = 'blur(5px)';
+            maskCtx.drawImage(mask, 0, 0, canvas.width, canvas.height);
+
+            // Apply the mask to keep only the person
+            ctx.globalCompositeOperation = 'destination-in';
+            ctx.drawImage(maskCanvas, 0, 0, canvas.width, canvas.height);
+
+            // Reset blending mode to normal
+            ctx.globalCompositeOperation = 'source-over';
+        };
+
+        console.log('✅ Apply Transparent Background');
+        return this.processStreamWithSegmentation(videoTrack, maskHandler);
+    }
+
    async loadImage(src) {
        // Load an image from the provided source URL
        return new Promise((resolve, reject) => {