ghaymah_dev/codepill-sfu
1
0
نسخ 0
الشفرة المشكلات طلبات السحب Actions Packages المشاريع الإصدارات الويكي النشاط

[mirotalksfu] - add Video AI

تصفح المصدر
هذا الالتزام موجود في:
Miroslav Pejic
2024-05-29 14:14:32 +02:00
الأصل 11185ba69f
التزام 3a50fe1513
12 ملفات معدلة مع 844 إضافات و6 حذوفات
تنزيل ملف التصحيح تنزيل ملف الاختلاف توسيع جميع الملفات طي جميع الملفات

1
README.md
عرض الملف

@@ -57,6 +57,7 @@
- Snapshot video frames and save them as PNG images.
- Chat with an Emoji Picker for expressing feelings, private messages, Markdown support, and conversation saving.
- ChatGPT (powered by OpenAI) for answering questions, providing information, and connecting users to relevant resources.
- VideoAI enables users to customize AI avatars to deliver messages, perform tasks, or act out scripts.
- Speech recognition, execute the app features simply with your voice.
- Push-to-talk functionality, similar to a walkie-talkie.
- Advanced collaborative whiteboard for teachers.

2
app/src/Room.js
عرض الملف

@@ -40,6 +40,7 @@ module.exports = class Room {
};
this.survey = config.survey;
this.redirect = config.redirect;
this.videoAIEnabled = config?.videoAI?.enabled || false;
this.peers = new Map();
this.bannedPeers = [];
this.webRtcTransport = config.mediasoup.webRtcTransport;
@@ -65,6 +66,7 @@ module.exports = class Room {
moderator: this._moderator,
survey: this.survey,
redirect: this.redirect,
videoAIEnabled: this.videoAIEnabled,
peers: JSON.stringify([...this.peers]),
};
}

215
app/src/Server.js
عرض الملف

@@ -42,7 +42,7 @@ dependencies: {
* @license For commercial or closed source, contact us at license.mirotalk@gmail.com or purchase directly via CodeCanyon
* @license CodeCanyon: https://codecanyon.net/item/mirotalk-sfu-webrtc-realtime-video-conferences/40769970
* @author Miroslav Pejic - miroslav.pejic.85@gmail.com
* @version 1.4.36
* @version 1.4.40
*
*/
@@ -1870,6 +1870,219 @@ function startServer() {
}
});
// https://docs.heygen.com/reference/overview-copy
socket.on('getAvatarList', async ({}, cb) => {
if (!config.videoAI.enabled || !config.videoAI.apiKey)
return cb({ error: 'Video AI seems disabled, try later!' });
try {
const response = await axios.get(`${config.videoAI.basePath}/v1/avatar.list`, {
headers: {
'Content-Type': 'application/json',
'X-Api-Key': config.videoAI.apiKey,
},
});
const data = { response: response.data.data };
//log.debug('getAvatarList', data);
cb(data);
} catch (error) {
cb({ error: error.response?.status === 500 ? 'Internal server error' : error.message });
}
});
socket.on('getVoiceList', async ({}, cb) => {
if (!config.videoAI.enabled || !config.videoAI.apiKey)
return cb({ error: 'Video AI seems disabled, try later!' });
try {
const response = await axios.get(`${config.videoAI.basePath}/v1/voice.list`, {
headers: {
'Content-Type': 'application/json',
'X-Api-Key': config.videoAI.apiKey,
},
});
const data = { response: response.data.data };
//log.debug('getVoiceList', data);
cb(data);
} catch (error) {
cb({ error: error.response?.status === 500 ? 'Internal server error' : error.message });
}
});
socket.on('streamingNew', async ({ quality, avatar_name, voice_id }, cb) => {
if (!roomList.has(socket.room_id)) return;
if (!config.videoAI.enabled || !config.videoAI.apiKey)
return cb({ error: 'Video AI seems disabled, try later!' });
try {
const response = await axios.post(
`${config.videoAI.basePath}/v1/streaming.new`,
{
quality,
avatar_name,
voice: {
voice_id: voice_id,
},
},
{
headers: {
'Content-Type': 'application/json',
'X-Api-Key': config.videoAI.apiKey,
},
},
);
const data = { response: response.data };
log.debug('streamingNew', data);
cb(data);
} catch (error) {
cb({ error: error.response?.status === 500 ? 'Internal server error' : error });
}
});
socket.on('streamingStart', async ({ session_id, sdp }, cb) => {
if (!roomList.has(socket.room_id)) return;
if (!config.videoAI.enabled || !config.videoAI.apiKey)
return cb({ error: 'Video AI seems disabled, try later!' });
try {
const response = await axios.post(
`${config.videoAI.basePath}/v1/streaming.start`,
{ session_id, sdp },
{
headers: {
'Content-Type': 'application/json',
'X-Api-Key': config.videoAI.apiKey,
},
},
);
const data = { response: response.data.data };
log.debug('startSessionAi', data);
cb(data);
} catch (error) {
cb({ error: error.response?.status === 500 ? 'server error' : error });
}
});
socket.on('streamingICE', async ({ session_id, candidate }, cb) => {
if (!roomList.has(socket.room_id)) return;
if (!config.videoAI.enabled || !config.videoAI.apiKey)
return cb({ error: 'Video AI seems disabled, try later!' });
try {
const response = await axios.post(
`${config.videoAI.basePath}/v1/streaming.ice`,
{ session_id, candidate },
{
headers: {
'Content-Type': 'application/json',
'X-Api-Key': config.videoAI.apiKey,
},
},
);
const data = { response: response.data };
log.debug('streamingICE', data);
cb(data);
} catch (error) {
log.error('Error in streamingICE:', error.response?.data || error.message); // Log detailed error
cb({ error: error.response?.status === 500 ? 'Internal server error' : error });
}
});
socket.on('streamingTask', async ({ session_id, text }, cb) => {
if (!roomList.has(socket.room_id)) return;
if (!config.videoAI.enabled || !config.videoAI.apiKey)
return cb({ error: 'Video AI seems disabled, try later!' });
try {
const response = await axios.post(
`${config.videoAI.basePath}/v1/streaming.task`,
{
session_id,
text,
},
{
headers: {
'Content-Type': 'application/json',
'X-Api-Key': config.videoAI.apiKey,
},
},
);
const data = { response: response.data };
log.debug('streamingTask', data);
cb(data);
} catch (error) {
cb({ error: error.response?.status === 500 ? 'server error' : error });
}
});
socket.on('talkToOpenAI', async ({ text, context }, cb) => {
if (!roomList.has(socket.room_id)) return;
if (!config.videoAI.enabled || !config.videoAI.apiKey)
return cb({ error: 'Video AI seems disabled, try later!' });
try {
const systemLimit = config.videoAI.systemLimit;
const arr = {
messages: [...context, { role: 'system', content: systemLimit }, { role: 'user', content: text }],
model: 'gpt-3.5-turbo',
};
const chatCompletion = await chatGPT.chat.completions.create(arr);
const chatText = chatCompletion.choices[0].message.content;
context.push({ role: 'system', content: chatText });
context.push({ role: 'assistant', content: chatText });
const data = { response: chatText, context: context };
log.debug('talkToOpenAI', data);
cb(data);
} catch (error) {
cb({ error: error.message });
}
});
socket.on('streamingStop', async ({ session_id }, cb) => {
if (!roomList.has(socket.room_id)) return;
if (!config.videoAI.enabled || !config.videoAI.apiKey)
return cb({ error: 'Video AI seems disabled, try later!' });
try {
const response = await axios.post(
`${config.videoAI.basePath}/v1/streaming.stop`,
{
session_id,
},
{
headers: {
'Content-Type': 'application/json',
'X-Api-Key': config.videoAI.apiKey,
},
},
);
const data = { response: response.data };
log.debug('streamingStop', data);
cb(data);
} catch (error) {
cb({ error: error.response?.status === 500 ? 'Internal server error' : error });
}
});
socket.on('disconnect', async () => {
if (!roomList.has(socket.room_id)) return;

13
app/src/config.template.js
عرض الملف

@@ -171,6 +171,19 @@ module.exports = {
max_tokens: 1000,
temperature: 0,
},
videoAI: {
/*
HeyGen Video AI
1. Goto https://app.heygen.com
2. Create your account
3. Generate your APIKey https://app.heygen.com/settings?nav=API
*/
enabled: false,
basePath: 'https://api.heygen.com',
apiKey: '',
systemLimit:
'You are a streaming avatar from MiroTalk SFU, an industry-leading product that specialize in videos communications. Audience will try to have a conversation with you, please try answer the questions or respond their comments naturally, and concisely. - please try your best to response with short answers, and only answer the last question.',
},
email: {
/*
Configure email settings for notifications or alerts

6
package.json
عرض الملف

@@ -1,6 +1,6 @@
{
"name": "mirotalksfu",
"version": "1.4.36",
"version": "1.4.40",
"description": "WebRTC SFU browser-based video calls",
"main": "Server.js",
"scripts": {
@@ -43,7 +43,7 @@
"dependencies": {
"@sentry/integrations": "7.114.0",
"@sentry/node": "7.114.0",
"axios": "^1.7.1",
"axios": "^1.7.2",
"body-parser": "1.20.2",
"colors": "1.4.0",
"compression": "1.7.4",
@@ -67,7 +67,7 @@
},
"devDependencies": {
"node-fetch": "^3.3.2",
"nodemon": "^3.1.0",
"nodemon": "^3.1.1",
"prettier": "3.2.5"
}
}

64
public/css/Room.css
عرض الملف

@@ -1320,6 +1320,70 @@ hr {
}
}
/*--------------------------------------------------------------
# Video AI Avatars
--------------------------------------------------------------*/
.avatarsVideoAI {
min-height: 480px;
max-height: 600px;
color: white;
overflow: auto;
justify-content: center; /* Center horizontally */
align-items: center; /* Center vertically */
}
/* Custom avatar audio UI */
#audio-container {
background: var(--select-bg);
color: #fff;
padding: 10px;
border-radius: 5px;
}
#audio-container audio {
width: 100%;
outline: none;
}
#audio-container audio::-webkit-media-controls-panel {
background: var(--body-bg);
border-radius: 5px;
padding: 5px;
}
#audio-container audio::-webkit-media-controls-timeline {
background: var(--body-bg);
border-radius: 5px;
}
/* Styling for the audio volume controls */
#audio-container audio::-webkit-media-controls-volume-slider-container {
color: #fff;
}
#audio-container audio::-webkit-media-controls-volume-slider {
background: var(--body-bg);
border-radius: 5px;
}
/* Styling for the play/pause button */
#audio-container audio::-webkit-media-controls-play-button,
#audio-container audio::-webkit-media-controls-pause-button {
background-color: #fff;
border-radius: 50%;
width: 30px;
height: 30px;
margin: 0 10px;
}
/* Styling for the time display */
#audio-container audio::-webkit-media-controls-current-time-display,
#audio-container audio::-webkit-media-controls-time-remaining-display {
color: #fff;
}
/*
z-index:
- 1 videoMediaContainer

8
public/css/VideoGrid.css
عرض الملف

@@ -191,6 +191,14 @@ video {
cursor: pointer;
}
#canvasAIElement {
width: 100%;
height: 100%;
object-fit: var(--videoObjFit);
border-radius: 10px;
cursor: pointer;
}
video:hover {
opacity: 0.9;
}

ثنائية
public/images/virtual/1.jpg Normal file
عرض الملف

ملف ثنائي غير معروض.
جنبًا إلى جنب

بعد

العرض:  |  الارتفاع:  |  الحجم: 3.2 MiB

ثنائية
public/images/virtual/2.jpg Normal file
عرض الملف

ملف ثنائي غير معروض.
جنبًا إلى جنب

بعد

العرض:  |  الارتفاع:  |  الحجم: 6.4 MiB

18
public/js/Room.js
عرض الملف

@@ -11,7 +11,7 @@ if (location.href.substr(0, 5) !== 'https') location.href = 'https' + location.h
* @license For commercial or closed source, contact us at license.mirotalk@gmail.com or purchase directly via CodeCanyon
* @license CodeCanyon: https://codecanyon.net/item/mirotalk-sfu-webrtc-realtime-video-conferences/40769970
* @author Miroslav Pejic - miroslav.pejic.85@gmail.com
* @version 1.4.36
* @version 1.4.40
*
*/
@@ -244,6 +244,8 @@ const RoomURL = window.location.origin + '/join/' + room_id; // window.location.
let transcription;
let showFreeAvatars = true;
// ####################################################
// INIT ROOM
// ####################################################
@@ -1444,6 +1446,20 @@ function handleButtons() {
tabLanguagesBtn.onclick = (e) => {
rc.openTab(e, 'tabLanguages');
};
tabVideoAIBtn.onclick = (e) => {
rc.openTab(e, 'tabVideoAI');
rc.getAvatarList();
rc.getVoiceList();
};
avatarVideoAIStart.onclick = (e) => {
rc.stopSession();
rc.handleVideoAI();
rc.toggleMySettings();
};
switchAvatars.onchange = (e) => {
showFreeAvatars = e.currentTarget.checked;
rc.getAvatarList();
};
refreshVideoDevices.onclick = async () => {
await refreshMyVideoDevices();
userLog('info', 'Refreshed video devices', 'top-end');

481
public/js/RoomClient.js
عرض الملف

@@ -9,7 +9,7 @@
* @license For commercial or closed source, contact us at license.mirotalk@gmail.com or purchase directly via CodeCanyon
* @license CodeCanyon: https://codecanyon.net/item/mirotalk-sfu-webrtc-realtime-video-conferences/40769970
* @author Miroslav Pejic - miroslav.pejic.85@gmail.com
* @version 1.4.36
* @version 1.4.40
*
*/
@@ -149,6 +149,19 @@ const enums = {
//...
};
// HeyGen config
const VideoAI = {
enabled: true,
active: false,
info: {},
avatar: null,
avatarName: 'Monica',
avatarVoice: '',
quality: 'medium',
virtualBackground: true,
background: '../images/virtual/1.jpg',
};
// Recording
let recordedBlobs = [];
@@ -205,6 +218,13 @@ class RoomClient {
this.chatMessageSpamCount = 0;
this.chatMessageSpamCountToBan = 10;
// HeyGen Video AI
this.videoAIContainer = null;
this.videoAIElement = null;
this.canvasAIElement = null;
this.renderAIToken = null;
this.peerConnection = null;
this.isAudioAllowed = isAudioAllowed;
this.isVideoAllowed = isVideoAllowed;
this.isScreenAllowed = isScreenAllowed;
@@ -490,7 +510,13 @@ class RoomClient {
this._moderator.audio_cant_unmute ? hide(tabAudioDevicesBtn) : show(tabAudioDevicesBtn);
this._moderator.video_cant_unhide ? hide(tabVideoDevicesBtn) : show(tabVideoDevicesBtn);
}
// Check if VideoAI is enabled
if (!room.videoAIEnabled) {
VideoAI.enabled = false;
elemDisplay('tabVideoAIBtn', false);
}
}
// PARTICIPANTS
for (let peer of Array.from(peers.keys()).filter((id) => id !== this.peer_id)) {
let peer_info = peers.get(peer).peer_info;
@@ -3682,6 +3708,7 @@ class RoomClient {
this.setMsgAvatar('right', 'ChatGPT');
this.appendMessage('right', image.chatgpt, 'ChatGPT', this.peer_id, message, 'ChatGPT', 'ChatGPT');
this.cleanMessage();
this.streamingTask(message); // Video AI avatar speak
this.speechInMessages ? this.speechMessage(true, 'ChatGPT', message) : this.sound('message');
})
.catch((err) => {
@@ -3778,6 +3805,7 @@ class RoomClient {
this.userLog('info', `💬 New message from: ${data.peer_name}`, 'top-end');
}
this.speechInMessages ? this.speechMessage(true, data.peer_name, data.peer_msg) : this.sound('message');
//this.speechInMessages ? this.streamingTask(data.peer_msg) : this.sound('message');
const participantsList = this.getId('participantsList');
const participantsListItems = participantsList.getElementsByTagName('li');
@@ -6769,6 +6797,457 @@ class RoomClient {
});
}
// ##############################################
// HeyGen Video AI
// ##############################################
getAvatarList() {
this.socket
.request('getAvatarList')
.then(function (completion) {
const avatarVideoAIPreview = document.getElementById('avatarVideoAIPreview');
const avatarVideoAIcontainer = document.getElementById('avatarVideoAIcontainer');
avatarVideoAIcontainer.innerHTML = ''; // cleanup the avatar container
const excludedIds = [
'josh_lite3_20230714',
'josh_lite_20230714',
'Lily_public_lite1_20230601',
'Brian_public_lite1_20230601',
'Brian_public_lite2_20230601',
'Eric_public_lite1_20230601',
'Mido-lite-20221128',
];
const freeAvatars = [
'Kristin in Black Suit',
'Angela in Black Dress',
'Kayla in Casual Suit',
'Anna in Brown T-shirt',
'Briana in Brown suit',
'Justin in White Shirt',
'Leah in Black Suit',
'Wade in Black Jacket',
'Tyler in Casual Suit',
'Tyler in Shirt',
'Tyler in Suit',
'default',
];
completion.response.avatars.forEach((avatar) => {
avatar.avatar_states.forEach((avatarUi) => {
if (
!excludedIds.includes(avatarUi.id) &&
(showFreeAvatars ? freeAvatars.includes(avatarUi.pose_name) : true)
) {
const div = document.createElement('div');
div.style.float = 'left';
div.style.padding = '5px';
div.style.width = '100px';
div.style.height = '200px';
const img = document.createElement('img');
const hr = document.createElement('hr');
const label = document.createElement('label');
const textContent = document.createTextNode(avatarUi.pose_name);
label.appendChild(textContent);
//label.style.fontSize = '12px';
img.setAttribute('id', avatarUi.id);
img.setAttribute('class', 'avatarImg');
img.setAttribute('src', avatarUi.normal_thumbnail_medium);
img.setAttribute('width', '100%');
img.setAttribute('height', 'auto');
img.setAttribute('alt', avatarUi.pose_name);
img.setAttribute('style', 'cursor:pointer; padding: 2px; border-radius: 5px;');
img.setAttribute(
'avatarData',
avatarUi.id +
'|' +
avatar.name +
'|' +
avatarUi.default_voice.free.voice_id +
'|' +
avatarUi.video_url.grey,
);
img.onclick = () => {
const avatarImages = document.querySelectorAll('.avatarImg');
avatarImages.forEach((image) => {
image.style.border = 'none';
});
img.style.border = 'var(--border)';
const avatarData = img.getAttribute('avatarData');
const avatarDataArr = avatarData.split('|');
VideoAI.avatar = avatarDataArr[0];
VideoAI.avatarName = avatarDataArr[1];
VideoAI.avatarVoice = avatarDataArr[2] ? avatarDataArr[2] : '';
avatarVideoAIPreview.src = avatarUi.video_url.grey;
avatarVideoAIPreview.play();
console.log('Avatar image click event', {
avatar,
avatarUi,
avatarDataArr,
});
};
div.append(img);
div.append(hr);
div.append(label);
avatarVideoAIcontainer.append(div);
// Show the first available free avatar
if (showFreeAvatars && avatarUi.pose_name === 'Kristin in Black Suit') {
avatarVideoAIPreview.src = avatarUi.video_url.grey;
avatarVideoAIPreview.play();
}
}
});
});
})
.catch((err) => {
console.error('Video AI getAvatarList error:', err);
});
}
getVoiceList() {
this.socket
.request('getVoiceList')
.then(function (completion) {
const selectElement = document.getElementById('avatarVoiceIDs');
selectElement.innerHTML = '<option value="">Select Avatar Voice</option>'; // Reset options with default
// Sort the list alphabetically by language
const sortedList = completion.response.list.sort((a, b) => a.language.localeCompare(b.language));
sortedList.forEach((flag) => {
// console.log('flag', flag);
const { is_paid, voice_id, language, display_name, gender } = flag;
if (showFreeAvatars ? is_paid == false : true) {
const option = document.createElement('option');
option.value = voice_id;
option.text = `${language}, ${display_name} (${gender})`; // You can customize the display text
selectElement.appendChild(option);
}
});
// Event listener for changes on select element
selectElement.addEventListener('change', (event) => {
const selectedVoiceID = event.target.value;
const selectedPreviewURL = completion.response.list.find(
(flag) => flag.voice_id === selectedVoiceID,
)?.preview?.movio;
VideoAI.avatarVoice = selectedVoiceID;
if (selectedPreviewURL) {
const previewAudio = document.getElementById('previewAudio');
previewAudio.src = selectedPreviewURL;
previewAudio.play();
}
});
})
.catch((err) => {
console.error('Video AI getVoiceList error:', err);
});
}
async handleVideoAI() {
const vb = document.createElement('div');
vb.setAttribute('id', 'avatar__vb');
vb.className = 'videoMenuBar fadein';
const fs = document.createElement('button');
fs.id = 'avatar__fs';
fs.className = html.fullScreen;
const pin = document.createElement('button');
pin.id = 'avatar__pin';
pin.className = html.pin;
const ss = document.createElement('button');
ss.id = 'avatar__stopSession';
ss.className = html.kickOut;
const avatarName = document.createElement('div');
const an = document.createElement('span');
an.id = 'avatar__name';
an.className = html.userName;
an.innerText = VideoAI.avatarName;
// Create video container element
this.videoAIContainer = document.createElement('div');
this.videoAIContainer.className = 'Camera';
this.videoAIContainer.id = 'videoAIContainer';
// Create canvas element for video rendering
this.canvasAIElement = document.createElement('canvas');
this.canvasAIElement.className = '';
this.canvasAIElement.id = 'canvasAIElement';
this.canvasAIElement.style.objectFit = this.isMobileDevice ? 'cover' : 'contain';
// Create video element for avatar
this.videoAIElement = document.createElement('video');
this.videoAIElement.id = 'videoAIElement';
this.videoAIElement.setAttribute('playsinline', true);
this.videoAIElement.autoplay = true;
this.videoAIElement.className = '';
this.videoAIElement.poster = image.poster;
this.videoAIElement.style.objectFit = this.isMobileDevice ? 'cover' : 'contain';
// Append elements to video container
vb.appendChild(ss);
this.isVideoFullScreenSupported && vb.appendChild(fs);
!this.isMobileDevice && vb.appendChild(pin);
avatarName.appendChild(an);
this.videoAIContainer.appendChild(this.videoAIElement);
VideoAI.virtualBackground && this.videoAIContainer.appendChild(this.canvasAIElement);
this.videoAIContainer.appendChild(vb);
this.videoAIContainer.appendChild(avatarName);
this.videoMediaContainer.appendChild(this.videoAIContainer);
// Hide canvas initially
this.canvasAIElement.hidden = true;
// Use video avatar virtual background
if (VideoAI.virtualBackground) {
this.isVideoFullScreenSupported && this.handleFS(this.canvasAIElement.id, fs.id);
this.handlePN(this.canvasAIElement.id, pin.id, this.videoAIContainer.id, true);
} else {
this.isVideoFullScreenSupported && this.handleFS(this.videoAIElement.id, fs.id);
this.handlePN(this.videoAIElement.id, pin.id, this.videoAIContainer.id, true);
}
ss.onclick = () => {
this.stopSession();
};
if (!this.isMobileDevice) {
this.setTippy(pin.id, 'Toggle Pin', 'bottom');
this.setTippy(fs.id, 'Toggle full screen', 'bottom');
this.setTippy(ss.id, 'Stop VideoAI session', 'bottom');
}
handleAspectRatio();
await this.streamingNew();
}
async streamingNew() {
try {
const { quality, avatar, avatarVoice } = VideoAI;
const response = await this.socket.request('streamingNew', {
quality: quality,
avatar_name: avatar,
voice_id: avatarVoice,
});
if (!response || Object.keys(response).length === 0 || response.error) {
this.userLog('error', 'Error to creating the avatar', 'top-end');
this.stopSession();
return;
}
if (response.response.code !== 100) {
this.userLog('warning', response.response.message, 'top-end');
this.stopSession();
return;
}
VideoAI.info = response.response.data;
console.log('Video AI streamingNew', VideoAI);
const { sdp, ice_servers2 } = VideoAI.info;
await this.setupPeerConnection(sdp, ice_servers2);
await this.startSession();
} catch (error) {
this.userLog('error', error, 'top-end');
console.error('Video AI streamingNew error:', error);
this.stopSession();
}
}
async setupPeerConnection(sdp, iceServers) {
this.peerConnection = new RTCPeerConnection({ iceServers: iceServers });
this.peerConnection.ontrack = (event) => {
if (event.track.kind === 'audio' || event.track.kind === 'video') {
this.videoAIElement.srcObject = event.streams[0];
}
};
this.peerConnection.ondatachannel = (event) => {
event.channel.onmessage = this.handleVideoAIMessage;
};
const remoteDescription = new RTCSessionDescription(sdp);
this.peerConnection.setRemoteDescription(remoteDescription);
}
handleVideoAIMessage(event) {
console.log('handleVideoAIMessage', event.data);
}
async startSession() {
if (!VideoAI.info) {
this.userLog('warning', 'Please create a connection first', 'top-end');
return;
}
this.userLog('info', 'Starting session... please wait', 'top-end');
try {
const answer = await this.peerConnection.createAnswer();
await this.peerConnection.setLocalDescription(answer);
await this.streamingStart(VideoAI.info.session_id, answer);
this.peerConnection.onicecandidate = async ({ candidate }) => {
if (candidate) {
await this.streamingICE(candidate);
}
};
} catch (error) {
this.userLog('error', error, 'top-end');
console.error('Video AI startSession error:', error);
}
}
async streamingICE(candidate) {
try {
const response = await this.socket.request('streamingICE', {
session_id: VideoAI.info.session_id,
candidate: candidate.toJSON(),
});
if (response && !response.error) {
return response.response;
}
} catch (error) {
console.error('Video AI streamingICE error:', error);
}
}
async streamingStart(sessionId, sdp) {
try {
const response = await this.socket.request('streamingStart', {
session_id: sessionId,
sdp: sdp,
});
if (!response || response.error) return;
this.startRendering();
VideoAI.active = true;
this.userLog('info', 'Video AI streaming started', 'top-end');
} catch (error) {
console.error('Video AI streamingStart error:', error);
}
}
streamingTask(message) {
if (VideoAI.enabled && VideoAI.active && message) {
const response = this.socket.request('streamingTask', {
session_id: VideoAI.info.session_id,
text: message,
});
console.log('Video AI streamingTask', response);
}
}
startRendering() {
if (!VideoAI.virtualBackground) return;
let frameCounter = 0;
this.renderAIToken = Math.trunc(1e9 * Math.random());
frameCounter = this.renderAIToken;
this.videoAIElement.hidden = true;
this.canvasAIElement.hidden = false;
const context = this.canvasAIElement.getContext('2d', { willReadFrequently: true });
const renderFrame = () => {
if (this.renderAIToken !== frameCounter) return;
this.canvasAIElement.width = this.videoAIElement.videoWidth;
this.canvasAIElement.height = this.videoAIElement.videoHeight;
context.drawImage(this.videoAIElement, 0, 0, this.canvasAIElement.width, this.canvasAIElement.height);
const imageData = context.getImageData(0, 0, this.canvasAIElement.width, this.canvasAIElement.height);
const pixels = imageData.data;
for (let i = 0; i < pixels.length; i += 4) {
if (shouldHidePixel([pixels[i], pixels[i + 1], pixels[i + 2]])) {
pixels[i + 3] = 0; // Make pixel transparent
}
}
function shouldHidePixel([r, g, b]) {
// Adjust the thresholds to match the green screen background
const greenThreshold = 90;
const redThreshold = 90;
const blueThreshold = 90;
return g > greenThreshold && r < redThreshold && b < blueThreshold;
}
context.putImageData(imageData, 0, 0);
requestAnimationFrame(renderFrame);
};
// Set the background of the canvas' parent element to an image or color of your choice
this.canvasAIElement.parentElement.style.background = `url("${VideoAI.background}") center / cover no-repeat`;
setTimeout(renderFrame, 1000);
}
stopRendering() {
this.renderAIToken = null;
}
stopSession() {
const videoAIElement = this.getId('videoAIElement');
if (videoAIElement) {
videoAIElement.parentNode.removeChild(videoAIElement);
}
const videoAIContainer = this.getId('videoAIContainer');
if (videoAIContainer) {
videoAIContainer.parentNode.removeChild(videoAIContainer);
const removeVideoAI = ['videoAIElement', 'canvasAIElement'];
if (this.isVideoPinned && removeVideoAI.includes(this.pinnedVideoPlayerId)) {
this.removeVideoPinMediaContainer();
}
}
handleAspectRatio();
this.streamingStop();
}
streamingStop() {
if (this.peerConnection) {
console.info('Video AI streamingStop peerConnection close done!');
this.peerConnection.close();
this.peerConnection = null;
}
if (VideoAI.info && VideoAI.info.session_id) {
const sessionId = VideoAI.info.session_id;
this.socket
.request('streamingStop', { session_id: sessionId })
.then(() => {
console.info('Video AI streamingStop done!');
})
.catch((error) => {
console.error('Video AI streamingStop error:', error);
});
}
this.stopRendering();
VideoAI.active = false;
}
sleep(ms) {
return new Promise((resolve) => setTimeout(resolve, ms));
}

42
public/views/Room.html
عرض الملف

@@ -209,6 +209,9 @@ access to use this app.
<button id="tabRecordingBtn" class="fas fa-record-vinyl tablinks">
<p class="tabButtonText">Recording</p>
</button>
<button id="tabVideoAIBtn" class="fab fa-airbnb tablinks">
<p class="tabButtonText">VideoAI</p>
</button>
<button id="tabVideoShareBtn" class="fab fa-youtube tablinks">
<p class="tabButtonText">Media</p>
</button>
@@ -878,6 +881,45 @@ access to use this app.
<div id="lastRecordingInfo" class="hidden"></div>
</div>
<div id="tabVideoAI" class="tabcontent">
<div class="avatarsVideoAI">
<video id="avatarVideoAIPreview" src="" controls="yes"></video>
<table class="settingsTable">
<tr id="avatars">
<td class=".width-180">
<div class="title">
<i class="fa-solid fa-users-line"></i>
<p>Show free avatars</p>
</div>
</td>
<td>
<div class="form-check form-switch form-switch-md">
<input
id="switchAvatars"
class="form-check-input"
type="checkbox"
checked
/>
</div>
</td>
</tr>
</table>
<div id="avatarVideoAIcontainer"></div>
<div>
<select id="avatarVoiceIDs" name="avatarVoiceIDs">
<option value="">Select Avatar Voice</option></select
><br /><br />
<div id="audio-container">
<audio id="previewAudio" controls></audio>
</div>
</div>
<br />
<button id="avatarVideoAIStart" class="btn-primary">
<i class="fas fa-play"></i> Start Video Avatar Session
</button>
</div>
</div>
<div id="tabVideoShare" class="tabcontent">
<button id="videoShareButton">
<i class="fab fa-youtube"></i>