Improve AI music prompts and audio metadata
هذا الالتزام موجود في:
boutmoun123
@@ -96,9 +96,16 @@ EMAIL_SMTP_USER=
|
||||
EMAIL_SMTP_PASS=
|
||||
EMAIL_FROM_NAME=Oudelaa
|
||||
EMAIL_FROM_EMAIL=
|
||||
# Enables POST /api/v1/media/ai/text-to-music.
|
||||
# Uses Google Vertex AI publisher model endpoint, not GEMINI_API_KEY directly.
|
||||
# Auth priority:
|
||||
# 1. AI_MUSIC_API_KEY when set.
|
||||
# 2. GOOGLE_APPLICATION_CREDENTIALS_JSON_BASE64 when set.
|
||||
# 3. GOOGLE_APPLICATION_CREDENTIALS file path handled by google-auth-library.
|
||||
AI_MUSIC_ENABLED=false
|
||||
AI_MUSIC_API_KEY=
|
||||
AI_MUSIC_PROJECT_ID=
|
||||
GOOGLE_APPLICATION_CREDENTIALS_JSON_BASE64=
|
||||
AI_MUSIC_PROJECT_ID=accordev
|
||||
AI_MUSIC_LOCATION=us-central1
|
||||
AI_MUSIC_MODEL=lyria-002
|
||||
|
||||
|
||||
@@ -1591,6 +1591,56 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Get My Artist Dashboard",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [
|
||||
{
|
||||
"key": "Authorization",
|
||||
"value": "Bearer {{accessToken}}"
|
||||
},
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"url": "{{baseUrl}}/users/me/dashboard",
|
||||
"description": "Private Flutter artist dashboard endpoint.\n\nReturns profile, aggregate post stats, audience summary, top content, recent activity, and availability flags.\n\nDoes not accept a userId and only uses the authenticated user token."
|
||||
},
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"pm.test('Status is 200', function () { pm.response.to.have.status(200); });",
|
||||
"const json = pm.response.json();",
|
||||
"pm.test('Dashboard response shape is Flutter-ready', function () {",
|
||||
" pm.expect(json).to.have.property('profile');",
|
||||
" pm.expect(json.profile).to.have.property('_id');",
|
||||
" pm.expect(json.profile).to.have.property('coverImage');",
|
||||
" pm.expect(json.profile).to.have.property('avatar');",
|
||||
" pm.expect(json.profile).to.have.property('name');",
|
||||
" pm.expect(json.profile).to.have.property('stageName');",
|
||||
" pm.expect(json.profile).to.have.property('bio');",
|
||||
" pm.expect(json.profile).to.have.property('isVerified');",
|
||||
" pm.expect(json).to.have.property('stats');",
|
||||
" ['followersCount','followingCount','postsCount','collaborationsCount','viewCount','playCount','listenCount','likesCount','commentsCount','savesCount','sharesCount','engagementRate','scorePercentage','earnings'].forEach((key) => pm.expect(json.stats).to.have.property(key));",
|
||||
" pm.expect(json).to.have.property('audience');",
|
||||
" ['followers','nonFollowers','newFollowersThisWeek','newFollowersThisMonth'].forEach((key) => pm.expect(json.audience).to.have.property(key));",
|
||||
" pm.expect(json).to.have.property('engagementChart').that.is.an('array');",
|
||||
" pm.expect(json).to.have.property('topContent').that.is.an('array');",
|
||||
" pm.expect(json).to.have.property('recentActivity').that.is.an('array');",
|
||||
" pm.expect(json).to.have.property('meta');",
|
||||
" ['generatedAt','chartAvailable','earningsAvailable','audienceAnalyticsAvailable'].forEach((key) => pm.expect(json.meta).to.have.property(key));",
|
||||
"});",
|
||||
"pm.environment.set('artistDashboardProfileId', json.profile._id);"
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Admin Discover Users",
|
||||
"request": {
|
||||
|
||||
@@ -975,6 +975,56 @@
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Get My Artist Dashboard",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [
|
||||
{
|
||||
"key": "Authorization",
|
||||
"value": "Bearer {{accessToken}}"
|
||||
},
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"url": "{{baseUrl}}/users/me/dashboard",
|
||||
"description": "Private Flutter artist dashboard endpoint.\n\nReturns profile, aggregate post stats, audience summary, top content, recent activity, and availability flags.\n\nDoes not accept a userId and only uses the authenticated user token."
|
||||
},
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"pm.test('Status is 200', function () { pm.response.to.have.status(200); });",
|
||||
"const json = pm.response.json();",
|
||||
"pm.test('Dashboard response shape is Flutter-ready', function () {",
|
||||
" pm.expect(json).to.have.property('profile');",
|
||||
" pm.expect(json.profile).to.have.property('_id');",
|
||||
" pm.expect(json.profile).to.have.property('coverImage');",
|
||||
" pm.expect(json.profile).to.have.property('avatar');",
|
||||
" pm.expect(json.profile).to.have.property('name');",
|
||||
" pm.expect(json.profile).to.have.property('stageName');",
|
||||
" pm.expect(json.profile).to.have.property('bio');",
|
||||
" pm.expect(json.profile).to.have.property('isVerified');",
|
||||
" pm.expect(json).to.have.property('stats');",
|
||||
" ['followersCount','followingCount','postsCount','collaborationsCount','viewCount','playCount','listenCount','likesCount','commentsCount','savesCount','sharesCount','engagementRate','scorePercentage','earnings'].forEach((key) => pm.expect(json.stats).to.have.property(key));",
|
||||
" pm.expect(json).to.have.property('audience');",
|
||||
" ['followers','nonFollowers','newFollowersThisWeek','newFollowersThisMonth'].forEach((key) => pm.expect(json.audience).to.have.property(key));",
|
||||
" pm.expect(json).to.have.property('engagementChart').that.is.an('array');",
|
||||
" pm.expect(json).to.have.property('topContent').that.is.an('array');",
|
||||
" pm.expect(json).to.have.property('recentActivity').that.is.an('array');",
|
||||
" pm.expect(json).to.have.property('meta');",
|
||||
" ['generatedAt','chartAvailable','earningsAvailable','audienceAnalyticsAvailable'].forEach((key) => pm.expect(json.meta).to.have.property(key));",
|
||||
"});",
|
||||
"pm.environment.set('artistDashboardProfileId', json.profile._id);"
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -1113,6 +1113,56 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Get My Artist Dashboard",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [
|
||||
{
|
||||
"key": "Authorization",
|
||||
"value": "Bearer {{accessToken}}"
|
||||
},
|
||||
{
|
||||
"key": "Content-Type",
|
||||
"value": "application/json"
|
||||
}
|
||||
],
|
||||
"url": "{{baseUrl}}/users/me/dashboard",
|
||||
"description": "Private Flutter artist dashboard endpoint.\n\nReturns profile, aggregate post stats, audience summary, top content, recent activity, and availability flags.\n\nDoes not accept a userId and only uses the authenticated user token."
|
||||
},
|
||||
"event": [
|
||||
{
|
||||
"listen": "test",
|
||||
"script": {
|
||||
"type": "text/javascript",
|
||||
"exec": [
|
||||
"pm.test('Status is 200', function () { pm.response.to.have.status(200); });",
|
||||
"const json = pm.response.json();",
|
||||
"pm.test('Dashboard response shape is Flutter-ready', function () {",
|
||||
" pm.expect(json).to.have.property('profile');",
|
||||
" pm.expect(json.profile).to.have.property('_id');",
|
||||
" pm.expect(json.profile).to.have.property('coverImage');",
|
||||
" pm.expect(json.profile).to.have.property('avatar');",
|
||||
" pm.expect(json.profile).to.have.property('name');",
|
||||
" pm.expect(json.profile).to.have.property('stageName');",
|
||||
" pm.expect(json.profile).to.have.property('bio');",
|
||||
" pm.expect(json.profile).to.have.property('isVerified');",
|
||||
" pm.expect(json).to.have.property('stats');",
|
||||
" ['followersCount','followingCount','postsCount','collaborationsCount','viewCount','playCount','listenCount','likesCount','commentsCount','savesCount','sharesCount','engagementRate','scorePercentage','earnings'].forEach((key) => pm.expect(json.stats).to.have.property(key));",
|
||||
" pm.expect(json).to.have.property('audience');",
|
||||
" ['followers','nonFollowers','newFollowersThisWeek','newFollowersThisMonth'].forEach((key) => pm.expect(json.audience).to.have.property(key));",
|
||||
" pm.expect(json).to.have.property('engagementChart').that.is.an('array');",
|
||||
" pm.expect(json).to.have.property('topContent').that.is.an('array');",
|
||||
" pm.expect(json).to.have.property('recentActivity').that.is.an('array');",
|
||||
" pm.expect(json).to.have.property('meta');",
|
||||
" ['generatedAt','chartAvailable','earningsAvailable','audienceAnalyticsAvailable'].forEach((key) => pm.expect(json.meta).to.have.property(key));",
|
||||
"});",
|
||||
"pm.environment.set('artistDashboardProfileId', json.profile._id);"
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Get User Presence",
|
||||
"request": {
|
||||
|
||||
@@ -10,6 +10,38 @@ const expectDisplaySafeWaveform = (peaks: number[], expectedLength = 100) => {
|
||||
expect(peaks.every((value) => value >= 0 && value <= 100)).toBe(true);
|
||||
};
|
||||
|
||||
const createPcmWavBuffer = (amplitudeByFrame: (frame: number) => number, frames = 8000) => {
|
||||
const sampleRate = 8000;
|
||||
const channels = 1;
|
||||
const bitsPerSample = 16;
|
||||
const bytesPerSample = bitsPerSample / 8;
|
||||
const dataSize = frames * channels * bytesPerSample;
|
||||
const buffer = Buffer.alloc(44 + dataSize);
|
||||
|
||||
buffer.write('RIFF', 0, 'ascii');
|
||||
buffer.writeUInt32LE(36 + dataSize, 4);
|
||||
buffer.write('WAVE', 8, 'ascii');
|
||||
buffer.write('fmt ', 12, 'ascii');
|
||||
buffer.writeUInt32LE(16, 16);
|
||||
buffer.writeUInt16LE(1, 20);
|
||||
buffer.writeUInt16LE(channels, 22);
|
||||
buffer.writeUInt32LE(sampleRate, 24);
|
||||
buffer.writeUInt32LE(sampleRate * channels * bytesPerSample, 28);
|
||||
buffer.writeUInt16LE(channels * bytesPerSample, 32);
|
||||
buffer.writeUInt16LE(bitsPerSample, 34);
|
||||
buffer.write('data', 36, 'ascii');
|
||||
buffer.writeUInt32LE(dataSize, 40);
|
||||
|
||||
for (let frame = 0; frame < frames; frame += 1) {
|
||||
const tone = Math.sin((frame / sampleRate) * Math.PI * 2 * 440);
|
||||
const amplitude = amplitudeByFrame(frame);
|
||||
const sample = Math.max(-32768, Math.min(32767, Math.round(tone * amplitude * 32767)));
|
||||
buffer.writeInt16LE(sample, 44 + frame * bytesPerSample);
|
||||
}
|
||||
|
||||
return buffer;
|
||||
};
|
||||
|
||||
describe('waveform util', () => {
|
||||
const originalAudioWaveformPeaks = process.env.AUDIO_WAVEFORM_PEAKS;
|
||||
|
||||
@@ -67,4 +99,14 @@ describe('waveform util', () => {
|
||||
expectDisplaySafeWaveform(generateWaveformPeaksFromBuffer(Buffer.from('fake audio bytes')));
|
||||
expectDisplaySafeWaveform(generateWaveformPeaksFromSeed('audio-url'));
|
||||
});
|
||||
|
||||
it('generates varied normalized peaks from actual WAV PCM samples', () => {
|
||||
const wavBuffer = createPcmWavBuffer((frame) => 0.12 + (frame / 7999) * 0.78);
|
||||
const peaks = generateWaveformPeaksFromBuffer(wavBuffer);
|
||||
|
||||
expectDisplaySafeWaveform(peaks);
|
||||
expect(new Set(peaks).size).toBeGreaterThan(8);
|
||||
expect(peaks.slice(0, 40).some((value) => value < 60)).toBe(true);
|
||||
expect(peaks.slice(60).some((value) => value > 80)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -29,12 +29,125 @@ const scaleToRange = (values: number[]): number[] => {
|
||||
return [];
|
||||
}
|
||||
|
||||
const max = Math.max(...values);
|
||||
const sorted = [...values].sort((left, right) => left - right);
|
||||
const max = sorted[sorted.length - 1] ?? 0;
|
||||
if (max <= 0) {
|
||||
return values.map((value) => clampPeak(value));
|
||||
}
|
||||
|
||||
return values.map((value) => clampPeak((value / max) * 100));
|
||||
const percentileIndex = Math.max(0, Math.min(sorted.length - 1, Math.floor(sorted.length * 0.95)));
|
||||
const scaleMax = Math.max(sorted[percentileIndex] ?? max, 1);
|
||||
|
||||
return values.map((value) => clampPeak((Math.min(value, scaleMax) / scaleMax) * 100));
|
||||
};
|
||||
|
||||
const readPcmSample = (
|
||||
buffer: Buffer,
|
||||
offset: number,
|
||||
bitsPerSample: number,
|
||||
audioFormat: number,
|
||||
): number => {
|
||||
if (audioFormat === 3) {
|
||||
if (bitsPerSample === 32 && offset + 4 <= buffer.length) {
|
||||
return Math.max(-1, Math.min(1, buffer.readFloatLE(offset)));
|
||||
}
|
||||
if (bitsPerSample === 64 && offset + 8 <= buffer.length) {
|
||||
return Math.max(-1, Math.min(1, buffer.readDoubleLE(offset)));
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
switch (bitsPerSample) {
|
||||
case 8:
|
||||
return (buffer.readUInt8(offset) - 128) / 128;
|
||||
case 16:
|
||||
return buffer.readInt16LE(offset) / 32768;
|
||||
case 24: {
|
||||
if (offset + 3 > buffer.length) {
|
||||
return 0;
|
||||
}
|
||||
let value = buffer.readUIntLE(offset, 3);
|
||||
if (value & 0x800000) {
|
||||
value |= 0xff000000;
|
||||
}
|
||||
return value / 8388608;
|
||||
}
|
||||
case 32:
|
||||
return buffer.readInt32LE(offset) / 2147483648;
|
||||
default:
|
||||
return 0;
|
||||
}
|
||||
};
|
||||
|
||||
const generateWaveformPeaksFromWavBuffer = (buffer: Buffer, samples: number): number[] | null => {
|
||||
if (buffer.length < 44 || buffer.toString('ascii', 0, 4) !== 'RIFF' || buffer.toString('ascii', 8, 12) !== 'WAVE') {
|
||||
return null;
|
||||
}
|
||||
|
||||
let offset = 12;
|
||||
let audioFormat = 0;
|
||||
let channels = 0;
|
||||
let bitsPerSample = 0;
|
||||
let dataOffset = 0;
|
||||
let dataSize = 0;
|
||||
|
||||
while (offset + 8 <= buffer.length) {
|
||||
const chunkId = buffer.toString('ascii', offset, offset + 4);
|
||||
const chunkSize = buffer.readUInt32LE(offset + 4);
|
||||
const chunkDataOffset = offset + 8;
|
||||
|
||||
if (chunkId === 'fmt ' && chunkDataOffset + 16 <= buffer.length) {
|
||||
audioFormat = buffer.readUInt16LE(chunkDataOffset);
|
||||
channels = buffer.readUInt16LE(chunkDataOffset + 2);
|
||||
bitsPerSample = buffer.readUInt16LE(chunkDataOffset + 14);
|
||||
} else if (chunkId === 'data') {
|
||||
dataOffset = chunkDataOffset;
|
||||
dataSize = Math.min(chunkSize, buffer.length - chunkDataOffset);
|
||||
}
|
||||
|
||||
offset = chunkDataOffset + chunkSize + (chunkSize % 2);
|
||||
}
|
||||
|
||||
const bytesPerSample = bitsPerSample / 8;
|
||||
const frameSize = bytesPerSample * channels;
|
||||
if (
|
||||
!dataOffset ||
|
||||
!dataSize ||
|
||||
!channels ||
|
||||
!Number.isInteger(bytesPerSample) ||
|
||||
frameSize <= 0 ||
|
||||
(audioFormat !== 1 && audioFormat !== 3)
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const frameCount = Math.floor(dataSize / frameSize);
|
||||
if (frameCount <= 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const bucketSize = Math.max(1, Math.ceil(frameCount / samples));
|
||||
const peaks: number[] = [];
|
||||
|
||||
for (let bucketStart = 0; bucketStart < frameCount; bucketStart += bucketSize) {
|
||||
const bucketEnd = Math.min(frameCount, bucketStart + bucketSize);
|
||||
let sumSquares = 0;
|
||||
let count = 0;
|
||||
|
||||
for (let frameIndex = bucketStart; frameIndex < bucketEnd; frameIndex += 1) {
|
||||
const frameOffset = dataOffset + frameIndex * frameSize;
|
||||
for (let channel = 0; channel < channels; channel += 1) {
|
||||
const sampleOffset = frameOffset + channel * bytesPerSample;
|
||||
const value = readPcmSample(buffer, sampleOffset, bitsPerSample, audioFormat);
|
||||
sumSquares += value * value;
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
peaks.push(count ? Math.sqrt(sumSquares / count) * 100 : 0);
|
||||
}
|
||||
|
||||
return normalizeWaveformPeaks(peaks, samples);
|
||||
};
|
||||
|
||||
export const normalizeWaveformPeaks = (
|
||||
@@ -93,6 +206,11 @@ export const generateWaveformPeaksFromBuffer = (
|
||||
return normalizeWaveformPeaks([], samples);
|
||||
}
|
||||
|
||||
const wavPeaks = generateWaveformPeaksFromWavBuffer(buffer, samples);
|
||||
if (wavPeaks) {
|
||||
return wavPeaks;
|
||||
}
|
||||
|
||||
const chunkSize = Math.max(1, Math.ceil(buffer.length / samples));
|
||||
const peaks: number[] = [];
|
||||
|
||||
|
||||
@@ -49,6 +49,8 @@ export default () => ({
|
||||
aiMusic: {
|
||||
enabled: (process.env.AI_MUSIC_ENABLED ?? 'false').toLowerCase() === 'true',
|
||||
apiKey: process.env.AI_MUSIC_API_KEY ?? '',
|
||||
googleApplicationCredentialsJsonBase64:
|
||||
process.env.GOOGLE_APPLICATION_CREDENTIALS_JSON_BASE64 ?? '',
|
||||
projectId: process.env.AI_MUSIC_PROJECT_ID ?? '',
|
||||
location: process.env.AI_MUSIC_LOCATION ?? 'us-central1',
|
||||
model: process.env.AI_MUSIC_MODEL ?? 'lyria-002',
|
||||
|
||||
@@ -32,6 +32,7 @@ export const validationSchema = Joi.object({
|
||||
EMAIL_FROM_EMAIL: Joi.string().allow('').optional(),
|
||||
AI_MUSIC_ENABLED: Joi.boolean().truthy('true').falsy('false').default(false),
|
||||
AI_MUSIC_API_KEY: Joi.string().allow('').optional(),
|
||||
GOOGLE_APPLICATION_CREDENTIALS_JSON_BASE64: Joi.string().allow('').optional(),
|
||||
AI_MUSIC_PROJECT_ID: Joi.string().allow('').optional(),
|
||||
AI_MUSIC_LOCATION: Joi.string().default('us-central1'),
|
||||
AI_MUSIC_MODEL: Joi.string().default('lyria-002'),
|
||||
|
||||
@@ -1,5 +1,15 @@
|
||||
import { Type } from 'class-transformer';
|
||||
import { IsInt, IsNotEmpty, IsOptional, IsString, MaxLength, Max, Min } from 'class-validator';
|
||||
import {
|
||||
IsArray,
|
||||
IsIn,
|
||||
IsInt,
|
||||
IsNotEmpty,
|
||||
IsOptional,
|
||||
IsString,
|
||||
MaxLength,
|
||||
Max,
|
||||
Min,
|
||||
} from 'class-validator';
|
||||
|
||||
export class TextToMusicDto {
|
||||
@IsString()
|
||||
@@ -20,4 +30,33 @@ export class TextToMusicDto {
|
||||
@Min(0)
|
||||
@Max(2147483647)
|
||||
seed?: number;
|
||||
|
||||
@IsOptional()
|
||||
@IsString()
|
||||
@MaxLength(80)
|
||||
maqam?: string;
|
||||
|
||||
@IsOptional()
|
||||
@IsString()
|
||||
@MaxLength(80)
|
||||
mood?: string;
|
||||
|
||||
@IsOptional()
|
||||
@IsArray()
|
||||
@IsString({ each: true })
|
||||
@MaxLength(80, { each: true })
|
||||
instruments?: string[];
|
||||
|
||||
@IsOptional()
|
||||
@IsString()
|
||||
@MaxLength(120)
|
||||
style?: string;
|
||||
|
||||
@IsOptional()
|
||||
@IsIn(['slow', 'medium', 'fast'])
|
||||
tempo?: 'slow' | 'medium' | 'fast';
|
||||
|
||||
@IsOptional()
|
||||
@IsIn(['none', 'light', 'medium'])
|
||||
percussion?: 'none' | 'light' | 'medium';
|
||||
}
|
||||
|
||||
222
src/modules/media/media.service.spec.ts
Normal file
222
src/modules/media/media.service.spec.ts
Normal file
@@ -0,0 +1,222 @@
|
||||
import { ServiceUnavailableException } from '@nestjs/common';
|
||||
import { MediaService } from './media.service';
|
||||
|
||||
const createPcmWavBuffer = (frames = 8000) => {
|
||||
const sampleRate = 8000;
|
||||
const channels = 1;
|
||||
const bitsPerSample = 16;
|
||||
const bytesPerSample = bitsPerSample / 8;
|
||||
const dataSize = frames * channels * bytesPerSample;
|
||||
const buffer = Buffer.alloc(44 + dataSize);
|
||||
|
||||
buffer.write('RIFF', 0, 'ascii');
|
||||
buffer.writeUInt32LE(36 + dataSize, 4);
|
||||
buffer.write('WAVE', 8, 'ascii');
|
||||
buffer.write('fmt ', 12, 'ascii');
|
||||
buffer.writeUInt32LE(16, 16);
|
||||
buffer.writeUInt16LE(1, 20);
|
||||
buffer.writeUInt16LE(channels, 22);
|
||||
buffer.writeUInt32LE(sampleRate, 24);
|
||||
buffer.writeUInt32LE(sampleRate * channels * bytesPerSample, 28);
|
||||
buffer.writeUInt16LE(channels * bytesPerSample, 32);
|
||||
buffer.writeUInt16LE(bitsPerSample, 34);
|
||||
buffer.write('data', 36, 'ascii');
|
||||
buffer.writeUInt32LE(dataSize, 40);
|
||||
|
||||
for (let frame = 0; frame < frames; frame += 1) {
|
||||
const amplitude = frame < frames / 2 ? 0.2 : 0.9;
|
||||
const tone = Math.sin((frame / sampleRate) * Math.PI * 2 * 440);
|
||||
buffer.writeInt16LE(Math.round(tone * amplitude * 32767), 44 + frame * bytesPerSample);
|
||||
}
|
||||
|
||||
return buffer;
|
||||
};
|
||||
|
||||
describe('MediaService Google credentials resolution', () => {
|
||||
const createService = (
|
||||
googleCredentialsBase64 = '',
|
||||
overrides: {
|
||||
config?: Record<string, unknown>;
|
||||
storageService?: Record<string, unknown>;
|
||||
mediaProbeService?: Record<string, unknown>;
|
||||
} = {},
|
||||
) => {
|
||||
const configService = {
|
||||
get: jest.fn((key: string) => {
|
||||
if (key === 'aiMusic.googleApplicationCredentialsJsonBase64') {
|
||||
return googleCredentialsBase64;
|
||||
}
|
||||
if (key in (overrides.config ?? {})) {
|
||||
return overrides.config?.[key];
|
||||
}
|
||||
return undefined;
|
||||
}),
|
||||
};
|
||||
|
||||
return new MediaService(
|
||||
configService as any,
|
||||
(overrides.storageService ?? {}) as any,
|
||||
(overrides.mediaProbeService ?? {}) as any,
|
||||
) as any;
|
||||
};
|
||||
|
||||
it('falls back to default Google application credentials when base64 JSON is missing', () => {
|
||||
const service = createService();
|
||||
|
||||
expect(service.resolveGoogleApplicationCredentials()).toEqual({});
|
||||
});
|
||||
|
||||
it('decodes service account credentials from base64 JSON', () => {
|
||||
const credentials = {
|
||||
type: 'service_account',
|
||||
client_email: 'ai-music@example.iam.gserviceaccount.com',
|
||||
private_key: '-----BEGIN PRIVATE KEY-----\nredacted\n-----END PRIVATE KEY-----\n',
|
||||
};
|
||||
const encoded = Buffer.from(JSON.stringify(credentials), 'utf8').toString('base64');
|
||||
const service = createService(encoded);
|
||||
|
||||
expect(service.resolveGoogleApplicationCredentials()).toEqual({ credentials });
|
||||
});
|
||||
|
||||
it('throws a safe configuration error for invalid base64 credentials', () => {
|
||||
const service = createService(Buffer.from('not-json', 'utf8').toString('base64'));
|
||||
|
||||
expect(() => service.resolveGoogleApplicationCredentials()).toThrow(
|
||||
ServiceUnavailableException,
|
||||
);
|
||||
expect(() => service.resolveGoogleApplicationCredentials()).toThrow(
|
||||
'Invalid GOOGLE_APPLICATION_CREDENTIALS_JSON_BASE64',
|
||||
);
|
||||
});
|
||||
|
||||
it('returns the probed audio duration instead of the requested duration', async () => {
|
||||
const audioBuffer = createPcmWavBuffer();
|
||||
const originalFetch = global.fetch;
|
||||
global.fetch = jest.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: jest.fn().mockResolvedValue({
|
||||
predictions: [
|
||||
{
|
||||
bytesBase64Encoded: audioBuffer.toString('base64'),
|
||||
mimeType: 'audio/wav',
|
||||
},
|
||||
],
|
||||
}),
|
||||
}) as any;
|
||||
const storageService = {
|
||||
saveFile: jest.fn().mockResolvedValue('/uploads/ai-music/generated.wav'),
|
||||
};
|
||||
const mediaProbeService = {
|
||||
extractDurationSecondsFromBuffer: jest.fn().mockResolvedValue(32),
|
||||
};
|
||||
const service = createService('', {
|
||||
config: {
|
||||
'aiMusic.enabled': true,
|
||||
'aiMusic.apiKey': 'test-api-key',
|
||||
'aiMusic.projectId': 'accordev',
|
||||
'aiMusic.location': 'us-central1',
|
||||
'aiMusic.model': 'lyria-002',
|
||||
},
|
||||
storageService,
|
||||
mediaProbeService,
|
||||
});
|
||||
|
||||
try {
|
||||
const result = await service.generateMusicFromText('user-1', {
|
||||
prompt: 'test prompt',
|
||||
durationSeconds: 12,
|
||||
});
|
||||
|
||||
expect(result.durationSeconds).toBe(32);
|
||||
expect(result.prompt).toBe('test prompt');
|
||||
expect(result.originalPrompt).toBe('test prompt');
|
||||
expect(result.enhancedPrompt).toContain('Instrumental only, no vocals');
|
||||
expect(result.audioUrl).toBe('/uploads/ai-music/generated.wav');
|
||||
expect(result.mimeType).toBe('audio/wav');
|
||||
expect(result.sizeBytes).toBe(audioBuffer.length);
|
||||
expect(result.waveformPeaks).toHaveLength(100);
|
||||
expect(mediaProbeService.extractDurationSecondsFromBuffer).toHaveBeenCalledWith(
|
||||
audioBuffer,
|
||||
{
|
||||
originalname: 'ai-music.wav',
|
||||
mimetype: 'audio/wav',
|
||||
},
|
||||
);
|
||||
} finally {
|
||||
global.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
it('enhances Arabic Middle Eastern music prompts before sending them to Lyria', async () => {
|
||||
const audioBuffer = createPcmWavBuffer();
|
||||
const originalFetch = global.fetch;
|
||||
global.fetch = jest.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: jest.fn().mockResolvedValue({
|
||||
predictions: [
|
||||
{
|
||||
bytesBase64Encoded: audioBuffer.toString('base64'),
|
||||
mimeType: 'audio/wav',
|
||||
},
|
||||
],
|
||||
}),
|
||||
}) as any;
|
||||
const service = createService('', {
|
||||
config: {
|
||||
'aiMusic.enabled': true,
|
||||
'aiMusic.apiKey': 'test-api-key',
|
||||
'aiMusic.projectId': 'accordev',
|
||||
'aiMusic.location': 'us-central1',
|
||||
'aiMusic.model': 'lyria-002',
|
||||
},
|
||||
storageService: {
|
||||
saveFile: jest.fn().mockResolvedValue('/uploads/ai-music/generated.wav'),
|
||||
},
|
||||
mediaProbeService: {
|
||||
extractDurationSecondsFromBuffer: jest.fn().mockResolvedValue(12),
|
||||
},
|
||||
});
|
||||
|
||||
try {
|
||||
const result = await service.generateMusicFromText('user-1', {
|
||||
prompt: 'بدي لحن عود حزين مقام حجاز مع إيقاع خفيف',
|
||||
durationSeconds: 12,
|
||||
});
|
||||
const fetchBody = JSON.parse((global.fetch as jest.Mock).mock.calls[0][1].body);
|
||||
const sentPrompt = fetchBody.instances[0].prompt as string;
|
||||
|
||||
expect(result.prompt).toBe('بدي لحن عود حزين مقام حجاز مع إيقاع خفيف');
|
||||
expect(result.originalPrompt).toBe('بدي لحن عود حزين مقام حجاز مع إيقاع خفيف');
|
||||
expect(result.enhancedPrompt).toBe(sentPrompt);
|
||||
expect(sentPrompt).toContain('Instrumental Arabic oud melody');
|
||||
expect(sentPrompt).toContain('Hijaz maqam, emotional Middle Eastern sound');
|
||||
expect(sentPrompt).toContain('sad, emotional, expressive mood');
|
||||
expect(sentPrompt).toContain('light riq percussion');
|
||||
expect(sentPrompt).toContain('slow tempo');
|
||||
expect(sentPrompt).toContain(
|
||||
'Instrumental only, no vocals, no lyrics, no artist imitation, clean studio recording, balanced mix.',
|
||||
);
|
||||
} finally {
|
||||
global.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
it('uses optional maqam mood instruments and percussion fields in the enhanced prompt', () => {
|
||||
const service = createService();
|
||||
const enhancedPrompt = service.buildEnhancedMusicPrompt({
|
||||
prompt: 'warm intro',
|
||||
maqam: 'Nahawand',
|
||||
mood: 'romantic',
|
||||
instruments: ['qanun', 'ney'],
|
||||
style: 'cinematic Levantine arrangement',
|
||||
tempo: 'medium',
|
||||
percussion: 'light',
|
||||
});
|
||||
|
||||
expect(enhancedPrompt).toContain('Nahawand maqam, dramatic minor Arabic mood');
|
||||
expect(enhancedPrompt).toContain('romantic, warm mood');
|
||||
expect(enhancedPrompt).toContain('Instrumental Arabic qanun melody');
|
||||
expect(enhancedPrompt).toContain('light percussion');
|
||||
expect(enhancedPrompt).toContain('cinematic Levantine arrangement');
|
||||
});
|
||||
});
|
||||
@@ -6,6 +6,36 @@ import { ManagedStorageService } from '../../infrastructure/storage/managed-stor
|
||||
import { MediaProbeService } from '../../infrastructure/storage/media-probe.service';
|
||||
import { TextToMusicDto } from './dto/text-to-music.dto';
|
||||
|
||||
const AI_MUSIC_SAFE_PRODUCTION_INSTRUCTIONS =
|
||||
'Instrumental only, no vocals, no lyrics, no artist imitation, clean studio recording, balanced mix.';
|
||||
|
||||
const ARABIC_MAQAM_MAPPINGS: Array<{ terms: string[]; description: string }> = [
|
||||
{ terms: ['حجاز', 'hijaz'], description: 'Hijaz maqam, emotional Middle Eastern sound' },
|
||||
{ terms: ['بياتي', 'bayati'], description: 'Bayati maqam, warm Levantine folk mood' },
|
||||
{ terms: ['راست', 'rast'], description: 'Rast maqam, classical Arabic optimistic character' },
|
||||
{ terms: ['نهاوند', 'nahawand'], description: 'Nahawand maqam, dramatic minor Arabic mood' },
|
||||
{ terms: ['كرد', 'kurd'], description: 'Kurd maqam, simple minor Arabic mood' },
|
||||
{ terms: ['صبا', 'saba'], description: 'Saba maqam, deeply sad Arabic mood' },
|
||||
];
|
||||
|
||||
const ARABIC_INSTRUMENT_MAPPINGS: Array<{ terms: string[]; description: string }> = [
|
||||
{ terms: ['عود', 'oud'], description: 'oud' },
|
||||
{ terms: ['قانون', 'qanun'], description: 'qanun' },
|
||||
{ terms: ['ناي', 'ney'], description: 'ney flute' },
|
||||
{ terms: ['كمان', 'violin'], description: 'violin' },
|
||||
{ terms: ['رق', 'riq'], description: 'riq frame drum' },
|
||||
{ terms: ['طبلة', 'دربكة', 'darbuka'], description: 'darbuka' },
|
||||
];
|
||||
|
||||
const ARABIC_MOOD_MAPPINGS: Array<{ terms: string[]; description: string }> = [
|
||||
{ terms: ['حزين', 'sad'], description: 'sad, emotional, expressive' },
|
||||
{ terms: ['هادئ', 'هادي', 'calm'], description: 'calm, soft, slow' },
|
||||
{ terms: ['حماسي', 'energetic'], description: 'energetic, heroic' },
|
||||
{ terms: ['رومانسي', 'romantic'], description: 'romantic, warm' },
|
||||
{ terms: ['صوفي', 'spiritual'], description: 'spiritual, meditative' },
|
||||
{ terms: ['تراثي', 'traditional'], description: 'traditional Arabic folk' },
|
||||
];
|
||||
|
||||
@Injectable()
|
||||
export class MediaService {
|
||||
constructor(
|
||||
@@ -123,21 +153,34 @@ export class MediaService {
|
||||
} else {
|
||||
const auth = new GoogleAuth({
|
||||
scopes: ['https://www.googleapis.com/auth/cloud-platform'],
|
||||
...this.resolveGoogleApplicationCredentials(),
|
||||
});
|
||||
|
||||
const client = await auth.getClient();
|
||||
const accessTokenRaw = await client.getAccessToken();
|
||||
const accessToken =
|
||||
typeof accessTokenRaw === 'string' ? accessTokenRaw : (accessTokenRaw?.token ?? '');
|
||||
let accessToken = '';
|
||||
try {
|
||||
const client = await auth.getClient();
|
||||
const accessTokenRaw = await client.getAccessToken();
|
||||
accessToken =
|
||||
typeof accessTokenRaw === 'string' ? accessTokenRaw : (accessTokenRaw?.token ?? '');
|
||||
} catch (error) {
|
||||
throw new ServiceUnavailableException(
|
||||
`Failed to authenticate with Google Cloud: ${
|
||||
error instanceof Error ? error.message : 'invalid credentials'
|
||||
}`,
|
||||
);
|
||||
}
|
||||
|
||||
if (!accessToken) {
|
||||
throw new ServiceUnavailableException('Failed to authenticate with Google Cloud');
|
||||
throw new ServiceUnavailableException(
|
||||
'Failed to authenticate with Google Cloud: access token was not returned',
|
||||
);
|
||||
}
|
||||
authorizationHeader = `Bearer ${accessToken}`;
|
||||
}
|
||||
|
||||
const enhancedPrompt = this.buildEnhancedMusicPrompt(dto);
|
||||
const requestBody: Record<string, unknown> = {
|
||||
instances: [{ prompt: dto.prompt }],
|
||||
instances: [{ prompt: enhancedPrompt }],
|
||||
parameters: {
|
||||
sampleCount: 1,
|
||||
durationSeconds: dto.durationSeconds ?? 12,
|
||||
@@ -189,10 +232,19 @@ export class MediaService {
|
||||
contentType: mimeType,
|
||||
fileNamePrefix: `ai-${userId}`,
|
||||
});
|
||||
const actualDurationSeconds = await this.mediaProbeService.extractDurationSecondsFromBuffer(
|
||||
buffer,
|
||||
{
|
||||
originalname: `ai-music.${extension}`,
|
||||
mimetype: mimeType,
|
||||
},
|
||||
);
|
||||
|
||||
return {
|
||||
prompt: dto.prompt,
|
||||
durationSeconds: dto.durationSeconds ?? 12,
|
||||
originalPrompt: dto.prompt,
|
||||
enhancedPrompt,
|
||||
durationSeconds: actualDurationSeconds ?? dto.durationSeconds ?? 12,
|
||||
mimeType,
|
||||
sizeBytes: buffer.length,
|
||||
audioUrl,
|
||||
@@ -215,4 +267,233 @@ export class MediaService {
|
||||
}
|
||||
return 'wav';
|
||||
}
|
||||
|
||||
private buildEnhancedMusicPrompt(dto: TextToMusicDto): string {
|
||||
const originalPrompt = dto.prompt.trim();
|
||||
const normalizedPrompt = this.normalizeArabicText(originalPrompt).toLowerCase();
|
||||
const isArabicPrompt = /[\u0600-\u06ff]/.test(originalPrompt);
|
||||
|
||||
const maqam = this.resolveMappedDescription(
|
||||
dto.maqam,
|
||||
normalizedPrompt,
|
||||
ARABIC_MAQAM_MAPPINGS,
|
||||
);
|
||||
const moods = this.resolveMappedDescriptions(
|
||||
dto.mood ? [dto.mood] : [],
|
||||
normalizedPrompt,
|
||||
ARABIC_MOOD_MAPPINGS,
|
||||
);
|
||||
const instruments = this.resolveMappedDescriptions(
|
||||
dto.instruments ?? [],
|
||||
normalizedPrompt,
|
||||
ARABIC_INSTRUMENT_MAPPINGS,
|
||||
);
|
||||
const tempo = dto.tempo ?? this.inferTempo(normalizedPrompt, moods);
|
||||
const percussion = dto.percussion ?? this.inferPercussion(normalizedPrompt, instruments);
|
||||
const style = dto.style?.trim();
|
||||
|
||||
const parts: string[] = [];
|
||||
if (isArabicPrompt || maqam || instruments.length || moods.length) {
|
||||
const mainInstruments = instruments.filter(
|
||||
(instrument) => !instrument.includes('drum') && !instrument.includes('riq'),
|
||||
);
|
||||
const melodyInstrument = mainInstruments[0] ?? instruments[0];
|
||||
parts.push(
|
||||
melodyInstrument
|
||||
? `Instrumental Arabic ${melodyInstrument} melody`
|
||||
: 'Instrumental Arabic music piece',
|
||||
);
|
||||
} else {
|
||||
parts.push(`Instrumental music based on this idea: ${originalPrompt}`);
|
||||
}
|
||||
|
||||
if (maqam) {
|
||||
parts.push(`in ${maqam}`);
|
||||
}
|
||||
if (moods.length) {
|
||||
parts.push(`${moods.join(', ')} mood`);
|
||||
}
|
||||
if (percussion !== 'none') {
|
||||
parts.push(this.describePercussion(percussion, instruments, isArabicPrompt));
|
||||
}
|
||||
if (tempo) {
|
||||
parts.push(`${tempo} tempo`);
|
||||
}
|
||||
if (style) {
|
||||
parts.push(style);
|
||||
} else if (isArabicPrompt) {
|
||||
parts.push('traditional Arabic atmosphere');
|
||||
}
|
||||
|
||||
parts.push(AI_MUSIC_SAFE_PRODUCTION_INSTRUCTIONS);
|
||||
return this.dedupePromptParts(parts).join(', ');
|
||||
}
|
||||
|
||||
private normalizeArabicText(value: string): string {
|
||||
return value
|
||||
.replace(/[أإآ]/g, 'ا')
|
||||
.replace(/ى/g, 'ي')
|
||||
.replace(/ة/g, 'ه')
|
||||
.replace(/[ًٌٍَُِّْـ]/g, '');
|
||||
}
|
||||
|
||||
private resolveMappedDescription(
|
||||
explicitValue: string | undefined,
|
||||
normalizedPrompt: string,
|
||||
mappings: Array<{ terms: string[]; description: string }>,
|
||||
): string | null {
|
||||
return this.resolveMappedDescriptions(explicitValue ? [explicitValue] : [], normalizedPrompt, mappings)[0] ?? null;
|
||||
}
|
||||
|
||||
private resolveMappedDescriptions(
|
||||
explicitValues: string[],
|
||||
normalizedPrompt: string,
|
||||
mappings: Array<{ terms: string[]; description: string }>,
|
||||
): string[] {
|
||||
const values = new Set<string>();
|
||||
const normalizedExplicitValues = explicitValues.map((value) =>
|
||||
this.normalizeArabicText(value).toLowerCase(),
|
||||
);
|
||||
|
||||
for (const mapping of mappings) {
|
||||
const normalizedTerms = mapping.terms.map((term) =>
|
||||
this.normalizeArabicText(term).toLowerCase(),
|
||||
);
|
||||
if (
|
||||
normalizedTerms.some(
|
||||
(term) =>
|
||||
normalizedPrompt.includes(term) ||
|
||||
normalizedExplicitValues.some((value) => value.includes(term)),
|
||||
)
|
||||
) {
|
||||
values.add(mapping.description);
|
||||
}
|
||||
}
|
||||
|
||||
for (const value of explicitValues) {
|
||||
const trimmed = value.trim();
|
||||
if (trimmed && ![...values].some((mappedValue) => mappedValue.toLowerCase().includes(trimmed.toLowerCase()))) {
|
||||
values.add(trimmed);
|
||||
}
|
||||
}
|
||||
|
||||
return [...values];
|
||||
}
|
||||
|
||||
private inferTempo(normalizedPrompt: string, moods: string[]): 'slow' | 'medium' | 'fast' {
|
||||
if (
|
||||
normalizedPrompt.includes('سريع') ||
|
||||
normalizedPrompt.includes('fast') ||
|
||||
moods.some((mood) => mood.includes('energetic'))
|
||||
) {
|
||||
return 'fast';
|
||||
}
|
||||
if (
|
||||
normalizedPrompt.includes('بطي') ||
|
||||
normalizedPrompt.includes('هاد') ||
|
||||
normalizedPrompt.includes('slow') ||
|
||||
moods.some((mood) => mood.includes('sad') || mood.includes('calm'))
|
||||
) {
|
||||
return 'slow';
|
||||
}
|
||||
return 'medium';
|
||||
}
|
||||
|
||||
private inferPercussion(
|
||||
normalizedPrompt: string,
|
||||
instruments: string[],
|
||||
): 'none' | 'light' | 'medium' {
|
||||
if (normalizedPrompt.includes('بدون ايقاع') || normalizedPrompt.includes('no percussion')) {
|
||||
return 'none';
|
||||
}
|
||||
if (
|
||||
normalizedPrompt.includes('خفيف') ||
|
||||
normalizedPrompt.includes('light') ||
|
||||
instruments.includes('riq frame drum')
|
||||
) {
|
||||
return 'light';
|
||||
}
|
||||
if (
|
||||
normalizedPrompt.includes('ايقاع') ||
|
||||
normalizedPrompt.includes('دربكه') ||
|
||||
normalizedPrompt.includes('طبله') ||
|
||||
normalizedPrompt.includes('percussion') ||
|
||||
instruments.includes('darbuka')
|
||||
) {
|
||||
return 'medium';
|
||||
}
|
||||
return 'none';
|
||||
}
|
||||
|
||||
private describePercussion(
|
||||
percussion: 'light' | 'medium',
|
||||
instruments: string[],
|
||||
isArabicPrompt: boolean,
|
||||
): string {
|
||||
const hasDarbuka = instruments.includes('darbuka');
|
||||
const hasRiq = instruments.includes('riq frame drum');
|
||||
if (percussion === 'light') {
|
||||
if (hasDarbuka) {
|
||||
return 'light darbuka percussion';
|
||||
}
|
||||
if (hasRiq || isArabicPrompt) {
|
||||
return 'light riq percussion';
|
||||
}
|
||||
return 'light percussion';
|
||||
}
|
||||
|
||||
if (hasRiq && !hasDarbuka) {
|
||||
return 'medium riq percussion';
|
||||
}
|
||||
return isArabicPrompt || hasDarbuka ? 'medium darbuka percussion' : 'medium percussion';
|
||||
}
|
||||
|
||||
private dedupePromptParts(parts: string[]): string[] {
|
||||
const seen = new Set<string>();
|
||||
const cleanParts: string[] = [];
|
||||
for (const part of parts) {
|
||||
const clean = part.trim().replace(/\s+/g, ' ');
|
||||
const key = clean.toLowerCase();
|
||||
if (clean && !seen.has(key)) {
|
||||
cleanParts.push(clean);
|
||||
seen.add(key);
|
||||
}
|
||||
}
|
||||
return cleanParts;
|
||||
}
|
||||
|
||||
private resolveGoogleApplicationCredentials(): {
|
||||
credentials?: any;
|
||||
} {
|
||||
const encodedCredentials = (
|
||||
this.configService.get<string>('aiMusic.googleApplicationCredentialsJsonBase64', {
|
||||
infer: true,
|
||||
}) ?? ''
|
||||
).trim();
|
||||
if (!encodedCredentials) {
|
||||
return {};
|
||||
}
|
||||
|
||||
try {
|
||||
const decoded = Buffer.from(encodedCredentials, 'base64').toString('utf8');
|
||||
const credentials = JSON.parse(decoded) as Record<string, unknown>;
|
||||
if (!credentials || typeof credentials !== 'object') {
|
||||
throw new Error('decoded credentials are not a JSON object');
|
||||
}
|
||||
if (typeof credentials.client_email !== 'string' || !credentials.client_email) {
|
||||
throw new Error('missing client_email');
|
||||
}
|
||||
if (typeof credentials.private_key !== 'string' || !credentials.private_key) {
|
||||
throw new Error('missing private_key');
|
||||
}
|
||||
|
||||
return { credentials };
|
||||
} catch (error) {
|
||||
throw new ServiceUnavailableException(
|
||||
`Invalid GOOGLE_APPLICATION_CREDENTIALS_JSON_BASE64: ${
|
||||
error instanceof Error ? error.message : 'unable to decode credentials'
|
||||
}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
المرجع في مشكلة جديدة
حظر مستخدم