diff --git a/.env.example b/.env.example index da05707..032c9d9 100644 --- a/.env.example +++ b/.env.example @@ -96,9 +96,16 @@ EMAIL_SMTP_USER= EMAIL_SMTP_PASS= EMAIL_FROM_NAME=Oudelaa EMAIL_FROM_EMAIL= +# Enables POST /api/v1/media/ai/text-to-music. +# Uses Google Vertex AI publisher model endpoint, not GEMINI_API_KEY directly. +# Auth priority: +# 1. AI_MUSIC_API_KEY when set. +# 2. GOOGLE_APPLICATION_CREDENTIALS_JSON_BASE64 when set. +# 3. GOOGLE_APPLICATION_CREDENTIALS file path handled by google-auth-library. AI_MUSIC_ENABLED=false AI_MUSIC_API_KEY= -AI_MUSIC_PROJECT_ID= +GOOGLE_APPLICATION_CREDENTIALS_JSON_BASE64= +AI_MUSIC_PROJECT_ID=accordev AI_MUSIC_LOCATION=us-central1 AI_MUSIC_MODEL=lyria-002 diff --git a/postman/Oudelaa-Auth-Users-Posts.postman_collection.json b/postman/Oudelaa-Auth-Users-Posts.postman_collection.json index b94e0ae..522cde8 100644 --- a/postman/Oudelaa-Auth-Users-Posts.postman_collection.json +++ b/postman/Oudelaa-Auth-Users-Posts.postman_collection.json @@ -1591,6 +1591,56 @@ } ] }, + { + "name": "Get My Artist Dashboard", + "request": { + "method": "GET", + "header": [ + { + "key": "Authorization", + "value": "Bearer {{accessToken}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": "{{baseUrl}}/users/me/dashboard", + "description": "Private Flutter artist dashboard endpoint.\n\nReturns profile, aggregate post stats, audience summary, top content, recent activity, and availability flags.\n\nDoes not accept a userId and only uses the authenticated user token." + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('Status is 200', function () { pm.response.to.have.status(200); });", + "const json = pm.response.json();", + "pm.test('Dashboard response shape is Flutter-ready', function () {", + " pm.expect(json).to.have.property('profile');", + " pm.expect(json.profile).to.have.property('_id');", + " pm.expect(json.profile).to.have.property('coverImage');", + " pm.expect(json.profile).to.have.property('avatar');", + " pm.expect(json.profile).to.have.property('name');", + " pm.expect(json.profile).to.have.property('stageName');", + " pm.expect(json.profile).to.have.property('bio');", + " pm.expect(json.profile).to.have.property('isVerified');", + " pm.expect(json).to.have.property('stats');", + " ['followersCount','followingCount','postsCount','collaborationsCount','viewCount','playCount','listenCount','likesCount','commentsCount','savesCount','sharesCount','engagementRate','scorePercentage','earnings'].forEach((key) => pm.expect(json.stats).to.have.property(key));", + " pm.expect(json).to.have.property('audience');", + " ['followers','nonFollowers','newFollowersThisWeek','newFollowersThisMonth'].forEach((key) => pm.expect(json.audience).to.have.property(key));", + " pm.expect(json).to.have.property('engagementChart').that.is.an('array');", + " pm.expect(json).to.have.property('topContent').that.is.an('array');", + " pm.expect(json).to.have.property('recentActivity').that.is.an('array');", + " pm.expect(json).to.have.property('meta');", + " ['generatedAt','chartAvailable','earningsAvailable','audienceAnalyticsAvailable'].forEach((key) => pm.expect(json.meta).to.have.property(key));", + "});", + "pm.environment.set('artistDashboardProfileId', json.profile._id);" + ] + } + } + ] + }, { "name": "Admin Discover Users", "request": { @@ -10176,4 +10226,4 @@ "value": "{{accessToken}}" } ] -} +} \ No newline at end of file diff --git a/postman/Oudelaa-Dashboard.postman_collection.json b/postman/Oudelaa-Dashboard.postman_collection.json index 5743050..d264a3d 100644 --- a/postman/Oudelaa-Dashboard.postman_collection.json +++ b/postman/Oudelaa-Dashboard.postman_collection.json @@ -975,6 +975,56 @@ ] } } + }, + { + "name": "Get My Artist Dashboard", + "request": { + "method": "GET", + "header": [ + { + "key": "Authorization", + "value": "Bearer {{accessToken}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": "{{baseUrl}}/users/me/dashboard", + "description": "Private Flutter artist dashboard endpoint.\n\nReturns profile, aggregate post stats, audience summary, top content, recent activity, and availability flags.\n\nDoes not accept a userId and only uses the authenticated user token." + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('Status is 200', function () { pm.response.to.have.status(200); });", + "const json = pm.response.json();", + "pm.test('Dashboard response shape is Flutter-ready', function () {", + " pm.expect(json).to.have.property('profile');", + " pm.expect(json.profile).to.have.property('_id');", + " pm.expect(json.profile).to.have.property('coverImage');", + " pm.expect(json.profile).to.have.property('avatar');", + " pm.expect(json.profile).to.have.property('name');", + " pm.expect(json.profile).to.have.property('stageName');", + " pm.expect(json.profile).to.have.property('bio');", + " pm.expect(json.profile).to.have.property('isVerified');", + " pm.expect(json).to.have.property('stats');", + " ['followersCount','followingCount','postsCount','collaborationsCount','viewCount','playCount','listenCount','likesCount','commentsCount','savesCount','sharesCount','engagementRate','scorePercentage','earnings'].forEach((key) => pm.expect(json.stats).to.have.property(key));", + " pm.expect(json).to.have.property('audience');", + " ['followers','nonFollowers','newFollowersThisWeek','newFollowersThisMonth'].forEach((key) => pm.expect(json.audience).to.have.property(key));", + " pm.expect(json).to.have.property('engagementChart').that.is.an('array');", + " pm.expect(json).to.have.property('topContent').that.is.an('array');", + " pm.expect(json).to.have.property('recentActivity').that.is.an('array');", + " pm.expect(json).to.have.property('meta');", + " ['generatedAt','chartAvailable','earningsAvailable','audienceAnalyticsAvailable'].forEach((key) => pm.expect(json.meta).to.have.property(key));", + "});", + "pm.environment.set('artistDashboardProfileId', json.profile._id);" + ] + } + } + ] } ] }, @@ -6346,4 +6396,4 @@ "value": "{{accessToken}}" } ] -} +} \ No newline at end of file diff --git a/postman/Oudelaa-Mobile.postman_collection.json b/postman/Oudelaa-Mobile.postman_collection.json index 5159f65..4341660 100644 --- a/postman/Oudelaa-Mobile.postman_collection.json +++ b/postman/Oudelaa-Mobile.postman_collection.json @@ -1113,6 +1113,56 @@ } ] }, + { + "name": "Get My Artist Dashboard", + "request": { + "method": "GET", + "header": [ + { + "key": "Authorization", + "value": "Bearer {{accessToken}}" + }, + { + "key": "Content-Type", + "value": "application/json" + } + ], + "url": "{{baseUrl}}/users/me/dashboard", + "description": "Private Flutter artist dashboard endpoint.\n\nReturns profile, aggregate post stats, audience summary, top content, recent activity, and availability flags.\n\nDoes not accept a userId and only uses the authenticated user token." + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('Status is 200', function () { pm.response.to.have.status(200); });", + "const json = pm.response.json();", + "pm.test('Dashboard response shape is Flutter-ready', function () {", + " pm.expect(json).to.have.property('profile');", + " pm.expect(json.profile).to.have.property('_id');", + " pm.expect(json.profile).to.have.property('coverImage');", + " pm.expect(json.profile).to.have.property('avatar');", + " pm.expect(json.profile).to.have.property('name');", + " pm.expect(json.profile).to.have.property('stageName');", + " pm.expect(json.profile).to.have.property('bio');", + " pm.expect(json.profile).to.have.property('isVerified');", + " pm.expect(json).to.have.property('stats');", + " ['followersCount','followingCount','postsCount','collaborationsCount','viewCount','playCount','listenCount','likesCount','commentsCount','savesCount','sharesCount','engagementRate','scorePercentage','earnings'].forEach((key) => pm.expect(json.stats).to.have.property(key));", + " pm.expect(json).to.have.property('audience');", + " ['followers','nonFollowers','newFollowersThisWeek','newFollowersThisMonth'].forEach((key) => pm.expect(json.audience).to.have.property(key));", + " pm.expect(json).to.have.property('engagementChart').that.is.an('array');", + " pm.expect(json).to.have.property('topContent').that.is.an('array');", + " pm.expect(json).to.have.property('recentActivity').that.is.an('array');", + " pm.expect(json).to.have.property('meta');", + " ['generatedAt','chartAvailable','earningsAvailable','audienceAnalyticsAvailable'].forEach((key) => pm.expect(json.meta).to.have.property(key));", + "});", + "pm.environment.set('artistDashboardProfileId', json.profile._id);" + ] + } + } + ] + }, { "name": "Get User Presence", "request": { @@ -7392,4 +7442,4 @@ "value": "{{accessToken}}" } ] -} +} \ No newline at end of file diff --git a/src/common/utils/waveform.util.spec.ts b/src/common/utils/waveform.util.spec.ts index 8fa90d6..ec04329 100644 --- a/src/common/utils/waveform.util.spec.ts +++ b/src/common/utils/waveform.util.spec.ts @@ -10,6 +10,38 @@ const expectDisplaySafeWaveform = (peaks: number[], expectedLength = 100) => { expect(peaks.every((value) => value >= 0 && value <= 100)).toBe(true); }; +const createPcmWavBuffer = (amplitudeByFrame: (frame: number) => number, frames = 8000) => { + const sampleRate = 8000; + const channels = 1; + const bitsPerSample = 16; + const bytesPerSample = bitsPerSample / 8; + const dataSize = frames * channels * bytesPerSample; + const buffer = Buffer.alloc(44 + dataSize); + + buffer.write('RIFF', 0, 'ascii'); + buffer.writeUInt32LE(36 + dataSize, 4); + buffer.write('WAVE', 8, 'ascii'); + buffer.write('fmt ', 12, 'ascii'); + buffer.writeUInt32LE(16, 16); + buffer.writeUInt16LE(1, 20); + buffer.writeUInt16LE(channels, 22); + buffer.writeUInt32LE(sampleRate, 24); + buffer.writeUInt32LE(sampleRate * channels * bytesPerSample, 28); + buffer.writeUInt16LE(channels * bytesPerSample, 32); + buffer.writeUInt16LE(bitsPerSample, 34); + buffer.write('data', 36, 'ascii'); + buffer.writeUInt32LE(dataSize, 40); + + for (let frame = 0; frame < frames; frame += 1) { + const tone = Math.sin((frame / sampleRate) * Math.PI * 2 * 440); + const amplitude = amplitudeByFrame(frame); + const sample = Math.max(-32768, Math.min(32767, Math.round(tone * amplitude * 32767))); + buffer.writeInt16LE(sample, 44 + frame * bytesPerSample); + } + + return buffer; +}; + describe('waveform util', () => { const originalAudioWaveformPeaks = process.env.AUDIO_WAVEFORM_PEAKS; @@ -67,4 +99,14 @@ describe('waveform util', () => { expectDisplaySafeWaveform(generateWaveformPeaksFromBuffer(Buffer.from('fake audio bytes'))); expectDisplaySafeWaveform(generateWaveformPeaksFromSeed('audio-url')); }); + + it('generates varied normalized peaks from actual WAV PCM samples', () => { + const wavBuffer = createPcmWavBuffer((frame) => 0.12 + (frame / 7999) * 0.78); + const peaks = generateWaveformPeaksFromBuffer(wavBuffer); + + expectDisplaySafeWaveform(peaks); + expect(new Set(peaks).size).toBeGreaterThan(8); + expect(peaks.slice(0, 40).some((value) => value < 60)).toBe(true); + expect(peaks.slice(60).some((value) => value > 80)).toBe(true); + }); }); diff --git a/src/common/utils/waveform.util.ts b/src/common/utils/waveform.util.ts index 448f332..e48000a 100644 --- a/src/common/utils/waveform.util.ts +++ b/src/common/utils/waveform.util.ts @@ -29,12 +29,125 @@ const scaleToRange = (values: number[]): number[] => { return []; } - const max = Math.max(...values); + const sorted = [...values].sort((left, right) => left - right); + const max = sorted[sorted.length - 1] ?? 0; if (max <= 0) { return values.map((value) => clampPeak(value)); } - return values.map((value) => clampPeak((value / max) * 100)); + const percentileIndex = Math.max(0, Math.min(sorted.length - 1, Math.floor(sorted.length * 0.95))); + const scaleMax = Math.max(sorted[percentileIndex] ?? max, 1); + + return values.map((value) => clampPeak((Math.min(value, scaleMax) / scaleMax) * 100)); +}; + +const readPcmSample = ( + buffer: Buffer, + offset: number, + bitsPerSample: number, + audioFormat: number, +): number => { + if (audioFormat === 3) { + if (bitsPerSample === 32 && offset + 4 <= buffer.length) { + return Math.max(-1, Math.min(1, buffer.readFloatLE(offset))); + } + if (bitsPerSample === 64 && offset + 8 <= buffer.length) { + return Math.max(-1, Math.min(1, buffer.readDoubleLE(offset))); + } + return 0; + } + + switch (bitsPerSample) { + case 8: + return (buffer.readUInt8(offset) - 128) / 128; + case 16: + return buffer.readInt16LE(offset) / 32768; + case 24: { + if (offset + 3 > buffer.length) { + return 0; + } + let value = buffer.readUIntLE(offset, 3); + if (value & 0x800000) { + value |= 0xff000000; + } + return value / 8388608; + } + case 32: + return buffer.readInt32LE(offset) / 2147483648; + default: + return 0; + } +}; + +const generateWaveformPeaksFromWavBuffer = (buffer: Buffer, samples: number): number[] | null => { + if (buffer.length < 44 || buffer.toString('ascii', 0, 4) !== 'RIFF' || buffer.toString('ascii', 8, 12) !== 'WAVE') { + return null; + } + + let offset = 12; + let audioFormat = 0; + let channels = 0; + let bitsPerSample = 0; + let dataOffset = 0; + let dataSize = 0; + + while (offset + 8 <= buffer.length) { + const chunkId = buffer.toString('ascii', offset, offset + 4); + const chunkSize = buffer.readUInt32LE(offset + 4); + const chunkDataOffset = offset + 8; + + if (chunkId === 'fmt ' && chunkDataOffset + 16 <= buffer.length) { + audioFormat = buffer.readUInt16LE(chunkDataOffset); + channels = buffer.readUInt16LE(chunkDataOffset + 2); + bitsPerSample = buffer.readUInt16LE(chunkDataOffset + 14); + } else if (chunkId === 'data') { + dataOffset = chunkDataOffset; + dataSize = Math.min(chunkSize, buffer.length - chunkDataOffset); + } + + offset = chunkDataOffset + chunkSize + (chunkSize % 2); + } + + const bytesPerSample = bitsPerSample / 8; + const frameSize = bytesPerSample * channels; + if ( + !dataOffset || + !dataSize || + !channels || + !Number.isInteger(bytesPerSample) || + frameSize <= 0 || + (audioFormat !== 1 && audioFormat !== 3) + ) { + return null; + } + + const frameCount = Math.floor(dataSize / frameSize); + if (frameCount <= 0) { + return null; + } + + const bucketSize = Math.max(1, Math.ceil(frameCount / samples)); + const peaks: number[] = []; + + for (let bucketStart = 0; bucketStart < frameCount; bucketStart += bucketSize) { + const bucketEnd = Math.min(frameCount, bucketStart + bucketSize); + let sumSquares = 0; + let count = 0; + + for (let frameIndex = bucketStart; frameIndex < bucketEnd; frameIndex += 1) { + const frameOffset = dataOffset + frameIndex * frameSize; + for (let channel = 0; channel < channels; channel += 1) { + const sampleOffset = frameOffset + channel * bytesPerSample; + const value = readPcmSample(buffer, sampleOffset, bitsPerSample, audioFormat); + sumSquares += value * value; + count += 1; + } + } + + peaks.push(count ? Math.sqrt(sumSquares / count) * 100 : 0); + } + + return normalizeWaveformPeaks(peaks, samples); }; export const normalizeWaveformPeaks = ( @@ -93,6 +206,11 @@ export const generateWaveformPeaksFromBuffer = ( return normalizeWaveformPeaks([], samples); } + const wavPeaks = generateWaveformPeaksFromWavBuffer(buffer, samples); + if (wavPeaks) { + return wavPeaks; + } + const chunkSize = Math.max(1, Math.ceil(buffer.length / samples)); const peaks: number[] = []; diff --git a/src/config/configuration.ts b/src/config/configuration.ts index f28e52a..f7c069c 100644 --- a/src/config/configuration.ts +++ b/src/config/configuration.ts @@ -49,6 +49,8 @@ export default () => ({ aiMusic: { enabled: (process.env.AI_MUSIC_ENABLED ?? 'false').toLowerCase() === 'true', apiKey: process.env.AI_MUSIC_API_KEY ?? '', + googleApplicationCredentialsJsonBase64: + process.env.GOOGLE_APPLICATION_CREDENTIALS_JSON_BASE64 ?? '', projectId: process.env.AI_MUSIC_PROJECT_ID ?? '', location: process.env.AI_MUSIC_LOCATION ?? 'us-central1', model: process.env.AI_MUSIC_MODEL ?? 'lyria-002', diff --git a/src/config/validation.schema.ts b/src/config/validation.schema.ts index 95974bb..9cb33ea 100644 --- a/src/config/validation.schema.ts +++ b/src/config/validation.schema.ts @@ -32,6 +32,7 @@ export const validationSchema = Joi.object({ EMAIL_FROM_EMAIL: Joi.string().allow('').optional(), AI_MUSIC_ENABLED: Joi.boolean().truthy('true').falsy('false').default(false), AI_MUSIC_API_KEY: Joi.string().allow('').optional(), + GOOGLE_APPLICATION_CREDENTIALS_JSON_BASE64: Joi.string().allow('').optional(), AI_MUSIC_PROJECT_ID: Joi.string().allow('').optional(), AI_MUSIC_LOCATION: Joi.string().default('us-central1'), AI_MUSIC_MODEL: Joi.string().default('lyria-002'), diff --git a/src/modules/media/dto/text-to-music.dto.ts b/src/modules/media/dto/text-to-music.dto.ts index 5d65051..1e82f0d 100644 --- a/src/modules/media/dto/text-to-music.dto.ts +++ b/src/modules/media/dto/text-to-music.dto.ts @@ -1,5 +1,15 @@ import { Type } from 'class-transformer'; -import { IsInt, IsNotEmpty, IsOptional, IsString, MaxLength, Max, Min } from 'class-validator'; +import { + IsArray, + IsIn, + IsInt, + IsNotEmpty, + IsOptional, + IsString, + MaxLength, + Max, + Min, +} from 'class-validator'; export class TextToMusicDto { @IsString() @@ -20,4 +30,33 @@ export class TextToMusicDto { @Min(0) @Max(2147483647) seed?: number; + + @IsOptional() + @IsString() + @MaxLength(80) + maqam?: string; + + @IsOptional() + @IsString() + @MaxLength(80) + mood?: string; + + @IsOptional() + @IsArray() + @IsString({ each: true }) + @MaxLength(80, { each: true }) + instruments?: string[]; + + @IsOptional() + @IsString() + @MaxLength(120) + style?: string; + + @IsOptional() + @IsIn(['slow', 'medium', 'fast']) + tempo?: 'slow' | 'medium' | 'fast'; + + @IsOptional() + @IsIn(['none', 'light', 'medium']) + percussion?: 'none' | 'light' | 'medium'; } diff --git a/src/modules/media/media.service.spec.ts b/src/modules/media/media.service.spec.ts new file mode 100644 index 0000000..501b777 --- /dev/null +++ b/src/modules/media/media.service.spec.ts @@ -0,0 +1,222 @@ +import { ServiceUnavailableException } from '@nestjs/common'; +import { MediaService } from './media.service'; + +const createPcmWavBuffer = (frames = 8000) => { + const sampleRate = 8000; + const channels = 1; + const bitsPerSample = 16; + const bytesPerSample = bitsPerSample / 8; + const dataSize = frames * channels * bytesPerSample; + const buffer = Buffer.alloc(44 + dataSize); + + buffer.write('RIFF', 0, 'ascii'); + buffer.writeUInt32LE(36 + dataSize, 4); + buffer.write('WAVE', 8, 'ascii'); + buffer.write('fmt ', 12, 'ascii'); + buffer.writeUInt32LE(16, 16); + buffer.writeUInt16LE(1, 20); + buffer.writeUInt16LE(channels, 22); + buffer.writeUInt32LE(sampleRate, 24); + buffer.writeUInt32LE(sampleRate * channels * bytesPerSample, 28); + buffer.writeUInt16LE(channels * bytesPerSample, 32); + buffer.writeUInt16LE(bitsPerSample, 34); + buffer.write('data', 36, 'ascii'); + buffer.writeUInt32LE(dataSize, 40); + + for (let frame = 0; frame < frames; frame += 1) { + const amplitude = frame < frames / 2 ? 0.2 : 0.9; + const tone = Math.sin((frame / sampleRate) * Math.PI * 2 * 440); + buffer.writeInt16LE(Math.round(tone * amplitude * 32767), 44 + frame * bytesPerSample); + } + + return buffer; +}; + +describe('MediaService Google credentials resolution', () => { + const createService = ( + googleCredentialsBase64 = '', + overrides: { + config?: Record; + storageService?: Record; + mediaProbeService?: Record; + } = {}, + ) => { + const configService = { + get: jest.fn((key: string) => { + if (key === 'aiMusic.googleApplicationCredentialsJsonBase64') { + return googleCredentialsBase64; + } + if (key in (overrides.config ?? {})) { + return overrides.config?.[key]; + } + return undefined; + }), + }; + + return new MediaService( + configService as any, + (overrides.storageService ?? {}) as any, + (overrides.mediaProbeService ?? {}) as any, + ) as any; + }; + + it('falls back to default Google application credentials when base64 JSON is missing', () => { + const service = createService(); + + expect(service.resolveGoogleApplicationCredentials()).toEqual({}); + }); + + it('decodes service account credentials from base64 JSON', () => { + const credentials = { + type: 'service_account', + client_email: 'ai-music@example.iam.gserviceaccount.com', + private_key: '-----BEGIN PRIVATE KEY-----\nredacted\n-----END PRIVATE KEY-----\n', + }; + const encoded = Buffer.from(JSON.stringify(credentials), 'utf8').toString('base64'); + const service = createService(encoded); + + expect(service.resolveGoogleApplicationCredentials()).toEqual({ credentials }); + }); + + it('throws a safe configuration error for invalid base64 credentials', () => { + const service = createService(Buffer.from('not-json', 'utf8').toString('base64')); + + expect(() => service.resolveGoogleApplicationCredentials()).toThrow( + ServiceUnavailableException, + ); + expect(() => service.resolveGoogleApplicationCredentials()).toThrow( + 'Invalid GOOGLE_APPLICATION_CREDENTIALS_JSON_BASE64', + ); + }); + + it('returns the probed audio duration instead of the requested duration', async () => { + const audioBuffer = createPcmWavBuffer(); + const originalFetch = global.fetch; + global.fetch = jest.fn().mockResolvedValue({ + ok: true, + json: jest.fn().mockResolvedValue({ + predictions: [ + { + bytesBase64Encoded: audioBuffer.toString('base64'), + mimeType: 'audio/wav', + }, + ], + }), + }) as any; + const storageService = { + saveFile: jest.fn().mockResolvedValue('/uploads/ai-music/generated.wav'), + }; + const mediaProbeService = { + extractDurationSecondsFromBuffer: jest.fn().mockResolvedValue(32), + }; + const service = createService('', { + config: { + 'aiMusic.enabled': true, + 'aiMusic.apiKey': 'test-api-key', + 'aiMusic.projectId': 'accordev', + 'aiMusic.location': 'us-central1', + 'aiMusic.model': 'lyria-002', + }, + storageService, + mediaProbeService, + }); + + try { + const result = await service.generateMusicFromText('user-1', { + prompt: 'test prompt', + durationSeconds: 12, + }); + + expect(result.durationSeconds).toBe(32); + expect(result.prompt).toBe('test prompt'); + expect(result.originalPrompt).toBe('test prompt'); + expect(result.enhancedPrompt).toContain('Instrumental only, no vocals'); + expect(result.audioUrl).toBe('/uploads/ai-music/generated.wav'); + expect(result.mimeType).toBe('audio/wav'); + expect(result.sizeBytes).toBe(audioBuffer.length); + expect(result.waveformPeaks).toHaveLength(100); + expect(mediaProbeService.extractDurationSecondsFromBuffer).toHaveBeenCalledWith( + audioBuffer, + { + originalname: 'ai-music.wav', + mimetype: 'audio/wav', + }, + ); + } finally { + global.fetch = originalFetch; + } + }); + + it('enhances Arabic Middle Eastern music prompts before sending them to Lyria', async () => { + const audioBuffer = createPcmWavBuffer(); + const originalFetch = global.fetch; + global.fetch = jest.fn().mockResolvedValue({ + ok: true, + json: jest.fn().mockResolvedValue({ + predictions: [ + { + bytesBase64Encoded: audioBuffer.toString('base64'), + mimeType: 'audio/wav', + }, + ], + }), + }) as any; + const service = createService('', { + config: { + 'aiMusic.enabled': true, + 'aiMusic.apiKey': 'test-api-key', + 'aiMusic.projectId': 'accordev', + 'aiMusic.location': 'us-central1', + 'aiMusic.model': 'lyria-002', + }, + storageService: { + saveFile: jest.fn().mockResolvedValue('/uploads/ai-music/generated.wav'), + }, + mediaProbeService: { + extractDurationSecondsFromBuffer: jest.fn().mockResolvedValue(12), + }, + }); + + try { + const result = await service.generateMusicFromText('user-1', { + prompt: 'بدي لحن عود حزين مقام حجاز مع إيقاع خفيف', + durationSeconds: 12, + }); + const fetchBody = JSON.parse((global.fetch as jest.Mock).mock.calls[0][1].body); + const sentPrompt = fetchBody.instances[0].prompt as string; + + expect(result.prompt).toBe('بدي لحن عود حزين مقام حجاز مع إيقاع خفيف'); + expect(result.originalPrompt).toBe('بدي لحن عود حزين مقام حجاز مع إيقاع خفيف'); + expect(result.enhancedPrompt).toBe(sentPrompt); + expect(sentPrompt).toContain('Instrumental Arabic oud melody'); + expect(sentPrompt).toContain('Hijaz maqam, emotional Middle Eastern sound'); + expect(sentPrompt).toContain('sad, emotional, expressive mood'); + expect(sentPrompt).toContain('light riq percussion'); + expect(sentPrompt).toContain('slow tempo'); + expect(sentPrompt).toContain( + 'Instrumental only, no vocals, no lyrics, no artist imitation, clean studio recording, balanced mix.', + ); + } finally { + global.fetch = originalFetch; + } + }); + + it('uses optional maqam mood instruments and percussion fields in the enhanced prompt', () => { + const service = createService(); + const enhancedPrompt = service.buildEnhancedMusicPrompt({ + prompt: 'warm intro', + maqam: 'Nahawand', + mood: 'romantic', + instruments: ['qanun', 'ney'], + style: 'cinematic Levantine arrangement', + tempo: 'medium', + percussion: 'light', + }); + + expect(enhancedPrompt).toContain('Nahawand maqam, dramatic minor Arabic mood'); + expect(enhancedPrompt).toContain('romantic, warm mood'); + expect(enhancedPrompt).toContain('Instrumental Arabic qanun melody'); + expect(enhancedPrompt).toContain('light percussion'); + expect(enhancedPrompt).toContain('cinematic Levantine arrangement'); + }); +}); diff --git a/src/modules/media/media.service.ts b/src/modules/media/media.service.ts index 289051f..756e9be 100644 --- a/src/modules/media/media.service.ts +++ b/src/modules/media/media.service.ts @@ -6,6 +6,36 @@ import { ManagedStorageService } from '../../infrastructure/storage/managed-stor import { MediaProbeService } from '../../infrastructure/storage/media-probe.service'; import { TextToMusicDto } from './dto/text-to-music.dto'; +const AI_MUSIC_SAFE_PRODUCTION_INSTRUCTIONS = + 'Instrumental only, no vocals, no lyrics, no artist imitation, clean studio recording, balanced mix.'; + +const ARABIC_MAQAM_MAPPINGS: Array<{ terms: string[]; description: string }> = [ + { terms: ['حجاز', 'hijaz'], description: 'Hijaz maqam, emotional Middle Eastern sound' }, + { terms: ['بياتي', 'bayati'], description: 'Bayati maqam, warm Levantine folk mood' }, + { terms: ['راست', 'rast'], description: 'Rast maqam, classical Arabic optimistic character' }, + { terms: ['نهاوند', 'nahawand'], description: 'Nahawand maqam, dramatic minor Arabic mood' }, + { terms: ['كرد', 'kurd'], description: 'Kurd maqam, simple minor Arabic mood' }, + { terms: ['صبا', 'saba'], description: 'Saba maqam, deeply sad Arabic mood' }, +]; + +const ARABIC_INSTRUMENT_MAPPINGS: Array<{ terms: string[]; description: string }> = [ + { terms: ['عود', 'oud'], description: 'oud' }, + { terms: ['قانون', 'qanun'], description: 'qanun' }, + { terms: ['ناي', 'ney'], description: 'ney flute' }, + { terms: ['كمان', 'violin'], description: 'violin' }, + { terms: ['رق', 'riq'], description: 'riq frame drum' }, + { terms: ['طبلة', 'دربكة', 'darbuka'], description: 'darbuka' }, +]; + +const ARABIC_MOOD_MAPPINGS: Array<{ terms: string[]; description: string }> = [ + { terms: ['حزين', 'sad'], description: 'sad, emotional, expressive' }, + { terms: ['هادئ', 'هادي', 'calm'], description: 'calm, soft, slow' }, + { terms: ['حماسي', 'energetic'], description: 'energetic, heroic' }, + { terms: ['رومانسي', 'romantic'], description: 'romantic, warm' }, + { terms: ['صوفي', 'spiritual'], description: 'spiritual, meditative' }, + { terms: ['تراثي', 'traditional'], description: 'traditional Arabic folk' }, +]; + @Injectable() export class MediaService { constructor( @@ -123,21 +153,34 @@ export class MediaService { } else { const auth = new GoogleAuth({ scopes: ['https://www.googleapis.com/auth/cloud-platform'], + ...this.resolveGoogleApplicationCredentials(), }); - const client = await auth.getClient(); - const accessTokenRaw = await client.getAccessToken(); - const accessToken = - typeof accessTokenRaw === 'string' ? accessTokenRaw : (accessTokenRaw?.token ?? ''); + let accessToken = ''; + try { + const client = await auth.getClient(); + const accessTokenRaw = await client.getAccessToken(); + accessToken = + typeof accessTokenRaw === 'string' ? accessTokenRaw : (accessTokenRaw?.token ?? ''); + } catch (error) { + throw new ServiceUnavailableException( + `Failed to authenticate with Google Cloud: ${ + error instanceof Error ? error.message : 'invalid credentials' + }`, + ); + } if (!accessToken) { - throw new ServiceUnavailableException('Failed to authenticate with Google Cloud'); + throw new ServiceUnavailableException( + 'Failed to authenticate with Google Cloud: access token was not returned', + ); } authorizationHeader = `Bearer ${accessToken}`; } + const enhancedPrompt = this.buildEnhancedMusicPrompt(dto); const requestBody: Record = { - instances: [{ prompt: dto.prompt }], + instances: [{ prompt: enhancedPrompt }], parameters: { sampleCount: 1, durationSeconds: dto.durationSeconds ?? 12, @@ -189,10 +232,19 @@ export class MediaService { contentType: mimeType, fileNamePrefix: `ai-${userId}`, }); + const actualDurationSeconds = await this.mediaProbeService.extractDurationSecondsFromBuffer( + buffer, + { + originalname: `ai-music.${extension}`, + mimetype: mimeType, + }, + ); return { prompt: dto.prompt, - durationSeconds: dto.durationSeconds ?? 12, + originalPrompt: dto.prompt, + enhancedPrompt, + durationSeconds: actualDurationSeconds ?? dto.durationSeconds ?? 12, mimeType, sizeBytes: buffer.length, audioUrl, @@ -215,4 +267,233 @@ export class MediaService { } return 'wav'; } + + private buildEnhancedMusicPrompt(dto: TextToMusicDto): string { + const originalPrompt = dto.prompt.trim(); + const normalizedPrompt = this.normalizeArabicText(originalPrompt).toLowerCase(); + const isArabicPrompt = /[\u0600-\u06ff]/.test(originalPrompt); + + const maqam = this.resolveMappedDescription( + dto.maqam, + normalizedPrompt, + ARABIC_MAQAM_MAPPINGS, + ); + const moods = this.resolveMappedDescriptions( + dto.mood ? [dto.mood] : [], + normalizedPrompt, + ARABIC_MOOD_MAPPINGS, + ); + const instruments = this.resolveMappedDescriptions( + dto.instruments ?? [], + normalizedPrompt, + ARABIC_INSTRUMENT_MAPPINGS, + ); + const tempo = dto.tempo ?? this.inferTempo(normalizedPrompt, moods); + const percussion = dto.percussion ?? this.inferPercussion(normalizedPrompt, instruments); + const style = dto.style?.trim(); + + const parts: string[] = []; + if (isArabicPrompt || maqam || instruments.length || moods.length) { + const mainInstruments = instruments.filter( + (instrument) => !instrument.includes('drum') && !instrument.includes('riq'), + ); + const melodyInstrument = mainInstruments[0] ?? instruments[0]; + parts.push( + melodyInstrument + ? `Instrumental Arabic ${melodyInstrument} melody` + : 'Instrumental Arabic music piece', + ); + } else { + parts.push(`Instrumental music based on this idea: ${originalPrompt}`); + } + + if (maqam) { + parts.push(`in ${maqam}`); + } + if (moods.length) { + parts.push(`${moods.join(', ')} mood`); + } + if (percussion !== 'none') { + parts.push(this.describePercussion(percussion, instruments, isArabicPrompt)); + } + if (tempo) { + parts.push(`${tempo} tempo`); + } + if (style) { + parts.push(style); + } else if (isArabicPrompt) { + parts.push('traditional Arabic atmosphere'); + } + + parts.push(AI_MUSIC_SAFE_PRODUCTION_INSTRUCTIONS); + return this.dedupePromptParts(parts).join(', '); + } + + private normalizeArabicText(value: string): string { + return value + .replace(/[أإآ]/g, 'ا') + .replace(/ى/g, 'ي') + .replace(/ة/g, 'ه') + .replace(/[ًٌٍَُِّْـ]/g, ''); + } + + private resolveMappedDescription( + explicitValue: string | undefined, + normalizedPrompt: string, + mappings: Array<{ terms: string[]; description: string }>, + ): string | null { + return this.resolveMappedDescriptions(explicitValue ? [explicitValue] : [], normalizedPrompt, mappings)[0] ?? null; + } + + private resolveMappedDescriptions( + explicitValues: string[], + normalizedPrompt: string, + mappings: Array<{ terms: string[]; description: string }>, + ): string[] { + const values = new Set(); + const normalizedExplicitValues = explicitValues.map((value) => + this.normalizeArabicText(value).toLowerCase(), + ); + + for (const mapping of mappings) { + const normalizedTerms = mapping.terms.map((term) => + this.normalizeArabicText(term).toLowerCase(), + ); + if ( + normalizedTerms.some( + (term) => + normalizedPrompt.includes(term) || + normalizedExplicitValues.some((value) => value.includes(term)), + ) + ) { + values.add(mapping.description); + } + } + + for (const value of explicitValues) { + const trimmed = value.trim(); + if (trimmed && ![...values].some((mappedValue) => mappedValue.toLowerCase().includes(trimmed.toLowerCase()))) { + values.add(trimmed); + } + } + + return [...values]; + } + + private inferTempo(normalizedPrompt: string, moods: string[]): 'slow' | 'medium' | 'fast' { + if ( + normalizedPrompt.includes('سريع') || + normalizedPrompt.includes('fast') || + moods.some((mood) => mood.includes('energetic')) + ) { + return 'fast'; + } + if ( + normalizedPrompt.includes('بطي') || + normalizedPrompt.includes('هاد') || + normalizedPrompt.includes('slow') || + moods.some((mood) => mood.includes('sad') || mood.includes('calm')) + ) { + return 'slow'; + } + return 'medium'; + } + + private inferPercussion( + normalizedPrompt: string, + instruments: string[], + ): 'none' | 'light' | 'medium' { + if (normalizedPrompt.includes('بدون ايقاع') || normalizedPrompt.includes('no percussion')) { + return 'none'; + } + if ( + normalizedPrompt.includes('خفيف') || + normalizedPrompt.includes('light') || + instruments.includes('riq frame drum') + ) { + return 'light'; + } + if ( + normalizedPrompt.includes('ايقاع') || + normalizedPrompt.includes('دربكه') || + normalizedPrompt.includes('طبله') || + normalizedPrompt.includes('percussion') || + instruments.includes('darbuka') + ) { + return 'medium'; + } + return 'none'; + } + + private describePercussion( + percussion: 'light' | 'medium', + instruments: string[], + isArabicPrompt: boolean, + ): string { + const hasDarbuka = instruments.includes('darbuka'); + const hasRiq = instruments.includes('riq frame drum'); + if (percussion === 'light') { + if (hasDarbuka) { + return 'light darbuka percussion'; + } + if (hasRiq || isArabicPrompt) { + return 'light riq percussion'; + } + return 'light percussion'; + } + + if (hasRiq && !hasDarbuka) { + return 'medium riq percussion'; + } + return isArabicPrompt || hasDarbuka ? 'medium darbuka percussion' : 'medium percussion'; + } + + private dedupePromptParts(parts: string[]): string[] { + const seen = new Set(); + const cleanParts: string[] = []; + for (const part of parts) { + const clean = part.trim().replace(/\s+/g, ' '); + const key = clean.toLowerCase(); + if (clean && !seen.has(key)) { + cleanParts.push(clean); + seen.add(key); + } + } + return cleanParts; + } + + private resolveGoogleApplicationCredentials(): { + credentials?: any; + } { + const encodedCredentials = ( + this.configService.get('aiMusic.googleApplicationCredentialsJsonBase64', { + infer: true, + }) ?? '' + ).trim(); + if (!encodedCredentials) { + return {}; + } + + try { + const decoded = Buffer.from(encodedCredentials, 'base64').toString('utf8'); + const credentials = JSON.parse(decoded) as Record; + if (!credentials || typeof credentials !== 'object') { + throw new Error('decoded credentials are not a JSON object'); + } + if (typeof credentials.client_email !== 'string' || !credentials.client_email) { + throw new Error('missing client_email'); + } + if (typeof credentials.private_key !== 'string' || !credentials.private_key) { + throw new Error('missing private_key'); + } + + return { credentials }; + } catch (error) { + throw new ServiceUnavailableException( + `Invalid GOOGLE_APPLICATION_CREDENTIALS_JSON_BASE64: ${ + error instanceof Error ? error.message : 'unable to decode credentials' + }`, + ); + } + } }