Add adaptive media variants for weak networks

src/infrastructure/storage/image-processing.service.ts

@@ -0,0 +1,245 @@
+import { Injectable, Logger } from '@nestjs/common';
+import { ConfigService } from '@nestjs/config';
+import { spawn } from 'child_process';
+import { mkdtemp, readFile, rm, writeFile } from 'fs/promises';
+import { tmpdir } from 'os';
+import { extname, join } from 'path';
+
+export type UploadedImageFile = {
+  mimetype?: string;
+  size: number;
+  buffer: Buffer;
+  originalname?: string;
+};
+
+export type ProcessedImageVariantName = 'original' | 'low' | 'medium' | 'high';
+
+export type ProcessedImageResult = {
+  primaryVariantName: ProcessedImageVariantName;
+  variants: Array<{
+    name: ProcessedImageVariantName;
+    relativePath: string;
+    extension: string;
+    buffer: Buffer;
+    contentType: string;
+  }>;
+};
+
+@Injectable()
+export class ImageProcessingService {
+  private readonly logger = new Logger(ImageProcessingService.name);
+  private ffmpegAvailabilityResolved = false;
+  private ffmpegAvailable = false;
+
+  constructor(private readonly configService: ConfigService) {}
+
+  async processForResponsiveDelivery(file: UploadedImageFile): Promise<ProcessedImageResult> {
+    const originalVariant: ProcessedImageResult['variants'][number] = {
+      name: 'original',
+      relativePath: `original${this.resolveInputExtension(file)}`,
+      extension: this.resolveInputExtension(file),
+      buffer: file.buffer,
+      contentType: this.resolveOriginalContentType(file),
+    };
+
+    if (!this.isEnabled()) {
+      return {
+        primaryVariantName: 'original',
+        variants: [originalVariant],
+      };
+    }
+
+    const ffmpegReady = await this.ensureFfmpegAvailable();
+    if (!ffmpegReady) {
+      return {
+        primaryVariantName: 'original',
+        variants: [originalVariant],
+      };
+    }
+
+    const workingDir = await mkdtemp(join(tmpdir(), 'oudelaa-image-'));
+    const inputPath = join(workingDir, `input${this.resolveInputExtension(file)}`);
+
+    try {
+      await writeFile(inputPath, file.buffer);
+
+      const lowVariant = await this.generateWebpVariant(inputPath, workingDir, 'low', this.getLowWidth());
+      const mediumVariant = await this.generateWebpVariant(
+        inputPath,
+        workingDir,
+        'medium',
+        this.getMediumWidth(),
+      );
+      const highVariant = await this.generateWebpVariant(inputPath, workingDir, 'high', this.getHighWidth());
+
+      return {
+        primaryVariantName: 'medium',
+        variants: [originalVariant, lowVariant, mediumVariant, highVariant],
+      };
+    } catch (error) {
+      this.logger.warn(
+        `Image optimization failed for "${file.originalname ?? 'upload'}": ${
+          error instanceof Error ? error.message : 'unknown error'
+        }`,
+      );
+
+      return {
+        primaryVariantName: 'original',
+        variants: [originalVariant],
+      };
+    } finally {
+      await rm(workingDir, { recursive: true, force: true });
+    }
+  }
+
+  private isEnabled(): boolean {
+    const explicit = this.configService.get<boolean>('imageProcessing.enabled', { infer: true });
+    if (typeof explicit === 'boolean') {
+      return explicit;
+    }
+
+    return this.configService.get<boolean>('videoProcessing.enabled', { infer: true }) ?? false;
+  }
+
+  private getFfmpegPath(): string {
+    return (
+      this.configService.get<string>('imageProcessing.ffmpegPath', { infer: true }) ??
+      this.configService.get<string>('videoProcessing.ffmpegPath', { infer: true }) ??
+      'ffmpeg'
+    ).trim();
+  }
+
+  private getLowWidth(): number {
+    return this.configService.get<number>('imageProcessing.lowWidth', { infer: true }) ?? 360;
+  }
+
+  private getMediumWidth(): number {
+    return this.configService.get<number>('imageProcessing.mediumWidth', { infer: true }) ?? 720;
+  }
+
+  private getHighWidth(): number {
+    return this.configService.get<number>('imageProcessing.highWidth', { infer: true }) ?? 1280;
+  }
+
+  private getQuality(): number {
+    return this.configService.get<number>('imageProcessing.quality', { infer: true }) ?? 78;
+  }
+
+  private resolveInputExtension(file: UploadedImageFile): string {
+    const extension = extname(file.originalname ?? '').toLowerCase();
+    if (extension) {
+      return extension;
+    }
+
+    switch (file.mimetype) {
+      case 'image/jpeg':
+        return '.jpg';
+      case 'image/png':
+        return '.png';
+      case 'image/webp':
+        return '.webp';
+      case 'image/gif':
+        return '.gif';
+      default:
+        return '.jpg';
+    }
+  }
+
+  private resolveOriginalContentType(file: UploadedImageFile): string {
+    if (file.mimetype?.trim()) {
+      return file.mimetype;
+    }
+
+    switch (this.resolveInputExtension(file)) {
+      case '.png':
+        return 'image/png';
+      case '.webp':
+        return 'image/webp';
+      case '.gif':
+        return 'image/gif';
+      case '.jpg':
+      case '.jpeg':
+      default:
+        return 'image/jpeg';
+    }
+  }
+
+  private async generateWebpVariant(
+    inputPath: string,
+    workingDir: string,
+    name: Exclude<ProcessedImageVariantName, 'original'>,
+    width: number,
+  ): Promise<ProcessedImageResult['variants'][number]> {
+    const outputPath = join(workingDir, `${name}.webp`);
+
+    await this.runFfmpeg([
+      '-y',
+      '-i',
+      inputPath,
+      '-frames:v',
+      '1',
+      '-vf',
+      `scale='min(${width},iw)':-2:force_original_aspect_ratio=decrease`,
+      '-c:v',
+      'libwebp',
+      '-compression_level',
+      '6',
+      '-q:v',
+      String(this.getQuality()),
+      outputPath,
+    ]);
+
+    return {
+      name,
+      relativePath: `${name}.webp`,
+      extension: '.webp',
+      buffer: await readFile(outputPath),
+      contentType: 'image/webp',
+    };
+  }
+
+  private async ensureFfmpegAvailable(): Promise<boolean> {
+    if (this.ffmpegAvailabilityResolved) {
+      return this.ffmpegAvailable;
+    }
+
+    try {
+      await this.runFfmpeg(['-version']);
+      this.ffmpegAvailable = true;
+    } catch (error) {
+      this.ffmpegAvailable = false;
+      this.logger.warn(
+        `Image processing is enabled, but ffmpeg is unavailable at "${this.getFfmpegPath()}": ${
+          error instanceof Error ? error.message : 'unknown error'
+        }`,
+      );
+    }
+
+    this.ffmpegAvailabilityResolved = true;
+    return this.ffmpegAvailable;
+  }
+
+  private async runFfmpeg(args: string[]): Promise<void> {
+    await new Promise<void>((resolve, reject) => {
+      const child = spawn(this.getFfmpegPath(), args, { windowsHide: true });
+      let stderr = '';
+
+      child.stderr.on('data', (chunk) => {
+        stderr += chunk.toString();
+      });
+
+      child.on('error', (error) => {
+        reject(error);
+      });
+
+      child.on('close', (code) => {
+        if (code === 0) {
+          resolve();
+          return;
+        }
+
+        reject(new Error(stderr.trim() || `ffmpeg exited with code ${code ?? 'unknown'}`));
+      });
+    });
+  }
+}

src/infrastructure/storage/storage.module.ts

@@ -1,10 +1,11 @@
 import { Global, Module } from '@nestjs/common';
+import { ImageProcessingService } from './image-processing.service';
 import { ManagedStorageService } from './managed-storage.service';
 import { VideoProcessingService } from './video-processing.service';
 
 @Global()
 @Module({
-  providers: [ManagedStorageService, VideoProcessingService],
-  exports: [ManagedStorageService, VideoProcessingService],
+  providers: [ManagedStorageService, VideoProcessingService, ImageProcessingService],
+  exports: [ManagedStorageService, VideoProcessingService, ImageProcessingService],
 })
 export class StorageModule {}

src/infrastructure/storage/video-processing.service.ts

@@ -30,6 +30,19 @@ export type OptimizedVideoResult = {
   };
 };
 
+type ProbedVideoInfo = {
+  hasAudio: boolean;
+  width: number;
+};
+
+type HlsRendition = {
+  width: number;
+  videoBitrateKbps: number;
+  maxRateKbps: number;
+  bufSizeKbps: number;
+  audioBitrateKbps: number;
+};
+
 @Injectable()
 export class VideoProcessingService {
   private readonly logger = new Logger(VideoProcessingService.name);
@@ -180,6 +193,24 @@ export class VideoProcessingService {
     ).trim();
   }
 
+  private getFfprobePath(): string {
+    const configured = (
+      this.configService.get<string>('videoProcessing.ffprobePath', { infer: true }) ?? ''
+    ).trim();
+    if (configured) {
+      return configured;
+    }
+
+    const ffmpegPath = this.getFfmpegPath();
+    if (/ffmpeg(?:\.exe)?$/i.test(ffmpegPath)) {
+      return ffmpegPath.replace(/ffmpeg(?:\.exe)?$/i, (match) =>
+        match.toLowerCase().endsWith('.exe') ? 'ffprobe.exe' : 'ffprobe',
+      );
+    }
+
+    return 'ffprobe';
+  }
+
   private getMaxWidth(): number {
     return this.configService.get<number>('videoProcessing.maxWidth', { infer: true }) ?? 1280;
   }
@@ -257,6 +288,112 @@ export class VideoProcessingService {
   ): Promise<NonNullable<OptimizedVideoResult['generatedHls']>> {
     await mkdir(hlsDir, { recursive: true });
 
+    let probe: ProbedVideoInfo | null = null;
+    try {
+      probe = await this.probeVideo(optimizedMp4Path);
+    } catch (error) {
+      this.logger.warn(
+        `Video probe failed before adaptive HLS generation: ${
+          error instanceof Error ? error.message : 'unknown error'
+        }`,
+      );
+    }
+
+    const renditions = this.buildHlsRenditions(probe?.width ?? this.getMaxWidth());
+    if (!probe || renditions.length <= 1) {
+      return this.generateSingleRenditionHlsPackage(optimizedMp4Path, hlsDir);
+    }
+
+    for (let index = 0; index < renditions.length; index += 1) {
+      await mkdir(join(hlsDir, `stream_${index}`), { recursive: true });
+    }
+
+    const args = ['-y', '-i', optimizedMp4Path];
+    args.push('-filter_complex', this.buildAdaptiveHlsFilterComplex(renditions));
+
+    for (let index = 0; index < renditions.length; index += 1) {
+      args.push('-map', `[v${index}out]`);
+      if (probe.hasAudio) {
+        args.push('-map', '0:a:0');
+      }
+    }
+
+    renditions.forEach((rendition, index) => {
+      args.push(
+        `-c:v:${index}`,
+        'libx264',
+        `-b:v:${index}`,
+        `${rendition.videoBitrateKbps}k`,
+        `-maxrate:v:${index}`,
+        `${rendition.maxRateKbps}k`,
+        `-bufsize:v:${index}`,
+        `${rendition.bufSizeKbps}k`,
+        `-g:v:${index}`,
+        String(this.getMaxFps() * 2),
+        `-keyint_min:v:${index}`,
+        String(this.getMaxFps() * 2),
+        `-sc_threshold:v:${index}`,
+        '0',
+        `-preset:v:${index}`,
+        this.getPreset(),
+      );
+
+      if (probe.hasAudio) {
+        args.push(
+          `-c:a:${index}`,
+          'aac',
+          `-b:a:${index}`,
+          `${rendition.audioBitrateKbps}k`,
+          `-ac:a:${index}`,
+          '2',
+          `-ar:a:${index}`,
+          '44100',
+        );
+      }
+    });
+
+    args.push(
+      '-f',
+      'hls',
+      '-hls_time',
+      String(this.getHlsSegmentDurationSeconds()),
+      '-hls_playlist_type',
+      'vod',
+      '-hls_list_size',
+      '0',
+      '-hls_flags',
+      'independent_segments',
+      '-hls_segment_type',
+      'fmp4',
+      '-master_pl_name',
+      'master.m3u8',
+      '-hls_fmp4_init_filename',
+      'stream_%v/init.mp4',
+      '-hls_segment_filename',
+      join(hlsDir, 'stream_%v/segment-%03d.m4s'),
+      '-var_stream_map',
+      probe.hasAudio
+        ? renditions.map((_, index) => `v:${index},a:${index}`).join(' ')
+        : renditions.map((_, index) => `v:${index}`).join(' '),
+      join(hlsDir, 'stream_%v/playlist.m3u8'),
+    );
+
+    await this.runFfmpeg(args);
+
+    const files = await this.readFilesRecursively(hlsDir);
+
+    return {
+      playlistRelativePath: 'master.m3u8',
+      files,
+    };
+  }
+
+  private async generateSingleRenditionHlsPackage(
+    optimizedMp4Path: string,
+    hlsDir: string,
+  ): Promise<NonNullable<OptimizedVideoResult['generatedHls']>> {
+    await mkdir(hlsDir, { recursive: true });
+
     await this.runFfmpeg([
       '-y',
       '-i',
@@ -288,14 +425,7 @@ export class VideoProcessingService {
       join(hlsDir, 'playlist.m3u8'),
     ]);
 
-    const fileNames = (await readdir(hlsDir)).sort((left, right) => left.localeCompare(right));
-    const files = await Promise.all(
-      fileNames.map(async (fileName) => ({
-        relativePath: fileName,
-        buffer: await readFile(join(hlsDir, fileName)),
-        contentType: this.resolveStreamingContentType(fileName),
-      })),
-    );
+    const files = await this.readFilesRecursively(hlsDir);
 
     return {
       playlistRelativePath: 'playlist.m3u8',
@@ -303,6 +433,119 @@ export class VideoProcessingService {
     };
   }
 
+  private buildAdaptiveHlsFilterComplex(renditions: HlsRendition[]): string {
+    if (renditions.length === 1) {
+      return `[0:v]${this.buildAdaptiveScaleFilter(renditions[0].width)}[v0out]`;
+    }
+
+    const splitOutputs = renditions.map((_, index) => `[v${index}]`).join('');
+    const split = `[0:v]split=${renditions.length}${splitOutputs}`;
+    const transforms = renditions.map(
+      (rendition, index) =>
+        `[v${index}]${this.buildAdaptiveScaleFilter(rendition.width)}[v${index}out]`,
+    );
+
+    return [split, ...transforms].join(';');
+  }
+
+  private buildAdaptiveScaleFilter(width: number): string {
+    return `scale='min(${width},iw)':-2:force_original_aspect_ratio=decrease,fps=${this.getMaxFps()},format=yuv420p`;
+  }
+
+  private buildHlsRenditions(sourceWidth: number): HlsRendition[] {
+    const requestedWidths = [Math.min(480, this.getMaxWidth()), Math.min(720, this.getMaxWidth()), this.getMaxWidth()];
+    const widths = Array.from(
+      new Set(
+        requestedWidths
+          .map((width) => Math.max(240, Math.min(width, sourceWidth)))
+          .filter((width) => Number.isFinite(width)),
+      ),
+    ).sort((left, right) => left - right);
+
+    return widths.map((width) => this.resolveHlsRendition(width));
+  }
+
+  private resolveHlsRendition(width: number): HlsRendition {
+    if (width <= 480) {
+      return {
+        width,
+        videoBitrateKbps: 800,
+        maxRateKbps: 960,
+        bufSizeKbps: 1600,
+        audioBitrateKbps: 96,
+      };
+    }
+
+    if (width <= 720) {
+      return {
+        width,
+        videoBitrateKbps: 1600,
+        maxRateKbps: 1920,
+        bufSizeKbps: 3200,
+        audioBitrateKbps: 128,
+      };
+    }
+
+    return {
+      width,
+      videoBitrateKbps: 2800,
+      maxRateKbps: 3360,
+      bufSizeKbps: 5600,
+      audioBitrateKbps: 128,
+    };
+  }
+
+  private async probeVideo(inputPath: string): Promise<ProbedVideoInfo> {
+    const stdout = await this.runCommand(this.getFfprobePath(), [
+      '-v',
+      'error',
+      '-show_entries',
+      'stream=codec_type,width',
+      '-of',
+      'json',
+      inputPath,
+    ]);
+
+    const parsed = JSON.parse(stdout) as {
+      streams?: Array<{ codec_type?: string; width?: number }>;
+    };
+    const streams = parsed.streams ?? [];
+    const videoStream = streams.find((stream) => stream.codec_type === 'video');
+    if (!videoStream?.width) {
+      throw new Error('ffprobe did not return a video width');
+    }
+
+    return {
+      width: videoStream.width,
+      hasAudio: streams.some((stream) => stream.codec_type === 'audio'),
+    };
+  }
+
+  private async readFilesRecursively(
+    baseDir: string,
+    currentDir = baseDir,
+  ): Promise<Array<{ relativePath: string; buffer: Buffer; contentType: string }>> {
+    const entries = await readdir(currentDir, { withFileTypes: true });
+    const files: Array<{ relativePath: string; buffer: Buffer; contentType: string }> = [];
+
+    for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name))) {
+      const absolutePath = join(currentDir, entry.name);
+      if (entry.isDirectory()) {
+        files.push(...(await this.readFilesRecursively(baseDir, absolutePath)));
+        continue;
+      }
+
+      const relativePath = absolutePath.slice(baseDir.length + 1).replace(/\\/g, '/');
+      files.push({
+        relativePath,
+        buffer: await readFile(absolutePath),
+        contentType: this.resolveStreamingContentType(relativePath),
+      });
+    }
+
+    return files;
+  }
+
   private resolveStreamingContentType(fileName: string): string {
     const extension = extname(fileName).toLowerCase();
 
@@ -341,11 +584,16 @@ export class VideoProcessingService {
     return this.ffmpegAvailable;
   }
 
-  private async runFfmpeg(args: string[]): Promise<void> {
-    await new Promise<void>((resolve, reject) => {
-      const child = spawn(this.getFfmpegPath(), args, { windowsHide: true });
+  private async runCommand(command: string, args: string[]): Promise<string> {
+    return new Promise<string>((resolve, reject) => {
+      const child = spawn(command, args, { windowsHide: true });
+      let stdout = '';
       let stderr = '';
 
+      child.stdout.on('data', (chunk) => {
+        stdout += chunk.toString();
+      });
+
       child.stderr.on('data', (chunk) => {
         stderr += chunk.toString();
       });
@@ -356,12 +604,16 @@ export class VideoProcessingService {
 
       child.on('close', (code) => {
         if (code === 0) {
-          resolve();
+          resolve(stdout);
           return;
         }
 
-        reject(new Error(stderr.trim() || `ffmpeg exited with code ${code ?? 'unknown'}`));
+        reject(new Error(stderr.trim() || `${command} exited with code ${code ?? 'unknown'}`));
       });
     });
   }
+
+  private async runFfmpeg(args: string[]): Promise<void> {
+    await this.runCommand(this.getFfmpegPath(), args);
+  }
 }