This commit is contained in:
+6
-1
@@ -1,5 +1,5 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { APP_GUARD } from '@nestjs/core';
|
||||
import { APP_GUARD, APP_INTERCEPTOR } from '@nestjs/core';
|
||||
import { ConfigModule } from '@nestjs/config';
|
||||
import { AppController } from './app.controller';
|
||||
import { AppService } from './app.service';
|
||||
@@ -34,6 +34,7 @@ import { SuperAdminModule } from './modules/superadmin/superadmin.module';
|
||||
import { SupportModule } from './modules/support/support.module';
|
||||
import { UsersModule } from './modules/users/users.module';
|
||||
import { ThrottleGuard } from './common/guards/throttle.guard';
|
||||
import { MediaUrlInterceptor } from './common/interceptors/media-url.interceptor';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
@@ -79,6 +80,10 @@ import { ThrottleGuard } from './common/guards/throttle.guard';
|
||||
provide: APP_GUARD,
|
||||
useClass: ThrottleGuard,
|
||||
},
|
||||
{
|
||||
provide: APP_INTERCEPTOR,
|
||||
useClass: MediaUrlInterceptor,
|
||||
},
|
||||
],
|
||||
})
|
||||
export class AppModule {}
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
import { CallHandler, ExecutionContext } from '@nestjs/common';
|
||||
import { lastValueFrom, of } from 'rxjs';
|
||||
import { MediaStorageService } from '../media/media-storage.service';
|
||||
import { MediaUrlInterceptor } from './media-url.interceptor';
|
||||
|
||||
describe('MediaUrlInterceptor', () => {
|
||||
const storageService = {
|
||||
shouldSignResponseUrls: jest.fn(() => true),
|
||||
resolveResponseUrl: jest.fn(async (url: string) =>
|
||||
url.includes('s3.cumin.dev') ? `${url}?signed=true` : url,
|
||||
),
|
||||
} as unknown as MediaStorageService;
|
||||
const interceptor = new MediaUrlInterceptor(storageService);
|
||||
const context = {
|
||||
getType: () => 'http',
|
||||
} as ExecutionContext;
|
||||
|
||||
beforeEach(() => {
|
||||
(storageService.resolveResponseUrl as jest.Mock).mockClear();
|
||||
});
|
||||
|
||||
it('signs nested media URLs and variants throughout a response', async () => {
|
||||
const response = {
|
||||
imageUrls: ['https://s3.cumin.dev/uploads/image.jpg'],
|
||||
media: {
|
||||
displayUrl: 'https://s3.cumin.dev/uploads/display.jpg',
|
||||
images: [
|
||||
{
|
||||
url: 'https://s3.cumin.dev/uploads/medium.jpg',
|
||||
originalUrl: 'https://s3.cumin.dev/uploads/original.jpg',
|
||||
variants: {
|
||||
lowUrl: 'https://s3.cumin.dev/uploads/low.jpg',
|
||||
},
|
||||
},
|
||||
],
|
||||
preferredPlaybackUrl: 'https://s3.cumin.dev/uploads/audio.mp3',
|
||||
},
|
||||
externalUrl: 'https://cdn.example.com/image.jpg',
|
||||
};
|
||||
const next = { handle: () => of(response) } as CallHandler;
|
||||
|
||||
const result = (await lastValueFrom(interceptor.intercept(context, next))) as typeof response;
|
||||
|
||||
expect(result.imageUrls[0]).toContain('?signed=true');
|
||||
expect(result.media.displayUrl).toContain('?signed=true');
|
||||
expect(result.media.images[0].originalUrl).toContain('?signed=true');
|
||||
expect(result.media.images[0].variants.lowUrl).toContain('?signed=true');
|
||||
expect(result.media.preferredPlaybackUrl).toContain('?signed=true');
|
||||
expect(result.externalUrl).toBe('https://cdn.example.com/image.jpg');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,63 @@
|
||||
import { CallHandler, ExecutionContext, Injectable, NestInterceptor } from '@nestjs/common';
|
||||
import { from, Observable, switchMap } from 'rxjs';
|
||||
import { MediaStorageService } from '../media/media-storage.service';
|
||||
|
||||
@Injectable()
|
||||
export class MediaUrlInterceptor implements NestInterceptor {
|
||||
constructor(private readonly mediaStorageService: MediaStorageService) {}
|
||||
|
||||
intercept(context: ExecutionContext, next: CallHandler): Observable<unknown> {
|
||||
if (context.getType() !== 'http' || !this.mediaStorageService.shouldSignResponseUrls()) {
|
||||
return next.handle();
|
||||
}
|
||||
|
||||
return next
|
||||
.handle()
|
||||
.pipe(switchMap((data) => from(this.resolveValue(data, new WeakMap<object, unknown>()))));
|
||||
}
|
||||
|
||||
private async resolveValue(value: unknown, seen: WeakMap<object, unknown>): Promise<unknown> {
|
||||
if (typeof value === 'string') {
|
||||
return this.mediaStorageService.resolveResponseUrl(value);
|
||||
}
|
||||
|
||||
if (
|
||||
value === null ||
|
||||
typeof value !== 'object' ||
|
||||
value instanceof Date ||
|
||||
Buffer.isBuffer(value)
|
||||
) {
|
||||
return value;
|
||||
}
|
||||
|
||||
const existing = seen.get(value);
|
||||
if (existing) {
|
||||
return existing;
|
||||
}
|
||||
|
||||
if (Array.isArray(value)) {
|
||||
const result: unknown[] = [];
|
||||
seen.set(value, result);
|
||||
const entries = await Promise.all(value.map((entry) => this.resolveValue(entry, seen)));
|
||||
result.push(...entries);
|
||||
return result;
|
||||
}
|
||||
|
||||
const serializable =
|
||||
typeof (value as { toJSON?: unknown }).toJSON === 'function'
|
||||
? (value as { toJSON: () => unknown }).toJSON()
|
||||
: value;
|
||||
if (serializable !== value) {
|
||||
return this.resolveValue(serializable, seen);
|
||||
}
|
||||
|
||||
const result: Record<string, unknown> = {};
|
||||
seen.set(value, result);
|
||||
await Promise.all(
|
||||
Object.entries(value).map(async ([key, entryValue]) => {
|
||||
result[key] = await this.resolveValue(entryValue, seen);
|
||||
}),
|
||||
);
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
|
||||
import { AppLoggerService } from '../../infrastructure/logging/app-logger.service';
|
||||
import { MediaStorageService } from './media-storage.service';
|
||||
|
||||
jest.mock('@aws-sdk/s3-request-presigner', () => ({
|
||||
getSignedUrl: jest.fn(),
|
||||
}));
|
||||
|
||||
const mockedGetSignedUrl = jest.mocked(getSignedUrl);
|
||||
|
||||
describe('MediaStorageService response URLs', () => {
|
||||
const config: Record<string, unknown> = {
|
||||
'storage.provider': 's3',
|
||||
'storage.mediaAccessMode': 'signed',
|
||||
'storage.signedUrlExpiresSeconds': 86400,
|
||||
'storage.basePath': 'uploads',
|
||||
'storage.publicBaseUrl': 'https://s3.cumin.dev',
|
||||
'storage.s3.bucket': 'oudelaa',
|
||||
'storage.s3.region': 'garage',
|
||||
'storage.s3.endpoint': 'https://s3.cumin.dev',
|
||||
'storage.s3.accessKeyId': 'test-access-key',
|
||||
'storage.s3.secretAccessKey': 'test-secret-key',
|
||||
'storage.s3.forcePathStyle': true,
|
||||
publicBaseUrl: 'https://api.oudelaa.test',
|
||||
};
|
||||
const configService = {
|
||||
get: jest.fn((key: string, defaultValue?: unknown) => config[key] ?? defaultValue),
|
||||
} as unknown as ConfigService;
|
||||
const appLogger = {
|
||||
error: jest.fn(),
|
||||
} as unknown as AppLoggerService;
|
||||
|
||||
beforeEach(() => {
|
||||
mockedGetSignedUrl.mockReset();
|
||||
(appLogger.error as jest.Mock).mockReset();
|
||||
});
|
||||
|
||||
it('signs a managed S3 URL with the configured expiry', async () => {
|
||||
mockedGetSignedUrl.mockResolvedValue(
|
||||
'https://s3.cumin.dev/oudelaa/uploads/posts/image.jpg?X-Amz-Signature=signed',
|
||||
);
|
||||
const service = new MediaStorageService(configService, appLogger);
|
||||
|
||||
await expect(
|
||||
service.resolveResponseUrl('https://s3.cumin.dev/uploads/posts/image.jpg'),
|
||||
).resolves.toContain('X-Amz-Signature=signed');
|
||||
|
||||
expect(mockedGetSignedUrl).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
input: {
|
||||
Bucket: 'oudelaa',
|
||||
Key: 'uploads/posts/image.jpg',
|
||||
},
|
||||
}),
|
||||
{ expiresIn: 86400 },
|
||||
);
|
||||
});
|
||||
|
||||
it('leaves external URLs unchanged', async () => {
|
||||
const service = new MediaStorageService(configService, appLogger);
|
||||
|
||||
await expect(service.resolveResponseUrl('https://cdn.example.com/external.jpg')).resolves.toBe(
|
||||
'https://cdn.example.com/external.jpg',
|
||||
);
|
||||
expect(mockedGetSignedUrl).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('extracts the key from a path-style URL that includes the bucket', async () => {
|
||||
config['storage.publicBaseUrl'] = '';
|
||||
mockedGetSignedUrl.mockResolvedValue(
|
||||
'https://s3.cumin.dev/oudelaa/uploads/posts/video.mp4?X-Amz-Signature=signed',
|
||||
);
|
||||
const service = new MediaStorageService(configService, appLogger);
|
||||
|
||||
await service.resolveResponseUrl('https://s3.cumin.dev/oudelaa/uploads/posts/video.mp4');
|
||||
|
||||
expect(mockedGetSignedUrl).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
input: {
|
||||
Bucket: 'oudelaa',
|
||||
Key: 'uploads/posts/video.mp4',
|
||||
},
|
||||
}),
|
||||
{ expiresIn: 86400 },
|
||||
);
|
||||
config['storage.publicBaseUrl'] = 'https://s3.cumin.dev';
|
||||
});
|
||||
|
||||
it('leaves local storage URLs unchanged', async () => {
|
||||
config['storage.provider'] = 'local';
|
||||
const service = new MediaStorageService(configService, appLogger);
|
||||
|
||||
await expect(service.resolveResponseUrl('/uploads/posts/image.jpg')).resolves.toBe(
|
||||
'/uploads/posts/image.jpg',
|
||||
);
|
||||
expect(mockedGetSignedUrl).not.toHaveBeenCalled();
|
||||
config['storage.provider'] = 's3';
|
||||
});
|
||||
|
||||
it('logs key, bucket and error details without returning a direct URL', async () => {
|
||||
mockedGetSignedUrl.mockRejectedValue(new Error('signing failed'));
|
||||
const service = new MediaStorageService(configService, appLogger);
|
||||
|
||||
await expect(
|
||||
service.resolveResponseUrl('https://s3.cumin.dev/uploads/posts/image.jpg'),
|
||||
).resolves.toBe('');
|
||||
expect(appLogger.error).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
event: 's3_presigned_url_failed',
|
||||
key: 'uploads/posts/image.jpg',
|
||||
bucket: 'oudelaa',
|
||||
errorName: 'Error',
|
||||
errorMessage: 'signing failed',
|
||||
}),
|
||||
expect.any(String),
|
||||
'MediaStorageService',
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1,8 +1,15 @@
|
||||
import { BadRequestException, Injectable, OnModuleDestroy } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { DeleteObjectCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
|
||||
import {
|
||||
DeleteObjectCommand,
|
||||
GetObjectCommand,
|
||||
PutObjectCommand,
|
||||
S3Client,
|
||||
} from '@aws-sdk/client-s3';
|
||||
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
|
||||
import { randomUUID } from 'crypto';
|
||||
import { posix } from 'path';
|
||||
import { AppLoggerService } from '../../infrastructure/logging/app-logger.service';
|
||||
import { getFileExtension } from './allowed-media';
|
||||
|
||||
export type MediaStorageUploadFile = {
|
||||
@@ -23,10 +30,17 @@ export type MediaStorageUploadResult = {
|
||||
@Injectable()
|
||||
export class MediaStorageService implements OnModuleDestroy {
|
||||
private s3Client: S3Client | null = null;
|
||||
private readonly signedUrlCache = new Map<string, { url: string; refreshAfterEpochMs: number }>();
|
||||
|
||||
constructor(private readonly configService: ConfigService) {}
|
||||
constructor(
|
||||
private readonly configService: ConfigService,
|
||||
private readonly appLogger: AppLoggerService,
|
||||
) {}
|
||||
|
||||
async uploadFile(file: MediaStorageUploadFile, folder: string): Promise<MediaStorageUploadResult> {
|
||||
async uploadFile(
|
||||
file: MediaStorageUploadFile,
|
||||
folder: string,
|
||||
): Promise<MediaStorageUploadResult> {
|
||||
const extension = getFileExtension(file.originalname);
|
||||
const fileName = `media-${randomUUID()}${extension}`;
|
||||
const key = posix.join(this.getBasePath(), this.normalizeFolder(folder), fileName);
|
||||
@@ -98,31 +112,92 @@ export class MediaStorageService implements OnModuleDestroy {
|
||||
|
||||
resolveKeyFromUrl(fileUrl: string): string | null {
|
||||
const normalizedUrl = fileUrl.split('?')[0].split('#')[0];
|
||||
const managedPathPrefix = `/${this.getBasePath()}/`;
|
||||
if (normalizedUrl.startsWith(managedPathPrefix)) {
|
||||
return this.normalizeKey(normalizedUrl.slice(1));
|
||||
}
|
||||
|
||||
const publicBaseUrl = this.getPublicBaseUrl();
|
||||
|
||||
if (publicBaseUrl && normalizedUrl.startsWith(`${publicBaseUrl}/`)) {
|
||||
return this.normalizeKey(normalizedUrl.slice(publicBaseUrl.length + 1));
|
||||
return this.normalizeS3UrlPath(normalizedUrl.slice(publicBaseUrl.length + 1));
|
||||
}
|
||||
|
||||
const endpoint = this.getEndpoint();
|
||||
const bucket = this.getBucket();
|
||||
if (!endpoint || !normalizedUrl.startsWith(`${endpoint}/`)) {
|
||||
return null;
|
||||
return this.resolveKeyFromApplicationUrl(normalizedUrl, managedPathPrefix);
|
||||
}
|
||||
|
||||
const pathPart = normalizedUrl.slice(endpoint.length + 1);
|
||||
if (this.getForcePathStyle()) {
|
||||
const expectedPrefix = `${bucket}/`;
|
||||
return pathPart.startsWith(expectedPrefix)
|
||||
? this.normalizeKey(pathPart.slice(expectedPrefix.length))
|
||||
: null;
|
||||
return this.normalizeS3UrlPath(normalizedUrl.slice(endpoint.length + 1));
|
||||
}
|
||||
|
||||
shouldSignResponseUrls(): boolean {
|
||||
return (
|
||||
this.getProvider() === 's3' &&
|
||||
this.configService.get<string>('storage.mediaAccessMode', { infer: true }) === 'signed'
|
||||
);
|
||||
}
|
||||
|
||||
async resolveResponseUrl(fileUrl: string): Promise<string> {
|
||||
if (!this.shouldSignResponseUrls() || !fileUrl.trim()) {
|
||||
return fileUrl;
|
||||
}
|
||||
|
||||
return this.normalizeKey(pathPart);
|
||||
const key = this.resolveKeyFromUrl(fileUrl);
|
||||
if (!key) {
|
||||
return fileUrl;
|
||||
}
|
||||
|
||||
return this.getSignedObjectUrl(key);
|
||||
}
|
||||
|
||||
async getSignedObjectUrl(key: string): Promise<string> {
|
||||
const normalizedKey = this.normalizeKey(key);
|
||||
if (!normalizedKey) {
|
||||
return '';
|
||||
}
|
||||
|
||||
const cached = this.signedUrlCache.get(normalizedKey);
|
||||
if (cached && cached.refreshAfterEpochMs > Date.now()) {
|
||||
return cached.url;
|
||||
}
|
||||
|
||||
const bucket = this.getBucket();
|
||||
const expiresIn = this.getSignedUrlExpiresSeconds();
|
||||
|
||||
try {
|
||||
const url = await getSignedUrl(
|
||||
this.getS3Client(),
|
||||
new GetObjectCommand({
|
||||
Bucket: bucket,
|
||||
Key: normalizedKey,
|
||||
}),
|
||||
{ expiresIn },
|
||||
);
|
||||
const refreshBufferSeconds = Math.min(60, Math.max(1, Math.floor(expiresIn / 10)));
|
||||
this.cacheSignedUrl(normalizedKey, url, expiresIn - refreshBufferSeconds);
|
||||
return url;
|
||||
} catch (error) {
|
||||
const errorName = error instanceof Error ? error.name : 'UnknownError';
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
this.appLogger.error(
|
||||
{
|
||||
event: 's3_presigned_url_failed',
|
||||
key: normalizedKey,
|
||||
bucket,
|
||||
errorName,
|
||||
errorMessage,
|
||||
},
|
||||
error instanceof Error ? error.stack : undefined,
|
||||
'MediaStorageService',
|
||||
);
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
onModuleDestroy(): void {
|
||||
this.s3Client = null;
|
||||
this.signedUrlCache.clear();
|
||||
}
|
||||
|
||||
private getS3Client(): S3Client {
|
||||
@@ -159,6 +234,15 @@ export class MediaStorageService implements OnModuleDestroy {
|
||||
.replace(/^\/+|\/+$/g, '');
|
||||
}
|
||||
|
||||
private getProvider(): 'local' | 's3' {
|
||||
return (
|
||||
(this.configService.get<string>('storage.provider', { infer: true }) as
|
||||
| 'local'
|
||||
| 's3'
|
||||
| undefined) ?? 'local'
|
||||
);
|
||||
}
|
||||
|
||||
private getBucket(): string {
|
||||
const bucket = this.configService.get<string>('storage.s3.bucket', { infer: true }) ?? '';
|
||||
if (!bucket) {
|
||||
@@ -185,6 +269,53 @@ export class MediaStorageService implements OnModuleDestroy {
|
||||
return this.configService.get<boolean>('storage.s3.forcePathStyle', { infer: true }) ?? false;
|
||||
}
|
||||
|
||||
private getSignedUrlExpiresSeconds(): number {
|
||||
const configured =
|
||||
this.configService.get<number>('storage.signedUrlExpiresSeconds', { infer: true }) ?? 86400;
|
||||
return Math.min(604800, Math.max(1, Math.floor(configured)));
|
||||
}
|
||||
|
||||
private resolveKeyFromApplicationUrl(fileUrl: string, managedPathPrefix: string): string | null {
|
||||
const applicationBaseUrl = (
|
||||
this.configService.get<string>('publicBaseUrl', { infer: true }) ?? ''
|
||||
).replace(/\/$/, '');
|
||||
if (!applicationBaseUrl || !fileUrl.startsWith(`${applicationBaseUrl}/`)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const pathname = new URL(fileUrl).pathname;
|
||||
return pathname.startsWith(managedPathPrefix) ? this.normalizeKey(pathname.slice(1)) : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private normalizeS3UrlPath(pathPart: string): string | null {
|
||||
if (!this.getForcePathStyle()) {
|
||||
return this.normalizeKey(pathPart);
|
||||
}
|
||||
|
||||
const bucketPrefix = `${this.getBucket()}/`;
|
||||
return this.normalizeKey(
|
||||
pathPart.startsWith(bucketPrefix) ? pathPart.slice(bucketPrefix.length) : pathPart,
|
||||
);
|
||||
}
|
||||
|
||||
private cacheSignedUrl(key: string, url: string, refreshAfterSeconds: number): void {
|
||||
if (this.signedUrlCache.size >= 10000) {
|
||||
const oldestKey = this.signedUrlCache.keys().next().value as string | undefined;
|
||||
if (oldestKey) {
|
||||
this.signedUrlCache.delete(oldestKey);
|
||||
}
|
||||
}
|
||||
|
||||
this.signedUrlCache.set(key, {
|
||||
url,
|
||||
refreshAfterEpochMs: Date.now() + Math.max(1, refreshAfterSeconds) * 1000,
|
||||
});
|
||||
}
|
||||
|
||||
private normalizeFolder(folder: string): string {
|
||||
const normalized = folder.replace(/\\/g, '/').replace(/^\/+|\/+$/g, '');
|
||||
if (
|
||||
|
||||
@@ -82,6 +82,8 @@ export default () => ({
|
||||
},
|
||||
storage: {
|
||||
provider: process.env.STORAGE_PROVIDER ?? 'local',
|
||||
mediaAccessMode: process.env.MEDIA_ACCESS_MODE ?? 'direct',
|
||||
signedUrlExpiresSeconds: Number(process.env.SIGNED_URL_EXPIRES_SECONDS ?? 86400),
|
||||
basePath: process.env.STORAGE_BASE_PATH ?? 'uploads',
|
||||
publicBaseUrl: process.env.S3_PUBLIC_BASE_URL ?? process.env.STORAGE_PUBLIC_BASE_URL ?? '',
|
||||
s3: {
|
||||
|
||||
@@ -54,6 +54,8 @@ export const validationSchema = Joi.object({
|
||||
QUEUE_REMOVE_ON_COMPLETE: Joi.boolean().truthy('true').falsy('false').default(true),
|
||||
QUEUE_WORKER_CONCURRENCY: Joi.number().min(1).max(100).default(5),
|
||||
STORAGE_PROVIDER: Joi.string().valid('local', 's3').default('local'),
|
||||
MEDIA_ACCESS_MODE: Joi.string().valid('direct', 'signed').default('direct'),
|
||||
SIGNED_URL_EXPIRES_SECONDS: Joi.number().integer().min(1).max(604800).default(86400),
|
||||
STORAGE_BASE_PATH: Joi.string().default('uploads'),
|
||||
STORAGE_PUBLIC_BASE_URL: Joi.string().allow('').optional(),
|
||||
S3_BUCKET: Joi.string().allow('').optional(),
|
||||
|
||||
Reference in New Issue
Block a user