Add presigned S3 media URLs
Deploy To Ghaymah / deploy (push) Has been cancelled

This commit is contained in:
boutmoun123
2026-06-25 17:38:08 +03:00
parent 6752ee5fbe
commit fc7c93ed3e
9 changed files with 463 additions and 198 deletions
+6 -1
View File
@@ -1,5 +1,5 @@
import { Module } from '@nestjs/common';
import { APP_GUARD } from '@nestjs/core';
import { APP_GUARD, APP_INTERCEPTOR } from '@nestjs/core';
import { ConfigModule } from '@nestjs/config';
import { AppController } from './app.controller';
import { AppService } from './app.service';
@@ -34,6 +34,7 @@ import { SuperAdminModule } from './modules/superadmin/superadmin.module';
import { SupportModule } from './modules/support/support.module';
import { UsersModule } from './modules/users/users.module';
import { ThrottleGuard } from './common/guards/throttle.guard';
import { MediaUrlInterceptor } from './common/interceptors/media-url.interceptor';
@Module({
imports: [
@@ -79,6 +80,10 @@ import { ThrottleGuard } from './common/guards/throttle.guard';
provide: APP_GUARD,
useClass: ThrottleGuard,
},
{
provide: APP_INTERCEPTOR,
useClass: MediaUrlInterceptor,
},
],
})
export class AppModule {}
@@ -0,0 +1,51 @@
import { CallHandler, ExecutionContext } from '@nestjs/common';
import { lastValueFrom, of } from 'rxjs';
import { MediaStorageService } from '../media/media-storage.service';
import { MediaUrlInterceptor } from './media-url.interceptor';
describe('MediaUrlInterceptor', () => {
const storageService = {
shouldSignResponseUrls: jest.fn(() => true),
resolveResponseUrl: jest.fn(async (url: string) =>
url.includes('s3.cumin.dev') ? `${url}?signed=true` : url,
),
} as unknown as MediaStorageService;
const interceptor = new MediaUrlInterceptor(storageService);
const context = {
getType: () => 'http',
} as ExecutionContext;
beforeEach(() => {
(storageService.resolveResponseUrl as jest.Mock).mockClear();
});
it('signs nested media URLs and variants throughout a response', async () => {
const response = {
imageUrls: ['https://s3.cumin.dev/uploads/image.jpg'],
media: {
displayUrl: 'https://s3.cumin.dev/uploads/display.jpg',
images: [
{
url: 'https://s3.cumin.dev/uploads/medium.jpg',
originalUrl: 'https://s3.cumin.dev/uploads/original.jpg',
variants: {
lowUrl: 'https://s3.cumin.dev/uploads/low.jpg',
},
},
],
preferredPlaybackUrl: 'https://s3.cumin.dev/uploads/audio.mp3',
},
externalUrl: 'https://cdn.example.com/image.jpg',
};
const next = { handle: () => of(response) } as CallHandler;
const result = (await lastValueFrom(interceptor.intercept(context, next))) as typeof response;
expect(result.imageUrls[0]).toContain('?signed=true');
expect(result.media.displayUrl).toContain('?signed=true');
expect(result.media.images[0].originalUrl).toContain('?signed=true');
expect(result.media.images[0].variants.lowUrl).toContain('?signed=true');
expect(result.media.preferredPlaybackUrl).toContain('?signed=true');
expect(result.externalUrl).toBe('https://cdn.example.com/image.jpg');
});
});
@@ -0,0 +1,63 @@
import { CallHandler, ExecutionContext, Injectable, NestInterceptor } from '@nestjs/common';
import { from, Observable, switchMap } from 'rxjs';
import { MediaStorageService } from '../media/media-storage.service';
@Injectable()
export class MediaUrlInterceptor implements NestInterceptor {
constructor(private readonly mediaStorageService: MediaStorageService) {}
intercept(context: ExecutionContext, next: CallHandler): Observable<unknown> {
if (context.getType() !== 'http' || !this.mediaStorageService.shouldSignResponseUrls()) {
return next.handle();
}
return next
.handle()
.pipe(switchMap((data) => from(this.resolveValue(data, new WeakMap<object, unknown>()))));
}
private async resolveValue(value: unknown, seen: WeakMap<object, unknown>): Promise<unknown> {
if (typeof value === 'string') {
return this.mediaStorageService.resolveResponseUrl(value);
}
if (
value === null ||
typeof value !== 'object' ||
value instanceof Date ||
Buffer.isBuffer(value)
) {
return value;
}
const existing = seen.get(value);
if (existing) {
return existing;
}
if (Array.isArray(value)) {
const result: unknown[] = [];
seen.set(value, result);
const entries = await Promise.all(value.map((entry) => this.resolveValue(entry, seen)));
result.push(...entries);
return result;
}
const serializable =
typeof (value as { toJSON?: unknown }).toJSON === 'function'
? (value as { toJSON: () => unknown }).toJSON()
: value;
if (serializable !== value) {
return this.resolveValue(serializable, seen);
}
const result: Record<string, unknown> = {};
seen.set(value, result);
await Promise.all(
Object.entries(value).map(async ([key, entryValue]) => {
result[key] = await this.resolveValue(entryValue, seen);
}),
);
return result;
}
}
@@ -0,0 +1,122 @@
import { ConfigService } from '@nestjs/config';
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
import { AppLoggerService } from '../../infrastructure/logging/app-logger.service';
import { MediaStorageService } from './media-storage.service';
jest.mock('@aws-sdk/s3-request-presigner', () => ({
getSignedUrl: jest.fn(),
}));
const mockedGetSignedUrl = jest.mocked(getSignedUrl);
describe('MediaStorageService response URLs', () => {
const config: Record<string, unknown> = {
'storage.provider': 's3',
'storage.mediaAccessMode': 'signed',
'storage.signedUrlExpiresSeconds': 86400,
'storage.basePath': 'uploads',
'storage.publicBaseUrl': 'https://s3.cumin.dev',
'storage.s3.bucket': 'oudelaa',
'storage.s3.region': 'garage',
'storage.s3.endpoint': 'https://s3.cumin.dev',
'storage.s3.accessKeyId': 'test-access-key',
'storage.s3.secretAccessKey': 'test-secret-key',
'storage.s3.forcePathStyle': true,
publicBaseUrl: 'https://api.oudelaa.test',
};
const configService = {
get: jest.fn((key: string, defaultValue?: unknown) => config[key] ?? defaultValue),
} as unknown as ConfigService;
const appLogger = {
error: jest.fn(),
} as unknown as AppLoggerService;
beforeEach(() => {
mockedGetSignedUrl.mockReset();
(appLogger.error as jest.Mock).mockReset();
});
it('signs a managed S3 URL with the configured expiry', async () => {
mockedGetSignedUrl.mockResolvedValue(
'https://s3.cumin.dev/oudelaa/uploads/posts/image.jpg?X-Amz-Signature=signed',
);
const service = new MediaStorageService(configService, appLogger);
await expect(
service.resolveResponseUrl('https://s3.cumin.dev/uploads/posts/image.jpg'),
).resolves.toContain('X-Amz-Signature=signed');
expect(mockedGetSignedUrl).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({
input: {
Bucket: 'oudelaa',
Key: 'uploads/posts/image.jpg',
},
}),
{ expiresIn: 86400 },
);
});
it('leaves external URLs unchanged', async () => {
const service = new MediaStorageService(configService, appLogger);
await expect(service.resolveResponseUrl('https://cdn.example.com/external.jpg')).resolves.toBe(
'https://cdn.example.com/external.jpg',
);
expect(mockedGetSignedUrl).not.toHaveBeenCalled();
});
it('extracts the key from a path-style URL that includes the bucket', async () => {
config['storage.publicBaseUrl'] = '';
mockedGetSignedUrl.mockResolvedValue(
'https://s3.cumin.dev/oudelaa/uploads/posts/video.mp4?X-Amz-Signature=signed',
);
const service = new MediaStorageService(configService, appLogger);
await service.resolveResponseUrl('https://s3.cumin.dev/oudelaa/uploads/posts/video.mp4');
expect(mockedGetSignedUrl).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({
input: {
Bucket: 'oudelaa',
Key: 'uploads/posts/video.mp4',
},
}),
{ expiresIn: 86400 },
);
config['storage.publicBaseUrl'] = 'https://s3.cumin.dev';
});
it('leaves local storage URLs unchanged', async () => {
config['storage.provider'] = 'local';
const service = new MediaStorageService(configService, appLogger);
await expect(service.resolveResponseUrl('/uploads/posts/image.jpg')).resolves.toBe(
'/uploads/posts/image.jpg',
);
expect(mockedGetSignedUrl).not.toHaveBeenCalled();
config['storage.provider'] = 's3';
});
it('logs key, bucket and error details without returning a direct URL', async () => {
mockedGetSignedUrl.mockRejectedValue(new Error('signing failed'));
const service = new MediaStorageService(configService, appLogger);
await expect(
service.resolveResponseUrl('https://s3.cumin.dev/uploads/posts/image.jpg'),
).resolves.toBe('');
expect(appLogger.error).toHaveBeenCalledWith(
expect.objectContaining({
event: 's3_presigned_url_failed',
key: 'uploads/posts/image.jpg',
bucket: 'oudelaa',
errorName: 'Error',
errorMessage: 'signing failed',
}),
expect.any(String),
'MediaStorageService',
);
});
});
+144 -13
View File
@@ -1,8 +1,15 @@
import { BadRequestException, Injectable, OnModuleDestroy } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { DeleteObjectCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
import {
DeleteObjectCommand,
GetObjectCommand,
PutObjectCommand,
S3Client,
} from '@aws-sdk/client-s3';
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
import { randomUUID } from 'crypto';
import { posix } from 'path';
import { AppLoggerService } from '../../infrastructure/logging/app-logger.service';
import { getFileExtension } from './allowed-media';
export type MediaStorageUploadFile = {
@@ -23,10 +30,17 @@ export type MediaStorageUploadResult = {
@Injectable()
export class MediaStorageService implements OnModuleDestroy {
private s3Client: S3Client | null = null;
private readonly signedUrlCache = new Map<string, { url: string; refreshAfterEpochMs: number }>();
constructor(private readonly configService: ConfigService) {}
constructor(
private readonly configService: ConfigService,
private readonly appLogger: AppLoggerService,
) {}
async uploadFile(file: MediaStorageUploadFile, folder: string): Promise<MediaStorageUploadResult> {
async uploadFile(
file: MediaStorageUploadFile,
folder: string,
): Promise<MediaStorageUploadResult> {
const extension = getFileExtension(file.originalname);
const fileName = `media-${randomUUID()}${extension}`;
const key = posix.join(this.getBasePath(), this.normalizeFolder(folder), fileName);
@@ -98,31 +112,92 @@ export class MediaStorageService implements OnModuleDestroy {
resolveKeyFromUrl(fileUrl: string): string | null {
const normalizedUrl = fileUrl.split('?')[0].split('#')[0];
const managedPathPrefix = `/${this.getBasePath()}/`;
if (normalizedUrl.startsWith(managedPathPrefix)) {
return this.normalizeKey(normalizedUrl.slice(1));
}
const publicBaseUrl = this.getPublicBaseUrl();
if (publicBaseUrl && normalizedUrl.startsWith(`${publicBaseUrl}/`)) {
return this.normalizeKey(normalizedUrl.slice(publicBaseUrl.length + 1));
return this.normalizeS3UrlPath(normalizedUrl.slice(publicBaseUrl.length + 1));
}
const endpoint = this.getEndpoint();
const bucket = this.getBucket();
if (!endpoint || !normalizedUrl.startsWith(`${endpoint}/`)) {
return null;
return this.resolveKeyFromApplicationUrl(normalizedUrl, managedPathPrefix);
}
const pathPart = normalizedUrl.slice(endpoint.length + 1);
if (this.getForcePathStyle()) {
const expectedPrefix = `${bucket}/`;
return pathPart.startsWith(expectedPrefix)
? this.normalizeKey(pathPart.slice(expectedPrefix.length))
: null;
return this.normalizeS3UrlPath(normalizedUrl.slice(endpoint.length + 1));
}
shouldSignResponseUrls(): boolean {
return (
this.getProvider() === 's3' &&
this.configService.get<string>('storage.mediaAccessMode', { infer: true }) === 'signed'
);
}
async resolveResponseUrl(fileUrl: string): Promise<string> {
if (!this.shouldSignResponseUrls() || !fileUrl.trim()) {
return fileUrl;
}
return this.normalizeKey(pathPart);
const key = this.resolveKeyFromUrl(fileUrl);
if (!key) {
return fileUrl;
}
return this.getSignedObjectUrl(key);
}
async getSignedObjectUrl(key: string): Promise<string> {
const normalizedKey = this.normalizeKey(key);
if (!normalizedKey) {
return '';
}
const cached = this.signedUrlCache.get(normalizedKey);
if (cached && cached.refreshAfterEpochMs > Date.now()) {
return cached.url;
}
const bucket = this.getBucket();
const expiresIn = this.getSignedUrlExpiresSeconds();
try {
const url = await getSignedUrl(
this.getS3Client(),
new GetObjectCommand({
Bucket: bucket,
Key: normalizedKey,
}),
{ expiresIn },
);
const refreshBufferSeconds = Math.min(60, Math.max(1, Math.floor(expiresIn / 10)));
this.cacheSignedUrl(normalizedKey, url, expiresIn - refreshBufferSeconds);
return url;
} catch (error) {
const errorName = error instanceof Error ? error.name : 'UnknownError';
const errorMessage = error instanceof Error ? error.message : String(error);
this.appLogger.error(
{
event: 's3_presigned_url_failed',
key: normalizedKey,
bucket,
errorName,
errorMessage,
},
error instanceof Error ? error.stack : undefined,
'MediaStorageService',
);
return '';
}
}
onModuleDestroy(): void {
this.s3Client = null;
this.signedUrlCache.clear();
}
private getS3Client(): S3Client {
@@ -159,6 +234,15 @@ export class MediaStorageService implements OnModuleDestroy {
.replace(/^\/+|\/+$/g, '');
}
private getProvider(): 'local' | 's3' {
return (
(this.configService.get<string>('storage.provider', { infer: true }) as
| 'local'
| 's3'
| undefined) ?? 'local'
);
}
private getBucket(): string {
const bucket = this.configService.get<string>('storage.s3.bucket', { infer: true }) ?? '';
if (!bucket) {
@@ -185,6 +269,53 @@ export class MediaStorageService implements OnModuleDestroy {
return this.configService.get<boolean>('storage.s3.forcePathStyle', { infer: true }) ?? false;
}
private getSignedUrlExpiresSeconds(): number {
const configured =
this.configService.get<number>('storage.signedUrlExpiresSeconds', { infer: true }) ?? 86400;
return Math.min(604800, Math.max(1, Math.floor(configured)));
}
private resolveKeyFromApplicationUrl(fileUrl: string, managedPathPrefix: string): string | null {
const applicationBaseUrl = (
this.configService.get<string>('publicBaseUrl', { infer: true }) ?? ''
).replace(/\/$/, '');
if (!applicationBaseUrl || !fileUrl.startsWith(`${applicationBaseUrl}/`)) {
return null;
}
try {
const pathname = new URL(fileUrl).pathname;
return pathname.startsWith(managedPathPrefix) ? this.normalizeKey(pathname.slice(1)) : null;
} catch {
return null;
}
}
private normalizeS3UrlPath(pathPart: string): string | null {
if (!this.getForcePathStyle()) {
return this.normalizeKey(pathPart);
}
const bucketPrefix = `${this.getBucket()}/`;
return this.normalizeKey(
pathPart.startsWith(bucketPrefix) ? pathPart.slice(bucketPrefix.length) : pathPart,
);
}
private cacheSignedUrl(key: string, url: string, refreshAfterSeconds: number): void {
if (this.signedUrlCache.size >= 10000) {
const oldestKey = this.signedUrlCache.keys().next().value as string | undefined;
if (oldestKey) {
this.signedUrlCache.delete(oldestKey);
}
}
this.signedUrlCache.set(key, {
url,
refreshAfterEpochMs: Date.now() + Math.max(1, refreshAfterSeconds) * 1000,
});
}
private normalizeFolder(folder: string): string {
const normalized = folder.replace(/\\/g, '/').replace(/^\/+|\/+$/g, '');
if (
+2
View File
@@ -82,6 +82,8 @@ export default () => ({
},
storage: {
provider: process.env.STORAGE_PROVIDER ?? 'local',
mediaAccessMode: process.env.MEDIA_ACCESS_MODE ?? 'direct',
signedUrlExpiresSeconds: Number(process.env.SIGNED_URL_EXPIRES_SECONDS ?? 86400),
basePath: process.env.STORAGE_BASE_PATH ?? 'uploads',
publicBaseUrl: process.env.S3_PUBLIC_BASE_URL ?? process.env.STORAGE_PUBLIC_BASE_URL ?? '',
s3: {
+2
View File
@@ -54,6 +54,8 @@ export const validationSchema = Joi.object({
QUEUE_REMOVE_ON_COMPLETE: Joi.boolean().truthy('true').falsy('false').default(true),
QUEUE_WORKER_CONCURRENCY: Joi.number().min(1).max(100).default(5),
STORAGE_PROVIDER: Joi.string().valid('local', 's3').default('local'),
MEDIA_ACCESS_MODE: Joi.string().valid('direct', 'signed').default('direct'),
SIGNED_URL_EXPIRES_SECONDS: Joi.number().integer().min(1).max(604800).default(86400),
STORAGE_BASE_PATH: Joi.string().default('uploads'),
STORAGE_PUBLIC_BASE_URL: Joi.string().allow('').optional(),
S3_BUCKET: Joi.string().allow('').optional(),