import os import sys import boto3 import json from botocore.client import Config from botocore.exceptions import ClientError from boto3.s3.transfer import TransferConfig # ✅ الإضافة الأولى # =========================== # ENV VARIABLES # =========================== AWS_SRC_ACCESS_KEY = os.getenv("AWS_SRC_ACCESS_KEY") AWS_SRC_SECRET_KEY = os.getenv("AWS_SRC_SECRET_KEY") AWS_SRC_REGION = os.getenv("AWS_SRC_REGION", "us-east-1") AWS_SRC_BUCKET = os.getenv("AWS_SRC_BUCKET") CUMIN_DEST_ACCESS_KEY = os.getenv("CUMIN_DEST_ACCESS_KEY") CUMIN_DEST_SECRET_KEY = os.getenv("CUMIN_DEST_SECRET_KEY") CUMIN_DEST_ENDPOINT = os.getenv("CUMIN_DEST_ENDPOINT") CUMIN_DEST_BUCKET = os.getenv("CUMIN_DEST_BUCKET") # =========================== # VALIDATION # =========================== required = [ AWS_SRC_ACCESS_KEY, AWS_SRC_SECRET_KEY, AWS_SRC_BUCKET, CUMIN_DEST_ACCESS_KEY, CUMIN_DEST_SECRET_KEY, CUMIN_DEST_ENDPOINT, CUMIN_DEST_BUCKET ] if not all(required): print("❌ Missing environment variables") sys.exit(1) # =========================== # TEST CONNECTIONS AND BUCKETS # =========================== def test_connections(): try: # Test source buckets src_buckets = src_s3.list_buckets() print("✅ Source S3 buckets:", [b['Name'] for b in src_buckets['Buckets']]) if AWS_SRC_BUCKET not in [b['Name'] for b in src_buckets['Buckets']]: print(f"❌ Source bucket '{AWS_SRC_BUCKET}' not found in available buckets") return False except ClientError as e: print(f"❌ Source S3 error: {e}") return False try: # Test dest buckets (if applicable, but since it's custom endpoint, list_buckets might not work) print("✅ Destination S3 connected (endpoint: {CUMIN_DEST_ENDPOINT})") except ClientError as e: print(f"❌ Destination S3 error: {e}") return False return True if not test_connections(): sys.exit(1) # =========================== # TRANSFER CONFIG (⭐ المهم) # =========================== transfer_config = TransferConfig( multipart_threshold=5 * 1024 * 1024, # 5MB multipart_chunksize=5 * 1024 * 1024, max_concurrency=2, use_threads=True ) # =========================== # CLIENTS # =========================== src_s3 = boto3.client( "s3", aws_access_key_id=AWS_SRC_ACCESS_KEY, aws_secret_access_key=AWS_SRC_SECRET_KEY, region_name=AWS_SRC_REGION ) dest_s3 = boto3.client( "s3", aws_access_key_id=CUMIN_DEST_ACCESS_KEY, aws_secret_access_key=CUMIN_DEST_SECRET_KEY, endpoint_url=CUMIN_DEST_ENDPOINT ) # =========================== # PROGRESS # =========================== def update_progress(current, total, percent, last_file, status="running"): with open("migration_progress.json", "w") as f: json.dump({ "current": current, "total": total, "percent": percent, "last_file": last_file, "status": status }, f) # =========================== # MIGRATE # =========================== def migrate_s3_to_s3(): update_progress(0, 0, 0, "Initializing...", "running") try: # جلب قائمة الملفات objects = src_s3.list_objects_v2(Bucket=AWS_SRC_BUCKET) if 'Contents' not in objects: print(f"ℹ️ No files found in bucket '{AWS_SRC_BUCKET}'") update_progress(0, 0, 100, "No files to migrate", "completed") return files = objects['Contents'] total = len(files) print(f"📁 Found {total} files in bucket '{AWS_SRC_BUCKET}'") current = 0 for obj in files: key = obj['Key'] update_progress(current, total, int((current / total) * 100), key) # نسخ الملف copy_source = {'Bucket': AWS_SRC_BUCKET, 'Key': key} dest_s3.copy_object( CopySource=copy_source, Bucket=CUMIN_DEST_BUCKET, Key=key, Config=transfer_config ) current += 1 update_progress(total, total, 100, "Migration completed", "completed") print("✅ Migration completed successfully") except ClientError as e: error_msg = f"Migration failed: {e}" print(f"❌ {error_msg}") update_progress(0, 0, 0, error_msg, "failed") sys.exit(1) Config=transfer_config ) current += 1 update_progress(total, total, 100, "Migration completed", "completed") if __name__ == "__main__": migrate_s3_to_s3()