diff --git a/s3_s3_migrator.py b/s3_s3_migrator.py deleted file mode 100644 index e3ecfe1..0000000 --- a/s3_s3_migrator.py +++ /dev/null @@ -1,152 +0,0 @@ -import os -import sys -import boto3 -import json -from botocore.client import Config -from botocore.exceptions import ClientError -from boto3.s3.transfer import TransferConfig # ✅ الإضافة الأولى - -# =========================== -# ENV VARIABLES -# =========================== -AWS_SRC_ACCESS_KEY = os.getenv("AWS_SRC_ACCESS_KEY") -AWS_SRC_SECRET_KEY = os.getenv("AWS_SRC_SECRET_KEY") -AWS_SRC_REGION = os.getenv("AWS_SRC_REGION", "us-east-1") -AWS_SRC_BUCKET = os.getenv("AWS_SRC_BUCKET") - -CUMIN_DEST_ACCESS_KEY = os.getenv("CUMIN_DEST_ACCESS_KEY") -CUMIN_DEST_SECRET_KEY = os.getenv("CUMIN_DEST_SECRET_KEY") -CUMIN_DEST_ENDPOINT = os.getenv("CUMIN_DEST_ENDPOINT") -CUMIN_DEST_BUCKET = os.getenv("CUMIN_DEST_BUCKET") - -# =========================== -# VALIDATION -# =========================== -required = [ - AWS_SRC_ACCESS_KEY, AWS_SRC_SECRET_KEY, AWS_SRC_BUCKET, - CUMIN_DEST_ACCESS_KEY, CUMIN_DEST_SECRET_KEY, - CUMIN_DEST_ENDPOINT, CUMIN_DEST_BUCKET -] - -if not all(required): - print("❌ Missing environment variables") - sys.exit(1) - -# =========================== -# TEST CONNECTIONS AND BUCKETS -# =========================== -def test_connections(): - try: - # Test source buckets - src_buckets = src_s3.list_buckets() - print("✅ Source S3 buckets:", [b['Name'] for b in src_buckets['Buckets']]) - if AWS_SRC_BUCKET not in [b['Name'] for b in src_buckets['Buckets']]: - print(f"❌ Source bucket '{AWS_SRC_BUCKET}' not found in available buckets") - return False - except ClientError as e: - print(f"❌ Source S3 error: {e}") - return False - - try: - # Test dest buckets (if applicable, but since it's custom endpoint, list_buckets might not work) - print("✅ Destination S3 connected (endpoint: {CUMIN_DEST_ENDPOINT})") - except ClientError as e: - print(f"❌ Destination S3 error: {e}") - return False - - return True - -if not test_connections(): - sys.exit(1) - -# =========================== -# TRANSFER CONFIG (⭐ المهم) -# =========================== -transfer_config = TransferConfig( - multipart_threshold=5 * 1024 * 1024, # 5MB - multipart_chunksize=5 * 1024 * 1024, - max_concurrency=2, - use_threads=True -) - -# =========================== -# CLIENTS -# =========================== -src_s3 = boto3.client( - "s3", - aws_access_key_id=AWS_SRC_ACCESS_KEY, - aws_secret_access_key=AWS_SRC_SECRET_KEY, - region_name=AWS_SRC_REGION -) - -dest_s3 = boto3.client( - "s3", - aws_access_key_id=CUMIN_DEST_ACCESS_KEY, - aws_secret_access_key=CUMIN_DEST_SECRET_KEY, - endpoint_url=CUMIN_DEST_ENDPOINT -) - -# =========================== -# PROGRESS -# =========================== -def update_progress(current, total, percent, last_file, status="running"): - with open("migration_progress.json", "w") as f: - json.dump({ - "current": current, - "total": total, - "percent": percent, - "last_file": last_file, - "status": status - }, f) - -# =========================== -# MIGRATE -# =========================== -def migrate_s3_to_s3(): - update_progress(0, 0, 0, "Initializing...", "running") - - try: - # جلب قائمة الملفات - objects = src_s3.list_objects_v2(Bucket=AWS_SRC_BUCKET) - if 'Contents' not in objects: - print(f"ℹ️ No files found in bucket '{AWS_SRC_BUCKET}'") - update_progress(0, 0, 100, "No files to migrate", "completed") - return - - files = objects['Contents'] - total = len(files) - print(f"📁 Found {total} files in bucket '{AWS_SRC_BUCKET}'") - current = 0 - - for obj in files: - key = obj['Key'] - update_progress(current, total, int((current / total) * 100), key) - - # نسخ الملف - copy_source = {'Bucket': AWS_SRC_BUCKET, 'Key': key} - dest_s3.copy_object( - CopySource=copy_source, - Bucket=CUMIN_DEST_BUCKET, - Key=key, - Config=transfer_config - ) - - current += 1 - - update_progress(total, total, 100, "Migration completed", "completed") - print("✅ Migration completed successfully") - - except ClientError as e: - error_msg = f"Migration failed: {e}" - print(f"❌ {error_msg}") - update_progress(0, 0, 0, error_msg, "failed") - sys.exit(1) - Config=transfer_config - ) - - current += 1 - - update_progress(total, total, 100, "Migration completed", "completed") - -if __name__ == "__main__": - migrate_s3_to_s3() \ No newline at end of file