import os import re import uuid import mimetypes from typing import Optional import boto3 import requests # ---- ENV / S3 setup --------------------------------------------------------- AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID") AWS_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY") AWS_STORAGE_BUCKET_NAME = os.getenv("AWS_STORAGE_BUCKET_NAME") S3_ENDPOINT_URL = os.getenv("S3_ENDPOINT_URL") S3_REGION = os.getenv("S3_REGION", "us-east-1") MEDIA_BASE_PATH = os.getenv("MEDIA_BASE_PATH", "instagram") # Optimization toggles MEDIA_OPTIMIZE_IMAGES = os.getenv("MEDIA_OPTIMIZE_IMAGES", "true").strip().lower() in ("1", "true", "yes", "y") MEDIA_OPTIMIZE_VIDEOS = os.getenv("MEDIA_OPTIMIZE_VIDEOS", "true").strip().lower() in ("1", "true", "yes", "y") KEEP_ORIGINAL_IMAGES = os.getenv("KEEP_ORIGINAL_IMAGES", "false").strip().lower() in ("1", "true", "yes", "y") KEEP_ORIGINAL_VIDEOS = os.getenv("KEEP_ORIGINAL_VIDEOS", "false").strip().lower() in ("1", "true", "yes", "y") # HTTP DOWNLOAD_TIMEOUT = int(os.getenv("DOWNLOAD_TIMEOUT", "30")) # S3 client session = boto3.session.Session() s3 = session.client( service_name="s3", aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY, endpoint_url=S3_ENDPOINT_URL, region_name=S3_REGION, ) # ---- Media processing imports ---------------------------------------------- # These utilities implement your exact constraints: # - Images: scale 85%, cap 1600x1600, quality 85 → WebP # - Videos: H.264, 480p, CRF 23, +faststart (with SIMPLE mode support) from utils.media_processing import ( # type: ignore download_bytes, process_image_to_webp_single, transcode_video_480_h264, ) # ---- Helpers ---------------------------------------------------------------- def _safe_basename_from_url(url: str) -> str: name = url.split("?")[0].rstrip("/").split("/")[-1] name = name.rsplit(".", 1)[0] if "." in name else name name = re.sub(r"[^a-zA-Z0-9._-]+", "-", name).strip("-") return name or uuid.uuid4().hex[:8] def _guess_ext_from_content_type(content_type: str, fallback: str = ".bin") -> str: ext = mimetypes.guess_extension(content_type or "") or fallback # normalize oddballs if ext in [".jpe", ".jfif", ".bin"]: ext = ".jpg" return ext def _upload_bytes(data: bytes, content_type: str, seller_id: str, post_id: str, key: str) -> str: """ Upload a byte blob to S3 at: {MEDIA_BASE_PATH}/{seller_id}/{post_id}/{key} Returns a public URL. """ s3_path = f"{MEDIA_BASE_PATH}/{seller_id}/{post_id}/{key}" s3.put_object( Bucket=AWS_STORAGE_BUCKET_NAME, Key=s3_path, Body=data, ACL="public-read", ContentType=content_type or "application/octet-stream", ) return f"{S3_ENDPOINT_URL}/{AWS_STORAGE_BUCKET_NAME}/{s3_path}" def _stream_to_s3_from_url(url: str, seller_id: str, post_id: str, key: str, content_type: Optional[str] = None) -> str: """ Stream a remote URL directly to S3 (no processing). """ with requests.get(url, stream=True, timeout=DOWNLOAD_TIMEOUT) as r: r.raise_for_status() ctype = content_type or r.headers.get("Content-Type") or "application/octet-stream" s3_path = f"{MEDIA_BASE_PATH}/{seller_id}/{post_id}/{key}" s3.upload_fileobj( r.raw, Bucket=AWS_STORAGE_BUCKET_NAME, Key=s3_path, ExtraArgs={"ACL": "public-read", "ContentType": ctype}, ) return f"{S3_ENDPOINT_URL}/{AWS_STORAGE_BUCKET_NAME}/{s3_path}" # ---- Public API ------------------------------------------------------------- def upload_image_from_url_optimized(image_url: str, seller_id: str, post_id: str) -> Optional[str]: """ Download -> (optional) optimize to WebP -> upload -> return public URL. If optimization fails, falls back to direct upload. """ base = _safe_basename_from_url(image_url) try: if MEDIA_OPTIMIZE_IMAGES: # Download full bytes, run WebP pipeline (85% scale, max 1600x1600, quality 85) blob = download_bytes(image_url, timeout=DOWNLOAD_TIMEOUT) variant_name, data, ctype = process_image_to_webp_single(blob) url_out = _upload_bytes(data, ctype, seller_id, post_id, f"images/{base}-{variant_name}.webp") if KEEP_ORIGINAL_IMAGES: # Keep original (best-effort content type) _upload_bytes(blob, "application/octet-stream", seller_id, post_id, f"images/{base}-original.bin") return url_out # No optimization: stream to S3 as-is # Try to pick a reasonable extension based on content type head = requests.head(image_url, timeout=DOWNLOAD_TIMEOUT, allow_redirects=True) content_type = head.headers.get("Content-Type", "") ext = _guess_ext_from_content_type(content_type, ".jpg") return _stream_to_s3_from_url(image_url, seller_id, post_id, f"images/{base}{ext}", content_type) except Exception as e: print(f"❌ Image upload failed (optimized path). Falling back. url={image_url} err={e}") # Final fallback: best-effort stream try: return _stream_to_s3_from_url(image_url, seller_id, post_id, f"images/{base}.bin", "application/octet-stream") except Exception as e2: print(f"❌ Image upload fallback failed. url={image_url} err={e2}") return None def upload_video_from_url_optimized(video_url: str, seller_id: str, post_id: str) -> Optional[str]: """ Download -> (optional) transcode to MP4 (H.264, 480p, CRF 23, +faststart) -> upload -> return public URL. If optimization fails, falls back to direct upload. """ base = _safe_basename_from_url(video_url) try: if MEDIA_OPTIMIZE_VIDEOS: blob = download_bytes(video_url, timeout=DOWNLOAD_TIMEOUT) variant_name, data, ctype = transcode_video_480_h264(blob) url_out = _upload_bytes(data, ctype, seller_id, post_id, f"videos/{base}-{variant_name}.mp4") if KEEP_ORIGINAL_VIDEOS: _upload_bytes(blob, "application/octet-stream", seller_id, post_id, f"videos/{base}-original.bin") return url_out # No optimization: stream to S3 as-is head = requests.head(video_url, timeout=DOWNLOAD_TIMEOUT, allow_redirects=True) content_type = head.headers.get("Content-Type", "") ext = _guess_ext_from_content_type(content_type, ".mp4") return _stream_to_s3_from_url(video_url, seller_id, post_id, f"videos/{base}{ext}", content_type) except Exception as e: print(f"❌ Video upload failed (optimized path). Falling back. url={video_url} err={e}") # Final fallback: best-effort stream try: return _stream_to_s3_from_url(video_url, seller_id, post_id, f"videos/{base}.bin", "application/octet-stream") except Exception as e2: print(f"❌ Video upload fallback failed. url={video_url} err={e2}") return None def upload_media_from_url(media_url: str, seller_id: str, post_id: str) -> Optional[str]: """ Backward-compatible alias if callers don't know type. Heuristic: decide by Content-Type, optimize when possible. """ try: head = requests.head(media_url, timeout=DOWNLOAD_TIMEOUT, allow_redirects=True) content_type = head.headers.get("Content-Type", "").lower() if "image" in content_type: return upload_image_from_url_optimized(media_url, seller_id, post_id) if "video" in content_type: return upload_video_from_url_optimized(media_url, seller_id, post_id) # Unknown → just stream raw base = _safe_basename_from_url(media_url) return _stream_to_s3_from_url(media_url, seller_id, post_id, f"misc/{base}", content_type or "application/octet-stream") except Exception as e: print(f"❌ HEAD failed for {media_url}: {e}") # If HEAD fails, still try image path → then video path → then raw url = upload_image_from_url_optimized(media_url, seller_id, post_id) if url: return url url = upload_video_from_url_optimized(media_url, seller_id, post_id) if url: return url try: base = _safe_basename_from_url(media_url) return _stream_to_s3_from_url(media_url, seller_id, post_id, f"misc/{base}.bin", "application/octet-stream") except Exception as e2: print(f"❌ Ultimate fallback failed for {media_url}: {e2}") return None