From 20954291c0998c37c5aa78bd95b5dff5f4105a7c Mon Sep 17 00:00:00 2001 From: Hossein Date: Tue, 16 Sep 2025 20:14:01 +0330 Subject: [PATCH] add ffmpeg --- Dockerfile | 36 +++---- docker-compose.yml | 10 +- scraper.py | 86 ++++++++++------- utils/media_processing.py | 193 ++++++++++++++++++++++++++++++++++++++ utils/s3_uploader.py | 191 +++++++++++++++++++++++++++++++++---- 5 files changed, 445 insertions(+), 71 deletions(-) create mode 100644 utils/media_processing.py diff --git a/Dockerfile b/Dockerfile index a076288..2ed29bf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,26 +1,30 @@ # Use Python 3.11 slim image as base FROM python:3.11-slim +# ---- System setup ---- +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + FFMPEG_BIN=/usr/bin/ffmpeg \ + FFPROBE_BIN=/usr/bin/ffprobe + # Set working directory WORKDIR /app -# Set environment variables -ENV PYTHONDONTWRITEBYTECODE=1 -ENV PYTHONUNBUFFERED=1 - -# Install system dependencies +# Install system dependencies (ffmpeg + curl for healthcheck, build tools for pip) RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - gcc \ - g++ \ - && rm -rf /var/lib/apt/lists/* + && apt-get install -y --no-install-recommends \ + ffmpeg \ + curl \ + build-essential \ + && rm -rf /var/lib/apt/lists/* # Copy requirements first for better caching -COPY requirements.txt . +COPY requirements.txt ./requirements.txt # Install Python dependencies -RUN pip install --no-cache-dir --upgrade pip \ - && pip install --no-cache-dir -r requirements.txt +RUN python -m pip install --upgrade pip \ + && pip install -r requirements.txt # Copy application code COPY . . @@ -28,9 +32,9 @@ COPY . . # Expose port EXPOSE 8000 -# Health check +# Health check (uses curl we installed above) HEALTHCHECK --interval=30s --timeout=30s --start-period=5s --retries=3 \ - CMD curl -f http://localhost:8000/health || exit 1 + CMD curl -fsS http://localhost:8000/health || exit 1 -# Run the application -CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000", "--reload"] \ No newline at end of file +# Run the application (dev-friendly: --reload; change to gunicorn/uvicorn w/o reload for prod) +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000", "--reload"] diff --git a/docker-compose.yml b/docker-compose.yml index 34cea7f..a89a888 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,9 @@ version: '3.8' services: fastapi-app: - build: . + build: + context: . + dockerfile: Dockerfile ports: - "8000:8000" volumes: @@ -12,6 +14,8 @@ services: environment: - PYTHONPATH=/app - DATABASE_URL=${DATABASE_URL} + - FFMPEG_BIN=/usr/bin/ffmpeg + - FFPROBE_BIN=/usr/bin/ffprobe restart: unless-stopped depends_on: postgres: @@ -19,7 +23,7 @@ services: redis: condition: service_healthy healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8000/health"] + test: ["CMD", "curl", "-fsS", "http://localhost:8000/health"] interval: 30s timeout: 10s retries: 3 @@ -67,4 +71,4 @@ networks: volumes: postgres_data: - redis_data: \ No newline at end of file + redis_data: diff --git a/scraper.py b/scraper.py index 78c099e..3b4c7b1 100644 --- a/scraper.py +++ b/scraper.py @@ -1,9 +1,15 @@ +# --- imports --- from sqlalchemy.orm import Session from database import InstagramAccount, InstagramPost from utils.boxapi_client import get_media, BoxAPIError from utils.parser_fast import parse_boxapi_items_fast from utils.seen_store import filter_unseen, mark_seen -from utils.s3_uploader import upload_media_from_url +# use optimized uploaders; keep the generic fallback as well +from utils.s3_uploader import ( + upload_media_from_url, + upload_image_from_url_optimized, + upload_video_from_url_optimized, +) from schemas import ScrapeRequest, ScrapedPost from typing import List from datetime import datetime @@ -11,8 +17,6 @@ from datetime import datetime def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]: try: - # 1) one BoxAPI call (fast) — we keep this single round trip - # Ask for more to offset duplicates raw_items, log_path = get_media(req.username, max(req.max_count, 12)) except BoxAPIError as e: print(f"❌ BoxAPI error: {e}") @@ -21,32 +25,27 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]: if not raw_items: return [] - # 2) normalize quickly normalized = parse_boxapi_items_fast(raw_items) - # 3) apply cursor filter if provided (safe, optional) + # optional cursor filter since_ts = getattr(req, "since_taken_at", None) if since_ts: try: since_ts = int(since_ts) normalized = [n for n in normalized if (n.get("taken_at") or 0) > since_ts] except Exception: - # If since_taken_at is invalid, ignore it pass - # 4) filter already-seen + # filter seen ids_in_order = [n["ig_post_id"] for n in normalized if n.get("ig_post_id")] unseen_ids = set(filter_unseen(req.username, ids_in_order)) - unseen = [n for n in normalized if n["ig_post_id"] in unseen_ids] - # 5) take the next N unseen (newest first already) + # pick next N batch = unseen[:req.max_count] - - # 6) mark them seen (only what we return) mark_seen(req.username, [b["ig_post_id"] for b in batch]) - # 7) Get or create InstagramAccount + # ensure account account = db.query(InstagramAccount).filter_by(username=req.username).first() if not account: account = InstagramAccount( @@ -63,53 +62,72 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]: new_post_count = 0 for item in batch: - post_id = item["ig_post_id"] - - # Check if already in database + post_id = str(item["ig_post_id"]) # ENSURE clean, no slashes + seller_id = str(req.seller_id) + + # skip if already saved if db.query(InstagramPost).filter_by(ig_post_id=post_id).first(): continue - # Filter media URLs to avoid duplicates (especially for videos) + # media URL list (normalized parser gives remote_urls) media_urls = list(item.get("remote_urls", [])) + # If this is a video post, keep only true video URLs if item["media_type"] == "video": thumb = (item.get("thumbnail_url") or "").strip() def is_video(u: str) -> bool: base = u.split("?", 1)[0].lower() return base.endswith((".mp4", ".mov", ".m4v", ".webm")) - # keep only real video URLs; drop image covers and anything equal to the thumbnail media_urls = [u for u in media_urls if is_video(u) and u != thumb] - # Upload remote URLs to S3 (optional - can be done async later) + # Upload media to S3 (optimized path) uploaded_urls: List[str] = [] for remote_url in media_urls: try: - local_url = upload_media_from_url(remote_url, str(req.seller_id), f"{post_id}/") - if local_url: - uploaded_urls.append(local_url) + if item["media_type"] == "video": + # optimized 480p H.264 CRF23 (+faststart) + url = upload_video_from_url_optimized(remote_url, seller_id, post_id) + elif item["media_type"] in ("image", "carousel"): + # carousel items may be images; optimize to WebP + url = upload_image_from_url_optimized(remote_url, seller_id, post_id) + if url is None: + # some carousels include videos; try generic fallback + url = upload_media_from_url(remote_url, seller_id, post_id) + else: + # unknown → generic detection/fallback + url = upload_media_from_url(remote_url, seller_id, post_id) + + if url: + uploaded_urls.append(url) except Exception as e: print(f"⚠️ Error uploading media: {e}") continue - # Upload thumbnail (optional) - final_thumbnail = item["thumbnail_url"] - if item["thumbnail_url"]: + # Upload/optimize thumbnail (if present) + final_thumbnail = item.get("thumbnail_url") or None + if final_thumbnail: try: - thumb_uploaded = upload_media_from_url(item["thumbnail_url"], str(req.seller_id), f"{post_id}/thumbnail") - if thumb_uploaded: - final_thumbnail = thumb_uploaded + thumb_url = upload_image_from_url_optimized(final_thumbnail, seller_id, post_id) + if thumb_url: + final_thumbnail = thumb_url + else: + # fallback to generic (no optimize) + alt = upload_media_from_url(final_thumbnail, seller_id, post_id) + if alt: + final_thumbnail = alt except Exception as e: print(f"⚠️ Failed to upload thumbnail: {e}") - # Save to DB + # Save + created_at = datetime.fromtimestamp(item["taken_at"]) if item.get("taken_at") else datetime.utcnow() post_entry = InstagramPost( ig_post_id=post_id, account_id=account.id, media_type=item["media_type"], - caption=item["caption"], + caption=item.get("caption"), thumbnail_url=final_thumbnail, - local_media=uploaded_urls, # Use filtered S3 URLs - created_at=datetime.fromtimestamp(item["taken_at"]), + local_media=uploaded_urls, + created_at=created_at, ) db.add(post_entry) db.commit() @@ -117,10 +135,10 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]: result.append(ScrapedPost( ig_post_id=post_id, media_type=item["media_type"], - caption=item["caption"], + caption=item.get("caption"), thumbnail_url=final_thumbnail, - local_media=uploaded_urls, # Use filtered S3 URLs - created_at=datetime.fromtimestamp(item["taken_at"]) + local_media=uploaded_urls, + created_at=created_at, )) new_post_count += 1 diff --git a/utils/media_processing.py b/utils/media_processing.py new file mode 100644 index 0000000..6a0b0e3 --- /dev/null +++ b/utils/media_processing.py @@ -0,0 +1,193 @@ +import io +import os +import re +import subprocess +import tempfile +from typing import Tuple, List +from PIL import Image +import requests + +FFMPEG = os.getenv("FFMPEG_BIN", "ffmpeg") +FFPROBE = os.getenv("FFPROBE_BIN", "ffprobe") + +# ------------------------ +# Shared helpers +# ------------------------ + +def _bool(name: str, default: bool) -> bool: + v = os.getenv(name) + if v is None: + return default + return v.strip().lower() in ("1", "true", "yes", "y") + +def _int(name: str, default: int) -> int: + try: + return int(os.getenv(name, str(default))) + except Exception: + return default + +def _float(name: str, default: float) -> float: + try: + return float(os.getenv(name, str(default))) + except Exception: + return default + +def _parse_max_size(default: Tuple[int, int] = (1600, 1600)) -> Tuple[int, int]: + raw = os.getenv("IMAGE_MAX_SIZE", f"{default[0]}x{default[1]}").strip().lower() + m = re.match(r"^\s*(\d+)\s*[x,]\s*(\d+)\s*$", raw) + if not m: + return default + return (int(m.group(1)), int(m.group(2))) + +def download_bytes(url: str, timeout: int = 30) -> bytes: + r = requests.get(url, stream=True, timeout=timeout) + r.raise_for_status() + return r.content + +# ------------------------ +# Image processing (WebP) +# ------------------------ + +def process_image_to_webp_single(image_bytes: bytes) -> Tuple[str, bytes, str]: + """ + Resize image to: + - scale percent from .env (default 0.85) + - and cap to max size from .env (default 1600x1600) + Save as WebP with quality from .env (default 85). + + Returns (variant_name, data, content_type) + variant_name: 'webp' + """ + scale_percent = _float("IMAGE_SCALE_PERCENT", 0.85) + max_w, max_h = _parse_max_size((1600, 1600)) + quality = _int("IMAGE_QUALITY", 85) + + img = Image.open(io.BytesIO(image_bytes)).convert("RGB") + orig_w, orig_h = img.size + + # First apply percentage scaling + target_w = max(1, int(orig_w * scale_percent)) + target_h = max(1, int(orig_h * scale_percent)) + + # Then cap to max size if still too large + ratio_w = max_w / target_w if target_w > max_w else 1.0 + ratio_h = max_h / target_h if target_h > max_h else 1.0 + cap_ratio = min(ratio_w, ratio_h) + if cap_ratio < 1.0: + target_w = max(1, int(target_w * cap_ratio)) + target_h = max(1, int(target_h * cap_ratio)) + + # Resize using LANCZOS + resized = img.resize((target_w, target_h), Image.LANCZOS) + + out = io.BytesIO() + # method=6 is a good quality/speed balance for WebP + resized.save(out, format="WEBP", quality=quality, method=6) + return ("webp", out.getvalue(), "image/webp") + +# ------------------------ +# Video processing (H.264 @ 480p) +# ------------------------ + +def _probe_streams(path: str) -> dict: + """Return ffprobe json for streams (video/audio codecs, dimensions).""" + # Minimal probing; not strictly required but helps with SIMPLE mode decisions + cmd = [ + FFPROBE, "-v", "error", "-select_streams", "v:0", + "-show_entries", "stream=codec_name,width,height", + "-of", "json", path + ] + try: + out = subprocess.run(cmd, check=True, capture_output=True).stdout + import json + return json.loads(out.decode("utf-8")) + except Exception: + return {} + +def transcode_video_480_h264(video_bytes: bytes) -> Tuple[str, bytes, str]: + """ + Transcode to MP4 (H.264/AAC), height=VIDEO_HEIGHT (default 480), CRF=VIDEO_CRF (default 23). + Adds +faststart unless disabled. Uses PRESET (default veryfast). + If VIDEO_SIMPLE_MODE=true: + - Use 'ultrafast' preset automatically + - If input already H.264/AAC and <= target height, attempt stream-copy with -movflags +faststart + (very quick) — otherwise fallback to a minimal transcode. + Returns (variant_name, data, content_type) where variant_name='480p' + """ + height = _int("VIDEO_HEIGHT", 480) + crf = _int("VIDEO_CRF", 23) + preset = os.getenv("VIDEO_PRESET", "veryfast") + faststart = _bool("VIDEO_FASTSTART", True) + simple = _bool("VIDEO_SIMPLE_MODE", False) + + with tempfile.TemporaryDirectory() as td: + in_path = os.path.join(td, "in.bin") + out_path = os.path.join(td, f"out-{height}p.mp4") + with open(in_path, "wb") as f: + f.write(video_bytes) + + vf = f"scale=-2:{height}" + + # SIMPLE MODE quick path + if simple: + # Try to detect if we can safely stream copy + meta = _probe_streams(in_path) + v_ok = False + try: + st = meta.get("streams", [{}])[0] + vcodec = st.get("codec_name") + v_h = int(st.get("height", 10**9)) + v_ok = (vcodec == "h264" and v_h <= height) + except Exception: + v_ok = False + + if v_ok: + # Quick remux with faststart; try copy video/audio + cmd = [FFMPEG, "-y", "-i", in_path, "-c", "copy"] + if faststart: + cmd += ["-movflags", "+faststart"] + cmd += [out_path] + try: + subprocess.run(cmd, check=True, capture_output=True) + with open(out_path, "rb") as f: + return (f"{height}p", f.read(), "video/mp4") + except Exception: + # fall back to quick transcode below + pass + + # Quick transcode with ultrafast + cmd = [ + FFMPEG, "-y", + "-i", in_path, + "-vf", vf, + "-c:v", "libx264", + "-preset", "ultrafast", # force quickest in simple mode + "-crf", str(crf), + "-c:a", "aac", + "-b:a", "128k", + ] + if faststart: + cmd += ["-movflags", "+faststart"] + cmd += [out_path] + subprocess.run(cmd, check=True, capture_output=True) + with open(out_path, "rb") as f: + return (f"{height}p", f.read(), "video/mp4") + + # Normal path (preset configurable) + cmd = [ + FFMPEG, "-y", + "-i", in_path, + "-vf", vf, + "-c:v", "libx264", + "-preset", preset, + "-crf", str(crf), + "-c:a", "aac", + "-b:a", "128k", + ] + if faststart: + cmd += ["-movflags", "+faststart"] + cmd += [out_path] + + subprocess.run(cmd, check=True, capture_output=True) + with open(out_path, "rb") as f: + return (f"{height}p", f.read(), "video/mp4") diff --git a/utils/s3_uploader.py b/utils/s3_uploader.py index d23495c..e12bc88 100644 --- a/utils/s3_uploader.py +++ b/utils/s3_uploader.py @@ -1,10 +1,14 @@ import os -import boto3 +import re import uuid import mimetypes -from urllib.parse import urljoin +from typing import Optional + +import boto3 import requests +# ---- ENV / S3 setup --------------------------------------------------------- + AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID") AWS_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY") AWS_STORAGE_BUCKET_NAME = os.getenv("AWS_STORAGE_BUCKET_NAME") @@ -12,6 +16,16 @@ S3_ENDPOINT_URL = os.getenv("S3_ENDPOINT_URL") S3_REGION = os.getenv("S3_REGION", "us-east-1") MEDIA_BASE_PATH = os.getenv("MEDIA_BASE_PATH", "instagram") +# Optimization toggles +MEDIA_OPTIMIZE_IMAGES = os.getenv("MEDIA_OPTIMIZE_IMAGES", "true").strip().lower() in ("1", "true", "yes", "y") +MEDIA_OPTIMIZE_VIDEOS = os.getenv("MEDIA_OPTIMIZE_VIDEOS", "true").strip().lower() in ("1", "true", "yes", "y") +KEEP_ORIGINAL_IMAGES = os.getenv("KEEP_ORIGINAL_IMAGES", "false").strip().lower() in ("1", "true", "yes", "y") +KEEP_ORIGINAL_VIDEOS = os.getenv("KEEP_ORIGINAL_VIDEOS", "false").strip().lower() in ("1", "true", "yes", "y") + +# HTTP +DOWNLOAD_TIMEOUT = int(os.getenv("DOWNLOAD_TIMEOUT", "30")) + +# S3 client session = boto3.session.Session() s3 = session.client( service_name="s3", @@ -21,30 +35,171 @@ s3 = session.client( region_name=S3_REGION, ) +# ---- Media processing imports ---------------------------------------------- +# These utilities implement your exact constraints: +# - Images: scale 85%, cap 1600x1600, quality 85 → WebP +# - Videos: H.264, 480p, CRF 23, +faststart (with SIMPLE mode support) +from utils.media_processing import ( # type: ignore + download_bytes, + process_image_to_webp_single, + transcode_video_480_h264, +) -def upload_media_from_url(media_url: str, seller_id: str, post_id: str) -> str | None: - try: - response = requests.get(media_url, stream=True, timeout=10) - response.raise_for_status() - content_type = response.headers.get("Content-Type", "") - ext = mimetypes.guess_extension(content_type) or ".jpg" - if ext in [".jpe", ".jfif", ".bin"]: - ext = ".jpg" +# ---- Helpers ---------------------------------------------------------------- - filename = f"{uuid.uuid4()}{ext}" - s3_path = f"{MEDIA_BASE_PATH}/{seller_id}/{post_id}/{filename}" +def _safe_basename_from_url(url: str) -> str: + name = url.split("?")[0].rstrip("/").split("/")[-1] + name = name.rsplit(".", 1)[0] if "." in name else name + name = re.sub(r"[^a-zA-Z0-9._-]+", "-", name).strip("-") + return name or uuid.uuid4().hex[:8] + +def _guess_ext_from_content_type(content_type: str, fallback: str = ".bin") -> str: + ext = mimetypes.guess_extension(content_type or "") or fallback + # normalize oddballs + if ext in [".jpe", ".jfif", ".bin"]: + ext = ".jpg" + return ext + + +def _upload_bytes(data: bytes, content_type: str, seller_id: str, post_id: str, key: str) -> str: + """ + Upload a byte blob to S3 at: + {MEDIA_BASE_PATH}/{seller_id}/{post_id}/{key} + Returns a public URL. + """ + s3_path = f"{MEDIA_BASE_PATH}/{seller_id}/{post_id}/{key}" + s3.put_object( + Bucket=AWS_STORAGE_BUCKET_NAME, + Key=s3_path, + Body=data, + ACL="public-read", + ContentType=content_type or "application/octet-stream", + ) + return f"{S3_ENDPOINT_URL}/{AWS_STORAGE_BUCKET_NAME}/{s3_path}" + + +def _stream_to_s3_from_url(url: str, seller_id: str, post_id: str, key: str, content_type: Optional[str] = None) -> str: + """ + Stream a remote URL directly to S3 (no processing). + """ + with requests.get(url, stream=True, timeout=DOWNLOAD_TIMEOUT) as r: + r.raise_for_status() + ctype = content_type or r.headers.get("Content-Type") or "application/octet-stream" + s3_path = f"{MEDIA_BASE_PATH}/{seller_id}/{post_id}/{key}" s3.upload_fileobj( - response.raw, + r.raw, Bucket=AWS_STORAGE_BUCKET_NAME, Key=s3_path, - ExtraArgs={"ACL": "public-read", "ContentType": content_type} + ExtraArgs={"ACL": "public-read", "ContentType": ctype}, ) + return f"{S3_ENDPOINT_URL}/{AWS_STORAGE_BUCKET_NAME}/{s3_path}" - public_url = f"{S3_ENDPOINT_URL}/{AWS_STORAGE_BUCKET_NAME}/{s3_path}" - return public_url + +# ---- Public API ------------------------------------------------------------- + +def upload_image_from_url_optimized(image_url: str, seller_id: str, post_id: str) -> Optional[str]: + """ + Download -> (optional) optimize to WebP -> upload -> return public URL. + If optimization fails, falls back to direct upload. + """ + base = _safe_basename_from_url(image_url) + + try: + if MEDIA_OPTIMIZE_IMAGES: + # Download full bytes, run WebP pipeline (85% scale, max 1600x1600, quality 85) + blob = download_bytes(image_url, timeout=DOWNLOAD_TIMEOUT) + variant_name, data, ctype = process_image_to_webp_single(blob) + url_out = _upload_bytes(data, ctype, seller_id, post_id, f"images/{base}-{variant_name}.webp") + + if KEEP_ORIGINAL_IMAGES: + # Keep original (best-effort content type) + _upload_bytes(blob, "application/octet-stream", seller_id, post_id, f"images/{base}-original.bin") + + return url_out + + # No optimization: stream to S3 as-is + # Try to pick a reasonable extension based on content type + head = requests.head(image_url, timeout=DOWNLOAD_TIMEOUT, allow_redirects=True) + content_type = head.headers.get("Content-Type", "") + ext = _guess_ext_from_content_type(content_type, ".jpg") + return _stream_to_s3_from_url(image_url, seller_id, post_id, f"images/{base}{ext}", content_type) except Exception as e: - print(f"❌ Failed to upload media from {media_url}: {e}") - return None \ No newline at end of file + print(f"❌ Image upload failed (optimized path). Falling back. url={image_url} err={e}") + # Final fallback: best-effort stream + try: + return _stream_to_s3_from_url(image_url, seller_id, post_id, f"images/{base}.bin", "application/octet-stream") + except Exception as e2: + print(f"❌ Image upload fallback failed. url={image_url} err={e2}") + return None + + +def upload_video_from_url_optimized(video_url: str, seller_id: str, post_id: str) -> Optional[str]: + """ + Download -> (optional) transcode to MP4 (H.264, 480p, CRF 23, +faststart) -> upload -> return public URL. + If optimization fails, falls back to direct upload. + """ + base = _safe_basename_from_url(video_url) + + try: + if MEDIA_OPTIMIZE_VIDEOS: + blob = download_bytes(video_url, timeout=DOWNLOAD_TIMEOUT) + variant_name, data, ctype = transcode_video_480_h264(blob) + url_out = _upload_bytes(data, ctype, seller_id, post_id, f"videos/{base}-{variant_name}.mp4") + + if KEEP_ORIGINAL_VIDEOS: + _upload_bytes(blob, "application/octet-stream", seller_id, post_id, f"videos/{base}-original.bin") + + return url_out + + # No optimization: stream to S3 as-is + head = requests.head(video_url, timeout=DOWNLOAD_TIMEOUT, allow_redirects=True) + content_type = head.headers.get("Content-Type", "") + ext = _guess_ext_from_content_type(content_type, ".mp4") + return _stream_to_s3_from_url(video_url, seller_id, post_id, f"videos/{base}{ext}", content_type) + + except Exception as e: + print(f"❌ Video upload failed (optimized path). Falling back. url={video_url} err={e}") + # Final fallback: best-effort stream + try: + return _stream_to_s3_from_url(video_url, seller_id, post_id, f"videos/{base}.bin", "application/octet-stream") + except Exception as e2: + print(f"❌ Video upload fallback failed. url={video_url} err={e2}") + return None + + +def upload_media_from_url(media_url: str, seller_id: str, post_id: str) -> Optional[str]: + """ + Backward-compatible alias if callers don't know type. + Heuristic: decide by Content-Type, optimize when possible. + """ + try: + head = requests.head(media_url, timeout=DOWNLOAD_TIMEOUT, allow_redirects=True) + content_type = head.headers.get("Content-Type", "").lower() + + if "image" in content_type: + return upload_image_from_url_optimized(media_url, seller_id, post_id) + if "video" in content_type: + return upload_video_from_url_optimized(media_url, seller_id, post_id) + + # Unknown → just stream raw + base = _safe_basename_from_url(media_url) + return _stream_to_s3_from_url(media_url, seller_id, post_id, f"misc/{base}", content_type or "application/octet-stream") + + except Exception as e: + print(f"❌ HEAD failed for {media_url}: {e}") + # If HEAD fails, still try image path → then video path → then raw + url = upload_image_from_url_optimized(media_url, seller_id, post_id) + if url: + return url + url = upload_video_from_url_optimized(media_url, seller_id, post_id) + if url: + return url + try: + base = _safe_basename_from_url(media_url) + return _stream_to_s3_from_url(media_url, seller_id, post_id, f"misc/{base}.bin", "application/octet-stream") + except Exception as e2: + print(f"❌ Ultimate fallback failed for {media_url}: {e2}") + return None