206 lines
8.4 KiB
Python
206 lines
8.4 KiB
Python
import os
|
|
import re
|
|
import uuid
|
|
import mimetypes
|
|
from typing import Optional
|
|
|
|
import boto3
|
|
import requests
|
|
|
|
# ---- ENV / S3 setup ---------------------------------------------------------
|
|
|
|
AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID")
|
|
AWS_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY")
|
|
AWS_STORAGE_BUCKET_NAME = os.getenv("AWS_STORAGE_BUCKET_NAME")
|
|
S3_ENDPOINT_URL = os.getenv("S3_ENDPOINT_URL")
|
|
S3_REGION = os.getenv("S3_REGION", "us-east-1")
|
|
MEDIA_BASE_PATH = os.getenv("MEDIA_BASE_PATH", "instagram")
|
|
|
|
# Optimization toggles
|
|
MEDIA_OPTIMIZE_IMAGES = os.getenv("MEDIA_OPTIMIZE_IMAGES", "true").strip().lower() in ("1", "true", "yes", "y")
|
|
MEDIA_OPTIMIZE_VIDEOS = os.getenv("MEDIA_OPTIMIZE_VIDEOS", "true").strip().lower() in ("1", "true", "yes", "y")
|
|
KEEP_ORIGINAL_IMAGES = os.getenv("KEEP_ORIGINAL_IMAGES", "false").strip().lower() in ("1", "true", "yes", "y")
|
|
KEEP_ORIGINAL_VIDEOS = os.getenv("KEEP_ORIGINAL_VIDEOS", "false").strip().lower() in ("1", "true", "yes", "y")
|
|
|
|
# HTTP
|
|
DOWNLOAD_TIMEOUT = int(os.getenv("DOWNLOAD_TIMEOUT", "30"))
|
|
|
|
# S3 client
|
|
session = boto3.session.Session()
|
|
s3 = session.client(
|
|
service_name="s3",
|
|
aws_access_key_id=AWS_ACCESS_KEY_ID,
|
|
aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
|
|
endpoint_url=S3_ENDPOINT_URL,
|
|
region_name=S3_REGION,
|
|
)
|
|
|
|
# ---- Media processing imports ----------------------------------------------
|
|
# These utilities implement your exact constraints:
|
|
# - Images: scale 85%, cap 1600x1600, quality 85 → WebP
|
|
# - Videos: H.264, 480p, CRF 23, +faststart (with SIMPLE mode support)
|
|
from utils.media_processing import ( # type: ignore
|
|
download_bytes,
|
|
process_image_to_webp_single,
|
|
transcode_video_480_h264,
|
|
)
|
|
|
|
|
|
# ---- Helpers ----------------------------------------------------------------
|
|
|
|
def _safe_basename_from_url(url: str) -> str:
|
|
name = url.split("?")[0].rstrip("/").split("/")[-1]
|
|
name = name.rsplit(".", 1)[0] if "." in name else name
|
|
name = re.sub(r"[^a-zA-Z0-9._-]+", "-", name).strip("-")
|
|
return name or uuid.uuid4().hex[:8]
|
|
|
|
|
|
def _guess_ext_from_content_type(content_type: str, fallback: str = ".bin") -> str:
|
|
ext = mimetypes.guess_extension(content_type or "") or fallback
|
|
# normalize oddballs
|
|
if ext in [".jpe", ".jfif", ".bin"]:
|
|
ext = ".jpg"
|
|
return ext
|
|
|
|
|
|
def _upload_bytes(data: bytes, content_type: str, seller_id: str, post_id: str, key: str) -> str:
|
|
"""
|
|
Upload a byte blob to S3 at:
|
|
{MEDIA_BASE_PATH}/{seller_id}/{post_id}/{key}
|
|
Returns a public URL.
|
|
"""
|
|
s3_path = f"{MEDIA_BASE_PATH}/{seller_id}/{post_id}/{key}"
|
|
s3.put_object(
|
|
Bucket=AWS_STORAGE_BUCKET_NAME,
|
|
Key=s3_path,
|
|
Body=data,
|
|
ACL="public-read",
|
|
ContentType=content_type or "application/octet-stream",
|
|
)
|
|
return f"{S3_ENDPOINT_URL}/{AWS_STORAGE_BUCKET_NAME}/{s3_path}"
|
|
|
|
|
|
def _stream_to_s3_from_url(url: str, seller_id: str, post_id: str, key: str, content_type: Optional[str] = None) -> str:
|
|
"""
|
|
Stream a remote URL directly to S3 (no processing).
|
|
"""
|
|
with requests.get(url, stream=True, timeout=DOWNLOAD_TIMEOUT) as r:
|
|
r.raise_for_status()
|
|
ctype = content_type or r.headers.get("Content-Type") or "application/octet-stream"
|
|
s3_path = f"{MEDIA_BASE_PATH}/{seller_id}/{post_id}/{key}"
|
|
s3.upload_fileobj(
|
|
r.raw,
|
|
Bucket=AWS_STORAGE_BUCKET_NAME,
|
|
Key=s3_path,
|
|
ExtraArgs={"ACL": "public-read", "ContentType": ctype},
|
|
)
|
|
return f"{S3_ENDPOINT_URL}/{AWS_STORAGE_BUCKET_NAME}/{s3_path}"
|
|
|
|
|
|
# ---- Public API -------------------------------------------------------------
|
|
|
|
def upload_image_from_url_optimized(image_url: str, seller_id: str, post_id: str) -> Optional[str]:
|
|
"""
|
|
Download -> (optional) optimize to WebP -> upload -> return public URL.
|
|
If optimization fails, falls back to direct upload.
|
|
"""
|
|
base = _safe_basename_from_url(image_url)
|
|
|
|
try:
|
|
if MEDIA_OPTIMIZE_IMAGES:
|
|
# Download full bytes, run WebP pipeline (85% scale, max 1600x1600, quality 85)
|
|
blob = download_bytes(image_url, timeout=DOWNLOAD_TIMEOUT)
|
|
variant_name, data, ctype = process_image_to_webp_single(blob)
|
|
url_out = _upload_bytes(data, ctype, seller_id, post_id, f"images/{base}-{variant_name}.webp")
|
|
|
|
if KEEP_ORIGINAL_IMAGES:
|
|
# Keep original (best-effort content type)
|
|
_upload_bytes(blob, "application/octet-stream", seller_id, post_id, f"images/{base}-original.bin")
|
|
|
|
return url_out
|
|
|
|
# No optimization: stream to S3 as-is
|
|
# Try to pick a reasonable extension based on content type
|
|
head = requests.head(image_url, timeout=DOWNLOAD_TIMEOUT, allow_redirects=True)
|
|
content_type = head.headers.get("Content-Type", "")
|
|
ext = _guess_ext_from_content_type(content_type, ".jpg")
|
|
return _stream_to_s3_from_url(image_url, seller_id, post_id, f"images/{base}{ext}", content_type)
|
|
|
|
except Exception as e:
|
|
print(f"❌ Image upload failed (optimized path). Falling back. url={image_url} err={e}")
|
|
# Final fallback: best-effort stream
|
|
try:
|
|
return _stream_to_s3_from_url(image_url, seller_id, post_id, f"images/{base}.bin", "application/octet-stream")
|
|
except Exception as e2:
|
|
print(f"❌ Image upload fallback failed. url={image_url} err={e2}")
|
|
return None
|
|
|
|
|
|
def upload_video_from_url_optimized(video_url: str, seller_id: str, post_id: str) -> Optional[str]:
|
|
"""
|
|
Download -> (optional) transcode to MP4 (H.264, 480p, CRF 23, +faststart) -> upload -> return public URL.
|
|
If optimization fails, falls back to direct upload.
|
|
"""
|
|
base = _safe_basename_from_url(video_url)
|
|
|
|
try:
|
|
if MEDIA_OPTIMIZE_VIDEOS:
|
|
blob = download_bytes(video_url, timeout=DOWNLOAD_TIMEOUT)
|
|
variant_name, data, ctype = transcode_video_480_h264(blob)
|
|
url_out = _upload_bytes(data, ctype, seller_id, post_id, f"videos/{base}-{variant_name}.mp4")
|
|
|
|
if KEEP_ORIGINAL_VIDEOS:
|
|
_upload_bytes(blob, "application/octet-stream", seller_id, post_id, f"videos/{base}-original.bin")
|
|
|
|
return url_out
|
|
|
|
# No optimization: stream to S3 as-is
|
|
head = requests.head(video_url, timeout=DOWNLOAD_TIMEOUT, allow_redirects=True)
|
|
content_type = head.headers.get("Content-Type", "")
|
|
ext = _guess_ext_from_content_type(content_type, ".mp4")
|
|
return _stream_to_s3_from_url(video_url, seller_id, post_id, f"videos/{base}{ext}", content_type)
|
|
|
|
except Exception as e:
|
|
print(f"❌ Video upload failed (optimized path). Falling back. url={video_url} err={e}")
|
|
# Final fallback: best-effort stream
|
|
try:
|
|
return _stream_to_s3_from_url(video_url, seller_id, post_id, f"videos/{base}.bin", "application/octet-stream")
|
|
except Exception as e2:
|
|
print(f"❌ Video upload fallback failed. url={video_url} err={e2}")
|
|
return None
|
|
|
|
|
|
def upload_media_from_url(media_url: str, seller_id: str, post_id: str) -> Optional[str]:
|
|
"""
|
|
Backward-compatible alias if callers don't know type.
|
|
Heuristic: decide by Content-Type, optimize when possible.
|
|
"""
|
|
try:
|
|
head = requests.head(media_url, timeout=DOWNLOAD_TIMEOUT, allow_redirects=True)
|
|
content_type = head.headers.get("Content-Type", "").lower()
|
|
|
|
if "image" in content_type:
|
|
return upload_image_from_url_optimized(media_url, seller_id, post_id)
|
|
if "video" in content_type:
|
|
return upload_video_from_url_optimized(media_url, seller_id, post_id)
|
|
|
|
# Unknown → just stream raw
|
|
base = _safe_basename_from_url(media_url)
|
|
return _stream_to_s3_from_url(media_url, seller_id, post_id, f"misc/{base}", content_type or "application/octet-stream")
|
|
|
|
except Exception as e:
|
|
print(f"❌ HEAD failed for {media_url}: {e}")
|
|
# If HEAD fails, still try image path → then video path → then raw
|
|
url = upload_image_from_url_optimized(media_url, seller_id, post_id)
|
|
if url:
|
|
return url
|
|
url = upload_video_from_url_optimized(media_url, seller_id, post_id)
|
|
if url:
|
|
return url
|
|
try:
|
|
base = _safe_basename_from_url(media_url)
|
|
return _stream_to_s3_from_url(media_url, seller_id, post_id, f"misc/{base}.bin", "application/octet-stream")
|
|
except Exception as e2:
|
|
print(f"❌ Ultimate fallback failed for {media_url}: {e2}")
|
|
return None
|