From ecebb100b771c07531ada60c89e9d10b83588efb Mon Sep 17 00:00:00 2001 From: fazli Date: Mon, 29 Jun 2026 11:01:29 +0000 Subject: [PATCH] Route carousel media by URL type to avoid double-downloading videos Carousels return a mixed image+video URL list. The uploader keyed on the post's media_type, so every carousel URL went through the image optimizer; videos failed PIL ('cannot identify image file') and were then re-downloaded for the raw fallback. That double download per video pushed the synchronous scrape past the backend's request timeout, so the backend imported 0 while the engine still saved posts to its own DB (causing a dedup desync on retry). Detect video URLs up front and send them straight to the video uploader. Co-Authored-By: Claude Opus 4.8 --- scraper.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/scraper.py b/scraper.py index 995ed11..269a35d 100644 --- a/scraper.py +++ b/scraper.py @@ -202,24 +202,32 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]: # Upload media media_urls = list(item.get("remote_urls", [])) + def is_video(u: str) -> bool: + base = u.split("?", 1)[0].lower() + return base.endswith((".mp4", ".mov", ".m4v", ".webm")) + if item["media_type"] == "video": thumb = (item.get("thumbnail_url") or "").strip() - def is_video(u: str) -> bool: - base = u.split("?", 1)[0].lower() - return base.endswith((".mp4", ".mov", ".m4v", ".webm")) media_urls = [u for u in media_urls if is_video(u) and u != thumb] uploaded_urls: List[str] = [] for remote_url in media_urls: try: - if item["media_type"] == "video": + # Route each URL by its actual type, not the post's media_type. + # Carousels mix images and videos in one remote_urls list; sending + # a video through the image optimizer makes PIL fail ("cannot + # identify image file") and then re-download it for the raw + # fallback — a double download per video that pushes the whole + # scrape past the backend's request timeout. Detecting video URLs + # up front sends them straight to the video uploader. + if is_video(remote_url): url = upload_video_from_url_optimized(remote_url, seller_id, post_id) - elif item["media_type"] in ("image", "carousel"): - url = upload_image_from_url_optimized(remote_url, seller_id, post_id) if not url: url = upload_media_from_url(remote_url, seller_id, post_id) else: - url = upload_media_from_url(remote_url, seller_id, post_id) + url = upload_image_from_url_optimized(remote_url, seller_id, post_id) + if not url: + url = upload_media_from_url(remote_url, seller_id, post_id) if url: uploaded_urls.append(url) except Exception as e: