fazli/FastApi-ISS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Route carousel media by URL type to avoid double-downloading videos

Browse Source 
Carousels return a mixed image+video URL list. The uploader keyed on the
post's media_type, so every carousel URL went through the image optimizer;
videos failed PIL ('cannot identify image file') and were then re-downloaded
for the raw fallback. That double download per video pushed the synchronous
scrape past the backend's request timeout, so the backend imported 0 while
the engine still saved posts to its own DB (causing a dedup desync on retry).
Detect video URLs up front and send them straight to the video uploader.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
fazli 2026-06-29 11:01:29 +00:00
parent 62a03c61fe
commit ecebb100b7
1 changed files with 15 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
scraper.py
View File

@ -202,24 +202,32 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]:
# Upload media # Upload media
media_urls = list(item.get("remote_urls", [])) media_urls = list(item.get("remote_urls", []))
def is_video(u: str) -> bool:
base = u.split("?", 1)[0].lower()
return base.endswith((".mp4", ".mov", ".m4v", ".webm"))
if item["media_type"] == "video": if item["media_type"] == "video":
thumb = (item.get("thumbnail_url") or "").strip() thumb = (item.get("thumbnail_url") or "").strip()
def is_video(u: str) -> bool:
base = u.split("?", 1)[0].lower()
return base.endswith((".mp4", ".mov", ".m4v", ".webm"))
media_urls = [u for u in media_urls if is_video(u) and u != thumb] media_urls = [u for u in media_urls if is_video(u) and u != thumb]
uploaded_urls: List[str] = [] uploaded_urls: List[str] = []
for remote_url in media_urls: for remote_url in media_urls:
try: try:
if item["media_type"] == "video": # Route each URL by its actual type, not the post's media_type.
# Carousels mix images and videos in one remote_urls list; sending
# a video through the image optimizer makes PIL fail ("cannot
# identify image file") and then re-download it for the raw
# fallback — a double download per video that pushes the whole
# scrape past the backend's request timeout. Detecting video URLs
# up front sends them straight to the video uploader.
if is_video(remote_url):
url = upload_video_from_url_optimized(remote_url, seller_id, post_id) url = upload_video_from_url_optimized(remote_url, seller_id, post_id)
elif item["media_type"] in ("image", "carousel"):
url = upload_image_from_url_optimized(remote_url, seller_id, post_id)
if not url: if not url:
url = upload_media_from_url(remote_url, seller_id, post_id) url = upload_media_from_url(remote_url, seller_id, post_id)
else: else:
url = upload_media_from_url(remote_url, seller_id, post_id) url = upload_image_from_url_optimized(remote_url, seller_id, post_id)
if not url:
url = upload_media_from_url(remote_url, seller_id, post_id)
if url: if url:
uploaded_urls.append(url) uploaded_urls.append(url)
except Exception as e: except Exception as e: