diff --git a/scraper.py b/scraper.py index a4ee758..78c099e 100644 --- a/scraper.py +++ b/scraper.py @@ -69,13 +69,24 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]: if db.query(InstagramPost).filter_by(ig_post_id=post_id).first(): continue + # Filter media URLs to avoid duplicates (especially for videos) + media_urls = list(item.get("remote_urls", [])) + + if item["media_type"] == "video": + thumb = (item.get("thumbnail_url") or "").strip() + def is_video(u: str) -> bool: + base = u.split("?", 1)[0].lower() + return base.endswith((".mp4", ".mov", ".m4v", ".webm")) + # keep only real video URLs; drop image covers and anything equal to the thumbnail + media_urls = [u for u in media_urls if is_video(u) and u != thumb] + # Upload remote URLs to S3 (optional - can be done async later) - media_urls: List[str] = [] - for remote_url in item["remote_urls"]: + uploaded_urls: List[str] = [] + for remote_url in media_urls: try: local_url = upload_media_from_url(remote_url, str(req.seller_id), f"{post_id}/") if local_url: - media_urls.append(local_url) + uploaded_urls.append(local_url) except Exception as e: print(f"⚠️ Error uploading media: {e}") continue @@ -97,7 +108,7 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]: media_type=item["media_type"], caption=item["caption"], thumbnail_url=final_thumbnail, - local_media=media_urls, + local_media=uploaded_urls, # Use filtered S3 URLs created_at=datetime.fromtimestamp(item["taken_at"]), ) db.add(post_entry) @@ -108,7 +119,7 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]: media_type=item["media_type"], caption=item["caption"], thumbnail_url=final_thumbnail, - local_media=media_urls, # Return S3 URLs only + local_media=uploaded_urls, # Use filtered S3 URLs created_at=datetime.fromtimestamp(item["taken_at"]) ))