From ce86d474a38cdd150e909303120a3c8a4980b97d Mon Sep 17 00:00:00 2001 From: Hossein Date: Mon, 11 Aug 2025 22:58:46 +0330 Subject: [PATCH] update the scrapper.py --- scraper.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/scraper.py b/scraper.py index 2d05e7c..a4ee758 100644 --- a/scraper.py +++ b/scraper.py @@ -24,9 +24,15 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]: # 2) normalize quickly normalized = parse_boxapi_items_fast(raw_items) - # 3) apply cursor filter if provided - if req.since_taken_at: - normalized = [n for n in normalized if n["taken_at"] < req.since_taken_at] + # 3) apply cursor filter if provided (safe, optional) + since_ts = getattr(req, "since_taken_at", None) + if since_ts: + try: + since_ts = int(since_ts) + normalized = [n for n in normalized if (n.get("taken_at") or 0) > since_ts] + except Exception: + # If since_taken_at is invalid, ignore it + pass # 4) filter already-seen ids_in_order = [n["ig_post_id"] for n in normalized if n.get("ig_post_id")] @@ -102,9 +108,8 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]: media_type=item["media_type"], caption=item["caption"], thumbnail_url=final_thumbnail, - remote_urls=item["remote_urls"], # Return remote URLs for async processing - taken_at=item["taken_at"], - code=item.get("code") + local_media=media_urls, # Return S3 URLs only + created_at=datetime.fromtimestamp(item["taken_at"]) )) new_post_count += 1