fazli/FastApi-ISS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update the scrapper.py

Browse Source
This commit is contained in:
Hossein 2025-08-11 22:58:46 +03:30
parent 49edb4a8cf
commit ce86d474a3
1 changed files with 11 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
scraper.py
View File

@ -24,9 +24,15 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]:
# 2) normalize quickly # 2) normalize quickly
normalized = parse_boxapi_items_fast(raw_items) normalized = parse_boxapi_items_fast(raw_items)
# 3) apply cursor filter if provided # 3) apply cursor filter if provided (safe, optional)
if req.since_taken_at: since_ts = getattr(req, "since_taken_at", None)
normalized = [n for n in normalized if n["taken_at"] < req.since_taken_at] if since_ts:
try:
since_ts = int(since_ts)
normalized = [n for n in normalized if (n.get("taken_at") or 0) > since_ts]
except Exception:
# If since_taken_at is invalid, ignore it
pass
# 4) filter already-seen # 4) filter already-seen
ids_in_order = [n["ig_post_id"] for n in normalized if n.get("ig_post_id")] ids_in_order = [n["ig_post_id"] for n in normalized if n.get("ig_post_id")]
@ -102,9 +108,8 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]:
media_type=item["media_type"], media_type=item["media_type"],
caption=item["caption"], caption=item["caption"],
thumbnail_url=final_thumbnail, thumbnail_url=final_thumbnail,
remote_urls=item["remote_urls"], # Return remote URLs for async processing local_media=media_urls, # Return S3 URLs only
taken_at=item["taken_at"], created_at=datetime.fromtimestamp(item["taken_at"])
code=item.get("code")
)) ))
new_post_count += 1 new_post_count += 1