fix(scraper): always fetch newest posts first
Stop resuming from the persisted (older) BoxAPI cursor. Resuming made every sync walk further back in time and never re-fetch posts published since the last run, so a seller's newest posts were never imported. Always start from max_id=None (newest) and page older within a single run, de-duping against the DB: brand-new posts first, then older un-imported ones. Also emit timezone-aware UTC created_at so ordering is correct regardless of the container's local timezone (and Django no longer receives naive datetimes). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
bc965a0cbd
commit
62a03c61fe
22
scraper.py
22
scraper.py
@ -1,7 +1,7 @@
|
||||
# scraper.py
|
||||
import os
|
||||
from typing import List, Dict, Any, Optional, Tuple, Set
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy import select
|
||||
@ -100,9 +100,14 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]:
|
||||
# Account (needed for saving)
|
||||
account = _get_or_create_account(db, username, req.seller_id)
|
||||
|
||||
# Cursor: start where we left off (older pages)
|
||||
max_id: Optional[str] = _get_cursor(db, username)
|
||||
dprint(f"start_cursor={max_id!r}")
|
||||
# ALWAYS start from the newest page (max_id=None). We must not resume from a
|
||||
# persisted "older" cursor: doing so makes every sync walk further back in
|
||||
# time and never re-fetch posts published since the last run, so the seller's
|
||||
# newest posts never get imported. BoxAPI returns newest-first and the parser
|
||||
# sorts newest-first, so paging older within this single run and de-duping
|
||||
# against the DB yields: brand-new posts first, then older un-imported ones.
|
||||
max_id: Optional[str] = None
|
||||
dprint(f"start_cursor={max_id!r} (forced newest-first)")
|
||||
|
||||
# Page through BoxAPI until we collect enough NEW items (older than what we have)
|
||||
pages = 0
|
||||
@ -235,7 +240,14 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]:
|
||||
except Exception as e:
|
||||
print(f"⚠️ Failed to upload thumbnail: {e}")
|
||||
|
||||
created_at = datetime.fromtimestamp(item["taken_at"]) if item.get("taken_at") else datetime.utcnow()
|
||||
# taken_at is a Unix timestamp (UTC). Build a timezone-aware UTC datetime
|
||||
# so ordering is correct regardless of the container's local timezone and
|
||||
# so the Django side doesn't receive a naive datetime.
|
||||
created_at = (
|
||||
datetime.fromtimestamp(item["taken_at"], tz=timezone.utc)
|
||||
if item.get("taken_at")
|
||||
else datetime.now(timezone.utc)
|
||||
)
|
||||
|
||||
row = InstagramPost(
|
||||
ig_post_id=post_id,
|
||||
|
||||
Loading…
Reference in New Issue
Block a user