fix(scraper): always fetch newest posts first

Stop resuming from the persisted (older) BoxAPI cursor. Resuming made every
sync walk further back in time and never re-fetch posts published since the
last run, so a seller's newest posts were never imported. Always start from
max_id=None (newest) and page older within a single run, de-duping against
the DB: brand-new posts first, then older un-imported ones.

Also emit timezone-aware UTC created_at so ordering is correct regardless of
the container's local timezone (and Django no longer receives naive datetimes).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
fazli 2026-06-28 07:28:23 +00:00
parent bc965a0cbd
commit 62a03c61fe

View File

@ -1,7 +1,7 @@
# scraper.py
import os
from typing import List, Dict, Any, Optional, Tuple, Set
from datetime import datetime
from datetime import datetime, timezone
from sqlalchemy.orm import Session
from sqlalchemy import select
@ -100,9 +100,14 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]:
# Account (needed for saving)
account = _get_or_create_account(db, username, req.seller_id)
# Cursor: start where we left off (older pages)
max_id: Optional[str] = _get_cursor(db, username)
dprint(f"start_cursor={max_id!r}")
# ALWAYS start from the newest page (max_id=None). We must not resume from a
# persisted "older" cursor: doing so makes every sync walk further back in
# time and never re-fetch posts published since the last run, so the seller's
# newest posts never get imported. BoxAPI returns newest-first and the parser
# sorts newest-first, so paging older within this single run and de-duping
# against the DB yields: brand-new posts first, then older un-imported ones.
max_id: Optional[str] = None
dprint(f"start_cursor={max_id!r} (forced newest-first)")
# Page through BoxAPI until we collect enough NEW items (older than what we have)
pages = 0
@ -235,7 +240,14 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]:
except Exception as e:
print(f"⚠️ Failed to upload thumbnail: {e}")
created_at = datetime.fromtimestamp(item["taken_at"]) if item.get("taken_at") else datetime.utcnow()
# taken_at is a Unix timestamp (UTC). Build a timezone-aware UTC datetime
# so ordering is correct regardless of the container's local timezone and
# so the Django side doesn't receive a naive datetime.
created_at = (
datetime.fromtimestamp(item["taken_at"], tz=timezone.utc)
if item.get("taken_at")
else datetime.now(timezone.utc)
)
row = InstagramPost(
ig_post_id=post_id,