diff --git a/scraper.py b/scraper.py index c96bedd..995ed11 100644 --- a/scraper.py +++ b/scraper.py @@ -1,7 +1,7 @@ # scraper.py import os from typing import List, Dict, Any, Optional, Tuple, Set -from datetime import datetime +from datetime import datetime, timezone from sqlalchemy.orm import Session from sqlalchemy import select @@ -100,9 +100,14 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]: # Account (needed for saving) account = _get_or_create_account(db, username, req.seller_id) - # Cursor: start where we left off (older pages) - max_id: Optional[str] = _get_cursor(db, username) - dprint(f"start_cursor={max_id!r}") + # ALWAYS start from the newest page (max_id=None). We must not resume from a + # persisted "older" cursor: doing so makes every sync walk further back in + # time and never re-fetch posts published since the last run, so the seller's + # newest posts never get imported. BoxAPI returns newest-first and the parser + # sorts newest-first, so paging older within this single run and de-duping + # against the DB yields: brand-new posts first, then older un-imported ones. + max_id: Optional[str] = None + dprint(f"start_cursor={max_id!r} (forced newest-first)") # Page through BoxAPI until we collect enough NEW items (older than what we have) pages = 0 @@ -235,7 +240,14 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]: except Exception as e: print(f"⚠️ Failed to upload thumbnail: {e}") - created_at = datetime.fromtimestamp(item["taken_at"]) if item.get("taken_at") else datetime.utcnow() + # taken_at is a Unix timestamp (UTC). Build a timezone-aware UTC datetime + # so ordering is correct regardless of the container's local timezone and + # so the Django side doesn't receive a naive datetime. + created_at = ( + datetime.fromtimestamp(item["taken_at"], tz=timezone.utc) + if item.get("taken_at") + else datetime.now(timezone.utc) + ) row = InstagramPost( ig_post_id=post_id,