fazli/FastApi-ISS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(scraper): always fetch newest posts first

Browse Source 
Stop resuming from the persisted (older) BoxAPI cursor. Resuming made every
sync walk further back in time and never re-fetch posts published since the
last run, so a seller's newest posts were never imported. Always start from
max_id=None (newest) and page older within a single run, de-duping against
the DB: brand-new posts first, then older un-imported ones.

Also emit timezone-aware UTC created_at so ordering is correct regardless of
the container's local timezone (and Django no longer receives naive datetimes).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
fazli 2026-06-28 07:28:23 +00:00
parent bc965a0cbd
commit 62a03c61fe
1 changed files with 17 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
scraper.py
View File

@ -1,7 +1,7 @@
# scraper.py # scraper.py
import os import os
from typing import List, Dict, Any, Optional, Tuple, Set from typing import List, Dict, Any, Optional, Tuple, Set
from datetime import datetime from datetime import datetime, timezone
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from sqlalchemy import select from sqlalchemy import select
@ -100,9 +100,14 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]:
# Account (needed for saving) # Account (needed for saving)
account = _get_or_create_account(db, username, req.seller_id) account = _get_or_create_account(db, username, req.seller_id)
# Cursor: start where we left off (older pages) # ALWAYS start from the newest page (max_id=None). We must not resume from a
max_id: Optional[str] = _get_cursor(db, username) # persisted "older" cursor: doing so makes every sync walk further back in
dprint(f"start_cursor={max_id!r}") # time and never re-fetch posts published since the last run, so the seller's
# newest posts never get imported. BoxAPI returns newest-first and the parser
# sorts newest-first, so paging older within this single run and de-duping
# against the DB yields: brand-new posts first, then older un-imported ones.
max_id: Optional[str] = None
dprint(f"start_cursor={max_id!r} (forced newest-first)")
# Page through BoxAPI until we collect enough NEW items (older than what we have) # Page through BoxAPI until we collect enough NEW items (older than what we have)
pages = 0 pages = 0
@ -235,7 +240,14 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]:
except Exception as e: except Exception as e:
print(f"⚠️ Failed to upload thumbnail: {e}") print(f"⚠️ Failed to upload thumbnail: {e}")
created_at = datetime.fromtimestamp(item["taken_at"]) if item.get("taken_at") else datetime.utcnow() # taken_at is a Unix timestamp (UTC). Build a timezone-aware UTC datetime
# so ordering is correct regardless of the container's local timezone and
# so the Django side doesn't receive a naive datetime.
created_at = (
datetime.fromtimestamp(item["taken_at"], tz=timezone.utc)
if item.get("taken_at")
else datetime.now(timezone.utc)
)
row = InstagramPost( row = InstagramPost(
ig_post_id=post_id, ig_post_id=post_id,