ok

scraper.py

@@ -1,5 +1,6 @@
 # scraper.py
 
+import os
 from sqlalchemy.orm import Session
 from database import InstagramAccount, InstagramPost
 from utils.boxapi_client import get_media_page, BoxAPIError
@@ -14,63 +15,60 @@ from typing import List, Dict, Any, Optional, Set
 from datetime import datetime
 from scraper_cursor import get_cursor, set_cursor
 
-# Safety cap for very large accounts
-MAX_PAGES = 500  # adjust if needed
+DIAG = os.getenv("SCRAPER_DIAGNOSTICS", "0") == "1"
+MAX_PAGES = int(os.getenv("SCRAPER_MAX_PAGES", "400"))  # safety cap
 
+def dprint(msg: str) -> None:
+    if DIAG:
+        print(f"[SCRAPER] {msg}")
 
-def _ig_id_of(it: Dict[str, Any]) -> Optional[str]:
+def _ig_id(it: Dict[str, Any]) -> Optional[str]:
     v = it.get("id") or it.get("pk")
     return str(v) if v is not None else None
 
-
 def _db_existing_ids(db: Session, ids: List[str]) -> Set[str]:
     if not ids:
         return set()
-    rows = (
-        db.query(InstagramPost.ig_post_id)
-        .filter(InstagramPost.ig_post_id.in_(ids))
-        .all()
-    )
+    rows = db.query(InstagramPost.ig_post_id)\
+             .filter(InstagramPost.ig_post_id.in_(ids)).all()
     return {r[0] for r in rows}
 
-
-def _collect_raw_to_save(
+def _collect_raw_not_in_db(
     db: Session,
     username: str,
     needed: int,
-    stored_cursor: Optional[str],
     since_ts: Optional[int],
-) -> tuple[List[Dict[str, Any]], Optional[str]]:
+) -> tuple[List[Dict[str, Any]], Optional[str], int]:
     """
-    Keep fetching pages until we have `needed` posts that are NOT in DB.
-    Return (raw_items_to_save, final_cursor).
+    Walk pages linearly: newest first, then older via next_max_id.
+    Keep only posts not present in DB. Stop when we have `needed` or no more pages.
+    Returns (raw_to_save, final_cursor, pages_walked)
     """
-    to_save_raw: List[Dict[str, Any]] = []
+    raw_to_save: List[Dict[str, Any]] = []
     pages = 0
-    cursor = stored_cursor
-    first = True
-    last_seen_cursor: Optional[str] = stored_cursor
+    max_id = None  # None → newest page
+    final_cursor: Optional[str] = None
 
-    while len(to_save_raw) < needed and pages < MAX_PAGES:
-        max_id = None if first else cursor
-        count = min(12, needed - len(to_save_raw)) or 12
+    while len(raw_to_save) < needed and pages < MAX_PAGES:
+        count = min(12, max(1, needed - len(raw_to_save)))  # fetch up to 12
         try:
-            page, _log, next_id = get_media_page(username=username, count=count, max_id=max_id)
-        except BoxAPIError:
+            page, _log, next_max = get_media_page(username=username, count=count, max_id=max_id)
+        except BoxAPIError as e:
+            dprint(f"BoxAPI error: {e}")
             break
 
         pages += 1
-        last_seen_cursor = next_id
+        dprint(f"Page#{pages}: got {len(page)} items, next_max_id={next_max}")
 
         if not page:
             break
 
-        # Optional: since_taken_at filter (raw, cheap)
+        # Optional cutoff by timestamp (cheap check)
         filtered = []
-        ids_in_order: List[str] = []
+        ids_ordered: List[str] = []
         for it in page:
-            igid = _ig_id_of(it)
-            if not igid:
+            pid = _ig_id(it)
+            if not pid:
                 continue
             if since_ts is not None:
                 ts = it.get("taken_at") or it.get("timestamp")
@@ -81,37 +79,29 @@ def _collect_raw_to_save(
                 if ts is not None and ts <= since_ts:
                     continue
             filtered.append(it)
-            ids_in_order.append(igid)
+            ids_ordered.append(pid)
 
-        if not ids_in_order:
-            if not next_id:
-                break
-            first = False
-            cursor = next_id
-            continue
+        if ids_ordered:
+            already = _db_existing_ids(db, ids_ordered)
+            for it in filtered:
+                pid = _ig_id(it)
+                if pid and pid not in already:
+                    raw_to_save.append(it)
+                    if len(raw_to_save) >= needed:
+                        break
 
-        # DB check: keep only items not yet saved
-        already = _db_existing_ids(db, ids_in_order)
-        for it in filtered:
-            igid = _ig_id_of(it)
-            if igid and igid not in already:
-                to_save_raw.append(it)
-                if len(to_save_raw) >= needed:
-                    break
-
-        if not next_id:
+        # advance pagination
+        final_cursor = next_max
+        if not next_max:
             break
+        max_id = next_max
 
-        first = False
-        cursor = next_id
-
-    return to_save_raw, last_seen_cursor
-
+    return raw_to_save, final_cursor, pages
 
 def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]:
     target = int(req.max_count)
 
-    # Optional since_taken_at
+    # since_taken_at (optional)
     since_ts: Optional[int] = None
     if hasattr(req, "since_taken_at") and req.since_taken_at is not None:
         try:
@@ -119,25 +109,23 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]:
         except Exception:
             since_ts = None
 
-    # Start from stored backfill cursor (if any), but always try newest page first
-    stored_cursor = get_cursor(db, req.username)
-
-    # 1) Collect raw items to save (NOT in DB), paging as deep as needed
-    raw_to_save, final_cursor = _collect_raw_to_save(
+    # 1) BRUTE-FORCE scan: newest → older, pick posts NOT in DB until we reach target
+    raw_to_save, final_cursor, pages_walked = _collect_raw_not_in_db(
         db=db,
         username=req.username,
         needed=target,
-        stored_cursor=stored_cursor,
         since_ts=since_ts,
     )
 
-    # Persist traversal progress (so next run can continue deeper)
-    set_cursor(db, req.username, final_cursor)
+    # Persist cursor if we walked at least one page
+    if pages_walked > 0:
+        set_cursor(db, req.username, final_cursor)
 
     if not raw_to_save:
+        dprint("No new posts to save.")
         return []
 
-    # 2) Normalize only what we intend to save
+    # 2) Normalize only the ones we intend to save
     normalized = parse_boxapi_items_fast(raw_to_save)[:target]
 
     # 3) Ensure account row
@@ -159,13 +147,12 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]:
         post_id = str(item["ig_post_id"])
         seller_id = str(req.seller_id)
 
-        # DB guard (rare race)
+        # DB guard in case a parallel worker just saved it
         if db.query(InstagramPost).filter_by(ig_post_id=post_id).first():
             continue
 
         media_urls = list(item.get("remote_urls", []))
 
-        # If video, keep only actual video URLs (avoid thumbnail)
         if item["media_type"] == "video":
             thumb = (item.get("thumbnail_url") or "").strip()
             def is_video(u: str) -> bool:
@@ -229,4 +216,4 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]:
         if len(result) >= target:
             break
 
-    return result
+    return result

utils/boxapi_client.py

@@ -12,9 +12,16 @@ BOXAPI_ENDPOINT = os.environ.get(
     "https://boxapi.ir/api/instagram/user/get_media_by_username",
 )
 
+# NEW: simple debug flag
+BOXAPI_DEBUG = os.getenv("BOXAPI_DEBUG", "0") == "1"
+
 class BoxAPIError(Exception):
     pass
 
+def _dprint(msg: str) -> None:
+    if BOXAPI_DEBUG:
+        print(f"[BOXAPI] {msg}")
+
 
 def get_media_page(
     username: str,
@@ -38,6 +45,8 @@ def get_media_page(
     if max_id:
         payload["max_id"] = max_id
 
+    _dprint(f"→ POST {BOXAPI_ENDPOINT} payload={payload}")
+
     resp_json: Optional[Dict[str, Any]] = None
     resp_text: Optional[str] = None
     status_code: Optional[int] = None
@@ -86,6 +95,8 @@ def get_media_page(
                 f"Unexpected response format from BoxAPI ({snapshot}). See log: {log_path}"
             )
 
+        _dprint(f"← items={len(items)} next_max_id={next_max_id!r}")
+
         return items, log_path, next_max_id
 
     except BoxAPIError: