From 760568e57107ead095617fa1ad037816c7dd293f Mon Sep 17 00:00:00 2001
From: Hossein <fazlihossein6@gmail.com>
Date: Thu, 25 Sep 2025 03:03:19 +0330
Subject: [PATCH] fuck you

---
 scraper.py | 37 +++++++++++++++++++++++++++----------
 1 file changed, 27 insertions(+), 10 deletions(-)

diff --git a/scraper.py b/scraper.py
index 74ccbe0..9a8b42a 100644
--- a/scraper.py
+++ b/scraper.py
@@ -40,17 +40,18 @@ def _collect_raw_not_in_db(
     since_ts: Optional[int],
 ) -> tuple[List[Dict[str, Any]], Optional[str], int]:
     """
-    Walk pages linearly: newest first, then older via next_max_id.
-    Keep only posts not present in DB. Stop when we have `needed` or no more pages.
+    Walk pages linearly: newest first (max_id=None), then older via next_max_id.
+    Keep only posts NOT in DB. Stop when we have `needed` or no more pages.
     Returns (raw_to_save, final_cursor, pages_walked)
     """
     raw_to_save: List[Dict[str, Any]] = []
     pages = 0
-    max_id = None  # None → newest page
+    first = True
+    max_id: Optional[str] = None   # None → newest page
     final_cursor: Optional[str] = None
 
     while len(raw_to_save) < needed and pages < MAX_PAGES:
-        count = min(12, max(1, needed - len(raw_to_save)))  # fetch up to 12
+        count = min(12, max(1, needed - len(raw_to_save)))  # 1..12
         try:
             page, _log, next_max = get_media_page(username=username, count=count, max_id=max_id)
         except BoxAPIError as e:
@@ -58,13 +59,23 @@ def _collect_raw_not_in_db(
             break
 
         pages += 1
-        dprint(f"Page#{pages}: got {len(page)} items, next_max_id={next_max}")
+        dprint(f"Page#{pages}: request max_id={max_id!r} → got {len(page)} items, next_max_id={next_max!r}")
+
+        # after first fetch, always continue with next_max_id
+        first = False
 
         if not page:
-            break
+            # no items on this page, stop if no further cursor
+            if not next_max:
+                dprint("Empty page and no next_max_id → stopping")
+                break
+            # otherwise, advance cursor and continue (rare)
+            max_id = next_max
+            final_cursor = next_max
+            continue
 
-        # Optional cutoff by timestamp (cheap check)
-        filtered = []
+        # Optional time cutoff
+        filtered: List[Dict[str, Any]] = []
         ids_ordered: List[str] = []
         for it in page:
             pid = _ig_id(it)
@@ -83,19 +94,25 @@ def _collect_raw_not_in_db(
 
         if ids_ordered:
             already = _db_existing_ids(db, ids_ordered)
+            dprint(f"  from {len(ids_ordered)} IDs, already_in_db={len(already)}")
             for it in filtered:
                 pid = _ig_id(it)
                 if pid and pid not in already:
                     raw_to_save.append(it)
                     if len(raw_to_save) >= needed:
                         break
+        else:
+            dprint("  no valid IDs after filtering")
 
-        # advance pagination
+        # advance pagination cursor for the next loop
         final_cursor = next_max
         if not next_max:
+            dprint("No next_max_id → reached end of feed")
             break
-        max_id = next_max
 
+        max_id = next_max  # ← KEY: chase older page next round
+
+    dprint(f"Collected to_save={len(raw_to_save)} across pages={pages}, final_cursor={final_cursor!r}")
     return raw_to_save, final_cursor, pages
 
 def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]: