fazli/FastApi-ISS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fuck you

Browse Source
This commit is contained in:
Hossein 2025-09-25 03:03:19 +03:30
parent 812d9964b4
commit 760568e571
1 changed files with 27 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
scraper.py
View File

@ -40,17 +40,18 @@ def _collect_raw_not_in_db(
since_ts: Optional[int], since_ts: Optional[int],
) -> tuple[List[Dict[str, Any]], Optional[str], int]: ) -> tuple[List[Dict[str, Any]], Optional[str], int]:
""" """
Walk pages linearly: newest first, then older via next_max_id. Walk pages linearly: newest first (max_id=None), then older via next_max_id.
Keep only posts not present in DB. Stop when we have `needed` or no more pages. Keep only posts NOT in DB. Stop when we have `needed` or no more pages.
Returns (raw_to_save, final_cursor, pages_walked) Returns (raw_to_save, final_cursor, pages_walked)
""" """
raw_to_save: List[Dict[str, Any]] = [] raw_to_save: List[Dict[str, Any]] = []
pages = 0 pages = 0
max_id = None # None → newest page first = True
max_id: Optional[str] = None # None → newest page
final_cursor: Optional[str] = None final_cursor: Optional[str] = None
while len(raw_to_save) < needed and pages < MAX_PAGES: while len(raw_to_save) < needed and pages < MAX_PAGES:
count = min(12, max(1, needed - len(raw_to_save))) # fetch up to 12 count = min(12, max(1, needed - len(raw_to_save))) # 1..12
try: try:
page, _log, next_max = get_media_page(username=username, count=count, max_id=max_id) page, _log, next_max = get_media_page(username=username, count=count, max_id=max_id)
except BoxAPIError as e: except BoxAPIError as e:
@ -58,13 +59,23 @@ def _collect_raw_not_in_db(
break break
pages += 1 pages += 1
dprint(f"Page#{pages}: got {len(page)} items, next_max_id={next_max}") dprint(f"Page#{pages}: request max_id={max_id!r} → got {len(page)} items, next_max_id={next_max!r}")
# after first fetch, always continue with next_max_id
first = False
if not page: if not page:
# no items on this page, stop if no further cursor
if not next_max:
dprint("Empty page and no next_max_id → stopping")
break break
# otherwise, advance cursor and continue (rare)
max_id = next_max
final_cursor = next_max
continue
# Optional cutoff by timestamp (cheap check) # Optional time cutoff
filtered = [] filtered: List[Dict[str, Any]] = []
ids_ordered: List[str] = [] ids_ordered: List[str] = []
for it in page: for it in page:
pid = _ig_id(it) pid = _ig_id(it)
@ -83,19 +94,25 @@ def _collect_raw_not_in_db(
if ids_ordered: if ids_ordered:
already = _db_existing_ids(db, ids_ordered) already = _db_existing_ids(db, ids_ordered)
dprint(f" from {len(ids_ordered)} IDs, already_in_db={len(already)}")
for it in filtered: for it in filtered:
pid = _ig_id(it) pid = _ig_id(it)
if pid and pid not in already: if pid and pid not in already:
raw_to_save.append(it) raw_to_save.append(it)
if len(raw_to_save) >= needed: if len(raw_to_save) >= needed:
break break
else:
dprint(" no valid IDs after filtering")
# advance pagination # advance pagination cursor for the next loop
final_cursor = next_max final_cursor = next_max
if not next_max: if not next_max:
dprint("No next_max_id → reached end of feed")
break break
max_id = next_max
max_id = next_max # ← KEY: chase older page next round
dprint(f"Collected to_save={len(raw_to_save)} across pages={pages}, final_cursor={final_cursor!r}")
return raw_to_save, final_cursor, pages return raw_to_save, final_cursor, pages
def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]: def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]: