# scraper.py import os from sqlalchemy.orm import Session from database import InstagramAccount, InstagramPost from utils.boxapi_client import get_media_page, BoxAPIError from utils.parser_fast import parse_boxapi_items_fast from utils.s3_uploader import ( upload_media_from_url, upload_image_from_url_optimized, upload_video_from_url_optimized, ) from schemas import ScrapeRequest, ScrapedPost from typing import List, Dict, Any, Optional, Set from datetime import datetime from scraper_cursor import get_cursor, set_cursor DIAG = os.getenv("SCRAPER_DIAGNOSTICS", "0") == "1" MAX_PAGES = int(os.getenv("SCRAPER_MAX_PAGES", "400")) # safety cap def dprint(msg: str) -> None: if DIAG: print(f"[SCRAPER] {msg}") def _db_existing_ids_by_ig(db: Session, ig_ids: List[str]) -> Set[str]: if not ig_ids: return set() rows = ( db.query(InstagramPost.ig_post_id) .filter(InstagramPost.ig_post_id.in_(ig_ids)) .all() ) return {r[0] for r in rows} def _resolve_target(req: ScrapeRequest) -> int: # Accept both "count" and "max_count" return int((getattr(req, "count", None) if getattr(req, "count", None) is not None else req.max_count)) def _collect_normalized_not_in_db( db: Session, username: str, target: int, start_cursor: Optional[str], since_ts: Optional[int], ) -> tuple[List[Dict[str, Any]], Optional[str], int]: """ Walk pages: start from start_cursor (if any) else newest (None). For each page: - normalize - DB-check by normalized ig_post_id - accumulate unseen until reaching target or no next_max_id Returns (normalized_to_save, final_cursor, pages_walked) """ to_save_norm: List[Dict[str, Any]] = [] pages = 0 max_id = start_cursor # None => newest final_cursor: Optional[str] = start_cursor while len(to_save_norm) < target and pages < MAX_PAGES: # Always ask up to 12; progress even when many are duplicates per_req = 12 try: page, _log, next_max = get_media_page(username=username, count=per_req, max_id=max_id) except BoxAPIError as e: dprint(f"BoxAPI error: {e}") break pages += 1 dprint(f"Page#{pages}: request max_id={max_id!r} → got {len(page)} items, next_max_id={next_max!r}") if not page: # no data; if there's a next, follow it once; else stop if not next_max: break max_id = next_max final_cursor = next_max continue # Normalize THIS page (critical: now ig_post_id matches DB) norm_page = parse_boxapi_items_fast(page) # Optional timestamp cutoff if since_ts is not None: try: s = int(since_ts) except Exception: s = None if s is not None: norm_page = [n for n in norm_page if (n.get("taken_at") or 0) > s] # DB duplicate filter using normalized ig_post_id norm_ids = [str(n.get("ig_post_id")) for n in norm_page if n.get("ig_post_id")] already = _db_existing_ids_by_ig(db, norm_ids) dprint(f" normalized ids={len(norm_ids)} already_in_db={len(already)}") for n in norm_page: igid = str(n.get("ig_post_id")) if n.get("ig_post_id") else None if not igid or igid in already: continue to_save_norm.append(n) if len(to_save_norm) >= target: break # advance cursor final_cursor = next_max if not next_max: dprint("No next_max_id → end of feed") break max_id = next_max dprint(f"Collected to_save={len(to_save_norm)} across pages={pages}, final_cursor={final_cursor!r}") return to_save_norm, final_cursor, pages def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]: target = _resolve_target(req) dprint(f"requested count={getattr(req, 'count', None)} max_count={req.max_count} → resolved target={target}") # since_taken_at (optional) since_ts: Optional[int] = None if hasattr(req, "since_taken_at") and req.since_taken_at is not None: try: since_ts = int(req.since_taken_at) except Exception: since_ts = None # Start from stored cursor so the next call continues deeper start_cursor = get_cursor(db, req.username) dprint(f"start_cursor={start_cursor!r}") # 1) Collect normalized posts NOT in DB, paging deeper as needed normalized_to_save, final_cursor, pages_walked = _collect_normalized_not_in_db( db=db, username=req.username, target=target, start_cursor=start_cursor, since_ts=since_ts, ) # Persist cursor if we walked any page if pages_walked > 0: set_cursor(db, req.username, final_cursor) if not normalized_to_save: dprint("No new normalized posts to save.") return [] # 2) Ensure account row account = db.query(InstagramAccount).filter_by(username=req.username).first() if not account: account = InstagramAccount( username=req.username, seller_id=req.seller_id, is_active=True, last_synced=None ) db.add(account) db.commit() db.refresh(account) # 3) Upload + save result: List[ScrapedPost] = [] for item in normalized_to_save[:target]: post_id = str(item["ig_post_id"]) seller_id = str(req.seller_id) # DB guard (parallel runner safety) if db.query(InstagramPost).filter_by(ig_post_id=post_id).first(): continue # Media URLs media_urls = list(item.get("remote_urls", [])) # If video, keep only video URLs (avoid thumb as media) if item["media_type"] == "video": thumb = (item.get("thumbnail_url") or "").strip() def is_video(u: str) -> bool: base = u.split("?", 1)[0].lower() return base.endswith((".mp4", ".mov", ".m4v", ".webm")) media_urls = [u for u in media_urls if is_video(u) and u != thumb] # Upload uploaded_urls: List[str] = [] for remote_url in media_urls: try: if item["media_type"] == "video": url = upload_video_from_url_optimized(remote_url, seller_id, post_id) elif item["media_type"] in ("image", "carousel"): url = upload_image_from_url_optimized(remote_url, seller_id, post_id) if url is None: url = upload_media_from_url(remote_url, seller_id, post_id) else: url = upload_media_from_url(remote_url, seller_id, post_id) if url: uploaded_urls.append(url) except Exception as e: print(f"⚠️ Error uploading media: {e}") continue # Thumbnail final_thumbnail = item.get("thumbnail_url") or None if final_thumbnail: try: thumb_url = upload_image_from_url_optimized(final_thumbnail, seller_id, post_id) if thumb_url: final_thumbnail = thumb_url else: alt = upload_media_from_url(final_thumbnail, seller_id, post_id) if alt: final_thumbnail = alt except Exception as e: print(f"⚠️ Failed to upload thumbnail: {e}") created_at = datetime.fromtimestamp(item["taken_at"]) if item.get("taken_at") else datetime.utcnow() # Insert post_entry = InstagramPost( ig_post_id=post_id, account_id=account.id, media_type=item["media_type"], caption=item.get("caption"), thumbnail_url=final_thumbnail, local_media=uploaded_urls, created_at=created_at, ) db.add(post_entry) db.commit() result.append(ScrapedPost( ig_post_id=post_id, media_type=item["media_type"], caption=item.get("caption"), thumbnail_url=final_thumbnail, local_media=uploaded_urls, created_at=created_at, )) if len(result) >= target: break return result