# scraper.py from sqlalchemy.orm import Session from database import InstagramAccount, InstagramPost from utils.boxapi_client import get_media_page, BoxAPIError from utils.parser_fast import parse_boxapi_items_fast from utils.s3_uploader import ( upload_media_from_url, upload_image_from_url_optimized, upload_video_from_url_optimized, ) from schemas import ScrapeRequest, ScrapedPost from typing import List, Dict, Any, Optional, Set from datetime import datetime from scraper_cursor import get_cursor, set_cursor # Safety cap for very large accounts MAX_PAGES = 500 # adjust if needed def _ig_id_of(it: Dict[str, Any]) -> Optional[str]: v = it.get("id") or it.get("pk") return str(v) if v is not None else None def _db_existing_ids(db: Session, ids: List[str]) -> Set[str]: if not ids: return set() rows = ( db.query(InstagramPost.ig_post_id) .filter(InstagramPost.ig_post_id.in_(ids)) .all() ) return {r[0] for r in rows} def _collect_raw_to_save( db: Session, username: str, needed: int, stored_cursor: Optional[str], since_ts: Optional[int], ) -> tuple[List[Dict[str, Any]], Optional[str]]: """ Keep fetching pages until we have `needed` posts that are NOT in DB. Return (raw_items_to_save, final_cursor). """ to_save_raw: List[Dict[str, Any]] = [] pages = 0 cursor = stored_cursor first = True last_seen_cursor: Optional[str] = stored_cursor while len(to_save_raw) < needed and pages < MAX_PAGES: max_id = None if first else cursor count = min(12, needed - len(to_save_raw)) or 12 try: page, _log, next_id = get_media_page(username=username, count=count, max_id=max_id) except BoxAPIError: break pages += 1 last_seen_cursor = next_id if not page: break # Optional: since_taken_at filter (raw, cheap) filtered = [] ids_in_order: List[str] = [] for it in page: igid = _ig_id_of(it) if not igid: continue if since_ts is not None: ts = it.get("taken_at") or it.get("timestamp") try: ts = int(ts) if ts is not None else None except Exception: ts = None if ts is not None and ts <= since_ts: continue filtered.append(it) ids_in_order.append(igid) if not ids_in_order: if not next_id: break first = False cursor = next_id continue # DB check: keep only items not yet saved already = _db_existing_ids(db, ids_in_order) for it in filtered: igid = _ig_id_of(it) if igid and igid not in already: to_save_raw.append(it) if len(to_save_raw) >= needed: break if not next_id: break first = False cursor = next_id return to_save_raw, last_seen_cursor def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]: target = int(req.max_count) # Optional since_taken_at since_ts: Optional[int] = None if hasattr(req, "since_taken_at") and req.since_taken_at is not None: try: since_ts = int(req.since_taken_at) except Exception: since_ts = None # Start from stored backfill cursor (if any), but always try newest page first stored_cursor = get_cursor(db, req.username) # 1) Collect raw items to save (NOT in DB), paging as deep as needed raw_to_save, final_cursor = _collect_raw_to_save( db=db, username=req.username, needed=target, stored_cursor=stored_cursor, since_ts=since_ts, ) # Persist traversal progress (so next run can continue deeper) set_cursor(db, req.username, final_cursor) if not raw_to_save: return [] # 2) Normalize only what we intend to save normalized = parse_boxapi_items_fast(raw_to_save)[:target] # 3) Ensure account row account = db.query(InstagramAccount).filter_by(username=req.username).first() if not account: account = InstagramAccount( username=req.username, seller_id=req.seller_id, is_active=True, last_synced=None ) db.add(account) db.commit() db.refresh(account) # 4) Upload + save result: List[ScrapedPost] = [] for item in normalized: post_id = str(item["ig_post_id"]) seller_id = str(req.seller_id) # DB guard (rare race) if db.query(InstagramPost).filter_by(ig_post_id=post_id).first(): continue media_urls = list(item.get("remote_urls", [])) # If video, keep only actual video URLs (avoid thumbnail) if item["media_type"] == "video": thumb = (item.get("thumbnail_url") or "").strip() def is_video(u: str) -> bool: base = u.split("?", 1)[0].lower() return base.endswith((".mp4", ".mov", ".m4v", ".webm")) media_urls = [u for u in media_urls if is_video(u) and u != thumb] uploaded_urls: List[str] = [] for remote_url in media_urls: try: if item["media_type"] == "video": url = upload_video_from_url_optimized(remote_url, seller_id, post_id) elif item["media_type"] in ("image", "carousel"): url = upload_image_from_url_optimized(remote_url, seller_id, post_id) if url is None: url = upload_media_from_url(remote_url, seller_id, post_id) else: url = upload_media_from_url(remote_url, seller_id, post_id) if url: uploaded_urls.append(url) except Exception as e: print(f"⚠️ Error uploading media: {e}") continue final_thumbnail = item.get("thumbnail_url") or None if final_thumbnail: try: thumb_url = upload_image_from_url_optimized(final_thumbnail, seller_id, post_id) if thumb_url: final_thumbnail = thumb_url else: alt = upload_media_from_url(final_thumbnail, seller_id, post_id) if alt: final_thumbnail = alt except Exception as e: print(f"⚠️ Failed to upload thumbnail: {e}") created_at = datetime.fromtimestamp(item["taken_at"]) if item.get("taken_at") else datetime.utcnow() post_entry = InstagramPost( ig_post_id=post_id, account_id=account.id, media_type=item["media_type"], caption=item.get("caption"), thumbnail_url=final_thumbnail, local_media=uploaded_urls, created_at=created_at, ) db.add(post_entry) db.commit() result.append(ScrapedPost( ig_post_id=post_id, media_type=item["media_type"], caption=item.get("caption"), thumbnail_url=final_thumbnail, local_media=uploaded_urls, created_at=created_at, )) if len(result) >= target: break return result