# scraper.py # --- imports --- from sqlalchemy.orm import Session from database import InstagramAccount, InstagramPost from utils.boxapi_client import get_media_page, BoxAPIError # ← paged client from utils.parser_fast import parse_boxapi_items_fast from utils.seen_store import filter_unseen, mark_seen # use optimized uploaders; keep the generic fallback as well from utils.s3_uploader import ( upload_media_from_url, upload_image_from_url_optimized, upload_video_from_url_optimized, ) from schemas import ScrapeRequest, ScrapedPost from typing import List, Dict, Any, Optional from datetime import datetime from scraper_cursor import get_cursor, set_cursor # ← new helpers def _extract_minimal(item: Dict[str, Any]) -> tuple[Optional[str], Optional[int]]: """Return (ig_id, taken_at) from a raw BoxAPI item without full parsing.""" ig_id = item.get("id") or item.get("pk") if ig_id is not None: ig_id = str(ig_id) taken_at = item.get("taken_at") or item.get("timestamp") # might be int/epoch try: taken_at = int(taken_at) if taken_at is not None else None except Exception: taken_at = None return ig_id, taken_at def _collect_unseen_raw( username: str, target_unseen: int, stored_cursor: Optional[str], since_ts: Optional[int], ) -> tuple[List[Dict[str, Any]], Optional[str], int]: """ Keep requesting BoxAPI pages until we accumulate `target_unseen` UNSEEN items, or we run out of pages. We do minimal work per page to decide quickly. Returns: - selected_raw: only the unseen (and since_ts-passing) raw items in order - final_cursor: the last next_max_id we reached (to persist) - unseen_count: number of unseen items we actually collected """ selected_raw: List[Dict[str, Any]] = [] final_cursor: Optional[str] = stored_cursor # 1) newest page first next_id: Optional[str] = None try: first_page, _log, next_id = get_media_page(username=username, count=min(12, target_unseen), max_id=None) except Exception: first_page = [] def add_unseen_from_page(page: List[Dict[str, Any]]) -> int: """Filter a page to unseen (and since_ts), append to selected_raw, return how many added.""" if not page: return 0 # Extract ids in order ids_in_order: List[str] = [] since_mask: List[bool] = [] for it in page: ig_id, taken_at = _extract_minimal(it) if ig_id: # optional since_ts filter (skip if not passing) ok = True if since_ts is not None and taken_at is not None: ok = taken_at > since_ts ids_in_order.append(ig_id) since_mask.append(ok) if not ids_in_order: return 0 # Fast unseen check unseen_ids_set = set(filter_unseen(username, ids_in_order)) if not unseen_ids_set: return 0 # Append only ids that pass since_mask AND are unseen, preserving order added = 0 for it in page: ig_id, taken_at = _extract_minimal(it) if not ig_id: continue idx = ids_in_order.index(ig_id) # safe: ig_id exists in list if not since_mask[idx]: continue if ig_id in unseen_ids_set: selected_raw.append(it) added += 1 if len(selected_raw) >= target_unseen: break return added # Process newest page first add_unseen_from_page(first_page) # 2) then keep going with stored cursor (older pages). Seed with stored or newest next_id. cursor = stored_cursor or next_id # Only continue if we still need more while len(selected_raw) < target_unseen and cursor: need = min(12, target_unseen - len(selected_raw)) page, _log, cursor = get_media_page(username=username, count=need, max_id=cursor) added = add_unseen_from_page(page) final_cursor = cursor # we move the cursor forward as we traverse if not page: break # keep looping until we hit target or run out (cursor becomes None) return selected_raw, final_cursor, len(selected_raw) def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]: target = int(req.max_count) # Optional: allow caller to pass since_taken_at; coerce safely to int since_ts_raw = getattr(req, "since_taken_at", None) since_ts: Optional[int] = None if since_ts_raw is not None: try: since_ts = int(since_ts_raw) except Exception: since_ts = None # 1) Gather ONLY unseen raw items up to target (newest → older via cursor) stored_cursor = get_cursor(db, req.username) # may be None try: selected_raw, final_cursor, unseen_count = _collect_unseen_raw( username=req.username, target_unseen=target, stored_cursor=stored_cursor, since_ts=since_ts, ) except BoxAPIError as e: print(f"❌ BoxAPI error: {e}") return [] if not selected_raw: # Still store cursor progress (we may have advanced even if nothing new) set_cursor(db, req.username, final_cursor) return [] # 2) Normalize ONLY the selected unseen raw items normalized = parse_boxapi_items_fast(selected_raw) # 3) Defensive re-check: filter_unseen again after normalize by IG IDs (handles races) ids_in_order = [n.get("ig_post_id") for n in normalized if n.get("ig_post_id")] if not ids_in_order: set_cursor(db, req.username, final_cursor) return [] unseen_ids = set(filter_unseen(req.username, ids_in_order)) final_batch = [n for n in normalized if n.get("ig_post_id") in unseen_ids][:target] # Persist cursor progress AFTER we’ve walked pages set_cursor(db, req.username, final_cursor) # Mark as seen now to avoid future reprocessing mark_seen(req.username, [b["ig_post_id"] for b in final_batch]) # 4) Ensure account row account = db.query(InstagramAccount).filter_by(username=req.username).first() if not account: account = InstagramAccount( username=req.username, seller_id=req.seller_id, is_active=True, last_synced=None ) db.add(account) db.commit() db.refresh(account) # 5) Upload + save each post in the final batch result: List[ScrapedPost] = [] for item in final_batch: post_id = str(item["ig_post_id"]) seller_id = str(req.seller_id) # Skip if already saved (DB guard) if db.query(InstagramPost).filter_by(ig_post_id=post_id).first(): continue # Media URLs (from normalized) media_urls = list(item.get("remote_urls", [])) # If video, keep only true video URLs (avoid using the thumb URL) if item["media_type"] == "video": thumb = (item.get("thumbnail_url") or "").strip() def is_video(u: str) -> bool: base = u.split("?", 1)[0].lower() return base.endswith((".mp4", ".mov", ".m4v", ".webm")) media_urls = [u for u in media_urls if is_video(u) and u != thumb] # Upload media uploaded_urls: List[str] = [] for remote_url in media_urls: try: if item["media_type"] == "video": url = upload_video_from_url_optimized(remote_url, seller_id, post_id) elif item["media_type"] in ("image", "carousel"): url = upload_image_from_url_optimized(remote_url, seller_id, post_id) if url is None: url = upload_media_from_url(remote_url, seller_id, post_id) else: url = upload_media_from_url(remote_url, seller_id, post_id) if url: uploaded_urls.append(url) except Exception as e: print(f"⚠️ Error uploading media: {e}") continue # Upload/optimize thumbnail (if present) final_thumbnail = item.get("thumbnail_url") or None if final_thumbnail: try: thumb_url = upload_image_from_url_optimized(final_thumbnail, seller_id, post_id) if thumb_url: final_thumbnail = thumb_url else: alt = upload_media_from_url(final_thumbnail, seller_id, post_id) if alt: final_thumbnail = alt except Exception as e: print(f"⚠️ Failed to upload thumbnail: {e}") # Save row created_at = datetime.fromtimestamp(item["taken_at"]) if item.get("taken_at") else datetime.utcnow() post_entry = InstagramPost( ig_post_id=post_id, account_id=account.id, media_type=item["media_type"], caption=item.get("caption"), thumbnail_url=final_thumbnail, local_media=uploaded_urls, created_at=created_at, ) db.add(post_entry) db.commit() result.append(ScrapedPost( ig_post_id=post_id, media_type=item["media_type"], caption=item.get("caption"), thumbnail_url=final_thumbnail, local_media=uploaded_urls, created_at=created_at, )) if len(result) >= target: break return result