# scraper.py # --- imports --- from sqlalchemy.orm import Session from database import InstagramAccount, InstagramPost from utils.boxapi_client import get_media_page, BoxAPIError # ← paged client from utils.parser_fast import parse_boxapi_items_fast from utils.seen_store import filter_unseen, mark_seen # use optimized uploaders; keep the generic fallback as well from utils.s3_uploader import ( upload_media_from_url, upload_image_from_url_optimized, upload_video_from_url_optimized, ) from schemas import ScrapeRequest, ScrapedPost from typing import List, Dict, Any, Optional from datetime import datetime from scraper_cursor import get_cursor, set_cursor # ← new helpers def _collect_across_pages(username: str, target: int, seed_cursor: Optional[str]) -> tuple[list[Dict[str, Any]], Optional[str]]: """ 1) Fetch ONE newest page (max_id=None). 2) If still short, continue with `seed_cursor` (or newest next_max_id) to backfill older pages. Returns (raw_items, final_cursor_to_store). """ collected: list[Dict[str, Any]] = [] newest_next: Optional[str] = None # Step 1: newest page first (cheap check for new content) need = min(12, target) try: page, _log, newest_next = get_media_page(username=username, count=need, max_id=None) if page: collected.extend(page) except Exception: # ignore and continue with backfill-only path pass # Step 2: continue older pages from stored cursor (or newest_next if none stored) cursor = seed_cursor or newest_next while len(collected) < target and cursor: need = min(12, target - len(collected)) page, _log, cursor = get_media_page(username=username, count=need, max_id=cursor) if not page: break collected.extend(page) return collected, cursor # final cursor to persist def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]: target = int(req.max_count) # 1) Collect raw items across newest + backfill pages (no request schema change) stored_cursor = get_cursor(db, req.username) # may be None on first run try: raw_items, final_cursor = _collect_across_pages(req.username, target, stored_cursor) except BoxAPIError as e: print(f"❌ BoxAPI error: {e}") return [] if not raw_items: return [] # 2) Normalize once on the aggregated list normalized = parse_boxapi_items_fast(raw_items) # 3) Optional cursor filter (kept as-is) since_ts = getattr(req, "since_taken_at", None) if since_ts: try: since_ts = int(since_ts) normalized = [n for n in normalized if (n.get("taken_at") or 0) > since_ts] except Exception: pass # 4) Filter already-seen FIRST (avoid re-uploading) — ensure filter_unseen has NO hidden 5-cap ids_in_order = [n["ig_post_id"] for n in normalized if n.get("ig_post_id")] unseen_ids = set(filter_unseen(req.username, ids_in_order)) unseen = [n for n in normalized if n["ig_post_id"] in unseen_ids] # 5) Clip to requested count and persist the backfill cursor (progress) batch = unseen[:target] set_cursor(db, req.username, final_cursor) # 6) Mark these as seen so the next run skips them mark_seen(req.username, [b["ig_post_id"] for b in batch]) # 7) Ensure account row account = db.query(InstagramAccount).filter_by(username=req.username).first() if not account: account = InstagramAccount( username=req.username, seller_id=req.seller_id, is_active=True, last_synced=None ) db.add(account) db.commit() db.refresh(account) result: List[ScrapedPost] = [] new_post_count = 0 # 8) Upload + save each post in the batch for item in batch: post_id = str(item["ig_post_id"]) # ENSURE clean, no slashes seller_id = str(req.seller_id) # skip if already saved (race-safety) if db.query(InstagramPost).filter_by(ig_post_id=post_id).first(): continue # media URL list (normalized parser gives remote_urls) media_urls = list(item.get("remote_urls", [])) # If this is a video post, keep only true video URLs (avoid thumbnail duplication) if item["media_type"] == "video": thumb = (item.get("thumbnail_url") or "").strip() def is_video(u: str) -> bool: base = u.split("?", 1)[0].lower() return base.endswith((".mp4", ".mov", ".m4v", ".webm")) media_urls = [u for u in media_urls if is_video(u) and u != thumb] # Upload media to S3 (optimized path) uploaded_urls: List[str] = [] for remote_url in media_urls: try: if item["media_type"] == "video": # optimized 480p H.264 CRF23 (+faststart) url = upload_video_from_url_optimized(remote_url, seller_id, post_id) elif item["media_type"] in ("image", "carousel"): # carousel items may be images; optimize to WebP url = upload_image_from_url_optimized(remote_url, seller_id, post_id) if url is None: # some carousels include videos; try generic fallback url = upload_media_from_url(remote_url, seller_id, post_id) else: # unknown → generic detection/fallback url = upload_media_from_url(remote_url, seller_id, post_id) if url: uploaded_urls.append(url) except Exception as e: print(f"⚠️ Error uploading media: {e}") continue # Upload/optimize thumbnail (if present) final_thumbnail = item.get("thumbnail_url") or None if final_thumbnail: try: thumb_url = upload_image_from_url_optimized(final_thumbnail, seller_id, post_id) if thumb_url: final_thumbnail = thumb_url else: # fallback to generic (no optimize) alt = upload_media_from_url(final_thumbnail, seller_id, post_id) if alt: final_thumbnail = alt except Exception as e: print(f"⚠️ Failed to upload thumbnail: {e}") # Save created_at = datetime.fromtimestamp(item["taken_at"]) if item.get("taken_at") else datetime.utcnow() post_entry = InstagramPost( ig_post_id=post_id, account_id=account.id, media_type=item["media_type"], caption=item.get("caption"), thumbnail_url=final_thumbnail, local_media=uploaded_urls, created_at=created_at, ) db.add(post_entry) db.commit() result.append(ScrapedPost( ig_post_id=post_id, media_type=item["media_type"], caption=item.get("caption"), thumbnail_url=final_thumbnail, local_media=uploaded_urls, created_at=created_at, )) new_post_count += 1 if new_post_count >= target: break return result