FastApi-ISS/scraper.py

# scraper.py

# --- imports ---
from sqlalchemy.orm import Session
from database import InstagramAccount, InstagramPost
from utils.boxapi_client import get_media_page, BoxAPIError  # ← paged client
from utils.parser_fast import parse_boxapi_items_fast
from utils.seen_store import filter_unseen, mark_seen
# use optimized uploaders; keep the generic fallback as well
from utils.s3_uploader import (
    upload_media_from_url,
    upload_image_from_url_optimized,
    upload_video_from_url_optimized,
)
from schemas import ScrapeRequest, ScrapedPost
from typing import List, Dict, Any, Optional
from datetime import datetime
from scraper_cursor import get_cursor, set_cursor  # ← new helpers


def _collect_across_pages(username: str, target: int, seed_cursor: Optional[str]) -> tuple[list[Dict[str, Any]], Optional[str]]:
    """
    1) Fetch ONE newest page (max_id=None).
    2) If still short, continue with `seed_cursor` (or newest next_max_id) to backfill older pages.
    Returns (raw_items, final_cursor_to_store).
    """
    collected: list[Dict[str, Any]] = []
    newest_next: Optional[str] = None

    # Step 1: newest page first (cheap check for new content)
    need = min(12, target)
    try:
        page, _log, newest_next = get_media_page(username=username, count=need, max_id=None)
        if page:
            collected.extend(page)
    except Exception:
        # ignore and continue with backfill-only path
        pass

    # Step 2: continue older pages from stored cursor (or newest_next if none stored)
    cursor = seed_cursor or newest_next
    while len(collected) < target and cursor:
        need = min(12, target - len(collected))
        page, _log, cursor = get_media_page(username=username, count=need, max_id=cursor)
        if not page:
            break
        collected.extend(page)

    return collected, cursor  # final cursor to persist


def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]:
    target = int(req.max_count)

    # 1) Collect raw items across newest + backfill pages (no request schema change)
    stored_cursor = get_cursor(db, req.username)  # may be None on first run
    try:
        raw_items, final_cursor = _collect_across_pages(req.username, target, stored_cursor)
    except BoxAPIError as e:
        print(f"❌ BoxAPI error: {e}")
        return []

    if not raw_items:
        return []

    # 2) Normalize once on the aggregated list
    normalized = parse_boxapi_items_fast(raw_items)

    # 3) Optional cursor filter (kept as-is)
    since_ts = getattr(req, "since_taken_at", None)
    if since_ts:
        try:
            since_ts = int(since_ts)
            normalized = [n for n in normalized if (n.get("taken_at") or 0) > since_ts]
        except Exception:
            pass

    # 4) Filter already-seen FIRST (avoid re-uploading) — ensure filter_unseen has NO hidden 5-cap
    ids_in_order = [n["ig_post_id"] for n in normalized if n.get("ig_post_id")]
    unseen_ids = set(filter_unseen(req.username, ids_in_order))
    unseen = [n for n in normalized if n["ig_post_id"] in unseen_ids]

    # 5) Clip to requested count and persist the backfill cursor (progress)
    batch = unseen[:target]
    set_cursor(db, req.username, final_cursor)

    # 6) Mark these as seen so the next run skips them
    mark_seen(req.username, [b["ig_post_id"] for b in batch])

    # 7) Ensure account row
    account = db.query(InstagramAccount).filter_by(username=req.username).first()
    if not account:
        account = InstagramAccount(
            username=req.username,
            seller_id=req.seller_id,
            is_active=True,
            last_synced=None
        )
        db.add(account)
        db.commit()
        db.refresh(account)

    result: List[ScrapedPost] = []
    new_post_count = 0

    # 8) Upload + save each post in the batch
    for item in batch:
        post_id = str(item["ig_post_id"])          # ENSURE clean, no slashes
        seller_id = str(req.seller_id)

        # skip if already saved (race-safety)
        if db.query(InstagramPost).filter_by(ig_post_id=post_id).first():
            continue

        # media URL list (normalized parser gives remote_urls)
        media_urls = list(item.get("remote_urls", []))

        # If this is a video post, keep only true video URLs (avoid thumbnail duplication)
        if item["media_type"] == "video":
            thumb = (item.get("thumbnail_url") or "").strip()
            def is_video(u: str) -> bool:
                base = u.split("?", 1)[0].lower()
                return base.endswith((".mp4", ".mov", ".m4v", ".webm"))
            media_urls = [u for u in media_urls if is_video(u) and u != thumb]

        # Upload media to S3 (optimized path)
        uploaded_urls: List[str] = []
        for remote_url in media_urls:
            try:
                if item["media_type"] == "video":
                    # optimized 480p H.264 CRF23 (+faststart)
                    url = upload_video_from_url_optimized(remote_url, seller_id, post_id)
                elif item["media_type"] in ("image", "carousel"):
                    # carousel items may be images; optimize to WebP
                    url = upload_image_from_url_optimized(remote_url, seller_id, post_id)
                    if url is None:
                        # some carousels include videos; try generic fallback
                        url = upload_media_from_url(remote_url, seller_id, post_id)
                else:
                    # unknown → generic detection/fallback
                    url = upload_media_from_url(remote_url, seller_id, post_id)

                if url:
                    uploaded_urls.append(url)
            except Exception as e:
                print(f"⚠️ Error uploading media: {e}")
                continue

        # Upload/optimize thumbnail (if present)
        final_thumbnail = item.get("thumbnail_url") or None
        if final_thumbnail:
            try:
                thumb_url = upload_image_from_url_optimized(final_thumbnail, seller_id, post_id)
                if thumb_url:
                    final_thumbnail = thumb_url
                else:
                    # fallback to generic (no optimize)
                    alt = upload_media_from_url(final_thumbnail, seller_id, post_id)
                    if alt:
                        final_thumbnail = alt
            except Exception as e:
                print(f"⚠️ Failed to upload thumbnail: {e}")

        # Save
        created_at = datetime.fromtimestamp(item["taken_at"]) if item.get("taken_at") else datetime.utcnow()
        post_entry = InstagramPost(
            ig_post_id=post_id,
            account_id=account.id,
            media_type=item["media_type"],
            caption=item.get("caption"),
            thumbnail_url=final_thumbnail,
            local_media=uploaded_urls,
            created_at=created_at,
        )
        db.add(post_entry)
        db.commit()

        result.append(ScrapedPost(
            ig_post_id=post_id,
            media_type=item["media_type"],
            caption=item.get("caption"),
            thumbnail_url=final_thumbnail,
            local_media=uploaded_urls,
            created_at=created_at,
        ))

        new_post_count += 1
        if new_post_count >= target:
            break

    return result