FastApi-ISS/scraper.py

# scraper.py

# --- imports ---
from sqlalchemy.orm import Session
from database import InstagramAccount, InstagramPost
from utils.boxapi_client import get_media_page, BoxAPIError  # ← paged client
from utils.parser_fast import parse_boxapi_items_fast
from utils.seen_store import filter_unseen, mark_seen
# use optimized uploaders; keep the generic fallback as well
from utils.s3_uploader import (
    upload_media_from_url,
    upload_image_from_url_optimized,
    upload_video_from_url_optimized,
)
from schemas import ScrapeRequest, ScrapedPost
from typing import List, Dict, Any, Optional
from datetime import datetime
from scraper_cursor import get_cursor, set_cursor  # ← new helpers


def _extract_minimal(item: Dict[str, Any]) -> tuple[Optional[str], Optional[int]]:
    """Return (ig_id, taken_at) from a raw BoxAPI item without full parsing."""
    ig_id = item.get("id") or item.get("pk")
    if ig_id is not None:
        ig_id = str(ig_id)
    taken_at = item.get("taken_at") or item.get("timestamp")  # might be int/epoch
    try:
        taken_at = int(taken_at) if taken_at is not None else None
    except Exception:
        taken_at = None
    return ig_id, taken_at


def _collect_unseen_raw(
    username: str,
    target_unseen: int,
    stored_cursor: Optional[str],
    since_ts: Optional[int],
) -> tuple[List[Dict[str, Any]], Optional[str], int]:
    """
    Keep requesting BoxAPI pages until we accumulate `target_unseen` UNSEEN items,
    or we run out of pages. We do minimal work per page to decide quickly.

    Returns:
      - selected_raw: only the unseen (and since_ts-passing) raw items in order
      - final_cursor: the last next_max_id we reached (to persist)
      - unseen_count: number of unseen items we actually collected
    """
    selected_raw: List[Dict[str, Any]] = []
    final_cursor: Optional[str] = stored_cursor
    # 1) newest page first
    next_id: Optional[str] = None
    try:
        first_page, _log, next_id = get_media_page(username=username, count=min(12, target_unseen), max_id=None)
    except Exception:
        first_page = []

    def add_unseen_from_page(page: List[Dict[str, Any]]) -> int:
        """Filter a page to unseen (and since_ts), append to selected_raw, return how many added."""
        if not page:
            return 0
        # Extract ids in order
        ids_in_order: List[str] = []
        since_mask: List[bool] = []
        for it in page:
            ig_id, taken_at = _extract_minimal(it)
            if ig_id:
                # optional since_ts filter (skip if not passing)
                ok = True
                if since_ts is not None and taken_at is not None:
                    ok = taken_at > since_ts
                ids_in_order.append(ig_id)
                since_mask.append(ok)
        if not ids_in_order:
            return 0

        # Fast unseen check
        unseen_ids_set = set(filter_unseen(username, ids_in_order))
        if not unseen_ids_set:
            return 0

        # Append only ids that pass since_mask AND are unseen, preserving order
        added = 0
        for it in page:
            ig_id, taken_at = _extract_minimal(it)
            if not ig_id:
                continue
            idx = ids_in_order.index(ig_id)  # safe: ig_id exists in list
            if not since_mask[idx]:
                continue
            if ig_id in unseen_ids_set:
                selected_raw.append(it)
                added += 1
                if len(selected_raw) >= target_unseen:
                    break
        return added

    # Process newest page first
    add_unseen_from_page(first_page)

    # 2) then keep going with stored cursor (older pages). Seed with stored or newest next_id.
    cursor = stored_cursor or next_id
    # Only continue if we still need more
    while len(selected_raw) < target_unseen and cursor:
        need = min(12, target_unseen - len(selected_raw))
        page, _log, cursor = get_media_page(username=username, count=need, max_id=cursor)
        added = add_unseen_from_page(page)
        final_cursor = cursor  # we move the cursor forward as we traverse
        if not page:
            break
        # keep looping until we hit target or run out (cursor becomes None)

    return selected_raw, final_cursor, len(selected_raw)


def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]:
    target = int(req.max_count)

    # Optional: allow caller to pass since_taken_at; coerce safely to int
    since_ts_raw = getattr(req, "since_taken_at", None)
    since_ts: Optional[int] = None
    if since_ts_raw is not None:
        try:
            since_ts = int(since_ts_raw)
        except Exception:
            since_ts = None

    # 1) Gather ONLY unseen raw items up to target (newest → older via cursor)
    stored_cursor = get_cursor(db, req.username)  # may be None
    try:
        selected_raw, final_cursor, unseen_count = _collect_unseen_raw(
            username=req.username,
            target_unseen=target,
            stored_cursor=stored_cursor,
            since_ts=since_ts,
        )
    except BoxAPIError as e:
        print(f"❌ BoxAPI error: {e}")
        return []

    if not selected_raw:
        # Still store cursor progress (we may have advanced even if nothing new)
        set_cursor(db, req.username, final_cursor)
        return []

    # 2) Normalize ONLY the selected unseen raw items
    normalized = parse_boxapi_items_fast(selected_raw)

    # 3) Defensive re-check: filter_unseen again after normalize by IG IDs (handles races)
    ids_in_order = [n.get("ig_post_id") for n in normalized if n.get("ig_post_id")]
    if not ids_in_order:
        set_cursor(db, req.username, final_cursor)
        return []

    unseen_ids = set(filter_unseen(req.username, ids_in_order))
    final_batch = [n for n in normalized if n.get("ig_post_id") in unseen_ids][:target]

    # Persist cursor progress AFTER we’ve walked pages
    set_cursor(db, req.username, final_cursor)

    # Mark as seen now to avoid future reprocessing
    mark_seen(req.username, [b["ig_post_id"] for b in final_batch])

    # 4) Ensure account row
    account = db.query(InstagramAccount).filter_by(username=req.username).first()
    if not account:
        account = InstagramAccount(
            username=req.username,
            seller_id=req.seller_id,
            is_active=True,
            last_synced=None
        )
        db.add(account)
        db.commit()
        db.refresh(account)

    # 5) Upload + save each post in the final batch
    result: List[ScrapedPost] = []
    for item in final_batch:
        post_id = str(item["ig_post_id"])
        seller_id = str(req.seller_id)

        # Skip if already saved (DB guard)
        if db.query(InstagramPost).filter_by(ig_post_id=post_id).first():
            continue

        # Media URLs (from normalized)
        media_urls = list(item.get("remote_urls", []))

        # If video, keep only true video URLs (avoid using the thumb URL)
        if item["media_type"] == "video":
            thumb = (item.get("thumbnail_url") or "").strip()

            def is_video(u: str) -> bool:
                base = u.split("?", 1)[0].lower()
                return base.endswith((".mp4", ".mov", ".m4v", ".webm"))

            media_urls = [u for u in media_urls if is_video(u) and u != thumb]

        # Upload media
        uploaded_urls: List[str] = []
        for remote_url in media_urls:
            try:
                if item["media_type"] == "video":
                    url = upload_video_from_url_optimized(remote_url, seller_id, post_id)
                elif item["media_type"] in ("image", "carousel"):
                    url = upload_image_from_url_optimized(remote_url, seller_id, post_id)
                    if url is None:
                        url = upload_media_from_url(remote_url, seller_id, post_id)
                else:
                    url = upload_media_from_url(remote_url, seller_id, post_id)
                if url:
                    uploaded_urls.append(url)
            except Exception as e:
                print(f"⚠️ Error uploading media: {e}")
                continue

        # Upload/optimize thumbnail (if present)
        final_thumbnail = item.get("thumbnail_url") or None
        if final_thumbnail:
            try:
                thumb_url = upload_image_from_url_optimized(final_thumbnail, seller_id, post_id)
                if thumb_url:
                    final_thumbnail = thumb_url
                else:
                    alt = upload_media_from_url(final_thumbnail, seller_id, post_id)
                    if alt:
                        final_thumbnail = alt
            except Exception as e:
                print(f"⚠️ Failed to upload thumbnail: {e}")

        # Save row
        created_at = datetime.fromtimestamp(item["taken_at"]) if item.get("taken_at") else datetime.utcnow()
        post_entry = InstagramPost(
            ig_post_id=post_id,
            account_id=account.id,
            media_type=item["media_type"],
            caption=item.get("caption"),
            thumbnail_url=final_thumbnail,
            local_media=uploaded_urls,
            created_at=created_at,
        )
        db.add(post_entry)
        db.commit()

        result.append(ScrapedPost(
            ig_post_id=post_id,
            media_type=item["media_type"],
            caption=item.get("caption"),
            thumbnail_url=final_thumbnail,
            local_media=uploaded_urls,
            created_at=created_at,
        ))

        if len(result) >= target:
            break

    return result