from sqlalchemy.orm import Session from database import InstagramAccount, InstagramPost from utils.boxapi_client import get_media, BoxAPIError from utils.boxapi_parser import parse_boxapi_items from utils.s3_uploader import upload_media_from_url from schemas import ScrapeRequest, ScrapedPost from typing import List from datetime import datetime def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]: try: items, log_path = get_media(req.username, req.max_count) except BoxAPIError as e: print(f"❌ BoxAPI error: {e}") return [] if not items: return [] # Parse and normalize the items normalized_items = parse_boxapi_items(items) # Get or create InstagramAccount account = db.query(InstagramAccount).filter_by(username=req.username).first() if not account: account = InstagramAccount( username=req.username, seller_id=req.seller_id, is_active=True, last_synced=None ) db.add(account) db.commit() db.refresh(account) result: List[ScrapedPost] = [] new_post_count = 0 for item in normalized_items: post_id = item["ig_post_id"] if db.query(InstagramPost).filter_by(ig_post_id=post_id).first(): continue # Upload remote URLs to S3 media_urls: List[str] = [] for remote_url in item["remote_urls"]: try: local_url = upload_media_from_url(remote_url, str(req.seller_id), f"{post_id}/") if local_url: media_urls.append(local_url) except Exception as e: print(f"⚠️ Error uploading media: {e}") continue if not media_urls: continue # Upload thumbnail (optional) final_thumbnail = item["thumbnail_url"] if item["thumbnail_url"]: try: thumb_uploaded = upload_media_from_url(item["thumbnail_url"], str(req.seller_id), f"{post_id}/thumbnail") if thumb_uploaded: final_thumbnail = thumb_uploaded except Exception as e: print(f"⚠️ Failed to upload thumbnail: {e}") # Save to DB post_entry = InstagramPost( ig_post_id=post_id, account_id=account.id, media_type=item["media_type"], caption=item["caption"], thumbnail_url=final_thumbnail, local_media=media_urls, created_at=item["taken_at"], ) db.add(post_entry) db.commit() result.append(ScrapedPost( ig_post_id=post_id, media_type=item["media_type"], caption=item["caption"], thumbnail_url=final_thumbnail, local_media=media_urls, created_at=item["taken_at"] )) new_post_count += 1 if new_post_count >= req.max_count: break return result