119 lines
4.1 KiB
Python
119 lines
4.1 KiB
Python
from instagrapi.exceptions import LoginRequired
|
|
from sqlalchemy.orm import Session
|
|
from database import InstagramAccount, InstagramPost
|
|
from utils.instagram_client import get_logged_in_client
|
|
from utils.s3_uploader import upload_media_from_url
|
|
from schemas import ScrapeRequest, ScrapedPost
|
|
from typing import List
|
|
from datetime import datetime
|
|
|
|
|
|
def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]:
|
|
client = get_logged_in_client()
|
|
|
|
try:
|
|
user_id = client.user_id_from_username(req.username)
|
|
posts = client.user_medias_v1(user_id, amount=30)
|
|
except Exception as e:
|
|
print(f"❌ Failed to fetch posts for @{req.username}: {e}")
|
|
return []
|
|
|
|
# Get or create InstagramAccount
|
|
account = db.query(InstagramAccount).filter_by(username=req.username).first()
|
|
if not account:
|
|
account = InstagramAccount(
|
|
username=req.username,
|
|
seller_id=req.seller_id,
|
|
is_active=True,
|
|
last_synced=None
|
|
)
|
|
db.add(account)
|
|
db.commit()
|
|
db.refresh(account)
|
|
|
|
result: List[ScrapedPost] = []
|
|
new_post_count = 0
|
|
|
|
for post in posts:
|
|
post_id = str(post.id)
|
|
|
|
if db.query(InstagramPost).filter_by(ig_post_id=post_id).first():
|
|
continue
|
|
|
|
media_urls: List[str] = []
|
|
|
|
try:
|
|
# Carousel
|
|
if post.media_type == 8 and hasattr(post, "resources"):
|
|
for res in post.resources:
|
|
media_url = res.video_url if res.media_type == 2 else res.thumbnail_url
|
|
if not media_url:
|
|
continue
|
|
try:
|
|
local_url = upload_media_from_url(media_url, str(req.seller_id), f"{post_id}/")
|
|
if local_url:
|
|
media_urls.append(local_url)
|
|
except Exception as e:
|
|
print(f"⚠️ Error uploading media: {e}")
|
|
continue
|
|
else:
|
|
media_url = post.video_url if post.media_type == 2 else post.thumbnail_url
|
|
if not media_url:
|
|
continue
|
|
try:
|
|
local_url = upload_media_from_url(media_url, str(req.seller_id), f"{post_id}/")
|
|
if local_url:
|
|
media_urls.append(local_url)
|
|
except Exception as e:
|
|
print(f"⚠️ Error uploading media: {e}")
|
|
continue
|
|
|
|
except LoginRequired:
|
|
print("⛔ Login expired or failed mid-request.")
|
|
break
|
|
except Exception as e:
|
|
print(f"⚠️ Unexpected error while handling media: {e}")
|
|
continue
|
|
|
|
if not media_urls:
|
|
continue
|
|
|
|
# ✅ Upload and override thumbnail
|
|
final_thumbnail = post.thumbnail_url or ""
|
|
if post.thumbnail_url:
|
|
try:
|
|
uploaded_thumb = upload_media_from_url(post.thumbnail_url, str(req.seller_id), f"{post_id}/thumbnail")
|
|
if uploaded_thumb:
|
|
final_thumbnail = uploaded_thumb
|
|
except Exception as e:
|
|
print(f"⚠️ Failed to upload thumbnail: {e}")
|
|
|
|
# Save to DB
|
|
post_entry = InstagramPost(
|
|
ig_post_id=post_id,
|
|
account_id=account.id,
|
|
media_type="carousel" if post.media_type == 8 else ("video" if post.media_type == 2 else "image"),
|
|
caption=post.caption_text or "",
|
|
thumbnail_url=final_thumbnail,
|
|
local_media=media_urls, # ✅ Now a list of strings (S3 URLs)
|
|
created_at=post.taken_at or datetime.utcnow(),
|
|
)
|
|
db.add(post_entry)
|
|
db.commit()
|
|
|
|
# Add to response
|
|
result.append(ScrapedPost(
|
|
ig_post_id=post_id,
|
|
media_type=post_entry.media_type,
|
|
caption=post_entry.caption,
|
|
thumbnail_url=final_thumbnail,
|
|
local_media=media_urls, # ✅ Again, list of strings
|
|
created_at=post_entry.created_at
|
|
))
|
|
|
|
new_post_count += 1
|
|
if new_post_count >= req.max_count:
|
|
break
|
|
|
|
return result
|