149 lines
5.3 KiB
Python
149 lines
5.3 KiB
Python
# --- imports ---
|
|
from sqlalchemy.orm import Session
|
|
from database import InstagramAccount, InstagramPost
|
|
from utils.boxapi_client import get_media, BoxAPIError
|
|
from utils.parser_fast import parse_boxapi_items_fast
|
|
from utils.seen_store import filter_unseen, mark_seen
|
|
# use optimized uploaders; keep the generic fallback as well
|
|
from utils.s3_uploader import (
|
|
upload_media_from_url,
|
|
upload_image_from_url_optimized,
|
|
upload_video_from_url_optimized,
|
|
)
|
|
from schemas import ScrapeRequest, ScrapedPost
|
|
from typing import List
|
|
from datetime import datetime
|
|
|
|
|
|
def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]:
|
|
try:
|
|
raw_items, log_path = get_media(req.username, max(req.max_count, 12))
|
|
except BoxAPIError as e:
|
|
print(f"❌ BoxAPI error: {e}")
|
|
return []
|
|
|
|
if not raw_items:
|
|
return []
|
|
|
|
normalized = parse_boxapi_items_fast(raw_items)
|
|
|
|
# optional cursor filter
|
|
since_ts = getattr(req, "since_taken_at", None)
|
|
if since_ts:
|
|
try:
|
|
since_ts = int(since_ts)
|
|
normalized = [n for n in normalized if (n.get("taken_at") or 0) > since_ts]
|
|
except Exception:
|
|
pass
|
|
|
|
# filter seen
|
|
ids_in_order = [n["ig_post_id"] for n in normalized if n.get("ig_post_id")]
|
|
unseen_ids = set(filter_unseen(req.username, ids_in_order))
|
|
unseen = [n for n in normalized if n["ig_post_id"] in unseen_ids]
|
|
|
|
# pick next N
|
|
batch = unseen[:req.max_count]
|
|
mark_seen(req.username, [b["ig_post_id"] for b in batch])
|
|
|
|
# ensure account
|
|
account = db.query(InstagramAccount).filter_by(username=req.username).first()
|
|
if not account:
|
|
account = InstagramAccount(
|
|
username=req.username,
|
|
seller_id=req.seller_id,
|
|
is_active=True,
|
|
last_synced=None
|
|
)
|
|
db.add(account)
|
|
db.commit()
|
|
db.refresh(account)
|
|
|
|
result: List[ScrapedPost] = []
|
|
new_post_count = 0
|
|
|
|
for item in batch:
|
|
post_id = str(item["ig_post_id"]) # ENSURE clean, no slashes
|
|
seller_id = str(req.seller_id)
|
|
|
|
# skip if already saved
|
|
if db.query(InstagramPost).filter_by(ig_post_id=post_id).first():
|
|
continue
|
|
|
|
# media URL list (normalized parser gives remote_urls)
|
|
media_urls = list(item.get("remote_urls", []))
|
|
|
|
# If this is a video post, keep only true video URLs
|
|
if item["media_type"] == "video":
|
|
thumb = (item.get("thumbnail_url") or "").strip()
|
|
def is_video(u: str) -> bool:
|
|
base = u.split("?", 1)[0].lower()
|
|
return base.endswith((".mp4", ".mov", ".m4v", ".webm"))
|
|
media_urls = [u for u in media_urls if is_video(u) and u != thumb]
|
|
|
|
# Upload media to S3 (optimized path)
|
|
uploaded_urls: List[str] = []
|
|
for remote_url in media_urls:
|
|
try:
|
|
if item["media_type"] == "video":
|
|
# optimized 480p H.264 CRF23 (+faststart)
|
|
url = upload_video_from_url_optimized(remote_url, seller_id, post_id)
|
|
elif item["media_type"] in ("image", "carousel"):
|
|
# carousel items may be images; optimize to WebP
|
|
url = upload_image_from_url_optimized(remote_url, seller_id, post_id)
|
|
if url is None:
|
|
# some carousels include videos; try generic fallback
|
|
url = upload_media_from_url(remote_url, seller_id, post_id)
|
|
else:
|
|
# unknown → generic detection/fallback
|
|
url = upload_media_from_url(remote_url, seller_id, post_id)
|
|
|
|
if url:
|
|
uploaded_urls.append(url)
|
|
except Exception as e:
|
|
print(f"⚠️ Error uploading media: {e}")
|
|
continue
|
|
|
|
# Upload/optimize thumbnail (if present)
|
|
final_thumbnail = item.get("thumbnail_url") or None
|
|
if final_thumbnail:
|
|
try:
|
|
thumb_url = upload_image_from_url_optimized(final_thumbnail, seller_id, post_id)
|
|
if thumb_url:
|
|
final_thumbnail = thumb_url
|
|
else:
|
|
# fallback to generic (no optimize)
|
|
alt = upload_media_from_url(final_thumbnail, seller_id, post_id)
|
|
if alt:
|
|
final_thumbnail = alt
|
|
except Exception as e:
|
|
print(f"⚠️ Failed to upload thumbnail: {e}")
|
|
|
|
# Save
|
|
created_at = datetime.fromtimestamp(item["taken_at"]) if item.get("taken_at") else datetime.utcnow()
|
|
post_entry = InstagramPost(
|
|
ig_post_id=post_id,
|
|
account_id=account.id,
|
|
media_type=item["media_type"],
|
|
caption=item.get("caption"),
|
|
thumbnail_url=final_thumbnail,
|
|
local_media=uploaded_urls,
|
|
created_at=created_at,
|
|
)
|
|
db.add(post_entry)
|
|
db.commit()
|
|
|
|
result.append(ScrapedPost(
|
|
ig_post_id=post_id,
|
|
media_type=item["media_type"],
|
|
caption=item.get("caption"),
|
|
thumbnail_url=final_thumbnail,
|
|
local_media=uploaded_urls,
|
|
created_at=created_at,
|
|
))
|
|
|
|
new_post_count += 1
|
|
if new_post_count >= req.max_count:
|
|
break
|
|
|
|
return result
|