fazli/FastApi-ISS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

scrapper hai hai

Browse Source
This commit is contained in:
Hossein 2025-08-07 02:46:57 +03:30
parent 02f6ed5b9b
commit 4583e08ca5
1 changed files with 35 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
scraper.py
View File

@ -13,7 +13,7 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]:
try: try:
user_id = client.user_id_from_username(req.username) user_id = client.user_id_from_username(req.username)
posts = client.user_medias_v1(user_id, amount=30) # fetch more to skip duplicates posts = client.user_medias_v1(user_id, amount=30)
except Exception as e: except Exception as e:
print(f"❌ Failed to fetch posts for @{req.username}: {e}") print(f"❌ Failed to fetch posts for @{req.username}: {e}")
return [] return []
@ -37,50 +37,64 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]:
for post in posts: for post in posts:
post_id = str(post.id) post_id = str(post.id)
# Skip if already in DB
if db.query(InstagramPost).filter_by(ig_post_id=post_id).first(): if db.query(InstagramPost).filter_by(ig_post_id=post_id).first():
continue continue
media_items = [] media_items = []
try: try:
# Handle carousel # Carousel
if post.media_type == 8 and hasattr(post, "resources"): if post.media_type == 8 and hasattr(post, "resources"):
for res in post.resources: for res in post.resources:
media_url = res.video_url if res.media_type == 2 else res.thumbnail_url media_url = res.video_url if res.media_type == 2 else res.thumbnail_url
if not media_url: if not media_url:
continue continue
local_url = upload_media_from_url(media_url, str(req.seller_id), post_id) try:
if local_url: local_url = upload_media_from_url(media_url, str(req.seller_id), f"{post_id}/")
media_items.append(MediaItem( if local_url:
media_type="video" if res.media_type == 2 else "image", media_items.append(MediaItem(
media_url=media_url, media_type="video" if res.media_type == 2 else "image",
local_media=local_url media_url=media_url,
)) local_media=local_url
))
except Exception as e:
print(f"⚠️ Error uploading media: {e}")
continue
else: else:
media_url = post.video_url if post.media_type == 2 else post.thumbnail_url media_url = post.video_url if post.media_type == 2 else post.thumbnail_url
if not media_url: if not media_url:
continue continue
local_url = upload_media_from_url(media_url, str(req.seller_id), post_id) try:
if local_url: local_url = upload_media_from_url(media_url, str(req.seller_id), f"{post_id}/")
media_items.append(MediaItem( if local_url:
media_type="video" if post.media_type == 2 else "image", media_items.append(MediaItem(
media_url=media_url, media_type="video" if post.media_type == 2 else "image",
local_media=local_url media_url=media_url,
)) local_media=local_url
))
except Exception as e:
print(f"⚠️ Error uploading media: {e}")
continue
except LoginRequired: except LoginRequired:
print("⛔ Login expired or failed mid-request.") print("⛔ Login expired or failed mid-request.")
break break
except Exception as e: except Exception as e:
print(f"⚠️ Error downloading media: {e}") print(f"⚠️ Unexpected error while handling media: {e}")
continue continue
if not media_items: if not media_items:
continue continue
# ✅ Upload thumbnail separately and store local version # ✅ Upload thumbnail safely
uploaded_thumb = upload_media_from_url(post.thumbnail_url, str(req.seller_id), f"{post_id}/thumbnail") final_thumbnail = post.thumbnail_url or ""
final_thumbnail = uploaded_thumb or post.thumbnail_url # fallback if upload fails if post.thumbnail_url:
try:
uploaded_thumb = upload_media_from_url(post.thumbnail_url, str(req.seller_id), f"{post_id}/thumbnail")
if uploaded_thumb:
final_thumbnail = uploaded_thumb
except Exception as e:
print(f"⚠️ Failed to upload thumbnail: {e}")
# Save to DB # Save to DB
post_entry = InstagramPost( post_entry = InstagramPost(