fazli/FastApi-ISS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

scrapper hai hai

Browse Source
This commit is contained in:
Hossein 2025-08-07 02:46:57 +03:30
parent 02f6ed5b9b
commit 4583e08ca5
1 changed files with 35 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
scraper.py
View File

@ -13,7 +13,7 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]:
try:
user_id = client.user_id_from_username(req.username)
posts = client.user_medias_v1(user_id, amount=30) # fetch more to skip duplicates
posts = client.user_medias_v1(user_id, amount=30)
except Exception as e:
print(f"❌ Failed to fetch posts for @{req.username}: {e}")
return []
@ -37,50 +37,64 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]:
for post in posts:
post_id = str(post.id)
# Skip if already in DB
if db.query(InstagramPost).filter_by(ig_post_id=post_id).first():
continue
media_items = []
try:
# Handle carousel
# Carousel
if post.media_type == 8 and hasattr(post, "resources"):
for res in post.resources:
media_url = res.video_url if res.media_type == 2 else res.thumbnail_url
if not media_url:
continue
local_url = upload_media_from_url(media_url, str(req.seller_id), post_id)
if local_url:
media_items.append(MediaItem(
media_type="video" if res.media_type == 2 else "image",
media_url=media_url,
local_media=local_url
))
try:
local_url = upload_media_from_url(media_url, str(req.seller_id), f"{post_id}/")
if local_url:
media_items.append(MediaItem(
media_type="video" if res.media_type == 2 else "image",
media_url=media_url,
local_media=local_url
))
except Exception as e:
print(f"⚠️ Error uploading media: {e}")
continue
else:
media_url = post.video_url if post.media_type == 2 else post.thumbnail_url
if not media_url:
continue
local_url = upload_media_from_url(media_url, str(req.seller_id), post_id)
if local_url:
media_items.append(MediaItem(
media_type="video" if post.media_type == 2 else "image",
media_url=media_url,
local_media=local_url
))
try:
local_url = upload_media_from_url(media_url, str(req.seller_id), f"{post_id}/")
if local_url:
media_items.append(MediaItem(
media_type="video" if post.media_type == 2 else "image",
media_url=media_url,
local_media=local_url
))
except Exception as e:
print(f"⚠️ Error uploading media: {e}")
continue
except LoginRequired:
print("⛔ Login expired or failed mid-request.")
break
except Exception as e:
print(f"⚠️ Error downloading media: {e}")
print(f"⚠️ Unexpected error while handling media: {e}")
continue
if not media_items:
continue
# ✅ Upload thumbnail separately and store local version
uploaded_thumb = upload_media_from_url(post.thumbnail_url, str(req.seller_id), f"{post_id}/thumbnail")
final_thumbnail = uploaded_thumb or post.thumbnail_url # fallback if upload fails
# ✅ Upload thumbnail safely
final_thumbnail = post.thumbnail_url or ""
if post.thumbnail_url:
try:
uploaded_thumb = upload_media_from_url(post.thumbnail_url, str(req.seller_id), f"{post_id}/thumbnail")
if uploaded_thumb:
final_thumbnail = uploaded_thumb
except Exception as e:
print(f"⚠️ Failed to upload thumbnail: {e}")
# Save to DB
post_entry = InstagramPost(