From 4583e08ca5a80d515571e3f863a9859a3452d022 Mon Sep 17 00:00:00 2001
From: Hossein <fazlihossein6@gmail.com>
Date: Thu, 7 Aug 2025 02:46:57 +0330
Subject: [PATCH] scrapper hai hai

---
 scraper.py | 56 ++++++++++++++++++++++++++++++++++--------------------
 1 file changed, 35 insertions(+), 21 deletions(-)

diff --git a/scraper.py b/scraper.py
index 1d28794..300c174 100644
--- a/scraper.py
+++ b/scraper.py
@@ -13,7 +13,7 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]:
 
     try:
         user_id = client.user_id_from_username(req.username)
-        posts = client.user_medias_v1(user_id, amount=30)  # fetch more to skip duplicates
+        posts = client.user_medias_v1(user_id, amount=30)
     except Exception as e:
         print(f"❌ Failed to fetch posts for @{req.username}: {e}")
         return []
@@ -37,50 +37,64 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]:
     for post in posts:
         post_id = str(post.id)
 
-        # Skip if already in DB
         if db.query(InstagramPost).filter_by(ig_post_id=post_id).first():
             continue
 
         media_items = []
 
         try:
-            # Handle carousel
+            # Carousel
             if post.media_type == 8 and hasattr(post, "resources"):
                 for res in post.resources:
                     media_url = res.video_url if res.media_type == 2 else res.thumbnail_url
                     if not media_url:
                         continue
-                    local_url = upload_media_from_url(media_url, str(req.seller_id), post_id)
-                    if local_url:
-                        media_items.append(MediaItem(
-                            media_type="video" if res.media_type == 2 else "image",
-                            media_url=media_url,
-                            local_media=local_url
-                        ))
+                    try:
+                        local_url = upload_media_from_url(media_url, str(req.seller_id), f"{post_id}/")
+                        if local_url:
+                            media_items.append(MediaItem(
+                                media_type="video" if res.media_type == 2 else "image",
+                                media_url=media_url,
+                                local_media=local_url
+                            ))
+                    except Exception as e:
+                        print(f"⚠️ Error uploading media: {e}")
+                        continue
             else:
                 media_url = post.video_url if post.media_type == 2 else post.thumbnail_url
                 if not media_url:
                     continue
-                local_url = upload_media_from_url(media_url, str(req.seller_id), post_id)
-                if local_url:
-                    media_items.append(MediaItem(
-                        media_type="video" if post.media_type == 2 else "image",
-                        media_url=media_url,
-                        local_media=local_url
-                    ))
+                try:
+                    local_url = upload_media_from_url(media_url, str(req.seller_id), f"{post_id}/")
+                    if local_url:
+                        media_items.append(MediaItem(
+                            media_type="video" if post.media_type == 2 else "image",
+                            media_url=media_url,
+                            local_media=local_url
+                        ))
+                except Exception as e:
+                    print(f"⚠️ Error uploading media: {e}")
+                    continue
+
         except LoginRequired:
             print("⛔ Login expired or failed mid-request.")
             break
         except Exception as e:
-            print(f"⚠️ Error downloading media: {e}")
+            print(f"⚠️ Unexpected error while handling media: {e}")
             continue
 
         if not media_items:
             continue
 
-        # ✅ Upload thumbnail separately and store local version
-        uploaded_thumb = upload_media_from_url(post.thumbnail_url, str(req.seller_id), f"{post_id}/thumbnail")
-        final_thumbnail = uploaded_thumb or post.thumbnail_url  # fallback if upload fails
+        # ✅ Upload thumbnail safely
+        final_thumbnail = post.thumbnail_url or ""
+        if post.thumbnail_url:
+            try:
+                uploaded_thumb = upload_media_from_url(post.thumbnail_url, str(req.seller_id), f"{post_id}/thumbnail")
+                if uploaded_thumb:
+                    final_thumbnail = uploaded_thumb
+            except Exception as e:
+                print(f"⚠️ Failed to upload thumbnail: {e}")
 
         # Save to DB
         post_entry = InstagramPost(