change the system to boxapi client

README.md

@@ -12,6 +12,8 @@ A FastAPI application with Docker and docker-compose support.
 - Health check endpoints
 - CORS middleware enabled
 - Auto-generated API documentation
+- Instagram post scraping via BoxAPI
+- S3 media upload and storage
 
 ## API Endpoints
 
@@ -22,6 +24,7 @@ A FastAPI application with Docker and docker-compose support.
 - `POST /items` - Create new item
 - `PUT /items/{item_id}` - Update item
 - `DELETE /items/{item_id}` - Delete item
+- `POST /scrape/` - Scrape Instagram posts (requires API key)
 
 ## Quick Start with Docker Compose
 
@@ -30,10 +33,18 @@ A FastAPI application with Docker and docker-compose support.
    # Copy the example environment file
    cp env.example .env
    
-   # Edit the .env file with your secure passwords
+   # Edit the .env file with your secure passwords and BoxAPI credentials
    # nano .env  # or use your preferred editor
    ```
 
+2. **Configure BoxAPI (for Instagram scraping):**
+   ```bash
+   # Add your BoxAPI credentials to .env
+   BOXAPI_USERNAME=your_boxapi_username
+   BOXAPI_PASSWORD=your_boxapi_password
+   BOXAPI_ENDPOINT=https://boxapi.ir/api/instagram/user/get_media
+   ```
+
 2. **Build and start the application:**
    ```bash
    docker-compose up --build
@@ -111,6 +122,14 @@ curl -X PUT "http://localhost:8000/items/1" \
 curl -X DELETE "http://localhost:8000/items/1"
 ```
 
+### Scrape Instagram posts
+```bash
+curl -X POST "http://localhost:8000/scrape/" \
+     -H "Content-Type: application/json" \
+     -H "X-API-Key: your_api_key_here" \
+     -d '{"username": "instagram_username", "seller_id": "your-seller-uuid", "max_count": 5}'
+```
+
 ### Database Connection
 
 You can connect to the PostgreSQL database directly:
@@ -129,6 +148,8 @@ docker exec -it fastapi-iss-postgres-1 psql -U ${POSTGRES_USER} -d ${POSTGRES_DB
 FastApi-ISS/
 ├── main.py              # FastAPI application
 ├── database.py          # Database configuration and models
+├── scraper.py           # Instagram scraping logic
+├── schemas.py           # Pydantic schemas
 ├── requirements.txt     # Python dependencies
 ├── Dockerfile          # Docker configuration
 ├── docker-compose.yml  # Docker Compose configuration
@@ -138,6 +159,11 @@ FastApi-ISS/
 ├── alembic/            # Database migrations
 │   ├── env.py          # Alembic environment
 │   └── script.py.mako  # Migration template
+├── utils/              # Utility modules
+│   ├── __init__.py
+│   ├── boxapi_client.py # BoxAPI client for Instagram scraping
+│   ├── instagram_client.py # Legacy Instagram client
+│   └── s3_uploader.py  # S3 media upload functionality
 ├── .dockerignore       # Docker ignore file
 ├── .gitignore          # Git ignore file
 └── README.md          # This file
@@ -147,6 +173,11 @@ FastApi-ISS/
 
 The application uses environment variables for configuration. Copy `env.example` to `.env` and customize the values:
 
+### BoxAPI Configuration (Instagram Scraping)
+- `BOXAPI_USERNAME`: Your BoxAPI username
+- `BOXAPI_PASSWORD`: Your BoxAPI password  
+- `BOXAPI_ENDPOINT`: BoxAPI endpoint URL (default: `https://boxapi.ir/api/instagram/user/get_media`)
+
 ### Database Configuration
 - `POSTGRES_DB`: Database name (default: `fastapi_db`)
 - `POSTGRES_USER`: Database user (default: `fastapi_user`)

main.py

@@ -5,10 +5,14 @@ from typing import List, Optional
 from sqlalchemy.orm import Session
 import uvicorn
 import os
+from dotenv import load_dotenv
 from database import get_db, create_tables, Item as ItemModel
 from schemas import ScrapeRequest, ScrapedPost
 from scraper import scrape_and_store
 
+# Load environment variables
+load_dotenv()
+
 # API Key for Django backend authentication
 API_KEY = os.getenv("ENGINE_API_KEY")
 

scraper.py

@@ -1,7 +1,6 @@
-from instagrapi.exceptions import LoginRequired
 from sqlalchemy.orm import Session
 from database import InstagramAccount, InstagramPost
-from utils.instagram_client import get_logged_in_client
+from utils.boxapi_client import fetch_boxapi_posts
 from utils.s3_uploader import upload_media_from_url
 from schemas import ScrapeRequest, ScrapedPost
 from typing import List
@@ -9,13 +8,8 @@ from datetime import datetime
 
 
 def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]:
-    client = get_logged_in_client()
-
-    try:
-        user_id = client.user_id_from_username(req.username)
-        posts = client.user_medias_v1(user_id, amount=30)
-    except Exception as e:
-        print(f"❌ Failed to fetch posts for @{req.username}: {e}")
+    posts = fetch_boxapi_posts(req.username, req.max_count)
+    if not posts:
         return []
 
     # Get or create InstagramAccount
@@ -35,56 +29,38 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]:
     new_post_count = 0
 
     for post in posts:
-        post_id = str(post.id)
+        post_id = str(post.get("id") or post.get("pk"))
+        caption = post.get("caption", "")
+        thumbnail_url = post.get("thumbnail_url")
+        taken_at = post.get("taken_at") or datetime.utcnow()
+        media_type = post.get("media_type") or "image"
 
         if db.query(InstagramPost).filter_by(ig_post_id=post_id).first():
             continue
 
         media_urls: List[str] = []
-
-        try:
-            # Carousel
-            if post.media_type == 8 and hasattr(post, "resources"):
-                for res in post.resources:
-                    media_url = res.video_url if res.media_type == 2 else res.thumbnail_url
-                    if not media_url:
-                        continue
-                    try:
-                        local_url = upload_media_from_url(media_url, str(req.seller_id), f"{post_id}/")
-                        if local_url:
-                            media_urls.append(local_url)
-                    except Exception as e:
-                        print(f"⚠️ Error uploading media: {e}")
-                        continue
-            else:
-                media_url = post.video_url if post.media_type == 2 else post.thumbnail_url
-                if not media_url:
-                    continue
-                try:
-                    local_url = upload_media_from_url(media_url, str(req.seller_id), f"{post_id}/")
-                    if local_url:
-                        media_urls.append(local_url)
-                except Exception as e:
-                    print(f"⚠️ Error uploading media: {e}")
-                    continue
-
-        except LoginRequired:
-            print("⛔ Login expired or failed mid-request.")
-            break
-        except Exception as e:
-            print(f"⚠️ Unexpected error while handling media: {e}")
-            continue
+        for media in post.get("media", []):
+            media_url = media.get("media_url")
+            if not media_url:
+                continue
+            try:
+                local_url = upload_media_from_url(media_url, str(req.seller_id), f"{post_id}/")
+                if local_url:
+                    media_urls.append(local_url)
+            except Exception as e:
+                print(f"⚠️ Error uploading media: {e}")
+                continue
 
         if not media_urls:
             continue
 
-        # ✅ Upload and override thumbnail
-        final_thumbnail = post.thumbnail_url or ""
-        if post.thumbnail_url:
+        # Upload thumbnail (optional)
+        final_thumbnail = thumbnail_url
+        if thumbnail_url:
             try:
-                uploaded_thumb = upload_media_from_url(post.thumbnail_url, str(req.seller_id), f"{post_id}/thumbnail")
-                if uploaded_thumb:
-                    final_thumbnail = uploaded_thumb
+                thumb_uploaded = upload_media_from_url(thumbnail_url, str(req.seller_id), f"{post_id}/thumbnail")
+                if thumb_uploaded:
+                    final_thumbnail = thumb_uploaded
             except Exception as e:
                 print(f"⚠️ Failed to upload thumbnail: {e}")
 
@@ -92,23 +68,22 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]:
         post_entry = InstagramPost(
             ig_post_id=post_id,
             account_id=account.id,
-            media_type="carousel" if post.media_type == 8 else ("video" if post.media_type == 2 else "image"),
-            caption=post.caption_text or "",
+            media_type=media_type,
+            caption=caption,
             thumbnail_url=final_thumbnail,
-            local_media=media_urls,  # ✅ Now a list of strings (S3 URLs)
-            created_at=post.taken_at or datetime.utcnow(),
+            local_media=media_urls,
+            created_at=taken_at,
         )
         db.add(post_entry)
         db.commit()
 
-        # Add to response
         result.append(ScrapedPost(
             ig_post_id=post_id,
-            media_type=post_entry.media_type,
-            caption=post_entry.caption,
+            media_type=media_type,
+            caption=caption,
             thumbnail_url=final_thumbnail,
-            local_media=media_urls,  # ✅ Again, list of strings
-            created_at=post_entry.created_at
+            local_media=media_urls,
+            created_at=taken_at
         ))
 
         new_post_count += 1

utils/boxapi_client.py

@@ -0,0 +1,29 @@
+import os
+import requests
+from typing import List, Dict
+
+BOXAPI_USERNAME = os.getenv("BOXAPI_USERNAME")
+BOXAPI_PASSWORD = os.getenv("BOXAPI_PASSWORD")
+BOXAPI_ENDPOINT = os.getenv("BOXAPI_ENDPOINT", "https://boxapi.ir/api/instagram/user/get_media")
+
+def fetch_boxapi_posts(username: str, count: int = 5) -> List[Dict]:
+    payload = {
+        "username": username,  # 👈 Do NOT replace with user ID
+        "count": count
+    }
+
+    try:
+        response = requests.post(
+            BOXAPI_ENDPOINT,
+            json=payload,
+            auth=(BOXAPI_USERNAME, BOXAPI_PASSWORD),
+            timeout=15
+        )
+        response.raise_for_status()
+        data = response.json()
+        if not isinstance(data, list):
+            raise ValueError("Unexpected response format from BoxAPI")
+        return data
+    except Exception as e:
+        print(f"❌ Failed to fetch posts from BoxAPI: {e}")
+        return []