FastHOSSEIN

env.example

@@ -1,32 +0,0 @@
-# Database Configuration
-POSTGRES_DB=fastapi_db
-POSTGRES_USER=fastapi_user
-POSTGRES_PASSWORD=your_secure_password_here
-POSTGRES_HOST=postgres
-POSTGRES_PORT=5432
-
-# Database URL (constructed from above variables)
-DATABASE_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}
-
-# FastAPI Configuration
-FASTAPI_TITLE=FastAPI ISS Project
-FASTAPI_DESCRIPTION=A FastAPI project with Docker and PostgreSQL support
-FASTAPI_VERSION=1.0.0
-
-# Application Configuration
-PYTHONPATH=/app
-PYTHONUNBUFFERED=1
-
-# Security (for production, change these values)
-SECRET_KEY=your_super_secret_key_here_change_in_production
-ALGORITHM=HS256
-ACCESS_TOKEN_EXPIRE_MINUTES=30
-
-# CORS Configuration
-ALLOWED_ORIGINS=["*"]
-ALLOWED_CREDENTIALS=true
-ALLOWED_METHODS=["*"]
-ALLOWED_HEADERS=["*"]
-
-# Logging
-LOG_LEVEL=INFO 

main.py

@@ -6,6 +6,8 @@ from sqlalchemy.orm import Session
 import uvicorn
 import os
 from database import get_db, create_tables, Item as ItemModel
+from schemas import ScrapeRequest, ScrapedPost
+from scraper import scrape_and_store
 
 app = FastAPI(
     title=os.getenv("FASTAPI_TITLE", "FastAPI ISS Project"),
@@ -92,5 +94,17 @@ async def delete_item(item_id: int, db: Session = Depends(get_db)):
     db.commit()
     return {"message": "Item deleted successfully"}
 
+# ============================================================================
+# Instagram Scraper Endpoint
+# ============================================================================
+
+@app.post("/scrape/", response_model=List[ScrapedPost])
+async def scrape_posts(request: ScrapeRequest, db: Session = Depends(get_db)):
+    """Scrape Instagram posts for a given username"""
+    try:
+        return scrape_and_store(db, request)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
 if __name__ == "__main__":
     uvicorn.run(app, host="0.0.0.0", port=8000) 

requirements.txt

@@ -5,3 +5,7 @@ python-multipart==0.0.6
 sqlalchemy==2.0.23
 psycopg2-binary==2.9.9
 alembic==1.13.0
+instagrapi==2.0.0
+python-dotenv==1.0.0
+boto3==1.34.0
+requests==2.31.0 

schemas.py

@@ -0,0 +1,25 @@
+from pydantic import BaseModel, Field
+from typing import List, Optional
+from uuid import UUID
+from datetime import datetime
+
+
+class ScrapeRequest(BaseModel):
+    username: str
+    seller_id: UUID
+    max_count: int = Field(default=5, ge=1, le=50)
+
+
+class MediaItem(BaseModel):
+    media_type: str  # image | video
+    media_url: str   # original Instagram URL
+    local_media: str # full S3/MinIO URL
+
+
+class ScrapedPost(BaseModel):
+    ig_post_id: str
+    media_type: str  # image | video | carousel
+    caption: Optional[str]
+    thumbnail_url: Optional[str]
+    local_media: List[MediaItem]
+    created_at: datetime 

scraper.py

@@ -0,0 +1,108 @@
+from instagrapi.exceptions import LoginRequired
+from sqlalchemy.orm import Session
+from database import InstagramAccount, InstagramPost
+from utils.instagram_client import get_logged_in_client
+from utils.s3_uploader import upload_media_from_url
+from schemas import ScrapeRequest, ScrapedPost, MediaItem
+from typing import List
+from datetime import datetime
+
+
+def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]:
+    client = get_logged_in_client()
+
+    try:
+        user_id = client.user_id_from_username(req.username)
+        posts = client.user_medias_v1(user_id, amount=30)  # fetch more to skip duplicates
+    except Exception as e:
+        print(f"❌ Failed to fetch posts for @{req.username}: {e}")
+        return []
+
+    # Get or create InstagramAccount
+    account = db.query(InstagramAccount).filter_by(username=req.username).first()
+    if not account:
+        account = InstagramAccount(
+            username=req.username,
+            seller_id=req.seller_id,
+            is_active=True,
+            last_synced=None
+        )
+        db.add(account)
+        db.commit()
+        db.refresh(account)
+
+    result: List[ScrapedPost] = []
+    new_post_count = 0
+
+    for post in posts:
+        post_id = str(post.id)
+
+        # Skip if already in DB
+        if db.query(InstagramPost).filter_by(ig_post_id=post_id).first():
+            continue
+
+        media_items = []
+
+        try:
+            # Handle carousel
+            if post.media_type == 8 and hasattr(post, "resources"):
+                for res in post.resources:
+                    media_url = res.video_url if res.media_type == 2 else res.thumbnail_url
+                    if not media_url:
+                        continue
+                    local_url = upload_media_from_url(media_url, str(req.seller_id), post_id)
+                    if local_url:
+                        media_items.append(MediaItem(
+                            media_type="video" if res.media_type == 2 else "image",
+                            media_url=media_url,
+                            local_media=local_url
+                        ))
+            else:
+                media_url = post.video_url if post.media_type == 2 else post.thumbnail_url
+                if not media_url:
+                    continue
+                local_url = upload_media_from_url(media_url, str(req.seller_id), post_id)
+                if local_url:
+                    media_items.append(MediaItem(
+                        media_type="video" if post.media_type == 2 else "image",
+                        media_url=media_url,
+                        local_media=local_url
+                    ))
+        except LoginRequired:
+            print("⛔ Login expired or failed mid-request.")
+            break
+        except Exception as e:
+            print(f"⚠️ Error downloading media: {e}")
+            continue
+
+        if not media_items:
+            continue
+
+        # Save to DB
+        post_entry = InstagramPost(
+            ig_post_id=post_id,
+            account_id=account.id,
+            media_type="carousel" if post.media_type == 8 else ("video" if post.media_type == 2 else "image"),
+            caption=post.caption_text or "",
+            thumbnail_url=post.thumbnail_url,
+            local_media=[m.dict() for m in media_items],
+            created_at=post.taken_at or datetime.utcnow(),
+        )
+        db.add(post_entry)
+        db.commit()
+
+        # Add to result for response
+        result.append(ScrapedPost(
+            ig_post_id=post_id,
+            media_type=post_entry.media_type,
+            caption=post_entry.caption,
+            thumbnail_url=post_entry.thumbnail_url,
+            local_media=media_items,
+            created_at=post_entry.created_at
+        ))
+
+        new_post_count += 1
+        if new_post_count >= req.max_count:
+            break
+
+    return result 

utils/__init__.py

@@ -0,0 +1 @@
+# Utils package for Instagram scraper utilities 

utils/instagram_client.py

@@ -0,0 +1,44 @@
+import os
+import json
+import uuid
+from pathlib import Path
+from instagrapi import Client
+from dotenv import dotenv_values
+
+# Load IG accounts from env (expects a JSON list string)
+INSTAGRAM_ACCOUNTS = json.loads(os.getenv("INSTAGRAM_ACCOUNTS", "[]"))
+
+# Path for storing session files
+SESSION_DIR = os.path.join(os.getcwd(), "instagram_sessions")
+os.makedirs(SESSION_DIR, exist_ok=True)
+
+
+def get_logged_in_client() -> Client:
+    for account in INSTAGRAM_ACCOUNTS:
+        username = account["username"]
+        password = account["password"]
+        session_path = os.path.join(SESSION_DIR, f"{username}.json")
+
+        client = Client()
+
+        try:
+            if os.path.exists(session_path):
+                with open(session_path, "r") as f:
+                    settings = json.load(f)
+                client.load_settings(settings)
+                client.login(username, password)
+                print(f"✅ Reused session: {username}")
+            else:
+                client.login(username, password)
+                with open(session_path, "w") as f:
+                    json.dump(client.get_settings(), f)
+                print(f"✅ Fresh login: {username}")
+            return client
+
+        except Exception as e:
+            print(f"❌ Login failed for {username}: {e}")
+            if os.path.exists(session_path):
+                os.remove(session_path)
+                print(f"🧹 Removed invalid session for {username}")
+
+    raise Exception("❌ All Instagram accounts failed login.") 

utils/s3_uploader.py

@@ -0,0 +1,50 @@
+import os
+import boto3
+import uuid
+import mimetypes
+from urllib.parse import urljoin
+import requests
+
+AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID")
+AWS_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY")
+AWS_STORAGE_BUCKET_NAME = os.getenv("AWS_STORAGE_BUCKET_NAME")
+S3_ENDPOINT_URL = os.getenv("S3_ENDPOINT_URL")
+S3_REGION = os.getenv("S3_REGION", "us-east-1")
+MEDIA_BASE_PATH = os.getenv("MEDIA_BASE_PATH", "instagram")
+
+session = boto3.session.Session()
+s3 = session.client(
+    service_name="s3",
+    aws_access_key_id=AWS_ACCESS_KEY_ID,
+    aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
+    endpoint_url=S3_ENDPOINT_URL,
+    region_name=S3_REGION,
+)
+
+
+def upload_media_from_url(media_url: str, seller_id: str, post_id: str) -> str | None:
+    try:
+        response = requests.get(media_url, stream=True, timeout=10)
+        response.raise_for_status()
+
+        content_type = response.headers.get("Content-Type", "")
+        ext = mimetypes.guess_extension(content_type) or ".jpg"
+        if ext in [".jpe", ".jfif", ".bin"]:
+            ext = ".jpg"
+
+        filename = f"{uuid.uuid4()}{ext}"
+        s3_path = f"{MEDIA_BASE_PATH}/{seller_id}/{post_id}/{filename}"
+
+        s3.upload_fileobj(
+            response.raw,
+            Bucket=AWS_STORAGE_BUCKET_NAME,
+            Key=s3_path,
+            ExtraArgs={"ACL": "public-read", "ContentType": content_type}
+        )
+
+        public_url = f"{S3_ENDPOINT_URL}/{AWS_STORAGE_BUCKET_NAME}/{s3_path}"
+        return public_url
+
+    except Exception as e:
+        print(f"❌ Failed to upload media from {media_url}: {e}")
+        return None