From 3761de8de09392cbdb757e57ac91d5fd10861d68 Mon Sep 17 00:00:00 2001 From: Hossein Date: Thu, 7 Aug 2025 00:53:03 +0330 Subject: [PATCH] FastHOSSEIN --- env.example | 32 ----------- main.py | 14 +++++ requirements.txt | 6 ++- schemas.py | 25 +++++++++ scraper.py | 108 ++++++++++++++++++++++++++++++++++++++ utils/__init__.py | 1 + utils/instagram_client.py | 44 ++++++++++++++++ utils/s3_uploader.py | 50 ++++++++++++++++++ 8 files changed, 247 insertions(+), 33 deletions(-) delete mode 100644 env.example create mode 100644 schemas.py create mode 100644 scraper.py create mode 100644 utils/__init__.py create mode 100644 utils/instagram_client.py create mode 100644 utils/s3_uploader.py diff --git a/env.example b/env.example deleted file mode 100644 index 494106d..0000000 --- a/env.example +++ /dev/null @@ -1,32 +0,0 @@ -# Database Configuration -POSTGRES_DB=fastapi_db -POSTGRES_USER=fastapi_user -POSTGRES_PASSWORD=your_secure_password_here -POSTGRES_HOST=postgres -POSTGRES_PORT=5432 - -# Database URL (constructed from above variables) -DATABASE_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} - -# FastAPI Configuration -FASTAPI_TITLE=FastAPI ISS Project -FASTAPI_DESCRIPTION=A FastAPI project with Docker and PostgreSQL support -FASTAPI_VERSION=1.0.0 - -# Application Configuration -PYTHONPATH=/app -PYTHONUNBUFFERED=1 - -# Security (for production, change these values) -SECRET_KEY=your_super_secret_key_here_change_in_production -ALGORITHM=HS256 -ACCESS_TOKEN_EXPIRE_MINUTES=30 - -# CORS Configuration -ALLOWED_ORIGINS=["*"] -ALLOWED_CREDENTIALS=true -ALLOWED_METHODS=["*"] -ALLOWED_HEADERS=["*"] - -# Logging -LOG_LEVEL=INFO \ No newline at end of file diff --git a/main.py b/main.py index f9f2d93..5977c70 100644 --- a/main.py +++ b/main.py @@ -6,6 +6,8 @@ from sqlalchemy.orm import Session import uvicorn import os from database import get_db, create_tables, Item as ItemModel +from schemas import ScrapeRequest, ScrapedPost +from scraper import scrape_and_store app = FastAPI( title=os.getenv("FASTAPI_TITLE", "FastAPI ISS Project"), @@ -92,5 +94,17 @@ async def delete_item(item_id: int, db: Session = Depends(get_db)): db.commit() return {"message": "Item deleted successfully"} +# ============================================================================ +# Instagram Scraper Endpoint +# ============================================================================ + +@app.post("/scrape/", response_model=List[ScrapedPost]) +async def scrape_posts(request: ScrapeRequest, db: Session = Depends(get_db)): + """Scrape Instagram posts for a given username""" + try: + return scrape_and_store(db, request) + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + if __name__ == "__main__": uvicorn.run(app, host="0.0.0.0", port=8000) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 1295531..64bdea5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,8 @@ pydantic==2.5.0 python-multipart==0.0.6 sqlalchemy==2.0.23 psycopg2-binary==2.9.9 -alembic==1.13.0 \ No newline at end of file +alembic==1.13.0 +instagrapi==2.0.0 +python-dotenv==1.0.0 +boto3==1.34.0 +requests==2.31.0 \ No newline at end of file diff --git a/schemas.py b/schemas.py new file mode 100644 index 0000000..6ed1e48 --- /dev/null +++ b/schemas.py @@ -0,0 +1,25 @@ +from pydantic import BaseModel, Field +from typing import List, Optional +from uuid import UUID +from datetime import datetime + + +class ScrapeRequest(BaseModel): + username: str + seller_id: UUID + max_count: int = Field(default=5, ge=1, le=50) + + +class MediaItem(BaseModel): + media_type: str # image | video + media_url: str # original Instagram URL + local_media: str # full S3/MinIO URL + + +class ScrapedPost(BaseModel): + ig_post_id: str + media_type: str # image | video | carousel + caption: Optional[str] + thumbnail_url: Optional[str] + local_media: List[MediaItem] + created_at: datetime \ No newline at end of file diff --git a/scraper.py b/scraper.py new file mode 100644 index 0000000..fdf7067 --- /dev/null +++ b/scraper.py @@ -0,0 +1,108 @@ +from instagrapi.exceptions import LoginRequired +from sqlalchemy.orm import Session +from database import InstagramAccount, InstagramPost +from utils.instagram_client import get_logged_in_client +from utils.s3_uploader import upload_media_from_url +from schemas import ScrapeRequest, ScrapedPost, MediaItem +from typing import List +from datetime import datetime + + +def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]: + client = get_logged_in_client() + + try: + user_id = client.user_id_from_username(req.username) + posts = client.user_medias_v1(user_id, amount=30) # fetch more to skip duplicates + except Exception as e: + print(f"❌ Failed to fetch posts for @{req.username}: {e}") + return [] + + # Get or create InstagramAccount + account = db.query(InstagramAccount).filter_by(username=req.username).first() + if not account: + account = InstagramAccount( + username=req.username, + seller_id=req.seller_id, + is_active=True, + last_synced=None + ) + db.add(account) + db.commit() + db.refresh(account) + + result: List[ScrapedPost] = [] + new_post_count = 0 + + for post in posts: + post_id = str(post.id) + + # Skip if already in DB + if db.query(InstagramPost).filter_by(ig_post_id=post_id).first(): + continue + + media_items = [] + + try: + # Handle carousel + if post.media_type == 8 and hasattr(post, "resources"): + for res in post.resources: + media_url = res.video_url if res.media_type == 2 else res.thumbnail_url + if not media_url: + continue + local_url = upload_media_from_url(media_url, str(req.seller_id), post_id) + if local_url: + media_items.append(MediaItem( + media_type="video" if res.media_type == 2 else "image", + media_url=media_url, + local_media=local_url + )) + else: + media_url = post.video_url if post.media_type == 2 else post.thumbnail_url + if not media_url: + continue + local_url = upload_media_from_url(media_url, str(req.seller_id), post_id) + if local_url: + media_items.append(MediaItem( + media_type="video" if post.media_type == 2 else "image", + media_url=media_url, + local_media=local_url + )) + except LoginRequired: + print("⛔ Login expired or failed mid-request.") + break + except Exception as e: + print(f"⚠️ Error downloading media: {e}") + continue + + if not media_items: + continue + + # Save to DB + post_entry = InstagramPost( + ig_post_id=post_id, + account_id=account.id, + media_type="carousel" if post.media_type == 8 else ("video" if post.media_type == 2 else "image"), + caption=post.caption_text or "", + thumbnail_url=post.thumbnail_url, + local_media=[m.dict() for m in media_items], + created_at=post.taken_at or datetime.utcnow(), + ) + db.add(post_entry) + db.commit() + + # Add to result for response + result.append(ScrapedPost( + ig_post_id=post_id, + media_type=post_entry.media_type, + caption=post_entry.caption, + thumbnail_url=post_entry.thumbnail_url, + local_media=media_items, + created_at=post_entry.created_at + )) + + new_post_count += 1 + if new_post_count >= req.max_count: + break + + return result \ No newline at end of file diff --git a/utils/__init__.py b/utils/__init__.py new file mode 100644 index 0000000..05bed71 --- /dev/null +++ b/utils/__init__.py @@ -0,0 +1 @@ +# Utils package for Instagram scraper utilities \ No newline at end of file diff --git a/utils/instagram_client.py b/utils/instagram_client.py new file mode 100644 index 0000000..e530746 --- /dev/null +++ b/utils/instagram_client.py @@ -0,0 +1,44 @@ +import os +import json +import uuid +from pathlib import Path +from instagrapi import Client +from dotenv import dotenv_values + +# Load IG accounts from env (expects a JSON list string) +INSTAGRAM_ACCOUNTS = json.loads(os.getenv("INSTAGRAM_ACCOUNTS", "[]")) + +# Path for storing session files +SESSION_DIR = os.path.join(os.getcwd(), "instagram_sessions") +os.makedirs(SESSION_DIR, exist_ok=True) + + +def get_logged_in_client() -> Client: + for account in INSTAGRAM_ACCOUNTS: + username = account["username"] + password = account["password"] + session_path = os.path.join(SESSION_DIR, f"{username}.json") + + client = Client() + + try: + if os.path.exists(session_path): + with open(session_path, "r") as f: + settings = json.load(f) + client.load_settings(settings) + client.login(username, password) + print(f"✅ Reused session: {username}") + else: + client.login(username, password) + with open(session_path, "w") as f: + json.dump(client.get_settings(), f) + print(f"✅ Fresh login: {username}") + return client + + except Exception as e: + print(f"❌ Login failed for {username}: {e}") + if os.path.exists(session_path): + os.remove(session_path) + print(f"🧹 Removed invalid session for {username}") + + raise Exception("❌ All Instagram accounts failed login.") \ No newline at end of file diff --git a/utils/s3_uploader.py b/utils/s3_uploader.py new file mode 100644 index 0000000..d23495c --- /dev/null +++ b/utils/s3_uploader.py @@ -0,0 +1,50 @@ +import os +import boto3 +import uuid +import mimetypes +from urllib.parse import urljoin +import requests + +AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID") +AWS_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY") +AWS_STORAGE_BUCKET_NAME = os.getenv("AWS_STORAGE_BUCKET_NAME") +S3_ENDPOINT_URL = os.getenv("S3_ENDPOINT_URL") +S3_REGION = os.getenv("S3_REGION", "us-east-1") +MEDIA_BASE_PATH = os.getenv("MEDIA_BASE_PATH", "instagram") + +session = boto3.session.Session() +s3 = session.client( + service_name="s3", + aws_access_key_id=AWS_ACCESS_KEY_ID, + aws_secret_access_key=AWS_SECRET_ACCESS_KEY, + endpoint_url=S3_ENDPOINT_URL, + region_name=S3_REGION, +) + + +def upload_media_from_url(media_url: str, seller_id: str, post_id: str) -> str | None: + try: + response = requests.get(media_url, stream=True, timeout=10) + response.raise_for_status() + + content_type = response.headers.get("Content-Type", "") + ext = mimetypes.guess_extension(content_type) or ".jpg" + if ext in [".jpe", ".jfif", ".bin"]: + ext = ".jpg" + + filename = f"{uuid.uuid4()}{ext}" + s3_path = f"{MEDIA_BASE_PATH}/{seller_id}/{post_id}/{filename}" + + s3.upload_fileobj( + response.raw, + Bucket=AWS_STORAGE_BUCKET_NAME, + Key=s3_path, + ExtraArgs={"ACL": "public-read", "ContentType": content_type} + ) + + public_url = f"{S3_ENDPOINT_URL}/{AWS_STORAGE_BUCKET_NAME}/{s3_path}" + return public_url + + except Exception as e: + print(f"❌ Failed to upload media from {media_url}: {e}") + return None \ No newline at end of file