From f3b5532abbfbae302a3609e2395b9211f2dbfed5 Mon Sep 17 00:00:00 2001 From: Hossein Date: Mon, 11 Aug 2025 21:42:00 +0330 Subject: [PATCH] change the system to boxapi client --- README.md | 33 ++++++++++++++- main.py | 4 ++ scraper.py | 91 +++++++++++++++--------------------------- utils/boxapi_client.py | 29 ++++++++++++++ 4 files changed, 98 insertions(+), 59 deletions(-) create mode 100644 utils/boxapi_client.py diff --git a/README.md b/README.md index bcd02d8..85dab5a 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,8 @@ A FastAPI application with Docker and docker-compose support. - Health check endpoints - CORS middleware enabled - Auto-generated API documentation +- Instagram post scraping via BoxAPI +- S3 media upload and storage ## API Endpoints @@ -22,6 +24,7 @@ A FastAPI application with Docker and docker-compose support. - `POST /items` - Create new item - `PUT /items/{item_id}` - Update item - `DELETE /items/{item_id}` - Delete item +- `POST /scrape/` - Scrape Instagram posts (requires API key) ## Quick Start with Docker Compose @@ -30,10 +33,18 @@ A FastAPI application with Docker and docker-compose support. # Copy the example environment file cp env.example .env - # Edit the .env file with your secure passwords + # Edit the .env file with your secure passwords and BoxAPI credentials # nano .env # or use your preferred editor ``` +2. **Configure BoxAPI (for Instagram scraping):** + ```bash + # Add your BoxAPI credentials to .env + BOXAPI_USERNAME=your_boxapi_username + BOXAPI_PASSWORD=your_boxapi_password + BOXAPI_ENDPOINT=https://boxapi.ir/api/instagram/user/get_media + ``` + 2. **Build and start the application:** ```bash docker-compose up --build @@ -111,6 +122,14 @@ curl -X PUT "http://localhost:8000/items/1" \ curl -X DELETE "http://localhost:8000/items/1" ``` +### Scrape Instagram posts +```bash +curl -X POST "http://localhost:8000/scrape/" \ + -H "Content-Type: application/json" \ + -H "X-API-Key: your_api_key_here" \ + -d '{"username": "instagram_username", "seller_id": "your-seller-uuid", "max_count": 5}' +``` + ### Database Connection You can connect to the PostgreSQL database directly: @@ -129,6 +148,8 @@ docker exec -it fastapi-iss-postgres-1 psql -U ${POSTGRES_USER} -d ${POSTGRES_DB FastApi-ISS/ ├── main.py # FastAPI application ├── database.py # Database configuration and models +├── scraper.py # Instagram scraping logic +├── schemas.py # Pydantic schemas ├── requirements.txt # Python dependencies ├── Dockerfile # Docker configuration ├── docker-compose.yml # Docker Compose configuration @@ -138,6 +159,11 @@ FastApi-ISS/ ├── alembic/ # Database migrations │ ├── env.py # Alembic environment │ └── script.py.mako # Migration template +├── utils/ # Utility modules +│ ├── __init__.py +│ ├── boxapi_client.py # BoxAPI client for Instagram scraping +│ ├── instagram_client.py # Legacy Instagram client +│ └── s3_uploader.py # S3 media upload functionality ├── .dockerignore # Docker ignore file ├── .gitignore # Git ignore file └── README.md # This file @@ -147,6 +173,11 @@ FastApi-ISS/ The application uses environment variables for configuration. Copy `env.example` to `.env` and customize the values: +### BoxAPI Configuration (Instagram Scraping) +- `BOXAPI_USERNAME`: Your BoxAPI username +- `BOXAPI_PASSWORD`: Your BoxAPI password +- `BOXAPI_ENDPOINT`: BoxAPI endpoint URL (default: `https://boxapi.ir/api/instagram/user/get_media`) + ### Database Configuration - `POSTGRES_DB`: Database name (default: `fastapi_db`) - `POSTGRES_USER`: Database user (default: `fastapi_user`) diff --git a/main.py b/main.py index a03b4b6..e679b13 100644 --- a/main.py +++ b/main.py @@ -5,10 +5,14 @@ from typing import List, Optional from sqlalchemy.orm import Session import uvicorn import os +from dotenv import load_dotenv from database import get_db, create_tables, Item as ItemModel from schemas import ScrapeRequest, ScrapedPost from scraper import scrape_and_store +# Load environment variables +load_dotenv() + # API Key for Django backend authentication API_KEY = os.getenv("ENGINE_API_KEY") diff --git a/scraper.py b/scraper.py index 3254f07..eef01e6 100644 --- a/scraper.py +++ b/scraper.py @@ -1,7 +1,6 @@ -from instagrapi.exceptions import LoginRequired from sqlalchemy.orm import Session from database import InstagramAccount, InstagramPost -from utils.instagram_client import get_logged_in_client +from utils.boxapi_client import fetch_boxapi_posts from utils.s3_uploader import upload_media_from_url from schemas import ScrapeRequest, ScrapedPost from typing import List @@ -9,13 +8,8 @@ from datetime import datetime def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]: - client = get_logged_in_client() - - try: - user_id = client.user_id_from_username(req.username) - posts = client.user_medias_v1(user_id, amount=30) - except Exception as e: - print(f"❌ Failed to fetch posts for @{req.username}: {e}") + posts = fetch_boxapi_posts(req.username, req.max_count) + if not posts: return [] # Get or create InstagramAccount @@ -35,56 +29,38 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]: new_post_count = 0 for post in posts: - post_id = str(post.id) + post_id = str(post.get("id") or post.get("pk")) + caption = post.get("caption", "") + thumbnail_url = post.get("thumbnail_url") + taken_at = post.get("taken_at") or datetime.utcnow() + media_type = post.get("media_type") or "image" if db.query(InstagramPost).filter_by(ig_post_id=post_id).first(): continue media_urls: List[str] = [] - - try: - # Carousel - if post.media_type == 8 and hasattr(post, "resources"): - for res in post.resources: - media_url = res.video_url if res.media_type == 2 else res.thumbnail_url - if not media_url: - continue - try: - local_url = upload_media_from_url(media_url, str(req.seller_id), f"{post_id}/") - if local_url: - media_urls.append(local_url) - except Exception as e: - print(f"⚠️ Error uploading media: {e}") - continue - else: - media_url = post.video_url if post.media_type == 2 else post.thumbnail_url - if not media_url: - continue - try: - local_url = upload_media_from_url(media_url, str(req.seller_id), f"{post_id}/") - if local_url: - media_urls.append(local_url) - except Exception as e: - print(f"⚠️ Error uploading media: {e}") - continue - - except LoginRequired: - print("⛔ Login expired or failed mid-request.") - break - except Exception as e: - print(f"⚠️ Unexpected error while handling media: {e}") - continue + for media in post.get("media", []): + media_url = media.get("media_url") + if not media_url: + continue + try: + local_url = upload_media_from_url(media_url, str(req.seller_id), f"{post_id}/") + if local_url: + media_urls.append(local_url) + except Exception as e: + print(f"⚠️ Error uploading media: {e}") + continue if not media_urls: continue - # ✅ Upload and override thumbnail - final_thumbnail = post.thumbnail_url or "" - if post.thumbnail_url: + # Upload thumbnail (optional) + final_thumbnail = thumbnail_url + if thumbnail_url: try: - uploaded_thumb = upload_media_from_url(post.thumbnail_url, str(req.seller_id), f"{post_id}/thumbnail") - if uploaded_thumb: - final_thumbnail = uploaded_thumb + thumb_uploaded = upload_media_from_url(thumbnail_url, str(req.seller_id), f"{post_id}/thumbnail") + if thumb_uploaded: + final_thumbnail = thumb_uploaded except Exception as e: print(f"⚠️ Failed to upload thumbnail: {e}") @@ -92,23 +68,22 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]: post_entry = InstagramPost( ig_post_id=post_id, account_id=account.id, - media_type="carousel" if post.media_type == 8 else ("video" if post.media_type == 2 else "image"), - caption=post.caption_text or "", + media_type=media_type, + caption=caption, thumbnail_url=final_thumbnail, - local_media=media_urls, # ✅ Now a list of strings (S3 URLs) - created_at=post.taken_at or datetime.utcnow(), + local_media=media_urls, + created_at=taken_at, ) db.add(post_entry) db.commit() - # Add to response result.append(ScrapedPost( ig_post_id=post_id, - media_type=post_entry.media_type, - caption=post_entry.caption, + media_type=media_type, + caption=caption, thumbnail_url=final_thumbnail, - local_media=media_urls, # ✅ Again, list of strings - created_at=post_entry.created_at + local_media=media_urls, + created_at=taken_at )) new_post_count += 1 diff --git a/utils/boxapi_client.py b/utils/boxapi_client.py new file mode 100644 index 0000000..7386f96 --- /dev/null +++ b/utils/boxapi_client.py @@ -0,0 +1,29 @@ +import os +import requests +from typing import List, Dict + +BOXAPI_USERNAME = os.getenv("BOXAPI_USERNAME") +BOXAPI_PASSWORD = os.getenv("BOXAPI_PASSWORD") +BOXAPI_ENDPOINT = os.getenv("BOXAPI_ENDPOINT", "https://boxapi.ir/api/instagram/user/get_media") + +def fetch_boxapi_posts(username: str, count: int = 5) -> List[Dict]: + payload = { + "username": username, # 👈 Do NOT replace with user ID + "count": count + } + + try: + response = requests.post( + BOXAPI_ENDPOINT, + json=payload, + auth=(BOXAPI_USERNAME, BOXAPI_PASSWORD), + timeout=15 + ) + response.raise_for_status() + data = response.json() + if not isinstance(data, list): + raise ValueError("Unexpected response format from BoxAPI") + return data + except Exception as e: + print(f"❌ Failed to fetch posts from BoxAPI: {e}") + return [] \ No newline at end of file