From 5504b1160d72ec9ec0ddef48d41519761b32c585 Mon Sep 17 00:00:00 2001 From: Hossein Date: Mon, 11 Aug 2025 21:54:56 +0330 Subject: [PATCH] update it --- README.md | 14 ++++--- main.py | 2 + scraper.py | 51 ++++++++++++------------ utils/boxapi_client.py | 82 ++++++++++++++++++++++++++++----------- utils/boxapi_log.py | 33 ++++++++++++++++ utils/boxapi_parser.py | 88 ++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 218 insertions(+), 52 deletions(-) create mode 100644 utils/boxapi_log.py create mode 100644 utils/boxapi_parser.py diff --git a/README.md b/README.md index 85dab5a..36dc3ac 100644 --- a/README.md +++ b/README.md @@ -40,9 +40,9 @@ A FastAPI application with Docker and docker-compose support. 2. **Configure BoxAPI (for Instagram scraping):** ```bash # Add your BoxAPI credentials to .env - BOXAPI_USERNAME=your_boxapi_username - BOXAPI_PASSWORD=your_boxapi_password - BOXAPI_ENDPOINT=https://boxapi.ir/api/instagram/user/get_media + BOXAPI_KEY=your_boxapi_bearer_token + BOXAPI_BASE=https://boxapi.ir + BOXAPI_LOG_DIR=/app/logs/boxapi ``` 2. **Build and start the application:** @@ -162,6 +162,8 @@ FastApi-ISS/ ├── utils/ # Utility modules │ ├── __init__.py │ ├── boxapi_client.py # BoxAPI client for Instagram scraping +│ ├── boxapi_parser.py # BoxAPI response parser +│ ├── boxapi_log.py # BoxAPI response logging │ ├── instagram_client.py # Legacy Instagram client │ └── s3_uploader.py # S3 media upload functionality ├── .dockerignore # Docker ignore file @@ -174,9 +176,9 @@ FastApi-ISS/ The application uses environment variables for configuration. Copy `env.example` to `.env` and customize the values: ### BoxAPI Configuration (Instagram Scraping) -- `BOXAPI_USERNAME`: Your BoxAPI username -- `BOXAPI_PASSWORD`: Your BoxAPI password -- `BOXAPI_ENDPOINT`: BoxAPI endpoint URL (default: `https://boxapi.ir/api/instagram/user/get_media`) +- `BOXAPI_KEY`: Your BoxAPI authentication key (Bearer token) +- `BOXAPI_BASE`: BoxAPI base URL (default: `https://boxapi.ir`) +- `BOXAPI_LOG_DIR`: Directory for BoxAPI response logs (default: `/app/logs/boxapi`) ### Database Configuration - `POSTGRES_DB`: Database name (default: `fastapi_db`) diff --git a/main.py b/main.py index e679b13..180b16e 100644 --- a/main.py +++ b/main.py @@ -118,6 +118,8 @@ async def scrape_posts( try: return scrape_and_store(db, request) except Exception as e: + # Log the error for debugging + print(f"❌ Scraping error for {request.username}: {e}") raise HTTPException(status_code=500, detail=str(e)) if __name__ == "__main__": diff --git a/scraper.py b/scraper.py index eef01e6..3d19114 100644 --- a/scraper.py +++ b/scraper.py @@ -1,6 +1,7 @@ from sqlalchemy.orm import Session from database import InstagramAccount, InstagramPost -from utils.boxapi_client import fetch_boxapi_posts +from utils.boxapi_client import get_media, BoxAPIError +from utils.boxapi_parser import parse_boxapi_items from utils.s3_uploader import upload_media_from_url from schemas import ScrapeRequest, ScrapedPost from typing import List @@ -8,10 +9,18 @@ from datetime import datetime def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]: - posts = fetch_boxapi_posts(req.username, req.max_count) - if not posts: + try: + items, log_path = get_media(req.username, req.max_count) + except BoxAPIError as e: + print(f"❌ BoxAPI error: {e}") return [] + if not items: + return [] + + # Parse and normalize the items + normalized_items = parse_boxapi_items(items) + # Get or create InstagramAccount account = db.query(InstagramAccount).filter_by(username=req.username).first() if not account: @@ -28,23 +37,17 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]: result: List[ScrapedPost] = [] new_post_count = 0 - for post in posts: - post_id = str(post.get("id") or post.get("pk")) - caption = post.get("caption", "") - thumbnail_url = post.get("thumbnail_url") - taken_at = post.get("taken_at") or datetime.utcnow() - media_type = post.get("media_type") or "image" - + for item in normalized_items: + post_id = item["ig_post_id"] + if db.query(InstagramPost).filter_by(ig_post_id=post_id).first(): continue + # Upload remote URLs to S3 media_urls: List[str] = [] - for media in post.get("media", []): - media_url = media.get("media_url") - if not media_url: - continue + for remote_url in item["remote_urls"]: try: - local_url = upload_media_from_url(media_url, str(req.seller_id), f"{post_id}/") + local_url = upload_media_from_url(remote_url, str(req.seller_id), f"{post_id}/") if local_url: media_urls.append(local_url) except Exception as e: @@ -55,10 +58,10 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]: continue # Upload thumbnail (optional) - final_thumbnail = thumbnail_url - if thumbnail_url: + final_thumbnail = item["thumbnail_url"] + if item["thumbnail_url"]: try: - thumb_uploaded = upload_media_from_url(thumbnail_url, str(req.seller_id), f"{post_id}/thumbnail") + thumb_uploaded = upload_media_from_url(item["thumbnail_url"], str(req.seller_id), f"{post_id}/thumbnail") if thumb_uploaded: final_thumbnail = thumb_uploaded except Exception as e: @@ -68,22 +71,22 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]: post_entry = InstagramPost( ig_post_id=post_id, account_id=account.id, - media_type=media_type, - caption=caption, + media_type=item["media_type"], + caption=item["caption"], thumbnail_url=final_thumbnail, local_media=media_urls, - created_at=taken_at, + created_at=item["taken_at"], ) db.add(post_entry) db.commit() result.append(ScrapedPost( ig_post_id=post_id, - media_type=media_type, - caption=caption, + media_type=item["media_type"], + caption=item["caption"], thumbnail_url=final_thumbnail, local_media=media_urls, - created_at=taken_at + created_at=item["taken_at"] )) new_post_count += 1 diff --git a/utils/boxapi_client.py b/utils/boxapi_client.py index ee32644..4bc331a 100644 --- a/utils/boxapi_client.py +++ b/utils/boxapi_client.py @@ -1,29 +1,67 @@ -import os -import requests -from typing import List, Dict +import os, requests +from typing import Any, Dict, List, Tuple, Optional +from utils.boxapi_log import log_boxapi_response -BOXAPI_USERNAME = os.getenv("BOXAPI_USERNAME") -BOXAPI_PASSWORD = os.getenv("BOXAPI_PASSWORD") -BOXAPI_ENDPOINT = os.getenv("BOXAPI_ENDPOINT", "https://boxapi.ir/api/instagram/user/get_media_by_username") +BOXAPI_BASE = os.environ.get("BOXAPI_BASE", "https://boxapi.ir") +BOXAPI_KEY = os.environ.get("BOXAPI_KEY", "") # put your key in .env -def fetch_boxapi_posts(username: str, count: int = 5) -> List[Dict]: - payload = { - "username": username, # 👈 Do NOT replace with user ID - "count": count - } +class BoxAPIError(Exception): + pass + +def get_media(username: str, count: int = 5) -> Tuple[List[Dict[str, Any]], str]: + """ + Returns (items, log_path) + items is the array under response.body.items + """ + url = f"{BOXAPI_BASE}/api/instagram/user/get_media" + payload = {"username": username, "count": count} + headers = {"Authorization": f"Bearer {BOXAPI_KEY}"} if BOXAPI_KEY else {} + + resp = None + resp_json: Optional[Dict[str, Any]] = None + status_code: Optional[int] = None try: - response = requests.post( - BOXAPI_ENDPOINT, - json=payload, - auth=(BOXAPI_USERNAME, BOXAPI_PASSWORD), - timeout=15 - ) - response.raise_for_status() - data = response.json() - if not isinstance(data, list): - raise ValueError("Unexpected response format from BoxAPI") - return data + resp = requests.post(url, json=payload, headers=headers, timeout=60) + status_code = resp.status_code + resp.raise_for_status() + resp_json = resp.json() except Exception as e: + log_path = log_boxapi_response(username, payload, resp_json, status_code, note=f"HTTP/parse error: {e}") + raise BoxAPIError(f"BoxAPI request failed ({e}). See log: {log_path}") + + # ✅ Real structure according to your sample + # { + # "status": "done", + # "response": { + # "status_code": 200, + # "content_type": "application/json", + # "body": { + # "items": [ ... ] + # } + # } + # } + try: + top_status = resp_json.get("status") + inner = resp_json.get("response") or {} + inner_code = inner.get("status_code") + body = inner.get("body") or {} + items = body.get("items") or [] + + log_path = log_boxapi_response(username, payload, resp_json, inner_code, note="OK") + + if top_status != "done" or inner_code != 200 or not isinstance(items, list): + raise ValueError(f"Unexpected shape: status={top_status}, inner_code={inner_code}, items_type={type(items)}") + return items, log_path + except Exception as e: + log_path = log_boxapi_response(username, payload, resp_json, status_code, note=f"schema error: {e}") + raise BoxAPIError(f"Unexpected response format from BoxAPI. See log: {log_path}") + +# Legacy function for backward compatibility +def fetch_boxapi_posts(username: str, count: int = 5) -> List[Dict]: + try: + items, _ = get_media(username, count) + return items + except BoxAPIError as e: print(f"❌ Failed to fetch posts from BoxAPI: {e}") return [] \ No newline at end of file diff --git a/utils/boxapi_log.py b/utils/boxapi_log.py new file mode 100644 index 0000000..b9300bb --- /dev/null +++ b/utils/boxapi_log.py @@ -0,0 +1,33 @@ +# utils/boxapi_log.py +import json, os, datetime +from typing import Any, Dict, Optional + +LOG_ROOT = os.environ.get("BOXAPI_LOG_DIR", "/app/logs/boxapi") + +def _ensure_dir(path: str) -> None: + os.makedirs(path, exist_ok=True) + +def log_boxapi_response( + username: str, + request_payload: Dict[str, Any], + response_json: Optional[Dict[str, Any]], + status_code: Optional[int], + note: str = "" +) -> str: + ts = datetime.datetime.utcnow().strftime("%Y%m%dT%H%M%SZ") + safe_user = (username or "unknown").replace("/", "_") + dirpath = os.path.join(LOG_ROOT, safe_user) + _ensure_dir(dirpath) + filepath = os.path.join(dirpath, f"{ts}.json") + + doc = { + "timestamp_utc": ts, + "username": username, + "status_code": status_code, + "note": note, + "request_payload": request_payload, + "response": response_json, + } + with open(filepath, "w", encoding="utf-8") as f: + json.dump(doc, f, ensure_ascii=False, indent=2) + return filepath \ No newline at end of file diff --git a/utils/boxapi_parser.py b/utils/boxapi_parser.py new file mode 100644 index 0000000..1edae0f --- /dev/null +++ b/utils/boxapi_parser.py @@ -0,0 +1,88 @@ +# utils/boxapi_parser.py +from typing import Any, Dict, List, Optional +from datetime import datetime + +def _ts_to_dt(ts: Optional[int]) -> datetime: + # BoxAPI/IG timestamps appear to be epoch seconds + if ts is None: + return datetime.utcnow() + return datetime.utcfromtimestamp(int(ts)) + +def pick_image_urls(item: Dict[str, Any]) -> List[str]: + urls: List[str] = [] + image_versions2 = item.get("image_versions2") or {} + for c in image_versions2.get("candidates", []): + u = c.get("url") + if u: + urls.append(u) + # Fallback to additional_candidates.first_frame if exists + add = image_versions2.get("additional_candidates") or {} + first_frame = add.get("first_frame") or {} + if first_frame.get("url"): + urls.append(first_frame["url"]) + return list(dict.fromkeys(urls)) # dedupe + +def pick_video_urls(item: Dict[str, Any]) -> List[str]: + urls: List[str] = [] + for v in item.get("video_versions", []) or []: + u = v.get("url") + if u: + urls.append(u) + # Fallback: progressive_download_url if present + prog = (item.get("clips_metadata") or {}).get("original_sound_info") or {} + prog_url = prog.get("progressive_download_url") + if prog_url: + urls.append(prog_url) + return list(dict.fromkeys(urls)) + +def parse_boxapi_items(items: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """ + Normalized items for storage: + { + "ig_post_id": str, + "media_type": "image"|"video"|"carousel", + "caption": str, + "taken_at": datetime, + "thumbnail_url": Optional[str], + "remote_urls": List[str], # to be uploaded to S3 + "code": Optional[str] + } + """ + out: List[Dict[str, Any]] = [] + for it in items: + mt_raw = it.get("media_type") + if mt_raw == 1: + mtype = "image" + remote_urls = pick_image_urls(it) + elif mt_raw == 2: + mtype = "video" + remote_urls = pick_video_urls(it) + # Add a cover frame if available (first image candidate) + cover = pick_image_urls(it)[:1] + remote_urls = cover + remote_urls + elif mt_raw == 8: # carousel, if BoxAPI returns it + mtype = "carousel" + remote_urls = [] + for child in it.get("carousel_media", []) or []: + if child.get("media_type") == 1: + remote_urls += pick_image_urls(child) + elif child.get("media_type") == 2: + # put a cover first if exists + cover = pick_image_urls(child)[:1] + remote_urls += cover + pick_video_urls(child) + else: + # Unknown, skip + continue + + cap = (it.get("caption") or {}).get("text") or "" + thumb = (it.get("image_versions2") or {}).get("candidates", [{}])[0].get("url") + out.append({ + "ig_post_id": str(it.get("pk") or it.get("id") or ""), + "media_type": mtype, + "caption": cap, + "taken_at": _ts_to_dt(it.get("taken_at")), + "thumbnail_url": thumb, + "remote_urls": remote_urls, + "code": it.get("code"), + }) + return out \ No newline at end of file