update it

README.md

@@ -40,9 +40,9 @@ A FastAPI application with Docker and docker-compose support.
 2. **Configure BoxAPI (for Instagram scraping):**
    ```bash
    # Add your BoxAPI credentials to .env
-   BOXAPI_USERNAME=your_boxapi_username
-   BOXAPI_PASSWORD=your_boxapi_password
-   BOXAPI_ENDPOINT=https://boxapi.ir/api/instagram/user/get_media
+   BOXAPI_KEY=your_boxapi_bearer_token
+   BOXAPI_BASE=https://boxapi.ir
+   BOXAPI_LOG_DIR=/app/logs/boxapi
    ```
 
 2. **Build and start the application:**
@@ -162,6 +162,8 @@ FastApi-ISS/
 ├── utils/              # Utility modules
 │   ├── __init__.py
 │   ├── boxapi_client.py # BoxAPI client for Instagram scraping
+│   ├── boxapi_parser.py # BoxAPI response parser
+│   ├── boxapi_log.py   # BoxAPI response logging
 │   ├── instagram_client.py # Legacy Instagram client
 │   └── s3_uploader.py  # S3 media upload functionality
 ├── .dockerignore       # Docker ignore file
@@ -174,9 +176,9 @@ FastApi-ISS/
 The application uses environment variables for configuration. Copy `env.example` to `.env` and customize the values:
 
 ### BoxAPI Configuration (Instagram Scraping)
-- `BOXAPI_USERNAME`: Your BoxAPI username
-- `BOXAPI_PASSWORD`: Your BoxAPI password  
-- `BOXAPI_ENDPOINT`: BoxAPI endpoint URL (default: `https://boxapi.ir/api/instagram/user/get_media`)
+- `BOXAPI_KEY`: Your BoxAPI authentication key (Bearer token)
+- `BOXAPI_BASE`: BoxAPI base URL (default: `https://boxapi.ir`)
+- `BOXAPI_LOG_DIR`: Directory for BoxAPI response logs (default: `/app/logs/boxapi`)
 
 ### Database Configuration
 - `POSTGRES_DB`: Database name (default: `fastapi_db`)

main.py

@@ -118,6 +118,8 @@ async def scrape_posts(
     try:
         return scrape_and_store(db, request)
     except Exception as e:
+        # Log the error for debugging
+        print(f"❌ Scraping error for {request.username}: {e}")
         raise HTTPException(status_code=500, detail=str(e))
 
 if __name__ == "__main__":

scraper.py

@@ -1,6 +1,7 @@
 from sqlalchemy.orm import Session
 from database import InstagramAccount, InstagramPost
-from utils.boxapi_client import fetch_boxapi_posts
+from utils.boxapi_client import get_media, BoxAPIError
+from utils.boxapi_parser import parse_boxapi_items
 from utils.s3_uploader import upload_media_from_url
 from schemas import ScrapeRequest, ScrapedPost
 from typing import List
@@ -8,10 +9,18 @@ from datetime import datetime
 
 
 def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]:
-    posts = fetch_boxapi_posts(req.username, req.max_count)
-    if not posts:
+    try:
+        items, log_path = get_media(req.username, req.max_count)
+    except BoxAPIError as e:
+        print(f"❌ BoxAPI error: {e}")
         return []
 
+    if not items:
+        return []
+
+    # Parse and normalize the items
+    normalized_items = parse_boxapi_items(items)
+
     # Get or create InstagramAccount
     account = db.query(InstagramAccount).filter_by(username=req.username).first()
     if not account:
@@ -28,23 +37,17 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]:
     result: List[ScrapedPost] = []
     new_post_count = 0
 
-    for post in posts:
-        post_id = str(post.get("id") or post.get("pk"))
-        caption = post.get("caption", "")
-        thumbnail_url = post.get("thumbnail_url")
-        taken_at = post.get("taken_at") or datetime.utcnow()
-        media_type = post.get("media_type") or "image"
-
+    for item in normalized_items:
+        post_id = item["ig_post_id"]
+        
         if db.query(InstagramPost).filter_by(ig_post_id=post_id).first():
             continue
 
+        # Upload remote URLs to S3
         media_urls: List[str] = []
-        for media in post.get("media", []):
-            media_url = media.get("media_url")
-            if not media_url:
-                continue
+        for remote_url in item["remote_urls"]:
             try:
-                local_url = upload_media_from_url(media_url, str(req.seller_id), f"{post_id}/")
+                local_url = upload_media_from_url(remote_url, str(req.seller_id), f"{post_id}/")
                 if local_url:
                     media_urls.append(local_url)
             except Exception as e:
@@ -55,10 +58,10 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]:
             continue
 
         # Upload thumbnail (optional)
-        final_thumbnail = thumbnail_url
-        if thumbnail_url:
+        final_thumbnail = item["thumbnail_url"]
+        if item["thumbnail_url"]:
             try:
-                thumb_uploaded = upload_media_from_url(thumbnail_url, str(req.seller_id), f"{post_id}/thumbnail")
+                thumb_uploaded = upload_media_from_url(item["thumbnail_url"], str(req.seller_id), f"{post_id}/thumbnail")
                 if thumb_uploaded:
                     final_thumbnail = thumb_uploaded
             except Exception as e:
@@ -68,22 +71,22 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]:
         post_entry = InstagramPost(
             ig_post_id=post_id,
             account_id=account.id,
-            media_type=media_type,
-            caption=caption,
+            media_type=item["media_type"],
+            caption=item["caption"],
             thumbnail_url=final_thumbnail,
             local_media=media_urls,
-            created_at=taken_at,
+            created_at=item["taken_at"],
         )
         db.add(post_entry)
         db.commit()
 
         result.append(ScrapedPost(
             ig_post_id=post_id,
-            media_type=media_type,
-            caption=caption,
+            media_type=item["media_type"],
+            caption=item["caption"],
             thumbnail_url=final_thumbnail,
             local_media=media_urls,
-            created_at=taken_at
+            created_at=item["taken_at"]
         ))
 
         new_post_count += 1

utils/boxapi_client.py

@@ -1,29 +1,67 @@
-import os
-import requests
-from typing import List, Dict
+import os, requests
+from typing import Any, Dict, List, Tuple, Optional
+from utils.boxapi_log import log_boxapi_response
 
-BOXAPI_USERNAME = os.getenv("BOXAPI_USERNAME")
-BOXAPI_PASSWORD = os.getenv("BOXAPI_PASSWORD")
-BOXAPI_ENDPOINT = os.getenv("BOXAPI_ENDPOINT", "https://boxapi.ir/api/instagram/user/get_media_by_username")
+BOXAPI_BASE = os.environ.get("BOXAPI_BASE", "https://boxapi.ir")
+BOXAPI_KEY  = os.environ.get("BOXAPI_KEY", "")  # put your key in .env
 
-def fetch_boxapi_posts(username: str, count: int = 5) -> List[Dict]:
-    payload = {
-        "username": username,  # 👈 Do NOT replace with user ID
-        "count": count
-    }
+class BoxAPIError(Exception):
+    pass
+
+def get_media(username: str, count: int = 5) -> Tuple[List[Dict[str, Any]], str]:
+    """
+    Returns (items, log_path)
+    items is the array under response.body.items
+    """
+    url = f"{BOXAPI_BASE}/api/instagram/user/get_media"
+    payload = {"username": username, "count": count}
+    headers = {"Authorization": f"Bearer {BOXAPI_KEY}"} if BOXAPI_KEY else {}
+
+    resp = None
+    resp_json: Optional[Dict[str, Any]] = None
+    status_code: Optional[int] = None
 
     try:
-        response = requests.post(
-            BOXAPI_ENDPOINT,
-            json=payload,
-            auth=(BOXAPI_USERNAME, BOXAPI_PASSWORD),
-            timeout=15
-        )
-        response.raise_for_status()
-        data = response.json()
-        if not isinstance(data, list):
-            raise ValueError("Unexpected response format from BoxAPI")
-        return data
+        resp = requests.post(url, json=payload, headers=headers, timeout=60)
+        status_code = resp.status_code
+        resp.raise_for_status()
+        resp_json = resp.json()
     except Exception as e:
+        log_path = log_boxapi_response(username, payload, resp_json, status_code, note=f"HTTP/parse error: {e}")
+        raise BoxAPIError(f"BoxAPI request failed ({e}). See log: {log_path}")
+
+    # ✅ Real structure according to your sample
+    # {
+    #   "status": "done",
+    #   "response": {
+    #     "status_code": 200,
+    #     "content_type": "application/json",
+    #     "body": {
+    #       "items": [ ... ]
+    #     }
+    #   }
+    # }
+    try:
+        top_status = resp_json.get("status")
+        inner = resp_json.get("response") or {}
+        inner_code = inner.get("status_code")
+        body = inner.get("body") or {}
+        items = body.get("items") or []
+
+        log_path = log_boxapi_response(username, payload, resp_json, inner_code, note="OK")
+
+        if top_status != "done" or inner_code != 200 or not isinstance(items, list):
+            raise ValueError(f"Unexpected shape: status={top_status}, inner_code={inner_code}, items_type={type(items)}")
+        return items, log_path
+    except Exception as e:
+        log_path = log_boxapi_response(username, payload, resp_json, status_code, note=f"schema error: {e}")
+        raise BoxAPIError(f"Unexpected response format from BoxAPI. See log: {log_path}")
+
+# Legacy function for backward compatibility
+def fetch_boxapi_posts(username: str, count: int = 5) -> List[Dict]:
+    try:
+        items, _ = get_media(username, count)
+        return items
+    except BoxAPIError as e:
         print(f"❌ Failed to fetch posts from BoxAPI: {e}")
         return [] 

utils/boxapi_log.py

@@ -0,0 +1,33 @@
+# utils/boxapi_log.py
+import json, os, datetime
+from typing import Any, Dict, Optional
+
+LOG_ROOT = os.environ.get("BOXAPI_LOG_DIR", "/app/logs/boxapi")
+
+def _ensure_dir(path: str) -> None:
+    os.makedirs(path, exist_ok=True)
+
+def log_boxapi_response(
+    username: str,
+    request_payload: Dict[str, Any],
+    response_json: Optional[Dict[str, Any]],
+    status_code: Optional[int],
+    note: str = ""
+) -> str:
+    ts = datetime.datetime.utcnow().strftime("%Y%m%dT%H%M%SZ")
+    safe_user = (username or "unknown").replace("/", "_")
+    dirpath = os.path.join(LOG_ROOT, safe_user)
+    _ensure_dir(dirpath)
+    filepath = os.path.join(dirpath, f"{ts}.json")
+
+    doc = {
+        "timestamp_utc": ts,
+        "username": username,
+        "status_code": status_code,
+        "note": note,
+        "request_payload": request_payload,
+        "response": response_json,
+    }
+    with open(filepath, "w", encoding="utf-8") as f:
+        json.dump(doc, f, ensure_ascii=False, indent=2)
+    return filepath 

utils/boxapi_parser.py

@@ -0,0 +1,88 @@
+# utils/boxapi_parser.py
+from typing import Any, Dict, List, Optional
+from datetime import datetime
+
+def _ts_to_dt(ts: Optional[int]) -> datetime:
+    # BoxAPI/IG timestamps appear to be epoch seconds
+    if ts is None:
+        return datetime.utcnow()
+    return datetime.utcfromtimestamp(int(ts))
+
+def pick_image_urls(item: Dict[str, Any]) -> List[str]:
+    urls: List[str] = []
+    image_versions2 = item.get("image_versions2") or {}
+    for c in image_versions2.get("candidates", []):
+        u = c.get("url")
+        if u:
+            urls.append(u)
+    # Fallback to additional_candidates.first_frame if exists
+    add = image_versions2.get("additional_candidates") or {}
+    first_frame = add.get("first_frame") or {}
+    if first_frame.get("url"):
+        urls.append(first_frame["url"])
+    return list(dict.fromkeys(urls))  # dedupe
+
+def pick_video_urls(item: Dict[str, Any]) -> List[str]:
+    urls: List[str] = []
+    for v in item.get("video_versions", []) or []:
+        u = v.get("url")
+        if u:
+            urls.append(u)
+    # Fallback: progressive_download_url if present
+    prog = (item.get("clips_metadata") or {}).get("original_sound_info") or {}
+    prog_url = prog.get("progressive_download_url")
+    if prog_url:
+        urls.append(prog_url)
+    return list(dict.fromkeys(urls))
+
+def parse_boxapi_items(items: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    """
+    Normalized items for storage:
+    {
+      "ig_post_id": str,
+      "media_type": "image"|"video"|"carousel",
+      "caption": str,
+      "taken_at": datetime,
+      "thumbnail_url": Optional[str],
+      "remote_urls": List[str],  # to be uploaded to S3
+      "code": Optional[str]
+    }
+    """
+    out: List[Dict[str, Any]] = []
+    for it in items:
+        mt_raw = it.get("media_type")
+        if mt_raw == 1:
+            mtype = "image"
+            remote_urls = pick_image_urls(it)
+        elif mt_raw == 2:
+            mtype = "video"
+            remote_urls = pick_video_urls(it)
+            # Add a cover frame if available (first image candidate)
+            cover = pick_image_urls(it)[:1]
+            remote_urls = cover + remote_urls
+        elif mt_raw == 8:  # carousel, if BoxAPI returns it
+            mtype = "carousel"
+            remote_urls = []
+            for child in it.get("carousel_media", []) or []:
+                if child.get("media_type") == 1:
+                    remote_urls += pick_image_urls(child)
+                elif child.get("media_type") == 2:
+                    # put a cover first if exists
+                    cover = pick_image_urls(child)[:1]
+                    remote_urls += cover + pick_video_urls(child)
+        else:
+            # Unknown, skip
+            continue
+
+        cap = (it.get("caption") or {}).get("text") or ""
+        thumb = (it.get("image_versions2") or {}).get("candidates", [{}])[0].get("url")
+        out.append({
+            "ig_post_id": str(it.get("pk") or it.get("id") or ""),
+            "media_type": mtype,
+            "caption": cap,
+            "taken_at": _ts_to_dt(it.get("taken_at")),
+            "thumbnail_url": thumb,
+            "remote_urls": remote_urls,
+            "code": it.get("code"),
+        })
+    return out