fazli/FastApi-ISS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

dodododdo

Browse Source
This commit is contained in:
Hossein 2025-08-07 03:10:33 +03:30
parent 65ccffd6e2
commit 6ce50b72ad
2 changed files with 9 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
schemas.py
View File

@ -10,15 +10,10 @@ class ScrapeRequest(BaseModel):
max_count: int = Field(default=5, ge=1, le=50) max_count: int = Field(default=5, ge=1, le=50)
class MediaItem(BaseModel):
media_type: str # "image" or "video"
url: str # ✅ only S3/MinIO URL (renamed from local_media)
class ScrapedPost(BaseModel): class ScrapedPost(BaseModel):
ig_post_id: str ig_post_id: str
media_type: str # "image", "video", or "carousel" media_type: str # "image", "video", or "carousel"
caption: Optional[str] caption: Optional[str]
thumbnail_url: Optional[str] thumbnail_url: Optional[str]
local_media: List[MediaItem] # ✅ now only contains {"media_type", "url"} local_media: List[str] # ✅ list of S3 URLs only
created_at: datetime created_at: datetime

22
scraper.py
View File

@ -40,10 +40,10 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]:
if db.query(InstagramPost).filter_by(ig_post_id=post_id).first(): if db.query(InstagramPost).filter_by(ig_post_id=post_id).first():
continue continue
media_items = [] media_urls: List[str] = []
try: try:
# Handle carousel # Carousel
if post.media_type == 8 and hasattr(post, "resources"): if post.media_type == 8 and hasattr(post, "resources"):
for res in post.resources: for res in post.resources:
media_url = res.video_url if res.media_type == 2 else res.thumbnail_url media_url = res.video_url if res.media_type == 2 else res.thumbnail_url
@ -52,10 +52,7 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]:
try: try:
local_url = upload_media_from_url(media_url, str(req.seller_id), f"{post_id}/") local_url = upload_media_from_url(media_url, str(req.seller_id), f"{post_id}/")
if local_url: if local_url:
media_items.append({ media_urls.append(local_url)
"media_type": "video" if res.media_type == 2 else "image",
"url": local_url
})
except Exception as e: except Exception as e:
print(f"⚠️ Error uploading media: {e}") print(f"⚠️ Error uploading media: {e}")
continue continue
@ -66,10 +63,7 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]:
try: try:
local_url = upload_media_from_url(media_url, str(req.seller_id), f"{post_id}/") local_url = upload_media_from_url(media_url, str(req.seller_id), f"{post_id}/")
if local_url: if local_url:
media_items.append({ media_urls.append(local_url)
"media_type": "video" if post.media_type == 2 else "image",
"url": local_url
})
except Exception as e: except Exception as e:
print(f"⚠️ Error uploading media: {e}") print(f"⚠️ Error uploading media: {e}")
continue continue
@ -81,10 +75,10 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]:
print(f"⚠️ Unexpected error while handling media: {e}") print(f"⚠️ Unexpected error while handling media: {e}")
continue continue
if not media_items: if not media_urls:
continue continue
# ✅ Upload thumbnail safely # ✅ Upload and override thumbnail
final_thumbnail = post.thumbnail_url or "" final_thumbnail = post.thumbnail_url or ""
if post.thumbnail_url: if post.thumbnail_url:
try: try:
@ -101,7 +95,7 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]:
media_type="carousel" if post.media_type == 8 else ("video" if post.media_type == 2 else "image"), media_type="carousel" if post.media_type == 8 else ("video" if post.media_type == 2 else "image"),
caption=post.caption_text or "", caption=post.caption_text or "",
thumbnail_url=final_thumbnail, thumbnail_url=final_thumbnail,
local_media=media_items, local_media=media_urls, # ✅ Now a list of strings (S3 URLs)
created_at=post.taken_at or datetime.utcnow(), created_at=post.taken_at or datetime.utcnow(),
) )
db.add(post_entry) db.add(post_entry)
@ -113,7 +107,7 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]:
media_type=post_entry.media_type, media_type=post_entry.media_type,
caption=post_entry.caption, caption=post_entry.caption,
thumbnail_url=final_thumbnail, thumbnail_url=final_thumbnail,
local_media=media_items, local_media=media_urls, # ✅ Again, list of strings
created_at=post_entry.created_at created_at=post_entry.created_at
)) ))