fazli/FastApi-ISS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

scrapper update

Browse Source
This commit is contained in:
Hossein 2025-08-07 02:41:10 +03:30
parent 8f60e9bcdf
commit 02f6ed5b9b
1 changed files with 8 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
scraper.py
View File

@ -78,25 +78,29 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]:
if not media_items: if not media_items:
continue continue
# ✅ Upload thumbnail separately and store local version
uploaded_thumb = upload_media_from_url(post.thumbnail_url, str(req.seller_id), f"{post_id}/thumbnail")
final_thumbnail = uploaded_thumb or post.thumbnail_url # fallback if upload fails
# Save to DB # Save to DB
post_entry = InstagramPost( post_entry = InstagramPost(
ig_post_id=post_id, ig_post_id=post_id,
account_id=account.id, account_id=account.id,
media_type="carousel" if post.media_type == 8 else ("video" if post.media_type == 2 else "image"), media_type="carousel" if post.media_type == 8 else ("video" if post.media_type == 2 else "image"),
caption=post.caption_text or "", caption=post.caption_text or "",
thumbnail_url=post.thumbnail_url, thumbnail_url=final_thumbnail,
local_media=[m.dict() for m in media_items], local_media=[m.dict() for m in media_items],
created_at=post.taken_at or datetime.utcnow(), created_at=post.taken_at or datetime.utcnow(),
) )
db.add(post_entry) db.add(post_entry)
db.commit() db.commit()
# Add to result for response # Add to response
result.append(ScrapedPost( result.append(ScrapedPost(
ig_post_id=post_id, ig_post_id=post_id,
media_type=post_entry.media_type, media_type=post_entry.media_type,
caption=post_entry.caption, caption=post_entry.caption,
thumbnail_url=post_entry.thumbnail_url, thumbnail_url=final_thumbnail,
local_media=media_items, local_media=media_items,
created_at=post_entry.created_at created_at=post_entry.created_at
)) ))
@ -105,4 +109,4 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]:
if new_post_count >= req.max_count: if new_post_count >= req.max_count:
break break
return result return result