fazli/FastApi-ISS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update the scrapper

Browse Source
This commit is contained in:
Hossein 2025-08-11 23:12:14 +03:30
parent ce86d474a3
commit 9349633feb
1 changed files with 16 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
scraper.py
View File

@ -69,13 +69,24 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]:
if db.query(InstagramPost).filter_by(ig_post_id=post_id).first(): if db.query(InstagramPost).filter_by(ig_post_id=post_id).first():
continue continue
# Filter media URLs to avoid duplicates (especially for videos)
media_urls = list(item.get("remote_urls", []))
if item["media_type"] == "video":
thumb = (item.get("thumbnail_url") or "").strip()
def is_video(u: str) -> bool:
base = u.split("?", 1)[0].lower()
return base.endswith((".mp4", ".mov", ".m4v", ".webm"))
# keep only real video URLs; drop image covers and anything equal to the thumbnail
media_urls = [u for u in media_urls if is_video(u) and u != thumb]
# Upload remote URLs to S3 (optional - can be done async later) # Upload remote URLs to S3 (optional - can be done async later)
media_urls: List[str] = [] uploaded_urls: List[str] = []
for remote_url in item["remote_urls"]: for remote_url in media_urls:
try: try:
local_url = upload_media_from_url(remote_url, str(req.seller_id), f"{post_id}/") local_url = upload_media_from_url(remote_url, str(req.seller_id), f"{post_id}/")
if local_url: if local_url:
media_urls.append(local_url) uploaded_urls.append(local_url)
except Exception as e: except Exception as e:
print(f"⚠️ Error uploading media: {e}") print(f"⚠️ Error uploading media: {e}")
continue continue
@ -97,7 +108,7 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]:
media_type=item["media_type"], media_type=item["media_type"],
caption=item["caption"], caption=item["caption"],
thumbnail_url=final_thumbnail, thumbnail_url=final_thumbnail,
local_media=media_urls, local_media=uploaded_urls, # Use filtered S3 URLs
created_at=datetime.fromtimestamp(item["taken_at"]), created_at=datetime.fromtimestamp(item["taken_at"]),
) )
db.add(post_entry) db.add(post_entry)
@ -108,7 +119,7 @@ def scrape_and_store(db: Session, req: ScrapeRequest) -> List[ScrapedPost]:
media_type=item["media_type"], media_type=item["media_type"],
caption=item["caption"], caption=item["caption"],
thumbnail_url=final_thumbnail, thumbnail_url=final_thumbnail,
local_media=media_urls, # Return S3 URLs only local_media=uploaded_urls, # Use filtered S3 URLs
created_at=datetime.fromtimestamp(item["taken_at"]) created_at=datetime.fromtimestamp(item["taken_at"])
)) ))