From 6b172b4fc0c2c5cbdde2ed9f7cf76aa98bb7e656 Mon Sep 17 00:00:00 2001 From: Hossein Date: Mon, 11 Aug 2025 23:35:16 +0330 Subject: [PATCH] migrate the data --- admin_sync.py | 17 +++++ already_scraped_posts.txt | 74 +++++++++++++++++++ main.py | 4 ++ sample_already_scraped.txt | 10 +++ sync_seen_from_table.py | 142 +++++++++++++++++++++++++++++++++++++ test_admin_sync.py | 137 +++++++++++++++++++++++++++++++++++ 6 files changed, 384 insertions(+) create mode 100644 admin_sync.py create mode 100644 already_scraped_posts.txt create mode 100644 sample_already_scraped.txt create mode 100644 sync_seen_from_table.py create mode 100644 test_admin_sync.py diff --git a/admin_sync.py b/admin_sync.py new file mode 100644 index 0000000..e000d6d --- /dev/null +++ b/admin_sync.py @@ -0,0 +1,17 @@ +# admin_sync.py +from fastapi import APIRouter +from pydantic import BaseModel +from typing import List +from utils.seen_store import mark_seen # uses your existing Redis "seen" store + +router = APIRouter(prefix="/admin", tags=["admin"]) + +class SyncSeenRequest(BaseModel): + username: str + ig_post_ids: List[str] # media PKs only (no "_" suffix) + +@router.post("/sync-seen") +def sync_seen(req: SyncSeenRequest): + # We namespace by username in seen_store, so seller_id not needed here + mark_seen(req.username, req.ig_post_ids) + return {"marked_seen": len(req.ig_post_ids)} \ No newline at end of file diff --git a/already_scraped_posts.txt b/already_scraped_posts.txt new file mode 100644 index 0000000..3240e04 --- /dev/null +++ b/already_scraped_posts.txt @@ -0,0 +1,74 @@ +# Already scraped posts data +# Format: +# The _ suffix will be automatically stripped + +75 bambii.land 3673126637020530684_43001846923 +74 bambii.land 3674975679815926931_43001846923 +73 bambii.land 3676008781820636104_43001846923 +72 bambii.land 3678171609666974359_43001846923 +71 bambii.land 3680351463476956634_43001846923 +70 bambii.land 3681071781787032379_43001846923 +69 bambii.land 3682478142664084665_43001846923 +68 bambii.land 3683263417589774721_43001846923 +67 bambii.land 3683970684668781069_43001846923 +66 bambii.land 3685407034634797242_43001846923 +65 eldo.shop 3674556514211785358_13504651965 +64 eldo.shop 3676011632963658864_13504651965 +63 eldo.shop 3676729147490599937_13504651965 +62 eldo.shop 3677427864173598853_13504651965 +61 eldo.shop 3680337705624013084_13504651965 +60 eldo.shop 3681069210083650155_13504651965 +59 eldo.shop 3682529221818779477_13504651965 +58 eldo.shop 3683259794270987192_13504651965 +57 eldo.shop 3683965638762725117_13504651965 +56 eldo.shop 3684666290820177200_13504651965 +55 frenzypop.studio 3651271249789146959_62244145726 +54 frenzypop.studio 3662742497565832943_62244145726 +52 frenzypop.studio 3667982533821070119_62244145726 +51 frenzypop.studio 3696016364309610273_74213546628 +50 frenzypop.studio 3670011113831102551_62244145726 +49 frenzypop.studio 3674168168494127979_62244145726 +48 frenzypop.studio 3675029090410229723_62244145726 +46 frenzypop.studio 3684472989742645172_62244145726 +45 bambii.land 3686181830335194766_43001846923 +44 bambii.land 3686849250855195216_43001846923 +43 bambii.land 3687586193280700420_43001846923 +42 bambii.land 3688335044836870452_43001846923 +41 bambii.land 3689765681917733851_43001846923 +40 bambii.land 3691209083960020293_43001846923 +39 bambii.land 3692649345223558497_43001846923 +38 bambii.land 3694109966839471024_43001846923 +37 bambii.land 3694872996896505908_43001846923 +36 bambii.land 3695553302871914441_43001846923 +35 eldo.shop 3687597028558266940_13504651965 +34 eldo.shop 3689744954071171348_13504651965 +33 eldo.shop 3691202569166700193_13504651965 +32 eldo.shop 3691947405202409628_13504651965 +31 eldo.shop 3692663324336579992_13504651965 +30 eldo.shop 3694107029903458044_13504651965 +29 eldo.shop 3694861176212032454_13504651965 +28 eldo.shop 3623833846270864278_13504651965 +27 eldo.shop 3667999364404482486_13504651965 +26 eldo.shop 3673813187535321144_13504651965 +25 Oramanwear 3653419297625760405_62991475962 +24 Oramanwear 3666473201552607643_62991475962 +23 Oramanwear 3667176171340810807_62991475962 +22 Oramanwear 3668000116533017188_62991475962 +21 Oramanwear 3669468838573381968_62991475962 +20 Oramanwear 3677412237746193161_62991475962 +19 Oramanwear 3678772906974486310_62991475962 +18 Oramanwear 3686828736454010810_62991475962 +17 Oramanwear 3688255097156509937_62991475962 +16 Oramanwear 3693233937951983122_62991475962 +15 Oramanwear 3665641771876682564_62991475962 +14 weaverroom_ 3634571165469923808_64452280860 +13 weaverroom_ 3636596539691118457_64452280860 +12 weaverroom_ 3643040332451837372_64452280860 +11 weaverroom_ 3644572934799339847_64452280860 +10 weaverroom_ 3662697027687679682_64452280860 +9 frenzypop.studio 3302015317167002143_62244145726 +8 frenzypop.studio 3567137991105266721_62244145726 +7 weaverroom_ 3672966427795968426_64452280860 +6 weaverroom_ 3680021684858304718_64452280860 +5 weaverroom_ 3681681058576104237_64452280860 +4 weaverroom_ 3693181708887583702_64452280860 \ No newline at end of file diff --git a/main.py b/main.py index 180b16e..aea29df 100644 --- a/main.py +++ b/main.py @@ -9,6 +9,7 @@ from dotenv import load_dotenv from database import get_db, create_tables, Item as ItemModel from schemas import ScrapeRequest, ScrapedPost from scraper import scrape_and_store +from admin_sync import router as admin_sync_router # Load environment variables load_dotenv() @@ -101,6 +102,9 @@ async def delete_item(item_id: int, db: Session = Depends(get_db)): db.commit() return {"message": "Item deleted successfully"} +# Include admin router +app.include_router(admin_sync_router) + # ============================================================================ # Instagram Scraper Endpoint # ============================================================================ diff --git a/sample_already_scraped.txt b/sample_already_scraped.txt new file mode 100644 index 0000000..4e062a1 --- /dev/null +++ b/sample_already_scraped.txt @@ -0,0 +1,10 @@ +# Sample data file showing the expected format +# Each line should have: ... +# The ig_post_id may have a suffix like _43001846923, which will be stripped + +1 weaverroom_ 3686849250855195216_43001846923 +2 weaverroom_ 3686849250855195217_43001846923 +3 weaverroom_ 3686849250855195218_43001846923 +4 test_account 1234567890123456789_123456789 +5 test_account 1234567890123456790_123456789 +6 another_user 9876543210987654321_987654321 \ No newline at end of file diff --git a/sync_seen_from_table.py b/sync_seen_from_table.py new file mode 100644 index 0000000..a287ab5 --- /dev/null +++ b/sync_seen_from_table.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +""" +Sync script to mark already scraped posts as seen in Redis +Parses table data and calls the admin endpoint to sync seen posts. +""" + +import re +import sys +import requests +from collections import defaultdict +from typing import List, Tuple + +# Configuration - CHANGE THIS TO YOUR FASTAPI HOST +API_BASE = "http://localhost:8000" # Change to your FastAPI host +API_ENDPOINT = f"{API_BASE}/admin/sync-seen" + +def parse_table_data(file_path: str) -> List[Tuple[str, str]]: + """ + Parse table data from file and extract account/ig_post_id pairs. + + Expected format: lines with ... + Ig post id may have suffix like _43001846923, we extract the base part. + """ + rows = [] + + try: + with open(file_path, encoding="utf-8") as f: + text = f.read() + except FileNotFoundError: + print(f"โŒ File not found: {file_path}") + return [] + except Exception as e: + print(f"โŒ Error reading file: {e}") + return [] + + for line_num, line in enumerate(text.splitlines(), 1): + # Skip empty lines + if not line.strip(): + continue + + # Match lines with: ... + # Pattern: whitespace + number + whitespace + account + whitespace + ig_post_id (with optional suffix) + m = re.search(r"\s*(\d+)\s+([A-Za-z0-9._]+)\s+(\d+)(?:_\d+)?\s", line) + if m: + _, account, ig_id = m.groups() + rows.append((account, ig_id)) + else: + print(f"โš ๏ธ Skipping line {line_num}: {line.strip()}") + + return rows + +def sync_seen_posts(account_data: dict) -> None: + """ + Call the admin endpoint to mark posts as seen for each account. + """ + total_synced = 0 + + for account, ids in account_data.items(): + if not ids: + continue + + payload = { + "username": account, + "ig_post_ids": sorted(list(ids)) # Convert set to sorted list + } + + try: + print(f"๐Ÿ“ค Syncing {len(ids)} posts for @{account}...") + response = requests.post(API_ENDPOINT, json=payload, timeout=30) + response.raise_for_status() + + result = response.json() + marked_count = result.get("marked_seen", 0) + print(f"โœ… @{account} -> marked_seen: {marked_count}") + total_synced += marked_count + + except requests.exceptions.ConnectionError: + print(f"โŒ Connection error for @{account}. Is the API running at {API_BASE}?") + except requests.exceptions.Timeout: + print(f"โŒ Timeout error for @{account}") + except requests.exceptions.RequestException as e: + print(f"โŒ Request error for @{account}: {e}") + except Exception as e: + print(f"โŒ Unexpected error for @{account}: {e}") + + print(f"\n๐ŸŽ‰ Total posts synced: {total_synced}") + +def main(): + """Main function to process the table file and sync seen posts.""" + + if len(sys.argv) != 2: + print("Usage: python sync_seen_from_table.py ") + print("Example: python sync_seen_from_table.py ./already_scraped.txt") + sys.exit(1) + + file_path = sys.argv[1] + + print("๐Ÿงช Instagram Seen Posts Sync Tool") + print("=" * 50) + print(f"๐Ÿ“ Processing file: {file_path}") + print(f"๐ŸŒ API endpoint: {API_ENDPOINT}") + print() + + # Parse the table data + print("๐Ÿ“ Parsing table data...") + rows = parse_table_data(file_path) + + if not rows: + print("โŒ No valid data found in file") + sys.exit(1) + + print(f"โœ… Found {len(rows)} rows") + + # Group by account and deduplicate + by_account = defaultdict(set) + for account, ig_id in rows: + by_account[account].add(ig_id) # dedupe per account + + print(f"๐Ÿ“Š Grouped into {len(by_account)} accounts:") + for account, ids in by_account.items(): + print(f" - @{account}: {len(ids)} posts") + + print() + + # Confirm before proceeding + total_posts = sum(len(ids) for ids in by_account.values()) + response = input(f"Proceed to sync {total_posts} posts across {len(by_account)} accounts? (y/N): ") + + if response.lower() not in ['y', 'yes']: + print("โŒ Sync cancelled") + sys.exit(0) + + print() + + # Sync the seen posts + sync_seen_posts(by_account) + + print() + print("๐ŸŽ‰ Sync completed!") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/test_admin_sync.py b/test_admin_sync.py new file mode 100644 index 0000000..48cebd4 --- /dev/null +++ b/test_admin_sync.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +""" +Test script to verify the admin sync endpoint works correctly +This script tests the /admin/sync-seen endpoint. +""" + +import requests +import json +from typing import List + +# Configuration - CHANGE THIS TO YOUR FASTAPI HOST +API_BASE = "http://localhost:8000" # Change to your FastAPI host +API_ENDPOINT = f"{API_BASE}/admin/sync-seen" + +def test_admin_sync(): + """Test the admin sync endpoint""" + + print("๐Ÿงช Testing Admin Sync Endpoint") + print("=" * 50) + print(f"๐ŸŒ API endpoint: {API_ENDPOINT}") + print() + + # Test data + test_cases = [ + { + "name": "Single account, multiple posts", + "username": "test_user_1", + "ig_post_ids": ["1234567890123456789", "1234567890123456790", "1234567890123456791"] + }, + { + "name": "Single account, single post", + "username": "test_user_2", + "ig_post_ids": ["9876543210987654321"] + }, + { + "name": "Empty post list", + "username": "test_user_3", + "ig_post_ids": [] + } + ] + + for i, case in enumerate(test_cases, 1): + print(f"๐Ÿ“ Test {i}: {case['name']}") + print(f" - Username: @{case['username']}") + print(f" - Post IDs: {len(case['ig_post_ids'])} posts") + + try: + payload = { + "username": case["username"], + "ig_post_ids": case["ig_post_ids"] + } + + print(f" ๐Ÿ“ค Sending request...") + response = requests.post(API_ENDPOINT, json=payload, timeout=30) + + if response.status_code == 200: + result = response.json() + marked_count = result.get("marked_seen", 0) + print(f" โœ… Success! marked_seen: {marked_count}") + + if marked_count == len(case["ig_post_ids"]): + print(f" โœ… All posts marked correctly") + else: + print(f" โš ๏ธ Expected {len(case['ig_post_ids'])}, got {marked_count}") + else: + print(f" โŒ HTTP {response.status_code}: {response.text}") + + except requests.exceptions.ConnectionError: + print(f" โŒ Connection error. Is the API running at {API_BASE}?") + except requests.exceptions.Timeout: + print(f" โŒ Timeout error") + except requests.exceptions.RequestException as e: + print(f" โŒ Request error: {e}") + except Exception as e: + print(f" โŒ Unexpected error: {e}") + + print() + + print("๐ŸŽ‰ Admin sync test completed!") + return True + +def test_health_check(): + """Test if the API is running""" + + print("๐Ÿฅ Testing API Health Check") + print("=" * 30) + + try: + health_url = f"{API_BASE}/health" + response = requests.get(health_url, timeout=10) + + if response.status_code == 200: + print(f"โœ… API is running at {API_BASE}") + return True + else: + print(f"โŒ API health check failed: {response.status_code}") + return False + + except requests.exceptions.ConnectionError: + print(f"โŒ Cannot connect to API at {API_BASE}") + print(" Make sure the FastAPI server is running:") + print(" docker-compose up") + return False + except Exception as e: + print(f"โŒ Health check error: {e}") + return False + +if __name__ == "__main__": + print("๐Ÿงช Admin Sync Endpoint Test Suite") + print("=" * 60) + + # First check if API is running + if not test_health_check(): + print("\nโŒ API is not running. Please start the FastAPI server first.") + print(" docker-compose up") + exit(1) + + print() + + # Test the admin sync endpoint + success = test_admin_sync() + + print("=" * 60) + if success: + print("โœ… Admin sync test passed!") + print("\n๐Ÿ“ Summary:") + print("- Admin endpoint is working correctly") + print("- Posts are being marked as seen in Redis") + print("- Endpoint handles various input sizes") + print("- Ready to use with sync_seen_from_table.py") + else: + print("โŒ Admin sync test failed!") + print("\nTroubleshooting:") + print("1. Check that the API is running") + print("2. Verify Redis connection") + print("3. Check admin_sync.py is properly imported") + print("4. Ensure the endpoint is accessible") \ No newline at end of file