migrate the data

admin_sync.py

@@ -0,0 +1,17 @@
+# admin_sync.py
+from fastapi import APIRouter
+from pydantic import BaseModel
+from typing import List
+from utils.seen_store import mark_seen  # uses your existing Redis "seen" store
+
+router = APIRouter(prefix="/admin", tags=["admin"])
+
+class SyncSeenRequest(BaseModel):
+    username: str
+    ig_post_ids: List[str]  # media PKs only (no "_<user_id>" suffix)
+
+@router.post("/sync-seen")
+def sync_seen(req: SyncSeenRequest):
+    # We namespace by username in seen_store, so seller_id not needed here
+    mark_seen(req.username, req.ig_post_ids)
+    return {"marked_seen": len(req.ig_post_ids)} 

already_scraped_posts.txt

@@ -0,0 +1,74 @@
+# Already scraped posts data
+# Format: <id> <account> <ig_post_id>
+# The _<user_id> suffix will be automatically stripped
+
+75	bambii.land	3673126637020530684_43001846923
+74	bambii.land	3674975679815926931_43001846923
+73	bambii.land	3676008781820636104_43001846923
+72	bambii.land	3678171609666974359_43001846923
+71	bambii.land	3680351463476956634_43001846923
+70	bambii.land	3681071781787032379_43001846923
+69	bambii.land	3682478142664084665_43001846923
+68	bambii.land	3683263417589774721_43001846923
+67	bambii.land	3683970684668781069_43001846923
+66	bambii.land	3685407034634797242_43001846923
+65	eldo.shop	3674556514211785358_13504651965
+64	eldo.shop	3676011632963658864_13504651965
+63	eldo.shop	3676729147490599937_13504651965
+62	eldo.shop	3677427864173598853_13504651965
+61	eldo.shop	3680337705624013084_13504651965
+60	eldo.shop	3681069210083650155_13504651965
+59	eldo.shop	3682529221818779477_13504651965
+58	eldo.shop	3683259794270987192_13504651965
+57	eldo.shop	3683965638762725117_13504651965
+56	eldo.shop	3684666290820177200_13504651965
+55	frenzypop.studio	3651271249789146959_62244145726
+54	frenzypop.studio	3662742497565832943_62244145726
+52	frenzypop.studio	3667982533821070119_62244145726
+51	frenzypop.studio	3696016364309610273_74213546628
+50	frenzypop.studio	3670011113831102551_62244145726
+49	frenzypop.studio	3674168168494127979_62244145726
+48	frenzypop.studio	3675029090410229723_62244145726
+46	frenzypop.studio	3684472989742645172_62244145726
+45	bambii.land	3686181830335194766_43001846923
+44	bambii.land	3686849250855195216_43001846923
+43	bambii.land	3687586193280700420_43001846923
+42	bambii.land	3688335044836870452_43001846923
+41	bambii.land	3689765681917733851_43001846923
+40	bambii.land	3691209083960020293_43001846923
+39	bambii.land	3692649345223558497_43001846923
+38	bambii.land	3694109966839471024_43001846923
+37	bambii.land	3694872996896505908_43001846923
+36	bambii.land	3695553302871914441_43001846923
+35	eldo.shop	3687597028558266940_13504651965
+34	eldo.shop	3689744954071171348_13504651965
+33	eldo.shop	3691202569166700193_13504651965
+32	eldo.shop	3691947405202409628_13504651965
+31	eldo.shop	3692663324336579992_13504651965
+30	eldo.shop	3694107029903458044_13504651965
+29	eldo.shop	3694861176212032454_13504651965
+28	eldo.shop	3623833846270864278_13504651965
+27	eldo.shop	3667999364404482486_13504651965
+26	eldo.shop	3673813187535321144_13504651965
+25	Oramanwear	3653419297625760405_62991475962
+24	Oramanwear	3666473201552607643_62991475962
+23	Oramanwear	3667176171340810807_62991475962
+22	Oramanwear	3668000116533017188_62991475962
+21	Oramanwear	3669468838573381968_62991475962
+20	Oramanwear	3677412237746193161_62991475962
+19	Oramanwear	3678772906974486310_62991475962
+18	Oramanwear	3686828736454010810_62991475962
+17	Oramanwear	3688255097156509937_62991475962
+16	Oramanwear	3693233937951983122_62991475962
+15	Oramanwear	3665641771876682564_62991475962
+14	weaverroom_	3634571165469923808_64452280860
+13	weaverroom_	3636596539691118457_64452280860
+12	weaverroom_	3643040332451837372_64452280860
+11	weaverroom_	3644572934799339847_64452280860
+10	weaverroom_	3662697027687679682_64452280860
+9	frenzypop.studio	3302015317167002143_62244145726
+8	frenzypop.studio	3567137991105266721_62244145726
+7	weaverroom_	3672966427795968426_64452280860
+6	weaverroom_	3680021684858304718_64452280860
+5	weaverroom_	3681681058576104237_64452280860
+4	weaverroom_	3693181708887583702_64452280860 

main.py

@@ -9,6 +9,7 @@ from dotenv import load_dotenv
 from database import get_db, create_tables, Item as ItemModel
 from schemas import ScrapeRequest, ScrapedPost
 from scraper import scrape_and_store
+from admin_sync import router as admin_sync_router
 
 # Load environment variables
 load_dotenv()
@@ -101,6 +102,9 @@ async def delete_item(item_id: int, db: Session = Depends(get_db)):
     db.commit()
     return {"message": "Item deleted successfully"}
 
+# Include admin router
+app.include_router(admin_sync_router)
+
 # ============================================================================
 # Instagram Scraper Endpoint
 # ============================================================================

sample_already_scraped.txt

@@ -0,0 +1,10 @@
+# Sample data file showing the expected format
+# Each line should have: <id> <account> <ig_post_id> ...
+# The ig_post_id may have a suffix like _43001846923, which will be stripped
+
+1 weaverroom_ 3686849250855195216_43001846923
+2 weaverroom_ 3686849250855195217_43001846923
+3 weaverroom_ 3686849250855195218_43001846923
+4 test_account 1234567890123456789_123456789
+5 test_account 1234567890123456790_123456789
+6 another_user 9876543210987654321_987654321 

sync_seen_from_table.py

@@ -0,0 +1,142 @@
+#!/usr/bin/env python3
+"""
+Sync script to mark already scraped posts as seen in Redis
+Parses table data and calls the admin endpoint to sync seen posts.
+"""
+
+import re
+import sys
+import requests
+from collections import defaultdict
+from typing import List, Tuple
+
+# Configuration - CHANGE THIS TO YOUR FASTAPI HOST
+API_BASE = "http://localhost:8000"  # Change to your FastAPI host
+API_ENDPOINT = f"{API_BASE}/admin/sync-seen"
+
+def parse_table_data(file_path: str) -> List[Tuple[str, str]]:
+    """
+    Parse table data from file and extract account/ig_post_id pairs.
+    
+    Expected format: lines with <id> <account> <ig_post_id> ...
+    Ig post id may have suffix like _43001846923, we extract the base part.
+    """
+    rows = []
+    
+    try:
+        with open(file_path, encoding="utf-8") as f:
+            text = f.read()
+    except FileNotFoundError:
+        print(f"❌ File not found: {file_path}")
+        return []
+    except Exception as e:
+        print(f"❌ Error reading file: {e}")
+        return []
+    
+    for line_num, line in enumerate(text.splitlines(), 1):
+        # Skip empty lines
+        if not line.strip():
+            continue
+            
+        # Match lines with: <id> <account> <ig_post_id> ...
+        # Pattern: whitespace + number + whitespace + account + whitespace + ig_post_id (with optional suffix)
+        m = re.search(r"\s*(\d+)\s+([A-Za-z0-9._]+)\s+(\d+)(?:_\d+)?\s", line)
+        if m:
+            _, account, ig_id = m.groups()
+            rows.append((account, ig_id))
+        else:
+            print(f"⚠️ Skipping line {line_num}: {line.strip()}")
+    
+    return rows
+
+def sync_seen_posts(account_data: dict) -> None:
+    """
+    Call the admin endpoint to mark posts as seen for each account.
+    """
+    total_synced = 0
+    
+    for account, ids in account_data.items():
+        if not ids:
+            continue
+            
+        payload = {
+            "username": account, 
+            "ig_post_ids": sorted(list(ids))  # Convert set to sorted list
+        }
+        
+        try:
+            print(f"📤 Syncing {len(ids)} posts for @{account}...")
+            response = requests.post(API_ENDPOINT, json=payload, timeout=30)
+            response.raise_for_status()
+            
+            result = response.json()
+            marked_count = result.get("marked_seen", 0)
+            print(f"✅ @{account} -> marked_seen: {marked_count}")
+            total_synced += marked_count
+            
+        except requests.exceptions.ConnectionError:
+            print(f"❌ Connection error for @{account}. Is the API running at {API_BASE}?")
+        except requests.exceptions.Timeout:
+            print(f"❌ Timeout error for @{account}")
+        except requests.exceptions.RequestException as e:
+            print(f"❌ Request error for @{account}: {e}")
+        except Exception as e:
+            print(f"❌ Unexpected error for @{account}: {e}")
+    
+    print(f"\n🎉 Total posts synced: {total_synced}")
+
+def main():
+    """Main function to process the table file and sync seen posts."""
+    
+    if len(sys.argv) != 2:
+        print("Usage: python sync_seen_from_table.py <table_file>")
+        print("Example: python sync_seen_from_table.py ./already_scraped.txt")
+        sys.exit(1)
+    
+    file_path = sys.argv[1]
+    
+    print("🧪 Instagram Seen Posts Sync Tool")
+    print("=" * 50)
+    print(f"📁 Processing file: {file_path}")
+    print(f"🌐 API endpoint: {API_ENDPOINT}")
+    print()
+    
+    # Parse the table data
+    print("📝 Parsing table data...")
+    rows = parse_table_data(file_path)
+    
+    if not rows:
+        print("❌ No valid data found in file")
+        sys.exit(1)
+    
+    print(f"✅ Found {len(rows)} rows")
+    
+    # Group by account and deduplicate
+    by_account = defaultdict(set)
+    for account, ig_id in rows:
+        by_account[account].add(ig_id)  # dedupe per account
+    
+    print(f"📊 Grouped into {len(by_account)} accounts:")
+    for account, ids in by_account.items():
+        print(f"   - @{account}: {len(ids)} posts")
+    
+    print()
+    
+    # Confirm before proceeding
+    total_posts = sum(len(ids) for ids in by_account.values())
+    response = input(f"Proceed to sync {total_posts} posts across {len(by_account)} accounts? (y/N): ")
+    
+    if response.lower() not in ['y', 'yes']:
+        print("❌ Sync cancelled")
+        sys.exit(0)
+    
+    print()
+    
+    # Sync the seen posts
+    sync_seen_posts(by_account)
+    
+    print()
+    print("🎉 Sync completed!")
+
+if __name__ == "__main__":
+    main() 

test_admin_sync.py

@@ -0,0 +1,137 @@
+#!/usr/bin/env python3
+"""
+Test script to verify the admin sync endpoint works correctly
+This script tests the /admin/sync-seen endpoint.
+"""
+
+import requests
+import json
+from typing import List
+
+# Configuration - CHANGE THIS TO YOUR FASTAPI HOST
+API_BASE = "http://localhost:8000"  # Change to your FastAPI host
+API_ENDPOINT = f"{API_BASE}/admin/sync-seen"
+
+def test_admin_sync():
+    """Test the admin sync endpoint"""
+    
+    print("🧪 Testing Admin Sync Endpoint")
+    print("=" * 50)
+    print(f"🌐 API endpoint: {API_ENDPOINT}")
+    print()
+    
+    # Test data
+    test_cases = [
+        {
+            "name": "Single account, multiple posts",
+            "username": "test_user_1",
+            "ig_post_ids": ["1234567890123456789", "1234567890123456790", "1234567890123456791"]
+        },
+        {
+            "name": "Single account, single post",
+            "username": "test_user_2", 
+            "ig_post_ids": ["9876543210987654321"]
+        },
+        {
+            "name": "Empty post list",
+            "username": "test_user_3",
+            "ig_post_ids": []
+        }
+    ]
+    
+    for i, case in enumerate(test_cases, 1):
+        print(f"📝 Test {i}: {case['name']}")
+        print(f"   - Username: @{case['username']}")
+        print(f"   - Post IDs: {len(case['ig_post_ids'])} posts")
+        
+        try:
+            payload = {
+                "username": case["username"],
+                "ig_post_ids": case["ig_post_ids"]
+            }
+            
+            print(f"   📤 Sending request...")
+            response = requests.post(API_ENDPOINT, json=payload, timeout=30)
+            
+            if response.status_code == 200:
+                result = response.json()
+                marked_count = result.get("marked_seen", 0)
+                print(f"   ✅ Success! marked_seen: {marked_count}")
+                
+                if marked_count == len(case["ig_post_ids"]):
+                    print(f"   ✅ All posts marked correctly")
+                else:
+                    print(f"   ⚠️ Expected {len(case['ig_post_ids'])}, got {marked_count}")
+            else:
+                print(f"   ❌ HTTP {response.status_code}: {response.text}")
+                
+        except requests.exceptions.ConnectionError:
+            print(f"   ❌ Connection error. Is the API running at {API_BASE}?")
+        except requests.exceptions.Timeout:
+            print(f"   ❌ Timeout error")
+        except requests.exceptions.RequestException as e:
+            print(f"   ❌ Request error: {e}")
+        except Exception as e:
+            print(f"   ❌ Unexpected error: {e}")
+        
+        print()
+    
+    print("🎉 Admin sync test completed!")
+    return True
+
+def test_health_check():
+    """Test if the API is running"""
+    
+    print("🏥 Testing API Health Check")
+    print("=" * 30)
+    
+    try:
+        health_url = f"{API_BASE}/health"
+        response = requests.get(health_url, timeout=10)
+        
+        if response.status_code == 200:
+            print(f"✅ API is running at {API_BASE}")
+            return True
+        else:
+            print(f"❌ API health check failed: {response.status_code}")
+            return False
+            
+    except requests.exceptions.ConnectionError:
+        print(f"❌ Cannot connect to API at {API_BASE}")
+        print("   Make sure the FastAPI server is running:")
+        print("   docker-compose up")
+        return False
+    except Exception as e:
+        print(f"❌ Health check error: {e}")
+        return False
+
+if __name__ == "__main__":
+    print("🧪 Admin Sync Endpoint Test Suite")
+    print("=" * 60)
+    
+    # First check if API is running
+    if not test_health_check():
+        print("\n❌ API is not running. Please start the FastAPI server first.")
+        print("   docker-compose up")
+        exit(1)
+    
+    print()
+    
+    # Test the admin sync endpoint
+    success = test_admin_sync()
+    
+    print("=" * 60)
+    if success:
+        print("✅ Admin sync test passed!")
+        print("\n📝 Summary:")
+        print("- Admin endpoint is working correctly")
+        print("- Posts are being marked as seen in Redis")
+        print("- Endpoint handles various input sizes")
+        print("- Ready to use with sync_seen_from_table.py")
+    else:
+        print("❌ Admin sync test failed!")
+        print("\nTroubleshooting:")
+        print("1. Check that the API is running")
+        print("2. Verify Redis connection")
+        print("3. Check admin_sync.py is properly imported")
+        print("4. Ensure the endpoint is accessible")