migrate the data
This commit is contained in:
parent
9349633feb
commit
6b172b4fc0
17
admin_sync.py
Normal file
17
admin_sync.py
Normal file
@ -0,0 +1,17 @@
|
||||
# admin_sync.py
|
||||
from fastapi import APIRouter
|
||||
from pydantic import BaseModel
|
||||
from typing import List
|
||||
from utils.seen_store import mark_seen # uses your existing Redis "seen" store
|
||||
|
||||
router = APIRouter(prefix="/admin", tags=["admin"])
|
||||
|
||||
class SyncSeenRequest(BaseModel):
|
||||
username: str
|
||||
ig_post_ids: List[str] # media PKs only (no "_<user_id>" suffix)
|
||||
|
||||
@router.post("/sync-seen")
|
||||
def sync_seen(req: SyncSeenRequest):
|
||||
# We namespace by username in seen_store, so seller_id not needed here
|
||||
mark_seen(req.username, req.ig_post_ids)
|
||||
return {"marked_seen": len(req.ig_post_ids)}
|
||||
74
already_scraped_posts.txt
Normal file
74
already_scraped_posts.txt
Normal file
@ -0,0 +1,74 @@
|
||||
# Already scraped posts data
|
||||
# Format: <id> <account> <ig_post_id>
|
||||
# The _<user_id> suffix will be automatically stripped
|
||||
|
||||
75 bambii.land 3673126637020530684_43001846923
|
||||
74 bambii.land 3674975679815926931_43001846923
|
||||
73 bambii.land 3676008781820636104_43001846923
|
||||
72 bambii.land 3678171609666974359_43001846923
|
||||
71 bambii.land 3680351463476956634_43001846923
|
||||
70 bambii.land 3681071781787032379_43001846923
|
||||
69 bambii.land 3682478142664084665_43001846923
|
||||
68 bambii.land 3683263417589774721_43001846923
|
||||
67 bambii.land 3683970684668781069_43001846923
|
||||
66 bambii.land 3685407034634797242_43001846923
|
||||
65 eldo.shop 3674556514211785358_13504651965
|
||||
64 eldo.shop 3676011632963658864_13504651965
|
||||
63 eldo.shop 3676729147490599937_13504651965
|
||||
62 eldo.shop 3677427864173598853_13504651965
|
||||
61 eldo.shop 3680337705624013084_13504651965
|
||||
60 eldo.shop 3681069210083650155_13504651965
|
||||
59 eldo.shop 3682529221818779477_13504651965
|
||||
58 eldo.shop 3683259794270987192_13504651965
|
||||
57 eldo.shop 3683965638762725117_13504651965
|
||||
56 eldo.shop 3684666290820177200_13504651965
|
||||
55 frenzypop.studio 3651271249789146959_62244145726
|
||||
54 frenzypop.studio 3662742497565832943_62244145726
|
||||
52 frenzypop.studio 3667982533821070119_62244145726
|
||||
51 frenzypop.studio 3696016364309610273_74213546628
|
||||
50 frenzypop.studio 3670011113831102551_62244145726
|
||||
49 frenzypop.studio 3674168168494127979_62244145726
|
||||
48 frenzypop.studio 3675029090410229723_62244145726
|
||||
46 frenzypop.studio 3684472989742645172_62244145726
|
||||
45 bambii.land 3686181830335194766_43001846923
|
||||
44 bambii.land 3686849250855195216_43001846923
|
||||
43 bambii.land 3687586193280700420_43001846923
|
||||
42 bambii.land 3688335044836870452_43001846923
|
||||
41 bambii.land 3689765681917733851_43001846923
|
||||
40 bambii.land 3691209083960020293_43001846923
|
||||
39 bambii.land 3692649345223558497_43001846923
|
||||
38 bambii.land 3694109966839471024_43001846923
|
||||
37 bambii.land 3694872996896505908_43001846923
|
||||
36 bambii.land 3695553302871914441_43001846923
|
||||
35 eldo.shop 3687597028558266940_13504651965
|
||||
34 eldo.shop 3689744954071171348_13504651965
|
||||
33 eldo.shop 3691202569166700193_13504651965
|
||||
32 eldo.shop 3691947405202409628_13504651965
|
||||
31 eldo.shop 3692663324336579992_13504651965
|
||||
30 eldo.shop 3694107029903458044_13504651965
|
||||
29 eldo.shop 3694861176212032454_13504651965
|
||||
28 eldo.shop 3623833846270864278_13504651965
|
||||
27 eldo.shop 3667999364404482486_13504651965
|
||||
26 eldo.shop 3673813187535321144_13504651965
|
||||
25 Oramanwear 3653419297625760405_62991475962
|
||||
24 Oramanwear 3666473201552607643_62991475962
|
||||
23 Oramanwear 3667176171340810807_62991475962
|
||||
22 Oramanwear 3668000116533017188_62991475962
|
||||
21 Oramanwear 3669468838573381968_62991475962
|
||||
20 Oramanwear 3677412237746193161_62991475962
|
||||
19 Oramanwear 3678772906974486310_62991475962
|
||||
18 Oramanwear 3686828736454010810_62991475962
|
||||
17 Oramanwear 3688255097156509937_62991475962
|
||||
16 Oramanwear 3693233937951983122_62991475962
|
||||
15 Oramanwear 3665641771876682564_62991475962
|
||||
14 weaverroom_ 3634571165469923808_64452280860
|
||||
13 weaverroom_ 3636596539691118457_64452280860
|
||||
12 weaverroom_ 3643040332451837372_64452280860
|
||||
11 weaverroom_ 3644572934799339847_64452280860
|
||||
10 weaverroom_ 3662697027687679682_64452280860
|
||||
9 frenzypop.studio 3302015317167002143_62244145726
|
||||
8 frenzypop.studio 3567137991105266721_62244145726
|
||||
7 weaverroom_ 3672966427795968426_64452280860
|
||||
6 weaverroom_ 3680021684858304718_64452280860
|
||||
5 weaverroom_ 3681681058576104237_64452280860
|
||||
4 weaverroom_ 3693181708887583702_64452280860
|
||||
4
main.py
4
main.py
@ -9,6 +9,7 @@ from dotenv import load_dotenv
|
||||
from database import get_db, create_tables, Item as ItemModel
|
||||
from schemas import ScrapeRequest, ScrapedPost
|
||||
from scraper import scrape_and_store
|
||||
from admin_sync import router as admin_sync_router
|
||||
|
||||
# Load environment variables
|
||||
load_dotenv()
|
||||
@ -101,6 +102,9 @@ async def delete_item(item_id: int, db: Session = Depends(get_db)):
|
||||
db.commit()
|
||||
return {"message": "Item deleted successfully"}
|
||||
|
||||
# Include admin router
|
||||
app.include_router(admin_sync_router)
|
||||
|
||||
# ============================================================================
|
||||
# Instagram Scraper Endpoint
|
||||
# ============================================================================
|
||||
|
||||
10
sample_already_scraped.txt
Normal file
10
sample_already_scraped.txt
Normal file
@ -0,0 +1,10 @@
|
||||
# Sample data file showing the expected format
|
||||
# Each line should have: <id> <account> <ig_post_id> ...
|
||||
# The ig_post_id may have a suffix like _43001846923, which will be stripped
|
||||
|
||||
1 weaverroom_ 3686849250855195216_43001846923
|
||||
2 weaverroom_ 3686849250855195217_43001846923
|
||||
3 weaverroom_ 3686849250855195218_43001846923
|
||||
4 test_account 1234567890123456789_123456789
|
||||
5 test_account 1234567890123456790_123456789
|
||||
6 another_user 9876543210987654321_987654321
|
||||
142
sync_seen_from_table.py
Normal file
142
sync_seen_from_table.py
Normal file
@ -0,0 +1,142 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Sync script to mark already scraped posts as seen in Redis
|
||||
Parses table data and calls the admin endpoint to sync seen posts.
|
||||
"""
|
||||
|
||||
import re
|
||||
import sys
|
||||
import requests
|
||||
from collections import defaultdict
|
||||
from typing import List, Tuple
|
||||
|
||||
# Configuration - CHANGE THIS TO YOUR FASTAPI HOST
|
||||
API_BASE = "http://localhost:8000" # Change to your FastAPI host
|
||||
API_ENDPOINT = f"{API_BASE}/admin/sync-seen"
|
||||
|
||||
def parse_table_data(file_path: str) -> List[Tuple[str, str]]:
|
||||
"""
|
||||
Parse table data from file and extract account/ig_post_id pairs.
|
||||
|
||||
Expected format: lines with <id> <account> <ig_post_id> ...
|
||||
Ig post id may have suffix like _43001846923, we extract the base part.
|
||||
"""
|
||||
rows = []
|
||||
|
||||
try:
|
||||
with open(file_path, encoding="utf-8") as f:
|
||||
text = f.read()
|
||||
except FileNotFoundError:
|
||||
print(f"❌ File not found: {file_path}")
|
||||
return []
|
||||
except Exception as e:
|
||||
print(f"❌ Error reading file: {e}")
|
||||
return []
|
||||
|
||||
for line_num, line in enumerate(text.splitlines(), 1):
|
||||
# Skip empty lines
|
||||
if not line.strip():
|
||||
continue
|
||||
|
||||
# Match lines with: <id> <account> <ig_post_id> ...
|
||||
# Pattern: whitespace + number + whitespace + account + whitespace + ig_post_id (with optional suffix)
|
||||
m = re.search(r"\s*(\d+)\s+([A-Za-z0-9._]+)\s+(\d+)(?:_\d+)?\s", line)
|
||||
if m:
|
||||
_, account, ig_id = m.groups()
|
||||
rows.append((account, ig_id))
|
||||
else:
|
||||
print(f"⚠️ Skipping line {line_num}: {line.strip()}")
|
||||
|
||||
return rows
|
||||
|
||||
def sync_seen_posts(account_data: dict) -> None:
|
||||
"""
|
||||
Call the admin endpoint to mark posts as seen for each account.
|
||||
"""
|
||||
total_synced = 0
|
||||
|
||||
for account, ids in account_data.items():
|
||||
if not ids:
|
||||
continue
|
||||
|
||||
payload = {
|
||||
"username": account,
|
||||
"ig_post_ids": sorted(list(ids)) # Convert set to sorted list
|
||||
}
|
||||
|
||||
try:
|
||||
print(f"📤 Syncing {len(ids)} posts for @{account}...")
|
||||
response = requests.post(API_ENDPOINT, json=payload, timeout=30)
|
||||
response.raise_for_status()
|
||||
|
||||
result = response.json()
|
||||
marked_count = result.get("marked_seen", 0)
|
||||
print(f"✅ @{account} -> marked_seen: {marked_count}")
|
||||
total_synced += marked_count
|
||||
|
||||
except requests.exceptions.ConnectionError:
|
||||
print(f"❌ Connection error for @{account}. Is the API running at {API_BASE}?")
|
||||
except requests.exceptions.Timeout:
|
||||
print(f"❌ Timeout error for @{account}")
|
||||
except requests.exceptions.RequestException as e:
|
||||
print(f"❌ Request error for @{account}: {e}")
|
||||
except Exception as e:
|
||||
print(f"❌ Unexpected error for @{account}: {e}")
|
||||
|
||||
print(f"\n🎉 Total posts synced: {total_synced}")
|
||||
|
||||
def main():
|
||||
"""Main function to process the table file and sync seen posts."""
|
||||
|
||||
if len(sys.argv) != 2:
|
||||
print("Usage: python sync_seen_from_table.py <table_file>")
|
||||
print("Example: python sync_seen_from_table.py ./already_scraped.txt")
|
||||
sys.exit(1)
|
||||
|
||||
file_path = sys.argv[1]
|
||||
|
||||
print("🧪 Instagram Seen Posts Sync Tool")
|
||||
print("=" * 50)
|
||||
print(f"📁 Processing file: {file_path}")
|
||||
print(f"🌐 API endpoint: {API_ENDPOINT}")
|
||||
print()
|
||||
|
||||
# Parse the table data
|
||||
print("📝 Parsing table data...")
|
||||
rows = parse_table_data(file_path)
|
||||
|
||||
if not rows:
|
||||
print("❌ No valid data found in file")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"✅ Found {len(rows)} rows")
|
||||
|
||||
# Group by account and deduplicate
|
||||
by_account = defaultdict(set)
|
||||
for account, ig_id in rows:
|
||||
by_account[account].add(ig_id) # dedupe per account
|
||||
|
||||
print(f"📊 Grouped into {len(by_account)} accounts:")
|
||||
for account, ids in by_account.items():
|
||||
print(f" - @{account}: {len(ids)} posts")
|
||||
|
||||
print()
|
||||
|
||||
# Confirm before proceeding
|
||||
total_posts = sum(len(ids) for ids in by_account.values())
|
||||
response = input(f"Proceed to sync {total_posts} posts across {len(by_account)} accounts? (y/N): ")
|
||||
|
||||
if response.lower() not in ['y', 'yes']:
|
||||
print("❌ Sync cancelled")
|
||||
sys.exit(0)
|
||||
|
||||
print()
|
||||
|
||||
# Sync the seen posts
|
||||
sync_seen_posts(by_account)
|
||||
|
||||
print()
|
||||
print("🎉 Sync completed!")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
137
test_admin_sync.py
Normal file
137
test_admin_sync.py
Normal file
@ -0,0 +1,137 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Test script to verify the admin sync endpoint works correctly
|
||||
This script tests the /admin/sync-seen endpoint.
|
||||
"""
|
||||
|
||||
import requests
|
||||
import json
|
||||
from typing import List
|
||||
|
||||
# Configuration - CHANGE THIS TO YOUR FASTAPI HOST
|
||||
API_BASE = "http://localhost:8000" # Change to your FastAPI host
|
||||
API_ENDPOINT = f"{API_BASE}/admin/sync-seen"
|
||||
|
||||
def test_admin_sync():
|
||||
"""Test the admin sync endpoint"""
|
||||
|
||||
print("🧪 Testing Admin Sync Endpoint")
|
||||
print("=" * 50)
|
||||
print(f"🌐 API endpoint: {API_ENDPOINT}")
|
||||
print()
|
||||
|
||||
# Test data
|
||||
test_cases = [
|
||||
{
|
||||
"name": "Single account, multiple posts",
|
||||
"username": "test_user_1",
|
||||
"ig_post_ids": ["1234567890123456789", "1234567890123456790", "1234567890123456791"]
|
||||
},
|
||||
{
|
||||
"name": "Single account, single post",
|
||||
"username": "test_user_2",
|
||||
"ig_post_ids": ["9876543210987654321"]
|
||||
},
|
||||
{
|
||||
"name": "Empty post list",
|
||||
"username": "test_user_3",
|
||||
"ig_post_ids": []
|
||||
}
|
||||
]
|
||||
|
||||
for i, case in enumerate(test_cases, 1):
|
||||
print(f"📝 Test {i}: {case['name']}")
|
||||
print(f" - Username: @{case['username']}")
|
||||
print(f" - Post IDs: {len(case['ig_post_ids'])} posts")
|
||||
|
||||
try:
|
||||
payload = {
|
||||
"username": case["username"],
|
||||
"ig_post_ids": case["ig_post_ids"]
|
||||
}
|
||||
|
||||
print(f" 📤 Sending request...")
|
||||
response = requests.post(API_ENDPOINT, json=payload, timeout=30)
|
||||
|
||||
if response.status_code == 200:
|
||||
result = response.json()
|
||||
marked_count = result.get("marked_seen", 0)
|
||||
print(f" ✅ Success! marked_seen: {marked_count}")
|
||||
|
||||
if marked_count == len(case["ig_post_ids"]):
|
||||
print(f" ✅ All posts marked correctly")
|
||||
else:
|
||||
print(f" ⚠️ Expected {len(case['ig_post_ids'])}, got {marked_count}")
|
||||
else:
|
||||
print(f" ❌ HTTP {response.status_code}: {response.text}")
|
||||
|
||||
except requests.exceptions.ConnectionError:
|
||||
print(f" ❌ Connection error. Is the API running at {API_BASE}?")
|
||||
except requests.exceptions.Timeout:
|
||||
print(f" ❌ Timeout error")
|
||||
except requests.exceptions.RequestException as e:
|
||||
print(f" ❌ Request error: {e}")
|
||||
except Exception as e:
|
||||
print(f" ❌ Unexpected error: {e}")
|
||||
|
||||
print()
|
||||
|
||||
print("🎉 Admin sync test completed!")
|
||||
return True
|
||||
|
||||
def test_health_check():
|
||||
"""Test if the API is running"""
|
||||
|
||||
print("🏥 Testing API Health Check")
|
||||
print("=" * 30)
|
||||
|
||||
try:
|
||||
health_url = f"{API_BASE}/health"
|
||||
response = requests.get(health_url, timeout=10)
|
||||
|
||||
if response.status_code == 200:
|
||||
print(f"✅ API is running at {API_BASE}")
|
||||
return True
|
||||
else:
|
||||
print(f"❌ API health check failed: {response.status_code}")
|
||||
return False
|
||||
|
||||
except requests.exceptions.ConnectionError:
|
||||
print(f"❌ Cannot connect to API at {API_BASE}")
|
||||
print(" Make sure the FastAPI server is running:")
|
||||
print(" docker-compose up")
|
||||
return False
|
||||
except Exception as e:
|
||||
print(f"❌ Health check error: {e}")
|
||||
return False
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("🧪 Admin Sync Endpoint Test Suite")
|
||||
print("=" * 60)
|
||||
|
||||
# First check if API is running
|
||||
if not test_health_check():
|
||||
print("\n❌ API is not running. Please start the FastAPI server first.")
|
||||
print(" docker-compose up")
|
||||
exit(1)
|
||||
|
||||
print()
|
||||
|
||||
# Test the admin sync endpoint
|
||||
success = test_admin_sync()
|
||||
|
||||
print("=" * 60)
|
||||
if success:
|
||||
print("✅ Admin sync test passed!")
|
||||
print("\n📝 Summary:")
|
||||
print("- Admin endpoint is working correctly")
|
||||
print("- Posts are being marked as seen in Redis")
|
||||
print("- Endpoint handles various input sizes")
|
||||
print("- Ready to use with sync_seen_from_table.py")
|
||||
else:
|
||||
print("❌ Admin sync test failed!")
|
||||
print("\nTroubleshooting:")
|
||||
print("1. Check that the API is running")
|
||||
print("2. Verify Redis connection")
|
||||
print("3. Check admin_sync.py is properly imported")
|
||||
print("4. Ensure the endpoint is accessible")
|
||||
Loading…
Reference in New Issue
Block a user