142 lines
4.5 KiB
Python
142 lines
4.5 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Sync script to mark already scraped posts as seen in Redis
|
|
Parses table data and calls the admin endpoint to sync seen posts.
|
|
"""
|
|
|
|
import re
|
|
import sys
|
|
import requests
|
|
from collections import defaultdict
|
|
from typing import List, Tuple
|
|
|
|
# Configuration - CHANGE THIS TO YOUR FASTAPI HOST
|
|
API_BASE = "http://localhost:8000" # Change to your FastAPI host
|
|
API_ENDPOINT = f"{API_BASE}/admin/sync-seen"
|
|
|
|
def parse_table_data(file_path: str) -> List[Tuple[str, str]]:
|
|
"""
|
|
Parse table data from file and extract account/ig_post_id pairs.
|
|
|
|
Expected format: lines with <id> <account> <ig_post_id> ...
|
|
Ig post id may have suffix like _43001846923, we extract the base part.
|
|
"""
|
|
rows = []
|
|
|
|
try:
|
|
with open(file_path, encoding="utf-8") as f:
|
|
text = f.read()
|
|
except FileNotFoundError:
|
|
print(f"❌ File not found: {file_path}")
|
|
return []
|
|
except Exception as e:
|
|
print(f"❌ Error reading file: {e}")
|
|
return []
|
|
|
|
for line_num, line in enumerate(text.splitlines(), 1):
|
|
# Skip empty lines
|
|
if not line.strip():
|
|
continue
|
|
|
|
# Match lines with: <id> <account> <ig_post_id> ...
|
|
# Pattern: whitespace/tabs + number + whitespace/tabs + account + whitespace/tabs + ig_post_id (with optional suffix)
|
|
m = re.search(r"\s*(\d+)\s+([A-Za-z0-9._]+)\s+(\d+)(?:_\d+)?", line)
|
|
if m:
|
|
_, account, ig_id = m.groups()
|
|
rows.append((account, ig_id))
|
|
else:
|
|
print(f"⚠️ Skipping line {line_num}: {line.strip()}")
|
|
|
|
return rows
|
|
|
|
def sync_seen_posts(account_data: dict) -> None:
|
|
"""
|
|
Call the admin endpoint to mark posts as seen for each account.
|
|
"""
|
|
total_synced = 0
|
|
|
|
for account, ids in account_data.items():
|
|
if not ids:
|
|
continue
|
|
|
|
payload = {
|
|
"username": account,
|
|
"ig_post_ids": sorted(list(ids)) # Convert set to sorted list
|
|
}
|
|
|
|
try:
|
|
print(f"📤 Syncing {len(ids)} posts for @{account}...")
|
|
response = requests.post(API_ENDPOINT, json=payload, timeout=30)
|
|
response.raise_for_status()
|
|
|
|
result = response.json()
|
|
marked_count = result.get("marked_seen", 0)
|
|
print(f"✅ @{account} -> marked_seen: {marked_count}")
|
|
total_synced += marked_count
|
|
|
|
except requests.exceptions.ConnectionError:
|
|
print(f"❌ Connection error for @{account}. Is the API running at {API_BASE}?")
|
|
except requests.exceptions.Timeout:
|
|
print(f"❌ Timeout error for @{account}")
|
|
except requests.exceptions.RequestException as e:
|
|
print(f"❌ Request error for @{account}: {e}")
|
|
except Exception as e:
|
|
print(f"❌ Unexpected error for @{account}: {e}")
|
|
|
|
print(f"\n🎉 Total posts synced: {total_synced}")
|
|
|
|
def main():
|
|
"""Main function to process the table file and sync seen posts."""
|
|
|
|
if len(sys.argv) != 2:
|
|
print("Usage: python sync_seen_from_table.py <table_file>")
|
|
print("Example: python sync_seen_from_table.py ./already_scraped.txt")
|
|
sys.exit(1)
|
|
|
|
file_path = sys.argv[1]
|
|
|
|
print("🧪 Instagram Seen Posts Sync Tool")
|
|
print("=" * 50)
|
|
print(f"📁 Processing file: {file_path}")
|
|
print(f"🌐 API endpoint: {API_ENDPOINT}")
|
|
print()
|
|
|
|
# Parse the table data
|
|
print("📝 Parsing table data...")
|
|
rows = parse_table_data(file_path)
|
|
|
|
if not rows:
|
|
print("❌ No valid data found in file")
|
|
sys.exit(1)
|
|
|
|
print(f"✅ Found {len(rows)} rows")
|
|
|
|
# Group by account and deduplicate
|
|
by_account = defaultdict(set)
|
|
for account, ig_id in rows:
|
|
by_account[account].add(ig_id) # dedupe per account
|
|
|
|
print(f"📊 Grouped into {len(by_account)} accounts:")
|
|
for account, ids in by_account.items():
|
|
print(f" - @{account}: {len(ids)} posts")
|
|
|
|
print()
|
|
|
|
# Confirm before proceeding
|
|
total_posts = sum(len(ids) for ids in by_account.values())
|
|
response = input(f"Proceed to sync {total_posts} posts across {len(by_account)} accounts? (y/N): ")
|
|
|
|
if response.lower() not in ['y', 'yes']:
|
|
print("❌ Sync cancelled")
|
|
sys.exit(0)
|
|
|
|
print()
|
|
|
|
# Sync the seen posts
|
|
sync_seen_posts(by_account)
|
|
|
|
print()
|
|
print("🎉 Sync completed!")
|
|
|
|
if __name__ == "__main__":
|
|
main() |