#!/usr/bin/env python3
"""
4chan Multi-Board Keyword Harvester
===================================
Downloads images from 4chan boards based on keyword matching.
Features:
- Multi-board support with per-board keyword lists
- Semantic directory structure: /base/board/keyword/YYYYMMDD/
- First-match-only filing (threads with multiple keyword matches filed under first match)
- Single proxy support (configured at top)
- Permanent retention (no automatic deletion)
- Duplicate detection via perceptual hashing (both during download and post-processing)
- Metadata embedding in media files (EXIF/XMP)
- Associated .nfo files for Jellyfin/Kodi/Emby compatibility using proper XML DOM
- Automatic deduplication: keeps only the oldest file when duplicates are found
- SQLite database for fast startup and caching of perceptual hashes
- Automatic schema versioning and migration
- File integrity checking: detects and deletes corrupted files including zero-byte files
- ONLY scans directories defined in BOARD_KEYWORDS configuration
- HEAVY database reliance for maximum speed
- Optimized video checking with fast header verification
- Detailed progress reporting

Copyright (c) 2026 Wizardry and Steamworks
"""

import os
import sys
import time
import re
import hashlib
import sqlite3
import json
from datetime import datetime
import subprocess
import tempfile
from xml.dom import minidom
from xml.etree import ElementTree as ET
from html.parser import HTMLParser
import html
from collections import defaultdict
from contextlib import contextmanager
import threading

# Suppress BeautifulSoup URL warning
try:
    from bs4 import BeautifulSoup, MarkupResemblesLocatorWarning
    import warnings
    warnings.filterwarnings("ignore", category=MarkupResemblesLocatorWarning)
    HAS_BS4 = True
except ImportError:
    HAS_BS4 = False
    print("WARNING: BeautifulSoup4 not installed. Using fallback HTML parser.", flush=True)

# Try importing optional dependencies with fallbacks
try:
    from PIL import Image
    from PIL.ExifTags import TAGS
    HAS_PIL = True
except ImportError:
    HAS_PIL = False
    print("WARNING: PIL/Pillow not installed. Image metadata embedding disabled.", flush=True)

try:
    import imagehash
    HAS_IMAGEHASH = True
except ImportError:
    HAS_IMAGEHASH = False
    print("WARNING: imagehash not installed. Duplicate detection disabled.", flush=True)

try:
    from curl_cffi import requests
    HAS_CURL_CFI = True
except ImportError:
    HAS_CURL_CFI = False
    print("WARNING: curl_cffi not installed. Using standard requests.", flush=True)
    import requests

# ==========================================
# CONFIGURATION (GLOBAL VARIABLES)
# ==========================================

# Base download directory - all boards and keywords will be organized here
BASE_DOWNLOAD_DIR = "/mnt/archie/Pictures/4chan"

# SQLite database path for caching metadata and hashes
DATABASE_PATH = "/mnt/docker-applications/scalp-4chan/harvester.db"

# Database schema version - increment when schema changes
DB_SCHEMA_VERSION = 4

# Image similarity threshold (0-100, lower = more strict matching)
SIMILARITY_THRESHOLD = 85

# Enable automatic deduplication of existing files
ENABLE_DEDUPLICATION = True

# Enable file integrity checking
ENABLE_INTEGRITY_CHECK = True

# Maximum retry attempts for failed downloads
MAX_RETRY_ATTEMPTS = 3

# Minimum file size in bytes (files smaller than this are considered corrupted)
MIN_FILE_SIZE = 1024  # 1KB

# Board configuration with keyword mappings
BOARD_KEYWORDS = {
    "pol": ["humor", "humour"],
    "wsg": ["animals"]
}

# ==========================================
# PROXY CONFIGURATION (SINGLE PROXY)
# ==========================================
# Set to None to disable proxy and connect directly
# Format: {"http": "http://HOSTNAME:PORT", "https": "http://HOSTNAME:PORT"}
# Example: {"http": "http://127.0.0.1:8080", "https": "http://127.0.0.1:8080"}
PROXY = {"http": "http://proxydrum:8080"}

# 4chan API endpoints
API_CATALOG_URL = "https://a.4cdn.org/{}/catalog.json"
THREAD_BASE_URL = "https://a.4cdn.org/{}/thread/{}.json"
IMAGE_BASE_URL = "https://i.4cdn.org/{}/{}{}"

# Database lock for thread safety
_db_lock = threading.Lock()

# ==========================================
# VALID PATH FUNCTIONS
# ==========================================

def get_valid_boards_and_keywords():
    """Get all valid board/keyword directory combinations."""
    combinations = []
    for board, keywords in BOARD_KEYWORDS.items():
        for keyword in keywords:
            combinations.append((board, keyword))
    return combinations

def get_valid_paths():
    """Get all valid directory paths based on BOARD_KEYWORDS."""
    paths = []
    for board, keywords in BOARD_KEYWORDS.items():
        base_dir = os.path.join(BASE_DOWNLOAD_DIR, board)
        for keyword in keywords:
            paths.append(os.path.join(base_dir, keyword))
    return paths

def is_valid_path(filepath):
    """Check if a filepath belongs to a valid board/keyword directory."""
    valid_paths = get_valid_paths()
    return any(filepath.startswith(valid_path) for valid_path in valid_paths)

# ==========================================
# DATABASE FUNCTIONS
# ==========================================

def get_schema_version(conn):
    """Get the current schema version from the database."""
    try:
        cursor = conn.execute("SELECT value FROM config WHERE key = 'schema_version'")
        row = cursor.fetchone()
        return int(row[0]) if row else 0
    except:
        return 0

def set_schema_version(conn, version):
    """Set the schema version in the database."""
    conn.execute(
        "INSERT OR REPLACE INTO config (key, value) VALUES ('schema_version', ?)",
        (str(version),)
    )
    conn.commit()

def get_db_schema():
    """Return the current database schema as a dictionary of SQL statements."""
    return {
        'files': """
            CREATE TABLE IF NOT EXISTS files (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                filepath TEXT UNIQUE NOT NULL,
                board TEXT NOT NULL,
                keyword TEXT,
                date_folder TEXT,
                file_size INTEGER,
                modified_time INTEGER,
                sha256 TEXT,
                perceptual_hash TEXT,
                media_type TEXT,
                last_seen INTEGER,
                created_at INTEGER DEFAULT (strftime('%s', 'now')),
                integrity_checked INTEGER DEFAULT 0,
                integrity_status TEXT DEFAULT 'unknown',
                retry_count INTEGER DEFAULT 0,
                quick_hash TEXT,
                header_hash TEXT
            )
        """,
        'files_indexes': [
            "CREATE INDEX IF NOT EXISTS idx_files_path ON files(filepath)",
            "CREATE INDEX IF NOT EXISTS idx_files_hash ON files(perceptual_hash)",
            "CREATE INDEX IF NOT EXISTS idx_files_board ON files(board, keyword)",
            "CREATE INDEX IF NOT EXISTS idx_files_last_seen ON files(last_seen)",
            "CREATE INDEX IF NOT EXISTS idx_files_integrity ON files(integrity_status)",
            "CREATE INDEX IF NOT EXISTS idx_files_quick_hash ON files(quick_hash)",
            "CREATE INDEX IF NOT EXISTS idx_files_sha256 ON files(sha256)"
        ],
        'proxy_cache': """
            CREATE TABLE IF NOT EXISTS proxy_cache (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                proxy_url TEXT UNIQUE,
                success_count INTEGER DEFAULT 0,
                fail_count INTEGER DEFAULT 0,
                last_used INTEGER,
                last_success INTEGER,
                response_time REAL,
                is_working BOOLEAN DEFAULT 1
            )
        """,
        'config': """
            CREATE TABLE IF NOT EXISTS config (
                key TEXT PRIMARY KEY,
                value TEXT,
                updated_at INTEGER DEFAULT (strftime('%s', 'now'))
            )
        """
    }

def check_and_migrate_database():
    """Check database schema version and migrate/drop if needed."""
    print(f"[DB] Checking database schema at: {DATABASE_PATH}", flush=True)

    # Ensure directory exists
    db_dir = os.path.dirname(DATABASE_PATH)
    if not os.path.exists(db_dir):
        print(f"[DB] Creating database directory: {db_dir}", flush=True)
        os.makedirs(db_dir, exist_ok=True)

    db_exists = os.path.exists(DATABASE_PATH)

    if db_exists:
        try:
            # Try to connect and check version
            conn = sqlite3.connect(DATABASE_PATH, timeout=30)
            conn.row_factory = sqlite3.Row
            current_version = get_schema_version(conn)
            conn.close()

            print(f"[DB] Current schema version: {current_version}", flush=True)
            print(f"[DB] Required schema version: {DB_SCHEMA_VERSION}", flush=True)

            if current_version != DB_SCHEMA_VERSION:
                print(f"[DB] Schema version mismatch. Recreating database...", flush=True)
                recreate_database()
                return
            else:
                print(f"[DB] Schema version matches. Database ready.", flush=True)
                return
        except Exception as e:
            print(f"[DB] Error checking schema: {e}", flush=True)
            print(f"[DB] Recreating database...", flush=True)
            recreate_database()
            return
    else:
        print(f"[DB] Database does not exist. Creating new database...", flush=True)
        recreate_database()

def recreate_database():
    """Recreate the database with current schema."""
    try:
        if os.path.exists(DATABASE_PATH):
            os.remove(DATABASE_PATH)
            print(f"[DB] Removed old database file", flush=True)

        print(f"[DB] Creating new database schema...", flush=True)

        # Create database directly without using context manager
        conn = sqlite3.connect(DATABASE_PATH, timeout=30)
        conn.row_factory = sqlite3.Row

        # Enable WAL mode
        conn.execute("PRAGMA journal_mode=WAL")
        conn.execute("PRAGMA synchronous=NORMAL")
        conn.execute("PRAGMA cache_size=-65536")

        # Create tables
        schema = get_db_schema()
        tables = [t for t in schema.keys() if t != 'files_indexes']
        print(f"[DB] Creating {len(tables)} tables...", flush=True)

        for table_name in tables:
            print(f"[DB]   Creating table: {table_name}", flush=True)
            conn.execute(schema[table_name])

        # Create indexes
        print(f"[DB] Creating {len(schema['files_indexes'])} indexes...", flush=True)
        for idx, index_sql in enumerate(schema['files_indexes']):
            print(f"[DB]   Index {idx+1}/{len(schema['files_indexes'])}", flush=True)
            conn.execute(index_sql)

        # Set schema version
        print(f"[DB] Setting schema version to {DB_SCHEMA_VERSION}...", flush=True)
        set_schema_version(conn, DB_SCHEMA_VERSION)

        conn.commit()
        conn.close()

        print(f"[DB] Database created successfully with schema version {DB_SCHEMA_VERSION}", flush=True)

    except Exception as e:
        print(f"[DB] Error recreating database: {e}", flush=True)
        raise

@contextmanager
def get_db():
    """Get database connection with WAL mode and proper locking."""
    with _db_lock:
        conn = sqlite3.connect(DATABASE_PATH, timeout=30, isolation_level=None)
        conn.execute("PRAGMA journal_mode=WAL")
        conn.execute("PRAGMA synchronous=NORMAL")
        conn.execute("PRAGMA cache_size=-65536")
        conn.execute("PRAGMA foreign_keys=ON")
        conn.execute("PRAGMA temp_store=MEMORY")
        conn.row_factory = sqlite3.Row

        try:
            yield conn
            conn.commit()
        except Exception as e:
            conn.rollback()
            raise e
        finally:
            conn.close()

def get_quick_hash(filepath):
    """Get a quick hash of the first 1KB of a file for fast lookup."""
    try:
        with open(filepath, 'rb') as f:
            data = f.read(1024)
            return hashlib.md5(data).hexdigest()
    except:
        return None

def get_header_hash(filepath):
    """Get hash of the file header (first 4KB) for fast identification."""
    try:
        with open(filepath, 'rb') as f:
            data = f.read(4096)
            return hashlib.sha256(data).hexdigest()
    except:
        return None

def insert_file_db_fast(filepath, board, keyword, date_folder, sha256, perceptual_hash, media_type):
    """Fast insert/update of file record using prepared statements."""
    try:
        with get_db() as conn:
            file_size = os.path.getsize(filepath)
            modified_time = int(os.path.getmtime(filepath))
            quick_hash = get_quick_hash(filepath)
            header_hash = get_header_hash(filepath)

            conn.execute("""
                INSERT OR REPLACE INTO files
                (filepath, board, keyword, date_folder, file_size, modified_time,
                 sha256, perceptual_hash, media_type, last_seen, integrity_checked,
                 integrity_status, quick_hash, header_hash)
                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, strftime('%s', 'now'), 0, 'unknown', ?, ?)
            """, (filepath, board, keyword, date_folder, file_size, modified_time,
                  sha256, perceptual_hash, media_type, quick_hash, header_hash))

            conn.commit()
    except Exception as e:
        print(f"      [DB WARNING] Could not update file: {e}", flush=True)

def file_exists_in_db(filepath):
    """Quick check if a file exists in the database."""
    try:
        with get_db() as conn:
            cursor = conn.execute(
                "SELECT 1 FROM files WHERE filepath = ? LIMIT 1",
                (filepath,)
            )
            return cursor.fetchone() is not None
    except:
        return False

def get_file_from_db_fast(filepath):
    """Get file record from database by path using optimized query."""
    try:
        with get_db() as conn:
            cursor = conn.execute(
                "SELECT filepath, sha256, perceptual_hash, integrity_status, modified_time, file_size FROM files WHERE filepath = ?",
                (filepath,)
            )
            row = cursor.fetchone()
            return dict(row) if row else None
    except Exception:
        return None

def get_hash_from_db_fast(perceptual_hash, threshold=SIMILARITY_THRESHOLD):
    """
    Fast duplicate check using database with perceptual hash.
    """
    if not HAS_IMAGEHASH:
        return []

    try:
        with get_db() as conn:
            cursor = conn.execute(
                "SELECT filepath, perceptual_hash FROM files WHERE perceptual_hash IS NOT NULL"
            )
            results = []

            try:
                h1 = imagehash.hex_to_hash(perceptual_hash)
            except:
                return []

            for row in cursor.fetchall():
                if not is_valid_path(row['filepath']):
                    continue

                existing_hash = row['perceptual_hash']
                if existing_hash:
                    try:
                        h2 = imagehash.hex_to_hash(existing_hash)
                        distance = h1 - h2
                        max_distance = 256
                        similarity = (1 - (distance / max_distance)) * 100
                        if similarity >= threshold:
                            results.append({
                                'filepath': row['filepath'],
                                'similarity': similarity
                            })
                            return results
                    except:
                        continue
            return results
    except Exception as e:
        print(f"[DB WARNING] Error in get_hash_from_db_fast: {e}", flush=True)
        return []

def update_integrity_batch(filepaths, status):
    """Batch update integrity status for multiple files."""
    if not filepaths:
        return

    try:
        with get_db() as conn:
            placeholders = ','.join(['?'] * len(filepaths))
            conn.execute(f"""
                UPDATE files
                SET integrity_checked = 1, integrity_status = ?
                WHERE filepath IN ({placeholders})
            """, (status, *filepaths))
            conn.commit()
    except Exception as e:
        print(f"[DB WARNING] Batch integrity update failed: {e}", flush=True)

def delete_files_batch(filepaths):
    """Batch delete files from database."""
    if not filepaths:
        return

    try:
        with get_db() as conn:
            placeholders = ','.join(['?'] * len(filepaths))
            conn.execute(f"DELETE FROM files WHERE filepath IN ({placeholders})", filepaths)
            conn.commit()
    except Exception as e:
        print(f"[DB WARNING] Batch delete failed: {e}", flush=True)

def cleanup_database_fast():
    """Fast cleanup of orphaned entries."""
    print("[DB] Cleaning up orphaned database entries...", flush=True)

    removed_count = 0

    try:
        with get_db() as conn:
            cursor = conn.execute("SELECT filepath FROM files")
            files_in_db = [row['filepath'] for row in cursor.fetchall()]

            total_files = len(files_in_db)
            print(f"[DB] Found {total_files} files in database", flush=True)

            if total_files == 0:
                print("[DB] No files to clean up", flush=True)
                return

            to_delete = []
            for filepath in files_in_db:
                if not is_valid_path(filepath) or not os.path.exists(filepath):
                    to_delete.append(filepath)

            if to_delete:
                print(f"[DB] Removing {len(to_delete)} orphaned entries...", flush=True)
                placeholders = ','.join(['?'] * len(to_delete))
                conn.execute(f"DELETE FROM files WHERE filepath IN ({placeholders})", to_delete)
                removed_count = len(to_delete)
                conn.commit()

            print(f"[DB] Removed {removed_count} orphaned entries", flush=True)
    except Exception as e:
        print(f"[DB] Error during cleanup: {e}", flush=True)

def get_database_stats_fast():
    """Get fast statistics from the database."""
    try:
        with get_db() as conn:
            cursor = conn.execute("SELECT COUNT(*) as count FROM files")
            total = cursor.fetchone()['count']

            cursor = conn.execute("SELECT COUNT(*) as count FROM files WHERE perceptual_hash IS NOT NULL")
            hashed = cursor.fetchone()['count']

            cursor = conn.execute("SELECT COUNT(*) as count FROM files WHERE integrity_status = 'valid'")
            valid = cursor.fetchone()['count']

            return {
                'total_files': total,
                'hashed_files': hashed,
                'valid_files': valid
            }
    except Exception as e:
        print(f"[DB WARNING] Could not get stats: {e}", flush=True)
        return {'total_files': 0, 'hashed_files': 0, 'valid_files': 0}

# ==========================================
# BUILD DATABASE FROM FILESYSTEM
# ==========================================

def build_database_from_filesystem():
    """
    Scan the filesystem and populate the database with existing files.
    This is the initial population step.
    """
    print("\n[SCAN] Scanning filesystem and building database...", flush=True)

    valid_paths = get_valid_paths()
    if not valid_paths:
        print("[SCAN] No valid paths configured in BOARD_KEYWORDS", flush=True)
        return

    total_files = 0
    processed = 0
    skipped = 0

    # First, count total files
    print("[SCAN] Counting files in configured directories...", flush=True)
    media_files = []
    for valid_path in valid_paths:
        if not os.path.exists(valid_path):
            print(f"[SCAN] Directory does not exist: {valid_path}", flush=True)
            continue

        print(f"[SCAN] Scanning: {valid_path}", flush=True)
        for root, dirs, files in os.walk(valid_path):
            for file in files:
                ext = os.path.splitext(file)[1].lower()
                if ext in {'.jpg', '.jpeg', '.png', '.gif', '.mp4', '.webm'}:
                    filepath = os.path.join(root, file)
                    # Check if already in database
                    if not file_exists_in_db(filepath):
                        media_files.append(filepath)
                        total_files += 1

    if not media_files:
        print("[SCAN] No new files found to add to database", flush=True)
        return

    print(f"[SCAN] Found {total_files} new files to process", flush=True)

    # Process files with progress
    for idx, filepath in enumerate(media_files):
        processed += 1
        if processed % 10 == 0:
            print(f"[SCAN] Progress: {processed}/{total_files} files processed", flush=True)

        # Extract board and keyword from path
        rel_path = os.path.relpath(filepath, BASE_DOWNLOAD_DIR)
        parts = rel_path.split(os.sep)

        board = parts[0] if len(parts) > 0 else "unknown"
        keyword = parts[1] if len(parts) > 1 else "unknown"
        date_folder = parts[2] if len(parts) > 2 else "unknown"

        # Check integrity
        is_valid, _ = check_file_integrity_fast(filepath)

        if not is_valid:
            print(f"[SCAN] Skipping corrupted file: {os.path.basename(filepath)}", flush=True)
            skipped += 1
            continue

        # Get hashes
        perceptual_hash = get_media_hash_fast(filepath)
        sha256 = get_file_hash(filepath)

        ext = os.path.splitext(filepath)[1].lower()
        media_type = 'image' if ext in {'.jpg', '.jpeg', '.png', '.gif'} else 'video'

        # Insert into database
        insert_file_db_fast(
            filepath, board, keyword, date_folder,
            sha256,
            str(perceptual_hash) if perceptual_hash else None,
            media_type
        )

        # Mark as valid
        update_integrity_batch([filepath], 'valid')

    print(f"[SCAN] Database build complete!", flush=True)
    print(f"[SCAN] Processed: {processed}, Skipped: {skipped}", flush=True)

# ==========================================
# FAST FILE INTEGRITY CHECKING
# ==========================================

def check_file_integrity_fast(filepath):
    """Fast integrity check using quick methods first."""
    if not os.path.exists(filepath):
        return False, "File does not exist"

    try:
        file_size = os.path.getsize(filepath)
        if file_size == 0:
            return False, "Zero-byte file"
        if file_size < MIN_FILE_SIZE:
            return False, f"File too small ({file_size} bytes)"
    except Exception as e:
        return False, f"Cannot get file size: {e}"

    ext = os.path.splitext(filepath)[1].lower()

    try:
        with open(filepath, 'rb') as f:
            header = f.read(20)
            if not header:
                return False, "Empty file"

            if ext in {'.jpg', '.jpeg'}:
                if header.startswith(b'\xFF\xD8'):
                    return True, "OK"
                return False, "Invalid JPEG header"
            elif ext == '.png':
                if header.startswith(b'\x89PNG'):
                    return True, "OK"
                return False, "Invalid PNG header"
            elif ext == '.gif':
                if header.startswith(b'GIF8'):
                    return True, "OK"
                return False, "Invalid GIF header"
            elif ext in {'.mp4', '.webm'}:
                if header.startswith(b'\x00\x00\x00') or header.startswith(b'\x1A\x45\xDF\xA3'):
                    return True, "OK"
                return False, "Invalid video header"
            else:
                return True, "OK"
    except Exception as e:
        return False, f"Cannot read file: {e}"

def check_video_integrity_fast(filepath):
    """Fast video integrity check using header and quick ffmpeg."""
    try:
        with open(filepath, 'rb') as f:
            header = f.read(50)
            if not header.startswith(b'\x00\x00\x00') and not header.startswith(b'\x1A\x45\xDF\xA3'):
                return False, "Invalid video header"
    except:
        return False, "Cannot read video header"

    try:
        cmd = [
            'ffmpeg', '-v', 'error',
            '-probesize', '4M',
            '-analyzeduration', '2M',
            '-i', filepath,
            '-f', 'null', '-'
        ]
        result = subprocess.run(cmd, capture_output=True, text=True, timeout=3)

        if result.returncode == 0:
            return True, "OK"
        elif result.stderr:
            stderr_lower = result.stderr.lower()
            non_critical = ['invalid data found', 'non-monotonous', 'missing reference',
                          'co located', 'header missing', 'max delay', 'vbv buffer']
            if any(p in stderr_lower for p in non_critical):
                return True, "OK (non-critical warning)"
            else:
                return False, f"Video corruption: {result.stderr[:100]}"
        else:
            return True, "OK"
    except subprocess.TimeoutExpired:
        return True, "OK (timeout, header valid)"
    except FileNotFoundError:
        return True, "OK (ffmpeg not available)"
    except Exception as e:
        return False, f"Video check failed: {e}"

# ==========================================
# FAST HASH FUNCTIONS
# ==========================================

def get_media_hash_fast(filepath):
    """Get perceptual hash quickly with caching."""
    if not HAS_IMAGEHASH or not HAS_PIL:
        return None

    ext = os.path.splitext(filepath)[1].lower()

    try:
        if ext in {'.jpg', '.jpeg', '.png', '.gif'}:
            img = Image.open(filepath)
            if img.mode != 'RGB':
                img = img.convert('RGB')
            return imagehash.phash(img, hash_size=16)
        elif ext in {'.mp4', '.webm'}:
            try:
                with tempfile.NamedTemporaryFile(suffix='.jpg', delete=False) as f:
                    temp_frame = f.name

                cmd = [
                    'ffmpeg', '-v', 'error',
                    '-probesize', '2M',
                    '-analyzeduration', '1M',
                    '-i', filepath,
                    '-vframes', '1',
                    '-f', 'image2',
                    '-y', temp_frame
                ]
                result = subprocess.run(cmd, capture_output=True, text=True, timeout=5)

                if result.returncode == 0 and os.path.exists(temp_frame):
                    img = Image.open(temp_frame)
                    if img.mode != 'RGB':
                        img = img.convert('RGB')
                    img_hash = imagehash.phash(img, hash_size=16)
                    os.unlink(temp_frame)
                    return img_hash

                if os.path.exists(temp_frame):
                    os.unlink(temp_frame)
            except:
                pass
    except Exception:
        pass

    return None

# ==========================================
# FAST DEDUPLICATION
# ==========================================

def deduplicate_all_fast(directory=BASE_DOWNLOAD_DIR, threshold=SIMILARITY_THRESHOLD, dry_run=False):
    """Fast deduplication using database with optimized queries."""
    if not HAS_IMAGEHASH:
        print("\n[DEDUP] ImageHash not available. Skipping deduplication.", flush=True)
        return {}

    print("\n" + "="*60)
    print("STARTING FAST DEDUPLICATION PASS")
    print("="*60, flush=True)
    print(f"Similarity threshold: {threshold}%", flush=True)
    print("", flush=True)

    total_deleted = 0
    total_kept = 0

    print("[DEDUP] Fetching files from database...", flush=True)
    try:
        with get_db() as conn:
            cursor = conn.execute("""
                SELECT filepath, perceptual_hash, modified_time
                FROM files
                WHERE perceptual_hash IS NOT NULL
                ORDER BY modified_time ASC
            """)
            all_files = cursor.fetchall()
    except Exception as e:
        print(f"[DEDUP] Error fetching files: {e}", flush=True)
        return {}

    if not all_files:
        print("[DEDUP] No files found in database", flush=True)
        return {}

    print(f"[DEDUP] Found {len(all_files)} files in database", flush=True)

    files = []
    for row in all_files:
        if is_valid_path(row['filepath']) and os.path.exists(row['filepath']):
            files.append({
                'filepath': row['filepath'],
                'perceptual_hash': row['perceptual_hash'],
                'modified_time': row['modified_time']
            })

    if not files:
        print("[DEDUP] No valid files found", flush=True)
        return {}

    print(f"[DEDUP] Found {len(files)} valid files to process", flush=True)

    hash_groups = defaultdict(list)
    for f in files:
        hash_groups[f['perceptual_hash']].append(f)

    total_groups = len(hash_groups)
    print(f"[DEDUP] Found {total_groups} unique hash groups", flush=True)

    to_delete = []
    processed = 0
    for perceptual_hash, file_entries in hash_groups.items():
        processed += 1
        if processed % 10 == 0:
            print(f"[DEDUP] Processing group {processed}/{total_groups}", flush=True)

        if len(file_entries) <= 1:
            total_kept += len(file_entries)
            continue

        file_entries.sort(key=lambda x: x['modified_time'])
        keep_file = file_entries[0]['filepath']
        duplicate_files = [entry['filepath'] for entry in file_entries[1:]]

        if dry_run:
            print(f"[DEDUP] Would keep: {os.path.basename(keep_file)}", flush=True)
            for dup in duplicate_files:
                print(f"[DEDUP] Would delete: {os.path.basename(dup)}", flush=True)
            total_kept += 1
            total_deleted += len(duplicate_files)
        else:
            for dup in duplicate_files:
                try:
                    if os.path.exists(dup):
                        os.remove(dup)
                        total_deleted += 1
                        print(f"[DEDUP] Deleted: {os.path.basename(dup)}", flush=True)

                        nfo_file = os.path.splitext(dup)[0] + '.nfo'
                        if os.path.exists(nfo_file):
                            os.remove(nfo_file)

                        to_delete.append(dup)
                except Exception as e:
                    print(f"[DEDUP] Error deleting {os.path.basename(dup)}: {e}", flush=True)

            total_kept += 1
            print(f"[DEDUP] Kept: {os.path.basename(keep_file)}", flush=True)

    if to_delete and not dry_run:
        print(f"[DEDUP] Removing {len(to_delete)} entries from database...", flush=True)
        delete_files_batch(to_delete)

    print("\n" + "="*60)
    print("DEDUPLICATION SUMMARY")
    print("="*60, flush=True)
    print(f"Groups processed: {total_groups}", flush=True)
    print(f"Files kept: {total_kept}", flush=True)
    print(f"Files deleted: {total_deleted}", flush=True)
    print("", flush=True)

    return {}

# ==========================================
# FAST INTEGRITY SCAN
# ==========================================

def scan_and_delete_corrupted_fast(directory=BASE_DOWNLOAD_DIR, dry_run=False):
    """Fast integrity scan using database caching and batch operations."""
    if not ENABLE_INTEGRITY_CHECK:
        print("\n[INTEGRITY] Integrity checking disabled.", flush=True)
        return {}

    print("\n" + "="*60)
    print("STARTING FAST INTEGRITY SCAN")
    print("="*60, flush=True)
    print("Mode: CORRUPTED FILES WILL BE DELETED", flush=True)
    print("", flush=True)

    stats = {
        'checked': 0,
        'valid': 0,
        'corrupted': 0,
        'zero_byte': 0,
        'deleted': 0,
        'skipped': 0
    }

    print("[INTEGRITY] Fetching files from database...", flush=True)
    try:
        with get_db() as conn:
            cursor = conn.execute("""
                SELECT filepath, integrity_status, file_size
                FROM files
                WHERE integrity_status != 'valid' OR integrity_checked = 0
            """)
            files_to_check = cursor.fetchall()
    except Exception as e:
        print(f"[INTEGRITY] Error fetching files: {e}", flush=True)
        return stats

    if not files_to_check:
        print("[INTEGRITY] All files already verified", flush=True)
        return stats

    print(f"[INTEGRITY] Checking {len(files_to_check)} files...", flush=True)

    valid_files = []
    to_delete = []

    for idx, row in enumerate(files_to_check):
        if idx % 10 == 0:
            print(f"[INTEGRITY] Checking {idx}/{len(files_to_check)} files", flush=True)

        filepath = row['filepath']

        if not is_valid_path(filepath):
            continue

        if not os.path.exists(filepath):
            to_delete.append(filepath)
            stats['deleted'] += 1
            continue

        try:
            file_size = os.path.getsize(filepath)
            if file_size == 0:
                stats['zero_byte'] += 1
                stats['corrupted'] += 1
                to_delete.append(filepath)
                print(f"[INTEGRITY] Zero-byte file: {os.path.basename(filepath)}", flush=True)
                continue
            if file_size < MIN_FILE_SIZE:
                stats['corrupted'] += 1
                to_delete.append(filepath)
                print(f"[INTEGRITY] File too small: {os.path.basename(filepath)} ({file_size} bytes)", flush=True)
                continue
        except:
            continue

        stats['checked'] += 1

        ext = os.path.splitext(filepath)[1].lower()
        if ext in {'.mp4', '.webm'}:
            is_valid, _ = check_video_integrity_fast(filepath)
        else:
            is_valid, _ = check_file_integrity_fast(filepath)

        if is_valid:
            stats['valid'] += 1
            valid_files.append(filepath)
        else:
            stats['corrupted'] += 1
            to_delete.append(filepath)
            print(f"[INTEGRITY] Corrupted file: {os.path.basename(filepath)}", flush=True)

    if valid_files and not dry_run:
        print(f"[INTEGRITY] Marking {len(valid_files)} files as valid...", flush=True)
        update_integrity_batch(valid_files, 'valid')

    if to_delete and not dry_run:
        print(f"[INTEGRITY] Deleting {len(to_delete)} corrupted files...", flush=True)
        for filepath in to_delete:
            try:
                if os.path.exists(filepath):
                    os.remove(filepath)
                    stats['deleted'] += 1
                    print(f"[INTEGRITY] Deleted: {os.path.basename(filepath)}", flush=True)

                    nfo_file = os.path.splitext(filepath)[0] + '.nfo'
                    if os.path.exists(nfo_file):
                        os.remove(nfo_file)
            except Exception as e:
                print(f"[INTEGRITY] Error deleting {os.path.basename(filepath)}: {e}", flush=True)

        print(f"[INTEGRITY] Removing {len(to_delete)} entries from database...", flush=True)
        delete_files_batch(to_delete)

    print("\n" + "="*60)
    print("INTEGRITY SCAN SUMMARY")
    print("="*60, flush=True)
    print(f"Files checked: {stats['checked']}", flush=True)
    print(f"Valid files: {stats['valid']}", flush=True)
    print(f"Corrupted files: {stats['corrupted']}", flush=True)
    print(f"Zero-byte files: {stats['zero_byte']}", flush=True)
    print(f"Files deleted: {stats['deleted']}", flush=True)
    print("", flush=True)

    return stats

# ==========================================
# HTML/XML PARSER UTILITIES
# ==========================================

class HTMLTextExtractor(HTMLParser):
    """Custom HTML parser that extracts plain text from HTML content."""
    def __init__(self):
        super().__init__()
        self.text_parts = []
        self.skip_tags = {'script', 'style', 'head', 'title'}
        self.in_skip_tag = False

    def handle_starttag(self, tag, attrs):
        if tag in self.skip_tags:
            self.in_skip_tag = True

    def handle_endtag(self, tag):
        if tag in self.skip_tags:
            self.in_skip_tag = False

    def handle_data(self, data):
        if not self.in_skip_tag and data.strip():
            self.text_parts.append(data.strip())

    def get_text(self):
        return ' '.join(self.text_parts)

def extract_text_from_html(html_content):
    """Extract plain text from HTML content using a proper DOM parser."""
    if not html_content:
        return "No content"

    if HAS_BS4:
        try:
            soup = BeautifulSoup(html_content, 'html.parser')
            for script in soup(["script", "style"]):
                script.decompose()
            text = soup.get_text(separator=' ', strip=True)
            text = re.sub(r'\s+', ' ', text).strip()
            return text if text else "No content"
        except Exception:
            pass

    try:
        parser = HTMLTextExtractor()
        parser.feed(html_content)
        text = parser.get_text()
        return text if text else "No content"
    except:
        text = re.sub(r'<[^>]+>', ' ', html_content)
        text = re.sub(r'\s+', ' ', text).strip()
        return text if text else "No content"

def clean_html_content(html_content, max_length=1000):
    """Clean HTML content and extract plain text with length limit."""
    text = extract_text_from_html(html_content)
    if len(text) > max_length:
        text = text[:max_length - 3] + "..."
    return text

# ==========================================
# METADATA EMBEDDING FUNCTIONS
# ==========================================

def get_file_hash(filepath):
    """Calculate SHA-256 hash of a file."""
    sha256 = hashlib.sha256()
    with open(filepath, 'rb') as f:
        for chunk in iter(lambda: f.read(4096), b''):
            sha256.update(chunk)
    return sha256.hexdigest()

def embed_metadata_in_image(filepath, thread_data):
    """Embed metadata as EXIF comments in image files."""
    if not HAS_PIL:
        return

    try:
        metadata_text = f"""
4chan Thread Metadata
---------------------
Board: /{thread_data.get('board', 'unknown')}/
Thread ID: {thread_data.get('thread_id', 'unknown')}
Thread Title: {thread_data.get('title', 'No title')}
Thread Content: {thread_data.get('content', 'No content')}
Post ID: {thread_data.get('post_id', 'unknown')}
Download Date: {thread_data.get('download_date', 'unknown')}
Original Filename: {thread_data.get('original_filename', 'unknown')}
Original URL: {thread_data.get('original_url', 'unknown')}
File SHA-256: {thread_data.get('file_hash', 'unknown')}
        """.strip()

        img = Image.open(filepath)
        exif = img.getexif()

        USER_COMMENT = 0x9286
        IMAGE_DESCRIPTION = 0x010E

        exif[USER_COMMENT] = metadata_text
        exif[IMAGE_DESCRIPTION] = f"4chan /{thread_data.get('board', '')}/ - {thread_data.get('title', '')[:100]}"

        img.save(filepath, exif=exif)

    except Exception as e:
        try:
            img = Image.open(filepath)
            img.save(filepath)
        except:
            pass
        print(f"      [METADATA WARNING]: Could not embed metadata in image: {e}")

def embed_metadata_in_video(filepath, thread_data):
    """Embed metadata in video files using ffmpeg metadata."""
    try:
        subprocess.run(['ffmpeg', '-version'], capture_output=True, check=True)
    except (subprocess.CalledProcessError, FileNotFoundError):
        print(f"      [METADATA WARNING]: ffmpeg not installed.", flush=True)
        return

    try:
        metadata_text = f"""
4chan Thread Metadata
Board: /{thread_data.get('board', 'unknown')}/
Thread ID: {thread_data.get('thread_id', 'unknown')}
Title: {thread_data.get('title', 'No title')}
Content: {thread_data.get('content', 'No content')}
Post ID: {thread_data.get('post_id', 'unknown')}
Download Date: {thread_data.get('download_date', 'unknown')}
        """.strip()

        escaped_metadata = metadata_text.replace('\n', '\\n')

        with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False, encoding='utf-8') as f:
            f.write(f";FFMETADATA1\ncomment={escaped_metadata}\n")
            meta_file = f.name

        temp_output = filepath + '.temp'
        cmd = [
            'ffmpeg', '-i', filepath,
            '-i', meta_file,
            '-map_metadata', '1',
            '-codec', 'copy',
            '-y', temp_output
        ]

        result = subprocess.run(cmd, capture_output=True, text=True)

        if result.returncode == 0:
            os.replace(temp_output, filepath)
            print(f"      [METADATA]: Embedded metadata in video file")
        else:
            if os.path.exists(temp_output):
                os.unlink(temp_output)

        if os.path.exists(meta_file):
            os.unlink(meta_file)

    except Exception as e:
        print(f"      [METADATA WARNING]: Could not embed metadata in video: {e}")

# ==========================================
# NFO FILE GENERATION
# ==========================================

def create_nfo_file(filepath, thread_data):
    """Create an .nfo file for Jellyfin/Kodi/Emby compatibility."""
    nfo_path = os.path.splitext(filepath)[0] + '.nfo'

    try:
        root = ET.Element("movie")

        title = f"4chan /{thread_data.get('board', 'unknown')}/ - {thread_data.get('title', 'No title')}"
        ET.SubElement(root, "title").text = title
        ET.SubElement(root, "originaltitle").text = f"Thread {thread_data.get('thread_id', 'unknown')}"
        ET.SubElement(root, "plot").text = thread_data.get('content', 'No content')
        ET.SubElement(root, "tagline").text = "4chan Thread Download"
        ET.SubElement(root, "year").text = str(datetime.now().year)
        ET.SubElement(root, "studio").text = "4chan"
        ET.SubElement(root, "genre").text = f"4chan /{thread_data.get('board', 'unknown')}/"

        fileinfo = ET.SubElement(root, "fileinfo")
        streamdetails = ET.SubElement(fileinfo, "streamdetails")
        video = ET.SubElement(streamdetails, "video")
        codec = ET.SubElement(video, "codec")
        codec.text = "Original 4chan Media"

        custom = ET.SubElement(root, "custom")
        metadata = ET.SubElement(custom, "metadata")

        metadata_fields = [
            ("board", f"/{thread_data.get('board', 'unknown')}/"),
            ("thread_id", thread_data.get('thread_id', 'unknown')),
            ("post_id", thread_data.get('post_id', 'unknown')),
            ("download_date", thread_data.get('download_date', 'unknown')),
            ("original_filename", thread_data.get('original_filename', 'unknown')),
            ("original_url", thread_data.get('original_url', 'unknown')),
            ("file_hash", thread_data.get('file_hash', 'unknown')),
            ("keyword", thread_data.get('keyword', 'unknown'))
        ]

        for field_name, field_value in metadata_fields:
            elem = ET.SubElement(metadata, field_name)
            elem.text = str(field_value)

        xml_str = '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>\n'
        xml_str += ET.tostring(root, encoding='unicode')

        try:
            dom = minidom.parseString(xml_str)
            pretty_xml = dom.toprettyxml(indent="    ")
            pretty_xml = pretty_xml.replace('<?xml version="1.0" ?>\n', '')
            pretty_xml = '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>\n' + pretty_xml.lstrip()
        except:
            pretty_xml = xml_str

        with open(nfo_path, 'w', encoding='utf-8') as f:
            f.write(pretty_xml)

        print(f"      [NFO CREATED]: {nfo_path}")

    except Exception as e:
        print(f"      [NFO ERROR]: Could not create .nfo file: {e}")

# ==========================================
# CORE FUNCTIONS
# ==========================================

def enforce_retention_policy(download_dir):
    """Retention policy - now does nothing (permanent retention)."""
    return 0

# ==========================================
# REQUEST FUNCTIONS (SINGLE PROXY)
# ==========================================

def execute_request(url, headers):
    """Execute HTTP request using the configured single proxy (or direct if none)."""
    try:
        if HAS_CURL_CFI:
            if PROXY:
                res = requests.get(url, impersonate="chrome120", headers=headers, proxies=PROXY, timeout=15)
            else:
                res = requests.get(url, impersonate="chrome120", headers=headers, timeout=15)
        else:
            if PROXY:
                res = requests.get(url, headers=headers, proxies=PROXY, timeout=15)
            else:
                res = requests.get(url, headers=headers, timeout=15)
        return res
    except Exception as e:
        print(f"[REQUEST ERROR] {e}", flush=True)
        return None

def download_with_retry_fast(url, filepath, headers, max_retries=MAX_RETRY_ATTEMPTS):
    """Fast download with retry logic. Uses database for duplicate checking."""
    existing = get_file_from_db_fast(filepath)
    if existing and existing.get('integrity_status') == 'valid':
        return True, "File already exists and is valid"

    for attempt in range(max_retries):
        try:
            if attempt > 0:
                print(f"      [RETRY] Attempt {attempt + 1}/{max_retries}", flush=True)
                time.sleep(1 * (attempt + 1))

            img_res = execute_request(url, headers)

            if img_res and img_res.status_code == 200:
                with open(filepath, "wb") as f_writer:
                    f_writer.write(img_res.content)

                is_valid, error_msg = check_file_integrity_fast(filepath)

                if is_valid:
                    return True, "Download successful"
                else:
                    if os.path.exists(filepath):
                        os.remove(filepath)
                    if attempt < max_retries - 1:
                        continue
                    else:
                        return False, f"File corrupted: {error_msg}"
            else:
                if attempt < max_retries - 1:
                    continue
                else:
                    return False, f"Download failed"
        except Exception as e:
            if attempt < max_retries - 1:
                continue
            else:
                return False, f"Error: {e}"

    return False, "Unknown error"

def process_board_fast(board, keywords, browser_headers):
    """Fast board processing with database-first approach."""
    print(f"\n{'='*60}")
    print(f"PROCESSING BOARD: /{board}/ with keywords: {keywords}")
    print(f"{'='*60}\n", flush=True)

    date_stamp = datetime.now().strftime("%Y%m%d")
    download_date = datetime.now().strftime("%Y-%m-%d %H:%M:%S")

    catalog_url = API_CATALOG_URL.format(board)
    print(f"[CATALOG] Fetching from: {catalog_url}", flush=True)

    catalog_res = execute_request(catalog_url, browser_headers)

    if not catalog_res:
        print(f"[ERROR]: Failed to fetch catalog for /{board}/", file=sys.stderr, flush=True)
        return 0

    try:
        pages_list = catalog_res.json()
    except Exception as parse_err:
        print(f"[ERROR]: Invalid JSON: {parse_err}", file=sys.stderr, flush=True)
        return 0

    print(f"[MATCH] Scanning catalog for keywords...", flush=True)
    matched_threads = {}
    processed_threads = set()

    for page in pages_list:
        for thread_item in page.get("threads", []):
            subject = thread_item.get("sub", "").lower()
            teaser = thread_item.get("com", "").lower()
            combined_text = f"{subject} {teaser}"

            thread_no = thread_item.get("no")
            if not thread_no:
                continue

            if thread_no in processed_threads:
                continue

            for keyword in keywords:
                if keyword.lower() in combined_text:
                    if keyword not in matched_threads:
                        matched_threads[keyword] = []
                    matched_threads[keyword].append(thread_item)
                    processed_threads.add(thread_no)
                    break

    total_matches = sum(len(threads) for threads in matched_threads.values())
    print(f"[MATCH] Found {total_matches} matching threads", flush=True)

    if total_matches == 0:
        return 0

    new_downloads_count = 0

    for keyword, thread_items in matched_threads.items():
        keyword_dir = os.path.join(BASE_DOWNLOAD_DIR, board, keyword, date_stamp)
        os.makedirs(keyword_dir, exist_ok=True)
        print(f"\n[KEYWORD: {keyword}] Saving to: {keyword_dir}", flush=True)

        for index, thread_item in enumerate(thread_items):
            thread_id = thread_item.get("no")
            thread_title = thread_item.get("sub", "No title")
            thread_content = thread_item.get("com", "")

            cleaned_title = clean_html_content(thread_title, max_length=200)
            cleaned_content = clean_html_content(thread_content, max_length=1000)

            print(f"  Processing thread [{index + 1}/{len(thread_items)}] (ID: {thread_id})...", flush=True)

            thread_endpoint = THREAD_BASE_URL.format(board, thread_id)
            thread_res = execute_request(thread_endpoint, browser_headers)

            if not thread_res:
                print(f"    -> [ERROR]: Could not open thread.", flush=True)
                continue

            try:
                posts_data = thread_res.json().get("posts", [])
            except Exception:
                continue

            for post in posts_data:
                tim_identifier = post.get("tim")
                ext_suffix = post.get("ext")
                post_id = post.get("no")

                if not tim_identifier or not ext_suffix:
                    continue

                filename = f"{tim_identifier}{ext_suffix}"
                save_path = os.path.join(keyword_dir, filename)

                if file_exists_in_db(save_path):
                    continue

                img_url = IMAGE_BASE_URL.format(board, tim_identifier, ext_suffix)
                print(f"    -> [DISCOVERY]: {filename}", flush=True)

                success, message = download_with_retry_fast(
                    img_url, save_path, browser_headers, MAX_RETRY_ATTEMPTS
                )

                if not success:
                    print(f"       -> [ERROR]: {message}", flush=True)
                    if os.path.exists(save_path):
                        os.remove(save_path)
                    continue

                perceptual_hash = get_media_hash_fast(save_path)
                sha256 = get_file_hash(save_path)
                media_type = 'image' if ext_suffix in {'.jpg', '.jpeg', '.png', '.gif'} else 'video'

                thread_data = {
                    'board': board,
                    'thread_id': str(thread_id),
                    'title': cleaned_title,
                    'content': cleaned_content,
                    'post_id': str(post_id),
                    'download_date': download_date,
                    'original_filename': filename,
                    'original_url': img_url,
                    'file_hash': sha256,
                    'keyword': keyword
                }

                if ext_suffix in {'.jpg', '.jpeg', '.png', '.gif'}:
                    embed_metadata_in_image(save_path, thread_data)
                elif ext_suffix in {'.mp4', '.webm'}:
                    embed_metadata_in_video(save_path, thread_data)

                create_nfo_file(save_path, thread_data)

                insert_file_db_fast(
                    save_path, board, keyword, date_stamp,
                    sha256,
                    str(perceptual_hash) if perceptual_hash else None,
                    media_type
                )

                print(f"       -> [SUCCESS]: Archived", flush=True)
                new_downloads_count += 1

    return new_downloads_count

# ==========================================
# MAIN FUNCTION
# ==========================================

def main():
    """
    Main entry point for the 4chan multi-board harvester.
    """
    print("=== 4CHAN MULTI-BOARD KEYWORD HARVESTER ===", flush=True)
    print(f"[TIME]  {time.strftime('%Y-%m-%d %H:%M:%S')}", flush=True)
    print(f"[BOARDS]: {', '.join(BOARD_KEYWORDS.keys())}", flush=True)
    print(f"[SIMILARITY THRESHOLD]: {SIMILARITY_THRESHOLD}%", flush=True)
    print(f"[DEDUPLICATION]: {'Enabled' if ENABLE_DEDUPLICATION else 'Disabled'}", flush=True)
    print(f"[INTEGRITY CHECK]: {'Enabled' if ENABLE_INTEGRITY_CHECK else 'Disabled'}", flush=True)
    if PROXY:
        print(f"[PROXY]: {PROXY.get('http', 'N/A')}", flush=True)
    else:
        print(f"[PROXY]: None (direct connection)", flush=True)
    print("")

    # Create base directory
    print("[INIT] Creating base directories...", flush=True)
    os.makedirs(BASE_DOWNLOAD_DIR, exist_ok=True)

    # Database setup
    print("[INIT] Setting up database...", flush=True)
    check_and_migrate_database()

    # Clean up database
    print("[INIT] Cleaning up database...", flush=True)
    cleanup_database_fast()

    # BUILD DATABASE FROM FILESYSTEM FIRST
    build_database_from_filesystem()

    # Get database stats
    print("[INIT] Getting database statistics...", flush=True)
    stats = get_database_stats_fast()
    print(f"[DB] {stats['total_files']} files, {stats['hashed_files']} hashes", flush=True)
    print(f"[DB] {stats['valid_files']} valid files", flush=True)

    # Fast integrity scan (only checks files that need it)
    if ENABLE_INTEGRITY_CHECK:
        print("[INIT] Running integrity scan...", flush=True)
        scan_and_delete_corrupted_fast(BASE_DOWNLOAD_DIR, dry_run=False)

    browser_headers = {
        "Accept": "application/json, text/plain, */*",
        "Accept-Language": "en-US,en;q=0.9",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
    }

    total_downloads = 0

    # Process boards
    for board, keywords in BOARD_KEYWORDS.items():
        downloads = process_board_fast(board, keywords, browser_headers)
        total_downloads += downloads
        print(f"\nCompleted /{board}/ - Downloaded {downloads} new assets", flush=True)

    # Final integrity scan
    if ENABLE_INTEGRITY_CHECK:
        print("[FINAL] Running final integrity scan...", flush=True)
        scan_and_delete_corrupted_fast(BASE_DOWNLOAD_DIR, dry_run=False)

    # Final deduplication
    if ENABLE_DEDUPLICATION:
        print("[FINAL] Running final deduplication...", flush=True)
        deduplicate_all_fast(BASE_DOWNLOAD_DIR, SIMILARITY_THRESHOLD, dry_run=False)

    # Final summary
    print(f"\n{'='*60}")
    print(f"FINAL SUMMARY: Total new assets: {total_downloads}")
    print(f"{'='*60}", flush=True)
    print(f"\nCopyright (c) 2026 Wizardry and Steamworks")
    print(f"MIT License - https://opensource.org/licenses/MIT")

    return 0

if __name__ == "__main__":
    try:
        sys.exit(main())
    except KeyboardInterrupt:
        print("\n[INFO]: Interrupted by user. Exiting gracefully...", flush=True)
        sys.exit(1)
    except Exception as e:
        print(f"\n[FATAL ERROR]: {e}", file=sys.stderr, flush=True)
        import traceback
        traceback.print_exc()
        sys.exit(1)