#!/usr/bin/env python3
"""
4chan Multi-Board Keyword Monitor
=================================
Monitors 4chan boards for keywords and sends Gotify notifications when matches are found.
Features:
- Multi-board support with per-board keyword lists
- Gotify notifications with post content and image previews
- Single proxy support (configured at top)
- Duplicate detection to avoid repeated notifications
- SQLite database for tracking already-notified posts
- Automatic schema versioning and migration
- Runs once and terminates
- Only scans thread title and first post (OP) for keyword matches
- Proper Markdown rendering in Gotify notifications

Copyright (c) 2026 Wizardry and Steamworks
"""

import os
import sys
import time
import re
import hashlib
import sqlite3
import json
from datetime import datetime
import subprocess
import tempfile
from html.parser import HTMLParser
import html
from collections import defaultdict
from contextlib import contextmanager
import threading
import base64

# Suppress BeautifulSoup URL warning
try:
    from bs4 import BeautifulSoup, MarkupResemblesLocatorWarning
    import warnings
    warnings.filterwarnings("ignore", category=MarkupResemblesLocatorWarning)
    HAS_BS4 = True
except ImportError:
    HAS_BS4 = False
    print("WARNING: BeautifulSoup4 not installed. Using fallback HTML parser.", flush=True)

try:
    from curl_cffi import requests
    HAS_CURL_CFI = True
except ImportError:
    HAS_CURL_CFI = False
    print("WARNING: curl_cffi not installed. Using standard requests.", flush=True)
    import requests

# ==========================================
# CONFIGURATION (GLOBAL VARIABLES)
# ==========================================

# Gotify Configuration
GOTIFY_URL = "https://gotify.0x90.duckdns.org/message"  # Change this to your Gotify server URL
GOTIFY_TOKEN = "gtfya.l7zZq0UadK_zJI-kJQUA3LTVcBtyFS0SHT3wrXMzIXk"  # Change this to your Gotify app token
GOTIFY_PRIORITY = 5  # 0-10, higher = more important

# Board configuration with keyword mappings
BOARD_KEYWORDS = {
    "pol": ["games", "flock"],
    "x": ["succubus"]
}

# ==========================================
# PROXY CONFIGURATION (SINGLE PROXY)
# ==========================================
# Set to None to disable proxy and connect directly
# Format: {"http": "http://HOSTNAME:PORT", "https": "http://HOSTNAME:PORT"}
# Example: {"http": "http://127.0.0.1:8080", "https": "http://127.0.0.1:8080"}
PROXY = {"http": "http://proxydrum:8080"}

# 4chan API endpoints
API_CATALOG_URL = "https://a.4cdn.org/{}/catalog.json"
THREAD_BASE_URL = "https://a.4cdn.org/{}/thread/{}.json"
IMAGE_BASE_URL = "https://i.4cdn.org/{}/{}{}"

# SQLite database path for caching metadata and hashes
DATABASE_PATH = "/mnt/docker-applications/scan-4chan/monitor.db"

# Database schema version - increment when schema changes
DB_SCHEMA_VERSION = 1

# Maximum retry attempts for failed requests
MAX_RETRY_ATTEMPTS = 3

# Database lock for thread safety
_db_lock = threading.Lock()

# ==========================================
# DATABASE FUNCTIONS
# ==========================================

def get_schema_version(conn):
    """Get the current schema version from the database."""
    try:
        cursor = conn.execute("SELECT value FROM config WHERE key = 'schema_version'")
        row = cursor.fetchone()
        return int(row[0]) if row else 0
    except:
        return 0

def set_schema_version(conn, version):
    """Set the schema version in the database."""
    conn.execute(
        "INSERT OR REPLACE INTO config (key, value) VALUES ('schema_version', ?)",
        (str(version),)
    )
    conn.commit()

def get_db_schema():
    """Return the current database schema as a dictionary of SQL statements."""
    return {
        'notifications': """
            CREATE TABLE IF NOT EXISTS notifications (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                post_id TEXT NOT NULL,
                thread_id TEXT NOT NULL,
                board TEXT NOT NULL,
                keyword TEXT NOT NULL,
                notified_at INTEGER DEFAULT (strftime('%s', 'now')),
                post_content TEXT,
                image_url TEXT,
                UNIQUE(post_id, board, keyword)
            )
        """,
        'notifications_indexes': [
            "CREATE INDEX IF NOT EXISTS idx_notifications_post_id ON notifications(post_id)",
            "CREATE INDEX IF NOT EXISTS idx_notifications_board ON notifications(board, keyword)",
            "CREATE INDEX IF NOT EXISTS idx_notifications_notified_at ON notifications(notified_at)"
        ],
        'proxy_cache': """
            CREATE TABLE IF NOT EXISTS proxy_cache (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                proxy_url TEXT UNIQUE,
                success_count INTEGER DEFAULT 0,
                fail_count INTEGER DEFAULT 0,
                last_used INTEGER,
                last_success INTEGER,
                response_time REAL,
                is_working BOOLEAN DEFAULT 1
            )
        """,
        'config': """
            CREATE TABLE IF NOT EXISTS config (
                key TEXT PRIMARY KEY,
                value TEXT,
                updated_at INTEGER DEFAULT (strftime('%s', 'now'))
            )
        """
    }

def check_and_migrate_database():
    """Check database schema version and migrate/drop if needed."""
    print(f"[DB] Checking database schema at: {DATABASE_PATH}", flush=True)

    # Ensure directory exists
    db_dir = os.path.dirname(DATABASE_PATH)
    if not os.path.exists(db_dir):
        print(f"[DB] Creating database directory: {db_dir}", flush=True)
        os.makedirs(db_dir, exist_ok=True)

    db_exists = os.path.exists(DATABASE_PATH)

    if db_exists:
        try:
            conn = sqlite3.connect(DATABASE_PATH, timeout=30)
            conn.row_factory = sqlite3.Row
            current_version = get_schema_version(conn)
            conn.close()

            print(f"[DB] Current schema version: {current_version}", flush=True)
            print(f"[DB] Required schema version: {DB_SCHEMA_VERSION}", flush=True)

            if current_version != DB_SCHEMA_VERSION:
                print(f"[DB] Schema version mismatch. Recreating database...", flush=True)
                recreate_database()
                return
            else:
                print(f"[DB] Schema version matches. Database ready.", flush=True)
                return
        except Exception as e:
            print(f"[DB] Error checking schema: {e}", flush=True)
            print(f"[DB] Recreating database...", flush=True)
            recreate_database()
            return
    else:
        print(f"[DB] Database does not exist. Creating new database...", flush=True)
        recreate_database()

def recreate_database():
    """Recreate the database with current schema."""
    try:
        if os.path.exists(DATABASE_PATH):
            os.remove(DATABASE_PATH)
            print(f"[DB] Removed old database file", flush=True)

        print(f"[DB] Creating new database schema...", flush=True)

        conn = sqlite3.connect(DATABASE_PATH, timeout=30)
        conn.row_factory = sqlite3.Row

        conn.execute("PRAGMA journal_mode=WAL")
        conn.execute("PRAGMA synchronous=NORMAL")
        conn.execute("PRAGMA cache_size=-65536")

        schema = get_db_schema()
        tables = [t for t in schema.keys() if t != 'notifications_indexes']
        print(f"[DB] Creating {len(tables)} tables...", flush=True)

        for table_name in tables:
            print(f"[DB]   Creating table: {table_name}", flush=True)
            conn.execute(schema[table_name])

        print(f"[DB] Creating {len(schema['notifications_indexes'])} indexes...", flush=True)
        for idx, index_sql in enumerate(schema['notifications_indexes']):
            print(f"[DB]   Index {idx+1}/{len(schema['notifications_indexes'])}", flush=True)
            conn.execute(index_sql)

        print(f"[DB] Setting schema version to {DB_SCHEMA_VERSION}...", flush=True)
        set_schema_version(conn, DB_SCHEMA_VERSION)

        conn.commit()
        conn.close()

        print(f"[DB] Database created successfully with schema version {DB_SCHEMA_VERSION}", flush=True)

    except Exception as e:
        print(f"[DB] Error recreating database: {e}", flush=True)
        raise

@contextmanager
def get_db():
    """Get database connection with WAL mode and proper locking."""
    with _db_lock:
        conn = sqlite3.connect(DATABASE_PATH, timeout=30, isolation_level=None)
        conn.execute("PRAGMA journal_mode=WAL")
        conn.execute("PRAGMA synchronous=NORMAL")
        conn.execute("PRAGMA cache_size=-65536")
        conn.execute("PRAGMA foreign_keys=ON")
        conn.execute("PRAGMA temp_store=MEMORY")
        conn.row_factory = sqlite3.Row

        try:
            yield conn
            conn.commit()
        except Exception as e:
            conn.rollback()
            raise e
        finally:
            conn.close()

def is_post_notified(post_id, board, keyword):
    """Check if a post has already been notified."""
    try:
        with get_db() as conn:
            cursor = conn.execute(
                "SELECT 1 FROM notifications WHERE post_id = ? AND board = ? AND keyword = ? LIMIT 1",
                (str(post_id), board, keyword)
            )
            return cursor.fetchone() is not None
    except:
        return False

def mark_post_notified(post_id, thread_id, board, keyword, post_content, image_url):
    """Mark a post as notified in the database."""
    try:
        with get_db() as conn:
            conn.execute("""
                INSERT OR REPLACE INTO notifications
                (post_id, thread_id, board, keyword, post_content, image_url)
                VALUES (?, ?, ?, ?, ?, ?)
            """, (str(post_id), str(thread_id), board, keyword, post_content, image_url))
            conn.commit()
            return True
    except Exception as e:
        print(f"[DB WARNING] Could not mark post as notified: {e}", flush=True)
        return False

# ==========================================
# HTML/XML PARSER UTILITIES
# ==========================================

class HTMLTextExtractor(HTMLParser):
    """Custom HTML parser that extracts plain text from HTML content."""
    def __init__(self):
        super().__init__()
        self.text_parts = []
        self.skip_tags = {'script', 'style', 'head', 'title'}
        self.in_skip_tag = False

    def handle_starttag(self, tag, attrs):
        if tag in self.skip_tags:
            self.in_skip_tag = True

    def handle_endtag(self, tag):
        if tag in self.skip_tags:
            self.in_skip_tag = False

    def handle_data(self, data):
        if not self.in_skip_tag and data.strip():
            self.text_parts.append(data.strip())

    def get_text(self):
        return ' '.join(self.text_parts)

def extract_text_from_html(html_content):
    """Extract plain text from HTML content using a proper DOM parser."""
    if not html_content:
        return "No content"

    if HAS_BS4:
        try:
            soup = BeautifulSoup(html_content, 'html.parser')
            for script in soup(["script", "style"]):
                script.decompose()
            text = soup.get_text(separator=' ', strip=True)
            text = re.sub(r'\s+', ' ', text).strip()
            return text if text else "No content"
        except Exception:
            pass

    try:
        parser = HTMLTextExtractor()
        parser.feed(html_content)
        text = parser.get_text()
        return text if text else "No content"
    except:
        text = re.sub(r'<[^>]+>', ' ', html_content)
        text = re.sub(r'\s+', ' ', text).strip()
        return text if text else "No content"

def clean_html_content(html_content, max_length=500):
    """Clean HTML content and extract plain text with length limit."""
    text = extract_text_from_html(html_content)
    if len(text) > max_length:
        text = text[:max_length - 3] + "..."
    return text

def truncate_text(text, max_length=200):
    """Truncate text to a maximum length."""
    if len(text) > max_length:
        return text[:max_length - 3] + "..."
    return text

# ==========================================
# GOTIFY NOTIFICATION FUNCTIONS
# ==========================================

def send_gotify_notification(title, message, priority=GOTIFY_PRIORITY, image_url=None):
    """Send a notification to Gotify with Markdown support."""
    try:
        headers = {
            "X-Gotify-Key": GOTIFY_TOKEN,
            "Content-Type": "application/json"
        }

        # Build the payload with Markdown support
        payload = {
            "title": title,
            "message": message,
            "priority": priority,
            "extras": {
                "client::display": {
                    "contentType": "text/markdown"
                }
            }
        }

        # If image URL is provided, try to download and attach it
        if image_url:
            try:
                if PROXY:
                    img_response = requests.get(image_url, proxies=PROXY, timeout=10)
                else:
                    img_response = requests.get(image_url, timeout=10)
                if img_response.status_code == 200:
                    # Convert image to base64 for attachment
                    img_data = base64.b64encode(img_response.content).decode('utf-8')
                    # Determine content type
                    ext = os.path.splitext(image_url)[1].lower()
                    content_type = "image/jpeg"
                    if ext in ['.png']:
                        content_type = "image/png"
                    elif ext in ['.gif']:
                        content_type = "image/gif"

                    payload["attachments"] = [{
                        "name": f"image{ext}",
                        "data": img_data,
                        "content_type": content_type
                    }]
            except Exception as e:
                print(f"      [GOTIFY WARNING] Could not fetch image for attachment: {e}", flush=True)

        if HAS_CURL_CFI:
            response = requests.post(GOTIFY_URL, json=payload, headers=headers, timeout=10)
        else:
            response = requests.post(GOTIFY_URL, json=payload, headers=headers, timeout=10)

        if response.status_code == 200:
            print(f"      [GOTIFY] Notification sent successfully", flush=True)
            return True
        else:
            print(f"      [GOTIFY ERROR] Failed to send notification: {response.status_code}", flush=True)
            if response.text:
                print(f"      [GOTIFY ERROR] Response: {response.text[:200]}", flush=True)
            return False

    except Exception as e:
        print(f"      [GOTIFY ERROR] Could not send notification: {e}", flush=True)
        return False

def create_notification_content(thread_data, post_data):
    """Create the notification title and message from thread and post data."""
    board = thread_data.get('board', 'unknown')
    thread_id = thread_data.get('thread_id', 'unknown')
    thread_title = thread_data.get('title', 'No title')

    # Clean up post content
    post_content = post_data.get('content', 'No content')
    post_content_clean = clean_html_content(post_content, max_length=500)

    # Get first image from post if available
    image_url = post_data.get('image_url', None)

    # Build title
    title = f"[{board}] Keyword Match: {truncate_text(thread_title, 60)}"

    # Build message with Markdown links - only thread link, no post link
    message = f"""**Board:** /{board}/
**Thread ID:** {thread_id}
**Keyword:** {thread_data.get('keyword', 'unknown')}

**OP Post Content:**
{post_content_clean}

**Thread:** [https://boards.4chan.org/{board}/thread/{thread_id}](https://boards.4chan.org/{board}/thread/{thread_id})
"""

    return title, message, image_url

# ==========================================
# REQUEST FUNCTIONS (SINGLE PROXY)
# ==========================================

def execute_request(url, headers):
    """Execute HTTP request using the configured single proxy (or direct if none)."""
    try:
        if HAS_CURL_CFI:
            if PROXY:
                res = requests.get(url, impersonate="chrome120", headers=headers, proxies=PROXY, timeout=15)
            else:
                res = requests.get(url, impersonate="chrome120", headers=headers, timeout=15)
        else:
            if PROXY:
                res = requests.get(url, headers=headers, proxies=PROXY, timeout=15)
            else:
                res = requests.get(url, headers=headers, timeout=15)
        return res
    except Exception as e:
        print(f"[REQUEST ERROR] {e}", flush=True)
        return None

# ==========================================
# CORE MONITORING FUNCTIONS
# ==========================================

def get_thread_first_post(board, thread_id, browser_headers):
    """Fetch only the first post (OP) from a thread."""
    thread_endpoint = THREAD_BASE_URL.format(board, thread_id)
    thread_res = execute_request(thread_endpoint, browser_headers)

    if not thread_res:
        print(f"    -> [ERROR]: Could not fetch thread.", flush=True)
        return None, None

    try:
        posts_data = thread_res.json().get("posts", [])
        if posts_data:
            # First post is always the OP
            return posts_data[0], thread_id
        return None, None
    except Exception:
        return None, None

def process_op_for_notification(board, keyword, op_post, thread_id, thread_title, thread_content, browser_headers):
    """Process the OP (first post) and send notification if it matches."""
    post_id = op_post.get("no")
    if not post_id:
        return False

    # Check if already notified
    if is_post_notified(post_id, board, keyword):
        return False

    # Get OP content
    post_content = op_post.get("com", "No content")

    # Check if the OP contains the keyword in its content
    content_text = extract_text_from_html(post_content).lower()
    if keyword.lower() not in content_text:
        return False

    # Get image if available
    tim_identifier = op_post.get("tim")
    ext_suffix = op_post.get("ext")
    image_url = None
    if tim_identifier and ext_suffix:
        image_url = IMAGE_BASE_URL.format(board, tim_identifier, ext_suffix)

    # Prepare thread data - use the thread_id passed from the catalog
    thread_info = {
        'board': board,
        'thread_id': thread_id,
        'title': thread_title,
        'keyword': keyword
    }

    post_info = {
        'post_id': post_id,
        'content': post_content,
        'image_url': image_url
    }

    # Create notification content
    title, message, img_url = create_notification_content(thread_info, post_info)

    # Send notification
    success = send_gotify_notification(title, message, GOTIFY_PRIORITY, img_url)

    if success:
        # Mark as notified
        mark_post_notified(
            post_id,
            thread_id,
            board,
            keyword,
            post_content,
            image_url
        )
        return True

    return False

def scan_board(board, keywords, browser_headers):
    """Scan a board for keyword matches and send notifications for matching OPs."""
    print(f"\n{'='*60}")
    print(f"SCANNING BOARD: /{board}/ with keywords: {keywords}")
    print(f"{'='*60}\n", flush=True)

    catalog_url = API_CATALOG_URL.format(board)
    print(f"[CATALOG] Fetching from: {catalog_url}", flush=True)

    catalog_res = execute_request(catalog_url, browser_headers)

    if not catalog_res:
        print(f"[ERROR]: Failed to fetch catalog for /{board}/", file=sys.stderr, flush=True)
        return 0

    try:
        pages_list = catalog_res.json()
    except Exception as parse_err:
        print(f"[ERROR]: Invalid JSON: {parse_err}", file=sys.stderr, flush=True)
        return 0

    print(f"[MATCH] Scanning catalog for keywords in titles and OP content...", flush=True)
    matched_threads = {}
    processed_threads = set()

    # Find threads with matching keywords in title OR OP content preview
    for page in pages_list:
        for thread_item in page.get("threads", []):
            subject = thread_item.get("sub", "").lower()  # Title
            teaser = thread_item.get("com", "").lower()   # OP content preview
            combined_text = f"{subject} {teaser}"         # Check both

            thread_no = thread_item.get("no")
            if not thread_no:
                continue

            if thread_no in processed_threads:
                continue

            for keyword in keywords:
                if keyword.lower() in combined_text:
                    if keyword not in matched_threads:
                        matched_threads[keyword] = []
                    matched_threads[keyword].append(thread_item)
                    processed_threads.add(thread_no)
                    break

    total_matches = sum(len(threads) for threads in matched_threads.values())
    print(f"[MATCH] Found {total_matches} matching threads", flush=True)

    if total_matches == 0:
        return 0

    notifications_sent = 0

    for keyword, thread_items in matched_threads.items():
        print(f"\n[KEYWORD: {keyword}] Processing {len(thread_items)} matching threads...", flush=True)

        for index, thread_item in enumerate(thread_items):
            thread_id = thread_item.get("no")
            thread_title = thread_item.get("sub", "No title")
            thread_content = thread_item.get("com", "")

            cleaned_title = clean_html_content(thread_title, max_length=200)

            print(f"  Processing thread [{index + 1}/{len(thread_items)}] (ID: {thread_id})...", flush=True)

            # Fetch only the OP (first post) - returns the post and the thread_id
            op_post, _ = get_thread_first_post(board, thread_id, browser_headers)

            if not op_post:
                print(f"    -> [WARNING]: Could not fetch OP for thread {thread_id}", flush=True)
                continue

            # Process the OP - pass the thread_id explicitly from the catalog
            if process_op_for_notification(
                board, keyword, op_post, str(thread_id), cleaned_title, thread_content,
                browser_headers
            ):
                notifications_sent += 1
                print(f"    -> [NOTIFICATION] Sent for OP post {op_post.get('no', 'unknown')}", flush=True)

            else:
                print(f"    -> [INFO] OP did not contain keyword '{keyword}'", flush=True)

    return notifications_sent

def run_monitor_pass(browser_headers):
    """Run a single monitoring pass."""
    print(f"\n{'='*60}")
    print(f"STARTING MONITORING PASS - {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
    print(f"{'='*60}\n", flush=True)

    total_notifications = 0

    # Process boards
    for board, keywords in BOARD_KEYWORDS.items():
        notifications = scan_board(board, keywords, browser_headers)
        total_notifications += notifications
        print(f"\nCompleted /{board}/ - Sent {notifications} notifications", flush=True)

    # Print summary
    print(f"\n{'='*60}")
    print(f"SUMMARY: Sent {total_notifications} notifications total")
    print(f"{'='*60}\n", flush=True)

    return total_notifications

# ==========================================
# MAIN FUNCTION
# ==========================================

def main():
    """
    Main entry point for the 4chan keyword monitor.
    Runs once and terminates.
    """
    print("=== 4CHAN MULTI-BOARD KEYWORD MONITOR ===", flush=True)
    print(f"[TIME]  {time.strftime('%Y-%m-%d %H:%M:%S')}", flush=True)
    print(f"[BOARDS]: {', '.join(BOARD_KEYWORDS.keys())}", flush=True)
    print(f"[GOTIFY URL]: {GOTIFY_URL}", flush=True)
    if PROXY:
        print(f"[PROXY]: {PROXY.get('http', 'N/A')}", flush=True)
    else:
        print(f"[PROXY]: None (direct connection)", flush=True)
    print("")

    # Validate Gotify configuration
    if GOTIFY_URL == "https://gotify.example.com/message" or GOTIFY_TOKEN == "your-app-token-here":
        print("[ERROR] Please configure GOTIFY_URL and GOTIFY_TOKEN in the configuration section.", flush=True)
        print("[ERROR] Exiting...", flush=True)
        return 1

    # Database setup
    print("[INIT] Setting up database...", flush=True)
    check_and_migrate_database()

    browser_headers = {
        "Accept": "application/json, text/plain, */*",
        "Accept-Language": "en-US,en;q=0.9",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
    }

    # Run a single monitoring pass
    total_notifications = run_monitor_pass(browser_headers)

    print(f"\n[COMPLETE] Monitoring pass finished. Sent {total_notifications} notifications.", flush=True)
    return 0

if __name__ == "__main__":
    try:
        sys.exit(main())
    except KeyboardInterrupt:
        print("\n[INFO]: Interrupted by user. Exiting gracefully...", flush=True)
        sys.exit(1)
    except Exception as e:
        print(f"\n[FATAL ERROR]: {e}", file=sys.stderr, flush=True)
        import traceback
        traceback.print_exc()
        sys.exit(1)