#!/usr/bin/env python3
"""
4chan /news/ Board Monitor (JSON API)
======================================
Monitors the 4chan /news/ board via JSON API and sends Gotify notifications for new threads.
Features:
- JSON API for reliable data fetching
- Gotify notifications with thread title, content, and image
- Thread ID tracking to avoid duplicate notifications
- SQLite database for persistent tracking
- Proxy rotation with failover
- Runs once and terminates
- Proper Markdown rendering in Gotify notifications
 
Copyright (c) 2026 Wizardry and Steamworks
"""
 
import os
import sys
import time
import re
import sqlite3
from datetime import datetime
from contextlib import contextmanager
import threading
import base64
 
# Suppress BeautifulSoup URL warning
try:
    from bs4 import BeautifulSoup, MarkupResemblesLocatorWarning
    import warnings
    warnings.filterwarnings("ignore", category=MarkupResemblesLocatorWarning)
    HAS_BS4 = True
except ImportError:
    HAS_BS4 = False
    print("WARNING: BeautifulSoup4 not installed. Using fallback HTML parser.", flush=True)
 
try:
    from curl_cffi import requests
    HAS_CURL_CFI = True
except ImportError:
    HAS_CURL_CFI = False
    print("WARNING: curl_cffi not installed. Using standard requests.", flush=True)
    import requests
 
# ==========================================
# CONFIGURATION (GLOBAL VARIABLES)
# ==========================================
 
# Gotify Configuration
GOTIFY_URL = "https://gotify.TLD/message"
GOTIFY_TOKEN = "g...k"
GOTIFY_PRIORITY = 5
 
# 4chan /news/ board
BOARD = "news"
 
# 4chan API endpoints (same as original script)
API_CATALOG_URL = "https://a.4cdn.org/{}/catalog.json"
THREAD_BASE_URL = "https://a.4cdn.org/{}/thread/{}.json"
IMAGE_BASE_URL = "https://i.4cdn.org/{}/{}{}"
 
# Proxy sources for rotation and failover (same as original script)
PROXY_SOURCES = [
    "https://raw.githubusercontent.com/proxygenerator1/ProxyGenerator/main/MostStable/http.txt"
]
 
# SQLite database path
DATABASE_PATH = "/mnt/docker-applications/scan-4chan-news/monitor.db"
 
# Database schema version
DB_SCHEMA_VERSION = 1
 
# Maximum retry attempts for failed requests
MAX_RETRY_ATTEMPTS = 3
 
# Database lock for thread safety
_db_lock = threading.Lock()
 
# ==========================================
# DATABASE FUNCTIONS
# ==========================================
 
def get_schema_version(conn):
    """Get the current schema version from the database."""
    try:
        cursor = conn.execute("SELECT value FROM config WHERE key = 'schema_version'")
        row = cursor.fetchone()
        return int(row[0]) if row else 0
    except:
        return 0
 
def set_schema_version(conn, version):
    """Set the schema version in the database."""
    conn.execute(
        "INSERT OR REPLACE INTO config (key, value) VALUES ('schema_version', ?)",
        (str(version),)
    )
    conn.commit()
 
def get_db_schema():
    """Return the current database schema as a dictionary of SQL statements."""
    return {
        'threads': """
            CREATE TABLE IF NOT EXISTS threads (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                thread_id TEXT UNIQUE NOT NULL,
                title TEXT,
                link TEXT,
                published_at INTEGER,
                content TEXT,
                image_url TEXT,
                notified_at INTEGER DEFAULT (strftime('%s', 'now')),
                created_at INTEGER DEFAULT (strftime('%s', 'now'))
            )
        """,
        'threads_indexes': [
            "CREATE INDEX IF NOT EXISTS idx_threads_thread_id ON threads(thread_id)",
            "CREATE INDEX IF NOT EXISTS idx_threads_published_at ON threads(published_at)",
            "CREATE INDEX IF NOT EXISTS idx_threads_notified_at ON threads(notified_at)"
        ],
        'proxy_cache': """
            CREATE TABLE IF NOT EXISTS proxy_cache (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                proxy_url TEXT UNIQUE,
                success_count INTEGER DEFAULT 0,
                fail_count INTEGER DEFAULT 0,
                last_used INTEGER,
                last_success INTEGER,
                response_time REAL,
                is_working BOOLEAN DEFAULT 1
            )
        """,
        'config': """
            CREATE TABLE IF NOT EXISTS config (
                key TEXT PRIMARY KEY,
                value TEXT,
                updated_at INTEGER DEFAULT (strftime('%s', 'now'))
            )
        """
    }
 
def check_and_migrate_database():
    """Check database schema version and migrate/drop if needed."""
    print(f"[DB] Checking database schema at: {DATABASE_PATH}", flush=True)
 
    db_dir = os.path.dirname(DATABASE_PATH)
    if not os.path.exists(db_dir):
        print(f"[DB] Creating database directory: {db_dir}", flush=True)
        os.makedirs(db_dir, exist_ok=True)
 
    db_exists = os.path.exists(DATABASE_PATH)
 
    if db_exists:
        try:
            conn = sqlite3.connect(DATABASE_PATH, timeout=30)
            conn.row_factory = sqlite3.Row
            current_version = get_schema_version(conn)
            conn.close()
 
            print(f"[DB] Current schema version: {current_version}", flush=True)
            print(f"[DB] Required schema version: {DB_SCHEMA_VERSION}", flush=True)
 
            if current_version != DB_SCHEMA_VERSION:
                print(f"[DB] Schema version mismatch. Recreating database...", flush=True)
                recreate_database()
                return
            else:
                print(f"[DB] Schema version matches. Database ready.", flush=True)
                return
        except Exception as e:
            print(f"[DB] Error checking schema: {e}", flush=True)
            print(f"[DB] Recreating database...", flush=True)
            recreate_database()
            return
    else:
        print(f"[DB] Database does not exist. Creating new database...", flush=True)
        recreate_database()
 
def recreate_database():
    """Recreate the database with current schema."""
    try:
        if os.path.exists(DATABASE_PATH):
            os.remove(DATABASE_PATH)
            print(f"[DB] Removed old database file", flush=True)
 
        print(f"[DB] Creating new database schema...", flush=True)
 
        conn = sqlite3.connect(DATABASE_PATH, timeout=30)
        conn.row_factory = sqlite3.Row
 
        conn.execute("PRAGMA journal_mode=WAL")
        conn.execute("PRAGMA synchronous=NORMAL")
        conn.execute("PRAGMA cache_size=-65536")
 
        schema = get_db_schema()
        tables = [t for t in schema.keys() if t != 'threads_indexes']
        print(f"[DB] Creating {len(tables)} tables...", flush=True)
 
        for table_name in tables:
            print(f"[DB]   Creating table: {table_name}", flush=True)
            conn.execute(schema[table_name])
 
        print(f"[DB] Creating {len(schema['threads_indexes'])} indexes...", flush=True)
        for idx, index_sql in enumerate(schema['threads_indexes']):
            print(f"[DB]   Index {idx+1}/{len(schema['threads_indexes'])}", flush=True)
            conn.execute(index_sql)
 
        print(f"[DB] Setting schema version to {DB_SCHEMA_VERSION}...", flush=True)
        set_schema_version(conn, DB_SCHEMA_VERSION)
 
        conn.commit()
        conn.close()
 
        print(f"[DB] Database created successfully with schema version {DB_SCHEMA_VERSION}", flush=True)
 
    except Exception as e:
        print(f"[DB] Error recreating database: {e}", flush=True)
        raise
 
@contextmanager
def get_db():
    """Get database connection with WAL mode and proper locking."""
    with _db_lock:
        conn = sqlite3.connect(DATABASE_PATH, timeout=30, isolation_level=None)
        conn.execute("PRAGMA journal_mode=WAL")
        conn.execute("PRAGMA synchronous=NORMAL")
        conn.execute("PRAGMA cache_size=-65536")
        conn.execute("PRAGMA foreign_keys=ON")
        conn.execute("PRAGMA temp_store=MEMORY")
        conn.row_factory = sqlite3.Row
 
        try:
            yield conn
            conn.commit()
        except Exception as e:
            conn.rollback()
            raise e
        finally:
            conn.close()
 
def is_thread_notified(thread_id):
    """Check if a thread has already been notified."""
    try:
        with get_db() as conn:
            cursor = conn.execute(
                "SELECT 1 FROM threads WHERE thread_id = ? LIMIT 1",
                (str(thread_id),)
            )
            return cursor.fetchone() is not None
    except:
        return False
 
def mark_thread_notified(thread_id, title, link, published_at, content, image_url):
    """Mark a thread as notified in the database."""
    try:
        with get_db() as conn:
            conn.execute("""
                INSERT OR REPLACE INTO threads
                (thread_id, title, link, published_at, content, image_url)
                VALUES (?, ?, ?, ?, ?, ?)
            """, (str(thread_id), title, link, published_at, content, image_url))
            conn.commit()
            return True
    except Exception as e:
        print(f"[DB WARNING] Could not mark thread as notified: {e}", flush=True)
        return False
 
def get_last_published_time():
    """Get the last published timestamp from the database."""
    try:
        with get_db() as conn:
            cursor = conn.execute(
                "SELECT MAX(published_at) as last_time FROM threads"
            )
            row = cursor.fetchone()
            return row['last_time'] if row and row['last_time'] else 0
    except:
        return 0
 
# ==========================================
# HTML/XML PARSER UTILITIES
# ==========================================
 
def extract_text_from_html(html_content):
    """Extract plain text from HTML content."""
    if not html_content:
        return "No content"
 
    if HAS_BS4:
        try:
            soup = BeautifulSoup(html_content, 'html.parser')
            for script in soup(["script", "style"]):
                script.decompose()
            text = soup.get_text(separator=' ', strip=True)
            text = re.sub(r'\s+', ' ', text).strip()
            return text if text else "No content"
        except Exception:
            pass
 
    text = re.sub(r'<[^>]+>', ' ', html_content)
    text = re.sub(r'\s+', ' ', text).strip()
    return text if text else "No content"
 
def clean_html_content(html_content, max_length=1000):
    """Clean HTML content and extract plain text with length limit."""
    text = extract_text_from_html(html_content)
    if len(text) > max_length:
        text = text[:max_length - 3] + "..."
    return text
 
def truncate_text(text, max_length=200):
    """Truncate text to a maximum length."""
    if len(text) > max_length:
        return text[:max_length - 3] + "..."
    return text
 
# ==========================================
# GOTIFY NOTIFICATION FUNCTIONS
# ==========================================
 
def send_gotify_notification(title, message, priority=GOTIFY_PRIORITY, image_url=None):
    """Send a notification to Gotify with Markdown support."""
    try:
        headers = {
            "X-Gotify-Key": GOTIFY_TOKEN,
            "Content-Type": "application/json"
        }
 
        payload = {
            "title": title,
            "message": message,
            "priority": priority,
            "extras": {
                "client::display": {
                    "contentType": "text/markdown"
                }
            }
        }
 
        # If image URL is provided, try to download and attach it
        if image_url:
            try:
                img_response = requests.get(image_url, timeout=10)
                if img_response.status_code == 200:
                    # Convert image to base64 for attachment
                    img_data = base64.b64encode(img_response.content).decode('utf-8')
                    # Determine content type
                    ext = os.path.splitext(image_url)[1].lower()
                    content_type = "image/jpeg"
                    if ext in ['.png']:
                        content_type = "image/png"
                    elif ext in ['.gif']:
                        content_type = "image/gif"
 
                    payload["attachments"] = [{
                        "name": f"image{ext}",
                        "data": img_data,
                        "content_type": content_type
                    }]
            except Exception as e:
                print(f"      [GOTIFY WARNING] Could not fetch image for attachment: {e}", flush=True)
 
        if HAS_CURL_CFI:
            response = requests.post(GOTIFY_URL, json=payload, headers=headers, timeout=10)
        else:
            response = requests.post(GOTIFY_URL, json=payload, headers=headers, timeout=10)
 
        if response.status_code == 200:
            print(f"      [GOTIFY] Notification sent successfully", flush=True)
            return True
        else:
            print(f"      [GOTIFY ERROR] Failed to send notification: {response.status_code}", flush=True)
            if response.text:
                print(f"      [GOTIFY ERROR] Response: {response.text[:200]}", flush=True)
            return False
 
    except Exception as e:
        print(f"      [GOTIFY ERROR] Could not send notification: {e}", flush=True)
        return False
 
# ==========================================
# PROXY FUNCTIONS (same as original script)
# ==========================================
 
def gather_proxy_pool():
    """Harvest proxies from configured sources."""
    print("[PROXY] Initializing proxy harvesting...", flush=True)
    proxies = []
    for url in PROXY_SOURCES:
        try:
            print(f"[PROXY] Fetching from: {url}", flush=True)
            if HAS_CURL_CFI:
                res = requests.get(url, timeout=8)
            else:
                res = requests.get(url, timeout=8)
            if res.status_code == 200:
                found = re.findall(r'[0-9]+(?:\.[0-9]+){3}:[0-9]+', res.text)
                proxies.extend(found)
                print(f"[PROXY] Found {len(found)} proxies from {url}", flush=True)
        except Exception as e:
            print(f"[PROXY] Error fetching from {url}: {e}", flush=True)
            continue
 
    unique_proxies = list(set(proxies))
    print(f"[PROXY] Total unique proxies: {len(unique_proxies)}", flush=True)
    return unique_proxies[:60]
 
def execute_request_with_failover(url, headers, proxy_pool, current_proxy_container):
    """Execute HTTP request with proxy failover support."""
    active_proxy = current_proxy_container[0]
 
    if active_proxy:
        try:
            if HAS_CURL_CFI:
                res = requests.get(url, impersonate="chrome120", headers=headers, proxies=active_proxy, timeout=12)
            else:
                res = requests.get(url, headers=headers, proxies=active_proxy, timeout=12)
            if res.status_code == 200:
                return res
        except Exception:
            pass
 
    for proxy in proxy_pool:
        proxy_config = {"http": f"http://{proxy}", "https": f"http://{proxy}"}
        try:
            if HAS_CURL_CFI:
                res = requests.get(url, impersonate="chrome120", headers=headers, proxies=proxy_config, timeout=8)
            else:
                res = requests.get(url, headers=headers, proxies=proxy_config, timeout=8)
            if res.status_code == 200:
                current_proxy_container[0] = proxy_config
                return res
        except Exception:
            continue
    return None
 
# ==========================================
# CORE MONITORING FUNCTIONS
# ==========================================
 
def fetch_catalog(board, proxy_pool, proxy_container):
    """Fetch the catalog for a board using the JSON API."""
    headers = {
        "Accept": "application/json, text/plain, */*",
        "Accept-Language": "en-US,en;q=0.9",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
    }
 
    catalog_url = API_CATALOG_URL.format(board)
    print(f"[API] Fetching catalog from: {catalog_url}", flush=True)
 
    response = execute_request_with_failover(catalog_url, headers, proxy_pool, proxy_container)
 
    if not response:
        print(f"[API ERROR] Failed to fetch catalog", flush=True)
        return None
 
    if response.status_code != 200:
        print(f"[API ERROR] HTTP {response.status_code}", flush=True)
        return None
 
    return response.json()
 
def get_thread_posts(board, thread_id, proxy_pool, proxy_container):
    """Fetch all posts from a thread using the JSON API."""
    headers = {
        "Accept": "application/json, text/plain, */*",
        "Accept-Language": "en-US,en;q=0.9",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
    }
 
    thread_url = THREAD_BASE_URL.format(board, thread_id)
    response = execute_request_with_failover(thread_url, headers, proxy_pool, proxy_container)
 
    if not response or response.status_code != 200:
        return None
 
    return response.json().get("posts", [])
 
def process_new_threads(catalog_data, proxy_pool, proxy_container):
    """Process new threads from the catalog and send notifications."""
    if not catalog_data:
        print("[INFO] No catalog data", flush=True)
        return 0
 
    # Extract all threads from catalog
    all_threads = []
    for page in catalog_data:
        for thread in page.get("threads", []):
            all_threads.append(thread)
 
    if not all_threads:
        print("[INFO] No threads found in catalog", flush=True)
        return 0
 
    print(f"[API] Found {len(all_threads)} threads in catalog", flush=True)
 
    # Sort by thread number (higher = newer)
    all_threads.sort(key=lambda x: x.get("no", 0), reverse=True)
 
    # Get last processed thread ID
    last_time = get_last_published_time()
    print(f"[DB] Last processed timestamp: {datetime.fromtimestamp(last_time).strftime('%Y-%m-%d %H:%M:%S') if last_time else 'Never'}", flush=True)
 
    new_threads = []
    for thread in all_threads:
        thread_id = str(thread.get("no", ""))
 
        if not thread_id:
            continue
 
        # Check if already notified
        if is_thread_notified(thread_id):
            continue
 
        # Get thread creation time (from first post)
        thread_posts = get_thread_posts(BOARD, thread_id, proxy_pool, proxy_container)
        if not thread_posts:
            continue
 
        first_post = thread_posts[0]
        published_at = first_post.get("time", int(time.time()))
 
        # Check if this is newer than our last processed
        if published_at > last_time or last_time == 0:
            new_threads.append({
                'thread_id': thread_id,
                'title': thread.get("sub", "No title"),
                'thread_data': thread,
                'first_post': first_post,
                'published_at': published_at
            })
 
    if not new_threads:
        print("[INFO] No new threads found", flush=True)
        return 0
 
    print(f"[INFO] Found {len(new_threads)} new threads to process", flush=True)
 
    # Process new threads (oldest first)
    new_threads.sort(key=lambda x: x.get('published_at', 0))
    notifications_sent = 0
 
    for thread_info in new_threads:
        thread_id = thread_info['thread_id']
        title = thread_info['title']
        first_post = thread_info['first_post']
        published_at = thread_info['published_at']
 
        # Get post content
        post_content = first_post.get("com", "No content")
        content_clean = clean_html_content(post_content, max_length=1000)
 
        # Get image if available
        tim_identifier = first_post.get("tim")
        ext_suffix = first_post.get("ext")
        image_url = None
        if tim_identifier and ext_suffix:
            image_url = IMAGE_BASE_URL.format(BOARD, tim_identifier, ext_suffix)
 
        # Create notification with Markdown
        notification_title = f"📰 /news/: {truncate_text(title, 80)}"
        notification_body = f"""**Title:** {title}
 
**Content:**
{content_clean}
 
**Thread:** [https://boards.4chan.org/{BOARD}/thread/{thread_id}](https://boards.4chan.org/{BOARD}/thread/{thread_id})
**Published:** {datetime.fromtimestamp(published_at).strftime('%Y-%m-%d %H:%M:%S')}
"""
 
        print(f"[PROCESS] Notifying about thread: {title[:50]}...", flush=True)
        success = send_gotify_notification(
            notification_title,
            notification_body,
            GOTIFY_PRIORITY,
            image_url
        )
 
        if success:
            mark_thread_notified(
                thread_id,
                title,
                f"https://boards.4chan.org/{BOARD}/thread/{thread_id}",
                published_at,
                post_content,
                image_url
            )
            notifications_sent += 1
            print(f"    -> [SUCCESS] Notification sent for thread {thread_id}", flush=True)
        else:
            print(f"    -> [ERROR] Failed to send notification for thread {thread_id}", flush=True)
 
        time.sleep(0.5)
 
    return notifications_sent
 
def run_monitor_pass(proxy_pool, proxy_container):
    """Run a single monitoring pass."""
    print(f"\n{'='*60}")
    print(f"STARTING /news/ MONITORING PASS - {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
    print(f"{'='*60}\n", flush=True)
 
    # Fetch catalog
    catalog_data = fetch_catalog(BOARD, proxy_pool, proxy_container)
 
    if not catalog_data:
        print("[ERROR] Could not fetch catalog", flush=True)
        return 0
 
    # Process new threads
    notifications_sent = process_new_threads(catalog_data, proxy_pool, proxy_container)
 
    print(f"\n{'='*60}")
    print(f"SUMMARY: Sent {notifications_sent} notifications")
    print(f"{'='*60}\n", flush=True)
 
    return notifications_sent
 
# ==========================================
# MAIN FUNCTION
# ==========================================
 
def main():
    """
    Main entry point for the /news/ board monitor.
    Runs once and terminates.
    """
    print("=== 4CHAN /news/ BOARD MONITOR (JSON API) ===", flush=True)
    print(f"[TIME]  {time.strftime('%Y-%m-%d %H:%M:%S')}", flush=True)
    print(f"[BOARD]: /{BOARD}/", flush=True)
    print(f"[GOTIFY URL]: {GOTIFY_URL}", flush=True)
    print("")
 
    if GOTIFY_URL == "https://gotify.example.com/message" or GOTIFY_TOKEN == "your-app-token-here":
        print("[ERROR] Please configure GOTIFY_URL and GOTIFY_TOKEN in the configuration section.", flush=True)
        print("[ERROR] Exiting...", flush=True)
        return 1
 
    print("[INIT] Setting up database...", flush=True)
    check_and_migrate_database()
 
    proxy_pool = gather_proxy_pool()
    proxy_container = [None]
 
    total_notifications = run_monitor_pass(proxy_pool, proxy_container)
 
    print(f"\n[COMPLETE] Monitoring pass finished. Sent {total_notifications} notifications.", flush=True)
    return 0
 
if __name__ == "__main__":
    try:
        sys.exit(main())
    except KeyboardInterrupt:
        print("\n[INFO]: Interrupted by user. Exiting gracefully...", flush=True)
        sys.exit(1)
    except Exception as e:
        print(f"\n[FATAL ERROR]: {e}", file=sys.stderr, flush=True)
        import traceback
        traceback.print_exc()
        sys.exit(1)