#!/usr/bin/env python3
"""
RT News Rumble Downloader
=========================
Scrapes the RT News Rumble channel page for the most recent individual videos
(excluding livestreams) and downloads them via yt-dlp.

Features:
- Auto-selects a working curl_cffi impersonation profile to defeat Cloudflare.
- Pre-resolves each video page to its Rumble embed URL using curl_cffi, so
  yt-dlp only talks to the embed endpoint (which is not as aggressively
  Cloudflare-gated as the main video page).
- Translates the winning profile to yt-dlp's naming scheme and rotates
  through profiles on failure, using --download-archive to skip succeeded
  videos on retry passes.
- Extracts recent /v<id>-<slug>.html links from Page 1 only.
- Excludes livestreams via URL filter and yt-dlp's !is_live match filter.
- Single outbound HTTP proxy configured by hostname and port.
- All yt-dlp embedding/metadata/subtitle options preserved.

Copyright (C) Wizardry and Steamworks 2024 - License: GNU GPLv3
"""

import os
import re
import subprocess
import sys
from hashlib import md5

from curl_cffi import requests

###########################################################################
#                            CONFIGURATION                                #
###########################################################################

PATH_PREFIX = "/mnt/archie/YouTube/Actual"
OUTPUT_PATH_RT = (
    "%(uploader)s/Season %(upload_date>%Y)s/"
    "%(uploader)s - S%(upload_date>%Y)sE%(upload_date>%Y%m%d)s - %(title)s.%(ext)s"
)
ARCHIVE_DIRECTORY = "/mnt/docker-applications/scratch/yt-dlp/archives/"
URL = "https://rumble.com/c/RTNews/videos"
CACHE_DIR = "/mnt/docker-applications/scratch/yt-dlp/cache"

###########################################################################
#                       PROXY CONFIGURATION                               #
###########################################################################
PROXY_HOST = "proxydrum"
PROXY_PORT = "8080"

SOCKET_TIMEOUT = 30
MAX_LINKS = 15
MAX_DOWNLOADS = 5

###########################################################################
#                    IMPERSONATION PROFILE CANDIDATES                     #
###########################################################################
IMPERSONATE_PROFILES = [
    "safari18_0",
    "safari17_2_ios",
    "safari17_0",
    "safari15_5",
    "firefox135",
    "firefox133",
    "chrome136",
    "chrome133a",
    "chrome131",
    "chrome124",
    "chrome123",
    "chrome120",
]

CURL_CFFI_TO_YTDLP = {
    "chrome110":      "Chrome-110",
    "chrome116":      "Chrome-116",
    "chrome119":      "Chrome-119",
    "chrome120":      "Chrome-120",
    "chrome123":      "Chrome-123",
    "chrome124":      "Chrome-124",
    "chrome131":      "Chrome-131",
    "chrome133a":     "Chrome-133",
    "chrome136":      "Chrome-136",
    "safari15_3":     "Safari-15.3",
    "safari15_5":     "Safari-15.5",
    "safari17_0":     "Safari-17.0",
    "safari17_2_ios": "Safari-17.2",
    "safari18_0":     "Safari-18.0",
    "firefox133":     "Firefox-133",
    "firefox135":     "Firefox-135",
    "edge99":         "Edge-99",
    "edge101":        "Edge-101",
}

if PROXY_HOST:
    PROXY_URL = f"http://{PROXY_HOST}:{PROXY_PORT}"
    PROXIES = {"http": PROXY_URL, "https": PROXY_URL}
    YTDLP_PROXY_ARGS = ["--proxy", PROXY_URL]
else:
    PROXY_URL = ""
    PROXIES = None
    YTDLP_PROXY_ARGS = []

URL_HASH = md5(URL.encode("utf-8")).hexdigest()
ARCHIVE_FILE = os.path.join(ARCHIVE_DIRECTORY, f"{URL_HASH}.txt")

###########################################################################
#                              SCRAPING                                   #
###########################################################################

VIDEO_LINK_PATTERN = re.compile(r"/v[a-z0-9]+-[a-zA-Z0-9._-]+(?:\.html)?")
EMBED_LINK_PATTERNS = [
    re.compile(r'"embedUrl"\s*:\s*"https?://rumble\.com/embed/(v[a-z0-9]+)/?"'),
    re.compile(r'rumble\.com/embed/(v[a-z0-9]+)'),
    re.compile(r'"video_id"\s*:\s*"?(v?[a-z0-9]{6,8})"?'),
]


def _looks_like_rumble_page(text):
    if not text:
        return False
    lower = text.lower()
    if "just a moment" in lower or "cf-chl" in lower or "challenges.cloudflare.com" in lower:
        return False
    if "rumble" in lower and ("followers" in lower or "/v" in text):
        return True
    return bool(VIDEO_LINK_PATTERN.search(text))


def fetch_with_profile(url, profile, proxies):
    """
    Fetch a URL with curl_cffi using a specific impersonate profile.
    Returns the response or None on error.
    """
    try:
        return requests.get(
            url,
            impersonate=profile,
            proxies=proxies,
            timeout=SOCKET_TIMEOUT,
        )
    except Exception as e:
        print(f"    [fetch error] {profile}: {str(e)[:80]}")
        return None


def fetch_rumble_page(target_url, proxies):
    """
    Try each impersonation profile until one returns a real Rumble page.
    Returns (html, first_profile, working_profiles).
    """
    print("Trying impersonation profiles against Cloudflare...")
    working_profiles = []
    first_html = None
    first_profile = None

    for profile in IMPERSONATE_PROFILES:
        r = fetch_with_profile(target_url, profile, proxies)
        if r is None:
            continue

        title = ""
        if "<title>" in r.text:
            title = r.text.split("<title>")[1].split("</title>")[0][:50]

        if r.status_code == 200 and _looks_like_rumble_page(r.text):
            print(f"  {profile:16s} -> HTTP {r.status_code}  {title}  [OK]")
            working_profiles.append(profile)
            if first_html is None:
                first_html = r.text
                first_profile = profile
        else:
            print(f"  {profile:16s} -> HTTP {r.status_code}  {title}")

    if first_html is None:
        print("[scrape error] All impersonation profiles were rejected.", file=sys.stderr)
        return None, None, []

    print(f"Selected impersonation profile: {first_profile}")
    print(f"Working profiles this run: {working_profiles}")
    return first_html, first_profile, working_profiles


def scrape_video_links(target_url, proxies):
    html, _first, working = fetch_rumble_page(target_url, proxies)
    if html is None:
        return [], []

    seen = []
    for match in VIDEO_LINK_PATTERN.findall(html):
        if "livestream" in match.lower():
            continue
        if match not in seen:
            seen.append(match)
        if len(seen) >= MAX_LINKS:
            break

    return seen, working


def resolve_embed_url(page_url, working_profiles, proxies):
    """
    Fetch a Rumble video page and extract its embed URL.

    Returns an embed URL (https://rumble.com/embed/vXXXXXX) on success,
    or None if the embed ID could not be extracted.
    """
    for profile in working_profiles:
        r = fetch_with_profile(page_url, profile, proxies)
        if r is None or r.status_code != 200:
            continue
        if not _looks_like_rumble_page(r.text):
            continue

        for pat in EMBED_LINK_PATTERNS:
            m = pat.search(r.text)
            if m:
                embed_id = m.group(1)
                if not embed_id.startswith("v"):
                    embed_id = "v" + embed_id
                return f"https://rumble.com/embed/{embed_id}"

    return None


###########################################################################
#                              DOWNLOAD                                   #
###########################################################################

def _count_lines(path):
    if not os.path.exists(path):
        return 0
    try:
        with open(path, "r", encoding="utf-8", errors="ignore") as f:
            return sum(1 for _ in f)
    except Exception:
        return 0


def run_ytdlp_once(urls, ytdlp_profile):
    """Run yt-dlp once with the given yt-dlp profile over the given URLs."""
    input_data = "\n".join(urls)

    cmd = [
        "yt-dlp",
        *YTDLP_PROXY_ARGS,
        "--socket-timeout", str(SOCKET_TIMEOUT),
        "--retries", "2",
        "--fragment-retries", "2",
        "--extractor-retries", "2",
        "--force-ipv4",
        "--match-filter", "!is_live",
        "--output-na-placeholder", "",
        "--impersonate", ytdlp_profile,
        "--print", "after_move:channel,title,filepath",
        "--cache-dir", CACHE_DIR,
        "--download-archive", ARCHIVE_FILE,
        "--output", os.path.join(PATH_PREFIX, OUTPUT_PATH_RT),
        "--max-downloads", str(MAX_DOWNLOADS),
        "--no-write-playlist-metafiles",
        "--write-info-json",
        "--write-subs",
        "--write-auto-sub",
        "--embed-subs",
        "--embed-metadata",
        "--embed-chapters",
        "--sponsorblock-mark", "all",
        "--sub-lang", "en",
        "--color", "never",
        "--ignore-errors",
        "-a", "-",
    ]

    try:
        result = subprocess.run(cmd, input=input_data, text=True)
        return result.returncode
    except FileNotFoundError:
        print("[fatal] yt-dlp not found in PATH.", file=sys.stderr)
        return 127
    except Exception as e:
        print(f"[fatal] Failed to run yt-dlp: {e}", file=sys.stderr)
        return 1


def build_rotation_list(working_profiles):
    rotation = []
    seen = set()
    for curl_name in working_profiles:
        ytdlp_name = CURL_CFFI_TO_YTDLP.get(curl_name)
        if ytdlp_name and ytdlp_name not in seen:
            rotation.append(ytdlp_name)
            seen.add(ytdlp_name)
    for curl_name in IMPERSONATE_PROFILES:
        ytdlp_name = CURL_CFFI_TO_YTDLP.get(curl_name)
        if ytdlp_name and ytdlp_name not in seen:
            rotation.append(ytdlp_name)
            seen.add(ytdlp_name)
    return rotation


def run_ytdlp(urls, working_profiles):
    rotation = build_rotation_list(working_profiles)
    if not rotation:
        print("[fatal] No yt-dlp impersonation profiles available.", file=sys.stderr)
        return 1

    print(f"yt-dlp will rotate through {len(rotation)} profile(s) on failure:")
    print(f"  {rotation}")

    last_rc = 0
    for attempt, profile in enumerate(rotation, start=1):
        before = _count_lines(ARCHIVE_FILE)
        print()
        print(f"=== yt-dlp attempt {attempt}/{len(rotation)}: impersonate={profile} ===")

        rc = run_ytdlp_once(urls, profile)
        last_rc = rc

        after = _count_lines(ARCHIVE_FILE)
        added = after - before
        print(f"    -> exit={rc}, {added} new video(s) added to archive (total: {after})")

        if added == 0:
            # No progress this pass; try the next profile
            continue

    print(f"[info] Finished rotation. Last exit code: {last_rc}")
    return last_rc


###########################################################################
#                              MAIN                                       #
###########################################################################

def main():
    if PROXY_URL:
        print(f"=== Using proxy: {PROXY_URL} (socket timeout: {SOCKET_TIMEOUT}s) ===")
    else:
        print("=== No proxy configured (direct connection) ===")

    print(f"Starting verbose download for: {URL}")
    print(f"Archive file: {ARCHIVE_FILE}")
    print("Scraping fresh video links from Rumble Page 1...")

    video_links, working_profiles = scrape_video_links(URL, PROXIES)
    if not video_links:
        print("Warning: No matching video links found on Rumble page 1.")
        print("Process completed.")
        return 0

    print(f"Extracted {len(video_links)} video link(s) from page 1.")

    # Pre-resolve each video page to its embed URL using curl_cffi. This
    # avoids handing yt-dlp the Cloudflare-protected main video page URL.
    print("Pre-resolving embed URLs via curl_cffi...")
    resolved_urls = []
    for link in video_links:
        page_url = f"https://rumble.com{link}"
        embed_url = resolve_embed_url(page_url, working_profiles, PROXIES)
        if embed_url:
            print(f"  {link} -> {embed_url}")
            resolved_urls.append(embed_url)
        else:
            print(f"  {link} -> [embed resolution failed, using page URL]")
            resolved_urls.append(page_url)

    if not resolved_urls:
        print("Warning: No URLs to download.")
        print("Process completed.")
        return 0

    print(f"Handing {len(resolved_urls)} URL(s) to yt-dlp.")
    run_ytdlp(resolved_urls, working_profiles)

    print("Process completed.")
    return 0


if __name__ == "__main__":
    try:
        sys.exit(main())
    except KeyboardInterrupt:
        print("\n[INFO] Interrupted by user. Exiting gracefully...", flush=True)
        sys.exit(1)
    except Exception as e:
        print(f"\n[FATAL ERROR] {e}", file=sys.stderr, flush=True)
        import traceback
        traceback.print_exc()
        sys.exit(1)