#!/usr/bin/env python3
"""Audyt http:// -> https:// w polach source/sources/url receptur. Wersja POPRAWNA.

Poprzednia wersja miala blad: sondowala URL http:// i wynik zapisywala jako wynik
https. Tutaj jawnie sondowane sa OBA warianty, przez curl (ten sam klient, ktorego
uzywa pagbuild), z cache i zapisem przyrostowym.

Klasyfikacja kazdego wystapienia:
  SAFE_HTTPS   - https dziala, a tresc wyglada na ta sama co http  -> podmienic
  SAFE_HTTPS_UNVERIFIED - https dziala, ale http nie (nie ma z czym porownac)
  CONTENT_DIFF - https dziala, ale inne ctype/clen niz http        -> NIE podmieniac automatycznie
  HTTP_ONLY    - tylko http dziala                                 -> NIE podmieniac
  BOTH_DEAD    - oba nie dzialaja                                  -> zrodlo martwe
  UNRESOLVED   - nie rozwiazano placeholderow
"""
import collections
import concurrent.futures as cf
import json
import os
import re
import subprocess
import sys
import threading

import yaml

R = "/var/lib/pagan-sync/recipes"
OUT = "/tmp/https-audit.json"
CACHE = "/tmp/curl-probe-cache.json"
KEYS = ("source", "sources", "url")
PH = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}")
UA = "curl/8.7.1"
FMT = "%{http_code}|%{size_download}|%{content_type}|%{url_effective}"

_lock = threading.Lock()
cache = {}


def as_list(v):
    if v is None:
        return []
    return v if isinstance(v, list) else [v]


def expand(s, d):
    missing = []

    def rep(m):
        k = m.group(1)
        if k in d and not isinstance(d[k], (dict, list)):
            return str(d[k])
        missing.append(k)
        return m.group(0)

    return PH.sub(rep, s), bool(missing)


def to_https(u):
    return "https://" + u[len("http://"):]


def curl_once(url, head, timeout=25):
    args = ["curl", "-sS", "-o", "/dev/null", "-L", "--max-time", str(timeout),
            "-A", UA, "-w", FMT]
    if head:
        args.append("-I")
    else:
        args += ["-r", "0-0"]
    args.append(url)
    p = subprocess.run(args, capture_output=True, text=True)
    out = (p.stdout or "").strip().splitlines()
    if not out:
        return {"status": 0, "ok": False, "curl_rc": p.returncode, "err": (p.stderr or "").strip()[:120]}
    parts = out[-1].split("|")
    if len(parts) != 4:
        return {"status": 0, "ok": False, "curl_rc": p.returncode, "err": "bad-format"}
    code = int(parts[0]) if parts[0].isdigit() else 0
    return {
        "status": code,
        "ok": 200 <= code < 400,
        "size": parts[1],
        "ctype": parts[2],
        "final": parts[3],
        "curl_rc": p.returncode,
    }


def probe(url):
    """HEAD, potem (jesli HEAD odrzucony) GET z Range 0-0."""
    r = curl_once(url, head=True)
    if r["ok"]:
        return r
    if r["status"] in (400, 403, 405, 501) or r["status"] == 0:
        r2 = curl_once(url, head=False)
        if r2["ok"]:
            return r2
        if r2["status"] and r2["status"] not in (0,):
            return r2
        # zostaw wersje z HEAD, jesli GET tez padl (zwykle URLError/TLS)
        return r2 if r2["status"] else r
    return r


def cached_probe(url):
    with _lock:
        if url in cache:
            return cache[url]
    res = probe(url)
    with _lock:
        cache[url] = res
    return res


def save_cache():
    with _lock:
        snap = dict(cache)
    tmp = CACHE + ".tmp"
    json.dump(snap, open(tmp, "w"))
    os.replace(tmp, CACHE)


def main():
    global cache
    if os.path.exists(CACHE):
        try:
            cache = json.load(open(CACHE))
        except Exception:
            cache = {}

    occ = []
    for dp, _d, fs in os.walk(R):
        if "PAGBUILD.yaml" not in fs:
            continue
        p = os.path.join(dp, "PAGBUILD.yaml")
        rel = p[len(R) + 1:]
        try:
            d = yaml.safe_load(open(p, encoding="utf-8")) or {}
        except Exception:
            continue
        for k in KEYS:
            for v in as_list(d.get(k)):
                s = str(v)
                if s.startswith("http://"):
                    ex, unres = expand(s, d)
                    occ.append((rel, k, s, ex, unres))

    http_urls = sorted({o[3] for o in occ if not o[4]})
    https_urls = [to_https(u) for u in http_urls]
    print(f"wystapien http://: {len(occ)} | unikalnych http URL: {len(http_urls)}", flush=True)

    todo_h = [u for u in http_urls if u not in cache]
    todo_s = [u for u in https_urls if u not in cache]
    print(f"do sondowania: http={len(todo_h)} https={len(todo_s)}", flush=True)

    def run_pass(urls, label):
        done = 0
        t = 0
        with cf.ThreadPoolExecutor(max_workers=12) as pool:
            futs = {pool.submit(cached_probe, u): u for u in urls}
            for fut in cf.as_completed(futs):
                done += 1
                if done % 50 == 0:
                    save_cache()
                    print(f"  [{label}] {done}/{len(urls)}", flush=True)
        save_cache()

    run_pass(todo_h, "http ")
    run_pass(todo_s, "https")

    out = collections.defaultdict(list)
    for rel, k, raw, ex, unres in occ:
        if unres:
            out["UNRESOLVED"].append({"recipe": rel, "key": k, "raw": raw,
                                      "expanded": ex, "detail": "placeholdery"})
            continue
        h = cache.get(ex, {})
        s = cache.get(to_https(ex), {})
        hok, sok = h.get("ok", False), s.get("ok", False)
        if sok and hok:
            same = (h.get("size") == s.get("size") and h.get("ctype") == s.get("ctype"))
            cls = "SAFE_HTTPS" if same else "CONTENT_DIFF"
        elif sok and not hok:
            cls = "SAFE_HTTPS_UNVERIFIED"
        elif hok and not sok:
            cls = "HTTP_ONLY"
        else:
            cls = "BOTH_DEAD"
        out[cls].append({"recipe": rel, "key": k, "raw": raw, "expanded": ex,
                         "http": h, "https": s})

    json.dump(out, open(OUT, "w"), indent=1, ensure_ascii=False)
    print()
    for cls in ("SAFE_HTTPS", "SAFE_HTTPS_UNVERIFIED", "CONTENT_DIFF", "HTTP_ONLY",
                "BOTH_DEAD", "UNRESOLVED"):
        print(f"{cls:24s}: {len(out.get(cls, []))}")
    print(f"\nzapisano {OUT}")


if __name__ == "__main__":
    sys.exit(main())