"""A small local viewer for the parsed AI-crawler data.

Start it with ./run.sh, then open http://127.0.0.1:8765

This binds to 127.0.0.1 only, so nothing outside this machine can reach it. The
raw events contain visitor IPs and browsed URLs, so every page requires login.
S3 is read server-side with your AWS credentials; the browser never talks to
AWS and never sees a credential.

Read-only by construction: this app only ever calls GetObject/ListObjectsV2.
"""

import csv
import functools
import hmac
import io
import math
import os
import secrets
import threading
from datetime import date, timedelta

from flask import (Flask, Response, redirect, render_template, request, session,
                   url_for)

import s3data
import sites
from categories import (AI_GROUPS, ALL_GROUPS, GROUP_INFO, GROUP_LABELS,
                        bot_info, category_for, platforms_by_group)

PER_PAGE = 100

# Landing range when nothing is asked for. The overview now reads the daily
# summaries for any range (the parser precomputes the content-only counts too),
# so a wide range no longer means a slow first paint -- see s3data and
# handler.rebuild_summary. Latest Day stays the default as the most useful first
# scope: the newest complete day, loaded from a single summary file. Wider ranges
# are one dropdown away and open just as fast.
DEFAULT_PRESET = "latest_day"

PRESETS = [
    ("latest_day", "Latest Day"),
    ("this_week", "This Week"),
    ("this_month", "This Month"),
    ("last_month", "Last Month"),
    ("last_90", "Last 90 Days"),
    ("custom", "Custom Date Range"),
]


def load_env(path):
    """Minimal .env reader -- avoids a dependency for six lines of parsing."""
    if not os.path.exists(path):
        return
    with open(path) as fh:
        for line in fh:
            line = line.strip()
            if not line or line.startswith("#") or "=" not in line:
                continue
            key, val = line.split("=", 1)
            os.environ.setdefault(key.strip(), val.strip().strip('"').strip("'"))


load_env(os.path.join(os.path.dirname(os.path.abspath(__file__)), ".env"))

app = Flask(__name__)
app.secret_key = os.environ.get("SECRET_KEY") or secrets.token_hex(32)

USERNAME = os.environ.get("VIEWER_USER", "")
PASSWORD = os.environ.get("VIEWER_PASSWORD", "")


def login_required(fn):
    @functools.wraps(fn)
    def wrapper(*a, **kw):
        if not session.get("user"):
            return redirect(url_for("login", next=request.full_path))
        return fn(*a, **kw)
    return wrapper


@app.route("/login", methods=["GET", "POST"])
def login():
    error = None
    if request.method == "POST":
        user = request.form.get("username", "")
        pw = request.form.get("password", "")
        ok = (hmac.compare_digest(user, USERNAME)
              and hmac.compare_digest(pw, PASSWORD))
        if ok and USERNAME and PASSWORD:
            session["user"] = user
            dest = request.args.get("next") or url_for("overview")
            if not dest.startswith("/"):
                dest = url_for("overview")
            return redirect(dest)
        error = "Wrong username or password."
    return render_template("login.html", error=error)


@app.route("/logout")
def logout():
    session.clear()
    return redirect(url_for("login"))


# --- date range -------------------------------------------------------------

def _today():
    """'Today' means the newest day we have data for, not the wall clock.

    The parser runs nightly, so the calendar is usually ahead of the data. A
    preset anchored on the wall clock would show an empty page every morning.
    """
    dates = s3data.available_dates()
    return date.fromisoformat(dates[0]) if dates else date.today()


def _preset_range(preset, anchor):
    """(start, end) for a named preset, as dates."""
    if preset == "latest_day":                      # just the newest day we hold
        return anchor, anchor
    if preset == "this_week":                       # Monday -> anchor
        return anchor - timedelta(days=anchor.weekday()), anchor
    if preset == "this_month":
        return anchor.replace(day=1), anchor
    if preset == "last_month":
        first_this = anchor.replace(day=1)
        last_prev = first_this - timedelta(days=1)
        return last_prev.replace(day=1), last_prev
    if preset == "last_90":
        return anchor - timedelta(days=89), anchor
    return None


def _selected_range():
    """(preset, start, end, days) from the query string.

    `days` only ever contains dates we actually hold data for, so a preset that
    reaches back before the data starts simply yields fewer days.
    """
    dates = s3data.available_dates()
    if not dates:
        return DEFAULT_PRESET, None, None, []

    preset = request.args.get("preset", "")
    if preset not in dict(PRESETS):
        # No preset given: honour explicit dates as a custom range, else default.
        preset = "custom" if (request.args.get("start") or request.args.get("end")) \
            else DEFAULT_PRESET

    if preset == "custom":
        start = request.args.get("start")
        end = request.args.get("end") or dates[0]
        if not start:
            # Default custom window is 7 days ending on the newest day with data
            # (6 days back, inclusive). Anchored on the data, not the wall clock,
            # for the same reason the presets are -- see _today. Clamped so it
            # never starts before the data does.
            try:
                anchor = date.fromisoformat(end)
            except ValueError:
                anchor = date.fromisoformat(dates[0])
            earliest = date.fromisoformat(dates[-1])
            start = max(anchor - timedelta(days=6), earliest).isoformat()
    else:
        lo, hi = _preset_range(preset, _today())
        start, end = lo.isoformat(), hi.isoformat()

    return preset, start, end, s3data.dates_in_range(start, end)


def _groups_from_request():
    """Which groups to show. Default: all AI, no conventional search."""
    if request.args.get("include_search") == "1":
        return list(ALL_GROUPS)
    return list(AI_GROUPS)


def _content_only():
    """Default ON: hide assets, APIs, robots.txt and scanner probes.

    An unticked checkbox sends nothing at all, which is indistinguishable from
    a fresh page load -- and this defaults ON, so "sent nothing" must not mean
    "on". The filter form pairs the checkbox with a hidden content=0 that comes
    first, so an unticked box submits just "0" while a ticked one submits
    "0","1". Hence: read the LAST value, not the first (request.args.get would
    return the "0" and the box would never turn on).
    """
    values = request.args.getlist("content")
    return (values[-1] if values else "1") != "0"


def _resolve_site(site, days):
    """Accept either an install (canonical) or a vhost (older links).

    `site=` used to mean the vhost, so bookmarked and shared links carry things
    like site=allmandlaw.com. Those now match no install, and a filter that
    matches nothing renders a confident, wrong "0 hits". Map a known vhost onto
    its install instead. An unrecognised value is passed through untouched so it
    still filters to nothing rather than silently widening to everything.
    """
    if not site or site in set(s3data.tracked_installs()):
        return site
    return s3data.domain_to_install(days).get(site, site)


def _filters():
    """Everything the two pages and the export agree on."""
    preset, start, end, days = _selected_range()
    groups = _groups_from_request()
    return {
        "preset": preset, "start": start, "end": end, "days": days,
        "groups": groups, "include_search": "search" in groups,
        "content_only": _content_only(),
        "site": _resolve_site(request.args.get("site", "").strip(), days),
        "platform": request.args.get("platform", "").strip(),
        "url_query": request.args.get("q", "").strip(),
    }


def _shared(f):
    """Template vars every page needs."""
    return {
        "dates": s3data.available_dates(), "presets": PRESETS,
        "group_labels": GROUP_LABELS, "group_info": GROUP_INFO,
        "all_groups": ALL_GROUPS, "bot_info": bot_info,
        "preset": f["preset"], "start": f["start"], "end": f["end"],
        "days": f["days"], "include_search": f["include_search"],
        "content_only": f["content_only"],
    }


# --- pages ------------------------------------------------------------------

@app.route("/")
@login_required
def overview():
    f = _filters()
    groups = f["groups"]
    docs = s3data.summaries(f["days"])

    # Classify each bot ONCE, here, and use that single answer everywhere on
    # the page. Previously the by_site rollup re-derived the group without the
    # stored-category fallback, so an unknown bot could land in one category in
    # the stat boxes and a different one in the site table -- the same hit
    # counted under two categories. See test_viewer.py.
    plat_group = {}
    for _, doc in docs:
        for plat, info in doc.get("by_platform", {}).items():
            if plat not in plat_group:
                plat_group[plat] = category_for(plat, info.get("category"))

    by_platform = {}
    by_install = {}
    content_mode = f["content_only"]

    # The overview reads only the small daily summaries, never the event shards.
    # The parser precomputes BOTH an all-hits count and a content-only count into
    # each summary (see handler.rebuild_summary), so the content filter no longer
    # forces a live scan -- picking "This Month" was ~6,600 S3 reads and minutes
    # of hang; now it is a dozen tiny files. A day whose summary predates those
    # fields (parsed before this change and not yet rebuilt) cannot answer the
    # content filter, so count ITS events -- but only its. This used to fall back
    # to scanning the whole range: one stale day made every Apply on a wide range
    # take minutes, which is exactly what happened when the deployed parser
    # lagged this code and kept writing old-format summaries nightly.
    stale_days = [d for d, doc in docs
                  if content_mode and "content_hits" not in doc.get("totals", {})]
    if stale_days:
        # Older summaries cannot say which hits were real pages: count events.
        plats, installs = s3data.rollup(stale_days, groups=groups, content_only=True)
        for plat, hits in plats.items():
            grp = plat_group.get(plat) or category_for(plat)
            by_platform[plat] = {"hits": hits, "group": grp}
        by_install = dict(installs)

    # The summaries are keyed by vhost, so fold each one onto its install
    # before counting -- otherwise buckfirelaw.com and buckfirelaw.wpengine.com
    # are two sites. A vhost we have no mapping for keeps its own name rather
    # than being dropped. content_mode just swaps in the content-only counts.
    stale = set(stale_days)
    plat_field = "content_hits" if content_mode else "hits"
    site_field = "by_platform_content" if content_mode else "by_platform"
    to_install = s3data.domain_to_install(f["days"])
    for day, doc in docs:
        if day in stale:
            continue  # already counted from its events above
        for plat, info in doc.get("by_platform", {}).items():
            grp = plat_group[plat]
            if grp not in groups:
                continue
            row = by_platform.setdefault(plat, {"hits": 0, "group": grp})
            row["hits"] += info.get(plat_field, 0)
        for site, info in doc.get("by_site", {}).items():
            install = to_install.get(site, site)
            for plat, hits in info.get(site_field, {}).items():
                grp = plat_group.get(plat) or category_for(plat)
                if grp not in groups:
                    continue
                by_install[install] = by_install.get(install, 0) + hits
    # A bot with hits but no *content* hits (all assets/probes) leaves a 0 row
    # here; the old event scan simply never emitted it. Drop the zeros so the
    # platform table and its "distinct bots" count match what the scan showed.
    if content_mode:
        by_platform = {p: r for p, r in by_platform.items() if r["hits"]}
        by_install = {i: h for i, h in by_install.items() if h}

    # Total every bot into the group boxes the page will actually render, and
    # keep anything that lands elsewhere instead of dropping it. Hits is then
    # the sum of exactly those boxes, so the headline cannot disagree with them.
    group_totals = {g: 0 for g in groups}
    stranded = {}
    for plat, row in by_platform.items():
        if row["group"] in group_totals:
            group_totals[row["group"]] += row["hits"]
        else:
            # A bot resolving outside the displayed groups would be counted in
            # the total but shown in no box -- the hits would silently vanish.
            # Surface it on the page instead.
            stranded[plat] = row
    total_hits = sum(group_totals.values())

    # Every install we hold logs for, not just the ones with hits. The union with
    # by_install is belt-and-braces: an install can only have events if it had a
    # log file, so it should already be in the manifest -- but if the manifest is
    # ever behind, this keeps the total a real superset instead of letting the
    # headline claim more sites with hits than sites tracked.
    tracked = set(s3data.tracked_installs()) | set(by_install)
    domains = s3data.install_domains(f["days"])
    site_rows = [
        {
            "name": sites.display_name(i, domains.get(i, set())),
            "hits": by_install.get(i, 0),
            "extra": sites.extra_domain_count(i, domains.get(i, set())),
            "install": i,
        }
        for i in tracked
    ]
    # Zero-hit sites sort last by construction; name breaks ties so the order is
    # stable rather than set-iteration order.
    site_rows.sort(key=lambda r: (-r["hits"], r["name"]))

    return render_template(
        "overview.html",
        totals={"hits": total_hits},
        platform_rows=sorted(by_platform.items(), key=lambda kv: -kv[1]["hits"]),
        site_rows=site_rows,
        sites_with_hits=sum(1 for r in site_rows if r["hits"]),
        sites_tracked=len(tracked),
        group_totals=group_totals, stranded=stranded,
        **_shared(f)
    )


@app.route("/detail")
@login_required
def detail():
    f = _filters()
    try:
        page = max(1, int(request.args.get("page", 1)))
    except ValueError:
        page = 1

    rows, total, by_group, _ = s3data.query(
        f["days"], sites=[f["site"]] if f["site"] else None,
        platforms=[f["platform"]] if f["platform"] else None,
        groups=f["groups"], url_query=f["url_query"],
        content_only=f["content_only"],
    )

    pages = max(1, math.ceil(total / PER_PAGE))
    page = min(page, pages)

    return render_template(
        "detail.html",
        rows=rows[(page - 1) * PER_PAGE: page * PER_PAGE],
        total=total, page=page, pages=pages, by_group=by_group,
        site=f["site"], platform=f["platform"], url_query=f["url_query"],
        # Only the dates in view; keep the current pick listed even if it has no
        # data in range, so the dropdown never silently drops your filter.
        all_sites=sorted(set(s3data.sites_for(f["days"])) | ({f["site"]} if f["site"] else set())),
        platforms_by_group=platforms_by_group(),
        cache=s3data.cache_stats(),
        **_shared(f)
    )


@app.route("/trend")
@login_required
def trend():
    """AI access to a URL (or URL pattern) over time, one point per day.

    Served entirely from the per-day urlagg files -- no event shards -- so a
    90-day range costs ~90 small gzipped reads the first time and nothing after.
    An empty URL query is deliberately valid: it charts all matching traffic,
    which makes the page double as a site-wide trend view.
    """
    f = _filters()
    per_day, top_urls, by_platform, missing = s3data.trend(
        f["days"], url_query=f["url_query"],
        sites=[f["site"]] if f["site"] else None,
        platforms=[f["platform"]] if f["platform"] else None,
        groups=f["groups"], content_only=f["content_only"],
    )
    total = sum(hits for _, hits in per_day)
    return render_template(
        "trend.html",
        per_day=per_day, total=total,
        peak=max((hits for _, hits in per_day), default=0),
        top_urls=top_urls,
        platform_rows=sorted(by_platform.items(), key=lambda kv: -kv[1]),
        missing=missing,
        site=f["site"], platform=f["platform"], url_query=f["url_query"],
        all_sites=sorted(set(s3data.sites_for(f["days"])) | ({f["site"]} if f["site"] else set())),
        platforms_by_group=platforms_by_group(),
        **_shared(f)
    )


@app.route("/export.csv")
@login_required
def export_csv():
    """The currently-filtered rows, all of them -- not just this page."""
    f = _filters()
    rows, total, _, _ = s3data.query(
        f["days"], sites=[f["site"]] if f["site"] else None,
        platforms=[f["platform"]] if f["platform"] else None,
        groups=f["groups"], url_query=f["url_query"],
        content_only=f["content_only"],
    )

    buf = io.StringIO()
    w = csv.writer(buf)
    w.writerow(["timestamp_utc", "site", "ai_platform", "category", "url", "status_code"])
    for r in rows:
        w.writerow([r[s3data.TS], r[s3data.SITE], r[s3data.PLATFORM],
                    GROUP_LABELS.get(r[s3data.GROUP], r[s3data.GROUP]),
                    r[s3data.URL], r[s3data.STATUS]])

    bits = ["ai-crawler-hits", f["start"] or "all"]
    if f["end"] and f["end"] != f["start"]:
        bits.append("to-" + f["end"])
    if f["site"]:
        bits.append(f["site"])
    if f["platform"]:
        bits.append(f["platform"])
    if not f["include_search"]:
        bits.append("ai-only")
    if f["content_only"]:
        bits.append("content-pages")
    name = "-".join(bits) + ".csv"

    # utf-8-sig so Excel opens accented URLs correctly on a double-click.
    return Response(
        buf.getvalue().encode("utf-8-sig"),
        mimetype="text/csv",
        headers={"Content-Disposition": 'attachment; filename="%s"' % name,
                 "X-Row-Count": str(total)},
    )


# --- template helpers -------------------------------------------------------

@app.template_filter("commafy")
def commafy(n):
    try:
        return "{:,}".format(int(n))
    except (TypeError, ValueError):
        return n


@app.template_filter("prettytime")
def prettytime(ts):
    return (ts or "").replace("T", " ").replace("Z", "")


@app.template_global("keep")
def keep(**over):
    """Current filters as a query dict, with overrides. Keeps links consistent."""
    f = _filters()
    out = {"preset": f["preset"], "start": f["start"], "end": f["end"],
           "site": f["site"], "platform": f["platform"], "q": f["url_query"],
           "include_search": "1" if f["include_search"] else "",
           "content": "" if f["content_only"] else "0"}
    out.update(over)
    return {k: v for k, v in out.items() if v not in (None, "")}


def _prewarm():
    """Load the default range into cache in the background at startup.

    "This week" is a few days, which is ~30s of S3 reads the first time. Doing
    it here means that cost lands while you are still typing your password,
    rather than on your first click into the detail table. Best-effort only: if
    it fails, the page just loads it on demand as before.
    """
    try:
        anchor = _today()
        lo, hi = _preset_range(DEFAULT_PRESET, anchor)
        days = s3data.dates_in_range(lo.isoformat(), hi.isoformat())
        s3data.summaries(days)
        s3data.sites_for(days)
        s3data.query(days, groups=list(AI_GROUPS), content_only=True)
        print("  (cached %s, ready)" % DEFAULT_PRESET.replace("_", " "))
        # The trend page reads one small aggregate per day, but a cold 90-day
        # range is still ~45s of fetch+parse. Pay that here, in the background,
        # so the widest range the UI offers opens warm. Runs after the default
        # range above on purpose: the landing page wins the race for the pool.
        trend_days = s3data.available_dates()[:90]
        s3data.trend(trend_days)
        print("  (trend cache ready, %d days)" % len(trend_days))
    except Exception as exc:
        print("  (prewarm skipped: %s)" % exc)


def main():
    if not USERNAME or not PASSWORD:
        raise SystemExit(
            "No login configured.\n"
            "Copy viewer/.env.example to viewer/.env and set VIEWER_USER and "
            "VIEWER_PASSWORD, then start again."
        )
    ok, msg = s3data.check_access()
    if not ok:
        raise SystemExit(
            "Cannot read s3://%s\n  %s\n"
            "Check your AWS CLI login (aws sts get-caller-identity)." % (s3data.BUCKET, msg)
        )
    port = int(os.environ.get("PORT", "8765"))
    print("\n  AI crawler viewer -> http://127.0.0.1:%d" % port)
    print("  Log in as: %s" % USERNAME)
    print("  Stop with Ctrl-C\n")
    threading.Thread(target=_prewarm, daemon=True).start()
    app.run(host="127.0.0.1", port=port, debug=False)


if __name__ == "__main__":
    main()