"""What counts as a "content page", for the Content-pages-only toggle.

Shared by both trees, and it must be: the parser (handler.py) tallies content
hits into each daily summary so the overview never has to scan events, and the
viewer classifies individual rows for the detail table and its export. They MUST
agree, so this lives at the repo root -- the parser imports it directly and the
viewer imports it via the sys.path insert in viewer/categories.py, exactly as
that module already borrows the parser's crawlers.py. deploy.sh ships it in the
Lambda zip; keep it there or the parser cannot classify.

A crawler hitting a site fetches three quite different things, and only one of
them is your writing:

  1. content     the page itself                     /blog/dog-bite-claims/
  2. furniture   assets and APIs the page needs      /js/main.js, /wp-json/...
  3. probing     someone looking for a way in        /.env, /.ssh/id_ed25519

Counting all three together overstates how much of your *content* is being
read. The toggle hides 2 and 3; turning it off shows everything. Nothing is
filtered out of S3 -- this only affects what the viewer counts and displays.

MATCHING IS ON PATH SEGMENTS, NOT SUBSTRINGS. `".env" in url` also matches
"/uploads/flyer.envelope.pdf", and `"secret" in url` matches a perfectly real
law-firm article at "/blog/the-secret-to-winning". Splitting the path and
testing whole segments avoids quietly deleting real traffic from the numbers.
"""

# --- 1. static assets, by extension ----------------------------------------
STATIC_EXTENSIONS = (
    ".js", ".mjs", ".cjs", ".css", ".map",                       # scripts, styles
    ".png", ".jpg", ".jpeg", ".gif", ".svg", ".ico",             # images
    ".webp", ".avif", ".bmp", ".tif", ".tiff",
    ".woff", ".woff2", ".ttf", ".otf", ".eot",                   # fonts
    ".mp4", ".webm", ".mov", ".mp3", ".wav", ".ogg",             # media
)

# --- 2. config / secret file extensions: scanner bait, never content -------
CONFIG_EXTENSIONS = (".yml", ".yaml", ".ini", ".bak", ".sql", ".env")

# --- 3. API endpoints (path contains this segment) -------------------------
API_SEGMENTS = ("wp-json",)

# --- 4. exact filenames that are instructions to robots, not content -------
NON_CONTENT_FILES = ("robots.txt",)

# --- 5. probe markers ------------------------------------------------------
# A path segment starting with any of these is someone rattling the handles.
PROBE_SEGMENT_PREFIXES = (
    ".env",              # /.env, /.env.bak, /.env.local
    ".git",              # /.git/config, /.gitignore
    "wp-config",         # /wp-config.php, /wp-config.php.bak
    "service-account",   # /service-account.json
)
# A path segment exactly equal to any of these.
PROBE_SEGMENTS_EXACT = (
    ".ssh", ".aws", ".svn", ".vscode", ".idea",
    "secrets", "secret", "credentials",
    "id_rsa", "id_dsa", "id_ecdsa", "id_ed25519",
)
# A FILENAME (last segment, and only when it has an extension) starting with
# one of these: catches secrets.yml / secret.json without eating an article
# called "secrets-of-a-good-claim".
PROBE_FILENAME_PREFIXES = ("secret", "credential")


def _path(url):
    """The path part of a request, lowercased, without query or fragment."""
    return (url or "").split("?")[0].split("#")[0].lower()


def classify(url):
    """Why a request is not content: 'asset' | 'api' | 'robots' | 'probe',
    or None when it IS content."""
    path = _path(url)
    segments = [s for s in path.split("/") if s]
    last = segments[-1] if segments else ""

    if last in NON_CONTENT_FILES:
        return "robots"

    for seg in segments:
        if seg in PROBE_SEGMENTS_EXACT:
            return "probe"
        if seg.startswith(PROBE_SEGMENT_PREFIXES):
            return "probe"
        if seg in API_SEGMENTS:
            return "api"

    if "." in last:
        if last.endswith(CONFIG_EXTENSIONS):
            return "probe"
        if last.startswith(PROBE_FILENAME_PREFIXES):
            return "probe"
        if last.endswith(STATIC_EXTENSIONS):
            return "asset"

    return None


def is_content(url):
    """True when this request is for an actual page."""
    return classify(url) is None


def pattern_summary():
    """The filter, as data, so the UI and the docs cannot drift from the code."""
    return [
        ("Static assets", "file extension", list(STATIC_EXTENSIONS)),
        ("Config / secret files", "file extension", list(CONFIG_EXTENSIONS)),
        ("API endpoints", "path segment", ["/%s/" % s for s in API_SEGMENTS]),
        ("Robots instructions", "filename", list(NON_CONTENT_FILES)),
        ("Scanner probes", "path segment starts with",
         list(PROBE_SEGMENT_PREFIXES)),
        ("Scanner probes", "path segment equals", list(PROBE_SEGMENTS_EXACT)),
        ("Scanner probes", "filename starts with",
         [p + ".*" for p in PROBE_FILENAME_PREFIXES]),
    ]