"""What counts as a "content page", for the Content-pages-only toggle.

A crawler hitting a site fetches three quite different things, and only one of
them is your writing:

  1. content     the page itself                     /blog/dog-bite-claims/
  2. furniture   assets and APIs the page needs      /js/main.js, /wp-json/...
  3. probing     someone looking for a way in        /.env, /.ssh/id_ed25519

Counting all three together overstates how much of your *content* is being
read. The toggle hides 2 and 3; turning it off shows everything. Nothing is
filtered out of S3 -- this only affects what the viewer counts and displays.

MATCHING IS ON PATH SEGMENTS, NOT SUBSTRINGS. `".env" in url` also matches
"/uploads/flyer.envelope.pdf", and `"secret" in url` matches a perfectly real
law-firm article at "/blog/the-secret-to-winning". Splitting the path and
testing whole segments avoids quietly deleting real traffic from the numbers.
"""

# --- 1. static assets, by extension ----------------------------------------
STATIC_EXTENSIONS = (
    ".js", ".mjs", ".cjs", ".css", ".map",                       # scripts, styles
    ".png", ".jpg", ".jpeg", ".gif", ".svg", ".ico",             # images
    ".webp", ".avif", ".bmp", ".tif", ".tiff",
    ".woff", ".woff2", ".ttf", ".otf", ".eot",                   # fonts
    ".mp4", ".webm", ".mov", ".mp3", ".wav", ".ogg",             # media
)

# --- 2. config / secret file extensions: scanner bait, never content -------
CONFIG_EXTENSIONS = (".yml", ".yaml", ".ini", ".bak", ".sql", ".env")

# --- 3. API endpoints (path contains this segment) -------------------------
API_SEGMENTS = ("wp-json",)

# --- 4. exact filenames that are instructions to robots, not content -------
NON_CONTENT_FILES = ("robots.txt",)

# --- 5. probe markers ------------------------------------------------------
# A path segment starting with any of these is someone rattling the handles.
PROBE_SEGMENT_PREFIXES = (
    ".env",              # /.env, /.env.bak, /.env.local
    ".git",              # /.git/config, /.gitignore
    "wp-config",         # /wp-config.php, /wp-config.php.bak
    "service-account",   # /service-account.json
)
# A path segment exactly equal to any of these.
PROBE_SEGMENTS_EXACT = (
    ".ssh", ".aws", ".svn", ".vscode", ".idea",
    "secrets", "secret", "credentials",
    "id_rsa", "id_dsa", "id_ecdsa", "id_ed25519",
)
# A FILENAME (last segment, and only when it has an extension) starting with
# one of these: catches secrets.yml / secret.json without eating an article
# called "secrets-of-a-good-claim".
PROBE_FILENAME_PREFIXES = ("secret", "credential")


def _path(url):
    """The path part of a request, lowercased, without query or fragment."""
    return (url or "").split("?")[0].split("#")[0].lower()


def classify(url):
    """Why a request is not content: 'asset' | 'api' | 'robots' | 'probe',
    or None when it IS content."""
    path = _path(url)
    segments = [s for s in path.split("/") if s]
    last = segments[-1] if segments else ""

    if last in NON_CONTENT_FILES:
        return "robots"

    for seg in segments:
        if seg in PROBE_SEGMENTS_EXACT:
            return "probe"
        if seg.startswith(PROBE_SEGMENT_PREFIXES):
            return "probe"
        if seg in API_SEGMENTS:
            return "api"

    if "." in last:
        if last.endswith(CONFIG_EXTENSIONS):
            return "probe"
        if last.startswith(PROBE_FILENAME_PREFIXES):
            return "probe"
        if last.endswith(STATIC_EXTENSIONS):
            return "asset"

    return None


def is_content(url):
    """True when this request is for an actual page."""
    return classify(url) is None


def pattern_summary():
    """The filter, as data, so the UI and the docs cannot drift from the code."""
    return [
        ("Static assets", "file extension", list(STATIC_EXTENSIONS)),
        ("Config / secret files", "file extension", list(CONFIG_EXTENSIONS)),
        ("API endpoints", "path segment", ["/%s/" % s for s in API_SEGMENTS]),
        ("Robots instructions", "filename", list(NON_CONTENT_FILES)),
        ("Scanner probes", "path segment starts with",
         list(PROBE_SEGMENT_PREFIXES)),
        ("Scanner probes", "path segment equals", list(PROBE_SEGMENTS_EXACT)),
        ("Scanner probes", "filename starts with",
         [p + ".*" for p in PROBE_FILENAME_PREFIXES]),
    ]