"""The bot -> group mapping the viewer displays.

Derived from the parser's own AI_CRAWLERS registry, with one change: the
parser's `search` bucket conflated two different things, so it is split in two.

  ai_training   collects a corpus to train models on
  ai_assistant  fetches a page live, because a user asked an assistant about it
  ai_search     an AI company building an index      <- split out of `search`
  search        a conventional search engine indexer  <- the only hidden group

Only the last group is hidden by default; the other three are "AI activity".

WHY WE RE-DERIVE THIS INSTEAD OF READING THE EVENT'S OWN `category` FIELD

The ~350k events already in S3 have the OLD three-way category baked in at
parse time -- OAI-SearchBot is literally stored as "search". Deriving the group
from `ai_platform` on read applies this grouping to all existing data with no
reprocessing and no writes to the bucket. It also means old and newly parsed
data always agree, even though crawlers.py still emits the old categories.

Adding a crawler to crawlers.py picks it up here automatically. Only add a name
to AI_SEARCH_PLATFORMS below if it is an AI company's *indexing* bot.
"""

import os
import sys

# crawlers.py is the parser's registry, one directory up. It is the source of
# truth for which bots exist; we only override how `search` is subdivided.
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from crawlers import AI_CRAWLERS  # noqa: E402

# AI companies that index pages. The parser files these under `search`, which
# buries them with Bingbot -- they are the whole reason this split exists.
# Confirmed 2026-07-16.
AI_SEARCH_PLATFORMS = {
    "OAI-SearchBot",     # OpenAI
    "PerplexityBot",     # Perplexity
    "YouBot",            # You.com
    "Claude-SearchBot",  # Anthropic (configured; not seen in the data yet)
}

# Every group except `search` counts as AI activity and is shown by default.
AI_GROUPS = ("ai_training", "ai_assistant", "ai_search")
ALL_GROUPS = AI_GROUPS + ("search",)

GROUP_LABELS = {
    "ai_training": "AI training",
    "ai_assistant": "AI assistant",
    "ai_search": "AI search",
    "search": "Search engines",
}

# Plain-English tooltips, written for someone reading this dashboard for the
# first time with no background. No jargon, one sentence.
GROUP_INFO = {
    "ai_training": "Bots that copy your pages to help train an AI model. "
                   "Your content becomes part of what the model learns from.",
    "ai_assistant": "Bots that fetch one of your pages right now, because a "
                    "person just asked an AI assistant something about it.",
    "ai_search": "AI companies indexing your pages so their AI can find you "
                 "and link to you when it answers a question.",
    "search": "Ordinary search engines building their normal search index. "
              "Hidden by default, because this is not AI activity.",
}

BOT_INFO = {
    # OpenAI
    "GPTBot": "OpenAI's collector. Copies pages to help train future ChatGPT models.",
    "OAI-SearchBot": "OpenAI's indexer. Lists your pages so ChatGPT can find and link to them.",
    "ChatGPT-User": "Visits one page because someone just asked ChatGPT about it.",
    # Anthropic
    "ClaudeBot": "Anthropic's collector. Copies pages to help train Claude.",
    "Claude-User": "Visits one page because someone just asked Claude about it.",
    "Claude-SearchBot": "Anthropic's indexer. Lists your pages so Claude can find and cite them.",
    "anthropic-ai": "An older Anthropic collector that gathers pages for training.",
    # Perplexity
    "PerplexityBot": "Perplexity's indexer. Lists your pages so its AI answers can cite them.",
    "Perplexity-User": "Visits one page because someone just asked Perplexity about it.",
    # Google
    "Google-Extended": "Not a visitor as such: Google's switch for whether Gemini may "
                       "learn from your pages.",
    "GoogleOther": "A general-purpose Google crawler used by its internal teams. "
                   "This is not Google Search.",
    # Apple
    "Applebot-Extended": "Apple's AI collector. Copies pages to help train Apple Intelligence.",
    "Applebot": "Apple's ordinary search crawler, behind Siri and Spotlight results.",
    # Amazon
    "Amazonbot": "Amazon's crawler, mostly feeding Alexa's answers.",
    # Others
    "Bytespider": "ByteDance's collector (TikTok's owner). Copies pages for AI training.",
    "meta-externalagent": "Meta's collector. Copies pages to help train Meta AI across "
                          "Facebook, Instagram and WhatsApp.",
    "FacebookBot": "Meta's collector, gathering pages to train its language models.",
    "CCBot": "Common Crawl, a non-profit that archives the web. Many AI companies "
             "train on its archive, so this feeds others downstream.",
    "cohere-ai": "Cohere's collector. Copies pages to help train its business AI models.",
    "Diffbot": "Diffbot's crawler. Turns pages into a knowledge database it sells on.",
    "ImagesiftBot": "Collects images from your pages, used for AI image training.",
    "Timpibot": "Timpi's crawler, building a community-run search index.",
    "YouBot": "You.com's indexer. Lists your pages so its AI search can cite them.",
    "DuckAssistBot": "Visits one page because someone just asked DuckDuckGo's AI assistant.",
    "Bingbot": "Microsoft's ordinary search crawler -- the one behind Bing results.",
}


def bot_info(platform):
    """One-line explanation of a bot, falling back to its group's."""
    if platform in BOT_INFO:
        return BOT_INFO[platform]
    return GROUP_INFO.get(category_for(platform), "An automated crawler.")

# label -> parser category, e.g. {"GPTBot": "ai_training"}
_PARSER_CATEGORY = {label: category for label, category in AI_CRAWLERS.values()}


class UncategorisedBot(Exception):
    """A bot does not resolve to exactly one of the four display groups.

    Raised loudly rather than letting the hit vanish. Every hit belongs to
    exactly one group, so the category boxes must always sum to total Hits; a
    bot that resolves to something else would be counted in the total but shown
    in no box, and the page would quietly stop adding up.
    """


def _check_registry():
    """Fail at startup if crawlers.py has a category we cannot display.

    The parser's registry is the source of truth for which bots exist, so it can
    grow a category (say "ai_agent") that this viewer has no box for. That must
    be a loud error here, not two hits silently missing from a dashboard.
    """
    rogue = sorted({(label, cat) for label, cat in AI_CRAWLERS.values()
                    if cat not in ALL_GROUPS})
    if rogue:
        raise UncategorisedBot(
            "crawlers.py has categories this viewer cannot display: %s.\n"
            "Every bot must map to one of %s. Either add a group here (and a "
            "stat box in overview.html), or fix the category in crawlers.py."
            % (", ".join("%s=%r" % r for r in rogue), ", ".join(ALL_GROUPS)))


_check_registry()


def category_for(platform, stored=None):
    """The display group for a bot name. Always exactly one of ALL_GROUPS.

    `stored` is the category recorded in the event, used only as a fallback for
    a bot this viewer does not know about (i.e. one added to the parser after
    this file was last read). Unknown bots are never silently called AI.
    """
    if platform in AI_SEARCH_PLATFORMS:
        return "ai_search"
    known = _PARSER_CATEGORY.get(platform)
    if known is not None:
        if known not in ALL_GROUPS:  # _check_registry should have caught this
            raise UncategorisedBot(
                "%r has category %r, which is not one of %s"
                % (platform, known, ", ".join(ALL_GROUPS)))
        return known
    if stored in ALL_GROUPS:
        return stored
    return "search"


def is_ai(group):
    """True for anything except a conventional search engine."""
    return group in AI_GROUPS


def platforms_by_group():
    """{group: [platform, ...]} over every configured crawler, for the UI."""
    out = {group: [] for group in ALL_GROUPS}
    for label in _PARSER_CATEGORY:
        out[category_for(label)].append(label)
    for label in AI_SEARCH_PLATFORMS:
        if label not in _PARSER_CATEGORY:
            out["ai_search"].append(label)
    return {group: sorted(labels) for group, labels in out.items()}