"""Registry of AI crawler user-agents we care about.

To add a crawler: add one row to AI_CRAWLERS. The key is the case-insensitive
substring to look for in the User-Agent; `label` is what shows up in reports;
`category` is one of ai_training / ai_assistant / search.

  ai_training   collects a corpus to train models on
  ai_assistant  fetches a page live, because a user asked an assistant about it
  search        builds a search index (including AI-flavoured search indexes)

Longer patterns win over shorter ones, so "Applebot-Extended" is matched
before the "Applebot" that is a substring of it. That ordering is handled
automatically in _compile_patterns() -- you do not need to sort this dict.
"""

AI_CRAWLERS = {
    # OpenAI
    "GPTBot": ("GPTBot", "ai_training"),
    "OAI-SearchBot": ("OAI-SearchBot", "search"),
    "ChatGPT-User": ("ChatGPT-User", "ai_assistant"),
    # Anthropic
    "ClaudeBot": ("ClaudeBot", "ai_training"),
    "Claude-User": ("Claude-User", "ai_assistant"),
    "Claude-SearchBot": ("Claude-SearchBot", "search"),
    "anthropic-ai": ("anthropic-ai", "ai_training"),
    # Perplexity
    "PerplexityBot": ("PerplexityBot", "search"),
    "Perplexity-User": ("Perplexity-User", "ai_assistant"),
    # Google
    "Google-Extended": ("Google-Extended", "ai_training"),
    "GoogleOther": ("GoogleOther", "ai_training"),
    # Apple
    "Applebot-Extended": ("Applebot-Extended", "ai_training"),
    "Applebot": ("Applebot", "search"),
    # Amazon
    "Amazonbot": ("Amazonbot", "search"),
    # ByteDance
    "Bytespider": ("Bytespider", "ai_training"),
    # Meta
    "meta-externalagent": ("meta-externalagent", "ai_training"),
    "FacebookBot": ("FacebookBot", "ai_training"),
    # Common Crawl (feeds many downstream model trainers)
    "CCBot": ("CCBot", "ai_training"),
    # Others
    "cohere-ai": ("cohere-ai", "ai_training"),
    "Diffbot": ("Diffbot", "ai_training"),
    "ImagesiftBot": ("ImagesiftBot", "ai_training"),
    "Timpibot": ("Timpibot", "ai_training"),
    "YouBot": ("YouBot", "search"),
    "DuckAssistBot": ("DuckAssistBot", "ai_assistant"),
    "Bingbot": ("Bingbot", "search"),
}


def _compile_patterns():
    """Lowercased (pattern, label, category), longest pattern first."""
    rows = [
        (pattern.lower(), label, category)
        for pattern, (label, category) in AI_CRAWLERS.items()
    ]
    rows.sort(key=lambda row: len(row[0]), reverse=True)
    return rows


_PATTERNS = _compile_patterns()


def match_crawler(user_agent):
    """Return (label, category) for the first AI crawler in `user_agent`.

    Returns None when the user-agent is not a known AI crawler.
    """
    if not user_agent:
        return None
    haystack = user_agent.lower()
    for pattern, label, category in _PATTERNS:
        if pattern in haystack:
            return (label, category)
    return None