403Webshell
Server IP : 138.197.107.151  /  Your IP : 216.73.217.7
Web Server : Apache/2.4.58 (Ubuntu)
System : Linux BloxBy-Builder 6.8.0-71-generic #71-Ubuntu SMP PREEMPT_DYNAMIC Tue Jul 22 16:52:38 UTC 2025 x86_64
User : wpbetasites_mrakzqskir ( 1022)
PHP Version : 8.3.6
Disable Function : NONE
MySQL : OFF  |  cURL : ON  |  WGET : ON  |  Perl : ON  |  Python : OFF  |  Sudo : ON  |  Pkexec : OFF
Directory :  /var/www/bsd-crawler-parser/

Upload File :
current_dir [ Writeable ] document_root [ Writeable ]

 

Command :


[ Back ]     

Current File : /var/www/bsd-crawler-parser//logparser.py
"""Parsing for WP Engine nginx logs in Apache combined format + vhost.

Line shape:
    IP vhost - [timestamp] "METHOD path HTTP/x" status bytes "referrer" "user-agent"
"""

import re
from datetime import datetime, timezone

from crawlers import match_crawler

# A quoted field: any run of non-quote/non-backslash chars, or a backslash escape.
_QUOTED = r'(?:[^"\\]|\\.)*'

LINE_RE = re.compile(
    r"^(?P<ip>\S+)\s+"
    r"(?P<vhost>\S+)\s+"
    r"(?P<ident>\S+)\s+"
    r"\[(?P<timestamp>[^\]]+)\]\s+"
    r'"(?P<request>' + _QUOTED + r')"\s+'
    r"(?P<status>\d{3})\s+"
    r"(?P<bytes>-|\d+)\s+"
    r'"(?P<referrer>' + _QUOTED + r')"\s+'
    r'"(?P<user_agent>' + _QUOTED + r')"'
)

# Explicit month map: strptime's %b honours the C locale, which we do not
# control inside Lambda. Apache month abbreviations are always English.
_MONTHS = {
    "Jan": 1, "Feb": 2, "Mar": 3, "Apr": 4, "May": 5, "Jun": 6,
    "Jul": 7, "Aug": 8, "Sep": 9, "Oct": 10, "Nov": 11, "Dec": 12,
}

_TS_RE = re.compile(
    r"^(\d{2})/([A-Za-z]{3})/(\d{4}):(\d{2}):(\d{2}):(\d{2})\s+([+-])(\d{2})(\d{2})$"
)


def parse_timestamp(raw):
    """'08/Jul/2026:00:17:35 +0000' -> timezone-aware datetime in UTC."""
    m = _TS_RE.match(raw.strip())
    if not m:
        return None
    day, mon, year, hour, minute, sec, sign, oh, om = m.groups()
    month = _MONTHS.get(mon.title())
    if month is None:
        return None
    offset_minutes = int(oh) * 60 + int(om)
    if sign == "-":
        offset_minutes = -offset_minutes
    try:
        dt = datetime(
            int(year), month, int(day), int(hour), int(minute), int(sec),
            tzinfo=timezone.utc,
        )
    except ValueError:
        return None
    # Shift by the logged offset to land on true UTC.
    return dt - _minutes(offset_minutes)


def _minutes(n):
    from datetime import timedelta
    return timedelta(minutes=n)


def split_request(request):
    """'GET /path HTTP/1.1' -> ('GET', '/path', 'HTTP/1.1'), tolerating junk."""
    parts = request.split(" ")
    if len(parts) >= 3 and parts[-1].startswith("HTTP/"):
        return parts[0], " ".join(parts[1:-1]), parts[-1]
    if len(parts) == 2:
        return parts[0], parts[1], ""
    if len(parts) == 1:
        return "", parts[0], ""
    return "", request, ""


def normalize_site(vhost):
    """Fold www. into the bare domain; leave everything else alone."""
    site = vhost.strip().lower().rstrip(".")
    if site.startswith("www."):
        site = site[4:]
    return site


def parse_line(line, source_key):
    """Parse one log line; return an event dict only for AI crawler hits.

    Returns None for non-matching lines and for lines we cannot parse.
    """
    m = LINE_RE.match(line.strip())
    if not m:
        return None

    user_agent = m.group("user_agent")
    matched = match_crawler(user_agent)
    if matched is None:
        return None
    label, category = matched

    dt = parse_timestamp(m.group("timestamp"))
    if dt is None:
        return None

    _method, path, _proto = split_request(m.group("request"))
    raw_bytes = m.group("bytes")

    return {
        "site": normalize_site(m.group("vhost")),
        "ai_platform": label,
        "category": category,
        "user_agent": user_agent,
        "url": path,
        "status_code": int(m.group("status")),
        "bytes": 0 if raw_bytes == "-" else int(raw_bytes),
        "timestamp": dt.strftime("%Y-%m-%dT%H:%M:%SZ"),
        "source_file": source_key,
    }


def iter_events(lines, source_key):
    """Yield (event, stats_key) for AI hits; stats_key tracks why lines dropped."""
    for line in lines:
        if not line.strip():
            continue
        event = parse_line(line, source_key)
        if event is not None:
            yield event

Youez - 2016 - github.com/yon3zu
LinuXploit