| Server IP : 138.197.107.151 / Your IP : 216.73.217.7 Web Server : Apache/2.4.58 (Ubuntu) System : Linux BloxBy-Builder 6.8.0-71-generic #71-Ubuntu SMP PREEMPT_DYNAMIC Tue Jul 22 16:52:38 UTC 2025 x86_64 User : wpbetasites_mrakzqskir ( 1022) PHP Version : 8.3.6 Disable Function : NONE MySQL : OFF | cURL : ON | WGET : ON | Perl : ON | Python : OFF | Sudo : ON | Pkexec : OFF Directory : /var/www/bsd-crawler-parser/ |
Upload File : |
"""Parsing for WP Engine nginx logs in Apache combined format + vhost.
Line shape:
IP vhost - [timestamp] "METHOD path HTTP/x" status bytes "referrer" "user-agent"
"""
import re
from datetime import datetime, timezone
from crawlers import match_crawler
# A quoted field: any run of non-quote/non-backslash chars, or a backslash escape.
_QUOTED = r'(?:[^"\\]|\\.)*'
LINE_RE = re.compile(
r"^(?P<ip>\S+)\s+"
r"(?P<vhost>\S+)\s+"
r"(?P<ident>\S+)\s+"
r"\[(?P<timestamp>[^\]]+)\]\s+"
r'"(?P<request>' + _QUOTED + r')"\s+'
r"(?P<status>\d{3})\s+"
r"(?P<bytes>-|\d+)\s+"
r'"(?P<referrer>' + _QUOTED + r')"\s+'
r'"(?P<user_agent>' + _QUOTED + r')"'
)
# Explicit month map: strptime's %b honours the C locale, which we do not
# control inside Lambda. Apache month abbreviations are always English.
_MONTHS = {
"Jan": 1, "Feb": 2, "Mar": 3, "Apr": 4, "May": 5, "Jun": 6,
"Jul": 7, "Aug": 8, "Sep": 9, "Oct": 10, "Nov": 11, "Dec": 12,
}
_TS_RE = re.compile(
r"^(\d{2})/([A-Za-z]{3})/(\d{4}):(\d{2}):(\d{2}):(\d{2})\s+([+-])(\d{2})(\d{2})$"
)
def parse_timestamp(raw):
"""'08/Jul/2026:00:17:35 +0000' -> timezone-aware datetime in UTC."""
m = _TS_RE.match(raw.strip())
if not m:
return None
day, mon, year, hour, minute, sec, sign, oh, om = m.groups()
month = _MONTHS.get(mon.title())
if month is None:
return None
offset_minutes = int(oh) * 60 + int(om)
if sign == "-":
offset_minutes = -offset_minutes
try:
dt = datetime(
int(year), month, int(day), int(hour), int(minute), int(sec),
tzinfo=timezone.utc,
)
except ValueError:
return None
# Shift by the logged offset to land on true UTC.
return dt - _minutes(offset_minutes)
def _minutes(n):
from datetime import timedelta
return timedelta(minutes=n)
def split_request(request):
"""'GET /path HTTP/1.1' -> ('GET', '/path', 'HTTP/1.1'), tolerating junk."""
parts = request.split(" ")
if len(parts) >= 3 and parts[-1].startswith("HTTP/"):
return parts[0], " ".join(parts[1:-1]), parts[-1]
if len(parts) == 2:
return parts[0], parts[1], ""
if len(parts) == 1:
return "", parts[0], ""
return "", request, ""
def normalize_site(vhost):
"""Fold www. into the bare domain; leave everything else alone."""
site = vhost.strip().lower().rstrip(".")
if site.startswith("www."):
site = site[4:]
return site
def parse_line(line, source_key):
"""Parse one log line; return an event dict only for AI crawler hits.
Returns None for non-matching lines and for lines we cannot parse.
"""
m = LINE_RE.match(line.strip())
if not m:
return None
user_agent = m.group("user_agent")
matched = match_crawler(user_agent)
if matched is None:
return None
label, category = matched
dt = parse_timestamp(m.group("timestamp"))
if dt is None:
return None
_method, path, _proto = split_request(m.group("request"))
raw_bytes = m.group("bytes")
return {
"site": normalize_site(m.group("vhost")),
"ai_platform": label,
"category": category,
"user_agent": user_agent,
"url": path,
"status_code": int(m.group("status")),
"bytes": 0 if raw_bytes == "-" else int(raw_bytes),
"timestamp": dt.strftime("%Y-%m-%dT%H:%M:%SZ"),
"source_file": source_key,
}
def iter_events(lines, source_key):
"""Yield (event, stats_key) for AI hits; stats_key tracks why lines dropped."""
for line in lines:
if not line.strip():
continue
event = parse_line(line, source_key)
if event is not None:
yield event