| Server IP : 138.197.107.151 / Your IP : 216.73.217.7 Web Server : Apache/2.4.58 (Ubuntu) System : Linux BloxBy-Builder 6.8.0-71-generic #71-Ubuntu SMP PREEMPT_DYNAMIC Tue Jul 22 16:52:38 UTC 2025 x86_64 User : wpbetasites_mrakzqskir ( 1022) PHP Version : 8.3.6 Disable Function : NONE MySQL : OFF | cURL : ON | WGET : ON | Perl : ON | Python : OFF | Sudo : ON | Pkexec : OFF Directory : /var/www/bsd-crawler-parser/ |
Upload File : |
"""End-to-end dry run: reads REAL logs from S3, writes output to a LOCAL folder.
Nothing is written to AWS. Use this to eyeball the output before deploying.
python3 test_local.py # 5 files -> ./local_out
python3 test_local.py --files 20 # more files
python3 test_local.py --out /tmp/foo # different output folder
"""
import argparse
import json
import os
import boto3
from handler import LocalSink, SOURCE_BUCKET, SOURCE_PREFIX, run
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--files", type=int, default=5, help="max source files to process")
ap.add_argument("--out", default="local_out", help="local output directory")
ap.add_argument("--bucket", default=SOURCE_BUCKET)
ap.add_argument("--prefix", default=SOURCE_PREFIX)
ap.add_argument("--region", default="us-east-1")
args = ap.parse_args()
s3 = boto3.client("s3", region_name=args.region)
sink = LocalSink(args.out)
os.makedirs(args.out, exist_ok=True)
print("reading s3://%s/%s" % (args.bucket, args.prefix))
print("writing %s/ (local only -- no AWS writes)\n" % os.path.abspath(args.out))
result = run(s3, sink, args.bucket, args.prefix, args.files)
print("\n" + "=" * 60)
print(json.dumps(result, indent=2))
for date in result["dates"]:
body = sink.get("summary/date=%s.json" % date)
summary = json.loads(body)
print("\n--- summary/date=%s.json ---" % date)
print(" totals: %s" % json.dumps(summary["totals"]))
print(" top platforms:")
for platform, v in list(summary["by_platform"].items())[:10]:
print(" %-22s %6d hits [%s]" % (platform, v["hits"], v["category"]))
print(" top sites:")
for site, v in list(summary["by_site"].items())[:5]:
print(" %-38s %6d hits" % (site, v["hits"]))
print("\n--- sample event ---")
for key in sorted(sink.list_keys("events/")):
body = sink.get(key)
if body and body.strip():
print(" %s" % key)
print(json.dumps(json.loads(body.splitlines()[0]), indent=2))
break
if __name__ == "__main__":
main()