| Server IP : 138.197.107.151 / Your IP : 216.73.217.120 Web Server : Apache/2.4.58 (Ubuntu) System : Linux BloxBy-Builder 6.8.0-71-generic #71-Ubuntu SMP PREEMPT_DYNAMIC Tue Jul 22 16:52:38 UTC 2025 x86_64 User : wpbetasites_mrakzqskir ( 1022) PHP Version : 8.3.6 Disable Function : NONE MySQL : OFF | cURL : ON | WGET : ON | Perl : ON | Python : OFF | Sudo : ON | Pkexec : OFF Directory : /proc/698955/root/var/www/bsd-crawler-parser/ |
Upload File : |
#!/usr/bin/env bash
#
# Creates/updates everything in AWS. Safe to re-run: every step is idempotent
# and nothing here deletes data. The IAM policy grants no s3:DeleteObject on
# either bucket, so the Lambda cannot destroy logs even if it has a bug.
#
# ./deploy.sh # create/update all resources, schedule DISABLED
# ./deploy.sh --enable # same, but turn the nightly schedule on
#
set -euo pipefail
REGION="us-east-1"
FUNCTION="wpe-log-parser"
ROLE="wpe-log-parser-role"
POLICY="wpe-log-parser-policy"
RULE="wpe-log-parser-nightly"
SRC_BUCKET="bsd-wpe-logs-private"
DST_BUCKET="bsd-wpe-logs-parsed"
SCHEDULE="cron(0 9 * * ? *)" # 09:00 UTC daily
ENABLE_SCHEDULE="no"
[[ "${1:-}" == "--enable" ]] && ENABLE_SCHEDULE="yes"
ACCOUNT=$(aws sts get-caller-identity --query Account --output text)
echo "account $ACCOUNT / region $REGION"
# --- 1. destination bucket -------------------------------------------------
if aws s3api head-bucket --bucket "$DST_BUCKET" --region "$REGION" 2>/dev/null; then
echo "[=] bucket $DST_BUCKET already exists"
else
echo "[+] creating bucket $DST_BUCKET"
# us-east-1 is the one region that rejects a LocationConstraint.
aws s3api create-bucket --bucket "$DST_BUCKET" --region "$REGION"
aws s3api wait bucket-exists --bucket "$DST_BUCKET" --region "$REGION"
fi
echo "[+] hardening $DST_BUCKET (block public access, encryption, versioning)"
aws s3api put-public-access-block --bucket "$DST_BUCKET" --region "$REGION" \
--public-access-block-configuration \
"BlockPublicAcls=true,IgnorePublicAcls=true,BlockPublicPolicy=true,RestrictPublicBuckets=true"
aws s3api put-bucket-encryption --bucket "$DST_BUCKET" --region "$REGION" \
--server-side-encryption-configuration \
'{"Rules":[{"ApplyServerSideEncryptionByDefault":{"SSEAlgorithm":"AES256"},"BucketKeyEnabled":true}]}'
aws s3api put-bucket-versioning --bucket "$DST_BUCKET" --region "$REGION" \
--versioning-configuration Status=Enabled
# --- 2. IAM role -----------------------------------------------------------
if aws iam get-role --role-name "$ROLE" >/dev/null 2>&1; then
echo "[=] role $ROLE already exists"
else
echo "[+] creating role $ROLE"
aws iam create-role --role-name "$ROLE" \
--description "Execution role for the $FUNCTION Lambda" \
--assume-role-policy-document '{
"Version":"2012-10-17",
"Statement":[{"Effect":"Allow","Principal":{"Service":"lambda.amazonaws.com"},"Action":"sts:AssumeRole"}]
}' >/dev/null
fi
POLICY_ARN="arn:aws:iam::${ACCOUNT}:policy/${POLICY}"
if aws iam get-policy --policy-arn "$POLICY_ARN" >/dev/null 2>&1; then
echo "[+] updating policy $POLICY to a new default version"
# Keep only the newest version; IAM caps a policy at 5 versions.
for v in $(aws iam list-policy-versions --policy-arn "$POLICY_ARN" \
--query 'Versions[?!IsDefaultVersion].VersionId' --output text); do
aws iam delete-policy-version --policy-arn "$POLICY_ARN" --version-id "$v" || true
done
aws iam create-policy-version --policy-arn "$POLICY_ARN" \
--policy-document file://iam-policy.json --set-as-default >/dev/null
else
echo "[+] creating policy $POLICY"
aws iam create-policy --policy-name "$POLICY" \
--description "Least-privilege S3 access for $FUNCTION" \
--policy-document file://iam-policy.json >/dev/null
fi
aws iam attach-role-policy --role-name "$ROLE" --policy-arn "$POLICY_ARN"
aws iam attach-role-policy --role-name "$ROLE" \
--policy-arn "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
ROLE_ARN=$(aws iam get-role --role-name "$ROLE" --query 'Role.Arn' --output text)
# --- 3. package ------------------------------------------------------------
echo "[+] packaging (boto3 ships with the Lambda runtime, so no vendored deps)"
rm -rf build function.zip && mkdir -p build
cp handler.py logparser.py crawlers.py content.py build/
(cd build && zip -qr ../function.zip .)
echo " $(du -h function.zip | cut -f1) zip"
# --- 4. lambda -------------------------------------------------------------
ENV_VARS="Variables={SOURCE_BUCKET=$SRC_BUCKET,DEST_BUCKET=$DST_BUCKET,SOURCE_PREFIX=logs/nginx/,MAX_FILES_PER_RUN=200}"
if aws lambda get-function --function-name "$FUNCTION" --region "$REGION" >/dev/null 2>&1; then
echo "[=] updating existing function $FUNCTION"
aws lambda update-function-code --function-name "$FUNCTION" --region "$REGION" \
--zip-file fileb://function.zip >/dev/null
aws lambda wait function-updated --function-name "$FUNCTION" --region "$REGION"
aws lambda update-function-configuration --function-name "$FUNCTION" --region "$REGION" \
--timeout 900 --memory-size 1024 --environment "$ENV_VARS" >/dev/null
else
echo "[+] creating function $FUNCTION"
# IAM role propagation to Lambda is eventually consistent; retry briefly.
for i in 1 2 3 4 5 6; do
if aws lambda create-function --function-name "$FUNCTION" --region "$REGION" \
--runtime python3.12 --architectures arm64 \
--role "$ROLE_ARN" --handler handler.lambda_handler \
--zip-file fileb://function.zip \
--timeout 900 --memory-size 1024 \
--description "Parses WP Engine nginx logs for AI crawler traffic" \
--environment "$ENV_VARS" >/dev/null 2>&1; then
break
fi
echo " waiting for IAM role to propagate (attempt $i)..."
sleep 10
done
fi
aws lambda wait function-updated --function-name "$FUNCTION" --region "$REGION"
# --- 5. schedule -----------------------------------------------------------
STATE=$([[ "$ENABLE_SCHEDULE" == "yes" ]] && echo ENABLED || echo DISABLED)
echo "[+] EventBridge rule $RULE ($SCHEDULE) -> $STATE"
aws events put-rule --name "$RULE" --region "$REGION" \
--schedule-expression "$SCHEDULE" --state "$STATE" \
--description "Nightly 09:00 UTC trigger for $FUNCTION" >/dev/null
aws lambda add-permission --function-name "$FUNCTION" --region "$REGION" \
--statement-id "${RULE}-invoke" \
--action lambda:InvokeFunction --principal events.amazonaws.com \
--source-arn "arn:aws:events:${REGION}:${ACCOUNT}:rule/${RULE}" >/dev/null 2>&1 || true
aws events put-targets --rule "$RULE" --region "$REGION" \
--targets "Id=1,Arn=arn:aws:lambda:${REGION}:${ACCOUNT}:function:${FUNCTION}" >/dev/null
echo
echo "done. schedule is $STATE."
[[ "$STATE" == "DISABLED" ]] && echo "enable it with: aws events enable-rule --name $RULE --region $REGION"
echo "invoke once: aws lambda invoke --function-name $FUNCTION --region $REGION /dev/stdout"