AWS DevOps Agent 検証 — Generic Webhook でアラームから調査を自動起動する

WEBHOOK_URL="https://event-ai.ap-northeast-1.api.aws/webhook/generic/(webhook-id)"
SECRET="(your-webhook-secret)"
 
TIMESTAMP=$(date -u +%Y-%m-%dT%H:%M:%S.000Z)
INCIDENT_ID="test-webhook-$(date +%s)"
 
PAYLOAD=$(cat <<EOF
{
  "eventType": "incident",
  "incidentId": "$INCIDENT_ID",
  "action": "created",
  "priority": "HIGH",
  "title": "High CPU usage on production server",
  "description": "CPU utilization exceeded 80% on instance i-0123456789abcdef0 in ap-northeast-1",
  "service": "web-app-prod",
  "timestamp": "$TIMESTAMP",
  "affectedResources": ["i-0123456789abcdef0"],
  "data": {
    "metadata": {
      "region": "ap-northeast-1",
      "environment": "production"
    }
  }
}
EOF
)
 
SIGNATURE=$(echo -n "${TIMESTAMP}:${PAYLOAD}" \
  | openssl dgst -sha256 -hmac "$SECRET" -binary \
  | base64)
 
curl -s -X POST "$WEBHOOK_URL" \
  -H "Content-Type: application/json" \
  -H "x-amzn-event-timestamp: $TIMESTAMP" \
  -H "x-amzn-event-signature: $SIGNATURE" \
  -d "$PAYLOAD"

aws logs filter-log-events \
  --log-group-name "/aws/events/devops-agent" \
  --filter-pattern "Investigation" \
  --start-time $(date -d '10 minutes ago' +%s)000 \
  --limit 10 \
  --region ap-northeast-1 \
  --query 'events[*].message' --output json \
  | python3 -c "
import sys, json
for m in json.load(sys.stdin):
    msg = json.loads(m)
    print(f'{msg[\"time\"]}  {msg[\"detail-type\"]}')
"

import json
import hashlib
import hmac
import os
import urllib.request
import base64
from datetime import datetime, timezone
 
WEBHOOK_URL = os.environ["WEBHOOK_URL"]
WEBHOOK_SECRET = os.environ["WEBHOOK_SECRET"]
 
def lambda_handler(event, context):
    sns_message = json.loads(event["Records"][0]["Sns"]["Message"])
 
    alarm_name = sns_message.get("AlarmName", "Unknown Alarm")
    reason = sns_message.get("NewStateReason", "")
    region = sns_message.get("Region", "")
 
    # Dimensions からインスタンス ID を抽出
    resources = []
    trigger = sns_message.get("Trigger", {})
    for dim in trigger.get("Dimensions", []):
        if dim.get("name") == "InstanceId":
            resources.append(dim["value"])
 
    timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.000Z")
    incident_id = f"cw-alarm-{alarm_name}-{int(datetime.now(timezone.utc).timestamp())}"
 
    payload = {
        "eventType": "incident",
        "incidentId": incident_id,
        "action": "created",
        "priority": "HIGH",
        "title": f"CloudWatch Alarm: {alarm_name}",
        "description": f"{alarm_name}: {reason}",
        "service": "cloudwatch-alarm",
        "timestamp": timestamp,
        "affectedResources": resources,
        "data": {
            "metadata": {
                "region": region,
                "alarmName": alarm_name
            }
        }
    }
 
    payload_str = json.dumps(payload)
 
    # HMAC 署名
    sign_input = f"{timestamp}:{payload_str}"
    signature = hmac.new(
        WEBHOOK_SECRET.encode("utf-8"),
        sign_input.encode("utf-8"),
        hashlib.sha256
    ).digest()
    signature_b64 = base64.b64encode(signature).decode("utf-8")
 
    req = urllib.request.Request(
        WEBHOOK_URL,
        data=payload_str.encode("utf-8"),
        headers={
            "Content-Type": "application/json",
            "x-amzn-event-timestamp": timestamp,
            "x-amzn-event-signature": signature_b64
        },
        method="POST"
    )
 
    with urllib.request.urlopen(req) as resp:
        status = resp.status
        body = resp.read().decode("utf-8")
 
    print(json.dumps({
        "incident_id": incident_id,
        "webhook_status": status,
        "webhook_response": body,
        "affected_resources": resources
    }))
 
    return {"statusCode": status, "body": body}

cat > /tmp/webhook-lambda-trust.json << 'EOF'
{
  "Version": "2012-10-17",
  "Statement": [{
    "Effect": "Allow",
    "Principal": {"Service": "lambda.amazonaws.com"},
    "Action": "sts:AssumeRole"
  }]
}
EOF
 
aws iam create-role \
  --role-name DevOpsAgentWebhookLambdaRole \
  --assume-role-policy-document file:///tmp/webhook-lambda-trust.json
 
aws iam attach-role-policy \
  --role-name DevOpsAgentWebhookLambdaRole \
  --policy-arn arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole

zip -j /tmp/webhook-lambda.zip lambda_function.py
 
aws lambda create-function \
  --function-name devops-agent-webhook-trigger \
  --runtime python3.13 \
  --handler lambda_function.lambda_handler \
  --role arn:aws:iam::(account-id):role/DevOpsAgentWebhookLambdaRole \
  --zip-file fileb:///tmp/webhook-lambda.zip \
  --timeout 30 \
  --environment "Variables={WEBHOOK_URL=https://event-ai.ap-northeast-1.api.aws/webhook/generic/(webhook-id),WEBHOOK_SECRET=(your-secret)}" \
  --region ap-northeast-1

aws sns create-topic --name devops-agent-webhook-trigger --region ap-northeast-1
 
aws lambda add-permission \
  --function-name devops-agent-webhook-trigger \
  --statement-id sns-trigger \
  --action lambda:InvokeFunction \
  --principal sns.amazonaws.com \
  --source-arn arn:aws:sns:ap-northeast-1:(account-id):devops-agent-webhook-trigger \
  --region ap-northeast-1
 
aws sns subscribe \
  --topic-arn arn:aws:sns:ap-northeast-1:(account-id):devops-agent-webhook-trigger \
  --protocol lambda \
  --notification-endpoint arn:aws:lambda:ap-northeast-1:(account-id):function:devops-agent-webhook-trigger \
  --region ap-northeast-1

aws cloudwatch put-metric-alarm \
  --alarm-name prod-web-high-cpu \
  --namespace AWS/EC2 \
  --metric-name CPUUtilization \
  --statistic Average \
  --period 60 \
  --evaluation-periods 2 \
  --threshold 80 \
  --comparison-operator GreaterThanThreshold \
  --dimensions Name=InstanceId,Value=i-0123456789abcdef0 \
  --alarm-actions arn:aws:sns:ap-northeast-1:(account-id):devops-agent-webhook-trigger \
  --region ap-northeast-1

aws ssm send-command \
  --instance-ids i-0123456789abcdef0 \
  --document-name "AWS-RunShellScript" \
  --parameters 'commands=["yum install -y stress-ng"]' \
  --region ap-northeast-1

aws ssm send-command \
  --instance-ids i-0123456789abcdef0 \
  --document-name "AWS-RunShellScript" \
  --parameters 'commands=["stress-ng --cpu 2 --cpu-load 99 --timeout 600s"]' \
  --region ap-northeast-1

aws cloudwatch describe-alarms \
  --alarm-names prod-web-high-cpu \
  --query 'MetricAlarms[0].{State:StateValue,Updated:StateUpdatedTimestamp}' \
  --output table --region ap-northeast-1

WEBHOOK_URL="https://event-ai.ap-northeast-1.api.aws/webhook/generic/(webhook-id)"
SECRET="(your-webhook-secret)"
INCIDENT_ID="dedup-test-001"
 
# 共通ペイロード（incidentId を固定）
PAYLOAD=$(cat <<EOF
{
  "eventType": "incident",
  "incidentId": "$INCIDENT_ID",
  "action": "created",
  "priority": "HIGH",
  "title": "High CPU usage on production server",
  "description": "CPU utilization exceeded 80% on instance i-0123456789abcdef0 in ap-northeast-1",
  "service": "web-app-prod",
  "timestamp": "$(date -u +%Y-%m-%dT%H:%M:%S.000Z)",
  "affectedResources": ["i-0123456789abcdef0"]
}
EOF
)
 
# 1回目
TS1=$(date -u +%Y-%m-%dT%H:%M:%S.000Z)
SIG1=$(echo -n "${TS1}:${PAYLOAD}" | openssl dgst -sha256 -hmac "$SECRET" -binary | base64)
curl -s -X POST "$WEBHOOK_URL" \
  -H "Content-Type: application/json" \
  -H "x-amzn-event-timestamp: $TS1" \
  -H "x-amzn-event-signature: $SIG1" \
  -d "$PAYLOAD"
 
sleep 5
 
# 2回目（同じ incidentId、新しいタイムスタンプ）
TS2=$(date -u +%Y-%m-%dT%H:%M:%S.000Z)
SIG2=$(echo -n "${TS2}:${PAYLOAD}" | openssl dgst -sha256 -hmac "$SECRET" -binary | base64)
curl -s -X POST "$WEBHOOK_URL" \
  -H "Content-Type: application/json" \
  -H "x-amzn-event-timestamp: $TS2" \
  -H "x-amzn-event-signature: $SIG2" \
  -d "$PAYLOAD"

# Lambda 関数削除
aws lambda delete-function \
  --function-name devops-agent-webhook-trigger \
  --region ap-northeast-1
 
# SNS サブスクリプション・トピック削除
SUB_ARN=$(aws sns list-subscriptions-by-topic \
  --topic-arn arn:aws:sns:ap-northeast-1:(account-id):devops-agent-webhook-trigger \
  --query 'Subscriptions[0].SubscriptionArn' --output text \
  --region ap-northeast-1)
aws sns unsubscribe --subscription-arn "$SUB_ARN" --region ap-northeast-1
aws sns delete-topic \
  --topic-arn arn:aws:sns:ap-northeast-1:(account-id):devops-agent-webhook-trigger \
  --region ap-northeast-1
 
# IAM ロール削除
aws iam detach-role-policy \
  --role-name DevOpsAgentWebhookLambdaRole \
  --policy-arn arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
aws iam delete-role --role-name DevOpsAgentWebhookLambdaRole
 
# CloudWatch アラームから SNS アクションを削除（アラーム自体は残す場合）
aws cloudwatch put-metric-alarm \
  --alarm-name prod-web-high-cpu \
  --namespace AWS/EC2 \
  --metric-name CPUUtilization \
  --statistic Average \
  --period 60 \
  --evaluation-periods 2 \
  --threshold 80 \
  --comparison-operator GreaterThanThreshold \
  --dimensions Name=InstanceId,Value=i-0123456789abcdef0 \
  --alarm-actions "" \
  --region ap-northeast-1
 
# Webhook はコンソールから削除
# エージェントスペース → Capabilities → Webhook → Remove