S3 Files を実機検証 — S3 バケットをファイルシステムとしてマウントする新機能の同期遅延と読み取り性能

BUCKET="<bucket-name>"
REGION="ap-northeast-1"
 
aws s3api create-bucket \
  --bucket $BUCKET \
  --region $REGION \
  --create-bucket-configuration LocationConstraint=$REGION
 
aws s3api put-bucket-versioning \
  --bucket $BUCKET \
  --versioning-configuration Status=Enabled

ACCOUNT_ID="<account-id>"
 
aws iam create-role \
  --role-name S3FilesVerifyFSRole \
  --assume-role-policy-document '{
    "Version": "2012-10-17",
    "Statement": [{
      "Effect": "Allow",
      "Principal": {"Service": "elasticfilesystem.amazonaws.com"},
      "Action": "sts:AssumeRole",
      "Condition": {
        "StringEquals": {"aws:SourceAccount": "'$ACCOUNT_ID'"},
        "ArnLike": {"aws:SourceArn": "arn:aws:s3files:'$REGION':'$ACCOUNT_ID':file-system/*"}
      }
    }]
  }'

aws iam put-role-policy \
  --role-name S3FilesVerifyFSRole \
  --policy-name S3FilesVerifyFSPolicy \
  --policy-document '{
    "Version": "2012-10-17",
    "Statement": [
      {
        "Sid": "S3BucketPermissions",
        "Effect": "Allow",
        "Action": ["s3:ListBucket", "s3:ListBucketVersions"],
        "Resource": "arn:aws:s3:::'$BUCKET'",
        "Condition": {"StringEquals": {"aws:ResourceAccount": "'$ACCOUNT_ID'"}}
      },
      {
        "Sid": "S3ObjectPermissions",
        "Effect": "Allow",
        "Action": ["s3:AbortMultipartUpload", "s3:DeleteObject*", "s3:GetObject*", "s3:List*", "s3:PutObject*"],
        "Resource": "arn:aws:s3:::'$BUCKET'/*",
        "Condition": {"StringEquals": {"aws:ResourceAccount": "'$ACCOUNT_ID'"}}
      },
      {
        "Sid": "EventBridgeManage",
        "Effect": "Allow",
        "Action": ["events:DeleteRule", "events:DisableRule", "events:EnableRule", "events:PutRule", "events:PutTargets", "events:RemoveTargets"],
        "Condition": {"StringEquals": {"events:ManagedBy": "elasticfilesystem.amazonaws.com"}},
        "Resource": ["arn:aws:events:*:*:rule/DO-NOT-DELETE-S3-Files*"]
      },
      {
        "Sid": "EventBridgeRead",
        "Effect": "Allow",
        "Action": ["events:DescribeRule", "events:ListRuleNamesByTarget", "events:ListRules", "events:ListTargetsByRule"],
        "Resource": ["arn:aws:events:*:*:rule/*"]
      }
    ]
  }'

aws iam create-role \
  --role-name S3FilesVerifyEC2Role \
  --assume-role-policy-document '{
    "Version": "2012-10-17",
    "Statement": [{
      "Effect": "Allow",
      "Principal": {"Service": "ec2.amazonaws.com"},
      "Action": "sts:AssumeRole"
    }]
  }'
 
# マネージドポリシーのアタッチ
aws iam attach-role-policy --role-name S3FilesVerifyEC2Role \
  --policy-arn arn:aws:iam::aws:policy/AmazonS3FilesClientFullAccess
aws iam attach-role-policy --role-name S3FilesVerifyEC2Role \
  --policy-arn arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore
 
# S3 バケットへの読み取りアクセス（S3 Files の読み取り最適化に必要）
aws iam put-role-policy \
  --role-name S3FilesVerifyEC2Role \
  --policy-name S3FilesVerifyEC2S3Policy \
  --policy-document '{
    "Version": "2012-10-17",
    "Statement": [
      {"Effect": "Allow", "Action": ["s3:GetObject", "s3:GetObjectVersion"], "Resource": "arn:aws:s3:::'$BUCKET'/*"},
      {"Effect": "Allow", "Action": "s3:ListBucket", "Resource": "arn:aws:s3:::'$BUCKET'"}
    ]
  }'
 
# インスタンスプロファイルの作成
aws iam create-instance-profile --instance-profile-name S3FilesVerifyEC2Profile
aws iam add-role-to-instance-profile \
  --instance-profile-name S3FilesVerifyEC2Profile \
  --role-name S3FilesVerifyEC2Role

VPC_ID=$(aws ec2 describe-vpcs --filters Name=isDefault,Values=true \
  --query 'Vpcs[0].VpcId' --output text --region $REGION)
 
# EC2 用セキュリティグループ
EC2_SG=$(aws ec2 create-security-group \
  --group-name s3files-verify-ec2-sg \
  --description "S3 Files verify - EC2" \
  --vpc-id $VPC_ID --region $REGION \
  --query 'GroupId' --output text)
 
# マウントターゲット用セキュリティグループ
MT_SG=$(aws ec2 create-security-group \
  --group-name s3files-verify-mt-sg \
  --description "S3 Files verify - mount target" \
  --vpc-id $VPC_ID --region $REGION \
  --query 'GroupId' --output text)
 
# NFS ポート（TCP 2049）の許可
aws ec2 authorize-security-group-egress --group-id $EC2_SG --region $REGION \
  --ip-permissions "IpProtocol=tcp,FromPort=2049,ToPort=2049,UserIdGroupPairs=[{GroupId=$MT_SG}]"
aws ec2 authorize-security-group-ingress --group-id $MT_SG --region $REGION \
  --ip-permissions "IpProtocol=tcp,FromPort=2049,ToPort=2049,UserIdGroupPairs=[{GroupId=$EC2_SG}]"
 
echo "EC2_SG=$EC2_SG  MT_SG=$MT_SG"

SUBNET_ID=$(aws ec2 describe-subnets \
  --filters Name=vpc-id,Values=$VPC_ID Name=availability-zone,Values=${REGION}a \
  --query 'Subnets[0].SubnetId' --output text --region $REGION)
 
AMI_ID=$(aws ssm get-parameters-by-path \
  --path /aws/service/ami-amazon-linux-latest \
  --query "Parameters[?contains(Name,'al2023-ami-kernel-default-x86_64')].Value" \
  --output text --region $REGION)
 
INSTANCE_ID=$(aws ec2 run-instances \
  --image-id $AMI_ID \
  --instance-type t3.medium \
  --subnet-id $SUBNET_ID \
  --security-group-ids $EC2_SG \
  --iam-instance-profile Name=S3FilesVerifyEC2Profile \
  --metadata-options HttpTokens=required \
  --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=s3files-verify}]' \
  --query 'Instances[0].InstanceId' --output text --region $REGION)
 
aws ec2 wait instance-running --instance-ids $INSTANCE_ID --region $REGION
echo "INSTANCE_ID=$INSTANCE_ID"

curl -s https://amazon-efs-utils.aws.com/efs-utils-installer.sh | sudo sh -s -- --install

Installed:
  amazon-efs-utils-3.0.0-1.amzn2023.x86_64

BUCKET="<bucket-name>"
REGION="ap-northeast-1"
 
for SIZE_LABEL in "1k:1024:1" "1m:1M:1" "10m:1M:10"; do
  IFS=: read -r NAME BS COUNT <<< "$SIZE_LABEL"
  FILE="export-${NAME}.bin"
  dd if=/dev/urandom of=/mnt/s3files/$FILE bs=$BS count=$COUNT 2>/dev/null
  WRITE_TIME=$(date +%s)
  echo "$FILE written at $(date -d @$WRITE_TIME '+%H:%M:%S')"
  while true; do
    aws s3api head-object --bucket $BUCKET --key $FILE --region $REGION >/dev/null 2>&1 && break
    sleep 5
  done
  SYNC_TIME=$(date +%s)
  echo "$FILE synced to S3 at $(date -d @$SYNC_TIME '+%H:%M:%S') ($((SYNC_TIME - WRITE_TIME))s)"
done

BUCKET="<bucket-name>"
REGION="ap-northeast-1"
 
for SIZE_LABEL in "1k:1024:1" "1m:1M:1" "10m:1M:10"; do
  IFS=: read -r NAME BS COUNT <<< "$SIZE_LABEL"
  FILE="import-${NAME}.bin"
  dd if=/dev/urandom of=/tmp/$FILE bs=$BS count=$COUNT 2>/dev/null
  aws s3 cp /tmp/$FILE s3://$BUCKET/$FILE --region $REGION --quiet
  UPLOAD_TIME=$(date +%s)
  echo "$FILE uploaded at $(date -d @$UPLOAD_TIME '+%H:%M:%S')"
  while true; do
    ls /mnt/s3files/$FILE >/dev/null 2>&1 && break
    sleep 2
  done
  SYNC_TIME=$(date +%s)
  echo "$FILE visible in FS at $(date -d @$SYNC_TIME '+%H:%M:%S') ($((SYNC_TIME - UPLOAD_TIME))s)"
done

BUCKET="<bucket-name>"
REGION="ap-northeast-1"
 
# テストファイルを S3 API でアップロード
dd if=/dev/urandom of=/tmp/read-4k.bin bs=4096 count=1 2>/dev/null
dd if=/dev/urandom of=/tmp/read-128k.bin bs=131072 count=1 2>/dev/null
dd if=/dev/urandom of=/tmp/read-1m.bin bs=1M count=1 2>/dev/null
dd if=/dev/urandom of=/tmp/read-10m.bin bs=1M count=10 2>/dev/null
for f in read-4k.bin read-128k.bin read-1m.bin read-10m.bin; do
  aws s3 cp /tmp/$f s3://$BUCKET/$f --region $REGION --quiet
done
echo "Waiting 30s for import..."
sleep 30
 
# コールド読み取り（ページキャッシュをドロップ）
echo "--- Cold reads ---"
for f in read-4k.bin read-128k.bin read-1m.bin read-10m.bin; do
  sync && echo 3 | sudo tee /proc/sys/vm/drop_caches >/dev/null
  START=$(date +%s%N)
  cat /mnt/s3files/$f > /dev/null
  END=$(date +%s%N)
  echo "Cold $f: $(( (END - START) / 1000000 ))ms"
done
 
# ウォーム読み取り（キャッシュあり）
echo "--- Warm reads ---"
for f in read-4k.bin read-128k.bin read-1m.bin read-10m.bin; do
  START=$(date +%s%N)
  cat /mnt/s3files/$f > /dev/null
  END=$(date +%s%N)
  echo "Warm $f: $(( (END - START) / 1000000 ))ms"
done

# EC2 インスタンスの削除
aws ec2 terminate-instances --instance-ids $INSTANCE_ID --region $REGION
 
# マウントターゲットの削除
aws s3files delete-mount-target \
  --region $REGION \
  --mount-target-id $MT_ID
 
# ファイルシステムの削除
aws s3files delete-file-system \
  --region $REGION \
  --file-system-id $FS_ID
 
# セキュリティグループの削除（EC2 終了完了後に実行）
aws ec2 wait instance-terminated --instance-ids $INSTANCE_ID --region $REGION
aws ec2 delete-security-group --group-id $EC2_SG --region $REGION
aws ec2 delete-security-group --group-id $MT_SG --region $REGION
 
# IAM リソースの削除
aws iam remove-role-from-instance-profile \
  --instance-profile-name S3FilesVerifyEC2Profile \
  --role-name S3FilesVerifyEC2Role
aws iam delete-instance-profile --instance-profile-name S3FilesVerifyEC2Profile
aws iam detach-role-policy --role-name S3FilesVerifyEC2Role \
  --policy-arn arn:aws:iam::aws:policy/AmazonS3FilesClientFullAccess
aws iam detach-role-policy --role-name S3FilesVerifyEC2Role \
  --policy-arn arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore
aws iam delete-role-policy --role-name S3FilesVerifyEC2Role --policy-name S3FilesVerifyEC2S3Policy
aws iam delete-role --role-name S3FilesVerifyEC2Role
aws iam delete-role-policy --role-name S3FilesVerifyFSRole --policy-name S3FilesVerifyFSPolicy
aws iam delete-role --role-name S3FilesVerifyFSRole
 
# S3 バケットの削除（バージョニング有効のため全バージョン削除が必要）
aws s3api list-object-versions --bucket $BUCKET --region $REGION \
  --query '{Objects: [].{Key:Key,VersionId:VersionId}}' --output json | \
  aws s3api delete-objects --bucket $BUCKET --region $REGION --delete file:///dev/stdin
aws s3api delete-bucket --bucket $BUCKET --region $REGION