AWSTemplateFormatVersion: '2010-09-09'
Description: >
  Test K8s cluster (master + 1~3 workers) on EC2 + EFS for CloudNativePG validation.
  Improvements vs v1: WorkerCount param, EBS unified gp3 (Size/Iops), efs.txt order fix
  (B1), admin.conf via S3 (no sshpass / no root SSH password), cfn-signal, expiry-date
  parameterized, AllowedValues extended for m5/m6i. Throughput must be set post-launch
  via "aws ec2 modify-volume --throughput N" since EC2::Instance BlockDeviceMappings
  does not support Throughput.

Metadata:
  AWS::CloudFormation::Interface:
    ParameterGroups:
      - Label: { default: "<<<<< Kubernetes >>>>>" }
        Parameters: [ KubernetesVersion ]
      - Label: { default: "<<<<< EC2 Node >>>>>" }
        Parameters:
          - KeyName
          - SgIngressCidr
          - WorkerCount
          - MasterNodeInstanceType
          - WorkerNodeInstanceType
          - Ec2EbsVolumeSize
          - Ec2EbsVolumeIops
          - LatestAmiId
      - Label: { default: "<<<<< Tags >>>>>" }
        Parameters: [ OwnerTag, ExpiryDate ]
      - Label: { default: "<<<<< Region AZ >>>>>" }
        Parameters: [ TargetRegion, AvailabilityZone1, AvailabilityZone2 ]
      - Label: { default: "<<<<< VPC Subnet >>>>>" }
        Parameters:
          - VpcBlock
          - PublicSubnet1Block
          - PublicSubnet2Block
          - PrivateSubnet1Block
          - PrivateSubnet2Block

Parameters:
  KubernetesVersion:
    Type: String
    Default: 1.28.0
    AllowedValues: [1.23.5, 1.23.6, 1.23.7, 1.24.0, 1.24.1, 1.24.2, 1.24.3, 1.24.4, 1.28.0, 1.29.0, 1.30.0]
    Description: Target kube* version. init.sh resolves to the latest available patch.

  KeyName:
    Type: AWS::EC2::KeyPair::KeyName
    Description: Existing EC2 KeyPair for SSH access
    ConstraintDescription: must be the name of an existing EC2 KeyPair.

  SgIngressCidr:
    Type: String
    Default: 0.0.0.0/0
    MinLength: '9'
    MaxLength: '18'
    AllowedPattern: (\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2})
    Description: Source CIDR allowed to reach EC2 (SSH/443)

  WorkerCount:
    Type: String
    Default: "2"
    AllowedValues: ["1", "2", "3"]
    Description: Number of worker nodes (default 2)

  MasterNodeInstanceType:
    Type: String
    Default: t3.large
    AllowedValues:
      - t2.micro
      - t2.small
      - t2.medium
      - t3.micro
      - t3.small
      - t3.medium
      - t3.large
      - t3.xlarge
      - m5.large
      - m5.xlarge
      - m5.2xlarge
      - m6i.large
      - m6i.xlarge
      - m6i.2xlarge

  WorkerNodeInstanceType:
    Type: String
    Default: m5.xlarge
    AllowedValues:
      - t2.micro
      - t2.small
      - t2.medium
      - t3.micro
      - t3.small
      - t3.medium
      - t3.large
      - t3.xlarge
      - m5.large
      - m5.xlarge
      - m5.2xlarge
      - m6i.large
      - m6i.xlarge
      - m6i.2xlarge

  Ec2EbsVolumeSize:
    Type: Number
    Default: 200
    MinValue: 50
    MaxValue: 16384
    Description: EBS gp3 size in GiB (default 200)

  Ec2EbsVolumeIops:
    Type: Number
    Default: 6000
    MinValue: 3000
    MaxValue: 16000
    Description: EBS gp3 IOPS (3000 ~ 16000)

  LatestAmiId:
    Type: 'AWS::SSM::Parameter::Value<AWS::EC2::Image::Id>'
    Default: '/aws/service/canonical/ubuntu/server/22.04/stable/current/amd64/hvm/ebs-gp2/ami-id'
    AllowedValues:
      - /aws/service/canonical/ubuntu/server/22.04/stable/current/amd64/hvm/ebs-gp2/ami-id
    Description: Ubuntu 22.04 LTS AMI (DO NOT CHANGE)

  OwnerTag:
    Type: String
    Default: bjh
    Description: Tag value for "owner"

  ExpiryDate:
    Type: String
    Default: 2026-12-31
    Description: Tag value for "expiry-date" (YYYY-MM-DD)

  TargetRegion:
    Type: String
    Default: us-east-1
  AvailabilityZone1:
    Type: String
    Default: us-east-1a
  AvailabilityZone2:
    Type: String
    Default: us-east-1b

  VpcBlock:
    Type: String
    Default: 192.168.0.0/16
    Description: (DO NOT CHANGE)
  PublicSubnet1Block:
    Type: String
    Default: 192.168.10.0/24
    Description: (DO NOT CHANGE)
  PublicSubnet2Block:
    Type: String
    Default: 192.168.20.0/24
    Description: (DO NOT CHANGE)
  PrivateSubnet1Block:
    Type: String
    Default: 192.168.30.0/24
    Description: (DO NOT CHANGE)
  PrivateSubnet2Block:
    Type: String
    Default: 192.168.40.0/24
    Description: (DO NOT CHANGE)

Conditions:
  CreateWorker2: !Or
    - !Equals [ !Ref WorkerCount, "2" ]
    - !Equals [ !Ref WorkerCount, "3" ]
  CreateWorker3: !Equals [ !Ref WorkerCount, "3" ]

Resources:
  # IAM Role for EC2: read scripts + read/write runtime/* (admin.conf) + EBS CSI driver
  EC2Role:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal: { Service: ec2.amazonaws.com }
            Action: sts:AssumeRole
      ManagedPolicyArns:
        # EBS CSI driver needs ec2:CreateVolume/AttachVolume/DescribeVolumes/etc.
        - arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy
      Policies:
        - PolicyName: S3ScriptsAccess
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action: s3:GetObject
                Resource: "arn:aws:s3:::my-k8s-scripts/*"
              - Effect: Allow
                Action:
                  - s3:PutObject
                  - s3:GetObject
                  - s3:ListBucket
                Resource:
                  - "arn:aws:s3:::my-k8s-scripts"
                  - "arn:aws:s3:::my-k8s-scripts/runtime/*"
        - PolicyName: EFSDescribe
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - elasticfilesystem:DescribeMountTargets
                  - elasticfilesystem:DescribeFileSystems
                Resource: "*"

  EC2InstanceProfile:
    Type: AWS::IAM::InstanceProfile
    Properties:
      Roles: [ !Ref EC2Role ]

  # VPC
  MyVPC:
    Type: AWS::EC2::VPC
    Properties:
      CidrBlock: !Ref VpcBlock
      EnableDnsSupport: true
      EnableDnsHostnames: true
      Tags:
        - { Key: Name, Value: !Sub "${AWS::StackName}-VPC" }

  PublicSubnet1:
    Type: AWS::EC2::Subnet
    Properties:
      AvailabilityZone: !Ref AvailabilityZone1
      CidrBlock: !Ref PublicSubnet1Block
      VpcId: !Ref MyVPC
      MapPublicIpOnLaunch: true
      Tags:
        - { Key: Name, Value: !Sub "${AWS::StackName}-PublicSubnet1" }
        - { Key: kubernetes.io/role/elb, Value: 1 }

  PublicSubnet2:
    Type: AWS::EC2::Subnet
    Properties:
      AvailabilityZone: !Ref AvailabilityZone2
      CidrBlock: !Ref PublicSubnet2Block
      VpcId: !Ref MyVPC
      MapPublicIpOnLaunch: true
      Tags:
        - { Key: Name, Value: !Sub "${AWS::StackName}-PublicSubnet2" }
        - { Key: kubernetes.io/role/elb, Value: 1 }

  InternetGateway:
    Type: AWS::EC2::InternetGateway

  VPCGatewayAttachment:
    Type: AWS::EC2::VPCGatewayAttachment
    Properties:
      InternetGatewayId: !Ref InternetGateway
      VpcId: !Ref MyVPC

  PublicSubnetRouteTable:
    Type: AWS::EC2::RouteTable
    Properties:
      VpcId: !Ref MyVPC
      Tags:
        - { Key: Name, Value: !Sub "${AWS::StackName}-PublicSubnetRouteTable" }

  PublicSubnetRoute:
    Type: AWS::EC2::Route
    Properties:
      RouteTableId: !Ref PublicSubnetRouteTable
      DestinationCidrBlock: 0.0.0.0/0
      GatewayId: !Ref InternetGateway

  PublicSubnet1RouteTableAssociation:
    Type: AWS::EC2::SubnetRouteTableAssociation
    Properties:
      SubnetId: !Ref PublicSubnet1
      RouteTableId: !Ref PublicSubnetRouteTable

  PublicSubnet2RouteTableAssociation:
    Type: AWS::EC2::SubnetRouteTableAssociation
    Properties:
      SubnetId: !Ref PublicSubnet2
      RouteTableId: !Ref PublicSubnetRouteTable

  PrivateSubnet1:
    Type: AWS::EC2::Subnet
    Properties:
      AvailabilityZone: !Ref AvailabilityZone1
      CidrBlock: !Ref PrivateSubnet1Block
      VpcId: !Ref MyVPC
      Tags:
        - { Key: Name, Value: !Sub "${AWS::StackName}-PrivateSubnet1" }
        - { Key: kubernetes.io/role/internal-elb, Value: 1 }

  PrivateSubnet2:
    Type: AWS::EC2::Subnet
    Properties:
      AvailabilityZone: !Ref AvailabilityZone2
      CidrBlock: !Ref PrivateSubnet2Block
      VpcId: !Ref MyVPC
      Tags:
        - { Key: Name, Value: !Sub "${AWS::StackName}-PrivateSubnet2" }
        - { Key: kubernetes.io/role/internal-elb, Value: 1 }

  PrivateSubnetRouteTable:
    Type: AWS::EC2::RouteTable
    Properties:
      VpcId: !Ref MyVPC
      Tags:
        - { Key: Name, Value: !Sub "${AWS::StackName}-PrivateSubnetRouteTable" }

  PrivateSubnet1RouteTableAssociation:
    Type: AWS::EC2::SubnetRouteTableAssociation
    Properties:
      SubnetId: !Ref PrivateSubnet1
      RouteTableId: !Ref PrivateSubnetRouteTable

  PrivateSubnet2RouteTableAssociation:
    Type: AWS::EC2::SubnetRouteTableAssociation
    Properties:
      SubnetId: !Ref PrivateSubnet2
      RouteTableId: !Ref PrivateSubnetRouteTable

  # Security Group
  EC2SG:
    Type: AWS::EC2::SecurityGroup
    Properties:
      GroupDescription: !Sub "${AWS::StackName}-EC2-SG"
      VpcId: !Ref MyVPC
      Tags:
        - { Key: Name, Value: !Sub "${AWS::StackName}-EC2-SG" }
      SecurityGroupIngress:
        - IpProtocol: '-1'
          CidrIp: !Ref SgIngressCidr
        - IpProtocol: '-1'
          CidrIp: !Ref VpcBlock
        - IpProtocol: '-1'
          CidrIp: 172.16.0.0/16
        - IpProtocol: '-1'
          CidrIp: 10.200.10.0/24

  EFSSG:
    Type: AWS::EC2::SecurityGroup
    Properties:
      VpcId: !Ref MyVPC
      GroupDescription: !Sub "${AWS::StackName}-EFS-SG"
      Tags:
        - { Key: Name, Value: !Sub "${AWS::StackName}-EFS-SG" }
      SecurityGroupIngress:
        - IpProtocol: tcp
          FromPort: '2049'
          ToPort: '2049'
          SourceSecurityGroupId: !Ref EC2SG

  # EFS
  ElasticFileSystem:
    Type: AWS::EFS::FileSystem
    Properties:
      FileSystemTags:
        - { Key: Name, Value: !Sub "${AWS::StackName}-EFS" }

  ElasticFileSystemMountTarget0:
    Type: AWS::EFS::MountTarget
    Properties:
      FileSystemId: !Ref ElasticFileSystem
      SecurityGroups: [ !Ref EFSSG ]
      SubnetId: !Ref PublicSubnet1

  ElasticFileSystemMountTarget1:
    Type: AWS::EFS::MountTarget
    Properties:
      FileSystemId: !Ref ElasticFileSystem
      SecurityGroups: [ !Ref EFSSG ]
      SubnetId: !Ref PublicSubnet2

  # ============================================================
  # Master Node
  # ============================================================
  Instance1ENI1:
    Type: AWS::EC2::NetworkInterface
    Properties:
      SubnetId: !Ref PublicSubnet1
      Description: !Sub "${AWS::StackName}-Instance1ENI1"
      GroupSet: [ !Ref EC2SG ]
      PrivateIpAddress: 192.168.10.10
      SourceDestCheck: false
      Tags:
        - { Key: Name, Value: !Sub "${AWS::StackName}-Instance1ENI1" }

  MEC2:
    Type: AWS::EC2::Instance
    DependsOn:
      - ElasticFileSystemMountTarget0
      - ElasticFileSystemMountTarget1
    CreationPolicy:
      ResourceSignal:
        Timeout: PT25M
    Properties:
      InstanceType: !Ref MasterNodeInstanceType
      ImageId: !Ref LatestAmiId
      KeyName: !Ref KeyName
      IamInstanceProfile: !Ref EC2InstanceProfile
      Tags:
        - { Key: Name, Value: !Sub "${AWS::StackName}-Master" }
        - { Key: owner, Value: !Ref OwnerTag }
        - { Key: expiry-date, Value: !Ref ExpiryDate }
      NetworkInterfaces:
        - NetworkInterfaceId: !Ref Instance1ENI1
          DeviceIndex: '0'
      BlockDeviceMappings:
        - DeviceName: /dev/sda1
          Ebs:
            VolumeType: gp3
            VolumeSize: !Ref Ec2EbsVolumeSize
            Iops: !Ref Ec2EbsVolumeIops
            DeleteOnTermination: true
      UserData:
        Fn::Base64:
          !Sub |
            #!/bin/bash
            hostnamectl --static set-hostname k8s-m
            export STACK_NAME=${AWS::StackName}
            export AWS_DEFAULT_REGION=${AWS::Region}
            echo "STACK_NAME=${AWS::StackName}" >> /etc/environment
            echo "AWS_DEFAULT_REGION=${AWS::Region}" >> /etc/environment
            export KUBERNETES_VERSION=${KubernetesVersion}
            echo "Kubernetes Version : $KUBERNETES_VERSION" > /root/kubernetes-version.txt

            # Install awscli + cfn-signal first so failures can be reported.
            apt-get update -y
            apt-get install -y awscli python3-pip
            pip3 install --break-system-packages https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-py3-latest.tar.gz \
              || pip3 install https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-py3-latest.tar.gz

            # Trace + capture failing line for cfn-signal --reason
            set -x
            exec > >(tee -a /var/log/userdata.log) 2>&1
            on_err() {
              local rc=$?
              echo "FAIL: line=$1 rc=$rc cmd=$2" > /tmp/userdata-fail.txt
              # Belt-and-suspenders: upload ALL logs to S3 since console may not have them,
              # CFN events don't propagate cfn-signal --reason, and instance is destroyed on rollback.
              local PFX="s3://my-k8s-scripts/runtime/${AWS::StackName}"
              aws s3 cp /var/log/userdata.log $PFX/MEC2-userdata.log 2>/dev/null || true
              aws s3 cp /tmp/userdata-fail.txt $PFX/MEC2-fail-reason.txt 2>/dev/null || true
              [ -f /root/init.log   ] && aws s3 cp /root/init.log   $PFX/MEC2-init.log   2>/dev/null || true
              [ -f /root/master.log ] && aws s3 cp /root/master.log $PFX/MEC2-master.log 2>/dev/null || true
              [ -f /root/final.log  ] && aws s3 cp /root/final.log  $PFX/MEC2-final.log  2>/dev/null || true
              cfn-signal --success false --reason "line=$1 rc=$rc cmd=$2" \
                --stack ${AWS::StackName} --resource MEC2 --region ${AWS::Region} || true
            }
            trap 'on_err $LINENO "$BASH_COMMAND"' ERR
            set -eE

            # ---- B1 fix: efs.txt BEFORE master.sh runs ----
            export EFS_FILE_SYSTEM_ID=${ElasticFileSystem}
            EFS_DNS="$EFS_FILE_SYSTEM_ID.efs.${AWS::Region}.amazonaws.com"
            echo "$EFS_DNS" > /root/efs.txt

            # Wait for EFS mount target available (CFN DependsOn already ensures this, but double-check)
            for i in $(seq 1 30); do
              STATE=$(aws efs describe-mount-targets --region ${AWS::Region} --file-system-id $EFS_FILE_SYSTEM_ID \
                        --query 'MountTargets[0].LifeCycleState' --output text 2>&1) || STATE="error: $STATE"
              if [ "$STATE" = "available" ]; then break; fi
              echo "Waiting for EFS mount target ($STATE)..."
              sleep 10
            done

            # Pre-create NFS mount point and mount
            mkdir -p /nfs4-share
            apt-get install -y nfs-common
            mount -t nfs4 -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport \
              $EFS_DNS:/ /nfs4-share || true
            echo "$EFS_DNS:/ /nfs4-share nfs4 nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport,_netdev 0 0" \
              >> /etc/fstab

            # Fetch + run scripts
            aws s3 cp s3://my-k8s-scripts/init_v2.sh    /root/init.sh
            aws s3 cp s3://my-k8s-scripts/master_v2.sh  /root/master.sh
            chmod +x /root/init.sh /root/master.sh
            sed -i 's/\r$//' /root/init.sh /root/master.sh
            /root/init.sh   > /root/init.log   2>&1
            /root/master.sh > /root/master.log 2>&1

            # Success
            cfn-signal --success true --stack ${AWS::StackName} --resource MEC2 --region ${AWS::Region}

  # ============================================================
  # Worker Node 1 (always created)
  # ============================================================
  Instance2ENI1:
    Type: AWS::EC2::NetworkInterface
    Properties:
      SubnetId: !Ref PublicSubnet1
      Description: !Sub "${AWS::StackName}-Instance2ENI1"
      GroupSet: [ !Ref EC2SG ]
      PrivateIpAddress: 192.168.10.101
      SourceDestCheck: false
      Tags:
        - { Key: Name, Value: !Sub "${AWS::StackName}-Instance2ENI1" }

  W1EC2:
    Type: AWS::EC2::Instance
    DependsOn: MEC2
    CreationPolicy:
      ResourceSignal:
        Timeout: PT25M
    Properties:
      InstanceType: !Ref WorkerNodeInstanceType
      ImageId: !Ref LatestAmiId
      KeyName: !Ref KeyName
      IamInstanceProfile: !Ref EC2InstanceProfile
      Tags:
        - { Key: Name, Value: !Sub "${AWS::StackName}-Worker1" }
        - { Key: owner, Value: !Ref OwnerTag }
        - { Key: expiry-date, Value: !Ref ExpiryDate }
      NetworkInterfaces:
        - NetworkInterfaceId: !Ref Instance2ENI1
          DeviceIndex: '0'
      BlockDeviceMappings:
        - DeviceName: /dev/sda1
          Ebs:
            VolumeType: gp3
            VolumeSize: !Ref Ec2EbsVolumeSize
            Iops: !Ref Ec2EbsVolumeIops
            DeleteOnTermination: true
      UserData:
        Fn::Base64:
          !Sub |
            #!/bin/bash
            hostnamectl --static set-hostname k8s-w1
            export STACK_NAME=${AWS::StackName}
            export AWS_DEFAULT_REGION=${AWS::Region}
            echo "STACK_NAME=${AWS::StackName}" >> /etc/environment
            echo "AWS_DEFAULT_REGION=${AWS::Region}" >> /etc/environment
            export KUBERNETES_VERSION=${KubernetesVersion}

            apt-get update -y
            apt-get install -y awscli python3-pip
            pip3 install --break-system-packages https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-py3-latest.tar.gz \
              || pip3 install https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-py3-latest.tar.gz

            set -x
            exec > >(tee -a /var/log/userdata.log) 2>&1
            on_err() {
              local rc=$?
              echo "FAIL: line=$1 rc=$rc cmd=$2" > /tmp/userdata-fail.txt
              local PFX="s3://my-k8s-scripts/runtime/${AWS::StackName}"
              aws s3 cp /var/log/userdata.log $PFX/W1EC2-userdata.log 2>/dev/null || true
              aws s3 cp /tmp/userdata-fail.txt $PFX/W1EC2-fail-reason.txt 2>/dev/null || true
              [ -f /root/init.log   ] && aws s3 cp /root/init.log   $PFX/W1EC2-init.log   2>/dev/null || true
              [ -f /root/worker.log ] && aws s3 cp /root/worker.log $PFX/W1EC2-worker.log 2>/dev/null || true
              cfn-signal --success false --reason "line=$1 rc=$rc cmd=$2" \
                --stack ${AWS::StackName} --resource W1EC2 --region ${AWS::Region} || true
            }
            trap 'on_err $LINENO "$BASH_COMMAND"' ERR
            set -eE

            aws s3 cp s3://my-k8s-scripts/init_v2.sh   /root/init.sh
            aws s3 cp s3://my-k8s-scripts/worker_v2.sh /root/worker.sh
            chmod +x /root/init.sh /root/worker.sh
            sed -i 's/\r$//' /root/init.sh /root/worker.sh
            /root/init.sh   > /root/init.log   2>&1
            /root/worker.sh > /root/worker.log 2>&1

            cfn-signal --success true --stack ${AWS::StackName} --resource W1EC2 --region ${AWS::Region}

  # ============================================================
  # Worker Node 2 (only if WorkerCount >= 2)
  # ============================================================
  Instance3ENI1:
    Type: AWS::EC2::NetworkInterface
    Condition: CreateWorker2
    Properties:
      SubnetId: !Ref PublicSubnet1
      Description: !Sub "${AWS::StackName}-Instance3ENI1"
      GroupSet: [ !Ref EC2SG ]
      PrivateIpAddress: 192.168.10.102
      SourceDestCheck: false
      Tags:
        - { Key: Name, Value: !Sub "${AWS::StackName}-Instance3ENI1" }

  W2EC2:
    Type: AWS::EC2::Instance
    Condition: CreateWorker2
    DependsOn: MEC2
    CreationPolicy:
      ResourceSignal:
        Timeout: PT25M
    Properties:
      InstanceType: !Ref WorkerNodeInstanceType
      ImageId: !Ref LatestAmiId
      KeyName: !Ref KeyName
      IamInstanceProfile: !Ref EC2InstanceProfile
      Tags:
        - { Key: Name, Value: !Sub "${AWS::StackName}-Worker2" }
        - { Key: owner, Value: !Ref OwnerTag }
        - { Key: expiry-date, Value: !Ref ExpiryDate }
      NetworkInterfaces:
        - NetworkInterfaceId: !Ref Instance3ENI1
          DeviceIndex: '0'
      BlockDeviceMappings:
        - DeviceName: /dev/sda1
          Ebs:
            VolumeType: gp3
            VolumeSize: !Ref Ec2EbsVolumeSize
            Iops: !Ref Ec2EbsVolumeIops
            DeleteOnTermination: true
      UserData:
        Fn::Base64:
          !Sub |
            #!/bin/bash
            hostnamectl --static set-hostname k8s-w2
            export STACK_NAME=${AWS::StackName}
            export AWS_DEFAULT_REGION=${AWS::Region}
            echo "STACK_NAME=${AWS::StackName}" >> /etc/environment
            echo "AWS_DEFAULT_REGION=${AWS::Region}" >> /etc/environment
            export KUBERNETES_VERSION=${KubernetesVersion}

            apt-get update -y
            apt-get install -y awscli python3-pip
            pip3 install --break-system-packages https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-py3-latest.tar.gz \
              || pip3 install https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-py3-latest.tar.gz

            set -x
            exec > >(tee -a /var/log/userdata.log) 2>&1
            on_err() {
              local rc=$?
              echo "FAIL: line=$1 rc=$rc cmd=$2" > /tmp/userdata-fail.txt
              local PFX="s3://my-k8s-scripts/runtime/${AWS::StackName}"
              aws s3 cp /var/log/userdata.log $PFX/W2EC2-userdata.log 2>/dev/null || true
              aws s3 cp /tmp/userdata-fail.txt $PFX/W2EC2-fail-reason.txt 2>/dev/null || true
              [ -f /root/init.log   ] && aws s3 cp /root/init.log   $PFX/W2EC2-init.log   2>/dev/null || true
              [ -f /root/worker.log ] && aws s3 cp /root/worker.log $PFX/W2EC2-worker.log 2>/dev/null || true
              cfn-signal --success false --reason "line=$1 rc=$rc cmd=$2" \
                --stack ${AWS::StackName} --resource W2EC2 --region ${AWS::Region} || true
            }
            trap 'on_err $LINENO "$BASH_COMMAND"' ERR
            set -eE

            aws s3 cp s3://my-k8s-scripts/init_v2.sh   /root/init.sh
            aws s3 cp s3://my-k8s-scripts/worker_v2.sh /root/worker.sh
            chmod +x /root/init.sh /root/worker.sh
            sed -i 's/\r$//' /root/init.sh /root/worker.sh
            /root/init.sh   > /root/init.log   2>&1
            /root/worker.sh > /root/worker.log 2>&1

            cfn-signal --success true --stack ${AWS::StackName} --resource W2EC2 --region ${AWS::Region}

  # ============================================================
  # Worker Node 3 (only if WorkerCount = 3) - in PublicSubnet2/AZ2
  # ============================================================
  Instance4ENI1:
    Type: AWS::EC2::NetworkInterface
    Condition: CreateWorker3
    Properties:
      SubnetId: !Ref PublicSubnet2
      Description: !Sub "${AWS::StackName}-Instance4ENI1"
      GroupSet: [ !Ref EC2SG ]
      PrivateIpAddress: 192.168.20.103
      SourceDestCheck: false
      Tags:
        - { Key: Name, Value: !Sub "${AWS::StackName}-Instance4ENI1" }

  W3EC2:
    Type: AWS::EC2::Instance
    Condition: CreateWorker3
    DependsOn: MEC2
    CreationPolicy:
      ResourceSignal:
        Timeout: PT25M
    Properties:
      InstanceType: !Ref WorkerNodeInstanceType
      ImageId: !Ref LatestAmiId
      KeyName: !Ref KeyName
      IamInstanceProfile: !Ref EC2InstanceProfile
      Tags:
        - { Key: Name, Value: !Sub "${AWS::StackName}-Worker3" }
        - { Key: owner, Value: !Ref OwnerTag }
        - { Key: expiry-date, Value: !Ref ExpiryDate }
      NetworkInterfaces:
        - NetworkInterfaceId: !Ref Instance4ENI1
          DeviceIndex: '0'
      BlockDeviceMappings:
        - DeviceName: /dev/sda1
          Ebs:
            VolumeType: gp3
            VolumeSize: !Ref Ec2EbsVolumeSize
            Iops: !Ref Ec2EbsVolumeIops
            DeleteOnTermination: true
      UserData:
        Fn::Base64:
          !Sub |
            #!/bin/bash
            hostnamectl --static set-hostname k8s-w3
            export STACK_NAME=${AWS::StackName}
            export AWS_DEFAULT_REGION=${AWS::Region}
            echo "STACK_NAME=${AWS::StackName}" >> /etc/environment
            echo "AWS_DEFAULT_REGION=${AWS::Region}" >> /etc/environment
            export KUBERNETES_VERSION=${KubernetesVersion}

            apt-get update -y
            apt-get install -y awscli python3-pip
            pip3 install --break-system-packages https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-py3-latest.tar.gz \
              || pip3 install https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-py3-latest.tar.gz

            set -x
            exec > >(tee -a /var/log/userdata.log) 2>&1
            on_err() {
              local rc=$?
              echo "FAIL: line=$1 rc=$rc cmd=$2" > /tmp/userdata-fail.txt
              local PFX="s3://my-k8s-scripts/runtime/${AWS::StackName}"
              aws s3 cp /var/log/userdata.log $PFX/W3EC2-userdata.log 2>/dev/null || true
              aws s3 cp /tmp/userdata-fail.txt $PFX/W3EC2-fail-reason.txt 2>/dev/null || true
              [ -f /root/init.log   ] && aws s3 cp /root/init.log   $PFX/W3EC2-init.log   2>/dev/null || true
              [ -f /root/worker.log ] && aws s3 cp /root/worker.log $PFX/W3EC2-worker.log 2>/dev/null || true
              cfn-signal --success false --reason "line=$1 rc=$rc cmd=$2" \
                --stack ${AWS::StackName} --resource W3EC2 --region ${AWS::Region} || true
            }
            trap 'on_err $LINENO "$BASH_COMMAND"' ERR
            set -eE

            aws s3 cp s3://my-k8s-scripts/init_v2.sh   /root/init.sh
            aws s3 cp s3://my-k8s-scripts/worker_v2.sh /root/worker.sh
            chmod +x /root/init.sh /root/worker.sh
            sed -i 's/\r$//' /root/init.sh /root/worker.sh
            /root/init.sh   > /root/init.log   2>&1
            /root/worker.sh > /root/worker.log 2>&1

            cfn-signal --success true --stack ${AWS::StackName} --resource W3EC2 --region ${AWS::Region}

Outputs:
  MasterNodeIP:
    Description: Public IP of master node
    Value: !GetAtt MEC2.PublicIp
  MasterNodePrivateIP:
    Description: Private IP of master node (kubeadm advertise)
    Value: !GetAtt MEC2.PrivateIp
  WorkerNode1IP:
    Description: Public IP of worker1
    Value: !GetAtt W1EC2.PublicIp
  WorkerNode2IP:
    Condition: CreateWorker2
    Value: !GetAtt W2EC2.PublicIp
  WorkerNode3IP:
    Condition: CreateWorker3
    Value: !GetAtt W3EC2.PublicIp
  EfsFileSystemID:
    Value: !Ref ElasticFileSystem
  EfsDnsName:
    Description: EFS DNS for NFS mount
    Value: !Sub "${ElasticFileSystem}.efs.${AWS::Region}.amazonaws.com"
  S3RuntimePrefix:
    Description: Where master uploads admin.conf for workers
    Value: !Sub "s3://my-k8s-scripts/runtime/${AWS::StackName}/"