existing-vpc-single-kafka.yml

ROSTemplateFormatVersion: '2015-09-01'
Description:
  zh-cn: 在现有VPC环境下，创建并配置ECS运行Kafka中间件，搭载Java 1.8.0、Scala 2.12、Kafka 0.10.2.2，自动挂载数据盘存储数据，设置公网访问，实现资源自动化部署。
  en: In the existing VPC environment, create and configure ECS instances to run Kafka
    middleware, equipped with Java 1.8.0, Scala 2.12, and Kafka 0.10.2.2. Automatically
    mount data disks for data storage, enable public network access, and implement
    automated resource deployment.
Parameters:
  VSwitchZoneId:
    Type: String
    Label:
      en: VSwitch Zone ID
      zh-cn: 交换机可用区
    Description:
      en: Availability ID for existing switches
      zh-cn: 现有交换机的可用区ID
    AssociationProperty: ALIYUN::ECS::Instance::ZoneId
  VpcId:
    Type: String
    Label:
      en: Existing VPC Instance ID
      zh-cn: 现有VPC的实例ID
    Description:
      en: Please search the ID starting with (vpc-xxx)from console-Virtual Private
        Cloud
      zh-cn: 现有虚拟专有网络的实例ID,控制台-VPC-专有网络下查询
    AssociationProperty: ALIYUN::ECS::VPC::VPCId
  VSwitchId:
    Type: String
    Label:
      en: VSwitch ID
      zh-cn: 网络交换机ID
    Description:
      en: Please search the business vswitch ID starting with(vsw-xxx)from console-Virtual
        Private Cloud-VSwitches
      zh-cn: 现有业务网络交换机的实例ID,控制台-VPC-专有网络-交换机下查询
    AssociationProperty: ALIYUN::ECS::VSwitch::VSwitchId
    AssociationPropertyMetadata:
      VpcId: VpcId
      ZoneId: VSwitchZoneId
  SecurityGroupId:
    Type: String
    Label:
      en: Business Security Group ID
      zh-cn: 业务安全组ID
    Description:
      en: Please search the business security group ID starting with(sg-xxx)from console-ECS-Network
        & Security
      zh-cn: 现有业务安全组的实例ID,控制台-ECS-网络与安全-安全组下查询
    AssociationProperty: ALIYUN::ECS::SecurityGroup::SecurityGroupId
    AssociationPropertyMetadata:
      VpcId: VpcId
  InstanceType:
    Type: String
    Label:
      en: Instance Type
      zh-cn: 实例规格
    Description:
      en: <font color='blue'><b>1.Before selecting the model please confirm that the
        current available zone under the model is in stock, some models need to be
        reported in advance</b></font>]<br><font color='blue'><b>2.List of optional
        models</font>]<br></b></font>[ecs.c5.large <font color='green'>2vCPU 4GiB
        Intranet bandwidth1Gbps In-grid sending and receiving packages30MillionPPSS</font>]<br></b>[ecs.c5.xlarge
        <font color='green'>4vCPU 8GiB Intranet bandwidth1.5Gbps In-grid sending and
        receiving packages50MillionPPS</font>]<br></b>[ecs.c5.2xlarge <font color='green'>8vCPU
        16GiB Intranet bandwidth2.5Gbps In-grid sending and receiving packages80MillionPPS</font>]
      zh-cn: <font color='blue'><b>1.选择机型前请先确认当前可用区下该机型是否有货，部分机型需要提前报备</b></font><br><font
        color='blue'><b>2.可选机型列表</font><br></b></font>[ecs.c5.large <font color='green'>2vCPU
        4GiB 内网带宽1Gbps 内网收发包30万PPS</font>]<br></b>[ecs.c5.xlarge <font color='green'>4vCPU
        8GiB 内网带宽1.5Gbps 内网收发包50万PPS</font>]<br></b>[ecs.c5.2xlarge <font color='green'>8vCPU
        16GiB 内网带宽2.5Gbps 内网收发包80万PPS</font>]
    AssociationProperty: ALIYUN::ECS::Instance::InstanceType
    AssociationPropertyMetadata:
      ZoneId: VSwitchZoneId
  InstancePassword:
    Type: String
    Label:
      en: Instance Password
      zh-cn: 实例密码
    Description:
      en: Server login password, Length 8-30, must contain three(Capital letters,
        lowercase letters, numbers, ()`~!@#$%^&*_-+=|{}[]:;'<>,.?/ Special symbol
        in)
      zh-cn: 服务器登录密码,长度8-30，必须包含三项（大写字母、小写字母、数字、 ()`~!@#$%^&*_-+=|{}[]:;'<>,.?/ 中的特殊符号）
    ConstraintDescription:
      en: Length 8-30, must contain three(Capital letters, lowercase letters, numbers,
        ()`~!@#$%^&*_-+=|{}[]:;'<>,.?/ Special symbol in).
      zh-cn: 长度8-30，必须包含三项（大写字母、小写字母、数字、 ()`~!@#$%^&*_-+=|{}[]:;'<>,.?/ 中的特殊符号）。
    AllowedPattern: '[0-9A-Za-z\_\-\&:;''<>,=%`~!@#\(\)\$\^\*\+\|\{\}\[\]\.\?\/]+$'
    MinLength: 8
    MaxLength: 30
    NoEcho: true
  BindWidth:
    Type: Number
    Label:
      en: Public IP Bandwidth
      zh-cn: 公网IP带宽值
    Description:
      en: 'Public network IP bandwidth，unit: Mbps'
      zh-cn: 公网IP带宽值，单位：Mbps
    Default: 5
    MinValue: 1
    MaxValue: 100
  DiskCategory:
    Type: String
    Label:
      en: Disk Type
      zh-cn: 磁盘类型
  DiskSize:
    Type: Number
    Label:
      en: Data Disk Space
      zh-cn: 数据盘空间
    Description:
      en: 'Instance disk size, range of values: 20-32768, units: GB'
      zh-cn: 实例数据盘大小，单位为GiB。取值范围：20~32768
    Default: 100
    MinValue: 20
    MaxValue: 32768
  ListenPort:
    Type: Number
    Label:
      en: Kafka Listeners Port
      zh-cn: Kafka监听端口
    Description:
      en: Kafka Listeners Port, please use more than 1000 port Numbers
      zh-cn: Kafka监听端口，请使用1000以上的端口号
    Default: 9092
    MinValue: 1000
    MaxValue: 65535
  MsgTime:
    Type: Number
    Label:
      en: Message Retention Time
      zh-cn: 消息保留时间
    Description:
      en: Maximum message retention time in hours
      zh-cn: 消息保留最长时间, 单位：小时
    Default: 24
    AllowedValues:
    - 24
    - 48
    - 72
    - 96
    - 120
    - 144
    - 168
  TopicClass:
    Type: Number
    Label:
      en: Topic Specifications
      zh-cn: Topic规格
    Description:
      en: number of topics
      zh-cn: topic数量
    Default: 50
    MinValue: 50
    MaxValue: 450
  InstanceImageId:
    Type: String
    Label:
      en: Image ID
      zh-cn: 镜像ID
    Description:
      en: Image ID，See detail：<b><a href='https://www.alibabacloud.com/help/doc-detail/112977.html'
        target='_blank'><font color='blue'>Find the mirror</font></a></b>
      zh-cn: 镜像ID, 详见：<b><a href='https://help.aliyun.com/document_detail/112977.html'
        target='_blank'><font color='blue'>查找镜像</font></a></b>
    AssociationProperty: ALIYUN::ECS::Image::ImageId
    AssociationPropertyMetadata:
      InstanceType: ${InstanceType}
      SupportedImageOwnerAlias:
        - system
        - self
        - others
    Default: centos_7_04_64_20G_alibase_201701015.vhd
Resources:
  EcsInstanceGroup:
    Type: ALIYUN::ECS::InstanceGroup
    Properties:
      VpcId:
        Ref: VpcId
      VSwitchId:
        Ref: VSwitchId
      SecurityGroupId:
        Ref: SecurityGroupId
      ImageId:
        Ref: InstanceImageId
      AllocatePublicIP: false
      DiskMappings:
      - Category:
          Ref: DiskCategory
        DiskName: kafka_data_disk
        Size:
          Ref: DiskSize
      HostName:
        Fn::Join:
        - ''
        - - Kafka
          - -[0,3]
      InstanceChargeType: PostPaid
      InstanceName:
        Fn::Join:
        - ''
        - - Kafka
          - -[0,3]
      InstanceType:
        Ref: InstanceType
      InternetMaxBandwidthIn:
        Ref: BindWidth
      InternetMaxBandwidthOut:
        Ref: BindWidth
      IoOptimized: optimized
      MaxAmount: 1
      Password:
        Ref: InstancePassword
      SystemDiskCategory:
        Ref: DiskCategory
      UserData:
        Fn::Replace:
        - ros-notify:
            Fn::GetAtt:
            - RosWaitConditionHandle
            - CurlCli
        - Fn::Join:
          - ''
          - - "#!/bin/sh \n"
            - "set -e \n"
            - "sleep 10 \n"
            - "FOLDER_NAME=\"/home/software\" \n"
            - "mkdir -p ${FOLDER_NAME} \n"
            - "### mount data disk \n"
            - "LOGS=/root/mount_logs \n"
            - "DISK_SUFFIX=\"b\" \n"
            - "touch ~/test_ftab \n"
            - 'cat << ESXU > ~/test_ftab

              '
            - 'fdisk -S 56 /dev/vd${DISK_SUFFIX} <<ESXU

              '
            - 'n

              '
            - 'p

              '
            - '1

              '
            - '

              '
            - '

              '
            - 'w

              '
            - 'ESXU

              '
            - "echo \"/dev/vd${DISK_SUFFIX} is fdisked\" >> ${LOGS} \n"
            - "mkfs.ext4 /dev/vd${DISK_SUFFIX} \n"
            - "if [[ $? -eq 0 ]];then \n"
            - "  echo \"/dev/vd${DISK_SUFFIX} is formated\" >> ${LOGS} \n"
            - "fi \n"
            - "touch ~/test_ftab \n"
            - 'cat << ESXU > ~/test_ftab

              '
            - '/dev/vd${DISK_SUFFIX}         ${FOLDER_NAME}       ext4       defaults        0
              0

              '
            - 'ESXU

              '
            - "cat ~/test_ftab >> /etc/fstab \n"
            - "mount -a \n"
            - "rm -rf ~/test_ftab \n"
            - "chmod -R 777 ${FOLDER_NAME} \n"
            - "echo \"/dev/vd${DISK_SUFFIX}1 is mounted\" \n"
            - "echo `df -h` >> 123.text \n"
            - "echo `fdisk -l` >> 123.text \n"
            - "#mount /dev/xvdb /software \n"
            - "### kafka full name = \"kafka_2.12-0.10.2.2.tgz\" \n"
            - "### jdk full name = \"jdk-8u251-linux-i586.tar.gz\" \n"
            - "HOST_IP=`ifconfig eth0 | awk '/inet /{print $2}'` \n"
            - "### --- install and config java \n"
            - "sudo yum -y install glibc.i686 \n"
            - "BASH_PROFILE=\"/etc/profile\" \n"
            - "OSS_NAME=\"ros-template-resources\" \n"
            - "OSS_REGION=\"cn-beijing\" \n"
            - "JDK_FOLDER_NAME=\"jdk-8u251-linux-i586\" \n"
            - "JDK_PACKAGE_NAME=\"${JDK_FOLDER_NAME}.rpm\" \n"
            - "cd ${FOLDER_NAME} \n"
            - "wget https://${OSS_NAME}.oss-${OSS_REGION}.aliyuncs.com/Kafka/${JDK_PACKAGE_NAME}\
              \ \n"
            - "#tar -zxvf ${JDK_PACKAGE_NAME} && rm -f ${JDK_PACKAGE_NAME} \n"
            - "rpm -Uvh jdk-8u251-linux-i586.rpm && rm -f ${JDK_PACKAGE_NAME} \n"
            - "JAVA_HOME=`find / -name jdk1.8.0_*` \n"
            - "echo export JAVA_HOME=${JAVA_HOME} >> /etc/profile \n"
            - "echo export JRE_HOME=${JAVA_HOME}/jre >> /etc/profile \n"
            - "echo export CLASSPATH=.:${JAVA_HOME}/lib:${JAVA_HOME}/jre/lib >> /etc/profile\
              \ \n"
            - "echo export PATH=${JAVA_HOME}/bin:$PATH >> /etc/profile \n"
            - "source ${BASH_PROFILE} \n"
            - "### install kafka \n"
            - "cd ${FOLDER_NAME} \n"
            - "KAFKA_FOLDER_NAME=\"kafka_2.12-0.10.2.2\" \n"
            - "KAFKA_NAME=\"kafka\" \n"
            - "KAFKA_PACKAGE_NAME=\"${KAFKA_FOLDER_NAME}.tgz\" \n"
            - "wget https://ros-template-resources.oss-cn-beijing.aliyuncs.com/Kafka/${KAFKA_PACKAGE_NAME}\
              \ \n"
            - "tar -zvxf ${KAFKA_PACKAGE_NAME} && rm -f ${KAFKA_PACKAGE_NAME} \n"
            - "mv ${KAFKA_FOLDER_NAME} ${KAFKA_NAME} && cd ${KAFKA_NAME} \n"
            - "### config zookeeper  port=2181 \n"
            - "mkdir -p ${FOLDER_NAME}/kafka/data/zookeeper && cd ${FOLDER_NAME}/kafka/data/zookeeper\
              \ \n"
            - "ZOOKEEPER_DATA_DIR=`pwd` \n"
            - "sed -i \"s%dataDir=/tmp/zookeeper%dataDir=${ZOOKEEPER_DATA_DIR}%\"\
              \ ${FOLDER_NAME}/kafka/config/zookeeper.properties \n"
            - "mkdir -p ${FOLDER_NAME}/kafka/log/zookeeper && cd ${FOLDER_NAME}/kafka/log/zookeeper\
              \ \n"
            - "ZOOKEEPER_DATA_LOG_DIR=`pwd` \n"
            - "echo dataLogDir=${ZOOKEEPER_DATA_LOG_DIR} >> ${FOLDER_NAME}/kafka/config/zookeeper.properties\
              \ \n"
            - "### config kafka \n"
            - 'sed -i "s%#listeners=PLAINTEXT://:9092%listeners=PLAINTEXT://$HOST_IP:'
            - Ref: ListenPort
            - "%\" ${FOLDER_NAME}/kafka/config/server.properties \n"
            - 'sed -i "s%#advertised.listeners=PLAINTEXT://your.host.name:9092%advertised.listeners=PLAINTEXT://$HOST_IP:'
            - Ref: ListenPort
            - "%\" ${FOLDER_NAME}/kafka/config/server.properties \n"
            - "mkdir -p ${FOLDER_NAME}/log/kafka && cd ${FOLDER_NAME}/log/kafka \n"
            - "KAFKA_LOG_DIR=`pwd` \n"
            - "sed -i \"s%log.dirs=/tmp/kafka-logs%log.dirs=${KAFKA_LOG_DIR}%\" ${FOLDER_NAME}/kafka/config/server.properties\
              \ \n"
            - sed -i "s%num.partitions=1%num.partitions=
            - Ref: TopicClass
            - "%\" ${FOLDER_NAME}/kafka/config/server.properties \n"
            - sed -i "s%log.retention.hours=168%log.retention.hours=
            - Ref: MsgTime
            - "%\" ${FOLDER_NAME}/kafka/config/server.properties \n"
            - "sed -i \"s%zookeeper.connect=localhost:2181%zookeeper.connect=${HOST_IP}:2181%\"\
              \ ${FOLDER_NAME}/kafka/config/server.properties \n"
            - "### install and config kafka \n"
            - "yum install -y unzip \n"
            - "wget https://${OSS_NAME}.oss-${OSS_REGION}.aliyuncs.com/Kafka/kafka-manager-1.3.3.23.zip\
              \ \n"
            - "unzip kafka-manager-1.3.3.23.zip -d ${FOLDER_NAME} && rm -f kafka-manager-1.3.3.23.zip\
              \ \n"
            - "cd ${FOLDER_NAME} && mv kafka-manager-1.3.3.23 kafka-manager && cd\
              \ kafka-manager \n"
            - "sed -i \"s%kafka-manager-zookeeper%${HOST_IP}%\" ${FOLDER_NAME}/kafka-manager/conf/application.conf\
              \ \n"
            - "### set start up and start kafka \n"
            - "cd ${FOLDER_NAME}/kafka \n"
            - "touch start_up.sh && chmod +x start_up.sh \n"
            - "echo '#!/bin/bash' >> start_up.sh \n"
            - "echo \"cd ${FOLDER_NAME}/kafka/bin\" >> start_up.sh \n"
            - "echo \"/usr/bin/nohup ./zookeeper-server-start.sh ../config/zookeeper.properties\
              \  > ${FOLDER_NAME}/kafka/zookeeper.start.log 2>&1 &\" >> start_up.sh\
              \ \n"
            - "sleep 5 \n"
            - "echo \"/usr/bin/nohup ./kafka-server-start.sh ../config/server.properties\
              \ > ${FOLDER_NAME}/kafka/kafka.start.log 2>&1 &\" >> start_up.sh \n"
            - "echo \"cd ${FOLDER_NAME}/kafka-manager/bin\" >> start_up.sh \n"
            - "sleep 5 \n"
            - "echo \"/usr/bin/nohup ./kafka-manager -Dconfig.file=../conf/application.conf\
              \ -Dhttp.port=9000 &\" >> start_up.sh \n"
            - "echo \"bash ${FOLDER_NAME}/kafka/start_up.sh\" >> /etc/rc.local \n"
            - "chmod +x /etc/rc.d/rc.local \n"
            - "./start_up.sh \n"
            - "sleep 5 \n"
            - "ros-notify -d \"{\\\"Data\\\" : \\\"${HOST_IP}\\\", \\\"Status\\\"\
              \ : \\\"Success\\\"}\" \n"
  VpcEip:
    Type: ALIYUN::VPC::EIP
    Properties:
      Bandwidth:
        Ref: BindWidth
      InternetChargeType: PayByTraffic
  EipAssociation:
    Type: ALIYUN::VPC::EIPAssociation
    Properties:
      InstanceId:
        Fn::Select:
        - '0'
        - Fn::GetAtt:
          - EcsInstanceGroup
          - InstanceIds
      AllocationId:
        Ref: VpcEip
    DependsOn:
    - VpcEip
  RosWaitConditionHandle:
    Type: ALIYUN::ROS::WaitConditionHandle
    Properties: {}
  RosWaitCondition:
    Type: ALIYUN::ROS::WaitCondition
    Properties:
      Count: 1
      Handle:
        Ref: RosWaitConditionHandle
      Timeout: 1800
Outputs:
  AddClusterZookeeperHostsInfo:
    Value:
      Fn::Join:
      - ''
      - - Fn::Select:
          - '0'
          - Fn::GetAtt:
            - EcsInstanceGroup
            - PrivateIps
        - :2181
  EcsEip:
    Value:
      Fn::GetAtt:
      - VpcEip
      - EipAddress
  EcsInstanceIds:
    Value:
      Fn::Select:
      - '0'
      - Fn::GetAtt:
        - EcsInstanceGroup
        - InstanceIds
  EcsPrivateIp:
    Value:
      Fn::Select:
      - '0'
      - Fn::GetAtt:
        - EcsInstanceGroup
        - PrivateIps
  KafkaManagerUrl:
    Value:
      Fn::Join:
      - ''
      - - http://
        - Fn::GetAtt:
          - VpcEip
          - EipAddress
        - :9000
Metadata:
  ALIYUN::ROS::Interface:
    ParameterGroups:
    - Parameters:
      - VpcId
      - VSwitchZoneId
      - VSwitchId
      - SecurityGroupId
      Label:
        default:
          en: Infrastructure Configuration
          zh-cn: 基础资源配置（必填）
    - Parameters:
      - InstanceType
      - InstanceImageId
      - BindWidth
      - DiskCategory
      - DiskSize
      - ListenPort
      - MsgTime
      - TopicClass
      - InstancePassword
      Label:
        default:
          en: Kafka Configuration
          zh-cn: Kafka 配置（必填）
    TemplateTags:
    - acs:example:弹性计算:Kafka单机版(已有VPC)
  ALIYUN::ROS::Composer:
    '65243119':
      Parent: 32d72df7
      Edge:
        - ebb10332
        - '4587e712'
      Line: 0:0:0:gray:0
    0ed017d0:
      Rect:
        - 659
        - 613
        - 40
        - 30
        - 1
        - 0
      ResT: Composer::ROSParameter::AlibabaCloud
    32d72df7:
      Parent: 0ed017d0
      Rect:
        - 609
        - 551
        - 60
        - 70
        - 2
        - 0
      ResT: Composer::ROSParameter::Region
    ca3bda3c:
      Res:
        - VpcEip
        - EipAssociation
      Parent: 32d72df7
      Rect:
        - 40
        - 40
        - 300
        - 110
        - 3
        - 0
    ebb10332:
      Res:
        - RosWaitCondition
      Parent: 32d72df7
      Rect:
        - 40
        - 40
        - 594
        - 200
        - 3
        - 0
    fc06ada9:
      Res:
        - VpcId
      Parent: 32d72df7
      Rect:
        - 480
        - 340
        - 80
        - 200
        - 3
        - 0
      ResT: ALIYUN::ECS::VPC
    '4587e712':
      Res:
        - RosWaitConditionHandle
      Parent: 32d72df7
      Rect:
        - 40
        - 40
        - 594
        - 420
        - 3
        - 0
    7b0a1d4b:
      Res:
        - VSwitchZoneId
      Parent: fc06ada9
      Rect:
        - 440
        - 270
        - 100
        - 250
        - 4
        - 0
      ResT: Composer::ROSParameter::Zone
    e5bc6dc1:
      Res:
        - VSwitchId
      Parent: 7b0a1d4b
      Rect:
        - 400
        - 200
        - 120
        - 300
        - 5
        - 0
      ResT: ALIYUN::ECS::VSwitch
    f9faa6f7:
      Res:
        - EcsInstanceGroup
      Parent: e5bc6dc1
      Rect:
        - 40
        - 40
        - 300
        - 380
        - 6
        - 0
    b253ab15:
      Parent: 32d72df7
      Edge:
        - ca3bda3c
        - f9faa6f7
      Line: 0:0:0:gray:0