cloud-config.yml

#cloud-config

# Instance
  - %(LOCAL_SSH_KEY)s
bootcmd:
- set -ex
- cloud-init-per once ssh-users-ca echo "TrustedUserCAKeys /etc/ssh/users_ca.pub" >> /etc/ssh/sshd_config
# - cloud-init-per once accepted-oracle-license-v1-1 echo "oracle-java8-installer shared/accepted-oracle-license-v1-1 select true" | debconf-set-selections
- cloud-init-per once fallocate-swapfile fallocate -l 4G /swapfile
- cloud-init-per once chmod-swapfile chmod 600 /swapfile
- cloud-init-per once mkswap-swapfile mkswap /swapfile

output:
  all: '| tee -a /var/log/cloud-init-output.log'
runcmd:
# install_packages_all
- set -ex
- sleep 30
- sudo apt-get update
- sudo apt-get -y install awscli
- sudo apt-get -y install git
- sudo apt-get -y install build-essential
- sudo apt-get -y install unattended-upgrades
- sudo apt-get -y install libbz2-dev
- sudo apt-get -y install libdb-dev
- sudo apt-get -y install libffi-dev
- sudo apt-get -y install libgdbm-dev
- sudo apt-get -y install liblzma-dev
- sudo apt-get -y install libncurses5-dev
- sudo apt-get -y install libnss3-dev
- sudo apt-get -y install libreadline-dev
- sudo apt-get -y install libssl-dev
- sudo apt-get -y install libsqlite3-dev
- sudo apt-get -y install python2
- sudo apt-get -y install python3-dev
- sudo apt-get -y install python3-pip
- sudo apt-get -y install python3-venv
- sudo apt-get -y install python3-testresources
- sudo apt-get -y install zlib1g-dev
- sudo apt-get -y install redis-server
- sudo apt-get -y install nagios-nrpe-plugin
- sudo apt-get -y install monitoring-plugins
- sudo apt-get -y install nagios-nrpe-server
- sudo apt-get -y install libjson-perl
- sudo apt-get -y install libmonitoring-plugin-perl
- sudo apt-get -y install liblwp-useragent-determined-perl
- sudo apt-get -y install sysstat
- sudo apt-get -y install libarchive-tools
# Manually install java
- sudo -u ubuntu aws s3 cp --region=us-west-2 --recursive s3://t2depi-conf-prod/.aws ~ubuntu/.aws
- sudo mkdir -p /usr/lib/jvm
- sudo chmod 777 /usr/lib/jvm
- sudo tar -xzvf /home/ubuntu/.aws/jdk-11.0.3_linux-x64_bin.tar.gz --directory /usr/lib/jvm/
- sudo -u ubuntu rm -r /home/ubuntu/.aws
- sudo chmod 755 /usr/lib/jvm
- sudo update-alternatives --install /usr/bin/java java /usr/lib/jvm/jdk-11.0.3/bin/java 100
- sudo -u ubuntu mv /home/ubuntu/.ssh/authorized_keys /home/ubuntu/.ssh/authorized_keys2
# add team public ssh keys
- sudo -u ubuntu aws s3 cp --region=us-west-2 s3://t2depi-conf-prod/ssh-keys/prod-authorized_keys/authorized_keys /home/ubuntu/.ssh/authorized_keys
# install_packages_encoded_app
- sudo apt-get -y install apache2
- sudo apt-get -y install apache2-dev
- sudo apt-get -y install apache2-utils
- sudo apt-get -y install debhelper
- sudo apt-get -y install dh-autoreconf
- sudo apt-get -y install ssl-cert
- sudo apt-get -y install libapache2-mod-wsgi-py3
- sudo apt-get -y install w3m
- sudo apt-get -y install libjpeg8-dev
- sudo apt-get -y install libpq-dev
- sudo apt-get -y install graphviz
# install_and_setup_postgres
- sudo wget -qO - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo tee /etc/apt/pg_gpg_key
- sudo apt-key add /etc/apt/pg_gpg_key
- echo "deb http://apt.postgresql.org/pub/repos/apt/ focal-pgdg main" | sudo tee -a /etc/apt/sources.list
- sudo apt-get update
- sudo apt-get -y install lzop
- sudo apt-get -y install pv
- sudo apt-get -y install daemontools
- sudo apt-get -y install libicu66
- sudo apt-get -y install postgresql-11
# install_and_setup_elasticsearch
- sudo wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo tee /etc/apt/es_gpg_key
- sudo apt-key add /etc/apt/es_gpg_key
- echo "deb https://artifacts.elastic.co/packages/5.x/apt stable main" | sudo tee -a /etc/apt/sources.list
- sudo apt-get -y install apt-transport-https
- sudo apt update
- yes |sudo apt-get -y install elasticsearch
# install_libmagic_packages
- wget http://archive.ubuntu.com/ubuntu/pool/main/f/file/libmagic-mgc_5.38-4_amd64.deb && sudo dpkg -i libmagic-mgc_5.38-4_amd64.deb && sudo apt-get install -yf
- wget http://archive.ubuntu.com/ubuntu/pool/main/f/file/libmagic1_5.38-4_amd64.deb && sudo dpkg -i libmagic1_5.38-4_amd64.deb && sudo apt-get install -yf
- wget http://archive.ubuntu.com/ubuntu/pool/main/f/file/libmagic-dev_5.38-4_amd64.deb && sudo dpkg -i libmagic-dev_5.38-4_amd64.deb && sudo apt-get install -yf
- sudo apt --fix-broken -y  install && sudo apt autoremove -y
# configure_apache2_logrotation
- sudo mkdir -p /etc/systemd/system/logrotate.timer.d

- echo "[Timer]" >> override.conf
- echo "OnCalendar=" >> override.conf
- echo "OnCalendar=*-*-* 14:00:00" >> override.conf

- sudo mv override.conf /etc/systemd/system/logrotate.timer.d/


# Ideally this would build as a different user so encoded only has read
# permissions  
- MEMGIGS=$(awk '/MemTotal/{printf "%%.0f", $2 / 1024**2}' /proc/meminfo)
- if [ "$MEMGIGS" -gt 32 ]
- then
-    echo "ES_HEAP_SIZE=16g" > /etc/default/elasticsearch
- elif [ "$MEMGIGS" -gt 12 ]
- then
-    echo "ES_HEAP_SIZE=8g" > /etc/default/elasticsearch
- else
-    echo "ES_HEAP_SIZE=4g" > /etc/default/elasticsearch
-    sysctl "vm.swappiness=1"
-    swapon /swapfile
- fi
- update-rc.d elasticsearch defaults
- service elasticsearch start
- chown postgres:postgres /etc/postgresql/11/main/*.conf
- echo "include 'custom.conf'" >> /etc/postgresql/11/main/postgresql.conf
- if test "%(ROLE)s" = "demo"
- then
-   echo "standby_mode = off" >> /etc/postgresql/11/main/recovery.conf
-   echo "include 'demo.conf'" >> /etc/postgresql/11/main/postgresql.conf
- fi
- sudo -u postgres createuser snowflakes
- sudo -u postgres createdb --owner=snowflakes snowflakes
- mkdir /srv/snowflakes
- chown snowflakes:snowflakes /srv/snowflakes
- cd /srv/snowflakes
- sudo -u snowflakes git clone --no-checkout https://github.com/T2DREAM/snovault.git .
- sudo -u snowflakes git checkout %(COMMIT)s
- yes |sudo apt install software-properties-common
- sudo add-apt-repository -y ppa:deadsnakes/ppa
- sudo apt-get update
- sudo apt-get install -y python3.6 python3.6-dev python3.6-venv
- curl -sL https://deb.nodesource.com/setup_16.x | sudo -E bash -
- sudo apt-get -y install nodejs
- sudo apt-get -y install g++
- sudo apt-get -y install gcc
- sudo apt-get -y install make
- cd /srv/snowflakes
- sudo pip install --upgrade botocore==1.12.36
- mkdir /opt/wal-e
- sudo chown postgres:postgres /opt/wal-e
- sudo pip install virtualenv
- sudo -u postgres virtualenv --python=python2.7 /opt/wal-e
- sudo -u postgres /opt/wal-e/bin/pip install -r wal-e-requirements.txt --target=/opt/wal-e/
- /etc/init.d/postgresql start
- sudo pip3 install -U zc.buildout setuptools==43
- sudo -u snowflakes buildout bootstrap
- sudo -u snowflakes LANG=en_US.UTF-8 bin/buildout -c %(ROLE)s.cfg production-ini:region_search_instance=localhost:9200
- sudo apt remove -y python3-botocore
- sudo -u snowflakes bin/aws s3 cp --recursive s3://t2depi-conf-prod/.aws .aws
## TODO this conf needs be copied to something more generic
- until sudo -u postgres psql postgres -c ""; do sleep 10; done
- sudo -u snowflakes sh -c 'cat /dev/urandom | head -c 256 | base64 > session-secret.b64'
- sudo -u snowflakes bin/create-mapping production.ini --app-name app
- ln -s /srv/snowflakes/etc/snowflakes-apache.conf /etc/apache2/sites-available/snowflakes.conf
- ln -s /srv/snowflakes/etc/logging-apache.conf /etc/apache2/conf-available/logging.conf
- a2enmod headers
- a2enmod proxy_http
- a2enmod rewrite
- a2enmod ssl
- a2ensite snowflakes.conf
- a2dissite 000-default
- a2enconf logging
- a2disconf charset
- a2disconf security
- a2disconf localized-error-pages
- a2disconf other-vhosts-access-log
- a2disconf serve-cgi-bin
- if test "%(ROLE)s" = "demo"
- then
-   sudo -i -u snowflakes bin/batchupgrade production.ini --app-name app
- fi

users:
- default
- name: build
  gecos: Build user
  inactive: true
  system: true
- name: snowflakes
  gecos: Snowflake Database daemon user
  inactive: true
  system: true
  # Specified homedir must exist
  # https://github.com/rubygems/rubygems/issues/689
  homedir: /srv/snowflakes

write_files:
- path: /etc/apt/apt.conf.d/20auto-upgrades
  content: |
    APT::Periodic::Update-Package-Lists "1";
    APT::Periodic::Unattended-Upgrade "1";

- path: /etc/apt/apt.conf.d/50unattended-upgrades
  content: |
    Unattended-Upgrade::Allowed-Origins {
        "${distro_id} ${distro_codename}-security";
    };
    Unattended-Upgrade::Automatic-Reboot "true";


- path: /etc/elasticsearch/elasticsearch.yml
  content: |
    index.search.slowlog.threshold.query.warn: 8s
    index.search.slowlog.threshold.fetch.warn: 1s
    index.indexing.slowlog.threshold.index.warn: 25s

- path: /etc/postgresql/11/main/custom.conf
  content: |
    hot_standby = on
    max_standby_archive_delay = -1
    wal_level = hot_standby
    archive_mode = on
    archive_timeout = 60
    # http://www.postgresql.org/message-id/CAOycyLTm6X3mVLz+sLCex+W==WSMgu9giteV7efPoPXYDhPtzQ@mail.gmail.com
    checkpoint_timeout = 1h

- path: /etc/postgresql/11/main/demo.conf
  content: |
    archive_mode = off

- path: /etc/postgresql/11/main/master.conf
  content: |
    archive_command = '/opt/wal-e/bin/envfile --config ~postgres/.aws/credentials --section default --upper -- /opt/wal-e/bin/wal-e --s3-prefix="$(cat /etc/postgresql/11/main/wale_s3_prefix)" wal-push "%%p"'

- path: /etc/postgresql/11/main/recovery.conf
  content: |
    # recovery.conf must be linked into data dir to do anything
    recovery_target_timeline = 'latest'
    restore_command = '/opt/wal-e/bin/wal-e --aws-instance-profile --s3-prefix="$(cat /etc/postgresql/11/main/wale_s3_prefix)" wal-fetch "%%f" "%%p"'
    standby_mode = on

- path: /etc/postgresql/11/main/wale_s3_prefix
  content: "%(WALE_S3_PREFIX)s"

- path: /etc/ssh/users_ca.pub
  content: ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAv/ymOcnN4LhM4NACc3Or116XXJ6KytuOgB/+1qNkOFBqBosrn7cmJ35rsoNHRgYNrCsRE9ch74RKsN6H72FtSJgBhGh/9oUK7Os6Fqt3/ZZXxgxIx6ubs/MTgrxrAnujiBxUXMXQhLKMriNMpo8mt4nGYVtLk9PBjiyfncaS8H9ZKoNio9dhP8bmTuYvioAI35dqKdSlVLyzr/XkZxia8Ki+pQ0N6uuiEwMR3ToM+LSp8wpFOOAiu4PEAujRW7us/+1hlpKWfn0J7/V3826joHE+I967Vg/+ikcVhF77JjK1nib879VgCWfmn1HPQosIpk4yJfVgGvRVI7I2nfBPVw== encoded@demo-l.encodedcc.org