Merge pull request #230 from stfc/chatops_docs

Doc for ChatOps

Author: meoflynn

.github/workflows/chatops.yaml

@@ -1,4 +1,4 @@
-name: ChatOps checks
+name: Linting
 
 on:
   push:

chatops_deployment/INSTALL.md

@@ -1,4 +1,29 @@
 # ChatOps Deployment
 
-This project outlines the deployment of the Cloud ChatOps application found [here](https://github.com/stfc/cloud-docker-images/tree/master/cloud-chatops).
-The goal is to create an easily deployable and highly available infrastructure to run the Docker image on. 
+![Linting](https://github.com/stfc/SCD-OpenStack-Utils/actions/workflows/chatops.yaml/badge.svg)
+
+## Contents
+
+- [About](#about)
+
+### About
+
+This project outlines the deployment of the Cloud ChatOps application
+found [here](https://github.com/stfc/cloud-docker-images/tree/master/cloud-chatops). The goal is to create an easily
+deployable and highly available infrastructure to run the Docker container on. We achieve this by using Terraform and
+Ansible to provision and configure a virtual machine the services will run on.
+
+This includes:
+
+- Load balanced application traffic
+- Infrastructure-wide service logging to a central location
+- Service monitoring with visual dashboards and alerting notifications
+- Multi-environment deployment (e.g. dev, staging, prod)
+
+To get started with the deployment, see [INSTALL.md](docs/INSTALL.md).
+
+For information about what services are deployed, see [SERVICES.md](docs/SERVICES.md)
+
+To understand what the Terraform modules do, see [TERRAFORM.md](docs/TERRAFORM.md)
+
+To know what and where variables are stored, see [VARIABLES.md](docks/VARIABLES.md)

chatops_deployment/README.md

@@ -35,26 +35,26 @@
   become: true
   ansible.builtin.stat:
     path: /etc/haproxy/{{ domain }}.crt
-  register: certificate_file
+  register: haproxy_certificate_file
 
 - name: Generate the certificate for the first time
   become: true
   ansible.builtin.command: |
     certbot certonly --standalone --non-interactive --agree-tos --expand --domains \
     {{ domain }},chatops.{{ domain }},prometheus.{{ domain }},grafana.{{ domain }},alertmanager.{{ domain }},kibana.{{ domain }} \
     -m [email protected]
-  register: generate_cert
-  changed_when: generate_cert.rc == 0
-  when: not certificate_file.stat.exists
+  register: haproxy_generate_cert
+  changed_when: haproxy_generate_cert.rc == 0
+  when: not haproxy_certificate_file.stat.exists
 
 - name: Copy certificate for the first time
   become: true
   ansible.builtin.command: |
     cat /etc/letsencrypt/live/{{ domain }}/privkey.pem \
     /etc/letsencrypt/live/{{ domain }}/fullchain.pem > /etc/haproxy/{{ domain }}.crt
-  register: copy_cert
-  changed_when: copy_cert.rc == 0
-  when: not certificate_file.stat.exists
+  register: haproxy_copy_cert
+  changed_when: haproxy_copy_cert.rc != 0
+  when: not haproxy_certificate_file.stat.exists
 
 - name: Create a cron job for the renewal of certificates
   become: true
@@ -99,4 +99,4 @@
   ansible.builtin.systemd_service:
     state: restarted
     name: haproxy.service
-  when: copy_cert.rc == 0
+  when: haproxy_copy_cert.rc == 0

chatops_deployment/ansible/roles/haproxy/tasks/certbot.yml

@@ -19,11 +19,11 @@
   become: true
   ansible.builtin.stat:
     path: /etc/haproxy/{{ domain }}.crt
-  register: certificate_file
+  register: haproxy_certificate_file
 
 - name: Make sure haproxy.service is running
   become: true
   ansible.builtin.systemd_service:
     state: restarted
     name: haproxy.service
-  when: certificate_file.stat.exists
+  when: haproxy_certificate_file.stat.exists

chatops_deployment/ansible/roles/haproxy/tasks/haproxy.yml

@@ -18,50 +18,50 @@
           send "{{ bastion_key_passphrase }}\r"
           expect eof
         EOF
-      register: _
-      changed_when: _.rc == 0
+      register: ssh_known_hosts_
+      changed_when: ssh_known_hosts_.rc != 0
 
 - name: Remove FIP known hosts
   ansible.builtin.command: 'ssh-keygen -R "{{ terraform_floating_ip }}"'
-  register: _
-  changed_when: _.rc == 0
+  register: ssh_known_hosts_
+  changed_when: ssh_known_hosts_.rc != 0
 
 - name: Remove private VM known host entries
   ansible.builtin.command: "ssh-keygen -R {{ item }}"
   loop: "{{ groups['private'] }}"
-  register: _
-  changed_when: _.rc == 0
+  register: ssh_known_hosts_
+  changed_when: ssh_known_hosts_.rc != 0
 
 - name: Add FIP fingerprint to known hosts
   ansible.builtin.command: 'ssh-keyscan "{{ terraform_floating_ip }}" >> ~/.ssh/known_hosts'
-  register: _
-  changed_when: _.rc == 0
+  register: ssh_known_hosts_
+  changed_when: ssh_known_hosts_.rc != 0
 
 - name: Get private VM fingerprints and retrieve to local host
   delegate_to: "{{ terraform_floating_ip }}"
   block:
     - name: Add private VM fingerprints to known hosts on LB
       ansible.builtin.command: 'ssh-keyscan "{{ item }}" >> ~/.ssh/known_hosts'
       loop: "{{ groups['private'] }}"
-      register: _
-      changed_when: _.rc == 0
+      register: ssh_known_hosts_
+      changed_when: ssh_known_hosts_.rc != 0
 
     - name: Retrieve known hosts from LB
       ansible.builtin.fetch:
         src: "~/.ssh/known_hosts"
         dest: "private_known_hosts.tmp"
         flat: true
-      register: _
-      changed_when: _.rc == 0
+      register: ssh_known_hosts_
+      changed_when: ssh_known_hosts_.rc != 0
 
 - name: Append fetched known hosts to localhost
   ansible.builtin.command: "cat private_known_hosts.tmp >> ~/.ssh/known_hosts"
-  register: _
-  changed_when: _.rc == 0
+  register: ssh_known_hosts_
+  changed_when: ssh_known_hosts_.rc != 0
 
 - name: Remove private_known_hosts.tmp
   ansible.builtin.file:
     path: "private_known_hosts.tmp"
     state: absent
-  register: _
-  changed_when: _.rc == 0
+  register: ssh_known_hosts_
+  changed_when: ssh_known_hosts_.rc != 0

chatops_deployment/ansible/roles/ssh_known_hosts/tasks/main.yml

@@ -4,29 +4,29 @@
     - name: Check clouds.yaml
       ansible.builtin.stat:
         path: "~/.config/openstack/clouds.yaml"
-      register: clouds_yaml_state
+      register: terraform_clouds_yaml_state
 
     - name: Fail if clouds.yaml does not exist
       ansible.builtin.fail:
         msg: "Could not find a clouds.yaml in ~/.config/openstack/clouds.yaml"
-      when: not clouds_yaml_state.stat.exists
+      when: not terraform_clouds_yaml_state.stat.exists
 
 - name: Check public and private keys
   block:
     # We can ignore this warning as this command doesn't change anything when it runs.
     - name: Check Bastion public key is valid # noqa: no-changed-when
       ansible.builtin.command: "ssh-keygen -l -f '../terraform/bastion-key.pub'"
       ignore_errors: true
-      register: public_key_state
+      register: terraform_public_key_state
 
     # We can ignore this warning as this command doesn't change anything when it runs.
     - name: Check Bastion private key is valid # noqa: no-changed-when
       ansible.builtin.command: "ssh-keygen -l -f '../ansible/bastion-key'"
       ignore_errors: true
-      register: private_key_state
+      register: terraform_private_key_state
 
     - name: Generate an SSH key pair and copy to directories
-      when: public_key_state.rc != 0 or private_key_state.rc != 0
+      when: terraform_public_key_state.rc != 0 or terraform_private_key_state.rc != 0
       block:
         - name: Generate key
           community.crypto.openssh_keypair:

chatops_deployment/ansible/roles/terraform/tasks/deploy.yml

@@ -0,0 +1,181 @@
+# Deployment
+
+## Contents:
+
+- [Quick Start](#quick-start)
+
+## Quick Start:
+
+- If you are deploying from scratch, start at [Setting up localhost](#setting-up-localhost)
+- If you already have the repository cloned, the vault password saved and the projects clouds.yaml then start
+  at [Deploy infrastructure](#deploy-infrastructure).
+- If you only need to make changes to an existing deployment then start
+  at [Configure infrastructure](#configure-infrastructure)
+- To destroy all infrastructure, see [Destroy infrastructure](#destroy-infrastructure)
+
+## OpenStack Project Requirements:
+
+The project `Cloud-MicroServices` is already setup with all the required requisites. The variables in this repository 
+reference that project. If you are using a different project for a deployment not used by the Cloud Team you will 
+require the following:
+
+- A floating IP (e.g. 130.246.X.Y)
+- DNS records:
+  - `<your-domain> CNAME host-130-246-X-Y.nubes.stfc.ac.uk`
+  - **AND**
+  - ```
+    # EITHER
+    *.<your-domain> CNAME host-130-246-X-Y.nubes.stfc.ac.uk
+    # OR
+    kibana.<your-domain>.       CNAME   host-130-246-X-Y.nubes.stfc.ac.uk.
+    grafana.<your-domain>.      CNAME   host-130-246-X-Y.nubes.stfc.ac.uk.
+    prometheus.<your-domain>.   CNAME   host-130-246-X-Y.nubes.stfc.ac.uk.
+    alertmanager.<your-domain>. CNAME   host-130-246-X-Y.nubes.stfc.ac.uk.
+    chatops.<your-domain>.      CNAME   host-130-246-X-Y.nubes.stfc.ac.uk.
+    ```
+- Ports 80 and 443 open inbound from the internet
+- OpenStack Volume for the VM ~10GB
+
+### Deploying the Infrastructure:
+
+You can run the deployment from any machine (including your local laptop).
+However, we suggest you make a dedicated "seed VM" in OpenStack as the
+deployment will create files such as SSL certificates and SSH keys which you
+will need to keep for further maintenance.
+
+Machine requirements:
+
+- Python3
+- Snap (to install Terraform)
+- Pip or equivalent (to install Ansible)
+
+#### Setting up localhost:
+
+1. Install Ansible and collections
+   ```shell
+   # Install venv and Ansible
+   apt install python3-venv ansible
+   
+   # Create a virtual environment
+   python3 -m venv venv
+   source venv/bin/activate
+   
+   # Install collections using Ansible Galaxy
+   ansible-galaxy install -r requirements.yml  
+
+   # Install dependencies
+   pip install -r requirements.yml
+   ```
+
+2. Create a vault password file to avoid repeated inputs
+   ```shell
+   # Either
+   
+   echo "chatops_vault_password" >> ~/.chatops_vault_pass
+   
+   # or
+   
+   vim ~/.chatops_vault_pass # and enter the vault password as plain text
+   ```
+
+3. Change permissions and attributes to protect the file
+   ```shell
+   chmod 400 ~/.chatops_vault_pass
+   chattr +i ~/.chatops_vault_pass
+   ```
+   
+4. Copy the projects clouds.yaml to the `~/.config/openstack/clouds.yaml`
+   ```shell
+   cp <path-to>/clouds.yaml ~/.config/openstack/clouds.yaml
+   ```
+
+#### Deploy infrastructure:
+
+You can deploy both development and production environments on the same machine but not at the same time.
+
+1. Clone this repository
+   ```shell
+   git clone https://github.com/stfc/SCD-OpenStack-Utils
+   ```
+
+2. Change into the `ansible` directory
+   ```shell
+   cd SCD-OpenStack-Utils/chatops_deployment/ansible
+   ```
+
+3. Deploy infrastructure. Using -i to specify which inventory to use, dev or prod
+   ```shell
+   ansible-playbook deploy.yml --vault-password-file=~/.chatops_vault_pass -i <environment>
+   ```
+
+#### Configure infrastructure
+
+1. Configure the VMs. This step will take ~15 minutes
+   ```shell
+   ansible-playbook configure.yml --vault-password-file=~./chatops_vault_pass -i <environment>
+   ```
+
+#### Destroy infrastructure
+
+To destroy the infrastructure and all locally generated files run the destroy playbook.
+
+1. Destroy the infrastructure and locally generated files
+   ```shell
+   ansible-playbook destroy.yml --vault-password-file=~./chatops_vault_pass -i <environment>
+   ```
+
+## Debugging:
+
+### Terraform
+
+To debug the Terraform deployment, it is best to use the Terraform directly rather than through Ansible.
+When you run the deploy.yml playbook, a `terraform.tfvars` file is created which allows you to run the Terraform modules
+separate to Ansible.
+
+1. Ensure you have run deploy.yml at least once to generate the variables file `terraform.tfvars`
+
+2. Change to the terraform directory
+   ```shell
+   # Assuming you are in the ansible directory
+   cd ../terraform
+   ```
+
+3. Check and change Terraform workspace. Terraform separates environments into workspaces. Make sure you are using the
+   correct workspace before making changes.
+   ```shell
+   # List all workspaces. You should see at most "default, dev, prod"
+   terraform workspace list
+   
+   # Select the workspace you want to affect
+   terraform workspace select <environment>
+   ```
+
+4. Now you can make changes to the deployment. It is advisable you only use the Terraform commands directly if there is
+   something very wrong. The Ansible playbooks should be the first choice.
+   ```shell
+   # For example, plan and apply changes
+   terraform plan -out plan
+   terraform apply plan
+   
+   # Refresh the state to check API connections
+   terraform refresh
+   
+   # Validate the config
+   terraform validate
+   ```
+   
+### Ansible
+
+Each role in the Ansible playbook is tagged in its play. This enables you to run only parts of the playbooks. This is 
+important as it takes ~15 minutes to run the entire playbook. So, when you only want to make changes to certain parts 
+of the deployment you can use `--tags <some-tag>` to run only that part of the play.
+
+For example, if you change the Prometheus config file template you can just run the playbook with the **prometheus** tag
+.
+```shell
+ansible-playbook configure.yml --vault-password-file=~./chatops_vault_pass -i dev --tags prometheus
+```
+
+It is not recommended to use tags when making changes to the production deployment. As changes are promoted to 
+production the entire playbook should be run. This avoids any changes being missed out and ensures the entire deployment
+is running the latest configuration.