Frontend cluster deployment functional.

dokterbob · dokterbob · commit ebc1e246c4e7 · 2022-12-28T14:32:49.000+01:00
diff --git a/inventory b/inventory
@@ -1 +1 @@
-Subproject commit f0e51f158827d97a2d9e547177de7b51e4b744b6
+Subproject commit 5509843fd9f348eda7a793c540aff4d53de32f56
diff --git a/playbooks/api_nsfw_server.yml b/playbooks/api_nsfw_server.yml
@@ -1,4 +1,8 @@
 ---
+- name: Gather facts from frontend nodes
+  hosts: frontend
+  gather_facts: true
+  tags: nsfw_server
 - name: NSFW server
   hosts: nsfw_server
   become: true
diff --git a/playbooks/api_nyats.yml b/playbooks/api_nyats.yml
@@ -1,4 +1,8 @@
 ---
+- name: Gather facts from frontend nodes
+  hosts: frontend
+  gather_facts: true
+  tags: nyats
 - name: NYATS (thumbnailer)
   hosts: nyats
   tags: nyats
diff --git a/playbooks/api_pinservice.yml b/playbooks/api_pinservice.yml
@@ -1,7 +1,7 @@
 ---
-- name: Gather facts for ipfs nodes
+- name: Gather facts from ipfs and frontend nodes
   gather_facts: true
-  hosts: ipfs
+  hosts: ipfs:frontend
   tags: node_server
 - name: Pin service
   hosts: pinservice
diff --git a/playbooks/api_search.yml b/playbooks/api_search.yml
@@ -1,7 +1,11 @@
 ---
+- name: Gather facts from frontend nodes
+  gather_facts: true
+  hosts: frontend
+  tags: search_api
 - name: Search API
   hosts: search_api
-  tags: ipfs-search-api
+  tags: search_api
   become: true
   serial:
     - 1
diff --git a/playbooks/group_vars/all.yml b/playbooks/group_vars/all.yml
@@ -21,7 +21,6 @@ influxdb_backup_destination: "{{ storagebox_mountpoint }}/influxdb"
 influxdb_backup_user: "{{ lookup('env', 'USER') }}"
 influxdb_backup_bucket: "telegraf"
 storagebox_mountpoint_group: "{{ lookup('env', 'USER') }}"
-domain_name: "dwebsearch.org"
 nsfw_server_port: 3432
 nsfw_server_version: e903037676d4b721e67f6d2273b8a193a0a21bd7
 node_branch: node_16.x
diff --git a/playbooks/setup_01_inventory.yml b/playbooks/setup_01_inventory.yml
@@ -22,33 +22,17 @@
   tags: generate_vlan_ip
   roles:
     - role: setup/generate_vlan_ip
-- name: Create group of hosts without IPFS peer id
-  hosts: ipfs
-  gather_facts: false
-  tags: generate_ipfs_key
-  tasks:
-  - name: Create group for VLAN-less hosts
-    group_by:
-      key: "{% if ipfs_peer_id is defined %}hasipfspeerid{% else %}noipfspeerid{% endif %}"
-- name: Generate IPFS keys
-  hosts: noipfspeerid
-  gather_facts: false
-  tags: generate_ipfs_key
-  roles:
-    - role: setup/generate_ipfs_key
-- name: Explicitly gather facts
-  hosts: all
-  gather_facts: false
-  tags:
-    - dns
-    - reverse_dns
-  tasks:
-    - name: Gather facts
-      gather_facts:
 - name: Configure DNS
   hosts: all
+  tags: dns
+  gather_facts: true
   roles:
     - role: setup/cloudflare_dns
       tags: dns
-    - role: setup/hetzner_reverse_dns
-      tags: reverse_dns
+- name: Configure Hetzner robot reverse DNS
+  hosts: hetzner
+  tags: reverse_dns
+  gather_facts: true
+  roles:
+    - role: setup/hetzner_robot_reverse_dns
+# TODO: Hcloud reverse DNS
diff --git a/playbooks/setup_02_hetzner.yml b/playbooks/setup_02_hetzner.yml
@@ -11,7 +11,7 @@
 #   roles:
 #     - hetzner_add_to_switch
 - name: Install OS image
-  hosts: all # TODO: Filter by machines in rescue mode
+  hosts: hetzner # TODO: Filter by machines in rescue mode
   remote_user: root
   ignore_unreachable: true
   roles:
diff --git a/roles/services/frontend/defaults/main.yml b/roles/services/frontend/defaults/main.yml
@@ -1,2 +1,3 @@
 ---
 frontend_install_path: /usr/local/libexec/ipfs-search/frontend
+frontend_cache_path: /etc/nginx/cache
diff --git a/roles/services/frontend/tasks/main.yml b/roles/services/frontend/tasks/main.yml
@@ -1,11 +1,4 @@
 ---
-- name: Gather facts from backend nodes
-  setup:
-  delegate_to: "{{ item }}"
-  delegate_facts: true
-  when: hostvars[item]['ansible_default_ipv4'] is not defined
-  loop: "{{ groups['backend'] }}"
-  ignore_unreachable: true
 - name: Install nginx configuration
   template:
     src: nginx-frontend.conf.j2
@@ -18,3 +11,22 @@
     src: /etc/nginx/sites-available/frontend.conf
     state: link
   notify: Reload nginx
+- name: Create cache directory
+  file:
+    path: "{{ frontend_cache_path }}"
+    state: directory
+    owner: www-data
+    group: www-data
+    mode: 0755
+- name: Flush handlers
+  meta: flush_handlers
+- name: Ensure HTTP and HTTPS are listening
+  wait_for:
+    port: "{{ item }}"
+  loop:
+    - 80
+    - 443
+  timeout: 30
+- name: Test API call
+  uri:
+    url: "https://{{ api_server_name }}/v1/search?q=test"
diff --git a/roles/services/frontend/templates/nginx-frontend.conf.j2 b/roles/services/frontend/templates/nginx-frontend.conf.j2
@@ -1,13 +1,13 @@
-proxy_cache_path /etc/nginx/cache/search levels=1:2 keys_zone=search_cache:64m max_size=5g
+proxy_cache_path {{ frontend_cache_path }}/search levels=1:2 keys_zone=search_cache:64m max_size=5g
              inactive=8h use_temp_path=off;
 
-proxy_cache_path /etc/nginx/cache/metadata levels=1:2 keys_zone=metadata_cache:128m max_size=10g
+proxy_cache_path {{ frontend_cache_path }}/metadata levels=1:2 keys_zone=metadata_cache:128m max_size=10g
              inactive=1y use_temp_path=off;
 
-proxy_cache_path /etc/nginx/cache/nsfw levels=1:2 keys_zone=nsfw_cache:32m max_size=1g
+proxy_cache_path {{ frontend_cache_path }}/nsfw levels=1:2 keys_zone=nsfw_cache:32m max_size=1g
              inactive=1y use_temp_path=off;
 
-proxy_cache_path /etc/nginx/cache/thumbnail levels=1:2 keys_zone=thumbnail_cache:64m max_size=1g
+proxy_cache_path {{ frontend_cache_path }}/thumbnail levels=1:2 keys_zone=thumbnail_cache:64m max_size=1g
              inactive=1y use_temp_path=off;
 
 # Allow hole-punching in proxy with `?nocache=true`.
@@ -36,52 +36,44 @@ server {
     }
 }
 
-server {
-    listen 80;
-    listen [::]:80;
-
-    server_name {{ api_server_name }};
-    return 301 https://{{ api_server_name }}$request_uri;
-}
-
 upstream search {
     ip_hash;
     keepalive 32;
 
-    {% for host in groups['search_api'] if 'ansible_default_ipv4' in hostvars[host] %}
-        server {{ hostvars[host]['ansible_default_ipv4']['address'] }} max_fails=3 fail_timeout=10s;
+    {% for host in groups['search_api'] %}
+        server {{ hostvars[host]['vlan_ip'] | ipaddr('address') }} max_fails=3 fail_timeout=10s;
     {% endfor %}
 }
 
 upstream metadata {
     keepalive 32;
 
-    {% for host in groups['search_api'] if 'ansible_default_ipv4' in hostvars[host] %}
-        server {{ hostvars[host]['ansible_default_ipv4']['address'] }} max_fails=3 fail_timeout=10s;
+    {% for host in groups['search_api'] %}
+        server {{ hostvars[host]['vlan_ip'] | ipaddr('address') }} max_fails=3 fail_timeout=10s;
     {% endfor %}
 }
 
 upstream nsfw {
     keepalive 32;
 
-    {% for host in groups['nsfw_server'] if 'ansible_default_ipv4' in hostvars[host] %}
-        server {{ hostvars[host]['ansible_default_ipv4']['address'] }} max_fails=3 fail_timeout=10;
+    {% for host in groups['nsfw_server'] %}
+        server {{ hostvars[host]['vlan_ip'] | ipaddr('address') }} max_fails=3 fail_timeout=10;
     {% endfor %}
 }
 
 upstream pinservice {
     keepalive 32;
 
-    {% for host in groups['pinservice'] if 'ansible_default_ipv4' in hostvars[host] %}
-        server {{ hostvars[host]['ansible_default_ipv4']['address'] }} max_fails=3 fail_timeout=10;
+    {% for host in groups['pinservice'] %}
+        server {{ hostvars[host]['vlan_ip'] | ipaddr('address') }} max_fails=3 fail_timeout=10;
     {% endfor %}
 }
 
 upstream nyats {
     keepalive 32;
 
-    {% for host in groups['nyats'] if 'ansible_default_ipv4' in hostvars[host] %}
-        server {{ hostvars[host]['ansible_default_ipv4']['address'] }} max_fails=0 fail_timeout=120s; # Only one server, never disable server.
+    {% for host in groups['nyats'] %}
+        server {{ hostvars[host]['vlan_ip'] | ipaddr('address') }} max_fails=0 fail_timeout=120s;
     {% endfor %}
 }
 
@@ -224,6 +216,6 @@ server {
 
     include {{ options_ssl_nginx }};
 
-    ssl_certificate /etc/letsencrypt/live/{{ api_server_name }}/fullchain.pem;
-    ssl_certificate_key /etc/letsencrypt/live/{{ api_server_name }}/privkey.pem;
+    ssl_certificate /etc/letsencrypt/live/{{ certbot_name }}/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/{{ certbot_name }}/privkey.pem;
 }
diff --git a/roles/services/location_proxy/templates/nginx-api.conf.j2 b/roles/services/location_proxy/templates/nginx-api.conf.j2
@@ -1,10 +1,17 @@
 server {
+    # TODO: Listen only in private interface once switched to frontend cluster.
     listen 80 default_server;
     listen [::]:80 default_server;
 
     include /etc/nginx/locations-enabled/*.conf;
 
     allow 127.0.0.1;
+
+    # Remove once switched to frontend cluster.
     allow {{ frontend_ip }};
+
+    {% for host in groups['frontend'] if 'ansible_all_ipv4_addresses' in hostvars[host] %}
+    allow {{ hostvars[host].ansible_all_ipv4_addresses | ipaddr('private') | ipaddr('address') | first }};
+    {% endfor %}
     deny all;
 }
diff --git a/roles/setup/cloudflare_dns/defaults/main.yml b/roles/setup/cloudflare_dns/defaults/main.yml
@@ -1,4 +1,4 @@
 hostname: "{{ inventory_hostname_short }}"
 domain_name: "{{ inventory_hostname.split('.').1 }}"
-ipv4_address: "{{ ansible_default_ipv4.address }}"
-ipv6_address: "{{ ansible_default_ipv6.address }}"
+ipv4_address: "{{ ansible_default_ipv4.address | default(None) }}"
+ipv6_address: "{{ ansible_default_ipv6.address | default(None)  }}"
diff --git a/roles/setup/cloudflare_dns/tasks/main.yml b/roles/setup/cloudflare_dns/tasks/main.yml
@@ -13,7 +13,7 @@
       type: A
       value: "{{ ipv4_address }}"
       api_token: "{{ cloudflare_token }}"
-    when: ipv4_address != a_record
+    when: ipv4_address and ipv4_address != a_record
     retries: 3
     delay: 3
   - name: AAAA record for host
@@ -23,6 +23,6 @@
       type: AAAA
       value: "{{ ipv6_address }}"
       api_token: "{{ cloudflare_token }}"
-    when: ipv6_address != aaaa_record
+    when: ipv6_address and ipv6_address != aaaa_record
     retries: 3
     delay: 3
diff --git a/roles/setup/hetzner_reverse_dns/defaults/main.yml b/roles/setup/hetzner_reverse_dns/defaults/main.yml
diff --git a/roles/setup/hetzner_robot_reverse_dns/defaults/main.yml b/roles/setup/hetzner_robot_reverse_dns/defaults/main.yml
@@ -0,0 +1,4 @@
+hostname: "{{ inventory_hostname_short }}"
+domain_name: "{{ inventory_hostname.split('.').1 }}"
+ipv4_address: "{{ ansible_default_ipv4.address | default(None) }}"
+ipv6_address: "{{ ansible_default_ipv6.address | default(None) }}"
diff --git a/roles/setup/hetzner_robot_reverse_dns/tasks/main.yml b/roles/setup/hetzner_robot_reverse_dns/tasks/main.yml
@@ -1,6 +1,5 @@
 ---
 - name: Set PTR records
-  when: inventory_hostname in groups['hetzner']
   delegate_to: localhost
   block:
   - name: Get existing records
@@ -14,7 +13,7 @@
       value: "{{ fqdn }}"
       hetzner_user: "{{ hetzner_user }}"
       hetzner_password: "{{ hetzner_password }}"
-    when: fqdn != ptr4_record
+    when: ipv4_address and fqdn != ptr4_record
     retries: 3
     delay: 3
   - name: IPv6 reverse DNS
@@ -23,6 +22,6 @@
       value: "{{ fqdn }}"
       hetzner_user: "{{ hetzner_user }}"
       hetzner_password: "{{ hetzner_password }}"
-    when: fqdn != ptr6_record
+    when: ipv6_address and fqdn != ptr6_record
     retries: 3
     delay: 3
diff --git a/roles/vendor/certbot/tasks/main.yml b/roles/vendor/certbot/tasks/main.yml
@@ -9,22 +9,62 @@
     name: certbot
     classic: true
 - name: Confirm plugin containment level 
-  shell: snap set certbot trust-plugin-with-root=ok
+  command: snap set certbot trust-plugin-with-root=ok
+  changed_when: false
 - name: Install certbot cloudflare DNS plugin
   snap:
     name: certbot-dns-cloudflare
+- name: Create letsencrypt folder
+  file:
+    path: /etc/letsencrypt
+    state: directory
+    owner: root
+    group: root
+    mode: 0644
 - name: Prepare Cloudflare credentials .ini file
   template:
     src: cloudflare.ini.j2
-    dest: ~/cloudflare.ini
+    dest: /etc/letsencrypt/cloudflare.ini
     mode: 0600
+    owner: root
+    group: root
+- name: Prepare certbot configuration file
+  template:
+    src: cli.ini.j2
+    dest: /etc/letsencrypt/cli.ini
+    mode: 0644
+    owner: root
+    group: root
 - name: Setup certificates
-  command: certbot certonly -a dns-cloudflare --dns-cloudflare-credentials ~/cloudflare.ini -i nginx {% if certbot_test %}--test-cert {% endif %}-m '{{ certbot_email }}' --agree-tos -n -d '{{ certbot_domains }}'
+  command:
+    argv:
+      - certbot
+      - certonly
+      - -n
+      - --cert-name
+      - "{{ certbot_name }}"
+      - -d
+      - "{{ certbot_domains }}"
   changed_when: true
+# TODO: Why does this hang?
+# - name: Test certificate renewal
+#   command:
+#     argv:
+#       - certbot
+#       - renew
+#       - -n
+#       - --dry-run
+#       - --test-cert
 - name: Generate Diffie-Hellman params
   command: openssl dhparam -out {{ dh_params }} 4096
   args:
-    creates: '{{ dh_params }}'
+    creates: "{{ dh_params }}"
+- name: Set permissions on dh_params
+  file:
+    path: "{{ dh_params }}"
+    owner: root
+    group: root
+    mode: 0600
 - name: Install nginx SSL params
   template:
     src: options-ssl-nginx.conf.j2
diff --git a/roles/vendor/certbot/templates/cli.ini.j2 b/roles/vendor/certbot/templates/cli.ini.j2
@@ -0,0 +1,17 @@
+# Because we are using logrotate for greater flexibility, disable the
+# internal certbot logrotation.
+max-log-backups = 0
+
+# Adjust interactive output regarding automated renewal
+preconfigured-renewal = True
+
+email = {{ certbot_email }}
+
+authenticator = dns-cloudflare
+dns-cloudflare-credentials = /etc/letsencrypt/cloudflare.ini
+
+installer = nginx
+
+agree-tos = true
+
+{% if certbot_test %}test-cert = true{% endif %}

Original file line number	Diff line number	Diff line change
`@@ -1,2 +1,3 @@`
`1`	`1`	`---`
`2`	`2`	`frontend_install_path: /usr/local/libexec/ipfs-search/frontend`
	`3`	`+frontend_cache_path: /etc/nginx/cache`