Merge branch 'master' into dejal

ansible/inventories/digital_ocean.yml

@@ -2,7 +2,6 @@ plugin: constructed
 strict: False
 
 groups:
-
   haproxy: inventory_hostname.startswith('www')
 
   web: inventory_hostname.startswith('app')
@@ -12,6 +11,7 @@ groups:
   counts: inventory_hostname.startswith('app-counts')
   push: inventory_hostname.startswith('app-push')
   blogs: inventory_hostname.startswith('blog')
+  forum: inventory_hostname.startswith('forum')
 
   node: inventory_hostname.startswith('node')
   node_socket: inventory_hostname.startswith('node-socket')
@@ -25,9 +25,9 @@ groups:
   task: inventory_hostname.startswith('task')
   celery: inventory_hostname.startswith('task-celery')
   work: inventory_hostname.startswith('task-work')
-  
+
   staging: inventory_hostname.startswith('staging')
-  
+
   db: inventory_hostname.startswith('db')
   search: inventory_hostname.startswith('db-elasticsearch')
   elasticsearch: inventory_hostname.startswith('db-elasticsearch')

ansible/inventories/hetzner.yml

@@ -2,9 +2,8 @@ plugin: constructed
 strict: False
 
 groups:
-
   hall: inventory_hostname.startswith('h')
-  
+
   haproxy: inventory_hostname.startswith('hwww')
 
   app: inventory_hostname.startswith('happ')
@@ -20,6 +19,7 @@ groups:
   push: inventory_hostname.startswith('happ-push')
   hpush: inventory_hostname.startswith('happ-push')
   blogs: inventory_hostname.startswith('blog')
+  forum: inventory_hostname.startswith('hforum')
 
   node: inventory_hostname.startswith('hnode')
   hnode: inventory_hostname.startswith('hnode')
@@ -40,9 +40,9 @@ groups:
   task: inventory_hostname.startswith('htask')
   celery: inventory_hostname.startswith('htask-celery')
   work: inventory_hostname.startswith('htask-work')
-  
+
   staging: inventory_hostname.startswith('hstaging')
-  
+
   hdb: inventory_hostname.startswith('hdb')
   db: inventory_hostname.startswith('hdb')
   search: inventory_hostname.startswith('hdb-elasticsearch')

ansible/playbooks/setup_app.yml

@@ -7,16 +7,16 @@
   vars:
     - update_apt_cache: yes
     - motd_role: app
-    
+
   roles:
-    - {role: 'base', tags: 'base'}
-    - {role: 'ufw', tags: 'ufw'}
-    - {role: 'docker', tags: 'docker'}
-    - {role: 'repo', tags: ['repo', 'pull']}
-    - {role: 'dnsmasq', tags: 'dnsmasq'}
-    - {role: 'consul', tags: 'consul'}
-    - {role: 'consul-client', tags: 'consul'}
-    
-    - {role: 'web', tags: 'web'}
-    - {role: 'nginx', tags: 'nginx'}
-    - {role: 'node-exporter', tags: ['node-exporter', 'metrics']}
+    - { role: "base", tags: "base" }
+    - { role: "ufw", tags: "ufw" }
+    - { role: "docker", tags: "docker" }
+    - { role: "repo", tags: ["repo", "pull"] }
+    - { role: "dnsmasq", tags: "dnsmasq" }
+    - { role: "consul", tags: "consul" }
+    - { role: "consul-client", tags: "consul" }
+
+    - { role: "web", tags: "web" }
+    - { role: "nginx", tags: "nginx" }
+    - { role: "node-exporter", tags: ["node-exporter", "metrics"] }

ansible/playbooks/setup_forum.yml

@@ -0,0 +1,20 @@
+---
+- name: SETUP -> forum containers
+  hosts: forum
+  vars_files:
+    - ../env_vars/base.yml
+  vars:
+    - update_apt_cache: yes
+    - motd_role: app
+
+  roles:
+    - { role: "base", tags: "base" }
+    - { role: "ufw", tags: "ufw" }
+    - { role: "docker", tags: "docker" }
+    - { role: "repo", tags: ["repo", "pull"] }
+    - { role: "dnsmasq", tags: "dnsmasq" }
+    - { role: "consul", tags: "consul" }
+    - { role: "consul-client", tags: "consul" }
+
+    - { role: "forum", tags: "forum" }
+    - { role: "node-exporter", tags: ["node-exporter", "metrics"] }

ansible/playbooks/setup_metrics.yml

@@ -8,14 +8,14 @@
     - motd_role: db
 
   roles:
-    # - {role: 'base', tags: 'base'}
-    # - {role: 'ufw', tags: 'ufw'}
-    # - {role: 'docker', tags: 'docker'}
-    # - {role: 'repo', tags: ['repo', 'pull']}
-    # - {role: 'dnsmasq', tags: 'dnsmasq'}
-    # - {role: 'consul', tags: 'consul'}
-    # - {role: 'consul-client', tags: 'consul'}
-    # - {role: 'mongo-exporter', tags: 'mongo-exporter'}
+    - { role: "base", tags: "base" }
+    - { role: "ufw", tags: "ufw" }
+    - { role: "docker", tags: "docker" }
+    - { role: "repo", tags: ["repo", "pull"] }
+    - { role: "dnsmasq", tags: "dnsmasq" }
+    - { role: "consul", tags: "consul" }
+    - { role: "consul-client", tags: "consul" }
+    - { role: "mongo-exporter", tags: "mongo-exporter" }
     - { role: "postgres-exporter", tags: "postgres-exporter" }
     - { role: "redis-exporter", tags: "redis-exporter" }
     - { role: "node-exporter", tags: ["node-exporter", "metrics"] }

ansible/roles/forum/handlers/main.yml

@@ -0,0 +1,11 @@
+- name: restart discourse
+  become: yes
+  docker_container:
+    name: discourse
+    state: restarted
+
+- name: reload ufw
+  become: yes
+  service:
+    name: ufw
+    state: reloaded

ansible/roles/forum/tasks/main.yml

@@ -0,0 +1,20 @@
+---
+- name: Register forum in consul
+  tags: consul
+  become: yes
+  template:
+    src: consul_service.json
+    dest: /etc/consul.d/forum.json
+  when: disable_consul_services_ie_staging is not defined
+  notify:
+    - reload consul
+
+- name: Allow http
+  become: yes
+  ufw: rule=allow port=80
+  tags: ufw
+
+- name: Allow https
+  become: yes
+  ufw: rule=allow port=443
+  tags: ufw

ansible/roles/forum/templates/consul_service.json

@@ -0,0 +1,10 @@
+{
+    "service": {
+        "name": "{{ inventory_hostname|regex_replace('\d+', '') }}",
+        "id": "{{ inventory_hostname }}",
+        "tags": [
+            "forum"
+        ],
+        "port": 80
+    }
+}  

ansible/roles/prometheus/tasks/main.yml

@@ -1,5 +1,4 @@
 ---
-
 - name: Template file for prometheus
   vars:
     monitor_server: "{{ 'staging.newsblur.com' if disable_consul_services_ie_staging is defined else 'newsblur.com' }}"
@@ -27,12 +26,12 @@
     user: root
     networks_cli_compatible: yes
     network_mode: default
-    command: --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/prometheus --storage.tsdb.retention.time=10y
+    command: --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/prometheus --storage.tsdb.retention.time=10y --storage.tsdb.retention.size=500GB
     networks:
       - name: newsblurnet
     state: started
     ports:
-    - 9090:9090
+      - 9090:9090
     restart_policy: unless-stopped
     container_default_behavior: no_defaults
     volumes:

ansible/roles/ufw/tasks/main.yml

@@ -1,17 +1,16 @@
 ---
-
 - name: Set hosts
   set_fact:
     hetzner_hosts: "{{ groups['hall'] | map('extract', hostvars, ['ansible_host']) }}"
     do_hosts: "{{ groups['NewsBlur_Docker'] | map('extract', hostvars, ['ansible_host']) }}"
-    
+
 - name: Generate UFW batch script
   become: yes
   template:
     src: ufw_rules.sh.j2
     dest: /tmp/ufw_rules.sh
-    mode: '0755'
-    
+    mode: "0755"
+
 # - name: Stop ufw and delete all rules
 #   become: yes
 #   ufw: state=reset
@@ -51,14 +50,14 @@
 #   tags:
 #     - firewall
 #     - ufw
-  
+
 # - name: Allow all access from Hetzner inventory hosts
 #   become: yes
 #   ufw:
 #     rule: allow
 #     src: '{{ item }}'
 #   with_items: "{{ groups['hall'] | map('extract', hostvars, ['ansible_host']) }}"
-#   tags: 
+#   tags:
 #     - firewall
 #     - ufw
 #     - hetzner_firewall
@@ -71,7 +70,7 @@
 #     route: yes
 #     src: '{{ item }}'
 #   with_items: "{{ groups['hall'] | map('extract', hostvars, ['ansible_host']) }}"
-#   tags: 
+#   tags:
 #     - firewall
 #     - ufw
 #     - hetzner_firewall
@@ -84,7 +83,7 @@
 #     src: '{{ item }}'
 #   with_items: "{{ groups['oldandnew'] | map('extract', hostvars, ['ansible_host']) }}"
 #   when: "'oldandnew' in groups"
-#   tags: 
+#   tags:
 #     - firewall
 #     - ufw
 
@@ -95,7 +94,7 @@
 #     src: '{{ item }}'
 #   with_items: "{{ groups['NewsBlur_Docker'] | map('extract', hostvars, ['ansible_host']) }}"
 #   when: "'oldandnew' not in groups"
-#   tags: 
+#   tags:
 #     - firewall
 #     - ufw
 
@@ -107,7 +106,7 @@
 #     src: '{{ item }}'
 #   with_items: "{{ groups['NewsBlur_Docker'] | map('extract', hostvars, ['ansible_host']) }}"
 #   when: "'oldandnew' not in groups"
-#   tags: 
+#   tags:
 #     - firewall
 #     - ufw
 
@@ -145,7 +144,7 @@
 
       COMMIT
       # END UFW AND DOCKER
-  tags: docker
+  tags: docker ufw
   notify: restart ufw
 
 - name: Start ufw

ansible/setup.yml

@@ -3,6 +3,8 @@
   when: "'app' in group_names"
 - import_playbook: playbooks/setup_blog.yml
   when: "'blogs' in group_names"
+- import_playbook: playbooks/setup_forum.yml
+  when: "'forum' in group_names"
 - import_playbook: playbooks/setup_www.yml
   when: "'haproxy' in group_names"
 - import_playbook: playbooks/setup_node.yml

docker/haproxy/haproxy.consul.cfg.j2

@@ -71,6 +71,9 @@ frontend public
     use_backend node_page if { path_beg /original_page/ }
     use_backend blog if { hdr_end(host) -i blog.newsblur.com }
     use_backend sentry if { hdr_end(host) -i sentry.newsblur.com }
+    use_backend forum if { hdr_end(host) -i forum.newsblur.com }
+    use_backend forum if { hdr_end(host) -i forum2.newsblur.com }
+    use_backend forum if { hdr_end(host) -i forum3.newsblur.com }
     use_backend nginx if { path_beg /media/ }
     use_backend nginx if { path_beg /static/ }
     use_backend nginx if { path_beg /favicon }
@@ -196,6 +199,13 @@ backend sentry
         server {{host}} {{host}}.node.nyc1.consul:9000
     {% endfor %}
 
+backend forum
+    balance roundrobin
+    default-server check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none
+    {% for host in groups.forum %}
+        server {{host}} {{host}}.node.nyc1.consul:80
+    {% endfor %}
+
 backend db_metrics
     balance roundrobin
     # option httpchk GET /_haproxychk