Add citizenlab ec2 service to dev

README.md

@@ -7,6 +7,8 @@ flowchart TB
     apiorg([api.ooni.org])-->alb
     apiio([api.ooni.io])-->backend
     ecs[Backend API ECS]<-->ch[(Clickhouse Cluster)]
+    cz[Citizenlab API EC2]<-->ch
+
     subgraph Hetzner
         backend[OONI Backend Monolith]<-->ch
         monitoring[Monitoring host]
@@ -15,6 +17,7 @@ flowchart TB
     subgraph AWS
     alb[API Load Balancer]<-->ecs
     alb-->backend
+    alb-->cz
     ecs<-->s3[(OONI S3 Buckets)]
     s3<-->backend
     end

ansible/deploy-citizenlab.yml

@@ -0,0 +1,23 @@
+---
+- name: Deploy citizenlab
+  hosts:
+    - citizenlab.dev.ooni.io
+    - citizenlab.prod.ooni.io
+  become: true
+  roles:
+    - role: bootstrap
+    - role: nginx
+    - role: prometheus_node_exporter
+      vars:
+        use_https: false
+        node_exporter_port: 9100
+        node_exporter_host: "0.0.0.0"
+        prometheus_nginx_proxy_config:
+          - location: /metrics/node_exporter
+            proxy_pass: http://127.0.0.1:9100/metrics
+    - role: geerlingguy.docker
+      docker_users:
+        - citizenlab
+        - ubuntu
+      docker_package_state: latest
+    - role: citizenlab

ansible/deploy-tier2.yml

@@ -9,6 +9,9 @@
 - name: Include notebook playbook
   ansible.builtin.import_playbook: deploy-notebook.yml
 
+- name: Include citizenlab playbook
+  ansible.builtin.import_playbook: deploy-citizenlab.yml
+
 # commented out due to the fact it requires manual config of ~/.ssh/config
 #- name: Setup codesign box
 #  hosts: codesign-box

ansible/inventory

@@ -49,3 +49,5 @@ fastpath.prod.ooni.io
 anonc.dev.ooni.io
 jumphost.dev.ooni.io
 jumphost.prod.ooni.io
+citizenlab.dev.ooni.io
+citizenlab.prod.ooni.io

ansible/roles/citizenlab/defaults/main.yml

@@ -0,0 +1,8 @@
+tls_cert_dir: /var/lib/dehydrated/certs
+
+# citizenlab user
+citizenlab_user: citizenlab
+citizenlab_home: "/opt/{{ citizenlab_user }}"
+
+# citizenlab settings
+clickhouse_url: "" # fetch from aws secrets

ansible/roles/citizenlab/handlers/main.yml

@@ -0,0 +1,27 @@
+- name: test nginx config
+  command: /usr/sbin/nginx -t -c /etc/nginx/nginx.conf
+  listen:
+    - restart nginx
+    - reload nginx
+
+- name: restart nginx
+  service:
+    name: nginx
+    state: restarted
+
+- name: reload nginx
+  service:
+    name: nginx
+    state: reloaded
+
+- name: reload nftables
+  tags: nftables
+  ansible.builtin.systemd_service:
+    name: nftables
+    state: reloaded
+
+- name: restart docker
+  tags: docker
+  ansible.builtin.systemd_service:
+    name: docker
+    state: restarted

ansible/roles/citizenlab/tasks/main.yml

@@ -0,0 +1,166 @@
+---
+# For prometheus scrape requests
+- name: Flush all handlers
+  meta: flush_handlers
+
+- name: Allow traffic on port 9100
+  become: true
+  tags: prometheus-proxy
+  blockinfile:
+    path: /etc/ooni/nftables/tcp/9100.nft
+    create: yes
+    block: |
+      add rule inet filter input tcp dport 9100 counter accept comment "node exporter"
+  notify:
+   - reload nftables
+
+# For statsd importer metrics
+- name: Flush all handlers
+  meta: flush_handlers
+
+- name: Allow traffic on port 9102
+  become: true
+  tags: prometheus-proxy
+  blockinfile:
+    path: /etc/ooni/nftables/tcp/9102.nft
+    create: yes
+    block: |
+      add rule inet filter input tcp dport 9102 counter accept comment "node exporter"
+  notify:
+   - reload nftables
+
+# For incoming citizenlab traffic
+- name: Allow traffic on port 443
+  become: true
+  tags: citizenlab
+  blockinfile:
+    path: /etc/ooni/nftables/tcp/443.nft
+    create: yes
+    block: |
+      add rule inet filter input tcp dport 443 counter accept comment "citizenlab"
+  notify:
+   - reload nftables
+
+# Docker seems to have problems with nftables, so this command will translate all iptables
+# commands to nftables commands
+# - name: Update alternatives for iptables
+#   tags: docker
+#   become: yes
+#   ansible.builtin.command: "update-alternatives --set iptables /usr/sbin/iptables-nft"
+#   notify:
+#    - restart docker
+
+# - name: Update alternatives for iptables
+#   tags: docker
+#   become: yes
+#   ansible.builtin.command: "update-alternatives --set ip6tables /usr/sbin/ip6tables-nft"
+#   notify:
+#    - restart docker
+
+# - name: Flush all handlers # Required to apply iptables settings before docker runs
+#   meta: flush_handlers
+
+### Create citizenlab user
+- name: Ensure the citizenlab group exists
+  ansible.builtin.group:
+    name: "{{ citizenlab_user }}"
+    state: present
+  become: yes
+
+- name: Create the citizenlab user
+  ansible.builtin.user:
+    name: "{{ citizenlab_user }}"
+    home: "{{ citizenlab_home }}"
+    shell: "/bin/bash"
+    group: "{{ citizenlab_user }}"
+    create_home: yes
+    system: yes
+  become: yes
+
+- name: Set ownership of the citizenlab directory
+  ansible.builtin.file:
+    path: "{{ citizenlab_home }}"
+    owner: "{{ citizenlab_user }}"
+    group: "{{ citizenlab_user }}"
+    state: directory
+    mode: "0700"
+  become: yes
+
+### Run citizenlab
+- name: Make sure that the citizenlab configuration directory exists
+  ansible.builtin.file:
+    path: /opt/{{citizenlab_user}}/backend/citizenlab/
+    state: directory
+    mode: "0700"
+    owner: "{{citizenlab_user}}"
+    group: "{{citizenlab_user}}"
+
+- name: Create configuration file
+  tags: citizenlab
+  template:
+    src: templates/citizenlab.conf
+    dest: "/opt/{{citizenlab_user}}/backend/citizenlab/citizenlab.conf"
+    mode: "0400"
+    owner: "{{citizenlab_user}}"
+  become: yes
+
+- name: Ensure ooniapi directory existence
+  ansible.builtin.file:
+    path: /var/lib/ooniapi
+    state: directory
+    mode: "0711"
+    owner: "{{citizenlab_user}}"
+    group: "{{citizenlab_user}}"
+
+- name: Ensure citizenlab var dir exists
+  ansible.builtin.file:
+    path: /var/lib/citizenlab
+    state: directory
+    mode: "0700"
+    owner: "{{citizenlab_user}}"
+    group: "{{citizenlab_user}}"
+
+- name: Allow nginx access to the spool dir
+  become: true
+  ansible.builtin.user:
+    name: nginx
+    groups: "{{citizenlab_user}}"
+    append: yes
+
+- name: Get UID of a specific user
+  command: id -u {{citizenlab_user}}
+  register: user_uid
+  changed_when: false
+
+- name: Get GID of a specific user
+  command: id -g {{citizenlab_user}}
+  register: user_gid
+  changed_when: false
+
+- name: Ensure citizenlab is running
+  community.docker.docker_container:
+    name: citizenlab
+    image: ooni/citizenlab:v0.1.0rc0
+    state: started
+    user: "{{user_uid.stdout}}:{{user_gid.stdout}}"
+    # use network mode = host to allow traffic from citizenlab to the statsd exporter without
+    # creating a network with redirection rules to match the ports
+    network_mode: host
+    # published_ports: # unused on network_mode: host
+    #   - "8472:8472"
+    volumes:
+      - /opt/{{citizenlab_user}}/backend/citizenlab/citizenlab.conf:/etc/ooni/citizenlab.conf
+      - /var/lib/ooniapi:/var/lib/ooniapi
+      - /var/lib/citizenlab:/var/lib/citizenlab
+  tags:
+    - citizenlab
+
+- name: Ensure the statsd to prometheus exporter is running
+  community.docker.docker_container:
+    name: statsd-exporter
+    image: prom/statsd-exporter:v0.28.0
+    state: started
+    published_ports:
+      - "9102:9102" # for /metrics
+      - "8125:9125" # for reporting metrics
+      - "8125:9125/udp"

ansible/roles/citizenlab/templates/citizenlab.conf

@@ -0,0 +1,19 @@
+[DEFAULT]
+# Collector hostnames, comma separated
+collectors = localhost
+
+
+{% if psql_uri is defined %}
+# The password is already made public
+db_uri = {{ psql_uri }}
+{% else %}
+db_uri =
+{% endif %}
+
+# S3 access credentials
+# Currently unused
+s3_access_key =
+s3_secret_key =
+
+
+clickhouse_url = {{clickhouse_url}}

ansible/roles/prometheus/templates/prometheus.yml

@@ -370,4 +370,50 @@ scrape_configs:
         replacement: "/$2"
         target_label: "__metrics_path__"
         action: "replace"
+
+  - job_name: "citizenlab"
+    static_configs:
+    - targets:
+      - citizenlab.dev.ooni.io:9102
+      - citizenlab.prod.ooni.io:9102
+    scrape_interval: 5s
+    scheme: https
+    relabel_configs: # Change the host to the proxy host with relabeling
+      # Store ip in ecs_host
+      - source_labels: [__address__]
+        regex: "([a-z\\.]+):([0-9]+)" # <host>:<port>"
+        replacement: "$1"
+        target_label: "ec2_host"
+        action: "replace"
+      # Extract environment from address
+      - source_labels: [__address__]
+        regex: ".*(dev|prod).*"
+        replacement: "$1"
+        target_label: "env"
+        action: "replace"
+      # Store the full adress with path in proxy_host
+      - source_labels: [__address__]
+        regex: "([a-z\\.]+):([0-9]+)" # <host>:<port>
+        replacement: "{{monitoring_proxy_host}}:9200/${1}/${2}/metrics" # proxy.org:9200/<hostname>/<port>/metrics
+        target_label: "__proxy_host"
+        action: "replace"
+      # Change the environment part in proxy host
+      - source_labels: [__proxy_host, env]
+        separator: ";"
+        regex: "([^;]*)ENV([^;]*);(.*)" # __proxy_host;env
+        replacement: "$1$3$2"
+        target_label: "__proxy_host"
+        action: "replace"
+      # Change the address where to send the scrape request to
+      - source_labels: [__proxy_host]
+        regex: "([^/]*)/(.*)"
+        replacement: "$1"
+        target_label: "__address__"
+        action: "replace"
+      # Change the metrics path to include ip address and /metrics path
+      - source_labels: [__proxy_host]
+        regex: "([^/]*)/(.*)"
+        replacement: "/$2"
+        target_label: "__metrics_path__"
+        action: "replace"
 ...