diff --git a/README.md b/README.md index fa9b8c7a..45bb0076 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,8 @@ flowchart TB apiorg([api.ooni.org])-->alb apiio([api.ooni.io])-->backend ecs[Backend API ECS]<-->ch[(Clickhouse Cluster)] + cz[Citizenlab API EC2]<-->ch + subgraph Hetzner backend[OONI Backend Monolith]<-->ch monitoring[Monitoring host] @@ -15,6 +17,7 @@ flowchart TB subgraph AWS alb[API Load Balancer]<-->ecs alb-->backend + alb-->cz ecs<-->s3[(OONI S3 Buckets)] s3<-->backend end diff --git a/ansible/deploy-citizenlab.yml b/ansible/deploy-citizenlab.yml new file mode 100644 index 00000000..99dc234a --- /dev/null +++ b/ansible/deploy-citizenlab.yml @@ -0,0 +1,23 @@ +--- +- name: Deploy citizenlab + hosts: + - citizenlab.dev.ooni.io + - citizenlab.prod.ooni.io + become: true + roles: + - role: bootstrap + - role: nginx + - role: prometheus_node_exporter + vars: + use_https: false + node_exporter_port: 9100 + node_exporter_host: "0.0.0.0" + prometheus_nginx_proxy_config: + - location: /metrics/node_exporter + proxy_pass: http://127.0.0.1:9100/metrics + - role: geerlingguy.docker + docker_users: + - citizenlab + - ubuntu + docker_package_state: latest + - role: citizenlab diff --git a/ansible/deploy-tier2.yml b/ansible/deploy-tier2.yml index cff8d695..79df612e 100644 --- a/ansible/deploy-tier2.yml +++ b/ansible/deploy-tier2.yml @@ -9,6 +9,9 @@ - name: Include notebook playbook ansible.builtin.import_playbook: deploy-notebook.yml +- name: Include citizenlab playbook + ansible.builtin.import_playbook: deploy-citizenlab.yml + # commented out due to the fact it requires manual config of ~/.ssh/config #- name: Setup codesign box # hosts: codesign-box diff --git a/ansible/inventory b/ansible/inventory index 7437f1af..93eebb31 100644 --- a/ansible/inventory +++ b/ansible/inventory @@ -49,3 +49,5 @@ fastpath.prod.ooni.io anonc.dev.ooni.io jumphost.dev.ooni.io jumphost.prod.ooni.io +citizenlab.dev.ooni.io +citizenlab.prod.ooni.io diff --git a/ansible/roles/citizenlab/defaults/main.yml b/ansible/roles/citizenlab/defaults/main.yml new file mode 100644 index 00000000..6842b38c --- /dev/null +++ b/ansible/roles/citizenlab/defaults/main.yml @@ -0,0 +1,8 @@ +tls_cert_dir: /var/lib/dehydrated/certs + +# citizenlab user +citizenlab_user: citizenlab +citizenlab_home: "/opt/{{ citizenlab_user }}" + +# citizenlab settings +clickhouse_url: "" # fetch from aws secrets diff --git a/ansible/roles/citizenlab/handlers/main.yml b/ansible/roles/citizenlab/handlers/main.yml new file mode 100644 index 00000000..5c8f17dd --- /dev/null +++ b/ansible/roles/citizenlab/handlers/main.yml @@ -0,0 +1,27 @@ +- name: test nginx config + command: /usr/sbin/nginx -t -c /etc/nginx/nginx.conf + listen: + - restart nginx + - reload nginx + +- name: restart nginx + service: + name: nginx + state: restarted + +- name: reload nginx + service: + name: nginx + state: reloaded + +- name: reload nftables + tags: nftables + ansible.builtin.systemd_service: + name: nftables + state: reloaded + +- name: restart docker + tags: docker + ansible.builtin.systemd_service: + name: docker + state: restarted diff --git a/ansible/roles/citizenlab/tasks/main.yml b/ansible/roles/citizenlab/tasks/main.yml new file mode 100644 index 00000000..3069be11 --- /dev/null +++ b/ansible/roles/citizenlab/tasks/main.yml @@ -0,0 +1,166 @@ +--- +# For prometheus scrape requests +- name: Flush all handlers + meta: flush_handlers + +- name: Allow traffic on port 9100 + become: true + tags: prometheus-proxy + blockinfile: + path: /etc/ooni/nftables/tcp/9100.nft + create: yes + block: | + add rule inet filter input tcp dport 9100 counter accept comment "node exporter" + notify: + - reload nftables + +# For statsd importer metrics +- name: Flush all handlers + meta: flush_handlers + +- name: Allow traffic on port 9102 + become: true + tags: prometheus-proxy + blockinfile: + path: /etc/ooni/nftables/tcp/9102.nft + create: yes + block: | + add rule inet filter input tcp dport 9102 counter accept comment "node exporter" + notify: + - reload nftables + +# For incoming citizenlab traffic +- name: Allow traffic on port 443 + become: true + tags: citizenlab + blockinfile: + path: /etc/ooni/nftables/tcp/443.nft + create: yes + block: | + add rule inet filter input tcp dport 443 counter accept comment "citizenlab" + notify: + - reload nftables + +# Docker seems to have problems with nftables, so this command will translate all iptables +# commands to nftables commands +# - name: Update alternatives for iptables +# tags: docker +# become: yes +# ansible.builtin.command: "update-alternatives --set iptables /usr/sbin/iptables-nft" +# notify: +# - restart docker + +# - name: Update alternatives for iptables +# tags: docker +# become: yes +# ansible.builtin.command: "update-alternatives --set ip6tables /usr/sbin/ip6tables-nft" +# notify: +# - restart docker + +# - name: Flush all handlers # Required to apply iptables settings before docker runs +# meta: flush_handlers + +### Create citizenlab user +- name: Ensure the citizenlab group exists + ansible.builtin.group: + name: "{{ citizenlab_user }}" + state: present + become: yes + +- name: Create the citizenlab user + ansible.builtin.user: + name: "{{ citizenlab_user }}" + home: "{{ citizenlab_home }}" + shell: "/bin/bash" + group: "{{ citizenlab_user }}" + create_home: yes + system: yes + become: yes + +- name: Set ownership of the citizenlab directory + ansible.builtin.file: + path: "{{ citizenlab_home }}" + owner: "{{ citizenlab_user }}" + group: "{{ citizenlab_user }}" + state: directory + mode: "0700" + become: yes + +### Run citizenlab +- name: Make sure that the citizenlab configuration directory exists + ansible.builtin.file: + path: /opt/{{citizenlab_user}}/backend/citizenlab/ + state: directory + mode: "0700" + owner: "{{citizenlab_user}}" + group: "{{citizenlab_user}}" + +- name: Create configuration file + tags: citizenlab + template: + src: templates/citizenlab.conf + dest: "/opt/{{citizenlab_user}}/backend/citizenlab/citizenlab.conf" + mode: "0400" + owner: "{{citizenlab_user}}" + become: yes + +- name: Ensure ooniapi directory existence + ansible.builtin.file: + path: /var/lib/ooniapi + state: directory + mode: "0711" + owner: "{{citizenlab_user}}" + group: "{{citizenlab_user}}" + +- name: Ensure citizenlab var dir exists + ansible.builtin.file: + path: /var/lib/citizenlab + state: directory + mode: "0700" + owner: "{{citizenlab_user}}" + group: "{{citizenlab_user}}" + +- name: Allow nginx access to the spool dir + become: true + ansible.builtin.user: + name: nginx + groups: "{{citizenlab_user}}" + append: yes + +- name: Get UID of a specific user + command: id -u {{citizenlab_user}} + register: user_uid + changed_when: false + +- name: Get GID of a specific user + command: id -g {{citizenlab_user}} + register: user_gid + changed_when: false + +- name: Ensure citizenlab is running + community.docker.docker_container: + name: citizenlab + image: ooni/citizenlab:v0.1.0rc0 + state: started + user: "{{user_uid.stdout}}:{{user_gid.stdout}}" + # use network mode = host to allow traffic from citizenlab to the statsd exporter without + # creating a network with redirection rules to match the ports + network_mode: host + # published_ports: # unused on network_mode: host + # - "8472:8472" + volumes: + - /opt/{{citizenlab_user}}/backend/citizenlab/citizenlab.conf:/etc/ooni/citizenlab.conf + - /var/lib/ooniapi:/var/lib/ooniapi + - /var/lib/citizenlab:/var/lib/citizenlab + tags: + - citizenlab + +- name: Ensure the statsd to prometheus exporter is running + community.docker.docker_container: + name: statsd-exporter + image: prom/statsd-exporter:v0.28.0 + state: started + published_ports: + - "9102:9102" # for /metrics + - "8125:9125" # for reporting metrics + - "8125:9125/udp" diff --git a/ansible/roles/citizenlab/templates/citizenlab.conf b/ansible/roles/citizenlab/templates/citizenlab.conf new file mode 100644 index 00000000..54ec9d7e --- /dev/null +++ b/ansible/roles/citizenlab/templates/citizenlab.conf @@ -0,0 +1,19 @@ +[DEFAULT] +# Collector hostnames, comma separated +collectors = localhost + + +{% if psql_uri is defined %} +# The password is already made public +db_uri = {{ psql_uri }} +{% else %} +db_uri = +{% endif %} + +# S3 access credentials +# Currently unused +s3_access_key = +s3_secret_key = + + +clickhouse_url = {{clickhouse_url}} diff --git a/ansible/roles/prometheus/templates/prometheus.yml b/ansible/roles/prometheus/templates/prometheus.yml index 56f53a72..dc97adbb 100755 --- a/ansible/roles/prometheus/templates/prometheus.yml +++ b/ansible/roles/prometheus/templates/prometheus.yml @@ -370,4 +370,50 @@ scrape_configs: replacement: "/$2" target_label: "__metrics_path__" action: "replace" + + - job_name: "citizenlab" + static_configs: + - targets: + - citizenlab.dev.ooni.io:9102 + - citizenlab.prod.ooni.io:9102 + scrape_interval: 5s + scheme: https + relabel_configs: # Change the host to the proxy host with relabeling + # Store ip in ecs_host + - source_labels: [__address__] + regex: "([a-z\\.]+):([0-9]+)" # :" + replacement: "$1" + target_label: "ec2_host" + action: "replace" + # Extract environment from address + - source_labels: [__address__] + regex: ".*(dev|prod).*" + replacement: "$1" + target_label: "env" + action: "replace" + # Store the full adress with path in proxy_host + - source_labels: [__address__] + regex: "([a-z\\.]+):([0-9]+)" # : + replacement: "{{monitoring_proxy_host}}:9200/${1}/${2}/metrics" # proxy.org:9200///metrics + target_label: "__proxy_host" + action: "replace" + # Change the environment part in proxy host + - source_labels: [__proxy_host, env] + separator: ";" + regex: "([^;]*)ENV([^;]*);(.*)" # __proxy_host;env + replacement: "$1$3$2" + target_label: "__proxy_host" + action: "replace" + # Change the address where to send the scrape request to + - source_labels: [__proxy_host] + regex: "([^/]*)/(.*)" + replacement: "$1" + target_label: "__address__" + action: "replace" + # Change the metrics path to include ip address and /metrics path + - source_labels: [__proxy_host] + regex: "([^/]*)/(.*)" + replacement: "/$2" + target_label: "__metrics_path__" + action: "replace" ... diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 1aca9365..1f381fbf 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -1087,6 +1087,100 @@ module "ooniapi_oonimeasurements" { ) } +### Tier2 Citizenlab service +module "ooniapi_citizenlab" { + source = "../../modules/ec2" + + stage = local.environment + + vpc_id = module.network.vpc_id + subnet_id = module.network.vpc_subnet_public[0].id + private_subnet_cidr = module.network.vpc_subnet_private[*].cidr_block + dns_zone_ooni_io = local.dns_zone_ooni_io + + key_name = module.adm_iam_roles.oonidevops_key_name + instance_type = "t3a.nano" + + name = "oonictzlab" + ingress_rules = [{ + from_port = 22, + to_port = 22, + protocol = "tcp", + cidr_blocks = ["0.0.0.0/0"], + }, { + from_port = 80, # for dehydrated challenge + to_port = 80, + protocol = "tcp", + cidr_blocks = ["0.0.0.0/0"], + }, { + // API endpoint + from_port = 443, + to_port = 443, + protocol = "tcp", + cidr_blocks = ["0.0.0.0/0"], + }, { + // For the prometheus proxy: + from_port = 9200, + to_port = 9200, + protocol = "tcp" + cidr_blocks = [for ip in flatten(data.dns_a_record_set.monitoring_host.*.addrs) : "${tostring(ip)}/32"] + }, { + from_port = 9100, + to_port = 9100, + protocol = "tcp" + cidr_blocks = ["${module.ooni_monitoring_proxy.aws_instance_private_ip}/32"] + }] + + egress_rules = [{ + from_port = 0, + to_port = 0, + protocol = "-1", + cidr_blocks = ["0.0.0.0/0"], + }, { + from_port = 0, + to_port = 0, + protocol = "-1", + ipv6_cidr_blocks = ["::/0"] + }] + + sg_prefix = "ooniciti" + tg_prefix = "citi" + + disk_size = 20 + + tags = merge( + local.tags, + { Name = "ooni-tier2-citizenlab" } + ) +} + +resource "aws_route53_record" "citizenlab_alias" { + zone_id = local.dns_zone_ooni_io + name = "citizenlab.${local.environment}.ooni.io" + type = "CNAME" + ttl = 300 + + records = [ + module.ooniapi_citizenlab.aws_instance_public_dns + ] +} + +module "citizenlab_builder" { + source = "../../modules/ooni_docker_build" + trigger_tag = "" + + service_name = "citizenlab" + repo = "ooni/backend" + branch_name = "add_citizenlab_url_management_with_porcelain" + buildspec_path = "ooniapi/services/citizenlab/buildspec.yml" + trigger_path = "ooniapi/services/citizenlab/**" + codestar_connection_arn = aws_codestarconnections_connection.oonidevops.arn + + codepipeline_bucket = aws_s3_bucket.ooniapi_codepipeline_bucket.bucket + + ecs_cluster_name = module.ooniapi_cluster.cluster_name +} + #### OONI Tier0 API Frontend module "ooniapi_frontend" { @@ -1101,6 +1195,7 @@ module "ooniapi_frontend" { ooniapi_ooniprobe_target_group_arn = module.ooniapi_ooniprobe.alb_target_group_id ooniapi_oonifindings_target_group_arn = module.ooniapi_oonifindings.alb_target_group_id ooniapi_oonimeasurements_target_group_arn = module.ooniapi_oonimeasurements.alb_target_group_id + ooniapi_citizenlab_target_group_arn = module.ooniapi_citizenlab.aws_instance_id ooniapi_service_security_groups = [ module.ooniapi_cluster.web_security_group_id, diff --git a/tf/modules/ooniapi_frontend/main.tf b/tf/modules/ooniapi_frontend/main.tf index 93de70fc..70635854 100644 --- a/tf/modules/ooniapi_frontend/main.tf +++ b/tf/modules/ooniapi_frontend/main.tf @@ -457,3 +457,24 @@ resource "aws_lb_listener_rule" "ooniapi_oonimeasurements_rule_2" { } } } + +#resource "aws_lb_listener_rule" "ooniapi_citizenlab_rule" { +# listener_arn = aws_alb_listener.ooniapi_listener_https.arn +# priority = 143 +# +# action { +# type = "forward" +# target_group_arn = var.ooniapi_citizenlab_target_group_arn +# } +# condition { +# path_pattern { +# values = [ +# "/api/_/url-submission/test-list/*", +# "/api/_/url-priorities/list", +# "/api/_/url-priorities/update", +# "/api/v1/url-submission/submit", +# "/api/v1/url-submission/update-url", +# ] +# } +# } +#} diff --git a/tf/modules/ooniapi_frontend/variables.tf b/tf/modules/ooniapi_frontend/variables.tf index d4ec3dd0..2c676694 100644 --- a/tf/modules/ooniapi_frontend/variables.tf +++ b/tf/modules/ooniapi_frontend/variables.tf @@ -37,6 +37,10 @@ variable "ooniapi_oonimeasurements_target_group_arn" { default = null } +variable "ooniapi_citizenlab_target_group_arn" { + description = "arn for the target group of the citizenlab service" +} + variable "dns_zone_ooni_io" { description = "id of the DNS zone for ooni_io" }