From c3c54ed05d779cd1e88395af9c3c0501d0c315e7 Mon Sep 17 00:00:00 2001 From: Nicky Gerritsen Date: Sun, 21 Aug 2022 20:16:54 +0200 Subject: [PATCH 1/2] Allow to set up load balancing when installing DOMjudge. --- provision-contest/ansible/domserver.yml | 12 ++--- .../ansible/group_vars/all/all.yml.example | 4 ++ .../ansible/group_vars/all/secret.yml.example | 4 ++ .../ansible/roles/domserver/tasks/main.yml | 20 ++++++--- .../domserver/templates/dbpasswords.secret.j2 | 4 ++ .../templates/nginx-domjudge-inner.j2 | 5 +++ .../templates/nginx-domjudge.conf.j2 | 45 +++++++++++++++++++ .../ansible/roles/mysql_server/tasks/main.yml | 20 +++++++++ 8 files changed, 103 insertions(+), 11 deletions(-) diff --git a/provision-contest/ansible/domserver.yml b/provision-contest/ansible/domserver.yml index 3c1ff1fb..f467dd5f 100644 --- a/provision-contest/ansible/domserver.yml +++ b/provision-contest/ansible/domserver.yml @@ -37,18 +37,18 @@ tags: ssh - role: mysql_server tags: mysql_server - - role: domjudge_checkout - tags: domjudge_checkout - - role: domjudge_build - tags: domjudge_build - - role: domserver - tags: domserver - role: mysql_replication tags: mysql_replication when: REPLICATION_PASSWORD is defined - role: keepalived tags: keepalived when: KEEPALIVED_PRIORITY is defined + - role: domjudge_checkout + tags: domjudge_checkout + - role: domjudge_build + tags: domjudge_build + - role: domserver + tags: domserver - role: prometheus_target_web tags: prometheus_target_web vars: diff --git a/provision-contest/ansible/group_vars/all/all.yml.example b/provision-contest/ansible/group_vars/all/all.yml.example index b2e08b3d..fcea1e81 100644 --- a/provision-contest/ansible/group_vars/all/all.yml.example +++ b/provision-contest/ansible/group_vars/all/all.yml.example @@ -52,6 +52,10 @@ HOSTS: domjudge-laptop: 10.3.3.200 pc2: 10.3.3.241 +# Set this to true to enable load balancing instead of (automatic) failover +# DBA_PASSWORD needs to be set for this in secret.yml +DOMSERVER_LOADBALANCING: false + TIMEZONE: "Asia/Dhaka" PHP_FPM_MAX_CHILDREN: 400 diff --git a/provision-contest/ansible/group_vars/all/secret.yml.example b/provision-contest/ansible/group_vars/all/secret.yml.example index 2b1a0932..bd06e2bd 100644 --- a/provision-contest/ansible/group_vars/all/secret.yml.example +++ b/provision-contest/ansible/group_vars/all/secret.yml.example @@ -8,6 +8,10 @@ # Set this to enable master-master replication between two domservers. #REPLICATION_PASSWORD: {some-strong-replication-password} +# Database administrator password. Needed when DOMSERVER_LOADBALANCING is true +# Note: this user will have access from the whole SERVER_IP_PREFIX range +#DBA_PASSWORD: some-dba-password + # Database user password. DB_PASSWORD: {some-strong-database-password} diff --git a/provision-contest/ansible/roles/domserver/tasks/main.yml b/provision-contest/ansible/roles/domserver/tasks/main.yml index 57415436..8e17d7cb 100644 --- a/provision-contest/ansible/roles/domserver/tasks/main.yml +++ b/provision-contest/ansible/roles/domserver/tasks/main.yml @@ -19,17 +19,27 @@ owner: domjudge notify: Fix permissions on domjudge inplace-install +- name: set the DBA credentials + set_fact: + dba_credentials: | + {% if DBA_PASSWORD is defined %} + -u domjudge_dba -p {{ DBA_PASSWORD }} + {% else %} + -s -u root + {% endif %} + # When using replication, the DB will be dropped and recreated on the slave later. -- name: Check if the database is configured - command: "{{ DJ_DIR }}/bin/dj_setup_database -s -u root status" +- name: check if the database is configured + command: "{{ DJ_DIR }}/bin/dj_setup_database {{ dba_credentials }} status" register: db_status ignore_errors: true changed_when: false check_mode: no + when: not DOMSERVER_LOADBALANCING or groups['domserver'][0] == inventory_hostname -- name: Make sure the database is configured - command: "{{ DJ_DIR }}/bin/dj_setup_database -s -u root bare-install" - when: "'failed' in db_status.stdout" +- name: make sure the database is configured + command: "{{ DJ_DIR }}/bin/dj_setup_database {{ dba_credentials }} bare-install" + when: "(not DOMSERVER_LOADBALANCING or groups['domserver'][0] == inventory_hostname) and 'failed' in db_status.stdout" - name: Install required packages apt: diff --git a/provision-contest/ansible/roles/domserver/templates/dbpasswords.secret.j2 b/provision-contest/ansible/roles/domserver/templates/dbpasswords.secret.j2 index 15706d84..711b48a1 100644 --- a/provision-contest/ansible/roles/domserver/templates/dbpasswords.secret.j2 +++ b/provision-contest/ansible/roles/domserver/templates/dbpasswords.secret.j2 @@ -1,3 +1,7 @@ # {{ansible_managed}} # Format: 'unused:::::' +{% if DOMSERVER_LOADBALANCING %} +unused:{{DOMSERVER_IP}}:domjudge:domjudge:{{DB_PASSWORD}}:3306 +{% else %} unused:localhost:domjudge:domjudge:{{DB_PASSWORD}}:3306 +{% endif %} diff --git a/provision-contest/ansible/roles/domserver/templates/nginx-domjudge-inner.j2 b/provision-contest/ansible/roles/domserver/templates/nginx-domjudge-inner.j2 index 7e0d1904..5db3567b 100644 --- a/provision-contest/ansible/roles/domserver/templates/nginx-domjudge-inner.j2 +++ b/provision-contest/ansible/roles/domserver/templates/nginx-domjudge-inner.j2 @@ -11,6 +11,11 @@ set $domjudgeRoot {{ DJ_DIR }}/webapp/public; set $prefix ''; location / { +{% if DOMSERVER_LOADBALANCING %} + if ($access_allowed = false) { + return 403; + } +{% endif %} root $domjudgeRoot; try_files $uri @domjudgeFront; diff --git a/provision-contest/ansible/roles/domserver/templates/nginx-domjudge.conf.j2 b/provision-contest/ansible/roles/domserver/templates/nginx-domjudge.conf.j2 index 3e005e7b..16f923e0 100644 --- a/provision-contest/ansible/roles/domserver/templates/nginx-domjudge.conf.j2 +++ b/provision-contest/ansible/roles/domserver/templates/nginx-domjudge.conf.j2 @@ -7,6 +7,36 @@ upstream domjudge { server unix:/var/run/php-fpm-domjudge.sock; # if using with etc/domjudge-fpm.conf } +{% if DOMSERVER_LOADBALANCING %} +upstream domjudge-loadbalanced { + least_conn; +{% for host in groups['domserver'] %} + server {{ hostvars[host].ansible_host }}:81; +{% endfor %} +} + +server { + listen 81; + listen [::]:81; + server_name _default_; + + add_header Strict-Transport-Security max-age=31556952; + include /etc/nginx/snippets/domjudge-inner; + + set_real_ip_from {{ SERVER_IP_PREFIX }}.0/24; + real_ip_header X-Forwarded-For; + real_ip_recursive on; +} + +map $realip_remote_addr $access_allowed { + default false; +{% for host in groups['domserver'] %} + {{ hostvars[host].ansible_host }} true; +{% endfor %} +} + +{% endif %} + server { listen 80 default; server_name _default_; @@ -24,5 +54,20 @@ server { add_header Strict-Transport-Security max-age=31556952; send_timeout 36000s; +{% if DOMSERVER_LOADBALANCING %} + location / { + proxy_pass http://domjudge-loadbalanced; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_request_buffering off; + proxy_buffering off; + } +{% else %} include /etc/nginx/snippets/domjudge-inner; +{% endif %} } diff --git a/provision-contest/ansible/roles/mysql_server/tasks/main.yml b/provision-contest/ansible/roles/mysql_server/tasks/main.yml index 0e9a6a9a..2c54e770 100644 --- a/provision-contest/ansible/roles/mysql_server/tasks/main.yml +++ b/provision-contest/ansible/roles/mysql_server/tasks/main.yml @@ -72,3 +72,23 @@ loop: - load-db - dump-db + +- name: create mysql user for for DOMjudge database administration + mysql_user: + name: domjudge_dba + host: '{{ SERVER_IP_PREFIX }}.%' + password: "{{ DBA_PASSWORD }}" + append_privs: true + priv: 'domjudge.*:ALL,GRANT/*.*:CREATE USER,RELOAD' + state: present + when: DBA_PASSWORD is defined + +- name: create mysql user for for DOMjudge when we are doing loadbalancing + mysql_user: + name: domjudge + host: '{{ SERVER_IP_PREFIX }}.%' + password: "{{ DB_PASSWORD }}" + append_privs: true + priv: 'domjudge.*:SELECT,INSERT,UPDATE,DELETE' + state: present + when: DOMSERVER_LOADBALANCING From e7216371a69bcf8f553ef3e33860ce22b33933f6 Mon Sep 17 00:00:00 2001 From: Nicky Gerritsen Date: Mon, 22 Aug 2022 09:47:37 +0200 Subject: [PATCH 2/2] Make sure admin machines still work, add MySQL users only for needed hosts and use SSL for LB communication. --- .../ansible/roles/domserver/tasks/main.yml | 6 ++--- .../domserver/templates/dbpasswords.secret.j2 | 2 +- .../templates/nginx-domjudge-inner.j2 | 2 +- .../templates/nginx-domjudge.conf.j2 | 22 ++++++++++++------- .../ansible/roles/mysql_server/tasks/main.yml | 10 +++++---- 5 files changed, 25 insertions(+), 17 deletions(-) diff --git a/provision-contest/ansible/roles/domserver/tasks/main.yml b/provision-contest/ansible/roles/domserver/tasks/main.yml index 8e17d7cb..6add053a 100644 --- a/provision-contest/ansible/roles/domserver/tasks/main.yml +++ b/provision-contest/ansible/roles/domserver/tasks/main.yml @@ -22,7 +22,7 @@ - name: set the DBA credentials set_fact: dba_credentials: | - {% if DBA_PASSWORD is defined %} + {% if host_type == 'domserver' and DBA_PASSWORD is defined %} -u domjudge_dba -p {{ DBA_PASSWORD }} {% else %} -s -u root @@ -35,11 +35,11 @@ ignore_errors: true changed_when: false check_mode: no - when: not DOMSERVER_LOADBALANCING or groups['domserver'][0] == inventory_hostname + when: not DOMSERVER_LOADBALANCING or groups['domserver'][0] == inventory_hostname or host_type != 'domserver' - name: make sure the database is configured command: "{{ DJ_DIR }}/bin/dj_setup_database {{ dba_credentials }} bare-install" - when: "(not DOMSERVER_LOADBALANCING or groups['domserver'][0] == inventory_hostname) and 'failed' in db_status.stdout" + when: "(not DOMSERVER_LOADBALANCING or groups['domserver'][0] == inventory_hostname or host_type != 'domserver') and 'failed' in db_status.stdout" - name: Install required packages apt: diff --git a/provision-contest/ansible/roles/domserver/templates/dbpasswords.secret.j2 b/provision-contest/ansible/roles/domserver/templates/dbpasswords.secret.j2 index 711b48a1..0b8b5865 100644 --- a/provision-contest/ansible/roles/domserver/templates/dbpasswords.secret.j2 +++ b/provision-contest/ansible/roles/domserver/templates/dbpasswords.secret.j2 @@ -1,6 +1,6 @@ # {{ansible_managed}} # Format: 'unused:::::' -{% if DOMSERVER_LOADBALANCING %} +{% if host_type == 'domserver' and DOMSERVER_LOADBALANCING %} unused:{{DOMSERVER_IP}}:domjudge:domjudge:{{DB_PASSWORD}}:3306 {% else %} unused:localhost:domjudge:domjudge:{{DB_PASSWORD}}:3306 diff --git a/provision-contest/ansible/roles/domserver/templates/nginx-domjudge-inner.j2 b/provision-contest/ansible/roles/domserver/templates/nginx-domjudge-inner.j2 index 5db3567b..1756013f 100644 --- a/provision-contest/ansible/roles/domserver/templates/nginx-domjudge-inner.j2 +++ b/provision-contest/ansible/roles/domserver/templates/nginx-domjudge-inner.j2 @@ -11,7 +11,7 @@ set $domjudgeRoot {{ DJ_DIR }}/webapp/public; set $prefix ''; location / { -{% if DOMSERVER_LOADBALANCING %} +{% if host_type == 'domserver' and DOMSERVER_LOADBALANCING %} if ($access_allowed = false) { return 403; } diff --git a/provision-contest/ansible/roles/domserver/templates/nginx-domjudge.conf.j2 b/provision-contest/ansible/roles/domserver/templates/nginx-domjudge.conf.j2 index 16f923e0..76577501 100644 --- a/provision-contest/ansible/roles/domserver/templates/nginx-domjudge.conf.j2 +++ b/provision-contest/ansible/roles/domserver/templates/nginx-domjudge.conf.j2 @@ -7,19 +7,25 @@ upstream domjudge { server unix:/var/run/php-fpm-domjudge.sock; # if using with etc/domjudge-fpm.conf } -{% if DOMSERVER_LOADBALANCING %} +{% if host_type == 'domserver' and DOMSERVER_LOADBALANCING %} upstream domjudge-loadbalanced { least_conn; + keepalive 100; {% for host in groups['domserver'] %} - server {{ hostvars[host].ansible_host }}:81; + server {{ hostvars[host].ansible_host }}:444; {% endfor %} } server { - listen 81; - listen [::]:81; + listen 444 ssl http2; + listen [::]:444 ssl http2; server_name _default_; + ssl_certificate {{DOMSERVER_SSL_CERT}}; + ssl_certificate_key {{DOMSERVER_SSL_KEY}}; + ssl_session_timeout 5m; + ssl_prefer_server_ciphers on; + add_header Strict-Transport-Security max-age=31556952; include /etc/nginx/snippets/domjudge-inner; @@ -30,6 +36,7 @@ server { map $realip_remote_addr $access_allowed { default false; + {{ DOMSERVER_IP }} true; {% for host in groups['domserver'] %} {{ hostvars[host].ansible_host }} true; {% endfor %} @@ -54,12 +61,11 @@ server { add_header Strict-Transport-Security max-age=31556952; send_timeout 36000s; -{% if DOMSERVER_LOADBALANCING %} +{% if host_type == 'domserver' and DOMSERVER_LOADBALANCING %} location / { - proxy_pass http://domjudge-loadbalanced; + proxy_pass https://domjudge-loadbalanced; proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; + proxy_set_header Connection ""; proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header Host $http_host; proxy_set_header X-Real-IP $remote_addr; diff --git a/provision-contest/ansible/roles/mysql_server/tasks/main.yml b/provision-contest/ansible/roles/mysql_server/tasks/main.yml index 2c54e770..a2477398 100644 --- a/provision-contest/ansible/roles/mysql_server/tasks/main.yml +++ b/provision-contest/ansible/roles/mysql_server/tasks/main.yml @@ -76,19 +76,21 @@ - name: create mysql user for for DOMjudge database administration mysql_user: name: domjudge_dba - host: '{{ SERVER_IP_PREFIX }}.%' + host: '{{ item }}' password: "{{ DBA_PASSWORD }}" append_privs: true priv: 'domjudge.*:ALL,GRANT/*.*:CREATE USER,RELOAD' state: present - when: DBA_PASSWORD is defined + when: host_type == 'domserver' and DBA_PASSWORD is defined + loop: "{{ groups['domserver'] | map('extract', hostvars, 'ansible_host') + [DOMSERVER_IP] }}" - name: create mysql user for for DOMjudge when we are doing loadbalancing mysql_user: name: domjudge - host: '{{ SERVER_IP_PREFIX }}.%' + host: '{{ item }}' password: "{{ DB_PASSWORD }}" append_privs: true priv: 'domjudge.*:SELECT,INSERT,UPDATE,DELETE' state: present - when: DOMSERVER_LOADBALANCING + when: host_type == 'domserver' and DOMSERVER_LOADBALANCING + loop: "{{ groups['domserver'] | map('extract', hostvars, 'ansible_host') + [DOMSERVER_IP] }}"