diff --git a/inventory/groups.yaml b/inventory/groups.yaml index ecb07291e0..bfa81ff57c 100644 --- a/inventory/groups.yaml +++ b/inventory/groups.yaml @@ -28,6 +28,8 @@ groups: # backup-server: # - backup[0-9]*.opendev.org cacti: cacti[0-9]*.open*.org + certcheck: + - cacti[0-9]*.open*.org cloud-launcher: - bridge.openstack.org codesearch: diff --git a/manifests/site.pp b/manifests/site.pp index cd6ac92154..71bebd89a4 100644 --- a/manifests/site.pp +++ b/manifests/site.pp @@ -28,7 +28,9 @@ node /^health\d*\.openstack\.org$/ { # Node-OS: xenial node /^cacti\d+\.open.*\.org$/ { $group = "cacti" - include openstack_project::ssl_cert_check + # NOTE(ianw) 2020-05 : disabled pending removal, migrated to + # ansible. + # include openstack_project::ssl_cert_check class { 'openstack_project::cacti': cacti_hosts => hiera_array('cacti_hosts'), vhost_name => 'cacti.openstack.org', diff --git a/playbooks/group_vars/certcheck.yaml b/playbooks/group_vars/certcheck.yaml new file mode 100644 index 0000000000..1e176b273d --- /dev/null +++ b/playbooks/group_vars/certcheck.yaml @@ -0,0 +1,17 @@ +letsencrypt_certcheck_additional_domains: + - ask.openstack.org 443 + - ethercalc.openstack.org 443 + - etherpad.openstack.org 443 + - firehose.openstack.org 8883 + - git.openstack.org 443 + - openstackid-dev.openstack.org 443 + - openstackid.org 443 + - refstack.openstack.org 443 + - review.openstack.org 443 + - storyboard.openstack.org 443 + - survey.openstack.org 443 + - static.openstack.org 443 + - translate.openstack.org 443 + - wiki.openstack.org 443 + - www.openstack.org 443 + - zuul.openstack.org 443 diff --git a/playbooks/host_vars/gitea01.opendev.org.yaml b/playbooks/host_vars/gitea01.opendev.org.yaml index 08cf199982..8c1b273ab4 100644 --- a/playbooks/host_vars/gitea01.opendev.org.yaml +++ b/playbooks/host_vars/gitea01.opendev.org.yaml @@ -1,4 +1,4 @@ letsencrypt_certs: gitea01-main: - - gitea01.opendev.org + - gitea01.opendev.org:3000 - opendev.org diff --git a/playbooks/host_vars/gitea02.opendev.org.yaml b/playbooks/host_vars/gitea02.opendev.org.yaml index 44cabb7ff9..9caa50573e 100644 --- a/playbooks/host_vars/gitea02.opendev.org.yaml +++ b/playbooks/host_vars/gitea02.opendev.org.yaml @@ -1,4 +1,4 @@ letsencrypt_certs: gitea02-main: - - gitea02.opendev.org + - gitea02.opendev.org:3000 - opendev.org diff --git a/playbooks/host_vars/gitea03.opendev.org.yaml b/playbooks/host_vars/gitea03.opendev.org.yaml index 02c58c7fdb..bfaadf6e27 100644 --- a/playbooks/host_vars/gitea03.opendev.org.yaml +++ b/playbooks/host_vars/gitea03.opendev.org.yaml @@ -1,4 +1,4 @@ letsencrypt_certs: gitea03-main: - - gitea03.opendev.org + - gitea03.opendev.org:3000 - opendev.org diff --git a/playbooks/host_vars/gitea04.opendev.org.yaml b/playbooks/host_vars/gitea04.opendev.org.yaml index 4a9ad0e17b..769eb10950 100644 --- a/playbooks/host_vars/gitea04.opendev.org.yaml +++ b/playbooks/host_vars/gitea04.opendev.org.yaml @@ -1,4 +1,4 @@ letsencrypt_certs: gitea04-main: - - gitea04.opendev.org + - gitea04.opendev.org:3000 - opendev.org diff --git a/playbooks/host_vars/gitea05.opendev.org.yaml b/playbooks/host_vars/gitea05.opendev.org.yaml index 20920feb71..8f90cb44f6 100644 --- a/playbooks/host_vars/gitea05.opendev.org.yaml +++ b/playbooks/host_vars/gitea05.opendev.org.yaml @@ -1,4 +1,4 @@ letsencrypt_certs: gitea05-main: - - gitea05.opendev.org + - gitea05.opendev.org:3000 - opendev.org diff --git a/playbooks/host_vars/gitea06.opendev.org.yaml b/playbooks/host_vars/gitea06.opendev.org.yaml index 8634b9283a..b8e6d4f3eb 100644 --- a/playbooks/host_vars/gitea06.opendev.org.yaml +++ b/playbooks/host_vars/gitea06.opendev.org.yaml @@ -1,4 +1,4 @@ letsencrypt_certs: gitea06-main: - - gitea06.opendev.org + - gitea06.opendev.org:3000 - opendev.org diff --git a/playbooks/host_vars/gitea07.opendev.org.yaml b/playbooks/host_vars/gitea07.opendev.org.yaml index 59e3e56d15..ebcc56f16c 100644 --- a/playbooks/host_vars/gitea07.opendev.org.yaml +++ b/playbooks/host_vars/gitea07.opendev.org.yaml @@ -1,4 +1,4 @@ letsencrypt_certs: gitea07-main: - - gitea07.opendev.org + - gitea07.opendev.org:3000 - opendev.org diff --git a/playbooks/host_vars/gitea08.opendev.org.yaml b/playbooks/host_vars/gitea08.opendev.org.yaml index 6105091315..64c34c4570 100644 --- a/playbooks/host_vars/gitea08.opendev.org.yaml +++ b/playbooks/host_vars/gitea08.opendev.org.yaml @@ -1,4 +1,4 @@ letsencrypt_certs: gitea08-main: - - gitea08.opendev.org + - gitea08.opendev.org:3000 - opendev.org diff --git a/playbooks/host_vars/insecure-ci-registry01.opendev.org.yaml b/playbooks/host_vars/insecure-ci-registry01.opendev.org.yaml index 05a656d83a..f923856211 100644 --- a/playbooks/host_vars/insecure-ci-registry01.opendev.org.yaml +++ b/playbooks/host_vars/insecure-ci-registry01.opendev.org.yaml @@ -1,4 +1,4 @@ letsencrypt_certs: insecure-ci-registry01-main: - - insecure-ci-registry01.opendev.org + - insecure-ci-registry01.opendev.org:5000 - insecure-ci-registry.opendev.org diff --git a/playbooks/roles/install-certcheck/README.rst b/playbooks/roles/install-certcheck/README.rst new file mode 100644 index 0000000000..1e9343b5ea --- /dev/null +++ b/playbooks/roles/install-certcheck/README.rst @@ -0,0 +1,24 @@ +Install ssl-cert-check + +Installs the ssl-cert-check tool and a cron job to check the freshness +of the SSL certificates for the configured domains daily. + +**Role Variables** + +.. zuul:rolevar:: ssl_cert_check_domain_list + :default: /var/lib/certcheck/domainlist + + The list of domains to check + +.. zuul:rolevar:: ssl_cert_check_days + :default: 30 + + Warn about certificates who have less than this number of days to + expiry. + +.. zuul:rolevar:: ssl_cert_check_email + :default: root + + The email to send reports to + + diff --git a/playbooks/roles/install-certcheck/defaults/main.yaml b/playbooks/roles/install-certcheck/defaults/main.yaml new file mode 100644 index 0000000000..0a27ea6dee --- /dev/null +++ b/playbooks/roles/install-certcheck/defaults/main.yaml @@ -0,0 +1,3 @@ +cert_check_domain_list: /var/lib/certcheck/domainlist +cert_check_days: 30 +cert_check_email: root diff --git a/playbooks/roles/install-certcheck/tasks/main.yaml b/playbooks/roles/install-certcheck/tasks/main.yaml new file mode 100644 index 0000000000..bfea1457bb --- /dev/null +++ b/playbooks/roles/install-certcheck/tasks/main.yaml @@ -0,0 +1,32 @@ +- name: Ensure dependencies + package: + name: + - openssl + - bsd-mailx + +- name: Ensure certcheck user + user: + name: certcheck + comment: User for SSL validation + +- name: Ensure certcheck config directory + file: + state: directory + path: '{{ cert_check_domain_list | dirname }}' + owner: certcheck + group: certcheck + mode: 0755 + +- name: Pull latest ssl-cert-check from git + git: + repo: 'https://github.com/Matty9191/ssl-cert-check' + dest: /opt/ssl-cert-check + +- name: Install cron job + cron: + user: certcheck + name: 'Run certcheck' + state: present + job: "/opt/ssl-cert-check/ssl-cert-check -a -q -f {{ cert_check_domain_list }} -x {{ cert_check_days }} -e {{ cert_check_email }}" + hour: 12 + minute: 04 diff --git a/playbooks/roles/letsencrypt-config-certcheck/README.rst b/playbooks/roles/letsencrypt-config-certcheck/README.rst new file mode 100644 index 0000000000..d3f7e63d95 --- /dev/null +++ b/playbooks/roles/letsencrypt-config-certcheck/README.rst @@ -0,0 +1,27 @@ +Generate SSL check list + +This role automatically generates a list of domains for the +certificate validation checks. This ensures our certificates are +valid and are being renewed as expected. + +This role must run after ``letsencrypt-request-certs`` role, as that +builds the ``letsencrypt_certcheck_domains`` variable for each +host and certificate. It must also run on a host that has already run +the ``install-certcheck`` role. + +**Role Variables** + +.. zuul:rolevar:: letsencrypt_certcheck_domain_list + :default: /var/lib/certcheck/ssldomains + + The ssl-cert-check domain configuration file to write. See also + the ``install-certcheck`` role. + +.. zuul:rolevar:: letsencrypt_certcheck_additional_domains + :default: [] + + A list of additional domains to check for hosts not using the + ``letsencrypt-*`` roles. Each entry should be in the format + ``hostname port``. + + diff --git a/playbooks/roles/letsencrypt-config-certcheck/defaults/main.yaml b/playbooks/roles/letsencrypt-config-certcheck/defaults/main.yaml new file mode 100644 index 0000000000..0f1857b125 --- /dev/null +++ b/playbooks/roles/letsencrypt-config-certcheck/defaults/main.yaml @@ -0,0 +1 @@ +letsencrypt_certcheck_domain_list: /var/lib/certcheck/ssldomains diff --git a/playbooks/roles/letsencrypt-config-certcheck/tasks/main.yaml b/playbooks/roles/letsencrypt-config-certcheck/tasks/main.yaml new file mode 100644 index 0000000000..70c8c44225 --- /dev/null +++ b/playbooks/roles/letsencrypt-config-certcheck/tasks/main.yaml @@ -0,0 +1,17 @@ +- name: Make domain list + set_fact: + letsencrypt_certcheck_domains: [] + +- name: Build SSL domain list + set_fact: + letsencrypt_certcheck_domains: '{{ letsencrypt_certcheck_domains }} + {{ hostvars[item]["letsencrypt_certcheck_domains" ] }}' + with_inventory_hostnames: + - letsencrypt:!disabled + +- name: Write configuration file + template: + dest: '{{ letsencrypt_certcheck_domain_list }}' + src: ssldomains.j2 + owner: certcheck + group: certcheck + mode: 0644 diff --git a/playbooks/roles/letsencrypt-config-certcheck/templates/ssldomains.j2 b/playbooks/roles/letsencrypt-config-certcheck/templates/ssldomains.j2 new file mode 100644 index 0000000000..8fd0221d05 --- /dev/null +++ b/playbooks/roles/letsencrypt-config-certcheck/templates/ssldomains.j2 @@ -0,0 +1,6 @@ +{% for domain in letsencrypt_certcheck_domains %} +{{ domain }} +{% endfor %} +{% for domain in letsencrypt_certcheck_additional_domains %} +{{ domain }} +{% endfor %} diff --git a/playbooks/roles/letsencrypt-create-certs/tasks/acme.yaml b/playbooks/roles/letsencrypt-create-certs/tasks/acme.yaml index 278b284db8..2c2d9a1a87 100644 --- a/playbooks/roles/letsencrypt-create-certs/tasks/acme.yaml +++ b/playbooks/roles/letsencrypt-create-certs/tasks/acme.yaml @@ -1,6 +1,6 @@ - name: 'Build arguments for letsencrypt acme.sh driver for: {{ item.key }}' set_fact: - acme_args: '"{% for domain in item.value %}-d {{ domain }} {% endfor %}"' + acme_args: '"{% for domain in item.value %}-d {{ domain.split(":")[0] }} {% endfor %}"' - name: 'Run acme.sh driver for {{ item.key }} certificate issue' shell: @@ -12,4 +12,4 @@ LETSENCRYPT_STAGING: '{{ "1" if letsencrypt_use_staging else "0" }}' notify: 'letsencrypt updated {{ item.key }}' -# Keys generated! \ No newline at end of file +# Keys generated! diff --git a/playbooks/roles/letsencrypt-request-certs/README.rst b/playbooks/roles/letsencrypt-request-certs/README.rst index d03216263d..57cb25e0fa 100644 --- a/playbooks/roles/letsencrypt-request-certs/README.rst +++ b/playbooks/roles/letsencrypt-request-certs/README.rst @@ -62,3 +62,12 @@ provision process. _acme-challenge.hostname01.opendev.org. IN CNAME acme.opendev.org. _acme-challenge.hostname.opendev.org. IN CNAME acme.opendev.org. _acme-challenge.foo.opendev.org. IN CNAME acme.opendev.org. + + The hostname in the first entry for each certificate will be + registered with the ``letsencrypt-config-certcheck`` for periodic + freshness tests; from the example above, ``hostname01.opendev.org`` + and ``foo.opendev.org`` would be checked. By default this will + check on port 443; if the certificate is actually active on another + port you can specify it after a colon; + e.g. ``foo.opendev.org:5000`` would indicate this host listens with + this certificate on port 5000. diff --git a/playbooks/roles/letsencrypt-request-certs/tasks/acme.yaml b/playbooks/roles/letsencrypt-request-certs/tasks/acme.yaml index 8c79f8ac43..a3f759a16a 100644 --- a/playbooks/roles/letsencrypt-request-certs/tasks/acme.yaml +++ b/playbooks/roles/letsencrypt-request-certs/tasks/acme.yaml @@ -2,7 +2,7 @@ set_fact: # NOTE(ianw): note the domains are passed in one string (between # ") as it makes argument parsing a little easier in the driver.sh - acme_args: '"{% for domain in cert.value %}-d {{ domain }} {% endfor %}"' + acme_args: '"{% for domain in cert.value %}-d {{ domain.split(":")[0] }} {% endfor %}"' - name: Run acme.sh driver for certificate issue shell: diff --git a/playbooks/roles/letsencrypt-request-certs/tasks/main.yaml b/playbooks/roles/letsencrypt-request-certs/tasks/main.yaml index 3e252ef6ca..4276485609 100644 --- a/playbooks/roles/letsencrypt-request-certs/tasks/main.yaml +++ b/playbooks/roles/letsencrypt-request-certs/tasks/main.yaml @@ -1,10 +1,6 @@ - set_fact: acme_txt_required: [] -- name: Show cert list - debug: - var: letsencrypt_certs - # Handle multiple certs for a single host; like # # letsencrypt_certs: @@ -22,5 +18,25 @@ loop_control: loop_var: cert -- debug: - var: acme_txt_required +- name: Create ssl check domain list + # For each generated certificate get the first entry as the domain + # to run the certificate validation tests against. If it specifies + # a port explicitly, use that, otherwise assume 443. + # + # Later in ssl-check role, the final certificate validation list is + # generated by walking the letsencrypt_certcheck_domains variable + # for each host in the letsencrypt group. + set_fact: + letsencrypt_certcheck_domains: >- + {%- set d = [] -%} + {%- for cert in letsencrypt_certs.keys() -%} + {%- for host in letsencrypt_certs[cert] -%} + {%- if loop.first -%} + {%- if not ":" in host -%} + {%- set host = host+":443" -%} + {%- endif -%} + {%- set d = d.append(host.replace(":"," ")) -%} + {% endif %} + {% endfor %} + {% endfor %} + {{- d -}} diff --git a/playbooks/service-letsencrypt.yaml b/playbooks/service-letsencrypt.yaml index b85dff3caf..1033611c92 100644 --- a/playbooks/service-letsencrypt.yaml +++ b/playbooks/service-letsencrypt.yaml @@ -1,6 +1,9 @@ # This needs to happen in order. letsencrypt hosts export their TXT # authentication records which is installed onto adns1, and then the # hosts verify to issue/renew keys +- hosts: "certcheck:!disabled" + roles: + - install-certcheck - hosts: "letsencrypt:!disabled" name: "Base: deploy and renew certificates" roles: @@ -14,3 +17,6 @@ name: "Create certs" roles: - letsencrypt-create-certs +- hosts: "certcheck:!disabled" + roles: + - letsencrypt-config-certcheck diff --git a/playbooks/zuul/templates/gate-groups.yaml.j2 b/playbooks/zuul/templates/gate-groups.yaml.j2 index 97079316a6..847b91db38 100644 --- a/playbooks/zuul/templates/gate-groups.yaml.j2 +++ b/playbooks/zuul/templates/gate-groups.yaml.j2 @@ -5,6 +5,9 @@ groups: docker: - bionic-docker + certcheck: + - bridge.openstack.org + letsencrypt: - letsencrypt01.opendev.org - letsencrypt02.opendev.org diff --git a/playbooks/zuul/templates/host_vars/letsencrypt01.opendev.org.yaml.j2 b/playbooks/zuul/templates/host_vars/letsencrypt01.opendev.org.yaml.j2 index b2b4974cb2..b6c7daa49b 100644 --- a/playbooks/zuul/templates/host_vars/letsencrypt01.opendev.org.yaml.j2 +++ b/playbooks/zuul/templates/host_vars/letsencrypt01.opendev.org.yaml.j2 @@ -1,7 +1,7 @@ letsencrypt_certs: letsencrypt01-main-service: - - letsencrypt01.opendev.org + - letsencrypt01.opendev.org:5000 - letsencrypt.opendev.org - alias.opendev.org letsencrypt01-other-service: - - someotherservice.opendev.org \ No newline at end of file + - someotherservice.opendev.org diff --git a/testinfra/test_letsencrypt.py b/testinfra/test_letsencrypt.py index 083236dae9..f7595389c2 100644 --- a/testinfra/test_letsencrypt.py +++ b/testinfra/test_letsencrypt.py @@ -15,6 +15,7 @@ import pytest testinfra_hosts = ['adns-letsencrypt.opendev.org', + 'bridge.openstack.org', 'letsencrypt01.opendev.org', 'letsencrypt02.opendev.org'] @@ -138,3 +139,25 @@ def test_acme_sh_config(host): config = host.file('/root/.acme.sh/account.conf') assert config.exists assert config.contains("^ACCOUNT_EMAIL='le-test@opendev.org'") + +def test_certcheck_config(host, zuul_data): + if host.backend.get_hostname() != 'bridge.openstack.org': + pytest.skip() + + if zuul_data['extra']['zuul']['job'] != 'system-config-run-letsencrypt': + pytest.skip() + + domainlist = host.file('/var/lib/certcheck/ssldomains') + + # TODO(ianw): figure out a flag or something from the + # system-config-run-letsencrypt test so that we can assert this + # file exists only in that case. + if not domainlist.exists: + pytest.skip() + + assert domainlist.exists + assert domainlist.user == 'certcheck' + # from variables + assert domainlist.contains('^letsencrypt01.opendev.org 5000') + # from extra list; may need to change if list is modified + assert domainlist.contains('^wiki.openstack.org 443') diff --git a/zuul.d/system-config-run.yaml b/zuul.d/system-config-run.yaml index 33da8699e5..e266a97ab9 100644 --- a/zuul.d/system-config-run.yaml +++ b/zuul.d/system-config-run.yaml @@ -191,6 +191,9 @@ - playbooks/service-nameserver.yaml - playbooks/service-letsencrypt.yaml host-vars: + bridge.openstack.org: + host_copy_output: + '/var/lib/certcheck': logs letsencrypt01.opendev.org: host_copy_output: '/var/log/acme.sh': logs