From 4b031f9f2477b71a44d11acf24b5759788f2da85 Mon Sep 17 00:00:00 2001
From: "James E. Blair" <jeblair@redhat.com>
Date: Tue, 19 Feb 2019 14:41:54 -0800
Subject: [PATCH] Run an haproxy load balancer for gitea
This runs an haproxy which is strikingly similar to the one we
currently run for git.openstack.org, but it is run in a docker
container.
Change-Id: I647ae8c02eb2cd4f3db2b203d61a181f7eb632d2
---
.zuul.yaml | 9 +++++
inventory/groups.yaml | 2 ++
playbooks/base.yaml | 6 ++++
playbooks/group_vars/gitea-lb.yaml | 16 +++++++++
playbooks/group_vars/gitea.yaml | 4 +++
playbooks/roles/haproxy/README.rst | 1 +
.../haproxy/files/docker/docker-compose.yaml | 12 +++++++
playbooks/roles/haproxy/tasks/main.yaml | 26 ++++++++++++++
.../roles/haproxy/templates/haproxy.cfg.j2 | 34 +++++++++++++++++++
playbooks/zuul/run-base.yaml | 1 +
.../templates/group_vars/gitea-lb.yaml.j2 | 13 +++++++
testinfra/test_gitea_lb.py | 23 +++++++++++++
12 files changed, 147 insertions(+)
create mode 100644 playbooks/group_vars/gitea-lb.yaml
create mode 100644 playbooks/roles/haproxy/README.rst
create mode 100644 playbooks/roles/haproxy/files/docker/docker-compose.yaml
create mode 100644 playbooks/roles/haproxy/tasks/main.yaml
create mode 100644 playbooks/roles/haproxy/templates/haproxy.cfg.j2
create mode 100644 playbooks/zuul/templates/group_vars/gitea-lb.yaml.j2
create mode 100644 testinfra/test_gitea_lb.py
diff --git a/.zuul.yaml b/.zuul.yaml
index 91df421dea..33d2f04ed6 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -509,6 +509,8 @@
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
+ - name: gitea-lb01.opendev.org
+ label: ubuntu-bionic
- name: gitea01.opendev.org
label: ubuntu-bionic
host-vars:
@@ -517,12 +519,19 @@
'/var/gitea/conf': logs
'/var/gitea/certs': logs
'/var/gitea/logs': logs
+ gitea-lb01.opendev.org:
+ host_copy_output:
+ '/var/haproxy/etc': logs
files:
- .zuul.yaml
- playbooks/group_vars/gitea.yaml
+ - playbooks/group_vars/gitea-lb.yaml
- playbooks/zuul/templates/group_vars/gitea.yaml.j2
+ - playbooks/zuul/templates/group_vars/gitea-lb.yaml.j2
- playbooks/roles/gitea/
+ - playbooks/roles/haproxy/
- testinfra/test_gitea.py
+ - testinfra/test_gitea_lb.py
- job:
name: infra-prod-playbook
diff --git a/inventory/groups.yaml b/inventory/groups.yaml
index acc9223571..73d16068bf 100644
--- a/inventory/groups.yaml
+++ b/inventory/groups.yaml
@@ -88,6 +88,8 @@ groups:
- git[0-9]*.openstack.org
gitea:
- gitea[0-9]*.opendev.org
+ gitea-lb:
+ - gitea-lb[0-9]*.opendev.org
grafana:
- grafana[0-9]*.open*.org
graphite:
diff --git a/playbooks/base.yaml b/playbooks/base.yaml
index 44fd0b1a76..253ff78fdc 100644
--- a/playbooks/base.yaml
+++ b/playbooks/base.yaml
@@ -79,3 +79,9 @@
roles:
- install-docker
- gitea
+
+- hosts: "gitea-lb:!disabled"
+ name: "Base: configure gitea load balancer"
+ roles:
+ - install-docker
+ - haproxy
diff --git a/playbooks/group_vars/gitea-lb.yaml b/playbooks/group_vars/gitea-lb.yaml
new file mode 100644
index 0000000000..cf3e117a6d
--- /dev/null
+++ b/playbooks/group_vars/gitea-lb.yaml
@@ -0,0 +1,16 @@
+haproxy_listeners:
+ - name: balance_git_http
+ bind:
+ - '*:80'
+ servers:
+ - name: 'gitea01.opendev.org'
+ address: '127.0.0.1:80' # placeholder
+ - name: balance_git_https
+ bind:
+ - '*:443'
+ servers:
+ - name: 'gitea01.opendev.org'
+ address: '127.0.0.1:443' # placeholder
+iptables_extra_public_tcp_ports:
+ - 443
+ - 80
diff --git a/playbooks/group_vars/gitea.yaml b/playbooks/group_vars/gitea.yaml
index daf5e205d9..1934a54338 100644
--- a/playbooks/group_vars/gitea.yaml
+++ b/playbooks/group_vars/gitea.yaml
@@ -1,2 +1,6 @@
gitea_root_email: infra-root@openstack.org
gitea_gerrit_public_key: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDVuhTMAz1H2Jr9AC3py9A0vlNna6Sdt4yrvZOayxukPqQ7GPZd+Mo7MVyypxLD479N2mA09JAdsbq1eTiPP8ksEkB+dNxZzw8mY1653R/IXSW6J9xPcoDa88HF2s/xHN24IWzgiDjNNe79AQ+sKleByEQZ++xXny3MRpy258hKUvAtjjOLOnM1PBs8JNOzBL+UPgWRgSX6GG0qywJZqjD1Qx5kvH9RTRLi+tcMhEi4laN7BYvn4csY0sYzTzPG4ZTu3ootIJoRlQGtQ0LmoFO1vSwyEJUags6/ZZGjgy3jl3kwcU/b8ZnFlF4MDw1OB1QqMb4r6bMHbXNIupp4zJbz gerrit-replication-2014-04-25
+iptables_extra_public_tcp_ports:
+ - 222
+ - 3000
+ - 3080
diff --git a/playbooks/roles/haproxy/README.rst b/playbooks/roles/haproxy/README.rst
new file mode 100644
index 0000000000..3e5aa70398
--- /dev/null
+++ b/playbooks/roles/haproxy/README.rst
@@ -0,0 +1 @@
+Install, configure, and run a haproxy server.
diff --git a/playbooks/roles/haproxy/files/docker/docker-compose.yaml b/playbooks/roles/haproxy/files/docker/docker-compose.yaml
new file mode 100644
index 0000000000..eff4cd2e1c
--- /dev/null
+++ b/playbooks/roles/haproxy/files/docker/docker-compose.yaml
@@ -0,0 +1,12 @@
+# Version 2 is the latest that is supported by docker-compose in
+# Ubuntu Xenial.
+version: '2'
+
+services:
+ haproxy:
+ restart: always
+ image: haproxy:latest
+ network_mode: host
+ volumes:
+ - /var/haproxy/etc:/usr/local/etc/haproxy
+ - /var/haproxy/run:/var/haproxy/run
diff --git a/playbooks/roles/haproxy/tasks/main.yaml b/playbooks/roles/haproxy/tasks/main.yaml
new file mode 100644
index 0000000000..5ffe0a2397
--- /dev/null
+++ b/playbooks/roles/haproxy/tasks/main.yaml
@@ -0,0 +1,26 @@
+- name: Synchronize docker-compose directory
+ synchronize:
+ src: docker/
+ dest: /etc/haproxy-docker/
+- name: Ensure registry volume directories exists
+ file:
+ state: directory
+ path: "/var/haproxy/{{ item }}"
+ owner: 1000
+ group: 1000
+ loop:
+ - etc
+ - run
+- name: Write haproxy config file
+ template:
+ src: haproxy.cfg.j2
+ dest: /var/haproxy/etc/haproxy.cfg
+- name: Install docker-compose
+ package:
+ name:
+ - docker-compose
+ state: present
+- name: Run docker-compose up
+ shell:
+ cmd: docker-compose up -d
+ chdir: /etc/haproxy-docker/
diff --git a/playbooks/roles/haproxy/templates/haproxy.cfg.j2 b/playbooks/roles/haproxy/templates/haproxy.cfg.j2
new file mode 100644
index 0000000000..5715383b15
--- /dev/null
+++ b/playbooks/roles/haproxy/templates/haproxy.cfg.j2
@@ -0,0 +1,34 @@
+global
+ uid 1000
+ gid 1000
+ log 127.0.0.1 local0
+ maxconn 4000
+ pidfile /var/haproxy/run/haproxy.pid
+ stats socket /var/haproxy/run/stats uid 1000 gid 1000 mode 0600 level admin
+
+defaults
+ log global
+ maxconn 8000
+ option redispatch
+ retries 3
+ stats enable
+ timeout http-request 10s
+ timeout queue 1m
+ timeout connect 10s
+ timeout client 2m
+ timeout server 2m
+ timeout check 10s
+
+{% for listener in haproxy_listeners %}
+listen {{ listener.name }}
+ {% for bind in listener.bind %}
+ bind {{ bind }}
+ {% endfor %}
+ mode tcp
+ balance leastconn
+ option tcplog
+
+ {% for server in listener.servers %}
+ server {{ server.name }} {{ server.address }}
+ {% endfor %}
+{% endfor %}
diff --git a/playbooks/zuul/run-base.yaml b/playbooks/zuul/run-base.yaml
index 31351b55c8..46df223717 100644
--- a/playbooks/zuul/run-base.yaml
+++ b/playbooks/zuul/run-base.yaml
@@ -63,6 +63,7 @@
- group_vars/ns.yaml
- group_vars/registry.yaml
- group_vars/gitea.yaml
+ - group_vars/gitea-lb.yaml
- host_vars/bridge.openstack.org.yaml
- name: Display group membership
command: ansible localhost -m debug -a 'var=groups'
diff --git a/playbooks/zuul/templates/group_vars/gitea-lb.yaml.j2 b/playbooks/zuul/templates/group_vars/gitea-lb.yaml.j2
new file mode 100644
index 0000000000..5841b9a57d
--- /dev/null
+++ b/playbooks/zuul/templates/group_vars/gitea-lb.yaml.j2
@@ -0,0 +1,13 @@
+haproxy_listeners:
+ - name: balance_git_http
+ bind:
+ - "*:80"
+ servers:
+ - name: "gitea01.opendev.org"
+ address: "{{ (hostvars['gitea01.opendev.org'] | default({})).get('nodepool', {}).get('public_ipv4', '') }}:3080"
+ - name: balance_git_https
+ bind:
+ - "*:443"
+ servers:
+ - name: "gitea01.opendev.org"
+ address: "{{ (hostvars['gitea01.opendev.org'] | default({})).get('nodepool', {}).get('public_ipv4', '') }}:3000"
diff --git a/testinfra/test_gitea_lb.py b/testinfra/test_gitea_lb.py
new file mode 100644
index 0000000000..e330a8e7f6
--- /dev/null
+++ b/testinfra/test_gitea_lb.py
@@ -0,0 +1,23 @@
+# Copyright 2018 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+testinfra_hosts = ['gitea-lb01.opendev.org']
+
+
+def test_gitea_listening(host):
+ gitea_https = host.socket("tcp://0.0.0.0:443")
+ assert gitea_https.is_listening
+ gitea_http = host.socket("tcp://0.0.0.0:80")
+ assert gitea_http.is_listening