From 42420830f65c0a76ba8e7e1b6a8d2974dabb9cfa Mon Sep 17 00:00:00 2001
From: Jeffrey Zhang <jeffrey.zhang@99cloud.net>
Date: Sun, 27 Mar 2016 00:39:07 +0800
Subject: [PATCH] Implement nova-ssh container
Add a nova-ssh container to handle the `nova migrate` and
`nova resize` case, in which the nova will use ssh to copy
files between machines.
Change-Id: Ie6675943f3aeabfbba8589d308d55b9c89d732db
Closes-Bug: #1562141
---
ansible/roles/nova/defaults/main.yml | 6 ++++
ansible/roles/nova/tasks/config.yml | 12 ++++++++
ansible/roles/nova/tasks/do_reconfigure.yml | 15 ++++++----
ansible/roles/nova/tasks/pull.yml | 7 +++++
ansible/roles/nova/tasks/start_compute.yml | 15 ++++++++++
ansible/roles/nova/templates/id_rsa | 1 +
ansible/roles/nova/templates/id_rsa.pub | 1 +
ansible/roles/nova/templates/nova-ssh.json.j2 | 29 +++++++++++++++++++
ansible/roles/nova/templates/ssh_config.j2 | 4 +++
ansible/roles/nova/templates/sshd_config.j2 | 5 ++++
docker/nova/nova-base/Dockerfile.j2 | 6 ++--
docker/nova/nova-ssh/Dockerfile.j2 | 23 +++++++++++++++
docker/nova/nova-ssh/extend_start.sh | 20 +++++++++++++
etc/kolla/passwords.yml | 7 +++++
kolla/cmd/genpwd.py | 23 +++++++++++++++
requirements.txt | 1 +
16 files changed, 167 insertions(+), 8 deletions(-)
create mode 100644 ansible/roles/nova/templates/id_rsa
create mode 100644 ansible/roles/nova/templates/id_rsa.pub
create mode 100644 ansible/roles/nova/templates/nova-ssh.json.j2
create mode 100644 ansible/roles/nova/templates/ssh_config.j2
create mode 100644 ansible/roles/nova/templates/sshd_config.j2
create mode 100644 docker/nova/nova-ssh/Dockerfile.j2
create mode 100644 docker/nova/nova-ssh/extend_start.sh
diff --git a/ansible/roles/nova/defaults/main.yml b/ansible/roles/nova/defaults/main.yml
index bb81cdeac2..7c5a54f151 100644
--- a/ansible/roles/nova/defaults/main.yml
+++ b/ansible/roles/nova/defaults/main.yml
@@ -32,6 +32,10 @@ nova_libvirt_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ do
nova_libvirt_tag: "{{ openstack_release }}"
nova_libvirt_image_full: "{{ nova_libvirt_image }}:{{ nova_libvirt_tag }}"
+nova_ssh_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-nova-ssh"
+nova_ssh_tag: "{{ openstack_release }}"
+nova_ssh_image_full: "{{ nova_ssh_image }}:{{ nova_ssh_tag }}"
+
nova_conductor_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-nova-conductor"
nova_conductor_tag: "{{ openstack_release }}"
nova_conductor_image_full: "{{ nova_conductor_image }}:{{ nova_conductor_tag }}"
@@ -74,3 +78,5 @@ nova_public_endpoint: "{{ public_protocol }}://{{ kolla_external_fqdn }}:{{ nova
nova_logging_debug: "{{ openstack_logging_debug }}"
openstack_nova_auth: "{'auth_url':'{{ openstack_auth.auth_url }}','username':'{{ openstack_auth.username }}','password':'{{ openstack_auth.password }}','project_name':'{{ openstack_auth.project_name }}'}"
+
+nova_ssh_port: "8022"
diff --git a/ansible/roles/nova/tasks/config.yml b/ansible/roles/nova/tasks/config.yml
index bd83120c22..8e2e7a8756 100644
--- a/ansible/roles/nova/tasks/config.yml
+++ b/ansible/roles/nova/tasks/config.yml
@@ -25,6 +25,7 @@
- "nova-novncproxy"
- "nova-scheduler"
- "nova-spicehtml5proxy"
+ - "nova-ssh"
- name: Copying over config.json files for services
template:
@@ -40,6 +41,7 @@
- "nova-novncproxy"
- "nova-scheduler"
- "nova-spicehtml5proxy"
+ - "nova-ssh"
- name: Copying over nova.conf
merge_configs:
@@ -68,3 +70,13 @@
template:
src: "libvirtd.conf.j2"
dest: "{{ node_config_directory }}/nova-libvirt/libvirtd.conf"
+
+- name: Copying files for nova-ssh
+ template:
+ src: "{{ item.src }}"
+ dest: "{{ node_config_directory }}/nova-ssh/{{ item.dest }}"
+ with_items:
+ - { src: "sshd_config.j2", dest: "sshd_config" }
+ - { src: "id_rsa", dest: "id_rsa" }
+ - { src: "id_rsa.pub", dest: "id_rsa.pub" }
+ - { src: "ssh_config.j2", dest: "ssh_config" }
diff --git a/ansible/roles/nova/tasks/do_reconfigure.yml b/ansible/roles/nova/tasks/do_reconfigure.yml
index 8891a4dea1..8bbd369d70 100644
--- a/ansible/roles/nova/tasks/do_reconfigure.yml
+++ b/ansible/roles/nova/tasks/do_reconfigure.yml
@@ -1,5 +1,5 @@
---
-- name: Ensuring the nova libvirt, conductor, api, consoleauth and scheduler containers are up
+- name: Ensuring the nova libvirt, ssh, conductor, api, consoleauth and scheduler containers are up
kolla_docker:
name: "{{ item.name }}"
action: "get_container_state"
@@ -8,6 +8,7 @@
when: inventory_hostname in groups[item.group]
with_items:
- { name: nova_libvirt, group: compute }
+ - { name: nova_ssh, group: compute }
- { name: nova_conductor, group: nova-conductor }
- { name: nova_api, group: nova-api }
- { name: nova_consoleauth, group: nova-consoleauth }
@@ -55,7 +56,7 @@
- include: config.yml
-- name: Check the configs for nova libvirt, conductor, api, consoleauth and scheduler containers
+- name: Check the configs for nova libvirt, ssh, conductor, api, consoleauth and scheduler containers
command: docker exec {{ item.name }} /usr/local/bin/kolla_set_configs --check
changed_when: false
failed_when: false
@@ -63,6 +64,7 @@
when: inventory_hostname in groups[item.group]
with_items:
- { name: nova_libvirt, group: compute }
+ - { name: nova_ssh, group: compute }
- { name: nova_conductor, group: nova-conductor }
- { name: nova_api, group: nova-api }
- { name: nova_consoleauth, group: nova-consoleauth }
@@ -107,7 +109,7 @@
# NOTE(jeffrey4l): when config_strategy == 'COPY_ALWAYS'
# and container env['KOLLA_CONFIG_STRATEGY'] == 'COPY_ONCE',
# just remove the container and start again
-- name: Containers config strategy for nova libvirt, conductor, api, consoleauth and scheduler containers
+- name: Containers config strategy for nova libvirt, ssh, conductor, api, consoleauth and scheduler containers
kolla_docker:
name: "{{ item.name }}"
action: "get_container_env"
@@ -115,6 +117,7 @@
when: inventory_hostname in groups[item.group]
with_items:
- { name: nova_libvirt, group: compute }
+ - { name: nova_ssh, group: compute }
- { name: nova_conductor, group: nova-conductor }
- { name: nova_api, group: nova-api }
- { name: nova_consoleauth, group: nova-consoleauth }
@@ -156,7 +159,7 @@
- nova_console == 'spice'
- inventory_hostname in groups['nova-spicehtml5proxy']
-- name: Remove the nova libvirt, conductor, api, consoleauth and scheduler containers
+- name: Remove the nova libvirt, ssh, conductor, api, consoleauth and scheduler containers
kolla_docker:
name: "{{ item[0]['name'] }}"
action: "remove_container"
@@ -167,6 +170,7 @@
- item[2]['rc'] == 1
with_together:
- [{ name: nova_libvirt, group: compute },
+ { name: nova_ssh, group: compute },
{ name: nova_conductor, group: nova-conductor },
{ name: nova_api, group: nova-api },
{ name: nova_consoleauth, group: nova-consoleauth },
@@ -246,7 +250,7 @@
- nova_console == 'spice'
- remove_nova_spicehtml5proxy_container.changed
-- name: Restart the nova libvirt, conductor, api, consoleauth and scheduler containers
+- name: Restart the nova libvirt, ssh, conductor, api, consoleauth and scheduler containers
kolla_docker:
name: "{{ item[0]['name'] }}"
action: "restart_container"
@@ -257,6 +261,7 @@
- item[2]['rc'] == 1
with_together:
- [{ name: nova_libvirt, group: compute },
+ { name: nova_ssh, group: compute },
{ name: nova_conductor, group: nova-conductor },
{ name: nova_api, group: nova-api },
{ name: nova_consoleauth, group: nova-consoleauth },
diff --git a/ansible/roles/nova/tasks/pull.yml b/ansible/roles/nova/tasks/pull.yml
index a8d0a9f52a..b761f0c451 100644
--- a/ansible/roles/nova/tasks/pull.yml
+++ b/ansible/roles/nova/tasks/pull.yml
@@ -45,6 +45,13 @@
image: "{{ nova_libvirt_image_full }}"
when: inventory_hostname in groups['compute']
+- name: Pulling nova-ssh image
+ kolla_docker:
+ action: "pull_image"
+ common_options: "{{ docker_common_options }}"
+ image: "{{ nova_ssh_image_full }}"
+ when: inventory_hostname in groups['compute']
+
- name: Pulling nova-novncproxy image
kolla_docker:
action: "pull_image"
diff --git a/ansible/roles/nova/tasks/start_compute.yml b/ansible/roles/nova/tasks/start_compute.yml
index 1540f4673f..f14b2b0c49 100644
--- a/ansible/roles/nova/tasks/start_compute.yml
+++ b/ansible/roles/nova/tasks/start_compute.yml
@@ -64,3 +64,18 @@
when:
- inventory_hostname in groups['compute']
- enable_nova_fake | bool
+
+- name: Staring nova-ssh container
+ kolla_docker:
+ action: "start_container"
+ common_options: "{{ docker_common_options }}"
+ image: "{{ nova_ssh_image_full }}"
+ name: "nova_ssh"
+ volumes:
+ - "{{ node_config_directory }}/nova-ssh/:{{ container_config_directory }}/:ro"
+ - "kolla_logs:/var/log/kolla"
+ - "nova_compute:/var/lib/nova"
+ - "heka_socket:/var/lib/kolla/heka/"
+ # TODO(jeffrey4l): how to handle the nova-compute-fake and
+ # nova-compute-ironic
+ when: inventory_hostname in groups['compute']
diff --git a/ansible/roles/nova/templates/id_rsa b/ansible/roles/nova/templates/id_rsa
new file mode 100644
index 0000000000..173a4b3e12
--- /dev/null
+++ b/ansible/roles/nova/templates/id_rsa
@@ -0,0 +1 @@
+{{ nova_ssh_key.private_key }}
diff --git a/ansible/roles/nova/templates/id_rsa.pub b/ansible/roles/nova/templates/id_rsa.pub
new file mode 100644
index 0000000000..16bd674f22
--- /dev/null
+++ b/ansible/roles/nova/templates/id_rsa.pub
@@ -0,0 +1 @@
+{{ nova_ssh_key.public_key }}
diff --git a/ansible/roles/nova/templates/nova-ssh.json.j2 b/ansible/roles/nova/templates/nova-ssh.json.j2
new file mode 100644
index 0000000000..1fb041ecc9
--- /dev/null
+++ b/ansible/roles/nova/templates/nova-ssh.json.j2
@@ -0,0 +1,29 @@
+{
+ "command": "/usr/sbin/sshd -D",
+ "config_files": [
+ {
+ "source": "{{ container_config_directory }}/sshd_config",
+ "dest": "/etc/ssh/sshd_config",
+ "owner": "root",
+ "perm": "0644"
+ },
+ {
+ "source": "{{ container_config_directory }}/ssh_config",
+ "dest": "/var/lib/nova/.ssh/config",
+ "owner": "nova",
+ "perm": "0600"
+ },
+ {
+ "source": "{{ container_config_directory }}/id_rsa",
+ "dest": "/var/lib/nova/.ssh/id_rsa",
+ "owner": "nova",
+ "perm": "0600"
+ },
+ {
+ "source": "{{ container_config_directory }}/id_rsa.pub",
+ "dest": "/var/lib/nova/.ssh/authorized_keys",
+ "owner": "nova",
+ "perm": "0600"
+ }
+ ]
+}
diff --git a/ansible/roles/nova/templates/ssh_config.j2 b/ansible/roles/nova/templates/ssh_config.j2
new file mode 100644
index 0000000000..7c5c962f9d
--- /dev/null
+++ b/ansible/roles/nova/templates/ssh_config.j2
@@ -0,0 +1,4 @@
+Host *
+ StrictHostKeyChecking no
+ UserKnownHostsFile /dev/null
+ port {{ nova_ssh_port }}
diff --git a/ansible/roles/nova/templates/sshd_config.j2 b/ansible/roles/nova/templates/sshd_config.j2
new file mode 100644
index 0000000000..ba6c8df8cc
--- /dev/null
+++ b/ansible/roles/nova/templates/sshd_config.j2
@@ -0,0 +1,5 @@
+Port {{ nova_ssh_port }}
+ListenAddress {{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}
+
+SyslogFacility AUTHPRIV
+UsePAM yes
diff --git a/docker/nova/nova-base/Dockerfile.j2 b/docker/nova/nova-base/Dockerfile.j2
index f9f465f16a..e34f3f228d 100644
--- a/docker/nova/nova-base/Dockerfile.j2
+++ b/docker/nova/nova-base/Dockerfile.j2
@@ -45,11 +45,11 @@ RUN apt-get install -y --no-install-recommends \
ADD nova-base-archive /nova-base-source
RUN ln -s nova-base-source/* nova \
- && useradd --user-group nova \
+ && useradd --user-group --home-dir /var/lib/nova nova \
&& /var/lib/kolla/venv/bin/pip --no-cache-dir install --upgrade -c requirements/upper-constraints.txt /nova \
- && mkdir -p /etc/nova /home/nova /var/lib/nova \
+ && mkdir -p /etc/nova /var/lib/nova \
&& cp -r /nova/etc/nova/* /etc/nova/ \
- && chown -R nova: /etc/nova /home/nova /var/lib/nova \
+ && chown -R nova: /etc/nova /var/lib/nova \
&& sed -i 's|^exec_dirs.*|exec_dirs=/var/lib/kolla/venv/bin,/sbin,/usr/sbin,/bin,/usr/bin,/usr/local/bin,/usr/local/sbin|g' /etc/nova/rootwrap.conf
COPY nova_sudoers /etc/sudoers.d/nova_sudoers
diff --git a/docker/nova/nova-ssh/Dockerfile.j2 b/docker/nova/nova-ssh/Dockerfile.j2
new file mode 100644
index 0000000000..7bc859ef1b
--- /dev/null
+++ b/docker/nova/nova-ssh/Dockerfile.j2
@@ -0,0 +1,23 @@
+FROM {{ namespace }}/{{ image_prefix }}nova-base:{{ tag }}
+MAINTAINER {{ maintainer }}
+
+{% if base_distro in ['centos', 'fedora', 'oraclelinux', 'rhel'] %}
+
+RUN yum -y install \
+ openssh-server \
+ && yum clean all
+
+{% elif base_distro in ['ubuntu', 'debian'] %}
+
+RUN apt-get install -y --no-install-recommends \
+ openssh-server \
+ && apt-get clean \
+ && mkdir -p /var/run/sshd \
+ && chmod 0755 /var/run/sshd
+
+{% endif %}
+
+COPY extend_start.sh /usr/local/bin/kolla_extend_start
+RUN chmod 755 /usr/local/bin/kolla_extend_start
+
+{{ include_footer }}
diff --git a/docker/nova/nova-ssh/extend_start.sh b/docker/nova/nova-ssh/extend_start.sh
new file mode 100644
index 0000000000..da05e4379a
--- /dev/null
+++ b/docker/nova/nova-ssh/extend_start.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+if [[ ! -L /dev/log ]]; then
+ ln -sf /var/lib/kolla/heka/log /dev/log
+fi
+
+SSH_HOST_KEY_TYPES=( "rsa" "dsa" "ecdsa" "ed25519" )
+
+for key_type in ${SSH_HOST_KEY_TYPES[@]}; do
+ KEY_PATH=/etc/ssh/ssh_host_${key_type}_key
+ if [[ ! -f "${KEY_PATH}" ]]; then
+ ssh-keygen -q -t ${key_type} -f ${KEY_PATH} -N ""
+ fi
+done
+
+mkdir -p /var/lib/nova/.ssh
+
+if [[ $(stat -c %U:%G /var/lib/nova/.ssh) != "nova:nova" ]]; then
+ chown nova: /var/lib/nova/.ssh
+fi
diff --git a/etc/kolla/passwords.yml b/etc/kolla/passwords.yml
index 12b2d6bf5e..de41e700c4 100644
--- a/etc/kolla/passwords.yml
+++ b/etc/kolla/passwords.yml
@@ -65,6 +65,13 @@ manila_keystone_password:
memcache_secret_key:
+nova_ssh_private_key:
+nova_ssh_public_key:
+
+nova_ssh_key:
+ private_key:
+ public_key:
+
####################
# RabbitMQ options
####################
diff --git a/kolla/cmd/genpwd.py b/kolla/cmd/genpwd.py
index 728dd458b9..70dbc0357c 100755
--- a/kolla/cmd/genpwd.py
+++ b/kolla/cmd/genpwd.py
@@ -12,16 +12,29 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import os
import random
import string
import uuid
import yaml
+from Crypto.PublicKey import RSA
+
+
+def generate_RSA(bits=2048):
+ new_key = RSA.generate(bits, os.urandom)
+ private_key = new_key.exportKey("PEM")
+ public_key = new_key.publickey().exportKey("OpenSSH")
+ return private_key, public_key
+
def main():
# These keys should be random uuids
uuid_keys = ['ceph_cluster_fsid', 'rbd_secret_uuid']
+ # SSH key pair
+ ssh_keys = ['nova_ssh_key']
+
# If these keys are None, leave them as None
blank_keys = ['docker_registry_password']
@@ -32,6 +45,16 @@ def main():
passwords = yaml.load(f.read())
for k, v in passwords.items():
+ if (k in ssh_keys and
+ (v is None
+ or v.get('public_key') is None
+ and v.get('private_key') is None)):
+ private_key, public_key = generate_RSA()
+ passwords[k] = {
+ 'private_key': private_key,
+ 'public_key': public_key
+ }
+ continue
if v is None:
if k in blank_keys and v is None:
continue
diff --git a/requirements.txt b/requirements.txt
index 7995bf1523..9646d20463 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -11,3 +11,4 @@ oslo.config>=3.7.0 # Apache-2.0
graphviz>=0.4.0 # MIT License
beautifulsoup4 # MIT
setuptools>=16.0 # PSF/ZPL
+pycrypto>=2.6 # Public Domain