diff --git a/dib-elements/README.rst b/dib-elements/README.rst new file mode 100644 index 0000000..e701fce --- /dev/null +++ b/dib-elements/README.rst @@ -0,0 +1,74 @@ +Using diskimage-builder to build opendev-ci nodes +==================================================== + +In addition to being able to just download and consume images that are the +same as what run devstack, it's easy to make your own for local dev or +testing - or just for fun. + +Install diskimage-builder +------------------------- + +Install the dependencies: + +:: + + sudo apt-get install kpartx qemu-utils curl python-yaml debootstrap + +Install diskimage-builder: + +:: + + sudo -H pip install diskimage-builder + + +Build an image +-------------- + +Building an image is simple, we have a script! + +:: + + bash tools/build-image.sh + +See the script for environment variables to set distribution, etc. By default +it builds an ubuntu-minimal based image. You should be left with a .qcow2 +image file of your selected distribution. + +Infra uses the -minimal build type for building Ubuntu/CentOS/Fedora. For +example: ubuntu-minimal. + +It is a good idea to set ``TMP_DIR`` to somewhere with plenty of space +to avoid the disappointment of a full-disk mid-way through the script +run. + +While testing, consider exporting DIB_OFFLINE=true, to skip updating the cache. + +Mounting the image +------------------ + +If you would like to examine the contents of the image, you can mount it on +a loopback device using qemu-nbd. + +:: + + sudo apt-get install qemu-utils + sudo modprobe nbd max_part=16 + sudo mkdir -p /tmp/newimage + sudo qemu-nbd -c /dev/nbd1 /path/to/opendev-ci-node-precise.qcow2 + sudo mount /dev/nbd1p1 /tmp/newimage + +or use the scripts + +:: + + sudo apt-get install qemu-utils + sudo modprobe nbd max_part=16 + sudo tools/mount-image.sh opendev-ci-node-precise.qcow2 + sudo tools/umount-image.sh + +Other things +------------ + +It's a qcow2 image, so you can do tons of things with it. You can upload it +to glance, you can boot it using kvm, and you can even copy it to a cloud +server, replace the contents of the server with it and kexec the new kernel. diff --git a/dib-elements/bindep-fallback.txt b/dib-elements/bindep-fallback.txt new file mode 100644 index 0000000..5936a19 --- /dev/null +++ b/dib-elements/bindep-fallback.txt @@ -0,0 +1,114 @@ +# This is the fallback list for packages to install. Do not add +# additional packages here. Repositories should use bindep and create +# their own bindep.txt files if the list below is not +# working for them. + +build-essential [platform:dpkg] +curl [!platform:gentoo] +net-misc/curl [platform:gentoo] +dev-libs/cyrus-sasl [platform:gentoo] +cyrus-sasl-devel [platform:rpm] +media-fonts/nanumfont [platform:gentoo] +fonts-nanum [platform:dpkg] +media-fonts/takao-fonts [platform:gentoo] +fonts-takao [platform:dpkg] +gawk +gettext [!platform:suse] +gettext-runtime [platform:suse] +graphviz [!platform:gentoo] +media-gfx/graphviz [platform:gentoo] +language-pack-en [platform:ubuntu] +libcurl-devel [platform:rpm] +libcurl4-gnutls-dev [platform:dpkg] +libevent-dev [platform:dpkg] +libevent-devel [platform:rpm] +dev-libs/libevent [platform:gentoo] +libffi-dev [platform:dpkg] +libffi-devel [platform:redhat platform:suse] +virtual/libffi [platform:gentoo] +libjerasure-dev [platform:ubuntu] +dev-libs/jerasure [platform:gentoo] +libjpeg-dev [platform:dpkg] +libjpeg8-devel [platform:suse] +libjpeg-turbo-devel [platform:redhat] +media-libs/libjpeg-turbo [platform:gentoo] +libldap2-dev [platform:dpkg] +net-nds/openldap [platform:gentoo] +libmysqlclient-dev [platform:dpkg] +libpcap-dev [platform:dpkg] +libpcap-devel [platform:rpm] +net-libs/libpcap [platform:gentoo] +libpq-dev [platform:dpkg] +librrd-dev [platform:dpkg] +net-analyzer/rrdtool [platform:gentoo] +libsasl2-dev [platform:dpkg] +libselinux-python [platform:redhat] +python-selinux [platform:suse] +sys-libs/libselinux [platform:gentoo] +libsqlite3-dev [platform:dpkg] +libuuid-devel [platform:rpm] +libvirt-dev [platform:dpkg] +libvirt-devel [platform:rpm] +app-emulation/libvirt [platform:gentoo] +libvirt-python [platform:rpm !platform:suse] +dev-python/libvirt-python [platform:gentoo] +libxml2-dev [platform:dpkg] +libxml2-devel [platform:rpm] +libxml2-utils [platform:dpkg] +dev-libs/libxml2 [platform:gentoo] +libxslt-devel [platform:rpm] +libxslt1-dev [platform:dpkg] +dev-libs/libxslt [platform:gentoo] +locales [platform:debian] +mariadb [platform:rpm] +mariadb-server [platform:redhat] +mariadb-devel [platform:redhat] +libmysqlclient-devel [platform:suse] +dev-db/mariadb [platform:gentoo] +memcached +mysql-client [platform:dpkg] +mysql-server [platform:dpkg] +openldap-devel [platform:redhat] +openldap2-devel [platform:suse] +pkg-config [platform:dpkg platform:suse] +pkgconfig [platform:redhat] +virtual/pkgconfig [platform:gentoo] +postgresql +postgresql-client [platform:dpkg] +postgresql-devel [platform:rpm] +postgresql-server [platform:rpm] +pypy [platform:ubuntu] +pypy-dev [platform:ubuntu] +python-dev [platform:dpkg] +python-devel [platform:rpm] +dev-lang/python [platform:gentoo] +python-libvirt [platform:dpkg] +python-lxml [!platform:gentoo !platform:fedora] +python2-lxml [platform:fedora] +dev-python/lxml [platform:gentoo] +# Note that python3-all-dev includes python3-all, added +# both here for documentary purpose. +python3-all [platform:dpkg] +python3-all-dev [platform:dpkg] +python3-devel [platform:fedora platform:suse] +# python3-devel does not pull in the python3 package on openSUSE so +# we need to be explicit. The python3 package contains the XML module +# which is required by a python3 virtualenv. Similarly, in python2, +# the XML module is located in python-xml which is not pulled in +# by python-devel as well. See https://bugzilla.suse.com/show_bug.cgi?id=1046990 +python3 [platform:suse] +python-xml [platform:suse] +rrdtool-devel [platform:rpm] +sqlite [platform:redhat] +sqlite-devel [platform:redhat] +sqlite3-devel [platform:suse] +sqlite3 [platform:dpkg] +dev-db/sqlite [platform:gentoo] +swig +unzip +uuid-dev [platform:dpkg] +xsltproc [platform:dpkg] +zip +zlib-devel [platform:rpm] +zlib1g-dev [platform:dpkg] +sys-libs/zlib [platform:gentoo] diff --git a/dib-elements/cache-devstack/README.rst b/dib-elements/cache-devstack/README.rst new file mode 100644 index 0000000..073f045 --- /dev/null +++ b/dib-elements/cache-devstack/README.rst @@ -0,0 +1,17 @@ +cache-devstack +============== + +Pre-cache a range of things into CI images. This element uses the +``source-repositories`` element to acquire files to be cached. The +standard cache location is ``/opt/cache/files``. + +A number of strategies are used to get the files to be cached. + +We have a number of ``source-repository-*`` files for each package +package that should be cached into images. + +``extra-data.d/55-cache-devstack-repos`` goes through each devstack +branch and runs the ``tools/image_list.sh`` script to dynamically +build a list of files to cache as requested by devstack. This is +mostly virtual machine images, but also some other peripheral packages. + diff --git a/dib-elements/cache-devstack/element-deps b/dib-elements/cache-devstack/element-deps new file mode 100644 index 0000000..7b2a984 --- /dev/null +++ b/dib-elements/cache-devstack/element-deps @@ -0,0 +1 @@ +openstack-repos diff --git a/dib-elements/cache-devstack/source-repository-dstat_graph b/dib-elements/cache-devstack/source-repository-dstat_graph new file mode 100644 index 0000000..eb49e25 --- /dev/null +++ b/dib-elements/cache-devstack/source-repository-dstat_graph @@ -0,0 +1 @@ +dstat_graph git /opt/cache/dstat_graph/ https://opendev.org/opendev/dstat_graph master diff --git a/dib-elements/cache-devstack/source-repository-images b/dib-elements/cache-devstack/source-repository-images new file mode 100644 index 0000000..4e56a16 --- /dev/null +++ b/dib-elements/cache-devstack/source-repository-images @@ -0,0 +1,15 @@ +cirros-0.5.2-x86_64-disk.img file /opt/cache/files/cirros-0.5.2-x86_64-disk.img https://download.cirros-cloud.net/0.5.2/cirros-0.5.2-x86_64-disk.img +cirros-0.5.2-x86_64-uec.tar.gz file /opt/cache/files/cirros-0.5.2-x86_64-uec.tar.gz https://download.cirros-cloud.net/0.5.2/cirros-0.5.2-x86_64-uec.tar.gz +cirros-0.5.2-aarch64-disk.img file /opt/cache/files/cirros-0.5.2-aarch64-disk.img https://download.cirros-cloud.net/0.5.2/cirros-0.5.2-aarch64-disk.img +cirros-0.5.3-x86_64-disk.img file /opt/cache/files/cirros-0.5.3-x86_64-disk.img https://download.cirros-cloud.net/0.5.3/cirros-0.5.3-x86_64-disk.img +cirros-0.5.3-x86_64-uec.tar.gz file /opt/cache/files/cirros-0.5.3-x86_64-uec.tar.gz https://download.cirros-cloud.net/0.5.3/cirros-0.5.3-x86_64-uec.tar.gz +cirros-0.5.3-aarch64-disk.img file /opt/cache/files/cirros-0.5.3-aarch64-disk.img https://download.cirros-cloud.net/0.5.3/cirros-0.5.3-aarch64-disk.img +cirros-0.6.1-x86_64-disk.img file /opt/cache/files/cirros-0.6.1-x86_64-disk.img https://download.cirros-cloud.net/0.6.1/cirros-0.6.1-x86_64-disk.img +cirros-0.6.1-x86_64-uec.tar.gz file /opt/cache/files/cirros-0.6.1-x86_64-uec.tar.gz https://download.cirros-cloud.net/0.6.1/cirros-0.6.1-x86_64-uec.tar.gz +cirros-0.6.1-aarch64-disk.img file /opt/cache/files/cirros-0.6.1-aarch64-disk.img https://download.cirros-cloud.net/0.6.1/cirros-0.6.1-aarch64-disk.img +cirros-0.6.2-x86_64-disk.img file /opt/cache/files/cirros-0.6.2-x86_64-disk.img https://download.cirros-cloud.net/0.6.2/cirros-0.6.2-x86_64-disk.img +cirros-0.6.2-x86_64-uec.tar.gz file /opt/cache/files/cirros-0.6.2-x86_64-uec.tar.gz https://download.cirros-cloud.net/0.6.2/cirros-0.6.2-x86_64-uec.tar.gz +cirros-0.6.2-aarch64-disk.img file /opt/cache/files/cirros-0.6.2-aarch64-disk.img https://download.cirros-cloud.net/0.6.2/cirros-0.6.2-aarch64-disk.img +etcd-v3.1.10-linux-amd64.tar.gz file /opt/cache/files/etcd-v3.1.10-linux-amd64.tar.gz https://github.com/etcd-io/etcd/releases/download/v3.1.10/etcd-v3.1.10-linux-amd64.tar.gz +etcd-v3.2.17-linux-amd64.tar.gz file /opt/cache/files/etcd-v3.2.17-linux-amd64.tar.gz https://github.com/etcd-io/etcd/releases/download/v3.2.17/etcd-v3.2.17-linux-amd64.tar.gz +etcd-v3.3.12-linux-amd64.tar.gz file /opt/cache/files/etcd-v3.3.12-linux-amd64.tar.gz https://github.com/etcd-io/etcd/releases/download/v3.3.12/etcd-v3.3.12-linux-amd64.tar.gz diff --git a/dib-elements/cache-devstack/source-repository-pip b/dib-elements/cache-devstack/source-repository-pip new file mode 100644 index 0000000..9c309aa --- /dev/null +++ b/dib-elements/cache-devstack/source-repository-pip @@ -0,0 +1,2 @@ +get-pip-py file /opt/cache/files/get-pip.py https://bootstrap.pypa.io/get-pip.py +get-pip-py-py36 file /opt/cache/files/get-pip.py-py36 https://bootstrap.pypa.io/pip/3.6/get-pip.py diff --git a/dib-elements/cache-devstack/source-repository-stackviz b/dib-elements/cache-devstack/source-repository-stackviz new file mode 100644 index 0000000..48a4647 --- /dev/null +++ b/dib-elements/cache-devstack/source-repository-stackviz @@ -0,0 +1 @@ +stackviz file /opt/cache/files/stackviz-latest.tar.gz https://tarballs.openstack.org/stackviz/dist/stackviz-latest.tar.gz diff --git a/dib-elements/cache-devstack/source-repository-zanatacli b/dib-elements/cache-devstack/source-repository-zanatacli new file mode 100644 index 0000000..63e9bcb --- /dev/null +++ b/dib-elements/cache-devstack/source-repository-zanatacli @@ -0,0 +1 @@ +zanata-cli file /opt/cache/files/zanata-cli-4.3.3-dist.tar.gz https://search.maven.org/remotecontent?filepath=org/zanata/zanata-cli/4.3.3/zanata-cli-4.3.3-dist.tar.gz diff --git a/dib-elements/control-plane-minimal/README.rst b/dib-elements/control-plane-minimal/README.rst new file mode 100644 index 0000000..9cc95e6 --- /dev/null +++ b/dib-elements/control-plane-minimal/README.rst @@ -0,0 +1,4 @@ +control-plane-minimal +--------------------- + +Essential packages for an OpenDev infra control-plane image. diff --git a/dib-elements/control-plane-minimal/element-deps b/dib-elements/control-plane-minimal/element-deps new file mode 100644 index 0000000..89d13d0 --- /dev/null +++ b/dib-elements/control-plane-minimal/element-deps @@ -0,0 +1,2 @@ +openssh-server +package-installs diff --git a/dib-elements/control-plane-minimal/package-installs.yaml b/dib-elements/control-plane-minimal/package-installs.yaml new file mode 100644 index 0000000..96097aa --- /dev/null +++ b/dib-elements/control-plane-minimal/package-installs.yaml @@ -0,0 +1,11 @@ +acpid: +coreutils: +cron: +util-linux: +python3: + phase: pre-install.d +ntp: +ntpdate: +lvm2: +haveged: +iptables: diff --git a/dib-elements/control-plane-minimal/post-install.d/80-enable-haveged b/dib-elements/control-plane-minimal/post-install.d/80-enable-haveged new file mode 100755 index 0000000..49f6031 --- /dev/null +++ b/dib-elements/control-plane-minimal/post-install.d/80-enable-haveged @@ -0,0 +1,17 @@ +#!/bin/bash + +if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +case "$DIB_INIT_SYSTEM" in + systemd) + systemctl enable haveged.service + ;; + *) + echo "Unsupported init system $DIB_INIT_SYSTEM" + exit 1 + ;; +esac diff --git a/dib-elements/control-plane-minimal/post-install.d/80-enable-infra-services b/dib-elements/control-plane-minimal/post-install.d/80-enable-infra-services new file mode 100755 index 0000000..e7698dc --- /dev/null +++ b/dib-elements/control-plane-minimal/post-install.d/80-enable-infra-services @@ -0,0 +1,17 @@ +#!/bin/bash + +if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +case "$DIB_INIT_SYSTEM" in + systemd) + systemctl enable ntp.service + ;; + *) + echo "Unsupported init system $DIB_INIT_SYSTEM" + exit 1 + ;; +esac diff --git a/dib-elements/control-plane-minimal/post-install.d/89-sshd b/dib-elements/control-plane-minimal/post-install.d/89-sshd new file mode 100755 index 0000000..46cc60f --- /dev/null +++ b/dib-elements/control-plane-minimal/post-install.d/89-sshd @@ -0,0 +1,33 @@ +#!/bin/bash +# Copyright (C) 2011-2013 OpenStack Foundation +# Copyright 2016 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +# NOTE(pabelanger): Glean configures access for root user, so allow us to +# properly login. +sed -i -e'/PermitRootLogin/d' /etc/ssh/sshd_config \ + && echo "PermitRootLogin yes" >> /etc/ssh/sshd_config + +# NOTE(clarkb): Glean configures ssh keys only and not passwords. Disable +# unnecessary password auth. +sed -i -e '/PasswordAuthentication/d' /etc/ssh/sshd_config \ + && echo "PasswordAuthentication no" >> /etc/ssh/sshd_config diff --git a/dib-elements/infra-package-needs/README.rst b/dib-elements/infra-package-needs/README.rst new file mode 100644 index 0000000..55fa581 --- /dev/null +++ b/dib-elements/infra-package-needs/README.rst @@ -0,0 +1,4 @@ +infra-package-needs +------------------- + +Install required packages for a OpenDev infra host diff --git a/dib-elements/infra-package-needs/element-deps b/dib-elements/infra-package-needs/element-deps new file mode 100644 index 0000000..77cf654 --- /dev/null +++ b/dib-elements/infra-package-needs/element-deps @@ -0,0 +1,3 @@ +ensure-venv +openssh-server +package-installs diff --git a/dib-elements/infra-package-needs/install.d/10-packages b/dib-elements/infra-package-needs/install.d/10-packages new file mode 100755 index 0000000..2fe1b7e --- /dev/null +++ b/dib-elements/infra-package-needs/install.d/10-packages @@ -0,0 +1,27 @@ +#!/bin/bash +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +# dib-lint: disable=setu setpipefail +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -e + +if [[ "$DISTRO_NAME" =~ ^(centos|rocky)$ && ! ${DIB_RELEASE} =~ '9' ]] ; then + # 9-stream we are just using the default rngd + # Note: $YUM exposed by centos|rhel-like environment, correct across releases + ${YUM} -y install --enablerepo=epel haveged +fi diff --git a/dib-elements/infra-package-needs/install.d/40-install-bindep b/dib-elements/infra-package-needs/install.d/40-install-bindep new file mode 100755 index 0000000..3cea319 --- /dev/null +++ b/dib-elements/infra-package-needs/install.d/40-install-bindep @@ -0,0 +1,33 @@ +#!/bin/bash +# Copyright (C) 2015 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# Install bindep into a virtualenv +# This is in /usr instead of /usr/local due to this bug on precise: +# https://bugs.launchpad.net/ubuntu/+source/python2.7/+bug/839588 + +# dib-lint: disable=setu setpipefail +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -e + +python3 -m venv /usr/bindep-env +if [[ ${DIB_RELEASE} == 'xenial' ]]; then + # The pip on xenial can't figure out it shouldn't install + # the latest pip; this is the last to support 3.5 + /usr/bindep-env/bin/pip install --upgrade pip==20.3.4 +else + /usr/bindep-env/bin/pip install --upgrade pip +fi +/usr/bindep-env/bin/pip install bindep diff --git a/dib-elements/infra-package-needs/install.d/89-rsyslog b/dib-elements/infra-package-needs/install.d/89-rsyslog new file mode 100755 index 0000000..fe3b0d6 --- /dev/null +++ b/dib-elements/infra-package-needs/install.d/89-rsyslog @@ -0,0 +1,26 @@ +#!/bin/bash +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +# dib-lint: disable=setu setpipefail +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -e + +if [ "$DISTRO_NAME" == "ubuntu" ]; then + rsyslog_dir="$(dirname $0)/../rsyslog.d" + cp -RP $rsyslog_dir/* /etc/rsyslog.d/ +fi diff --git a/dib-elements/infra-package-needs/package-installs.yaml b/dib-elements/infra-package-needs/package-installs.yaml new file mode 100644 index 0000000..3bf1a69 --- /dev/null +++ b/dib-elements/infra-package-needs/package-installs.yaml @@ -0,0 +1,28 @@ +acl: +acpid: +coreutils: +cron: +util-linux: +build-essential: +python3-dev: +uuid-runtime: +traceroute: +ntp: +ntpdate: +gentoolkit: +at: +strace: +tcpdump: +rsyslog: +git: +rsync: +parted: +wget: +iputils-ping: +iproute2: +dnsutils: +haveged: +iptables: +redhat-rpm-config: +redhat-lsb-core: +gnupg2: diff --git a/dib-elements/infra-package-needs/pkg-map b/dib-elements/infra-package-needs/pkg-map new file mode 100644 index 0000000..2742099 --- /dev/null +++ b/dib-elements/infra-package-needs/pkg-map @@ -0,0 +1,117 @@ +{ + "release": { + "centos": { + "8": { + "ntp": "chrony", + "ntpdate": "" + }, + "9-stream": { + "ntp": "chrony", + "ntpdate": "", + "iptables": "iptables-services" + } + }, + "debian": { + "bookworm": { + "ntp": "", + "ntpdate": "" + } + }, + "rocky": { + "8": { + "ntp": "chrony", + "ntpdate": "", + "coreutils": "" + }, + "9": { + "ntp": "chrony", + "ntpdate": "", + "iptables": "iptables-services", + "coreutils": "" + } + }, + "ubuntu": { + "focal": { + "ntp": "", + "ntpdate": "" + }, + "jammy": { + "ntp": "", + "ntpdate": "" + }, + "noble": { + "ntp": "", + "ntpdate": "" + } + } + }, + "distro": { + "fedora": { + "python3-dev": "python3-devel", + "iptables": "iptables-services", + "haveged": "haveged", + "ntp": "chrony", + "ntpdate": "", + "redhat-rpm-config": "redhat-rpm-config", + "redhat-lsb-core": "redhat-lsb-core" + }, + "openeuler": { + "cron": "cronie", + "build-essential": "glibc-devel gcc make", + "dnsutils": "bind-utils", + "iproute2": "iproute", + "iputils-ping": "iputils", + "ntp": "ntp ntp-perl", + "python3-dev": "python3-devel", + "iptables": "iptables-services", + "haveged": "haveged" + }, + "gentoo": { + "acl": "sys-apps/acl", + "build-essential": "", + "cron": "sys-process/cronie", + "dnsutils": "net-dns/bind-tools", + "gentoolkit": "app-portage/gentoolkit", + "git": "dev-vcs/git", + "gnupg2": "app-crypt/gnupg", + "iptables": "net-firewall/iptables", + "iputils-ping": "net-misc/iputils", + "ntp": "net-misc/ntp", + "ntpdate": "net-misc/ntp", + "python3-dev": "", + "traceroute": "net-analyzer/traceroute", + "uuid-runtime": "" + } + }, + "family": { + "redhat": { + "cron": "cronie", + "build-essential": "glibc-devel gcc make", + "dnsutils": "bind-utils", + "iproute2": "iproute", + "iputils-ping": "iputils", + "ntp": "ntp ntp-perl", + "python3-dev": "", + "uuid-runtime": "", + "iptables": "iptables-services", + "haveged": "" + }, + "suse": { + "build-essential": "glibc-devel gcc make", + "dnsutils": "bind-utils", + "git": "git-core", + "gnupg2": "gpg2", + "iputils-ping": "iputils", + "ntpdate": "", + "python3-dev": "python3-devel", + "iptables": "iptables", + "uuid-runtime": "uuidd" + } + }, + "default": { + "iptables": "iptables-persistent", + "gentoolkit": "", + "redhat-rpm-config": "", + "redhat-lsb-core": "" + } +} diff --git a/dib-elements/infra-package-needs/post-install.d/80-enable-haveged b/dib-elements/infra-package-needs/post-install.d/80-enable-haveged new file mode 100755 index 0000000..b07a4d9 --- /dev/null +++ b/dib-elements/infra-package-needs/post-install.d/80-enable-haveged @@ -0,0 +1,32 @@ +#!/bin/bash + +if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +if [[ "$DISTRO_NAME" =~ ^(centos|rocky)$ && ${DIB_RELEASE} =~ '9' ]]; then + # 9/9-stream we are just using the default rngd + exit 0 +fi + +case "$DIB_INIT_SYSTEM" in + upstart) + # nothing to do + exit 0 + ;; + systemd) + systemctl enable haveged.service + ;; + openrc) + rc-update add haveged default + ;; + sysv) + exit 0 + ;; + *) + echo "Unsupported init system $DIB_INIT_SYSTEM" + exit 1 + ;; +esac diff --git a/dib-elements/infra-package-needs/post-install.d/80-enable-infra-services b/dib-elements/infra-package-needs/post-install.d/80-enable-infra-services new file mode 100755 index 0000000..cb8767e --- /dev/null +++ b/dib-elements/infra-package-needs/post-install.d/80-enable-infra-services @@ -0,0 +1,38 @@ +#!/bin/bash + +if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +case "$DIB_INIT_SYSTEM" in + upstart) + # nothing to do + exit 0 + ;; + systemd) + # stick with default systemd timesyncd on bookworm, focal and beyond + if [[ ":bookworm: :focal: :jammy: :noble:" =~ :${DIB_RELEASE}: ]]; then + exit 0 + elif [[ $DISTRO_NAME = "ubuntu" || $DISTRO_NAME = "debian" ]]; then + systemctl enable ntp.service + elif [[ ( $DISTRO_NAME == "centos" && $DIB_RELEASE > 7 ) || $DISTRO_NAME == "fedora" || $DISTRO_NAME == "rocky" ]]; then + systemctl enable chronyd + else + systemctl enable ntpd.service + fi + ;; + openrc) + rc-update add ntp-client default + rc-update add acpid default + ;; + sysv) + # ntp is enabled by default, nothing to do + exit 0 + ;; + *) + echo "Unsupported init system $DIB_INIT_SYSTEM" + exit 1 + ;; +esac diff --git a/dib-elements/infra-package-needs/post-install.d/89-sshd b/dib-elements/infra-package-needs/post-install.d/89-sshd new file mode 100755 index 0000000..0f1b374 --- /dev/null +++ b/dib-elements/infra-package-needs/post-install.d/89-sshd @@ -0,0 +1,46 @@ +#!/bin/bash +# Copyright (C) 2011-2013 OpenStack Foundation +# Copyright 2016 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +# NOTE(pabelanger): Glean configures access for root user, so allow us to +# properly login. +sed -i -e'/PermitRootLogin/d' /etc/ssh/sshd_config \ + && echo "PermitRootLogin yes" >> /etc/ssh/sshd_config + +# NOTE(clarkb): Glean configures ssh keys only and not passwords. Disable +# unnecessary password auth. +sed -i -e '/PasswordAuthentication/d' /etc/ssh/sshd_config \ + && echo "PasswordAuthentication no" >> /etc/ssh/sshd_config + +# NOTE(clarkb): SSH scanners may be affecting Zuul ssh connectivity +# Default LoginGraceTime is 120. Reduce that to 30 to cycle connections more +# quickly. +sed -i -e '/LoginGraceTime/d' /etc/ssh/sshd_config \ + && echo "LoginGraceTime 30" >> /etc/ssh/sshd_config + +# NOTE(clarkb): SSH scanners may be affecting Zuul ssh connectivity +# Default MaxStartups is 10:30:100 which means after 10 unauthenticated +# connections randomly drop 30% of connections with an increasing +# percentage until 100 connections is reached. +sed -i -e '/MaxStartups/d' /etc/ssh/sshd_config \ + && echo "MaxStartups 30:10:100" >> /etc/ssh/sshd_config diff --git a/dib-elements/infra-package-needs/pre-install.d/00-gentoo-useflags b/dib-elements/infra-package-needs/pre-install.d/00-gentoo-useflags new file mode 100755 index 0000000..b5c3bb7 --- /dev/null +++ b/dib-elements/infra-package-needs/pre-install.d/00-gentoo-useflags @@ -0,0 +1,18 @@ +#!/bin/bash + +if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +# needed for unbound +if [ "${DISTRO_NAME}" == "gentoo" ]; then + mkdir -p /etc/portage/package.use + echo "app-admin/rsyslog openssl" >> /etc/portage/package.use/rsyslog + echo "dev-lang/python sqlite" >> /etc/portage/package.use/python + echo "dev-libs/openssl static-libs -bindist" >> /etc/portage/package.use/openssl + echo "net-misc/openssh -bindist" >> /etc/portage/package.use/openssh + echo "sys-apps/systemd audit curl gcrypt importd lzma nat" >> /etc/portage/package.use/systemd + echo "sys-libs/zlib static-libs" >> /etc/portage/package.use/zlib +fi diff --git a/dib-elements/infra-package-needs/rsyslog.d/50-default.conf b/dib-elements/infra-package-needs/rsyslog.d/50-default.conf new file mode 100644 index 0000000..528af29 --- /dev/null +++ b/dib-elements/infra-package-needs/rsyslog.d/50-default.conf @@ -0,0 +1,68 @@ +# Default rules for rsyslog. +# +# For more information see rsyslog.conf(5) and /etc/rsyslog.conf + +# +# First some standard log files. Log by facility. +# +auth,authpriv.* /var/log/auth.log +*.*;auth,authpriv.none -/var/log/syslog +#cron.* /var/log/cron.log +#daemon.* -/var/log/daemon.log +kern.* -/var/log/kern.log +#lpr.* -/var/log/lpr.log +mail.* -/var/log/mail.log +#user.* -/var/log/user.log + +# +# Logging for the mail system. Split it up so that +# it is easy to write scripts to parse these files. +# +#mail.info -/var/log/mail.info +#mail.warn -/var/log/mail.warn +mail.err /var/log/mail.err + +# +# Logging for INN news system. +# +news.crit /var/log/news/news.crit +news.err /var/log/news/news.err +news.notice -/var/log/news/news.notice + +# +# Some "catch-all" log files. +# +#*.=debug;\ +# auth,authpriv.none;\ +# news.none;mail.none -/var/log/debug +#*.=info;*.=notice;*.=warn;\ +# auth,authpriv.none;\ +# cron,daemon.none;\ +# mail,news.none -/var/log/messages + +# +# Emergencies are sent to everybody logged in. +# +*.emerg :omusrmsg:* + +# +# I like to have messages displayed on the console, but only on a virtual +# console I usually leave idle. +# +#daemon,mail.*;\ +# news.=crit;news.=err;news.=notice;\ +# *.=debug;*.=info;\ +# *.=notice;*.=warn /dev/tty8 + +# The named pipe /dev/xconsole is for the `xconsole' utility. To use it, +# you must invoke `xconsole' with the `-file' option: +# +# $ xconsole -file /dev/xconsole [...] +# +# NOTE: adjust the list below, or you'll go crazy if you have a reasonably +# busy site.. +# +#daemon.*;mail.*;\ +# news.err;\ +# *.=debug;*.=info;\ +# *.=notice;*.=warn |/dev/xconsole diff --git a/dib-elements/initialize-urandom/README.rst b/dib-elements/initialize-urandom/README.rst new file mode 100644 index 0000000..c44a20b --- /dev/null +++ b/dib-elements/initialize-urandom/README.rst @@ -0,0 +1,2 @@ +This uses haveged to quickly initialize the nonblocking kernel random +number generator at boot. diff --git a/dib-elements/initialize-urandom/element-deps b/dib-elements/initialize-urandom/element-deps new file mode 100644 index 0000000..69a71fd --- /dev/null +++ b/dib-elements/initialize-urandom/element-deps @@ -0,0 +1,2 @@ +dib-init-system +install-static diff --git a/dib-elements/initialize-urandom/init-scripts/systemd/initialize-urandom.service b/dib-elements/initialize-urandom/init-scripts/systemd/initialize-urandom.service new file mode 100644 index 0000000..cc4489e --- /dev/null +++ b/dib-elements/initialize-urandom/init-scripts/systemd/initialize-urandom.service @@ -0,0 +1,13 @@ +[Unit] +Description=Quickly initialize the nonblocking kernel random number generator at boot. +Before=network-pre.target +Wants=network-pre.target + +[Service] +Type=oneshot +User=root +ExecStart=/usr/local/bin/initialize-urandom.py +RemainAfterExit=true + +[Install] +WantedBy=multi-user.target diff --git a/dib-elements/initialize-urandom/post-install.d/80-initialize-urandom b/dib-elements/initialize-urandom/post-install.d/80-initialize-urandom new file mode 100755 index 0000000..6174349 --- /dev/null +++ b/dib-elements/initialize-urandom/post-install.d/80-initialize-urandom @@ -0,0 +1,18 @@ +#!/bin/bash + +if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +case "$DIB_INIT_SYSTEM" in + systemd) + sudo chmod 0644 /usr/lib/systemd/system/initialize-urandom.service + systemctl enable initialize-urandom.service + ;; + *) + echo "Unsupported init system" + exit 1 + ;; +esac diff --git a/dib-elements/initialize-urandom/static/usr/local/bin/initialize-urandom.py b/dib-elements/initialize-urandom/static/usr/local/bin/initialize-urandom.py new file mode 100755 index 0000000..fa21529 --- /dev/null +++ b/dib-elements/initialize-urandom/static/usr/local/bin/initialize-urandom.py @@ -0,0 +1,273 @@ +#!/usr/bin/env python3 + +# Copyright 2016 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import ctypes +import errno +import fcntl +import os +import struct +import subprocess + +"""Add entropy to the kernel until the nonblocking pool is +initialized. + +The Linux kernel has 3 entropy pools: input, blocking, and +nonblocking. Normally entropy accumulates in the input pool and as it +is depleted by the other pools, it is transferred from the input pool +to the others. + +The blocking pool corresponds to /dev/random, where reads from that +device return random numbers only as long as there is sufficient +entropy in the blocking pool. When that entropy is depleted, further +reads from /dev/random block until it is replenished. + +The nonblocking pool corresponds to /dev/urandom, where reads never +block. Even if there is no entropy in the nonblocking pool, random +numbers are still returned. + +The algorithms in use in Linux 3.17 require 128 bits of entropy in +order to initialize the random number generators associated with each +pool. Naturally, reads from /dev/random will not return until the +associated generator is initialized. Reads from /dev/urandom will not +block -- even if the generator is not initialized. The kernel will +output a notice[1] if this happens. + +In order to avoid the situation where urandom is used when +uninitialized, the kernel diverts entropy from timers and interrupts +to the nonblocking pool (instead of the input pool) until it is +initialized. In this way, as the system boots, the nonblocking pool +accumulates entropy first, reducing the time period during which +urandom might produce numbers from an uninitialized generator, and +then the input and blocking pools are filled. + +Beginning with Linux 3.17, the getrandom(2) syscall was added[2] so +that user-space programs that generally would like to use /dev/urandom +can do so without opening a file descriptor and, more relevant here, +can ensure that they do so only after the generator is initialized +(which otherwise is not possible with the /dev/urandom interface). + +Unfortunately, programs which use this interface during early boot may +need to wait some time for the nonblocking pool to accumulate enough +entropy to initialize, and therefore for getrandom to return. +Particularly in the case of a VM, this may take considerable time. + +There are many methods of addressing this shortcoming: + +* Store data from /dev/random at shutdown and use it to seed the + entropy pool at the next boot. Most GNU/Linux distributions do + this. On Ubuntu Xenial, this task is performed by systemd[3]. + Unfortunately, while writes to /dev/random (which is the method + systemd uses to seed the system at boot) do add data to the pool, + they do not increase the internal tracking of the amount of entropy + in the pool. Therefore, for the purposes of determining whether the + nonblocking pool has accumulated 128 bits of entropy, they are not + counted. + +* Use haveged to maintain a sufficient amount of entropy. Haveged can + produce entropy very quickly, and when run at boot, will typically + immediately fill the entropy pool. Haveged performs an ioctl + operation on /dev/random rather than writing data to it, and this + ioctl allows it to specify how much entropy the data it supplies + contains. Therefore, unlike writes to /dev/random, ioctls do + increment the entropy counter. Unfortunately, data from ioctls are + *always* directed to the input pool. While entropy from timers and + interrupts are diverted to the nonblocking pool to speed its + initialization, data arriving from the ioctl instead end up in the + input pool for later use. + + When more entropy than is needed is supplied to the input pool, the + kernel will preemptively transfer some of that entropy to the + secondary (including nonblocking) pools. Since haveged supplies so + much data on startup, some of this entropy should be able to spill + over into the nonblocking pool to aid it in achieving the + initialization threshold. Unfortunately, at the stage of early boot + we are considering, the input pool's generator also has not been + initialized. When the kernel receives a large amount of data from + haveged over the ioctl, it pushes the input pool's generator over + the 128 bit threshold, and initializes the input pool's generator. + When a pool's generator is initialized, the entropy counter for that + pool is reset to zero. This leaves no entropy to spill over to the + nonblocking pool. Haveged is only able to see the entropy count for + the input pool, and therefore is unaware that further contributions + of entropy would aid (via spill-over) in seeding the nonblocking + pool. + + At this point it's worth discussing why the nonblocking pool is + still not initialized despite a full input pool. When a secondary + pool needs more entropy, it can pull from the input pool. However, + there is a timer that only allows the nonblocking pool to withdraw + entropy from the input pool every 60 seconds by default (this can be + adjusted via proc). If something during very early boot reads data + from /dev/urandom, a transfer (from the very likely empty) input + pool is initiated, starting the timer that will prevent another + transfer for 60 seconds, even if the input pool is later filled + (such as by haveged). This means that even with haveged running at + boot the delay due to a blocking getrandom(2) call may still be as + long as 60 seconds. + +* Use rng-tools for the same purpose as haveged. rng-tools operates + in a similar manner to haveged, supplying entropic data to the + kernel via ioctl. However, it does so in smaller chunks. This + means that once the input pool's generator surpasses the 128 bit + threshold for initialization, entropy from the next ioctl from + rng-tools will be available to spill over to the nonblocking pool, + and may be sufficient to initialize it. + + Because of this behavior, use of rng-tools may cause getrandom(2) to + return more quickly at boot, however, this may only happen due to a + quirk of implementation and relies on some specific values and + conditions for the amount of entropy in the input pool at the time + it is run. + +This program speeds initialization of the nonblocking pool by adding +entropy to the input pool in small chunks. To determine when the +nonblocking pool is initialized, it performs the nonblocking +getrandom(2) syscall requesting one byte of random data. As long as +the nonblocking pool is uninitialized, that call will fail and set +errno to EAGAIN. In that case, the program reads 64 bytes of data +from haveged and sends it to the kernel using the ioctl interface, +then repeats this in a loop. That will cause entropy to accumulate in +the input pool until it is initialized and reaches the spill-over +threshold. Further data will accumulate in the nonblocking pool until +it is initialized. Once that occurs, the getrandom(2) call will +return successfully, and the program will exit the loop. + +There are other ways this problem could be addressed (changes to +haveged or rng-tools to support behavior like this, or changes to the +kernel to direct entropy received via ioctl to the nonblocking pool +during initialization), however, this problem is likely to be +short-lived as the nonblocking generator is being replaced[4] in +current kernel versions and should not suffer from the same problem. + +[1] http://lxr.free-electrons.com/source/drivers/char/random.c?v=3.17#L1385 +[2] https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/commit/\ +?id=c6e9d6f38894798696f23c8084ca7edbf16ee895 +[3] https://www.freedesktop.org/software/systemd/man/systemd-random-seed.\ +service.html +[4] https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/commit/\ +?id=e192be9d9a30555aae2ca1dc3aad37cba484cd4a +""" + + +class GeneratorNotInitializedError(Exception): + pass + + +class InterruptedError(Exception): + pass + + +class Pump(object): + # How much data, in bytes, to move at once. 64 is the size of the + # internal kernel buffer, so we match it. + CHUNK_SIZE = 64 + + # The syscall number for getrandom(2). + SYS_getrandom = 318 + + # The IOCTL to add entropy. + OP_RNDADDENTROPY = 0x40085203 + + # Flags for getrandom: + GRND_NONBLOCK = 0x0001 # Do not block + GRND_RANDOM = 0x0002 # Use /dev/random instead of urandom + + def __init__(self): + # Use ctypes to invoke getrandom since it is not available in + # python. os.urandom may call getrandom in some versions of + # python3, however, the blocking on initialization behavior is + # seen as a bug and so os.urandom will never block, even if + # getrandom would. See http://bugs.python.org/issue26839 + self._getrandom = ctypes.CDLL(None, use_errno=True).syscall + self._getrandom.restype = ctypes.c_long + # The arguments are syscall number, void *buf, + # size_t buflen, unsigned int flags. + self._getrandom.argtypes = (ctypes.c_long, ctypes.c_void_p, + ctypes.c_size_t, ctypes.c_uint) + + def getrandom(self, length, random=False, nonblock=False): + flags = 0 + if random: + flags |= self.GRND_RANDOM + if nonblock: + flags |= self.GRND_NONBLOCK + buf = ctypes.ARRAY(ctypes.c_char, length)() + r = self._getrandom(self.SYS_getrandom, buf, len(buf), flags) + if r == -1: + err = ctypes.get_errno() + if err == errno.EINVAL: + raise Exception("getrandom: Invalid argument") + elif err == errno.EFAULT: + raise Exception("getrandom: Buffer is outside " + "accessible address space") + elif err == errno.EAGAIN: + raise GeneratorNotInitializedError() + elif err == errno.EINTR: + raise InterruptedError() + return buf[:r] + + def isInitialized(self): + # Read one byte from getrandom to determine whether the + # nonblocking pool is initialized. + try: + r = self.getrandom(1, nonblock=True) + if len(r) != 1: + raise Exception("No data returned from getrandom") + print("Nonblocking pool initialized") + return True + except GeneratorNotInitializedError: + return False + + def run(self): + """Move data from haveged to the kernel until the nonblocking pool is + initialized. + + """ + if self.isInitialized(): + return + + random_fd = os.open('/dev/random', os.O_RDWR) + # Start haveged and tell it to supply unlimited data on + # stdout, and print summary information. + p = subprocess.Popen(['/usr/sbin/haveged', '-f', '-', '-n', '0', + '-v', '1'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + while not self.isInitialized(): + # Read a chunk from haveged. + data = b'' + while len(data) < self.CHUNK_SIZE: + data += p.stdout.read(self.CHUNK_SIZE - len(data)) + # The data structure is: + # struct rand_pool_info { + # int entropy_count; + # int buf_size; + # __u32 buf[0]; + # }; + arg = struct.pack('iis', len(data) * 8, len(data), data) + print("Moving %s bytes" % len(data)) + fcntl.ioctl(random_fd, self.OP_RNDADDENTROPY, arg) + # Now that the generator is initialized, stop haveged and + # print the summary information. + p.send_signal(2) + p.stdout.read() + print(p.stderr.read().decode('utf-8')) + + +if __name__ == '__main__': + p = Pump() + p.run() diff --git a/dib-elements/nodepool-base/README.rst b/dib-elements/nodepool-base/README.rst new file mode 100644 index 0000000..e5052b9 --- /dev/null +++ b/dib-elements/nodepool-base/README.rst @@ -0,0 +1,24 @@ +============= +nodepool-base +============= + +Tasks to deal with image metadata and other Nodepool cloud specific tweaks. + +Name resolution +--------------- + +The image should have the unbound DNS resolver package installed, the +``nodepool-base`` element then configures it to forward DNS queries +to: + + ``NODEPOOL_STATIC_NAMESERVER_V4``, default: ``1.0.0.1`` + ``NODEPOOL_STATIC_NAMESERVER_V4_FALLBACK``, default: ``8.8.8.8``. + +If ``NODEPOOL_STATIC_NAMESERVER_POPULATE_IPV6`` is set to ``1`` then +the following two servers will be configured as forwarders too + + ``NODEPOOL_STATIC_NAMESERVER_V6``, default: ``2606:4700:4700::1111`` + ``NODEPOOL_STATIC_NAMESERVER_V6_FALLBACK``, default: ``2001:4860:4860::8888`` + +Note externally setting either of these values implies +``NODEPOOL_STATIC_NAMESERVER_POPULATE_IPV6=1`` diff --git a/dib-elements/nodepool-base/cleanup.d/50-root b/dib-elements/nodepool-base/cleanup.d/50-root new file mode 100755 index 0000000..79ea352 --- /dev/null +++ b/dib-elements/nodepool-base/cleanup.d/50-root @@ -0,0 +1,23 @@ +#!/bin/bash +# +# Copyright 2017 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +sudo rm -rf $TARGET_ROOT/root/.cache diff --git a/dib-elements/nodepool-base/element-deps b/dib-elements/nodepool-base/element-deps new file mode 100644 index 0000000..4bc83d4 --- /dev/null +++ b/dib-elements/nodepool-base/element-deps @@ -0,0 +1,3 @@ +ensure-venv +package-installs +zuul-worker diff --git a/dib-elements/nodepool-base/environment.d/75-nodepool-base-env b/dib-elements/nodepool-base/environment.d/75-nodepool-base-env new file mode 100644 index 0000000..4403426 --- /dev/null +++ b/dib-elements/nodepool-base/environment.d/75-nodepool-base-env @@ -0,0 +1,4 @@ +export NODEPOOL_STATIC_NAMESERVER_V6=${NODEPOOL_STATIC_NAMESERVER_V6:-2606:4700:4700::1111} +export NODEPOOL_STATIC_NAMESERVER_V4=${NODEPOOL_STATIC_NAMESERVER_V4:-1.0.0.1} +export NODEPOOL_STATIC_NAMESERVER_V6_FALLBACK=${NODEPOOL_STATIC_NAMESERVER_V6_FALLBACK:-2001:4860:4860::8888} +export NODEPOOL_STATIC_NAMESERVER_V4_FALLBACK=${NODEPOOL_STATIC_NAMESERVER_V4_FALLBACK:-8.8.8.8} diff --git a/dib-elements/nodepool-base/finalise.d/89-boot-settings b/dib-elements/nodepool-base/finalise.d/89-boot-settings new file mode 100755 index 0000000..0aa7469 --- /dev/null +++ b/dib-elements/nodepool-base/finalise.d/89-boot-settings @@ -0,0 +1,254 @@ +#!/bin/bash +# Copyright (C) 2011-2013 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +# dib-lint: disable=set setu setpipefail indent +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -e + +# +# Note that in OpenStack infra, the configure-unbound role [1] that is +# part of the base jobs will reconfigure unbound based on the host's +# ipv6 support very early in the job setup. Thus the following +# forwarder setup is only relevant to the initial boot and some parts +# of the integration-tests before configure-unbound role is used. +# +# [1] https://opendev.org/opendev/base-jobs/src/branch/master/roles/configure-unbound +# + +NODEPOOL_STATIC_NAMESERVER_V4=${NODEPOOL_STATIC_NAMESERVER_V4:-1.0.0.1} +NODEPOOL_STATIC_NAMESERVER_V4_FALLBACK=${NODEPOOL_STATIC_NAMESERVER_V4_FALLBACK:-8.8.8.8} + +# Explicitly setting a v6 nameserver implies you want ipv6 +if [[ -n ${NODEPOOL_STATIC_NAMESERVER_V6:-} || -n ${NODEPOOL_STATIC_NAMESERVER_V6_FALLBACK} ]]; then + NODEPOOL_STATIC_NAMESERVER_POPULATE_IPV6=1 +fi + +if [[ ${NODEPOOL_STATIC_NAMESERVER_POPULATE_IPV6:-0} == 1 ]]; then + NODEPOOL_STATIC_NAMESERVER_V6=${NODEPOOL_STATIC_NAMESERVER_V6:-2606:4700:4700::1111} + NODEPOOL_STATIC_NAMESERVER_V6_FALLBACK=${NODEPOOL_STATIC_NAMESERVER_V6_FALLBACK:-2001:4860:4860::8888} + + dd of=/tmp/forwarding.conf < /tmp/unbound-logging.conf + +if [[ "$DISTRO_NAME" =~ (centos|rhel7|fedora|opensuse|openeuler|rocky) ]] ; then + UNBOUND_CONFD=/etc/unbound/conf.d +elif [[ "$DISTRO_NAME" =~ 'gentoo' ]] ; then + UNBOUND_CONFD=/etc/unbound/conf.d + mkdir -p $UNBOUND_CONFD + echo "include: \"$UNBOUND_CONFD/*.conf\"" >> /etc/unbound/unbound.conf +else + UNBOUND_CONFD=/etc/unbound/unbound.conf.d +fi + +mv /tmp/unbound-logging.conf $UNBOUND_CONFD +chown root:root $UNBOUND_CONFD/unbound-logging.conf +chmod a+r $UNBOUND_CONFD/unbound-logging.conf + +touch /var/log/unbound.log +chown unbound /var/log/unbound.log +chmod 0644 /var/log/unbound.log +# NOTE(ianw) 2022-05-22 : not 100% sure why but emperically unbound +# runs under named context. Generally it doesn't log here on rhel-ish +# distros, but for consistency it's much easier if we just have one +# log file in the same place across distros. Thus set the context to +# allow it. +if [[ -e /usr/sbin/semanage ]]; then + semanage fcontext -a -t named_log_t /var/log/unbound.log + restorecon -v /var/log/unbound.log +fi + +if [[ "$DISTRO_NAME" =~ (opensuse) ]] ; then + # NOTE(ianw) 2021-03-17 : something about building the suse chroot + # has changed and /etc/init.d/ is no longer available. However, + # it rc-local.service (part of systemd) still runs. We should + # probably convert this all to a systemd job; but for now... + mkdir -p /etc/init.d + rclocal=/etc/init.d/boot.local +elif [[ "${DISTRO_NAME}" =~ "gentoo" ]]; then + rclocal=/etc/local.d/unbound.start + mkdir -p /etc/local.d +else + # You'd think rc.local would be simple ... + # + # On Redhat systems, systemd's rc-local service looks for an + # executable /etc/rc.d/rc.local file to run. On Debian/Ubuntu, the + # eqivalent file is /etc/rc.local, which is missing on Debian stretch. + # + # Centos' systemd package symlinks /etc/rc.local to /etc/rc.d/rc.local + # correctly. Fedora, however, does not come with an rc.local file at + # all. Thus if we have a rc.d directory, but no rc.local file, we + # need to create it (if you don't have an rc.d directory, and don't + # have /etc/rc.local, then it's not clear what platform you are on). + # + # Bug [1] is filed to bring Fedora in-line with Centos, and has more + # details on all this. As at 2016-10-18 is unresolved. + # + # [1] https://bugzilla.redhat.com/show_bug.cgi?id=1386052 + if [[ ! -e /etc/rc.local ]] && [[ ! -e /etc/debian_version ]]; then + if [[ ! -d /etc/rc.d ]]; then + echo "No rc.local and no rc.d directory! See comments in 89-boot-settings" + exit 1 + fi + touch /etc/rc.d/rc.local + ln -sf /etc/rc.d/rc.local /etc/rc.local + # permissions added below. selinux context will be fixed up at + # end of build. + fi + rclocal=/etc/rc.local +fi + +# Overwrite /etc/resolv.conf at boot (let's hope nothing else is using +# rc.local...) +cat >$rclocal < /etc/resolv.conf + +EOF + +# Configure NetworkManager to not manage /etc/resolv.conf +if [ -d /etc/NetworkManager/conf.d ]; then + cat <>$rclocal <>$rclocal <> /etc/unbound/unbound.conf + +# Disable dlv. Per the unbound.conf manpage this should not be used +# anymore but is in use by some of our distros. The problem here is it +# does lookasides for DNSSEC which increases the number of queries and +# introduces more points of lookup failure. Disable it to avoid these +# problems. +sed -i -e 's/dlv-anchor-file:/#dlv-anchor-file:/g' /etc/unbound/unbound.conf + +# Tripleo uses dhcp +dhcp_file='' +if [[ "$DISTRO_NAME" =~ (centos|rhel7|fedora|openeuler|rocky) ]] ; then + dhcp_file=/etc/dhcp/dhclient-eth0.conf +elif [[ "$DISTRO_NAME" =~ (debian|ubuntu|gentoo) ]] ; then + dhcp_file=/etc/dhcp/dhclient.conf +fi +if [ -n "$dhcp_file" ] ; then + cat > $dhcp_file <"; +request subnet-mask, broadcast-address, routers, + interface-mtu, rfc3442-classless-static-routes; +supersede domain-name-servers 127.0.0.1; +supersede domain-search ""; +supersede domain-name ""; +EOF +fi + +# On bionic and later, the install of the new systemd-resolved in the +# chroot will see that there is no /etc/resolv.conf and assume it is a +# blank system where it will be the nameserver provider. It thus +# creates /etc/resolv.conf as a link back to its compatability files. +# To configure systemd-resolved's resolvers you need to modify +# /etc/systemd/resolved.conf; which would be possible, but we'd prefer +# to be consistent across all our platforms. +# +# dib will copy whatever is in /etc/resolv.conf.ORIG to +# /etc/resolv.conf as one of the final steps in image creation. Thus +# we are hard-coding resolution to localhost (unbound) here. +# +# Note that with /etc/resolv.conf as a regular file, systemd-resolved +# will also obey it for nameserver info when it starts at boot. +rm -f /etc/resolv.conf.ORIG +echo "nameserver 127.0.0.1" > /etc/resolv.conf.ORIG + +case "$DIB_INIT_SYSTEM" in + upstart) + # nothing to do + ;; + systemd) + systemctl enable unbound.service + ;; + openrc) + rc-update add unbound default + ;; + sysv) + # nothing to do + ;; + *) + echo "Unsupported init system $DIB_INIT_SYSTEM" + exit 1 + ;; +esac diff --git a/dib-elements/nodepool-base/finalise.d/89-glean b/dib-elements/nodepool-base/finalise.d/89-glean new file mode 100755 index 0000000..312024c --- /dev/null +++ b/dib-elements/nodepool-base/finalise.d/89-glean @@ -0,0 +1,34 @@ +#!/bin/bash +# Copyright (C) 2011-2013 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +# dib-lint: disable=set setu setpipefail indent +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -e + +case "$DIB_INIT_SYSTEM" in + systemd) + glean_path_dib="/etc/systemd/system/glean@.service.d" + mkdir -p $glean_path_dib + nodepool_base="$(dirname $0)/../glean@.service.d" + cp -RP $nodepool_base/override.conf $glean_path_dib/override.conf + ;; + *) + echo "Skipping glean systemd configuration" + ;; +esac diff --git a/dib-elements/nodepool-base/finalise.d/89-journald-persistent b/dib-elements/nodepool-base/finalise.d/89-journald-persistent new file mode 100755 index 0000000..a2af142 --- /dev/null +++ b/dib-elements/nodepool-base/finalise.d/89-journald-persistent @@ -0,0 +1,31 @@ +#!/bin/bash +# Copyright (C) 2017 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +# dib-lint: disable=setu setpipefail +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -e + +# Journald default is to not persist logs to disk if /var/log/journal is +# not present. Update the configuration to set storage to persistent which +# will create /var/log/journal if necessary and store logs on disk. This +# avoids the situation where test runs can fill the journald ring buffer +# deleting older logs that may be important to the job. +if [ -f /etc/systemd/journald.conf ] ; then + sed -i -e 's/#Storage=auto/Storage=persistent/' /etc/systemd/journald.conf +fi diff --git a/dib-elements/nodepool-base/finalise.d/99-nodepool-dir b/dib-elements/nodepool-base/finalise.d/99-nodepool-dir new file mode 100755 index 0000000..f5567d0 --- /dev/null +++ b/dib-elements/nodepool-base/finalise.d/99-nodepool-dir @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright (C) 2011-2013 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +# dib-lint: disable=setu setpipefail +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -e + +# Nodepool expects this dir to exist when it boots slaves. +# Nodepool writes environment info to this dir. We set the mode +# to 0777 so that any user can access this env info. +mkdir /etc/nodepool +chmod 0777 /etc/nodepool diff --git a/dib-elements/nodepool-base/glean@.service.d/override.conf b/dib-elements/nodepool-base/glean@.service.d/override.conf new file mode 100644 index 0000000..202216c --- /dev/null +++ b/dib-elements/nodepool-base/glean@.service.d/override.conf @@ -0,0 +1,2 @@ +[Service] +Environment="ARGS=--interface %I --skip-dns" diff --git a/dib-elements/nodepool-base/install.d/05-record-details b/dib-elements/nodepool-base/install.d/05-record-details new file mode 100755 index 0000000..65aa3e5 --- /dev/null +++ b/dib-elements/nodepool-base/install.d/05-record-details @@ -0,0 +1,24 @@ +#!/bin/bash +# Copyright (C) 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +# dib-lint: disable=setu setpipefail +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -e + +echo $DIB_IMAGE_NAME > /etc/image-hostname.txt diff --git a/dib-elements/nodepool-base/install.d/06-record-builddate b/dib-elements/nodepool-base/install.d/06-record-builddate new file mode 100755 index 0000000..9db66e7 --- /dev/null +++ b/dib-elements/nodepool-base/install.d/06-record-builddate @@ -0,0 +1,12 @@ +#!/bin/bash + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +# Put a timestamp in the image file of the date the image was built. +# This is echoed into the logs on each run for easy cross-reference + +date --utc "+%Y-%m-%d %H:%M" > /etc/dib-builddate.txt diff --git a/dib-elements/nodepool-base/install.d/20-iptables b/dib-elements/nodepool-base/install.d/20-iptables new file mode 100755 index 0000000..ff60961 --- /dev/null +++ b/dib-elements/nodepool-base/install.d/20-iptables @@ -0,0 +1,92 @@ +#!/bin/bash +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +# dib-lint: disable=setu setpipefail +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -e + +if [[ "$DISTRO_NAME" =~ (debian|ubuntu) ]] ; then + rules_dir=/etc/iptables + ipv4_rules=${rules_dir}/rules.v4 + ipv6_rules=${rules_dir}/rules.v6 +elif [[ "$DISTRO_NAME" =~ (centos|rhel7|fedora|openeuler|rocky) ]] ; then + rules_dir=/etc/sysconfig + ipv4_rules=${rules_dir}/iptables + ipv6_rules=${rules_dir}/ip6tables +elif [[ "$DISTRO_NAME" =~ 'opensuse' ]] ; then + rules_dir=/etc/sysconfig + ipv4_rules=${rules_dir}/iptables + ipv6_rules=${rules_dir}/ip6tables +elif [[ "$DISTRO_NAME" =~ 'gentoo' ]] ; then + rules_dir=/var/lib/iptables # not needed, part of the package install + ipv4_rules=/var/lib/iptables/rules-save + ipv6_rules=/var/lib/ip6tables/rules-save +else + echo "Unsupported operating system $DISTRO_NAME" + exit 1 +fi + +mkdir -p $rules_dir + +cat > $ipv4_rules << EOF +*filter +:INPUT ACCEPT [0:0] +:FORWARD ACCEPT [0:0] +:OUTPUT ACCEPT [0:0] +:openstack-INPUT - [0:0] +-A INPUT -j openstack-INPUT +-A openstack-INPUT -i lo -j ACCEPT +-A openstack-INPUT -p icmp --icmp-type any -j ACCEPT +#-A openstack-INPUT -p udp --dport 5353 -d 224.0.0.251 -j ACCEPT +# SSH from anywhere without -m state to avoid hanging connections on iptables-restore +-A openstack-INPUT -m tcp -p tcp --dport 22 -j ACCEPT +-A openstack-INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT +# Public TCP ports +-A openstack-INPUT -p tcp -m state --state NEW -m tcp --dport 19885 -j ACCEPT +# Ports 69 and 6385 allow to allow ironic VM nodes to reach tftp and +# the ironic API from the neutron public net +-A openstack-INPUT -s 172.24.4.0/23 -p udp -m udp --dport 69 -j ACCEPT +-A openstack-INPUT -s 172.24.4.0/23 -p tcp -m tcp --dport 6385 -j ACCEPT +# Ports 80, 8000, 8003, 8004 from the devstack neutron public net to allow +# nova servers to reach heat-api-cfn, heat-api-cloudwatch, heat-api +-A openstack-INPUT -s 172.24.4.0/23 -p tcp -m tcp --dport 80 -j ACCEPT +-A openstack-INPUT -s 172.24.4.0/23 -p tcp -m tcp --dport 8000 -j ACCEPT +-A openstack-INPUT -s 172.24.4.0/23 -p tcp -m tcp --dport 8003 -j ACCEPT +-A openstack-INPUT -s 172.24.4.0/23 -p tcp -m tcp --dport 8004 -j ACCEPT +-A openstack-INPUT -m limit --limit 2/min -j LOG --log-prefix "iptables dropped: " +-A openstack-INPUT -j REJECT --reject-with icmp-host-prohibited +COMMIT +EOF + +cat > $ipv6_rules << EOF +*filter +:INPUT ACCEPT [0:0] +:FORWARD ACCEPT [0:0] +:OUTPUT ACCEPT [0:0] +:openstack-INPUT - [0:0] +-A INPUT -j openstack-INPUT +-A openstack-INPUT -i lo -j ACCEPT +-A openstack-INPUT -p ipv6-icmp -j ACCEPT +# SSH from anywhere without -m state to avoid hanging connections on iptables-restore +-A openstack-INPUT -m tcp -p tcp --dport 22 -j ACCEPT +-A openstack-INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT +# Public TCP ports +-A openstack-INPUT -p tcp -m state --state NEW -m tcp --dport 19885 -j ACCEPT +-A openstack-INPUT -j REJECT --reject-with icmp6-adm-prohibited +COMMIT +EOF diff --git a/dib-elements/nodepool-base/install.d/50-disable-metadata-cloudinit b/dib-elements/nodepool-base/install.d/50-disable-metadata-cloudinit new file mode 100755 index 0000000..376f2f4 --- /dev/null +++ b/dib-elements/nodepool-base/install.d/50-disable-metadata-cloudinit @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright (C) 2015 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. + +# dib-lint: disable=setu setpipefail +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -e + +# Make all cloud-init data sources match rackspace- only attempt to look +# at ConfigDrive, not at metadata service. This is not needed if there +# is no cloud-init +if [ -d /etc/cloud/cloud.cfg.d ] ; then + dd of=/etc/cloud/cloud.cfg.d/95_real_datasources.cfg < /etc/sysctl.d/99-cloudimg-ipv6.conf < $PRESEED <> $TMP_HOOKS_PATH/zuul-user-ssh-public-key +elif [ -n "$ZUUL_USER_SSH_PUBLIC_KEY" ]; then + # save the specific public key inside the chroot from env derectly + echo "$ZUUL_USER_SSH_PUBLIC_KEY" >> $TMP_HOOKS_PATH/zuul-user-ssh-public-key +else + die "Can not find public key for zuul user!" +fi diff --git a/dib-elements/zuul-worker/install.d/60-zuul-worker b/dib-elements/zuul-worker/install.d/60-zuul-worker new file mode 100755 index 0000000..7138bc1 --- /dev/null +++ b/dib-elements/zuul-worker/install.d/60-zuul-worker @@ -0,0 +1,60 @@ +#!/bin/bash + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +# Add zuul user and group. Note we don't want to rely on +# "useradd"'s group adding behaviour, because it might differ across +# distros. +groupadd zuul +useradd -m zuul -g zuul -s /bin/bash + +cat > /etc/sudoers.d/zuul << EOF +zuul ALL=(ALL) NOPASSWD:ALL +EOF +chmod 0440 /etc/sudoers.d/zuul + +visudo -c || die "Error setting zuul sudo!" + +# this was copied from outside the chroot by extras.d +_pub_key=/tmp/in_target.d/zuul-user-ssh-public-key +if [ ! -f $_pub_key ]; then + die "Can not find Zuul public key!" +fi + +mkdir -p /home/zuul/.ssh +chmod 700 /home/zuul/.ssh +cp $_pub_key /home/zuul/.ssh/authorized_keys + +# cleanup everything to the right owner +chown -R zuul:zuul /home/zuul + +# a lot of caching happens in extra-data.d (for "historical" reasons). +# We've put the cache stuff into /opt/cache/files, but again, for +# "historical" reasons, ensure this is available in /home/zuul +# +# We do this for zuul as relative paths to the current user's homedir +# are used in places like devstack. +# +# Check if the cache exists as we don't have a strict dependency on the +# devstack-cache element. This allows you to build an image without +# incurring the cost of caching all the things. +if [ -d /opt/cache/files ] ; then + mkdir -p /home/zuul/cache + chown zuul:zuul /home/zuul/cache + ln -sf /opt/cache/files /home/zuul/cache/files + # but make sure the cache is readable by everyone + chmod -R a+rX /opt/cache/files/* +fi + +# New versions of git don't let you clone repos as a different user +# than the user owning the repo by default for security reasons. +# As above we cache git repos during extra-data.d in /opt/git/ and they +# end up owned by root. Chown them to zuul here to avoid permissions +# issues with the most likely user to interact with the git cache( zuul). +if [ -d /opt/git ] ; then + chown -R zuul:zuul /opt/git +fi diff --git a/playbooks/opendev-build-diskimage-base/post-inner.yaml b/playbooks/opendev-build-diskimage-base/post-inner.yaml new file mode 100644 index 0000000..52a4929 --- /dev/null +++ b/playbooks/opendev-build-diskimage-base/post-inner.yaml @@ -0,0 +1,27 @@ +- name: Compress image + when: "upload_image_format in ['raw', 'vhd']" + command: zstd '{{ build_diskimage_image_root }}/{{ build_diskimage_image_name }}.{{ upload_image_format }}' +- name: Set extension + when: "upload_image_format in ['raw', 'vhd']" + set_fact: + upload_image_extension: '{{ upload_image_format }}.zst' +- name: Set extension + when: "upload_image_format not in ['raw', 'vhd']" + set_fact: + upload_image_extension: '{{ upload_image_format }}' +- name: Upload image + no_log: true + include_role: + name: image-upload-swift + vars: + cloud: + auth_type: 'v3applicationcredential' + auth: + auth_url: 'https://keystone.api.sjc3.rackspacecloud.com/v3' + application_credential_id: '{{ image_upload_secret.application_credential_id }}' + application_credential_secret: '{{ image_upload_secret.application_credential_secret }}' + user_domain_name: rackspace_cloud_domain + container: images-1f49951f5beb + filename: '{{ build_diskimage_image_root }}/{{ build_diskimage_image_name }}.{{ upload_image_extension }}' + name: '{{ zuul.build }}-{{ build_diskimage_image_name }}.{{ upload_image_extension }}' + delete_after: 259200 diff --git a/playbooks/opendev-build-diskimage-base/post.yaml b/playbooks/opendev-build-diskimage-base/post.yaml new file mode 100644 index 0000000..3c78c2c --- /dev/null +++ b/playbooks/opendev-build-diskimage-base/post.yaml @@ -0,0 +1,8 @@ +- hosts: all + tasks: + - name: Upload image + when: image_upload_secret is defined and zuul_success + with_items: '{{ build_diskimage_formats }}' + loop_control: + loop_var: upload_image_format + include_tasks: post-inner.yaml diff --git a/playbooks/opendev-build-diskimage-base/pre.yaml b/playbooks/opendev-build-diskimage-base/pre.yaml new file mode 100644 index 0000000..41e32f8 --- /dev/null +++ b/playbooks/opendev-build-diskimage-base/pre.yaml @@ -0,0 +1,38 @@ +- hosts: all + pre_tasks: + - name: Install custom element requirements + become: true + package: + state: present + name: + - python3-yaml + # The following are required for the image upload + - python3-openstacksdk + - python3-oslo.utils + - zstd + - name: Setup dib directories + become: true + shell: | + set -x + + mkdir -p /opt/dib_cache + chown {{ ansible_user }} /opt/dib_cache + chgrp {{ ansible_user }} /opt/dib_cache + + mkdir -p /opt/dib_tmp + + if [ -b /dev/vdb ]; then + DEV='/dev/vdb' + elif [ -b /dev/xvde ]; then + DEV='/dev/xvde' + fi + + if [ -n "$DEV" ]; then + mkfs.ext4 $DEV + mount $DEV /opt/dib_tmp + fi + + chown {{ ansible_user }} /opt/dib_tmp + chgrp {{ ansible_user }} /opt/dib_tmp + roles: + - make-source-repositories-cache diff --git a/roles/image-upload-swift/library/__init__.py b/roles/image-upload-swift/library/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/roles/image-upload-swift/library/image_upload_swift.py b/roles/image-upload-swift/library/image_upload_swift.py new file mode 100644 index 0000000..44081b0 --- /dev/null +++ b/roles/image-upload-swift/library/image_upload_swift.py @@ -0,0 +1,168 @@ +# Copyright 2014 Rackspace Australia +# Copyright 2018 Red Hat, Inc +# Copyright 2024 Acme Gating, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import argparse +import concurrent.futures +import datetime +import logging +import os +import sys +import traceback + +import openstack +import requests.exceptions +import keystoneauth1.exceptions + +from ansible.module_utils.basic import AnsibleModule + +SEGMENT_SIZE = 500000000 # 500MB + + +def get_cloud(cloud): + if isinstance(cloud, dict): + config = openstack.config.loader.OpenStackConfig().get_one(**cloud) + return openstack.connection.Connection( + config=config, + pool_executor=concurrent.futures.ThreadPoolExecutor( + max_workers=10 + )) + else: + return openstack.connect(cloud=cloud) + + +def _add_etag_to_manifest(self, *args, **kw): + return + + +def prune(cloud, container, delete_after): + # In case the automatic expiration doesn't work, manually prune old uploads + if not delete_after: + return + target = (datetime.datetime.now(datetime.UTC) - + datetime.timedelta(seconds=delete_after)) + endpoint = cloud.object_store.get_endpoint() + url = os.path.join(endpoint, container) + for obj in cloud.object_store.objects(container): + ts = datetime.datetime.fromisoformat(obj['last_modified']) + ts = ts.replace(tzinfo=datetime.UTC) + if ts < target: + path = os.path.join(url, obj.name) + try: + cloud.session.delete(path) + except keystoneauth1.exceptions.http.NotFound: + pass + + +def run(cloud, container, filename, name, delete_after=None): + # Monkey-patch sdk so that the SLO upload does not add the etag; + # this works around an issue with rackspace-flex. + cloud.object_store._add_etag_to_manifest = _add_etag_to_manifest + prune(cloud, container, delete_after) + headers = {} + if delete_after: + headers['X-Delete-After'] = str(delete_after) + endpoint = cloud.object_store.get_endpoint() + cloud.object_store.create_object( + container, + name=name, + filename=filename, + segment_size=SEGMENT_SIZE, + **headers) + url = os.path.join(endpoint, container, name) + return url + + +def ansible_main(): + module = AnsibleModule( + argument_spec=dict( + cloud=dict(required=True, type='raw'), + container=dict(required=True, type='str'), + filename=dict(required=True, type='path'), + name=dict(required=True, type='str'), + delete_after=dict(type='int'), + ) + ) + + p = module.params + cloud = get_cloud(p.get('cloud')) + try: + url = run( + cloud, + p.get('container'), + p.get('filename'), + p.get('name'), + delete_after=p.get('delete_after'), + ) + except (keystoneauth1.exceptions.http.HttpError, + requests.exceptions.RequestException): + s = "Error uploading to %s.%s" % (cloud.name, cloud.config.region_name) + s += "\n" + traceback.format_exc() + module.fail_json( + changed=False, + msg=s, + cloud=cloud.name, + region_name=cloud.config.region_name) + module.exit_json( + changed=True, + url=url, + ) + + +def cli_main(): + parser = argparse.ArgumentParser( + description="Upload image to swift" + ) + parser.add_argument('--verbose', action='store_true', + help='show debug information') + parser.add_argument('cloud', + help='Name of the cloud to use when uploading') + parser.add_argument('container', + help='Name of the container to use when uploading') + parser.add_argument('filename', + help='the file to upload') + parser.add_argument('name', + help='the object name') + parser.add_argument('--delete-after', + help='Number of seconds to delete object after ' + 'upload. Default is 3 days (259200 seconds) ' + 'and if set to 0 X-Delete-After will not be set', + type=int) + + args = parser.parse_args() + + if args.verbose: + logging.basicConfig(level=logging.DEBUG) + # Set requests log level accordingly + logging.getLogger("requests").setLevel(logging.DEBUG) + logging.getLogger("keystoneauth").setLevel(logging.INFO) + logging.getLogger("stevedore").setLevel(logging.INFO) + logging.captureWarnings(True) + + url = run( + get_cloud(args.cloud), + args.container, + args.filename, + args.name, + delete_after=args.delete_after, + ) + print(url) + + +if __name__ == '__main__': + if not sys.stdin.isatty(): + ansible_main() + else: + cli_main() diff --git a/roles/image-upload-swift/tasks/main.yaml b/roles/image-upload-swift/tasks/main.yaml new file mode 100644 index 0000000..c7fe4e4 --- /dev/null +++ b/roles/image-upload-swift/tasks/main.yaml @@ -0,0 +1,56 @@ +# Run the checksums in the background while we're uploading +- name: Get sha256 hash + stat: + path: '{{ filename }}' + checksum_algorithm: sha256 + async: 600 + poll: 0 + register: sha256_task + +- name: Get md5 hash + stat: + path: '{{ filename }}' + checksum_algorithm: md5 + async: 600 + poll: 0 + register: md5_task + +- name: Upload image to swift + no_log: true + image_upload_swift: + cloud: '{{ cloud }}' + container: '{{ container }}' + filename: '{{ filename }}' + name: '{{ name }}' + delete_after: '{{ delete_after }}' + register: upload_results + +- name: Wait for sha256 + async_status: + jid: "{{ sha256_task.ansible_job_id }}" + register: sha256 + until: sha256.finished + retries: 1 + delay: 10 + +- name: Wait for md5 + async_status: + jid: "{{ md5_task.ansible_job_id }}" + register: md5 + until: md5.finished + retries: 1 + delay: 10 + +- name: Return artifact to Zuul + zuul_return: + data: + zuul: + artifacts: + - name: '{{ upload_image_format }} image' + url: '{{ upload_results.url }}' + metadata: + type: 'zuul_image' + image_name: '{{ build_diskimage_image_name }}' + format: '{{ upload_image_format }}' + sha256: '{{ sha256.stat.checksum }}' + md5sum: '{{ md5.stat.checksum }}' diff --git a/roles/make-source-repositories-cache/README.rst b/roles/make-source-repositories-cache/README.rst new file mode 100644 index 0000000..73e3378 --- /dev/null +++ b/roles/make-source-repositories-cache/README.rst @@ -0,0 +1,12 @@ +Reformat the git repo cache for source-respositories + +This reverses the process of creating the /opt/git cache on worker +nodes. It turns that cache into the format used by the +source-repositories diskimage builder element. + +**Role Variables** + +.. zuul:rolevar:: cached_repos_root + :default: /opt/git + + The root of the cached repos. diff --git a/roles/make-source-repositories-cache/defaults/main.yaml b/roles/make-source-repositories-cache/defaults/main.yaml new file mode 100644 index 0000000..4c8a06d --- /dev/null +++ b/roles/make-source-repositories-cache/defaults/main.yaml @@ -0,0 +1,2 @@ +# Matches prepare_workspace_root +cached_repos_root: /opt/git diff --git a/roles/make-source-repositories-cache/tasks/main.yaml b/roles/make-source-repositories-cache/tasks/main.yaml new file mode 100644 index 0000000..4bf265f --- /dev/null +++ b/roles/make-source-repositories-cache/tasks/main.yaml @@ -0,0 +1,27 @@ +- name: Reformat the git repo cache for source-repositories + environment: "{{ build_diskimage_environment }}" + args: + executable: /bin/bash + shell: | + GIT_CACHE_ROOT="{{ cached_repos_root }}" + + REPOTYPE=git + CACHE_BASE=${DIB_IMAGE_CACHE}/source-repositories + mkdir -p ${CACHE_BASE} + + # Note: maxdepth 4 is based on the current opendev practice of + # not having repos deeper than two directory levels. This is + # fragile and should probably be replaced with something better, + # but as written it saves considerable time. + for NODE_CACHE_PATH in $(find $GIT_CACHE_ROOT -maxdepth 4 -type d -name .git); do + REPOPATH=$(dirname $NODE_CACHE_PATH) + REPONAME=$(basename $REPOPATH) + REPOLOCATION="$(echo $REPOPATH | sed "s,$GIT_CACHE_ROOT/,https://,").git" + + # From diskimage-builder element source-repositories + CACHE_NAME=$(echo "${REPOTYPE}_${REPOLOCATION}" | sha1sum | awk '{ print $1 }' ) + CACHE_NAME=${REPONAME//[^A-Za-z0-9]/_}_${CACHE_NAME} + CACHE_PATH=${CACHE_BASE}/${CACHE_NAME} + + mv ${REPOPATH} ${CACHE_PATH} + done diff --git a/zuul.d/image-build-jobs.yaml b/zuul.d/image-build-jobs.yaml new file mode 100644 index 0000000..d29a67c --- /dev/null +++ b/zuul.d/image-build-jobs.yaml @@ -0,0 +1,98 @@ +- job: + name: opendev-build-diskimage-base + description: | + The abstract base job for building VM images used in OpenDev's Zuul + + If the variable ``image_upload_secret`` is provided (as a + secret) this job will upload the resulting image (this should be + done in image build pipelines). Otherwise, it will only build + the image without uploading it (this can be useful for check + pipelines). + parent: build-diskimage + timeout: 7200 + post-timeout: 7200 + abstract: true + pre-run: + - playbooks/opendev-build-diskimage-base/pre.yaml + post-run: + - playbooks/opendev-build-diskimage-base/post.yaml + files: + - 'dib-elements/.*' + - 'playbooks/opendev-build-diskimage-base/.*' + - 'roles/make-source-repositories-cache/.*' + vars: + # TODO(corvus): This will eventually be supplied by Zuul + build_diskimage_formats: + - qcow2 + - raw + # /opt/dib_tmp is an ephemeral disk if present + build_diskimage_image_root: "/opt/dib_tmp/dib-images" + build_diskimage_elements: &base_elements + - vm + - simple-init + - openstack-repos + - nodepool-base + - growroot + - infra-package-needs + build_diskimage_environment: + TMPDIR: /opt/dib_tmp + ELEMENTS_PATH: "{{ ansible_user_dir }}/{{ zuul.projects['opendev.org/opendev/zuul-providers'].src_dir }}/dib-elements" + # TODO: Do we want streaming logs? + DIB_QUIET: '0' + DIB_NO_TMPFS: '1' + DIB_CHECKSUM: '1' + DIB_IMAGE_CACHE: /opt/dib_cache + DIB_JOURNAL_SIZE: '512' + DIB_GRUB_TIMEOUT: '0' + GIT_HTTP_LOW_SPEED_TIME: '300' + GIT_HTTP_LOW_SPEED_LIMIT: '1000' + DIB_SHOW_IMAGE_USAGE: '1' + ZUUL_USER_SSH_PUBLIC_KEY: "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDh5u0DWNi0d3uqI82izAxVTUTrGl36L3szEqV9WrilGmxaTtp9X7HrktJ5J+rvxQqz39llTf1v7iYA4CNKto/48RBAB0mKEEI4x4iw+fu/BLU7nu7ewSfXLUxHahxaTgIk2KcbegknD5NzMjalyfNfgTPDGv8BjwHeeNdZmJMBvPFGu6fO48M4yK1tiQn0kAkaH4oII/M4pyF8vy9tPTODAD7RvnMvQAb08LZZvE/IPzJAHNXFRb1v+DBa38fOvdyaz/nibrsxiOWZxQVLgjYciUeDy1xvXADaWlqvxmLy+90LHbJFbGxK4AN0mWfwBiUMVyxZjkun39pjTNl2k09OhOq+R52UqnehMc4eBdZCddnCUq4/efbFCJkqe5wY+SE8fYybJjauUL64zyrwf6yfWkXvPVHWa9Y+NCmvH8PCBUcsQnwO7l/Yb4N+8+u6zkODyuc9wLAY+DpnptE3plXtvUs5negC4fvJSnOHpWXuoi9yzp7IlPf6fSjMMDQo0JjCYJwazdzqrIH2VSCcfHAqWF0ECR8IgwZV1bp0xFe0UN0Gjsgkozqf8rvs1AYyTSeD19Wg9j+crTke8E1sfoI/qFzHwzBQFKJ+2l0cs7pZWJBARlhbt1j1IouS2aH+74xwsavRhBz4IsFTPqWiP6JTrgk5cgKRnTqInzNfdaLeUw== zuul-worker@openstack.org" + +- job: + name: opendev-build-diskimage-base-debuntu + description: | + An abstract base job for building Debian or Ubuntu based VM images + parent: opendev-build-diskimage-base + abstract: true + vars: + build_diskimage_environment: + DIB_APT_LOCAL_CACHE: '0' + DIB_DISABLE_APT_CLEANUP: '1' + DIB_DEBOOTSTRAP_EXTRA_ARGS: '--no-check-gpg' + +- job: + name: opendev-build-diskimage-debian-bullseye + description: | + Build the OpenDev Debian bullseye VM image + parent: opendev-build-diskimage-base-debuntu + image-build-name: debian-bullseye + vars: + build_diskimage_image_name: debian-bullseye + build_diskimage_elements: + - *base_elements + - debian-minimal + - cache-devstack + build_diskimage_environment: + DIB_RELEASE: 'bullseye' + DIB_DISTRIBUTION_MIRROR: 'https://{{ zuul_site_mirror_fqdn }}/debian' + DIB_DEBIAN_SECURITY_MIRROR: + 'https://{{ zuul_site_mirror_fqdn }}/debian-security' + DIB_BLOCK_DEVICE_CONFIG: | + # Default single partition loopback + - local_loop: + name: image0 + + - partitioning: + base: image0 + label: mbr + partitions: + - name: root + flags: [ boot, primary ] + size: 100% + mkfs: + mount: + mount_point: / + fstab: + options: "defaults,nobarrier,noatime" + fsck-passno: 1 diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index e30e3ff..7c3423a 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -1,7 +1,20 @@ - project: check: jobs: - - noop + - opendev-zuul-jobs-noop + - opendev-build-diskimage-debian-bullseye gate: jobs: - - noop + - opendev-zuul-jobs-noop + - opendev-build-diskimage-debian-bullseye: + secrets: + - name: image_upload_secret + secret: image-upload-secret + pass-to-parent: true + image-build: + jobs: + - opendev-build-diskimage-debian-bullseye: + secrets: + - name: image_upload_secret + secret: image-upload-secret + pass-to-parent: true diff --git a/zuul.d/secrets.yaml b/zuul.d/secrets.yaml new file mode 100644 index 0000000..d83f5cc --- /dev/null +++ b/zuul.d/secrets.yaml @@ -0,0 +1,25 @@ +- secret: + name: image-upload-secret + data: + application_credential_id: !encrypted/pkcs1-oaep + - bICM8SzvLnrFfa/fhCSVYhxrgysEQDL1UZNPxYilK2eqad7+heFoGV/eU6M2C1AT1uBfM + N0IdfXjIAmzk0TMZGLmRTAFLx08VcQp77+ny6DSi6WvQd1aVeTMl48/LCpmU8qcxy5wvE + lGVen39vNpUuPkNaHOAypRAHIwXs6nGGNMILOQ3nKlCosaNmnB3Ixzu0fxPdSoDiYhRb5 + 7555UzhwNNr81gMD2b0idGcWEfL9A/DIxEljsEhUgV95JPeOs+PTlAVZXNSLiZsOJyGHp + +vZNCvg7Pnk7PFUSkWcpE4hKcmx8C55bESfWlLYr1z24PILXntLoFvfAvTldx1Q8yIRsQ + ifwtmMSiNvIsx8hse0scQ2pAQNtN0WRBRrWbU0NlSkDO4x07KrVxpE6fX+Qi9nRmKRAkX + uyszGJVjsLB+Lxo5kdxXHZRlkeYix8LcZaJdr5vMJlDrdN1WbtNEF9DB3oT0VB7Va4Vpa + 9FvQ6Z8IH6xp/jwnt/2DUEYJ1pjgqXuV7TMo3w1qfUTkGxjiJCVQe6nN7JMZfq6gXrXhV + u8k4bMUc5O5Pm54yxfbGfBUWhokk5WpdnxY9J2ruDnxUshUJYVlUCe4yLo3ARe3vCeE50 + SQ1iDZ9ubl69ehZv/Ok1kgYXxgPqQP0uFQQgUwG1ePrPIE+U0haFDXphrjrOiM= + application_credential_secret: !encrypted/pkcs1-oaep + - s5mHiz8S+jT0PuFNd2567PTn9Mypo/pnkgEBIRj3czZCtfOprvT/atyFcKExYI+NU6/1P + tv6XgjeItKyH0Iof6PiGLZyQf/BMekPCu7YG+fkKGjoWX6cxl/ZNYwPXuhI8tfrClCKMR + FOueVRbqjJrliAfAi6PID5XBiuEX4r+F+EmezipeuT05Mw+4R0HY8dLwu9lxmvrHVIA2t + GeHiNc1aP9jLa8Pp1suNPHAedZvyBPLajI0jhNffBfo96cPR276JpccJsSQkbh/7nVATk + hzaha1z+rjU2bWaTHsuqk3M6MGKU4M06siynwhlHjWAGzh1CDEc/CHwA1hHeiK3nuA0By + 1la55jH/5GPGi0e/2wmaZNpm/UMZsbFvOr9zCq0eb0Op0d1kZSD0fFcn0ixrYY+4XZ2CL + IjD1P05mmi2VbIvCY8fbyX/+ucUzM/P1B9XWdyhlQgF1cLnRQ27r2oa82m0aRbTms0x38 + +XLe7q4bKdDJI7YR/FwKzP+93KkmXJA05pzU8N7ojZ72MtykuQgxv/3SRL85po6J+58M3 + JHyovg7/PHIwyRM/j9Mm/+o5yu6GWwYr44lgQP1KoRz+YEa7XuEy9NI4bRtNRPixlJrwP + 7LunKyaRoe1aX7pc1C3/onhCsoSWbPQGW9hpzrtLLVChobphxy9ESam6Lcsr9k=