From f831b3d0b6bbbc15afd4596c18982bc582d9a4bf Mon Sep 17 00:00:00 2001 From: Jay Faulkner Date: Thu, 11 Jul 2024 11:18:19 -0700 Subject: [PATCH] [gentoo] Fix+Update CI for 23.0 profile - Adjusts how we remove pacakges. Gentoo maintains a list of packages that the user has requested be installed called "world". By deselecting the packages, we remove them from this list, and at the end, call depclean which will uninstall packages no longer selected or needed as dependencies. - Updates profile logic. We should theoretically be able to support any new gentoo profile, without having to maintain a static list ourself by just updating the base. - Updates CI job to use default profile as determined by the gentoo element. This basically eliminates one more place we need to update profiles on change. - Ensures we install installkernel with USE=+grub so we actually install the kernel - Do not use testing (~amd64) packages unless absolutely neccessary - Fix growroot openrc initscript to use /sbin/openrc-run instead of deprecated-and-now-removed /sbin/runscript. Change-Id: Ie9d2ab67d72114603034374854bb3a3d52de8ca4 --- .zuul.d/jobs.yaml | 2 - .../elements/devuser/install.d/50-devuser | 1 + diskimage_builder/elements/gentoo/README.rst | 24 ++++--- .../elements/gentoo/bin/install-packages | 35 ++++++---- .../environment.d/00-gentoo-envars.bash | 10 ++- .../gentoo/pre-install.d/02-gentoo-02-flags | 10 +-- .../elements/gentoo/root.d/10-gentoo-image | 67 ++++++------------- .../growroot/init-scripts/openrc/growroot | 2 +- .../elements/install-static/pkg-map | 7 ++ .../gentoo-profile-23.0-99357c919639bd3f.yaml | 14 ++++ 10 files changed, 89 insertions(+), 83 deletions(-) create mode 100644 diskimage_builder/elements/install-static/pkg-map create mode 100644 releasenotes/notes/gentoo-profile-23.0-99357c919639bd3f.yaml diff --git a/.zuul.d/jobs.yaml b/.zuul.d/jobs.yaml index d9f8da9fc..2b7dd0f36 100644 --- a/.zuul.d/jobs.yaml +++ b/.zuul.d/jobs.yaml @@ -266,8 +266,6 @@ nodepool_diskimage: base_element: gentoo release: '' - env-vars: - GENTOO_PROFILE: 'default/linux/amd64/17.1/systemd/merged-usr' - job: name: dib-nodepool-functional-openstack-debian-stretch-src diff --git a/diskimage_builder/elements/devuser/install.d/50-devuser b/diskimage_builder/elements/devuser/install.d/50-devuser index 5e704ef22..8819b34cc 100755 --- a/diskimage_builder/elements/devuser/install.d/50-devuser +++ b/diskimage_builder/elements/devuser/install.d/50-devuser @@ -19,6 +19,7 @@ fi set -x if [ -n "${DIB_DEV_USER_PWDLESS_SUDO}" ]; then + mkdir -p /etc/sudoers.d/ cat > /etc/sudoers.d/${DIB_DEV_USER_USERNAME} << EOF ${DIB_DEV_USER_USERNAME} ALL=(ALL) NOPASSWD:ALL EOF diff --git a/diskimage_builder/elements/gentoo/README.rst b/diskimage_builder/elements/gentoo/README.rst index 1719076e2..5a81fec58 100644 --- a/diskimage_builder/elements/gentoo/README.rst +++ b/diskimage_builder/elements/gentoo/README.rst @@ -23,25 +23,29 @@ Notes: * In order to run the vm element you will need to make sure `sys-block/parted` is installed on the host. -* Other profiles can be used by exporting GENTOO_PROFILE with a valid profile. - A list of valid profiles follows: +* The default profile is ``default/linux/amd64/23.0``. - default/linux/amd64/17.1 - default/linux/amd64/17.1/no-multilib - default/linux/amd64/17.1/hardened - default/linux/amd64/17.1/no-multilib/hardened - default/linux/amd64/17.1/systemd - default/linux/arm64/17.0 - default/linux/arm64/17.0/systemd +* Any ``amd64`` or ``arm64`` profile with a stage tarball published by gentoo + in the ``autobuilds`` directory for that arch are supported. Warning: + the GENTOO_PROFILE environment variable will take precedence over the ARCH + environment variable. * You can set the `GENTOO_PORTAGE_CLEANUP` environment variable to False to disable the clean up of portage repositories (including overlays). This will make the image bigger if caching is also disabled. +* In many cases, the resulting image will not have a valid profile set. If + you need to interactively use portage in a machine created with DIB, you + will need to run `eselect profile set some/valid/profile` before interacting + with portage. + * Gentoo supports many different versions of python, in order to select one you may use the `GENTOO_PYTHON_TARGETS` environment variable to select the versions of python you want on your image. The format of this variable - is a string as follows `"python2_7 python3_6"`. + is a string as follows `"python3_10 python3_11"`. This variable only impacts + the python versions used for distribution-installed python packages; see + https://wiki.gentoo.org/wiki/Project:Python/PYTHON_TARGETS for more + information. * You can enable overlays using the `GENTOO_OVERLAYS` variable. In it you should put a space separated list of overlays. The overlays must be in the diff --git a/diskimage_builder/elements/gentoo/bin/install-packages b/diskimage_builder/elements/gentoo/bin/install-packages index 43e0c44ff..d22cc5281 100755 --- a/diskimage_builder/elements/gentoo/bin/install-packages +++ b/diskimage_builder/elements/gentoo/bin/install-packages @@ -87,7 +87,7 @@ while true; do install_gentoo_packages --usepkg=n @preserved-rebuild etc-update --automode -5 eselect news read new - exit 0; + exit 0 ;; -e ) ACTION='remove' @@ -127,24 +127,33 @@ else if [[ ! -f ${PORTDIR}/profiles ]]; then emerge-webrsync -q fi - install_gentoo_packages --changed-use "${PKGS}" + # --noreplace prevents us from rebuilding a package already installed + # --changed-use means that package will be rebuilt *if* USE flags for + # it (configuration) has changed + install_gentoo_packages --noreplace --changed-use "${PKGS}" elif [[ "${ACTION}" == 'remove' ]]; then if [[ ! -f ${PORTDIR}/profiles ]]; then emerge-webrsync -q fi - # remove packages from uninstall list that are not already installed - # this can result in false positives if not presented with full category/package names - CLEANED_PKGS=() + # A good practice for removing packages in gentoo is to deselect them, + # removing them from "world" set -- the equivalent of "unmark" in dnf. + # This tells portage we no longer care if the package is installed, + # and it can be removed if depedancies allow. + # This means a removal is two steps: + # - emerge --deselect $pkg + # - emerge --depclean + # + # The depclean step removes all packages that are not in the "world" + # set and are not in the dependency graph for any packages in "world" + # set. + # + # Other methods of removal may work; but this method sets us up to + # calculate the dependency graph exactly once and prevents portage + # from erroring if any of the packages were not already selected. for PKG in ${PKGS}; do - # the '^' and '$' in the search query are important so we don't get matched to - # packages that include our package name as part of their package name - if ! emerge --search "^${PKG}$" | grep -q 'Not Installed' ; then - CLEANED_PKGS+=("${PKG}") - fi + install_gentoo_packages --deselect $PKG done - if [ ${#CLEANED_PKGS[@]} -ne 0 ]; then - install_gentoo_packages -C "${CLEANED_PKGS[@]}" - fi + install_gentoo_packages --depclean else echo 'something went really wrong, install action is not install or remove' fi diff --git a/diskimage_builder/elements/gentoo/environment.d/00-gentoo-envars.bash b/diskimage_builder/elements/gentoo/environment.d/00-gentoo-envars.bash index 8cc9c7e54..92941eb60 100644 --- a/diskimage_builder/elements/gentoo/environment.d/00-gentoo-envars.bash +++ b/diskimage_builder/elements/gentoo/environment.d/00-gentoo-envars.bash @@ -1,12 +1,18 @@ export DIB_RELEASE=gentoo export DISTRO_NAME=gentoo export EFI_BOOT_DIR="EFI/gentoo" -export GENTOO_PROFILE=${GENTOO_PROFILE:-'default/linux/amd64/17.1'} + export GENTOO_PORTAGE_CLEANUP=${GENTOO_PORTAGE_CLEANUP:-'True'} export GENTOO_PYTHON_TARGETS=${GENTOO_PYTHON_TARGETS:-''} export GENTOO_OVERLAYS=${GENTOO_OVERLAYS:-''} export GENTOO_EMERGE_DEFAULT_OPTS=${GENTOO_EMERGE_DEFAULT_OPTS:-"--binpkg-respect-use --rebuilt-binaries=y --usepkg=y --with-bdeps=y --binpkg-changed-deps=y --quiet --jobs=2 --autounmask=n"} +# NOTE(JayF): This defines the base gentoo profile version supported +# in DIB. As gentoo is a rolling release distro, the older profiles +# are unsupported. +export GENTOO_BASE_PROFILE="default/linux/${ARCH}/23.0" +export GENTOO_PROFILE=${GENTOO_PROFILE:-$GENTOO_BASE_PROFILE} + # set the default bash array if GENTOO_EMERGE_ENV is not defined as an array if ! declare -p GENTOO_EMERGE_ENV 2> /dev/null | grep -q '^declare \-a'; then declare -a GENTOO_EMERGE_ENV @@ -17,7 +23,7 @@ if ! declare -p GENTOO_EMERGE_ENV 2> /dev/null | grep -q '^declare \-a'; then GENTOO_EMERGE_ENV+=("PORTDIR=\"/tmp/portage-portdir\"") export GENTOO_EMERGE_ENV fi -# itterate over the array, exporting each 'line' +# iterate over the array, exporting each 'line' for (( i=0; i<${#GENTOO_EMERGE_ENV[@]}; i++ )); do eval export "${GENTOO_EMERGE_ENV[i]}" done diff --git a/diskimage_builder/elements/gentoo/pre-install.d/02-gentoo-02-flags b/diskimage_builder/elements/gentoo/pre-install.d/02-gentoo-02-flags index 7cf8ebba1..6386afa1a 100755 --- a/diskimage_builder/elements/gentoo/pre-install.d/02-gentoo-02-flags +++ b/diskimage_builder/elements/gentoo/pre-install.d/02-gentoo-02-flags @@ -20,21 +20,13 @@ mkdir -p /etc/portage/package.use echo 'dev-python/pip vanilla' >> /etc/portage/package.use/pip # needed to create disk images echo 'sys-fs/lvm2 lvm -thin' >> /etc/portage/package.use/grub -echo 'sys-kernel/installkernel dracut' >> /etc/portage/package.use/kernel +echo 'sys-kernel/installkernel grub dracut' >> /etc/portage/package.use/kernel echo 'sys-boot/grub device-mapper' >> /etc/portage/package.use/grub echo 'sys-boot/grub grub_platforms_efi-64' >> /etc/portage/package.use/grub # always enable efi-64 if [[ 'amd64' == "${ARCH}" ]]; then echo 'sys-boot/grub grub_platforms_pc' >> /etc/portage/package.use/grub # bios support for bios systems fi -# needed to install static kernel -echo "sys-kernel/gentoo-kernel-bin ~${ARCH}" >> /etc/portage/package.accept_keywords/kernel -echo "virtual/dist-kernel ~${ARCH}" >> /etc/portage/package.accept_keywords/kernel - -# needed for gcc-10 support -echo "~sys-block/open-iscsi-2.1.4 ~${ARCH}" >> /etc/portage/package.accept_keywords/open-iscsi -echo "~sys-block/open-isns-0.101 ~${ARCH}" >> /etc/portage/package.accept_keywords/open-iscsi - # musl only valid for amd64 for now if [[ "${GENTOO_PROFILE}" == *"musl"* ]]; then echo "sys-libs/pam cracklib" >> /etc/portage/package.use/musl diff --git a/diskimage_builder/elements/gentoo/root.d/10-gentoo-image b/diskimage_builder/elements/gentoo/root.d/10-gentoo-image index 456e44c69..4017e95ec 100755 --- a/diskimage_builder/elements/gentoo/root.d/10-gentoo-image +++ b/diskimage_builder/elements/gentoo/root.d/10-gentoo-image @@ -24,56 +24,30 @@ set -o pipefail [ -n "${ARCH}" ] [ -n "${TARGET_ROOT}" ] -if [[ 'amd64' != "${ARCH}" ]] && [[ 'arm64' != "${ARCH}" ]]; then - echo "Only amd64 or arm64 images are currently available but ARCH is set to ${ARCH}." +P_SUFFIX="${GENTOO_PROFILE#$GENTOO_BASE_PROFILE}" +F_SUFFIX="${P_SUFFIX//\//\-}" +if [[ ${F_SUFFIX} != *"-systemd" ]]; then + # NOTE(JayF): OpenRC is implied, and appended to the filename, unless systemd is specified. + F_SUFFIX="${F_SUFFIX}-openrc" +fi + +DIB_CLOUD_SOURCE=${DIB_CLOUD_SOURCE:-"https://distfiles.gentoo.org/releases/${ARCH}/autobuilds/latest-stage3-${ARCH}${F_SUFFIX}.txt"} +echo "Fetching available stages from ${DIB_CLOUD_SOURCE} for profile ${GENTOO_PROFILE}" + +STAGE_LIST=$(curl "${DIB_CLOUD_SOURCE}" -s -f || true) +if [[ -z ${STAGE_LIST} ]]; then + echo "Unable to find a stage list for ${GENTOO_PROFILE} at ${DIB_CLOUD_SOURCE}." + echo "This element only currently supports profiles included in the periodic" + echo "Gentoo autobuilds." exit 1 fi -GENTOO_PROFILE=${GENTOO_PROFILE:-'default/linux/amd64/17.1'} -if [[ "${GENTOO_PROFILE}" == "default/linux/amd64/17.1" ]]; then - FILENAME_BASE='amd64_gentoo-stage3' - SIGNED_SOURCE_SUFFIX='-openrc' -elif [[ "${GENTOO_PROFILE}" == "default/linux/amd64/17.1/no-multilib" ]]; then - FILENAME_BASE='amd64_gentoo-stage3-nomultilib' - SIGNED_SOURCE_SUFFIX='-nomultilib-openrc' -elif [[ "${GENTOO_PROFILE}" == "default/linux/amd64/17.1/hardened" ]]; then - FILENAME_BASE='amd64_gentoo-stage3-hardened' - SIGNED_SOURCE_SUFFIX='-hardened-openrc' -elif [[ "${GENTOO_PROFILE}" == "default/linux/amd64/17.1/no-multilib/hardened" ]]; then - FILENAME_BASE='amd64_gentoo-stage3-hardened-nomultilib' - SIGNED_SOURCE_SUFFIX='-hardened-nomultilib-openrc' -elif [[ "${GENTOO_PROFILE}" == "default/linux/amd64/17.0/musl/hardened" ]]; then - FILENAME_BASE='amd64_gentoo-stage3-hardened-musl' - SIGNED_SOURCE_SUFFIX='-musl-hardened' -elif [[ "${GENTOO_PROFILE}" == "default/linux/amd64/17.1/systemd/merged-usr" ]]; then - FILENAME_BASE='amd64_gentoo-stage3-systemd-mergedusr' - SIGNED_SOURCE_SUFFIX='-systemd-mergedusr' -elif [[ "${GENTOO_PROFILE}" == "default/linux/arm64/17.0" ]]; then - FILENAME_BASE='arm64_gentoo-stage3' - SIGNED_SOURCE_SUFFIX='' -elif [[ "${GENTOO_PROFILE}" == "default/linux/arm64/17.0/systemd/merged-usr" ]]; then - FILENAME_BASE='arm64_gentoo-stage3-systemd-mergedusr' - SIGNED_SOURCE_SUFFIX='-systemd-mergedusr' -else - echo 'invalid profile, please select from the following profiles' - echo 'default/linux/amd64/17.1' - echo 'default/linux/amd64/17.1/no-multilib' - echo 'default/linux/amd64/17.1/hardened' - echo 'default/linux/amd64/17.1/no-multilib/hardened' - echo 'default/linux/amd64/17.1/systemd/merged-usr' - echo 'default/linux/arm64/17.0' - echo 'default/linux/arm64/17.0/systemd/merged-usr' - exit 1 -fi +UPSTREAM_FILENAME=$(echo "${STAGE_LIST}" | grep -B1 'BEGIN PGP SIGNATURE' | head -n1 | cut -d\ -f1) -if [[ "${GENTOO_PROFILE}" == *'amd64'* ]]; then - ARCH_PATH='amd64' -elif [[ "${GENTOO_PROFILE}" == *'arm64'* ]]; then - ARCH_PATH='arm64' -fi -DIB_CLOUD_SOURCE=${DIB_CLOUD_SOURCE:-"http://distfiles.gentoo.org/releases/${ARCH_PATH}/autobuilds/latest-stage3-${ARCH_PATH}${SIGNED_SOURCE_SUFFIX}.txt"} -BASE_IMAGE_FILE=${BASE_IMAGE_FILE:-"http://distfiles.gentoo.org/releases/${ARCH_PATH}/autobuilds/$(curl "${DIB_CLOUD_SOURCE}" -s -f | grep -B1 'BEGIN PGP SIGNATURE' | head -n 1 | cut -d\ -f 1)"} +echo "Chose ${UPSTREAM_FILENAME} as candidate stage tarball" +BASE_IMAGE_FILE=${BASE_IMAGE_FILE:-"https://distfiles.gentoo.org/releases/${ARCH}/autobuilds/${UPSTREAM_FILENAME}"} BASE_IMAGE_FILE_SUFFIX=${BASE_IMAGE_FILE_SUFFIX:-"$(basename "${BASE_IMAGE_FILE}" | cut -d. -f 2,3)"} +FILENAME_BASE="gentoo-${GENTOO_PROFILE//\//\-}.${BASE_IMAGE_FILE_SUFFIX}" SIGNATURE_FILE="${SIGNATURE_FILE:-${BASE_IMAGE_FILE}.asc}" CACHED_FILE="${DIB_IMAGE_CACHE}/${FILENAME_BASE}.${BASE_IMAGE_FILE_SUFFIX}" CACHED_SIGNATURE_FILE="${DIB_IMAGE_CACHE}/${FILENAME_BASE}.asc" @@ -89,7 +63,7 @@ else # this key can be verified at one of the following places # https://wiki.gentoo.org/wiki/Project:RelEng#Keys # https://dev.gentoo.org/~dolsen/releases/keyrings/gentoo-keys-*.tar.xz - # http://distfiles.gentoo.org/distfiles/gentoo-keys-*.tar.xz + # https://distfiles.gentoo.org/distfiles/gentoo-keys-*.tar.xz # check the sig file if ! gpgv --keyring "${TMP_HOOKS_PATH}"/extra-data.d/gentoo-releng.gpg "${CACHED_SIGNATURE_FILE}" "${CACHED_FILE}"; then echo 'invalid signature file' @@ -110,3 +84,4 @@ sudo tar -C "${TARGET_ROOT}" --numeric-owner --xattrs -xf "${CACHED_FILE}" # This broken link confuses things like dhclient. # [1] https://bugzilla.redhat.com/show_bug.cgi?id=1197204 echo -e "# This file intentionally left blank\n" | sudo tee "${TARGET_ROOT}"/etc/resolv.conf + diff --git a/diskimage_builder/elements/growroot/init-scripts/openrc/growroot b/diskimage_builder/elements/growroot/init-scripts/openrc/growroot index 4656fbcf7..1783b6124 100755 --- a/diskimage_builder/elements/growroot/init-scripts/openrc/growroot +++ b/diskimage_builder/elements/growroot/init-scripts/openrc/growroot @@ -1,4 +1,4 @@ -#!/sbin/runscript +#!/sbin/openrc-run start() { /usr/local/sbin/growroot diff --git a/diskimage_builder/elements/install-static/pkg-map b/diskimage_builder/elements/install-static/pkg-map new file mode 100644 index 000000000..2445bc098 --- /dev/null +++ b/diskimage_builder/elements/install-static/pkg-map @@ -0,0 +1,7 @@ +{ + "family":{ + "gentoo": { + "rsync": "net-misc/rsync" + } + } +} diff --git a/releasenotes/notes/gentoo-profile-23.0-99357c919639bd3f.yaml b/releasenotes/notes/gentoo-profile-23.0-99357c919639bd3f.yaml new file mode 100644 index 000000000..4a71ba3d6 --- /dev/null +++ b/releasenotes/notes/gentoo-profile-23.0-99357c919639bd3f.yaml @@ -0,0 +1,14 @@ +features: + - Supports Gentoo profile 23.0 and removes support for the nonworking + 17.1 and 17.0 profiles. + - Gentoo element updated to avoid using testing (~arch) packages. + - Gentoo element now uses upstream binary package host by default. +fixes: + - Fixed an issue where the growroot element on openrc init systems would + not function. + - Fixed an issue where the devuser element was unable to grant sudo + capabilities on gentoo images. + - Fixed an issue in Gentoo implmentation for install-packages element + where build time would grow linearly with each additional package removal. + Now, all removed packages are deselected and removed in a single + transaction.