From 313cccff27cd05ca9bdfe606a867204d86d29602 Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Thu, 26 Apr 2018 16:53:06 -0700 Subject: [PATCH] Use the distro package to determine distro Some distros don't have lsb_release and some aren't even linuxes. Lets work to support them by using the python distro package which has a consistent api to getting the distro id, codename, and version info that we need. This does its best to be backward compatible by adding atoms for the os-release reported name and the lsb_release reported name when they differ on distros; however, it is likely that my survey was incomplete and there may be other non matching distros. (For some reason it seems like this is super common with rpm based distros...). Note that the test for a missing lsb_release has been removed since that is not the distro package's responsibility and it has multiple fallbacks. Change-Id: I891f5d21accb3fbf6d56bd6b0c2ebc2601682442 --- bindep/depends.py | 108 +++++++++++++----- .../use-distro-library-db71244a0a5cf1dd.yaml | 12 ++ .../tests/fixtures/amazonami/etc/os-release | 9 ++ bindep/tests/fixtures/arch/etc/os-release | 7 ++ bindep/tests/fixtures/centos/etc/os-release | 15 +++ bindep/tests/fixtures/fedora/etc/os-release | 14 +++ .../fixtures/opensuseleap/etc/os-release | 10 ++ .../tests/fixtures/rhelserver/etc/os-release | 15 +++ .../fixtures/rhelworkstation/etc/os-release | 15 +++ bindep/tests/fixtures/sles/etc/os-release | 7 ++ bindep/tests/fixtures/ubuntu/etc/os-release | 11 ++ bindep/tests/test_depends.py | 76 +++++++----- requirements.txt | 1 + 13 files changed, 242 insertions(+), 58 deletions(-) create mode 100644 bindep/releasenotes/notes/use-distro-library-db71244a0a5cf1dd.yaml create mode 100644 bindep/tests/fixtures/amazonami/etc/os-release create mode 100644 bindep/tests/fixtures/arch/etc/os-release create mode 100644 bindep/tests/fixtures/centos/etc/os-release create mode 100644 bindep/tests/fixtures/fedora/etc/os-release create mode 100644 bindep/tests/fixtures/opensuseleap/etc/os-release create mode 100644 bindep/tests/fixtures/rhelserver/etc/os-release create mode 100644 bindep/tests/fixtures/rhelworkstation/etc/os-release create mode 100644 bindep/tests/fixtures/sles/etc/os-release create mode 100644 bindep/tests/fixtures/ubuntu/etc/os-release diff --git a/bindep/depends.py b/bindep/depends.py index b3b5e71..4551824 100644 --- a/bindep/depends.py +++ b/bindep/depends.py @@ -23,6 +23,8 @@ import platform import subprocess import sys +import distro + debversion_grammar = """ epoch = :d ':' -> d @@ -275,6 +277,20 @@ class Depends(object): profiles.add(selector) return sorted(profiles) + def codenamebits(self, distro_id, codename): + atoms = set() + codenamebits = codename.split() + for i in range(len(codenamebits)): + atoms.add("%s-%s" % (distro_id, "-".join(codenamebits[:i + 1]))) + return atoms + + def releasebits(self, distro_id, release): + atoms = set() + releasebits = release.split(".") + for i in range(len(releasebits)): + atoms.add("%s-%s" % (distro_id, ".".join(releasebits[:i + 1]))) + return atoms + def platform_profiles(self): if platform.system() == 'Darwin': atoms = set(['darwin']) @@ -283,51 +299,89 @@ class Depends(object): atoms.add('brew') self.platform = Brew() return ["platform:%s" % (atom,) for atom in sorted(atoms)] - try: - output = subprocess.check_output( - ["lsb_release", "-cirs"], - stderr=subprocess.STDOUT).decode(getpreferredencoding(False)) - except OSError: + distro_id = distro.id() + if not distro_id: log = logging.getLogger(__name__) - log.error('Unable to execute lsb_release. Is it installed?') - raise - lsbinfo = output.lower().split() + log.error('Unable to determine distro ID. ' + 'Does /etc/os-release exist or ' + 'is lsb_release installed?') + raise Exception('Distro name not found') # NOTE(toabctl): distro can be more than one string (i.e. "SUSE LINUX") - codename = lsbinfo[len(lsbinfo) - 1:len(lsbinfo)][0] - release = lsbinfo[len(lsbinfo) - 2:len(lsbinfo) - 1][0] + codename = distro.codename().lower() + release = distro.version().lower() # NOTE(toabctl): space is a delimiter for bindep, so remove the spaces - distro = "".join(lsbinfo[0:len(lsbinfo) - 2]) - atoms = set([distro]) - atoms.add("%s-%s" % (distro, codename)) - releasebits = release.split(".") - for i in range(len(releasebits)): - atoms.add("%s-%s" % (distro, ".".join(releasebits[:i + 1]))) - if distro in ["debian", "ubuntu"]: + distro_id = "".join(distro_id.split()).lower() + atoms = set([distro_id]) + atoms.update(self.codenamebits(distro_id, codename)) + atoms.update(self.releasebits(distro_id, release)) + if distro_id in ["debian", "ubuntu"]: atoms.add("dpkg") self.platform = Dpkg() - elif distro in ["amazonami", "centos", "redhatenterpriseserver", - "redhatenterpriseworkstation", - "fedora", "opensuseproject", "opensuse", - "suselinux"]: - if distro in ["redhatenterpriseserver", - "redhatenterpriseworkstation"]: + # RPM distros seem to be especially complicated + elif distro_id in ["amzn", "amazonami", + "centos", "rhel", + "redhatenterpriseserver", + "redhatenterpriseworkstation", + "fedora", + "opensuseproject", "opensuse", + "opensuse-tumbleweed", "sles", "suselinux"]: + # Distro aliases + if distro_id in ["redhatenterpriseserver", + "redhatenterpriseworkstation"]: # just short alias atoms.add("rhel") - elif distro == "opensuseproject": + atoms.update(self.codenamebits("rhel", codename)) + atoms.update(self.releasebits("rhel", release)) + elif distro_id == 'rhel' and 'server' in distro.name().lower(): + atoms.add("redhatenterpriseserver") + atoms.update(self.codenamebits("redhatenterpriseserver", + codename)) + atoms.update(self.releasebits("redhatenterpriseserver", + release)) + elif (distro_id == 'rhel' and + 'workstation' in distro.name().lower()): + atoms.add("redhatenterpriseworkstation") + atoms.update(self.codenamebits("redhatenterpriseworkstation", + codename)) + atoms.update(self.releasebits("redhatenterpriseworkstation", + release)) + elif "amzn" in distro_id: + atoms.add("amazonami") + atoms.update(self.codenamebits("amazonami", codename)) + atoms.update(self.releasebits("amazonami", release)) + elif "amazonami" in distro_id: + atoms.add("amzn") + atoms.update(self.codenamebits("amzn", codename)) + atoms.update(self.releasebits("amzn", release)) + elif "opensuse" in distro_id: # just short alias atoms.add("opensuse") + atoms.update(self.codenamebits("opensuse", codename)) + atoms.update(self.releasebits("opensuse", release)) + atoms.add("opensuseproject") + atoms.update(self.codenamebits("opensuseproject", codename)) + atoms.update(self.releasebits("opensuseproject", release)) + elif "sles" in distro_id: + atoms.add("suselinux") + atoms.update(self.codenamebits("suselinux", codename)) + atoms.update(self.releasebits("suselinux", release)) + elif "suselinux" in distro_id: + atoms.add("sles") + atoms.update(self.codenamebits("sles", codename)) + atoms.update(self.releasebits("sles", release)) + # Family aliases - if 'suse' in distro: + if 'suse' in distro_id or distro_id == 'sles': atoms.add("suse") else: atoms.add("redhat") atoms.add("rpm") self.platform = Rpm() - elif distro in ["gentoo"]: + elif distro_id in ["gentoo"]: atoms.add("emerge") self.platform = Emerge() - elif distro in ["arch"]: + elif distro_id in ["arch"]: atoms.add("pacman") self.platform = Pacman() return ["platform:%s" % (atom,) for atom in sorted(atoms)] diff --git a/bindep/releasenotes/notes/use-distro-library-db71244a0a5cf1dd.yaml b/bindep/releasenotes/notes/use-distro-library-db71244a0a5cf1dd.yaml new file mode 100644 index 0000000..df3c6c1 --- /dev/null +++ b/bindep/releasenotes/notes/use-distro-library-db71244a0a5cf1dd.yaml @@ -0,0 +1,12 @@ +--- +upgrade: + - | + Bindep now depends on the distro python library to determine details + about the current platform. This library looks at both /etc/os-release + and lsb_release to find platform info. The os-release file data is + preferred and at times has slightly different data than lsb_release. + Every effort has been made to make this transition backward compatible + but some things may have been missed. + + The motivation for this change is that not all distros have lsb_release + available and we can let the distro library sort that out for us. diff --git a/bindep/tests/fixtures/amazonami/etc/os-release b/bindep/tests/fixtures/amazonami/etc/os-release new file mode 100644 index 0000000..7d7f60f --- /dev/null +++ b/bindep/tests/fixtures/amazonami/etc/os-release @@ -0,0 +1,9 @@ +NAME="Amazon Linux AMI" +VERSION="2016.03" +ID="amzn" +ID_LIKE="rhel fedora" +VERSION_ID="2016.03" +PRETTY_NAME="Amazon Linux AMI 2016.03" +ANSI_COLOR="0;33" +CPE_NAME="cpe:/o:amazon:linux:2016.03:ga" +HOME_URL="http://aws.amazon.com/amazon-linux-ami/" diff --git a/bindep/tests/fixtures/arch/etc/os-release b/bindep/tests/fixtures/arch/etc/os-release new file mode 100644 index 0000000..30e5cf3 --- /dev/null +++ b/bindep/tests/fixtures/arch/etc/os-release @@ -0,0 +1,7 @@ +NAME="Arch Linux" +ID=arch +PRETTY_NAME="Arch Linux" +ANSI_COLOR="0;36" +HOME_URL="https://www.archlinux.org/" +SUPPORT_URL="https://bbs.archlinux.org/" +BUG_REPORT_URL="https://bugs.archlinux.org/" diff --git a/bindep/tests/fixtures/centos/etc/os-release b/bindep/tests/fixtures/centos/etc/os-release new file mode 100644 index 0000000..c276e3a --- /dev/null +++ b/bindep/tests/fixtures/centos/etc/os-release @@ -0,0 +1,15 @@ +NAME="CentOS Linux" +VERSION="7 (Core)" +ID="centos" +ID_LIKE="rhel fedora" +VERSION_ID="7" +PRETTY_NAME="CentOS Linux 7 (Core)" +ANSI_COLOR="0;31" +CPE_NAME="cpe:/o:centos:centos:7" +HOME_URL="https://www.centos.org/" +BUG_REPORT_URL="https://bugs.centos.org/" + +CENTOS_MANTISBT_PROJECT="CentOS-7" +CENTOS_MANTISBT_PROJECT_VERSION="7" +REDHAT_SUPPORT_PRODUCT="centos" +REDHAT_SUPPORT_PRODUCT_VERSION="7" diff --git a/bindep/tests/fixtures/fedora/etc/os-release b/bindep/tests/fixtures/fedora/etc/os-release new file mode 100644 index 0000000..f57b5fd --- /dev/null +++ b/bindep/tests/fixtures/fedora/etc/os-release @@ -0,0 +1,14 @@ +NAME=Fedora +VERSION="23 (Twenty Three)" +ID=fedora +VERSION_ID=23 +PRETTY_NAME="Fedora 23 (Twenty Three)" +ANSI_COLOR="0;34" +CPE_NAME="cpe:/o:fedoraproject:fedora:23" +HOME_URL="https://fedoraproject.org/" +BUG_REPORT_URL="https://bugzilla.redhat.com/" +REDHAT_BUGZILLA_PRODUCT="Fedora" +REDHAT_BUGZILLA_PRODUCT_VERSION=23 +REDHAT_SUPPORT_PRODUCT="Fedora" +REDHAT_SUPPORT_PRODUCT_VERSION=23 +PRIVACY_POLICY_URL=https://fedoraproject.org/wiki/Legal:PrivacyPolicy diff --git a/bindep/tests/fixtures/opensuseleap/etc/os-release b/bindep/tests/fixtures/opensuseleap/etc/os-release new file mode 100644 index 0000000..2f4fd34 --- /dev/null +++ b/bindep/tests/fixtures/opensuseleap/etc/os-release @@ -0,0 +1,10 @@ +NAME="openSUSE Leap" +VERSION="42.1" +VERSION_ID="42.1" +PRETTY_NAME="openSUSE Leap 42.1 (x86_64)" +ID=opensuse +ANSI_COLOR="0;32" +CPE_NAME="cpe:/o:opensuse:opensuse:42.1" +BUG_REPORT_URL="https://bugs.opensuse.org" +HOME_URL="https://opensuse.org/" +ID_LIKE="suse" diff --git a/bindep/tests/fixtures/rhelserver/etc/os-release b/bindep/tests/fixtures/rhelserver/etc/os-release new file mode 100644 index 0000000..5c3ae5a --- /dev/null +++ b/bindep/tests/fixtures/rhelserver/etc/os-release @@ -0,0 +1,15 @@ +NAME="Red Hat Enterprise Linux Server" +VERSION="7.0 (Maipo)" +ID="rhel" +ID_LIKE="fedora" +VERSION_ID="7.0" +PRETTY_NAME="Red Hat Enterprise Linux Server 7.0 (Maipo)" +ANSI_COLOR="0;31" +CPE_NAME="cpe:/o:redhat:enterprise_linux:7.0:GA:server" +HOME_URL="https://www.redhat.com/" +BUG_REPORT_URL="https://bugzilla.redhat.com/" + +REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 7" +REDHAT_BUGZILLA_PRODUCT_VERSION=7.0 +REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" +REDHAT_SUPPORT_PRODUCT_VERSION=7.0 diff --git a/bindep/tests/fixtures/rhelworkstation/etc/os-release b/bindep/tests/fixtures/rhelworkstation/etc/os-release new file mode 100644 index 0000000..6f277ef --- /dev/null +++ b/bindep/tests/fixtures/rhelworkstation/etc/os-release @@ -0,0 +1,15 @@ +NAME="Red Hat Enterprise Linux Workstation" +VERSION="7.3 (Maipo)" +ID="rhel" +ID_LIKE="fedora" +VERSION_ID="7.3" +PRETTY_NAME="Red Hat Enterprise Linux Workstation 7.3 (Maipo)" +ANSI_COLOR="0;31" +CPE_NAME="cpe:/o:redhat:enterprise_linux:7.3:GA:workstation" +HOME_URL="https://www.redhat.com/" +BUG_REPORT_URL="https://bugzilla.redhat.com/" + +REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 7" +REDHAT_BUGZILLA_PRODUCT_VERSION=7.3 +REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" +REDHAT_SUPPORT_PRODUCT_VERSION="7.3" diff --git a/bindep/tests/fixtures/sles/etc/os-release b/bindep/tests/fixtures/sles/etc/os-release new file mode 100644 index 0000000..b68860e --- /dev/null +++ b/bindep/tests/fixtures/sles/etc/os-release @@ -0,0 +1,7 @@ +NAME="SLES" +VERSION="12-SP3" +VERSION_ID="12.3" +PRETTY_NAME="SUSE Linux Enterprise Server 12 SP3" +ID="sles" +ANSI_COLOR="0;32" +CPE_NAME="cpe:/o:suse:sles:12:sp3" diff --git a/bindep/tests/fixtures/ubuntu/etc/os-release b/bindep/tests/fixtures/ubuntu/etc/os-release new file mode 100644 index 0000000..c167782 --- /dev/null +++ b/bindep/tests/fixtures/ubuntu/etc/os-release @@ -0,0 +1,11 @@ +NAME="Ubuntu" +VERSION="16.04.4 LTS (Xenial Xerus)" +ID=ubuntu +ID_LIKE=debian +PRETTY_NAME="Ubuntu 16.04.4 LTS" +VERSION_ID="16.04" +HOME_URL="http://www.ubuntu.com/" +SUPPORT_URL="http://help.ubuntu.com/" +BUG_REPORT_URL="http://bugs.launchpad.net/ubuntu/" +VERSION_CODENAME=xenial +UBUNTU_CODENAME=xenial diff --git a/bindep/tests/test_depends.py b/bindep/tests/test_depends.py index 65e85bb..34f4547 100644 --- a/bindep/tests/test_depends.py +++ b/bindep/tests/test_depends.py @@ -16,10 +16,12 @@ # limitations under the License. import contextlib +import os.path import platform import subprocess from textwrap import dedent +import distro import fixtures import mock import ometa.runtime @@ -40,6 +42,22 @@ from bindep.depends import Rpm # string. All mock calls for subprocess.check_output have been updated to # ensure bytes is used over string. In python 2 this is a no-op change. +FIXTURE_DIR = os.path.join(os.path.dirname(__file__), + 'fixtures') + + +class DistroFixture(fixtures.Fixture): + def __init__(self, distro_name): + self.distro_name = distro_name.lower() + + def _setUp(self): + # This type of monkey patching is borrowed from the distro test + # suite. + os_release = os.path.join(FIXTURE_DIR, self.distro_name, + 'etc', 'os-release') + mydistro = distro.LinuxDistribution(False, os_release, 'non') + self.useFixture(fixtures.MonkeyPatch('distro._distro', mydistro)) + class TestDepends(TestCase): @@ -48,7 +66,7 @@ class TestDepends(TestCase): self.assertEqual([], depends.profiles()) def test_platform_profiles_succeeds(self): - with self._mock_lsb('Ubuntu'): + with DistroFixture('Ubuntu'): depends = Depends("") self.assertIsInstance(depends.platform_profiles(), list) @@ -76,13 +94,13 @@ class TestDepends(TestCase): mock_checkoutput.assert_called_once_with() def test_detects_amazon_linux(self): - with self._mock_lsb("AmazonAMI"): + with DistroFixture("AmazonAMI"): depends = Depends("") self.assertThat( depends.platform_profiles(), Contains("platform:amazonami")) def test_detects_centos(self): - with self._mock_lsb("CentOS"): + with DistroFixture("CentOS"): depends = Depends("") platform_profiles = depends.platform_profiles() self.assertThat(platform_profiles, Contains("platform:centos")) @@ -95,7 +113,7 @@ class TestDepends(TestCase): self.assertThat(platform_profiles, Contains("platform:darwin")) def test_detects_rhel(self): - with self._mock_lsb("RedHatEnterpriseServer"): + with DistroFixture("RHELServer"): depends = Depends("") platform_profiles = depends.platform_profiles() self.assertThat( @@ -109,7 +127,7 @@ class TestDepends(TestCase): Contains("platform:redhat")) def test_detects_rhel_workstation(self): - with self._mock_lsb("RedHatEnterpriseWorkstation"): + with DistroFixture("RHELWorkstation"): depends = Depends("") platform_profiles = depends.platform_profiles() self.assertThat( @@ -123,14 +141,16 @@ class TestDepends(TestCase): Contains("platform:redhat")) def test_detects_fedora(self): - with self._mock_lsb("Fedora"): + with DistroFixture("Fedora"): depends = Depends("") platform_profiles = depends.platform_profiles() self.assertThat(platform_profiles, Contains("platform:fedora")) self.assertThat(platform_profiles, Contains("platform:redhat")) def test_detects_opensuse_project(self): - with self._mock_lsb("openSUSE Project"): + # TODO what does an os-release for opensuse project look like? + # Is this different than sles, leap, and tumbleweed? + with DistroFixture("openSUSEleap"): depends = Depends("") platform_profiles = depends.platform_profiles() self.assertThat(platform_profiles, @@ -138,12 +158,12 @@ class TestDepends(TestCase): self.assertThat(platform_profiles, Contains("platform:opensuse")) self.assertThat(platform_profiles, - Contains("platform:opensuseproject-14.04")) + Contains("platform:opensuseproject-42.1")) self.assertThat(platform_profiles, Contains("platform:suse")) def test_detects_opensuse(self): - with self._mock_lsb("openSUSE"): + with DistroFixture("openSUSEleap"): depends = Depends("") platform_profiles = depends.platform_profiles() self.assertThat(platform_profiles, @@ -152,99 +172,93 @@ class TestDepends(TestCase): Contains("platform:suse")) def test_detects_suse_linux(self): - with self._mock_lsb("SUSE Linux"): + with DistroFixture("SLES"): depends = Depends("") platform_profiles = depends.platform_profiles() self.assertThat(platform_profiles, Contains("platform:suselinux")) self.assertThat(platform_profiles, Contains("platform:suse")) def test_detects_ubuntu(self): - with self._mock_lsb("Ubuntu"): + with DistroFixture("Ubuntu"): depends = Depends("") self.assertThat( depends.platform_profiles(), Contains("platform:ubuntu")) def test_detects_release(self): - with self._mock_lsb("Ubuntu"): + with DistroFixture("Ubuntu"): depends = Depends("") self.assertThat( - depends.platform_profiles(), Contains("platform:ubuntu-14")) + depends.platform_profiles(), Contains("platform:ubuntu-16")) def test_detects_subrelease(self): - with self._mock_lsb("Ubuntu"): + with DistroFixture("Ubuntu"): depends = Depends("") self.assertThat( - depends.platform_profiles(), Contains("platform:ubuntu-14.04")) + depends.platform_profiles(), Contains("platform:ubuntu-16.04")) def test_detects_codename(self): - with self._mock_lsb("Ubuntu"): + with DistroFixture("Ubuntu"): depends = Depends("") self.assertThat( depends.platform_profiles(), - Contains("platform:ubuntu-trusty")) + Contains("platform:ubuntu-xenial")) def test_centos_implies_rpm(self): - with self._mock_lsb("CentOS"): + with DistroFixture("CentOS"): depends = Depends("") self.assertThat( depends.platform_profiles(), Contains("platform:rpm")) self.assertIsInstance(depends.platform, Rpm) def test_rhel_implies_rpm(self): - with self._mock_lsb("RedHatEnterpriseServer"): + with DistroFixture("RHELServer"): depends = Depends("") self.assertThat( depends.platform_profiles(), Contains("platform:rpm")) self.assertIsInstance(depends.platform, Rpm) def test_fedora_implies_rpm(self): - with self._mock_lsb("Fedora"): + with DistroFixture("Fedora"): depends = Depends("") self.assertThat( depends.platform_profiles(), Contains("platform:rpm")) self.assertIsInstance(depends.platform, Rpm) def test_opensuse_project_implies_rpm(self): - with self._mock_lsb("openSUSE Project"): + with DistroFixture("openSUSEleap"): depends = Depends("") self.assertThat( depends.platform_profiles(), Contains("platform:rpm")) self.assertIsInstance(depends.platform, Rpm) def test_opensuse_implies_rpm(self): - with self._mock_lsb("openSUSE"): + with DistroFixture("openSUSEleap"): depends = Depends("") self.assertThat( depends.platform_profiles(), Contains("platform:rpm")) self.assertIsInstance(depends.platform, Rpm) def test_suse_linux_implies_rpm(self): - with self._mock_lsb("SUSE LINUX"): + with DistroFixture("SLES"): depends = Depends("") self.assertThat( depends.platform_profiles(), Contains("platform:rpm")) self.assertIsInstance(depends.platform, Rpm) def test_ubuntu_implies_dpkg(self): - with self._mock_lsb("Ubuntu"): + with DistroFixture("Ubuntu"): depends = Depends("") self.assertThat( depends.platform_profiles(), Contains("platform:dpkg")) self.assertIsInstance(depends.platform, Dpkg) def test_arch_implies_pacman(self): - with self._mock_lsb("Arch"): + with DistroFixture("Arch"): depends = Depends("") self.assertThat( depends.platform_profiles(), Contains("platform:pacman")) self.assertIsInstance(depends.platform, Pacman) - def test_missing_lsb_release(self): - with mock.patch('subprocess.check_output') as mock_co: - mock_co.side_effect = OSError - depends = Depends("") - self.assertRaises(OSError, depends.platform_profiles) - def test_finds_profiles(self): depends = Depends(dedent("""\ foo diff --git a/requirements.txt b/requirements.txt index 7b24fc4..62c1ee4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ +distro pbr>=2.0.0 # Apache-2.0 Parsley