Merge pull request #255893 from illustris/cloud-init

cloud-init: 23.2.2 -> 23.3.1

3 files changed, 12 insertions, 429 deletions

diff --git a/pkgs/tools/virtualization/cloud-init/0001-add-nixos-support.patch b/pkgs/tools/virtualization/cloud-init/0001-add-nixos-support.patch
index f26690bacb7..2e293321ac0 100644
--- a/pkgs/tools/virtualization/cloud-init/0001-add-nixos-support.patch
+++ b/pkgs/tools/virtualization/cloud-init/0001-add-nixos-support.patch
@@ -1,10 +1,10 @@
 diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
-index b82852e1..c998b21e 100644
+index 7b83df8d..6d04de1a 100644
 --- a/cloudinit/distros/__init__.py
 +++ b/cloudinit/distros/__init__.py
-@@ -74,6 +74,7 @@ OSFAMILIES = {
+@@ -75,6 +75,7 @@ OSFAMILIES = {
      ],
-     "openEuler": ["openEuler"],
+     "openeuler": ["openeuler"],
      "OpenCloudOS": ["OpenCloudOS", "TencentOS"],
 +    "nixos": ["nixos"],
  }
@@ -12,7 +12,7 @@ index b82852e1..c998b21e 100644
  LOG = logging.getLogger(__name__)
 diff --git a/cloudinit/distros/nixos.py b/cloudinit/distros/nixos.py
 new file mode 100644
-index 00000000..d53d2a62
+index 00000000..954e564b
 --- /dev/null
 +++ b/cloudinit/distros/nixos.py
 @@ -0,0 +1,109 @@
diff --git a/pkgs/tools/virtualization/cloud-init/0002-Add-Udhcpc-support.patch b/pkgs/tools/virtualization/cloud-init/0002-Add-Udhcpc-support.patch
deleted file mode 100644
index 0df3f27a2c4..00000000000
--- a/pkgs/tools/virtualization/cloud-init/0002-Add-Udhcpc-support.patch
+++ /dev/null
@@ -1,421 +0,0 @@
-From 53260ce3bd70a0852d3e0d5569474214cea0ec0c Mon Sep 17 00:00:00 2001
-From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= <jfroche@pyxel.be>
-Date: Mon, 19 Jun 2023 15:56:46 +0200
-Subject: [PATCH] net/dhcp: add udhcpc support
-
-The currently used dhcp client, dhclient, is coming from the unmaintained package, isc-dhcp-client (refer https://www.isc.org/dhcp/) which ended support in 2022.
-
-This change introduce support for the dhcp client, udhcpc, from the busybox project. Busybox advantages are that it is available across many distributions and comes with lightweight executables.
----
- cloudinit/distros/__init__.py    |   8 +-
- cloudinit/net/dhcp.py            | 129 ++++++++++++++++++++++-
- tests/unittests/net/test_dhcp.py | 175 ++++++++++++++++++++++++++++++-
- tools/.github-cla-signers        |   1 +
- 4 files changed, 309 insertions(+), 4 deletions(-)
-
-diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
-index ec148939..0fab8945 100644
---- a/cloudinit/distros/__init__.py
-+++ b/cloudinit/distros/__init__.py
-@@ -110,14 +110,18 @@ class Distro(persistence.CloudInitPickleMixin, metaclass=abc.ABCMeta):
-     resolve_conf_fn = "/etc/resolv.conf"
- 
-     osfamily: str
--    dhcp_client_priority = [dhcp.IscDhclient, dhcp.Dhcpcd]
-+    dhcp_client_priority = [dhcp.IscDhclient, dhcp.Dhcpcd, dhcp.Udhcpc]
- 
-     def __init__(self, name, cfg, paths):
-         self._paths = paths
-         self._cfg = cfg
-         self.name = name
-         self.networking: Networking = self.networking_cls()
--        self.dhcp_client_priority = [dhcp.IscDhclient, dhcp.Dhcpcd]
-+        self.dhcp_client_priority = [
-+            dhcp.IscDhclient,
-+            dhcp.Dhcpcd,
-+            dhcp.Udhcpc,
-+        ]
- 
-     def _unpickle(self, ci_pkl_version: int) -> None:
-         """Perform deserialization fixes for Distro."""
-diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py
-index 6c8c2f54..f5586cea 100644
---- a/cloudinit/net/dhcp.py
-+++ b/cloudinit/net/dhcp.py
-@@ -21,6 +21,7 @@ from cloudinit import subp, temp_utils, util
- from cloudinit.net import (
-     find_fallback_nic,
-     get_devicelist,
-+    get_ib_interface_hwaddr,
-     get_interface_mac,
-     is_ib_interface,
- )
-@@ -28,6 +29,37 @@ from cloudinit.net import (
- LOG = logging.getLogger(__name__)
- 
- NETWORKD_LEASES_DIR = "/run/systemd/netif/leases"
-+UDHCPC_SCRIPT = """#!/bin/sh
-+log() {
-+    echo "udhcpc[$PPID]" "$interface: $2"
-+}
-+[ -z "$1" ] && echo "Error: should be called from udhcpc" && exit 1
-+case $1 in
-+    bound|renew)
-+    cat <<JSON > "$LEASE_FILE"
-+{
-+    "interface": "$interface",
-+    "fixed-address": "$ip",
-+    "subnet-mask": "$subnet",
-+    "routers": "${router%% *}",
-+    "static_routes" : "${staticroutes}"
-+}
-+JSON
-+    ;;
-+    deconfig)
-+    log err "Not supported"
-+    exit 1
-+    ;;
-+    leasefail | nak)
-+    log err "configuration failed: $1: $message"
-+    exit 1
-+    ;;
-+    *)
-+    echo "$0: Unknown udhcpc command: $1" >&2
-+    exit 1
-+    ;;
-+esac
-+"""
- 
- 
- class NoDHCPLeaseError(Exception):
-@@ -50,6 +82,10 @@ class NoDHCPLeaseMissingDhclientError(NoDHCPLeaseError):
-     """Raised when unable to find dhclient."""
- 
- 
-+class NoDHCPLeaseMissingUdhcpcError(NoDHCPLeaseError):
-+    """Raised when unable to find udhcpc client."""
-+
-+
- def select_dhcp_client(distro):
-     """distros set priority list, select based on this order which to use
- 
-@@ -60,7 +96,10 @@ def select_dhcp_client(distro):
-             dhcp_client = client()
-             LOG.debug("DHCP client selected: %s", client.client_name)
-             return dhcp_client
--        except NoDHCPLeaseMissingDhclientError:
-+        except (
-+            NoDHCPLeaseMissingDhclientError,
-+            NoDHCPLeaseMissingUdhcpcError,
-+        ):
-             LOG.warning("DHCP client not found: %s", client.client_name)
-     raise NoDHCPLeaseMissingDhclientError()
- 
-@@ -497,3 +536,91 @@ class Dhcpcd:
- 
-     def __init__(self):
-         raise NoDHCPLeaseMissingDhclientError("Dhcpcd not yet implemented")
-+
-+
-+class Udhcpc(DhcpClient):
-+    client_name = "udhcpc"
-+
-+    def __init__(self):
-+        self.udhcpc_path = subp.which("udhcpc")
-+        if not self.udhcpc_path:
-+            LOG.debug("Skip udhcpc configuration: No udhcpc command found.")
-+            raise NoDHCPLeaseMissingUdhcpcError()
-+
-+    def dhcp_discovery(
-+        self,
-+        interface,
-+        dhcp_log_func=None,
-+        distro=None,
-+    ):
-+        """Run udhcpc on the interface without scripts or filesystem artifacts.
-+
-+        @param interface: Name of the network interface on which to run udhcpc.
-+        @param dhcp_log_func: A callable accepting the udhcpc output and
-+            error streams.
-+
-+        @return: A list of dicts of representing the dhcp leases parsed from
-+            the udhcpc lease file.
-+        """
-+        LOG.debug("Performing a dhcp discovery on %s", interface)
-+
-+        tmp_dir = temp_utils.get_tmp_ancestor(needs_exe=True)
-+        lease_file = os.path.join(tmp_dir, interface + ".lease.json")
-+        with contextlib.suppress(FileNotFoundError):
-+            os.remove(lease_file)
-+
-+        # udhcpc needs the interface up to send initial discovery packets
-+        subp.subp(["ip", "link", "set", "dev", interface, "up"], capture=True)
-+
-+        udhcpc_script = os.path.join(tmp_dir, "udhcpc_script")
-+        util.write_file(udhcpc_script, UDHCPC_SCRIPT, 0o755)
-+
-+        cmd = [
-+            self.udhcpc_path,
-+            "-O",
-+            "staticroutes",
-+            "-i",
-+            interface,
-+            "-s",
-+            udhcpc_script,
-+            "-n",  # Exit if lease is not obtained
-+            "-q",  # Exit after obtaining lease
-+            "-f",  # Run in foreground
-+            "-v",
-+        ]
-+
-+        # For INFINIBAND port the dhcpc must be running with
-+        # client id option. So here we are checking if the interface is
-+        # INFINIBAND or not. If yes, we are generating the the client-id to be
-+        # used with the udhcpc
-+        if is_ib_interface(interface):
-+            dhcp_client_identifier = get_ib_interface_hwaddr(
-+                interface, ethernet_format=True
-+            )
-+            cmd.extend(
-+                ["-x", "0x3d:%s" % dhcp_client_identifier.replace(":", "")]
-+            )
-+        try:
-+            out, err = subp.subp(
-+                cmd, update_env={"LEASE_FILE": lease_file}, capture=True
-+            )
-+        except subp.ProcessExecutionError as error:
-+            LOG.debug(
-+                "udhcpc exited with code: %s stderr: %r stdout: %r",
-+                error.exit_code,
-+                error.stderr,
-+                error.stdout,
-+            )
-+            raise NoDHCPLeaseError from error
-+
-+        if dhcp_log_func is not None:
-+            dhcp_log_func(out, err)
-+
-+        lease_json = util.load_json(util.load_file(lease_file))
-+        static_routes = lease_json["static_routes"].split()
-+        if static_routes:
-+            # format: dest1/mask gw1 ... destn/mask gwn
-+            lease_json["static_routes"] = [
-+                i for i in zip(static_routes[::2], static_routes[1::2])
-+            ]
-+        return [lease_json]
-diff --git a/tests/unittests/net/test_dhcp.py b/tests/unittests/net/test_dhcp.py
-index 55d4c6e9..9123cd15 100644
---- a/tests/unittests/net/test_dhcp.py
-+++ b/tests/unittests/net/test_dhcp.py
-@@ -13,6 +13,8 @@ from cloudinit.net.dhcp import (
-     NoDHCPLeaseError,
-     NoDHCPLeaseInterfaceError,
-     NoDHCPLeaseMissingDhclientError,
-+    NoDHCPLeaseMissingUdhcpcError,
-+    Udhcpc,
-     maybe_perform_dhcp_discovery,
-     networkd_load_leases,
- )
-@@ -388,11 +390,13 @@ class TestDHCPDiscoveryClean(CiTestCase):
-             self.logs.getvalue(),
-         )
- 
-+    @mock.patch("cloudinit.temp_utils.get_tmp_ancestor", return_value="/tmp")
-     @mock.patch("cloudinit.net.dhcp.find_fallback_nic", return_value="eth9")
-     @mock.patch("cloudinit.net.dhcp.os.remove")
-     @mock.patch("cloudinit.net.dhcp.subp.subp")
-     @mock.patch("cloudinit.net.dhcp.subp.which")
--    def test_dhcp_client_failover(self, m_which, m_subp, m_remove, m_fallback):
-+    def test_dhcp_client_failover(self, m_which, m_subp, m_remove, m_fallback,
-+                                  m_get_tmp_ancestor):
-         """Log and do nothing when nic is absent and no fallback is found."""
-         m_subp.side_effect = [
-             ("", ""),
-@@ -928,3 +932,172 @@ class TestEphemeralDhcpLeaseErrors:
-                 pass
- 
-         assert len(m_dhcp.mock_calls) == 1
-+
-+
-+class TestUDHCPCDiscoveryClean(CiTestCase):
-+    with_logs = True
-+    maxDiff = None
-+
-+    @mock.patch("cloudinit.temp_utils.get_tmp_ancestor", return_value="/tmp")
-+    @mock.patch("cloudinit.net.dhcp.subp.which")
-+    @mock.patch("cloudinit.net.dhcp.find_fallback_nic")
-+    def test_absent_udhcpc_command(self, m_fallback, m_which,
-+                                   m_get_tmp_ancestor):
-+        """When dhclient doesn't exist in the OS, log the issue and no-op."""
-+        m_fallback.return_value = "eth9"
-+        m_which.return_value = None  # udhcpc isn't found
-+
-+        distro = MockDistro()
-+        distro.dhcp_client_priority = [Udhcpc]
-+
-+        with pytest.raises(NoDHCPLeaseMissingDhclientError):
-+            maybe_perform_dhcp_discovery(distro)
-+
-+        self.assertIn(
-+            "Skip udhcpc configuration: No udhcpc command found.",
-+            self.logs.getvalue(),
-+        )
-+
-+    @mock.patch("cloudinit.temp_utils.get_tmp_ancestor", return_value="/tmp")
-+    @mock.patch("cloudinit.net.dhcp.is_ib_interface", return_value=False)
-+    @mock.patch("cloudinit.net.dhcp.subp.which", return_value="/sbin/udhcpc")
-+    @mock.patch("cloudinit.net.dhcp.os.remove")
-+    @mock.patch("cloudinit.net.dhcp.subp.subp")
-+    @mock.patch("cloudinit.util.load_json")
-+    @mock.patch("cloudinit.util.load_file")
-+    @mock.patch("cloudinit.util.write_file")
-+    def test_udhcpc_discovery(
-+        self,
-+        m_write_file,
-+        m_load_file,
-+        m_loadjson,
-+        m_subp,
-+        m_remove,
-+        m_which,
-+        mocked_is_ib_interface,
-+        m_get_tmp_ancestor,
-+    ):
-+        """dhcp_discovery runs udcpc and parse the dhcp leases."""
-+        m_subp.return_value = ("", "")
-+        m_loadjson.return_value = {
-+            "interface": "eth9",
-+            "fixed-address": "192.168.2.74",
-+            "subnet-mask": "255.255.255.0",
-+            "routers": "192.168.2.1",
-+            "static_routes": "10.240.0.1/32 0.0.0.0 0.0.0.0/0 10.240.0.1",
-+        }
-+        self.assertEqual(
-+            [
-+                {
-+                    "fixed-address": "192.168.2.74",
-+                    "interface": "eth9",
-+                    "routers": "192.168.2.1",
-+                    "static_routes": [
-+                        ("10.240.0.1/32", "0.0.0.0"),
-+                        ("0.0.0.0/0", "10.240.0.1"),
-+                    ],
-+                    "subnet-mask": "255.255.255.0",
-+                }
-+            ],
-+            Udhcpc().dhcp_discovery("eth9", distro=MockDistro()),
-+        )
-+        # Interface was brought up before dhclient called
-+        m_subp.assert_has_calls(
-+            [
-+                mock.call(
-+                    ["ip", "link", "set", "dev", "eth9", "up"],
-+                    capture=True,
-+                ),
-+                mock.call(
-+                    [
-+                        "/sbin/udhcpc",
-+                        "-O",
-+                        "staticroutes",
-+                        "-i",
-+                        "eth9",
-+                        "-s",
-+                        "/tmp/udhcpc_script",
-+                        "-n",
-+                        "-q",
-+                        "-f",
-+                        "-v",
-+                    ],
-+                    update_env={"LEASE_FILE": "/tmp/eth9.lease.json"},
-+                    capture=True,
-+                ),
-+            ]
-+        )
-+
-+    @mock.patch("cloudinit.temp_utils.get_tmp_ancestor", return_value="/tmp")
-+    @mock.patch("cloudinit.net.dhcp.is_ib_interface", return_value=True)
-+    @mock.patch("cloudinit.net.dhcp.get_ib_interface_hwaddr")
-+    @mock.patch("cloudinit.net.dhcp.subp.which", return_value="/sbin/udhcpc")
-+    @mock.patch("cloudinit.net.dhcp.os.remove")
-+    @mock.patch("cloudinit.net.dhcp.subp.subp")
-+    @mock.patch("cloudinit.util.load_json")
-+    @mock.patch("cloudinit.util.load_file")
-+    @mock.patch("cloudinit.util.write_file")
-+    def test_udhcpc_discovery_ib(
-+        self,
-+        m_write_file,
-+        m_load_file,
-+        m_loadjson,
-+        m_subp,
-+        m_remove,
-+        m_which,
-+        m_get_ib_interface_hwaddr,
-+        m_is_ib_interface,
-+        m_get_tmp_ancestor,
-+    ):
-+        """dhcp_discovery runs udcpc and parse the dhcp leases."""
-+        m_subp.return_value = ("", "")
-+        m_loadjson.return_value = {
-+            "interface": "ib0",
-+            "fixed-address": "192.168.2.74",
-+            "subnet-mask": "255.255.255.0",
-+            "routers": "192.168.2.1",
-+            "static_routes": "10.240.0.1/32 0.0.0.0 0.0.0.0/0 10.240.0.1",
-+        }
-+        m_get_ib_interface_hwaddr.return_value = "00:21:28:00:01:cf:4b:01"
-+        self.assertEqual(
-+            [
-+                {
-+                    "fixed-address": "192.168.2.74",
-+                    "interface": "ib0",
-+                    "routers": "192.168.2.1",
-+                    "static_routes": [
-+                        ("10.240.0.1/32", "0.0.0.0"),
-+                        ("0.0.0.0/0", "10.240.0.1"),
-+                    ],
-+                    "subnet-mask": "255.255.255.0",
-+                }
-+            ],
-+            Udhcpc().dhcp_discovery("ib0", distro=MockDistro()),
-+        )
-+        # Interface was brought up before dhclient called
-+        m_subp.assert_has_calls(
-+            [
-+                mock.call(
-+                    ["ip", "link", "set", "dev", "ib0", "up"], capture=True
-+                ),
-+                mock.call(
-+                    [
-+                        "/sbin/udhcpc",
-+                        "-O",
-+                        "staticroutes",
-+                        "-i",
-+                        "ib0",
-+                        "-s",
-+                        "/tmp/udhcpc_script",
-+                        "-n",
-+                        "-q",
-+                        "-f",
-+                        "-v",
-+                        "-x",
-+                        "0x3d:0021280001cf4b01",
-+                    ],
-+                    update_env={"LEASE_FILE": "/tmp/ib0.lease.json"},
-+                    capture=True,
-+                ),
-+            ]
-+        )
-diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers
-index b4a9326e..4d82a055 100644
---- a/tools/.github-cla-signers
-+++ b/tools/.github-cla-signers
-@@ -65,6 +65,7 @@ jacobsalmela
- jamesottinger
- Jehops
- jf
-+jfroche
- Jille
- JohnKepplers
- johnsonshi
--- 
-2.40.1
-
diff --git a/pkgs/tools/virtualization/cloud-init/default.nix b/pkgs/tools/virtualization/cloud-init/default.nix
index 5b85bae033a..edf456d3094 100644
--- a/pkgs/tools/virtualization/cloud-init/default.nix
+++ b/pkgs/tools/virtualization/cloud-init/default.nix
@@ -12,24 +12,23 @@
 , coreutils
 , gitUpdater
 , busybox
+, procps
 }:
 
 python3.pkgs.buildPythonApplication rec {
   pname = "cloud-init";
-  version = "23.2.2";
+  version = "23.3.1";
   namePrefix = "";
 
   src = fetchFromGitHub {
     owner = "canonical";
     repo = "cloud-init";
     rev = "refs/tags/${version}";
-    hash = "sha256-lOeLVgT/qTB6JhRcLv9QIfNLMnMyNlUp3dMCqva9Tes=";
+    hash = "sha256-3UxTqlhLZi/3/buWqDGto4cZN03uONbA8HEWQtaIRxU=";
   };
 
   patches = [
     ./0001-add-nixos-support.patch
-    # upstream: https://github.com/canonical/cloud-init/pull/4190
-    ./0002-Add-Udhcpc-support.patch
   ];
 
   prePatch = ''
@@ -71,10 +70,12 @@ python3.pkgs.buildPythonApplication rec {
     httpretty
     dmidecode
     # needed for tests; at runtime we rather want the setuid wrapper
+    passlib
     shadow
     responses
     pytest-mock
     coreutils
+    procps
   ];
 
   makeWrapperArgs = [
@@ -84,8 +85,11 @@ python3.pkgs.buildPythonApplication rec {
   disabledTests = [
     # tries to create /var
     "test_dhclient_run_with_tmpdir"
+    "test_dhcp_client_failover"
     # clears path and fails because mkdir is not found
     "test_path_env_gets_set_from_main"
+    # fails to find cat
+    "test_subp_combined_stderr_stdout"
     # tries to read from /etc/ca-certificates.conf while inside the sandbox
     "test_handler_ca_certs"
     "TestRemoveDefaultCaCerts"