Multipath Hardware path handling

Removes multipath base devices from consideration by
default, and instead allows the device-mapper device
managed by multipath to be picked up and utilized
instead.

In effect, allowing us to ignore standby paths *and*
leverage multiple concurrent IO paths if so offered
via ALUA.

In reality, anyone who has previously built IPA with
multipath tooling might not have encountered issues
previously because they used Active/Active SAN storage
environments. They would have worked because the IO lock
would have been exchanged between controllers and paths.
However, Active/Passive environments will block passive
paths from access, ultimately preventing new locks from
being established without proper negotiation. Ultimately
requiring multipathing *and* the agent to be smart enough
to know to disqualify underlying paths to backend storage
volumes.

An additional benefit of this is active/active MPIO devices
will, as long as ``multipath`` is present inside the ramdisk,
no longer possibly result in duplicate IO wipes occuring
accross numerous devices.

Story: #2010003
Task: #45108
Resolves: rhbz#2076622
Resolves: rhbz#2070519
Change-Id: I0fd6356f036d5ff17510fb838eaf418164cdfc92

Author: juliakreger

ironic_python_agent/hardware.py

@@ -80,6 +80,8 @@
     }
 }
 
+MULTIPATH_ENABLED = None
+
 
 def _get_device_info(dev, devclass, field):
     """Get the device info according to device class and field."""
@@ -126,6 +128,35 @@ def _udev_settle():
         return
 
 
+def _load_multipath_modules():
+    """Load multipath modules
+
+    This is required to be able to collect multipath information.
+    Two separate paths exist, one with a helper utility for Centos/RHEL
+    and another which is just load the modules, and trust multipathd
+    will do the needful.
+    """
+    if (os.path.isfile('/usr/sbin/mpathconf')
+        and not os.path.isfile('/etc/multipath.conf')):
+        # For Centos/Rhel/Etc which uses mpathconf, this does
+        # a couple different things, including configuration generation...
+        # which is not *really* required.. at least *shouldn't* be.
+        # WARNING(TheJulia): This command explicitly replaces local
+        # configuration.
+        il_utils.try_execute('/usr/sbin/mpathconf', '--enable',
+                             '--find_multipaths', 'yes',
+                             '--with_module', 'y',
+                             '--with_multipathd', 'y')
+    else:
+        # Ensure modules are loaded. Configuration is not required
+        # and implied based upon compiled in defaults.
+        # NOTE(TheJulia): Debian/Ubuntu specifically just document
+        # using `multipath -t` output to start a new configuration
+        # file, if needed.
+        il_utils.try_execute('modprobe', 'dm_multipath')
+        il_utils.try_execute('modprobe', 'multipath')
+
+
 def _check_for_iscsi():
     """Connect iSCSI shared connected via iBFT or OF.
 
@@ -169,6 +200,84 @@ def _get_md_uuid(raid_device):
             return match.group(1)
 
 
+def _enable_multipath():
+    """Initialize multipath IO if possible.
+
+    :returns: True if the multipathd daemon and multipath command to enumerate
+              devices was scucessfully able to be called.
+    """
+    try:
+        _load_multipath_modules()
+        # This might not work, ideally it *should* already be running...
+        # NOTE(TheJulia): Testing locally, a prior running multipathd, the
+        # explicit multipathd start just appears to silently exit with a
+        # result code of 0.
+        utils.execute('multipathd')
+        # This is mainly to get the system to actually do the needful and
+        # identify/enumerate paths by combining what it can detect and what
+        # it already knows. This may be useful, and in theory this should be
+        # logged in the IPA log should it be needed.
+        utils.execute('multipath', '-ll')
+        return True
+    except FileNotFoundError as e:
+        LOG.warning('Attempted to determine if multipath tools were present. '
+                    'Not detected. Error recorded: %s', e)
+        return False
+    except processutils.ProcessExecutionError as e:
+        LOG.warning('Attempted to invoke multipath utilities, but we '
+                    'encountered an error: %s', e)
+        return False
+
+
+def _get_multipath_parent_device(device):
+    """Check and return a multipath device."""
+    if not device:
+        # if lsblk provides invalid output, this can be None.
+        return
+    check_device = os.path.join('/dev', str(device))
+    try:
+        # Explicitly run the check as regardless of if the device is mpath or
+        # not, multipath tools when using list always exits with a return
+        # code of 0.
+        utils.execute('multipath', '-c', check_device)
+        # path check with return an exit code of 1 if you send it a multipath
+        # device mapper device, like dm-0.
+        # NOTE(TheJulia): -ll is supposed to load from all available
+        # information, but may not force a rescan. It may be -f if we need
+        # that. That being said, it has been about a decade since I was
+        # running multipath tools on SAN connected gear, so my memory is
+        # definitely fuzzy.
+        out, _ = utils.execute('multipath', '-ll', check_device)
+    except processutils.ProcessExecutionError as e:
+        # FileNotFoundError if the utility does not exist.
+        # -1 return code if the device is not valid.
+        LOG.debug('Checked device %(dev)s and determined it was '
+                  'not a multipath device. %(error)s',
+                  {'dev': check_device,
+                   'error': e})
+        return
+    except FileNotFoundError:
+        # This should never happen, as MULTIPATH_ENABLED would be False
+        # before this occurs.
+        LOG.warning('Attempted to check multipathing status, however '
+                    'the \'multipath\' binary is missing or not in the '
+                    'execution PATH.')
+        return
+    # Data format:
+    # MPATHDEVICENAME dm-0 TYPE,HUMANNAME
+    # size=56G features='1 retain_attached_hw_handler' hwhandler='0' wp=rw
+    # `-+- policy='service-time 0' prio=1 status=active
+    #   `- 0:0:0:0 sda 8:0  active ready running
+    try:
+        lines = out.splitlines()
+        mpath_device = lines[0].split(' ')[1]
+        # give back something like dm-0 so we can log it.
+        return mpath_device
+    except IndexError:
+        # We didn't get any command output, so Nope.
+        pass
+
+
 def _get_component_devices(raid_device):
     """Get the component devices of a Software RAID device.
 
@@ -359,7 +468,8 @@ def _md_scan_and_assemble():
 def list_all_block_devices(block_type='disk',
                            ignore_raid=False,
                            ignore_floppy=True,
-                           ignore_empty=True):
+                           ignore_empty=True,
+                           ignore_multipath=False):
     """List all physical block devices
 
     The switches we use for lsblk: P for KEY="value" output, b for size output
@@ -376,6 +486,9 @@ def list_all_block_devices(block_type='disk',
     :param ignore_floppy: Ignore floppy disk devices in the block device
                           list. By default, these devices are filtered out.
     :param ignore_empty: Whether to ignore disks with size equal 0.
+    :param ignore_multipath: Whether to ignore devices backing multipath
+                             devices. Default is to consider multipath
+                             devices, if possible.
     :return: A list of BlockDevices
     """
 
@@ -386,6 +499,8 @@ def _is_known_device(existing, new_device_name):
                 return True
         return False
 
+    check_multipath = not ignore_multipath and get_multipath_status()
+
     _udev_settle()
 
     # map device names to /dev/disk/by-path symbolic links that points to it
@@ -416,7 +531,6 @@ def _is_known_device(existing, new_device_name):
                            '-o{}'.format(','.join(columns)))[0]
     lines = report.splitlines()
     context = pyudev.Context()
-
     devices = []
     for line in lines:
         device = {}
@@ -438,16 +552,31 @@ def _is_known_device(existing, new_device_name):
             LOG.debug('Ignoring floppy disk device %s', device)
             continue
 
+        dev_kname = device.get('KNAME')
+        if check_multipath:
+            # Net effect is we ignore base devices, and their base devices
+            # to what would be the mapped device name which would not pass the
+            # validation, but would otherwise be match-able.
+            mpath_parent_dev = _get_multipath_parent_device(dev_kname)
+            if mpath_parent_dev:
+                LOG.warning(
+                    "We have identified a multipath device %(device)s, this "
+                    "is being ignored in favor of %(mpath_device)s and its "
+                    "related child devices.",
+                    {'device': dev_kname,
+                     'mpath_device': mpath_parent_dev})
+                continue
         # Search for raid in the reply type, as RAID is a
         # disk device, and we should honor it if is present.
         # Other possible type values, which we skip recording:
         #   lvm, part, rom, loop
+
         if devtype != block_type:
             if devtype is None or ignore_raid:
                 LOG.debug("Skipping: {!r}".format(line))
                 continue
             elif ('raid' in devtype
-                  and block_type in ['raid', 'disk']):
+                  and block_type in ['raid', 'disk', 'mpath']):
                 LOG.debug(
                     "TYPE detected to contain 'raid', signifying a "
                     "RAID volume. Found: {!r}".format(line))
@@ -461,6 +590,11 @@ def _is_known_device(existing, new_device_name):
                 LOG.debug(
                     "TYPE detected to contain 'md', signifying a "
                     "RAID partition. Found: {!r}".format(line))
+            elif devtype == 'mpath' and block_type == 'disk':
+                LOG.debug(
+                    "TYPE detected to contain 'mpath', "
+                    "signifing a device mapper multipath device. "
+                    "Found: %s", line)
             else:
                 LOG.debug(
                     "TYPE did not match. Wanted: {!r} but found: {!r}".format(
@@ -974,6 +1108,10 @@ def evaluate_hardware_support(self):
         # Do some initialization before we declare ourself ready
         _check_for_iscsi()
         _md_scan_and_assemble()
+        global MULTIPATH_ENABLED
+        if MULTIPATH_ENABLED is None:
+            MULTIPATH_ENABLED = _enable_multipath()
+
         self.wait_for_disks()
         return HardwareSupport.GENERIC
 
@@ -2609,3 +2747,11 @@ def deduplicate_steps(candidate_steps):
         deduped_steps[manager].append(winning_step)
 
     return deduped_steps
+
+
+def get_multipath_status():
+    """Return the status of multipath initialization."""
+    # NOTE(TheJulia): Provides a nice place to mock out and simplify testing
+    # as if we directly try and work with the global var, we will be racing
+    # tests endlessly.
+    return MULTIPATH_ENABLED

ironic_python_agent/tests/unit/extensions/test_image.py

@@ -787,6 +787,7 @@ def test__install_grub2_uefi(self, mock_get_part_uuid, mkdir_mock,
         mock_append_to_fstab.assert_called_with(self.fake_dir,
                                                 self.fake_efi_system_part_uuid)
 
+    @mock.patch.object(hardware, 'get_multipath_status', lambda *_: False)
     @mock.patch.object(os.path, 'ismount', lambda *_: False)
     @mock.patch.object(image, '_is_bootloader_loaded', lambda *_: True)
     @mock.patch.object(os.path, 'exists', autospec=True)
@@ -898,6 +899,7 @@ def test__install_grub2_uefi_fstab(self, mock_get_part_uuid, mkdir_mock,
                                            uuid=self.fake_efi_system_part_uuid)
         self.assertFalse(mock_dispatch.called)
 
+    @mock.patch.object(hardware, 'get_multipath_status', lambda *_: False)
     @mock.patch.object(image, '_efi_boot_setup', lambda *_: False)
     @mock.patch.object(os.path, 'ismount', lambda *_: False)
     @mock.patch.object(image, '_is_bootloader_loaded', lambda *_: True)

ironic_python_agent/tests/unit/samples/hardware_samples.py

@@ -121,7 +121,10 @@
     'KNAME="fd1" MODEL="magic" SIZE="4096" ROTA="1" TYPE="disk" UUID="" '
     'PARTUUID=""\n'
     'KNAME="sdf" MODEL="virtual floppy" SIZE="0" ROTA="1" TYPE="disk" UUID="" '
-    'PARTUUID=""'
+    'PARTUUID=""\n'
+    'KNAME="dm-0" MODEL="NWD-BLP4-1600   " SIZE="1765517033472" '
+    ' ROTA="0" TYPE="mpath" UUID="" PARTUUID=""\n'
+
 )
 
 # NOTE(pas-ha) largest device is 1 byte smaller than 4GiB
@@ -160,6 +163,49 @@
     'PARTUUID=""'
 )
 
+MULTIPATH_BLK_DEVICE_TEMPLATE = (
+    'KNAME="sda" MODEL="INTEL_SSDSC2CT060A3" SIZE="60022480896" ROTA="0" '
+    'TYPE="disk" UUID="" PARTUUID=""\n'
+    'KNAME="sda2" MODEL="" SIZE="59162722304" ROTA="0" TYPE="part" '
+    'UUID="f8b55d59-96c3-3982-b129-1b6b2ee8da86" '
+    'PARTUUID="c97c8aac-7796-4433-b1fc-9b5fac43edf3"\n'
+    'KNAME="sda3" MODEL="" SIZE="650002432" ROTA="0" TYPE="part" '
+    'UUID="b3b03565-5f13-3c93-b2a6-6d90e25be926" '
+    'PARTUUID="6c85beff-b2bd-4a1c-91b7-8abb5256459d"\n'
+    'KNAME="sda1" MODEL="" SIZE="209715200" ROTA="0" TYPE="part" '
+    'UUID="0a83355d-7500-3f5f-9abd-66f6fd03714c" '
+    'PARTUUID="eba28b26-b76a-402c-94dd-0b66a523a485"\n'
+    'KNAME="dm-0" MODEL="" SIZE="60022480896" ROTA="0" TYPE="mpath" '
+    'UUID="" PARTUUID=""\n'
+    'KNAME="dm-4" MODEL="" SIZE="650002432" ROTA="0" TYPE="part" '
+    'UUID="b3b03565-5f13-3c93-b2a6-6d90e25be926" '
+    'PARTUUID="6c85beff-b2bd-4a1c-91b7-8abb5256459d"\n'
+    'KNAME="dm-2" MODEL="" SIZE="209715200" ROTA="0" TYPE="part" '
+    'UUID="0a83355d-7500-3f5f-9abd-66f6fd03714c" '
+    'PARTUUID="eba28b26-b76a-402c-94dd-0b66a523a485"\n'
+    'KNAME="dm-3" MODEL="" SIZE="59162722304" ROTA="0" TYPE="part" '
+    'UUID="f8b55d59-96c3-3982-b129-1b6b2ee8da86" '
+    'PARTUUID="c97c8aac-7796-4433-b1fc-9b5fac43edf3"\n'
+    'KNAME="sdb" MODEL="INTEL_SSDSC2CT060A3" SIZE="60022480896" '
+    'ROTA="0" TYPE="disk" UUID="" PARTUUID=""\n'
+    'KNAME="sdb2" MODEL="" SIZE="59162722304" ROTA="0" TYPE="part" '
+    'UUID="f8b55d59-96c3-3982-b129-1b6b2ee8da86" '
+    'PARTUUID="c97c8aac-7796-4433-b1fc-9b5fac43edf3"\n'
+    'KNAME="sdb3" MODEL="" SIZE="650002432" ROTA="0" TYPE="part" '
+    'UUID="b3b03565-5f13-3c93-b2a6-6d90e25be926" '
+    'PARTUUID="6c85beff-b2bd-4a1c-91b7-8abb5256459d"\n'
+    'KNAME="sdb1" MODEL="" SIZE="209715200" ROTA="0" TYPE="part" '
+    'UUID="0a83355d-7500-3f5f-9abd-66f6fd03714c" '
+    'PARTUUID="eba28b26-b76a-402c-94dd-0b66a523a485"\n'
+    'KNAME="sdc" MODEL="ST1000DM003-1CH162" SIZE="1000204886016" '
+    'ROTA="1" TYPE="disk" UUID="" PARTUUID=""\n'
+    'KNAME="sdc1" MODEL="" SIZE="899999072256" ROTA="1" TYPE="part" '
+    'UUID="457f7d3c-9376-4997-89bd-d1a7c8b04060" '
+    'PARTUUID="c9433d2e-3bbc-47b4-92bf-43c1d80f06e0"\n'
+    'KNAME="dm-1" MODEL="" SIZE="1000204886016" ROTA="0" TYPE="mpath" '
+    'UUID="" PARTUUID=""\n'
+)
+
 PARTUUID_DEVICE_TEMPLATE = (
     'KNAME="sda" MODEL="DRIVE 0" SIZE="1765517033472" '
     'ROTA="1" TYPE="disk" UUID="" PARTUUID=""\n'
@@ -1501,7 +1547,6 @@
 }
 """)
 
-
 SGDISK_INFO_TEMPLATE = ("""
 Partition GUID code: C12A7328-F81F-11D2-BA4B-00A0C93EC93B (EFI system partition)
 Partition unique GUID: FAED7408-6D92-4FC6-883B-9069E2274ECA
@@ -1511,3 +1556,13 @@
 Attribute flags: 0000000000000000
 Partition name: 'EFI System Partition'
 """)  # noqa
+
+MULTIPATH_VALID_PATH = '%s is a valid multipath device path'
+MULTIPATH_INVALID_PATH = '%s is not a valid multipath device path'
+
+MULTIPATH_LINKS_DM = (
+    'SUPER_FRIENDLY_NAME %s ATA,INTEL SSDSC2CT06\n'
+    'size=56G features=\'1 retain_attached_hw_handler\' hwhandler=\'0\' wp=rw\n'  # noqa
+    ' `-+- policy=\'service-time 0\' prio=1 status=active\n'
+    '  `- 0:0:0:0 device s  8:0  active ready running\n'
+)