More hashlib robustness in strange "FIPS mode" environments.

Author: gpshead

Doc/library/hashlib.rst

@@ -55,11 +55,14 @@ hash supplied more than 2047 bytes of data at once in its constructor or
 .. index:: single: OpenSSL; (use in module hashlib)
 
 Constructors for hash algorithms that are always present in this module are
-:func:`sha1`, :func:`sha224`, :func:`sha256`, :func:`sha384`, :func:`sha512`,
+:func:`md5`, :func:`sha1`, :func:`sha224`, :func:`sha256`, :func:`sha384`, :func:`sha512`,
 :func:`sha3_224`, :func:`sha3_256`, :func:`sha3_384`, :func:`sha3_512`,
 :func:`shake_128`, :func:`shake_256`, :func:`blake2b`, and :func:`blake2s`.
-:func:`md5` is normally available as well, though it may be missing or blocked
-if you are using a rare "FIPS compliant" build of Python.
+Some of these may be missing or blocked if you are running in an environment
+with OpenSSL's "FIPS mode" configured to exclude some hash algorithms from its
+default provider and are using a Python runtime built with that in mind.  Such
+environments are unusual.
+
 These correspond to :data:`algorithms_guaranteed`.
 
 Additional algorithms may also be available if your Python distribution's
@@ -119,7 +122,7 @@ More condensed:
 Constructors
 ------------
 
-.. function:: new(name[, data], *, usedforsecurity=True)
+.. function:: new(name[, data], \*, usedforsecurity=True)
 
    Is a generic constructor that takes the string *name* of the desired
    algorithm as its first parameter.  It also exists to allow access to the
@@ -134,16 +137,16 @@ Using :func:`new` with an algorithm name:
    '031edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9406'
 
 
-.. function:: md5([, data], *, usedforsecurity=True)
-.. function:: sha1([, data], *, usedforsecurity=True)
-.. function:: sha224([, data], *, usedforsecurity=True)
-.. function:: sha256([, data], *, usedforsecurity=True)
-.. function:: sha384([, data], *, usedforsecurity=True)
-.. function:: sha512([, data], *, usedforsecurity=True)
-.. function:: sha3_224([, data], *, usedforsecurity=True)
-.. function:: sha3_256([, data], *, usedforsecurity=True)
-.. function:: sha3_384([, data], *, usedforsecurity=True)
-.. function:: sha3_512([, data], *, usedforsecurity=True)
+.. function:: md5([, data], \*, usedforsecurity=True)
+.. function:: sha1([, data], \*, usedforsecurity=True)
+.. function:: sha224([, data], \*, usedforsecurity=True)
+.. function:: sha256([, data], \*, usedforsecurity=True)
+.. function:: sha384([, data], \*, usedforsecurity=True)
+.. function:: sha512([, data], \*, usedforsecurity=True)
+.. function:: sha3_224([, data], \*, usedforsecurity=True)
+.. function:: sha3_256([, data], \*, usedforsecurity=True)
+.. function:: sha3_384([, data], \*, usedforsecurity=True)
+.. function:: sha3_512([, data], \*, usedforsecurity=True)
 
 Named constructors such as these are faster than passing an algorithm name to
 :func:`new`.
@@ -156,9 +159,10 @@ Hashlib provides the following constant module attributes:
 .. data:: algorithms_guaranteed
 
    A set containing the names of the hash algorithms guaranteed to be supported
-   by this module on all platforms.  Note that 'md5' is in this list despite
-   some upstream vendors offering an odd "FIPS compliant" Python build that
-   excludes it.
+   by this module on all platforms.  Note that the guarnatees do not hold true
+   in the face of vendors offering "FIPS compliant" Python builds that exclude
+   some algorithms entirely.  Similarly when OpenSSL is used and its FIPS mode
+   configuration disables some in the default provider.
 
    .. versionadded:: 3.2
 

Lib/hashlib.py

@@ -55,17 +55,18 @@
 
 # This tuple and __get_builtin_constructor() must be modified if a new
 # always available algorithm is added.
-__always_supported = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512',
-                      'blake2b', 'blake2s',
-                      'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512',
-                      'shake_128', 'shake_256')
+__always_supported = [
+    'md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512',
+    'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512',
+    'shake_128', 'shake_256', 'blake2b', 'blake2s'
+]
 
 
 algorithms_guaranteed = set(__always_supported)
 algorithms_available = set(__always_supported)
 
-__all__ = __always_supported + ('new', 'algorithms_guaranteed',
-                                'algorithms_available', 'file_digest')
+__all__ = __always_supported + [
+    'new', 'algorithms_guaranteed', 'algorithms_available', 'file_digest']
 
 
 __builtin_constructor_cache = {}
@@ -243,9 +244,11 @@ def file_digest(fileobj, digest, /, *, _bufsize=2**18):
     # version not supporting that algorithm.
     try:
         globals()[__func_name] = __get_hash(__func_name)
-    except ValueError:
-        import logging
-        logging.exception('code for hash %s was not found.', __func_name)
+    except ValueError as exc:
+        # Errors logged here would be seen as noise by most people.
+        # Code using a missing hash will get an obvious exception.
+        __all__.remove(__func_name)
+        algorithms_available.remove(__func_name)
 
 
 # Cleanup locals()

Lib/test/test_hashlib.py

@@ -143,7 +143,15 @@ def __init__(self, *args, **kwargs):
         # For each algorithm, test the direct constructor and the use
         # of hashlib.new given the algorithm name.
         for algorithm, constructors in self.constructors_to_test.items():
-            constructors.add(getattr(hashlib, algorithm))
+            if get_fips_mode():
+                # Arbitrary algorithms may be missing via openssl.cnf
+                try:
+                    constructor = getattr(hashlib, algorithm)
+                except AttributeError:
+                    continue
+                constructors.add(constructor)
+            else:
+                constructors.add(getattr(hashlib, algorithm))
             def _test_algorithm_via_hashlib_new(data=None, _alg=algorithm, **kwargs):
                 if data is None:
                     return hashlib.new(_alg, **kwargs)
@@ -219,15 +227,23 @@ def test_algorithms_guaranteed(self):
             set(_algo for _algo in self.supported_hash_names
                   if _algo.islower()))
 
+    @unittest.skipIf(get_fips_mode(), reason="guaranteed algorithms may not be available in FIPS mode")
     def test_algorithms_available(self):
+        print(f"{get_fips_mode()=}")
         self.assertTrue(set(hashlib.algorithms_guaranteed).
-                            issubset(hashlib.algorithms_available))
+                            issubset(hashlib.algorithms_available),
+                        msg=f"\n{sorted(hashlib.algorithms_guaranteed)=}\n{sorted(hashlib.algorithms_available)=}")
         # all available algorithms must be loadable, bpo-47101
         self.assertNotIn("undefined", hashlib.algorithms_available)
         for name in hashlib.algorithms_available:
-            digest = hashlib.new(name, usedforsecurity=False)
+            with self.subTest(name=name):
+                if name in self.blakes and not _blake2:
+                    self.skipTest("requires _blake2")
+                hashlib.new(name, usedforsecurity=False)
 
     @requires_usedforsecurity
+    @unittest.skipUnless(hasattr(hashlib, "sha256"), "sha256 unavailable")
+    @unittest.skipUnless(hasattr(hashlib, "md5"), "md5 unavailable")
     def test_usedforsecurity_true(self):
         hashlib.new("sha256", usedforsecurity=True)
         for cons in self.hash_constructors:
@@ -239,6 +255,8 @@ def test_usedforsecurity_true(self):
             self._hashlib.new("md5", usedforsecurity=True)
             self._hashlib.openssl_md5(usedforsecurity=True)
 
+    @unittest.skipUnless(hasattr(hashlib, "sha256"), "sha256 unavailable")
+    @unittest.skipUnless(hasattr(hashlib, "md5"), "md5 unavailable")
     def test_usedforsecurity_false(self):
         hashlib.new("sha256", usedforsecurity=False)
         for cons in self.hash_constructors:
@@ -254,6 +272,7 @@ def test_unknown_hash(self):
         self.assertRaises(ValueError, hashlib.new, 'spam spam spam spam spam')
         self.assertRaises(TypeError, hashlib.new, 1)
 
+    @unittest.skipUnless(hasattr(hashlib, "sha256"), "sha256 unavailable")
     def test_new_upper_to_lower(self):
         self.assertEqual(hashlib.new("SHA256", usedforsecurity=False).name, "sha256")
 
@@ -387,7 +406,8 @@ def check(self, name, data, hexdigest, shake=False, **kwargs):
         hexdigest = hexdigest.lower()
         constructors = self.constructors_to_test[name]
         # 2 is for hashlib.name(...) and hashlib.new(name, ...)
-        self.assertGreaterEqual(len(constructors), 2)
+        if get_fips_mode() == 0:
+            self.assertGreaterEqual(len(constructors), 2)
         for hash_object_constructor in constructors:
             m = hash_object_constructor(data, **kwargs)
             computed = m.hexdigest() if not shake else m.hexdigest(length)
@@ -407,8 +427,6 @@ def check(self, name, data, hexdigest, shake=False, **kwargs):
             # skip shake and blake2 extended parameter tests
             self.check_file_digest(name, data, hexdigest)
 
-    # defaults True because file_digest doesn't support the parameter.
-    @requires_usedforsecurity
     def check_file_digest(self, name, data, hexdigest):
         hexdigest = hexdigest.lower()
         try:
@@ -914,6 +932,7 @@ def test_case_shake256_vector(self):
         for msg, md in read_vectors('shake_256'):
             self.check('shake_256', msg, md, True)
 
+    @unittest.skipUnless(hasattr(hashlib, "sha256"), "sha256 unavailable")
     def test_gil(self):
         # Check things work fine with an input larger than the size required
         # for multithreaded operation (which is hardwired to 2048).
@@ -928,7 +947,7 @@ def test_gil(self):
             m = cons(b'x' * gil_minsize, usedforsecurity=False)
             m.update(b'1')
 
-        m = hashlib.sha256()
+        m = hashlib.sha256(usedforsecurity=False)
         m.update(b'1')
         m.update(b'#' * gil_minsize)
         m.update(b'1')
@@ -937,22 +956,24 @@ def test_gil(self):
             '1cfceca95989f51f658e3f3ffe7f1cd43726c9e088c13ee10b46f57cef135b94'
         )
 
-        m = hashlib.sha256(b'1' + b'#' * gil_minsize + b'1')
+        m = hashlib.sha256(b'1' + b'#' * gil_minsize + b'1',
+                           usedforsecurity=False)
         self.assertEqual(
             m.hexdigest(),
             '1cfceca95989f51f658e3f3ffe7f1cd43726c9e088c13ee10b46f57cef135b94'
         )
 
     @threading_helper.reap_threads
     @threading_helper.requires_working_threading()
+    @unittest.skipUnless(hasattr(hashlib, "sha1"), "sha1 unavailable")
     def test_threaded_hashing(self):
         # Updating the same hash object from several threads at once
         # using data chunk sizes containing the same byte sequences.
         #
         # If the internal locks are working to prevent multiple
         # updates on the same object from running at once, the resulting
         # hash will be the same as doing it single threaded upfront.
-        hasher = hashlib.sha1()
+        hasher = hashlib.sha1(usedforsecurity=False)
         num_threads = 5
         smallest_data = b'swineflu'
         data = smallest_data * 200000
@@ -1174,6 +1195,9 @@ def test_normalized_name(self):
         self.assertNotIn("blake2b512", hashlib.algorithms_available)
         self.assertNotIn("sha3-512", hashlib.algorithms_available)
 
+    # defaults True because file_digest doesn't support the parameter.
+    @requires_usedforsecurity
+    @unittest.skipUnless(hasattr(hashlib, "sha256"), "sha256 unavailable")
     def test_file_digest(self):
         data = b'a' * 65536
         d1 = hashlib.sha256()