Deal with test_cpio.py's test_xz failure on python3

The following assertion boiled down to a difference in the defaul CRC
algorithm between pylzma on python2 and the now-standard python3 lzma
module... but brought many improvements along the way.

  tests/test_cpio.py:89: in archiveCreate
      self.assertEqual(f.read(2), b'\x00\x01')
  E   AssertionError: b'\x00\x04' != b'\x00\x01'

Author: ydirson

tests/test_cpio.py

@@ -14,6 +14,7 @@
 import subprocess, shutil
 
 def writeRandomFile(fn, size, start=b'', add=b'a'):
+    "Create a pseudo-random reproducible file from seeds `start` amd `add`"
     with open(fn, 'wb') as f:
         m = md5()
         m.update(start)
@@ -43,7 +44,12 @@ def setUp(self):
         writeRandomFile('archive/data', 10491)
         with open('archive/data', 'rb') as fd:
             self.md5data = md5(fd.read()).hexdigest()
-        check_call("find archive | cpio -o -H newc > archive.cpio")
+        # fixed timestamps for cpio reproducibility
+        os.utime('archive/data', (0, 0))
+        os.utime('archive', (0, 0))
+
+        check_call(
+            "find archive | cpio --reproducible -o -H newc > archive.cpio")
         check_call("gzip -c < archive.cpio > archive.cpio.gz")
         check_call("bzip2 -c < archive.cpio > archive.cpio.bz2")
         try:
@@ -55,7 +61,7 @@ def setUp(self):
             self.doXZ = False
 
     def tearDown(self):
-        check_call("rm -rf archive archive.cpio*")
+        check_call("rm -rf archive archive.cpio* archive2.cpio*")
 
     # TODO check with file (like 'r:*')
     # TODO use cat to check properly for pipes
@@ -71,7 +77,8 @@ def archiveExtract(self, fn, fmt='r|*'):
         self.assertTrue(found)
 
     def archiveCreate(self, fn, fmt='w'):
-        os.unlink(fn)
+        if os.path.exists(fn):
+            os.unlink(fn)
         arc = CpioFile.open(fn, fmt)
         f = arc.getcpioinfo('archive/data')
         arc.addfile(f, open('archive/data', 'rb'))
@@ -80,15 +87,23 @@ def archiveCreate(self, fn, fmt='w'):
         # special case for XZ, test check type (crc32)
         if fmt.endswith('xz'):
             with open(fn, 'rb') as f:
-                f.seek(6)
-                self.assertEqual(f.read(2), b'\x00\x01')
+                # check xz magic
+                self.assertEqual(f.read(6), b"\xfd7zXZ\0")
+                # check stream flags
+                if sys.version_info < (3, 0):
+                    expected_flags = b'\x00\x01' # pylzma defaults to CRC32
+                else:
+                    expected_flags = b'\x00\x04' # python3 defaults to CRC64
+                self.assertEqual(f.read(2), expected_flags)
         self.archiveExtract(fn)
 
     def doArchive(self, fn, fmt=None):
         self.archiveExtract(fn)
-        self.archiveCreate(fn, fmt is None and 'w' or 'w|%s' % fmt)
-        if not fmt is None:
-            self.archiveExtract(fn, 'r|%s' % fmt)
+        fn2 = "archive2" + fn[len("archive"):]
+        print("creating %s" % fn2)
+        self.archiveCreate(fn2, fmt is None and 'w' or 'w|%s' % fmt)
+        if fmt is not None:
+            self.archiveExtract(fn2, 'r|%s' % fmt)
 
     def test_plain(self):
         self.doArchive('archive.cpio')
@@ -101,7 +116,7 @@ def test_bz2(self):
 
     def test_xz(self):
         if not self.doXZ:
-            return
+            raise unittest.SkipTest("lzma package or xz tool not available")
         print('Running test for XZ')
         self.doArchive('archive.cpio.xz', 'xz')
 

xcp/cpiofile.py

@@ -478,7 +478,7 @@ def getcomptype(self):
             return "gz"
         if self.buf.startswith(b"BZh91"):
             return "bz2"
-        if self.buf.startswith(b"\xfd7zXZ"):
+        if self.buf.startswith(b"\xfd7zXZ\0"):
             return "xz"
         return "cpio"