From 200bac07d4c7d62e2a0cef9141f62b09488125cb Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Wed, 24 Jun 2015 20:28:02 -0400 Subject: [PATCH 01/30] RF: Begin refactoring load into image classes --- nibabel/freesurfer/mghformat.py | 2 + nibabel/imageglobals.py | 10 +++ nibabel/loadsave.py | 105 ++++---------------------- nibabel/minc1.py | 24 ++++++ nibabel/minc2.py | 6 ++ nibabel/nifti1.py | 9 +++ nibabel/nifti2.py | 14 ++++ nibabel/parrec.py | 2 + nibabel/spatialimages.py | 21 +++++- nibabel/spm2analyze.py | 9 +++ nibabel/tests/test_image_load_save.py | 59 --------------- 11 files changed, 108 insertions(+), 153 deletions(-) diff --git a/nibabel/freesurfer/mghformat.py b/nibabel/freesurfer/mghformat.py index e84f8e2319..58f0f3ad5f 100644 --- a/nibabel/freesurfer/mghformat.py +++ b/nibabel/freesurfer/mghformat.py @@ -13,6 +13,7 @@ from os.path import splitext import numpy as np +from ..imageglobals import valid_exts from ..volumeutils import (array_to_file, array_from_file, Recoder) from ..spatialimages import HeaderDataError, SpatialImage from ..fileholders import FileHolder, copy_file_map @@ -454,6 +455,7 @@ def writeftr_to(self, fileobj): fileobj.write(ftr_nd.tostring()) +@valid_exts('.mgh', '.mgz') @ImageOpener.register_ext_from_image('.mgz', ImageOpener.gz_def) class MGHImage(SpatialImage): """ Class for MGH format image diff --git a/nibabel/imageglobals.py b/nibabel/imageglobals.py index 0fc6dd3033..3a5161a7f3 100644 --- a/nibabel/imageglobals.py +++ b/nibabel/imageglobals.py @@ -58,3 +58,13 @@ def __enter__(self): def __exit__(self, exc, value, tb): for handler in self.orig_handlers: logger.addHandler(handler) + +IMAGE_MAP = {} + + +def valid_exts(*exts): + def decorate(klass): + for ext in exts: + IMAGE_MAP.setdefault(ext, []).append(klass) + return klass + return decorate diff --git a/nibabel/loadsave.py b/nibabel/loadsave.py index 018907d7bb..9ad3c8f84a 100644 --- a/nibabel/loadsave.py +++ b/nibabel/loadsave.py @@ -11,17 +11,12 @@ import numpy as np -from .filename_parser import types_filenames, splitext_addext +from .filename_parser import splitext_addext from .openers import ImageOpener -from .analyze import AnalyzeImage -from .spm2analyze import Spm2AnalyzeImage -from .nifti1 import Nifti1Image, Nifti1Pair, header_dtype as ni1_hdr_dtype from .nifti2 import Nifti2Image, Nifti2Pair -from .minc1 import Minc1Image -from .minc2 import Minc2Image -from .freesurfer import MGHImage from .spatialimages import ImageFileError from .imageclasses import class_map, ext_map +from .imageglobals import IMAGE_MAP from .arrayproxy import is_proxy @@ -40,55 +35,21 @@ def load(filename, **kwargs): img : ``SpatialImage`` Image of guessed type ''' - return guessed_image_type(filename).from_filename(filename, **kwargs) + froot, ext, trailing = splitext_addext(filename, ('.gz', '.bz2')) + lext = ext.lower() -def guessed_image_type(filename): - """ Guess image type from file `filename` + potential_classes = IMAGE_MAP[lext] - Parameters - ---------- - filename : str - File name containing an image + if len(potential_classes) == 1: + return potential_classes[0].from_filename(filename, **kwargs) - Returns - ------- - image_class : class - Class corresponding to guessed image type - """ - froot, ext, trailing = splitext_addext(filename, ('.gz', '.bz2')) - lext = ext.lower() - try: - img_type = ext_map[lext] - except KeyError: - raise ImageFileError('Cannot work out file type of "%s"' % - filename) - if lext in ('.mgh', '.mgz', '.par'): - klass = class_map[img_type]['class'] - elif lext == '.mnc': - # Look for HDF5 signature for MINC2 - # https://www.hdfgroup.org/HDF5/doc/H5.format.html - with ImageOpener(filename) as fobj: - signature = fobj.read(4) - klass = Minc2Image if signature == b'\211HDF' else Minc1Image - elif lext == '.nii': - with ImageOpener(filename) as fobj: - binaryblock = fobj.read(348) - ft = which_analyze_type(binaryblock) - klass = Nifti2Image if ft == 'nifti2' else Nifti1Image - else: # might be nifti 1 or 2 pair or analyze of some sort - files_types = (('image', '.img'), ('header', '.hdr')) - filenames = types_filenames(filename, files_types) - with ImageOpener(filenames['header']) as fobj: - binaryblock = fobj.read(348) - ft = which_analyze_type(binaryblock) - if ft == 'nifti2': - klass = Nifti2Pair - elif ft == 'nifti1': - klass = Nifti1Pair - else: - klass = Spm2AnalyzeImage - return klass + # Allow image tests to cache data + sniff = None + for img_type in IMAGE_MAP[lext]: + is_valid, sniff = img_type.is_image(filename, sniff) + if is_valid: + return img_type.from_filename(filename, **kwargs) def save(img, filename): @@ -212,43 +173,3 @@ def read_img_data(img, prefer='scaled'): if prefer == 'scaled': return hdr.data_from_fileobj(fileobj) return hdr.raw_data_from_fileobj(fileobj) - - -def which_analyze_type(binaryblock): - """ Is `binaryblock` from NIfTI1, NIfTI2 or Analyze header? - - Parameters - ---------- - binaryblock : bytes - The `binaryblock` is 348 bytes that might be NIfTI1, NIfTI2, Analyze, - or None of the the above. - - Returns - ------- - hdr_type : str - * a nifti1 header (pair or single) -> return 'nifti1' - * a nifti2 header (pair or single) -> return 'nifti2' - * an Analyze header -> return 'analyze' - * None of the above -> return None - - Notes - ----- - Algorithm: - - * read in the first 4 bytes from the file as 32-bit int ``sizeof_hdr`` - * if ``sizeof_hdr`` is 540 or byteswapped 540 -> assume nifti2 - * Check for 'ni1', 'n+1' magic -> assume nifti1 - * if ``sizeof_hdr`` is 348 or byteswapped 348 assume Analyze - * Return None - """ - hdr = np.ndarray(shape=(), dtype=ni1_hdr_dtype, buffer=binaryblock) - bs_hdr = hdr.byteswap() - sizeof_hdr = hdr['sizeof_hdr'] - bs_sizeof_hdr = bs_hdr['sizeof_hdr'] - if 540 in (sizeof_hdr, bs_sizeof_hdr): - return 'nifti2' - if hdr['magic'] in (b'ni1', b'n+1'): - return 'nifti1' - if 348 in (sizeof_hdr, bs_sizeof_hdr): - return 'analyze' - return None diff --git a/nibabel/minc1.py b/nibabel/minc1.py index d646397ee5..196314d144 100644 --- a/nibabel/minc1.py +++ b/nibabel/minc1.py @@ -14,8 +14,11 @@ from .externals.netcdf import netcdf_file +from .filename_parser import splitext_addext +from .imageglobals import valid_exts from .spatialimages import Header, SpatialImage from .fileslice import canonical_slicers +from .volumeutils import BinOpener from .deprecated import FutureWarningMixin @@ -279,6 +282,7 @@ def data_from_fileobj(self, fileobj): raise NotImplementedError +@valid_exts('.mnc') class Minc1Image(SpatialImage): ''' Class for MINC1 format images @@ -306,6 +310,26 @@ def from_file_map(klass, file_map): data = klass.ImageArrayProxy(minc_file) return klass(data, affine, header, extra=None, file_map=file_map) + @classmethod + def is_image(klass, filename, sniff=None): + ftypes = dict(klass.files_types) + froot, ext, trailing = splitext_addext(filename, klass._compressed_exts) + lext = ext.lower() + + if lext not in ftypes.values(): + return False, sniff + + fname = froot + ftypes['header'] if 'header' in ftypes else filename + if not sniff: + with BinOpener(fname, 'rb') as fobj: + sniff = fobj.read(4) + + return klass._minctest(sniff), sniff + + @classmethod + def _minctest(klass, binaryblock): + return binaryblock != b'\211HDF' + load = Minc1Image.load diff --git a/nibabel/minc2.py b/nibabel/minc2.py index a8a69ebd23..9bd54e9481 100644 --- a/nibabel/minc2.py +++ b/nibabel/minc2.py @@ -30,6 +30,7 @@ from .optpkg import optional_package h5py, have_h5py, setup_module = optional_package('h5py') +from .imageglobals import valid_exts from .minc1 import Minc1File, Minc1Image, MincError @@ -134,6 +135,7 @@ def get_scaled_data(self, sliceobj=()): return self._normalize(raw_data, sliceobj) +@valid_exts('.mnc') class Minc2Image(Minc1Image): ''' Class for MINC2 images @@ -160,5 +162,9 @@ def from_file_map(klass, file_map): data = klass.ImageArrayProxy(minc_file) return klass(data, affine, header, extra=None, file_map=file_map) + @classmethod + def _minctest(klass, binaryblock): + return binaryblock[:4] == b'\211HDF' + load = Minc2Image.load diff --git a/nibabel/nifti1.py b/nibabel/nifti1.py index 82e4531f66..7c6f45045b 100644 --- a/nibabel/nifti1.py +++ b/nibabel/nifti1.py @@ -18,6 +18,7 @@ from .py3k import asstr from .volumeutils import Recoder, make_dt_codes, endian_codes +from .imageglobals import valid_exts from .spatialimages import HeaderDataError, ImageFileError from .batteryrunners import Report from .quaternions import fillpositive, quat2mat, mat2quat @@ -1611,6 +1612,12 @@ def _chk_xform_code(klass, code_type, hdr, fix): rep.fix_msg = 'setting to 0' return hdr, rep + @classmethod + def is_header(klass, binaryblock): + hdr = np.ndarray(shape=(), dtype=header_dtype, + buffer=binaryblock[:348]) + return hdr['magic'] in (b'ni1', b'n+1') + class Nifti1PairHeader(Nifti1Header): ''' Class for NIfTI1 pair header ''' @@ -1618,6 +1625,7 @@ class Nifti1PairHeader(Nifti1Header): is_single = False +@valid_exts('.img', '.hdr') class Nifti1Pair(analyze.AnalyzeImage): """ Class for NIfTI1 format image, header pair """ @@ -1841,6 +1849,7 @@ def set_sform(self, affine, code=None, **kwargs): self._affine[:] = self._header.get_best_affine() +@valid_exts('.nii') class Nifti1Image(Nifti1Pair): """ Class for single file NIfTI1 format image """ diff --git a/nibabel/nifti2.py b/nibabel/nifti2.py index 89fe3345e3..fdc0c52c84 100644 --- a/nibabel/nifti2.py +++ b/nibabel/nifti2.py @@ -20,6 +20,7 @@ import numpy as np from .analyze import AnalyzeHeader +from .imageglobals import valid_exts from .batteryrunners import Report from .spatialimages import HeaderDataError, ImageFileError from .nifti1 import Nifti1Header, Nifti1Pair, Nifti1Image @@ -221,6 +222,17 @@ def _chk_eol_check(hdr, fix=False): rep.fix_msg = 'setting EOL check to 13, 10, 26, 10' return hdr, rep + @classmethod + def is_header(klass, binaryblock): + if len(binaryblock) < 540: + return False + + hdr = np.ndarray(shape=(), dtype=header_dtype, + buffer=binaryblock[:540]) + bs_hdr = hdr.byteswap() + return 540 in (hdr['sizeof_hdr'], bs_hdr['sizeof_hdr']) + + class Nifti2PairHeader(Nifti2Header): ''' Class for NIfTI2 pair header ''' @@ -228,12 +240,14 @@ class Nifti2PairHeader(Nifti2Header): is_single = False +@valid_exts('.img', '.hdr') class Nifti2Pair(Nifti1Pair): """ Class for NIfTI2 format image, header pair """ header_class = Nifti2PairHeader +@valid_exts('.nii') class Nifti2Image(Nifti1Image): """ Class for single file NIfTI2 format image """ diff --git a/nibabel/parrec.py b/nibabel/parrec.py index ef4c11c698..1d981e0ad2 100644 --- a/nibabel/parrec.py +++ b/nibabel/parrec.py @@ -100,6 +100,7 @@ from .spatialimages import SpatialImage, Header from .eulerangles import euler2mat from .volumeutils import Recoder, array_from_file +from .imageglobals import valid_exts from .affines import from_matvec, dot_reduce, apply_affine from .nifti1 import unit_codes from .fileslice import fileslice, strided_scalar @@ -1017,6 +1018,7 @@ def get_sorted_slice_indices(self): return np.lexsort(keys)[:n_used] +@valid_exts('.par', '.rec') class PARRECImage(SpatialImage): """PAR/REC image""" header_class = PARRECHeader diff --git a/nibabel/spatialimages.py b/nibabel/spatialimages.py index eb4befa077..6b1e796ca0 100644 --- a/nibabel/spatialimages.py +++ b/nibabel/spatialimages.py @@ -141,9 +141,10 @@ import numpy as np -from .filename_parser import types_filenames, TypesFilenamesError +from .filename_parser import types_filenames, TypesFilenamesError, \ + splitext_addext from .fileholders import FileHolder -from .volumeutils import shape_zoom_affine +from .volumeutils import shape_zoom_affine, BinOpener class HeaderDataError(Exception): @@ -866,6 +867,22 @@ def from_image(klass, img): klass.header_class.from_header(img.header), extra=img.extra.copy()) + @classmethod + def is_image(klass, filename, sniff=None): + ftypes = dict(klass.files_types) + froot, ext, trailing = splitext_addext(filename, ('.gz', '.bz2')) + lext = ext.lower() + + if lext not in ftypes.values(): + return False, sniff + + fname = froot + ftypes['header'] if 'header' in ftypes else filename + if not sniff: + with BinOpener(fname, 'rb') as fobj: + sniff = fobj.read(1024) + + return klass.header_class.is_header(sniff), sniff + def __getitem__(self): ''' No slicing or dictionary interface for images ''' diff --git a/nibabel/spm2analyze.py b/nibabel/spm2analyze.py index 28d04bc7a1..879b0b8222 100644 --- a/nibabel/spm2analyze.py +++ b/nibabel/spm2analyze.py @@ -113,7 +113,16 @@ def get_slope_inter(self): return slope, inter return None, None + @classmethod + def is_header(klass, binaryblock): + hdr = np.ndarray(shape=(), dtype=header_dtype, + buffer=binaryblock[:348]) + bs_hdr = hdr.byteswap() + return (binaryblock[344:348] not in (b'ni1\x00', b'n+1\x00') and + 348 in (hdr['sizeof_hdr'], bs_hdr['sizeof_hdr'])) + +@valid_exts('.img', '.hdr') class Spm2AnalyzeImage(spm99.Spm99AnalyzeImage): """ Class for SPM2 variant of basic Analyze image """ diff --git a/nibabel/tests/test_image_load_save.py b/nibabel/tests/test_image_load_save.py index 7ade7d09c3..9394dcb655 100644 --- a/nibabel/tests/test_image_load_save.py +++ b/nibabel/tests/test_image_load_save.py @@ -265,62 +265,3 @@ def test_filename_save(): del rt_img finally: shutil.rmtree(pth) - - -def test_analyze_detection(): - # Test detection of Analyze, Nifti1 and Nifti2 - # Algorithm is as described in loadsave:which_analyze_type - def wat(hdr): - return nils.which_analyze_type(hdr.binaryblock) - n1_hdr = Nifti1Header(b'\0' * 348, check=False) - assert_equal(wat(n1_hdr), None) - n1_hdr['sizeof_hdr'] = 540 - assert_equal(wat(n1_hdr), 'nifti2') - assert_equal(wat(n1_hdr.as_byteswapped()), 'nifti2') - n1_hdr['sizeof_hdr'] = 348 - assert_equal(wat(n1_hdr), 'analyze') - assert_equal(wat(n1_hdr.as_byteswapped()), 'analyze') - n1_hdr['magic'] = b'n+1' - assert_equal(wat(n1_hdr), 'nifti1') - assert_equal(wat(n1_hdr.as_byteswapped()), 'nifti1') - n1_hdr['magic'] = b'ni1' - assert_equal(wat(n1_hdr), 'nifti1') - assert_equal(wat(n1_hdr.as_byteswapped()), 'nifti1') - # Doesn't matter what magic is if it's not a nifti1 magic - n1_hdr['magic'] = b'ni2' - assert_equal(wat(n1_hdr), 'analyze') - n1_hdr['sizeof_hdr'] = 0 - n1_hdr['magic'] = b'' - assert_equal(wat(n1_hdr), None) - n1_hdr['magic'] = 'n+1' - assert_equal(wat(n1_hdr), 'nifti1') - n1_hdr['magic'] = 'ni1' - assert_equal(wat(n1_hdr), 'nifti1') - - -def test_guessed_image_type(): - # Test whether we can guess the image type from example files - assert_equal(nils.guessed_image_type( - pjoin(DATA_PATH, 'example4d.nii.gz')), - Nifti1Image) - assert_equal(nils.guessed_image_type( - pjoin(DATA_PATH, 'nifti1.hdr')), - Nifti1Pair) - assert_equal(nils.guessed_image_type( - pjoin(DATA_PATH, 'example_nifti2.nii.gz')), - Nifti2Image) - assert_equal(nils.guessed_image_type( - pjoin(DATA_PATH, 'nifti2.hdr')), - Nifti2Pair) - assert_equal(nils.guessed_image_type( - pjoin(DATA_PATH, 'tiny.mnc')), - Minc1Image) - assert_equal(nils.guessed_image_type( - pjoin(DATA_PATH, 'small.mnc')), - Minc2Image) - assert_equal(nils.guessed_image_type( - pjoin(DATA_PATH, 'test.mgz')), - MGHImage) - assert_equal(nils.guessed_image_type( - pjoin(DATA_PATH, 'analyze.hdr')), - Spm2AnalyzeImage) From 2cb5c95c4fdde80d18e1b34475c89d5a073d4092 Mon Sep 17 00:00:00 2001 From: Ben Cipollini Date: Thu, 16 Jul 2015 13:00:58 -0700 Subject: [PATCH 02/30] Deprecate the use of class_map and ext_map. --- nibabel/imageclasses.py | 107 ++++++++++++++++++++++------------------ 1 file changed, 59 insertions(+), 48 deletions(-) diff --git a/nibabel/imageclasses.py b/nibabel/imageclasses.py index 31a219482c..0b99cff030 100644 --- a/nibabel/imageclasses.py +++ b/nibabel/imageclasses.py @@ -7,6 +7,8 @@ # ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## ''' Define supported image classes and names ''' +import warnings + from .analyze import AnalyzeImage from .spm99analyze import Spm99AnalyzeImage from .spm2analyze import Spm2AnalyzeImage @@ -19,57 +21,66 @@ _, have_scipy, _ = optional_package('scipy') -# mapping of names to classes and class functionality +# DEPRECATED: mapping of names to classes and class functionality +class ClassMapDict(dict): + def __getitem__(self, *args, **kwargs): + warnings.warn("class_map is deprecated.", DeprecationWarning) + return super(ClassMapDict, self).__getitem__(*args, **kwargs) + +class_map = ClassMapDict( + analyze={'class': AnalyzeImage, # Image class + 'ext': '.img', # characteristic image extension + 'has_affine': False, # class can store an affine + 'makeable': True, # empty image can be easily made in memory + 'rw': True}, # image can be written + spm99analyze={'class': Spm99AnalyzeImage, + 'ext': '.img', + 'has_affine': True, + 'makeable': True, + 'rw': have_scipy}, + spm2analyze={'class': Spm2AnalyzeImage, + 'ext': '.img', + 'has_affine': True, + 'makeable': True, + 'rw': have_scipy}, + nifti_pair={'class': Nifti1Pair, + 'ext': '.img', + 'has_affine': True, + 'makeable': True, + 'rw': True}, + nifti_single={'class': Nifti1Image, + 'ext': '.nii', + 'has_affine': True, + 'makeable': True, + 'rw': True}, + minc={'class': Minc1Image, + 'ext': '.mnc', + 'has_affine': True, + 'makeable': True, + 'rw': False}, + mgh={'class': MGHImage, + 'ext': '.mgh', + 'has_affine': True, + 'makeable': True, + 'rw': True}, + mgz={'class': MGHImage, + 'ext': '.mgz', + 'has_affine': True, + 'makeable': True, + 'rw': True}, + par={'class': PARRECImage, + 'ext': '.par', + 'has_affine': True, + 'makeable': False, + 'rw': False}) -class_map = { - 'analyze': {'class': AnalyzeImage, # Image class - 'ext': '.img', # characteristic image extension - 'has_affine': False, # class can store an affine - 'makeable': True, # empty image can be easily made in memory - 'rw': True}, # image can be written - 'spm99analyze': {'class': Spm99AnalyzeImage, - 'ext': '.img', - 'has_affine': True, - 'makeable': True, - 'rw': have_scipy}, - 'spm2analyze': {'class': Spm2AnalyzeImage, - 'ext': '.img', - 'has_affine': True, - 'makeable': True, - 'rw': have_scipy}, - 'nifti_pair': {'class': Nifti1Pair, - 'ext': '.img', - 'has_affine': True, - 'makeable': True, - 'rw': True}, - 'nifti_single': {'class': Nifti1Image, - 'ext': '.nii', - 'has_affine': True, - 'makeable': True, - 'rw': True}, - 'minc': {'class': Minc1Image, - 'ext': '.mnc', - 'has_affine': True, - 'makeable': True, - 'rw': False}, - 'mgh': {'class': MGHImage, - 'ext': '.mgh', - 'has_affine': True, - 'makeable': True, - 'rw': True}, - 'mgz': {'class': MGHImage, - 'ext': '.mgz', - 'has_affine': True, - 'makeable': True, - 'rw': True}, - 'par': {'class': PARRECImage, - 'ext': '.par', - 'has_affine': True, - 'makeable': False, - 'rw': False}} +class ExtMapRecoder(Recoder): + def __getitem__(self, *args, **kwargs): + warnings.warn("ext_map is deprecated.", DeprecationWarning) + return super(ExtMapRecoder, self).__getitem__(*args, **kwargs) # mapping of extensions to default image class names -ext_map = Recoder(( +ext_map = ExtMapRecoder(( ('nifti_single', '.nii'), ('nifti_pair', '.img', '.hdr'), ('minc', '.mnc'), From f3373cae430ba8d937403b551f49fe97c3442bca Mon Sep 17 00:00:00 2001 From: Ben Cipollini Date: Thu, 16 Jul 2015 16:00:44 -0700 Subject: [PATCH 03/30] Migrating imageclasses, away from IMAGE_MAP --- nibabel/analyze.py | 6 ++++- nibabel/filename_parser.py | 2 ++ nibabel/freesurfer/mghformat.py | 8 +++++-- nibabel/imageclasses.py | 9 +++++++ nibabel/imageglobals.py | 10 -------- nibabel/loadsave.py | 21 ++++++---------- nibabel/minc1.py | 37 +++++++++++------------------ nibabel/minc2.py | 11 ++++++--- nibabel/nifti1.py | 14 +++++++---- nibabel/nifti2.py | 13 +++++----- nibabel/parrec.py | 6 +++-- nibabel/spatialimages.py | 37 ++++++++++++++++++++++------- nibabel/spm2analyze.py | 11 ++++++--- nibabel/spm99analyze.py | 6 +++++ nibabel/tests/test_analyze.py | 7 +++--- nibabel/tests/test_filehandles.py | 6 ++++- nibabel/tests/test_spatialimages.py | 4 ++-- 17 files changed, 124 insertions(+), 84 deletions(-) diff --git a/nibabel/analyze.py b/nibabel/analyze.py index 982da58be9..02cd398eea 100644 --- a/nibabel/analyze.py +++ b/nibabel/analyze.py @@ -829,7 +829,7 @@ def _chk_datatype(klass, hdr, fix=False): dtype = klass._data_type_codes.dtype[code] except KeyError: rep.problem_level = 40 - rep.problem_msg = 'data code %d not recognized' % code + rep.problem_msg = 'data code %d not recognized by %s' % (code, klass.__name__) else: if dtype.itemsize == 0: rep.problem_level = 40 @@ -899,6 +899,10 @@ class AnalyzeImage(SpatialImage): header_class = AnalyzeHeader files_types = (('image', '.img'), ('header', '.hdr')) _compressed_exts = ('.gz', '.bz2') + has_affine = False + makeable = True + rw = True + nickname = 'analyze' ImageArrayProxy = ArrayProxy diff --git a/nibabel/filename_parser.py b/nibabel/filename_parser.py index bc21cbc872..6065716e7d 100644 --- a/nibabel/filename_parser.py +++ b/nibabel/filename_parser.py @@ -131,6 +131,8 @@ def types_filenames(template_fname, types_exts, elif found_ext == found_ext.lower(): proc_ext = lambda s: s.lower() for name, ext in types_exts: + if name in tfns: # priority to those found first. + continue if name == direct_set_name: tfns[name] = template_fname continue diff --git a/nibabel/freesurfer/mghformat.py b/nibabel/freesurfer/mghformat.py index 58f0f3ad5f..6e410641e9 100644 --- a/nibabel/freesurfer/mghformat.py +++ b/nibabel/freesurfer/mghformat.py @@ -13,7 +13,6 @@ from os.path import splitext import numpy as np -from ..imageglobals import valid_exts from ..volumeutils import (array_to_file, array_from_file, Recoder) from ..spatialimages import HeaderDataError, SpatialImage from ..fileholders import FileHolder, copy_file_map @@ -461,8 +460,13 @@ class MGHImage(SpatialImage): """ Class for MGH format image """ header_class = MGHHeader - files_types = (('image', '.mgh'),) + files_types = (('image', '.mgh'), + ('image', '.mgz')) _compressed_exts = (('.gz',)) + nickname = 'mgh' + has_affine = True + makeable = True + rw = True ImageArrayProxy = ArrayProxy diff --git a/nibabel/imageclasses.py b/nibabel/imageclasses.py index 0b99cff030..189fdd95b4 100644 --- a/nibabel/imageclasses.py +++ b/nibabel/imageclasses.py @@ -13,7 +13,9 @@ from .spm99analyze import Spm99AnalyzeImage from .spm2analyze import Spm2AnalyzeImage from .nifti1 import Nifti1Pair, Nifti1Image +from .nifti2 import Nifti2Pair, Nifti2Image from .minc1 import Minc1Image +from .minc2 import Minc2Image from .freesurfer import MGHImage from .parrec import PARRECImage from .volumeutils import Recoder @@ -21,6 +23,13 @@ _, have_scipy, _ = optional_package('scipy') +# Ordered by the load/save priority. +all_image_classes = [Nifti1Pair, Nifti1Image, Nifti2Pair, Nifti2Image, + Spm2AnalyzeImage, Spm99AnalyzeImage, AnalyzeImage, + Minc1Image, Minc2Image, MGHImage, + PARRECImage] + + # DEPRECATED: mapping of names to classes and class functionality class ClassMapDict(dict): def __getitem__(self, *args, **kwargs): diff --git a/nibabel/imageglobals.py b/nibabel/imageglobals.py index 3a5161a7f3..0fc6dd3033 100644 --- a/nibabel/imageglobals.py +++ b/nibabel/imageglobals.py @@ -58,13 +58,3 @@ def __enter__(self): def __exit__(self, exc, value, tb): for handler in self.orig_handlers: logger.addHandler(handler) - -IMAGE_MAP = {} - - -def valid_exts(*exts): - def decorate(klass): - for ext in exts: - IMAGE_MAP.setdefault(ext, []).append(klass) - return klass - return decorate diff --git a/nibabel/loadsave.py b/nibabel/loadsave.py index 9ad3c8f84a..0aa03cd829 100644 --- a/nibabel/loadsave.py +++ b/nibabel/loadsave.py @@ -15,8 +15,7 @@ from .openers import ImageOpener from .nifti2 import Nifti2Image, Nifti2Pair from .spatialimages import ImageFileError -from .imageclasses import class_map, ext_map -from .imageglobals import IMAGE_MAP +from .imageclasses import class_map, ext_map, all_image_classes from .arrayproxy import is_proxy @@ -36,20 +35,14 @@ def load(filename, **kwargs): Image of guessed type ''' - froot, ext, trailing = splitext_addext(filename, ('.gz', '.bz2')) - lext = ext.lower() - - potential_classes = IMAGE_MAP[lext] - - if len(potential_classes) == 1: - return potential_classes[0].from_filename(filename, **kwargs) - - # Allow image tests to cache data sniff = None - for img_type in IMAGE_MAP[lext]: - is_valid, sniff = img_type.is_image(filename, sniff) + for image_klass in all_image_classes: + is_valid, sniff = image_klass.is_image(filename, sniff) if is_valid: - return img_type.from_filename(filename, **kwargs) + return image_klass.from_filename(filename, **kwargs) + + raise ImageFileError('Cannot work out file type of "%s"' % + filename) def save(img, filename): diff --git a/nibabel/minc1.py b/nibabel/minc1.py index 196314d144..c54a137c1f 100644 --- a/nibabel/minc1.py +++ b/nibabel/minc1.py @@ -15,7 +15,6 @@ from .externals.netcdf import netcdf_file from .filename_parser import splitext_addext -from .imageglobals import valid_exts from .spatialimages import Header, SpatialImage from .fileslice import canonical_slicers from .volumeutils import BinOpener @@ -273,6 +272,9 @@ class MincHeader(Header): # We don't use the data layout - this just in case we do later data_layout = 'C' + # + sniff_size = 4 + def data_to_fileobj(self, data, fileobj, rescale=True): """ See Header class for an implementation we can't use """ raise NotImplementedError @@ -282,7 +284,12 @@ def data_from_fileobj(self, fileobj): raise NotImplementedError -@valid_exts('.mnc') +class Minc1Header(MincHeader): + @classmethod + def is_header(klass, binaryblock): + return binaryblock != b'\211HDF' + + class Minc1Image(SpatialImage): ''' Class for MINC1 format images @@ -290,9 +297,13 @@ class Minc1Image(SpatialImage): MINC header type - and reads the relevant information from the MINC file on load. ''' - header_class = MincHeader + header_class = Minc1Header files_types = (('image', '.mnc'),) _compressed_exts = ('.gz', '.bz2') + has_affine = True + makeable = True + rw = False + nickname = 'minc' ImageArrayProxy = MincImageArrayProxy @@ -310,26 +321,6 @@ def from_file_map(klass, file_map): data = klass.ImageArrayProxy(minc_file) return klass(data, affine, header, extra=None, file_map=file_map) - @classmethod - def is_image(klass, filename, sniff=None): - ftypes = dict(klass.files_types) - froot, ext, trailing = splitext_addext(filename, klass._compressed_exts) - lext = ext.lower() - - if lext not in ftypes.values(): - return False, sniff - - fname = froot + ftypes['header'] if 'header' in ftypes else filename - if not sniff: - with BinOpener(fname, 'rb') as fobj: - sniff = fobj.read(4) - - return klass._minctest(sniff), sniff - - @classmethod - def _minctest(klass, binaryblock): - return binaryblock != b'\211HDF' - load = Minc1Image.load diff --git a/nibabel/minc2.py b/nibabel/minc2.py index 9bd54e9481..9572c271c5 100644 --- a/nibabel/minc2.py +++ b/nibabel/minc2.py @@ -30,8 +30,7 @@ from .optpkg import optional_package h5py, have_h5py, setup_module = optional_package('h5py') -from .imageglobals import valid_exts -from .minc1 import Minc1File, Minc1Image, MincError +from .minc1 import Minc1File, MincHeader, Minc1Image, MincError class Hdf5Bunch(object): @@ -135,7 +134,12 @@ def get_scaled_data(self, sliceobj=()): return self._normalize(raw_data, sliceobj) -@valid_exts('.mnc') +class Minc2Header(MincHeader): + @classmethod + def is_header(klass, binaryblock): + return binaryblock == b'\211HDF' + + class Minc2Image(Minc1Image): ''' Class for MINC2 images @@ -145,6 +149,7 @@ class Minc2Image(Minc1Image): ''' # MINC2 does not do compressed whole files _compressed_exts = () + header_class = Minc2Header @classmethod def from_file_map(klass, file_map): diff --git a/nibabel/nifti1.py b/nibabel/nifti1.py index 7c6f45045b..0f3c516c1e 100644 --- a/nibabel/nifti1.py +++ b/nibabel/nifti1.py @@ -18,7 +18,6 @@ from .py3k import asstr from .volumeutils import Recoder, make_dt_codes, endian_codes -from .imageglobals import valid_exts from .spatialimages import HeaderDataError, ImageFileError from .batteryrunners import Report from .quaternions import fillpositive, quat2mat, mat2quat @@ -559,6 +558,9 @@ class Nifti1Header(SpmAnalyzeHeader): pair_magic = b'ni1' single_magic = b'n+1' + # for sniffing type + sniff_size = 348 + # Quaternion threshold near 0, based on float32 precision quaternion_threshold = -np.finfo(np.float32).eps * 3 @@ -1614,8 +1616,11 @@ def _chk_xform_code(klass, code_type, hdr, fix): @classmethod def is_header(klass, binaryblock): + if len(binaryblock) < klass.sniff_size: + raise ValueError('Must pass a binary block >= %d bytes' % klass.sniff_size) + hdr = np.ndarray(shape=(), dtype=header_dtype, - buffer=binaryblock[:348]) + buffer=binaryblock[:klass.sniff_size]) return hdr['magic'] in (b'ni1', b'n+1') @@ -1625,11 +1630,12 @@ class Nifti1PairHeader(Nifti1Header): is_single = False -@valid_exts('.img', '.hdr') class Nifti1Pair(analyze.AnalyzeImage): """ Class for NIfTI1 format image, header pair """ header_class = Nifti1PairHeader + nickname = 'nifti_pair' + rw = True def __init__(self, dataobj, affine, header=None, extra=None, file_map=None): @@ -1849,12 +1855,12 @@ def set_sform(self, affine, code=None, **kwargs): self._affine[:] = self._header.get_best_affine() -@valid_exts('.nii') class Nifti1Image(Nifti1Pair): """ Class for single file NIfTI1 format image """ header_class = Nifti1Header files_types = (('image', '.nii'),) + nickname = 'nifti_single' @staticmethod def _get_fileholders(file_map): diff --git a/nibabel/nifti2.py b/nibabel/nifti2.py index fdc0c52c84..54b760c02e 100644 --- a/nibabel/nifti2.py +++ b/nibabel/nifti2.py @@ -20,7 +20,6 @@ import numpy as np from .analyze import AnalyzeHeader -from .imageglobals import valid_exts from .batteryrunners import Report from .spatialimages import HeaderDataError, ImageFileError from .nifti1 import Nifti1Header, Nifti1Pair, Nifti1Image @@ -142,6 +141,9 @@ class Nifti2Header(Nifti1Header): # Size of header in sizeof_hdr field sizeof_hdr = 540 + # sniff size to determine type + sniff_size = 540 + # Quaternion threshold near 0, based on float64 preicision quaternion_threshold = -np.finfo(np.float64).eps * 3 @@ -224,30 +226,27 @@ def _chk_eol_check(hdr, fix=False): @classmethod def is_header(klass, binaryblock): - if len(binaryblock) < 540: - return False + if len(binaryblock) < klass.sniff_size: + raise ValueError('Must pass a binary block >= %d bytes' % klass.sniff_size) hdr = np.ndarray(shape=(), dtype=header_dtype, - buffer=binaryblock[:540]) + buffer=binaryblock[:klass.sniff_size]) bs_hdr = hdr.byteswap() return 540 in (hdr['sizeof_hdr'], bs_hdr['sizeof_hdr']) - class Nifti2PairHeader(Nifti2Header): ''' Class for NIfTI2 pair header ''' # Signal whether this is single (header + data) file is_single = False -@valid_exts('.img', '.hdr') class Nifti2Pair(Nifti1Pair): """ Class for NIfTI2 format image, header pair """ header_class = Nifti2PairHeader -@valid_exts('.nii') class Nifti2Image(Nifti1Image): """ Class for single file NIfTI2 format image """ diff --git a/nibabel/parrec.py b/nibabel/parrec.py index 1d981e0ad2..48eeeb765a 100644 --- a/nibabel/parrec.py +++ b/nibabel/parrec.py @@ -100,7 +100,6 @@ from .spatialimages import SpatialImage, Header from .eulerangles import euler2mat from .volumeutils import Recoder, array_from_file -from .imageglobals import valid_exts from .affines import from_matvec, dot_reduce, apply_affine from .nifti1 import unit_codes from .fileslice import fileslice, strided_scalar @@ -1018,11 +1017,14 @@ def get_sorted_slice_indices(self): return np.lexsort(keys)[:n_used] -@valid_exts('.par', '.rec') class PARRECImage(SpatialImage): """PAR/REC image""" header_class = PARRECHeader files_types = (('image', '.rec'), ('header', '.par')) + nickname = 'par' + has_affine = True + makeable = False + rw = False ImageArrayProxy = PARRECArrayProxy diff --git a/nibabel/spatialimages.py b/nibabel/spatialimages.py index 6b1e796ca0..4aa248e9ff 100644 --- a/nibabel/spatialimages.py +++ b/nibabel/spatialimages.py @@ -320,11 +320,21 @@ class ImageFileError(Exception): class SpatialImage(object): + ''' Template class for images ''' header_class = Header files_types = (('image', None),) _compressed_exts = () - ''' Template class for images ''' + @classmethod + def is_valid_extension(klass, lext): + return np.any([ft[1] == lext for ft in klass.files_types]) + + @classmethod + def is_valid_filename(klass, filename): + froot, ext, trailing = splitext_addext(filename, klass._compressed_exts) + lext = ext.lower() + return klass.is_valid_extension(lext) + def __init__(self, dataobj, affine, header=None, extra=None, file_map=None): ''' Initialize image @@ -869,19 +879,28 @@ def from_image(klass, img): @classmethod def is_image(klass, filename, sniff=None): - ftypes = dict(klass.files_types) - froot, ext, trailing = splitext_addext(filename, ('.gz', '.bz2')) + froot, ext, trailing = splitext_addext(filename, klass._compressed_exts) lext = ext.lower() - if lext not in ftypes.values(): + if not klass.is_valid_extension(lext): return False, sniff + elif (not hasattr(klass.header_class, 'sniff_size') or + not hasattr(klass.header_class, 'is_header')): + return True, sniff + + # Determine the metadata location, then sniff it + ftypes = dict(klass.files_types) + if 'header' not in ftypes: + metadata_filename = filename + else: + metadata_filename = froot + ftypes['header'] + trailing - fname = froot + ftypes['header'] if 'header' in ftypes else filename - if not sniff: - with BinOpener(fname, 'rb') as fobj: - sniff = fobj.read(1024) + sniff_size = 1024 # klass.header_class.sniff_size + if not sniff or len(sniff) < sniff_size: + with BinOpener(metadata_filename, 'rb') as fobj: + sniff = fobj.read(sniff_size) - return klass.header_class.is_header(sniff), sniff + return klass.header_class.is_header(sniff[:sniff_size]), sniff def __getitem__(self): ''' No slicing or dictionary interface for images diff --git a/nibabel/spm2analyze.py b/nibabel/spm2analyze.py index 879b0b8222..4864522338 100644 --- a/nibabel/spm2analyze.py +++ b/nibabel/spm2analyze.py @@ -35,6 +35,9 @@ class Spm2AnalyzeHeader(spm99.Spm99AnalyzeHeader): # Copies of module level definitions template_dtype = header_dtype + # binary read size to determine type + sniff_size = 348 + def get_slope_inter(self): ''' Get data scaling (slope) and intercept from header data @@ -115,19 +118,21 @@ def get_slope_inter(self): @classmethod def is_header(klass, binaryblock): + if len(binaryblock) < klass.sniff_size: + raise ValueError('Must pass a binary block >= %d bytes' % klass.sniff_size) + hdr = np.ndarray(shape=(), dtype=header_dtype, - buffer=binaryblock[:348]) + buffer=binaryblock[:klass.sniff_size]) bs_hdr = hdr.byteswap() return (binaryblock[344:348] not in (b'ni1\x00', b'n+1\x00') and 348 in (hdr['sizeof_hdr'], bs_hdr['sizeof_hdr'])) -@valid_exts('.img', '.hdr') class Spm2AnalyzeImage(spm99.Spm99AnalyzeImage): """ Class for SPM2 variant of basic Analyze image """ header_class = Spm2AnalyzeHeader - + nickname = 'spm2analyze' load = Spm2AnalyzeImage.load save = Spm2AnalyzeImage.instance_to_filename diff --git a/nibabel/spm99analyze.py b/nibabel/spm99analyze.py index f3d565e41d..92c1af7c51 100644 --- a/nibabel/spm99analyze.py +++ b/nibabel/spm99analyze.py @@ -17,6 +17,8 @@ from .batteryrunners import Report from . import analyze # module import from .keywordonly import kw_only_meth +from .optpkg import optional_package +have_scipy = optional_package('scipy')[1] ''' Support subtle variations of SPM version of Analyze ''' header_key_dtd = analyze.header_key_dtd @@ -237,6 +239,10 @@ class Spm99AnalyzeImage(analyze.AnalyzeImage): files_types = (('image', '.img'), ('header', '.hdr'), ('mat', '.mat')) + has_affine = True + makeable = True + rw = have_scipy + nickname = 'spm99analyze' @classmethod @kw_only_meth(1) diff --git a/nibabel/tests/test_analyze.py b/nibabel/tests/test_analyze.py index b567bc3c21..157b84dff9 100644 --- a/nibabel/tests/test_analyze.py +++ b/nibabel/tests/test_analyze.py @@ -32,7 +32,7 @@ from ..arraywriters import WriterError from nose.tools import (assert_equal, assert_not_equal, assert_true, - assert_false, assert_raises) + assert_false, assert_raises, assert_in) from numpy.testing import (assert_array_equal, assert_array_almost_equal) @@ -157,8 +157,9 @@ def test_log_checks(self): hdr['datatype'] = -1 # severity 40 with suppress_warnings(): fhdr, message, raiser = self.log_chk(hdr, 40) - assert_equal(message, 'data code -1 not recognized; ' - 'not attempting fix') + assert_in('data code -1 not recognized', message) + assert_in('not attempting fix', message) + assert_raises(*raiser) # datatype not supported hdr['datatype'] = 255 # severity 40 diff --git a/nibabel/tests/test_filehandles.py b/nibabel/tests/test_filehandles.py index 2ecadf5840..c1e6c718a1 100644 --- a/nibabel/tests/test_filehandles.py +++ b/nibabel/tests/test_filehandles.py @@ -34,7 +34,7 @@ def test_multiload(): if N > 5000: warn('It would take too long to test file handles, aborting') return - arr = np.arange(24).reshape((2,3,4)) + arr = np.arange(24).reshape((2, 3, 4)) img = Nifti1Image(arr, np.eye(4)) imgs = [] try: @@ -43,6 +43,10 @@ def test_multiload(): save(img, fname) for i in range(N): imgs.append(load(fname)) + except Exception as e: + if 'i' in locals(): + e.message += ' (i == %d)' % i + raise Exception(e.message) finally: del img, imgs shutil.rmtree(tmpdir) diff --git a/nibabel/tests/test_spatialimages.py b/nibabel/tests/test_spatialimages.py index d2d6d0a93e..29c227be73 100644 --- a/nibabel/tests/test_spatialimages.py +++ b/nibabel/tests/test_spatialimages.py @@ -371,9 +371,9 @@ def test_load_mmap(self): back_img = func(param1, **kwargs) back_data = back_img.get_data() if expected_mode is None: - assert_false(isinstance(back_data, np.memmap)) + assert_false(isinstance(back_data, np.memmap), 'Should not be a %s' % img_klass.__name__) else: - assert_true(isinstance(back_data, np.memmap)) + assert_true(isinstance(back_data, np.memmap), 'Not a %s' % img_klass.__name__) if self.check_mmap_mode: assert_equal(back_data.mode, expected_mode) del back_img, back_data From 45f50e365b4b5f841ac7a592e40c8610945961cb Mon Sep 17 00:00:00 2001 From: Ben Cipollini Date: Thu, 16 Jul 2015 16:29:45 -0700 Subject: [PATCH 04/30] Improve error handling, efficiency, and search more broadly over header extensions. --- nibabel/spatialimages.py | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/nibabel/spatialimages.py b/nibabel/spatialimages.py index 4aa248e9ff..843552af23 100644 --- a/nibabel/spatialimages.py +++ b/nibabel/spatialimages.py @@ -137,6 +137,7 @@ except NameError: # python 3 basestring = str +import os.path import warnings import numpy as np @@ -889,18 +890,30 @@ def is_image(klass, filename, sniff=None): return True, sniff # Determine the metadata location, then sniff it - ftypes = dict(klass.files_types) - if 'header' not in ftypes: + header_exts = [ft[1] for ft in klass.files_types if ft[0] == 'header'] + if len(header_exts) == 0: metadata_filename = filename else: - metadata_filename = froot + ftypes['header'] + trailing + # Search for an acceptable existing header; + # could be compressed or not... + for ext in header_exts: + for tr_ext in np.unique([trailing, ''] + list(klass._compressed_exts)): + metadata_filename = froot + ext + tr_ext + if os.path.exists(metadata_filename): + break - sniff_size = 1024 # klass.header_class.sniff_size - if not sniff or len(sniff) < sniff_size: - with BinOpener(metadata_filename, 'rb') as fobj: - sniff = fobj.read(sniff_size) - - return klass.header_class.is_header(sniff[:sniff_size]), sniff + try: + if not sniff or len(sniff) < klass.header_class.sniff_size: + # 1024 == large size, for efficiency (could iterate over imageclasses). + sniff_size = np.max([1024, klass.header_class.sniff_size]) + with BinOpener(metadata_filename, 'rb') as fobj: + sniff = fobj.read(sniff_size) + return klass.header_class.is_header(sniff[:klass.header_class.sniff_size]), sniff + except Exception as e: + # Can happen if: file doesn't exist, + # filesize < necessary sniff size (this happens!) + # other unexpected errors. + return False, sniff def __getitem__(self): ''' No slicing or dictionary interface for images From 20ad34714b7a7e1c4d9fe44d7e5156876d533245 Mon Sep 17 00:00:00 2001 From: Ben Cipollini Date: Thu, 16 Jul 2015 16:30:10 -0700 Subject: [PATCH 05/30] Modify save, remove vestigates of class_map / ext_map --- nibabel/__init__.py | 2 +- nibabel/freesurfer/mghformat.py | 1 - nibabel/loadsave.py | 24 +++++++++++---- nibabel/tests/test_files_interface.py | 44 +++++++++++++-------------- nibabel/tests/test_image_load_save.py | 12 +++----- 5 files changed, 45 insertions(+), 38 deletions(-) diff --git a/nibabel/__init__.py b/nibabel/__init__.py index 2df9a1c534..779f6e8587 100644 --- a/nibabel/__init__.py +++ b/nibabel/__init__.py @@ -61,7 +61,7 @@ from .orientations import (io_orientation, orientation_affine, flip_axis, OrientationError, apply_orientation, aff2axcodes) -from .imageclasses import class_map, ext_map +from .imageclasses import class_map, ext_map, all_image_classes from . import trackvis from . import mriutils diff --git a/nibabel/freesurfer/mghformat.py b/nibabel/freesurfer/mghformat.py index 6e410641e9..b6ed977695 100644 --- a/nibabel/freesurfer/mghformat.py +++ b/nibabel/freesurfer/mghformat.py @@ -454,7 +454,6 @@ def writeftr_to(self, fileobj): fileobj.write(ftr_nd.tostring()) -@valid_exts('.mgh', '.mgz') @ImageOpener.register_ext_from_image('.mgz', ImageOpener.gz_def) class MGHImage(SpatialImage): """ Class for MGH format image diff --git a/nibabel/loadsave.py b/nibabel/loadsave.py index 0aa03cd829..608eef1574 100644 --- a/nibabel/loadsave.py +++ b/nibabel/loadsave.py @@ -13,9 +13,8 @@ from .filename_parser import splitext_addext from .openers import ImageOpener -from .nifti2 import Nifti2Image, Nifti2Pair from .spatialimages import ImageFileError -from .imageclasses import class_map, ext_map, all_image_classes +from .imageclasses import all_image_classes from .arrayproxy import is_proxy @@ -59,14 +58,22 @@ def save(img, filename): ------- None ''' + + # Save the type as expected try: img.to_filename(filename) except ImageFileError: pass else: return - froot, ext, trailing = splitext_addext(filename, ('.gz', '.bz2')) + + # Be nice to users by making common implicit conversions + froot, ext, trailing = splitext_addext(filename, img._compressed_exts) + lext = ext.lower() + # Special-case Nifti singles and Pairs + from .nifti1 import Nifti1Image, Nifti1Pair # Inline imports, as this file + from .nifti2 import Nifti2Image, Nifti2Pair # really shouldn't reference any image type if type(img) == Nifti1Image and ext in ('.img', '.hdr'): klass = Nifti1Pair elif type(img) == Nifti2Image and ext in ('.img', '.hdr'): @@ -75,9 +82,14 @@ def save(img, filename): klass = Nifti1Image elif type(img) == Nifti2Pair and ext == '.nii': klass = Nifti2Image - else: - img_type = ext_map[ext] - klass = class_map[img_type]['class'] + else: # arbitrary conversion + valid_klasses = filter(lambda klass: klass.is_valid_extension(lext), + all_image_classes) + if len(valid_klasses) > 0: + klass = valid_klasses[0] + else: + raise ImageFileError('Cannot work out file type of "%s"' % + filename) converted = klass.from_image(img) converted.to_filename(filename) diff --git a/nibabel/tests/test_files_interface.py b/nibabel/tests/test_files_interface.py index 2c0bdfff0f..59839b3b96 100644 --- a/nibabel/tests/test_files_interface.py +++ b/nibabel/tests/test_files_interface.py @@ -12,7 +12,7 @@ import numpy as np -from .. import class_map, Nifti1Image, Nifti1Pair, MGHImage +from .. import Nifti1Image, Nifti1Pair, MGHImage, all_image_classes from ..externals.six import BytesIO from ..fileholders import FileHolderError @@ -25,15 +25,14 @@ def test_files_images(): # test files creation in image classes arr = np.zeros((2,3,4)) aff = np.eye(4) - for img_def in class_map.values(): - klass = img_def['class'] + for klass in all_image_classes: file_map = klass.make_file_map() for key, value in file_map.items(): assert_equal(value.filename, None) assert_equal(value.fileobj, None) assert_equal(value.pos, 0) # If we can't create new images in memory without loading, bail here - if not img_def['makeable']: + if not klass.makeable: continue # MGHImage accepts only a few datatypes # so we force a type change to float32 @@ -83,22 +82,21 @@ def test_files_interface(): def test_round_trip(): - # write an image to files - data = np.arange(24, dtype='i4').reshape((2,3,4)) - aff = np.eye(4) - klasses = [val['class'] for key, val in class_map.items() - if val['rw']] - for klass in klasses: - file_map = klass.make_file_map() - for key in file_map: - file_map[key].fileobj = BytesIO() - img = klass(data, aff) - img.file_map = file_map - img.to_file_map() - # read it back again from the written files - img2 = klass.from_file_map(file_map) - assert_array_equal(img2.get_data(), data) - # write, read it again - img2.to_file_map() - img3 = klass.from_file_map(file_map) - assert_array_equal(img3.get_data(), data) + # write an image to files + data = np.arange(24, dtype='i4').reshape((2,3,4)) + aff = np.eye(4) + klasses = filter(lambda klass: klass.rw, all_image_classes) + for klass in klasses: + file_map = klass.make_file_map() + for key in file_map: + file_map[key].fileobj = BytesIO() + img = klass(data, aff) + img.file_map = file_map + img.to_file_map() + # read it back again from the written files + img2 = klass.from_file_map(file_map) + assert_array_equal(img2.get_data(), data) + # write, read it again + img2.to_file_map() + img3 = klass.from_file_map(file_map) + assert_array_equal(img3.get_data(), data) diff --git a/nibabel/tests/test_image_load_save.py b/nibabel/tests/test_image_load_save.py index 9394dcb655..0d273ef5cb 100644 --- a/nibabel/tests/test_image_load_save.py +++ b/nibabel/tests/test_image_load_save.py @@ -26,7 +26,7 @@ from .. import loadsave as nils from .. import (Nifti1Image, Nifti1Header, Nifti1Pair, Nifti2Image, Nifti2Pair, Minc1Image, Minc2Image, Spm2AnalyzeImage, Spm99AnalyzeImage, - AnalyzeImage, MGHImage, class_map) + AnalyzeImage, MGHImage, all_image_classes) from ..tmpdirs import InTemporaryDirectory @@ -53,16 +53,14 @@ def test_conversion(): affine = np.diag([1, 2, 3, 1]) for npt in np.float32, np.int16: data = np.arange(np.prod(shape), dtype=npt).reshape(shape) - for r_class_def in class_map.values(): - r_class = r_class_def['class'] - if not r_class_def['makeable']: + for r_class in all_image_classes: + if not r_class.makeable: continue img = r_class(data, affine) img.set_data_dtype(npt) - for w_class_def in class_map.values(): - if not w_class_def['makeable']: + for w_class in all_image_classes: + if not w_class.makeable: continue - w_class = w_class_def['class'] img2 = w_class.from_image(img) assert_array_equal(img2.get_data(), data) assert_array_equal(img2.affine, affine) From 91202cf87637a4497e0413b8cd6082dc94f05a41 Mon Sep 17 00:00:00 2001 From: Ben Cipollini Date: Mon, 20 Jul 2015 09:13:16 -0700 Subject: [PATCH 06/30] Code cleanup after self code review, fix for Python 3 'filter' issue. --- nibabel/analyze.py | 5 ++--- nibabel/filename_parser.py | 4 ++-- nibabel/freesurfer/mghformat.py | 3 +-- nibabel/loadsave.py | 18 +++++++++--------- nibabel/minc1.py | 3 +-- nibabel/nifti1.py | 2 -- nibabel/parrec.py | 3 +-- nibabel/spatialimages.py | 27 ++++++++++++++------------- nibabel/spm2analyze.py | 1 - nibabel/spm99analyze.py | 1 - nibabel/tests/test_analyze.py | 6 +++--- nibabel/tests/test_filehandles.py | 6 +----- 12 files changed, 34 insertions(+), 45 deletions(-) diff --git a/nibabel/analyze.py b/nibabel/analyze.py index 02cd398eea..7aadcd423c 100644 --- a/nibabel/analyze.py +++ b/nibabel/analyze.py @@ -829,7 +829,7 @@ def _chk_datatype(klass, hdr, fix=False): dtype = klass._data_type_codes.dtype[code] except KeyError: rep.problem_level = 40 - rep.problem_msg = 'data code %d not recognized by %s' % (code, klass.__name__) + rep.problem_msg = 'data code %d not recognized' % code else: if dtype.itemsize == 0: rep.problem_level = 40 @@ -899,10 +899,9 @@ class AnalyzeImage(SpatialImage): header_class = AnalyzeHeader files_types = (('image', '.img'), ('header', '.hdr')) _compressed_exts = ('.gz', '.bz2') - has_affine = False + makeable = True rw = True - nickname = 'analyze' ImageArrayProxy = ArrayProxy diff --git a/nibabel/filename_parser.py b/nibabel/filename_parser.py index 6065716e7d..8965ed53e3 100644 --- a/nibabel/filename_parser.py +++ b/nibabel/filename_parser.py @@ -131,8 +131,8 @@ def types_filenames(template_fname, types_exts, elif found_ext == found_ext.lower(): proc_ext = lambda s: s.lower() for name, ext in types_exts: - if name in tfns: # priority to those found first. - continue + if name in tfns: # Allow multipe definitions of image, header, etc, + continue # giving priority to those found first. if name == direct_set_name: tfns[name] = template_fname continue diff --git a/nibabel/freesurfer/mghformat.py b/nibabel/freesurfer/mghformat.py index b6ed977695..54c1cfd92a 100644 --- a/nibabel/freesurfer/mghformat.py +++ b/nibabel/freesurfer/mghformat.py @@ -462,8 +462,7 @@ class MGHImage(SpatialImage): files_types = (('image', '.mgh'), ('image', '.mgz')) _compressed_exts = (('.gz',)) - nickname = 'mgh' - has_affine = True + makeable = True rw = True diff --git a/nibabel/loadsave.py b/nibabel/loadsave.py index 608eef1574..55bc7526cb 100644 --- a/nibabel/loadsave.py +++ b/nibabel/loadsave.py @@ -72,22 +72,22 @@ def save(img, filename): lext = ext.lower() # Special-case Nifti singles and Pairs - from .nifti1 import Nifti1Image, Nifti1Pair # Inline imports, as this file + from .nifti1 import Nifti1Image, Nifti1Pair # Inline imports, as this module from .nifti2 import Nifti2Image, Nifti2Pair # really shouldn't reference any image type - if type(img) == Nifti1Image and ext in ('.img', '.hdr'): + if type(img) == Nifti1Image and lext in ('.img', '.hdr'): klass = Nifti1Pair - elif type(img) == Nifti2Image and ext in ('.img', '.hdr'): + elif type(img) == Nifti2Image and lext in ('.img', '.hdr'): klass = Nifti2Pair - elif type(img) == Nifti1Pair and ext == '.nii': + elif type(img) == Nifti1Pair and lext == '.nii': klass = Nifti1Image - elif type(img) == Nifti2Pair and ext == '.nii': + elif type(img) == Nifti2Pair and lext == '.nii': klass = Nifti2Image else: # arbitrary conversion - valid_klasses = filter(lambda klass: klass.is_valid_extension(lext), + valid_klasses = filter(lambda klass: klass.is_valid_extension(ext), all_image_classes) - if len(valid_klasses) > 0: - klass = valid_klasses[0] - else: + try: + klass = next(iter(valid_klasses)) + except StopIteration: # if iterator is empty raise ImageFileError('Cannot work out file type of "%s"' % filename) converted = klass.from_image(img) diff --git a/nibabel/minc1.py b/nibabel/minc1.py index c54a137c1f..4eba0ed92e 100644 --- a/nibabel/minc1.py +++ b/nibabel/minc1.py @@ -300,10 +300,9 @@ class Minc1Image(SpatialImage): header_class = Minc1Header files_types = (('image', '.mnc'),) _compressed_exts = ('.gz', '.bz2') - has_affine = True + makeable = True rw = False - nickname = 'minc' ImageArrayProxy = MincImageArrayProxy diff --git a/nibabel/nifti1.py b/nibabel/nifti1.py index 0f3c516c1e..f59ba6b4d0 100644 --- a/nibabel/nifti1.py +++ b/nibabel/nifti1.py @@ -1634,7 +1634,6 @@ class Nifti1Pair(analyze.AnalyzeImage): """ Class for NIfTI1 format image, header pair """ header_class = Nifti1PairHeader - nickname = 'nifti_pair' rw = True def __init__(self, dataobj, affine, header=None, @@ -1860,7 +1859,6 @@ class Nifti1Image(Nifti1Pair): """ header_class = Nifti1Header files_types = (('image', '.nii'),) - nickname = 'nifti_single' @staticmethod def _get_fileholders(file_map): diff --git a/nibabel/parrec.py b/nibabel/parrec.py index 48eeeb765a..85fc30aa4e 100644 --- a/nibabel/parrec.py +++ b/nibabel/parrec.py @@ -1021,8 +1021,7 @@ class PARRECImage(SpatialImage): """PAR/REC image""" header_class = PARRECHeader files_types = (('image', '.rec'), ('header', '.par')) - nickname = 'par' - has_affine = True + makeable = False rw = False diff --git a/nibabel/spatialimages.py b/nibabel/spatialimages.py index 843552af23..a87aa9e6dd 100644 --- a/nibabel/spatialimages.py +++ b/nibabel/spatialimages.py @@ -326,15 +326,8 @@ class SpatialImage(object): files_types = (('image', None),) _compressed_exts = () - @classmethod - def is_valid_extension(klass, lext): - return np.any([ft[1] == lext for ft in klass.files_types]) - - @classmethod - def is_valid_filename(klass, filename): - froot, ext, trailing = splitext_addext(filename, klass._compressed_exts) - lext = ext.lower() - return klass.is_valid_extension(lext) + makeable = True # Used in test code + rw = True # Used in test code def __init__(self, dataobj, affine, header=None, extra=None, file_map=None): @@ -878,15 +871,23 @@ def from_image(klass, img): klass.header_class.from_header(img.header), extra=img.extra.copy()) + @classmethod + def is_valid_extension(klass, ext): + return np.any([ft[1] == ext.lower() for ft in klass.files_types]) + + @classmethod + def is_valid_filename(klass, filename): + froot, ext, trailing = splitext_addext(filename, klass._compressed_exts) + return klass.is_valid_extension(ext) + @classmethod def is_image(klass, filename, sniff=None): froot, ext, trailing = splitext_addext(filename, klass._compressed_exts) - lext = ext.lower() - if not klass.is_valid_extension(lext): + if not klass.is_valid_extension(ext): return False, sniff - elif (not hasattr(klass.header_class, 'sniff_size') or - not hasattr(klass.header_class, 'is_header')): + elif (getattr(klass.header_class, 'sniff_size', None) is None or + getattr(klass.header_class, 'is_header', None) is None): return True, sniff # Determine the metadata location, then sniff it diff --git a/nibabel/spm2analyze.py b/nibabel/spm2analyze.py index 4864522338..c3decf2af8 100644 --- a/nibabel/spm2analyze.py +++ b/nibabel/spm2analyze.py @@ -132,7 +132,6 @@ class Spm2AnalyzeImage(spm99.Spm99AnalyzeImage): """ Class for SPM2 variant of basic Analyze image """ header_class = Spm2AnalyzeHeader - nickname = 'spm2analyze' load = Spm2AnalyzeImage.load save = Spm2AnalyzeImage.instance_to_filename diff --git a/nibabel/spm99analyze.py b/nibabel/spm99analyze.py index 92c1af7c51..fdf6c2d31a 100644 --- a/nibabel/spm99analyze.py +++ b/nibabel/spm99analyze.py @@ -242,7 +242,6 @@ class Spm99AnalyzeImage(analyze.AnalyzeImage): has_affine = True makeable = True rw = have_scipy - nickname = 'spm99analyze' @classmethod @kw_only_meth(1) diff --git a/nibabel/tests/test_analyze.py b/nibabel/tests/test_analyze.py index 157b84dff9..55e7e39532 100644 --- a/nibabel/tests/test_analyze.py +++ b/nibabel/tests/test_analyze.py @@ -32,7 +32,7 @@ from ..arraywriters import WriterError from nose.tools import (assert_equal, assert_not_equal, assert_true, - assert_false, assert_raises, assert_in) + assert_false, assert_raises) from numpy.testing import (assert_array_equal, assert_array_almost_equal) @@ -157,8 +157,8 @@ def test_log_checks(self): hdr['datatype'] = -1 # severity 40 with suppress_warnings(): fhdr, message, raiser = self.log_chk(hdr, 40) - assert_in('data code -1 not recognized', message) - assert_in('not attempting fix', message) + assert_equal(message, 'data code -1 not recognized; ' + 'not attempting fix') assert_raises(*raiser) # datatype not supported diff --git a/nibabel/tests/test_filehandles.py b/nibabel/tests/test_filehandles.py index c1e6c718a1..2ecadf5840 100644 --- a/nibabel/tests/test_filehandles.py +++ b/nibabel/tests/test_filehandles.py @@ -34,7 +34,7 @@ def test_multiload(): if N > 5000: warn('It would take too long to test file handles, aborting') return - arr = np.arange(24).reshape((2, 3, 4)) + arr = np.arange(24).reshape((2,3,4)) img = Nifti1Image(arr, np.eye(4)) imgs = [] try: @@ -43,10 +43,6 @@ def test_multiload(): save(img, fname) for i in range(N): imgs.append(load(fname)) - except Exception as e: - if 'i' in locals(): - e.message += ' (i == %d)' % i - raise Exception(e.message) finally: del img, imgs shutil.rmtree(tmpdir) From 6f40ad8d49d56cb8dea26a45edac0542f35c2367 Mon Sep 17 00:00:00 2001 From: Ben Cipollini Date: Mon, 31 Aug 2015 18:43:48 -0700 Subject: [PATCH 07/30] Remove references to BinOpener --- nibabel/loadsave.py | 8 ++-- nibabel/minc1.py | 1 - nibabel/spatialimages.py | 5 ++- nibabel/tests/test_image_load_save.py | 65 ++++++++++++++++++++++++++- 4 files changed, 71 insertions(+), 8 deletions(-) diff --git a/nibabel/loadsave.py b/nibabel/loadsave.py index 55bc7526cb..20e9815122 100644 --- a/nibabel/loadsave.py +++ b/nibabel/loadsave.py @@ -83,11 +83,11 @@ def save(img, filename): elif type(img) == Nifti2Pair and lext == '.nii': klass = Nifti2Image else: # arbitrary conversion - valid_klasses = filter(lambda klass: klass.is_valid_extension(ext), - all_image_classes) + valid_klasses = [klass for klass in all_image_classes + if klass.is_valid_extension(ext)] try: - klass = next(iter(valid_klasses)) - except StopIteration: # if iterator is empty + klass = valid_klasses[0] + except IndexError: # if list is empty raise ImageFileError('Cannot work out file type of "%s"' % filename) converted = klass.from_image(img) diff --git a/nibabel/minc1.py b/nibabel/minc1.py index 4eba0ed92e..4f260215dc 100644 --- a/nibabel/minc1.py +++ b/nibabel/minc1.py @@ -17,7 +17,6 @@ from .filename_parser import splitext_addext from .spatialimages import Header, SpatialImage from .fileslice import canonical_slicers -from .volumeutils import BinOpener from .deprecated import FutureWarningMixin diff --git a/nibabel/spatialimages.py b/nibabel/spatialimages.py index a87aa9e6dd..85bc176863 100644 --- a/nibabel/spatialimages.py +++ b/nibabel/spatialimages.py @@ -145,7 +145,8 @@ from .filename_parser import types_filenames, TypesFilenamesError, \ splitext_addext from .fileholders import FileHolder -from .volumeutils import shape_zoom_affine, BinOpener +from .openers import ImageOpener +from .volumeutils import shape_zoom_affine class HeaderDataError(Exception): @@ -907,7 +908,7 @@ def is_image(klass, filename, sniff=None): if not sniff or len(sniff) < klass.header_class.sniff_size: # 1024 == large size, for efficiency (could iterate over imageclasses). sniff_size = np.max([1024, klass.header_class.sniff_size]) - with BinOpener(metadata_filename, 'rb') as fobj: + with ImageOpener(metadata_filename, 'rb') as fobj: sniff = fobj.read(sniff_size) return klass.header_class.is_header(sniff[:klass.header_class.sniff_size]), sniff except Exception as e: diff --git a/nibabel/tests/test_image_load_save.py b/nibabel/tests/test_image_load_save.py index 0d273ef5cb..f533f0515a 100644 --- a/nibabel/tests/test_image_load_save.py +++ b/nibabel/tests/test_image_load_save.py @@ -33,7 +33,7 @@ from ..volumeutils import native_code, swapped_code from numpy.testing import assert_array_equal, assert_array_almost_equal -from nose.tools import assert_true, assert_equal, assert_raises +from nose.tools import assert_true, assert_equal, assert_false, assert_raises DATA_PATH = pjoin(dirname(__file__), 'data') MGH_DATA_PATH = pjoin(dirname(__file__), '..', 'freesurfer', 'tests', 'data') @@ -65,6 +65,69 @@ def test_conversion(): assert_array_equal(img2.get_data(), data) assert_array_equal(img2.affine, affine) +def test_sniff_and_guessed_image_type(): + # Randomize the class order + + def test_image_class(img_path, expected_img_klass): + + def check_img(img_path, expected_img_klass, mode, sniff=None, expect_match=True, msg=''): + if mode == 'no_sniff': + is_img, _ = expected_img_klass.is_image(img_path) + else: + is_img, sniff = expected_img_klass.is_image(img_path, sniff) + + msg = '%s (%s) image is%s a %s image.' % ( + img_path, + msg, + '' if is_img else ' not', + klass.__name__) + from ..spatialimages import ImageFileError + try: + klass.from_filename(img_path) + # assert_true(is_img, msg) + print("Passed: " + msg) + except ImageFileError: + print("Failed (image load): " + msg) + except Exception as e: + print("Failed (%s): %s" % (str(e), msg)) + # if is_img: + # raise + # assert_false(is_img, msg) # , issubclass(expected_img_klass, klass) and expect_match, msg) + return sniff + + for mode in ['vanilla', 'no-sniff']: + if mode == 'random': + img_klasses = all_image_classes.copy() + np.random.shuffle(img_klasses) + else: + img_klasses = all_image_classes + + if mode == 'no_sniff': + all_sniffs = [None] + bad_sniff = None + else: + sizeof_hdr = getattr(expected_img_klass.header_class, 'sizeof_hdr', 0) + all_sniffs = [None, '', 'a' * (sizeof_hdr - 1)] + bad_sniff = 'a' * sizeof_hdr + + # Test that passing in different sniffs is OK + if bad_sniff is not None: + for klass in img_klasses: + check_img(img_path, expected_img_klass, mode=mode, + sniff=bad_sniff, expect_match=False, + msg='%s / %s / %s' % (expected_img_klass.__name__, mode, 'bad_sniff')) + + for si, sniff in enumerate(all_sniffs): + for klass in img_klasses: + sniff = check_img(img_path, expected_img_klass, mode=mode, + sniff=sniff, expect_match=True, + msg='%s / %s / %d' % (expected_img_klass.__name__, mode, si)) + + + + # Test whether we can guess the image type from example files + test_image_class(pjoin(DATA_PATH, 'analyze.hdr'), + Spm2AnalyzeImage) def test_save_load_endian(): shape = (2, 4, 6) From 7635ed229df19e8e28bf59dd6c597c5fc002f85f Mon Sep 17 00:00:00 2001 From: Ben Cipollini Date: Tue, 1 Sep 2015 10:27:04 -0700 Subject: [PATCH 08/30] Linting, cleaning up logic. --- nibabel/minc1.py | 2 -- nibabel/spatialimages.py | 16 ++++++++++------ nibabel/tests/test_analyze.py | 2 +- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/nibabel/minc1.py b/nibabel/minc1.py index 4f260215dc..13d148091e 100644 --- a/nibabel/minc1.py +++ b/nibabel/minc1.py @@ -270,8 +270,6 @@ class MincHeader(Header): """ # We don't use the data layout - this just in case we do later data_layout = 'C' - - # sniff_size = 4 def data_to_fileobj(self, data, fileobj, rescale=True): diff --git a/nibabel/spatialimages.py b/nibabel/spatialimages.py index 85bc176863..f9b82d54e5 100644 --- a/nibabel/spatialimages.py +++ b/nibabel/spatialimages.py @@ -887,8 +887,7 @@ def is_image(klass, filename, sniff=None): if not klass.is_valid_extension(ext): return False, sniff - elif (getattr(klass.header_class, 'sniff_size', None) is None or - getattr(klass.header_class, 'is_header', None) is None): + elif not hasattr(klass.header_class, 'is_header'): return True, sniff # Determine the metadata location, then sniff it @@ -905,17 +904,22 @@ def is_image(klass, filename, sniff=None): break try: - if not sniff or len(sniff) < klass.header_class.sniff_size: + klass_sniff_size = getattr(klass.header_class, 'sniff_size', 0) + + if not sniff or len(sniff) < klass_sniff_size: # 1024 == large size, for efficiency (could iterate over imageclasses). - sniff_size = np.max([1024, klass.header_class.sniff_size]) + sniff_size = np.max([1024, klass_sniff_size]) with ImageOpener(metadata_filename, 'rb') as fobj: sniff = fobj.read(sniff_size) - return klass.header_class.is_header(sniff[:klass.header_class.sniff_size]), sniff + + is_header = klass.header_class.is_header(sniff[:klass_sniff_size]) except Exception as e: # Can happen if: file doesn't exist, # filesize < necessary sniff size (this happens!) # other unexpected errors. - return False, sniff + is_header = False + + return is_header, sniff def __getitem__(self): ''' No slicing or dictionary interface for images diff --git a/nibabel/tests/test_analyze.py b/nibabel/tests/test_analyze.py index 55e7e39532..4a0a4180ab 100644 --- a/nibabel/tests/test_analyze.py +++ b/nibabel/tests/test_analyze.py @@ -141,7 +141,7 @@ def test_log_checks(self): # magic hdr = HC() with suppress_warnings(): - hdr['sizeof_hdr'] = 350 # severity 30 + hdr['sizeof_hdr'] = 350 # severity 30 fhdr, message, raiser = self.log_chk(hdr, 30) assert_equal(fhdr['sizeof_hdr'], self.sizeof_hdr) assert_equal(message, From fab693e5288fc6b81114bcb4c0096dcc161e658e Mon Sep 17 00:00:00 2001 From: Ben Cipollini Date: Tue, 1 Sep 2015 10:31:27 -0700 Subject: [PATCH 09/30] Adding tests for loading each image type. --- nibabel/tests/test_image_load_save.py | 78 ++------------- nibabel/tests/test_image_types.py | 133 ++++++++++++++++++++++++++ 2 files changed, 139 insertions(+), 72 deletions(-) create mode 100644 nibabel/tests/test_image_types.py diff --git a/nibabel/tests/test_image_load_save.py b/nibabel/tests/test_image_load_save.py index f533f0515a..261473d856 100644 --- a/nibabel/tests/test_image_load_save.py +++ b/nibabel/tests/test_image_load_save.py @@ -8,17 +8,14 @@ ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## ''' Tests for loader function ''' from __future__ import division, print_function, absolute_import -from os.path import join as pjoin, dirname +from ..externals.six import BytesIO + import shutil +from os.path import dirname, join as pjoin from tempfile import mkdtemp -from ..externals.six import BytesIO import numpy as np -# If we don't have scipy, then we cannot write SPM format files -from ..optpkg import optional_package -_, have_scipy, _ = optional_package('scipy') - from .. import analyze as ana from .. import spm99analyze as spm99 from .. import spm2analyze as spm2 @@ -27,14 +24,14 @@ from .. import (Nifti1Image, Nifti1Header, Nifti1Pair, Nifti2Image, Nifti2Pair, Minc1Image, Minc2Image, Spm2AnalyzeImage, Spm99AnalyzeImage, AnalyzeImage, MGHImage, all_image_classes) - from ..tmpdirs import InTemporaryDirectory - from ..volumeutils import native_code, swapped_code +from ..optpkg import optional_package from numpy.testing import assert_array_equal, assert_array_almost_equal -from nose.tools import assert_true, assert_equal, assert_false, assert_raises +from nose.tools import assert_true, assert_equal +_, have_scipy, _ = optional_package('scipy') # No scipy=>no SPM-format writing DATA_PATH = pjoin(dirname(__file__), 'data') MGH_DATA_PATH = pjoin(dirname(__file__), '..', 'freesurfer', 'tests', 'data') @@ -65,69 +62,6 @@ def test_conversion(): assert_array_equal(img2.get_data(), data) assert_array_equal(img2.affine, affine) -def test_sniff_and_guessed_image_type(): - # Randomize the class order - - def test_image_class(img_path, expected_img_klass): - - def check_img(img_path, expected_img_klass, mode, sniff=None, expect_match=True, msg=''): - if mode == 'no_sniff': - is_img, _ = expected_img_klass.is_image(img_path) - else: - is_img, sniff = expected_img_klass.is_image(img_path, sniff) - - msg = '%s (%s) image is%s a %s image.' % ( - img_path, - msg, - '' if is_img else ' not', - klass.__name__) - from ..spatialimages import ImageFileError - try: - klass.from_filename(img_path) - # assert_true(is_img, msg) - print("Passed: " + msg) - except ImageFileError: - print("Failed (image load): " + msg) - except Exception as e: - print("Failed (%s): %s" % (str(e), msg)) - # if is_img: - # raise - # assert_false(is_img, msg) # , issubclass(expected_img_klass, klass) and expect_match, msg) - return sniff - - for mode in ['vanilla', 'no-sniff']: - if mode == 'random': - img_klasses = all_image_classes.copy() - np.random.shuffle(img_klasses) - else: - img_klasses = all_image_classes - - if mode == 'no_sniff': - all_sniffs = [None] - bad_sniff = None - else: - sizeof_hdr = getattr(expected_img_klass.header_class, 'sizeof_hdr', 0) - all_sniffs = [None, '', 'a' * (sizeof_hdr - 1)] - bad_sniff = 'a' * sizeof_hdr - - # Test that passing in different sniffs is OK - if bad_sniff is not None: - for klass in img_klasses: - check_img(img_path, expected_img_klass, mode=mode, - sniff=bad_sniff, expect_match=False, - msg='%s / %s / %s' % (expected_img_klass.__name__, mode, 'bad_sniff')) - - for si, sniff in enumerate(all_sniffs): - for klass in img_klasses: - sniff = check_img(img_path, expected_img_klass, mode=mode, - sniff=sniff, expect_match=True, - msg='%s / %s / %d' % (expected_img_klass.__name__, mode, si)) - - - - # Test whether we can guess the image type from example files - test_image_class(pjoin(DATA_PATH, 'analyze.hdr'), - Spm2AnalyzeImage) def test_save_load_endian(): shape = (2, 4, 6) diff --git a/nibabel/tests/test_image_types.py b/nibabel/tests/test_image_types.py new file mode 100644 index 0000000000..979e0624b9 --- /dev/null +++ b/nibabel/tests/test_image_types.py @@ -0,0 +1,133 @@ +# emacs: -*- mode: python-mode; py-indent-offset: 4; indent-tabs-mode: nil -*- +# vi: set ft=python sts=4 ts=4 sw=4 et: +### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## +# +# See COPYING file distributed along with the NiBabel package for the +# copyright and license terms. +# +### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## +''' Tests for is_image / is_header functions ''' +from __future__ import division, print_function, absolute_import + +import copy +from os.path import dirname, basename, join as pjoin + +import numpy as np + +from .. import (Nifti1Image, Nifti1Header, Nifti1Pair, Nifti2Image, Nifti2Pair, + Minc1Image, Minc2Image, Spm2AnalyzeImage, Spm99AnalyzeImage, + AnalyzeImage, MGHImage, all_image_classes) + +from nose.tools import assert_true, assert_equal, assert_false, assert_raises + +DATA_PATH = pjoin(dirname(__file__), 'data') + + +def test_sniff_and_guessed_image_type(img_klasses=all_image_classes): + """ + Loop over all test cases: + * whether a sniff is provided or not + * randomizing the order of image classes + * over all known image types + + For each, we expect: + * When the file matches the expected class, things should + either work, or fail if we're doing bad stuff. + * When the file is a mismatch, it should either + * Fail to be loaded if the image type is unrelated to the expected class + * Load or fail in a consistent manner, if there is a relationship between + the image class and expected image class. + """ + + def test_image_class(img_path, expected_img_klass): + """ Embedded function to compare an image of one image class to all others. + + The function should make sure that it loads the image with the expected class, + but failing when given a bad sniff (when the sniff is used).""" + + def check_img(img_path, img_klass, sniff_mode, sniff, expect_success, msg): + """Embedded function to do the actual checks expected.""" + + if sniff_mode == 'no_sniff': + # Don't pass any sniff--not even "None" + is_img, new_sniff = img_klass.is_image(img_path) + else: + # Pass a sniff, but don't reuse across images. + is_img, new_sniff = img_klass.is_image(img_path, sniff) + + if expect_success: + new_msg = '%s returned sniff==None (%s)' % (img_klass.__name__, msg) + expected_sniff_size = getattr(img_klass.header_class, 'sniff_size', 0) + current_sniff_size = len(new_sniff) if new_sniff is not None else 0 + assert_true(current_sniff_size >= expected_sniff_size, new_msg) + + # Build a message to the user. + new_msg = '%s (%s) image is%s a %s image.' % ( + basename(img_path), + msg, + '' if is_img else ' not', + img_klass.__name__) + + if expect_success is None: + assert_true(True, new_msg) # No expectation, pass if no Exception + # elif is_img != expect_success: + # print('Failed! %s' % new_msg) + else: + assert_equal(is_img, expect_success, new_msg) + + if sniff_mode == 'vanilla': + return new_sniff + else: + return sniff + + sniff_size = getattr(expected_img_klass.header_class, 'sniff_size', 0) + + for sniff_mode, sniff in dict( + vanilla=None, # use the sniff of the previous item + no_sniff=None, # Don't pass a sniff + none=None, # pass None as the sniff, should query in fn + empty='', # pass an empty sniff, should query in fn + irrelevant='a' * (sniff_size - 1), # A too-small sniff, query + bad_sniff='a' * sniff_size).items(): # Bad sniff, should fail. + + for klass in img_klasses: + if klass == expected_img_klass: + expect_success = (sniff_mode not in ['bad_sniff'] or + sniff_size == 0 or + klass == Minc1Image) # special case... + elif (issubclass(klass, expected_img_klass) or + issubclass(expected_img_klass, klass)): + expect_success = None # Images are related; can't be sure. + else: + # Usually, if the two images are unrelated, they + # won't be able to be loaded. But here's a + # list of manually confirmed special cases + expect_success = ((expected_img_klass == Nifti1Pair and klass == Spm99AnalyzeImage) or + (expected_img_klass == Nifti2Pair and klass == Spm99AnalyzeImage)) + + msg = '%s / %s / %s' % (expected_img_klass.__name__, sniff_mode, str(expect_success)) + print(msg) + # Reuse the sniff... but it will only change for some + # sniff_mode values. + sniff = check_img(img_path, klass, sniff_mode=sniff_mode, + sniff=sniff, expect_success=expect_success, + msg=msg) + + # Test whether we can guess the image type from example files + for img_filename, image_klass in [('example4d.nii.gz', Nifti1Image), + ('nifti1.hdr', Nifti1Pair), + ('example_nifti2.nii.gz', Nifti2Image), + ('nifti2.hdr', Nifti2Pair), + ('tiny.mnc', Minc1Image), + ('small.mnc', Minc2Image), + ('test.mgz', MGHImage), + ('analyze.hdr', Spm2AnalyzeImage)]: + print('Testing: %s %s' % (img_filename, image_klass.__name__)) + test_image_class(pjoin(DATA_PATH, img_filename), image_klass) + + +def test_sniff_and_guessed_image_type_randomized(): + """Re-test image classes, but in a randomized order.""" + img_klasses = copy.copy(all_image_classes) + np.random.shuffle(img_klasses) + test_sniff_and_guessed_image_type(img_klasses=img_klasses) From 887423fcce79a7a513f34c19122e5838a6720325 Mon Sep 17 00:00:00 2001 From: Ben Cipollini Date: Tue, 1 Sep 2015 11:10:04 -0700 Subject: [PATCH 10/30] Add back analyze header tests. --- nibabel/analyze.py | 11 ++++++ nibabel/tests/test_image_types.py | 66 +++++++++++++++++++++++++++++-- 2 files changed, 74 insertions(+), 3 deletions(-) diff --git a/nibabel/analyze.py b/nibabel/analyze.py index 7aadcd423c..33c028bf1a 100644 --- a/nibabel/analyze.py +++ b/nibabel/analyze.py @@ -190,6 +190,7 @@ class AnalyzeHeader(LabeledWrapStruct): has_data_intercept = False sizeof_hdr = 348 + sniff_size = 348 def __init__(self, binaryblock=None, @@ -892,6 +893,16 @@ def _chk_pixdims(hdr, fix=False): rep.fix_msg = ' and '.join(fmsgs) return hdr, rep + @classmethod + def is_header(klass, binaryblock): + if len(binaryblock) < klass.sniff_size: + raise ValueError('Must pass a binary block >= %d bytes' % klass.sniff_size) + + hdr = np.ndarray(shape=(), dtype=header_dtype, + buffer=binaryblock[:klass.sniff_size]) + bs_hdr = hdr.byteswap() + return 348 in (hdr['sizeof_hdr'], bs_hdr['sizeof_hdr']) + class AnalyzeImage(SpatialImage): """ Class for basic Analyze format image diff --git a/nibabel/tests/test_image_types.py b/nibabel/tests/test_image_types.py index 979e0624b9..16b86512dc 100644 --- a/nibabel/tests/test_image_types.py +++ b/nibabel/tests/test_image_types.py @@ -14,15 +14,75 @@ import numpy as np -from .. import (Nifti1Image, Nifti1Header, Nifti1Pair, Nifti2Image, Nifti2Pair, - Minc1Image, Minc2Image, Spm2AnalyzeImage, Spm99AnalyzeImage, - AnalyzeImage, MGHImage, all_image_classes) +from .. import (Nifti1Image, Nifti1Header, Nifti1Pair, + Nifti2Image, Nifti2Header, Nifti2Pair, + AnalyzeImage, AnalyzeHeader, + Minc1Image, Minc2Image, + Spm2AnalyzeImage, Spm99AnalyzeImage, + MGHImage, all_image_classes) from nose.tools import assert_true, assert_equal, assert_false, assert_raises DATA_PATH = pjoin(dirname(__file__), 'data') + +def test_analyze_detection(): + # Test detection of Analyze, Nifti1 and Nifti2 + # Algorithm is as described in loadsave:which_analyze_type + def wat(hdr): + all_analyze_header_klasses = [Nifti1Header, Nifti2Header, + AnalyzeHeader] + for klass in all_analyze_header_klasses: + try: + if klass.is_header(hdr.binaryblock): + return klass + else: + print('checked completed, but failed.') + except ValueError as ve: + print(ve) + continue + return None + # return nils.which_analyze_type(hdr.binaryblock) + + n1_hdr = Nifti1Header(b'\0' * 348, check=False) + n2_hdr = Nifti2Header(b'\0' * 540, check=False) + assert_equal(wat(n1_hdr), None) + + n1_hdr['sizeof_hdr'] = 540 + n2_hdr['sizeof_hdr'] = 540 + assert_equal(wat(n1_hdr), None) + assert_equal(wat(n1_hdr.as_byteswapped()), None) + assert_equal(wat(n2_hdr), Nifti2Header) + assert_equal(wat(n2_hdr.as_byteswapped()), Nifti2Header) + + n1_hdr['sizeof_hdr'] = 348 + assert_equal(wat(n1_hdr), AnalyzeHeader) + assert_equal(wat(n1_hdr.as_byteswapped()), AnalyzeHeader) + + n1_hdr['magic'] = b'n+1' + assert_equal(wat(n1_hdr), Nifti1Header) + assert_equal(wat(n1_hdr.as_byteswapped()), Nifti1Header) + + n1_hdr['magic'] = b'ni1' + assert_equal(wat(n1_hdr), Nifti1Header) + assert_equal(wat(n1_hdr.as_byteswapped()), Nifti1Header) + + # Doesn't matter what magic is if it's not a nifti1 magic + n1_hdr['magic'] = b'ni2' + assert_equal(wat(n1_hdr), AnalyzeHeader) + + n1_hdr['sizeof_hdr'] = 0 + n1_hdr['magic'] = b'' + assert_equal(wat(n1_hdr), None) + + n1_hdr['magic'] = 'n+1' + assert_equal(wat(n1_hdr), Nifti1Header) + + n1_hdr['magic'] = 'ni1' + assert_equal(wat(n1_hdr), Nifti1Header) + + def test_sniff_and_guessed_image_type(img_klasses=all_image_classes): """ Loop over all test cases: From e2a6fd8cc2f955976badc7ef6f2392c10f401c6f Mon Sep 17 00:00:00 2001 From: Ben Cipollini Date: Tue, 1 Sep 2015 11:34:05 -0700 Subject: [PATCH 11/30] sniff_size => sizeof_hdr --- nibabel/analyze.py | 7 +++---- nibabel/minc1.py | 2 +- nibabel/nifti1.py | 9 +++------ nibabel/nifti2.py | 9 +++------ nibabel/spatialimages.py | 10 +++++----- nibabel/spm2analyze.py | 9 +++------ nibabel/tests/test_image_types.py | 14 +++++++------- 7 files changed, 25 insertions(+), 35 deletions(-) diff --git a/nibabel/analyze.py b/nibabel/analyze.py index 33c028bf1a..389d552083 100644 --- a/nibabel/analyze.py +++ b/nibabel/analyze.py @@ -190,7 +190,6 @@ class AnalyzeHeader(LabeledWrapStruct): has_data_intercept = False sizeof_hdr = 348 - sniff_size = 348 def __init__(self, binaryblock=None, @@ -895,11 +894,11 @@ def _chk_pixdims(hdr, fix=False): @classmethod def is_header(klass, binaryblock): - if len(binaryblock) < klass.sniff_size: - raise ValueError('Must pass a binary block >= %d bytes' % klass.sniff_size) + if len(binaryblock) < klass.sizeof_hdr: + raise ValueError('Must pass a binary block >= %d bytes' % klass.sizeof_hdr) hdr = np.ndarray(shape=(), dtype=header_dtype, - buffer=binaryblock[:klass.sniff_size]) + buffer=binaryblock[:klass.sizeof_hdr]) bs_hdr = hdr.byteswap() return 348 in (hdr['sizeof_hdr'], bs_hdr['sizeof_hdr']) diff --git a/nibabel/minc1.py b/nibabel/minc1.py index 13d148091e..dafda41a14 100644 --- a/nibabel/minc1.py +++ b/nibabel/minc1.py @@ -270,7 +270,7 @@ class MincHeader(Header): """ # We don't use the data layout - this just in case we do later data_layout = 'C' - sniff_size = 4 + sizeof_hdr = 4 def data_to_fileobj(self, data, fileobj, rescale=True): """ See Header class for an implementation we can't use """ diff --git a/nibabel/nifti1.py b/nibabel/nifti1.py index f59ba6b4d0..9a63f9e245 100644 --- a/nibabel/nifti1.py +++ b/nibabel/nifti1.py @@ -558,9 +558,6 @@ class Nifti1Header(SpmAnalyzeHeader): pair_magic = b'ni1' single_magic = b'n+1' - # for sniffing type - sniff_size = 348 - # Quaternion threshold near 0, based on float32 precision quaternion_threshold = -np.finfo(np.float32).eps * 3 @@ -1616,11 +1613,11 @@ def _chk_xform_code(klass, code_type, hdr, fix): @classmethod def is_header(klass, binaryblock): - if len(binaryblock) < klass.sniff_size: - raise ValueError('Must pass a binary block >= %d bytes' % klass.sniff_size) + if len(binaryblock) < klass.sizeof_hdr: + raise ValueError('Must pass a binary block >= %d bytes' % klass.sizeof_hdr) hdr = np.ndarray(shape=(), dtype=header_dtype, - buffer=binaryblock[:klass.sniff_size]) + buffer=binaryblock[:klass.sizeof_hdr]) return hdr['magic'] in (b'ni1', b'n+1') diff --git a/nibabel/nifti2.py b/nibabel/nifti2.py index 54b760c02e..87523bd0de 100644 --- a/nibabel/nifti2.py +++ b/nibabel/nifti2.py @@ -141,9 +141,6 @@ class Nifti2Header(Nifti1Header): # Size of header in sizeof_hdr field sizeof_hdr = 540 - # sniff size to determine type - sniff_size = 540 - # Quaternion threshold near 0, based on float64 preicision quaternion_threshold = -np.finfo(np.float64).eps * 3 @@ -226,11 +223,11 @@ def _chk_eol_check(hdr, fix=False): @classmethod def is_header(klass, binaryblock): - if len(binaryblock) < klass.sniff_size: - raise ValueError('Must pass a binary block >= %d bytes' % klass.sniff_size) + if len(binaryblock) < klass.sizeof_hdr: + raise ValueError('Must pass a binary block >= %d bytes' % klass.sizeof_hdr) hdr = np.ndarray(shape=(), dtype=header_dtype, - buffer=binaryblock[:klass.sniff_size]) + buffer=binaryblock[:klass.sizeof_hdr]) bs_hdr = hdr.byteswap() return 540 in (hdr['sizeof_hdr'], bs_hdr['sizeof_hdr']) diff --git a/nibabel/spatialimages.py b/nibabel/spatialimages.py index f9b82d54e5..bb4e9eb601 100644 --- a/nibabel/spatialimages.py +++ b/nibabel/spatialimages.py @@ -904,15 +904,15 @@ def is_image(klass, filename, sniff=None): break try: - klass_sniff_size = getattr(klass.header_class, 'sniff_size', 0) + klass_sizeof_hdr = getattr(klass.header_class, 'sizeof_hdr', 0) - if not sniff or len(sniff) < klass_sniff_size: + if not sniff or len(sniff) < klass_sizeof_hdr: # 1024 == large size, for efficiency (could iterate over imageclasses). - sniff_size = np.max([1024, klass_sniff_size]) + sizeof_hdr = np.max([1024, klass_sizeof_hdr]) with ImageOpener(metadata_filename, 'rb') as fobj: - sniff = fobj.read(sniff_size) + sniff = fobj.read(sizeof_hdr) - is_header = klass.header_class.is_header(sniff[:klass_sniff_size]) + is_header = klass.header_class.is_header(sniff[:klass_sizeof_hdr]) except Exception as e: # Can happen if: file doesn't exist, # filesize < necessary sniff size (this happens!) diff --git a/nibabel/spm2analyze.py b/nibabel/spm2analyze.py index c3decf2af8..7611212109 100644 --- a/nibabel/spm2analyze.py +++ b/nibabel/spm2analyze.py @@ -35,9 +35,6 @@ class Spm2AnalyzeHeader(spm99.Spm99AnalyzeHeader): # Copies of module level definitions template_dtype = header_dtype - # binary read size to determine type - sniff_size = 348 - def get_slope_inter(self): ''' Get data scaling (slope) and intercept from header data @@ -118,11 +115,11 @@ def get_slope_inter(self): @classmethod def is_header(klass, binaryblock): - if len(binaryblock) < klass.sniff_size: - raise ValueError('Must pass a binary block >= %d bytes' % klass.sniff_size) + if len(binaryblock) < klass.sizeof_hdr: + raise ValueError('Must pass a binary block >= %d bytes' % klass.sizeof_hdr) hdr = np.ndarray(shape=(), dtype=header_dtype, - buffer=binaryblock[:klass.sniff_size]) + buffer=binaryblock[:klass.sizeof_hdr]) bs_hdr = hdr.byteswap() return (binaryblock[344:348] not in (b'ni1\x00', b'n+1\x00') and 348 in (hdr['sizeof_hdr'], bs_hdr['sizeof_hdr'])) diff --git a/nibabel/tests/test_image_types.py b/nibabel/tests/test_image_types.py index 16b86512dc..b96fe23791 100644 --- a/nibabel/tests/test_image_types.py +++ b/nibabel/tests/test_image_types.py @@ -117,9 +117,9 @@ def check_img(img_path, img_klass, sniff_mode, sniff, expect_success, msg): if expect_success: new_msg = '%s returned sniff==None (%s)' % (img_klass.__name__, msg) - expected_sniff_size = getattr(img_klass.header_class, 'sniff_size', 0) - current_sniff_size = len(new_sniff) if new_sniff is not None else 0 - assert_true(current_sniff_size >= expected_sniff_size, new_msg) + expected_sizeof_hdr = getattr(img_klass.header_class, 'sizeof_hdr', 0) + current_sizeof_hdr = len(new_sniff) if new_sniff is not None else 0 + assert_true(current_sizeof_hdr >= expected_sizeof_hdr, new_msg) # Build a message to the user. new_msg = '%s (%s) image is%s a %s image.' % ( @@ -140,20 +140,20 @@ def check_img(img_path, img_klass, sniff_mode, sniff, expect_success, msg): else: return sniff - sniff_size = getattr(expected_img_klass.header_class, 'sniff_size', 0) + sizeof_hdr = getattr(expected_img_klass.header_class, 'sizeof_hdr', 0) for sniff_mode, sniff in dict( vanilla=None, # use the sniff of the previous item no_sniff=None, # Don't pass a sniff none=None, # pass None as the sniff, should query in fn empty='', # pass an empty sniff, should query in fn - irrelevant='a' * (sniff_size - 1), # A too-small sniff, query - bad_sniff='a' * sniff_size).items(): # Bad sniff, should fail. + irrelevant='a' * (sizeof_hdr - 1), # A too-small sniff, query + bad_sniff='a' * sizeof_hdr).items(): # Bad sniff, should fail. for klass in img_klasses: if klass == expected_img_klass: expect_success = (sniff_mode not in ['bad_sniff'] or - sniff_size == 0 or + sizeof_hdr == 0 or klass == Minc1Image) # special case... elif (issubclass(klass, expected_img_klass) or issubclass(expected_img_klass, klass)): From 17b8253229d28432fc3799e3a8b6c1bbd646663a Mon Sep 17 00:00:00 2001 From: Ben Cipollini Date: Tue, 1 Sep 2015 11:35:49 -0700 Subject: [PATCH 12/30] Simplify is_image test. --- nibabel/tests/test_image_types.py | 49 ++++++++++++------------------- 1 file changed, 19 insertions(+), 30 deletions(-) diff --git a/nibabel/tests/test_image_types.py b/nibabel/tests/test_image_types.py index b96fe23791..4316173135 100644 --- a/nibabel/tests/test_image_types.py +++ b/nibabel/tests/test_image_types.py @@ -93,10 +93,7 @@ def test_sniff_and_guessed_image_type(img_klasses=all_image_classes): For each, we expect: * When the file matches the expected class, things should either work, or fail if we're doing bad stuff. - * When the file is a mismatch, it should either - * Fail to be loaded if the image type is unrelated to the expected class - * Load or fail in a consistent manner, if there is a relationship between - the image class and expected image class. + * When the file is a mismatch, the functions should not throw. """ def test_image_class(img_path, expected_img_klass): @@ -116,24 +113,19 @@ def check_img(img_path, img_klass, sniff_mode, sniff, expect_success, msg): is_img, new_sniff = img_klass.is_image(img_path, sniff) if expect_success: + # Check that the sniff returned is appropriate. new_msg = '%s returned sniff==None (%s)' % (img_klass.__name__, msg) expected_sizeof_hdr = getattr(img_klass.header_class, 'sizeof_hdr', 0) current_sizeof_hdr = len(new_sniff) if new_sniff is not None else 0 assert_true(current_sizeof_hdr >= expected_sizeof_hdr, new_msg) - # Build a message to the user. - new_msg = '%s (%s) image is%s a %s image.' % ( - basename(img_path), - msg, - '' if is_img else ' not', - img_klass.__name__) - - if expect_success is None: - assert_true(True, new_msg) # No expectation, pass if no Exception - # elif is_img != expect_success: - # print('Failed! %s' % new_msg) - else: - assert_equal(is_img, expect_success, new_msg) + # Check that the image type was recognized. + new_msg = '%s (%s) image is%s a %s image.' % ( + basename(img_path), + msg, + '' if is_img else ' not', + img_klass.__name__) + assert_true(is_img, new_msg) if sniff_mode == 'vanilla': return new_sniff @@ -152,23 +144,20 @@ def check_img(img_path, img_klass, sniff_mode, sniff, expect_success, msg): for klass in img_klasses: if klass == expected_img_klass: - expect_success = (sniff_mode not in ['bad_sniff'] or + # Class will load unless you pass a bad sniff, + # the header actually uses the sniff, and the + # sniff check is actually something meaningful + # (we're looking at you, Minc1Header...) + expect_success = (sniff_mode != 'bad_sniff' or sizeof_hdr == 0 or klass == Minc1Image) # special case... - elif (issubclass(klass, expected_img_klass) or - issubclass(expected_img_klass, klass)): - expect_success = None # Images are related; can't be sure. else: - # Usually, if the two images are unrelated, they - # won't be able to be loaded. But here's a - # list of manually confirmed special cases - expect_success = ((expected_img_klass == Nifti1Pair and klass == Spm99AnalyzeImage) or - (expected_img_klass == Nifti2Pair and klass == Spm99AnalyzeImage)) - - msg = '%s / %s / %s' % (expected_img_klass.__name__, sniff_mode, str(expect_success)) - print(msg) + expect_success = False # Not sure the relationships + # Reuse the sniff... but it will only change for some # sniff_mode values. + msg = '%s/ %s/ %s' % (expected_img_klass.__name__, sniff_mode, + str(expect_success)) sniff = check_img(img_path, klass, sniff_mode=sniff_mode, sniff=sniff, expect_success=expect_success, msg=msg) @@ -182,7 +171,7 @@ def check_img(img_path, img_klass, sniff_mode, sniff, expect_success, msg): ('small.mnc', Minc2Image), ('test.mgz', MGHImage), ('analyze.hdr', Spm2AnalyzeImage)]: - print('Testing: %s %s' % (img_filename, image_klass.__name__)) + # print('Testing: %s %s' % (img_filename, image_klass.__name__)) test_image_class(pjoin(DATA_PATH, img_filename), image_klass) From 4eec23d4ef7ef99cab133bdf771c6549d37c4c96 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Tue, 4 Aug 2015 15:49:19 -0400 Subject: [PATCH 13/30] Relegate slicing to is_header, remove _minctest Use more definitive test for MINC1 images Remove unused import --- nibabel/minc1.py | 7 +++++-- nibabel/minc2.py | 10 +++++----- nibabel/spatialimages.py | 2 +- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/nibabel/minc1.py b/nibabel/minc1.py index dafda41a14..387f393a70 100644 --- a/nibabel/minc1.py +++ b/nibabel/minc1.py @@ -14,7 +14,6 @@ from .externals.netcdf import netcdf_file -from .filename_parser import splitext_addext from .spatialimages import Header, SpatialImage from .fileslice import canonical_slicers @@ -284,7 +283,11 @@ def data_from_fileobj(self, fileobj): class Minc1Header(MincHeader): @classmethod def is_header(klass, binaryblock): - return binaryblock != b'\211HDF' + if len(binaryblock) < klass.sizeof_hdr: + raise ValueError('Must pass a binary block >= %d bytes' % + klass.sizeof_hdr) + + return binaryblock[:klass.sizeof_hdr] == b'CDF\x01' class Minc1Image(SpatialImage): diff --git a/nibabel/minc2.py b/nibabel/minc2.py index 9572c271c5..c0a13f5584 100644 --- a/nibabel/minc2.py +++ b/nibabel/minc2.py @@ -137,7 +137,11 @@ def get_scaled_data(self, sliceobj=()): class Minc2Header(MincHeader): @classmethod def is_header(klass, binaryblock): - return binaryblock == b'\211HDF' + if len(binaryblock) < klass.sizeof_hdr: + raise ValueError('Must pass a binary block >= %d bytes' % + klass.sizeof_hdr) + + return binaryblock[:klass.sizeof_hdr] == b'\211HDF' class Minc2Image(Minc1Image): @@ -167,9 +171,5 @@ def from_file_map(klass, file_map): data = klass.ImageArrayProxy(minc_file) return klass(data, affine, header, extra=None, file_map=file_map) - @classmethod - def _minctest(klass, binaryblock): - return binaryblock[:4] == b'\211HDF' - load = Minc2Image.load diff --git a/nibabel/spatialimages.py b/nibabel/spatialimages.py index bb4e9eb601..0a1b3aafc2 100644 --- a/nibabel/spatialimages.py +++ b/nibabel/spatialimages.py @@ -912,7 +912,7 @@ def is_image(klass, filename, sniff=None): with ImageOpener(metadata_filename, 'rb') as fobj: sniff = fobj.read(sizeof_hdr) - is_header = klass.header_class.is_header(sniff[:klass_sizeof_hdr]) + is_header = klass.header_class.is_header(sniff) except Exception as e: # Can happen if: file doesn't exist, # filesize < necessary sniff size (this happens!) From 64e9ac1e135ecd53e9dcb1187e46b397309b6e16 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Wed, 2 Sep 2015 18:24:10 -0400 Subject: [PATCH 14/30] TST: Remove Minc1Header exception --- nibabel/tests/test_image_types.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/nibabel/tests/test_image_types.py b/nibabel/tests/test_image_types.py index 4316173135..28cb29d7b8 100644 --- a/nibabel/tests/test_image_types.py +++ b/nibabel/tests/test_image_types.py @@ -145,12 +145,9 @@ def check_img(img_path, img_klass, sniff_mode, sniff, expect_success, msg): for klass in img_klasses: if klass == expected_img_klass: # Class will load unless you pass a bad sniff, - # the header actually uses the sniff, and the - # sniff check is actually something meaningful - # (we're looking at you, Minc1Header...) + # or the header ignores the sniff expect_success = (sniff_mode != 'bad_sniff' or - sizeof_hdr == 0 or - klass == Minc1Image) # special case... + sizeof_hdr == 0) else: expect_success = False # Not sure the relationships From 4d77eb4eb3525d3de649746c14aa662dc48d1b2c Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Fri, 4 Sep 2015 15:17:03 -0400 Subject: [PATCH 15/30] STY: Unnecessary try block, comment lines too long --- nibabel/analyze.py | 3 ++- nibabel/imageclasses.py | 1 + nibabel/loadsave.py | 10 +++++----- nibabel/nifti1.py | 3 ++- nibabel/nifti2.py | 3 ++- nibabel/spatialimages.py | 12 +++++++----- nibabel/spm2analyze.py | 3 ++- nibabel/tests/test_files_interface.py | 2 +- nibabel/tests/test_image_types.py | 22 ++++++++++++---------- nibabel/tests/test_spatialimages.py | 6 ++++-- 10 files changed, 38 insertions(+), 27 deletions(-) diff --git a/nibabel/analyze.py b/nibabel/analyze.py index 389d552083..df39a4305c 100644 --- a/nibabel/analyze.py +++ b/nibabel/analyze.py @@ -895,7 +895,8 @@ def _chk_pixdims(hdr, fix=False): @classmethod def is_header(klass, binaryblock): if len(binaryblock) < klass.sizeof_hdr: - raise ValueError('Must pass a binary block >= %d bytes' % klass.sizeof_hdr) + raise ValueError('Must pass a binary block >= %d bytes' % + klass.sizeof_hdr) hdr = np.ndarray(shape=(), dtype=header_dtype, buffer=binaryblock[:klass.sizeof_hdr]) diff --git a/nibabel/imageclasses.py b/nibabel/imageclasses.py index 189fdd95b4..c74b7ed79a 100644 --- a/nibabel/imageclasses.py +++ b/nibabel/imageclasses.py @@ -83,6 +83,7 @@ def __getitem__(self, *args, **kwargs): 'makeable': False, 'rw': False}) + class ExtMapRecoder(Recoder): def __getitem__(self, *args, **kwargs): warnings.warn("ext_map is deprecated.", DeprecationWarning) diff --git a/nibabel/loadsave.py b/nibabel/loadsave.py index 20e9815122..bcd21c3ffc 100644 --- a/nibabel/loadsave.py +++ b/nibabel/loadsave.py @@ -72,8 +72,9 @@ def save(img, filename): lext = ext.lower() # Special-case Nifti singles and Pairs - from .nifti1 import Nifti1Image, Nifti1Pair # Inline imports, as this module - from .nifti2 import Nifti2Image, Nifti2Pair # really shouldn't reference any image type + # Inline imports, as this module really shouldn't reference any image type + from .nifti1 import Nifti1Image, Nifti1Pair + from .nifti2 import Nifti2Image, Nifti2Pair if type(img) == Nifti1Image and lext in ('.img', '.hdr'): klass = Nifti1Pair elif type(img) == Nifti2Image and lext in ('.img', '.hdr'): @@ -85,11 +86,10 @@ def save(img, filename): else: # arbitrary conversion valid_klasses = [klass for klass in all_image_classes if klass.is_valid_extension(ext)] - try: - klass = valid_klasses[0] - except IndexError: # if list is empty + if not valid_klasses: # if list is empty raise ImageFileError('Cannot work out file type of "%s"' % filename) + klass = valid_klasses[0] converted = klass.from_image(img) converted.to_filename(filename) diff --git a/nibabel/nifti1.py b/nibabel/nifti1.py index 9a63f9e245..4915943743 100644 --- a/nibabel/nifti1.py +++ b/nibabel/nifti1.py @@ -1614,7 +1614,8 @@ def _chk_xform_code(klass, code_type, hdr, fix): @classmethod def is_header(klass, binaryblock): if len(binaryblock) < klass.sizeof_hdr: - raise ValueError('Must pass a binary block >= %d bytes' % klass.sizeof_hdr) + raise ValueError('Must pass a binary block >= %d bytes' % + klass.sizeof_hdr) hdr = np.ndarray(shape=(), dtype=header_dtype, buffer=binaryblock[:klass.sizeof_hdr]) diff --git a/nibabel/nifti2.py b/nibabel/nifti2.py index 87523bd0de..6836b267e8 100644 --- a/nibabel/nifti2.py +++ b/nibabel/nifti2.py @@ -224,7 +224,8 @@ def _chk_eol_check(hdr, fix=False): @classmethod def is_header(klass, binaryblock): if len(binaryblock) < klass.sizeof_hdr: - raise ValueError('Must pass a binary block >= %d bytes' % klass.sizeof_hdr) + raise ValueError('Must pass a binary block >= %d bytes' % + klass.sizeof_hdr) hdr = np.ndarray(shape=(), dtype=header_dtype, buffer=binaryblock[:klass.sizeof_hdr]) diff --git a/nibabel/spatialimages.py b/nibabel/spatialimages.py index 0a1b3aafc2..55a6e4935e 100644 --- a/nibabel/spatialimages.py +++ b/nibabel/spatialimages.py @@ -878,12 +878,13 @@ def is_valid_extension(klass, ext): @classmethod def is_valid_filename(klass, filename): - froot, ext, trailing = splitext_addext(filename, klass._compressed_exts) + _, ext, _ = splitext_addext(filename, klass._compressed_exts) return klass.is_valid_extension(ext) @classmethod def is_image(klass, filename, sniff=None): - froot, ext, trailing = splitext_addext(filename, klass._compressed_exts) + froot, ext, trailing = splitext_addext(filename, + klass._compressed_exts) if not klass.is_valid_extension(ext): return False, sniff @@ -898,7 +899,8 @@ def is_image(klass, filename, sniff=None): # Search for an acceptable existing header; # could be compressed or not... for ext in header_exts: - for tr_ext in np.unique([trailing, ''] + list(klass._compressed_exts)): + for tr_ext in np.unique([trailing, ''] + + list(klass._compressed_exts)): metadata_filename = froot + ext + tr_ext if os.path.exists(metadata_filename): break @@ -907,13 +909,13 @@ def is_image(klass, filename, sniff=None): klass_sizeof_hdr = getattr(klass.header_class, 'sizeof_hdr', 0) if not sniff or len(sniff) < klass_sizeof_hdr: - # 1024 == large size, for efficiency (could iterate over imageclasses). + # 1024 bytes is currently larger than all headers sizeof_hdr = np.max([1024, klass_sizeof_hdr]) with ImageOpener(metadata_filename, 'rb') as fobj: sniff = fobj.read(sizeof_hdr) is_header = klass.header_class.is_header(sniff) - except Exception as e: + except Exception: # Can happen if: file doesn't exist, # filesize < necessary sniff size (this happens!) # other unexpected errors. diff --git a/nibabel/spm2analyze.py b/nibabel/spm2analyze.py index 7611212109..676e684bbc 100644 --- a/nibabel/spm2analyze.py +++ b/nibabel/spm2analyze.py @@ -116,7 +116,8 @@ def get_slope_inter(self): @classmethod def is_header(klass, binaryblock): if len(binaryblock) < klass.sizeof_hdr: - raise ValueError('Must pass a binary block >= %d bytes' % klass.sizeof_hdr) + raise ValueError('Must pass a binary block >= %d bytes' % + klass.sizeof_hdr) hdr = np.ndarray(shape=(), dtype=header_dtype, buffer=binaryblock[:klass.sizeof_hdr]) diff --git a/nibabel/tests/test_files_interface.py b/nibabel/tests/test_files_interface.py index 59839b3b96..2137de8d09 100644 --- a/nibabel/tests/test_files_interface.py +++ b/nibabel/tests/test_files_interface.py @@ -83,7 +83,7 @@ def test_files_interface(): def test_round_trip(): # write an image to files - data = np.arange(24, dtype='i4').reshape((2,3,4)) + data = np.arange(24, dtype='i4').reshape((2, 3, 4)) aff = np.eye(4) klasses = filter(lambda klass: klass.rw, all_image_classes) for klass in klasses: diff --git a/nibabel/tests/test_image_types.py b/nibabel/tests/test_image_types.py index 28cb29d7b8..c33474fc42 100644 --- a/nibabel/tests/test_image_types.py +++ b/nibabel/tests/test_image_types.py @@ -21,12 +21,11 @@ Spm2AnalyzeImage, Spm99AnalyzeImage, MGHImage, all_image_classes) -from nose.tools import assert_true, assert_equal, assert_false, assert_raises +from nose.tools import assert_true, assert_equal DATA_PATH = pjoin(dirname(__file__), 'data') - def test_analyze_detection(): # Test detection of Analyze, Nifti1 and Nifti2 # Algorithm is as described in loadsave:which_analyze_type @@ -36,7 +35,7 @@ def wat(hdr): for klass in all_analyze_header_klasses: try: if klass.is_header(hdr.binaryblock): - return klass + return klass else: print('checked completed, but failed.') except ValueError as ve: @@ -97,12 +96,13 @@ def test_sniff_and_guessed_image_type(img_klasses=all_image_classes): """ def test_image_class(img_path, expected_img_klass): - """ Embedded function to compare an image of one image class to all others. + """ Compare an image of one image class to all others. - The function should make sure that it loads the image with the expected class, - but failing when given a bad sniff (when the sniff is used).""" + The function should make sure that it loads the image with the expected + class, but failing when given a bad sniff (when the sniff is used).""" - def check_img(img_path, img_klass, sniff_mode, sniff, expect_success, msg): + def check_img(img_path, img_klass, sniff_mode, sniff, expect_success, + msg): """Embedded function to do the actual checks expected.""" if sniff_mode == 'no_sniff': @@ -114,9 +114,11 @@ def check_img(img_path, img_klass, sniff_mode, sniff, expect_success, msg): if expect_success: # Check that the sniff returned is appropriate. - new_msg = '%s returned sniff==None (%s)' % (img_klass.__name__, msg) - expected_sizeof_hdr = getattr(img_klass.header_class, 'sizeof_hdr', 0) - current_sizeof_hdr = len(new_sniff) if new_sniff is not None else 0 + new_msg = '%s returned sniff==None (%s)' % (img_klass.__name__, + msg) + expected_sizeof_hdr = getattr(img_klass.header_class, + 'sizeof_hdr', 0) + current_sizeof_hdr = 0 if new_sniff is None else len(new_sniff) assert_true(current_sizeof_hdr >= expected_sizeof_hdr, new_msg) # Check that the image type was recognized. diff --git a/nibabel/tests/test_spatialimages.py b/nibabel/tests/test_spatialimages.py index 29c227be73..7b684d1fd2 100644 --- a/nibabel/tests/test_spatialimages.py +++ b/nibabel/tests/test_spatialimages.py @@ -371,9 +371,11 @@ def test_load_mmap(self): back_img = func(param1, **kwargs) back_data = back_img.get_data() if expected_mode is None: - assert_false(isinstance(back_data, np.memmap), 'Should not be a %s' % img_klass.__name__) + assert_false(isinstance(back_data, np.memmap), + 'Should not be a %s' % img_klass.__name__) else: - assert_true(isinstance(back_data, np.memmap), 'Not a %s' % img_klass.__name__) + assert_true(isinstance(back_data, np.memmap), + 'Not a %s' % img_klass.__name__) if self.check_mmap_mode: assert_equal(back_data.mode, expected_mode) del back_img, back_data From afc9c25bdd13227dcef80dcd467f46a12e3a31ba Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Fri, 4 Sep 2015 16:51:52 -0400 Subject: [PATCH 16/30] TST: Test ValueErrors raised by is_header Remove unused is_valid_filename --- nibabel/spatialimages.py | 5 ----- nibabel/tests/test_image_types.py | 13 +++++++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/nibabel/spatialimages.py b/nibabel/spatialimages.py index 55a6e4935e..3cfcc650c9 100644 --- a/nibabel/spatialimages.py +++ b/nibabel/spatialimages.py @@ -876,11 +876,6 @@ def from_image(klass, img): def is_valid_extension(klass, ext): return np.any([ft[1] == ext.lower() for ft in klass.files_types]) - @classmethod - def is_valid_filename(klass, filename): - _, ext, _ = splitext_addext(filename, klass._compressed_exts) - return klass.is_valid_extension(ext) - @classmethod def is_image(klass, filename, sniff=None): froot, ext, trailing = splitext_addext(filename, diff --git a/nibabel/tests/test_image_types.py b/nibabel/tests/test_image_types.py index c33474fc42..6e3d914c78 100644 --- a/nibabel/tests/test_image_types.py +++ b/nibabel/tests/test_image_types.py @@ -21,7 +21,7 @@ Spm2AnalyzeImage, Spm99AnalyzeImage, MGHImage, all_image_classes) -from nose.tools import assert_true, assert_equal +from nose.tools import assert_true, assert_equal, assert_raises DATA_PATH = pjoin(dirname(__file__), 'data') @@ -105,6 +105,11 @@ def check_img(img_path, img_klass, sniff_mode, sniff, expect_success, msg): """Embedded function to do the actual checks expected.""" + if sniff_mode == 'empty' and \ + hasattr(img_klass.header_class, 'is_header'): + assert_raises(ValueError, img_klass.header_class.is_header, + sniff) + if sniff_mode == 'no_sniff': # Don't pass any sniff--not even "None" is_img, new_sniff = img_klass.is_image(img_path) @@ -140,9 +145,9 @@ def check_img(img_path, img_klass, sniff_mode, sniff, expect_success, vanilla=None, # use the sniff of the previous item no_sniff=None, # Don't pass a sniff none=None, # pass None as the sniff, should query in fn - empty='', # pass an empty sniff, should query in fn - irrelevant='a' * (sizeof_hdr - 1), # A too-small sniff, query - bad_sniff='a' * sizeof_hdr).items(): # Bad sniff, should fail. + empty=b'', # pass an empty sniff, should query in fn + irrelevant=b'a' * (sizeof_hdr - 1), # A too-small sniff, query + bad_sniff=b'a' * sizeof_hdr).items(): # Bad sniff, should fail for klass in img_klasses: if klass == expected_img_klass: From 1971efb3a679e253c0f8fd8981387a8605becca8 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Fri, 4 Sep 2015 17:53:38 -0400 Subject: [PATCH 17/30] RF: Rename is_header to may_contain_header Rename hdr variables to hdr_struct to reduce confusion Rename is_image to path_maybe_image --- nibabel/analyze.py | 10 +++++----- nibabel/loadsave.py | 2 +- nibabel/minc1.py | 2 +- nibabel/minc2.py | 2 +- nibabel/nifti1.py | 8 ++++---- nibabel/nifti2.py | 10 +++++----- nibabel/spatialimages.py | 10 +++++----- nibabel/spm2analyze.py | 10 +++++----- nibabel/tests/test_image_types.py | 16 ++++++++-------- 9 files changed, 35 insertions(+), 35 deletions(-) diff --git a/nibabel/analyze.py b/nibabel/analyze.py index df39a4305c..6408e876e3 100644 --- a/nibabel/analyze.py +++ b/nibabel/analyze.py @@ -893,15 +893,15 @@ def _chk_pixdims(hdr, fix=False): return hdr, rep @classmethod - def is_header(klass, binaryblock): + def may_contain_header(klass, binaryblock): if len(binaryblock) < klass.sizeof_hdr: raise ValueError('Must pass a binary block >= %d bytes' % klass.sizeof_hdr) - hdr = np.ndarray(shape=(), dtype=header_dtype, - buffer=binaryblock[:klass.sizeof_hdr]) - bs_hdr = hdr.byteswap() - return 348 in (hdr['sizeof_hdr'], bs_hdr['sizeof_hdr']) + hdr_struct = np.ndarray(shape=(), dtype=header_dtype, + buffer=binaryblock[:klass.sizeof_hdr]) + bs_hdr_struct = hdr_struct.byteswap() + return 348 in (hdr_struct['sizeof_hdr'], bs_hdr_struct['sizeof_hdr']) class AnalyzeImage(SpatialImage): diff --git a/nibabel/loadsave.py b/nibabel/loadsave.py index bcd21c3ffc..9799c7ee4d 100644 --- a/nibabel/loadsave.py +++ b/nibabel/loadsave.py @@ -36,7 +36,7 @@ def load(filename, **kwargs): sniff = None for image_klass in all_image_classes: - is_valid, sniff = image_klass.is_image(filename, sniff) + is_valid, sniff = image_klass.path_maybe_image(filename, sniff) if is_valid: return image_klass.from_filename(filename, **kwargs) diff --git a/nibabel/minc1.py b/nibabel/minc1.py index 387f393a70..3a24cc902a 100644 --- a/nibabel/minc1.py +++ b/nibabel/minc1.py @@ -282,7 +282,7 @@ def data_from_fileobj(self, fileobj): class Minc1Header(MincHeader): @classmethod - def is_header(klass, binaryblock): + def may_contain_header(klass, binaryblock): if len(binaryblock) < klass.sizeof_hdr: raise ValueError('Must pass a binary block >= %d bytes' % klass.sizeof_hdr) diff --git a/nibabel/minc2.py b/nibabel/minc2.py index c0a13f5584..d9c86ef6f9 100644 --- a/nibabel/minc2.py +++ b/nibabel/minc2.py @@ -136,7 +136,7 @@ def get_scaled_data(self, sliceobj=()): class Minc2Header(MincHeader): @classmethod - def is_header(klass, binaryblock): + def may_contain_header(klass, binaryblock): if len(binaryblock) < klass.sizeof_hdr: raise ValueError('Must pass a binary block >= %d bytes' % klass.sizeof_hdr) diff --git a/nibabel/nifti1.py b/nibabel/nifti1.py index 4915943743..fec037356b 100644 --- a/nibabel/nifti1.py +++ b/nibabel/nifti1.py @@ -1612,14 +1612,14 @@ def _chk_xform_code(klass, code_type, hdr, fix): return hdr, rep @classmethod - def is_header(klass, binaryblock): + def may_contain_header(klass, binaryblock): if len(binaryblock) < klass.sizeof_hdr: raise ValueError('Must pass a binary block >= %d bytes' % klass.sizeof_hdr) - hdr = np.ndarray(shape=(), dtype=header_dtype, - buffer=binaryblock[:klass.sizeof_hdr]) - return hdr['magic'] in (b'ni1', b'n+1') + hdr_struct = np.ndarray(shape=(), dtype=header_dtype, + buffer=binaryblock[:klass.sizeof_hdr]) + return hdr_struct['magic'] in (b'ni1', b'n+1') class Nifti1PairHeader(Nifti1Header): diff --git a/nibabel/nifti2.py b/nibabel/nifti2.py index 6836b267e8..e10a1d4fef 100644 --- a/nibabel/nifti2.py +++ b/nibabel/nifti2.py @@ -222,15 +222,15 @@ def _chk_eol_check(hdr, fix=False): return hdr, rep @classmethod - def is_header(klass, binaryblock): + def may_contain_header(klass, binaryblock): if len(binaryblock) < klass.sizeof_hdr: raise ValueError('Must pass a binary block >= %d bytes' % klass.sizeof_hdr) - hdr = np.ndarray(shape=(), dtype=header_dtype, - buffer=binaryblock[:klass.sizeof_hdr]) - bs_hdr = hdr.byteswap() - return 540 in (hdr['sizeof_hdr'], bs_hdr['sizeof_hdr']) + hdr_struct = np.ndarray(shape=(), dtype=header_dtype, + buffer=binaryblock[:klass.sizeof_hdr]) + bs_hdr_struct = hdr_struct.byteswap() + return 540 in (hdr_struct['sizeof_hdr'], bs_hdr_struct['sizeof_hdr']) class Nifti2PairHeader(Nifti2Header): diff --git a/nibabel/spatialimages.py b/nibabel/spatialimages.py index 3cfcc650c9..6d0ccc8d72 100644 --- a/nibabel/spatialimages.py +++ b/nibabel/spatialimages.py @@ -877,13 +877,13 @@ def is_valid_extension(klass, ext): return np.any([ft[1] == ext.lower() for ft in klass.files_types]) @classmethod - def is_image(klass, filename, sniff=None): + def path_maybe_image(klass, filename, sniff=None): froot, ext, trailing = splitext_addext(filename, klass._compressed_exts) if not klass.is_valid_extension(ext): return False, sniff - elif not hasattr(klass.header_class, 'is_header'): + elif not hasattr(klass.header_class, 'may_contain_header'): return True, sniff # Determine the metadata location, then sniff it @@ -909,14 +909,14 @@ def is_image(klass, filename, sniff=None): with ImageOpener(metadata_filename, 'rb') as fobj: sniff = fobj.read(sizeof_hdr) - is_header = klass.header_class.is_header(sniff) + may_contain_header = klass.header_class.may_contain_header(sniff) except Exception: # Can happen if: file doesn't exist, # filesize < necessary sniff size (this happens!) # other unexpected errors. - is_header = False + may_contain_header = False - return is_header, sniff + return may_contain_header, sniff def __getitem__(self): ''' No slicing or dictionary interface for images diff --git a/nibabel/spm2analyze.py b/nibabel/spm2analyze.py index 676e684bbc..a121ad631b 100644 --- a/nibabel/spm2analyze.py +++ b/nibabel/spm2analyze.py @@ -114,16 +114,16 @@ def get_slope_inter(self): return None, None @classmethod - def is_header(klass, binaryblock): + def may_contain_header(klass, binaryblock): if len(binaryblock) < klass.sizeof_hdr: raise ValueError('Must pass a binary block >= %d bytes' % klass.sizeof_hdr) - hdr = np.ndarray(shape=(), dtype=header_dtype, - buffer=binaryblock[:klass.sizeof_hdr]) - bs_hdr = hdr.byteswap() + hdr_struct = np.ndarray(shape=(), dtype=header_dtype, + buffer=binaryblock[:klass.sizeof_hdr]) + bs_hdr_struct = hdr_struct.byteswap() return (binaryblock[344:348] not in (b'ni1\x00', b'n+1\x00') and - 348 in (hdr['sizeof_hdr'], bs_hdr['sizeof_hdr'])) + 348 in (hdr_struct['sizeof_hdr'], bs_hdr_struct['sizeof_hdr'])) class Spm2AnalyzeImage(spm99.Spm99AnalyzeImage): diff --git a/nibabel/tests/test_image_types.py b/nibabel/tests/test_image_types.py index 6e3d914c78..bd04e0c238 100644 --- a/nibabel/tests/test_image_types.py +++ b/nibabel/tests/test_image_types.py @@ -6,7 +6,7 @@ # copyright and license terms. # ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## -''' Tests for is_image / is_header functions ''' +''' Tests for is_image / may_contain_header functions ''' from __future__ import division, print_function, absolute_import import copy @@ -21,7 +21,7 @@ Spm2AnalyzeImage, Spm99AnalyzeImage, MGHImage, all_image_classes) -from nose.tools import assert_true, assert_equal, assert_raises +from nose.tools import assert_true, assert_raises DATA_PATH = pjoin(dirname(__file__), 'data') @@ -34,7 +34,7 @@ def wat(hdr): AnalyzeHeader] for klass in all_analyze_header_klasses: try: - if klass.is_header(hdr.binaryblock): + if klass.may_contain_header(hdr.binaryblock): return klass else: print('checked completed, but failed.') @@ -106,16 +106,16 @@ def check_img(img_path, img_klass, sniff_mode, sniff, expect_success, """Embedded function to do the actual checks expected.""" if sniff_mode == 'empty' and \ - hasattr(img_klass.header_class, 'is_header'): - assert_raises(ValueError, img_klass.header_class.is_header, - sniff) + hasattr(img_klass.header_class, 'may_contain_header'): + assert_raises(ValueError, + img_klass.header_class.may_contain_header, sniff) if sniff_mode == 'no_sniff': # Don't pass any sniff--not even "None" - is_img, new_sniff = img_klass.is_image(img_path) + is_img, new_sniff = img_klass.path_maybe_image(img_path) else: # Pass a sniff, but don't reuse across images. - is_img, new_sniff = img_klass.is_image(img_path, sniff) + is_img, new_sniff = img_klass.path_maybe_image(img_path, sniff) if expect_success: # Check that the sniff returned is appropriate. From 4ddcbd278efe88a0e075668c8831f0e2258bcba3 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Fri, 4 Sep 2015 18:23:48 -0400 Subject: [PATCH 18/30] RF: Restore and deprecate loadsave helpers --- nibabel/loadsave.py | 70 ++++++++++++++++++++++++++- nibabel/tests/test_image_load_save.py | 59 ++++++++++++++++++++++ 2 files changed, 128 insertions(+), 1 deletion(-) diff --git a/nibabel/loadsave.py b/nibabel/loadsave.py index 9799c7ee4d..4500692c18 100644 --- a/nibabel/loadsave.py +++ b/nibabel/loadsave.py @@ -10,6 +10,7 @@ """ Utilities to load and save image objects """ import numpy as np +import warnings from .filename_parser import splitext_addext from .openers import ImageOpener @@ -33,7 +34,6 @@ def load(filename, **kwargs): img : ``SpatialImage`` Image of guessed type ''' - sniff = None for image_klass in all_image_classes: is_valid, sniff = image_klass.path_maybe_image(filename, sniff) @@ -44,6 +44,31 @@ def load(filename, **kwargs): filename) +@np.deprecate +def guessed_image_type(filename): + """ Guess image type from file `filename` + + Parameters + ---------- + filename : str + File name containing an image + + Returns + ------- + image_class : class + Class corresponding to guessed image type + """ + warnings.warn('guessed_image_type is deprecated', DeprecationWarning) + sniff = None + for image_klass in all_image_classes: + is_valid, sniff = image_klass.path_maybe_image(filename, sniff) + if is_valid: + return image_klass + + raise ImageFileError('Cannot work out file type of "%s"' % + filename) + + def save(img, filename): ''' Save an image to file adapting format to `filename` @@ -178,3 +203,46 @@ def read_img_data(img, prefer='scaled'): if prefer == 'scaled': return hdr.data_from_fileobj(fileobj) return hdr.raw_data_from_fileobj(fileobj) + + +@np.deprecate +def which_analyze_type(binaryblock): + """ Is `binaryblock` from NIfTI1, NIfTI2 or Analyze header? + + Parameters + ---------- + binaryblock : bytes + The `binaryblock` is 348 bytes that might be NIfTI1, NIfTI2, Analyze, + or None of the the above. + + Returns + ------- + hdr_type : str + * a nifti1 header (pair or single) -> return 'nifti1' + * a nifti2 header (pair or single) -> return 'nifti2' + * an Analyze header -> return 'analyze' + * None of the above -> return None + + Notes + ----- + Algorithm: + + * read in the first 4 bytes from the file as 32-bit int ``sizeof_hdr`` + * if ``sizeof_hdr`` is 540 or byteswapped 540 -> assume nifti2 + * Check for 'ni1', 'n+1' magic -> assume nifti1 + * if ``sizeof_hdr`` is 348 or byteswapped 348 assume Analyze + * Return None + """ + warnings.warn('which_analyze_type is deprecated', DeprecationWarning) + from .nifti1 import header_dtype + hdr_struct = np.ndarray(shape=(), dtype=header_dtype, buffer=binaryblock) + bs_hdr_struct = hdr_struct.byteswap() + sizeof_hdr = hdr_struct['sizeof_hdr'] + bs_sizeof_hdr = bs_hdr_struct['sizeof_hdr'] + if 540 in (sizeof_hdr, bs_sizeof_hdr): + return 'nifti2' + if hdr_struct['magic'] in (b'ni1', b'n+1'): + return 'nifti1' + if 348 in (sizeof_hdr, bs_sizeof_hdr): + return 'analyze' + return None diff --git a/nibabel/tests/test_image_load_save.py b/nibabel/tests/test_image_load_save.py index 261473d856..362c15313b 100644 --- a/nibabel/tests/test_image_load_save.py +++ b/nibabel/tests/test_image_load_save.py @@ -260,3 +260,62 @@ def test_filename_save(): del rt_img finally: shutil.rmtree(pth) + + +def test_analyze_detection(): + # Test detection of Analyze, Nifti1 and Nifti2 + # Algorithm is as described in loadsave:which_analyze_type + def wat(hdr): + return nils.which_analyze_type(hdr.binaryblock) + n1_hdr = Nifti1Header(b'\0' * 348, check=False) + assert_equal(wat(n1_hdr), None) + n1_hdr['sizeof_hdr'] = 540 + assert_equal(wat(n1_hdr), 'nifti2') + assert_equal(wat(n1_hdr.as_byteswapped()), 'nifti2') + n1_hdr['sizeof_hdr'] = 348 + assert_equal(wat(n1_hdr), 'analyze') + assert_equal(wat(n1_hdr.as_byteswapped()), 'analyze') + n1_hdr['magic'] = b'n+1' + assert_equal(wat(n1_hdr), 'nifti1') + assert_equal(wat(n1_hdr.as_byteswapped()), 'nifti1') + n1_hdr['magic'] = b'ni1' + assert_equal(wat(n1_hdr), 'nifti1') + assert_equal(wat(n1_hdr.as_byteswapped()), 'nifti1') + # Doesn't matter what magic is if it's not a nifti1 magic + n1_hdr['magic'] = b'ni2' + assert_equal(wat(n1_hdr), 'analyze') + n1_hdr['sizeof_hdr'] = 0 + n1_hdr['magic'] = b'' + assert_equal(wat(n1_hdr), None) + n1_hdr['magic'] = 'n+1' + assert_equal(wat(n1_hdr), 'nifti1') + n1_hdr['magic'] = 'ni1' + assert_equal(wat(n1_hdr), 'nifti1') + + +def test_guessed_image_type(): + # Test whether we can guess the image type from example files + assert_equal(nils.guessed_image_type( + pjoin(DATA_PATH, 'example4d.nii.gz')), + Nifti1Image) + assert_equal(nils.guessed_image_type( + pjoin(DATA_PATH, 'nifti1.hdr')), + Nifti1Pair) + assert_equal(nils.guessed_image_type( + pjoin(DATA_PATH, 'example_nifti2.nii.gz')), + Nifti2Image) + assert_equal(nils.guessed_image_type( + pjoin(DATA_PATH, 'nifti2.hdr')), + Nifti2Pair) + assert_equal(nils.guessed_image_type( + pjoin(DATA_PATH, 'tiny.mnc')), + Minc1Image) + assert_equal(nils.guessed_image_type( + pjoin(DATA_PATH, 'small.mnc')), + Minc2Image) + assert_equal(nils.guessed_image_type( + pjoin(DATA_PATH, 'test.mgz')), + MGHImage) + assert_equal(nils.guessed_image_type( + pjoin(DATA_PATH, 'analyze.hdr')), + Spm2AnalyzeImage) From 5570d368ad810f3e70a724b97bbb03b6394a076e Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Fri, 4 Sep 2015 19:16:43 -0400 Subject: [PATCH 19/30] TST: Kill extra test; doc string to comment --- nibabel/tests/test_image_types.py | 77 ++++--------------------------- 1 file changed, 9 insertions(+), 68 deletions(-) diff --git a/nibabel/tests/test_image_types.py b/nibabel/tests/test_image_types.py index bd04e0c238..47de54e4bb 100644 --- a/nibabel/tests/test_image_types.py +++ b/nibabel/tests/test_image_types.py @@ -26,75 +26,16 @@ DATA_PATH = pjoin(dirname(__file__), 'data') -def test_analyze_detection(): - # Test detection of Analyze, Nifti1 and Nifti2 - # Algorithm is as described in loadsave:which_analyze_type - def wat(hdr): - all_analyze_header_klasses = [Nifti1Header, Nifti2Header, - AnalyzeHeader] - for klass in all_analyze_header_klasses: - try: - if klass.may_contain_header(hdr.binaryblock): - return klass - else: - print('checked completed, but failed.') - except ValueError as ve: - print(ve) - continue - return None - # return nils.which_analyze_type(hdr.binaryblock) - - n1_hdr = Nifti1Header(b'\0' * 348, check=False) - n2_hdr = Nifti2Header(b'\0' * 540, check=False) - assert_equal(wat(n1_hdr), None) - - n1_hdr['sizeof_hdr'] = 540 - n2_hdr['sizeof_hdr'] = 540 - assert_equal(wat(n1_hdr), None) - assert_equal(wat(n1_hdr.as_byteswapped()), None) - assert_equal(wat(n2_hdr), Nifti2Header) - assert_equal(wat(n2_hdr.as_byteswapped()), Nifti2Header) - - n1_hdr['sizeof_hdr'] = 348 - assert_equal(wat(n1_hdr), AnalyzeHeader) - assert_equal(wat(n1_hdr.as_byteswapped()), AnalyzeHeader) - - n1_hdr['magic'] = b'n+1' - assert_equal(wat(n1_hdr), Nifti1Header) - assert_equal(wat(n1_hdr.as_byteswapped()), Nifti1Header) - - n1_hdr['magic'] = b'ni1' - assert_equal(wat(n1_hdr), Nifti1Header) - assert_equal(wat(n1_hdr.as_byteswapped()), Nifti1Header) - - # Doesn't matter what magic is if it's not a nifti1 magic - n1_hdr['magic'] = b'ni2' - assert_equal(wat(n1_hdr), AnalyzeHeader) - - n1_hdr['sizeof_hdr'] = 0 - n1_hdr['magic'] = b'' - assert_equal(wat(n1_hdr), None) - - n1_hdr['magic'] = 'n+1' - assert_equal(wat(n1_hdr), Nifti1Header) - - n1_hdr['magic'] = 'ni1' - assert_equal(wat(n1_hdr), Nifti1Header) - - def test_sniff_and_guessed_image_type(img_klasses=all_image_classes): - """ - Loop over all test cases: - * whether a sniff is provided or not - * randomizing the order of image classes - * over all known image types - - For each, we expect: - * When the file matches the expected class, things should - either work, or fail if we're doing bad stuff. - * When the file is a mismatch, the functions should not throw. - """ - + # Loop over all test cases: + # * whether a sniff is provided or not + # * randomizing the order of image classes + # * over all known image types + + # For each, we expect: + # * When the file matches the expected class, things should + # either work, or fail if we're doing bad stuff. + # * When the file is a mismatch, the functions should not throw. def test_image_class(img_path, expected_img_klass): """ Compare an image of one image class to all others. From 01af20f57e293ff886741cf210d6deccb7e557c9 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Sun, 6 Sep 2015 12:34:28 -0400 Subject: [PATCH 20/30] RF: Prefer alternate_exts over abusing files_types --- nibabel/filename_parser.py | 2 -- nibabel/freesurfer/mghformat.py | 3 +-- nibabel/openers.py | 8 +++++--- nibabel/spatialimages.py | 4 +++- nibabel/tests/test_openers.py | 2 ++ 5 files changed, 11 insertions(+), 8 deletions(-) diff --git a/nibabel/filename_parser.py b/nibabel/filename_parser.py index 8965ed53e3..bc21cbc872 100644 --- a/nibabel/filename_parser.py +++ b/nibabel/filename_parser.py @@ -131,8 +131,6 @@ def types_filenames(template_fname, types_exts, elif found_ext == found_ext.lower(): proc_ext = lambda s: s.lower() for name, ext in types_exts: - if name in tfns: # Allow multipe definitions of image, header, etc, - continue # giving priority to those found first. if name == direct_set_name: tfns[name] = template_fname continue diff --git a/nibabel/freesurfer/mghformat.py b/nibabel/freesurfer/mghformat.py index 54c1cfd92a..c63a7d9bac 100644 --- a/nibabel/freesurfer/mghformat.py +++ b/nibabel/freesurfer/mghformat.py @@ -459,8 +459,7 @@ class MGHImage(SpatialImage): """ Class for MGH format image """ header_class = MGHHeader - files_types = (('image', '.mgh'), - ('image', '.mgz')) + files_types = (('image', '.mgh'),) _compressed_exts = (('.gz',)) makeable = True diff --git a/nibabel/openers.py b/nibabel/openers.py index eb1c8d3708..ad6dfa7962 100644 --- a/nibabel/openers.py +++ b/nibabel/openers.py @@ -160,7 +160,9 @@ class ImageOpener(Opener): def register_ext_from_image(opener_klass, ext, func_def): """Decorator for adding extension / opener_function associations. - Should be used to decorate classes. + Should be used to decorate classes. Updates ImageOpener class with + desired extension / opener association. Updates decorated class by + adding ```ext``` to ```klass.alternate_exts```. Parameters ---------- @@ -176,12 +178,12 @@ def register_ext_from_image(opener_klass, ext, func_def): Returns ------- - opener_klass, with a side-effect of updating the ImageOpener class - with the desired extension / opener association. + opener_klass """ def decorate(klass): assert ext not in opener_klass.compress_ext_map, \ "Cannot redefine extension-function mappings." opener_klass.compress_ext_map[ext] = func_def + klass.alternate_exts += (ext,) return klass return decorate diff --git a/nibabel/spatialimages.py b/nibabel/spatialimages.py index 6d0ccc8d72..fc05e84cb2 100644 --- a/nibabel/spatialimages.py +++ b/nibabel/spatialimages.py @@ -325,6 +325,7 @@ class SpatialImage(object): ''' Template class for images ''' header_class = Header files_types = (('image', None),) + alternate_exts = () # Modified by @ImageOpener.register_ext_from_image _compressed_exts = () makeable = True # Used in test code @@ -874,7 +875,8 @@ def from_image(klass, img): @classmethod def is_valid_extension(klass, ext): - return np.any([ft[1] == ext.lower() for ft in klass.files_types]) + valid = tuple(ft[1] for ft in klass.files_types) + klass.alternate_exts + return ext.lower() in valid @classmethod def path_maybe_image(klass, filename, sniff=None): diff --git a/nibabel/tests/test_openers.py b/nibabel/tests/test_openers.py index 3bd5ccb98c..6dc61e3160 100644 --- a/nibabel/tests/test_openers.py +++ b/nibabel/tests/test_openers.py @@ -90,6 +90,7 @@ def test_BinOpener(): BinOpener, 'test.txt', 'r') class TestImageOpener: + alternate_exts = () def setUp(self): self.compress_ext_map = ImageOpener.compress_ext_map.copy() @@ -115,6 +116,7 @@ def file_opener(fileish, mode): dec(self.__class__) assert_equal(n_associations + 1, len(ImageOpener.compress_ext_map)) assert_true('.foo' in ImageOpener.compress_ext_map) + assert_true('.foo' in self.alternate_exts) with InTemporaryDirectory(): with ImageOpener('test.foo', 'w'): From 0c06464856b46954af3001b7083ae6ca2586b09d Mon Sep 17 00:00:00 2001 From: Matthew Brett Date: Fri, 18 Sep 2015 22:47:08 -0700 Subject: [PATCH 21/30] RF: refactor using valid_exts, _meta_sniff_len This is a refactor, trying to separate the idea of valid extensions for loading images, from compressed extra suffixes. --- nibabel/analyze.py | 2 + nibabel/ecat.py | 1 + nibabel/freesurfer/mghformat.py | 8 ++- nibabel/freesurfer/tests/test_mghformat.py | 2 +- nibabel/loadsave.py | 2 +- nibabel/minc1.py | 2 + nibabel/nifti1.py | 2 + nibabel/nifti2.py | 2 + nibabel/parrec.py | 1 + nibabel/spatialimages.py | 67 +++++++++------------- 10 files changed, 45 insertions(+), 44 deletions(-) diff --git a/nibabel/analyze.py b/nibabel/analyze.py index 6408e876e3..8b08c6cefe 100644 --- a/nibabel/analyze.py +++ b/nibabel/analyze.py @@ -908,7 +908,9 @@ class AnalyzeImage(SpatialImage): """ Class for basic Analyze format image """ header_class = AnalyzeHeader + _meta_sniff_len = header_class.sizeof_hdr files_types = (('image', '.img'), ('header', '.hdr')) + valid_exts = ('.img', '.hdr') _compressed_exts = ('.gz', '.bz2') makeable = True diff --git a/nibabel/ecat.py b/nibabel/ecat.py index 666a2d843f..495d9ab6c5 100644 --- a/nibabel/ecat.py +++ b/nibabel/ecat.py @@ -732,6 +732,7 @@ class EcatImage(SpatialImage): """ _header = EcatHeader header_class = _header + valid_exts = ('.v',) _subheader = EcatSubHeader files_types = (('image', '.v'), ('header', '.v')) diff --git a/nibabel/freesurfer/mghformat.py b/nibabel/freesurfer/mghformat.py index c63a7d9bac..bb11af253a 100644 --- a/nibabel/freesurfer/mghformat.py +++ b/nibabel/freesurfer/mghformat.py @@ -454,13 +454,17 @@ def writeftr_to(self, fileobj): fileobj.write(ftr_nd.tostring()) -@ImageOpener.register_ext_from_image('.mgz', ImageOpener.gz_def) +# Register .mgz extension as compressed +ImageOpener.compress_ext_map['.mgz'] = ImageOpener.gz_def + + class MGHImage(SpatialImage): """ Class for MGH format image """ header_class = MGHHeader + valid_exts = ('.mgh', '.mgz') files_types = (('image', '.mgh'),) - _compressed_exts = (('.gz',)) + _compressed_exts = () makeable = True rw = True diff --git a/nibabel/freesurfer/tests/test_mghformat.py b/nibabel/freesurfer/tests/test_mghformat.py index c174e3ef3e..cf7d26f20a 100644 --- a/nibabel/freesurfer/tests/test_mghformat.py +++ b/nibabel/freesurfer/tests/test_mghformat.py @@ -145,7 +145,7 @@ def test_filename_exts(): # and the default affine matrix (Note the "None") img = MGHImage(v, None) # Check if these extensions allow round trip - for ext in ('.mgh', '.mgz', '.mgh.gz'): + for ext in ('.mgh', '.mgz'): with InTemporaryDirectory(): fname = 'tmpname' + ext save(img, fname) diff --git a/nibabel/loadsave.py b/nibabel/loadsave.py index 4500692c18..bac67d4660 100644 --- a/nibabel/loadsave.py +++ b/nibabel/loadsave.py @@ -110,7 +110,7 @@ def save(img, filename): klass = Nifti2Image else: # arbitrary conversion valid_klasses = [klass for klass in all_image_classes - if klass.is_valid_extension(ext)] + if ext in klass.valid_exts] if not valid_klasses: # if list is empty raise ImageFileError('Cannot work out file type of "%s"' % filename) diff --git a/nibabel/minc1.py b/nibabel/minc1.py index 3a24cc902a..21be2c93be 100644 --- a/nibabel/minc1.py +++ b/nibabel/minc1.py @@ -298,6 +298,8 @@ class Minc1Image(SpatialImage): load. ''' header_class = Minc1Header + _meta_sniff_len = 4 + valid_exts = ('.mnc',) files_types = (('image', '.mnc'),) _compressed_exts = ('.gz', '.bz2') diff --git a/nibabel/nifti1.py b/nibabel/nifti1.py index fec037356b..9aabf4d6a0 100644 --- a/nibabel/nifti1.py +++ b/nibabel/nifti1.py @@ -1632,6 +1632,7 @@ class Nifti1Pair(analyze.AnalyzeImage): """ Class for NIfTI1 format image, header pair """ header_class = Nifti1PairHeader + _meta_sniff_len = header_class.sizeof_hdr rw = True def __init__(self, dataobj, affine, header=None, @@ -1856,6 +1857,7 @@ class Nifti1Image(Nifti1Pair): """ Class for single file NIfTI1 format image """ header_class = Nifti1Header + valid_exts = ('.nii',) files_types = (('image', '.nii'),) @staticmethod diff --git a/nibabel/nifti2.py b/nibabel/nifti2.py index e10a1d4fef..74f83b9458 100644 --- a/nibabel/nifti2.py +++ b/nibabel/nifti2.py @@ -243,12 +243,14 @@ class Nifti2Pair(Nifti1Pair): """ Class for NIfTI2 format image, header pair """ header_class = Nifti2PairHeader + _meta_sniff_len = header_class.sizeof_hdr class Nifti2Image(Nifti1Image): """ Class for single file NIfTI2 format image """ header_class = Nifti2Header + _meta_sniff_len = header_class.sizeof_hdr def load(filename): diff --git a/nibabel/parrec.py b/nibabel/parrec.py index 85fc30aa4e..cfbc77b1db 100644 --- a/nibabel/parrec.py +++ b/nibabel/parrec.py @@ -1020,6 +1020,7 @@ def get_sorted_slice_indices(self): class PARRECImage(SpatialImage): """PAR/REC image""" header_class = PARRECHeader + valid_exts = ('.rec', '.par') files_types = (('image', '.rec'), ('header', '.par')) makeable = False diff --git a/nibabel/spatialimages.py b/nibabel/spatialimages.py index fc05e84cb2..21e84b15b9 100644 --- a/nibabel/spatialimages.py +++ b/nibabel/spatialimages.py @@ -324,8 +324,9 @@ class ImageFileError(Exception): class SpatialImage(object): ''' Template class for images ''' header_class = Header + _meta_sniff_len = 0 files_types = (('image', None),) - alternate_exts = () # Modified by @ImageOpener.register_ext_from_image + valid_exts = () _compressed_exts = () makeable = True # Used in test code @@ -874,51 +875,37 @@ def from_image(klass, img): extra=img.extra.copy()) @classmethod - def is_valid_extension(klass, ext): - valid = tuple(ft[1] for ft in klass.files_types) + klass.alternate_exts - return ext.lower() in valid + def _sniff_meta_for(klass, filename, sniff_nbytes): + froot, ext, trailing = splitext_addext(filename, + klass._compressed_exts) + # Determine the metadata location, then sniff it + t_fnames = types_filenames(filename, + klass.files_types, + trailing_suffixes=klass._compressed_exts) + meta_fname = t_fnames.get('header', filename) + try: + with ImageOpener(meta_fname, 'rb') as fobj: + sniff = fobj.read(sniff_nbytes) + except IOError: + return None + return sniff @classmethod - def path_maybe_image(klass, filename, sniff=None): + def path_maybe_image(klass, filename, sniff=None, sniff_max=1024): froot, ext, trailing = splitext_addext(filename, klass._compressed_exts) - - if not klass.is_valid_extension(ext): + if ext.lower() not in klass.valid_exts: return False, sniff - elif not hasattr(klass.header_class, 'may_contain_header'): + if not hasattr(klass.header_class, 'may_contain_header'): return True, sniff - - # Determine the metadata location, then sniff it - header_exts = [ft[1] for ft in klass.files_types if ft[0] == 'header'] - if len(header_exts) == 0: - metadata_filename = filename - else: - # Search for an acceptable existing header; - # could be compressed or not... - for ext in header_exts: - for tr_ext in np.unique([trailing, ''] + - list(klass._compressed_exts)): - metadata_filename = froot + ext + tr_ext - if os.path.exists(metadata_filename): - break - - try: - klass_sizeof_hdr = getattr(klass.header_class, 'sizeof_hdr', 0) - - if not sniff or len(sniff) < klass_sizeof_hdr: - # 1024 bytes is currently larger than all headers - sizeof_hdr = np.max([1024, klass_sizeof_hdr]) - with ImageOpener(metadata_filename, 'rb') as fobj: - sniff = fobj.read(sizeof_hdr) - - may_contain_header = klass.header_class.may_contain_header(sniff) - except Exception: - # Can happen if: file doesn't exist, - # filesize < necessary sniff size (this happens!) - # other unexpected errors. - may_contain_header = False - - return may_contain_header, sniff + if sniff is None or len(sniff) < klass._meta_sniff_len: + sniff_nbytes = max(klass._meta_sniff_len, sniff_max) + sniff = klass._sniff_meta_for(filename, sniff_nbytes) + if sniff is None: # Can't sniff, won't sniff + return False, None + if len(sniff) < klass._meta_sniff_len: + return False, sniff + return klass.header_class.may_contain_header(sniff), sniff def __getitem__(self): ''' No slicing or dictionary interface for images From fce4cadbeb49e3d731c0e918c88600eb2012e58b Mon Sep 17 00:00:00 2001 From: Matthew Brett Date: Fri, 18 Sep 2015 23:49:21 -0700 Subject: [PATCH 22/30] RF: rename compressed_exts to compressed_suffxes Make clear that these are extra suffixes on the filename indicating the file is compressed, as opposed to something like '.mgz' which is not a suffix, but an alternative extension. --- nibabel/analyze.py | 2 +- nibabel/freesurfer/mghformat.py | 2 +- nibabel/loadsave.py | 2 +- nibabel/minc1.py | 2 +- nibabel/minc2.py | 2 +- nibabel/spatialimages.py | 20 ++++++++++---------- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/nibabel/analyze.py b/nibabel/analyze.py index 8b08c6cefe..feaba4addd 100644 --- a/nibabel/analyze.py +++ b/nibabel/analyze.py @@ -911,7 +911,7 @@ class AnalyzeImage(SpatialImage): _meta_sniff_len = header_class.sizeof_hdr files_types = (('image', '.img'), ('header', '.hdr')) valid_exts = ('.img', '.hdr') - _compressed_exts = ('.gz', '.bz2') + _compressed_suffixes = ('.gz', '.bz2') makeable = True rw = True diff --git a/nibabel/freesurfer/mghformat.py b/nibabel/freesurfer/mghformat.py index bb11af253a..b3ee1974c4 100644 --- a/nibabel/freesurfer/mghformat.py +++ b/nibabel/freesurfer/mghformat.py @@ -464,7 +464,7 @@ class MGHImage(SpatialImage): header_class = MGHHeader valid_exts = ('.mgh', '.mgz') files_types = (('image', '.mgh'),) - _compressed_exts = () + _compressed_suffixes = () makeable = True rw = True diff --git a/nibabel/loadsave.py b/nibabel/loadsave.py index bac67d4660..0c80864283 100644 --- a/nibabel/loadsave.py +++ b/nibabel/loadsave.py @@ -93,7 +93,7 @@ def save(img, filename): return # Be nice to users by making common implicit conversions - froot, ext, trailing = splitext_addext(filename, img._compressed_exts) + froot, ext, trailing = splitext_addext(filename, ('.gz', '.bz2')) lext = ext.lower() # Special-case Nifti singles and Pairs diff --git a/nibabel/minc1.py b/nibabel/minc1.py index 21be2c93be..34fcc7c75b 100644 --- a/nibabel/minc1.py +++ b/nibabel/minc1.py @@ -301,7 +301,7 @@ class Minc1Image(SpatialImage): _meta_sniff_len = 4 valid_exts = ('.mnc',) files_types = (('image', '.mnc'),) - _compressed_exts = ('.gz', '.bz2') + _compressed_suffixes = ('.gz', '.bz2') makeable = True rw = False diff --git a/nibabel/minc2.py b/nibabel/minc2.py index d9c86ef6f9..b681a95781 100644 --- a/nibabel/minc2.py +++ b/nibabel/minc2.py @@ -152,7 +152,7 @@ class Minc2Image(Minc1Image): the MINC file on load. ''' # MINC2 does not do compressed whole files - _compressed_exts = () + _compressed_suffixes = () header_class = Minc2Header @classmethod diff --git a/nibabel/spatialimages.py b/nibabel/spatialimages.py index 21e84b15b9..b6eb59d305 100644 --- a/nibabel/spatialimages.py +++ b/nibabel/spatialimages.py @@ -137,13 +137,12 @@ except NameError: # python 3 basestring = str -import os.path import warnings import numpy as np -from .filename_parser import types_filenames, TypesFilenamesError, \ - splitext_addext +from .filename_parser import (types_filenames, TypesFilenamesError, + splitext_addext) from .fileholders import FileHolder from .openers import ImageOpener from .volumeutils import shape_zoom_affine @@ -327,7 +326,7 @@ class SpatialImage(object): _meta_sniff_len = 0 files_types = (('image', None),) valid_exts = () - _compressed_exts = () + _compressed_suffixes = () makeable = True # Used in test code rw = True # Used in test code @@ -753,7 +752,7 @@ def filespec_to_file_map(klass, filespec): try: filenames = types_filenames( filespec, klass.files_types, - trailing_suffixes=klass._compressed_exts) + trailing_suffixes=klass._compressed_suffixes) except TypesFilenamesError: raise ImageFileError( 'Filespec "{0}" does not look right for class {1}'.format( @@ -877,11 +876,12 @@ def from_image(klass, img): @classmethod def _sniff_meta_for(klass, filename, sniff_nbytes): froot, ext, trailing = splitext_addext(filename, - klass._compressed_exts) + klass._compressed_suffixes) # Determine the metadata location, then sniff it - t_fnames = types_filenames(filename, - klass.files_types, - trailing_suffixes=klass._compressed_exts) + t_fnames = types_filenames( + filename, + klass.files_types, + trailing_suffixes=klass._compressed_suffixes) meta_fname = t_fnames.get('header', filename) try: with ImageOpener(meta_fname, 'rb') as fobj: @@ -893,7 +893,7 @@ def _sniff_meta_for(klass, filename, sniff_nbytes): @classmethod def path_maybe_image(klass, filename, sniff=None, sniff_max=1024): froot, ext, trailing = splitext_addext(filename, - klass._compressed_exts) + klass._compressed_suffixes) if ext.lower() not in klass.valid_exts: return False, sniff if not hasattr(klass.header_class, 'may_contain_header'): From 2ad562180fbd199e46fd6ee55a6663b6b8a0736a Mon Sep 17 00:00:00 2001 From: Matthew Brett Date: Sat, 19 Sep 2015 11:19:41 -0700 Subject: [PATCH 23/30] BF: isolate ImageOpener extension list ImageOpener extension list was pointer to dictionary from Opener class, so modifying ImageOpener modified Opener. --- nibabel/openers.py | 1 + nibabel/tests/test_openers.py | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/nibabel/openers.py b/nibabel/openers.py index ad6dfa7962..e4b3a61572 100644 --- a/nibabel/openers.py +++ b/nibabel/openers.py @@ -155,6 +155,7 @@ class ImageOpener(Opener): attributes, via the `register_ex_from_images`. The class can therefore change state when image classes are defined. """ + compress_ext_map = Opener.compress_ext_map.copy() @classmethod def register_ext_from_image(opener_klass, ext, func_def): diff --git a/nibabel/tests/test_openers.py b/nibabel/tests/test_openers.py index 6dc61e3160..9d0bf31c1b 100644 --- a/nibabel/tests/test_openers.py +++ b/nibabel/tests/test_openers.py @@ -57,6 +57,7 @@ def test_Opener(): # mode is gently ignored fobj = Opener(obj, mode='r') + def test_Opener_various(): # Check we can do all sorts of files here message = b"Oh what a giveaway" @@ -84,11 +85,13 @@ def test_Opener_various(): # Just check there is a fileno assert_not_equal(fobj.fileno(), 0) + def test_BinOpener(): with error_warnings(): assert_raises(DeprecationWarning, BinOpener, 'test.txt', 'r') + class TestImageOpener: alternate_exts = () def setUp(self): @@ -123,6 +126,9 @@ def file_opener(fileish, mode): pass assert_true(os.path.exists('test.foo')) + # Check this doesn't add anything to parent + assert_false('.foo' in Opener.compress_ext_map) + def test_file_like_wrapper(): # Test wrapper using BytesIO (full API) From 5600d4237a8d54b3f43f12d36b39701cc4292ab0 Mon Sep 17 00:00:00 2001 From: Matthew Brett Date: Sat, 19 Sep 2015 11:54:33 -0700 Subject: [PATCH 24/30] DOC: add docstrings for image loading classmethods Explain sniff and friends. --- nibabel/spatialimages.py | 50 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/nibabel/spatialimages.py b/nibabel/spatialimages.py index b6eb59d305..48b886f3e9 100644 --- a/nibabel/spatialimages.py +++ b/nibabel/spatialimages.py @@ -875,6 +875,25 @@ def from_image(klass, img): @classmethod def _sniff_meta_for(klass, filename, sniff_nbytes): + """ Sniff metadata for image represented by `filename` + + Parameters + ---------- + filename : str + Filename for an image, or an image header (metadata) file. + If `filename` points to an image data file, and the image type has + a separate "header" file, we work out the name of the header file, + and read from that instead of `filename`. + sniff_nbytes : int + Number of bytes to read from the image or metadata file + + Returns + ------- + meta_bytes : None or bytes + None if we could not read the image or metadata file. `meta_bytes` + is either length `sniff_nbytes` or the length of the image / + metadata file, whichever is the shorter. + """ froot, ext, trailing = splitext_addext(filename, klass._compressed_suffixes) # Determine the metadata location, then sniff it @@ -892,6 +911,37 @@ def _sniff_meta_for(klass, filename, sniff_nbytes): @classmethod def path_maybe_image(klass, filename, sniff=None, sniff_max=1024): + """ Return True if `filename` may be image matching this class + + Parameters + ---------- + filename : str + Filename for an image, or an image header (metadata) file. + If `filename` points to an image data file, and the image type has + a separate "header" file, we work out the name of the header file, + and read from that instead of `filename`. + sniff : None or bytes, optional + Bytes content read from a previous call to this method, on another + class. This allows us to read metadata bytes once from the image / + or header, and pass this read set of bytes to other image classes, + therefore saving a repeat read of the metadata. None forces this + method to read the metadata. + sniff_max : int, optional + The maximum number of bytes to read from the metadata. If the + metadata file is long enough, we read this many bytes from the + file, otherwise we read to the end of the file. Longer values + sniff more of the metadata / image file, making it more likely that + the returned sniff will be useful for later calls to + ``path_maybe_image`` for other image classes. + + Returns + ------- + maybe_image : bool + True if `filename` may be valid for an image of this class. + sniff : None or bytes + Read bytes content from found metadata. May be None if the file + does not appear to have useful metadata. + """ froot, ext, trailing = splitext_addext(filename, klass._compressed_suffixes) if ext.lower() not in klass.valid_exts: From d8f1be9f86ecc8a4cd6df574f1a5d43ceaf0cc66 Mon Sep 17 00:00:00 2001 From: Matthew Brett Date: Sat, 19 Sep 2015 15:24:01 -0700 Subject: [PATCH 25/30] RF: go back to using decorator for .mgz extension At Ben's suggestion, restore the decorator for now, to simplify discussion of this PR. --- nibabel/freesurfer/mghformat.py | 6 ++---- nibabel/openers.py | 2 +- nibabel/tests/test_openers.py | 5 +++-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/nibabel/freesurfer/mghformat.py b/nibabel/freesurfer/mghformat.py index b3ee1974c4..f63875b2c6 100644 --- a/nibabel/freesurfer/mghformat.py +++ b/nibabel/freesurfer/mghformat.py @@ -455,14 +455,12 @@ def writeftr_to(self, fileobj): # Register .mgz extension as compressed -ImageOpener.compress_ext_map['.mgz'] = ImageOpener.gz_def - - +@ImageOpener.register_ext_from_image('.mgz', ImageOpener.gz_def) class MGHImage(SpatialImage): """ Class for MGH format image """ header_class = MGHHeader - valid_exts = ('.mgh', '.mgz') + valid_exts = ('.mgh',) files_types = (('image', '.mgh'),) _compressed_suffixes = () diff --git a/nibabel/openers.py b/nibabel/openers.py index e4b3a61572..26bbb09d7b 100644 --- a/nibabel/openers.py +++ b/nibabel/openers.py @@ -185,6 +185,6 @@ def decorate(klass): assert ext not in opener_klass.compress_ext_map, \ "Cannot redefine extension-function mappings." opener_klass.compress_ext_map[ext] = func_def - klass.alternate_exts += (ext,) + klass.valid_exts += (ext,) return klass return decorate diff --git a/nibabel/tests/test_openers.py b/nibabel/tests/test_openers.py index 9d0bf31c1b..08f9730ace 100644 --- a/nibabel/tests/test_openers.py +++ b/nibabel/tests/test_openers.py @@ -93,7 +93,8 @@ def test_BinOpener(): class TestImageOpener: - alternate_exts = () + valid_exts = () + def setUp(self): self.compress_ext_map = ImageOpener.compress_ext_map.copy() @@ -119,7 +120,7 @@ def file_opener(fileish, mode): dec(self.__class__) assert_equal(n_associations + 1, len(ImageOpener.compress_ext_map)) assert_true('.foo' in ImageOpener.compress_ext_map) - assert_true('.foo' in self.alternate_exts) + assert_true('.foo' in self.valid_exts) with InTemporaryDirectory(): with ImageOpener('test.foo', 'w'): From ecee5617af6be2a48b82fbfd04f5611d0efbcfe1 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Mon, 21 Sep 2015 17:32:53 -0400 Subject: [PATCH 26/30] DOC: Deprecation warnings to stacklevel 2 --- nibabel/imageclasses.py | 6 ++++-- nibabel/loadsave.py | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/nibabel/imageclasses.py b/nibabel/imageclasses.py index c74b7ed79a..20e67c06ef 100644 --- a/nibabel/imageclasses.py +++ b/nibabel/imageclasses.py @@ -33,7 +33,8 @@ # DEPRECATED: mapping of names to classes and class functionality class ClassMapDict(dict): def __getitem__(self, *args, **kwargs): - warnings.warn("class_map is deprecated.", DeprecationWarning) + warnings.warn("class_map is deprecated.", DeprecationWarning, + stacklevel=2) return super(ClassMapDict, self).__getitem__(*args, **kwargs) class_map = ClassMapDict( @@ -86,7 +87,8 @@ def __getitem__(self, *args, **kwargs): class ExtMapRecoder(Recoder): def __getitem__(self, *args, **kwargs): - warnings.warn("ext_map is deprecated.", DeprecationWarning) + warnings.warn("ext_map is deprecated.", DeprecationWarning, + stacklevel=2) return super(ExtMapRecoder, self).__getitem__(*args, **kwargs) # mapping of extensions to default image class names diff --git a/nibabel/loadsave.py b/nibabel/loadsave.py index 0c80864283..f4ce286f61 100644 --- a/nibabel/loadsave.py +++ b/nibabel/loadsave.py @@ -58,7 +58,8 @@ def guessed_image_type(filename): image_class : class Class corresponding to guessed image type """ - warnings.warn('guessed_image_type is deprecated', DeprecationWarning) + warnings.warn('guessed_image_type is deprecated', DeprecationWarning, + stacklevel=2) sniff = None for image_klass in all_image_classes: is_valid, sniff = image_klass.path_maybe_image(filename, sniff) @@ -233,7 +234,8 @@ def which_analyze_type(binaryblock): * if ``sizeof_hdr`` is 348 or byteswapped 348 assume Analyze * Return None """ - warnings.warn('which_analyze_type is deprecated', DeprecationWarning) + warnings.warn('which_analyze_type is deprecated', DeprecationWarning, + stacklevel=2) from .nifti1 import header_dtype hdr_struct = np.ndarray(shape=(), dtype=header_dtype, buffer=binaryblock) bs_hdr_struct = hdr_struct.byteswap() From c3518d169a95d3b8cd6c33d3782c8fc53d23078d Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Tue, 22 Sep 2015 15:48:55 -0400 Subject: [PATCH 27/30] STY: Remove misleading 'sizeof_hdr' from Minc1/2 --- nibabel/minc1.py | 8 +++----- nibabel/minc2.py | 7 +++---- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/nibabel/minc1.py b/nibabel/minc1.py index 34fcc7c75b..f1f2a282ec 100644 --- a/nibabel/minc1.py +++ b/nibabel/minc1.py @@ -269,7 +269,6 @@ class MincHeader(Header): """ # We don't use the data layout - this just in case we do later data_layout = 'C' - sizeof_hdr = 4 def data_to_fileobj(self, data, fileobj, rescale=True): """ See Header class for an implementation we can't use """ @@ -283,11 +282,10 @@ def data_from_fileobj(self, fileobj): class Minc1Header(MincHeader): @classmethod def may_contain_header(klass, binaryblock): - if len(binaryblock) < klass.sizeof_hdr: - raise ValueError('Must pass a binary block >= %d bytes' % - klass.sizeof_hdr) + if len(binaryblock) < 4: + raise ValueError('Must pass a binary block >= 4 bytes') - return binaryblock[:klass.sizeof_hdr] == b'CDF\x01' + return binaryblock[:4] == b'CDF\x01' class Minc1Image(SpatialImage): diff --git a/nibabel/minc2.py b/nibabel/minc2.py index b681a95781..64e22b83d0 100644 --- a/nibabel/minc2.py +++ b/nibabel/minc2.py @@ -137,11 +137,10 @@ def get_scaled_data(self, sliceobj=()): class Minc2Header(MincHeader): @classmethod def may_contain_header(klass, binaryblock): - if len(binaryblock) < klass.sizeof_hdr: - raise ValueError('Must pass a binary block >= %d bytes' % - klass.sizeof_hdr) + if len(binaryblock) < 4: + raise ValueError('Must pass a binary block >= 4 bytes') - return binaryblock[:klass.sizeof_hdr] == b'\211HDF' + return binaryblock[:4] == b'\211HDF' class Minc2Image(Minc1Image): From 5814435790cdd1b531af4584eda46ceb4940656a Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Tue, 6 Oct 2015 12:09:25 -0400 Subject: [PATCH 28/30] RF: Tag sniffs with file name Currently doesn't occur, but possible that different images with a common extension could have different header extensions. This change checks that the header file name is the same before reusing a sniff. --- nibabel/spatialimages.py | 53 ++++++++++++++++++++----------- nibabel/tests/test_image_types.py | 8 +++-- 2 files changed, 40 insertions(+), 21 deletions(-) diff --git a/nibabel/spatialimages.py b/nibabel/spatialimages.py index 48b886f3e9..521f8cf307 100644 --- a/nibabel/spatialimages.py +++ b/nibabel/spatialimages.py @@ -874,7 +874,7 @@ def from_image(klass, img): extra=img.extra.copy()) @classmethod - def _sniff_meta_for(klass, filename, sniff_nbytes): + def _sniff_meta_for(klass, filename, sniff_nbytes, sniff=None): """ Sniff metadata for image represented by `filename` Parameters @@ -886,28 +886,39 @@ def _sniff_meta_for(klass, filename, sniff_nbytes): and read from that instead of `filename`. sniff_nbytes : int Number of bytes to read from the image or metadata file + sniff : (bytes, fname), optional + The result of a previous call to `_sniff_meta_for`. If fname + matches the computed header file name, `sniff` is returned without + rereading the file. Returns ------- - meta_bytes : None or bytes - None if we could not read the image or metadata file. `meta_bytes` + sniff : None or (bytes, fname) + None if we could not read the image or metadata file. `sniff[0]` is either length `sniff_nbytes` or the length of the image / - metadata file, whichever is the shorter. + metadata file, whichever is the shorter. `fname` is the name of + the sniffed file. """ froot, ext, trailing = splitext_addext(filename, klass._compressed_suffixes) - # Determine the metadata location, then sniff it + # Determine the metadata location t_fnames = types_filenames( filename, klass.files_types, trailing_suffixes=klass._compressed_suffixes) meta_fname = t_fnames.get('header', filename) + + # Do not re-sniff if it would be from the same file + if sniff is not None and sniff[1] == meta_fname: + return sniff + + # Attempt to sniff from metadata location try: with ImageOpener(meta_fname, 'rb') as fobj: - sniff = fobj.read(sniff_nbytes) + binaryblock = fobj.read(sniff_nbytes) except IOError: return None - return sniff + return (binaryblock, meta_fname) @classmethod def path_maybe_image(klass, filename, sniff=None, sniff_max=1024): @@ -920,11 +931,13 @@ def path_maybe_image(klass, filename, sniff=None, sniff_max=1024): If `filename` points to an image data file, and the image type has a separate "header" file, we work out the name of the header file, and read from that instead of `filename`. - sniff : None or bytes, optional + sniff : None or (bytes, filename), optional Bytes content read from a previous call to this method, on another - class. This allows us to read metadata bytes once from the image / - or header, and pass this read set of bytes to other image classes, - therefore saving a repeat read of the metadata. None forces this + class, with metadata filename. This allows us to read metadata + bytes once from the image or header, and pass this read set of + bytes to other image classes, therefore saving a repeat read of the + metadata. `filename` is used to validate that metadata would be + read from the same file, re-reading if not. None forces this method to read the metadata. sniff_max : int, optional The maximum number of bytes to read from the metadata. If the @@ -938,7 +951,7 @@ def path_maybe_image(klass, filename, sniff=None, sniff_max=1024): ------- maybe_image : bool True if `filename` may be valid for an image of this class. - sniff : None or bytes + sniff : None or (bytes, filename) Read bytes content from found metadata. May be None if the file does not appear to have useful metadata. """ @@ -948,14 +961,16 @@ def path_maybe_image(klass, filename, sniff=None, sniff_max=1024): return False, sniff if not hasattr(klass.header_class, 'may_contain_header'): return True, sniff - if sniff is None or len(sniff) < klass._meta_sniff_len: - sniff_nbytes = max(klass._meta_sniff_len, sniff_max) - sniff = klass._sniff_meta_for(filename, sniff_nbytes) - if sniff is None: # Can't sniff, won't sniff - return False, None - if len(sniff) < klass._meta_sniff_len: + + # Force re-sniff on too-short sniff + if sniff is not None and len(sniff[0]) < klass._meta_sniff_len: + sniff = None + sniff = klass._sniff_meta_for(filename, + max(klass._meta_sniff_len, sniff_max), + sniff) + if sniff is None or len(sniff[0]) < klass._meta_sniff_len: return False, sniff - return klass.header_class.may_contain_header(sniff), sniff + return klass.header_class.may_contain_header(sniff[0]), sniff def __getitem__(self): ''' No slicing or dictionary interface for images diff --git a/nibabel/tests/test_image_types.py b/nibabel/tests/test_image_types.py index 47de54e4bb..42f4c266e4 100644 --- a/nibabel/tests/test_image_types.py +++ b/nibabel/tests/test_image_types.py @@ -54,6 +54,9 @@ def check_img(img_path, img_klass, sniff_mode, sniff, expect_success, if sniff_mode == 'no_sniff': # Don't pass any sniff--not even "None" is_img, new_sniff = img_klass.path_maybe_image(img_path) + elif sniff_mode in ('empty', 'irrelevant', 'bad_sniff'): + # Add img_path to binaryblock sniff parameters + is_img, new_sniff = img_klass.path_maybe_image(img_path, (sniff, img_path)) else: # Pass a sniff, but don't reuse across images. is_img, new_sniff = img_klass.path_maybe_image(img_path, sniff) @@ -64,7 +67,7 @@ def check_img(img_path, img_klass, sniff_mode, sniff, expect_success, msg) expected_sizeof_hdr = getattr(img_klass.header_class, 'sizeof_hdr', 0) - current_sizeof_hdr = 0 if new_sniff is None else len(new_sniff) + current_sizeof_hdr = 0 if new_sniff is None else len(new_sniff[0]) assert_true(current_sizeof_hdr >= expected_sizeof_hdr, new_msg) # Check that the image type was recognized. @@ -88,7 +91,8 @@ def check_img(img_path, img_klass, sniff_mode, sniff, expect_success, none=None, # pass None as the sniff, should query in fn empty=b'', # pass an empty sniff, should query in fn irrelevant=b'a' * (sizeof_hdr - 1), # A too-small sniff, query - bad_sniff=b'a' * sizeof_hdr).items(): # Bad sniff, should fail + bad_sniff=b'a' * sizeof_hdr, # Bad sniff, should fail + ).items(): for klass in img_klasses: if klass == expected_img_klass: From 76440f29cc2cc234907152026c557fa51688d52d Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Tue, 6 Oct 2015 12:35:55 -0400 Subject: [PATCH 29/30] RF: Fail quietly on too-short header blocks --- nibabel/analyze.py | 3 +-- nibabel/minc1.py | 3 --- nibabel/minc2.py | 3 --- nibabel/nifti1.py | 3 +-- nibabel/nifti2.py | 3 +-- nibabel/spm2analyze.py | 3 +-- nibabel/tests/test_image_types.py | 13 +++++-------- 7 files changed, 9 insertions(+), 22 deletions(-) diff --git a/nibabel/analyze.py b/nibabel/analyze.py index feaba4addd..ee0d127a24 100644 --- a/nibabel/analyze.py +++ b/nibabel/analyze.py @@ -895,8 +895,7 @@ def _chk_pixdims(hdr, fix=False): @classmethod def may_contain_header(klass, binaryblock): if len(binaryblock) < klass.sizeof_hdr: - raise ValueError('Must pass a binary block >= %d bytes' % - klass.sizeof_hdr) + return False hdr_struct = np.ndarray(shape=(), dtype=header_dtype, buffer=binaryblock[:klass.sizeof_hdr]) diff --git a/nibabel/minc1.py b/nibabel/minc1.py index f1f2a282ec..8a155712df 100644 --- a/nibabel/minc1.py +++ b/nibabel/minc1.py @@ -282,9 +282,6 @@ def data_from_fileobj(self, fileobj): class Minc1Header(MincHeader): @classmethod def may_contain_header(klass, binaryblock): - if len(binaryblock) < 4: - raise ValueError('Must pass a binary block >= 4 bytes') - return binaryblock[:4] == b'CDF\x01' diff --git a/nibabel/minc2.py b/nibabel/minc2.py index 64e22b83d0..393fa02180 100644 --- a/nibabel/minc2.py +++ b/nibabel/minc2.py @@ -137,9 +137,6 @@ def get_scaled_data(self, sliceobj=()): class Minc2Header(MincHeader): @classmethod def may_contain_header(klass, binaryblock): - if len(binaryblock) < 4: - raise ValueError('Must pass a binary block >= 4 bytes') - return binaryblock[:4] == b'\211HDF' diff --git a/nibabel/nifti1.py b/nibabel/nifti1.py index 9aabf4d6a0..fc188d9201 100644 --- a/nibabel/nifti1.py +++ b/nibabel/nifti1.py @@ -1614,8 +1614,7 @@ def _chk_xform_code(klass, code_type, hdr, fix): @classmethod def may_contain_header(klass, binaryblock): if len(binaryblock) < klass.sizeof_hdr: - raise ValueError('Must pass a binary block >= %d bytes' % - klass.sizeof_hdr) + return False hdr_struct = np.ndarray(shape=(), dtype=header_dtype, buffer=binaryblock[:klass.sizeof_hdr]) diff --git a/nibabel/nifti2.py b/nibabel/nifti2.py index 74f83b9458..b2f4be0054 100644 --- a/nibabel/nifti2.py +++ b/nibabel/nifti2.py @@ -224,8 +224,7 @@ def _chk_eol_check(hdr, fix=False): @classmethod def may_contain_header(klass, binaryblock): if len(binaryblock) < klass.sizeof_hdr: - raise ValueError('Must pass a binary block >= %d bytes' % - klass.sizeof_hdr) + return False hdr_struct = np.ndarray(shape=(), dtype=header_dtype, buffer=binaryblock[:klass.sizeof_hdr]) diff --git a/nibabel/spm2analyze.py b/nibabel/spm2analyze.py index a121ad631b..7ab93d4514 100644 --- a/nibabel/spm2analyze.py +++ b/nibabel/spm2analyze.py @@ -116,8 +116,7 @@ def get_slope_inter(self): @classmethod def may_contain_header(klass, binaryblock): if len(binaryblock) < klass.sizeof_hdr: - raise ValueError('Must pass a binary block >= %d bytes' % - klass.sizeof_hdr) + return False hdr_struct = np.ndarray(shape=(), dtype=header_dtype, buffer=binaryblock[:klass.sizeof_hdr]) diff --git a/nibabel/tests/test_image_types.py b/nibabel/tests/test_image_types.py index 42f4c266e4..b6ca7ea938 100644 --- a/nibabel/tests/test_image_types.py +++ b/nibabel/tests/test_image_types.py @@ -21,7 +21,7 @@ Spm2AnalyzeImage, Spm99AnalyzeImage, MGHImage, all_image_classes) -from nose.tools import assert_true, assert_raises +from nose.tools import assert_true DATA_PATH = pjoin(dirname(__file__), 'data') @@ -46,17 +46,13 @@ def check_img(img_path, img_klass, sniff_mode, sniff, expect_success, msg): """Embedded function to do the actual checks expected.""" - if sniff_mode == 'empty' and \ - hasattr(img_klass.header_class, 'may_contain_header'): - assert_raises(ValueError, - img_klass.header_class.may_contain_header, sniff) - if sniff_mode == 'no_sniff': # Don't pass any sniff--not even "None" is_img, new_sniff = img_klass.path_maybe_image(img_path) elif sniff_mode in ('empty', 'irrelevant', 'bad_sniff'): # Add img_path to binaryblock sniff parameters - is_img, new_sniff = img_klass.path_maybe_image(img_path, (sniff, img_path)) + is_img, new_sniff = img_klass.path_maybe_image( + img_path, (sniff, img_path)) else: # Pass a sniff, but don't reuse across images. is_img, new_sniff = img_klass.path_maybe_image(img_path, sniff) @@ -67,7 +63,8 @@ def check_img(img_path, img_klass, sniff_mode, sniff, expect_success, msg) expected_sizeof_hdr = getattr(img_klass.header_class, 'sizeof_hdr', 0) - current_sizeof_hdr = 0 if new_sniff is None else len(new_sniff[0]) + current_sizeof_hdr = 0 if new_sniff is None else \ + len(new_sniff[0]) assert_true(current_sizeof_hdr >= expected_sizeof_hdr, new_msg) # Check that the image type was recognized. From 8a926a4f184250d479ddf29942400689379121d1 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Tue, 6 Oct 2015 18:40:16 -0400 Subject: [PATCH 30/30] TEST: path_maybe_image part of image API Not all tests have image_maker attributes, so add klass attribute to test path_maybe_image. Checks structure of return values on calls to example images. --- nibabel/tests/test_image_api.py | 34 ++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/nibabel/tests/test_image_api.py b/nibabel/tests/test_image_api.py index fed9f85d5b..f5a081fd8b 100644 --- a/nibabel/tests/test_image_api.py +++ b/nibabel/tests/test_image_api.py @@ -24,6 +24,7 @@ import warnings from functools import partial +from ..externals.six import string_types import numpy as np @@ -283,11 +284,21 @@ class LoadImageAPI(GenericImageAPI): # Sequence of dictionaries, where dictionaries have keys # 'fname" in addition to keys for ``params`` (see obj_params docstring) example_images = () + # Class of images to be tested + klass = None def obj_params(self): for img_params in self.example_images: yield lambda : self.loader(img_params['fname']), img_params + def validate_path_maybe_image(self, imaker, params): + for img_params in self.example_images: + test, sniff = self.klass.path_maybe_image(img_params['fname']) + assert_true(isinstance(test, bool)) + if sniff is not None: + assert isinstance(sniff[0], bytes) + assert isinstance(sniff[1], string_types) + class MakeImageAPI(LoadImageAPI): """ Validation for images we can make with ``func(data, affine, header)`` @@ -346,48 +357,48 @@ def header_maker(self): class TestAnalyzeAPI(ImageHeaderAPI): """ General image validation API instantiated for Analyze images """ - image_maker = AnalyzeImage + klass = image_maker = AnalyzeImage has_scaling = False can_save = True standard_extension = '.img' class TestSpatialImageAPI(TestAnalyzeAPI): - image_maker = SpatialImage + klass = image_maker = SpatialImage can_save = False class TestSpm99AnalyzeAPI(TestAnalyzeAPI): # SPM-type analyze need scipy for mat file IO - image_maker = Spm99AnalyzeImage + klass = image_maker = Spm99AnalyzeImage has_scaling = True can_save = have_scipy class TestSpm2AnalyzeAPI(TestSpm99AnalyzeAPI): - image_maker = Spm2AnalyzeImage + klass = image_maker = Spm2AnalyzeImage class TestNifti1PairAPI(TestSpm99AnalyzeAPI): - image_maker = Nifti1Pair + klass = image_maker = Nifti1Pair can_save = True class TestNifti1API(TestNifti1PairAPI): - image_maker = Nifti1Image + klass = image_maker = Nifti1Image standard_extension = '.nii' class TestNifti2PairAPI(TestNifti1PairAPI): - image_maker = Nifti2Pair + klass = image_maker = Nifti2Pair class TestNifti2API(TestNifti1API): - image_maker = Nifti2Image + klass = image_maker = Nifti2Image class TestMinc1API(ImageHeaderAPI): - image_maker = Minc1Image + klass = image_maker = Minc1Image loader = minc1.load example_images = MINC1_EXAMPLE_IMAGES @@ -397,7 +408,7 @@ def __init__(self): if not have_h5py: raise SkipTest('Need h5py for these tests') - image_maker = Minc2Image + klass = image_maker = Minc2Image loader = minc2.load example_images = MINC2_EXAMPLE_IMAGES @@ -406,6 +417,7 @@ class TestPARRECAPI(LoadImageAPI): def loader(self, fname): return parrec.load(fname) + klass = parrec.PARRECImage example_images = PARREC_EXAMPLE_IMAGES @@ -418,7 +430,7 @@ def loader(self, fname): class TestMGHAPI(ImageHeaderAPI): - image_maker = MGHImage + klass = image_maker = MGHImage example_shapes = ((2, 3, 4), (2, 3, 4, 5)) # MGH can only do >= 3D has_scaling = True can_save = True