h5py-2.7.1/0000755000175000017500000000000013152363123014063 5ustar tcaswelltcaswell00000000000000h5py-2.7.1/setup_build.py0000644000175000017500000001602713114361073016762 0ustar tcaswelltcaswell00000000000000 """ Implements a custom Distutils build_ext replacement, which handles the full extension module build process, from api_gen to C compilation and linking. """ try: from setuptools import Extension except ImportError: from distutils.extension import Extension from distutils.command.build_ext import build_ext import sys import os import os.path as op import subprocess from functools import reduce import api_gen def localpath(*args): return op.abspath(reduce(op.join, (op.dirname(__file__),)+args)) MODULES = ['defs','_errors','_objects','_proxy', 'h5fd', 'h5z', 'h5','h5i','h5r','utils', '_conv', 'h5t','h5s', 'h5p', 'h5d', 'h5a', 'h5f', 'h5g', 'h5l', 'h5o', 'h5ds', 'h5ac'] EXTRA_SRC = {'h5z': [ localpath("lzf/lzf_filter.c"), localpath("lzf/lzf/lzf_c.c"), localpath("lzf/lzf/lzf_d.c")]} COMPILER_SETTINGS = { 'libraries' : ['hdf5', 'hdf5_hl'], 'include_dirs' : [localpath('lzf')], 'library_dirs' : [], 'define_macros' : [('H5_USE_16_API', None)] } if sys.platform.startswith('win'): COMPILER_SETTINGS['include_dirs'].append(localpath('windows')) COMPILER_SETTINGS['define_macros'].extend([ ('_HDF5USEDLL_', None), ('H5_BUILT_AS_DYNAMIC_LIB', None) ]) else: COMPILER_SETTINGS['include_dirs'].extend(['/opt/local/include', '/usr/local/include']) COMPILER_SETTINGS['library_dirs'].extend(['/opt/local/include', '/usr/local/include']) class h5py_build_ext(build_ext): """ Custom distutils command which encapsulates api_gen pre-building, Cython building, and C compilation. Also handles making the Extension modules, since we can't rely on NumPy being present in the main body of the setup script. """ @staticmethod def _make_extensions(config): """ Produce a list of Extension instances which can be passed to cythonize(). This is the point at which custom directories, MPI options, etc. enter the build process. """ import numpy import pkgconfig settings = COMPILER_SETTINGS.copy() try: if pkgconfig.exists('hdf5'): pkgcfg = pkgconfig.parse("hdf5") settings['include_dirs'].extend(pkgcfg['include_dirs']) settings['library_dirs'].extend(pkgcfg['library_dirs']) settings['define_macros'].extend(pkgcfg['define_macros']) except EnvironmentError: pass try: numpy_includes = numpy.get_include() except AttributeError: # if numpy is not installed get the headers from the .egg directory import numpy.core numpy_includes = os.path.join(os.path.dirname(numpy.core.__file__), 'include') settings['include_dirs'] += [numpy_includes] if config.mpi: import mpi4py settings['include_dirs'] += [mpi4py.get_include()] # Ensure a custom location appears first, so we don't get a copy of # HDF5 from some default location in COMPILER_SETTINGS if config.hdf5 is not None: settings['include_dirs'].insert(0, op.join(config.hdf5, 'include')) settings['library_dirs'].insert(0, op.join(config.hdf5, 'lib')) # TODO: should this only be done on UNIX? if os.name != 'nt': settings['runtime_library_dirs'] = settings['library_dirs'] def make_extension(module): sources = [localpath('h5py', module+'.pyx')] + EXTRA_SRC.get(module, []) return Extension('h5py.'+module, sources, **settings) return [make_extension(m) for m in MODULES] @staticmethod def run_system_cython(pyx_files): try: retcode = subprocess.call(['cython', '--fast-fail', '--verbose'] + pyx_files) if not retcode == 0: raise Exception('ERROR: Cython failed') except OSError as e: print("ERROR: cython exec failed. Is cython not in the path? ", str(e)) raise except Exception as e: print("ERROR: cython exec failed", str(e)) raise def check_rerun_cythonize(self): """ Check whether the cythonize() call produced the expected .c files. If the expected .c files are not found then cython from the system path will be executed in order to produce the missing files. """ missing_c_src_files = [] for c_src_file in [ext.sources[0] for ext in self.extensions]: if not op.isfile(c_src_file): missing_c_src_files.append( c_src_file ) if missing_c_src_files: print("WARNING: cythonize() failed to create all .c files (setuptools too old?)") pyx_files = [os.path.splitext(fname)[0] + ".pyx" for fname in missing_c_src_files] print(" Executing system cython on pyx files: ", str(pyx_files)) self.run_system_cython(pyx_files) def run(self): """ Distutils calls this method to run the command """ from Cython.Build import cythonize import numpy # Provides all of our build options config = self.distribution.get_command_obj('configure') config.run() defs_file = localpath('h5py', 'defs.pyx') func_file = localpath('h5py', 'api_functions.txt') config_file = localpath('h5py', 'config.pxi') # Rebuild low-level defs if missing or stale if not op.isfile(defs_file) or os.stat(func_file).st_mtime > os.stat(defs_file).st_mtime: print("Executing api_gen rebuild of defs") api_gen.run() # Rewrite config.pxi file if needed if not op.isfile(config_file) or config.rebuild_required: with open(config_file, 'wb') as f: if config.mpi: import mpi4py from distutils.version import StrictVersion v2 = StrictVersion(mpi4py.__version__) > StrictVersion("1.3.1") else: v2 = False s = """\ # This file is automatically generated by the h5py setup script. Don't modify. DEF MPI = %(mpi)s DEF MPI4PY_V2 = %(mpi4py_v2)s DEF HDF5_VERSION = %(version)s DEF SWMR_MIN_HDF5_VERSION = (1,9,178) DEF VDS_MIN_HDF5_VERSION = (1,9,233) DEF COMPLEX256_SUPPORT = %(complex256_support)s """ s %= { 'mpi': bool(config.mpi), 'mpi4py_v2': bool(v2), 'version': tuple(int(x) for x in config.hdf5_version.split('.')), 'complex256_support': hasattr(numpy, 'complex256') } s = s.encode('utf-8') f.write(s) # Run Cython print("Executing cythonize()") self.extensions = cythonize(self._make_extensions(config), force=config.rebuild_required or self.force) self.check_rerun_cythonize() # Perform the build build_ext.run(self) # Mark the configuration as built config.reset_rebuild() h5py-2.7.1/setup.py0000755000175000017500000001225613152363105015606 0ustar tcaswelltcaswell00000000000000#!/usr/bin/env python """ This is the main setup script for h5py (http://www.h5py.org). Most of the functionality is provided in two separate modules: setup_configure, which manages compile-time/Cython-time build options for h5py, and setup_build, which handles the actual compilation process. """ try: from setuptools import Extension, setup except ImportError: from distutils.core import setup from distutils.extension import Extension from distutils.cmd import Command from distutils.dist import Distribution import sys import os import os.path as op import setup_build, setup_configure VERSION = '2.7.1' NUMPY_DEP = 'numpy>=1.7' # these are required to use h5py RUN_REQUIRES = [NUMPY_DEP, 'six'] # these are required to build h5py # RUN_REQUIRES is included as setup.py test needs RUN_REQUIRES for testing # RUN_REQUIRES can be removed when setup.py test is removed SETUP_REQUIRES = RUN_REQUIRES + [NUMPY_DEP, 'Cython>=0.19', 'pkgconfig'] # Needed to avoid trying to install numpy/cython on pythons which the latest # versions don't support if ("sdist" in sys.argv and "bdist_wheel" not in sys.argv and "install" not in sys.argv) or "check" in sys.argv: use_setup_requires = False else: use_setup_requires = True # --- Custom Distutils commands ----------------------------------------------- class test(Command): """ Custom Distutils command to run the h5py test suite. This command will invoke build/build_ext if the project has not already been built. It then patches in the build directory to sys.path and runs the test suite directly. """ description = "Run the test suite" user_options = [('detail', 'd', 'Display additional test information')] def initialize_options(self): self.detail = False def finalize_options(self): self.detail = bool(self.detail) def run(self): """ Called by Distutils when this command is run """ import sys py_version = sys.version_info[:2] if py_version != (2, 6): import unittest else: try: import unittest2 as unittest except ImportError: raise ImportError( "unittest2 is required to run tests with Python 2.6") buildobj = self.distribution.get_command_obj('build') buildobj.run() oldpath = sys.path try: sys.path = [op.abspath(buildobj.build_lib)] + oldpath import h5py result = h5py.run_tests(verbose=self.detail) if not result.wasSuccessful(): sys.exit(1) finally: sys.path = oldpath CMDCLASS = {'build_ext': setup_build.h5py_build_ext, 'configure': setup_configure.configure, 'test': test, } # --- Distutils setup and metadata -------------------------------------------- cls_txt = \ """ Development Status :: 5 - Production/Stable Intended Audience :: Developers Intended Audience :: Information Technology Intended Audience :: Science/Research License :: OSI Approved :: BSD License Programming Language :: Cython Programming Language :: Python Programming Language :: Python :: 2 Programming Language :: Python :: 2.6 Programming Language :: Python :: 2.7 Programming Language :: Python :: 3 Programming Language :: Python :: 3.3 Programming Language :: Python :: 3.4 Programming Language :: Python :: 3.5 Programming Language :: Python :: 3.6 Programming Language :: Python :: Implementation :: CPython Topic :: Scientific/Engineering Topic :: Database Topic :: Software Development :: Libraries :: Python Modules Operating System :: Unix Operating System :: POSIX :: Linux Operating System :: MacOS :: MacOS X Operating System :: Microsoft :: Windows """ short_desc = "Read and write HDF5 files from Python" long_desc = \ """ The h5py package provides both a high- and low-level interface to the HDF5 library from Python. The low-level interface is intended to be a complete wrapping of the HDF5 API, while the high-level component supports access to HDF5 files, datasets and groups using established Python and NumPy concepts. A strong emphasis on automatic conversion between Python (Numpy) datatypes and data structures and their HDF5 equivalents vastly simplifies the process of reading and writing data from Python. Supports HDF5 versions 1.8.4 and higher. On Windows, HDF5 is included with the installer. """ if os.name == 'nt': package_data = {'h5py': ['*.dll']} else: package_data = {'h5py': []} setup( name = 'h5py', version = VERSION, description = short_desc, long_description = long_desc, classifiers = [x for x in cls_txt.split("\n") if x], author = 'Andrew Collette', author_email = 'andrew.collette@gmail.com', maintainer = 'Andrew Collette', maintainer_email = 'andrew.collette@gmail.com', url = 'http://www.h5py.org', download_url = 'https://pypi.python.org/pypi/h5py', packages = ['h5py', 'h5py._hl', 'h5py.tests', 'h5py.tests.old', 'h5py.tests.hl'], package_data = package_data, ext_modules = [Extension('h5py.x',['x.c'])], # To trick build into running build_ext install_requires = RUN_REQUIRES, setup_requires = SETUP_REQUIRES if use_setup_requires else [], cmdclass = CMDCLASS, ) h5py-2.7.1/setup_configure.py0000644000175000017500000001745013114362755017655 0ustar tcaswelltcaswell00000000000000 """ Implements a new custom Distutils command for handling library configuration. The "configure" command here doesn't directly affect things like config.pxi; rather, it exists to provide a set of attributes that are used by the build_ext replacement in setup_build.py. Options from the command line and environment variables are stored between invocations in a pickle file. This allows configuring the library once and e.g. calling "build" and "test" without recompiling everything or explicitly providing the same options every time. This module also contains the auto-detection logic for figuring out the currently installed HDF5 version. """ from distutils.cmd import Command import os import os.path as op import sys import pickle def loadpickle(): """ Load settings dict from the pickle file """ try: with open('h5config.pkl','rb') as f: cfg = pickle.load(f) if not isinstance(cfg, dict): raise TypeError except Exception: return {} return cfg def savepickle(dct): """ Save settings dict to the pickle file """ with open('h5config.pkl','wb') as f: pickle.dump(dct, f, protocol=0) def validate_version(s): """ Ensure that s contains an X.Y.Z format version string, or ValueError. """ try: tpl = tuple(int(x) for x in s.split('.')) if len(tpl) != 3: raise ValueError except Exception: raise ValueError("HDF5 version string must be in X.Y.Z format") class EnvironmentOptions(object): """ Convenience class representing the current environment variables. """ def __init__(self): self.hdf5 = os.environ.get('HDF5_DIR') self.hdf5_version = os.environ.get('HDF5_VERSION') self.mpi = os.environ.get('HDF5_MPI') == "ON" if self.hdf5_version is not None: validate_version(self.hdf5_version) class configure(Command): """ Configure build options for h5py: custom path to HDF5, version of the HDF5 library, and whether MPI is enabled. Options come from the following sources, in order of priority: 1. Current command-line options 2. Old command-line options 3. Current environment variables 4. Old environment variables 5. Autodetection When options change, the rebuild_required attribute is set, and may only be reset by calling reset_rebuild(). The custom build_ext command does this.s """ description = "Configure h5py build options" user_options = [('hdf5=', 'h', 'Custom path to HDF5'), ('hdf5-version=', '5', 'HDF5 version "X.Y.Z"'), ('mpi', 'm', 'Enable MPI building'), ('reset', 'r', 'Reset config options') ] def initialize_options(self): self.hdf5 = None self.hdf5_version = None self.mpi = None self.reset = None def finalize_options(self): if self.hdf5_version is not None: validate_version(self.hdf5_version) def reset_rebuild(self): """ Mark this configuration as built """ dct = loadpickle() dct['rebuild'] = False savepickle(dct) def run(self): """ Distutils calls this when the command is run """ env = EnvironmentOptions() # Step 1: determine if settings have changed and update cache oldsettings = {} if self.reset else loadpickle() dct = oldsettings.copy() # Only update settings which have actually been specified this # round; ignore the others (which have value None). if self.hdf5 is not None: dct['cmd_hdf5'] = self.hdf5 if env.hdf5 is not None: dct['env_hdf5'] = env.hdf5 if self.hdf5_version is not None: dct['cmd_hdf5_version'] = self.hdf5_version if env.hdf5_version is not None: dct['env_hdf5_version'] = env.hdf5_version if self.mpi is not None: dct['cmd_mpi'] = self.mpi if env.mpi is not None: dct['env_mpi'] = env.mpi self.rebuild_required = dct.get('rebuild') or dct != oldsettings # Corner case: rebuild if options reset, but only if they previously # had non-default values (to handle multiple resets in a row) if self.reset and any(loadpickle().values()): self.rebuild_required = True dct['rebuild'] = self.rebuild_required savepickle(dct) # Step 2: update public config attributes according to priority rules if self.hdf5 is None: self.hdf5 = oldsettings.get('cmd_hdf5') if self.hdf5 is None: self.hdf5 = env.hdf5 if self.hdf5 is None: self.hdf5 = oldsettings.get('env_hdf5') if self.hdf5_version is None: self.hdf5_version = oldsettings.get('cmd_hdf5_version') if self.hdf5_version is None: self.hdf5_version = env.hdf5_version if self.hdf5_version is None: self.hdf5_version = oldsettings.get('env_hdf5_version') if self.hdf5_version is None: try: self.hdf5_version = autodetect_version(self.hdf5) print("Autodetected HDF5 %s" % self.hdf5_version) except Exception as e: sys.stderr.write("Autodetection skipped [%s]\n" % e) self.hdf5_version = '1.8.4' if self.mpi is None: self.mpi = oldsettings.get('cmd_mpi') if self.mpi is None: self.mpi = env.mpi if self.mpi is None: self.mpi = oldsettings.get('env_mpi') # Step 3: print the resulting configuration to stdout print('*' * 80) print(' ' * 23 + "Summary of the h5py configuration") print('') print(" Path to HDF5: " + repr(self.hdf5)) print(" HDF5 Version: " + repr(self.hdf5_version)) print(" MPI Enabled: " + repr(bool(self.mpi))) print("Rebuild Required: " + repr(bool(self.rebuild_required))) print('') print('*' * 80) def autodetect_version(hdf5_dir=None): """ Detect the current version of HDF5, and return X.Y.Z version string. Intended for Unix-ish platforms (Linux, OS X, BSD). Does not support Windows. Raises an exception if anything goes wrong. hdf5_dir: optional HDF5 install directory to look in (containing "lib") """ import os import sys import os.path as op import re import ctypes from ctypes import byref import pkgconfig if sys.platform.startswith('darwin'): regexp = re.compile(r'^libhdf5.dylib') else: regexp = re.compile(r'^libhdf5.so') libdirs = ['/usr/local/lib', '/opt/local/lib'] try: if pkgconfig.exists("hdf5"): libdirs.extend(pkgconfig.parse("hdf5")['library_dirs']) except EnvironmentError: pass if hdf5_dir is not None: libdirs.insert(0, op.join(hdf5_dir, 'lib')) path = None for d in libdirs: try: candidates = [x for x in os.listdir(d) if regexp.match(x)] except Exception: continue # Skip invalid entries if len(candidates) != 0: candidates.sort(key=lambda x: len(x)) # Prefer libfoo.so to libfoo.so.X.Y.Z path = op.abspath(op.join(d, candidates[0])) break if path is None: path = "libhdf5.so" lib = ctypes.cdll.LoadLibrary(path) major = ctypes.c_uint() minor = ctypes.c_uint() release = ctypes.c_uint() lib.H5get_libversion(byref(major), byref(minor), byref(release)) return "{0}.{1}.{2}".format(int(major.value), int(minor.value), int(release.value)) h5py-2.7.1/setup.cfg0000644000175000017500000000004613152363123015704 0ustar tcaswelltcaswell00000000000000[egg_info] tag_build = tag_date = 0 h5py-2.7.1/examples/0000755000175000017500000000000013152363123015701 5ustar tcaswelltcaswell00000000000000h5py-2.7.1/examples/collective_io.py0000644000175000017500000000342513114362767021112 0ustar tcaswelltcaswell00000000000000# This file is to test collective io in h5py """ Author: Jialin Liu, jalnliu@lbl.gov Date: Nov 17, 2015 Prerequisites: python 2.5.0, mpi4py and numpy Source Codes: Already submit this 'collective io' branch to h5py master, meanwhile, can download this branch at https://github.com/valiantljk/h5py.git Note: Must build the h5py with parallel hdf5 """ from mpi4py import MPI import numpy as np import h5py import time import sys #"run as "mpirun -np 64 python-mpi collective_io.py 1 file.h5" #(1 is for collective write, ohter number for non-collective write)" colw=1 #default is collective write filename="parallel_test.hdf5" if len(sys.argv)>2: colw = int(sys.argv[1]) filename=str(sys.argv[2]) comm =MPI.COMM_WORLD nproc = comm.Get_size() f = h5py.File(filename, 'w', driver='mpio', comm=MPI.COMM_WORLD) rank = comm.Get_rank() length_x = 6400*1024 length_y = 1024 dset = f.create_dataset('test', (length_x,length_y), dtype='f8') #data type should be consistent in numpy and h5py, e.g., 64 bits #otherwise, hdf5 layer will fall back to independent io. f.atomic = False length_rank=length_x / nproc length_last_rank=length_x -length_rank*(nproc-1) comm.Barrier() timestart=MPI.Wtime() start=rank*length_rank end=start+length_rank if rank==nproc-1: #last rank end=start+length_last_rank temp=np.random.random((end-start,length_y)) if colw==1: with dset.collective: dset[start:end,:] = temp else : dset[start:end,:] = temp comm.Barrier() timeend=MPI.Wtime() if rank==0: if colw==1: print "collective write time %f" %(timeend-timestart) else : print "independent write time %f" %(timeend-timestart) print "data size x: %d y: %d" %(length_x, length_y) print "file size ~%d GB" % (length_x*length_y/1024.0/1024.0/1024.0*8.0) print "number of processes %d" %nproc f.close() h5py-2.7.1/examples/swmr_multiprocess.py0000644000175000017500000000763013114361073022062 0ustar tcaswelltcaswell00000000000000""" Demonstrate the use of h5py in SWMR mode to write to a dataset (appending) from one process while monitoring the growing dataset from another process. Usage: swmr_multiprocess.py [FILENAME [DATASETNAME]] FILENAME: name of file to monitor. Default: swmrmp.h5 DATASETNAME: name of dataset to monitor in DATAFILE. Default: data This script will start up two processes: a writer and a reader. The writer will open/create the file (FILENAME) in SWMR mode, create a dataset and start appending data to it. After each append the dataset is flushed and an event sent to the reader process. Meanwhile the reader process will wait for events from the writer and when triggered it will refresh the dataset and read the current shape of it. """ import sys, time import h5py import numpy as np import logging from multiprocessing import Process, Event class SwmrReader(Process): def __init__(self, event, fname, dsetname, timeout = 2.0): super(SwmrReader, self).__init__() self._event = event self._fname = fname self._dsetname = dsetname self._timeout = timeout def run(self): self.log = logging.getLogger('reader') self.log.info("Waiting for initial event") assert self._event.wait( self._timeout ) self._event.clear() self.log.info("Opening file %s", self._fname) f = h5py.File(self._fname, 'r', libver='latest', swmr=True) assert f.swmr_mode dset = f[self._dsetname] try: # monitor and read loop while self._event.wait( self._timeout ): self._event.clear() self.log.debug("Refreshing dataset") dset.refresh() shape = dset.shape self.log.info("Read dset shape: %s"%str(shape)) finally: f.close() class SwmrWriter(Process): def __init__(self, event, fname, dsetname): super(SwmrWriter, self).__init__() self._event = event self._fname = fname self._dsetname = dsetname def run(self): self.log = logging.getLogger('writer') self.log.info("Creating file %s", self._fname) f = h5py.File(self._fname, 'w', libver='latest') try: arr = np.array([1,2,3,4]) dset = f.create_dataset(self._dsetname, chunks=(2,), maxshape=(None,), data=arr) assert not f.swmr_mode self.log.info("SWMR mode") f.swmr_mode = True assert f.swmr_mode self.log.debug("Sending initial event") self._event.set() # Write loop for i in range(5): new_shape = ((i+1) * len(arr), ) self.log.info("Resizing dset shape: %s"%str(new_shape)) dset.resize( new_shape ) self.log.debug("Writing data") dset[i*len(arr):] = arr #dset.write_direct( arr, np.s_[:], np.s_[i*len(arr):] ) self.log.debug("Flushing data") dset.flush() self.log.info("Sending event") self._event.set() finally: f.close() if __name__ == "__main__": logging.basicConfig(format='%(levelname)10s %(asctime)s %(name)10s %(message)s',level=logging.INFO) fname = 'swmrmp.h5' dsetname = 'data' if len(sys.argv) > 1: fname = sys.argv[1] if len(sys.argv) > 2: dsetname = sys.argv[2] event = Event() reader = SwmrReader(event, fname, dsetname) writer = SwmrWriter(event, fname, dsetname) logging.info("Starting reader") reader.start() logging.info("Starting reader") writer.start() logging.info("Waiting for writer to finish") writer.join() logging.info("Waiting for reader to finish") reader.join() h5py-2.7.1/examples/swmr_inotify_example.py0000644000175000017500000000533313114361073022523 0ustar tcaswelltcaswell00000000000000 """ Demonstrate the use of h5py in SWMR mode to monitor the growth of a dataset on nofication of file modifications. This demo uses pyinotify as a wrapper of Linux inotify. https://pypi.python.org/pypi/pyinotify Usage: swmr_inotify_example.py [FILENAME [DATASETNAME]] FILENAME: name of file to monitor. Default: swmr.h5 DATASETNAME: name of dataset to monitor in DATAFILE. Default: data This script will open the file in SWMR mode and monitor the shape of the dataset on every write event (from inotify). If another application is concurrently writing data to the file, the writer must have have switched the file into SWMR mode before this script can open the file. """ import asyncore import pyinotify import sys import h5py import logging #assert h5py.version.hdf5_version_tuple >= (1,9,178), "SWMR requires HDF5 version >= 1.9.178" class EventHandler(pyinotify.ProcessEvent): def monitor_dataset(self, filename, datasetname): logging.info("Opening file %s", filename) self.f = h5py.File(filename, 'r', libver='latest', swmr=True) logging.debug("Looking up dataset %s"%datasetname) self.dset = self.f[datasetname] self.get_dset_shape() def get_dset_shape(self): logging.debug("Refreshing dataset") self.dset.refresh() logging.debug("Getting shape") shape = self.dset.shape logging.info("Read data shape: %s"%str(shape)) return shape def read_dataset(self, latest): logging.info("Reading out dataset [%d]"%latest) self.dset[latest:] def process_IN_MODIFY(self, event): logging.debug("File modified!") shape = self.get_dset_shape() self.read_dataset(shape[0]) def process_IN_CLOSE_WRITE(self, event): logging.info("File writer closed file") self.get_dset_shape() logging.debug("Good bye!") sys.exit(0) if __name__ == "__main__": logging.basicConfig(format='%(asctime)s %(levelname)s\t%(message)s',level=logging.INFO) file_name = "swmr.h5" if len(sys.argv) > 1: file_name = sys.argv[1] dataset_name = "data" if len(sys.argv) > 2: dataset_name = sys.argv[2] wm = pyinotify.WatchManager() # Watch Manager mask = pyinotify.IN_MODIFY | pyinotify.IN_CLOSE_WRITE evh = EventHandler() evh.monitor_dataset( file_name, dataset_name ) notifier = pyinotify.AsyncNotifier(wm, evh) wdd = wm.add_watch(file_name, mask, rec=False) # Sit in this loop() until the file writer closes the file # or the user hits ctrl-c asyncore.loop() h5py-2.7.1/examples/threading_example.py0000644000175000017500000002513713114362771021751 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. """ Demonstrates use of h5py in a multi-threaded GUI program. In a perfect world, multi-threaded programs would practice strict separation of tasks, with separate threads for HDF5, user interface, processing, etc, communicating via queues. In the real world, shared state is frequently encountered, especially in the world of GUIs. It's quite common to initialize a shared resource (in this case an HDF5 file), and pass it around between threads. One must then be careful to regulate access using locks, to ensure that each thread sees the file in a consistent fashion. This program demonstrates how to use h5py in a medium-sized "shared-state" threading application. Two threads exist: a GUI thread (Tkinter) which takes user input and displays results, and a calculation thread which is used to perform computation in the background, leaving the GUI responsive to user input. The computation thread calculates portions of the Mandelbrot set and stores them in an HDF5 file. The visualization/control thread reads datasets from the same file and displays them using matplotlib. """ import Tkinter as tk import threading import numpy as np import pylab as p from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg from matplotlib.figure import Figure import h5py file_lock = threading.RLock() # Protects the file from concurrent access t = None # We'll use this to store the active computation thread class ComputeThread(threading.Thread): """ Computes a slice of the Mandelbrot set, and saves it to the HDF5 file. """ def __init__(self, f, shape, escape, startcoords, extent, eventcall): """ Set up a computation thread. f: HDF5 File object shape: 2-tuple (NX, NY) escape: Integer giving max iterations to escape start: Complex number giving initial location on the plane extent: Complex number giving calculation extent on the plane """ self.f = f self.shape = shape self.escape = escape self.startcoords = startcoords self.extent = extent self.eventcall = eventcall threading.Thread.__init__(self) def run(self): """ Perform computations and record the result to file """ nx, ny = self.shape arr = np.ndarray((nx,ny), dtype='i') xincr = self.extent.real/nx yincr = self.extent.imag/ny def compute_escape(pos, escape): """ Compute the number of steps required to escape """ z = 0+0j; for i in xrange(escape): z = z**2 + pos if abs(z) > 2: break return i for x in xrange(nx): if x%25 == 0: print "Computing row %d" % x for y in xrange(ny): pos = self.startcoords + complex(x*xincr, y*yincr) arr[x,y] = compute_escape(pos, self.escape) with file_lock: dsname = "slice%03d" % len(self.f) dset = self.f.create_dataset(dsname, (nx, ny), 'i') dset.attrs['shape'] = self.shape dset.attrs['start'] = self.startcoords dset.attrs['extent'] = self.extent dset.attrs['escape'] = self.escape dset[...] = arr print "Calculation for %s done" % dsname self.eventcall() class ComputeWidget(object): """ Responsible for input widgets, and starting new computation threads. """ def __init__(self, f, master, eventcall): self.f = f self.eventcall = eventcall self.mainframe = tk.Frame(master=master) entryframe = tk.Frame(master=self.mainframe) nxlabel = tk.Label(entryframe, text="NX") nylabel = tk.Label(entryframe, text="NY") escapelabel = tk.Label(entryframe, text="Escape") startxlabel = tk.Label(entryframe, text="Start X") startylabel = tk.Label(entryframe, text="Start Y") extentxlabel = tk.Label(entryframe, text="Extent X") extentylabel = tk.Label(entryframe, text="Extent Y") self.nxfield = tk.Entry(entryframe) self.nyfield = tk.Entry(entryframe) self.escapefield = tk.Entry(entryframe) self.startxfield = tk.Entry(entryframe) self.startyfield = tk.Entry(entryframe) self.extentxfield = tk.Entry(entryframe) self.extentyfield = tk.Entry(entryframe) nxlabel.grid(row=0, column=0, sticky=tk.E) nylabel.grid(row=1, column=0, sticky=tk.E) escapelabel.grid(row=2, column=0, sticky=tk.E) startxlabel.grid(row=3, column=0, sticky=tk.E) startylabel.grid(row=4, column=0, sticky=tk.E) extentxlabel.grid(row=5, column=0, sticky=tk.E) extentylabel.grid(row=6, column=0, sticky=tk.E) self.nxfield.grid(row=0, column=1) self.nyfield.grid(row=1, column=1) self.escapefield.grid(row=2, column=1) self.startxfield.grid(row=3, column=1) self.startyfield.grid(row=4, column=1) self.extentxfield.grid(row=5, column=1) self.extentyfield.grid(row=6, column=1) entryframe.grid(row=0, rowspan=2, column=0) self.suggestbutton = tk.Button(master=self.mainframe, text="Suggest", command=self.suggest) self.computebutton = tk.Button(master=self.mainframe, text="Compute", command=self.compute) self.suggestbutton.grid(row=0, column=1) self.computebutton.grid(row=1, column=1) self.suggest = 0 def compute(self, *args): """ Validate input and start calculation thread. We use a global variable "t" to store the current thread, to make sure old threads are properly joined before they are discarded. """ global t try: nx = int(self.nxfield.get()) ny = int(self.nyfield.get()) escape = int(self.escapefield.get()) start = complex(float(self.startxfield.get()), float(self.startyfield.get())) extent = complex(float(self.extentxfield.get()), float(self.extentyfield.get())) if (nx<=0) or (nx<=0) or (escape<=0): raise ValueError("NX, NY and ESCAPE must be positive") if abs(extent)==0: raise ValueError("Extent must be finite") except (ValueError, TypeError), e: print e return if t is not None: t.join() t = ComputeThread(self.f, (nx,ny), escape, start, extent, self.eventcall) t.start() def suggest(self, *args): """ Populate the input fields with interesting locations """ suggestions = [(200,200,50, -2, -1, 3, 2), (500, 500, 200, 0.110, -0.680, 0.05, 0.05), (200, 200, 1000, -0.16070135-5e-8, 1.0375665-5e-8, 1e-7, 1e-7), (500, 500, 100, -1, 0, 0.5, 0.5)] for entry, val in zip((self.nxfield, self.nyfield, self.escapefield, self.startxfield, self.startyfield, self.extentxfield, self.extentyfield), suggestions[self.suggest]): entry.delete(0, 999) entry.insert(0, repr(val)) self.suggest = (self.suggest+1)%len(suggestions) class ViewWidget(object): """ Draws images using the datasets recorded in the HDF5 file. Also provides widgets to pick which dataset is displayed. """ def __init__(self, f, master): self.f = f self.mainframe = tk.Frame(master=master) self.lbutton = tk.Button(self.mainframe, text="<= Back", command=self.back) self.rbutton = tk.Button(self.mainframe, text="Next =>", command=self.forward) self.loclabel = tk.Label(self.mainframe, text='To start, enter values and click "compute"') self.infolabel = tk.Label(self.mainframe, text='Or, click the "suggest" button for interesting locations') self.fig = Figure(figsize=(5,5), dpi=100) self.plot = self.fig.add_subplot(111) self.canvas = FigureCanvasTkAgg(self.fig, master=self.mainframe) self.canvas.show() self.loclabel.grid(row=0, column=1) self.infolabel.grid(row=1, column=1) self.lbutton.grid(row=2, column=0) self.canvas.get_tk_widget().grid(row=2, column=1) self.rbutton.grid(row=2, column=2) self.index = 0 self.jumptolast() def draw_fractal(self): """ Read a dataset from the HDF5 file and display it """ with file_lock: name = self.f.keys()[self.index] dset = self.f[name] arr = dset[...] start = dset.attrs['start'] extent = dset.attrs['extent'] self.loclabel["text"] = 'Displaying dataset "%s" (%d of %d)' % (dset.name, self.index+1, len(self.f)) self.infolabel["text"] = "%(shape)s pixels, starts at %(start)s, extent %(extent)s" % dset.attrs self.plot.clear() self.plot.imshow(arr.transpose(), cmap='jet', aspect='auto', origin='lower', extent=(start.real, (start.real+extent.real), start.imag, (start.imag+extent.imag))) self.canvas.show() def back(self): """ Go to the previous dataset (in ASCII order) """ if self.index == 0: print "Can't go back" return self.index -= 1 self.draw_fractal() def forward(self): """ Go to the next dataset (in ASCII order) """ if self.index == (len(self.f)-1): print "Can't go forward" return self.index += 1 self.draw_fractal() def jumptolast(self,*args): """ Jump to the last (ASCII order) dataset and display it """ with file_lock: if len(self.f) == 0: print "can't jump to last (no datasets)" return index = len(self.f)-1 self.index = index self.draw_fractal() if __name__ == '__main__': f = h5py.File('mandelbrot_gui.hdf5','a') root = tk.Tk() display = ViewWidget(f, root) root.bind("<>", display.jumptolast) def callback(): root.event_generate("<>") compute = ComputeWidget(f, root, callback) display.mainframe.grid(row=0, column=0) compute.mainframe.grid(row=1, column=0) try: root.mainloop() finally: if t is not None: t.join() f.close() h5py-2.7.1/examples/multiprocessing_example.py0000644000175000017500000000700413114361073023216 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. """ Demonstrates how to use h5py with the multiprocessing module. This module implements a simple multi-process program to generate Mandelbrot set images. It uses a process pool to do the computations, and a single process to save the results to file. Importantly, only one process actually reads/writes the HDF5 file. Remember that when a process is fork()ed, the child inherits the HDF5 state from its parent, which can be dangerous if you already have a file open. Trying to interact with the same file on disk from multiple processes results in undefined behavior. If matplotlib is available, the program will read from the HDF5 file and display an image of the fractal in a window. To re-run the calculation, delete the file "mandelbrot.hdf5". """ from __future__ import print_function import numpy as np import multiprocessing as mp import h5py # === Parameters for Mandelbrot calculation =================================== NX = 512 NY = 512 ESCAPE = 1000 XSTART = -0.16070135 - 5e-8 YSTART = 1.0375665 -5e-8 XEXTENT = 1.0E-7 YEXTENT = 1.0E-7 xincr = XEXTENT*1.0/NX yincr = YEXTENT*1.0/NY # === Functions to compute set ================================================ def compute_escape(pos): """ Compute the number of steps required to escape from a point on the complex plane """ z = 0+0j; for i in xrange(ESCAPE): z = z**2 + pos if abs(z) > 2: break return i def compute_row(xpos): """ Compute a 1-D array containing escape step counts for each y-position. """ a = np.ndarray((NY,), dtype='i') for y in xrange(NY): pos = complex(XSTART,YSTART) + complex(xpos, y*yincr) a[y] = compute_escape(pos) return a # === Functions to run process pool & visualize =============================== def run_calculation(): """ Begin multi-process calculation, and save to file """ print("Creating %d-process pool" % mp.cpu_count()) pool = mp.Pool(mp.cpu_count()) f = h5py.File('mandelbrot.hdf5','w') print("Creating output dataset with shape %s x %s" % (NX, NY)) dset = f.create_dataset('mandelbrot', (NX,NY), 'i') dset.attrs['XSTART'] = XSTART dset.attrs['YSTART'] = YSTART dset.attrs['XEXTENT'] = XEXTENT dset.attrs['YEXTENT'] = YEXTENT result = pool.imap(compute_row, (x*xincr for x in xrange(NX))) for idx, arr in enumerate(result): if idx%25 == 0: print("Recording row %s" % idx) dset[idx] = arr print("Closing HDF5 file") f.close() print("Shutting down process pool") pool.close() pool.join() def visualize_file(): """ Open the HDF5 file and display the result """ try: import pylab as p except ImportError: print("Whoops! Matplotlib is required to view the fractal.") raise f = h5py.File('mandelbrot.hdf5','r') dset = f['mandelbrot'] a = dset[...] p.imshow(a.transpose()) print("Displaying fractal. Close window to exit program.") try: p.show() finally: f.close() if __name__ == '__main__': if not h5py.is_hdf5('mandelbrot.hdf5'): run_calculation() else: print('Fractal found in "mandelbrot.hdf5". Delete file to re-run calculation.') visualize_file() h5py-2.7.1/PKG-INFO0000644000175000017500000000410513152363123015160 0ustar tcaswelltcaswell00000000000000Metadata-Version: 1.1 Name: h5py Version: 2.7.1 Summary: Read and write HDF5 files from Python Home-page: http://www.h5py.org Author: Andrew Collette Author-email: andrew.collette@gmail.com License: UNKNOWN Download-URL: https://pypi.python.org/pypi/h5py Description: The h5py package provides both a high- and low-level interface to the HDF5 library from Python. The low-level interface is intended to be a complete wrapping of the HDF5 API, while the high-level component supports access to HDF5 files, datasets and groups using established Python and NumPy concepts. A strong emphasis on automatic conversion between Python (Numpy) datatypes and data structures and their HDF5 equivalents vastly simplifies the process of reading and writing data from Python. Supports HDF5 versions 1.8.4 and higher. On Windows, HDF5 is included with the installer. Platform: UNKNOWN Classifier: Development Status :: 5 - Production/Stable Classifier: Intended Audience :: Developers Classifier: Intended Audience :: Information Technology Classifier: Intended Audience :: Science/Research Classifier: License :: OSI Approved :: BSD License Classifier: Programming Language :: Cython Classifier: Programming Language :: Python Classifier: Programming Language :: Python :: 2 Classifier: Programming Language :: Python :: 2.6 Classifier: Programming Language :: Python :: 2.7 Classifier: Programming Language :: Python :: 3 Classifier: Programming Language :: Python :: 3.3 Classifier: Programming Language :: Python :: 3.4 Classifier: Programming Language :: Python :: 3.5 Classifier: Programming Language :: Python :: 3.6 Classifier: Programming Language :: Python :: Implementation :: CPython Classifier: Topic :: Scientific/Engineering Classifier: Topic :: Database Classifier: Topic :: Software Development :: Libraries :: Python Modules Classifier: Operating System :: Unix Classifier: Operating System :: POSIX :: Linux Classifier: Operating System :: MacOS :: MacOS X Classifier: Operating System :: Microsoft :: Windows h5py-2.7.1/windows/0000755000175000017500000000000013152363123015555 5ustar tcaswelltcaswell00000000000000h5py-2.7.1/windows/cacheinit.cmake0000644000175000017500000000636413025343121020512 0ustar tcaswelltcaswell00000000000000# This is the official h5py CMakeCache file. # Please see README.txt in the "windows" directory of the h5py package for more info. ######################## # EXTERNAL cache entries ######################## SET (BUILD_SHARED_LIBS ON CACHE BOOL "Build Shared Libraries" FORCE) SET (BUILD_TESTING ON CACHE BOOL "Build HDF5 Unit Testing" FORCE) SET (HDF_PACKAGE_EXT "" CACHE STRING "Name of HDF package extension" FORCE) SET (HDF5_EXTERNAL_LIB_PREFIX "h5py_" CACHE STRING "docstring" FORCE) SET (HDF5_BUILD_CPP_LIB OFF CACHE BOOL "Build HDF5 C++ Library" FORCE) SET (HDF5_BUILD_EXAMPLES OFF CACHE BOOL "Build HDF5 Library Examples" FORCE) SET (HDF5_BUILD_FORTRAN OFF CACHE BOOL "Build FORTRAN support" FORCE) SET (HDF5_ENABLE_F2003 OFF CACHE BOOL "Enable FORTRAN 2003 Standard" FORCE) SET (HDF5_BUILD_HL_LIB ON CACHE BOOL "Build HIGH Level HDF5 Library" FORCE) SET (HDF5_BUILD_TOOLS ON CACHE BOOL "Build HDF5 Tools" FORCE) SET (HDF5_BUILD_GENERATORS OFF CACHE BOOL "Build Test Generators" FORCE) SET (HDF5_ENABLE_Z_LIB_SUPPORT ON CACHE BOOL "Enable Zlib Filters" FORCE) SET (HDF5_ENABLE_SZIP_SUPPORT ON CACHE BOOL "Use SZip Filter" FORCE) SET (HDF5_ENABLE_SZIP_ENCODING OFF CACHE BOOL "Use SZip Encoding" FORCE) SET (HDF5_ENABLE_HSIZET ON CACHE BOOL "Enable datasets larger than memory" FORCE) SET (HDF5_ENABLE_UNSUPPORTED OFF CACHE BOOL "Enable unsupported combinations of configuration options" FORCE) SET (HDF5_ENABLE_DEPRECATED_SYMBOLS ON CACHE BOOL "Enable deprecated public API symbols" FORCE) SET (HDF5_ENABLE_DIRECT_VFD OFF CACHE BOOL "Build the Direct I/O Virtual File Driver" FORCE) SET (HDF5_ENABLE_PARALLEL OFF CACHE BOOL "Enable parallel build (requires MPI)" FORCE) SET (MPIEXEC_MAX_NUMPROCS "3" CACHE STRING "Minimum number of processes for HDF parallel tests" FORCE) SET (HDF5_BUILD_PARALLEL_ALL OFF CACHE BOOL "Build Parallel Programs" FORCE) SET (HDF5_ENABLE_COVERAGE OFF CACHE BOOL "Enable code coverage for Libraries and Programs" FORCE) SET (HDF5_ENABLE_USING_MEMCHECKER OFF CACHE BOOL "Indicate that a memory checker is used" FORCE) SET (HDF5_DISABLE_COMPILER_WARNINGS OFF CACHE BOOL "Disable compiler warnings" FORCE) SET (HDF5_USE_FOLDERS ON CACHE BOOL "Enable folder grouping of projects in IDEs." FORCE) SET (HDF5_USE_16_API_DEFAULT OFF CACHE BOOL "Use the HDF5 1.6.x API by default" FORCE) SET (HDF5_ENABLE_THREADSAFE OFF CACHE BOOL "(WINDOWS)Enable Threadsafety" FORCE) SET (HDF5_PACKAGE_EXTLIBS ON CACHE BOOL "(WINDOWS)CPACK - include external libraries" FORCE) SET (HDF5_NO_PACKAGES OFF CACHE BOOL "CPACK - Disable packaging" FORCE) SET (HDF5_ALLOW_EXTERNAL_SUPPORT "SVN" CACHE STRING "Allow External Library Building (NO SVN TGZ)" FORCE) SET_PROPERTY(CACHE HDF5_ALLOW_EXTERNAL_SUPPORT PROPERTY STRINGS NO SVN TGZ) SET (ZLIB_SVN_URL "http://svn.hdfgroup.uiuc.edu/zlib/trunk" CACHE STRING "Use ZLib from HDF repository" FORCE) SET (SZIP_SVN_URL "http://svn.hdfgroup.uiuc.edu/szip/trunk" CACHE STRING "Use SZip from HDF repository" FORCE) SET (ZLIB_TGZ_NAME "ZLib.tar.gz" CACHE STRING "Use ZLib from compressed file" FORCE) SET (SZIP_TGZ_NAME "SZip.tar.gz" CACHE STRING "Use SZip from compressed file" FORCE) SET (ZLIB_PACKAGE_NAME "zlib" CACHE STRING "Name of ZLIB package" FORCE) SET (SZIP_PACKAGE_NAME "szip" CACHE STRING "Name of SZIP package" FORCE) h5py-2.7.1/windows/README.txt0000644000175000017500000000143113025343121017245 0ustar tcaswelltcaswell00000000000000Build instructions for h5py on Windows: Build h5py in the normal fashion, except you are required to provide both the --hdf5 and --hdf5-version arguments to setup.py. Build HDF5 for distribution with a single command, using the pavement file in this directory. You will need to install paver first. CMake 2.8 (NOT 3.0 or higher), and a SVN client, must be installed and on the path. To build HDF5 with Visual Studio 2008 (required for Python 2.6, 2.7 and 3.2): paver build_2008 To build with Visual Studio 2010 (required for Python 3.3): paver build_2010 These commands will each produce a zip file containing the appropriate build of HDF5. Unpack them and supply the appropriate directory to --hdf5. Check pavement.py for the current HDF5 version to pass to --hdf5-version.h5py-2.7.1/windows/unistd.h0000644000175000017500000000000013025343121017215 0ustar tcaswelltcaswell00000000000000h5py-2.7.1/windows/pavement.py0000644000175000017500000000470713114362756017767 0ustar tcaswelltcaswell00000000000000import shutil, zipfile, tempfile, glob, urllib import os import os.path as op from paver.easy import * from paver.path import pushd # Directory containing pavement.py ROOTPATH = op.dirname(op.abspath(__file__)) def archive(fname): """ Currently just copies a single file to the current directory """ print "Archiving %s" % str(fname) if op.exists(op.join(ROOTPATH, fname)): os.remove(op.join(ROOTPATH, fname)) shutil.copy(fname, ROOTPATH) # --- Tasks to download and build HDF5 ---------------------------------------- CACHENAME = op.join(ROOTPATH, "cacheinit.cmake") ZIPDIR = "HDF5-1.8.13" ZIPFILE_NAME = "hdf5-1.8.13.zip" ZIPFILE_URL = "http://www.hdfgroup.org/ftp/HDF5/releases/hdf5-1.8.13/src/hdf5-1.8.13.zip" def build(vs_version): """ Build HDF5. Intended to be called with os.cwd inside the root of the HDF5 checkout. This means a place containing a directory called "src". vs_version: Integer giving Visual Studio version (2008 or 2010). """ build_dir = {2008: "build_2008", 2010: "build_2010"}[vs_version] vs_name = {2008: "Visual Studio 9 2008", 2010: "Visual Studio 10"}[vs_version] if not op.exists(build_dir): os.mkdir(build_dir) os.chdir(build_dir) sh('cmake -C "{}" -G "{}" ..'.format(CACHENAME, vs_name)) sh('cmake --build . --config Release', ignore_error=True) sh('copy bin\Release\* bin /Y') sh('cmake --build . --config Release') sh('cpack -G ZIP') fname = glob.glob("HDF5-*.zip")[0] new_fname = 'hdf5-h5py-vs%d.zip' % vs_version if op.exists(new_fname): os.remove(new_fname) os.rename(fname, new_fname) archive(new_fname) def check_zip(): if not op.exists(ZIPFILE_NAME): print "Downloading HDF5 source code..." urllib.urlretrieve(ZIPFILE_URL, ZIPFILE_NAME) try: sh('cmake --version') except Exception: raise ValueError("CMake must be installed to build HDF5") @task def build_2008(): """ Build HDF5 using Visual Studio 2008 """ check_zip() if not op.exists(ZIPDIR): with zipfile.ZipFile(ZIPFILE_NAME) as z: z.extractall('.') with pushd(ZIPDIR): build(2008) @task def build_2010(): """ Build HDF5 using Visual Studio 2010 """ check_zip() if not op.exists(ZIPDIR): with zipfile.ZipFile(ZIPFILE_NAME) as z: z.extractall('.') with pushd(ZIPDIR): build(2010) h5py-2.7.1/windows/stdint.h0000644000175000017500000001620113025343121017226 0ustar tcaswelltcaswell00000000000000// ISO C9x compliant stdint.h for Microsoft Visual Studio // Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 // // Copyright (c) 2006-2008 Alexander Chemeris // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are met: // // 1. Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimer. // // 2. Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // // 3. The name of the author may be used to endorse or promote products // derived from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED // WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO // EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; // OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, // WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR // OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF // ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // /////////////////////////////////////////////////////////////////////////////// #ifndef _MSC_VER // [ #error "Use this header only with Microsoft Visual C++ compilers!" #endif // _MSC_VER ] #ifndef _MSC_STDINT_H_ // [ #define _MSC_STDINT_H_ #if _MSC_VER > 1000 #pragma once #endif #include // For Visual Studio 6 in C++ mode wrap include with 'extern "C++" {}' // or compiler give many errors like this: // error C2733: second C linkage of overloaded function 'wmemchr' not allowed #if (_MSC_VER < 1300) && defined(__cplusplus) extern "C++" { #endif # include #if (_MSC_VER < 1300) && defined(__cplusplus) } #endif // Define _W64 macros to mark types changing their size, like intptr_t. #ifndef _W64 # if !defined(__midl) && (defined(_X86_) || defined(_M_IX86)) && _MSC_VER >= 1300 # define _W64 __w64 # else # define _W64 # endif #endif // 7.18.1 Integer types // 7.18.1.1 Exact-width integer types typedef __int8 int8_t; typedef __int16 int16_t; typedef __int32 int32_t; typedef __int64 int64_t; typedef unsigned __int8 uint8_t; typedef unsigned __int16 uint16_t; typedef unsigned __int32 uint32_t; typedef unsigned __int64 uint64_t; // 7.18.1.2 Minimum-width integer types typedef int8_t int_least8_t; typedef int16_t int_least16_t; typedef int32_t int_least32_t; typedef int64_t int_least64_t; typedef uint8_t uint_least8_t; typedef uint16_t uint_least16_t; typedef uint32_t uint_least32_t; typedef uint64_t uint_least64_t; // 7.18.1.3 Fastest minimum-width integer types typedef int8_t int_fast8_t; typedef int16_t int_fast16_t; typedef int32_t int_fast32_t; typedef int64_t int_fast64_t; typedef uint8_t uint_fast8_t; typedef uint16_t uint_fast16_t; typedef uint32_t uint_fast32_t; typedef uint64_t uint_fast64_t; // 7.18.1.4 Integer types capable of holding object pointers #ifdef _WIN64 // [ typedef __int64 intptr_t; typedef unsigned __int64 uintptr_t; #else // _WIN64 ][ typedef _W64 int intptr_t; typedef _W64 unsigned int uintptr_t; #endif // _WIN64 ] // 7.18.1.5 Greatest-width integer types typedef int64_t intmax_t; typedef uint64_t uintmax_t; // 7.18.2 Limits of specified-width integer types #if !defined(__cplusplus) || defined(__STDC_LIMIT_MACROS) // [ See footnote 220 at page 257 and footnote 221 at page 259 // 7.18.2.1 Limits of exact-width integer types #define INT8_MIN ((int8_t)_I8_MIN) #define INT8_MAX _I8_MAX #define INT16_MIN ((int16_t)_I16_MIN) #define INT16_MAX _I16_MAX #define INT32_MIN ((int32_t)_I32_MIN) #define INT32_MAX _I32_MAX #define INT64_MIN ((int64_t)_I64_MIN) #define INT64_MAX _I64_MAX #define UINT8_MAX _UI8_MAX #define UINT16_MAX _UI16_MAX #define UINT32_MAX _UI32_MAX #define UINT64_MAX _UI64_MAX // 7.18.2.2 Limits of minimum-width integer types #define INT_LEAST8_MIN INT8_MIN #define INT_LEAST8_MAX INT8_MAX #define INT_LEAST16_MIN INT16_MIN #define INT_LEAST16_MAX INT16_MAX #define INT_LEAST32_MIN INT32_MIN #define INT_LEAST32_MAX INT32_MAX #define INT_LEAST64_MIN INT64_MIN #define INT_LEAST64_MAX INT64_MAX #define UINT_LEAST8_MAX UINT8_MAX #define UINT_LEAST16_MAX UINT16_MAX #define UINT_LEAST32_MAX UINT32_MAX #define UINT_LEAST64_MAX UINT64_MAX // 7.18.2.3 Limits of fastest minimum-width integer types #define INT_FAST8_MIN INT8_MIN #define INT_FAST8_MAX INT8_MAX #define INT_FAST16_MIN INT16_MIN #define INT_FAST16_MAX INT16_MAX #define INT_FAST32_MIN INT32_MIN #define INT_FAST32_MAX INT32_MAX #define INT_FAST64_MIN INT64_MIN #define INT_FAST64_MAX INT64_MAX #define UINT_FAST8_MAX UINT8_MAX #define UINT_FAST16_MAX UINT16_MAX #define UINT_FAST32_MAX UINT32_MAX #define UINT_FAST64_MAX UINT64_MAX // 7.18.2.4 Limits of integer types capable of holding object pointers #ifdef _WIN64 // [ # define INTPTR_MIN INT64_MIN # define INTPTR_MAX INT64_MAX # define UINTPTR_MAX UINT64_MAX #else // _WIN64 ][ # define INTPTR_MIN INT32_MIN # define INTPTR_MAX INT32_MAX # define UINTPTR_MAX UINT32_MAX #endif // _WIN64 ] // 7.18.2.5 Limits of greatest-width integer types #define INTMAX_MIN INT64_MIN #define INTMAX_MAX INT64_MAX #define UINTMAX_MAX UINT64_MAX // 7.18.3 Limits of other integer types #ifdef _WIN64 // [ # define PTRDIFF_MIN _I64_MIN # define PTRDIFF_MAX _I64_MAX #else // _WIN64 ][ # define PTRDIFF_MIN _I32_MIN # define PTRDIFF_MAX _I32_MAX #endif // _WIN64 ] #define SIG_ATOMIC_MIN INT_MIN #define SIG_ATOMIC_MAX INT_MAX #ifndef SIZE_MAX // [ # ifdef _WIN64 // [ # define SIZE_MAX _UI64_MAX # else // _WIN64 ][ # define SIZE_MAX _UI32_MAX # endif // _WIN64 ] #endif // SIZE_MAX ] // WCHAR_MIN and WCHAR_MAX are also defined in #ifndef WCHAR_MIN // [ # define WCHAR_MIN 0 #endif // WCHAR_MIN ] #ifndef WCHAR_MAX // [ # define WCHAR_MAX _UI16_MAX #endif // WCHAR_MAX ] #define WINT_MIN 0 #define WINT_MAX _UI16_MAX #endif // __STDC_LIMIT_MACROS ] // 7.18.4 Limits of other integer types #if !defined(__cplusplus) || defined(__STDC_CONSTANT_MACROS) // [ See footnote 224 at page 260 // 7.18.4.1 Macros for minimum-width integer constants #define INT8_C(val) val##i8 #define INT16_C(val) val##i16 #define INT32_C(val) val##i32 #define INT64_C(val) val##i64 #define UINT8_C(val) val##ui8 #define UINT16_C(val) val##ui16 #define UINT32_C(val) val##ui32 #define UINT64_C(val) val##ui64 // 7.18.4.2 Macros for greatest-width integer constants #define INTMAX_C INT64_C #define UINTMAX_C UINT64_C #endif // __STDC_CONSTANT_MACROS ] #endif // _MSC_STDINT_H_ ] h5py-2.7.1/lzf/0000755000175000017500000000000013152363123014656 5ustar tcaswelltcaswell00000000000000h5py-2.7.1/lzf/LICENSE.txt0000644000175000017500000000303613025343033016500 0ustar tcaswelltcaswell00000000000000Copyright Notice and Statement for LZF filter Copyright (c) 2008-2009 Andrew Collette http://h5py.alfven.org All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: a. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. b. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. c. Neither the name of the author nor the names of contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. h5py-2.7.1/lzf/README.txt0000644000175000017500000000633213025343121016353 0ustar tcaswelltcaswell00000000000000=============================== LZF filter for HDF5, revision 3 =============================== The LZF filter provides high-speed compression with acceptable compression performance, resulting in much faster performance than DEFLATE, at the cost of a slightly lower compression ratio. It's appropriate for large datasets of low to moderate complexity, for which some compression is much better than none, but for which the speed of DEFLATE is unacceptable. This filter has been tested against HDF5 versions 1.6.5 through 1.8.3. It is released under the BSD license (see LICENSE.txt for details). Using the filter from HDF5 -------------------------- With HDF5 version 1.8.11 or later the filter can be loaded dynamically by the HDF5 library. The filter needs to be compiled as a plugin as described below that is placed in the default plugin path /usr/local/hdf5/lib/plugin/. The plugin path can be overridden with the environment variable HDF5_PLUGIN_PATH. With older HDF5 versions, or when statically linking the filter to your program, the filter must be registered manually. There is exactly one new public function declared in lzf_filter.h, with the following signature: int register_lzf(void) Calling this will register the filter with the HDF5 library. A non-negative return value indicates success. If the registration fails, an error is pushed onto the current error stack and a negative value is returned. It's strongly recommended to use the SHUFFLE filter with LZF, as it's cheap, supported by all current versions of HDF5, and can significantly improve the compression ratio. An example C program ("example.c") is included which demonstrates the proper use of the filter. Compiling --------- The filter consists of a single .c file and header, along with an embedded version of the LZF compression library. Since the filter is stateless, it's recommended to statically link the entire thing into your program; for example: $ gcc -O2 lzf/*.c lzf_filter.c myprog.c -lhdf5 -o myprog It can also be built as a shared library, although you will have to install the resulting library somewhere the runtime linker can find it: $ gcc -O2 -fPIC -shared lzf/*.c lzf_filter.c -lhdf5 -o liblzf_filter.so A similar procedure should be used for building C++ code. As in these examples, using option -O1 or higher is strongly recommended for increased performance. With HDF5 version 1.8.11 or later the filter can be dynamically loaded as a plugin. The filter is built as a shared library that is *not* linked against the HDF5 library: $ gcc -O2 -fPIC -shared lzf/*.c lzf_filter.c -o liblzf_filter.so Contact ------- This filter is maintained as part of the HDF5 for Python (h5py) project. The goal of h5py is to provide access to the majority of the HDF5 C API and feature set from Python. The most recent version of h5py (1.1) includes the LZF filter by default. * Downloads and bug tracker: http://h5py.googlecode.com * Main web site and documentation: http://h5py.alfven.org * Contact email: h5py at alfven dot org History of changes ------------------ Revision 3 (6/25/09) Fix issue with changed filter struct definition under HDF5 1.8.3. Revision 2 Minor speed enhancement. Revision 1 Initial release. h5py-2.7.1/lzf/example.c0000644000175000017500000000526613025343033016463 0ustar tcaswelltcaswell00000000000000/* Copyright (C) 2009 Andrew Collette http://h5py.alfven.org License: BSD (see LICENSE.txt) Example program demonstrating use of the LZF filter from C code. To compile this program: h5cc -DH5_USE_16_API lzf/*.c lzf_filter.c example.c -o example To run: $ ./example Success! $ h5ls -v test_lzf.hdf5 Opened "test_lzf.hdf5" with sec2 driver. dset Dataset {100/100, 100/100, 100/100} Location: 0:1:0:976 Links: 1 Modified: 2009-02-15 16:35:11 PST Chunks: {1, 100, 100} 40000 bytes Storage: 4000000 logical bytes, 174288 allocated bytes, 2295.05% utilization Filter-0: shuffle-2 OPT {4} Filter-1: lzf-32000 OPT {1, 261, 40000} Type: native float */ #include #include "hdf5.h" #include "lzf_filter.h" #define SIZE 100*100*100 #define SHAPE {100,100,100} #define CHUNKSHAPE {1,100,100} int main(){ static float data[SIZE]; static float data_out[SIZE]; const hsize_t shape[] = SHAPE; const hsize_t chunkshape[] = CHUNKSHAPE; int r, i; int return_code = 1; hid_t fid, sid, dset, plist = 0; for(i=0; i0) H5Dclose(dset); if(sid>0) H5Sclose(sid); if(plist>0) H5Pclose(plist); if(fid>0) H5Fclose(fid); return return_code; } h5py-2.7.1/lzf/lzf_filter.h0000644000175000017500000000155213025343033017167 0ustar tcaswelltcaswell00000000000000/***** Preamble block ********************************************************* * * This file is part of h5py, a low-level Python interface to the HDF5 library. * * Copyright (C) 2008 Andrew Collette * http://h5py.alfven.org * License: BSD (See LICENSE.txt for full license) * * $Date$ * ****** End preamble block ****************************************************/ #ifndef H5PY_LZF_H #define H5PY_LZF_H #ifdef __cplusplus extern "C" { #endif /* Filter revision number, starting at 1 */ #define H5PY_FILTER_LZF_VERSION 4 /* Filter ID registered with the HDF Group as of 2/6/09. For maintenance requests, contact the filter author directly. */ #define H5PY_FILTER_LZF 32000 /* Register the filter with the library. Returns a negative value on failure, and a non-negative value on success. */ int register_lzf(void); #ifdef __cplusplus } #endif #endif h5py-2.7.1/lzf/lzf_filter.c0000644000175000017500000001600113025343121017153 0ustar tcaswelltcaswell00000000000000/***** Preamble block ********************************************************* * * This file is part of h5py, a low-level Python interface to the HDF5 library. * * Copyright (C) 2008 Andrew Collette * http://h5py.alfven.org * License: BSD (See LICENSE.txt for full license) * * $Date$ * ****** End preamble block ****************************************************/ /* Implements an LZF filter module for HDF5, using the BSD-licensed library by Marc Alexander Lehmann (http://www.goof.com/pcg/marc/liblzf.html). No Python-specific code is used. The filter behaves like the DEFLATE filter, in that it is called for every type and space, and returns 0 if the data cannot be compressed. The only public function is (int) register_lzf(void), which passes on the result from H5Zregister. */ #include #include #include #include "hdf5.h" #include "lzf/lzf.h" #include "lzf_filter.h" /* Our own versions of H5Epush_sim, as it changed in 1.8 */ #if H5_VERS_MAJOR == 1 && H5_VERS_MINOR < 7 #define PUSH_ERR(func, minor, str) H5Epush(__FILE__, func, __LINE__, H5E_PLINE, minor, str) #define H5PY_GET_FILTER H5Pget_filter_by_id #else #define PUSH_ERR(func, minor, str) H5Epush1(__FILE__, func, __LINE__, H5E_PLINE, minor, str) #define H5PY_GET_FILTER(a,b,c,d,e,f,g) H5Pget_filter_by_id2(a,b,c,d,e,f,g,NULL) #endif /* Deal with the mutiple definitions for H5Z_class_t. Note: Only HDF5 1.6 and 1.8 are supported. (1) The old class should always be used for HDF5 1.6 (2) The new class should always be used for HDF5 1.8 < 1.8.3 (3) The old class should be used for HDF5 1.8 >= 1.8.3 only if the macro H5_USE_16_API is set */ #if H5_VERS_MAJOR == 1 && H5_VERS_MINOR == 8 && (H5_VERS_RELEASE < 3 || !H5_USE_16_API) #define H5PY_H5Z_NEWCLS 1 #else #define H5PY_H5Z_NEWCLS 0 #endif size_t lzf_filter(unsigned flags, size_t cd_nelmts, const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf); herr_t lzf_set_local(hid_t dcpl, hid_t type, hid_t space); #if H5PY_H5Z_NEWCLS static const H5Z_class_t filter_class = { H5Z_CLASS_T_VERS, (H5Z_filter_t)(H5PY_FILTER_LZF), 1, 1, "lzf", NULL, (H5Z_set_local_func_t)(lzf_set_local), (H5Z_func_t)(lzf_filter) }; #else static const H5Z_class_t filter_class = { (H5Z_filter_t)(H5PY_FILTER_LZF), "lzf", NULL, (H5Z_set_local_func_t)(lzf_set_local), (H5Z_func_t)(lzf_filter) }; #endif /* Support dynamical loading of LZF filter plugin */ #if defined(H5_VERSION_GE) #if H5_VERSION_GE(1, 8, 11) #include "H5PLextern.h" H5PL_type_t H5PLget_plugin_type(void){ return H5PL_TYPE_FILTER; } const void *H5PLget_plugin_info(void){ return &filter_class; } #endif #endif /* Try to register the filter, passing on the HDF5 return value */ int register_lzf(void){ int retval; retval = H5Zregister(&filter_class); if(retval<0){ PUSH_ERR("register_lzf", H5E_CANTREGISTER, "Can't register LZF filter"); } return retval; } /* Filter setup. Records the following inside the DCPL: 1. If version information is not present, set slots 0 and 1 to the filter revision and LZF API version, respectively. 2. Compute the chunk size in bytes and store it in slot 2. */ herr_t lzf_set_local(hid_t dcpl, hid_t type, hid_t space){ int ndims; int i; herr_t r; unsigned int bufsize; hsize_t chunkdims[32]; unsigned int flags; size_t nelements = 8; unsigned values[] = {0,0,0,0,0,0,0,0}; r = H5PY_GET_FILTER(dcpl, H5PY_FILTER_LZF, &flags, &nelements, values, 0, NULL); if(r<0) return -1; if(nelements < 3) nelements = 3; /* First 3 slots reserved. If any higher slots are used, preserve the contents. */ /* It seems the H5Z_FLAG_REVERSE flag doesn't work here, so we have to be careful not to clobber any existing version info */ if(values[0]==0) values[0] = H5PY_FILTER_LZF_VERSION; if(values[1]==0) values[1] = LZF_VERSION; ndims = H5Pget_chunk(dcpl, 32, chunkdims); if(ndims<0) return -1; if(ndims>32){ PUSH_ERR("lzf_set_local", H5E_CALLBACK, "Chunk rank exceeds limit"); return -1; } bufsize = H5Tget_size(type); if(bufsize==0) return -1; for(i=0;i=3)&&(cd_values[2]!=0)){ outbuf_size = cd_values[2]; /* Precomputed buffer guess */ }else{ outbuf_size = (*buf_size); } #ifdef H5PY_LZF_DEBUG fprintf(stderr, "Decompress %d chunk w/buffer %d\n", nbytes, outbuf_size); #endif while(!status){ free(outbuf); outbuf = malloc(outbuf_size); if(outbuf == NULL){ PUSH_ERR("lzf_filter", H5E_CALLBACK, "Can't allocate decompression buffer"); goto failed; } status = lzf_decompress(*buf, nbytes, outbuf, outbuf_size); if(!status){ /* compression failed */ if(errno == E2BIG){ outbuf_size += (*buf_size); #ifdef H5PY_LZF_DEBUG fprintf(stderr, " Too small: %d\n", outbuf_size); #endif } else if(errno == EINVAL) { PUSH_ERR("lzf_filter", H5E_CALLBACK, "Invalid data for LZF decompression"); goto failed; } else { PUSH_ERR("lzf_filter", H5E_CALLBACK, "Unknown LZF decompression error"); goto failed; } } /* if !status */ } /* while !status */ } /* compressing vs decompressing */ if(status != 0){ free(*buf); *buf = outbuf; *buf_size = outbuf_size; return status; /* Size of compressed/decompressed data */ } failed: free(outbuf); return 0; } /* End filter function */ h5py-2.7.1/lzf/lzf/0000755000175000017500000000000013152363123015451 5ustar tcaswelltcaswell00000000000000h5py-2.7.1/lzf/lzf/lzfP.h0000644000175000017500000001251013025343033016531 0ustar tcaswelltcaswell00000000000000/* * Copyright (c) 2000-2007 Marc Alexander Lehmann * * Redistribution and use in source and binary forms, with or without modifica- * tion, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER- * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE- * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH- * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED * OF THE POSSIBILITY OF SUCH DAMAGE. * * Alternatively, the contents of this file may be used under the terms of * the GNU General Public License ("GPL") version 2 or any later version, * in which case the provisions of the GPL are applicable instead of * the above. If you wish to allow the use of your version of this file * only under the terms of the GPL and not to allow others to use your * version of this file under the BSD license, indicate your decision * by deleting the provisions above and replace them with the notice * and other provisions required by the GPL. If you do not delete the * provisions above, a recipient may use your version of this file under * either the BSD or the GPL. */ #ifndef LZFP_h #define LZFP_h #define STANDALONE 1 /* at the moment, this is ok. */ #ifndef STANDALONE # include "lzf.h" #endif /* * Size of hashtable is (1 << HLOG) * sizeof (char *) * decompression is independent of the hash table size * the difference between 15 and 14 is very small * for small blocks (and 14 is usually a bit faster). * For a low-memory/faster configuration, use HLOG == 13; * For best compression, use 15 or 16 (or more, up to 23). */ #ifndef HLOG # define HLOG 17 /* Avoid pathological case at HLOG=16 A.C. 2/15/09 */ #endif /* * Sacrifice very little compression quality in favour of compression speed. * This gives almost the same compression as the default code, and is * (very roughly) 15% faster. This is the preferred mode of operation. */ #ifndef VERY_FAST # define VERY_FAST 1 #endif /* * Sacrifice some more compression quality in favour of compression speed. * (roughly 1-2% worse compression for large blocks and * 9-10% for small, redundant, blocks and >>20% better speed in both cases) * In short: when in need for speed, enable this for binary data, * possibly disable this for text data. */ #ifndef ULTRA_FAST # define ULTRA_FAST 1 #endif /* * Unconditionally aligning does not cost very much, so do it if unsure */ #ifndef STRICT_ALIGN # define STRICT_ALIGN !(defined(__i386) || defined (__amd64)) #endif /* * You may choose to pre-set the hash table (might be faster on some * modern cpus and large (>>64k) blocks, and also makes compression * deterministic/repeatable when the configuration otherwise is the same). */ #ifndef INIT_HTAB # define INIT_HTAB 0 #endif /* ======================================================================= Changing things below this line may break the HDF5 LZF filter. A.C. 2/15/09 ======================================================================= */ /* * Avoid assigning values to errno variable? for some embedding purposes * (linux kernel for example), this is neccessary. NOTE: this breaks * the documentation in lzf.h. */ #ifndef AVOID_ERRNO # define AVOID_ERRNO 0 #endif /* * Wether to pass the LZF_STATE variable as argument, or allocate it * on the stack. For small-stack environments, define this to 1. * NOTE: this breaks the prototype in lzf.h. */ #ifndef LZF_STATE_ARG # define LZF_STATE_ARG 0 #endif /* * Wether to add extra checks for input validity in lzf_decompress * and return EINVAL if the input stream has been corrupted. This * only shields against overflowing the input buffer and will not * detect most corrupted streams. * This check is not normally noticable on modern hardware * (<1% slowdown), but might slow down older cpus considerably. */ #ifndef CHECK_INPUT # define CHECK_INPUT 1 #endif /*****************************************************************************/ /* nothing should be changed below */ typedef unsigned char u8; typedef const u8 *LZF_STATE[1 << (HLOG)]; #if !STRICT_ALIGN /* for unaligned accesses we need a 16 bit datatype. */ # include # if USHRT_MAX == 65535 typedef unsigned short u16; # elif UINT_MAX == 65535 typedef unsigned int u16; # else # undef STRICT_ALIGN # define STRICT_ALIGN 1 # endif #endif #if ULTRA_FAST # if defined(VERY_FAST) # undef VERY_FAST # endif #endif #if INIT_HTAB # ifdef __cplusplus # include # else # include # endif #endif #endif h5py-2.7.1/lzf/lzf/lzf_c.c0000644000175000017500000002145713025343121016716 0ustar tcaswelltcaswell00000000000000/* * Copyright (c) 2000-2008 Marc Alexander Lehmann * * Redistribution and use in source and binary forms, with or without modifica- * tion, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER- * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE- * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH- * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED * OF THE POSSIBILITY OF SUCH DAMAGE. * * Alternatively, the contents of this file may be used under the terms of * the GNU General Public License ("GPL") version 2 or any later version, * in which case the provisions of the GPL are applicable instead of * the above. If you wish to allow the use of your version of this file * only under the terms of the GPL and not to allow others to use your * version of this file under the BSD license, indicate your decision * by deleting the provisions above and replace them with the notice * and other provisions required by the GPL. If you do not delete the * provisions above, a recipient may use your version of this file under * either the BSD or the GPL. */ #include "lzfP.h" #define HSIZE (1 << (HLOG)) /* * don't play with this unless you benchmark! * decompression is not dependent on the hash function * the hashing function might seem strange, just believe me * it works ;) */ #ifndef FRST # define FRST(p) (((p[0]) << 8) | p[1]) # define NEXT(v,p) (((v) << 8) | p[2]) # if ULTRA_FAST # define IDX(h) ((( h >> (3*8 - HLOG)) - h ) & (HSIZE - 1)) # elif VERY_FAST # define IDX(h) ((( h >> (3*8 - HLOG)) - h*5) & (HSIZE - 1)) # else # define IDX(h) ((((h ^ (h << 5)) >> (3*8 - HLOG)) - h*5) & (HSIZE - 1)) # endif #endif /* * IDX works because it is very similar to a multiplicative hash, e.g. * ((h * 57321 >> (3*8 - HLOG)) & (HSIZE - 1)) * the latter is also quite fast on newer CPUs, and compresses similarly. * * the next one is also quite good, albeit slow ;) * (int)(cos(h & 0xffffff) * 1e6) */ #if 0 /* original lzv-like hash function, much worse and thus slower */ # define FRST(p) (p[0] << 5) ^ p[1] # define NEXT(v,p) ((v) << 5) ^ p[2] # define IDX(h) ((h) & (HSIZE - 1)) #endif #define MAX_LIT (1 << 5) #define MAX_OFF (1 << 13) #define MAX_REF ((1 << 8) + (1 << 3)) #if __GNUC__ >= 3 # define expect(expr,value) __builtin_expect ((expr),(value)) # define inline inline #else # define expect(expr,value) (expr) # define inline static #endif #define expect_false(expr) expect ((expr) != 0, 0) #define expect_true(expr) expect ((expr) != 0, 1) /* * compressed format * * 000LLLLL ; literal * LLLooooo oooooooo ; backref L * 111ooooo LLLLLLLL oooooooo ; backref L+7 * */ unsigned int lzf_compress (const void *const in_data, unsigned int in_len, void *out_data, unsigned int out_len #if LZF_STATE_ARG , LZF_STATE htab #endif ) { #if !LZF_STATE_ARG LZF_STATE htab; #endif const u8 **hslot; const u8 *ip = (const u8 *)in_data; u8 *op = (u8 *)out_data; const u8 *in_end = ip + in_len; u8 *out_end = op + out_len; const u8 *ref; /* off requires a type wide enough to hold a general pointer difference. * ISO C doesn't have that (size_t might not be enough and ptrdiff_t only * works for differences within a single object). We also assume that no * no bit pattern traps. Since the only platform that is both non-POSIX * and fails to support both assumptions is windows 64 bit, we make a * special workaround for it. */ #if ( defined (WIN32) && defined (_M_X64) ) || defined (_WIN64) unsigned _int64 off; /* workaround for missing POSIX compliance */ #else unsigned long off; #endif unsigned int hval; int lit; if (!in_len || !out_len) return 0; #if INIT_HTAB memset (htab, 0, sizeof (htab)); # if 0 for (hslot = htab; hslot < htab + HSIZE; hslot++) *hslot++ = ip; # endif #endif lit = 0; op++; /* start run */ hval = FRST (ip); while (ip < in_end - 2) { hval = NEXT (hval, ip); hslot = htab + IDX (hval); ref = *hslot; *hslot = ip; if (1 #if INIT_HTAB && ref < ip /* the next test will actually take care of this, but this is faster */ #endif && (off = ip - ref - 1) < MAX_OFF && ip + 4 < in_end && ref > (u8 *)in_data #if STRICT_ALIGN && ref[0] == ip[0] && ref[1] == ip[1] && ref[2] == ip[2] #else && *(u16 *)ref == *(u16 *)ip && ref[2] == ip[2] #endif ) { /* match found at *ref++ */ unsigned int len = 2; unsigned int maxlen = in_end - ip - len; maxlen = maxlen > MAX_REF ? MAX_REF : maxlen; if (expect_false (op + 3 + 1 >= out_end)) /* first a faster conservative test */ if (op - !lit + 3 + 1 >= out_end) /* second the exact but rare test */ return 0; op [- lit - 1] = lit - 1; /* stop run */ op -= !lit; /* undo run if length is zero */ for (;;) { if (expect_true (maxlen > 16)) { len++; if (ref [len] != ip [len]) break; len++; if (ref [len] != ip [len]) break; len++; if (ref [len] != ip [len]) break; len++; if (ref [len] != ip [len]) break; len++; if (ref [len] != ip [len]) break; len++; if (ref [len] != ip [len]) break; len++; if (ref [len] != ip [len]) break; len++; if (ref [len] != ip [len]) break; len++; if (ref [len] != ip [len]) break; len++; if (ref [len] != ip [len]) break; len++; if (ref [len] != ip [len]) break; len++; if (ref [len] != ip [len]) break; len++; if (ref [len] != ip [len]) break; len++; if (ref [len] != ip [len]) break; len++; if (ref [len] != ip [len]) break; len++; if (ref [len] != ip [len]) break; } do len++; while (len < maxlen && ref[len] == ip[len]); break; } len -= 2; /* len is now #octets - 1 */ ip++; if (len < 7) { *op++ = (off >> 8) + (len << 5); } else { *op++ = (off >> 8) + ( 7 << 5); *op++ = len - 7; } *op++ = off; lit = 0; op++; /* start run */ ip += len + 1; if (expect_false (ip >= in_end - 2)) break; #if ULTRA_FAST || VERY_FAST --ip; # if VERY_FAST && !ULTRA_FAST --ip; # endif hval = FRST (ip); hval = NEXT (hval, ip); htab[IDX (hval)] = ip; ip++; # if VERY_FAST && !ULTRA_FAST hval = NEXT (hval, ip); htab[IDX (hval)] = ip; ip++; # endif #else ip -= len + 1; do { hval = NEXT (hval, ip); htab[IDX (hval)] = ip; ip++; } while (len--); #endif } else { /* one more literal byte we must copy */ if (expect_false (op >= out_end)) return 0; lit++; *op++ = *ip++; if (expect_false (lit == MAX_LIT)) { op [- lit - 1] = lit - 1; /* stop run */ lit = 0; op++; /* start run */ } } } if (op + 3 > out_end) /* at most 3 bytes can be missing here */ return 0; while (ip < in_end) { lit++; *op++ = *ip++; if (expect_false (lit == MAX_LIT)) { op [- lit - 1] = lit - 1; /* stop run */ lit = 0; op++; /* start run */ } } op [- lit - 1] = lit - 1; /* end run */ op -= !lit; /* undo run if length is zero */ return op - (u8 *)out_data; } h5py-2.7.1/lzf/lzf/lzf_d.c0000644000175000017500000001050513025343033016711 0ustar tcaswelltcaswell00000000000000/* * Copyright (c) 2000-2007 Marc Alexander Lehmann * * Redistribution and use in source and binary forms, with or without modifica- * tion, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER- * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE- * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH- * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED * OF THE POSSIBILITY OF SUCH DAMAGE. * * Alternatively, the contents of this file may be used under the terms of * the GNU General Public License ("GPL") version 2 or any later version, * in which case the provisions of the GPL are applicable instead of * the above. If you wish to allow the use of your version of this file * only under the terms of the GPL and not to allow others to use your * version of this file under the BSD license, indicate your decision * by deleting the provisions above and replace them with the notice * and other provisions required by the GPL. If you do not delete the * provisions above, a recipient may use your version of this file under * either the BSD or the GPL. */ #include "lzfP.h" #if AVOID_ERRNO # define SET_ERRNO(n) #else # include # define SET_ERRNO(n) errno = (n) #endif /* ASM is slower than C in HDF5 tests -- A.C. 2/5/09 #ifndef __STRICT_ANSI__ #ifndef H5PY_DISABLE_LZF_ASM #if (__i386 || __amd64) && __GNUC__ >= 3 # define lzf_movsb(dst, src, len) \ asm ("rep movsb" \ : "=D" (dst), "=S" (src), "=c" (len) \ : "0" (dst), "1" (src), "2" (len)); #endif #endif #endif */ unsigned int lzf_decompress (const void *const in_data, unsigned int in_len, void *out_data, unsigned int out_len) { u8 const *ip = (const u8 *)in_data; u8 *op = (u8 *)out_data; u8 const *const in_end = ip + in_len; u8 *const out_end = op + out_len; do { unsigned int ctrl = *ip++; if (ctrl < (1 << 5)) /* literal run */ { ctrl++; if (op + ctrl > out_end) { SET_ERRNO (E2BIG); return 0; } #if CHECK_INPUT if (ip + ctrl > in_end) { SET_ERRNO (EINVAL); return 0; } #endif #ifdef lzf_movsb lzf_movsb (op, ip, ctrl); #else do *op++ = *ip++; while (--ctrl); #endif } else /* back reference */ { unsigned int len = ctrl >> 5; u8 *ref = op - ((ctrl & 0x1f) << 8) - 1; #if CHECK_INPUT if (ip >= in_end) { SET_ERRNO (EINVAL); return 0; } #endif if (len == 7) { len += *ip++; #if CHECK_INPUT if (ip >= in_end) { SET_ERRNO (EINVAL); return 0; } #endif } ref -= *ip++; if (op + len + 2 > out_end) { SET_ERRNO (E2BIG); return 0; } if (ref < (u8 *)out_data) { SET_ERRNO (EINVAL); return 0; } #ifdef lzf_movsb len += 2; lzf_movsb (op, ref, len); #else *op++ = *ref++; *op++ = *ref++; do *op++ = *ref++; while (--len); #endif } } while (ip < in_end); return op - (u8 *)out_data; } h5py-2.7.1/lzf/lzf/lzf.h0000644000175000017500000001047513025343033016421 0ustar tcaswelltcaswell00000000000000/* * Copyright (c) 2000-2008 Marc Alexander Lehmann * * Redistribution and use in source and binary forms, with or without modifica- * tion, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MER- * CHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPE- * CIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTH- * ERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED * OF THE POSSIBILITY OF SUCH DAMAGE. * * Alternatively, the contents of this file may be used under the terms of * the GNU General Public License ("GPL") version 2 or any later version, * in which case the provisions of the GPL are applicable instead of * the above. If you wish to allow the use of your version of this file * only under the terms of the GPL and not to allow others to use your * version of this file under the BSD license, indicate your decision * by deleting the provisions above and replace them with the notice * and other provisions required by the GPL. If you do not delete the * provisions above, a recipient may use your version of this file under * either the BSD or the GPL. */ #ifndef LZF_H #define LZF_H /*********************************************************************** ** ** lzf -- an extremely fast/free compression/decompression-method ** http://liblzf.plan9.de/ ** ** This algorithm is believed to be patent-free. ** ***********************************************************************/ #define LZF_VERSION 0x0105 /* 1.5, API version */ /* * Compress in_len bytes stored at the memory block starting at * in_data and write the result to out_data, up to a maximum length * of out_len bytes. * * If the output buffer is not large enough or any error occurs return 0, * otherwise return the number of bytes used, which might be considerably * more than in_len (but less than 104% of the original size), so it * makes sense to always use out_len == in_len - 1), to ensure _some_ * compression, and store the data uncompressed otherwise (with a flag, of * course. * * lzf_compress might use different algorithms on different systems and * even different runs, thus might result in different compressed strings * depending on the phase of the moon or similar factors. However, all * these strings are architecture-independent and will result in the * original data when decompressed using lzf_decompress. * * The buffers must not be overlapping. * * If the option LZF_STATE_ARG is enabled, an extra argument must be * supplied which is not reflected in this header file. Refer to lzfP.h * and lzf_c.c. * */ unsigned int lzf_compress (const void *const in_data, unsigned int in_len, void *out_data, unsigned int out_len); /* * Decompress data compressed with some version of the lzf_compress * function and stored at location in_data and length in_len. The result * will be stored at out_data up to a maximum of out_len characters. * * If the output buffer is not large enough to hold the decompressed * data, a 0 is returned and errno is set to E2BIG. Otherwise the number * of decompressed bytes (i.e. the original length of the data) is * returned. * * If an error in the compressed data is detected, a zero is returned and * errno is set to EINVAL. * * This function is very fast, about as fast as a copying loop. */ unsigned int lzf_decompress (const void *const in_data, unsigned int in_len, void *out_data, unsigned int out_len); #endif h5py-2.7.1/h5py/0000755000175000017500000000000013152363123014750 5ustar tcaswelltcaswell00000000000000h5py-2.7.1/h5py/_hl/0000755000175000017500000000000013152363123015512 5ustar tcaswelltcaswell00000000000000h5py-2.7.1/h5py/_hl/group.py0000644000175000017500000004464413143100476017234 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. """ Implements support for high-level access to HDF5 groups. """ from __future__ import absolute_import import posixpath as pp import six import numpy from .compat import filename_decode, filename_encode from .. import h5g, h5i, h5o, h5r, h5t, h5l, h5p from . import base from .base import HLObject, MutableMappingHDF5, phil, with_phil from . import dataset from . import datatype class Group(HLObject, MutableMappingHDF5): """ Represents an HDF5 group. """ def __init__(self, bind): """ Create a new Group object by binding to a low-level GroupID. """ with phil: if not isinstance(bind, h5g.GroupID): raise ValueError("%s is not a GroupID" % bind) HLObject.__init__(self, bind) def create_group(self, name): """ Create and return a new subgroup. Name may be absolute or relative. Fails if the target name already exists. """ with phil: name, lcpl = self._e(name, lcpl=True) gid = h5g.create(self.id, name, lcpl=lcpl) return Group(gid) def create_dataset(self, name, shape=None, dtype=None, data=None, **kwds): """ Create a new HDF5 dataset name Name of the dataset (absolute or relative). Provide None to make an anonymous dataset. shape Dataset shape. Use "()" for scalar datasets. Required if "data" isn't provided. dtype Numpy dtype or string. If omitted, dtype('f') will be used. Required if "data" isn't provided; otherwise, overrides data array's dtype. data Provide data to initialize the dataset. If used, you can omit shape and dtype arguments. Keyword-only arguments: chunks (Tuple) Chunk shape, or True to enable auto-chunking. maxshape (Tuple) Make the dataset resizable up to this shape. Use None for axes you want to be unlimited. compression (String or int) Compression strategy. Legal values are 'gzip', 'szip', 'lzf'. If an integer in range(10), this indicates gzip compression level. Otherwise, an integer indicates the number of a dynamically loaded compression filter. compression_opts Compression settings. This is an integer for gzip, 2-tuple for szip, etc. If specifying a dynamically loaded compression filter number, this must be a tuple of values. scaleoffset (Integer) Enable scale/offset filter for (usually) lossy compression of integer or floating-point data. For integer data, the value of scaleoffset is the number of bits to retain (pass 0 to let HDF5 determine the minimum number of bits necessary for lossless compression). For floating point data, scaleoffset is the number of digits after the decimal place to retain; stored values thus have absolute error less than 0.5*10**(-scaleoffset). shuffle (T/F) Enable shuffle filter. fletcher32 (T/F) Enable fletcher32 error detection. Not permitted in conjunction with the scale/offset filter. fillvalue (Scalar) Use this value for uninitialized parts of the dataset. track_times (T/F) Enable dataset creation timestamps. """ with phil: dsid = dataset.make_new_dset(self, shape, dtype, data, **kwds) dset = dataset.Dataset(dsid) if name is not None: self[name] = dset return dset def require_dataset(self, name, shape, dtype, exact=False, **kwds): """ Open a dataset, creating it if it doesn't exist. If keyword "exact" is False (default), an existing dataset must have the same shape and a conversion-compatible dtype to be returned. If True, the shape and dtype must match exactly. Other dataset keywords (see create_dataset) may be provided, but are only used if a new dataset is to be created. Raises TypeError if an incompatible object already exists, or if the shape or dtype don't match according to the above rules. """ with phil: if not name in self: return self.create_dataset(name, *(shape, dtype), **kwds) dset = self[name] if not isinstance(dset, dataset.Dataset): raise TypeError("Incompatible object (%s) already exists" % dset.__class__.__name__) if not shape == dset.shape: raise TypeError("Shapes do not match (existing %s vs new %s)" % (dset.shape, shape)) if exact: if not dtype == dset.dtype: raise TypeError("Datatypes do not exactly match (existing %s vs new %s)" % (dset.dtype, dtype)) elif not numpy.can_cast(dtype, dset.dtype): raise TypeError("Datatypes cannot be safely cast (existing %s vs new %s)" % (dset.dtype, dtype)) return dset def require_group(self, name): """ Return a group, creating it if it doesn't exist. TypeError is raised if something with that name already exists that isn't a group. """ with phil: if not name in self: return self.create_group(name) grp = self[name] if not isinstance(grp, Group): raise TypeError("Incompatible object (%s) already exists" % grp.__class__.__name__) return grp @with_phil def __getitem__(self, name): """ Open an object in the file """ if isinstance(name, h5r.Reference): oid = h5r.dereference(name, self.id) if oid is None: raise ValueError("Invalid HDF5 object reference") else: oid = h5o.open(self.id, self._e(name), lapl=self._lapl) otype = h5i.get_type(oid) if otype == h5i.GROUP: return Group(oid) elif otype == h5i.DATASET: return dataset.Dataset(oid) elif otype == h5i.DATATYPE: return datatype.Datatype(oid) else: raise TypeError("Unknown object type") def get(self, name, default=None, getclass=False, getlink=False): """ Retrieve an item or other information. "name" given only: Return the item, or "default" if it doesn't exist "getclass" is True: Return the class of object (Group, Dataset, etc.), or "default" if nothing with that name exists "getlink" is True: Return HardLink, SoftLink or ExternalLink instances. Return "default" if nothing with that name exists. "getlink" and "getclass" are True: Return HardLink, SoftLink and ExternalLink classes. Return "default" if nothing with that name exists. Example: >>> cls = group.get('foo', getclass=True) >>> if cls == SoftLink: ... print '"foo" is a soft link!' """ # pylint: disable=arguments-differ with phil: if not (getclass or getlink): try: return self[name] except KeyError: return default if not name in self: return default elif getclass and not getlink: typecode = h5o.get_info(self.id, self._e(name)).type try: return {h5o.TYPE_GROUP: Group, h5o.TYPE_DATASET: dataset.Dataset, h5o.TYPE_NAMED_DATATYPE: datatype.Datatype}[typecode] except KeyError: raise TypeError("Unknown object type") elif getlink: typecode = self.id.links.get_info(self._e(name)).type if typecode == h5l.TYPE_SOFT: if getclass: return SoftLink linkbytes = self.id.links.get_val(self._e(name)) return SoftLink(self._d(linkbytes)) elif typecode == h5l.TYPE_EXTERNAL: if getclass: return ExternalLink filebytes, linkbytes = self.id.links.get_val(self._e(name)) return ExternalLink( filename_decode(filebytes), self._d(linkbytes) ) elif typecode == h5l.TYPE_HARD: return HardLink if getclass else HardLink() else: raise TypeError("Unknown link type") def __setitem__(self, name, obj): """ Add an object to the group. The name must not already be in use. The action taken depends on the type of object assigned: Named HDF5 object (Dataset, Group, Datatype) A hard link is created at "name" which points to the given object. SoftLink or ExternalLink Create the corresponding link. Numpy ndarray The array is converted to a dataset object, with default settings (contiguous storage, etc.). Numpy dtype Commit a copy of the datatype as a named datatype in the file. Anything else Attempt to convert it to an ndarray and store it. Scalar values are stored as scalar datasets. Raise ValueError if we can't understand the resulting array dtype. """ do_link = False with phil: name, lcpl = self._e(name, lcpl=True) if isinstance(obj, HLObject): h5o.link(obj.id, self.id, name, lcpl=lcpl, lapl=self._lapl) elif isinstance(obj, SoftLink): self.id.links.create_soft(name, self._e(obj.path), lcpl=lcpl, lapl=self._lapl) elif isinstance(obj, ExternalLink): do_link = True elif isinstance(obj, numpy.dtype): htype = h5t.py_create(obj, logical=True) htype.commit(self.id, name, lcpl=lcpl) else: ds = self.create_dataset(None, data=obj, dtype=base.guess_dtype(obj)) h5o.link(ds.id, self.id, name, lcpl=lcpl) if do_link: fn = filename_encode(obj.filename) with phil: self.id.links.create_external(name, fn, self._e(obj.path), lcpl=lcpl, lapl=self._lapl) @with_phil def __delitem__(self, name): """ Delete (unlink) an item from this group. """ self.id.unlink(self._e(name)) @with_phil def __len__(self): """ Number of members attached to this group """ return self.id.get_num_objs() @with_phil def __iter__(self): """ Iterate over member names """ for x in self.id.__iter__(): yield self._d(x) @with_phil def __contains__(self, name): """ Test if a member name exists """ return self._e(name) in self.id def copy(self, source, dest, name=None, shallow=False, expand_soft=False, expand_external=False, expand_refs=False, without_attrs=False): """Copy an object or group. The source can be a path, Group, Dataset, or Datatype object. The destination can be either a path or a Group object. The source and destinations need not be in the same file. If the source is a Group object, all objects contained in that group will be copied recursively. When the destination is a Group object, by default the target will be created in that group with its current name (basename of obj.name). You can override that by setting "name" to a string. There are various options which all default to "False": - shallow: copy only immediate members of a group. - expand_soft: expand soft links into new objects. - expand_external: expand external links into new objects. - expand_refs: copy objects that are pointed to by references. - without_attrs: copy object without copying attributes. Example: >>> f = File('myfile.hdf5') >>> f.listnames() ['MyGroup'] >>> f.copy('MyGroup', 'MyCopy') >>> f.listnames() ['MyGroup', 'MyCopy'] """ with phil: if isinstance(source, HLObject): source_path = '.' else: # Interpret source as a path relative to this group source_path = source source = self if isinstance(dest, Group): if name is not None: dest_path = name else: # copy source into dest group: dest_name/source_name dest_path = pp.basename(h5i.get_name(source[source_path].id)) elif isinstance(dest, HLObject): raise TypeError("Destination must be path or Group object") else: # Interpret destination as a path relative to this group dest_path = dest dest = self flags = 0 if shallow: flags |= h5o.COPY_SHALLOW_HIERARCHY_FLAG if expand_soft: flags |= h5o.COPY_EXPAND_SOFT_LINK_FLAG if expand_external: flags |= h5o.COPY_EXPAND_EXT_LINK_FLAG if expand_refs: flags |= h5o.COPY_EXPAND_REFERENCE_FLAG if without_attrs: flags |= h5o.COPY_WITHOUT_ATTR_FLAG if flags: copypl = h5p.create(h5p.OBJECT_COPY) copypl.set_copy_object(flags) else: copypl = None h5o.copy(source.id, self._e(source_path), dest.id, self._e(dest_path), copypl, base.dlcpl) def move(self, source, dest): """ Move a link to a new location in the file. If "source" is a hard link, this effectively renames the object. If "source" is a soft or external link, the link itself is moved, with its value unmodified. """ with phil: if source == dest: return self.id.links.move(self._e(source), self.id, self._e(dest), lapl=self._lapl, lcpl=self._lcpl) def visit(self, func): """ Recursively visit all names in this group and subgroups (HDF5 1.8). You supply a callable (function, method or callable object); it will be called exactly once for each link in this group and every group below it. Your callable must conform to the signature: func() => Returning None continues iteration, returning anything else stops and immediately returns that value from the visit method. No particular order of iteration within groups is guranteed. Example: >>> # List the entire contents of the file >>> f = File("foo.hdf5") >>> list_of_names = [] >>> f.visit(list_of_names.append) """ with phil: def proxy(name): """ Call the function with the text name, not bytes """ return func(self._d(name)) return h5o.visit(self.id, proxy) def visititems(self, func): """ Recursively visit names and objects in this group (HDF5 1.8). You supply a callable (function, method or callable object); it will be called exactly once for each link in this group and every group below it. Your callable must conform to the signature: func(, ) => Returning None continues iteration, returning anything else stops and immediately returns that value from the visit method. No particular order of iteration within groups is guranteed. Example: # Get a list of all datasets in the file >>> mylist = [] >>> def func(name, obj): ... if isinstance(obj, Dataset): ... mylist.append(name) ... >>> f = File('foo.hdf5') >>> f.visititems(func) """ with phil: def proxy(name): """ Use the text name of the object, not bytes """ name = self._d(name) return func(name, self[name]) return h5o.visit(self.id, proxy) @with_phil def __repr__(self): if not self: r = u"" else: namestr = ( u'"%s"' % self.name ) if self.name is not None else u"(anonymous)" r = u'' % (namestr, len(self)) if six.PY2: return r.encode('utf8') return r class HardLink(object): """ Represents a hard link in an HDF5 file. Provided only so that Group.get works in a sensible way. Has no other function. """ pass class SoftLink(object): """ Represents a symbolic ("soft") link in an HDF5 file. The path may be absolute or relative. No checking is performed to ensure that the target actually exists. """ @property def path(self): """ Soft link value. Not guaranteed to be a valid path. """ return self._path def __init__(self, path): self._path = str(path) def __repr__(self): return '' % self.path class ExternalLink(object): """ Represents an HDF5 external link. Paths may be absolute or relative. No checking is performed to ensure either the target or file exists. """ @property def path(self): """ Soft link path, i.e. the part inside the HDF5 file. """ return self._path @property def filename(self): """ Path to the external HDF5 file in the filesystem. """ return self._filename def __init__(self, filename, path): self._filename = filename_decode(filename_encode(filename)) self._path = str(path) def __repr__(self): return '>> dset = myfile['dataset'] >>> myref = dset.regionref[0:100,20:30] >>> data = dset[myref] """ def __init__(self, dsid): """ Supply a h5py.h5d.DatasetID instance """ self.id = dsid def __getitem__(self, args): """ Takes arbitrary selection terms and produces a RegionReference object. Selection must be compatible with the dataset. """ selection = select(self.id.shape, args, self.id) return h5r.create(self.id, '.', h5r.DATASET_REGION, selection.id) class Selection(object): """ Base class for HDF5 dataspace selections. Subclasses support the "selection protocol", which means they have at least the following members: __init__(shape) => Create a new selection on "shape"-tuple __getitem__(args) => Perform a selection with the range specified. What args are allowed depends on the particular subclass in use. id (read-only) => h5py.h5s.SpaceID instance shape (read-only) => The shape of the dataspace. mshape (read-only) => The shape of the selection region. Not guaranteed to fit within "shape", although the total number of points is less than product(shape). nselect (read-only) => Number of selected points. Always equal to product(mshape). broadcast(target_shape) => Return an iterable which yields dataspaces for read, based on target_shape. The base class represents "unshaped" selections (1-D). """ def __init__(self, shape, spaceid=None): """ Create a selection. Shape may be None if spaceid is given. """ if spaceid is not None: self._id = spaceid self._shape = spaceid.shape else: shape = tuple(shape) self._shape = shape self._id = h5s.create_simple(shape, (h5s.UNLIMITED,)*len(shape)) self._id.select_all() @property def id(self): """ SpaceID instance """ return self._id @property def shape(self): """ Shape of whole dataspace """ return self._shape @property def nselect(self): """ Number of elements currently selected """ return self._id.get_select_npoints() @property def mshape(self): """ Shape of selection (always 1-D for this class) """ return (self.nselect,) def broadcast(self, target_shape): """ Get an iterable for broadcasting """ if np.product(target_shape) != self.nselect: raise TypeError("Broadcasting is not supported for point-wise selections") yield self._id def __getitem__(self, args): raise NotImplementedError("This class does not support indexing") class PointSelection(Selection): """ Represents a point-wise selection. You can supply sequences of points to the three methods append(), prepend() and set(), or a single boolean array to __getitem__. """ def _perform_selection(self, points, op): """ Internal method which actually performs the selection """ points = np.asarray(points, order='C', dtype='u8') if len(points.shape) == 1: points.shape = (1,points.shape[0]) if self._id.get_select_type() != h5s.SEL_POINTS: op = h5s.SELECT_SET if len(points) == 0: self._id.select_none() else: self._id.select_elements(points, op) def __getitem__(self, arg): """ Perform point-wise selection from a NumPy boolean array """ if not (isinstance(arg, np.ndarray) and arg.dtype.kind == 'b'): raise TypeError("PointSelection __getitem__ only works with bool arrays") if not arg.shape == self.shape: raise TypeError("Boolean indexing array has incompatible shape") points = np.transpose(arg.nonzero()) self.set(points) return self def append(self, points): """ Add the sequence of points to the end of the current selection """ self._perform_selection(points, h5s.SELECT_APPEND) def prepend(self, points): """ Add the sequence of points to the beginning of the current selection """ self._perform_selection(points, h5s.SELECT_PREPEND) def set(self, points): """ Replace the current selection with the given sequence of points""" self._perform_selection(points, h5s.SELECT_SET) class SimpleSelection(Selection): """ A single "rectangular" (regular) selection composed of only slices and integer arguments. Can participate in broadcasting. """ @property def mshape(self): """ Shape of current selection """ return self._mshape def __init__(self, shape, *args, **kwds): Selection.__init__(self, shape, *args, **kwds) rank = len(self.shape) self._sel = ((0,)*rank, self.shape, (1,)*rank, (False,)*rank) self._mshape = self.shape def __getitem__(self, args): if not isinstance(args, tuple): args = (args,) if self.shape == (): if len(args) > 0 and args[0] not in (Ellipsis, ()): raise TypeError("Invalid index for scalar dataset (only ..., () allowed)") self._id.select_all() return self start, count, step, scalar = _handle_simple(self.shape,args) self._id.select_hyperslab(start, count, step) self._sel = (start, count, step, scalar) self._mshape = tuple(x for x, y in zip(count, scalar) if not y) return self def broadcast(self, target_shape): """ Return an iterator over target dataspaces for broadcasting. Follows the standard NumPy broadcasting rules against the current selection shape (self.mshape). """ if self.shape == (): if np.product(target_shape) != 1: raise TypeError("Can't broadcast %s to scalar" % target_shape) self._id.select_all() yield self._id return start, count, step, scalar = self._sel rank = len(count) target = list(target_shape) tshape = [] for idx in xrange(1,rank+1): if len(target) == 0 or scalar[-idx]: # Skip scalar axes tshape.append(1) else: t = target.pop() if t == 1 or count[-idx] == t: tshape.append(t) else: raise TypeError("Can't broadcast %s -> %s" % (target_shape, count)) tshape.reverse() tshape = tuple(tshape) chunks = tuple(x//y for x, y in zip(count, tshape)) nchunks = int(np.product(chunks)) if nchunks == 1: yield self._id else: sid = self._id.copy() sid.select_hyperslab((0,)*rank, tshape, step) for idx in xrange(nchunks): offset = tuple(x*y*z + s for x, y, z, s in zip(np.unravel_index(idx, chunks), tshape, step, start)) sid.offset_simple(offset) yield sid class FancySelection(Selection): """ Implements advanced NumPy-style selection operations in addition to the standard slice-and-int behavior. Indexing arguments may be ints, slices, lists of indicies, or per-axis (1D) boolean arrays. Broadcasting is not supported for these selections. """ @property def mshape(self): return self._mshape def __init__(self, shape, *args, **kwds): Selection.__init__(self, shape, *args, **kwds) self._mshape = self.shape def __getitem__(self, args): if not isinstance(args, tuple): args = (args,) args = _expand_ellipsis(args, len(self.shape)) # First build up a dictionary of (position:sequence) pairs sequenceargs = {} for idx, arg in enumerate(args): if not isinstance(arg, slice): if hasattr(arg, 'dtype') and arg.dtype == np.dtype('bool'): if len(arg.shape) != 1: raise TypeError("Boolean indexing arrays must be 1-D") arg = arg.nonzero()[0] try: sequenceargs[idx] = list(arg) except TypeError: pass else: list_arg = list(arg) adjacent = zip(list_arg[:-1], list_arg[1:]) if any(fst >= snd for fst, snd in adjacent): raise TypeError("Indexing elements must be in increasing order") if len(sequenceargs) > 1: raise TypeError("Only one indexing vector or array is currently allowed for advanced selection") if len(sequenceargs) == 0: raise TypeError("Advanced selection inappropriate") vectorlength = len(list(sequenceargs.values())[0]) if not all(len(x) == vectorlength for x in sequenceargs.values()): raise TypeError("All sequence arguments must have the same length %s" % sequenceargs) # Now generate a vector of selection lists, # consisting only of slices and ints argvector = [] for idx in xrange(vectorlength): entry = list(args) for position, seq in six.iteritems(sequenceargs): entry[position] = seq[idx] argvector.append(entry) # "OR" all these selection lists together to make the final selection self._id.select_none() for idx, vector in enumerate(argvector): start, count, step, scalar = _handle_simple(self.shape, vector) self._id.select_hyperslab(start, count, step, op=h5s.SELECT_OR) # Final shape excludes scalars, except where # they correspond to sequence entries mshape = list(count) for idx in xrange(len(mshape)): if idx in sequenceargs: mshape[idx] = len(sequenceargs[idx]) elif scalar[idx]: mshape[idx] = 0 self._mshape = tuple(x for x in mshape if x != 0) def broadcast(self, target_shape): if not target_shape == self.mshape: raise TypeError("Broadcasting is not supported for complex selections") yield self._id def _expand_ellipsis(args, rank): """ Expand ellipsis objects and fill in missing axes. """ n_el = sum(1 for arg in args if arg is Ellipsis) if n_el > 1: raise ValueError("Only one ellipsis may be used.") elif n_el == 0 and len(args) != rank: args = args + (Ellipsis,) final_args = [] n_args = len(args) for arg in args: if arg is Ellipsis: final_args.extend( (slice(None,None,None),)*(rank-n_args+1) ) else: final_args.append(arg) if len(final_args) > rank: raise TypeError("Argument sequence too long") return final_args def _handle_simple(shape, args): """ Process a "simple" selection tuple, containing only slices and integer objects. Return is a 4-tuple with tuples for start, count, step, and a flag which tells if the axis is a "scalar" selection (indexed by an integer). If "args" is shorter than "shape", the remaining axes are fully selected. """ args = _expand_ellipsis(args, len(shape)) start = [] count = [] step = [] scalar = [] for arg, length in zip(args, shape): if isinstance(arg, slice): x,y,z = _translate_slice(arg, length) s = False else: try: x,y,z = _translate_int(int(arg), length) s = True except TypeError: raise TypeError('Illegal index "%s" (must be a slice or number)' % arg) start.append(x) count.append(y) step.append(z) scalar.append(s) return tuple(start), tuple(count), tuple(step), tuple(scalar) def _translate_int(exp, length): """ Given an integer index, return a 3-tuple (start, count, step) for hyperslab selection """ if exp < 0: exp = length+exp if not 0<=exp 0, then start and stop are in [0, length]; # if step < 0, they are in [-1, length - 1] (Python 2.6b2 and later; # Python issue 3004). if step < 1: raise ValueError("Step must be >= 1 (got %d)" % step) if stop < start: raise ValueError("Reverse-order selections are not allowed") count = 1 + (stop - start - 1) // step return start, count, step def guess_shape(sid): """ Given a dataspace, try to deduce the shape of the selection. Returns one of: * A tuple with the selection shape, same length as the dataspace * A 1D selection shape for point-based and multiple-hyperslab selections * None, for unselected scalars and for NULL dataspaces """ sel_class = sid.get_simple_extent_type() # Dataspace class sel_type = sid.get_select_type() # Flavor of selection in use if sel_class == h5s.NULL: # NULL dataspaces don't support selections return None elif sel_class == h5s.SCALAR: # NumPy has no way of expressing empty 0-rank selections, so we use None if sel_type == h5s.SEL_NONE: return None if sel_type == h5s.SEL_ALL: return tuple() elif sel_class != h5s.SIMPLE: raise TypeError("Unrecognized dataspace class %s" % sel_class) # We have a "simple" (rank >= 1) dataspace N = sid.get_select_npoints() rank = len(sid.shape) if sel_type == h5s.SEL_NONE: return (0,)*rank elif sel_type == h5s.SEL_ALL: return sid.shape elif sel_type == h5s.SEL_POINTS: # Like NumPy, point-based selections yield 1D arrays regardless of # the dataspace rank return (N,) elif sel_type != h5s.SEL_HYPERSLABS: raise TypeError("Unrecognized selection method %s" % sel_type) # We have a hyperslab-based selection if N == 0: return (0,)*rank bottomcorner, topcorner = (np.array(x) for x in sid.get_select_bounds()) # Shape of full selection box boxshape = topcorner - bottomcorner + np.ones((rank,)) def get_n_axis(sid, axis): """ Determine the number of elements selected along a particular axis. To do this, we "mask off" the axis by making a hyperslab selection which leaves only the first point along the axis. For a 2D dataset with selection box shape (X, Y), for axis 1, this would leave a selection of shape (X, 1). We count the number of points N_leftover remaining in the selection and compute the axis selection length by N_axis = N/N_leftover. """ if(boxshape[axis]) == 1: return 1 start = bottomcorner.copy() start[axis] += 1 count = boxshape.copy() count[axis] -= 1 # Throw away all points along this axis masked_sid = sid.copy() masked_sid.select_hyperslab(tuple(start), tuple(count), op=h5s.SELECT_NOTB) N_leftover = masked_sid.get_select_npoints() return N//N_leftover shape = tuple(get_n_axis(sid, x) for x in xrange(rank)) if np.product(shape) != N: # This means multiple hyperslab selections are in effect, # so we fall back to a 1D shape return (N,) return shape h5py-2.7.1/h5py/_hl/selections2.py0000644000175000017500000000535413114361073020325 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. """ Implements a portion of the selection operations. """ from __future__ import absolute_import import numpy as np from .. import h5s def read_dtypes(dataset_dtype, names): """ Returns a 2-tuple containing: 1. Output dataset dtype 2. Dtype containing HDF5-appropriate description of destination """ if len(names) == 0: # Not compound, or all fields needed format_dtype = dataset_dtype elif dataset_dtype.names is None: raise ValueError("Field names only allowed for compound types") elif any(x not in dataset_dtype.names for x in names): raise ValueError("Field does not appear in this type.") else: format_dtype = np.dtype([(name, dataset_dtype.fields[name][0]) for name in names]) if len(names) == 1: # We don't preserve the field information if only one explicitly selected. output_dtype = format_dtype.fields[names[0]][0] else: output_dtype = format_dtype return output_dtype, format_dtype def read_selections_scalar(dsid, args): """ Returns a 2-tuple containing: 1. Output dataset shape 2. HDF5 dataspace containing source selection. Works for scalar datasets. """ if dsid.shape != (): raise RuntimeError("Illegal selection function for non-scalar dataset") if args == (): # This is a signal that an array scalar should be returned instead # of an ndarray with shape () out_shape = None elif args == (Ellipsis,): out_shape = () else: raise ValueError("Illegal slicing argument for scalar dataspace") source_space = dsid.get_space() source_space.select_all() return out_shape, source_space class ScalarReadSelection(object): """ Implements slicing for scalar datasets. """ def __init__(self, fspace, args): if args == (): self.mshape = None elif args == (Ellipsis,): self.mshape = () else: raise ValueError("Illegal slicing argument for scalar dataspace") self.mspace = h5s.create(h5s.SCALAR) self.fspace = fspace def __iter__(self): self.mspace.select_all() yield self.fspace, self.mspace def select_read(fspace, args): """ Top-level dispatch function for reading. At the moment, only supports reading from scalar datasets. """ if fspace.shape == (): return ScalarReadSelection(fspace, args) raise NotImplementedError() h5py-2.7.1/h5py/_hl/__init__.py0000644000175000017500000000076613114361073017634 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. """ This subpackage implements the high-level interface for h5py. Don't manually import things from here; the public API lives directly in the top-level package namespace. """ from __future__ import absolute_import h5py-2.7.1/h5py/_hl/base.py0000644000175000017500000002732013114361073017002 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. """ Implements operations common to all high-level objects (File, etc.). """ from __future__ import absolute_import import posixpath import os import six from collections import (Mapping, MutableMapping, KeysView, ValuesView, ItemsView) from .compat import fspath, filename_encode from .. import h5d, h5i, h5r, h5p, h5f, h5t, h5s # The high-level interface is serialized; every public API function & method # is wrapped in a lock. We re-use the low-level lock because (1) it's fast, # and (2) it eliminates the possibility of deadlocks due to out-of-order # lock acquisition. from .._objects import phil, with_phil def is_hdf5(fname): """ Determine if a file is valid HDF5 (False if it doesn't exist). """ with phil: fname = os.path.abspath(fspath(fname)) if os.path.isfile(fname): return h5f.is_hdf5(filename_encode(fname)) return False def guess_dtype(data): """ Attempt to guess an appropriate dtype for the object, returning None if nothing is appropriate (or if it should be left up the the array constructor to figure out) """ with phil: if isinstance(data, h5r.RegionReference): return h5t.special_dtype(ref=h5r.RegionReference) if isinstance(data, h5r.Reference): return h5t.special_dtype(ref=h5r.Reference) if type(data) == bytes: return h5t.special_dtype(vlen=bytes) if type(data) == six.text_type: return h5t.special_dtype(vlen=six.text_type) return None def default_lapl(): """ Default link access property list """ lapl = h5p.create(h5p.LINK_ACCESS) fapl = h5p.create(h5p.FILE_ACCESS) fapl.set_fclose_degree(h5f.CLOSE_STRONG) lapl.set_elink_fapl(fapl) return lapl def default_lcpl(): """ Default link creation property list """ lcpl = h5p.create(h5p.LINK_CREATE) lcpl.set_create_intermediate_group(True) return lcpl dlapl = default_lapl() dlcpl = default_lcpl() def is_empty_dataspace(obj): """ Check if an object's dataspace is empty """ if obj.get_space().get_simple_extent_type() == h5s.NULL: return True return False class CommonStateObject(object): """ Mixin class that allows sharing information between objects which reside in the same HDF5 file. Requires that the host class have a ".id" attribute which returns a low-level ObjectID subclass. Also implements Unicode operations. """ @property def _lapl(self): """ Fetch the link access property list appropriate for this object """ return dlapl @property def _lcpl(self): """ Fetch the link creation property list appropriate for this object """ return dlcpl def _e(self, name, lcpl=None): """ Encode a name according to the current file settings. Returns name, or 2-tuple (name, lcpl) if lcpl is True - Binary strings are always passed as-is, h5t.CSET_ASCII - Unicode strings are encoded utf8, h5t.CSET_UTF8 If name is None, returns either None or (None, None) appropriately. """ def get_lcpl(coding): """ Create an appropriate link creation property list """ lcpl = self._lcpl.copy() lcpl.set_char_encoding(coding) return lcpl if name is None: return (None, None) if lcpl else None if isinstance(name, bytes): coding = h5t.CSET_ASCII else: try: name = name.encode('ascii') coding = h5t.CSET_ASCII except UnicodeEncodeError: name = name.encode('utf8') coding = h5t.CSET_UTF8 if lcpl: return name, get_lcpl(coding) return name def _d(self, name): """ Decode a name according to the current file settings. - Try to decode utf8 - Failing that, return the byte string If name is None, returns None. """ if name is None: return None try: return name.decode('utf8') except UnicodeDecodeError: pass return name class _RegionProxy(object): """ Proxy object which handles region references. To create a new region reference (datasets only), use slicing syntax: >>> newref = obj.regionref[0:10:2] To determine the target dataset shape from an existing reference: >>> shape = obj.regionref.shape(existingref) where may be any object in the file. To determine the shape of the selection in use on the target dataset: >>> selection_shape = obj.regionref.selection(existingref) """ def __init__(self, obj): self.id = obj.id def __getitem__(self, args): if not isinstance(self.id, h5d.DatasetID): raise TypeError("Region references can only be made to datasets") from . import selections with phil: selection = selections.select(self.id.shape, args, dsid=self.id) return h5r.create(self.id, b'.', h5r.DATASET_REGION, selection.id) def shape(self, ref): """ Get the shape of the target dataspace referred to by *ref*. """ with phil: sid = h5r.get_region(ref, self.id) return sid.shape def selection(self, ref): """ Get the shape of the target dataspace selection referred to by *ref* """ from . import selections with phil: sid = h5r.get_region(ref, self.id) return selections.guess_shape(sid) class HLObject(CommonStateObject): """ Base class for high-level interface objects. """ @property def file(self): """ Return a File instance associated with this object """ from . import files with phil: return files.File(self.id) @property @with_phil def name(self): """ Return the full name of this object. None if anonymous. """ return self._d(h5i.get_name(self.id)) @property @with_phil def parent(self): """Return the parent group of this object. This is always equivalent to obj.file[posixpath.dirname(obj.name)]. ValueError if this object is anonymous. """ if self.name is None: raise ValueError("Parent of an anonymous object is undefined") return self.file[posixpath.dirname(self.name)] @property @with_phil def id(self): """ Low-level identifier appropriate for this object """ return self._id @property @with_phil def ref(self): """ An (opaque) HDF5 reference to this object """ return h5r.create(self.id, b'.', h5r.OBJECT) @property @with_phil def regionref(self): """Create a region reference (Datasets only). The syntax is regionref[]. For example, dset.regionref[...] creates a region reference in which the whole dataset is selected. Can also be used to determine the shape of the referenced dataset (via .shape property), or the shape of the selection (via the .selection property). """ return _RegionProxy(self) @property def attrs(self): """ Attributes attached to this object """ from . import attrs with phil: return attrs.AttributeManager(self) @with_phil def __init__(self, oid): """ Setup this object, given its low-level identifier """ self._id = oid @with_phil def __hash__(self): return hash(self.id) @with_phil def __eq__(self, other): if hasattr(other, 'id'): return self.id == other.id return False @with_phil def __ne__(self, other): return not self.__eq__(other) def __bool__(self): with phil: return bool(self.id) __nonzero__ = __bool__ # --- Dictionary-style interface ---------------------------------------------- # To implement the dictionary-style interface from groups and attributes, # we inherit from the appropriate abstract base classes in collections. # # All locking is taken care of by the subclasses. # We have to override ValuesView and ItemsView here because Group and # AttributeManager can only test for key names. class ValuesViewHDF5(ValuesView): """ Wraps e.g. a Group or AttributeManager to provide a value view. Note that __contains__ will have poor performance as it has to scan all the links or attributes. """ def __contains__(self, value): with phil: for key in self._mapping: if value == self._mapping.get(key): return True return False def __iter__(self): with phil: for key in self._mapping: yield self._mapping.get(key) class ItemsViewHDF5(ItemsView): """ Wraps e.g. a Group or AttributeManager to provide an items view. """ def __contains__(self, item): with phil: key, val = item if key in self._mapping: return val == self._mapping.get(key) return False def __iter__(self): with phil: for key in self._mapping: yield (key, self._mapping.get(key)) class MappingHDF5(Mapping): """ Wraps a Group, AttributeManager or DimensionManager object to provide an immutable mapping interface. We don't inherit directly from MutableMapping because certain subclasses, for example DimensionManager, are read-only. """ if six.PY2: def keys(self): """ Get a list containing member names """ with phil: return list(self) def values(self): """ Get a list containing member objects """ with phil: return [self.get(x) for x in self] def itervalues(self): """ Get an iterator over member objects """ for x in self: yield self.get(x) def items(self): """ Get a list of tuples containing (name, object) pairs """ with phil: return [(x, self.get(x)) for x in self] def iteritems(self): """ Get an iterator over (name, object) pairs """ for x in self: yield (x, self.get(x)) else: def keys(self): """ Get a view object on member names """ return KeysView(self) def values(self): """ Get a view object on member objects """ return ValuesViewHDF5(self) def items(self): """ Get a view object on member items """ return ItemsViewHDF5(self) class MutableMappingHDF5(MappingHDF5, MutableMapping): """ Wraps a Group or AttributeManager object to provide a mutable mapping interface, in contrast to the read-only mapping of MappingHDF5. """ pass class Empty(object): """ Proxy object to represent empty/null dataspaces (a.k.a H5S_NULL). This can have an associated dtype, but has no shape or data. This is not the same as an array with shape (0,). """ shape = None def __init__(self, dtype): self.dtype = dtype def __eq__(self, other): if isinstance(other, Empty) and self.dtype == other.dtype: return True return False def __repr__(self): return "Empty(dtype={0!r})".format(self.dtype) h5py-2.7.1/h5py/_hl/dims.py0000644000175000017500000001232313114361073017021 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. """ Implements support for HDF5 dimension scales. """ from __future__ import absolute_import from .. import h5ds from . import base from .base import phil, with_phil from .dataset import Dataset class DimensionProxy(base.CommonStateObject): """ Represents an HDF5 "dimension". """ @property @with_phil def label(self): """ Get or set the dimension scale label """ #return h5ds.get_label(self._id, self._dimension) # Produces a segfault for a non-existent label (Fixed in hdf5-1.8.8). # Here is a workaround: try: dset = Dataset(self._id) return self._d(dset.attrs['DIMENSION_LABELS'][self._dimension]) except (KeyError, IndexError): return '' @label.setter @with_phil def label(self, val): # pylint: disable=missing-docstring h5ds.set_label(self._id, self._dimension, self._e(val)) @with_phil def __init__(self, id_, dimension): self._id = id_ self._dimension = dimension @with_phil def __hash__(self): return hash((type(self), self._id, self._dimension)) @with_phil def __eq__(self, other): return hash(self) == hash(other) @with_phil def __iter__(self): for k in self.keys(): yield k @with_phil def __len__(self): return h5ds.get_num_scales(self._id, self._dimension) @with_phil def __getitem__(self, item): if isinstance(item, int): scales = [] h5ds.iterate(self._id, self._dimension, scales.append, 0) return Dataset(scales[item]) else: def f(dsid): """ Iterate over scales to find a matching name """ if h5ds.get_scale_name(dsid) == self._e(item): return dsid res = h5ds.iterate(self._id, self._dimension, f, 0) if res is None: raise KeyError(item) return Dataset(res) def attach_scale(self, dset): """ Attach a scale to this dimension. Provide the Dataset of the scale you would like to attach. """ with phil: h5ds.attach_scale(self._id, dset.id, self._dimension) def detach_scale(self, dset): """ Remove a scale from this dimension. Provide the Dataset of the scale you would like to remove. """ with phil: h5ds.detach_scale(self._id, dset.id, self._dimension) def items(self): """ Get a list of (name, Dataset) pairs with all scales on this dimension. """ with phil: scales = [] # H5DSiterate raises an error if there are no dimension scales, # rather than iterating 0 times. See #483. if len(self) > 0: h5ds.iterate(self._id, self._dimension, scales.append, 0) return [ (self._d(h5ds.get_scale_name(x)), Dataset(x)) for x in scales ] def keys(self): """ Get a list of names for the scales on this dimension. """ with phil: return [key for (key, _) in self.items()] def values(self): """ Get a list of Dataset for scales on this dimension. """ with phil: return [val for (_, val) in self.items()] @with_phil def __repr__(self): if not self._id: return "" return ('<"%s" dimension %d of HDF5 dataset at %s>' % (self.label, self._dimension, id(self._id))) class DimensionManager(base.MappingHDF5, base.CommonStateObject): """ Represents a collection of dimension associated with a dataset. Like AttributeManager, an instance of this class is returned when accessing the ".dims" property on a Dataset. """ @with_phil def __init__(self, parent): """ Private constructor. """ self._id = parent.id @with_phil def __getitem__(self, index): """ Return a Dimension object """ if index > len(self) - 1: raise IndexError('Index out of range') return DimensionProxy(self._id, index) @with_phil def __len__(self): """ Number of dimensions associated with the dataset. """ return len(Dataset(self._id).shape) @with_phil def __iter__(self): """ Iterate over the dimensions. """ for i in range(len(self)): yield self[i] @with_phil def __repr__(self): if not self._id: return "" return "" % id(self._id) def create_scale(self, dset, name=''): """ Create a new dimension, from an initial scale. Provide the dataset and a name for the scale. """ with phil: h5ds.set_scale(dset.id, self._e(name)) h5py-2.7.1/h5py/_hl/datatype.py0000644000175000017500000000307313114361073017702 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. """ Implements high-level access to committed datatypes in the file. """ from __future__ import absolute_import import posixpath as pp from ..h5t import TypeID from .base import HLObject, with_phil class Datatype(HLObject): """ Represents an HDF5 named datatype stored in a file. To store a datatype, simply assign it to a name in a group: >>> MyGroup["name"] = numpy.dtype("f") >>> named_type = MyGroup["name"] >>> assert named_type.dtype == numpy.dtype("f") """ @property @with_phil def dtype(self): """Numpy dtype equivalent for this datatype""" return self.id.dtype @with_phil def __init__(self, bind): """ Create a new Datatype object by binding to a low-level TypeID. """ if not isinstance(bind, TypeID): raise ValueError("%s is not a TypeID" % bind) HLObject.__init__(self, bind) @with_phil def __repr__(self): if not self.id: return "" if self.name is None: namestr = '("anonymous")' else: name = pp.basename(pp.normpath(self.name)) namestr = '"%s"' % (name if name != '' else '/') return '' % \ (namestr, self.dtype.str) h5py-2.7.1/h5py/_hl/dataset.py0000644000175000017500000006277413146406212017531 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. """ Implements support for high-level dataset access. """ from __future__ import absolute_import import posixpath as pp import sys from threading import local import six from six.moves import xrange # pylint: disable=redefined-builtin import numpy from .. import h5, h5s, h5t, h5r, h5d, h5p, h5fd from .base import HLObject, phil, with_phil, Empty, is_empty_dataspace from . import filters from . import selections as sel from . import selections2 as sel2 from .datatype import Datatype _LEGACY_GZIP_COMPRESSION_VALS = frozenset(range(10)) MPI = h5.get_config().mpi def readtime_dtype(basetype, names): """ Make a NumPy dtype appropriate for reading """ if len(names) == 0: # Not compound, or we want all fields return basetype if basetype.names is None: # Names provided, but not compound raise ValueError("Field names only allowed for compound types") for name in names: # Check all names are legal if not name in basetype.names: raise ValueError("Field %s does not appear in this type." % name) return numpy.dtype([(name, basetype.fields[name][0]) for name in names]) def make_new_dset(parent, shape=None, dtype=None, data=None, chunks=None, compression=None, shuffle=None, fletcher32=None, maxshape=None, compression_opts=None, fillvalue=None, scaleoffset=None, track_times=None): """ Return a new low-level dataset identifier Only creates anonymous datasets. """ # Convert data to a C-contiguous ndarray if data is not None and not isinstance(data, Empty): from . import base data = numpy.asarray(data, order="C", dtype=base.guess_dtype(data)) # Validate shape if shape is None: if data is None: if dtype is None: raise TypeError("One of data, shape or dtype must be specified") data = Empty(dtype) shape = data.shape else: shape = tuple(shape) if data is not None and (numpy.product(shape) != numpy.product(data.shape)): raise ValueError("Shape tuple is incompatible with data") tmp_shape = maxshape if maxshape is not None else shape # Validate chunk shape if isinstance(chunks, tuple) and any( chunk > dim for dim, chunk in zip(tmp_shape,chunks) if dim is not None ): errmsg = "Chunk shape must not be greater than data shape in any dimension. "\ "{} is not compatible with {}".format(chunks, shape) raise ValueError(errmsg) if isinstance(dtype, Datatype): # Named types are used as-is tid = dtype.id dtype = tid.dtype # Following code needs this else: # Validate dtype if dtype is None and data is None: dtype = numpy.dtype("=f4") elif dtype is None and data is not None: dtype = data.dtype else: dtype = numpy.dtype(dtype) tid = h5t.py_create(dtype, logical=1) # Legacy if any((compression, shuffle, fletcher32, maxshape,scaleoffset)) and chunks is False: raise ValueError("Chunked format required for given storage options") # Legacy if compression is True: if compression_opts is None: compression_opts = 4 compression = 'gzip' # Legacy if compression in _LEGACY_GZIP_COMPRESSION_VALS: if compression_opts is not None: raise TypeError("Conflict in compression options") compression_opts = compression compression = 'gzip' dcpl = filters.generate_dcpl(shape, dtype, chunks, compression, compression_opts, shuffle, fletcher32, maxshape, scaleoffset) if fillvalue is not None: fillvalue = numpy.array(fillvalue) dcpl.set_fill_value(fillvalue) if track_times in (True, False): dcpl.set_obj_track_times(track_times) elif track_times is not None: raise TypeError("track_times must be either True or False") if maxshape is not None: maxshape = tuple(m if m is not None else h5s.UNLIMITED for m in maxshape) if isinstance(data, Empty): sid = h5s.create(h5s.NULL) else: sid = h5s.create_simple(shape, maxshape) dset_id = h5d.create(parent.id, None, tid, sid, dcpl=dcpl) if (data is not None) and (not isinstance(data, Empty)): dset_id.write(h5s.ALL, h5s.ALL, data) return dset_id class AstypeContext(object): """ Context manager which allows changing the type read from a dataset. """ def __init__(self, dset, dtype): self._dset = dset self._dtype = numpy.dtype(dtype) def __enter__(self): # pylint: disable=protected-access self._dset._local.astype = self._dtype def __exit__(self, *args): # pylint: disable=protected-access self._dset._local.astype = None if MPI: class CollectiveContext(object): """ Manages collective I/O in MPI mode """ # We don't bother with _local as threads are forbidden in MPI mode def __init__(self, dset): self._dset = dset def __enter__(self): # pylint: disable=protected-access self._dset._dxpl.set_dxpl_mpio(h5fd.MPIO_COLLECTIVE) def __exit__(self, *args): # pylint: disable=protected-access self._dset._dxpl.set_dxpl_mpio(h5fd.MPIO_INDEPENDENT) class Dataset(HLObject): """ Represents an HDF5 dataset """ def astype(self, dtype): """ Get a context manager allowing you to perform reads to a different destination type, e.g.: >>> with dataset.astype('f8'): ... double_precision = dataset[0:100:2] """ return AstypeContext(self, dtype) if MPI: @property @with_phil def collective(self): """ Context manager for MPI collective reads & writes """ return CollectiveContext(self) @property def dims(self): """ Access dimension scales attached to this dataset. """ from .dims import DimensionManager with phil: return DimensionManager(self) @property @with_phil def ndim(self): """Numpy-style attribute giving the number of dimensions""" return self.id.rank @property @with_phil def shape(self): """Numpy-style shape tuple giving dataset dimensions""" return self.id.shape @shape.setter @with_phil def shape(self, shape): # pylint: disable=missing-docstring self.resize(shape) @property @with_phil def size(self): """Numpy-style attribute giving the total dataset size""" return numpy.prod(self.shape, dtype=numpy.intp) @property @with_phil def dtype(self): """Numpy dtype representing the datatype""" return self.id.dtype @property @with_phil def value(self): """ Alias for dataset[()] """ DeprecationWarning("dataset.value has been deprecated. " "Use dataset[()] instead.") return self[()] @property @with_phil def chunks(self): """Dataset chunks (or None)""" dcpl = self._dcpl if dcpl.get_layout() == h5d.CHUNKED: return dcpl.get_chunk() return None @property @with_phil def compression(self): """Compression strategy (or None)""" for x in ('gzip','lzf','szip'): if x in self._filters: return x return None @property @with_phil def compression_opts(self): """ Compression setting. Int(0-9) for gzip, 2-tuple for szip. """ return self._filters.get(self.compression, None) @property @with_phil def shuffle(self): """Shuffle filter present (T/F)""" return 'shuffle' in self._filters @property @with_phil def fletcher32(self): """Fletcher32 filter is present (T/F)""" return 'fletcher32' in self._filters @property @with_phil def scaleoffset(self): """Scale/offset filter settings. For integer data types, this is the number of bits stored, or 0 for auto-detected. For floating point data types, this is the number of decimal places retained. If the scale/offset filter is not in use, this is None.""" try: return self._filters['scaleoffset'][1] except KeyError: return None @property @with_phil def maxshape(self): """Shape up to which this dataset can be resized. Axes with value None have no resize limit. """ space = self.id.get_space() dims = space.get_simple_extent_dims(True) return tuple(x if x != h5s.UNLIMITED else None for x in dims) @property @with_phil def fillvalue(self): """Fill value for this dataset (0 by default)""" arr = numpy.ndarray((1,), dtype=self.dtype) self._dcpl.get_fill_value(arr) return arr[0] @with_phil def __init__(self, bind): """ Create a new Dataset object by binding to a low-level DatasetID. """ if not isinstance(bind, h5d.DatasetID): raise ValueError("%s is not a DatasetID" % bind) HLObject.__init__(self, bind) self._dcpl = self.id.get_create_plist() self._dxpl = h5p.create(h5p.DATASET_XFER) self._filters = filters.get_filters(self._dcpl) self._local = local() self._local.astype = None def resize(self, size, axis=None): """ Resize the dataset, or the specified axis. The dataset must be stored in chunked format; it can be resized up to the "maximum shape" (keyword maxshape) specified at creation time. The rank of the dataset cannot be changed. "Size" should be a shape tuple, or if an axis is specified, an integer. BEWARE: This functions differently than the NumPy resize() method! The data is not "reshuffled" to fit in the new shape; each axis is grown or shrunk independently. The coordinates of existing data are fixed. """ with phil: if self.chunks is None: raise TypeError("Only chunked datasets can be resized") if axis is not None: if not (axis >=0 and axis < self.id.rank): raise ValueError("Invalid axis (0 to %s allowed)" % (self.id.rank-1)) try: newlen = int(size) except TypeError: raise TypeError("Argument must be a single int if axis is specified") size = list(self.shape) size[axis] = newlen size = tuple(size) self.id.set_extent(size) #h5f.flush(self.id) # THG recommends @with_phil def __len__(self): """ The size of the first axis. TypeError if scalar. Limited to 2**32 on 32-bit systems; Dataset.len() is preferred. """ size = self.len() if size > sys.maxsize: raise OverflowError("Value too big for Python's __len__; use Dataset.len() instead.") return size def len(self): """ The size of the first axis. TypeError if scalar. Use of this method is preferred to len(dset), as Python's built-in len() cannot handle values greater then 2**32 on 32-bit systems. """ with phil: shape = self.shape if len(shape) == 0: raise TypeError("Attempt to take len() of scalar dataset") return shape[0] @with_phil def __iter__(self): """ Iterate over the first axis. TypeError if scalar. BEWARE: Modifications to the yielded data are *NOT* written to file. """ shape = self.shape if len(shape) == 0: raise TypeError("Can't iterate over a scalar dataset") for i in xrange(shape[0]): yield self[i] @with_phil def __getitem__(self, args): """ Read a slice from the HDF5 dataset. Takes slices and recarray-style field names (more than one is allowed!) in any order. Obeys basic NumPy rules, including broadcasting. Also supports: * Boolean "mask" array indexing """ args = args if isinstance(args, tuple) else (args,) if is_empty_dataspace(self.id): if not (args == tuple() or args == (Ellipsis,)): raise ValueError("Empty datasets cannot be sliced") return Empty(self.dtype) # Sort field indices from the rest of the args. names = tuple(x for x in args if isinstance(x, six.string_types)) args = tuple(x for x in args if not isinstance(x, six.string_types)) if six.PY2: names = tuple(x.encode('utf-8') if isinstance(x, six.text_type) else x for x in names) new_dtype = getattr(self._local, 'astype', None) if new_dtype is not None: new_dtype = readtime_dtype(new_dtype, names) else: # This is necessary because in the case of array types, NumPy # discards the array information at the top level. new_dtype = readtime_dtype(self.id.dtype, names) mtype = h5t.py_create(new_dtype) # === Special-case region references ==== if len(args) == 1 and isinstance(args[0], h5r.RegionReference): obj = h5r.dereference(args[0], self.id) if obj != self.id: raise ValueError("Region reference must point to this dataset") sid = h5r.get_region(args[0], self.id) mshape = sel.guess_shape(sid) if mshape is None: return numpy.array((0,), dtype=new_dtype) if numpy.product(mshape) == 0: return numpy.array(mshape, dtype=new_dtype) out = numpy.empty(mshape, dtype=new_dtype) sid_out = h5s.create_simple(mshape) sid_out.select_all() self.id.read(sid_out, sid, out, mtype) return out # === Check for zero-sized datasets ===== if numpy.product(self.shape) == 0: # These are the only access methods NumPy allows for such objects if args == (Ellipsis,) or args == tuple(): return numpy.empty(self.shape, dtype=new_dtype) # === Scalar dataspaces ================= if self.shape == (): fspace = self.id.get_space() selection = sel2.select_read(fspace, args) arr = numpy.ndarray(selection.mshape, dtype=new_dtype) for mspace, fspace in selection: self.id.read(mspace, fspace, arr, mtype) if len(names) == 1: arr = arr[names[0]] if selection.mshape is None: return arr[()] return arr # === Everything else =================== # Perform the dataspace selection. selection = sel.select(self.shape, args, dsid=self.id) if selection.nselect == 0: return numpy.ndarray(selection.mshape, dtype=new_dtype) # Up-converting to (1,) so that numpy.ndarray correctly creates # np.void rows in case of multi-field dtype. (issue 135) single_element = selection.mshape == () mshape = (1,) if single_element else selection.mshape arr = numpy.ndarray(mshape, new_dtype, order='C') # HDF5 has a bug where if the memory shape has a different rank # than the dataset, the read is very slow if len(mshape) < len(self.shape): # pad with ones mshape = (1,)*(len(self.shape)-len(mshape)) + mshape # Perfom the actual read mspace = h5s.create_simple(mshape) fspace = selection.id self.id.read(mspace, fspace, arr, mtype, dxpl=self._dxpl) # Patch up the output for NumPy if len(names) == 1: arr = arr[names[0]] # Single-field recarray convention if arr.shape == (): arr = numpy.asscalar(arr) if single_element: arr = arr[0] return arr @with_phil def __setitem__(self, args, val): """ Write to the HDF5 dataset from a Numpy array. NumPy's broadcasting rules are honored, for "simple" indexing (slices and integers). For advanced indexing, the shapes must match. """ args = args if isinstance(args, tuple) else (args,) # Sort field indices from the slicing names = tuple(x for x in args if isinstance(x, six.string_types)) args = tuple(x for x in args if not isinstance(x, six.string_types)) if six.PY2: names = tuple(x.encode('utf-8') if isinstance(x, six.text_type) else x for x in names) # Generally we try to avoid converting the arrays on the Python # side. However, for compound literals this is unavoidable. vlen = h5t.check_dtype(vlen=self.dtype) if vlen is not None and vlen not in (bytes, six.text_type): try: val = numpy.asarray(val, dtype=vlen) except ValueError: try: val = numpy.array([numpy.array(x, dtype=vlen) for x in val], dtype=self.dtype) except ValueError: pass if vlen == val.dtype: if val.ndim > 1: tmp = numpy.empty(shape=val.shape[:-1], dtype=object) tmp.ravel()[:] = [i for i in val.reshape( (numpy.product(val.shape[:-1]), val.shape[-1]))] else: tmp = numpy.array([None], dtype=object) tmp[0] = val val = tmp elif self.dtype.kind == "O" or \ (self.dtype.kind == 'V' and \ (not isinstance(val, numpy.ndarray) or val.dtype.kind != 'V') and \ (self.dtype.subdtype == None)): if len(names) == 1 and self.dtype.fields is not None: # Single field selected for write, from a non-array source if not names[0] in self.dtype.fields: raise ValueError("No such field for indexing: %s" % names[0]) dtype = self.dtype.fields[names[0]][0] cast_compound = True else: dtype = self.dtype cast_compound = False val = numpy.asarray(val, dtype=dtype.base, order='C') if cast_compound: val = val.view(numpy.dtype([(names[0], dtype)])) val = val.reshape(val.shape[:len(val.shape) - len(dtype.shape)]) else: val = numpy.asarray(val, order='C') # Check for array dtype compatibility and convert if self.dtype.subdtype is not None: shp = self.dtype.subdtype[1] valshp = val.shape[-len(shp):] if valshp != shp: # Last dimension has to match raise TypeError("When writing to array types, last N dimensions have to match (got %s, but should be %s)" % (valshp, shp,)) mtype = h5t.py_create(numpy.dtype((val.dtype, shp))) mshape = val.shape[0:len(val.shape)-len(shp)] # Make a compound memory type if field-name slicing is required elif len(names) != 0: mshape = val.shape # Catch common errors if self.dtype.fields is None: raise TypeError("Illegal slicing argument (not a compound dataset)") mismatch = [x for x in names if x not in self.dtype.fields] if len(mismatch) != 0: mismatch = ", ".join('"%s"'%x for x in mismatch) raise ValueError("Illegal slicing argument (fields %s not in dataset type)" % mismatch) # Write non-compound source into a single dataset field if len(names) == 1 and val.dtype.fields is None: subtype = h5t.py_create(val.dtype) mtype = h5t.create(h5t.COMPOUND, subtype.get_size()) mtype.insert(self._e(names[0]), 0, subtype) # Make a new source type keeping only the requested fields else: fieldnames = [x for x in val.dtype.names if x in names] # Keep source order mtype = h5t.create(h5t.COMPOUND, val.dtype.itemsize) for fieldname in fieldnames: subtype = h5t.py_create(val.dtype.fields[fieldname][0]) offset = val.dtype.fields[fieldname][1] mtype.insert(self._e(fieldname), offset, subtype) # Use mtype derived from array (let DatasetID.write figure it out) else: mshape = val.shape mtype = None # Perform the dataspace selection selection = sel.select(self.shape, args, dsid=self.id) if selection.nselect == 0: return # Broadcast scalars if necessary. if mshape == () and selection.mshape != (): if self.dtype.subdtype is not None: raise TypeError("Scalar broadcasting is not supported for array dtypes") val2 = numpy.empty(selection.mshape[-1], dtype=val.dtype) val2[...] = val val = val2 mshape = val.shape # Perform the write, with broadcasting # Be careful to pad memory shape with ones to avoid HDF5 chunking # glitch, which kicks in for mismatched memory/file selections if len(mshape) < len(self.shape): mshape_pad = (1,)*(len(self.shape)-len(mshape)) + mshape else: mshape_pad = mshape mspace = h5s.create_simple(mshape_pad, (h5s.UNLIMITED,)*len(mshape_pad)) for fspace in selection.broadcast(mshape): self.id.write(mspace, fspace, val, mtype, dxpl=self._dxpl) def read_direct(self, dest, source_sel=None, dest_sel=None): """ Read data directly from HDF5 into an existing NumPy array. The destination array must be C-contiguous and writable. Selections must be the output of numpy.s_[]. Broadcasting is supported for simple indexing. """ with phil: if is_empty_dataspace(self.id): raise TypeError("Empty datasets have no numpy representation") if source_sel is None: source_sel = sel.SimpleSelection(self.shape) else: source_sel = sel.select(self.shape, source_sel, self.id) # for numpy.s_ fspace = source_sel.id if dest_sel is None: dest_sel = sel.SimpleSelection(dest.shape) else: dest_sel = sel.select(dest.shape, dest_sel, self.id) for mspace in dest_sel.broadcast(source_sel.mshape): self.id.read(mspace, fspace, dest, dxpl=self._dxpl) def write_direct(self, source, source_sel=None, dest_sel=None): """ Write data directly to HDF5 from a NumPy array. The source array must be C-contiguous. Selections must be the output of numpy.s_[]. Broadcasting is supported for simple indexing. """ with phil: if is_empty_dataspace(self.id): raise TypeError("Empty datasets cannot be written to") if source_sel is None: source_sel = sel.SimpleSelection(source.shape) else: source_sel = sel.select(source.shape, source_sel, self.id) # for numpy.s_ mspace = source_sel.id if dest_sel is None: dest_sel = sel.SimpleSelection(self.shape) else: dest_sel = sel.select(self.shape, dest_sel, self.id) for fspace in dest_sel.broadcast(source_sel.mshape): self.id.write(mspace, fspace, source, dxpl=self._dxpl) @with_phil def __array__(self, dtype=None): """ Create a Numpy array containing the whole dataset. DON'T THINK THIS MEANS DATASETS ARE INTERCHANGABLE WITH ARRAYS. For one thing, you have to read the whole dataset everytime this method is called. """ arr = numpy.empty(self.shape, dtype=self.dtype if dtype is None else dtype) # Special case for (0,)*-shape datasets if numpy.product(self.shape) == 0: return arr self.read_direct(arr) return arr @with_phil def __repr__(self): if not self: r = u'' else: if self.name is None: namestr = u'("anonymous")' else: name = pp.basename(pp.normpath(self.name)) namestr = u'"%s"' % (name if name != u'' else u'/') r = u'' % ( namestr, self.shape, self.dtype.str ) if six.PY2: return r.encode('utf8') return r if hasattr(h5d.DatasetID, "refresh"): @with_phil def refresh(self): """ Refresh the dataset metadata by reloading from the file. This is part of the SWMR features and only exist when the HDF5 librarary version >=1.9.178 """ self._id.refresh() if hasattr(h5d.DatasetID, "flush"): @with_phil def flush(self): """ Flush the dataset data and metadata to the file. If the dataset is chunked, raw data chunks are written to the file. This is part of the SWMR features and only exist when the HDF5 librarary version >=1.9.178 """ self._id.flush() h5py-2.7.1/h5py/_hl/compat.py0000644000175000017500000001034413114361073017351 0ustar tcaswelltcaswell00000000000000""" Compatibility module for high-level h5py """ import sys import six WINDOWS_ENCODING = "mbcs" try: from os import fspath except ImportError: def fspath(path): """ Return the string representation of the path. If str or bytes is passed in, it is returned unchanged. This code comes from PEP 519, modified to support earlier versions of python. This is required for python < 3.6. """ if isinstance(path, (six.text_type, six.binary_type)): return path # Work from the object's type to match method resolution of other magic # methods. path_type = type(path) try: return path_type.__fspath__(path) except AttributeError: if hasattr(path_type, '__fspath__'): raise try: import pathlib except ImportError: pass else: if isinstance(path, pathlib.PurePath): return six.text_type(path) raise TypeError("expected str, bytes or os.PathLike object, not " + path_type.__name__) # This is from python 3.5 stdlib (hence lacks PEP 519 changes) # This was introduced into python 3.2, so python < 3.2 does not have this # Effectively, this is only required for python 2.6 and 2.7, and can be removed # once support for them is dropped def _fscodec(): encoding = sys.getfilesystemencoding() if encoding == 'mbcs': errors = 'strict' else: try: from codecs import lookup_error lookup_error('surrogateescape') except LookupError: errors = 'strict' else: errors = 'surrogateescape' def fsencode(filename): """ Encode filename to the filesystem encoding with 'surrogateescape' error handler, return bytes unchanged. On Windows, use 'strict' error handler if the file system encoding is 'mbcs' (which is the default encoding). """ if isinstance(filename, six.binary_type): return filename elif isinstance(filename, six.text_type): return filename.encode(encoding, errors) else: raise TypeError("expect bytes or str, not %s" % type(filename).__name__) def fsdecode(filename): """ Decode filename from the filesystem encoding with 'surrogateescape' error handler, return str unchanged. On Windows, use 'strict' error handler if the file system encoding is 'mbcs' (which is the default encoding). """ if isinstance(filename, six.text_type): return filename elif isinstance(filename, six.binary_type): return filename.decode(encoding, errors) else: raise TypeError("expect bytes or str, not %s" % type(filename).__name__) return fsencode, fsdecode _fsencode, _fsdecode = _fscodec() del _fscodec try: from os import fsencode except ImportError: fsencode = _fsencode try: from os import fsdecode except ImportError: fsdecode = _fsdecode def filename_encode(filename): """ Encode filename for use in the HDF5 library. Due to how HDF5 handles filenames on different systems, this should be called on any filenames passed to the HDF5 library. See the documentation on filenames in h5py for more information. """ filename = fspath(filename) if sys.platform == "win32": if isinstance(filename, six.text_type): return filename.encode(WINDOWS_ENCODING, "strict") return filename return fsencode(filename) def filename_decode(filename): """ Decode filename used by HDF5 library. Due to how HDF5 handles filenames on different systems, this should be called on any filenames passed from the HDF5 library. See the documentation on filenames in h5py for more information. """ if sys.platform == "win32": if isinstance(filename, six.binary_type): return filename.decode(WINDOWS_ENCODING, "strict") elif isinstance(filename, six.text_type): return filename else: raise TypeError("expect bytes or str, not %s" % type(filename).__name__) return fsdecode(filename) h5py-2.7.1/h5py/_hl/files.py0000644000175000017500000002531113146376227017204 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. """ Implements high-level support for HDF5 file objects. """ from __future__ import absolute_import import sys import os from .compat import filename_decode, filename_encode import six from .base import phil, with_phil from .group import Group from .. import h5, h5f, h5p, h5i, h5fd, _objects from .. import version mpi = h5.get_config().mpi hdf5_version = version.hdf5_version_tuple[0:3] swmr_support = False if hdf5_version >= h5.get_config().swmr_min_hdf5_version: swmr_support = True if mpi: import mpi4py libver_dict = {'earliest': h5f.LIBVER_EARLIEST, 'latest': h5f.LIBVER_LATEST} libver_dict_r = dict((y, x) for x, y in six.iteritems(libver_dict)) def make_fapl(driver, libver, **kwds): """ Set up a file access property list """ plist = h5p.create(h5p.FILE_ACCESS) if libver is not None: if libver in libver_dict: low = libver_dict[libver] high = h5f.LIBVER_LATEST else: low, high = (libver_dict[x] for x in libver) plist.set_libver_bounds(low, high) if driver is None or (driver == 'windows' and sys.platform == 'win32'): # Prevent swallowing unused key arguments if kwds: msg = "'{key}' is an invalid keyword argument for this function" \ .format(key=next(iter(kwds))) raise TypeError(msg) return plist if driver == 'sec2': plist.set_fapl_sec2(**kwds) elif driver == 'stdio': plist.set_fapl_stdio(**kwds) elif driver == 'core': plist.set_fapl_core(**kwds) elif driver == 'family': plist.set_fapl_family(memb_fapl=plist.copy(), **kwds) elif driver == 'mpio': kwds.setdefault('info', mpi4py.MPI.Info()) plist.set_fapl_mpio(**kwds) else: raise ValueError('Unknown driver type "%s"' % driver) return plist def make_fid(name, mode, userblock_size, fapl, fcpl=None, swmr=False): """ Get a new FileID by opening or creating a file. Also validates mode argument.""" if userblock_size is not None: if mode in ('r', 'r+'): raise ValueError("User block may only be specified " "when creating a file") try: userblock_size = int(userblock_size) except (TypeError, ValueError): raise ValueError("User block size must be an integer") if fcpl is None: fcpl = h5p.create(h5p.FILE_CREATE) fcpl.set_userblock(userblock_size) if mode == 'r': flags = h5f.ACC_RDONLY if swmr and swmr_support: flags |= h5f.ACC_SWMR_READ fid = h5f.open(name, flags, fapl=fapl) elif mode == 'r+': fid = h5f.open(name, h5f.ACC_RDWR, fapl=fapl) elif mode in ['w-', 'x']: fid = h5f.create(name, h5f.ACC_EXCL, fapl=fapl, fcpl=fcpl) elif mode == 'w': fid = h5f.create(name, h5f.ACC_TRUNC, fapl=fapl, fcpl=fcpl) elif mode == 'a': # Open in append mode (read/write). # If that fails, create a new file only if it won't clobber an # existing one (ACC_EXCL) try: fid = h5f.open(name, h5f.ACC_RDWR, fapl=fapl) except IOError: fid = h5f.create(name, h5f.ACC_EXCL, fapl=fapl, fcpl=fcpl) elif mode is None: # Try to open in append mode (read/write). # If that fails, try readonly, and finally create a new file only # if it won't clobber an existing file (ACC_EXCL). try: fid = h5f.open(name, h5f.ACC_RDWR, fapl=fapl) except IOError: try: fid = h5f.open(name, h5f.ACC_RDONLY, fapl=fapl) except IOError: fid = h5f.create(name, h5f.ACC_EXCL, fapl=fapl, fcpl=fcpl) else: raise ValueError("Invalid mode; must be one of r, r+, w, w-, x, a") try: if userblock_size is not None: existing_fcpl = fid.get_create_plist() if existing_fcpl.get_userblock() != userblock_size: raise ValueError("Requested userblock size (%d) does not match that of existing file (%d)" % (userblock_size, existing_fcpl.get_userblock())) except: fid.close() raise return fid class File(Group): """ Represents an HDF5 file. """ @property def attrs(self): """ Attributes attached to this object """ # hdf5 complains that a file identifier is an invalid location for an # attribute. Instead of self, pass the root group to AttributeManager: from . import attrs with phil: return attrs.AttributeManager(self['/']) @property @with_phil def filename(self): """File name on disk""" return filename_decode(h5f.get_name(self.fid)) @property @with_phil def driver(self): """Low-level HDF5 file driver used to open file""" drivers = {h5fd.SEC2: 'sec2', h5fd.STDIO: 'stdio', h5fd.CORE: 'core', h5fd.FAMILY: 'family', h5fd.WINDOWS: 'windows', h5fd.MPIO: 'mpio', h5fd.MPIPOSIX: 'mpiposix'} return drivers.get(self.fid.get_access_plist().get_driver(), 'unknown') @property @with_phil def mode(self): """ Python mode used to open file """ return {h5f.ACC_RDONLY: 'r', h5f.ACC_RDWR: 'r+'}.get(self.fid.get_intent()) @property @with_phil def fid(self): """File ID (backwards compatibility) """ return self.id @property @with_phil def libver(self): """File format version bounds (2-tuple: low, high)""" bounds = self.id.get_access_plist().get_libver_bounds() return tuple(libver_dict_r[x] for x in bounds) @property @with_phil def userblock_size(self): """ User block size (in bytes) """ fcpl = self.fid.get_create_plist() return fcpl.get_userblock() if mpi and hdf5_version >= (1, 8, 9): @property @with_phil def atomic(self): """ Set/get MPI-IO atomic mode """ return self.id.get_mpi_atomicity() @atomic.setter @with_phil def atomic(self, value): # pylint: disable=missing-docstring self.id.set_mpi_atomicity(value) if swmr_support: @property def swmr_mode(self): """ Controls single-writer multiple-reader mode """ return self._swmr_mode @swmr_mode.setter @with_phil def swmr_mode(self, value): # pylint: disable=missing-docstring if value: self.id.start_swmr_write() self._swmr_mode = True else: raise ValueError("It is not possible to forcibly switch SWMR mode off.") def __init__(self, name, mode=None, driver=None, libver=None, userblock_size=None, swmr=False, **kwds): """Create a new file object. See the h5py user guide for a detailed explanation of the options. name Name of the file on disk. Note: for files created with the 'core' driver, HDF5 still requires this be non-empty. mode r Readonly, file must exist r+ Read/write, file must exist w Create file, truncate if exists w- or x Create file, fail if exists a Read/write if exists, create otherwise (default) driver Name of the driver to use. Legal values are None (default, recommended), 'core', 'sec2', 'stdio', 'mpio'. libver Library version bounds. Currently only the strings 'earliest' and 'latest' are defined. userblock Desired size of user block. Only allowed when creating a new file (mode w, w- or x). swmr Open the file in SWMR read mode. Only used when mode = 'r'. Additional keywords Passed on to the selected file driver. """ if swmr and not swmr_support: raise ValueError("The SWMR feature is not available in this version of the HDF5 library") if isinstance(name, _objects.ObjectID): with phil: fid = h5i.get_file_id(name) else: name = filename_encode(name) with phil: fapl = make_fapl(driver, libver, **kwds) fid = make_fid(name, mode, userblock_size, fapl, swmr=swmr) if swmr_support: self._swmr_mode = False if swmr and mode == 'r': self._swmr_mode = True Group.__init__(self, fid) def close(self): """ Close the file. All open objects become invalid """ with phil: # Check that the file is still open, otherwise skip if self.id.valid: # We have to explicitly murder all open objects related to the file # Close file-resident objects first, then the files. # Otherwise we get errors in MPI mode. id_list = h5f.get_obj_ids(self.id, ~h5f.OBJ_FILE) file_list = h5f.get_obj_ids(self.id, h5f.OBJ_FILE) id_list = [x for x in id_list if h5i.get_file_id(x).id == self.id.id] file_list = [x for x in file_list if h5i.get_file_id(x).id == self.id.id] for id_ in id_list: while id_.valid: h5i.dec_ref(id_) for id_ in file_list: while id_.valid: h5i.dec_ref(id_) self.id.close() _objects.nonlocal_close() def flush(self): """ Tell the HDF5 library to flush its buffers. """ with phil: h5f.flush(self.fid) @with_phil def __enter__(self): return self @with_phil def __exit__(self, *args): if self.id: self.close() @with_phil def __repr__(self): if not self.id: r = u'' else: # Filename has to be forced to Unicode if it comes back bytes # Mode is always a "native" string filename = self.filename if isinstance(filename, bytes): # Can't decode fname filename = filename.decode('utf8', 'replace') r = u'' % (os.path.basename(filename), self.mode) if six.PY2: return r.encode('utf8') return r h5py-2.7.1/h5py/_hl/attrs.py0000644000175000017500000002227213114361073017226 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. """ Implements high-level operations for attributes. Provides the AttributeManager class, available on high-level objects as .attrs. """ from __future__ import absolute_import import numpy import uuid from .. import h5s, h5t, h5a from . import base from .base import phil, with_phil, Empty, is_empty_dataspace from .dataset import readtime_dtype from .datatype import Datatype class AttributeManager(base.MutableMappingHDF5, base.CommonStateObject): """ Allows dictionary-style access to an HDF5 object's attributes. These are created exclusively by the library and are available as a Python attribute at .attrs Like Group objects, attributes provide a minimal dictionary- style interface. Anything which can be reasonably converted to a Numpy array or Numpy scalar can be stored. Attributes are automatically created on assignment with the syntax .attrs[name] = value, with the HDF5 type automatically deduced from the value. Existing attributes are overwritten. To modify an existing attribute while preserving its type, use the method modify(). To specify an attribute of a particular type and shape, use create(). """ def __init__(self, parent): """ Private constructor. """ self._id = parent.id @with_phil def __getitem__(self, name): """ Read the value of an attribute. """ attr = h5a.open(self._id, self._e(name)) if is_empty_dataspace(attr): return Empty(attr.dtype) dtype = readtime_dtype(attr.dtype, []) shape = attr.shape # Do this first, as we'll be fiddling with the dtype for top-level # array types htype = h5t.py_create(dtype) # NumPy doesn't support top-level array types, so we have to "fake" # the correct type and shape for the array. For example, consider # attr.shape == (5,) and attr.dtype == '(3,)f'. Then: if dtype.subdtype is not None: subdtype, subshape = dtype.subdtype shape = attr.shape + subshape # (5, 3) dtype = subdtype # 'f' arr = numpy.ndarray(shape, dtype=dtype, order='C') attr.read(arr, mtype=htype) if len(arr.shape) == 0: return arr[()] return arr @with_phil def __setitem__(self, name, value): """ Set a new attribute, overwriting any existing attribute. The type and shape of the attribute are determined from the data. To use a specific type or shape, or to preserve the type of an attribute, use the methods create() and modify(). """ self.create(name, data=value, dtype=base.guess_dtype(value)) @with_phil def __delitem__(self, name): """ Delete an attribute (which must already exist). """ h5a.delete(self._id, self._e(name)) def create(self, name, data, shape=None, dtype=None): """ Create a new attribute, overwriting any existing attribute. name Name of the new attribute (required) data An array to initialize the attribute (required) shape Shape of the attribute. Overrides data.shape if both are given, in which case the total number of points must be unchanged. dtype Data type of the attribute. Overrides data.dtype if both are given. """ with phil: # First, make sure we have a NumPy array. We leave the data # type conversion for HDF5 to perform. if not isinstance(data, Empty): data = numpy.asarray(data, order='C') if shape is None: shape = data.shape use_htype = None # If a committed type is given, we must use it # in the call to h5a.create. if isinstance(dtype, Datatype): use_htype = dtype.id dtype = dtype.dtype elif dtype is None: dtype = data.dtype else: dtype = numpy.dtype(dtype) # In case a string, e.g. 'i8' is passed original_dtype = dtype # We'll need this for top-level array types # Where a top-level array type is requested, we have to do some # fiddling around to present the data as a smaller array of # subarrays. if dtype.subdtype is not None: subdtype, subshape = dtype.subdtype # Make sure the subshape matches the last N axes' sizes. if shape[-len(subshape):] != subshape: raise ValueError("Array dtype shape %s is incompatible with data shape %s" % (subshape, shape)) # New "advertised" shape and dtype shape = shape[0:len(shape)-len(subshape)] dtype = subdtype # Not an array type; make sure to check the number of elements # is compatible, and reshape if needed. else: if shape is not None and numpy.product(shape) != numpy.product(data.shape): raise ValueError("Shape of new attribute conflicts with shape of data") if shape != data.shape: data = data.reshape(shape) # We need this to handle special string types. if not isinstance(data, Empty): data = numpy.asarray(data, dtype=dtype) # Make HDF5 datatype and dataspace for the H5A calls if use_htype is None: htype = h5t.py_create(original_dtype, logical=True) htype2 = h5t.py_create(original_dtype) # Must be bit-for-bit representation rather than logical else: htype = use_htype htype2 = None if isinstance(data, Empty): space = h5s.create(h5s.NULL) else: space = h5s.create_simple(shape) # This mess exists because you can't overwrite attributes in HDF5. # So we write to a temporary attribute first, and then rename. tempname = uuid.uuid4().hex try: attr = h5a.create(self._id, self._e(tempname), htype, space) except: raise else: try: if not isinstance(data, Empty): attr.write(data, mtype=htype2) except: attr.close() h5a.delete(self._id, self._e(tempname)) raise else: try: # No atomic rename in HDF5 :( if h5a.exists(self._id, self._e(name)): h5a.delete(self._id, self._e(name)) h5a.rename(self._id, self._e(tempname), self._e(name)) except: attr.close() h5a.delete(self._id, self._e(tempname)) raise def modify(self, name, value): """ Change the value of an attribute while preserving its type. Differs from __setitem__ in that if the attribute already exists, its type is preserved. This can be very useful for interacting with externally generated files. If the attribute doesn't exist, it will be automatically created. """ with phil: if not name in self: self[name] = value else: value = numpy.asarray(value, order='C') attr = h5a.open(self._id, self._e(name)) if is_empty_dataspace(attr): raise IOError("Empty attributes can't be modified") # Allow the case of () <-> (1,) if (value.shape != attr.shape) and not \ (numpy.product(value.shape) == 1 and numpy.product(attr.shape) == 1): raise TypeError("Shape of data is incompatible with existing attribute") attr.write(value) @with_phil def __len__(self): """ Number of attributes attached to the object. """ # I expect we will not have more than 2**32 attributes return h5a.get_num_attrs(self._id) def __iter__(self): """ Iterate over the names of attributes. """ with phil: attrlist = [] def iter_cb(name, *args): """ Callback to gather attribute names """ attrlist.append(self._d(name)) h5a.iterate(self._id, iter_cb) for name in attrlist: yield name @with_phil def __contains__(self, name): """ Determine if an attribute exists, by name. """ return h5a.exists(self._id, self._e(name)) @with_phil def __repr__(self): if not self._id: return "" return "" % id(self._id) h5py-2.7.1/h5py/_hl/filters.py0000644000175000017500000002527013114361073017542 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. """ Implements support for HDF5 compression filters via the high-level interface. The following types of filter are available: "gzip" Standard DEFLATE-based compression, at integer levels from 0 to 9. Built-in to all public versions of HDF5. Use this if you want a decent-to-good ratio, good portability, and don't mind waiting. "lzf" Custom compression filter for h5py. This filter is much, much faster than gzip (roughly 10x in compression vs. gzip level 4, and 3x faster in decompressing), but at the cost of a worse compression ratio. Use this if you want cheap compression and portability is not a concern. "szip" Access to the HDF5 SZIP encoder. SZIP is a non-mainstream compression format used in space science on integer and float datasets. SZIP is subject to license requirements, which means the encoder is not guaranteed to be always available. However, it is also much faster than gzip. The following constants in this module are also useful: decode Tuple of available filter names for decoding encode Tuple of available filter names for encoding """ from __future__ import absolute_import, division import numpy as np from .. import h5z, h5p, h5d _COMP_FILTERS = {'gzip': h5z.FILTER_DEFLATE, 'szip': h5z.FILTER_SZIP, 'lzf': h5z.FILTER_LZF, 'shuffle': h5z.FILTER_SHUFFLE, 'fletcher32': h5z.FILTER_FLETCHER32, 'scaleoffset': h5z.FILTER_SCALEOFFSET } DEFAULT_GZIP = 4 DEFAULT_SZIP = ('nn', 8) def _gen_filter_tuples(): """ Bootstrap function to figure out what filters are available. """ dec = [] enc = [] for name, code in _COMP_FILTERS.items(): if h5z.filter_avail(code): info = h5z.get_filter_info(code) if info & h5z.FILTER_CONFIG_ENCODE_ENABLED: enc.append(name) if info & h5z.FILTER_CONFIG_DECODE_ENABLED: dec.append(name) return tuple(dec), tuple(enc) decode, encode = _gen_filter_tuples() def generate_dcpl(shape, dtype, chunks, compression, compression_opts, shuffle, fletcher32, maxshape, scaleoffset): """ Generate a dataset creation property list. Undocumented and subject to change without warning. """ if shape == (): if any((chunks, compression, compression_opts, shuffle, fletcher32, scaleoffset is not None)): raise TypeError("Scalar datasets don't support chunk/filter options") if maxshape and maxshape != (): raise TypeError("Scalar datasets cannot be extended") return h5p.create(h5p.DATASET_CREATE) def rq_tuple(tpl, name): """ Check if chunks/maxshape match dataset rank """ if tpl in (None, True): return try: tpl = tuple(tpl) except TypeError: raise TypeError('"%s" argument must be None or a sequence object' % name) if len(tpl) != len(shape): raise ValueError('"%s" must have same rank as dataset shape' % name) rq_tuple(chunks, 'chunks') rq_tuple(maxshape, 'maxshape') if compression is not None: if compression not in encode and not isinstance(compression, int): raise ValueError('Compression filter "%s" is unavailable' % compression) if compression == 'gzip': if compression_opts is None: gzip_level = DEFAULT_GZIP elif compression_opts in range(10): gzip_level = compression_opts else: raise ValueError("GZIP setting must be an integer from 0-9, not %r" % compression_opts) elif compression == 'lzf': if compression_opts is not None: raise ValueError("LZF compression filter accepts no options") elif compression == 'szip': if compression_opts is None: compression_opts = DEFAULT_SZIP err = "SZIP options must be a 2-tuple ('ec'|'nn', even integer 0-32)" try: szmethod, szpix = compression_opts except TypeError: raise TypeError(err) if szmethod not in ('ec', 'nn'): raise ValueError(err) if not (0= 0') if dtype.kind == 'f': if scaleoffset is True: raise ValueError('integer scaleoffset must be provided for ' 'floating point types') elif dtype.kind in ('u', 'i'): if scaleoffset is True: scaleoffset = h5z.SO_INT_MINBITS_DEFAULT else: raise TypeError('scale/offset filter only supported for integer ' 'and floating-point types') # Scale/offset following fletcher32 in the filter chain will (almost?) # always triggera a read error, as most scale/offset settings are # lossy. Since fletcher32 must come first (see comment below) we # simply prohibit the combination of fletcher32 and scale/offset. if fletcher32: raise ValueError('fletcher32 cannot be used with potentially lossy' ' scale/offset filter') # End argument validation if (chunks is True) or \ (chunks is None and any((shuffle, fletcher32, compression, maxshape, scaleoffset is not None))): chunks = guess_chunk(shape, maxshape, dtype.itemsize) if maxshape is True: maxshape = (None,)*len(shape) plist = h5p.create(h5p.DATASET_CREATE) if chunks is not None: plist.set_chunk(chunks) plist.set_fill_time(h5d.FILL_TIME_ALLOC) # prevent resize glitch # MUST be first, to prevent 1.6/1.8 compatibility glitch if fletcher32: plist.set_fletcher32() # scale-offset must come before shuffle and compression if scaleoffset is not None: if dtype.kind in ('u', 'i'): plist.set_scaleoffset(h5z.SO_INT, scaleoffset) else: # dtype.kind == 'f' plist.set_scaleoffset(h5z.SO_FLOAT_DSCALE, scaleoffset) if shuffle: plist.set_shuffle() if compression == 'gzip': plist.set_deflate(gzip_level) elif compression == 'lzf': plist.set_filter(h5z.FILTER_LZF, h5z.FLAG_OPTIONAL) elif compression == 'szip': opts = {'ec': h5z.SZIP_EC_OPTION_MASK, 'nn': h5z.SZIP_NN_OPTION_MASK} plist.set_szip(opts[szmethod], szpix) elif isinstance(compression, int): if not h5z.filter_avail(compression): raise ValueError("Unknown compression filter number: %s" % compression) plist.set_filter(compression, h5z.FLAG_OPTIONAL, compression_opts) return plist def get_filters(plist): """ Extract a dictionary of active filters from a DCPL, along with their settings. Undocumented and subject to change without warning. """ filters = {h5z.FILTER_DEFLATE: 'gzip', h5z.FILTER_SZIP: 'szip', h5z.FILTER_SHUFFLE: 'shuffle', h5z.FILTER_FLETCHER32: 'fletcher32', h5z.FILTER_LZF: 'lzf', h5z.FILTER_SCALEOFFSET: 'scaleoffset'} pipeline = {} nfilters = plist.get_nfilters() for i in range(nfilters): code, _, vals, _ = plist.get_filter(i) if code == h5z.FILTER_DEFLATE: vals = vals[0] # gzip level elif code == h5z.FILTER_SZIP: mask, pixels = vals[0:2] if mask & h5z.SZIP_EC_OPTION_MASK: mask = 'ec' elif mask & h5z.SZIP_NN_OPTION_MASK: mask = 'nn' else: raise TypeError("Unknown SZIP configuration") vals = (mask, pixels) elif code == h5z.FILTER_LZF: vals = None else: if len(vals) == 0: vals = None pipeline[filters.get(code, str(code))] = vals return pipeline CHUNK_BASE = 16*1024 # Multiplier by which chunks are adjusted CHUNK_MIN = 8*1024 # Soft lower limit (8k) CHUNK_MAX = 1024*1024 # Hard upper limit (1M) def guess_chunk(shape, maxshape, typesize): """ Guess an appropriate chunk layout for a dataset, given its shape and the size of each element in bytes. Will allocate chunks only as large as MAX_SIZE. Chunks are generally close to some power-of-2 fraction of each axis, slightly favoring bigger values for the last index. Undocumented and subject to change without warning. """ # pylint: disable=unused-argument # For unlimited dimensions we have to guess 1024 shape = tuple((x if x!=0 else 1024) for i, x in enumerate(shape)) ndims = len(shape) if ndims == 0: raise ValueError("Chunks not allowed for scalar datasets.") chunks = np.array(shape, dtype='=f8') if not np.all(np.isfinite(chunks)): raise ValueError("Illegal value in chunk tuple") # Determine the optimal chunk size in bytes using a PyTables expression. # This is kept as a float. dset_size = np.product(chunks)*typesize target_size = CHUNK_BASE * (2**np.log10(dset_size/(1024.*1024))) if target_size > CHUNK_MAX: target_size = CHUNK_MAX elif target_size < CHUNK_MIN: target_size = CHUNK_MIN idx = 0 while True: # Repeatedly loop over the axes, dividing them by 2. Stop when: # 1a. We're smaller than the target chunk size, OR # 1b. We're within 50% of the target chunk size, AND # 2. The chunk is smaller than the maximum chunk size chunk_bytes = np.product(chunks)*typesize if (chunk_bytes < target_size or \ abs(chunk_bytes-target_size)/target_size < 0.5) and \ chunk_bytes < CHUNK_MAX: break if np.product(chunks) == 1: break # Element size larger than CHUNK_MAX chunks[idx%ndims] = np.ceil(chunks[idx%ndims] / 2.0) idx += 1 return tuple(int(x) for x in chunks) h5py-2.7.1/h5py/_errors.pyx0000644000175000017500000001377313146406212017200 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. # Python-style minor error classes. If the minor error code matches an entry # in this dict, the generated exception will be used. _minor_table = { H5E_SEEKERROR: IOError, # Seek failed H5E_READERROR: IOError, # Read failed H5E_WRITEERROR: IOError, # Write failed H5E_CLOSEERROR: IOError, # Close failed H5E_OVERFLOW: IOError, # Address overflowed H5E_FCNTL: IOError, # File control (fcntl) failed H5E_FILEEXISTS: IOError, # File already exists H5E_FILEOPEN: IOError, # File already open H5E_CANTCREATE: IOError, # Unable to create file H5E_CANTOPENFILE: IOError, # Unable to open file H5E_CANTCLOSEFILE: IOError, # Unable to close file H5E_NOTHDF5: IOError, # Not an HDF5 file H5E_BADFILE: ValueError, # Bad file ID accessed H5E_TRUNCATED: IOError, # File has been truncated H5E_MOUNT: IOError, # File mount error H5E_NOFILTER: IOError, # Requested filter is not available H5E_CALLBACK: IOError, # Callback failed H5E_CANAPPLY: IOError, # Error from filter 'can apply' callback H5E_SETLOCAL: IOError, # Error from filter 'set local' callback H5E_NOENCODER: IOError, # Filter present but encoding disabled H5E_BADATOM: ValueError, # Unable to find atom information (already closed?) H5E_BADGROUP: ValueError, # Unable to find ID group information H5E_BADSELECT: ValueError, # Invalid selection (hyperslabs) H5E_UNINITIALIZED: ValueError, # Information is uinitialized H5E_UNSUPPORTED: NotImplementedError, # Feature is unsupported H5E_NOTFOUND: KeyError, # Object not found H5E_CANTINSERT: ValueError, # Unable to insert object H5E_BADTYPE: TypeError, # Inappropriate type H5E_BADRANGE: ValueError, # Out of range H5E_BADVALUE: ValueError, # Bad value H5E_EXISTS: ValueError, # Object already exists H5E_ALREADYEXISTS: ValueError, # Object already exists, part II H5E_CANTCONVERT: TypeError, # Can't convert datatypes H5E_CANTDELETE: KeyError, # Can't delete message H5E_CANTOPENOBJ: KeyError, H5E_CANTMOVE: ValueError, # Can't move a link } # "Fudge" table to accomodate annoying inconsistencies in HDF5's use # of the minor error codes. If a (major, minor) entry appears here, # it will override any entry in the minor error table. _exact_table = { (H5E_CACHE, H5E_BADVALUE): IOError, # obj create w/o write intent 1.8 (H5E_RESOURCE, H5E_CANTINIT): IOError, # obj create w/o write intent 1.6 (H5E_INTERNAL, H5E_SYSERRSTR): IOError, # e.g. wrong file permissions (H5E_DATATYPE, H5E_CANTINIT): TypeError, # No conversion path (H5E_DATASET, H5E_CANTINIT): ValueError, # bad param for dataset setup (H5E_ARGS, H5E_CANTINIT): TypeError, # Illegal operation on object (H5E_SYM, H5E_CANTINIT): ValueError, # Object already exists/1.8 (H5E_ARGS, H5E_BADTYPE): ValueError, # Invalid location in file (H5E_REFERENCE, H5E_CANTINIT): ValueError, # Dereferencing invalid ref } cdef struct err_data_t: H5E_error_t err int n cdef herr_t walk_cb(int n, H5E_error_t *desc, void *e): cdef err_data_t *ee = e ee[0].err.maj_num = desc[0].maj_num ee[0].err.min_num = desc[0].min_num ee[0].err.desc = desc[0].desc ee[0].n = n cdef int set_exception() except -1: cdef err_data_t err cdef const char *desc = NULL # Note: HDF5 forbids freeing these cdef const char *desc_bottom = NULL # First, extract the major & minor error codes from the top of the # stack, along with the top-level error description err.n = -1 if H5Ewalk(H5E_WALK_UPWARD, walk_cb, &err) < 0: raise RuntimeError("Failed to walk error stack") if err.n < 0: # No HDF5 exception information found return 0 eclass = _minor_table.get(err.err.min_num, RuntimeError) eclass = _exact_table.get((err.err.maj_num, err.err.min_num), eclass) desc = err.err.desc if desc is NULL: raise RuntimeError("Failed to extract top-level error description") # Second, retrieve the bottom-most error description for additional info err.n = -1 if H5Ewalk(H5E_WALK_DOWNWARD, walk_cb, &err) < 0: raise RuntimeError("Failed to walk error stack") desc_bottom = err.err.desc if desc_bottom is NULL: raise RuntimeError("Failed to extract bottom-level error description") msg = ("%s (%s)" % (desc.decode('utf-8').capitalize(), desc_bottom.decode('utf-8'))).encode('utf-8') # Finally, set the exception. We do this with the Python C function # so that the traceback doesn't point here. PyErr_SetString(eclass, msg) return 1 cdef extern from "stdio.h": void *stderr def silence_errors(): """ Disable HDF5's automatic error printing in this thread """ if H5Eset_auto(NULL, NULL) < 0: raise RuntimeError("Failed to disable automatic error printing") def unsilence_errors(): """ Re-enable HDF5's automatic error printing in this thread """ if H5Eset_auto( H5Eprint, stderr) < 0: raise RuntimeError("Failed to enable automatic error printing") cdef err_cookie set_error_handler(err_cookie handler): # Note: exceptions here will be printed instead of raised. cdef err_cookie old_handler if H5Eget_auto(&old_handler.func, &old_handler.data) < 0: raise RuntimeError("Failed to retrieve old handler") if H5Eset_auto(handler.func, handler.data) < 0: raise RuntimeError("Failed to install new handler") return old_handler h5py-2.7.1/h5py/h5z.pyx0000644000175000017500000000535413025343121016222 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. """ Filter API and constants. """ from ._objects import phil, with_phil # === Public constants and data structures ==================================== FILTER_LZF = H5PY_FILTER_LZF FILTER_ERROR = H5Z_FILTER_ERROR FILTER_NONE = H5Z_FILTER_NONE FILTER_ALL = H5Z_FILTER_ALL FILTER_DEFLATE = H5Z_FILTER_DEFLATE FILTER_SHUFFLE = H5Z_FILTER_SHUFFLE FILTER_FLETCHER32 = H5Z_FILTER_FLETCHER32 FILTER_SZIP = H5Z_FILTER_SZIP FILTER_NBIT = H5Z_FILTER_NBIT FILTER_SCALEOFFSET = H5Z_FILTER_SCALEOFFSET FILTER_RESERVED = H5Z_FILTER_RESERVED FILTER_MAX = H5Z_FILTER_MAX FLAG_DEFMASK = H5Z_FLAG_DEFMASK FLAG_MANDATORY = H5Z_FLAG_MANDATORY FLAG_OPTIONAL = H5Z_FLAG_OPTIONAL FLAG_INVMASK = H5Z_FLAG_INVMASK FLAG_REVERSE = H5Z_FLAG_REVERSE FLAG_SKIP_EDC = H5Z_FLAG_SKIP_EDC SZIP_ALLOW_K13_OPTION_MASK = H5_SZIP_ALLOW_K13_OPTION_MASK #1 SZIP_CHIP_OPTION_MASK = H5_SZIP_CHIP_OPTION_MASK #2 SZIP_EC_OPTION_MASK = H5_SZIP_EC_OPTION_MASK #4 SZIP_NN_OPTION_MASK = H5_SZIP_NN_OPTION_MASK #32 SZIP_MAX_PIXELS_PER_BLOCK = H5_SZIP_MAX_PIXELS_PER_BLOCK #32 SO_FLOAT_DSCALE = H5Z_SO_FLOAT_DSCALE SO_FLOAT_ESCALE = H5Z_SO_FLOAT_ESCALE SO_INT = H5Z_SO_INT SO_INT_MINBITS_DEFAULT = H5Z_SO_INT_MINBITS_DEFAULT FILTER_CONFIG_ENCODE_ENABLED = H5Z_FILTER_CONFIG_ENCODE_ENABLED FILTER_CONFIG_DECODE_ENABLED = H5Z_FILTER_CONFIG_DECODE_ENABLED ERROR_EDC = H5Z_ERROR_EDC DISABLE_EDC = H5Z_DISABLE_EDC ENABLE_EDC = H5Z_ENABLE_EDC NO_EDC = H5Z_NO_EDC # === Filter API ============================================================= @with_phil def filter_avail(int filter_code): """(INT filter_code) => BOOL Determine if the given filter is available to the library. The filter code should be one of: - FILTER_DEFLATE - FILTER_SHUFFLE - FILTER_FLETCHER32 - FILTER_SZIP """ return H5Zfilter_avail(filter_code) @with_phil def get_filter_info(int filter_code): """(INT filter_code) => INT filter_flags Retrieve a bitfield with information about the given filter. The filter code should be one of: - FILTER_DEFLATE - FILTER_SHUFFLE - FILTER_FLETCHER32 - FILTER_SZIP Valid bitmasks for use with the returned bitfield are: - FILTER_CONFIG_ENCODE_ENABLED - FILTER_CONFIG_DECODE_ENABLED """ cdef unsigned int flags H5Zget_filter_info(filter_code, &flags) return flags def _register_lzf(): register_lzf() h5py-2.7.1/h5py/h5t.pxd0000644000175000017500000000260513025343121016163 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. from defs cimport * from _objects cimport class ObjectID cdef class TypeID(ObjectID): cdef object py_dtype(self) # --- Top-level classes --- cdef class TypeArrayID(TypeID): pass cdef class TypeOpaqueID(TypeID): pass cdef class TypeStringID(TypeID): # Both vlen and fixed-len strings pass cdef class TypeVlenID(TypeID): # Non-string vlens pass cdef class TypeTimeID(TypeID): pass cdef class TypeBitfieldID(TypeID): pass cdef class TypeReferenceID(TypeID): pass # --- Numeric atomic types --- cdef class TypeAtomicID(TypeID): pass cdef class TypeIntegerID(TypeAtomicID): pass cdef class TypeFloatID(TypeAtomicID): pass # --- Enums & compound types --- cdef class TypeCompositeID(TypeID): pass cdef class TypeEnumID(TypeCompositeID): cdef int enum_convert(self, long long *buf, int reverse) except -1 cdef class TypeCompoundID(TypeCompositeID): pass # === C API for other modules ================================================= cpdef TypeID typewrap(hid_t id_) cdef hid_t H5PY_OBJ cpdef TypeID py_create(object dtype, bint logical=*, bint aligned=*) h5py-2.7.1/h5py/h5l.pxd0000644000175000017500000000051413025343121016150 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. from defs cimport * cdef class LinkProxy: cdef hid_t id h5py-2.7.1/h5py/h5ac.pxd0000644000175000017500000000054313025343121016302 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. from defs cimport * cdef class CacheConfig: cdef H5AC_cache_config_t cache_config h5py-2.7.1/h5py/version.py0000644000175000017500000000312513152363105017010 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. """ Versioning module for h5py. """ from __future__ import absolute_import from collections import namedtuple from . import h5 as _h5 import sys import numpy # All should be integers, except pre, as validating versions is more than is # needed for our use case _H5PY_VERSION_CLS = namedtuple("_H5PY_VERSION_CLS", "major minor bugfix pre post dev") hdf5_built_version_tuple = _h5.HDF5_VERSION_COMPILED_AGAINST version_tuple = _H5PY_VERSION_CLS(2, 7, 1, None, None, None) version = "{0.major:d}.{0.minor:d}.{0.bugfix:d}".format(version_tuple) if version_tuple.pre is not None: version += version_tuple.pre if version_tuple.post is not None: version += ".post{0.post:d}".format(version_tuple) if version_tuple.dev is not None: version += ".dev{0.dev:d}".format(version_tuple) hdf5_version_tuple = _h5.get_libversion() hdf5_version = "%d.%d.%d" % hdf5_version_tuple api_version_tuple = (1,8) api_version = "%d.%d" % api_version_tuple info = """\ Summary of the h5py configuration --------------------------------- h5py %(h5py)s HDF5 %(hdf5)s Python %(python)s sys.platform %(platform)s sys.maxsize %(maxsize)s numpy %(numpy)s """ % { 'h5py': version, 'hdf5': hdf5_version, 'python': sys.version, 'platform': sys.platform, 'maxsize': sys.maxsize, 'numpy': numpy.__version__ } h5py-2.7.1/h5py/h5.pxd0000644000175000017500000000110413025343121015770 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. from defs cimport * cdef class H5PYConfig: cdef readonly object _r_name cdef readonly object _i_name cdef readonly object _f_name cdef readonly object _t_name cdef readonly object API_16 cdef readonly object API_18 cdef readonly object _bytestrings cpdef H5PYConfig get_config() h5py-2.7.1/h5py/h5l.pyx0000644000175000017500000002340613025343121016202 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. """ API for the "H5L" family of link-related operations. Defines the class LinkProxy, which comes attached to GroupID objects as .links. """ from _objects cimport pdefault from h5p cimport PropID from h5g cimport GroupID from utils cimport emalloc, efree from ._objects import phil, with_phil # === Public constants ======================================================== TYPE_HARD = H5L_TYPE_HARD TYPE_SOFT = H5L_TYPE_SOFT TYPE_EXTERNAL = H5L_TYPE_EXTERNAL cdef class LinkInfo: cdef H5L_info_t infostruct property type: """ Integer type code for link (h5l.TYPE_*) """ def __get__(self): return self.infostruct.type property corder_valid: """ Indicates if the creation order is valid """ def __get__(self): return self.infostruct.corder_valid property corder: """ Creation order """ def __get__(self): return self.infostruct.corder property cset: """ Integer type code for character set (h5t.CSET_*) """ def __get__(self): return self.infostruct.cset property u: """ Either the address of a hard link or the size of a soft/UD link """ def __get__(self): if self.infostruct.type == H5L_TYPE_HARD: return self.infostruct.u.address else: return self.infostruct.u.val_size cdef class _LinkVisitor: """ Helper class for iteration callback """ cdef object func cdef object retval cdef LinkInfo info def __init__(self, func): self.func = func self.retval = None self.info = LinkInfo() cdef herr_t cb_link_iterate(hid_t grp, const char* name, const H5L_info_t *istruct, void* data) except 2: # Standard iteration callback for iterate/visit routines cdef _LinkVisitor it = <_LinkVisitor?>data it.info.infostruct = istruct[0] it.retval = it.func(name, it.info) if (it.retval is None) or (not it.retval): return 0 return 1 cdef herr_t cb_link_simple(hid_t grp, const char* name, const H5L_info_t *istruct, void* data) except 2: # Simplified iteration callback which only provides the name cdef _LinkVisitor it = <_LinkVisitor?>data it.retval = it.func(name) if (it.retval is None) or (not it.retval): return 0 return 1 cdef class LinkProxy: """ Proxy class which provides access to the HDF5 "H5L" API. These come attached to GroupID objects as "obj.links". Since every H5L function operates on at least one group, the methods provided operate on their parent group identifier. For example:: >>> g = h5g.open(fid, '/') >>> g.links.exists("MyGroup") True >>> g.links.exists("FooBar") False * Hashable: No * Equality: Undefined You will note that this class does *not* inherit from ObjectID. """ def __init__(self, hid_t id_): # The identifier in question is the hid_t for the parent GroupID. # We "borrow" this reference. self.id = id_ def __richcmp__(self, object other, int how): return NotImplemented def __hash__(self): raise TypeError("Link proxies are unhashable; use the parent group instead.") @with_phil def create_hard(self, char* new_name, GroupID cur_loc not None, char* cur_name, PropID lcpl=None, PropID lapl=None): """ (STRING new_name, GroupID cur_loc, STRING cur_name, PropID lcpl=None, PropID lapl=None) Create a new hard link in this group pointing to an existing link in another group. """ H5Lcreate_hard(cur_loc.id, cur_name, self.id, new_name, pdefault(lcpl), pdefault(lapl)) @with_phil def create_soft(self, char* new_name, char* target, PropID lcpl=None, PropID lapl=None): """(STRING new_name, STRING target, PropID lcpl=None, PropID lapl=None) Create a new soft link in this group, with the given string value. The link target does not need to exist. """ H5Lcreate_soft(target, self.id, new_name, pdefault(lcpl), pdefault(lapl)) @with_phil def create_external(self, char* link_name, char* file_name, char* obj_name, PropID lcpl=None, PropID lapl=None): """(STRING link_name, STRING file_name, STRING obj_name, PropLCID lcpl=None, PropLAID lapl=None) Create a new external link, pointing to an object in another file. """ H5Lcreate_external(file_name, obj_name, self.id, link_name, pdefault(lcpl), pdefault(lapl)) @with_phil def get_val(self, char* name, PropID lapl=None): """(STRING name, PropLAID lapl=None) => STRING or TUPLE(file, obj) Get the string value of a soft link, or a 2-tuple representing the contents of an external link. """ cdef hid_t plist = pdefault(lapl) cdef H5L_info_t info cdef size_t buf_size cdef char* buf = NULL cdef char* ext_file_name = NULL cdef char* ext_obj_name = NULL cdef unsigned int wtf = 0 H5Lget_info(self.id, name, &info, plist) if info.type != H5L_TYPE_SOFT and info.type != H5L_TYPE_EXTERNAL: raise TypeError("Link must be either a soft or external link") buf_size = info.u.val_size buf = emalloc(buf_size) try: H5Lget_val(self.id, name, buf, buf_size, plist) if info.type == H5L_TYPE_SOFT: py_retval = buf else: H5Lunpack_elink_val(buf, buf_size, &wtf, &ext_file_name, &ext_obj_name) py_retval = (bytes(ext_file_name), bytes(ext_obj_name)) finally: efree(buf) return py_retval @with_phil def move(self, char* src_name, GroupID dst_loc not None, char* dst_name, PropID lcpl=None, PropID lapl=None): """ (STRING src_name, GroupID dst_loc, STRING dst_name) Move a link to a new location in the file. """ H5Lmove(self.id, src_name, dst_loc.id, dst_name, pdefault(lcpl), pdefault(lapl)) @with_phil def exists(self, char* name, PropID lapl=None): """ (STRING name, PropID lapl=None) => BOOL Check if a link of the specified name exists in this group. """ return (H5Lexists(self.id, name, pdefault(lapl))) @with_phil def get_info(self, char* name, int index=-1, *, PropID lapl=None): """(STRING name=, INT index=, **kwds) => LinkInfo instance Get information about a link, either by name or its index. Keywords: """ cdef LinkInfo info = LinkInfo() H5Lget_info(self.id, name, &info.infostruct, pdefault(lapl)) return info @with_phil def visit(self, object func, *, int idx_type=H5_INDEX_NAME, int order=H5_ITER_NATIVE, char* obj_name='.', PropID lapl=None, bint info=0): """(CALLABLE func, **kwds) => Iterate a function or callable object over all groups below this one. Your callable should conform to the signature:: func(STRING name) => Result or if the keyword argument "info" is True:: func(STRING name, LinkInfo info) => Result Returning None or a logical False continues iteration; returning anything else aborts iteration and returns that value. BOOL info (False) Provide a LinkInfo instance to callback STRING obj_name (".") Visit this subgroup instead PropLAID lapl (None) Link access property list for "obj_name" INT idx_type (h5.INDEX_NAME) INT order (h5.ITER_NATIVE) """ cdef _LinkVisitor it = _LinkVisitor(func) cdef H5L_iterate_t cfunc if info: cfunc = cb_link_iterate else: cfunc = cb_link_simple H5Lvisit_by_name(self.id, obj_name, idx_type, order, cfunc, it, pdefault(lapl)) return it.retval @with_phil def iterate(self, object func, *, int idx_type=H5_INDEX_NAME, int order=H5_ITER_NATIVE, char* obj_name='.', PropID lapl=None, bint info=0, hsize_t idx=0): """(CALLABLE func, **kwds) => , Iterate a function or callable object over all groups in this one. Your callable should conform to the signature:: func(STRING name) => Result or if the keyword argument "info" is True:: func(STRING name, LinkInfo info) => Result Returning None or a logical False continues iteration; returning anything else aborts iteration and returns that value. BOOL info (False) Provide a LinkInfo instance to callback STRING obj_name (".") Visit this subgroup instead PropLAID lapl (None) Link access property list for "obj_name" INT idx_type (h5.INDEX_NAME) INT order (h5.ITER_NATIVE) hsize_t idx (0) The index to start iterating at """ cdef _LinkVisitor it = _LinkVisitor(func) cdef H5L_iterate_t cfunc if info: cfunc = cb_link_iterate else: cfunc = cb_link_simple H5Literate_by_name(self.id, obj_name, idx_type, order, &idx, cfunc, it, pdefault(lapl)) return it.retval, idx h5py-2.7.1/h5py/h5fd.pxd0000644000175000017500000000544313025343121016314 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. # This file contains code or comments from the HDF5 library. See the file # licenses/hdf5.txt for the full HDF5 software license. from defs cimport * cdef extern from "hdf5.h": ctypedef enum H5FD_mem_t: H5FD_MEM_NOLIST = -1, H5FD_MEM_DEFAULT = 0, H5FD_MEM_SUPER = 1, H5FD_MEM_BTREE = 2, H5FD_MEM_DRAW = 3, H5FD_MEM_GHEAP = 4, H5FD_MEM_LHEAP = 5, H5FD_MEM_OHDR = 6, H5FD_MEM_NTYPES # HDF5 uses a clever scheme wherein these are actually init() calls # Hopefully Pyrex won't have a problem with this. # Thankfully they are defined but -1 if unavailable hid_t H5FD_CORE hid_t H5FD_FAMILY # hid_t H5FD_GASS not in 1.8.X hid_t H5FD_LOG hid_t H5FD_MPIO # hid_t H5FD_MPIPOSIX removed in 1.8.13 hid_t H5FD_MULTI hid_t H5FD_SEC2 hid_t H5FD_STDIO IF UNAME_SYSNAME == "Windows": hid_t H5FD_WINDOWS hid_t H5FD_MPIO_COLLECTIVE hid_t H5FD_MPIO_INDEPENDENT int H5FD_LOG_LOC_READ # 0x0001 int H5FD_LOG_LOC_WRITE # 0x0002 int H5FD_LOG_LOC_SEEK # 0x0004 int H5FD_LOG_LOC_IO # (H5FD_LOG_LOC_READ|H5FD_LOG_LOC_WRITE|H5FD_LOG_LOC_SEEK) # /* Flags for tracking number of times each byte is read/written */ int H5FD_LOG_FILE_READ # 0x0008 int H5FD_LOG_FILE_WRITE # 0x0010 int H5FD_LOG_FILE_IO # (H5FD_LOG_FILE_READ|H5FD_LOG_FILE_WRITE) # /* Flag for tracking "flavor" (type) of information stored at each byte */ int H5FD_LOG_FLAVOR # 0x0020 # /* Flags for tracking total number of reads/writes/seeks */ int H5FD_LOG_NUM_READ # 0x0040 int H5FD_LOG_NUM_WRITE # 0x0080 int H5FD_LOG_NUM_SEEK # 0x0100 int H5FD_LOG_NUM_IO # (H5FD_LOG_NUM_READ|H5FD_LOG_NUM_WRITE|H5FD_LOG_NUM_SEEK) # /* Flags for tracking time spent in open/read/write/seek/close */ int H5FD_LOG_TIME_OPEN # 0x0200 # /* Not implemented yet */ int H5FD_LOG_TIME_READ # 0x0400 # /* Not implemented yet */ int H5FD_LOG_TIME_WRITE # 0x0800 # /* Partially implemented (need to track total time) */ int H5FD_LOG_TIME_SEEK # 0x1000 # /* Partially implemented (need to track total time & track time for seeks during reading) */ int H5FD_LOG_TIME_CLOSE # 0x2000 # /* Fully implemented */ int H5FD_LOG_TIME_IO # (H5FD_LOG_TIME_OPEN|H5FD_LOG_TIME_READ|H5FD_LOG_TIME_WRITE|H5FD_LOG_TIME_SEEK|H5FD_LOG_TIME_CLOSE) # /* Flag for tracking allocation of space in file */ int H5FD_LOG_ALLOC # 0x4000 int H5FD_LOG_ALL # (H5FD_LOG_ALLOC|H5FD_LOG_TIME_IO|H5FD_LOG_NUM_IO|H5FD_LOG_FLAVOR|H5FD_LOG_FILE_IO|H5FD_LOG_LOC_IO) h5py-2.7.1/h5py/h5p.pxd0000644000175000017500000000355613025343121016165 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. from defs cimport * from _objects cimport ObjectID # --- Base classes --- cdef class PropID(ObjectID): """ Base class for all property lists """ pass cdef class PropClassID(PropID): """ Represents an HDF5 property list class. These can be either (locked) library-defined classes or user-created classes. """ pass cdef class PropInstanceID(PropID): """ Represents an instance of a property list class (i.e. an actual list which can be passed on to other API functions). """ pass cdef class PropCreateID(PropInstanceID): """ Base class for all object creation lists. Also includes string character set methods. """ pass cdef class PropCopyID(PropInstanceID): """ Property list for copying objects (as in h5o.copy) """ # --- Object creation --- cdef class PropOCID(PropCreateID): """ Object creation property list """ pass cdef class PropDCID(PropOCID): """ Dataset creation property list """ pass cdef class PropFCID(PropCreateID): """ File creation property list """ pass # --- Object access --- cdef class PropFAID(PropInstanceID): """ File access property list """ pass cdef class PropDXID(PropInstanceID): """ Dataset transfer property list """ pass # --- New in 1.8 --- cdef class PropLCID(PropCreateID): """ Link creation property list """ pass cdef class PropLAID(PropInstanceID): """ Link access property list """ cdef char* _buf cdef class PropGCID(PropOCID): """ Group creation property list """ pass cdef hid_t pdefault(PropID pid) cdef object propwrap(hid_t id_in) h5py-2.7.1/h5py/h5.pyx0000644000175000017500000001156113114357361016037 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. include "config.pxi" from defs cimport * from ._objects import phil, with_phil ITER_INC = H5_ITER_INC # Increasing order ITER_DEC = H5_ITER_DEC # Decreasing order ITER_NATIVE = H5_ITER_NATIVE # No particular order, whatever is fastest INDEX_NAME = H5_INDEX_NAME # Index on names INDEX_CRT_ORDER = H5_INDEX_CRT_ORDER # Index on creation order HDF5_VERSION_COMPILED_AGAINST = HDF5_VERSION class ByteStringContext(object): def __init__(self): self._readbytes = False def __bool__(self): return self._readbytes def __nonzero__(self): return self.__bool__() def __enter__(self): self._readbytes = True def __exit__(self, *args): self._readbytes = False cdef class H5PYConfig: """ Provides runtime access to global library settings. You retrieve the master copy of this object by calling h5py.get_config(). complex_names (tuple, r/w) Settable 2-tuple controlling how complex numbers are saved. Defaults to ('r','i'). bool_names (tuple, r/w) Settable 2-tuple controlling the HDF5 enum names used for boolean values. Defaults to ('FALSE', 'TRUE') for values 0 and 1. """ def __init__(self): self._r_name = b'r' self._i_name = b'i' self._f_name = b'FALSE' self._t_name = b'TRUE' self._bytestrings = ByteStringContext() property complex_names: """ Settable 2-tuple controlling how complex numbers are saved. Format is (real_name, imag_name), defaulting to ('r','i'). """ def __get__(self): with phil: import sys def handle_val(val): if sys.version[0] == '3': return val.decode('utf8') return val return (handle_val(self._r_name), handle_val(self._i_name)) def __set__(self, val): with phil: def handle_val(val): if isinstance(val, unicode): return val.encode('utf8') elif isinstance(val, bytes): return val else: return bytes(val) try: if len(val) != 2: raise TypeError() r = handle_val(val[0]) i = handle_val(val[1]) except Exception: raise TypeError("complex_names must be a length-2 sequence of strings (real, img)") self._r_name = r self._i_name = i property bool_names: """ Settable 2-tuple controlling HDF5 ENUM names for boolean types. Format is (false_name, real_name), defaulting to ('FALSE', 'TRUE'). """ def __get__(self): with phil: return (self._f_name, self._t_name) def __set__(self, val): with phil: try: if len(val) != 2: raise TypeError() f = str(val[0]) t = str(val[1]) except Exception: raise TypeError("bool_names must be a length-2 sequence of of names (false, true)") self._f_name = f self._t_name = t property read_byte_strings: """ Returns a context manager which forces all strings to be returned as byte strings. """ def __get__(self): with phil: return self._bytestrings property mpi: """ Boolean indicating if Parallel HDF5 is available """ def __get__(self): IF MPI: return True ELSE: return False property swmr_min_hdf5_version: """ Tuple indicating the minimum HDF5 version required for SWMR features""" def __get__(self): return SWMR_MIN_HDF5_VERSION property vds_min_hdf5_version: """Tuple indicating the minimum HDF5 version required for virtual dataset (VDS) features""" def __get__(self): return VDS_MIN_HDF5_VERSION cdef H5PYConfig cfg = H5PYConfig() cpdef H5PYConfig get_config(): """() => H5PYConfig Get a reference to the global library configuration object. """ return cfg @with_phil def get_libversion(): """ () => TUPLE (major, minor, release) Retrieve the HDF5 library version as a 3-tuple. """ cdef unsigned int major cdef unsigned int minor cdef unsigned int release cdef herr_t retval H5get_libversion(&major, &minor, &release) return (major, minor, release) h5py-2.7.1/h5py/h5g.pyx0000644000175000017500000003462213025343121016177 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. """ Low-level HDF5 "H5G" group interface. """ include "config.pxi" # Compile-time imports from _objects cimport pdefault from utils cimport emalloc, efree from h5p cimport PropID cimport _hdf5 # to implement container testing for 1.6 from _errors cimport set_error_handler, err_cookie from h5py import _objects from ._objects import phil, with_phil # === Public constants and data structures ==================================== # Enumerated object types for groups "H5G_obj_t" UNKNOWN = H5G_UNKNOWN LINK = H5G_LINK GROUP = H5G_GROUP DATASET = H5G_DATASET TYPE = H5G_TYPE # Enumerated link types "H5G_link_t" LINK_ERROR = H5G_LINK_ERROR LINK_HARD = H5G_LINK_HARD LINK_SOFT = H5G_LINK_SOFT cdef class GroupStat: """Represents the H5G_stat_t structure containing group member info. Fields (read-only): * fileno: 2-tuple uniquely identifying the current file * objno: 2-tuple uniquely identifying this object * nlink: Number of hard links to this object * mtime: Modification time of this object * linklen: Length of the symbolic link name, or 0 if not a link. "Uniquely identifying" means unique among currently open files, not universally unique. * Hashable: Yes * Equality: Yes """ cdef H5G_stat_t infostruct property fileno: def __get__(self): return (self.infostruct.fileno[0], self.infostruct.fileno[1]) property objno: def __get__(self): return (self.infostruct.objno[0], self.infostruct.objno[1]) property nlink: def __get__(self): return self.infostruct.nlink property type: def __get__(self): return self.infostruct.type property mtime: def __get__(self): return self.infostruct.mtime property linklen: def __get__(self): return self.infostruct.linklen def _hash(self): return hash((self.fileno, self.objno, self.nlink, self.type, self.mtime, self.linklen)) cdef class GroupIter: """ Iterator over the names of group members. After this iterator is exhausted, it releases its reference to the group ID. """ cdef unsigned long idx cdef unsigned long nobjs cdef GroupID grp cdef list names def __init__(self, GroupID grp not None): self.idx = 0 self.grp = grp self.nobjs = grp.get_num_objs() self.names = [] def __iter__(self): return self def __next__(self): if self.idx == self.nobjs: self.grp = None self.names = None raise StopIteration if self.idx == 0: self.grp.links.iterate(self.names.append) retval = self.names[self.idx] self.idx += 1 return retval # === Basic group management ================================================== @with_phil def open(ObjectID loc not None, char* name): """(ObjectID loc, STRING name) => GroupID Open an existing HDF5 group, attached to some other group. """ return GroupID(H5Gopen(loc.id, name)) @with_phil def create(ObjectID loc not None, object name, PropID lcpl=None, PropID gcpl=None): """(ObjectID loc, STRING name or None, PropLCID lcpl=None, PropGCID gcpl=None) => GroupID Create a new group, under a given parent group. If name is None, an anonymous group will be created in the file. """ cdef hid_t gid cdef char* cname = NULL if name is not None: cname = name if cname != NULL: gid = H5Gcreate2(loc.id, cname, pdefault(lcpl), pdefault(gcpl), H5P_DEFAULT) else: gid = H5Gcreate_anon(loc.id, pdefault(gcpl), H5P_DEFAULT) return GroupID(gid) cdef class _GroupVisitor: cdef object func cdef object retval def __init__(self, func): self.func = func self.retval = None cdef herr_t cb_group_iter(hid_t gid, char *name, void* vis_in) except 2: cdef _GroupVisitor vis = <_GroupVisitor>vis_in vis.retval = vis.func(name) if vis.retval is not None: return 1 return 0 @with_phil def iterate(GroupID loc not None, object func, int startidx=0, *, char* obj_name='.'): """ (GroupID loc, CALLABLE func, UINT startidx=0, **kwds) => Return value from func Iterate a callable (function, method or callable object) over the members of a group. Your callable should have the signature:: func(STRING name) => Result Returning None continues iteration; returning anything else aborts iteration and returns that value. Keywords: STRING obj_name (".") Iterate over this subgroup instead """ if startidx < 0: raise ValueError("Starting index must be non-negative") cdef int i = startidx cdef _GroupVisitor vis = _GroupVisitor(func) H5Giterate(loc.id, obj_name, &i, cb_group_iter, vis) return vis.retval @with_phil def get_objinfo(ObjectID obj not None, object name=b'.', int follow_link=1): """(ObjectID obj, STRING name='.', BOOL follow_link=True) => GroupStat object Obtain information about a named object. If "name" is provided, "obj" is taken to be a GroupID object containing the target. The return value is a GroupStat object; see that class's docstring for a description of its attributes. If follow_link is True (default) and the object is a symbolic link, the information returned describes its target. Otherwise the information describes the link itself. """ cdef GroupStat statobj statobj = GroupStat() cdef char* _name _name = name H5Gget_objinfo(obj.id, _name, follow_link, &statobj.infostruct) return statobj # === Group member management ================================================= cdef class GroupID(ObjectID): """ Represents an HDF5 group identifier Python extensions: __contains__ Test for group member ("if name in grpid") __iter__ Get an iterator over member names __len__ Number of members in this group; len(grpid) = N If HDF5 1.8.X is used, the attribute "links" contains a proxy object providing access to the H5L family of routines. See the docs for h5py.h5l.LinkProxy for more information. * Hashable: Yes, unless anonymous * Equality: True HDF5 identity unless anonymous """ def __init__(self, hid_t id_): with phil: import h5l self.links = h5l.LinkProxy(id_) @with_phil def link(self, char* current_name, char* new_name, int link_type=H5G_LINK_HARD, GroupID remote=None): """(STRING current_name, STRING new_name, INT link_type=LINK_HARD, GroupID remote=None) Create a new hard or soft link. current_name identifies the link target (object the link will point to). The new link is identified by new_name and (optionally) another group "remote". Link types are: LINK_HARD Hard link to existing object (default) LINK_SOFT Symbolic link; link target need not exist. """ cdef hid_t remote_id if remote is None: remote_id = self.id else: remote_id = remote.id H5Glink2(self.id, current_name, link_type, remote_id, new_name) @with_phil def unlink(self, char* name): """(STRING name) Remove a link to an object from this group. """ H5Gunlink(self.id, name) @with_phil def move(self, char* current_name, char* new_name, GroupID remote=None): """(STRING current_name, STRING new_name, GroupID remote=None) Relink an object. current_name identifies the object. new_name and (optionally) another group "remote" determine where it should be moved. """ cdef hid_t remote_id if remote is None: remote_id = self.id else: remote_id = remote.id H5Gmove2(self.id, current_name, remote_id, new_name) @with_phil def get_num_objs(self): """() => INT number_of_objects Get the number of objects directly attached to a given group. """ cdef hsize_t size H5Gget_num_objs(self.id, &size) return size @with_phil def get_objname_by_idx(self, hsize_t idx): """(INT idx) => STRING Get the name of a group member given its zero-based index. """ cdef int size cdef char* buf buf = NULL size = H5Gget_objname_by_idx(self.id, idx, NULL, 0) buf = emalloc(sizeof(char)*(size+1)) try: H5Gget_objname_by_idx(self.id, idx, buf, size+1) pystring = buf return pystring finally: efree(buf) @with_phil def get_objtype_by_idx(self, hsize_t idx): """(INT idx) => INT object_type_code Get the type of an object attached to a group, given its zero-based index. Possible return values are: - LINK - GROUP - DATASET - TYPE """ return H5Gget_objtype_by_idx(self.id, idx) @with_phil def get_linkval(self, char* name): """(STRING name) => STRING link_value Retrieve the value (target name) of a symbolic link. Limited to 2048 characters on Windows. """ cdef char* value cdef H5G_stat_t statbuf value = NULL H5Gget_objinfo(self.id, name, 0, &statbuf) if statbuf.type != H5G_LINK: raise ValueError('"%s" is not a symbolic link.' % name) IF UNAME_SYSNAME == "Windows": linklen = 2049 # Windows statbuf.linklen seems broken ELSE: linklen = statbuf.linklen+1 value = emalloc(sizeof(char)*linklen) try: H5Gget_linkval(self.id, name, linklen, value) value[linklen-1] = c'\0' # in case HDF5 doesn't null terminate on Windows pyvalue = value return pyvalue finally: efree(value) @with_phil def set_comment(self, char* name, char* comment): """(STRING name, STRING comment) Set the comment on a group member. """ H5Gset_comment(self.id, name, comment) @with_phil def get_comment(self, char* name): """(STRING name) => STRING comment Retrieve the comment for a group member. """ cdef int cmnt_len cdef char* cmnt cmnt = NULL cmnt_len = H5Gget_comment(self.id, name, 0, NULL) assert cmnt_len >= 0 cmnt = emalloc(sizeof(char)*(cmnt_len+1)) try: H5Gget_comment(self.id, name, cmnt_len+1, cmnt) py_cmnt = cmnt return py_cmnt finally: efree(cmnt) # === Special methods ===================================================== def __contains__(self, name): """(STRING name) Determine if a group member of the given name is present """ cdef err_cookie old_handler cdef err_cookie new_handler cdef herr_t retval new_handler.func = NULL new_handler.data = NULL if not self: return False IF HDF5_VERSION >= (1, 8, 5): # New system is more robust but requires H5Oexists_by_name with phil: return _path_valid(self, name) ELSE: with phil: old_handler = set_error_handler(new_handler) retval = _hdf5.H5Gget_objinfo(self.id, name, 0, NULL) set_error_handler(old_handler) return bool(retval >= 0) def __iter__(self): """ Return an iterator over the names of group members. """ with phil: return GroupIter(self) def __len__(self): """ Number of group members """ cdef hsize_t size with phil: H5Gget_num_objs(self.id, &size) return size IF HDF5_VERSION >= (1, 8, 5): @with_phil def _path_valid(GroupID grp not None, object path not None, PropID lapl=None): """ Determine if *path* points to an object in the file. If *path* represents an external or soft link, the link's validity is not checked. """ import h5o if isinstance(path, bytes): path = path.decode('utf-8') else: path = unicode(path) # Empty names are not allowed by HDF5 if len(path) == 0: return False # Note: we cannot use pp.normpath as it resolves ".." components, # which don't exist in HDF5 path_parts = path.split('/') # Absolute path (started with slash) if path_parts[0] == '': current_loc = h5o.open(grp, b'/', lapl=lapl) else: current_loc = grp # HDF5 ignores duplicate or trailing slashes path_parts = [x for x in path_parts if x != ''] # Special case: path was entirely composed of slashes! if len(path_parts) == 0: path_parts = ['.'] # i.e. the root group path_parts = [x.encode('utf-8') for x in path_parts] nparts = len(path_parts) for idx, p in enumerate(path_parts): # Special case; '.' always refers to the present group if p == b'.': continue # Is there any kind of link by that name in this group? if not current_loc.links.exists(p, lapl=lapl): return False # If we're at the last link in the chain, we're done. # We don't check to see if the last part points to a valid object; # it's enough that it exists. if idx == nparts - 1: return True # Otherwise, does the link point to a real object? if not h5o.exists_by_name(current_loc, p, lapl=lapl): return False # Is that object a group? next_loc = h5o.open(current_loc, p, lapl=lapl) info = h5o.get_info(next_loc) if info.type != H5O_TYPE_GROUP: return False # Go into that group current_loc = next_loc return True h5py-2.7.1/h5py/_proxy.pyx0000644000175000017500000002603613025343121017034 0ustar tcaswelltcaswell00000000000000# cython: profile=False # This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. """ Proxy functions for read/write, to work around the HDF5 bogus type issue. """ cdef enum copy_dir: H5PY_SCATTER = 0, H5PY_GATHER cdef herr_t attr_rw(hid_t attr, hid_t mtype, void *progbuf, int read) except -1: cdef htri_t need_bkg cdef hid_t atype = -1 cdef hid_t aspace = -1 cdef hsize_t npoints cdef size_t msize, asize cdef void* conv_buf = NULL cdef void* back_buf = NULL try: atype = H5Aget_type(attr) if not (needs_proxy(atype) or needs_proxy(mtype)): if read: H5Aread(attr, mtype, progbuf) else: H5Awrite(attr, mtype, progbuf) else: asize = H5Tget_size(atype) msize = H5Tget_size(mtype) aspace = H5Aget_space(attr) npoints = H5Sget_select_npoints(aspace) conv_buf = create_buffer(asize, msize, npoints) if read: need_bkg = needs_bkg_buffer(atype, mtype) else: need_bkg = needs_bkg_buffer(mtype, atype) if need_bkg: back_buf = malloc(msize*npoints) memcpy(back_buf, progbuf, msize*npoints) if read: H5Aread(attr, atype, conv_buf) H5Tconvert(atype, mtype, npoints, conv_buf, back_buf, H5P_DEFAULT) memcpy(progbuf, conv_buf, msize*npoints) else: memcpy(conv_buf, progbuf, msize*npoints) H5Tconvert(mtype, atype, npoints, conv_buf, back_buf, H5P_DEFAULT) H5Awrite(attr, atype, conv_buf) H5Dvlen_reclaim(atype, aspace, H5P_DEFAULT, conv_buf) finally: free(conv_buf) free(back_buf) if atype > 0: H5Tclose(atype) if aspace > 0: H5Sclose(aspace) return 0 # ============================================================================= # Proxy functions to safely release the GIL around read/write operations cdef herr_t H5PY_H5Dread(hid_t dset, hid_t mtype, hid_t mspace, hid_t fspace, hid_t dxpl, void* buf) except -1: cdef herr_t retval #with nogil: retval = H5Dread(dset, mtype, mspace, fspace, dxpl, buf) if retval < 0: return -1 return retval cdef herr_t H5PY_H5Dwrite(hid_t dset, hid_t mtype, hid_t mspace, hid_t fspace, hid_t dxpl, void* buf) except -1: cdef herr_t retval #with nogil: retval = H5Dwrite(dset, mtype, mspace, fspace, dxpl, buf) if retval < 0: return -1 return retval # ============================================================================= # Proxy for vlen buf workaround cdef herr_t dset_rw(hid_t dset, hid_t mtype, hid_t mspace, hid_t fspace, hid_t dxpl, void* progbuf, int read) except -1: cdef htri_t need_bkg cdef hid_t dstype = -1 # Dataset datatype cdef hid_t rawdstype = -1 cdef hid_t dspace = -1 # Dataset dataspace cdef hid_t cspace = -1 # Temporary contiguous dataspaces cdef void* back_buf = NULL cdef void* conv_buf = NULL cdef hsize_t npoints try: # Issue 372: when a compound type is involved, using the dataset type # may result in uninitialized data being sent to H5Tconvert for fields # not present in the memory type. Limit the type used for the dataset # to only those fields present in the memory type. We can't use the # memory type directly because of course that triggers HDFFV-1063. if (H5Tget_class(mtype) == H5T_COMPOUND) and (not read): rawdstype = H5Dget_type(dset) dstype = make_reduced_type(mtype, rawdstype) H5Tclose(rawdstype) else: dstype = H5Dget_type(dset) if not (needs_proxy(dstype) or needs_proxy(mtype)): if read: H5PY_H5Dread(dset, mtype, mspace, fspace, dxpl, progbuf) else: H5PY_H5Dwrite(dset, mtype, mspace, fspace, dxpl, progbuf) else: if mspace == H5S_ALL and fspace != H5S_ALL: mspace = fspace elif mspace != H5S_ALL and fspace == H5S_ALL: fspace = mspace elif mspace == H5S_ALL and fspace == H5S_ALL: fspace = mspace = dspace = H5Dget_space(dset) npoints = H5Sget_select_npoints(mspace) cspace = H5Screate_simple(1, &npoints, NULL) conv_buf = create_buffer(H5Tget_size(dstype), H5Tget_size(mtype), npoints) # Only create a (contiguous) backing buffer if absolutely # necessary. Note this buffer always has memory type. if read: need_bkg = needs_bkg_buffer(dstype, mtype) else: need_bkg = needs_bkg_buffer(mtype, dstype) if need_bkg: back_buf = create_buffer(H5Tget_size(dstype), H5Tget_size(mtype), npoints) h5py_copy(mtype, mspace, back_buf, progbuf, H5PY_GATHER) if read: H5PY_H5Dread(dset, dstype, cspace, fspace, dxpl, conv_buf) H5Tconvert(dstype, mtype, npoints, conv_buf, back_buf, dxpl) h5py_copy(mtype, mspace, conv_buf, progbuf, H5PY_SCATTER) else: h5py_copy(mtype, mspace, conv_buf, progbuf, H5PY_GATHER) H5Tconvert(mtype, dstype, npoints, conv_buf, back_buf, dxpl) H5PY_H5Dwrite(dset, dstype, cspace, fspace, dxpl, conv_buf) H5Dvlen_reclaim(dstype, cspace, H5P_DEFAULT, conv_buf) finally: free(back_buf) free(conv_buf) if dstype > 0: H5Tclose(dstype) if dspace > 0: H5Sclose(dspace) if cspace > 0: H5Sclose(cspace) return 0 cdef hid_t make_reduced_type(hid_t mtype, hid_t dstype): # Go through dstype, pick out the fields which also appear in mtype, and # return a new compound type with the fields packed together # See also: issue 372 cdef hid_t newtype, temptype cdef hsize_t newtype_size, offset cdef char* member_name = NULL # Make a list of all names in the memory type. mtype_fields = [] for idx in xrange(H5Tget_nmembers(mtype)): member_name = H5Tget_member_name(mtype, idx) try: mtype_fields.append(member_name) finally: free(member_name) member_name = NULL # First pass: add up the sizes of matching fields so we know how large a # type to make newtype_size = 0 for idx in xrange(H5Tget_nmembers(dstype)): member_name = H5Tget_member_name(dstype, idx) try: if member_name not in mtype_fields: continue temptype = H5Tget_member_type(dstype, idx) newtype_size += H5Tget_size(temptype) H5Tclose(temptype) finally: free(member_name) member_name = NULL newtype = H5Tcreate(H5T_COMPOUND, newtype_size) # Second pass: pick out the matching fields and pack them in the new type offset = 0 for idx in xrange(H5Tget_nmembers(dstype)): member_name = H5Tget_member_name(dstype, idx) try: if member_name not in mtype_fields: continue temptype = H5Tget_member_type(dstype, idx) H5Tinsert(newtype, member_name, offset, temptype) offset += H5Tget_size(temptype) H5Tclose(temptype) finally: free(member_name) member_name = NULL return newtype cdef void* create_buffer(size_t ipt_size, size_t opt_size, size_t nl) except NULL: cdef size_t final_size cdef void* buf if ipt_size >= opt_size: final_size = ipt_size*nl else: final_size = opt_size*nl buf = malloc(final_size) if buf == NULL: raise MemoryError("Failed to allocate conversion buffer") return buf # ============================================================================= # Scatter/gather routines ctypedef struct h5py_scatter_t: size_t i size_t elsize void* buf cdef herr_t h5py_scatter_cb(void* elem, hid_t type_id, unsigned ndim, const hsize_t *point, void *operator_data) except -1: cdef h5py_scatter_t* info = operator_data memcpy(elem, (info[0].buf)+((info[0].i)*(info[0].elsize)), info[0].elsize) info[0].i += 1 return 0 cdef herr_t h5py_gather_cb(void* elem, hid_t type_id, unsigned ndim, const hsize_t *point, void *operator_data) except -1: cdef h5py_scatter_t* info = operator_data memcpy((info[0].buf)+((info[0].i)*(info[0].elsize)), elem, info[0].elsize) info[0].i += 1 return 0 # Copy between a contiguous and non-contiguous buffer, with the layout # of the latter specified by a dataspace selection. cdef herr_t h5py_copy(hid_t tid, hid_t space, void* contig, void* noncontig, copy_dir op) except -1: cdef h5py_scatter_t info cdef hsize_t elsize elsize = H5Tget_size(tid) info.i = 0 info.elsize = elsize info.buf = contig if op == H5PY_SCATTER: H5Diterate(noncontig, tid, space, h5py_scatter_cb, &info) elif op == H5PY_GATHER: H5Diterate(noncontig, tid, space, h5py_gather_cb, &info) else: raise RuntimeError("Illegal direction") return 0 # ============================================================================= # VLEN support routines cdef htri_t needs_bkg_buffer(hid_t src, hid_t dst) except -1: cdef H5T_cdata_t *info = NULL if H5Tdetect_class(src, H5T_COMPOUND) or H5Tdetect_class(dst, H5T_COMPOUND): return 1 try: H5Tfind(src, dst, &info) except: print "Failed to find converter for %s -> %s" % (H5Tget_size(src), H5Tget_tag(dst)) raise if info[0].need_bkg == H5T_BKG_YES: return 1 return 0 # Determine if the given type requires proxy buffering cdef htri_t needs_proxy(hid_t tid) except -1: cdef H5T_class_t cls cdef hid_t supertype cdef int i, n cdef htri_t result cls = H5Tget_class(tid) if cls == H5T_VLEN or cls == H5T_REFERENCE: return 1 elif cls == H5T_STRING: return H5Tis_variable_str(tid) elif cls == H5T_ARRAY: supertype = H5Tget_super(tid) try: return needs_proxy(supertype) finally: H5Tclose(supertype) elif cls == H5T_COMPOUND: n = H5Tget_nmembers(tid) for i from 0<=i 0: return 1 finally: H5Tclose(supertype) return 0 return 0 h5py-2.7.1/h5py/__init__.py0000644000175000017500000000620113143100476017060 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. """ This is the h5py package, a Python interface to the HDF5 scientific data format. """ from __future__ import absolute_import from warnings import warn as _warn # --- Library setup ----------------------------------------------------------- # When importing from the root of the unpacked tarball or git checkout, # Python sees the "h5py" source directory and tries to load it, which fails. # We tried working around this by using "package_dir" but that breaks Cython. try: from . import _errors except ImportError: import os.path as _op if _op.exists(_op.join(_op.dirname(__file__), '..', 'setup.py')): raise ImportError("You cannot import h5py from inside the install directory.\nChange to another directory first.") else: raise _errors.silence_errors() from ._conv import register_converters as _register_converters _register_converters() from .h5z import _register_lzf _register_lzf() # --- Public API -------------------------------------------------------------- from . import h5a, h5d, h5ds, h5f, h5fd, h5g, h5r, h5s, h5t, h5p, h5z from ._hl import filters from ._hl.base import is_hdf5, HLObject, Empty from ._hl.files import File from ._hl.group import Group, SoftLink, ExternalLink, HardLink from ._hl.dataset import Dataset from ._hl.datatype import Datatype from ._hl.attrs import AttributeManager from .h5 import get_config from .h5r import Reference, RegionReference from .h5t import special_dtype, check_dtype from . import version from .version import version as __version__ from .tests import run_tests if version.hdf5_version_tuple != version.hdf5_built_version_tuple: _warn(("h5py is running against HDF5 {0} when it was built against {1}, " "this may cause problems").format( '{0}.{1}.{2}'.format(*version.hdf5_version_tuple), '{0}.{1}.{2}'.format(*version.hdf5_built_version_tuple) )) def enable_ipython_completer(): """ Call this from an interactive IPython session to enable tab-completion of group and attribute names. """ import sys if 'IPython' in sys.modules: ip_running = False try: from IPython.core.interactiveshell import InteractiveShell ip_running = InteractiveShell.initialized() except ImportError: # support weakref(obj), where obj is an ObjectID instance. # Objects are added only via ObjectID.__cinit__, and removed only by # ObjectID.__dealloc__. cdef dict registry = {} @with_phil def print_reg(): import h5py refs = registry.values() objs = [r() for r in refs] none = len([x for x in objs if x is None]) files = len([x for x in objs if isinstance(x, h5py.h5f.FileID)]) groups = len([x for x in objs if isinstance(x, h5py.h5g.GroupID)]) print "REGISTRY: %d | %d None | %d FileID | %d GroupID" % (len(objs), none, files, groups) @with_phil def nonlocal_close(): """ Find dead ObjectIDs and set their integer identifiers to 0. """ cdef ObjectID obj cdef list reg_ids # create a cached list of ids whilst the gc is disabled to avoid hitting # the cyclic gc while iterating through the registry dict gc.disable() try: reg_ids = list(registry) finally: gc.enable() for python_id in reg_ids: ref = registry.get(python_id) # registry dict has changed underneath us, skip to next item if ref is None: continue obj = ref() # Object died while walking the registry list, presumably because # the cyclic GC kicked in. if obj is None: continue # Locked objects are immortal, as they generally are provided by # the HDF5 library itself (property list classes, etc.). if obj.locked: continue # Invalid object; set obj.id = 0 so it doesn't become a zombie if not H5Iis_valid(obj.id): IF DEBUG_ID: print("NONLOCAL - invalidating %d of kind %s HDF5 id %d" % (python_id, type(obj), obj.id) ) obj.id = 0 continue # --- End registry code ------------------------------------------------------- cdef class ObjectID: """ Represents an HDF5 identifier. """ property fileno: def __get__(self): cdef H5G_stat_t stat with _phil: H5Gget_objinfo(self.id, '.', 0, &stat) return (stat.fileno[0], stat.fileno[1]) property valid: def __get__(self): return is_h5py_obj_valid(self) def __cinit__(self, id_): with _phil: self.id = id_ self.locked = 0 self._pyid = id(self) IF DEBUG_ID: print("CINIT - registering %d of kind %s HDF5 id %d" % (self._pyid, type(self), self.id)) registry[self._pyid] = weakref.ref(self) def __dealloc__(self): with _phil: IF DEBUG_ID: print("DEALLOC - unregistering %d HDF5 id %d" % (self._pyid, self.id)) if is_h5py_obj_valid(self) and (not self.locked): if H5Idec_ref(self.id) < 0: warnings.warn( "Reference counting issue with HDF5 id {}".format( self.id ) ) if self._pyid is not None: del registry[self._pyid] def _close(self): """ Manually close this object. """ with _phil: IF DEBUG_ID: print("CLOSE - %d HDF5 id %d" % (self._pyid, self.id)) if is_h5py_obj_valid(self) and (not self.locked): if H5Idec_ref(self.id) < 0: warnings.warn( "Reference counting issue with HDF5 id {}".format( self.id ) ) self.id = 0 def close(self): """ Close this identifier. """ # Note this is the default close method. Subclasses, e.g. FileID, # which have nonlocal effects should override this. self._close() def __nonzero__(self): return self.valid def __copy__(self): cdef ObjectID cpy with _phil: cpy = type(self)(self.id) H5Iinc_ref(cpy.id) return cpy def __richcmp__(self, object other, int how): """ Default comparison mechanism for HDF5 objects (equal/not-equal) Default equality testing: 1. Objects which are not both ObjectIDs are unequal 2. Objects with the same HDF5 ID number are always equal 3. Objects which hash the same are equal """ cdef bint equal = 0 with _phil: if how != 2 and how != 3: return NotImplemented if isinstance(other, ObjectID): if self.id == other.id: equal = 1 else: try: equal = hash(self) == hash(other) except TypeError: pass if how == 2: return equal return not equal def __hash__(self): """ Default hashing mechanism for HDF5 objects Default hashing strategy: 1. Try to hash based on the object's fileno and objno records 2. If (1) succeeds, cache the resulting value 3. If (1) fails, raise TypeError """ cdef H5G_stat_t stat with _phil: if self._hash is None: try: H5Gget_objinfo(self.id, '.', 0, &stat) self._hash = hash((stat.fileno[0], stat.fileno[1], stat.objno[0], stat.objno[1])) except Exception: raise TypeError("Objects of class %s cannot be hashed" % self.__class__.__name__) return self._hash cdef hid_t pdefault(ObjectID pid): if pid is None: return H5P_DEFAULT return pid.id cdef int is_h5py_obj_valid(ObjectID obj): """ Check that h5py object is valid, i.e. HDF5 object wrapper is valid and HDF5 object is valid """ # MUST BE CALLABLE AT ANY TIME, CANNOT USE PROPERTIES ETC. AS PER # http://cython.readthedocs.io/en/latest/src/userguide/special_methods.html # Locked objects are always valid, regardless of obj.id if obj.locked: return True # Former zombie object if obj.id == 0: return False # Ask HDF5. Note that H5Iis_valid only works for "user" # identifiers, hence the above checks. with _phil: return H5Iis_valid(obj.id) h5py-2.7.1/h5py/utils.pxd0000644000175000017500000000153113025343121016620 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. from defs cimport * from numpy cimport ndarray cdef void* emalloc(size_t size) except? NULL cdef void efree(void* ptr) cpdef int check_numpy_read(ndarray arr, hid_t space_id=*) except -1 cpdef int check_numpy_write(ndarray arr, hid_t space_id=*) except -1 cdef int convert_tuple(object tuple, hsize_t *dims, hsize_t rank) except -1 cdef object convert_dims(hsize_t* dims, hsize_t rank) cdef int require_tuple(object tpl, int none_allowed, int size, char* name) except -1 cdef object create_numpy_hsize(int rank, hsize_t* dims) cdef object create_hsize_array(object arr) h5py-2.7.1/h5py/_conv.pyx0000644000175000017500000007461413143100507016625 0ustar tcaswelltcaswell00000000000000# cython: profile=False # This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. """ Low-level type-conversion routines. """ from h5r cimport Reference, RegionReference, hobj_ref_t, hdset_reg_ref_t from h5t cimport H5PY_OBJ, typewrap, py_create, TypeID cimport numpy as np from libc.stdlib cimport realloc # Initialization np.import_array() # Minimal interface for Python objects immune to Cython refcounting cdef extern from "Python.h": # From Cython declarations ctypedef int PyTypeObject ctypedef struct PyObject: Py_ssize_t ob_refcnt PyTypeObject *ob_type PyObject* PyBytes_FromString(char* str) except NULL int PyBytes_CheckExact(PyObject* str) except * int PyBytes_Size(PyObject* obj) except * PyObject* PyString_AsDecodedObject(PyObject* s, char *encoding, char *errors) except NULL PyObject* PyUnicode_DecodeUTF8(char *s, Py_ssize_t size, char *errors) except NULL int PyUnicode_CheckExact(PyObject* str) except * PyObject* PyUnicode_AsUTF8String(PyObject* s) except NULL PyObject* PyObject_Str(PyObject* obj) except NULL #PyObject* (PyObject* obj) except NULL char* PyBytes_AsString(PyObject* obj) except NULL PyObject* Py_None void Py_INCREF(PyObject* obj) void Py_DECREF(PyObject* obj) void Py_XDECREF(PyObject* obj) cdef object objectify(PyObject* o): Py_INCREF(o) return o cdef extern from "numpy/arrayobject.h": PyTypeObject PyArray_Type object PyArray_NewFromDescr(PyTypeObject* subtype, np.dtype descr, int nd, np.npy_intp* dims, np.npy_intp* strides, void* data, int flags, object obj) ctypedef int (*conv_operator_t)(void* ipt, void* opt, void* bkg, void* priv) except -1 ctypedef herr_t (*init_operator_t)(hid_t src, hid_t dst, void** priv) except -1 # Generic conversion callback # # The actual conversion routines are one-liners which plug the appropriate # operator callback into this function. This prevents us from having to # repeat all the conversion boilerplate for every single callback. # # While this is somewhat slower than a custom function, the added overhead is # likely small compared to the cost of the Python-side API calls required to # implement the conversions. cdef herr_t generic_converter(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nl, size_t buf_stride, size_t bkg_stride, void *buf_i, void *bkg_i, hid_t dxpl, conv_operator_t op, init_operator_t initop, H5T_bkg_t need_bkg) except -1: cdef int command = cdata[0].command cdef conv_size_t *sizes cdef int i cdef char* buf = buf_i cdef char* bkg = bkg_i if command == H5T_CONV_INIT: cdata[0].need_bkg = need_bkg return initop(src_id, dst_id, &(cdata[0].priv)) elif command == H5T_CONV_FREE: free(cdata[0].priv) cdata[0].priv = NULL elif command == H5T_CONV_CONV: sizes = cdata[0].priv if H5Tis_variable_str(src_id): sizes.cset = H5Tget_cset(src_id) elif H5Tis_variable_str(dst_id): sizes.cset = H5Tget_cset(dst_id) if bkg_stride==0: bkg_stride = sizes[0].dst_size; if buf_stride == 0: # No explicit stride seems to mean that the elements are packed # contiguously in the buffer. In this case we must be careful # not to "stomp on" input elements if the output elements are # of a larger size. if sizes[0].src_size >= sizes[0].dst_size: for i from 0<=ii>=0: op( buf + (i*sizes[0].src_size), buf + (i*sizes[0].dst_size), bkg + (i*bkg_stride), cdata[0].priv) else: # With explicit strides, we assume that the library knows the # alignment better than us. Therefore we use the given stride # offsets exclusively. for i from 0<=imalloc(sizeof(conv_size_t)) priv[0] = sizes sizes[0].src_size = H5Tget_size(src) sizes[0].dst_size = H5Tget_size(dst) return 0 # ============================================================================= # Vlen string conversion cdef int conv_vlen2str(void* ipt, void* opt, void* bkg, void* priv) except -1: cdef PyObject** buf_obj = opt cdef PyObject** bkg_obj = bkg cdef char** buf_cstring = ipt cdef PyObject* temp_obj = NULL cdef conv_size_t *sizes = priv cdef PyObject* bkg_obj0 cdef char* buf_cstring0 memcpy(&bkg_obj0, bkg_obj, sizeof(bkg_obj0)) memcpy(&buf_cstring0, buf_cstring, sizeof(buf_cstring0)) # When reading we identify H5T_CSET_ASCII as a byte string and # H5T_CSET_UTF8 as a utf8-encoded unicode string if sizes.cset == H5T_CSET_ASCII: if buf_cstring0 == NULL: temp_obj = PyBytes_FromString("") else: temp_obj = PyBytes_FromString(buf_cstring0) elif sizes.cset == H5T_CSET_UTF8: if buf_cstring0 == NULL: temp_obj = PyUnicode_DecodeUTF8("", 0, NULL) else: temp_obj = PyUnicode_DecodeUTF8(buf_cstring0, strlen(buf_cstring0), NULL) # Since all data conversions are by definition in-place, it # is our responsibility to free the memory used by the vlens. free(buf_cstring0) # HDF5 will eventuallly overwrite this target location, so we # make sure to decref the object there. Py_XDECREF(bkg_obj0) # Write the new string object to the buffer in-place memcpy(buf_obj, &temp_obj, sizeof(temp_obj)); return 0 cdef int conv_str2vlen(void* ipt, void* opt, void* bkg, void* priv) except -1: cdef PyObject** buf_obj = ipt cdef char** buf_cstring = opt cdef conv_size_t* sizes = priv cdef PyObject* temp_object = NULL cdef PyObject* temp_encoded = NULL cdef char* temp_string = NULL cdef size_t temp_string_len = 0 # Not including null term cdef PyObject* buf_obj0 cdef char* buf_cstring0 memcpy(&buf_obj0, buf_obj, sizeof(buf_obj0)) try: if buf_obj0 == NULL or buf_obj0 == Py_None: temp_string = "" temp_string_len = 0 else: if PyBytes_CheckExact(buf_obj0): # Input is a byte string. If we're using CSET_UTF8, make sure # it's valid UTF-8. Otherwise just store it. temp_object = buf_obj0 Py_INCREF(temp_object) if sizes.cset == H5T_CSET_UTF8: try: pass # disabled for Python 3 compatibility #temp_encoded = PyString_AsDecodedObject(temp_object, "utf8", NULL) except: raise ValueError("Byte string is not valid utf-8 and can't be stored in a utf-8 dataset") temp_string = PyBytes_AsString(temp_object) temp_string_len = PyBytes_Size(temp_object) # We are given a Unicode object. Encode it to utf-8 regardless of # the HDF5 character set. elif PyUnicode_CheckExact(buf_obj0): temp_object = buf_obj0 Py_INCREF(temp_object) temp_encoded = PyUnicode_AsUTF8String(temp_object) temp_string = PyBytes_AsString(temp_encoded) temp_string_len = PyBytes_Size(temp_encoded) else: if sizes.cset == H5T_CSET_ASCII: temp_object = PyObject_Str(buf_obj0) temp_string = PyBytes_AsString(temp_object) temp_string_len = PyBytes_Size(temp_object) elif sizes.cset == H5T_CSET_UTF8: temp_object = PyObject_Str(buf_obj0) Py_INCREF(temp_object) temp_encoded = PyUnicode_AsUTF8String(temp_object) Py_INCREF(temp_encoded) temp_string = PyBytes_AsString(temp_encoded) temp_string_len = PyBytes_Size(temp_encoded) else: raise TypeError("Unrecognized dataset encoding") if strlen(temp_string) != temp_string_len: raise ValueError("VLEN strings do not support embedded NULLs") buf_cstring0 = malloc(temp_string_len+1) memcpy(buf_cstring0, temp_string, temp_string_len+1) memcpy(buf_cstring, &buf_cstring0, sizeof(buf_cstring0)); return 0 finally: Py_XDECREF(temp_object) Py_XDECREF(temp_encoded) # ============================================================================= # VLEN to fixed-width strings cdef herr_t init_vlen2fixed(hid_t src, hid_t dst, void** priv) except -1: cdef conv_size_t *sizes if not (H5Tis_variable_str(src) and (not H5Tis_variable_str(dst))): return -2 sizes = malloc(sizeof(conv_size_t)) priv[0] = sizes sizes[0].src_size = H5Tget_size(src) sizes[0].dst_size = H5Tget_size(dst) return 0 cdef herr_t init_fixed2vlen(hid_t src, hid_t dst, void** priv) except -1: cdef conv_size_t *sizes if not (H5Tis_variable_str(dst) and (not H5Tis_variable_str(src))): return -2 sizes = malloc(sizeof(conv_size_t)) priv[0] = sizes sizes[0].src_size = H5Tget_size(src) sizes[0].dst_size = H5Tget_size(dst) return 0 cdef int conv_vlen2fixed(void* ipt, void* opt, void* bkg, void* priv) except -1: cdef char** buf_vlen = ipt cdef char* buf_fixed = opt cdef char* temp_string = NULL cdef size_t temp_string_len = 0 # Without null term cdef conv_size_t *sizes = priv cdef char* buf_vlen0 memcpy(&buf_vlen0, buf_vlen, sizeof(buf_vlen0)); if buf_vlen0 != NULL: temp_string = buf_vlen0 temp_string_len = strlen(temp_string) if temp_string_len <= sizes[0].dst_size: # Pad with zeros memcpy(buf_fixed, temp_string, temp_string_len) memset(buf_fixed + temp_string_len, c'\0', sizes[0].dst_size - temp_string_len) else: # Simply truncate the string memcpy(buf_fixed, temp_string, sizes[0].dst_size) else: memset(buf_fixed, c'\0', sizes[0].dst_size) return 0 cdef int conv_fixed2vlen(void* ipt, void* opt, void* bkg, void* priv) except -1: cdef char** buf_vlen = opt cdef char* buf_fixed = ipt cdef char* temp_string = NULL cdef conv_size_t *sizes = priv temp_string = malloc(sizes[0].src_size+1) memcpy(temp_string, buf_fixed, sizes[0].src_size) temp_string[sizes[0].src_size] = c'\0' memcpy(buf_vlen, &temp_string, sizeof(temp_string)); return 0 # ============================================================================= # HDF5 references to Python instances of h5r.Reference cdef int conv_objref2pyref(void* ipt, void* opt, void* bkg, void* priv) except -1: cdef PyObject** buf_obj = opt cdef hobj_ref_t* buf_ref = ipt cdef Reference ref = Reference() cdef PyObject* ref_ptr = NULL memcpy(&ref.ref.obj_ref, buf_ref, sizeof(ref.ref.obj_ref)) ref.typecode = H5R_OBJECT ref_ptr = ref Py_INCREF(ref_ptr) # because Cython discards its reference when the # function exits memcpy(buf_obj, &ref_ptr, sizeof(ref_ptr)) return 0 cdef int conv_pyref2objref(void* ipt, void* opt, void* bkg, void* priv) except -1: cdef PyObject** buf_obj = ipt cdef hobj_ref_t* buf_ref = opt cdef object obj cdef Reference ref cdef PyObject* buf_obj0 memcpy(&buf_obj0, buf_obj, sizeof(buf_obj0)); if buf_obj0 != NULL and buf_obj0 != Py_None: obj = (buf_obj0) if not isinstance(obj, Reference): raise TypeError("Can't convert incompatible object to HDF5 object reference") ref = (buf_obj0) memcpy(buf_ref, &ref.ref.obj_ref, sizeof(ref.ref.obj_ref)) else: memset(buf_ref, c'\0', sizeof(hobj_ref_t)) return 0 cdef int conv_regref2pyref(void* ipt, void* opt, void* bkg, void* priv) except -1: cdef PyObject** buf_obj = opt cdef PyObject** bkg_obj = bkg cdef hdset_reg_ref_t* buf_ref = ipt cdef RegionReference ref = RegionReference() cdef PyObject* ref_ptr = NULL cdef PyObject* bkg_obj0 memcpy(&bkg_obj0, bkg_obj, sizeof(bkg_obj0)); memcpy(ref.ref.reg_ref, buf_ref, sizeof(hdset_reg_ref_t)) ref.typecode = H5R_DATASET_REGION ref_ptr = ref Py_INCREF(ref_ptr) # because Cython discards its reference when the # function exits Py_XDECREF(bkg_obj0) memcpy(buf_obj, &ref_ptr, sizeof(ref_ptr)) return 0 cdef int conv_pyref2regref(void* ipt, void* opt, void* bkg, void* priv) except -1: cdef PyObject** buf_obj = ipt cdef hdset_reg_ref_t* buf_ref = opt cdef object obj cdef RegionReference ref cdef PyObject* buf_obj0 memcpy(&buf_obj0, buf_obj, sizeof(buf_obj0)); if buf_obj0 != NULL and buf_obj0 != Py_None: obj = (buf_obj0) if not isinstance(obj, RegionReference): raise TypeError("Can't convert incompatible object to HDF5 region reference") ref = (buf_obj0) memcpy(buf_ref, ref.ref.reg_ref, sizeof(hdset_reg_ref_t)) else: memset(buf_ref, c'\0', sizeof(hdset_reg_ref_t)) return 0 # ============================================================================= # Conversion functions cdef herr_t vlen2str(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nl, size_t buf_stride, size_t bkg_stride, void *buf_i, void *bkg_i, hid_t dxpl) except -1: return generic_converter(src_id, dst_id, cdata, nl, buf_stride, bkg_stride, buf_i, bkg_i, dxpl, conv_vlen2str, init_generic, H5T_BKG_YES) cdef herr_t str2vlen(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nl, size_t buf_stride, size_t bkg_stride, void *buf_i, void *bkg_i, hid_t dxpl) except -1: return generic_converter(src_id, dst_id, cdata, nl, buf_stride, bkg_stride, buf_i, bkg_i, dxpl, conv_str2vlen, init_generic, H5T_BKG_NO) cdef herr_t vlen2fixed(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nl, size_t buf_stride, size_t bkg_stride, void *buf_i, void *bkg_i, hid_t dxpl) except -1: return generic_converter(src_id, dst_id, cdata, nl, buf_stride, bkg_stride, buf_i, bkg_i, dxpl, conv_vlen2fixed, init_vlen2fixed, H5T_BKG_NO) cdef herr_t fixed2vlen(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nl, size_t buf_stride, size_t bkg_stride, void *buf_i, void *bkg_i, hid_t dxpl) except -1: return generic_converter(src_id, dst_id, cdata, nl, buf_stride, bkg_stride, buf_i, bkg_i, dxpl, conv_fixed2vlen, init_fixed2vlen, H5T_BKG_NO) cdef herr_t objref2pyref(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nl, size_t buf_stride, size_t bkg_stride, void *buf_i, void *bkg_i, hid_t dxpl) except -1: return generic_converter(src_id, dst_id, cdata, nl, buf_stride, bkg_stride, buf_i, bkg_i, dxpl, conv_objref2pyref, init_generic, H5T_BKG_NO) cdef herr_t pyref2objref(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nl, size_t buf_stride, size_t bkg_stride, void *buf_i, void *bkg_i, hid_t dxpl) except -1: return generic_converter(src_id, dst_id, cdata, nl, buf_stride, bkg_stride, buf_i, bkg_i, dxpl, conv_pyref2objref, init_generic, H5T_BKG_NO) cdef herr_t regref2pyref(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nl, size_t buf_stride, size_t bkg_stride, void *buf_i, void *bkg_i, hid_t dxpl) except -1: return generic_converter(src_id, dst_id, cdata, nl, buf_stride, bkg_stride, buf_i, bkg_i, dxpl, conv_regref2pyref, init_generic, H5T_BKG_YES) cdef herr_t pyref2regref(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nl, size_t buf_stride, size_t bkg_stride, void *buf_i, void *bkg_i, hid_t dxpl) except -1: return generic_converter(src_id, dst_id, cdata, nl, buf_stride, bkg_stride, buf_i, bkg_i, dxpl, conv_pyref2regref, init_generic, H5T_BKG_NO) # ============================================================================= # Enum to integer converter cdef struct conv_enum_t: size_t src_size size_t dst_size cdef int enum_int_converter_init(hid_t src, hid_t dst, H5T_cdata_t *cdata, int forward) except -1 with gil: cdef conv_enum_t *info cdata[0].need_bkg = H5T_BKG_NO cdata[0].priv = info = malloc(sizeof(conv_enum_t)) info[0].src_size = H5Tget_size(src) info[0].dst_size = H5Tget_size(dst) cdef void enum_int_converter_free(H5T_cdata_t *cdata): cdef conv_enum_t *info info = cdata[0].priv free(info) cdata[0].priv = NULL cdef int enum_int_converter_conv(hid_t src, hid_t dst, H5T_cdata_t *cdata, size_t nl, size_t buf_stride, size_t bkg_stride, void *buf_i, void *bkg_i, hid_t dxpl, int forward) except -1 with gil: cdef conv_enum_t *info cdef size_t nalloc cdef int i cdef char* cbuf = NULL cdef char* buf = buf_i cdef int identical cdef hid_t supertype = -1 info = cdata[0].priv try: if forward: supertype = H5Tget_super(src) identical = H5Tequal(supertype, dst) else: supertype = H5Tget_super(dst) identical = H5Tequal(supertype, src) # Short-circuit success if identical: return 0 if buf_stride == 0: # Contiguous case: call H5Tconvert directly if forward: H5Tconvert(supertype, dst, nl, buf, NULL, dxpl) else: H5Tconvert(src, supertype, nl, buf, NULL, dxpl) else: # Non-contiguous: gather, convert and then scatter if info[0].src_size > info[0].dst_size: nalloc = info[0].src_size*nl else: nalloc = info[0].dst_size*nl cbuf = malloc(nalloc) if cbuf == NULL: raise MemoryError() for i from 0<=i 0: H5Tclose(supertype) return 0 # Direction ("forward"): 1 = enum to int, 0 = int to enum cdef herr_t enum_int_converter(hid_t src, hid_t dst, H5T_cdata_t *cdata, size_t nl, size_t buf_stride, size_t bkg_stride, void *buf_i, void *bkg_i, hid_t dxpl, int forward) except -1: cdef int command = cdata[0].command if command == H5T_CONV_INIT: enum_int_converter_init(src, dst, cdata, forward) elif command == H5T_CONV_FREE: enum_int_converter_free(cdata) elif command == H5T_CONV_CONV: return enum_int_converter_conv(src, dst, cdata, nl, buf_stride, bkg_stride, buf_i, bkg_i, dxpl, forward) else: return -2 return 0 cdef herr_t enum2int(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nl, size_t buf_stride, size_t bkg_stride, void *buf_i, void *bkg_i, hid_t dxpl) except -1: return enum_int_converter(src_id, dst_id, cdata, nl, buf_stride, bkg_stride, buf_i, bkg_i, dxpl, 1) cdef herr_t int2enum(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nl, size_t buf_stride, size_t bkg_stride, void *buf_i, void *bkg_i, hid_t dxpl) except -1: return enum_int_converter(src_id, dst_id, cdata, nl, buf_stride, bkg_stride, buf_i, bkg_i, dxpl, 0) # ============================================================================= # ndarray to VLEN routines cdef herr_t vlen2ndarray(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nl, size_t buf_stride, size_t bkg_stride, void *buf_i, void *bkg_i, hid_t dxpl) except -1: cdef int command = cdata[0].command cdef size_t src_size, dst_size cdef TypeID supertype cdef TypeID outtype cdef np.dtype dt cdef int i cdef char* buf = buf_i if command == H5T_CONV_INIT: cdata[0].need_bkg = H5T_BKG_NO if H5Tget_class(src_id) != H5T_VLEN or H5Tget_class(dst_id) != H5T_OPAQUE: return -2 elif command == H5T_CONV_FREE: pass elif command == H5T_CONV_CONV: # need to pass element dtype to converter supertype = typewrap(H5Tget_super(src_id)) dt = supertype.dtype outtype = py_create(dt) if buf_stride == 0: # No explicit stride seems to mean that the elements are packed # contiguously in the buffer. In this case we must be careful # not to "stomp on" input elements if the output elements are # of a larger size. src_size = H5Tget_size(src_id) dst_size = H5Tget_size(dst_id) if src_size >= dst_size: for i from 0<=ii>=0: conv_vlen2ndarray(buf + (i*src_size), buf + (i*dst_size), dt, supertype, outtype) else: # With explicit strides, we assume that the library knows the # alignment better than us. Therefore we use the given stride # offsets exclusively. for i from 0<=iopt cdef vlen_t* in_vlen = ipt cdef int flags = np.NPY_WRITEABLE | np.NPY_C_CONTIGUOUS cdef np.npy_intp dims[1] cdef void* data cdef np.ndarray ndarray cdef PyObject* ndarray_obj cdef vlen_t in_vlen0 memcpy(&in_vlen0, in_vlen, sizeof(in_vlen0)) dims[0] = in_vlen0.len data = in_vlen0.ptr if outtype.get_size() > intype.get_size(): data = realloc(data, outtype.get_size() * in_vlen0.len) H5Tconvert(intype.id, outtype.id, in_vlen0.len, data, NULL, H5P_DEFAULT) Py_INCREF(elem_dtype) ndarray = PyArray_NewFromDescr(&PyArray_Type, elem_dtype, 1, dims, NULL, data, flags, NULL) ndarray.flags |= np.NPY_OWNDATA ndarray_obj = ndarray Py_INCREF(ndarray_obj) # Write the new object to the buffer in-place in_vlen0.ptr = NULL memcpy(buf_obj, &ndarray_obj, sizeof(ndarray_obj)) return 0 cdef herr_t ndarray2vlen(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nl, size_t buf_stride, size_t bkg_stride, void *buf_i, void *bkg_i, hid_t dxpl) except -1: cdef int command = cdata[0].command cdef size_t src_size, dst_size cdef TypeID supertype cdef TypeID outtype cdef np.dtype dt cdef int i cdef PyObject **pdata = buf_i cdef PyObject *pdata_elem cdef char* buf = buf_i if command == H5T_CONV_INIT: cdata[0].need_bkg = H5T_BKG_NO if not H5Tequal(src_id, H5PY_OBJ) or H5Tget_class(dst_id) != H5T_VLEN: return -2 supertype = typewrap(H5Tget_super(dst_id)) for i from 0 <= i < nl: memcpy(&pdata_elem, pdata+i, sizeof(pdata_elem)) if supertype != py_create(( pdata_elem).dtype, 1): return -2 if ( pdata_elem).ndim != 1: return -2 elif command == H5T_CONV_FREE: pass elif command == H5T_CONV_CONV: # need to pass element dtype to converter memcpy(&pdata_elem, pdata, sizeof(pdata_elem)) supertype = py_create(( pdata_elem).dtype) outtype = typewrap(H5Tget_super(dst_id)) if buf_stride == 0: # No explicit stride seems to mean that the elements are packed # contiguously in the buffer. In this case we must be careful # not to "stomp on" input elements if the output elements are # of a larger size. src_size = H5Tget_size(src_id) dst_size = H5Tget_size(dst_id) if src_size >= dst_size: for i from 0<=ii>=0: conv_ndarray2vlen(buf + (i*src_size), buf + (i*dst_size), supertype, outtype) else: # With explicit strides, we assume that the library knows the # alignment better than us. Therefore we use the given stride # offsets exclusively. for i from 0<=iipt cdef vlen_t* in_vlen = opt cdef int flags = np.NPY_WRITEABLE | np.NPY_C_CONTIGUOUS cdef np.npy_intp dims[1] cdef void* data cdef np.ndarray ndarray cdef size_t len cdef PyObject* buf_obj0 memcpy(&buf_obj0, buf_obj, sizeof(buf_obj0)) ndarray = buf_obj0 len = ndarray.shape[0] if outtype.get_size() > intype.get_size(): data = malloc(outtype.get_size() * len) else: data = malloc(intype.get_size() * len) memcpy(data, ndarray.data, intype.get_size() * len) H5Tconvert(intype.id, outtype.id, len, data, NULL, H5P_DEFAULT) memcpy(&in_vlen[0].len, &len, sizeof(len)) memcpy(&in_vlen[0].ptr, &data, sizeof(data)) return 0 # ============================================================================= cpdef int register_converters() except -1: cdef hid_t vlstring cdef hid_t vlentype cdef hid_t pyobj cdef hid_t enum vlstring = H5Tcopy(H5T_C_S1) H5Tset_size(vlstring, H5T_VARIABLE) enum = H5Tenum_create(H5T_STD_I32LE) vlentype = H5Tvlen_create(H5T_STD_I32LE) pyobj = H5PY_OBJ H5Tregister(H5T_PERS_HARD, "vlen2str", vlstring, pyobj, vlen2str) H5Tregister(H5T_PERS_HARD, "str2vlen", pyobj, vlstring, str2vlen) H5Tregister(H5T_PERS_SOFT, "vlen2fixed", vlstring, H5T_C_S1, vlen2fixed) H5Tregister(H5T_PERS_SOFT, "fixed2vlen", H5T_C_S1, vlstring, fixed2vlen) H5Tregister(H5T_PERS_HARD, "objref2pyref", H5T_STD_REF_OBJ, pyobj, objref2pyref) H5Tregister(H5T_PERS_HARD, "pyref2objref", pyobj, H5T_STD_REF_OBJ, pyref2objref) H5Tregister(H5T_PERS_HARD, "regref2pyref", H5T_STD_REF_DSETREG, pyobj, regref2pyref) H5Tregister(H5T_PERS_HARD, "pyref2regref", pyobj, H5T_STD_REF_DSETREG, pyref2regref) H5Tregister(H5T_PERS_SOFT, "enum2int", enum, H5T_STD_I32LE, enum2int) H5Tregister(H5T_PERS_SOFT, "int2enum", H5T_STD_I32LE, enum, int2enum) H5Tregister(H5T_PERS_SOFT, "vlen2ndarray", vlentype, pyobj, vlen2ndarray) H5Tregister(H5T_PERS_SOFT, "ndarray2vlen", pyobj, vlentype, ndarray2vlen) H5Tclose(vlstring) H5Tclose(vlentype) H5Tclose(enum) return 0 cpdef int unregister_converters() except -1: H5Tunregister(H5T_PERS_HARD, "vlen2str", -1, -1, vlen2str) H5Tunregister(H5T_PERS_HARD, "str2vlen", -1, -1, str2vlen) H5Tunregister(H5T_PERS_SOFT, "vlen2fixed", -1, -1, vlen2fixed) H5Tunregister(H5T_PERS_SOFT, "fixed2vlen", -1, -1, fixed2vlen) H5Tunregister(H5T_PERS_HARD, "objref2pyref", -1, -1, objref2pyref) H5Tunregister(H5T_PERS_HARD, "pyref2objref", -1, -1, pyref2objref) H5Tunregister(H5T_PERS_HARD, "regref2pyref", -1, -1, regref2pyref) H5Tunregister(H5T_PERS_HARD, "pyref2regref", -1, -1, pyref2regref) H5Tunregister(H5T_PERS_SOFT, "enum2int", -1, -1, enum2int) H5Tunregister(H5T_PERS_SOFT, "int2enum", -1, -1, int2enum) H5Tunregister(H5T_PERS_SOFT, "vlen2ndarray", -1, -1, vlen2ndarray) H5Tunregister(H5T_PERS_SOFT, "ndarray2vlen", -1, -1, ndarray2vlen) return 0 h5py-2.7.1/h5py/h5s.pyx0000644000175000017500000004337213025343121016215 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. """ Low-level interface to the "H5S" family of data-space functions. """ include "config.pxi" # Pyrex compile-time imports from utils cimport require_tuple, convert_dims, convert_tuple, \ emalloc, efree, create_numpy_hsize, create_hsize_array from numpy cimport ndarray from h5py import _objects from ._objects import phil, with_phil cdef object lockid(hid_t id_): cdef SpaceID space space = SpaceID(id_) space.locked = 1 return space # === Public constants and data structures ==================================== #enum H5S_seloper_t: SELECT_NOOP = H5S_SELECT_NOOP SELECT_SET = H5S_SELECT_SET SELECT_OR = H5S_SELECT_OR SELECT_AND = H5S_SELECT_AND SELECT_XOR = H5S_SELECT_XOR SELECT_NOTB = H5S_SELECT_NOTB SELECT_NOTA = H5S_SELECT_NOTA SELECT_APPEND = H5S_SELECT_APPEND SELECT_PREPEND = H5S_SELECT_PREPEND SELECT_INVALID = H5S_SELECT_INVALID ALL = lockid(H5S_ALL) # This is accepted in lieu of an actual identifier # in functions like H5Dread, so wrap it. UNLIMITED = H5S_UNLIMITED #enum H5S_class_t NO_CLASS = H5S_NO_CLASS SCALAR = H5S_SCALAR SIMPLE = H5S_SIMPLE globals()["NULL"] = H5S_NULL # "NULL" is reserved in Cython #enum H5S_sel_type SEL_ERROR = H5S_SEL_ERROR SEL_NONE = H5S_SEL_NONE SEL_POINTS = H5S_SEL_POINTS SEL_HYPERSLABS = H5S_SEL_HYPERSLABS SEL_ALL = H5S_SEL_ALL # === Basic dataspace operations ============================================== @with_phil def create(int class_code): """(INT class_code) => SpaceID Create a new HDF5 dataspace object, of the given class. Legal values are SCALAR and SIMPLE. """ return SpaceID(H5Screate(class_code)) @with_phil def create_simple(object dims_tpl, object max_dims_tpl=None): """(TUPLE dims_tpl, TUPLE max_dims_tpl) => SpaceID Create a simple (slab) dataspace from a tuple of dimensions. Every element of dims_tpl must be a positive integer. You can optionally specify the maximum dataspace size. The special value UNLIMITED, as an element of max_dims, indicates an unlimited dimension. """ cdef int rank cdef hsize_t* dims = NULL cdef hsize_t* max_dims = NULL require_tuple(dims_tpl, 0, -1, "dims_tpl") rank = len(dims_tpl) require_tuple(max_dims_tpl, 1, rank, "max_dims_tpl") try: dims = emalloc(sizeof(hsize_t)*rank) convert_tuple(dims_tpl, dims, rank) if max_dims_tpl is not None: max_dims = emalloc(sizeof(hsize_t)*rank) convert_tuple(max_dims_tpl, max_dims, rank) return SpaceID(H5Screate_simple(rank, dims, max_dims)) finally: efree(dims) efree(max_dims) @with_phil def decode(buf): """(STRING buf) => SpaceID Unserialize a dataspace. Bear in mind you can also use the native Python pickling machinery to do this. """ cdef char* buf_ = buf return SpaceID(H5Sdecode(buf_)) # === H5S class API =========================================================== cdef class SpaceID(ObjectID): """ Represents a dataspace identifier. Properties: shape Numpy-style shape tuple with dimensions. * Hashable: No * Equality: Unimplemented Can be pickled if HDF5 1.8 is available. """ property shape: """ Numpy-style shape tuple representing dimensions. () == scalar. """ def __get__(self): with phil: return self.get_simple_extent_dims() @with_phil def copy(self): """() => SpaceID Create a new copy of this dataspace. """ return SpaceID(H5Scopy(self.id)) @with_phil def encode(self): """() => STRING Serialize a dataspace, including its selection. Bear in mind you can also use the native Python pickling machinery to do this. """ cdef void* buf = NULL cdef size_t nalloc = 0 H5Sencode(self.id, NULL, &nalloc) buf = emalloc(nalloc) try: H5Sencode(self.id, buf, &nalloc) pystr = PyBytes_FromStringAndSize(buf, nalloc) finally: efree(buf) return pystr def __reduce__(self): with phil: return (type(self), (-1,), self.encode()) def __setstate__(self, state): cdef char* buf = state with phil: self.id = H5Sdecode(buf) # === Simple dataspaces =================================================== @with_phil def is_simple(self): """() => BOOL is_simple Determine if an existing dataspace is "simple" (including scalar dataspaces). Currently all HDF5 dataspaces are simple. """ return (H5Sis_simple(self.id)) @with_phil def offset_simple(self, object offset=None): """(TUPLE offset=None) Set the offset of a dataspace. The length of the given tuple must match the rank of the dataspace. If None is provided (default), the offsets on all axes will be set to 0. """ cdef int rank cdef int i cdef hssize_t *dims = NULL try: if not H5Sis_simple(self.id): raise ValueError("%d is not a simple dataspace" % self.id) rank = H5Sget_simple_extent_ndims(self.id) require_tuple(offset, 1, rank, "offset") dims = emalloc(sizeof(hssize_t)*rank) if(offset is not None): convert_tuple(offset, dims, rank) else: # The HDF5 docs say passing in NULL resets the offset to 0. # Instead it raises an exception. Imagine my surprise. We'll # do this manually. for i from 0<=i INT rank Determine the rank of a "simple" (slab) dataspace. """ return H5Sget_simple_extent_ndims(self.id) @with_phil def get_simple_extent_dims(self, int maxdims=0): """(BOOL maxdims=False) => TUPLE shape Determine the shape of a "simple" (slab) dataspace. If "maxdims" is True, retrieve the maximum dataspace size instead. """ cdef int rank cdef hsize_t* dims = NULL if self.get_simple_extent_type() == H5S_NULL: return None rank = H5Sget_simple_extent_dims(self.id, NULL, NULL) dims = emalloc(sizeof(hsize_t)*rank) try: if maxdims: H5Sget_simple_extent_dims(self.id, NULL, dims) else: H5Sget_simple_extent_dims(self.id, dims, NULL) return convert_dims(dims, rank) finally: efree(dims) @with_phil def get_simple_extent_npoints(self): """() => LONG npoints Determine the total number of elements in a dataspace. """ return H5Sget_simple_extent_npoints(self.id) @with_phil def get_simple_extent_type(self): """() => INT class_code Class code is either SCALAR or SIMPLE. """ return H5Sget_simple_extent_type(self.id) # === Extents ============================================================= @with_phil def extent_copy(self, SpaceID source not None): """(SpaceID source) Replace this dataspace's extent with another's, changing its typecode if necessary. """ H5Sextent_copy(self.id, source.id) @with_phil def set_extent_simple(self, object dims_tpl, object max_dims_tpl=None): """(TUPLE dims_tpl, TUPLE max_dims_tpl=None) Reset the dataspace extent via a tuple of dimensions. Every element of dims_tpl must be a positive integer. You can optionally specify the maximum dataspace size. The special value UNLIMITED, as an element of max_dims, indicates an unlimited dimension. """ cdef int rank cdef hsize_t* dims = NULL cdef hsize_t* max_dims = NULL require_tuple(dims_tpl, 0, -1, "dims_tpl") rank = len(dims_tpl) require_tuple(max_dims_tpl, 1, rank, "max_dims_tpl") try: dims = emalloc(sizeof(hsize_t)*rank) convert_tuple(dims_tpl, dims, rank) if max_dims_tpl is not None: max_dims = emalloc(sizeof(hsize_t)*rank) convert_tuple(max_dims_tpl, max_dims, rank) H5Sset_extent_simple(self.id, rank, dims, max_dims) finally: efree(dims) efree(max_dims) @with_phil def set_extent_none(self): """() Remove the dataspace extent; typecode changes to NO_CLASS. """ H5Sset_extent_none(self.id) # === General selection operations ======================================== @with_phil def get_select_type(self): """ () => INT select_code Determine selection type. Return values are: - SEL_NONE - SEL_ALL - SEL_POINTS - SEL_HYPERSLABS """ return H5Sget_select_type(self.id) @with_phil def get_select_npoints(self): """() => LONG npoints Determine the total number of points currently selected. Works for all selection techniques. """ return H5Sget_select_npoints(self.id) @with_phil def get_select_bounds(self): """() => (TUPLE start, TUPLE end) Determine the bounding box which exactly contains the current selection. """ cdef int rank cdef hsize_t *start = NULL cdef hsize_t *end = NULL rank = H5Sget_simple_extent_ndims(self.id) if H5Sget_select_npoints(self.id) == 0: return None start = emalloc(sizeof(hsize_t)*rank) end = emalloc(sizeof(hsize_t)*rank) try: H5Sget_select_bounds(self.id, start, end) start_tpl = convert_dims(start, rank) end_tpl = convert_dims(end, rank) return (start_tpl, end_tpl) finally: efree(start) efree(end) @with_phil def select_all(self): """() Select all points in the dataspace. """ H5Sselect_all(self.id) @with_phil def select_none(self): """() Deselect entire dataspace. """ H5Sselect_none(self.id) @with_phil def select_valid(self): """() => BOOL Determine if the current selection falls within the dataspace extent. """ return (H5Sselect_valid(self.id)) # === Point selection functions =========================================== @with_phil def get_select_elem_npoints(self): """() => LONG npoints Determine the number of elements selected in point-selection mode. """ return H5Sget_select_elem_npoints(self.id) @with_phil def get_select_elem_pointlist(self): """() => NDARRAY Get a list of all selected elements. Return is a Numpy array of unsigned ints, with shape ``(, buf.data) return buf @with_phil def select_elements(self, object coords, int op=H5S_SELECT_SET): """(SEQUENCE coords, INT op=SELECT_SET) Select elements by specifying coordinates points. The argument "coords" may be an ndarray or any nested sequence which can be converted to an array of uints with the shape:: (, ) Examples:: >>> obj.shape (10, 10) >>> obj.select_elements([(1,2), (3,4), (5,9)]) A zero-length selection (i.e. shape ``(0, )``) is not allowed by the HDF5 library. """ cdef ndarray hcoords cdef size_t nelements # The docs say the selection list should be an hsize_t**, but it seems # that HDF5 expects the coordinates to be a static, contiguous # array. We simulate that by creating a contiguous NumPy array of # a compatible type and initializing it to the input. hcoords = create_hsize_array(coords) if hcoords.nd != 2 or hcoords.dimensions[1] != H5Sget_simple_extent_ndims(self.id): raise ValueError("Coordinate array must have shape (, %d)" % self.get_simple_extent_ndims()) nelements = hcoords.dimensions[0] H5Sselect_elements(self.id, op, nelements, hcoords.data) # === Hyperslab selection functions ======================================= @with_phil def get_select_hyper_nblocks(self): """() => LONG nblocks Get the number of hyperslab blocks currently selected. """ return H5Sget_select_hyper_nblocks(self.id) @with_phil def get_select_hyper_blocklist(self): """() => NDARRAY Get the current hyperslab selection. The returned array has shape:: (, 2, ) and can be interpreted as a nested sequence:: [ (corner_coordinate_1, opposite_coordinate_1), ... ] with length equal to the total number of blocks. """ cdef hsize_t dims[3] # 0=nblocks 1=(#2), 2=rank cdef ndarray buf dims[0] = H5Sget_select_hyper_nblocks(self.id) dims[1] = 2 dims[2] = H5Sget_simple_extent_ndims(self.id) buf = create_numpy_hsize(3, dims) H5Sget_select_hyper_blocklist(self.id, 0, dims[0], buf.data) return buf @with_phil def select_hyperslab(self, object start, object count, object stride=None, object block=None, int op=H5S_SELECT_SET): """(TUPLE start, TUPLE count, TUPLE stride=None, TUPLE block=None, INT op=SELECT_SET) Select a block region from an existing dataspace. See the HDF5 documentation for the meaning of the "block" and "op" keywords. """ cdef int rank cdef hsize_t* start_array = NULL cdef hsize_t* count_array = NULL cdef hsize_t* stride_array = NULL cdef hsize_t* block_array = NULL # Dataspace rank. All provided tuples must match this. rank = H5Sget_simple_extent_ndims(self.id) require_tuple(start, 0, rank, "start") require_tuple(count, 0, rank, "count") require_tuple(stride, 1, rank, "stride") require_tuple(block, 1, rank, "block") try: start_array = emalloc(sizeof(hsize_t)*rank) count_array = emalloc(sizeof(hsize_t)*rank) convert_tuple(start, start_array, rank) convert_tuple(count, count_array, rank) if stride is not None: stride_array = emalloc(sizeof(hsize_t)*rank) convert_tuple(stride, stride_array, rank) if block is not None: block_array = emalloc(sizeof(hsize_t)*rank) convert_tuple(block, block_array, rank) H5Sselect_hyperslab(self.id, op, start_array, stride_array, count_array, block_array) finally: efree(start_array) efree(count_array) efree(stride_array) efree(block_array) # === Virtual dataset functions =========================================== IF HDF5_VERSION >= VDS_MIN_HDF5_VERSION: @with_phil def is_regular_hyperslab(self): """() => BOOL Determine whether a hyperslab selection is regular. """ return H5Sis_regular_hyperslab(self.id) @with_phil def get_regular_hyperslab(self): """() => (TUPLE start, TUPLE stride, TUPLE count, TUPLE block) Retrieve a regular hyperslab selection. """ cdef int rank cdef hsize_t* start_array = NULL cdef hsize_t* count_array = NULL cdef hsize_t* stride_array = NULL cdef hsize_t* block_array = NULL cdef list start = [] cdef list stride = [] cdef list count = [] cdef list block = [] cdef int i rank = H5Sget_simple_extent_ndims(self.id) try: start_array = emalloc(sizeof(hsize_t)*rank) stride_array = emalloc(sizeof(hsize_t)*rank) count_array = emalloc(sizeof(hsize_t)*rank) block_array = emalloc(sizeof(hsize_t)*rank) H5Sget_regular_hyperslab(self.id, start_array, stride_array, count_array, block_array) for i in range(rank): start.append(start_array[i]) stride.append(stride_array[i]) count.append(count_array[i]) block.append(block_array[i]) return (tuple(start), tuple(stride), tuple(count), tuple(block)) finally: efree(start_array) efree(stride_array) efree(count_array) efree(block_array) h5py-2.7.1/h5py/h5z.pxd0000644000175000017500000000044113025343121016165 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. from defs cimport * h5py-2.7.1/h5py/api_functions.txt0000644000175000017500000007107413025343121020356 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. # Defines the HDF5 C API functions wrapped by h5py. See also api_gen.py. # # Format of this file: # # header_declaration: # function_line # function_line # ... # # Header declarations are the basenames of the C header files (without the .h); # for example, functions from "hdf5.h" are under the "hdf5" declaration. # # Function lines are C function signatures, optionally preceeded by: # MPI If present, function requires an MPI-aware build of h5py # ERROR Explicit error checks are needed (HDF5 does not use error stack) # X.Y.Z Minimum version of HDF5 required for this function # # Blank/whitespace-only lines are ignored, as are lines whose first # non-whitespace character is "#". Indentation must be via spaces only. hdf5: # === H5 - General library functions ======================================== herr_t H5open() herr_t H5close() herr_t H5get_libversion(unsigned *majnum, unsigned *minnum, unsigned *relnum) # === H5A - Attributes API ================================================== hid_t H5Acreate(hid_t loc_id, char *name, hid_t type_id, hid_t space_id, hid_t create_plist) hid_t H5Aopen_idx(hid_t loc_id, unsigned int idx) hid_t H5Aopen_name(hid_t loc_id, char *name) herr_t H5Aclose(hid_t attr_id) herr_t H5Adelete(hid_t loc_id, char *name) herr_t H5Aread(hid_t attr_id, hid_t mem_type_id, void *buf) herr_t H5Awrite(hid_t attr_id, hid_t mem_type_id, void *buf ) int H5Aget_num_attrs(hid_t loc_id) ssize_t H5Aget_name(hid_t attr_id, size_t buf_size, char *buf) hid_t H5Aget_space(hid_t attr_id) hid_t H5Aget_type(hid_t attr_id) herr_t H5Aiterate(hid_t loc_id, unsigned * idx, H5A_operator_t op, void* op_data) herr_t H5Adelete_by_name(hid_t loc_id, char *obj_name, char *attr_name, hid_t lapl_id) herr_t H5Adelete_by_idx(hid_t loc_id, char *obj_name, H5_index_t idx_type, H5_iter_order_t order, hsize_t n, hid_t lapl_id) hid_t H5Acreate_by_name(hid_t loc_id, char *obj_name, char *attr_name, hid_t type_id, hid_t space_id, hid_t acpl_id, hid_t aapl_id, hid_t lapl_id) hid_t H5Aopen(hid_t obj_id, char *attr_name, hid_t aapl_id) hid_t H5Aopen_by_name( hid_t loc_id, char *obj_name, char *attr_name, hid_t aapl_id, hid_t lapl_id) hid_t H5Aopen_by_idx(hid_t loc_id, char *obj_name, H5_index_t idx_type, H5_iter_order_t order, hsize_t n, hid_t aapl_id, hid_t lapl_id) htri_t H5Aexists_by_name( hid_t loc_id, char *obj_name, char *attr_name, hid_t lapl_id) htri_t H5Aexists(hid_t obj_id, char *attr_name) herr_t H5Arename(hid_t loc_id, char *old_attr_name, char *new_attr_name) herr_t H5Arename_by_name(hid_t loc_id, char *obj_name, char *old_attr_name, char *new_attr_name, hid_t lapl_id) herr_t H5Aget_info( hid_t attr_id, H5A_info_t *ainfo) herr_t H5Aget_info_by_name(hid_t loc_id, char *obj_name, char *attr_name, H5A_info_t *ainfo, hid_t lapl_id) herr_t H5Aget_info_by_idx(hid_t loc_id, char *obj_name, H5_index_t idx_type, H5_iter_order_t order, hsize_t n, H5A_info_t *ainfo, hid_t lapl_id) herr_t H5Aiterate2(hid_t obj_id, H5_index_t idx_type, H5_iter_order_t order, hsize_t *n, H5A_operator2_t op, void *op_data) hsize_t H5Aget_storage_size(hid_t attr_id) # === H5D - Dataset API ===================================================== hid_t H5Dcreate2(hid_t loc_id, char *name, hid_t type_id, hid_t space_id, hid_t lcpl_id, hid_t dcpl_id, hid_t dapl_id) hid_t H5Dcreate_anon(hid_t file_id, hid_t type_id, hid_t space_id, hid_t plist_id, hid_t dapl_id) hid_t H5Dopen(hid_t file_id, char *name) hid_t H5Dopen2(hid_t loc_id, char *name, hid_t dapl_id ) herr_t H5Dclose(hid_t dset_id) hid_t H5Dget_space(hid_t dset_id) herr_t H5Dget_space_status(hid_t dset_id, H5D_space_status_t *status) hid_t H5Dget_type(hid_t dset_id) hid_t H5Dget_create_plist(hid_t dataset_id) hid_t H5Dget_access_plist(hid_t dataset_id) haddr_t H5Dget_offset(hid_t dset_id) hsize_t H5Dget_storage_size(hid_t dset_id) herr_t H5Dread(hid_t dset_id, hid_t mem_type_id, hid_t mem_space_id, hid_t file_space_id, hid_t plist_id, void *buf) herr_t H5Dwrite(hid_t dset_id, hid_t mem_type, hid_t mem_space, hid_t file_space, hid_t xfer_plist, void* buf) herr_t H5Dextend(hid_t dataset_id, hsize_t *size) herr_t H5Dfill(void *fill, hid_t fill_type_id, void *buf, hid_t buf_type_id, hid_t space_id ) herr_t H5Dvlen_get_buf_size(hid_t dset_id, hid_t type_id, hid_t space_id, hsize_t *size) herr_t H5Dvlen_reclaim(hid_t type_id, hid_t space_id, hid_t plist, void *buf) herr_t H5Diterate(void *buf, hid_t type_id, hid_t space_id, H5D_operator_t op, void* operator_data) herr_t H5Dset_extent(hid_t dset_id, hsize_t* size) # SWMR functions 1.9.178 herr_t H5Dflush(hid_t dataset_id) 1.9.178 herr_t H5Drefresh(hid_t dataset_id) # Direct Chunk Writing 1.8.11 herr_t H5DOwrite_chunk(hid_t dset_id, hid_t dxpl_id, uint32_t filters, const hsize_t *offset, size_t data_size, const void *buf) # === H5E - Minimal error-handling interface ================================ # This is used only for defs.py, to disable printing in new threads herr_t H5Eset_auto(void* a, void* b) # === H5F - File API ======================================================== hid_t H5Fcreate(char *filename, unsigned int flags, hid_t create_plist, hid_t access_plist) hid_t H5Fopen(char *name, unsigned flags, hid_t access_id) herr_t H5Fclose (hid_t file_id) htri_t H5Fis_hdf5(char *name) herr_t H5Fflush(hid_t object_id, H5F_scope_t scope) hid_t H5Freopen(hid_t file_id) herr_t H5Fmount(hid_t loc_id, char *name, hid_t child_id, hid_t plist_id) herr_t H5Funmount(hid_t loc_id, char *name) herr_t H5Fget_filesize(hid_t file_id, hsize_t *size) hid_t H5Fget_create_plist(hid_t file_id ) hid_t H5Fget_access_plist(hid_t file_id) hssize_t H5Fget_freespace(hid_t file_id) ssize_t H5Fget_name(hid_t obj_id, char *name, size_t size) int H5Fget_obj_count(hid_t file_id, unsigned int types) int H5Fget_obj_ids(hid_t file_id, unsigned int types, int max_objs, hid_t *obj_id_list) herr_t H5Fget_vfd_handle(hid_t file_id, hid_t fapl_id, void **file_handle) herr_t H5Fget_intent(hid_t file_id, unsigned int *intent) herr_t H5Fget_mdc_config(hid_t file_id, H5AC_cache_config_t *config_ptr) herr_t H5Fget_mdc_hit_rate(hid_t file_id, double *hit_rate_ptr) herr_t H5Fget_mdc_size(hid_t file_id, size_t *max_size_ptr, size_t *min_clean_size_ptr, size_t *cur_size_ptr, int *cur_num_entries_ptr) herr_t H5Freset_mdc_hit_rate_stats(hid_t file_id) herr_t H5Fset_mdc_config(hid_t file_id, H5AC_cache_config_t *config_ptr) # File Image Operations 1.8.9 ssize_t H5Fget_file_image(hid_t file_id, void *buf_ptr, size_t buf_len) # MPI functions MPI 1.8.9 herr_t H5Fset_mpi_atomicity(hid_t file_id, hbool_t flag) MPI 1.8.9 herr_t H5Fget_mpi_atomicity(hid_t file_id, hbool_t *flag) # SWMR functions 1.9.178 herr_t H5Fstart_swmr_write(hid_t file_id) # === H5G - Groups API ====================================================== hid_t H5Gcreate(hid_t loc_id, char *name, size_t size_hint) hid_t H5Gopen(hid_t loc_id, char *name) herr_t H5Gclose(hid_t group_id) herr_t H5Glink2( hid_t curr_loc_id, char *current_name, H5G_link_t link_type, hid_t new_loc_id, char *new_name) herr_t H5Gunlink (hid_t file_id, char *name) herr_t H5Gmove2(hid_t src_loc_id, char *src_name, hid_t dst_loc_id, char *dst_name) herr_t H5Gget_num_objs(hid_t loc_id, hsize_t* num_obj) int H5Gget_objname_by_idx(hid_t loc_id, hsize_t idx, char *name, size_t size) int H5Gget_objtype_by_idx(hid_t loc_id, hsize_t idx) herr_t H5Giterate(hid_t loc_id, char *name, int *idx, H5G_iterate_t op, void* data) herr_t H5Gget_objinfo(hid_t loc_id, char* name, int follow_link, H5G_stat_t *statbuf) herr_t H5Gget_linkval(hid_t loc_id, char *name, size_t size, char *value) herr_t H5Gset_comment(hid_t loc_id, char *name, char *comment) int H5Gget_comment(hid_t loc_id, char *name, size_t bufsize, char *comment) hid_t H5Gcreate_anon( hid_t loc_id, hid_t gcpl_id, hid_t gapl_id) hid_t H5Gcreate2(hid_t loc_id, char *name, hid_t lcpl_id, hid_t gcpl_id, hid_t gapl_id) hid_t H5Gopen2( hid_t loc_id, char * name, hid_t gapl_id) herr_t H5Gget_info( hid_t group_id, H5G_info_t *group_info) herr_t H5Gget_info_by_name( hid_t loc_id, char *group_name, H5G_info_t *group_info, hid_t lapl_id) hid_t H5Gget_create_plist(hid_t group_id) # === H5I - Identifier and reflection interface ============================= H5I_type_t H5Iget_type(hid_t obj_id) ssize_t H5Iget_name( hid_t obj_id, char *name, size_t size) hid_t H5Iget_file_id(hid_t obj_id) int H5Idec_ref(hid_t obj_id) int H5Iget_ref(hid_t obj_id) int H5Iinc_ref(hid_t obj_id) htri_t H5Iis_valid( hid_t obj_id ) # === H5L - Links interface ================================================= herr_t H5Lmove(hid_t src_loc, char *src_name, hid_t dst_loc, char *dst_name, hid_t lcpl_id, hid_t lapl_id) herr_t H5Lcopy(hid_t src_loc, char *src_name, hid_t dst_loc, char *dst_name, hid_t lcpl_id, hid_t lapl_id) herr_t H5Lcreate_hard(hid_t cur_loc, char *cur_name, hid_t dst_loc, char *dst_name, hid_t lcpl_id, hid_t lapl_id) herr_t H5Lcreate_soft(char *link_target, hid_t link_loc_id, char *link_name, hid_t lcpl_id, hid_t lapl_id) herr_t H5Ldelete(hid_t loc_id, char *name, hid_t lapl_id) herr_t H5Ldelete_by_idx(hid_t loc_id, char *group_name, H5_index_t idx_type, H5_iter_order_t order, hsize_t n, hid_t lapl_id) herr_t H5Lget_val(hid_t loc_id, char *name, void *bufout, size_t size, hid_t lapl_id) herr_t H5Lget_val_by_idx(hid_t loc_id, char *group_name, H5_index_t idx_type, H5_iter_order_t order, hsize_t n, void *bufout, size_t size, hid_t lapl_id) htri_t H5Lexists(hid_t loc_id, char *name, hid_t lapl_id) herr_t H5Lget_info(hid_t loc_id, char *name, H5L_info_t *linfo, hid_t lapl_id) herr_t H5Lget_info_by_idx(hid_t loc_id, char *group_name, H5_index_t idx_type, H5_iter_order_t order, hsize_t n, H5L_info_t *linfo, hid_t lapl_id) ssize_t H5Lget_name_by_idx(hid_t loc_id, char *group_name, H5_index_t idx_type, H5_iter_order_t order, hsize_t n, char *name, size_t size, hid_t lapl_id) herr_t H5Literate(hid_t grp_id, H5_index_t idx_type, H5_iter_order_t order, hsize_t *idx, H5L_iterate_t op, void *op_data) herr_t H5Literate_by_name(hid_t loc_id, char *group_name, H5_index_t idx_type, H5_iter_order_t order, hsize_t *idx, H5L_iterate_t op, void *op_data, hid_t lapl_id) herr_t H5Lvisit(hid_t grp_id, H5_index_t idx_type, H5_iter_order_t order, H5L_iterate_t op, void *op_data) herr_t H5Lvisit_by_name(hid_t loc_id, char *group_name, H5_index_t idx_type, H5_iter_order_t order, H5L_iterate_t op, void *op_data, hid_t lapl_id) herr_t H5Lunpack_elink_val(void *ext_linkval, size_t link_size, unsigned *flags, const char **filename, const char **obj_path) herr_t H5Lcreate_external(char *file_name, char *obj_name, hid_t link_loc_id, char *link_name, hid_t lcpl_id, hid_t lapl_id) # === H5O - General object operations ======================================= hid_t H5Oopen(hid_t loc_id, char *name, hid_t lapl_id) hid_t H5Oopen_by_addr(hid_t loc_id, haddr_t addr) hid_t H5Oopen_by_idx(hid_t loc_id, char *group_name, H5_index_t idx_type, H5_iter_order_t order, hsize_t n, hid_t lapl_id) herr_t H5Oget_info(hid_t loc_id, H5O_info_t *oinfo) herr_t H5Oget_info_by_name(hid_t loc_id, char *name, H5O_info_t *oinfo, hid_t lapl_id) herr_t H5Oget_info_by_idx(hid_t loc_id, char *group_name, H5_index_t idx_type, H5_iter_order_t order, hsize_t n, H5O_info_t *oinfo, hid_t lapl_id) herr_t H5Olink(hid_t obj_id, hid_t new_loc_id, char *new_name, hid_t lcpl_id, hid_t lapl_id) herr_t H5Ocopy(hid_t src_loc_id, char *src_name, hid_t dst_loc_id, char *dst_name, hid_t ocpypl_id, hid_t lcpl_id) herr_t H5Oincr_refcount(hid_t object_id) herr_t H5Odecr_refcount(hid_t object_id) herr_t H5Oset_comment(hid_t obj_id, char *comment) herr_t H5Oset_comment_by_name(hid_t loc_id, char *name, char *comment, hid_t lapl_id) ssize_t H5Oget_comment(hid_t obj_id, char *comment, size_t bufsize) ssize_t H5Oget_comment_by_name(hid_t loc_id, char *name, char *comment, size_t bufsize, hid_t lapl_id) herr_t H5Ovisit(hid_t obj_id, H5_index_t idx_type, H5_iter_order_t order, H5O_iterate_t op, void *op_data) herr_t H5Ovisit_by_name(hid_t loc_id, char *obj_name, H5_index_t idx_type, H5_iter_order_t order, H5O_iterate_t op, void *op_data, hid_t lapl_id) herr_t H5Oclose(hid_t object_id) # Version-limited functions 1.8.5 htri_t H5Oexists_by_name(hid_t loc_id, char * name, hid_t lapl_id ) # === H5P - Property list API =============================================== # General operations hid_t H5Pcreate(hid_t plist_id) hid_t H5Pcopy(hid_t plist_id) hid_t H5Pget_class(hid_t plist_id) herr_t H5Pclose(hid_t plist_id) htri_t H5Pequal( hid_t id1, hid_t id2 ) herr_t H5Pclose_class(hid_t id) # File creation herr_t H5Pget_version(hid_t plist, unsigned int *super_, unsigned int* freelist, unsigned int *stab, unsigned int *shhdr) herr_t H5Pset_userblock(hid_t plist, hsize_t size) herr_t H5Pget_userblock(hid_t plist, hsize_t * size) herr_t H5Pset_sizes(hid_t plist, size_t sizeof_addr, size_t sizeof_size) herr_t H5Pget_sizes(hid_t plist, size_t *sizeof_addr, size_t *sizeof_size) herr_t H5Pset_sym_k(hid_t plist, unsigned int ik, unsigned int lk) herr_t H5Pget_sym_k(hid_t plist, unsigned int *ik, unsigned int *lk) herr_t H5Pset_istore_k(hid_t plist, unsigned int ik) herr_t H5Pget_istore_k(hid_t plist, unsigned int *ik) # File access herr_t H5Pset_fclose_degree(hid_t fapl_id, H5F_close_degree_t fc_degree) herr_t H5Pget_fclose_degree(hid_t fapl_id, H5F_close_degree_t *fc_degree) herr_t H5Pset_fapl_core( hid_t fapl_id, size_t increment, hbool_t backing_store) herr_t H5Pget_fapl_core( hid_t fapl_id, size_t *increment, hbool_t *backing_store) herr_t H5Pset_fapl_family ( hid_t fapl_id, hsize_t memb_size, hid_t memb_fapl_id ) herr_t H5Pget_fapl_family ( hid_t fapl_id, hsize_t *memb_size, hid_t *memb_fapl_id ) herr_t H5Pset_family_offset ( hid_t fapl_id, hsize_t offset) herr_t H5Pget_family_offset ( hid_t fapl_id, hsize_t *offset) herr_t H5Pset_fapl_log(hid_t fapl_id, char *logfile, unsigned int flags, size_t buf_size) herr_t H5Pset_fapl_multi(hid_t fapl_id, H5FD_mem_t *memb_map, hid_t *memb_fapl, const char * const *memb_name, haddr_t *memb_addr, hbool_t relax) herr_t H5Pset_cache(hid_t plist_id, int mdc_nelmts, int rdcc_nelmts, size_t rdcc_nbytes, double rdcc_w0) herr_t H5Pget_cache(hid_t plist_id, int *mdc_nelmts, size_t *rdcc_nelmts, size_t *rdcc_nbytes, double *rdcc_w0) herr_t H5Pset_fapl_sec2(hid_t fapl_id) herr_t H5Pset_fapl_stdio(hid_t fapl_id) hid_t H5Pget_driver(hid_t fapl_id) herr_t H5Pget_mdc_config(hid_t plist_id, H5AC_cache_config_t *config_ptr) herr_t H5Pset_mdc_config(hid_t plist_id, H5AC_cache_config_t *config_ptr) 1.8.9 herr_t H5Pset_file_image(hid_t plist_id, void *buf_ptr, size_t buf_len) # Dataset creation herr_t H5Pset_layout(hid_t plist, int layout) H5D_layout_t H5Pget_layout(hid_t plist) herr_t H5Pset_chunk(hid_t plist, int ndims, hsize_t * dim) int H5Pget_chunk(hid_t plist, int max_ndims, hsize_t * dims ) herr_t H5Pset_deflate( hid_t plist, int level) herr_t H5Pset_fill_value(hid_t plist_id, hid_t type_id, void *value ) herr_t H5Pget_fill_value(hid_t plist_id, hid_t type_id, void *value ) herr_t H5Pfill_value_defined(hid_t plist_id, H5D_fill_value_t *status ) herr_t H5Pset_fill_time(hid_t plist_id, H5D_fill_time_t fill_time ) herr_t H5Pget_fill_time(hid_t plist_id, H5D_fill_time_t *fill_time ) herr_t H5Pset_alloc_time(hid_t plist_id, H5D_alloc_time_t alloc_time ) herr_t H5Pget_alloc_time(hid_t plist_id, H5D_alloc_time_t *alloc_time ) herr_t H5Pset_filter(hid_t plist, H5Z_filter_t filter, unsigned int flags, size_t cd_nelmts, unsigned int* cd_values ) htri_t H5Pall_filters_avail(hid_t dcpl_id) int H5Pget_nfilters(hid_t plist) H5Z_filter_t H5Pget_filter(hid_t plist, unsigned int filter_number, unsigned int *flags, size_t *cd_nelmts, unsigned int* cd_values, size_t namelen, char* name ) herr_t H5Pget_filter_by_id( hid_t plist_id, H5Z_filter_t filter, unsigned int *flags, size_t *cd_nelmts, unsigned int* cd_values, size_t namelen, char* name) herr_t H5Pmodify_filter(hid_t plist, H5Z_filter_t filter, unsigned int flags, size_t cd_nelmts, unsigned int *cd_values) herr_t H5Premove_filter(hid_t plist, H5Z_filter_t filter ) herr_t H5Pset_fletcher32(hid_t plist) herr_t H5Pset_shuffle(hid_t plist_id) herr_t H5Pset_szip(hid_t plist, unsigned int options_mask, unsigned int pixels_per_block) herr_t H5Pset_scaleoffset(hid_t plist, H5Z_SO_scale_type_t scale_type, int scale_factor) 1.9.233 ssize_t H5Pget_virtual_dsetname(hid_t dcpl_id, size_t index, char *name, size_t size) 1.9.233 ssize_t H5Pget_virtual_filename(hid_t dcpl_id, size_t index, char *name, size_t size) 1.9.233 herr_t H5Pget_virtual_count(hid_t dcpl_id, size_t *count) 1.9.233 herr_t H5Pset_virtual(hid_t dcpl_id, hid_t vspace_id, const char *src_file_name, const char *src_dset_name, hid_t src_space_id) 1.9.233 hid_t H5Pget_virtual_vspace(hid_t dcpl_id, size_t index) 1.9.233 hid_t H5Pget_virtual_srcspace(hid_t dcpl_id, size_t index) # Dataset access herr_t H5Pset_edc_check(hid_t plist, H5Z_EDC_t check) H5Z_EDC_t H5Pget_edc_check(hid_t plist) herr_t H5Pset_chunk_cache( hid_t dapl_id, size_t rdcc_nslots, size_t rdcc_nbytes, double rdcc_w0 ) herr_t H5Pget_chunk_cache( hid_t dapl_id, size_t *rdcc_nslots, size_t *rdcc_nbytes, double *rdcc_w0 ) 1.9.233 herr_t H5Pset_virtual_view(hid_t plist_id, H5D_vds_view_t view) 1.9.233 herr_t H5Pget_virtual_view(hid_t plist_id, H5D_vds_view_t *view) 1.9.233 herr_t H5Pset_virtual_printf_gap(hid_t plist_id, hsize_t gap_size) 1.9.233 herr_t H5Pget_virtual_printf_gap(hid_t plist_id, hsize_t *gap_size) MPI herr_t H5Pset_dxpl_mpio( hid_t dxpl_id, H5FD_mpio_xfer_t xfer_mode ) MPI herr_t H5Pget_dxpl_mpio( hid_t dxpl_id, H5FD_mpio_xfer_t* xfer_mode ) # Other properties herr_t H5Pset_sieve_buf_size(hid_t fapl_id, size_t size) herr_t H5Pget_sieve_buf_size(hid_t fapl_id, size_t *size) herr_t H5Pset_nlinks(hid_t plist_id, size_t nlinks) herr_t H5Pget_nlinks(hid_t plist_id, size_t *nlinks) herr_t H5Pset_elink_prefix(hid_t plist_id, char *prefix) ssize_t H5Pget_elink_prefix(hid_t plist_id, char *prefix, size_t size) hid_t H5Pget_elink_fapl(hid_t lapl_id) herr_t H5Pset_elink_fapl(hid_t lapl_id, hid_t fapl_id) herr_t H5Pset_create_intermediate_group(hid_t plist_id, unsigned crt_intmd) herr_t H5Pget_create_intermediate_group(hid_t plist_id, unsigned *crt_intmd) herr_t H5Pset_copy_object(hid_t plist_id, unsigned crt_intmd) herr_t H5Pget_copy_object(hid_t plist_id, unsigned *crt_intmd) herr_t H5Pset_char_encoding(hid_t plist_id, H5T_cset_t encoding) herr_t H5Pget_char_encoding(hid_t plist_id, H5T_cset_t *encoding) herr_t H5Pset_obj_track_times( hid_t ocpl_id, hbool_t track_times ) herr_t H5Pget_obj_track_times( hid_t ocpl_id, hbool_t *track_times ) herr_t H5Pset_local_heap_size_hint(hid_t plist_id, size_t size_hint) herr_t H5Pget_local_heap_size_hint(hid_t plist_id, size_t *size_hint) herr_t H5Pset_link_phase_change(hid_t plist_id, unsigned max_compact, unsigned min_dense) herr_t H5Pget_link_phase_change(hid_t plist_id, unsigned *max_compact , unsigned *min_dense) herr_t H5Pset_est_link_info(hid_t plist_id, unsigned est_num_entries, unsigned est_name_len) herr_t H5Pget_est_link_info(hid_t plist_id, unsigned *est_num_entries , unsigned *est_name_len) herr_t H5Pset_link_creation_order(hid_t plist_id, unsigned crt_order_flags) herr_t H5Pget_link_creation_order(hid_t plist_id, unsigned *crt_order_flags) herr_t H5Pset_libver_bounds(hid_t fapl_id, H5F_libver_t libver_low, H5F_libver_t libver_high) herr_t H5Pget_libver_bounds(hid_t fapl_id, H5F_libver_t *libver_low, H5F_libver_t *libver_high) herr_t H5Pset_alignment(hid_t plist_id, hsize_t threshold, hsize_t alignment) herr_t H5Pget_alignment(hid_t plist_id, hsize_t *threshold, hsize_t *alignment) # MPI functions MPI herr_t H5Pset_fapl_mpio(hid_t fapl_id, MPI_Comm comm, MPI_Info info) MPI herr_t H5Pget_fapl_mpio(hid_t fapl_id, MPI_Comm *comm, MPI_Info *info) # === H5R - Reference API =================================================== herr_t H5Rcreate(void *ref, hid_t loc_id, char *name, H5R_type_t ref_type, hid_t space_id) hid_t H5Rdereference(hid_t obj_id, H5R_type_t ref_type, void *ref) hid_t H5Rget_region(hid_t dataset, H5R_type_t ref_type, void *ref) H5G_obj_t H5Rget_obj_type(hid_t id, H5R_type_t ref_type, void *ref) ssize_t H5Rget_name(hid_t loc_id, H5R_type_t ref_type, void *ref, char *name, size_t size) # === H5S - Dataspaces ======================================================== # Basic operations hid_t H5Screate(H5S_class_t type) hid_t H5Scopy(hid_t space_id ) herr_t H5Sclose(hid_t space_id) # Simple dataspace operations hid_t H5Screate_simple(int rank, hsize_t *dims, hsize_t *maxdims) htri_t H5Sis_simple(hid_t space_id) herr_t H5Soffset_simple(hid_t space_id, hssize_t *offset ) int H5Sget_simple_extent_ndims(hid_t space_id) int H5Sget_simple_extent_dims(hid_t space_id, hsize_t *dims, hsize_t *maxdims) hssize_t H5Sget_simple_extent_npoints(hid_t space_id) H5S_class_t H5Sget_simple_extent_type(hid_t space_id) # Extents herr_t H5Sextent_copy(hid_t dest_space_id, hid_t source_space_id ) herr_t H5Sset_extent_simple(hid_t space_id, int rank, hsize_t *current_size, hsize_t *maximum_size ) herr_t H5Sset_extent_none(hid_t space_id) # Dataspace selection H5S_sel_type H5Sget_select_type(hid_t space_id) hssize_t H5Sget_select_npoints(hid_t space_id) herr_t H5Sget_select_bounds(hid_t space_id, hsize_t *start, hsize_t *end) herr_t H5Sselect_all(hid_t space_id) herr_t H5Sselect_none(hid_t space_id) htri_t H5Sselect_valid(hid_t space_id) hssize_t H5Sget_select_elem_npoints(hid_t space_id) herr_t H5Sget_select_elem_pointlist(hid_t space_id, hsize_t startpoint, hsize_t numpoints, hsize_t *buf) herr_t H5Sselect_elements(hid_t space_id, H5S_seloper_t op, size_t num_elements, const hsize_t *coord) hssize_t H5Sget_select_hyper_nblocks(hid_t space_id ) herr_t H5Sget_select_hyper_blocklist(hid_t space_id, hsize_t startblock, hsize_t numblocks, hsize_t *buf ) herr_t H5Sselect_hyperslab(hid_t space_id, H5S_seloper_t op, hsize_t *start, hsize_t *_stride, hsize_t *count, hsize_t *_block) herr_t H5Sencode(hid_t obj_id, void *buf, size_t *nalloc) hid_t H5Sdecode(void *buf) 1.9.233 htri_t H5Sis_regular_hyperslab(hid_t spaceid) 1.9.233 htri_t H5Sget_regular_hyperslab(hid_t spaceid, hsize_t* start, hsize_t* stride, hsize_t* count, hsize_t* block) # === H5T - Datatypes ========================================================= # General operations hid_t H5Tcreate(H5T_class_t type, size_t size) hid_t H5Topen(hid_t loc, char* name) herr_t H5Tcommit(hid_t loc_id, char* name, hid_t type) htri_t H5Tcommitted(hid_t type) hid_t H5Tcopy(hid_t type_id) htri_t H5Tequal(hid_t type_id1, hid_t type_id2 ) herr_t H5Tlock(hid_t type_id) H5T_class_t H5Tget_class(hid_t type_id) size_t H5Tget_size(hid_t type_id) hid_t H5Tget_super(hid_t type) htri_t H5Tdetect_class(hid_t type_id, H5T_class_t dtype_class) herr_t H5Tclose(hid_t type_id) hid_t H5Tget_native_type(hid_t type_id, H5T_direction_t direction) herr_t H5Tcommit2(hid_t loc_id, char *name, hid_t dtype_id, hid_t lcpl_id, hid_t tcpl_id, hid_t tapl_id) hid_t H5Tdecode(unsigned char *buf) herr_t H5Tencode(hid_t obj_id, unsigned char *buf, size_t *nalloc) H5T_conv_t H5Tfind(hid_t src_id, hid_t dst_id, H5T_cdata_t **pcdata) herr_t H5Tconvert(hid_t src_id, hid_t dst_id, size_t nelmts, void *buf, void *background, hid_t plist_id) herr_t H5Tregister(H5T_pers_t pers, char *name, hid_t src_id, hid_t dst_id, H5T_conv_t func) herr_t H5Tunregister(H5T_pers_t pers, char *name, hid_t src_id, hid_t dst_id, H5T_conv_t func) # Atomic datatypes herr_t H5Tset_size(hid_t type_id, size_t size) H5T_order_t H5Tget_order(hid_t type_id) herr_t H5Tset_order(hid_t type_id, H5T_order_t order) hsize_t H5Tget_precision(hid_t type_id) herr_t H5Tset_precision(hid_t type_id, size_t prec) int H5Tget_offset(hid_t type_id) herr_t H5Tset_offset(hid_t type_id, size_t offset) herr_t H5Tget_pad(hid_t type_id, H5T_pad_t * lsb, H5T_pad_t * msb ) herr_t H5Tset_pad(hid_t type_id, H5T_pad_t lsb, H5T_pad_t msb ) H5T_sign_t H5Tget_sign(hid_t type_id) herr_t H5Tset_sign(hid_t type_id, H5T_sign_t sign) herr_t H5Tget_fields(hid_t type_id, size_t *spos, size_t *epos, size_t *esize, size_t *mpos, size_t *msize ) herr_t H5Tset_fields(hid_t type_id, size_t spos, size_t epos, size_t esize, size_t mpos, size_t msize ) size_t H5Tget_ebias(hid_t type_id) herr_t H5Tset_ebias(hid_t type_id, size_t ebias) H5T_norm_t H5Tget_norm(hid_t type_id) herr_t H5Tset_norm(hid_t type_id, H5T_norm_t norm) H5T_pad_t H5Tget_inpad(hid_t type_id) herr_t H5Tset_inpad(hid_t type_id, H5T_pad_t inpad) H5T_cset_t H5Tget_cset(hid_t type_id) herr_t H5Tset_cset(hid_t type_id, H5T_cset_t cset) H5T_str_t H5Tget_strpad(hid_t type_id) herr_t H5Tset_strpad(hid_t type_id, H5T_str_t strpad) # VLENs hid_t H5Tvlen_create(hid_t base_type_id) htri_t H5Tis_variable_str(hid_t dtype_id) # Compound data types int H5Tget_nmembers(hid_t type_id) H5T_class_t H5Tget_member_class(hid_t type_id, int member_no) char* H5Tget_member_name(hid_t type_id, unsigned membno) hid_t H5Tget_member_type(hid_t type_id, unsigned membno) int H5Tget_member_offset(hid_t type_id, int membno) int H5Tget_member_index(hid_t type_id, char* name) herr_t H5Tinsert(hid_t parent_id, char *name, size_t offset, hid_t member_id) herr_t H5Tpack(hid_t type_id) # Enumerated types hid_t H5Tenum_create(hid_t base_id) herr_t H5Tenum_insert(hid_t type, char *name, void *value) herr_t H5Tenum_nameof( hid_t type, void *value, char *name, size_t size ) herr_t H5Tenum_valueof( hid_t type, char *name, void *value ) herr_t H5Tget_member_value(hid_t type, unsigned int memb_no, void *value ) # Array data types hid_t H5Tarray_create(hid_t base_id, int ndims, hsize_t *dims, int *perm) int H5Tget_array_ndims(hid_t type_id) int H5Tget_array_dims(hid_t type_id, hsize_t *dims, int *perm) # Opaque data types herr_t H5Tset_tag(hid_t type_id, char* tag) char* H5Tget_tag(hid_t type_id) # === H5Z - Filters ========================================================= htri_t H5Zfilter_avail(H5Z_filter_t id_) herr_t H5Zget_filter_info(H5Z_filter_t filter_, unsigned int *filter_config_flags) hdf5_hl: # === H5DS - Dimension Scales API ============================================= ERROR herr_t H5DSattach_scale(hid_t did, hid_t dsid, unsigned int idx) ERROR herr_t H5DSdetach_scale(hid_t did, hid_t dsid, unsigned int idx) ERROR htri_t H5DSis_attached(hid_t did, hid_t dsid, unsigned int idx) ERROR herr_t H5DSset_scale(hid_t dsid, char *dimname) ERROR int H5DSget_num_scales(hid_t did, unsigned int dim) ERROR herr_t H5DSset_label(hid_t did, unsigned int idx, char *label) ERROR ssize_t H5DSget_label(hid_t did, unsigned int idx, char *label, size_t size) ERROR ssize_t H5DSget_scale_name(hid_t did, char *name, size_t size) ERROR htri_t H5DSis_scale(hid_t did) ERROR herr_t H5DSiterate_scales(hid_t did, unsigned int dim, int *idx, H5DS_iterate_t visitor, void *visitor_data) h5py-2.7.1/h5py/h5ds.pyx0000644000175000017500000000731013025343121016351 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. """ Low-level HDF5 "H5G" group interface. """ # Compile-time imports from h5d cimport DatasetID from utils cimport emalloc, efree from ._objects import phil, with_phil @with_phil def set_scale(DatasetID dset not None, char* dimname=''): """(DatasetID dset, STRING dimname) Convert dataset dset to a dimension scale, with optional name dimname. """ H5DSset_scale(dset.id, dimname) @with_phil def is_scale(DatasetID dset not None): """(DatasetID dset) Determines whether dset is a dimension scale. """ return (H5DSis_scale(dset.id)) @with_phil def attach_scale(DatasetID dset not None, DatasetID dscale not None, unsigned int idx): H5DSattach_scale(dset.id, dscale.id, idx) @with_phil def is_attached(DatasetID dset not None, DatasetID dscale not None, unsigned int idx): return (H5DSis_attached(dset.id, dscale.id, idx)) @with_phil def detach_scale(DatasetID dset not None, DatasetID dscale not None, unsigned int idx): H5DSdetach_scale(dset.id, dscale.id, idx) @with_phil def get_num_scales(DatasetID dset not None, unsigned int dim): return H5DSget_num_scales(dset.id, dim) @with_phil def set_label(DatasetID dset not None, unsigned int idx, char* label): H5DSset_label(dset.id, idx, label) @with_phil def get_label(DatasetID dset not None, unsigned int idx): cdef ssize_t size cdef char* label label = NULL size = H5DSget_label(dset.id, idx, NULL, 0) if size <= 0: return b'' label = emalloc(sizeof(char)*(size+1)) try: H5DSget_label(dset.id, idx, label, size+1) plabel = label return plabel finally: efree(label) @with_phil def get_scale_name(DatasetID dscale not None): cdef ssize_t namelen cdef char* name = NULL namelen = H5DSget_scale_name(dscale.id, NULL, 0) if namelen <= 0: return b'' name = emalloc(sizeof(char)*(namelen+1)) try: H5DSget_scale_name(dscale.id, name, namelen+1) pname = name return pname finally: efree(name) cdef class _DimensionScaleVisitor: cdef object func cdef object retval def __init__(self, func): self.func = func self.retval = None cdef herr_t cb_ds_iter(hid_t dset, unsigned int dim, hid_t scale, void* vis_in) except 2: cdef _DimensionScaleVisitor vis = <_DimensionScaleVisitor>vis_in # we did not retrieve the scale identifier using the normal machinery, # so we need to inc_ref it before using it to create a DatasetID. H5Iinc_ref(scale) vis.retval = vis.func(DatasetID(scale)) if vis.retval is not None: return 1 return 0 @with_phil def iterate(DatasetID dset not None, unsigned int dim, object func, int startidx=0): """ (DatasetID loc, UINT dim, CALLABLE func, UINT startidx=0) => Return value from func Iterate a callable (function, method or callable object) over the members of a group. Your callable shoutld have the signature:: func(STRING name) => Result Returning None continues iteration; returning anything else aborts iteration and returns that value. Keywords: """ if startidx < 0: raise ValueError("Starting index must be non-negative") cdef int i = startidx cdef _DimensionScaleVisitor vis = _DimensionScaleVisitor(func) H5DSiterate_scales(dset.id, dim, &i, cb_ds_iter, vis) return vis.retval h5py-2.7.1/h5py/tests/0000755000175000017500000000000013152363123016112 5ustar tcaswelltcaswell00000000000000h5py-2.7.1/h5py/tests/old/0000755000175000017500000000000013152363123016670 5ustar tcaswelltcaswell00000000000000h5py-2.7.1/h5py/tests/old/test_objects.py0000644000175000017500000000167113114361073021737 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. from __future__ import absolute_import from h5py import _objects as o from ..common import TestCase class TestObjects(TestCase): def test_invalid(self): # Check for segfault on close oid = o.ObjectID(0) del oid oid = o.ObjectID(1) del oid def test_equality(self): # Identifier-based equality oid1 = o.ObjectID(42) oid2 = o.ObjectID(42) oid3 = o.ObjectID(43) self.assertEqual(oid1, oid2) self.assertNotEqual(oid1, oid3) def test_hash(self): # Default objects are not hashable oid = o.ObjectID(42) with self.assertRaises(TypeError): hash(oid) h5py-2.7.1/h5py/tests/old/test_attrs.py0000644000175000017500000001112713114361073021440 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. """ Attributes testing module Covers all operations which access the .attrs property, with the exception of data read/write and type conversion. Those operations are tested by module test_attrs_data. """ from __future__ import absolute_import import six import numpy as np import collections from ..common import TestCase, ut from h5py.highlevel import File from h5py import h5a, h5t from h5py.highlevel import AttributeManager class BaseAttrs(TestCase): def setUp(self): self.f = File(self.mktemp(), 'w') def tearDown(self): if self.f: self.f.close() class TestAccess(BaseAttrs): """ Feature: Attribute creation/retrieval via special methods """ def test_create(self): """ Attribute creation by direct assignment """ self.f.attrs['a'] = 4.0 self.assertEqual(list(self.f.attrs.keys()), ['a']) self.assertEqual(self.f.attrs['a'], 4.0) def test_overwrite(self): """ Attributes are silently overwritten """ self.f.attrs['a'] = 4.0 self.f.attrs['a'] = 5.0 self.assertEqual(self.f.attrs['a'], 5.0) def test_rank(self): """ Attribute rank is preserved """ self.f.attrs['a'] = (4.0, 5.0) self.assertEqual(self.f.attrs['a'].shape, (2,)) self.assertArrayEqual(self.f.attrs['a'], np.array((4.0,5.0))) def test_single(self): """ Attributes of shape (1,) don't become scalars """ self.f.attrs['a'] = np.ones((1,)) out = self.f.attrs['a'] self.assertEqual(out.shape, (1,)) self.assertEqual(out[()], 1) def test_access_exc(self): """ Attempt to access missing item raises KeyError """ with self.assertRaises(KeyError): self.f.attrs['a'] class TestDelete(BaseAttrs): """ Feature: Deletion of attributes using __delitem__ """ def test_delete(self): """ Deletion via "del" """ self.f.attrs['a'] = 4.0 self.assertIn('a', self.f.attrs) del self.f.attrs['a'] self.assertNotIn('a', self.f.attrs) def test_delete_exc(self): """ Attempt to delete missing item raises KeyError """ with self.assertRaises(KeyError): del self.f.attrs['a'] class TestUnicode(BaseAttrs): """ Feature: Attributes can be accessed via Unicode or byte strings """ def test_ascii(self): """ Access via pure-ASCII byte string """ self.f.attrs[b"ascii"] = 42 out = self.f.attrs[b"ascii"] self.assertEqual(out, 42) def test_raw(self): """ Access via non-ASCII byte string """ name = b"non-ascii\xfe" self.f.attrs[name] = 42 out = self.f.attrs[name] self.assertEqual(out, 42) def test_unicode(self): """ Access via Unicode string with non-ascii characters """ name = u"Omega" + six.unichr(0x03A9) self.f.attrs[name] = 42 out = self.f.attrs[name] self.assertEqual(out, 42) class TestCreate(BaseAttrs): """ Options for explicit attribute creation """ def test_named(self): """ Attributes created from named types link to the source type object """ self.f['type'] = np.dtype('u8') self.f.attrs.create('x', 42, dtype=self.f['type']) self.assertEqual(self.f.attrs['x'], 42) aid = h5a.open(self.f.id, b'x') htype = aid.get_type() htype2 = self.f['type'].id self.assertEqual(htype, htype2) self.assertTrue(htype.committed()) class TestMutableMapping(BaseAttrs): '''Tests if the registration of AttributeManager as a MutableMapping behaves as expected ''' def test_resolution(self): assert issubclass(AttributeManager, collections.MutableMapping) assert isinstance(self.f.attrs, collections.MutableMapping) def test_validity(self): ''' Test that the required functions are implemented. ''' AttributeManager.__getitem__ AttributeManager.__setitem__ AttributeManager.__delitem__ AttributeManager.__iter__ AttributeManager.__len__ class TestVlen(BaseAttrs): def test_vlen(self): a = np.array([np.arange(3), np.arange(4)], dtype=h5t.special_dtype(vlen=int)) self.f.attrs['a'] = a self.assertArrayEqual(self.f.attrs['a'][0], a[0]) h5py-2.7.1/h5py/tests/old/__init__.py0000644000175000017500000000173013114361073021002 0ustar tcaswelltcaswell00000000000000 from __future__ import absolute_import from . import ( test_attrs, test_attrs_data, test_base, test_dataset, test_datatype, test_dimension_scales, test_file, test_file_image, test_group, test_h5, test_h5f, test_h5p, test_h5t, test_objects, test_selections, test_slicing ) MODULES = ( test_attrs, test_attrs_data, test_base, test_dataset, test_datatype, test_dimension_scales, test_file, test_file_image, test_group, test_h5, test_h5f, test_h5p, test_h5t, test_objects, test_selections, test_slicing ) h5py-2.7.1/h5py/tests/old/test_attrs_data.py0000644000175000017500000001674213114361073022441 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. """ Attribute data transfer testing module Covers all data read/write and type-conversion operations for attributes. """ from __future__ import absolute_import import six import numpy as np from ..common import TestCase, ut import h5py from h5py import h5a, h5s, h5t from h5py.highlevel import File from h5py._hl.base import is_empty_dataspace class BaseAttrs(TestCase): def setUp(self): self.f = File(self.mktemp(), 'w') def tearDown(self): if self.f: self.f.close() class TestScalar(BaseAttrs): """ Feature: Scalar types map correctly to array scalars """ def test_int(self): """ Integers are read as correct NumPy type """ self.f.attrs['x'] = np.array(1, dtype=np.int8) out = self.f.attrs['x'] self.assertIsInstance(out, np.int8) def test_compound(self): """ Compound scalars are read as numpy.void """ dt = np.dtype([('a','i'),('b','f')]) data = np.array((1,4.2), dtype=dt) self.f.attrs['x'] = data out = self.f.attrs['x'] self.assertIsInstance(out, np.void) self.assertEqual(out, data) self.assertEqual(out['b'], data['b']) class TestArray(BaseAttrs): """ Feature: Non-scalar types are correctly retrieved as ndarrays """ def test_single(self): """ Single-element arrays are correctly recovered """ data = np.ndarray((1,), dtype='f') self.f.attrs['x'] = data out = self.f.attrs['x'] self.assertIsInstance(out, np.ndarray) self.assertEqual(out.shape, (1,)) def test_multi(self): """ Rank-1 arrays are correctly recovered """ data = np.ndarray((42,), dtype='f') data[:] = 42.0 data[10:35] = -47.0 self.f.attrs['x'] = data out = self.f.attrs['x'] self.assertIsInstance(out, np.ndarray) self.assertEqual(out.shape, (42,)) self.assertArrayEqual(out, data) class TestTypes(BaseAttrs): """ Feature: All supported types can be stored in attributes """ def test_int(self): """ Storage of integer types """ dtypes = (np.int8, np.int16, np.int32, np.int64, np.uint8, np.uint16, np.uint32, np.uint64) for dt in dtypes: data = np.ndarray((1,), dtype=dt) data[...] = 42 self.f.attrs['x'] = data out = self.f.attrs['x'] self.assertEqual(out.dtype, dt) self.assertArrayEqual(out, data) def test_float(self): """ Storage of floating point types """ dtypes = tuple(np.dtype(x) for x in ('f4','f8')) for dt in dtypes: data = np.ndarray((1,), dtype=dt) data[...] = 42.3 self.f.attrs['x'] = data out = self.f.attrs['x'] self.assertEqual(out.dtype, dt) self.assertArrayEqual(out, data) def test_complex(self): """ Storage of complex types """ dtypes = tuple(np.dtype(x) for x in ('c8','c16')) for dt in dtypes: data = np.ndarray((1,), dtype=dt) data[...] = -4.2j+35.9 self.f.attrs['x'] = data out = self.f.attrs['x'] self.assertEqual(out.dtype, dt) self.assertArrayEqual(out, data) def test_string(self): """ Storage of fixed-length strings """ dtypes = tuple(np.dtype(x) for x in ('|S1', '|S10')) for dt in dtypes: data = np.ndarray((1,), dtype=dt) data[...] = 'h' self.f.attrs['x'] = data out = self.f.attrs['x'] self.assertEqual(out.dtype, dt) self.assertEqual(out[0], data[0]) def test_bool(self): """ Storage of NumPy booleans """ data = np.ndarray((2,), dtype=np.bool_) data[...] = True, False self.f.attrs['x'] = data out = self.f.attrs['x'] self.assertEqual(out.dtype, data.dtype) self.assertEqual(out[0], data[0]) self.assertEqual(out[1], data[1]) def test_vlen_string_array(self): """ Storage of vlen byte string arrays""" dt = h5py.special_dtype(vlen=bytes) data = np.ndarray((2,), dtype=dt) data[...] = b"Hello", b"Hi there! This is HDF5!" self.f.attrs['x'] = data out = self.f.attrs['x'] self.assertEqual(out.dtype, dt) self.assertEqual(out[0], data[0]) self.assertEqual(out[1], data[1]) def test_string_scalar(self): """ Storage of variable-length byte string scalars (auto-creation) """ self.f.attrs['x'] = b'Hello' out = self.f.attrs['x'] self.assertEqual(out,b'Hello') self.assertEqual(type(out), bytes) aid = h5py.h5a.open(self.f.id, b"x") tid = aid.get_type() self.assertEqual(type(tid), h5py.h5t.TypeStringID) self.assertEqual(tid.get_cset(), h5py.h5t.CSET_ASCII) self.assertTrue(tid.is_variable_str()) def test_unicode_scalar(self): """ Storage of variable-length unicode strings (auto-creation) """ self.f.attrs['x'] = u"Hello" + six.unichr(0x2340) + u"!!" out = self.f.attrs['x'] self.assertEqual(out, u"Hello" + six.unichr(0x2340) + u"!!") self.assertEqual(type(out), six.text_type) aid = h5py.h5a.open(self.f.id, b"x") tid = aid.get_type() self.assertEqual(type(tid), h5py.h5t.TypeStringID) self.assertEqual(tid.get_cset(), h5py.h5t.CSET_UTF8) self.assertTrue(tid.is_variable_str()) class TestEmpty(BaseAttrs): def setUp(self): BaseAttrs.setUp(self) sid = h5s.create(h5s.NULL) tid = h5t.C_S1.copy() tid.set_size(10) aid = h5a.create(self.f.id, b'x', tid, sid) self.empty_obj = h5py.Empty(np.dtype("S10")) def test_read(self): self.assertEqual( self.empty_obj, self.f.attrs['x'] ) def test_write(self): self.f.attrs["y"] = self.empty_obj self.assertTrue(is_empty_dataspace(h5a.open(self.f.id, b'y'))) def test_modify(self): with self.assertRaises(IOError): self.f.attrs.modify('x', 1) def test_values(self): # list() is for Py3 where these are iterators values = list(self.f.attrs.values()) self.assertEqual( [self.empty_obj], values ) def test_items(self): items = list(self.f.attrs.items()) self.assertEqual( [(u"x", self.empty_obj)], items ) def test_itervalues(self): values = list(six.itervalues(self.f.attrs)) self.assertEqual( [self.empty_obj], values ) def test_iteritems(self): items = list(six.iteritems(self.f.attrs)) self.assertEqual( [(u"x", self.empty_obj)], items ) class TestWriteException(BaseAttrs): """ Ensure failed attribute writes don't leave garbage behind. """ def test_write(self): """ ValueError on string write wipes out attribute """ s = b"Hello\x00\Hello" try: self.f.attrs['x'] = s except ValueError: pass with self.assertRaises(KeyError): self.f.attrs['x'] h5py-2.7.1/h5py/tests/old/test_base.py0000644000175000017500000000425313114361073021217 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. """ Common high-level operations test Tests features common to all high-level objects, like the .name property. """ from __future__ import absolute_import import six from h5py import File from ..common import ut, TestCase, UNICODE_FILENAMES import numpy as np import os import tempfile class BaseTest(TestCase): def setUp(self): self.f = File(self.mktemp(), 'w') def tearDown(self): if self.f: self.f.close() class TestName(BaseTest): """ Feature: .name attribute returns the object name """ def test_anonymous(self): """ Anomymous objects have name None """ grp = self.f.create_group(None) self.assertIs(grp.name, None) class TestRepr(BaseTest): """ repr() works correctly with Unicode names """ USTRING = six.unichr(0xfc) + six.unichr(0xdf) def _check_type(self, obj): if six.PY2: self.assertIsInstance(repr(obj), bytes) else: self.assertIsInstance(repr(obj), six.text_type) def test_group(self): """ Group repr() with unicode """ grp = self.f.create_group(self.USTRING) self._check_type(grp) def test_dataset(self): """ Dataset repr() with unicode """ dset = self.f.create_dataset(self.USTRING, (1,)) self._check_type(dset) def test_namedtype(self): """ Named type repr() with unicode """ self.f['type'] = np.dtype('f') typ = self.f['type'] self._check_type(typ) @ut.skipIf(not UNICODE_FILENAMES, "Filesystem unicode support required") def test_file(self): """ File object repr() with unicode """ fname = tempfile.mktemp(self.USTRING+u'.hdf5') try: with File(fname,'w') as f: self._check_type(f) finally: try: os.unlink(fname) except Exception: pass h5py-2.7.1/h5py/tests/old/test_dataset.py0000644000175000017500000011314513114361073021733 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. """ Dataset testing operations. Tests all dataset operations, including creation, with the exception of: 1. Slicing operations for read and write, handled by module test_slicing 2. Type conversion for read and write (currently untested) """ from __future__ import absolute_import import sys import six import numpy as np from ..common import ut, TestCase from h5py.highlevel import File, Group, Dataset from h5py._hl.base import is_empty_dataspace from h5py import h5t import h5py class BaseDataset(TestCase): def setUp(self): self.f = File(self.mktemp(), 'w') def tearDown(self): if self.f: self.f.close() class TestRepr(BaseDataset): """ Feature: repr(Dataset) behaves sensibly """ def test_repr_open(self): """ repr() works on live and dead datasets """ ds = self.f.create_dataset('foo', (4,)) self.assertIsInstance(repr(ds), six.string_types) self.f.close() self.assertIsInstance(repr(ds), six.string_types) class TestCreateShape(BaseDataset): """ Feature: Datasets can be created from a shape only """ def test_create_scalar(self): """ Create a scalar dataset """ dset = self.f.create_dataset('foo', ()) self.assertEqual(dset.shape, ()) def test_create_simple(self): """ Create a size-1 dataset """ dset = self.f.create_dataset('foo', (1,)) self.assertEqual(dset.shape, (1,)) def test_create_extended(self): """ Create an extended dataset """ dset = self.f.create_dataset('foo', (63,)) self.assertEqual(dset.shape, (63,)) self.assertEqual(dset.size, 63) dset = self.f.create_dataset('bar', (6, 10)) self.assertEqual(dset.shape, (6, 10)) self.assertEqual(dset.size, (60)) def test_default_dtype(self): """ Confirm that the default dtype is float """ dset = self.f.create_dataset('foo', (63,)) self.assertEqual(dset.dtype, np.dtype('=f4')) def test_missing_shape(self): """ Missing shape raises TypeError """ with self.assertRaises(TypeError): self.f.create_dataset('foo') def test_long_double(self): """ Confirm that the default dtype is float """ dset = self.f.create_dataset('foo', (63,), dtype=np.longdouble) self.assertEqual(dset.dtype, np.longdouble) @ut.skipIf(not hasattr(np, "complex256"), "No support for complex256") def test_complex256(self): """ Confirm that the default dtype is float """ dset = self.f.create_dataset('foo', (63,), dtype=np.dtype('complex256')) self.assertEqual(dset.dtype, np.dtype('complex256')) class TestCreateData(BaseDataset): """ Feature: Datasets can be created from existing data """ def test_create_scalar(self): """ Create a scalar dataset from existing array """ data = np.ones((), 'f') dset = self.f.create_dataset('foo', data=data) self.assertEqual(dset.shape, data.shape) def test_create_extended(self): """ Create an extended dataset from existing data """ data = np.ones((63,), 'f') dset = self.f.create_dataset('foo', data=data) self.assertEqual(dset.shape, data.shape) def test_dataset_intermediate_group(self): """ Create dataset with missing intermediate groups """ ds = self.f.create_dataset("/foo/bar/baz", shape=(10, 10), dtype=' all fields out, format = sel.read_dtypes(dt, ()) self.assertEqual(out, format) self.assertEqual(out, dt) # Explicit selection of fields -> requested fields out, format = sel.read_dtypes(dt, ('a','b')) self.assertEqual(out, format) self.assertEqual(out, np.dtype( [('a','i'), ('b','f')] )) # Explicit selection of exactly one field -> no fields out, format = sel.read_dtypes(dt, ('a',)) self.assertEqual(out, np.dtype('i')) self.assertEqual(format, np.dtype( [('a','i')] )) class TestScalarSliceRules(TestCase): """ Internal feature: selections rules for scalar datasets """ def setUp(self): self.f = h5py.File(self.mktemp(), 'w') self.dsid = self.f.create_dataset('x', ()).id def tearDown(self): if self.f: self.f.close() def test_args(self): """ Permissible arguments for scalar slicing """ shape, selection = sel.read_selections_scalar(self.dsid, ()) self.assertEqual(shape, None) self.assertEqual(selection.get_select_npoints(), 1) shape, selection = sel.read_selections_scalar(self.dsid, (Ellipsis,)) self.assertEqual(shape, ()) self.assertEqual(selection.get_select_npoints(), 1) with self.assertRaises(ValueError): shape, selection = sel.read_selections_scalar(self.dsid, (1,)) h5py-2.7.1/h5py/tests/old/test_group.py0000644000175000017500000010162313114361073021440 0ustar tcaswelltcaswell00000000000000# -*- coding: utf-8 -*- # This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. """ Group test module. Tests all methods and properties of Group objects, with the following exceptions: 1. Method create_dataset is tested in module test_dataset """ from __future__ import absolute_import import collections import numpy as np import os import os.path import sys from tempfile import mkdtemp import six from ..common import ut, TestCase import h5py from h5py.highlevel import File, Group, SoftLink, HardLink, ExternalLink from h5py.highlevel import Dataset, Datatype from h5py import h5t from h5py._hl.compat import filename_encode # If we can't encode unicode filenames, there's not much point failing tests # which must fail try: filename_encode(u"α") except UnicodeEncodeError: NO_FS_UNICODE = True else: NO_FS_UNICODE = False class BaseGroup(TestCase): def setUp(self): self.f = File(self.mktemp(), 'w') def tearDown(self): if self.f: self.f.close() class TestRepr(BaseGroup): """ Feature: repr() works sensibly on Group objects """ def test_repr(self): """ repr() works on Group objects """ g = self.f.create_group('foo') self.assertIsInstance(g, six.string_types) self.f.close() self.assertIsInstance(g, six.string_types) class TestCreate(BaseGroup): """ Feature: New groups can be created via .create_group method """ def test_create(self): """ Simple .create_group call """ grp = self.f.create_group('foo') self.assertIsInstance(grp, Group) def test_create_intermediate(self): """ Intermediate groups can be created automatically """ grp = self.f.create_group('foo/bar/baz') self.assertEqual(grp.name, '/foo/bar/baz') def test_create_exception(self): """ Name conflict causes group creation to fail with ValueError """ self.f.create_group('foo') with self.assertRaises(ValueError): self.f.create_group('foo') def test_unicode(self): """ Unicode names are correctly stored """ name = u"/Name" + six.unichr(0x4500) group = self.f.create_group(name) self.assertEqual(group.name, name) self.assertEqual(group.id.links.get_info(name.encode('utf8')).cset, h5t.CSET_UTF8) def test_unicode_default(self): """ Unicode names convertible to ASCII are stored as ASCII (issue 239) """ name = u"/Hello, this is a name" group = self.f.create_group(name) self.assertEqual(group.name, name) self.assertEqual(group.id.links.get_info(name.encode('utf8')).cset, h5t.CSET_ASCII) def test_appropriate_low_level_id(self): " Binding a group to a non-group identifier fails with ValueError " dset = self.f.create_dataset('foo', [1]) with self.assertRaises(ValueError): Group(dset.id) class TestDatasetAssignment(BaseGroup): """ Feature: Datasets can be created by direct assignment of data """ def test_ndarray(self): """ Dataset auto-creation by direct assignment """ data = np.ones((4,4),dtype='f') self.f['a'] = data self.assertIsInstance(self.f['a'], Dataset) self.assertArrayEqual(self.f['a'][...], data) class TestDtypeAssignment(BaseGroup): """ Feature: Named types can be created by direct assignment of dtypes """ def test_dtype(self): """ Named type creation """ dtype = np.dtype('|S10') self.f['a'] = dtype self.assertIsInstance(self.f['a'], Datatype) self.assertEqual(self.f['a'].dtype, dtype) class TestRequire(BaseGroup): """ Feature: Groups can be auto-created, or opened via .require_group """ def test_open_existing(self): """ Existing group is opened and returned """ grp = self.f.create_group('foo') grp2 = self.f.require_group('foo') self.assertEqual(grp, grp2) def test_create(self): """ Group is created if it doesn't exist """ grp = self.f.require_group('foo') self.assertIsInstance(grp, Group) self.assertEqual(grp.name, '/foo') def test_require_exception(self): """ Opening conflicting object results in TypeError """ self.f.create_dataset('foo', (1,), 'f') with self.assertRaises(TypeError): self.f.require_group('foo') class TestDelete(BaseGroup): """ Feature: Objects can be unlinked via "del" operator """ def test_delete(self): """ Object deletion via "del" """ self.f.create_group('foo') self.assertIn('foo', self.f) del self.f['foo'] self.assertNotIn('foo', self.f) def test_nonexisting(self): """ Deleting non-existent object raises KeyError """ with self.assertRaises(KeyError): del self.f['foo'] def test_readonly_delete_exception(self): """ Deleting object in readonly file raises KeyError """ # Note: it is impossible to restore the old behavior (ValueError) # without breaking the above test (non-existing objects) fname = self.mktemp() hfile = File(fname,'w') try: hfile.create_group('foo') finally: hfile.close() hfile = File(fname, 'r') try: with self.assertRaises(KeyError): del hfile['foo'] finally: hfile.close() class TestOpen(BaseGroup): """ Feature: Objects can be opened via indexing syntax obj[name] """ def test_open(self): """ Simple obj[name] opening """ grp = self.f.create_group('foo') grp2 = self.f['foo'] grp3 = self.f['/foo'] self.assertEqual(grp, grp2) self.assertEqual(grp, grp3) def test_nonexistent(self): """ Opening missing objects raises KeyError """ with self.assertRaises(KeyError): self.f['foo'] def test_reference(self): """ Objects can be opened by HDF5 object reference """ grp = self.f.create_group('foo') grp2 = self.f[grp.ref] self.assertEqual(grp2, grp) def test_reference_numpyobj(self): """ Object can be opened by numpy.object_ containing object ref Test for issue 181, issue 202. """ g = self.f.create_group('test') rtype = h5py.special_dtype(ref=h5py.Reference) dt = np.dtype([('a', 'i'),('b',rtype)]) dset = self.f.create_dataset('test_dset', (1,), dt) dset[0] =(42,g.ref) data = dset[0] self.assertEqual(self.f[data[1]], g) def test_invalid_ref(self): """ Invalid region references should raise ValueError """ ref = h5py.h5r.Reference() with self.assertRaises(ValueError): self.f[ref] self.f.create_group('x') ref = self.f['x'].ref del self.f['x'] with self.assertRaises(ValueError): self.f[ref] # TODO: check that regionrefs also work with __getitem__ class TestRepr(BaseGroup): """ Feature: Opened and closed groups provide a useful __repr__ string """ def test_repr(self): """ Opened and closed groups provide a useful __repr__ string """ g = self.f.create_group('foo') self.assertIsInstance(repr(g), six.string_types) g.id._close() self.assertIsInstance(repr(g), six.string_types) class BaseMapping(BaseGroup): """ Base class for mapping tests """ def setUp(self): self.f = File(self.mktemp(), 'w') self.groups = ('a','b','c','d') for x in self.groups: self.f.create_group(x) self.f['x'] = h5py.SoftLink('/mongoose') self.groups = self.groups + ('x',) def tearDown(self): if self.f: self.f.close() class TestLen(BaseMapping): """ Feature: The Python len() function returns the number of groups """ def test_len(self): """ len() returns number of group members """ self.assertEqual(len(self.f), len(self.groups)) self.f.create_group('e') self.assertEqual(len(self.f), len(self.groups)+1) def test_exc(self): """ len() on closed group gives ValueError """ self.f.close() with self.assertRaises(ValueError): len(self.f) class TestContains(BaseGroup): """ Feature: The Python "in" builtin tests for containership """ def test_contains(self): """ "in" builtin works for containership (byte and Unicode) """ self.f.create_group('a') self.assertIn(b'a', self.f) self.assertIn(u'a', self.f) self.assertIn(b'/a', self.f) self.assertIn(u'/a', self.f) self.assertNotIn(b'mongoose', self.f) self.assertNotIn(u'mongoose', self.f) def test_exc(self): """ "in" on closed group returns False (see also issue 174) """ self.f.create_group('a') self.f.close() self.assertFalse(b'a' in self.f) self.assertFalse(u'a' in self.f) def test_empty(self): """ Empty strings work properly and aren't contained """ self.assertNotIn(u'', self.f) self.assertNotIn(b'', self.f) def test_dot(self): """ Current group "." is always contained """ self.assertIn(b'.', self.f) self.assertIn(u'.', self.f) def test_root(self): """ Root group (by itself) is contained """ self.assertIn(b'/', self.f) self.assertIn(u'/', self.f) def test_trailing_slash(self): """ Trailing slashes are unconditionally ignored """ self.f.create_group('group') self.f['dataset'] = 42 self.assertIn('/group/', self.f) self.assertIn('group/', self.f) self.assertIn('/dataset/', self.f) self.assertIn('dataset/', self.f) def test_softlinks(self): """ Broken softlinks are contained, but their members are not """ self.f.create_group('grp') self.f['/grp/soft'] = h5py.SoftLink('/mongoose') self.f['/grp/external'] = h5py.ExternalLink('mongoose.hdf5', '/mongoose') self.assertIn('/grp/soft', self.f) self.assertNotIn('/grp/soft/something', self.f) self.assertIn('/grp/external', self.f) self.assertNotIn('/grp/external/something', self.f) def test_oddball_paths(self): """ Technically legitimate (but odd-looking) paths """ self.f.create_group('x/y/z') self.f['dset'] = 42 self.assertIn('/', self.f) self.assertIn('//', self.f) self.assertIn('///', self.f) self.assertIn('.///', self.f) self.assertIn('././/', self.f) grp = self.f['x'] self.assertIn('.//x/y/z', self.f) self.assertNotIn('.//x/y/z', grp) self.assertIn('x///', self.f) self.assertIn('./x///', self.f) self.assertIn('dset///', self.f) self.assertIn('/dset//', self.f) class TestIter(BaseMapping): """ Feature: You can iterate over group members via "for x in y", etc. """ def test_iter(self): """ "for x in y" iteration """ lst = [x for x in self.f] self.assertSameElements(lst, self.groups) def test_iter_zero(self): """ Iteration works properly for the case with no group members """ hfile = File(self.mktemp(), 'w') try: lst = [x for x in hfile] self.assertEqual(lst, []) finally: hfile.close() @ut.skipIf(sys.version_info[0] != 2, "Py2") class TestPy2Dict(BaseMapping): """ Feature: Standard Python 2 .keys, .values, etc. methods are available """ def test_keys(self): """ .keys method """ self.assertIsInstance(self.f.keys(), list) self.assertSameElements(self.f.keys(), self.groups) def test_values(self): """ .values method """ self.assertIsInstance(self.f.values(), list) self.assertSameElements(self.f.values(), [self.f.get(x) for x in self.groups]) def test_items(self): """ .items method """ self.assertIsInstance(self.f.items(), list) self.assertSameElements(self.f.items(), [(x, self.f.get(x)) for x in self.groups]) def test_iterkeys(self): """ .iterkeys method """ self.assertSameElements([x for x in self.f.iterkeys()], self.groups) def test_itervalues(self): """ .itervalues method """ self.assertSameElements([x for x in self.f.itervalues()], [self.f.get(x) for x in self.groups]) def test_iteritems(self): """ .iteritems method """ self.assertSameElements([x for x in self.f.iteritems()], [(x, self.f.get(x)) for x in self.groups]) @ut.skipIf(six.PY2, "Py3") class TestPy3Dict(BaseMapping): def test_keys(self): """ .keys provides a key view """ kv = getattr(self.f, 'keys')() self.assertSameElements(list(kv), self.groups) for x in self.groups: self.assertIn(x, kv) self.assertEqual(len(kv), len(self.groups)) def test_values(self): """ .values provides a value view """ vv = getattr(self.f, 'values')() self.assertSameElements(list(vv), [self.f.get(x) for x in self.groups]) self.assertEqual(len(vv), len(self.groups)) for x in self.groups: self.assertIn(self.f.get(x), vv) def test_items(self): """ .items provides an item view """ iv = getattr(self.f, 'items')() self.assertSameElements(list(iv), [(x,self.f.get(x)) for x in self.groups]) self.assertEqual(len(iv), len(self.groups)) for x in self.groups: self.assertIn((x, self.f.get(x)), iv) class TestAdditionalMappingFuncs(BaseMapping): """ Feature: Other dict methods (pop, pop_item, clear, update, setdefault) are available. """ def setUp(self): self.f = File(self.mktemp(), 'w') for x in ('/test/a','/test/b','/test/c','/test/d'): self.f.create_group(x) self.group = self.f['test'] def tearDown(self): if self.f: self.f.close() def test_pop_item(self): """.pop_item exists and removes item""" key, val = self.group.popitem() self.assertNotIn(key, self.group) def test_pop(self): """.pop exists and removes specified item""" self.group.pop('a') self.assertNotIn('a', self.group) def test_pop_default(self): """.pop falls back to default""" # e shouldn't exist as a group value = self.group.pop('e', None) self.assertEqual(value, None) def test_pop_raises(self): """.pop raises KeyError for non-existence""" # e shouldn't exist as a group with self.assertRaises(KeyError): key = self.group.pop('e') def test_clear(self): """.clear removes groups""" self.group.clear() self.assertEqual(len(self.group), 0) def test_update_dict(self): """.update works with dict""" new_items = {'e': np.array([42])} self.group.update(new_items) self.assertIn('e', self.group) def test_update_iter(self): """.update works with list""" new_items = [ ('e', np.array([42])), ('f', np.array([42])) ] self.group.update(new_items) self.assertIn('e', self.group) def test_update_kwargs(self): """.update works with kwargs""" new_items = {'e': np.array([42])} self.group.update(**new_items) self.assertIn('e', self.group) def test_setdefault(self): """.setdefault gets group if it exists""" value = self.group.setdefault('a') self.assertEqual(value, self.group.get('a')) def test_setdefault_with_default(self): """.setdefault gets default if group doesn't exist""" # e shouldn't exist as a group # 42 used as groups should be strings value = self.group.setdefault('e', np.array([42])) self.assertEqual(value, 42) def test_setdefault_no_default(self): """ .setdefault gets None if group doesn't exist, but as None isn't defined as data for a dataset, this should raise a TypeError. """ # e shouldn't exist as a group with self.assertRaises(TypeError): self.group.setdefault('e') class TestGet(BaseGroup): """ Feature: The .get method allows access to objects and metadata """ def test_get_default(self): """ Object is returned, or default if it doesn't exist """ default = object() out = self.f.get('mongoose', default) self.assertIs(out, default) grp = self.f.create_group('a') out = self.f.get('a') self.assertEqual(out, grp) def test_get_class(self): """ Object class is returned with getclass option """ self.f.create_group('foo') out = self.f.get('foo', getclass=True) self.assertEqual(out, Group) self.f.create_dataset('bar', (4,)) out = self.f.get('bar', getclass=True) self.assertEqual(out, Dataset) self.f['baz'] = np.dtype('|S10') out = self.f.get('baz', getclass=True) self.assertEqual(out, Datatype) def test_get_link_class(self): """ Get link classes """ default = object() sl = SoftLink('/mongoose') el = ExternalLink('somewhere.hdf5', 'mongoose') self.f.create_group('hard') self.f['soft'] = sl self.f['external'] = el out_hl = self.f.get('hard', default, getlink=True, getclass=True) out_sl = self.f.get('soft', default, getlink=True, getclass=True) out_el = self.f.get('external', default, getlink=True, getclass=True) self.assertEqual(out_hl, HardLink) self.assertEqual(out_sl, SoftLink) self.assertEqual(out_el, ExternalLink) def test_get_link(self): """ Get link values """ sl = SoftLink('/mongoose') el = ExternalLink('somewhere.hdf5', 'mongoose') self.f.create_group('hard') self.f['soft'] = sl self.f['external'] = el out_hl = self.f.get('hard', getlink=True) out_sl = self.f.get('soft', getlink=True) out_el = self.f.get('external', getlink=True) #TODO: redo with SoftLink/ExternalLink built-in equality self.assertIsInstance(out_hl, HardLink) self.assertIsInstance(out_sl, SoftLink) self.assertEqual(out_sl._path, sl._path) self.assertIsInstance(out_el, ExternalLink) self.assertEqual(out_el._path, el._path) self.assertEqual(out_el._filename, el._filename) class TestVisit(TestCase): """ Feature: The .visit and .visititems methods allow iterative access to group and subgroup members """ def setUp(self): self.f = File(self.mktemp(), 'w') self.groups = [ 'grp1', 'grp1/sg1', 'grp1/sg2', 'grp2', 'grp2/sg1', 'grp2/sg1/ssg1' ] for x in self.groups: self.f.create_group(x) def tearDown(self): self.f.close() def test_visit(self): """ All subgroups are visited """ l = [] self.f.visit(l.append) self.assertSameElements(l, self.groups) def test_visititems(self): """ All subgroups and contents are visited """ l = [] comp = [(x, self.f[x]) for x in self.groups] self.f.visititems(lambda x, y: l.append((x,y))) self.assertSameElements(comp, l) def test_bailout(self): """ Returning a non-None value immediately aborts iteration """ x = self.f.visit(lambda x: x) self.assertEqual(x, self.groups[0]) x = self.f.visititems(lambda x, y: (x,y)) self.assertEqual(x, (self.groups[0], self.f[self.groups[0]])) class TestSoftLinks(BaseGroup): """ Feature: Create and manage soft links with the high-level interface """ def test_spath(self): """ SoftLink path attribute """ sl = SoftLink('/foo') self.assertEqual(sl.path, '/foo') def test_srepr(self): """ SoftLink path repr """ sl = SoftLink('/foo') self.assertIsInstance(repr(sl), six.string_types) def test_create(self): """ Create new soft link by assignment """ g = self.f.create_group('new') sl = SoftLink('/new') self.f['alias'] = sl g2 = self.f['alias'] self.assertEqual(g, g2) def test_exc(self): """ Opening dangling soft link results in KeyError """ self.f['alias'] = SoftLink('new') with self.assertRaises(KeyError): self.f['alias'] class TestExternalLinks(TestCase): """ Feature: Create and manage external links """ def setUp(self): self.f = File(self.mktemp(), 'w') self.ename = self.mktemp() self.ef = File(self.ename, 'w') self.ef.create_group('external') self.ef.close() def tearDown(self): if self.f: self.f.close() if self.ef: self.ef.close() def test_epath(self): """ External link paths attributes """ el = ExternalLink('foo.hdf5', '/foo') self.assertEqual(el.filename, 'foo.hdf5') self.assertEqual(el.path, '/foo') def test_erepr(self): """ External link repr """ el = ExternalLink('foo.hdf5','/foo') self.assertIsInstance(repr(el), six.string_types) def test_create(self): """ Creating external links """ self.f['ext'] = ExternalLink(self.ename, '/external') grp = self.f['ext'] self.ef = grp.file self.assertNotEqual(self.ef, self.f) self.assertEqual(grp.name, '/external') def test_exc(self): """ KeyError raised when attempting to open broken link """ self.f['ext'] = ExternalLink(self.ename, '/missing') with self.assertRaises(KeyError): self.f['ext'] # I would prefer IOError but there's no way to fix this as the exception # class is determined by HDF5. def test_exc_missingfile(self): """ KeyError raised when attempting to open missing file """ self.f['ext'] = ExternalLink('mongoose.hdf5','/foo') with self.assertRaises(KeyError): self.f['ext'] def test_close_file(self): """ Files opened by accessing external links can be closed Issue 189. """ self.f['ext'] = ExternalLink(self.ename, '/') grp = self.f['ext'] f2 = grp.file f2.close() self.assertFalse(f2) @ut.skipIf(NO_FS_UNICODE, "No unicode filename support") def test_unicode_encode(self): """ Check that external links encode unicode filenames properly Testing issue #732 """ ext_filename = os.path.join(mkdtemp(), u"α.hdf5") with File(ext_filename, "w") as ext_file: ext_file.create_group('external') self.f['ext'] = ExternalLink(ext_filename, '/external') @ut.skipIf(NO_FS_UNICODE, "No unicode filename support") def test_unicode_decode(self): """ Check that external links decode unicode filenames properly Testing issue #732 """ ext_filename = os.path.join(mkdtemp(), u"α.hdf5") with File(ext_filename, "w") as ext_file: ext_file.create_group('external') ext_file["external"].attrs["ext_attr"] = "test" self.f['ext'] = ExternalLink(ext_filename, '/external') self.assertEqual(self.f["ext"].attrs["ext_attr"], "test") class TestExtLinkBugs(TestCase): """ Bugs: Specific regressions for external links """ def test_issue_212(self): """ Issue 212 Fails with: AttributeError: 'SharedConfig' object has no attribute 'lapl' """ def closer(x): def w(): try: if x: x.close() except IOError: pass return w orig_name = self.mktemp() new_name = self.mktemp() f = File(orig_name, 'w') self.addCleanup(closer(f)) f.create_group('a') f.close() g = File(new_name, 'w') self.addCleanup(closer(g)) g['link'] = ExternalLink(orig_name, '/') # note root group g.close() h = File(new_name, 'r') self.addCleanup(closer(h)) self.assertIsInstance(h['link']['a'], Group) class TestCopy(TestCase): def setUp(self): self.f1 = File(self.mktemp(), 'w') self.f2 = File(self.mktemp(), 'w') def tearDown(self): if self.f1: self.f1.close() if self.f2: self.f2.close() @ut.skipIf(h5py.version.hdf5_version_tuple < (1,8,9), "Bug in HDF5<1.8.8 prevents copying open dataset") def test_copy_path_to_path(self): foo = self.f1.create_group('foo') foo['bar'] = [1,2,3] self.f1.copy('foo', 'baz') baz = self.f1['baz'] self.assertIsInstance(baz, Group) self.assertArrayEqual(baz['bar'], np.array([1,2,3])) @ut.skipIf(h5py.version.hdf5_version_tuple < (1,8,9), "Bug in HDF5<1.8.8 prevents copying open dataset") def test_copy_path_to_group(self): foo = self.f1.create_group('foo') foo['bar'] = [1,2,3] baz = self.f1.create_group('baz') self.f1.copy('foo', baz) baz = self.f1['baz'] self.assertIsInstance(baz, Group) self.assertArrayEqual(baz['foo/bar'], np.array([1,2,3])) self.f1.copy('foo', self.f2['/']) self.assertIsInstance(self.f2['/foo'], Group) self.assertArrayEqual(self.f2['foo/bar'], np.array([1,2,3])) @ut.skipIf(h5py.version.hdf5_version_tuple < (1,8,9), "Bug in HDF5<1.8.8 prevents copying open dataset") def test_copy_group_to_path(self): foo = self.f1.create_group('foo') foo['bar'] = [1,2,3] self.f1.copy(foo, 'baz') baz = self.f1['baz'] self.assertIsInstance(baz, Group) self.assertArrayEqual(baz['bar'], np.array([1,2,3])) self.f2.copy(foo, 'foo') self.assertIsInstance(self.f2['/foo'], Group) self.assertArrayEqual(self.f2['foo/bar'], np.array([1,2,3])) @ut.skipIf(h5py.version.hdf5_version_tuple < (1,8,9), "Bug in HDF5<1.8.8 prevents copying open dataset") def test_copy_group_to_group(self): foo = self.f1.create_group('foo') foo['bar'] = [1,2,3] baz = self.f1.create_group('baz') self.f1.copy(foo, baz) baz = self.f1['baz'] self.assertIsInstance(baz, Group) self.assertArrayEqual(baz['foo/bar'], np.array([1,2,3])) self.f1.copy(foo, self.f2['/']) self.assertIsInstance(self.f2['/foo'], Group) self.assertArrayEqual(self.f2['foo/bar'], np.array([1,2,3])) @ut.skipIf(h5py.version.hdf5_version_tuple < (1,8,9), "Bug in HDF5<1.8.8 prevents copying open dataset") def test_copy_dataset(self): self.f1['foo'] = [1,2,3] foo = self.f1['foo'] self.f1.copy(foo, 'bar') self.assertArrayEqual(self.f1['bar'], np.array([1,2,3])) self.f1.copy('foo', 'baz') self.assertArrayEqual(self.f1['baz'], np.array([1,2,3])) self.f1.copy('foo', self.f2) self.assertArrayEqual(self.f2['foo'], np.array([1,2,3])) self.f2.copy(self.f1['foo'], self.f2, 'bar') self.assertArrayEqual(self.f2['bar'], np.array([1,2,3])) @ut.skipIf(h5py.version.hdf5_version_tuple < (1,8,9), "Bug in HDF5<1.8.8 prevents copying open dataset") def test_copy_shallow(self): foo = self.f1.create_group('foo') bar = foo.create_group('bar') foo['qux'] = [1,2,3] bar['quux'] = [4,5,6] self.f1.copy(foo, 'baz', shallow=True) baz = self.f1['baz'] self.assertIsInstance(baz, Group) self.assertIsInstance(baz['bar'], Group) self.assertEqual(len(baz['bar']), 0) self.assertArrayEqual(baz['qux'], np.array([1,2,3])) self.f2.copy(foo, 'foo', shallow=True) self.assertIsInstance(self.f2['/foo'], Group) self.assertIsInstance(self.f2['foo/bar'], Group) self.assertEqual(len(self.f2['foo/bar']), 0) self.assertArrayEqual(self.f2['foo/qux'], np.array([1,2,3])) @ut.skipIf(h5py.version.hdf5_version_tuple < (1,8,9), "Bug in HDF5<1.8.8 prevents copying open dataset") def test_copy_without_attributes(self): self.f1['foo'] = [1,2,3] foo = self.f1['foo'] foo.attrs['bar'] = [4,5,6] self.f1.copy(foo, 'baz', without_attrs=True) self.assertArrayEqual(self.f1['baz'], np.array([1,2,3])) self.assert_('bar' not in self.f1['baz'].attrs) self.f2.copy(foo, 'baz', without_attrs=True) self.assertArrayEqual(self.f2['baz'], np.array([1,2,3])) self.assert_('bar' not in self.f2['baz'].attrs) @ut.skipIf(h5py.version.hdf5_version_tuple < (1,8,9), "Bug in HDF5<1.8.8 prevents copying open dataset") def test_copy_soft_links(self): self.f1['bar'] = [1,2,3] foo = self.f1.create_group('foo') foo['baz'] = SoftLink('/bar') self.f1.copy(foo, 'qux', expand_soft=True) self.f2.copy(foo, 'foo', expand_soft=True) del self.f1['bar'] self.assertIsInstance(self.f1['qux'], Group) self.assertArrayEqual(self.f1['qux/baz'], np.array([1,2,3])) self.assertIsInstance(self.f2['/foo'], Group) self.assertArrayEqual(self.f2['foo/baz'], np.array([1,2,3])) @ut.skipIf(h5py.version.hdf5_version_tuple < (1,8,9), "Bug in HDF5<1.8.8 prevents copying open dataset") def test_copy_external_links(self): filename = self.f1.filename self.f1['foo'] = [1,2,3] self.f2['bar'] = ExternalLink(filename, 'foo') self.f1.close() self.f1 = None self.assertArrayEqual(self.f2['bar'], np.array([1,2,3])) self.f2.copy('bar', 'baz', expand_external=True) os.unlink(filename) self.assertArrayEqual(self.f2['baz'], np.array([1,2,3])) @ut.skipIf(h5py.version.hdf5_version_tuple < (1,8,9), "Bug in HDF5<1.8.8 prevents copying open dataset") def test_copy_refs(self): self.f1['foo'] = [1,2,3] self.f1['bar'] = [4,5,6] foo = self.f1['foo'] bar = self.f1['bar'] foo.attrs['bar'] = bar.ref self.f1.copy(foo, 'baz', expand_refs=True) self.assertArrayEqual(self.f1['baz'], np.array([1,2,3])) baz_bar = self.f1['baz'].attrs['bar'] self.assertArrayEqual(self.f1[baz_bar], np.array([4,5,6])) # The reference points to a copy of bar, not to bar itself. self.assertNotEqual(self.f1[baz_bar].name, bar.name) self.f1.copy('foo', self.f2, 'baz', expand_refs=True) self.assertArrayEqual(self.f2['baz'], np.array([1,2,3])) baz_bar = self.f2['baz'].attrs['bar'] self.assertArrayEqual(self.f2[baz_bar], np.array([4,5,6])) self.f1.copy('/', self.f2, 'root', expand_refs=True) self.assertArrayEqual(self.f2['root/foo'], np.array([1,2,3])) self.assertArrayEqual(self.f2['root/bar'], np.array([4,5,6])) foo_bar = self.f2['root/foo'].attrs['bar'] self.assertArrayEqual(self.f2[foo_bar], np.array([4,5,6])) # There's only one copy of bar, which the reference points to. self.assertEqual(self.f2[foo_bar], self.f2['root/bar']) class TestMove(BaseGroup): """ Feature: Group.move moves links in a file """ def test_move_hardlink(self): """ Moving an object """ grp = self.f.create_group("X") self.f.move("X", "Y") self.assertEqual(self.f["Y"], grp) self.f.move("Y", "new/nested/path") self.assertEqual(self.f['new/nested/path'], grp) def test_move_softlink(self): """ Moving a soft link """ self.f['soft'] = h5py.SoftLink("relative/path") self.f.move('soft', 'new_soft') lnk = self.f.get('new_soft', getlink=True) self.assertEqual(lnk.path, "relative/path") def test_move_conflict(self): """ Move conflict raises ValueError """ self.f.create_group("X") self.f.create_group("Y") with self.assertRaises(ValueError): self.f.move("X", "Y") def test_short_circuit(self): ''' Test that a null-move works ''' self.f.create_group("X") self.f.move("X", "X") class TestMutableMapping(BaseGroup): '''Tests if the registration of Group as a MutableMapping behaves as expected ''' def test_resolution(self): assert issubclass(Group, collections.MutableMapping) grp = self.f.create_group("K") assert isinstance(grp, collections.MutableMapping) def test_validity(self): ''' Test that the required functions are implemented. ''' Group.__getitem__ Group.__setitem__ Group.__delitem__ Group.__iter__ Group.__len__ h5py-2.7.1/h5py/tests/old/test_file.py0000644000175000017500000004245513114361073021232 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. """ File object test module. Tests all aspects of File objects, including their creation. """ from __future__ import absolute_import, with_statement import os, stat from sys import platform import tempfile import six from ..common import ut, TestCase, UNICODE_FILENAMES, closed_tempfile from h5py.highlevel import File import h5py try: import pathlib except ImportError: pathlib = None mpi = h5py.get_config().mpi class TestFileOpen(TestCase): """ Feature: Opening files with Python-style modes. """ def test_default(self): """ Default semantics in the presence or absence of a file """ fname = self.mktemp() # No existing file; create a new file and open RW with File(fname) as f: self.assertTrue(f) self.assertEqual(f.mode, 'r+') # Existing readonly file; open read-only os.chmod(fname, stat.S_IREAD) # Running as root (e.g. in a docker container) gives 'r+' as the file # mode, even for a read-only file. See # https://github.com/h5py/h5py/issues/696 exp_mode = 'r+' if os.stat(fname).st_uid == 0 and platform != "win32" else 'r' try: with File(fname) as f: self.assertTrue(f) self.assertEqual(f.mode, exp_mode) finally: os.chmod(fname, stat.S_IWRITE) # File exists but is not HDF5; raise IOError with open(fname, 'wb') as f: f.write(b'\x00') with self.assertRaises(IOError): File(fname) def test_create(self): """ Mode 'w' opens file in overwrite mode """ fname = self.mktemp() fid = File(fname, 'w') self.assertTrue(fid) fid.create_group('foo') fid.close() fid = File(fname, 'w') self.assertNotIn('foo', fid) fid.close() def test_create_exclusive(self): """ Mode 'w-' opens file in exclusive mode """ fname = self.mktemp() fid = File(fname, 'w-') self.assert_(fid) fid.close() with self.assertRaises(IOError): File(fname, 'w-') def test_append(self): """ Mode 'a' opens file in append/readwrite mode, creating if necessary """ fname = self.mktemp() fid = File(fname, 'a') try: self.assert_(fid) fid.create_group('foo') self.assert_('foo' in fid) finally: fid.close() fid = File(fname, 'a') try: self.assert_('foo' in fid) fid.create_group('bar') self.assert_('bar' in fid) finally: fid.close() def test_readonly(self): """ Mode 'r' opens file in readonly mode """ fname = self.mktemp() fid = File(fname, 'w') fid.close() self.assert_(not fid) fid = File(fname, 'r') self.assert_(fid) with self.assertRaises(ValueError): fid.create_group('foo') fid.close() def test_readwrite(self): """ Mode 'r+' opens existing file in readwrite mode """ fname = self.mktemp() fid = File(fname, 'w') fid.create_group('foo') fid.close() fid = File(fname, 'r+') self.assert_('foo' in fid) fid.create_group('bar') self.assert_('bar' in fid) fid.close() def test_nonexistent_file(self): """ Modes 'r' and 'r+' do not create files """ fname = self.mktemp() with self.assertRaises(IOError): File(fname, 'r') with self.assertRaises(IOError): File(fname, 'r+') def test_invalid_mode(self): """ Invalid modes raise ValueError """ with self.assertRaises(ValueError): File(self.mktemp(), 'mongoose') class TestModes(TestCase): """ Feature: File mode can be retrieved via file.mode """ def test_mode_attr(self): """ Mode equivalent can be retrieved via property """ fname = self.mktemp() with File(fname, 'w') as f: self.assertEqual(f.mode, 'r+') with File(fname, 'r') as f: self.assertEqual(f.mode, 'r') def test_mode_external(self): """ Mode property works for files opened via external links Issue 190. """ fname1 = self.mktemp() fname2 = self.mktemp() f1 = File(fname1,'w') f1.close() f2 = File(fname2,'w') try: f2['External'] = h5py.ExternalLink(fname1, '/') f3 = f2['External'].file self.assertEqual(f3.mode, 'r+') finally: f2.close() f3.close() f2 = File(fname2,'r') try: f3 = f2['External'].file self.assertEqual(f3.mode, 'r') finally: f2.close() f3.close() class TestDrivers(TestCase): """ Feature: Files can be opened with low-level HDF5 drivers """ @ut.skipUnless(os.name == 'posix', "Stdio driver is supported on posix") def test_stdio(self): """ Stdio driver is supported on posix """ fid = File(self.mktemp(), 'w', driver='stdio') self.assertTrue(fid) self.assertEqual(fid.driver, 'stdio') fid.close() @ut.skipUnless(os.name == 'posix', "Sec2 driver is supported on posix") def test_sec2(self): """ Sec2 driver is supported on posix """ fid = File(self.mktemp(), 'w', driver='sec2') self.assert_(fid) self.assertEqual(fid.driver, 'sec2') fid.close() def test_core(self): """ Core driver is supported (no backing store) """ fname = self.mktemp() fid = File(fname, 'w', driver='core', backing_store=False) self.assert_(fid) self.assertEqual(fid.driver, 'core') fid.close() self.assertFalse(os.path.exists(fname)) def test_backing(self): """ Core driver saves to file when backing store used """ fname = self.mktemp() fid = File(fname, 'w', driver='core', backing_store=True) fid.create_group('foo') fid.close() fid = File(fname, 'r') self.assert_('foo' in fid) fid.close() def test_readonly(self): """ Core driver can be used to open existing files """ fname = self.mktemp() fid = File(fname, 'w') fid.create_group('foo') fid.close() fid = File(fname, 'r', driver='core') self.assert_(fid) self.assert_('foo' in fid) with self.assertRaises(ValueError): fid.create_group('bar') fid.close() def test_blocksize(self): """ Core driver supports variable block size """ fname = self.mktemp() fid = File(fname, 'w', driver='core', block_size=1024, backing_store=False) self.assert_(fid) fid.close() @ut.skipUnless(mpi, "Parallel HDF5 is required for MPIO driver test") def test_mpio(self): """ MPIO driver and options """ from mpi4py import MPI fname = self.mktemp() with File(fname, 'w', driver='mpio', comm=MPI.COMM_WORLD) as f: self.assertTrue(f) self.assertEqual(f.driver, 'mpio') @ut.skipUnless(mpi, "Parallel HDF5 required") @ut.skipIf(h5py.version.hdf5_version_tuple < (1,8,9), "mpio atomic file operations were added in HDF5 1.8.9+") def test_mpi_atomic(self): """ Enable atomic mode for MPIO driver """ from mpi4py import MPI fname = self.mktemp() with File(fname, 'w', driver='mpio', comm=MPI.COMM_WORLD) as f: self.assertFalse(f.atomic) f.atomic = True self.assertTrue(f.atomic) #TODO: family driver tests class TestLibver(TestCase): """ Feature: File format compatibility bounds can be specified when opening a file. """ def test_single(self): """ Opening with single libver arg """ f = File(self.mktemp(), 'w', libver='latest') self.assertEqual(f.libver, ('latest','latest')) f.close() def test_multiple(self): """ Opening with two libver args """ f = File(self.mktemp(), 'w', libver=('earliest','latest')) self.assertEqual(f.libver, ('earliest', 'latest')) f.close() def test_none(self): """ Omitting libver arg results in maximum compatibility """ f = File(self.mktemp(), 'w') self.assertEqual(f.libver, ('earliest', 'latest')) f.close() class TestUserblock(TestCase): """ Feature: Files can be create with user blocks """ def test_create_blocksize(self): """ User blocks created with w, w-, x and properties work correctly """ f = File(self.mktemp(),'w-', userblock_size=512) try: self.assertEqual(f.userblock_size, 512) finally: f.close() f = File(self.mktemp(),'x', userblock_size=512) try: self.assertEqual(f.userblock_size, 512) finally: f.close() f = File(self.mktemp(),'w', userblock_size=512) try: self.assertEqual(f.userblock_size, 512) finally: f.close() def test_write_only(self): """ User block only allowed for write """ name = self.mktemp() f = File(name, 'w') f.close() with self.assertRaises(ValueError): f = h5py.File(name, 'r', userblock_size=512) with self.assertRaises(ValueError): f = h5py.File(name, 'r+', userblock_size=512) def test_match_existing(self): """ User block size must match that of file when opening for append """ name = self.mktemp() f = File(name, 'w', userblock_size=512) f.close() with self.assertRaises(ValueError): f = File(name, 'a', userblock_size=1024) f = File(name, 'a', userblock_size=512) try: self.assertEqual(f.userblock_size, 512) finally: f.close() def test_power_of_two(self): """ User block size must be a power of 2 and at least 512 """ name = self.mktemp() with self.assertRaises(ValueError): f = File(name, 'w', userblock_size=128) with self.assertRaises(ValueError): f = File(name, 'w', userblock_size=513) with self.assertRaises(ValueError): f = File(name, 'w', userblock_size=1023) def test_write_block(self): """ Test that writing to a user block does not destroy the file """ name = self.mktemp() f = File(name, 'w', userblock_size=512) f.create_group("Foobar") f.close() pyfile = open(name, 'r+b') try: pyfile.write(b'X'*512) finally: pyfile.close() f = h5py.File(name, 'r') try: self.assert_("Foobar" in f) finally: f.close() pyfile = open(name, 'rb') try: self.assertEqual(pyfile.read(512), b'X'*512) finally: pyfile.close() class TestContextManager(TestCase): """ Feature: File objects can be used as context managers """ def test_context_manager(self): """ File objects can be used in with statements """ with File(self.mktemp(), 'w') as fid: self.assertTrue(fid) self.assertTrue(not fid) @ut.skipIf(not UNICODE_FILENAMES, "Filesystem unicode support required") class TestUnicode(TestCase): """ Feature: Unicode filenames are supported """ def test_unicode(self): """ Unicode filenames can be used, and retrieved properly via .filename """ fname = self.mktemp(prefix = six.unichr(0x201a)) fid = File(fname, 'w') try: self.assertEqual(fid.filename, fname) self.assertIsInstance(fid.filename, six.text_type) finally: fid.close() def test_unicode_hdf5_python_consistent(self): """ Unicode filenames can be used, and seen correctly from python """ fname = self.mktemp(prefix = six.unichr(0x201a)) with File(fname, 'w') as f: self.assertTrue(os.path.exists(fname)) class TestFileProperty(TestCase): """ Feature: A File object can be retrieved from any child object, via the .file property """ def test_property(self): """ File object can be retrieved from subgroup """ fname = self.mktemp() hfile = File(fname, 'w') try: hfile2 = hfile['/'].file self.assertEqual(hfile, hfile2) finally: hfile.close() def test_close(self): """ All retrieved File objects are closed at the same time """ fname = self.mktemp() hfile = File(fname, 'w') grp = hfile.create_group('foo') hfile2 = grp.file hfile3 = hfile['/'].file hfile2.close() self.assertFalse(hfile) self.assertFalse(hfile2) self.assertFalse(hfile3) def test_mode(self): """ Retrieved File objects have a meaningful mode attribute """ hfile = File(self.mktemp(),'w') try: grp = hfile.create_group('foo') self.assertEqual(grp.file.mode, hfile.mode) finally: hfile.close() class TestClose(TestCase): """ Feature: Files can be closed """ def test_close(self): """ Close file via .close method """ fid = File(self.mktemp()) self.assert_(fid) fid.close() self.assert_(not fid) def test_closed_file(self): """ Trying to modify closed file raises ValueError """ fid = File(self.mktemp(), 'w') fid.close() with self.assertRaises(ValueError): fid.create_group('foo') def test_close_multiple_default_driver(self): fname = self.mktemp() f = h5py.File(fname, 'w') f.create_group("test") f.close() f.close() @ut.skipUnless(mpi, "Parallel HDF5 is required for MPIO driver test") def test_close_multiple_mpio_driver(self): """ MPIO driver and options """ from mpi4py import MPI fname = self.mktemp() f = File(fname, 'w', driver='mpio', comm=MPI.COMM_WORLD) f.create_group("test") f.close() f.close() class TestFlush(TestCase): """ Feature: Files can be flushed """ def test_flush(self): """ Flush via .flush method """ fid = File(self.mktemp(), 'w') fid.flush() fid.close() class TestRepr(TestCase): """ Feature: File objects provide a helpful __repr__ string """ def test_repr(self): """ __repr__ behaves itself when files are open and closed """ fid = File(self.mktemp()) self.assertIsInstance(repr(fid), six.string_types) fid.close() self.assertIsInstance(repr(fid), six.string_types) class TestFilename(TestCase): """ Feature: The name of a File object can be retrieved via .filename """ def test_filename(self): """ .filename behaves properly for string data """ fname = self.mktemp() fid = File(fname, 'w') try: self.assertEqual(fid.filename, fname) self.assertIsInstance(fid.filename, six.text_type) finally: fid.close() class TestBackwardsCompat(TestCase): """ Feauture: Deprecated attributes are included to support 1.3 code """ def test_fid(self): """ File objects provide a .fid attribute aliased to the file ID """ with File(self.mktemp(), 'w') as hfile: self.assertIs(hfile.fid, hfile.id) class TestCloseInvalidatesOpenObjectIDs(TestCase): """ Ensure that closing a file invalidates object IDs, as appropriate """ def test_close(self): """ Closing a file invalidates any of the file's open objects """ with File(self.mktemp(), 'w') as f1: g1 = f1.create_group('foo') self.assertTrue(bool(f1.id)) self.assertTrue(bool(g1.id)) f1.close() self.assertFalse(bool(f1.id)) self.assertFalse(bool(g1.id)) with File(self.mktemp(), 'w') as f2: g2 = f2.create_group('foo') self.assertTrue(bool(f2.id)) self.assertTrue(bool(g2.id)) self.assertFalse(bool(f1.id)) self.assertFalse(bool(g1.id)) @ut.skipIf(pathlib is None, "pathlib module not installed") class TestPathlibSupport(TestCase): """ Check that h5py doesn't break on pathlib """ def test_pathlib_accepted_file(self): """ Check that pathlib is accepted by h5py.File """ with closed_tempfile() as f: path = pathlib.Path(f) with File(path) as f2: self.assertTrue(True) def test_pathlib_name_match(self): """ Check that using pathlib does not affect naming """ with closed_tempfile() as f: path = pathlib.Path(f) with File(path) as h5f1: pathlib_name = h5f1.filename with File(f) as h5f2: normal_name = h5f2.filename self.assertEqual(pathlib_name, normal_name) h5py-2.7.1/h5py/tests/old/test_datatype.py0000644000175000017500000000240313114361073022113 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. """ File-resident datatype tests. Tests "committed" file-resident datatype objects. """ from __future__ import absolute_import import six import numpy as np from ..common import ut, TestCase from h5py import File from h5py._hl.datatype import Datatype class BaseType(TestCase): def setUp(self): self.f = File(self.mktemp(), 'w') def tearDown(self): if self.f: self.f.close() class TestCreation(BaseType): """ Feature: repr() works sensibly on datatype objects """ def test_repr(self): """ repr() on datatype objects """ self.f['foo'] = np.dtype('S10') dt = self.f['foo'] self.assertIsInstance(repr(dt), six.string_types) self.f.close() self.assertIsInstance(repr(dt), six.string_types) def test_appropriate_low_level_id(self): " Binding a group to a non-TypeID identifier fails with ValueError " with self.assertRaises(ValueError): Datatype(self.f['/'].id) h5py-2.7.1/h5py/tests/old/test_h5f.py0000644000175000017500000000447013114361073020770 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. from __future__ import absolute_import import tempfile import shutil import os from h5py import File from ..common import TestCase class TestFileID(TestCase): def test_descriptor_core(self): with File('TestFileID.test_descriptor_core', driver='core', backing_store=False) as f: with self.assertRaises(NotImplementedError): f.id.get_vfd_handle() def test_descriptor_sec2(self): dn_tmp = tempfile.mkdtemp('h5py.lowtest.test_h5f.TestFileID.test_descriptor_sec2') fn_h5 = os.path.join(dn_tmp, 'test.h5') try: with File(fn_h5, driver='sec2') as f: descriptor = f.id.get_vfd_handle() self.assertNotEqual(descriptor, 0) os.fsync(descriptor) finally: shutil.rmtree(dn_tmp) class TestCacheConfig(TestCase): def test_simple_gets(self): dn_tmp = tempfile.mkdtemp('h5py.lowtest.test_h5f.TestFileID.TestCacheConfig.test_simple_gets') fn_h5 = os.path.join(dn_tmp, 'test.h5') try: with File(fn_h5) as f: hit_rate = f._id.get_mdc_hit_rate() mdc_size = f._id.get_mdc_size() finally: shutil.rmtree(dn_tmp) def test_hitrate_reset(self): dn_tmp = tempfile.mkdtemp('h5py.lowtest.test_h5f.TestFileID.TestCacheConfig.test_hitrate_reset') fn_h5 = os.path.join(dn_tmp, 'test.h5') try: with File(fn_h5) as f: hit_rate = f._id.get_mdc_hit_rate() f._id.reset_mdc_hit_rate_stats() hit_rate = f._id.get_mdc_hit_rate() assert hit_rate == 0 finally: shutil.rmtree(dn_tmp) def test_mdc_config_get(self): dn_tmp = tempfile.mkdtemp('h5py.lowtest.test_h5f.TestFileID.TestCacheConfig.test_mdc_config_get') fn_h5 = os.path.join(dn_tmp, 'test.h5') try: with File(fn_h5) as f: conf = f._id.get_mdc_config() f._id.set_mdc_config(conf) finally: shutil.rmtree(dn_tmp) h5py-2.7.1/h5py/tests/old/test_dimension_scales.py0000644000175000017500000001711413114361073023624 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. from __future__ import absolute_import import sys import numpy as np from ..common import ut, TestCase from h5py.highlevel import File, Group, Dataset import h5py class BaseDataset(TestCase): """ data is a 3-dimensional dataset with dimensions [z, y, x] The z dimension is labeled. It does not have any attached scales. The y dimension is not labeled. It has one attached scale. The x dimension is labeled. It has two attached scales. data2 is a 3-dimensional dataset with no associated dimension scales. """ def setUp(self): self.f = File(self.mktemp(), 'w') self.f['data'] = np.ones((4, 3, 2), 'f') self.f['data2'] = np.ones((4, 3, 2), 'f') self.f['x1'] = np.ones((2), 'f') h5py.h5ds.set_scale(self.f['x1'].id) h5py.h5ds.attach_scale(self.f['data'].id, self.f['x1'].id, 2) self.f['x2'] = np.ones((2), 'f') h5py.h5ds.set_scale(self.f['x2'].id, b'x2 name') h5py.h5ds.attach_scale(self.f['data'].id, self.f['x2'].id, 2) self.f['y1'] = np.ones((3), 'f') h5py.h5ds.set_scale(self.f['y1'].id, b'y1 name') h5py.h5ds.attach_scale(self.f['data'].id, self.f['y1'].id, 1) self.f['z1'] = np.ones((4), 'f') h5py.h5ds.set_label(self.f['data'].id, 0, b'z') h5py.h5ds.set_label(self.f['data'].id, 2, b'x') def tearDown(self): if self.f: self.f.close() class TestH5DSBindings(BaseDataset): """ Feature: Datasets can be created from existing data """ def test_create_dimensionscale(self): """ Create a dimension scale from existing dataset """ self.assertTrue(h5py.h5ds.is_scale(self.f['x1'].id)) self.assertEqual(h5py.h5ds.get_scale_name(self.f['x1'].id), b'') self.assertEqual(self.f['x1'].attrs['CLASS'], b"DIMENSION_SCALE") self.assertEqual(h5py.h5ds.get_scale_name(self.f['x2'].id), b'x2 name') def test_attach_dimensionscale(self): self.assertTrue( h5py.h5ds.is_attached(self.f['data'].id, self.f['x1'].id, 2) ) self.assertFalse( h5py.h5ds.is_attached(self.f['data'].id, self.f['x1'].id, 1)) self.assertEqual(h5py.h5ds.get_num_scales(self.f['data'].id, 0), 0) self.assertEqual(h5py.h5ds.get_num_scales(self.f['data'].id, 1), 1) self.assertEqual(h5py.h5ds.get_num_scales(self.f['data'].id, 2), 2) def test_detach_dimensionscale(self): self.assertTrue( h5py.h5ds.is_attached(self.f['data'].id, self.f['x1'].id, 2) ) h5py.h5ds.detach_scale(self.f['data'].id, self.f['x1'].id, 2) self.assertFalse( h5py.h5ds.is_attached(self.f['data'].id, self.f['x1'].id, 2) ) self.assertEqual(h5py.h5ds.get_num_scales(self.f['data'].id, 2), 1) # TODO: update condition once the bug is fixed upstream @ut.skipUnless( h5py.version.hdf5_version_tuple > (2, 0, 0), "Reading non-existent label segfaults" ) def test_label_dimensionscale(self): self.assertEqual(h5py.h5ds.get_label(self.f['data'].id, 0), b'z') self.assertEqual(h5py.h5ds.get_label(self.f['data'].id, 1), b'') self.assertEqual(h5py.h5ds.get_label(self.f['data'].id, 2), b'x') def test_iter_dimensionscales(self): def func(dsid): res = h5py.h5ds.get_scale_name(dsid) if res == b'x2 name': return dsid res = h5py.h5ds.iterate(self.f['data'].id, 2, func, 0) self.assertEqual(h5py.h5ds.get_scale_name(res), b'x2 name') class TestDimensionManager(BaseDataset): def test_create_scale(self): # test recreating or renaming an existing scale: self.f['data'].dims.create_scale(self.f['x1'], b'foobar') self.assertEqual(self.f['data'].dims[2]['foobar'], self.f['x1']) # test creating entirely new scale: self.f['data'].dims.create_scale(self.f['data2'], b'foobaz') self.f['data'].dims[2].attach_scale(self.f['data2']) self.assertEqual(self.f['data'].dims[2]['foobaz'], self.f['data2']) def test_get_dimension(self): with self.assertRaises(IndexError): self.f['data'].dims[3] def test_len(self): self.assertEqual(len(self.f['data'].dims), 3) self.assertEqual(len(self.f['data2'].dims), 3) def test_iter(self): dims = self.f['data'].dims self.assertEqual( [d for d in dims], [dims[0], dims[1], dims[2]] ) class TestDimensionsHighLevel(BaseDataset): def test_len(self): self.assertEqual(len(self.f['data'].dims[0]), 0) self.assertEqual(len(self.f['data'].dims[1]), 1) self.assertEqual(len(self.f['data'].dims[2]), 2) self.assertEqual(len(self.f['data2'].dims[0]), 0) self.assertEqual(len(self.f['data2'].dims[1]), 0) self.assertEqual(len(self.f['data2'].dims[2]), 0) def test_get_label(self): self.assertEqual(self.f['data'].dims[2].label, 'x') self.assertEqual(self.f['data'].dims[1].label, '') self.assertEqual(self.f['data'].dims[0].label, 'z') self.assertEqual(self.f['data2'].dims[2].label, '') self.assertEqual(self.f['data2'].dims[1].label, '') self.assertEqual(self.f['data2'].dims[0].label, '') def test_set_label(self): self.f['data'].dims[0].label = 'foo' self.assertEqual(self.f['data'].dims[2].label, 'x') self.assertEqual(self.f['data'].dims[1].label, '') self.assertEqual(self.f['data'].dims[0].label, 'foo') def test_detach_scale(self): self.f['data'].dims[2].detach_scale(self.f['x1']) self.assertEqual(len(self.f['data'].dims[2]), 1) self.assertEqual(self.f['data'].dims[2][0], self.f['x2']) self.f['data'].dims[2].detach_scale(self.f['x2']) self.assertEqual(len(self.f['data'].dims[2]), 0) def test_attach_scale(self): self.f['x3'] = self.f['x2'][...] self.f['data'].dims[2].attach_scale(self.f['x3']) self.assertEqual(len(self.f['data'].dims[2]), 3) self.assertEqual(self.f['data'].dims[2][2], self.f['x3']) def test_get_dimension_scale(self): self.assertEqual(self.f['data'].dims[2][0], self.f['x1']) with self.assertRaises(RuntimeError): self.f['data2'].dims[2][0], self.f['x2'] self.assertEqual(self.f['data'].dims[2][''], self.f['x1']) self.assertEqual(self.f['data'].dims[2]['x2 name'], self.f['x2']) def test_get_items(self): self.assertEqual( self.f['data'].dims[2].items(), [('', self.f['x1']), ('x2 name', self.f['x2'])] ) def test_get_keys(self): self.assertEqual(self.f['data'].dims[2].keys(), ['', 'x2 name']) def test_get_values(self): self.assertEqual( self.f['data'].dims[2].values(), [self.f['x1'], self.f['x2']] ) def test_iter(self): self.assertEqual([i for i in self.f['data'].dims[2]], ['', 'x2 name']) def test_repr(self): self.assertEqual(repr(self.f['data'].dims[2])[1:16], '"x" dimension 2') def test_attributes(self): self.f["data2"].attrs["DIMENSION_LIST"] = self.f["data"].attrs[ "DIMENSION_LIST"] self.assertEqual(len(self.f['data2'].dims[0]), 0) self.assertEqual(len(self.f['data2'].dims[1]), 1) self.assertEqual(len(self.f['data2'].dims[2]), 2) h5py-2.7.1/h5py/tests/old/test_h5p.py0000644000175000017500000000704113114361073020777 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. from __future__ import absolute_import try: import unittest2 as ut except ImportError: import unittest as ut from h5py import h5p, h5f from ..common import TestCase class TestLibver(TestCase): """ Feature: Setting/getting lib ver bounds """ def test_libver(self): """ Test libver bounds set/get """ plist = h5p.create(h5p.FILE_ACCESS) plist.set_libver_bounds(h5f.LIBVER_EARLIEST, h5f.LIBVER_LATEST) self.assertEqual((h5f.LIBVER_EARLIEST, h5f.LIBVER_LATEST), plist.get_libver_bounds()) class TestDA(TestCase): ''' Feature: setting/getting chunk cache size on a dataset access property list ''' def test_chunk_cache(self): '''test get/set chunk cache ''' dalist = h5p.create(h5p.DATASET_ACCESS) nslots = 10000 # 40kb hash table nbytes = 1000000 #1MB cache size w0 = .5 # even blend of eviction strategy dalist.set_chunk_cache(nslots, nbytes, w0) self.assertEqual((nslots, nbytes, w0), dalist.get_chunk_cache()) class TestFA(TestCase): ''' Feature: setting/getting mdc config on a file access property list ''' def test_mdc_config(self): '''test get/set mdc config ''' falist = h5p.create(h5p.FILE_ACCESS) config = falist.get_mdc_config() falist.set_mdc_config(config) def test_set_alignment(self): '''test get/set chunk cache ''' falist = h5p.create(h5p.FILE_ACCESS) threshold = 10 * 1024 # threshold of 10kiB alignment = 1024 * 1024 # threshold of 1kiB falist.set_alignment(threshold, alignment) self.assertEqual((threshold, alignment), falist.get_alignment()) class TestPL(TestCase): def test_obj_track_times(self): """ tests if the object track times set/get """ # test for groups gcid = h5p.create(h5p.GROUP_CREATE) gcid.set_obj_track_times(False) self.assertEqual(False,gcid.get_obj_track_times()) gcid.set_obj_track_times(True) self.assertEqual(True,gcid.get_obj_track_times()) # test for datasets dcid = h5p.create(h5p.DATASET_CREATE) dcid.set_obj_track_times(False) self.assertEqual(False,dcid.get_obj_track_times()) dcid.set_obj_track_times(True) self.assertEqual(True,dcid.get_obj_track_times()) # test for generic objects ocid = h5p.create(h5p.OBJECT_CREATE) ocid.set_obj_track_times(False) self.assertEqual(False,ocid.get_obj_track_times()) ocid.set_obj_track_times(True) self.assertEqual(True,ocid.get_obj_track_times()) def test_link_creation_tracking(self): """ tests the link creation order set/get """ gcid = h5p.create(h5p.GROUP_CREATE) gcid.set_link_creation_order(0) self.assertEqual(0, gcid.get_link_creation_order()) flags = h5p.CRT_ORDER_TRACKED|h5p.CRT_ORDER_INDEXED gcid.set_link_creation_order(flags) self.assertEqual(flags, gcid.get_link_creation_order()) # test for file creation fcpl = h5p.create(h5p.FILE_CREATE) fcpl.set_link_creation_order(flags) self.assertEqual(flags, fcpl.get_link_creation_order()) h5py-2.7.1/h5py/tests/old/test_h5.py0000644000175000017500000000235113114361073020616 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. from __future__ import absolute_import from h5py import h5 from ..common import TestCase def fixnames(): cfg = h5.get_config() cfg.complex_names = ('r','i') class TestH5(TestCase): def test_config(self): cfg = h5.get_config() self.assertIsInstance(cfg, h5.H5PYConfig) cfg2 = h5.get_config() self.assertIs(cfg, cfg2) def test_cnames_get(self): cfg = h5.get_config() self.assertEqual(cfg.complex_names, ('r','i')) def test_cnames_set(self): self.addCleanup(fixnames) cfg = h5.get_config() cfg.complex_names = ('q','x') self.assertEqual(cfg.complex_names, ('q','x')) def test_cnames_set_exc(self): self.addCleanup(fixnames) cfg = h5.get_config() with self.assertRaises(TypeError): cfg.complex_names = ('q','i','v') self.assertEqual(cfg.complex_names, ('r','i')) def test_repr(self): cfg = h5.get_config() repr(cfg) h5py-2.7.1/h5py/tests/old/test_file_image.py0000644000175000017500000000173113114361073022364 0ustar tcaswelltcaswell00000000000000from __future__ import absolute_import import h5py from h5py import h5f, h5p from ..common import ut, TestCase @ut.skipUnless(h5py.version.hdf5_version_tuple >= (1, 8, 9), 'file image operations require HDF5 >= 1.8.9') class TestFileImage(TestCase): def test_load_from_image(self): from binascii import a2b_base64 from zlib import decompress compressed_image = 'eJzr9HBx4+WS4mIAAQ4OBhYGAQZk8B8KKjhQ+TD5BCjNCKU7oPQKJpg4I1hOAiouCDUfXV1IkKsrSPV/NACzx4AFQnMwjIKRCDxcHQNAdASUD0ulJ5hQ1ZWkFpeAaFh69KDQXkYGNohZjDA+JCUzMkIEmKHqELQAWKkAByytOoBJViAPJM7ExATWyAE0B8RgZkyAJmlYDoEAIahukJoNU6+HMTA0UOgT6oBgP38XUI6G5UMFZrzKR8EoGAUjGMDKYVgxDSsuAHcfMK8=' image = decompress(a2b_base64(compressed_image)) fapl = h5p.create(h5py.h5p.FILE_ACCESS) fapl.set_fapl_core() fapl.set_file_image(image) fid = h5f.open(self.mktemp().encode(), h5py.h5f.ACC_RDONLY, fapl=fapl) f = h5py.File(fid) self.assertTrue('test' in f) h5py-2.7.1/h5py/tests/old/test_h5d_direct_chunk_write.py0000644000175000017500000000217113114361073024716 0ustar tcaswelltcaswell00000000000000from __future__ import absolute_import import h5py import numpy from ..common import ut, TestCase @ut.skipUnless(h5py.version.hdf5_version_tuple >= (1, 8, 11), 'Direct Chunk Writing requires HDF5 >= 1.8.11') class TestWriteDirectChunk(TestCase): def test_write_direct_chunk(self): filename = self.mktemp().encode() filehandle = h5py.File(filename, "w") dataset = filehandle.create_dataset("data", (100, 100, 100), maxshape=(None, 100, 100), chunks=(1, 100, 100), dtype='float32') # writing array = numpy.zeros((10, 100, 100)) for index in range(10): a = numpy.random.rand(100, 100).astype('float32') dataset.id.write_direct_chunk((index, 0, 0), a.tostring(), filter_mask=1) array[index] = a filehandle.close() # checking filehandle = h5py.File(filename, "r") for i in range(10): read_data = filehandle["data"][i] self.assertTrue((array[i] == read_data).all()) h5py-2.7.1/h5py/tests/old/test_slicing.py0000644000175000017500000002421113114361073021731 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. """ Dataset slicing test module. Tests all supported slicing operations, including read/write and broadcasting operations. Does not test type conversion except for corner cases overlapping with slicing; for example, when selecting specific fields of a compound type. """ from __future__ import absolute_import import six import numpy as np from ..common import ut, TestCase import h5py from h5py import h5s, h5t, h5d from h5py.highlevel import File class BaseSlicing(TestCase): def setUp(self): self.f = File(self.mktemp(), 'w') def tearDown(self): if self.f: self.f.close() class TestSingleElement(BaseSlicing): """ Feature: Retrieving a single element works with NumPy semantics """ def test_single_index(self): """ Single-element selection with [index] yields array scalar """ dset = self.f.create_dataset('x', (1,), dtype='i1') out = dset[0] self.assertIsInstance(out, np.int8) def test_single_null(self): """ Single-element selection with [()] yields ndarray """ dset = self.f.create_dataset('x', (1,), dtype='i1') out = dset[()] self.assertIsInstance(out, np.ndarray) self.assertEqual(out.shape, (1,)) def test_scalar_index(self): """ Slicing with [...] yields scalar ndarray """ dset = self.f.create_dataset('x', shape=(), dtype='f') out = dset[...] self.assertIsInstance(out, np.ndarray) self.assertEqual(out.shape, ()) def test_scalar_null(self): """ Slicing with [()] yields array scalar """ dset = self.f.create_dataset('x', shape=(), dtype='i1') out = dset[()] self.assertIsInstance(out, np.int8) def test_compound(self): """ Compound scalar is numpy.void, not tuple (issue 135) """ dt = np.dtype([('a','i4'),('b','f8')]) v = np.ones((4,), dtype=dt) dset = self.f.create_dataset('foo', (4,), data=v) self.assertEqual(dset[0], v[0]) self.assertIsInstance(dset[0], np.void) class TestObjectIndex(BaseSlicing): """ Feauture: numpy.object_ subtypes map to real Python objects """ def test_reference(self): """ Indexing a reference dataset returns a h5py.Reference instance """ dset = self.f.create_dataset('x', (1,), dtype=h5py.special_dtype(ref=h5py.Reference)) dset[0] = self.f.ref self.assertEqual(type(dset[0]), h5py.Reference) def test_regref(self): """ Indexing a region reference dataset returns a h5py.RegionReference """ dset1 = self.f.create_dataset('x', (10,10)) regref = dset1.regionref[...] dset2 = self.f.create_dataset('y', (1,), dtype=h5py.special_dtype(ref=h5py.RegionReference)) dset2[0] = regref self.assertEqual(type(dset2[0]), h5py.RegionReference) def test_reference_field(self): """ Compound types of which a reference is an element work right """ reftype = h5py.special_dtype(ref=h5py.Reference) dt = np.dtype([('a', 'i'),('b',reftype)]) dset = self.f.create_dataset('x', (1,), dtype=dt) dset[0] = (42, self.f['/'].ref) out = dset[0] self.assertEqual(type(out[1]), h5py.Reference) # isinstance does NOT work def test_scalar(self): """ Indexing returns a real Python object on scalar datasets """ dset = self.f.create_dataset('x', (), dtype=h5py.special_dtype(ref=h5py.Reference)) dset[()] = self.f.ref self.assertEqual(type(dset[()]), h5py.Reference) def test_bytestr(self): """ Indexing a byte string dataset returns a real python byte string """ dset = self.f.create_dataset('x', (1,), dtype=h5py.special_dtype(vlen=bytes)) dset[0] = b"Hello there!" self.assertEqual(type(dset[0]), bytes) class TestSimpleSlicing(TestCase): """ Feature: Simple NumPy-style slices (start:stop:step) are supported. """ def setUp(self): self.f = File(self.mktemp(), 'w') self.arr = np.arange(10) self.dset = self.f.create_dataset('x', data=self.arr) def tearDown(self): if self.f: self.f.close() def test_negative_stop(self): """ Negative stop indexes work as they do in NumPy """ self.assertArrayEqual(self.dset[2:-2], self.arr[2:-2]) class TestArraySlicing(BaseSlicing): """ Feature: Array types are handled appropriately """ def test_read(self): """ Read arrays tack array dimensions onto end of shape tuple """ dt = np.dtype('(3,)f8') dset = self.f.create_dataset('x',(10,),dtype=dt) self.assertEqual(dset.shape, (10,)) self.assertEqual(dset.dtype, dt) # Full read out = dset[...] self.assertEqual(out.dtype, np.dtype('f8')) self.assertEqual(out.shape, (10,3)) # Single element out = dset[0] self.assertEqual(out.dtype, np.dtype('f8')) self.assertEqual(out.shape, (3,)) # Range out = dset[2:8:2] self.assertEqual(out.dtype, np.dtype('f8')) self.assertEqual(out.shape, (3,3)) def test_write_broadcast(self): """ Array fill from constant is not supported (issue 211). """ dt = np.dtype('(3,)i') dset = self.f.create_dataset('x', (10,), dtype=dt) with self.assertRaises(TypeError): dset[...] = 42 def test_write_element(self): """ Write a single element to the array Issue 211. """ dt = np.dtype('(3,)f8') dset = self.f.create_dataset('x', (10,), dtype=dt) data = np.array([1,2,3.0]) dset[4] = data out = dset[4] self.assertTrue(np.all(out == data)) def test_write_slices(self): """ Write slices to array type """ dt = np.dtype('(3,)i') data1 = np.ones((2,), dtype=dt) data2 = np.ones((4,5), dtype=dt) dset = self.f.create_dataset('x', (10,9,11), dtype=dt) dset[0,0,2:4] = data1 self.assertArrayEqual(dset[0,0,2:4], data1) dset[3, 1:5, 6:11] = data2 self.assertArrayEqual(dset[3, 1:5, 6:11], data2) def test_roundtrip(self): """ Read the contents of an array and write them back Issue 211. """ dt = np.dtype('(3,)f8') dset = self.f.create_dataset('x', (10,), dtype=dt) out = dset[...] dset[...] = out self.assertTrue(np.all(dset[...] == out)) class TestZeroLengthSlicing(BaseSlicing): """ Slices resulting in empty arrays """ def test_slice_zero_length_dimension(self): """ Slice a dataset with a zero in its shape vector along the zero-length dimension """ for i, shape in enumerate([(0,), (0, 3), (0, 2, 1)]): dset = self.f.create_dataset('x%d'%i, shape, dtype=np.int, maxshape=(None,)*len(shape)) self.assertEqual(dset.shape, shape) out = dset[...] self.assertIsInstance(out, np.ndarray) self.assertEqual(out.shape, shape) out = dset[:] self.assertIsInstance(out, np.ndarray) self.assertEqual(out.shape, shape) if len(shape) > 1: out = dset[:, :1] self.assertIsInstance(out, np.ndarray) self.assertEqual(out.shape[:2], (0, 1)) def test_slice_other_dimension(self): """ Slice a dataset with a zero in its shape vector along a non-zero-length dimension """ for i, shape in enumerate([(3, 0), (1, 2, 0), (2, 0, 1)]): dset = self.f.create_dataset('x%d'%i, shape, dtype=np.int, maxshape=(None,)*len(shape)) self.assertEqual(dset.shape, shape) out = dset[:1] self.assertIsInstance(out, np.ndarray) self.assertEqual(out.shape, (1,)+shape[1:]) def test_slice_of_length_zero(self): """ Get a slice of length zero from a non-empty dataset """ for i, shape in enumerate([(3,), (2, 2,), (2, 1, 5)]): dset = self.f.create_dataset('x%d'%i, data=np.zeros(shape, np.int), maxshape=(None,)*len(shape)) self.assertEqual(dset.shape, shape) out = dset[1:1] self.assertIsInstance(out, np.ndarray) self.assertEqual(out.shape, (0,)+shape[1:]) class TestFieldNames(BaseSlicing): """ Field names for read & write """ dt = np.dtype([('a', 'f'), ('b', 'i'), ('c', 'f4')]) data = np.ones((100,), dtype=dt) def setUp(self): BaseSlicing.setUp(self) self.dset = self.f.create_dataset('x', (100,), dtype=self.dt) self.dset[...] = self.data def test_read(self): """ Test read with field selections (bytes and unicode) """ if six.PY2: # Byte strings are only allowed for field names on Py2 self.assertArrayEqual(self.dset[b'a'], self.data['a']) self.assertArrayEqual(self.dset[u'a'], self.data['a']) def test_unicode_names(self): """ Unicode field names for for read and write """ self.assertArrayEqual(self.dset[u'a'], self.data['a']) self.dset[u'a'] = 42 data = self.data.copy() data['a'] = 42 self.assertArrayEqual(self.dset[u'a'], data['a']) def test_write(self): """ Test write with field selections """ data2 = self.data.copy() data2['a'] *= 2 self.dset['a'] = data2 self.assertTrue(np.all(self.dset[...] == data2)) data2['b'] *= 4 self.dset['b'] = data2 self.assertTrue(np.all(self.dset[...] == data2)) data2['a'] *= 3 data2['c'] *= 3 self.dset['a','c'] = data2 self.assertTrue(np.all(self.dset[...] == data2)) def test_write_noncompound(self): """ Test write with non-compound source (single-field) """ data2 = self.data.copy() data2['b'] = 1.0 self.dset['b'] = 1.0 self.assertTrue(np.all(self.dset[...] == data2)) h5py-2.7.1/h5py/tests/common.py0000644000175000017500000001230413114361073017754 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. from __future__ import absolute_import import sys import os import shutil import tempfile from contextlib import contextmanager from six import unichr import numpy as np import h5py if sys.version_info[:2] == (2, 6): try: import unittest2 as ut except ImportError: raise ImportError( "unittest2 is required to run tests with Python 2.6") else: import unittest as ut # Check if non-ascii filenames are supported # Evidently this is the most reliable way to check # See also h5py issue #263 and ipython #466 # To test for this, run the testsuite with LC_ALL=C try: testfile, fname = tempfile.mkstemp(unichr(0x03b7)) except UnicodeError: UNICODE_FILENAMES = False else: UNICODE_FILENAMES = True os.close(testfile) os.unlink(fname) del fname del testfile class TestCase(ut.TestCase): """ Base class for unit tests. """ @classmethod def setUpClass(cls): cls.tempdir = tempfile.mkdtemp(prefix='h5py-test_') @classmethod def tearDownClass(cls): shutil.rmtree(cls.tempdir) def mktemp(self, suffix='.hdf5', prefix='', dir=None): if dir is None: dir = self.tempdir return tempfile.mktemp(suffix, prefix, dir=self.tempdir) def setUp(self): self.f = h5py.File(self.mktemp(), 'w') def tearDown(self): try: if self.f: self.f.close() except: pass def assertSameElements(self, a, b): for x in a: match = False for y in b: if x == y: match = True if not match: raise AssertionError("Item '%s' appears in a but not b" % x) for x in b: match = False for y in a: if x == y: match = True if not match: raise AssertionError("Item '%s' appears in b but not a" % x) def assertArrayEqual(self, dset, arr, message=None, precision=None): """ Make sure dset and arr have the same shape, dtype and contents, to within the given precision. Note that dset may be a NumPy array or an HDF5 dataset. """ if precision is None: precision = 1e-5 if message is None: message = '' else: message = ' (%s)' % message if np.isscalar(dset) or np.isscalar(arr): self.assert_( np.isscalar(dset) and np.isscalar(arr), 'Scalar/array mismatch ("%r" vs "%r")%s' % (dset, arr, message) ) self.assert_( dset - arr < precision, "Scalars differ by more than %.3f%s" % (precision, message) ) return self.assert_( dset.shape == arr.shape, "Shape mismatch (%s vs %s)%s" % (dset.shape, arr.shape, message) ) self.assert_( dset.dtype == arr.dtype, "Dtype mismatch (%s vs %s)%s" % (dset.dtype, arr.dtype, message) ) if arr.dtype.names is not None: for n in arr.dtype.names: message = '[FIELD %s] %s' % (n, message) self.assertArrayEqual(dset[n], arr[n], message=message, precision=precision) elif arr.dtype.kind in ('i', 'f'): self.assert_( np.all(np.abs(dset[...] - arr[...]) < precision), "Arrays differ by more than %.3f%s" % (precision, message) ) else: self.assert_( np.all(dset[...] == arr[...]), "Arrays are not equal (dtype %s) %s" % (arr.dtype.str, message) ) def assertNumpyBehavior(self, dset, arr, s): """ Apply slicing arguments "s" to both dset and arr. Succeeds if the results of the slicing are identical, or the exception raised is of the same type for both. "arr" must be a Numpy array; "dset" may be a NumPy array or dataset. """ exc = None try: arr_result = arr[s] except Exception as e: exc = type(e) if exc is None: self.assertArrayEqual(dset[s], arr_result) else: with self.assertRaises(exc): dset[s] NUMPY_RELEASE_VERSION = tuple([int(i) for i in np.__version__.split(".")[0:2]]) @contextmanager def closed_tempfile(suffix='', text=None): """ Context manager which yields the path to a closed temporary file with the suffix `suffix`. The file will be deleted on exiting the context. An additional argument `text` can be provided to have the file contain `text`. """ with tempfile.NamedTemporaryFile( 'w+t', suffix=suffix, delete=False ) as test_file: file_name = test_file.name if text is not None: test_file.write(text) test_file.flush() yield file_name shutil.rmtree(file_name, ignore_errors=True) h5py-2.7.1/h5py/tests/__init__.py0000644000175000017500000000224413114361073020225 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. from __future__ import print_function import sys from .common import ut from . import old, hl MODULES = old.MODULES + hl.MODULES def mname(obj): """ Get the full dotted name of the test method """ mod_name = obj.__class__.__module__.replace('h5py.tests.','') return "%s.%s.%s" % (mod_name, obj.__class__.__name__, obj._testMethodName) def run_tests(verbose=False): """ Run tests with TextTestRunner. Returns a TestResult instance. """ suite = ut.TestSuite() for m in MODULES: suite.addTests(ut.defaultTestLoader.loadTestsFromModule(m)) result = ut.TextTestRunner(verbosity=1).run(suite) if verbose: for (case, reason) in result.skipped: print("S %s (%s)" % (mname(case), reason), file=sys.stderr) for (case, reason) in result.expectedFailures: print("X %s" % mname(case), file=sys.stderr) return result h5py-2.7.1/h5py/tests/hl/0000755000175000017500000000000013152363123016515 5ustar tcaswelltcaswell00000000000000h5py-2.7.1/h5py/tests/hl/test_dataset_getitem.py0000644000175000017500000003522513146406212023300 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. """ Tests the h5py.Dataset.__getitem__ method. This module does not specifically test type conversion. The "type" axis therefore only tests objects which interact with the slicing system in unreliable ways; for example, compound and array types. See test_dataset_getitem_types for type-conversion tests. Tests are organized into TestCases by dataset shape and type. Test methods vary by slicing arg type. 1. Dataset shape: Empty Scalar 1D 3D 2. Type: Float Compound Array 3. Slicing arg types: Ellipsis Empty tuple Regular slice Indexing Index list Boolean mask Field names """ from __future__ import absolute_import import sys import numpy as np import h5py from ..common import ut, TestCase class TestEmpty(TestCase): def setUp(self): TestCase.setUp(self) sid = h5py.h5s.create(h5py.h5s.NULL) tid = h5py.h5t.C_S1.copy() tid.set_size(10) dsid = h5py.h5d.create(self.f.id, b'x', tid, sid) self.dset = h5py.Dataset(dsid) self.empty_obj = h5py.Empty(np.dtype("S10")) def test_ndim(self): """ Verify number of dimensions """ self.assertEquals(self.dset.ndim, 0) def test_shape(self): """ Verify shape """ self.assertEquals(self.dset.shape, None) def test_ellipsis(self): """ Ellipsis -> ValueError """ self.assertEquals(self.dset[...], self.empty_obj) def test_tuple(self): """ () -> IOError """ self.assertEquals(self.dset[()], self.empty_obj) def test_slice(self): """ slice -> ValueError """ with self.assertRaises(ValueError): self.dset[0:4] def test_index(self): """ index -> ValueError """ with self.assertRaises(ValueError): self.dset[0] def test_indexlist(self): """ index list -> ValueError """ with self.assertRaises(ValueError): self.dset[[1,2,5]] def test_mask(self): """ mask -> ValueError """ mask = np.array(True, dtype='bool') with self.assertRaises(ValueError): self.dset[mask] def test_fieldnames(self): """ field name -> ValueError """ with self.assertRaises(ValueError): self.dset['field'] class TestScalarFloat(TestCase): def setUp(self): TestCase.setUp(self) self.data = np.array(42.5, dtype='f') self.dset = self.f.create_dataset('x', data=self.data) def test_ndim(self): """ Verify number of dimensions """ self.assertEquals(self.dset.ndim, 0) def test_shape(self): """ Verify shape """ self.assertEquals(self.dset.shape, tuple()) def test_ellipsis(self): """ Ellipsis -> scalar ndarray """ out = self.dset[...] self.assertArrayEqual(out, self.data) def test_tuple(self): """ () -> bare item """ out = self.dset[()] self.assertArrayEqual(out, self.data.item()) def test_slice(self): """ slice -> ValueError """ with self.assertRaises(ValueError): self.dset[0:4] def test_index(self): """ index -> ValueError """ with self.assertRaises(ValueError): self.dset[0] # FIXME: NumPy has IndexError instead def test_indexlist(self): """ index list -> ValueError """ with self.assertRaises(ValueError): self.dset[[1,2,5]] # FIXME: NumPy permits this def test_mask(self): """ mask -> ValueError """ mask = np.array(True, dtype='bool') with self.assertRaises(ValueError): self.dset[mask] def test_fieldnames(self): """ field name -> ValueError (no fields) """ with self.assertRaises(ValueError): self.dset['field'] class TestScalarCompound(TestCase): def setUp(self): TestCase.setUp(self) self.data = np.array((42.5, -118, "Hello"), dtype=[('a', 'f'), ('b', 'i'), ('c', '|S10')]) self.dset = self.f.create_dataset('x', data=self.data) def test_ndim(self): """ Verify number of dimensions """ self.assertEquals(self.dset.ndim, 0) def test_shape(self): """ Verify shape """ self.assertEquals(self.dset.shape, tuple()) def test_ellipsis(self): """ Ellipsis -> scalar ndarray """ out = self.dset[...] # assertArrayEqual doesn't work with compounds; do manually self.assertIsInstance(out, np.ndarray) self.assertEqual(out.shape, self.data.shape) self.assertEqual(out.dtype, self.data.dtype) def test_tuple(self): """ () -> np.void instance """ out = self.dset[()] self.assertIsInstance(out, np.void) self.assertEqual(out.dtype, self.data.dtype) def test_slice(self): """ slice -> ValueError """ with self.assertRaises(ValueError): self.dset[0:4] def test_index(self): """ index -> ValueError """ with self.assertRaises(ValueError): self.dset[0] # FIXME: NumPy has IndexError instead def test_indexlist(self): """ index list -> ValueError """ with self.assertRaises(ValueError): self.dset[[1,2,5]] # FIXME: NumPy permits this def test_mask(self): """ mask -> ValueError """ mask = np.array(True, dtype='bool') with self.assertRaises(ValueError): self.dset[mask] # FIXME: NumPy returns a scalar ndarray def test_fieldnames(self): """ field name -> bare value """ out = self.dset['a'] self.assertIsInstance(out, np.float32) self.assertEqual(out, self.dset['a']) class TestScalarArray(TestCase): def setUp(self): TestCase.setUp(self) self.dt = np.dtype('(3,2)f') self.data = np.array([(3.2, -119), (42, 99.8), (3.14, 0)], dtype='f') self.dset = self.f.create_dataset('x', (), dtype=self.dt) self.dset[...] = self.data def test_ndim(self): """ Verify number of dimensions """ self.assertEquals(self.data.ndim, 2) self.assertEquals(self.dset.ndim, 0) def test_shape(self): """ Verify shape """ self.assertEquals(self.data.shape, (3, 2)) self.assertEquals(self.dset.shape, tuple()) def test_ellipsis(self): """ Ellipsis -> ndarray promoted to underlying shape """ out = self.dset[...] self.assertArrayEqual(out, self.data) def test_tuple(self): """ () -> same as ellipsis """ out = self.dset[...] self.assertArrayEqual(out, self.data) def test_slice(self): """ slice -> ValueError """ with self.assertRaises(ValueError): self.dset[0:4] def test_index(self): """ index -> ValueError """ with self.assertRaises(ValueError): self.dset[0] def test_indexlist(self): """ index list -> ValueError """ with self.assertRaises(ValueError): self.dset[[]] def test_mask(self): """ mask -> ValueError """ mask = np.array(True, dtype='bool') with self.assertRaises(ValueError): self.dset[mask] def test_fieldnames(self): """ field name -> ValueError (no fields) """ with self.assertRaises(ValueError): self.dset['field'] @ut.skipUnless(h5py.version.hdf5_version_tuple >= (1, 8, 7), 'HDF5 1.8.7+ required') class Test1DZeroFloat(TestCase): def setUp(self): TestCase.setUp(self) self.data = np.ones((0,), dtype='f') self.dset = self.f.create_dataset('x', data=self.data) def test_ndim(self): """ Verify number of dimensions """ self.assertEquals(self.dset.ndim, 1) def test_shape(self): """ Verify shape """ self.assertEquals(self.dset.shape, (0,)) def test_ellipsis(self): """ Ellipsis -> ndarray of matching shape """ self.assertNumpyBehavior(self.dset, self.data, np.s_[...]) def test_tuple(self): """ () -> same as ellipsis """ self.assertNumpyBehavior(self.dset, self.data, np.s_[()]) def test_slice(self): """ slice -> ndarray of shape (0,) """ self.assertNumpyBehavior(self.dset, self.data, np.s_[0:4]) # FIXME: NumPy raises IndexError def test_index(self): """ index -> out of range """ with self.assertRaises(ValueError): self.dset[0] # FIXME: Under NumPy this works and returns a shape-(0,) array # Also, at the moment it rasies UnboundLocalError (!) @ut.expectedFailure def test_indexlist(self): """ index list """ with self.assertRaises(ValueError): self.dset[[]] def test_mask(self): """ mask -> ndarray of matching shape """ mask = np.ones((0,), dtype='bool') self.assertNumpyBehavior(self.dset, self.data, np.s_[mask]) def test_fieldnames(self): """ field name -> ValueError (no fields) """ with self.assertRaises(ValueError): self.dset['field'] class Test1DFloat(TestCase): def setUp(self): TestCase.setUp(self) self.data = np.arange(13).astype('f') self.dset = self.f.create_dataset('x', data=self.data) def test_ndim(self): """ Verify number of dimensions """ self.assertEquals(self.dset.ndim, 1) def test_shape(self): """ Verify shape """ self.assertEquals(self.dset.shape, (13,)) def test_ellipsis(self): self.assertNumpyBehavior(self.dset, self.data, np.s_[...]) def test_tuple(self): self.assertNumpyBehavior(self.dset, self.data, np.s_[()]) def test_slice_simple(self): self.assertNumpyBehavior(self.dset, self.data, np.s_[0:4]) def test_slice_zerosize(self): self.assertNumpyBehavior(self.dset, self.data, np.s_[4:4]) def test_slice_strides(self): self.assertNumpyBehavior(self.dset, self.data, np.s_[1:7:3]) def test_slice_negindexes(self): self.assertNumpyBehavior(self.dset, self.data, np.s_[-8:-2:3]) def test_slice_outofrange(self): self.assertNumpyBehavior(self.dset, self.data, np.s_[100:400:3]) def test_slice_backwards(self): """ we disallow negative steps """ with self.assertRaises(ValueError): self.dset[::-1] def test_slice_zerostride(self): self.assertNumpyBehavior(self.dset, self.data, np.s_[::0]) def test_index_simple(self): self.assertNumpyBehavior(self.dset, self.data, np.s_[3]) def test_index_neg(self): self.assertNumpyBehavior(self.dset, self.data, np.s_[-4]) # FIXME: NumPy permits this... it adds a new axis in front def test_index_none(self): with self.assertRaises(TypeError): self.dset[None] # FIXME: NumPy raises IndexError # Also this currently raises UnboundLocalError. :( @ut.expectedFailure def test_index_illegal(self): """ Illegal slicing argument """ with self.assertRaises(TypeError): self.dset[{}] # FIXME: NumPy raises IndexError def test_index_outofrange(self): with self.assertRaises(ValueError): self.dset[100] def test_indexlist_simple(self): self.assertNumpyBehavior(self.dset, self.data, np.s_[[1,2,5]]) def test_indexlist_single_index_ellipsis(self): self.assertNumpyBehavior(self.dset, self.data, np.s_[[0], ...]) def test_indexlist_numpyarray_single_index_ellipsis(self): self.assertNumpyBehavior(self.dset, self.data, np.s_[np.array([0]), ...]) def test_indexlist_numpyarray_ellipsis(self): self.assertNumpyBehavior(self.dset, self.data, np.s_[np.array([1, 2, 5]), ...]) # Another UnboundLocalError @ut.expectedFailure def test_indexlist_empty(self): self.assertNumpyBehavior(self.dset, self.data, np.s_[[]]) # FIXME: NumPy has IndexError def test_indexlist_outofrange(self): with self.assertRaises(ValueError): self.dset[[100]] def test_indexlist_nonmonotonic(self): """ we require index list values to be strictly increasing """ with self.assertRaises(TypeError): self.dset[[1,3,2]] def test_indexlist_repeated(self): """ we forbid repeated index values """ with self.assertRaises(TypeError): self.dset[[1,1,2]] def test_mask_true(self): self.assertNumpyBehavior(self.dset, self.data, np.s_[self.data > -100]) def test_mask_false(self): self.assertNumpyBehavior(self.dset, self.data, np.s_[self.data > 100]) def test_mask_partial(self): self.assertNumpyBehavior(self.dset, self.data, np.s_[self.data > 5]) def test_mask_wrongsize(self): """ we require the boolean mask shape to match exactly """ with self.assertRaises(TypeError): self.dset[np.ones((2,), dtype='bool')] def test_fieldnames(self): """ field name -> ValueError (no fields) """ with self.assertRaises(ValueError): self.dset['field'] @ut.skipUnless(h5py.version.hdf5_version_tuple >= (1, 8, 7), 'HDF5 1.8.7+ required') class Test2DZeroFloat(TestCase): def setUp(self): TestCase.setUp(self) self.data = np.ones((0,3), dtype='f') self.dset = self.f.create_dataset('x', data=self.data) def test_ndim(self): """ Verify number of dimensions """ self.assertEquals(self.dset.ndim, 2) def test_shape(self): """ Verify shape """ self.assertEquals(self.dset.shape, (0, 3)) @ut.expectedFailure def test_indexlist(self): """ see issue #473 """ self.assertNumpyBehavior(self.dset, self.data, np.s_[:,[0,1,2]]) class TestVeryLargeArray(TestCase): def setUp(self): TestCase.setUp(self) self.dset = self.f.create_dataset('x', shape=(2**15, 2**16)) @ut.skipIf(sys.maxsize < 2**31, 'Maximum integer size >= 2**31 required') def test_size(self): self.assertEqual(self.dset.size, 2**31) h5py-2.7.1/h5py/tests/hl/__init__.py0000644000175000017500000000101313143100476020621 0ustar tcaswelltcaswell00000000000000 from __future__ import absolute_import from . import (test_dataset_getitem, test_dataset_swmr, test_dims_dimensionproxy, test_file, test_attribute_create, test_threads, test_datatype, ) MODULES = ( test_dataset_getitem, test_dataset_swmr, test_dims_dimensionproxy, test_file, test_attribute_create, test_threads, test_datatype, ) h5py-2.7.1/h5py/tests/hl/test_dataset_swmr.py0000644000175000017500000001317213120345673022634 0ustar tcaswelltcaswell00000000000000from __future__ import absolute_import import numpy as np import h5py from ..common import ut, TestCase @ut.skipUnless(h5py.version.hdf5_version_tuple < (1, 9, 178), 'SWMR is available. Skipping backwards compatible tests') class TestSwmrNotAvailable(TestCase): """ Test backwards compatibility behaviour when using SWMR functions with an older version of HDF5 which does not have this feature available. Skip this test if SWMR features *are* available in the HDF5 library. """ def setUp(self): TestCase.setUp(self) self.data = np.arange(13).astype('f') self.dset = self.f.create_dataset('data', chunks=(13,), maxshape=(None,), data=self.data) def test_open_swmr_raises(self): fname = self.f.filename self.f.close() with self.assertRaises(ValueError): self.f = h5py.File(fname, 'r', swmr=True) def test_refresh_raises(self): """ If the SWMR feature is not available then Dataset.refresh() should throw an AttributeError """ with self.assertRaises(AttributeError): self.dset.refresh() def test_flush_raises(self): """ If the SWMR feature is not available the Dataset.flush() should throw an AttributeError """ with self.assertRaises(AttributeError): self.dset.flush() def test_swmr_mode_raises(self): with self.assertRaises(AttributeError): self.f.swmr_mode @ut.skipUnless(h5py.version.hdf5_version_tuple >= (1, 9, 178), 'SWMR requires HDF5 >= 1.9.178') class TestDatasetSwmrRead(TestCase): """ Testing SWMR functions when reading a dataset. Skip this test if the HDF5 library does not have the SWMR features. """ def setUp(self): TestCase.setUp(self) self.data = np.arange(13).astype('f') self.dset = self.f.create_dataset('data', chunks=(13,), maxshape=(None,), data=self.data) fname = self.f.filename self.f.close() self.f = h5py.File(fname, 'r', swmr=True) self.dset = self.f['data'] def test_initial_swmr_mode_on(self): """ Verify that the file is initially in SWMR mode""" self.assertTrue(self.f.swmr_mode) def test_read_data(self): self.assertArrayEqual(self.dset, self.data) def test_refresh(self): self.dset.refresh() def test_force_swmr_mode_on_raises(self): """ Verify when reading a file cannot be forcibly switched to swmr mode. When reading with SWMR the file must be opened with swmr=True.""" with self.assertRaises(ValueError): self.f.swmr_mode = True self.assertTrue(self.f.swmr_mode) def test_force_swmr_mode_off_raises(self): """ Switching SWMR write mode off is only possible by closing the file. Attempts to forcibly switch off the SWMR mode should raise a ValueError. """ with self.assertRaises(ValueError): self.f.swmr_mode = False self.assertTrue(self.f.swmr_mode) @ut.skipUnless(h5py.version.hdf5_version_tuple >= (1, 9, 178), 'SWMR requires HDF5 >= 1.9.178') class TestDatasetSwmrWrite(TestCase): """ Testing SWMR functions when reading a dataset. Skip this test if the HDF5 library does not have the SWMR features. """ def setUp(self): """ First setup a file with a small chunked and empty dataset. No data written yet. """ # Note that when creating the file, the swmr=True is not required for # write, but libver='latest' is required. self.f = h5py.File(self.mktemp(), 'w', libver='latest') self.data = np.arange(4).astype('f') self.dset = self.f.create_dataset('data', shape=(0,), dtype=self.data.dtype, chunks=(2,), maxshape=(None,)) def test_initial_swmr_mode_off(self): """ Verify that the file is not initially in SWMR mode""" self.assertFalse(self.f.swmr_mode) def test_switch_swmr_mode_on(self): """ Switch to SWMR mode and verify """ self.f.swmr_mode = True self.assertTrue(self.f.swmr_mode) def test_switch_swmr_mode_off_raises(self): """ Switching SWMR write mode off is only possible by closing the file. Attempts to forcibly switch off the SWMR mode should raise a ValueError. """ self.f.swmr_mode = True self.assertTrue(self.f.swmr_mode) with self.assertRaises(ValueError): self.f.swmr_mode = False self.assertTrue(self.f.swmr_mode) def test_extend_dset(self): """ Extend and flush a SWMR dataset """ self.f.swmr_mode = True self.assertTrue(self.f.swmr_mode) self.dset.resize( self.data.shape ) self.dset[:] = self.data self.dset.flush() # Refresh and read back data for assertion self.dset.refresh() self.assertArrayEqual(self.dset, self.data) def test_extend_dset_multiple(self): self.f.swmr_mode = True self.assertTrue(self.f.swmr_mode) self.dset.resize( (4,) ) self.dset[0:] = self.data self.dset.flush() # Refresh and read back 1st data block for assertion self.dset.refresh() self.assertArrayEqual(self.dset, self.data) self.dset.resize( (8,) ) self.dset[4:] = self.data self.dset.flush() # Refresh and read back 1st data block for assertion self.dset.refresh() self.assertArrayEqual(self.dset[0:4], self.data) self.assertArrayEqual(self.dset[4:8], self.data) h5py-2.7.1/h5py/tests/hl/test_file.py0000644000175000017500000000332113114361073021044 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. """ Tests the h5py.File object. """ from __future__ import absolute_import import h5py from ..common import ut, TestCase def nfiles(): return h5py.h5f.get_obj_count(h5py.h5f.OBJ_ALL, h5py.h5f.OBJ_FILE) def ngroups(): return h5py.h5f.get_obj_count(h5py.h5f.OBJ_ALL, h5py.h5f.OBJ_GROUP) class TestDealloc(TestCase): """ Behavior on object dellocation. Note most of this behavior is delegated to FileID. """ def test_autoclose(self): """ File objects close automatically when out of scope, but other objects remain open. """ start_nfiles = nfiles() start_ngroups = ngroups() fname = self.mktemp() f = h5py.File(fname, 'w') g = f['/'] self.assertEqual(nfiles(), start_nfiles+1) self.assertEqual(ngroups(), start_ngroups+1) del f self.assertTrue(g) self.assertEqual(nfiles(), start_nfiles) self.assertEqual(ngroups(), start_ngroups+1) f = g.file self.assertTrue(f) self.assertEqual(nfiles(), start_nfiles+1) self.assertEqual(ngroups(), start_ngroups+1) del g self.assertEqual(nfiles(), start_nfiles+1) self.assertEqual(ngroups(), start_ngroups) del f self.assertEqual(nfiles(), start_nfiles) self.assertEqual(ngroups(), start_ngroups) h5py-2.7.1/h5py/tests/hl/test_datatype.py0000644000175000017500000001502013152157047021745 0ustar tcaswelltcaswell00000000000000""" Tests for the h5py.Datatype class. """ from __future__ import absolute_import from itertools import count import numpy as np import h5py from ..common import ut, TestCase class TestVlen(TestCase): """ Check that storage of vlen strings is carried out correctly. """ def assertVlenArrayEqual(self, dset, arr, message=None, precision=None): self.assert_( dset.shape == arr.shape, "Shape mismatch (%s vs %s)%s" % (dset.shape, arr.shape, message) ) for (i, d, a) in zip(count(), dset, arr): self.assertArrayEqual(d, a, message, precision) def test_compound(self): fields = [] fields.append(('field_1', h5py.special_dtype(vlen=str))) fields.append(('field_2', np.int32)) dt = np.dtype(fields) self.f['mytype'] = np.dtype(dt) dt_out = self.f['mytype'].dtype.fields['field_1'][0] self.assertEqual(h5py.check_dtype(vlen=dt_out), str) def test_compound_vlen_bool(self): vidt = h5py.special_dtype(vlen=np.uint8) def a(items): return np.array(items, dtype=np.uint8) f = self.f dt_vb = np.dtype([ ('foo', vidt), ('logical', np.bool)]) vb = f.create_dataset('dt_vb', shape=(4,), dtype=dt_vb) data = np.array([(a([1,2,3]), True), (a([1 ]), False), (a([1,5 ]), True), (a([], ), False),], dtype=dt_vb) vb[:] = data actual = f['dt_vb'][:] self.assertVlenArrayEqual(data['foo'], actual['foo']) self.assertArrayEqual(data['logical'], actual['logical']) dt_vv = np.dtype([ ('foo', vidt), ('bar', vidt)]) f.create_dataset('dt_vv', shape=(4,), dtype=dt_vv) dt_vvb = np.dtype([ ('foo', vidt), ('bar', vidt), ('logical', np.bool)]) vvb = f.create_dataset('dt_vvb', shape=(2,), dtype=dt_vvb) dt_bvv = np.dtype([ ('logical', np.bool), ('foo', vidt), ('bar', vidt)]) bvv = f.create_dataset('dt_bvv', shape=(2,), dtype=dt_bvv) data = np.array([(True, a([1,2,3]), a([1,2]) ), (False, a([]), a([2,4,6])),], dtype=bvv) bvv[:] = data actual = bvv[:] self.assertVlenArrayEqual(data['foo'], actual['foo']) self.assertVlenArrayEqual(data['bar'], actual['bar']) self.assertArrayEqual(data['logical'], actual['logical']) def test_compound_vlen_enum(self): eidt = h5py.special_dtype(enum=(np.uint8, {'OFF': 0, 'ON': 1})) vidt = h5py.special_dtype(vlen=np.uint8) def a(items): return np.array(items, dtype=np.uint8) f = self.f dt_vve = np.dtype([ ('foo', vidt), ('bar', vidt), ('switch', eidt)]) vve = f.create_dataset('dt_vve', shape=(2,), dtype=dt_vve) data = np.array([(a([1,2,3]), a([1,2]), 1), (a([]), a([2,4,6]), 0),], dtype=dt_vve) vve[:] = data actual = vve[:] self.assertVlenArrayEqual(data['foo'], actual['foo']) self.assertVlenArrayEqual(data['bar'], actual['bar']) self.assertArrayEqual(data['switch'], actual['switch']) def test_vlen_enum(self): fname = self.mktemp() arr1 = [[1],[1,2]] dt1 = h5py.special_dtype(vlen=h5py.special_dtype( enum=('i', dict(foo=1, bar=2)))) with h5py.File(fname,'w') as f: df1 = f.create_dataset('test', (len(arr1),), dtype=dt1) df1[:] = np.array(arr1) with h5py.File(fname,'r') as f: df2 = f['test'] dt2 = df2.dtype arr2 = [e.tolist() for e in df2[:]] self.assertEqual(arr1, arr2) self.assertEqual(h5py.check_dtype(enum=h5py.check_dtype(vlen=dt1)), h5py.check_dtype(enum=h5py.check_dtype(vlen=dt2))) class TestOffsets(TestCase): """ Check that compound members with aligned or manual offsets are handled correctly. """ def test_compound_vlen(self): vidt = h5py.special_dtype(vlen=np.uint8) eidt = h5py.special_dtype(enum=(np.uint8, {'OFF': 0, 'ON': 1})) for np_align in (False, True): dt = np.dtype([ ('a', eidt), ('foo', vidt), ('bar', vidt), ('switch', eidt)], align=np_align) np_offsets = [dt.fields[i][1] for i in dt.names] for logical in (False, True): if logical and np_align: # Vlen types have different size in the numpy struct self.assertRaises(TypeError, h5py.h5t.py_create, dt, logical=logical) else: ht = h5py.h5t.py_create(dt, logical=logical) offsets = [ht.get_member_offset(i) for i in range(ht.get_nmembers())] if np_align: self.assertEqual(np_offsets, offsets) def test_aligned_offsets(self): dt = np.dtype('i2,i8', align=True) ht = h5py.h5t.py_create(dt) self.assertEqual(dt.itemsize, ht.get_size()) self.assertEqual( [dt.fields[i][1] for i in dt.names], [ht.get_member_offset(i) for i in range(ht.get_nmembers())] ) def test_aligned_data(self): dt = np.dtype('i2,f8', align=True) data = np.empty(10, dtype=dt) data['f0'] = np.array(np.random.randint(-100, 100, size=data.size), dtype='i2') data['f1'] = np.random.rand(data.size) fname = self.mktemp() with h5py.File(fname, 'w') as f: f['data'] = data with h5py.File(fname, 'r') as f: self.assertArrayEqual(f['data'], data) def test_out_of_order_offsets(self): dt = np.dtype({ 'names' : ['f1', 'f2', 'f3'], 'formats' : [' empty list """ dset = self.f.create_dataset('x', (10,)) self.assertEqual(dset.dims[0].items(), []) h5py-2.7.1/h5py/tests/hl/test_threads.py0000644000175000017500000000315013114361073021557 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. """ Tests the h5py.File object. """ from __future__ import absolute_import import threading import h5py from ..common import ut, TestCase class TestErrorPrinting(TestCase): """ Verify the error printing is squashed in all threads. """ def test_printing(self): """ No console messages should be shown from containership tests """ # Unfortunately we can't have this test assert anything, as # HDF5 writes directly to stderr. But it will show up in the # console output. import threading def test(): with h5py.File(self.mktemp(), 'w') as newfile: try: doesnt_exist = newfile['doesnt_exist'].value except KeyError: pass th = threading.Thread(target=test) th.start() th.join() def test_attr_printing(self): """ No console messages should be shown for non-existing attributes """ def test(): with h5py.File(self.mktemp(), 'w') as newfile: newfile['newdata'] = [1,2,3] try: nonexistent_attr = newfile['newdata'].attrs['nonexistent_attr'] except KeyError: pass th = threading.Thread(target=test) th.start() th.join() h5py-2.7.1/h5py/h5fd.pyx0000644000175000017500000000576613025343121016351 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. # This file contains code or comments from the HDF5 library. See the file # licenses/hdf5.txt for the full HDF5 software license. """ File driver constants (H5FD*). """ # === Multi-file driver ======================================================= MEM_DEFAULT = H5FD_MEM_DEFAULT MEM_SUPER = H5FD_MEM_SUPER MEM_BTREE = H5FD_MEM_BTREE MEM_DRAW = H5FD_MEM_DRAW MEM_GHEAP = H5FD_MEM_GHEAP MEM_LHEAP = H5FD_MEM_LHEAP MEM_OHDR = H5FD_MEM_OHDR MEM_NTYPES = H5FD_MEM_NTYPES # === MPI driver ============================================================== MPIO_INDEPENDENT = H5FD_MPIO_INDEPENDENT MPIO_COLLECTIVE = H5FD_MPIO_COLLECTIVE # === Driver types ============================================================ CORE = H5FD_CORE FAMILY = H5FD_FAMILY LOG = H5FD_LOG MPIO = H5FD_MPIO MPIPOSIX = -1 MULTI = H5FD_MULTI SEC2 = H5FD_SEC2 STDIO = H5FD_STDIO IF UNAME_SYSNAME == "Windows": WINDOWS = H5FD_WINDOWS ELSE: WINDOWS = -1 # === Logging driver ========================================================== LOG_LOC_READ = H5FD_LOG_LOC_READ # 0x0001 LOG_LOC_WRITE = H5FD_LOG_LOC_WRITE # 0x0002 LOG_LOC_SEEK = H5FD_LOG_LOC_SEEK # 0x0004 LOG_LOC_IO = H5FD_LOG_LOC_IO # (H5FD_LOG_LOC_READ|H5FD_LOG_LOC_WRITE|H5FD_LOG_LOC_SEEK) # Flags for tracking number of times each byte is read/written LOG_FILE_READ = H5FD_LOG_FILE_READ # 0x0008 LOG_FILE_WRITE= H5FD_LOG_FILE_WRITE # 0x0010 LOG_FILE_IO = H5FD_LOG_FILE_IO # (H5FD_LOG_FILE_READ|H5FD_LOG_FILE_WRITE) # Flag for tracking "flavor" (type) of information stored at each byte LOG_FLAVOR = H5FD_LOG_FLAVOR # 0x0020 # Flags for tracking total number of reads/writes/seeks LOG_NUM_READ = H5FD_LOG_NUM_READ # 0x0040 LOG_NUM_WRITE = H5FD_LOG_NUM_WRITE # 0x0080 LOG_NUM_SEEK = H5FD_LOG_NUM_SEEK # 0x0100 LOG_NUM_IO = H5FD_LOG_NUM_IO # (H5FD_LOG_NUM_READ|H5FD_LOG_NUM_WRITE|H5FD_LOG_NUM_SEEK) # Flags for tracking time spent in open/read/write/seek/close LOG_TIME_OPEN = H5FD_LOG_TIME_OPEN # 0x0200 # Not implemented yet LOG_TIME_READ = H5FD_LOG_TIME_READ # 0x0400 # Not implemented yet LOG_TIME_WRITE= H5FD_LOG_TIME_WRITE # 0x0800 # Partially implemented (need to track total time) LOG_TIME_SEEK = H5FD_LOG_TIME_SEEK # 0x1000 # Partially implemented (need to track total time & track time for seeks during reading) LOG_TIME_CLOSE= H5FD_LOG_TIME_CLOSE # 0x2000 # Fully implemented LOG_TIME_IO = H5FD_LOG_TIME_IO # (H5FD_LOG_TIME_OPEN|H5FD_LOG_TIME_READ|H5FD_LOG_TIME_WRITE|H5FD_LOG_TIME_SEEK|H5FD_LOG_TIME_CLOSE) # Flag for tracking allocation of space in file LOG_ALLOC = H5FD_LOG_ALLOC # 0x4000 LOG_ALL = H5FD_LOG_ALL # (H5FD_LOG_ALLOC|H5FD_LOG_TIME_IO|H5FD_LOG_NUM_IO|H5FD_LOG_FLAVOR|H5FD_LOG_FILE_IO|H5FD_LOG_LOC_IO) h5py-2.7.1/h5py/h5t.pyx0000644000175000017500000014102713152157047016225 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. """ HDF5 "H5T" data-type API This module contains the datatype identifier class TypeID, and its subclasses which represent things like integer/float/compound identifiers. The majority of the H5T API is presented as methods on these identifiers. """ # Pyrex compile-time imports include "config.pxi" from _objects cimport pdefault from numpy cimport dtype, ndarray from h5r cimport Reference, RegionReference from utils cimport emalloc, efree, \ require_tuple, convert_dims, convert_tuple # Runtime imports import sys import operator from h5 import get_config import numpy as np from ._objects import phil, with_phil import platform cfg = get_config() PY3 = sys.version_info[0] == 3 MACHINE = platform.machine() # === Custom C API ============================================================ cpdef TypeID typewrap(hid_t id_): cdef H5T_class_t cls cls = H5Tget_class(id_) if cls == H5T_INTEGER: pcls = TypeIntegerID elif cls == H5T_FLOAT: pcls = TypeFloatID elif cls == H5T_TIME: pcls = TypeTimeID elif cls == H5T_STRING: pcls = TypeStringID elif cls == H5T_BITFIELD: pcls = TypeBitfieldID elif cls == H5T_OPAQUE: pcls = TypeOpaqueID elif cls == H5T_COMPOUND: pcls = TypeCompoundID elif cls == H5T_REFERENCE: pcls = TypeReferenceID elif cls == H5T_ENUM: pcls = TypeEnumID elif cls == H5T_VLEN: pcls = TypeVlenID elif cls == H5T_ARRAY: pcls = TypeArrayID else: pcls = TypeID return pcls(id_) cdef object lockid(hid_t id_in): cdef TypeID tid tid = typewrap(id_in) tid.locked = 1 return tid # === Public constants and data structures ==================================== # Enumeration H5T_class_t NO_CLASS = H5T_NO_CLASS INTEGER = H5T_INTEGER FLOAT = H5T_FLOAT TIME = H5T_TIME STRING = H5T_STRING BITFIELD = H5T_BITFIELD OPAQUE = H5T_OPAQUE COMPOUND = H5T_COMPOUND REFERENCE = H5T_REFERENCE ENUM = H5T_ENUM VLEN = H5T_VLEN ARRAY = H5T_ARRAY # Enumeration H5T_sign_t SGN_NONE = H5T_SGN_NONE SGN_2 = H5T_SGN_2 # Enumeration H5T_order_t ORDER_LE = H5T_ORDER_LE ORDER_BE = H5T_ORDER_BE ORDER_VAX = H5T_ORDER_VAX ORDER_NONE = H5T_ORDER_NONE DIR_DEFAULT = H5T_DIR_DEFAULT DIR_ASCEND = H5T_DIR_ASCEND DIR_DESCEND = H5T_DIR_DESCEND # Enumeration H5T_str_t STR_NULLTERM = H5T_STR_NULLTERM STR_NULLPAD = H5T_STR_NULLPAD STR_SPACEPAD = H5T_STR_SPACEPAD # Enumeration H5T_norm_t NORM_IMPLIED = H5T_NORM_IMPLIED NORM_MSBSET = H5T_NORM_MSBSET NORM_NONE = H5T_NORM_NONE # Enumeration H5T_cset_t: CSET_ASCII = H5T_CSET_ASCII # Enumeration H5T_pad_t: PAD_ZERO = H5T_PAD_ZERO PAD_ONE = H5T_PAD_ONE PAD_BACKGROUND = H5T_PAD_BACKGROUND if sys.byteorder == "little": # Custom python addition ORDER_NATIVE = H5T_ORDER_LE else: ORDER_NATIVE = H5T_ORDER_BE # For conversion BKG_NO = H5T_BKG_NO BKG_TEMP = H5T_BKG_TEMP BKG_YES = H5T_BKG_YES # --- Built-in HDF5 datatypes ------------------------------------------------- # IEEE floating-point IEEE_F32LE = lockid(H5T_IEEE_F32LE) IEEE_F32BE = lockid(H5T_IEEE_F32BE) IEEE_F64LE = lockid(H5T_IEEE_F64LE) IEEE_F64BE = lockid(H5T_IEEE_F64BE) # Signed 2's complement integer types STD_I8LE = lockid(H5T_STD_I8LE) STD_I16LE = lockid(H5T_STD_I16LE) STD_I32LE = lockid(H5T_STD_I32LE) STD_I64LE = lockid(H5T_STD_I64LE) STD_I8BE = lockid(H5T_STD_I8BE) STD_I16BE = lockid(H5T_STD_I16BE) STD_I32BE = lockid(H5T_STD_I32BE) STD_I64BE = lockid(H5T_STD_I64BE) # Unsigned integers STD_U8LE = lockid(H5T_STD_U8LE) STD_U16LE = lockid(H5T_STD_U16LE) STD_U32LE = lockid(H5T_STD_U32LE) STD_U64LE = lockid(H5T_STD_U64LE) STD_U8BE = lockid(H5T_STD_U8BE) STD_U16BE = lockid(H5T_STD_U16BE) STD_U32BE = lockid(H5T_STD_U32BE) STD_U64BE = lockid(H5T_STD_U64BE) # Native types by bytesize NATIVE_INT8 = lockid(H5T_NATIVE_INT8) NATIVE_UINT8 = lockid(H5T_NATIVE_UINT8) NATIVE_INT16 = lockid(H5T_NATIVE_INT16) NATIVE_UINT16 = lockid(H5T_NATIVE_UINT16) NATIVE_INT32 = lockid(H5T_NATIVE_INT32) NATIVE_UINT32 = lockid(H5T_NATIVE_UINT32) NATIVE_INT64 = lockid(H5T_NATIVE_INT64) NATIVE_UINT64 = lockid(H5T_NATIVE_UINT64) NATIVE_FLOAT = lockid(H5T_NATIVE_FLOAT) NATIVE_DOUBLE = lockid(H5T_NATIVE_DOUBLE) NATIVE_LDOUBLE = lockid(H5T_NATIVE_LDOUBLE) # Unix time types UNIX_D32LE = lockid(H5T_UNIX_D32LE) UNIX_D64LE = lockid(H5T_UNIX_D64LE) UNIX_D32BE = lockid(H5T_UNIX_D32BE) UNIX_D64BE = lockid(H5T_UNIX_D64BE) # Reference types STD_REF_OBJ = lockid(H5T_STD_REF_OBJ) STD_REF_DSETREG = lockid(H5T_STD_REF_DSETREG) # Null terminated (C) and Fortran string types C_S1 = lockid(H5T_C_S1) FORTRAN_S1 = lockid(H5T_FORTRAN_S1) VARIABLE = H5T_VARIABLE # Character sets CSET_ASCII = H5T_CSET_ASCII CSET_UTF8 = H5T_CSET_UTF8 # Mini floats IEEE_F16BE = IEEE_F32BE.copy() IEEE_F16BE.set_fields(15, 10, 5, 0, 10) IEEE_F16BE.set_size(2) IEEE_F16BE.set_ebias(15) IEEE_F16BE.lock() IEEE_F16LE = IEEE_F16BE.copy() IEEE_F16LE.set_order(H5T_ORDER_LE) IEEE_F16LE.lock() # Custom Python object pointer type cdef hid_t H5PY_OBJ = H5Tcreate(H5T_OPAQUE, sizeof(PyObject*)) H5Tset_tag(H5PY_OBJ, "PYTHON:OBJECT") H5Tlock(H5PY_OBJ) PYTHON_OBJECT = lockid(H5PY_OBJ) # Translation tables for HDF5 -> NumPy dtype conversion cdef dict _order_map = { H5T_ORDER_NONE: '|', H5T_ORDER_LE: '<', H5T_ORDER_BE: '>'} cdef dict _sign_map = { H5T_SGN_NONE: 'u', H5T_SGN_2: 'i' } # Available floating point types available_ftypes = dict() for ftype in np.typeDict.values(): if np.issubdtype(ftype, float): available_ftypes[np.dtype(ftype).itemsize] = np.finfo(ftype) # === General datatype operations ============================================= @with_phil def create(int classtype, size_t size): """(INT classtype, UINT size) => TypeID Create a new HDF5 type object. Legal class values are COMPOUND and OPAQUE. Use enum_create for enums. """ # HDF5 versions 1.6.X segfault with anything else if classtype != H5T_COMPOUND and classtype != H5T_OPAQUE: raise ValueError("Class must be COMPOUND or OPAQUE.") return typewrap(H5Tcreate(classtype, size)) @with_phil def open(ObjectID group not None, char* name): """(ObjectID group, STRING name) => TypeID Open a named datatype from a file. """ return typewrap(H5Topen(group.id, name)) @with_phil def array_create(TypeID base not None, object dims_tpl): """(TypeID base, TUPLE dimensions) => TypeArrayID Create a new array datatype, using and HDF5 parent type and dimensions given via a tuple of positive integers. "Unlimited" dimensions are not allowed. """ cdef hsize_t rank cdef hsize_t *dims = NULL require_tuple(dims_tpl, 0, -1, "dims_tpl") rank = len(dims_tpl) dims = emalloc(sizeof(hsize_t)*rank) try: convert_tuple(dims_tpl, dims, rank) return TypeArrayID(H5Tarray_create(base.id, rank, dims, NULL)) finally: efree(dims) @with_phil def enum_create(TypeID base not None): """(TypeID base) => TypeID Create a new enumerated type based on an (integer) parent type. """ return typewrap(H5Tenum_create(base.id)) @with_phil def vlen_create(TypeID base not None): """(TypeID base) => TypeID Create a new variable-length datatype, using any HDF5 type as a base. Although the Python interface can manipulate these types, there is no provision for reading/writing vlen data. """ return typewrap(H5Tvlen_create(base.id)) @with_phil def decode(char* buf): """(STRING buf) => TypeID Unserialize an HDF5 type. You can also do this with the native Python pickling machinery. """ return typewrap(H5Tdecode(buf)) # === Base type class ========================================================= cdef class TypeID(ObjectID): """ Base class for type identifiers (implements common operations) * Hashable: If committed; in HDF5 1.8.X, also if locked * Equality: Logical H5T comparison """ def __hash__(self): with phil: if self._hash is None: try: # Try to use object header first return ObjectID.__hash__(self) except TypeError: # It's a transient type object if self.locked: self._hash = hash(self.encode()) else: raise TypeError("Only locked or committed types can be hashed") return self._hash def __richcmp__(self, object other, int how): cdef bint truthval = 0 with phil: if how != 2 and how != 3: return NotImplemented if isinstance(other, TypeID): truthval = self.equal(other) if how == 2: return truthval return not truthval def __copy__(self): cdef TypeID cpy with phil: cpy = ObjectID.__copy__(self) return cpy property dtype: """ A Numpy-style dtype object representing this object. """ def __get__(self): with phil: return self.py_dtype() cdef object py_dtype(self): raise TypeError("No NumPy equivalent for %s exists" % self.__class__.__name__) @with_phil def commit(self, ObjectID group not None, char* name, ObjectID lcpl=None): """(ObjectID group, STRING name, PropID lcpl=None) Commit this (transient) datatype to a named datatype in a file. If present, lcpl may be a link creation property list. """ H5Tcommit2(group.id, name, self.id, pdefault(lcpl), H5P_DEFAULT, H5P_DEFAULT) @with_phil def committed(self): """() => BOOL is_comitted Determine if a given type object is named (T) or transient (F). """ return (H5Tcommitted(self.id)) @with_phil def copy(self): """() => TypeID Create a copy of this type object. """ return typewrap(H5Tcopy(self.id)) @with_phil def equal(self, TypeID typeid): """(TypeID typeid) => BOOL Logical comparison between datatypes. Also called by Python's "==" operator. """ return (H5Tequal(self.id, typeid.id)) @with_phil def lock(self): """() Lock this datatype, which makes it immutable and indestructible. Once locked, it can't be unlocked. """ H5Tlock(self.id) self.locked = 1 @with_phil def get_class(self): """() => INT classcode Determine the datatype's class code. """ return H5Tget_class(self.id) @with_phil def set_size(self, size_t size): """(UINT size) Set the total size of the datatype, in bytes. """ H5Tset_size(self.id, size) @with_phil def get_size(self): """ () => INT size Determine the total size of a datatype, in bytes. """ return H5Tget_size(self.id) @with_phil def get_super(self): """() => TypeID Determine the parent type of an array, enumeration or vlen datatype. """ return typewrap(H5Tget_super(self.id)) @with_phil def detect_class(self, int classtype): """(INT classtype) => BOOL class_is_present Determine if a member of the given class exists in a compound datatype. The search is recursive. """ return (H5Tdetect_class(self.id, classtype)) @with_phil def encode(self): """() => STRING Serialize an HDF5 type. Bear in mind you can also use the native Python pickle/unpickle machinery to do this. The returned string may contain binary values, including NULLs. """ cdef size_t nalloc = 0 cdef char* buf = NULL H5Tencode(self.id, NULL, &nalloc) buf = emalloc(sizeof(char)*nalloc) try: H5Tencode(self.id, buf, &nalloc) pystr = PyBytes_FromStringAndSize(buf, nalloc) finally: efree(buf) return pystr def __reduce__(self): with phil: return (type(self), (-1,), self.encode()) def __setstate__(self, char* state): with phil: self.id = H5Tdecode(state) # === Top-level classes (inherit directly from TypeID) ======================== cdef class TypeArrayID(TypeID): """ Represents an array datatype """ @with_phil def get_array_ndims(self): """() => INT rank Get the rank of the given array datatype. """ return H5Tget_array_ndims(self.id) @with_phil def get_array_dims(self): """() => TUPLE dimensions Get the dimensions of the given array datatype as a tuple of integers. """ cdef hsize_t rank cdef hsize_t* dims = NULL rank = H5Tget_array_dims(self.id, NULL, NULL) dims = emalloc(sizeof(hsize_t)*rank) try: H5Tget_array_dims(self.id, dims, NULL) return convert_dims(dims, rank) finally: efree(dims) cdef object py_dtype(self): # Numpy translation function for array types cdef TypeID tmp_type tmp_type = self.get_super() base_dtype = tmp_type.py_dtype() shape = self.get_array_dims() return dtype( (base_dtype, shape) ) cdef class TypeOpaqueID(TypeID): """ Represents an opaque type """ @with_phil def set_tag(self, char* tag): """(STRING tag) Set a string describing the contents of an opaque datatype. Limited to 256 characters. """ H5Tset_tag(self.id, tag) @with_phil def get_tag(self): """() => STRING tag Get the tag associated with an opaque datatype. """ cdef char* buf = NULL try: buf = H5Tget_tag(self.id) assert buf != NULL tag = buf return tag finally: free(buf) cdef object py_dtype(self): # Numpy translation function for opaque types return dtype("|V" + str(self.get_size())) cdef class TypeStringID(TypeID): """ String datatypes, both fixed and vlen. """ @with_phil def is_variable_str(self): """() => BOOL is_variable Determine if the given string datatype is a variable-length string. """ return (H5Tis_variable_str(self.id)) @with_phil def get_cset(self): """() => INT character_set Retrieve the character set used for a string. """ return H5Tget_cset(self.id) @with_phil def set_cset(self, int cset): """(INT character_set) Set the character set used for a string. """ H5Tset_cset(self.id, cset) @with_phil def get_strpad(self): """() => INT padding_type Get the padding type. Legal values are: STR_NULLTERM NULL termination only (C style) STR_NULLPAD Pad buffer with NULLs STR_SPACEPAD Pad buffer with spaces (FORTRAN style) """ return H5Tget_strpad(self.id) @with_phil def set_strpad(self, int pad): """(INT pad) Set the padding type. Legal values are: STR_NULLTERM NULL termination only (C style) STR_NULLPAD Pad buffer with NULLs STR_SPACEPAD Pad buffer with spaces (FORTRAN style) """ H5Tset_strpad(self.id, pad) cdef object py_dtype(self): # Numpy translation function for string types if self.is_variable_str(): if self.get_cset() == H5T_CSET_ASCII: return special_dtype(vlen=bytes) elif self.get_cset() == H5T_CSET_UTF8: return special_dtype(vlen=unicode) else: raise TypeError("Unknown string encoding (value %d)" % self.get_cset()) return dtype("|S" + str(self.get_size())) cdef class TypeVlenID(TypeID): """ Non-string vlen datatypes. """ cdef object py_dtype(self): # get base type id cdef TypeID base_type base_type = self.get_super() return special_dtype(vlen=base_type.dtype) cdef class TypeTimeID(TypeID): """ Unix-style time_t (deprecated) """ pass cdef class TypeBitfieldID(TypeID): """ HDF5 bitfield type """ pass cdef class TypeReferenceID(TypeID): """ HDF5 object or region reference """ cdef object py_dtype(self): if H5Tequal(self.id, H5T_STD_REF_OBJ): return special_dtype(ref=Reference) elif H5Tequal(self.id, H5T_STD_REF_DSETREG): return special_dtype(ref=RegionReference) else: raise TypeError("Unknown reference type") # === Numeric classes (integers and floats) =================================== cdef class TypeAtomicID(TypeID): """ Base class for atomic datatypes (float or integer) """ @with_phil def get_order(self): """() => INT order Obtain the byte order of the datatype; one of: - ORDER_LE - ORDER_BE """ return H5Tget_order(self.id) @with_phil def set_order(self, int order): """(INT order) Set the byte order of the datatype; one of: - ORDER_LE - ORDER_BE """ H5Tset_order(self.id, order) @with_phil def get_precision(self): """() => UINT precision Get the number of significant bits (excludes padding). """ return H5Tget_precision(self.id) @with_phil def set_precision(self, size_t precision): """(UINT precision) Set the number of significant bits (excludes padding). """ H5Tset_precision(self.id, precision) @with_phil def get_offset(self): """() => INT offset Get the offset of the first significant bit. """ return H5Tget_offset(self.id) @with_phil def set_offset(self, size_t offset): """(UINT offset) Set the offset of the first significant bit. """ H5Tset_offset(self.id, offset) @with_phil def get_pad(self): """() => (INT lsb_pad_code, INT msb_pad_code) Determine the padding type. Possible values are: - PAD_ZERO - PAD_ONE - PAD_BACKGROUND """ cdef H5T_pad_t lsb cdef H5T_pad_t msb H5Tget_pad(self.id, &lsb, &msb) return (lsb, msb) @with_phil def set_pad(self, int lsb, int msb): """(INT lsb_pad_code, INT msb_pad_code) Set the padding type. Possible values are: - PAD_ZERO - PAD_ONE - PAD_BACKGROUND """ H5Tset_pad(self.id, lsb, msb) cdef class TypeIntegerID(TypeAtomicID): """ Integer atomic datatypes """ @with_phil def get_sign(self): """() => INT sign Get the "signedness" of the datatype; one of: SGN_NONE Unsigned SGN_2 Signed 2's complement """ return H5Tget_sign(self.id) @with_phil def set_sign(self, int sign): """(INT sign) Set the "signedness" of the datatype; one of: SGN_NONE Unsigned SGN_2 Signed 2's complement """ H5Tset_sign(self.id, sign) cdef object py_dtype(self): # Translation function for integer types return dtype( _order_map[self.get_order()] + _sign_map[self.get_sign()] + str(self.get_size()) ) cdef class TypeFloatID(TypeAtomicID): """ Floating-point atomic datatypes """ @with_phil def get_fields(self): """() => TUPLE field_info Get information about floating-point bit fields. See the HDF5 docs for a full description. Tuple has the following members: 0. UINT spos 1. UINT epos 2. UINT esize 3. UINT mpos 4. UINT msize """ cdef size_t spos, epos, esize, mpos, msize H5Tget_fields(self.id, &spos, &epos, &esize, &mpos, &msize) return (spos, epos, esize, mpos, msize) @with_phil def set_fields(self, size_t spos, size_t epos, size_t esize, size_t mpos, size_t msize): """(UINT spos, UINT epos, UINT esize, UINT mpos, UINT msize) Set floating-point bit fields. Refer to the HDF5 docs for argument definitions. """ H5Tset_fields(self.id, spos, epos, esize, mpos, msize) @with_phil def get_ebias(self): """() => UINT ebias Get the exponent bias. """ return H5Tget_ebias(self.id) @with_phil def set_ebias(self, size_t ebias): """(UINT ebias) Set the exponent bias. """ H5Tset_ebias(self.id, ebias) @with_phil def get_norm(self): """() => INT normalization_code Get the normalization strategy. Legal values are: - NORM_IMPLIED - NORM_MSBSET - NORM_NONE """ return H5Tget_norm(self.id) @with_phil def set_norm(self, int norm): """(INT normalization_code) Set the normalization strategy. Legal values are: - NORM_IMPLIED - NORM_MSBSET - NORM_NONE """ H5Tset_norm(self.id, norm) @with_phil def get_inpad(self): """() => INT pad_code Determine the internal padding strategy. Legal values are: - PAD_ZERO - PAD_ONE - PAD_BACKGROUND """ return H5Tget_inpad(self.id) @with_phil def set_inpad(self, int pad_code): """(INT pad_code) Set the internal padding strategy. Legal values are: - PAD_ZERO - PAD_ONE - PAD_BACKGROUND """ H5Tset_inpad(self.id, pad_code) cdef object py_dtype(self): # Translation function for floating-point types order = _order_map[self.get_order()] # string with '<' or '>' s_offset, e_offset, e_size, m_offset, m_size = self.get_fields() e_bias = self.get_ebias() # Handle non-standard exponent and mantissa sizes. for size, finfo in sorted(available_ftypes.items()): nmant = finfo.nmant maxexp = finfo.maxexp minexp = finfo.minexp # workaround for numpy's buggy finfo on float128 on ppc64 archs if size == 16 and MACHINE == 'ppc64': # values reported by hdf5 nmant = 116 maxexp = 1024 minexp = -1022 elif size == 16 and MACHINE == 'ppc64le': # values reported by hdf5 nmant = 52 maxexp = 1024 minexp = -1022 elif nmant == 63 and finfo.nexp == 15: # This is an 80-bit float, correct mantissa size nmant += 1 if (size >= self.get_size() and m_size <= nmant and (2**e_size - e_bias - 1) <= maxexp and (1 - e_bias) >= minexp): break else: raise ValueError('Insufficient precision in available types to ' + 'represent ' + str(self.get_fields())) return dtype(order + "f" + str(size) ) # === Composite types (enums and compound) ==================================== cdef class TypeCompositeID(TypeID): """ Base class for enumerated and compound types. """ @with_phil def get_nmembers(self): """() => INT number_of_members Determine the number of members in a compound or enumerated type. """ return H5Tget_nmembers(self.id) @with_phil def get_member_name(self, int member): """(INT member) => STRING name Determine the name of a member of a compound or enumerated type, identified by its index (0 <= member < nmembers). """ cdef char* name name = NULL if member < 0: raise ValueError("Member index must be non-negative.") try: name = H5Tget_member_name(self.id, member) assert name != NULL pyname = name finally: free(name) return pyname @with_phil def get_member_index(self, char* name): """(STRING name) => INT index Determine the index of a member of a compound or enumerated datatype identified by a string name. """ return H5Tget_member_index(self.id, name) cdef class TypeCompoundID(TypeCompositeID): """ Represents a compound datatype """ @with_phil def get_member_class(self, int member): """(INT member) => INT class Determine the datatype class of the member of a compound type, identified by its index (0 <= member < nmembers). """ if member < 0: raise ValueError("Member index must be non-negative.") return H5Tget_member_class(self.id, member) @with_phil def get_member_offset(self, int member): """(INT member) => INT offset Determine the offset, in bytes, of the beginning of the specified member of a compound datatype. """ if member < 0: raise ValueError("Member index must be non-negative.") return H5Tget_member_offset(self.id, member) @with_phil def get_member_type(self, int member): """(INT member) => TypeID Create a copy of a member of a compound datatype, identified by its index. """ if member < 0: raise ValueError("Member index must be non-negative.") return typewrap(H5Tget_member_type(self.id, member)) @with_phil def insert(self, char* name, size_t offset, TypeID field not None): """(STRING name, UINT offset, TypeID field) Add a named member datatype to a compound datatype. The parameter offset indicates the offset from the start of the compound datatype, in bytes. """ H5Tinsert(self.id, name, offset, field.id) @with_phil def pack(self): """() Recursively removes padding (introduced on account of e.g. compiler alignment rules) from a compound datatype. """ H5Tpack(self.id) cdef object py_dtype(self): cdef TypeID tmp_type cdef list field_names cdef list field_types cdef int nfields field_names = [] field_types = [] field_offsets = [] nfields = self.get_nmembers() # First step: read field names and their Numpy dtypes into # two separate arrays. for i from 0 <= i < nfields: tmp_type = self.get_member_type(i) name = self.get_member_name(i) field_names.append(name) field_types.append(tmp_type.py_dtype()) field_offsets.append(self.get_member_offset(i)) # 1. Check if it should be converted to a complex number if len(field_names) == 2 and \ tuple(field_names) == (cfg._r_name, cfg._i_name) and \ field_types[0] == field_types[1] and \ field_types[0].kind == 'f': bstring = field_types[0].str blen = int(bstring[2:]) nstring = bstring[0] + "c" + str(2*blen) typeobj = dtype(nstring) # 2. Otherwise, read all fields of the compound type, in HDF5 order. else: if sys.version[0] == '3': field_names = [x.decode('utf8') for x in field_names] if len(field_names) > 0: collated_fields = zip(field_names, field_types, field_offsets) ordered_fields = sorted( collated_fields, key=operator.itemgetter(2)) field_names, field_types, field_offsets = \ map(list, zip(*ordered_fields)) typeobj = dtype({ 'names': field_names, 'formats': field_types, 'offsets': field_offsets }) return typeobj cdef class TypeEnumID(TypeCompositeID): """ Represents an enumerated type """ cdef int enum_convert(self, long long *buf, int reverse) except -1: # Convert the long long value in "buf" to the native representation # of this (enumerated) type. Conversion performed in-place. # Reverse: false => llong->type; true => type->llong cdef hid_t basetype cdef H5T_class_t class_code class_code = H5Tget_class(self.id) if class_code != H5T_ENUM: raise ValueError("This type (class %d) is not of class ENUM" % class_code) basetype = H5Tget_super(self.id) assert basetype > 0 try: if not reverse: H5Tconvert(H5T_NATIVE_LLONG, basetype, 1, buf, NULL, H5P_DEFAULT) else: H5Tconvert(basetype, H5T_NATIVE_LLONG, 1, buf, NULL, H5P_DEFAULT) finally: H5Tclose(basetype) @with_phil def enum_insert(self, char* name, long long value): """(STRING name, INT/LONG value) Define a new member of an enumerated type. The value will be automatically converted to the base type defined for this enum. If the conversion results in overflow, the value will be silently clipped. """ cdef long long buf buf = value self.enum_convert(&buf, 0) H5Tenum_insert(self.id, name, &buf) @with_phil def enum_nameof(self, long long value): """(LONG value) => STRING name Determine the name associated with the given value. Due to a limitation of the HDF5 library, this can only retrieve names up to 1023 characters in length. """ cdef herr_t retval cdef char name[1024] cdef long long buf buf = value self.enum_convert(&buf, 0) retval = H5Tenum_nameof(self.id, &buf, name, 1024) assert retval >= 0 retstring = name return retstring @with_phil def enum_valueof(self, char* name): """(STRING name) => LONG value Get the value associated with an enum name. """ cdef long long buf H5Tenum_valueof(self.id, name, &buf) self.enum_convert(&buf, 1) return buf @with_phil def get_member_value(self, int idx): """(UINT index) => LONG value Determine the value for the member at the given zero-based index. """ cdef herr_t retval cdef hid_t ptype cdef long long val ptype = 0 if idx < 0: raise ValueError("Index must be non-negative.") H5Tget_member_value(self.id, idx, &val) self.enum_convert(&val, 1) return val cdef object py_dtype(self): # Translation function for enum types cdef TypeID basetype = self.get_super() nmembers = self.get_nmembers() members = {} for idx in xrange(nmembers): name = self.get_member_name(idx) val = self.get_member_value(idx) members[name] = val ref = {cfg._f_name: 0, cfg._t_name: 1} # Boolean types have priority over standard enums if members == ref: return dtype('bool') # Convert strings to appropriate representation members_conv = {} for name, val in members.iteritems(): try: # ASCII; Py2 -> preserve bytes, Py3 -> make unicode uname = name.decode('ascii') if PY3: name = uname except UnicodeDecodeError: try: # Non-ascii; all platforms try unicode name = name.decode('utf8') except UnicodeDecodeError: pass # Last resort: return byte string members_conv[name] = val return special_dtype(enum=(basetype.py_dtype(), members_conv)) # === Translation from NumPy dtypes to HDF5 type objects ====================== # The following series of native-C functions each translate a specific class # of NumPy dtype into an HDF5 type object. The result is guaranteed to be # transient and unlocked. cdef dict _float_le = {2: H5Tcopy(IEEE_F16LE.id), 4: H5Tcopy(H5T_IEEE_F32LE), 8: H5Tcopy(H5T_IEEE_F64LE)} cdef dict _float_be = {2: H5Tcopy(IEEE_F16BE.id), 4: H5Tcopy(H5T_IEEE_F32BE), 8: H5Tcopy(H5T_IEEE_F64BE)} cdef dict _float_nt = dict(_float_le) if ORDER_NATIVE == H5T_ORDER_LE else dict(_float_be) _float_nt[sizeof(long double)] = H5Tcopy(H5T_NATIVE_LDOUBLE) cdef dict _int_le = {1: H5Tcopy(H5T_STD_I8LE), 2: H5Tcopy(H5T_STD_I16LE), 4: H5Tcopy(H5T_STD_I32LE), 8: H5Tcopy(H5T_STD_I64LE)} cdef dict _int_be = {1: H5Tcopy(H5T_STD_I8BE), 2: H5Tcopy(H5T_STD_I16BE), 4: H5Tcopy(H5T_STD_I32BE), 8: H5Tcopy(H5T_STD_I64BE)} cdef dict _int_nt = {1: H5Tcopy(H5T_NATIVE_INT8), 2: H5Tcopy(H5T_NATIVE_INT16), 4: H5Tcopy(H5T_NATIVE_INT32), 8: H5Tcopy(H5T_NATIVE_INT64)} cdef dict _uint_le = {1: H5Tcopy(H5T_STD_U8LE), 2: H5Tcopy(H5T_STD_U16LE), 4: H5Tcopy(H5T_STD_U32LE), 8: H5Tcopy(H5T_STD_U64LE)} cdef dict _uint_be = {1: H5Tcopy(H5T_STD_U8BE), 2: H5Tcopy(H5T_STD_U16BE), 4: H5Tcopy(H5T_STD_U32BE), 8: H5Tcopy(H5T_STD_U64BE)} cdef dict _uint_nt = {1: H5Tcopy(H5T_NATIVE_UINT8), 2: H5Tcopy(H5T_NATIVE_UINT16), 4: H5Tcopy(H5T_NATIVE_UINT32), 8: H5Tcopy(H5T_NATIVE_UINT64)} cdef TypeFloatID _c_float(dtype dt): # Floats (single and double) cdef hid_t tid try: if dt.byteorder == c'<': tid = _float_le[dt.elsize] elif dt.byteorder == c'>': tid = _float_be[dt.elsize] else: tid = _float_nt[dt.elsize] except KeyError: raise TypeError("Unsupported float size (%s)" % dt.elsize) return TypeFloatID(H5Tcopy(tid)) cdef TypeIntegerID _c_int(dtype dt): # Integers (ints and uints) cdef hid_t tid try: if dt.kind == c'i': if dt.byteorder == c'<': tid = _int_le[dt.elsize] elif dt.byteorder == c'>': tid = _int_be[dt.elsize] else: tid = _int_nt[dt.elsize] elif dt.kind == c'u': if dt.byteorder == c'<': tid = _uint_le[dt.elsize] elif dt.byteorder == c'>': tid = _uint_be[dt.elsize] else: tid = _uint_nt[dt.elsize] else: raise TypeError('Illegal int kind "%s"' % dt.kind) except KeyError: raise TypeError("Unsupported integer size (%s)" % dt.elsize) return TypeIntegerID(H5Tcopy(tid)) cdef TypeEnumID _c_enum(dtype dt, dict vals): # Enums cdef TypeIntegerID base cdef TypeEnumID out base = _c_int(dt) out = TypeEnumID(H5Tenum_create(base.id)) for name in sorted(vals): if isinstance(name, bytes): bname = name else: bname = unicode(name).encode('utf8') out.enum_insert(bname, vals[name]) return out cdef TypeEnumID _c_bool(dtype dt): # Booleans global cfg cdef TypeEnumID out out = TypeEnumID(H5Tenum_create(H5T_NATIVE_INT8)) out.enum_insert(cfg._f_name, 0) out.enum_insert(cfg._t_name, 1) return out cdef TypeArrayID _c_array(dtype dt, int logical): # Arrays cdef dtype base cdef TypeID type_base cdef object shape base, shape = dt.subdtype try: shape = tuple(shape) except TypeError: try: shape = (int(shape),) except TypeError: raise TypeError("Array shape for dtype must be a sequence or integer") type_base = py_create(base, logical=logical) return array_create(type_base, shape) cdef TypeOpaqueID _c_opaque(dtype dt): # Opaque return TypeOpaqueID(H5Tcreate(H5T_OPAQUE, dt.itemsize)) cdef TypeStringID _c_string(dtype dt): # Strings (fixed-length) cdef hid_t tid tid = H5Tcopy(H5T_C_S1) H5Tset_size(tid, dt.itemsize) H5Tset_strpad(tid, H5T_STR_NULLPAD) return TypeStringID(tid) cdef TypeCompoundID _c_complex(dtype dt): # Complex numbers (names depend on cfg) global cfg cdef hid_t tid, tid_sub cdef size_t size, off_r, off_i cdef size_t length = dt.itemsize cdef char byteorder = dt.byteorder if length == 8: size = h5py_size_n64 off_r = h5py_offset_n64_real off_i = h5py_offset_n64_imag if byteorder == c'<': tid_sub = H5T_IEEE_F32LE elif byteorder == c'>': tid_sub = H5T_IEEE_F32BE else: tid_sub = H5T_NATIVE_FLOAT elif length == 16: size = h5py_size_n128 off_r = h5py_offset_n128_real off_i = h5py_offset_n128_imag if byteorder == c'<': tid_sub = H5T_IEEE_F64LE elif byteorder == c'>': tid_sub = H5T_IEEE_F64BE else: tid_sub = H5T_NATIVE_DOUBLE elif length == 32: IF COMPLEX256_SUPPORT: size = h5py_size_n256 off_r = h5py_offset_n256_real off_i = h5py_offset_n256_imag tid_sub = H5T_NATIVE_LDOUBLE ELSE: raise TypeError("Illegal length %d for complex dtype" % length) else: raise TypeError("Illegal length %d for complex dtype" % length) tid = H5Tcreate(H5T_COMPOUND, size) H5Tinsert(tid, cfg._r_name, off_r, tid_sub) H5Tinsert(tid, cfg._i_name, off_i, tid_sub) return TypeCompoundID(tid) cdef TypeCompoundID _c_compound(dtype dt, int logical, int aligned): # Compound datatypes cdef hid_t tid cdef TypeID member_type cdef dtype member_dt cdef size_t member_offset = 0 cdef dict offsets = {} cdef list fields = [] # The challenge with correctly converting a numpy/h5py dtype to a HDF5 type # which is composed of subtypes has three aspects we must consider # 1. numpy/h5py dtypes do not always have the same size as HDF5, even when # equivalent (can result in overlapping elements if not careful) # 2. For correct round-tripping of aligned dtypes, we need to consider how # much padding we need by looking at the field offsets # 3. There is no requirement that the offsets be monotonically increasing # (so we start by sorting the names as a function of increasing offset) # # The code below tries to cover these aspects # Get offsets for each compound member for name, field in dt.fields.items(): offsets[name] = field[1] # Build list of names, offsets, and types, sorted by increasing offset # (i.e. the position of the member in the struct) for name in sorted(dt.names, key=offsets.__getitem__): field = dt.fields[name] name = name.encode('utf8') if isinstance(name, unicode) else name # Get HDF5 data types and set the offset for each member member_dt = field[0] member_offset = max(member_offset, field[1]) member_type = py_create(member_dt, logical=logical, aligned=aligned) if aligned and (member_offset > field[1] or member_dt.itemsize != member_type.get_size()): raise TypeError("Enforced alignment not compatible with HDF5 type") fields.append((name, member_offset, member_type)) # Update member offset based on the HDF5 type size member_offset += member_type.get_size() member_offset = max(member_offset, dt.itemsize) if aligned and member_offset > dt.itemsize: raise TypeError("Enforced alignment not compatible with HDF5 type") # Create compound with the necessary size, and insert its members tid = H5Tcreate(H5T_COMPOUND, member_offset) for (name, member_offset, member_type) in fields: H5Tinsert(tid, name, member_offset, member_type.id) return TypeCompoundID(tid) cdef TypeStringID _c_vlen_str(): # Variable-length strings cdef hid_t tid tid = H5Tcopy(H5T_C_S1) H5Tset_size(tid, H5T_VARIABLE) return TypeStringID(tid) cdef TypeStringID _c_vlen_unicode(): cdef hid_t tid tid = H5Tcopy(H5T_C_S1) H5Tset_size(tid, H5T_VARIABLE) H5Tset_cset(tid, H5T_CSET_UTF8) return TypeStringID(tid) cdef TypeReferenceID _c_ref(object refclass): if refclass is Reference: return STD_REF_OBJ elif refclass is RegionReference: return STD_REF_DSETREG raise TypeError("Unrecognized reference code") cpdef TypeID py_create(object dtype_in, bint logical=0, bint aligned=0): """(OBJECT dtype_in, BOOL logical=False) => TypeID Given a Numpy dtype object, generate a byte-for-byte memory-compatible HDF5 datatype object. The result is guaranteed to be transient and unlocked. Argument dtype_in may be a dtype object, or anything which can be converted to a dtype, including strings like ' TUPLE or None Determine if a conversion path exists from src to dst. Result is None or a tuple describing the conversion path. Currently tuple entries are: 1. INT need_bkg: Whether this routine requires a backing buffer. Values are BKG_NO, BKG_TEMP and BKG_YES. """ cdef H5T_cdata_t *data cdef H5T_conv_t result = NULL try: result = H5Tfind(src.id, dst.id, &data) if result == NULL: return None return (data[0].need_bkg,) except: return None # ============================================================================ # Deprecated functions import warnings cpdef dtype py_new_enum(object dt_in, dict enum_vals): """ (DTYPE dt_in, DICT enum_vals) => DTYPE Deprecated; use special_dtype() instead. """ #warnings.warn("Deprecated; use special_dtype(enum=(dtype, values)) instead", DeprecationWarning) return special_dtype(enum = (dt_in, enum_vals)) cpdef dict py_get_enum(object dt): """ (DTYPE dt_in) => DICT Deprecated; use check_dtype() instead. """ #warnings.warn("Deprecated; use check_dtype(enum=dtype) instead", DeprecationWarning) return check_dtype(enum=dt) cpdef dtype py_new_vlen(object kind): """ (OBJECT kind) => DTYPE Deprecated; use special_dtype() instead. """ #warnings.warn("Deprecated; use special_dtype(vlen=basetype) instead", DeprecationWarning) return special_dtype(vlen=kind) cpdef object py_get_vlen(object dt_in): """ (OBJECT dt_in) => TYPE Deprecated; use check_dtype() instead. """ #warnings.warn("Deprecated; use check_dtype(vlen=dtype) instead", DeprecationWarning) return check_dtype(vlen=dt_in) h5py-2.7.1/h5py/h5ds.pxd0000644000175000017500000000044013025343121016321 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. from defs cimport * h5py-2.7.1/h5py/h5d.pxd0000644000175000017500000000057113025343121016143 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. from defs cimport * from _objects cimport ObjectID cdef class DatasetID(ObjectID): cdef object _dtype h5py-2.7.1/h5py/h5r.pyx0000644000175000017500000001310613063246304016213 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. """ H5R API for object and region references. """ # Pyrex compile-time imports from _objects cimport ObjectID from ._objects import phil, with_phil # === Public constants and data structures ==================================== OBJECT = H5R_OBJECT DATASET_REGION = H5R_DATASET_REGION # === Reference API =========================================================== @with_phil def create(ObjectID loc not None, char* name, int ref_type, ObjectID space=None): """(ObjectID loc, STRING name, INT ref_type, SpaceID space=None) => ReferenceObject ref Create a new reference. The value of ref_type determines the kind of reference created: OBJECT Reference to an object in an HDF5 file. Parameters "loc" and "name" identify the object; "space" is unused. DATASET_REGION Reference to a dataset region. Parameters "loc" and "name" identify the dataset; the selection on "space" identifies the region. """ cdef hid_t space_id cdef Reference ref if ref_type == H5R_OBJECT: ref = Reference() elif ref_type == H5R_DATASET_REGION: if space is None: # work around segfault in HDF5 raise ValueError("Dataspace required for region reference") ref = RegionReference() else: raise ValueError("Unknown reference typecode") if space is None: space_id = -1 else: space_id = space.id H5Rcreate(&ref.ref, loc.id, name, ref_type, space_id) return ref @with_phil def dereference(Reference ref not None, ObjectID id not None): """(Reference ref, ObjectID id) => ObjectID or None Open the object pointed to by the reference and return its identifier. The file identifier (or the identifier for any object in the file) must also be provided. Returns None if the reference is zero-filled. The reference may be either Reference or RegionReference. """ import h5i if not ref: return None return h5i.wrap_identifier(H5Rdereference(id.id, ref.typecode, &ref.ref)) @with_phil def get_region(RegionReference ref not None, ObjectID id not None): """(Reference ref, ObjectID id) => SpaceID or None Retrieve the dataspace selection pointed to by the reference. Returns a copy of the dataset's dataspace, with the appropriate elements selected. The file identifier or the identifier of any object in the file (including the dataset itself) must also be provided. The reference object must be a RegionReference. If it is zero-filled, returns None. """ import h5s if ref.typecode != H5R_DATASET_REGION or not ref: return None return h5s.SpaceID(H5Rget_region(id.id, ref.typecode, &ref.ref)) @with_phil def get_obj_type(Reference ref not None, ObjectID id not None): """(Reference ref, ObjectID id) => INT obj_code or None Determine what type of object the reference points to. The reference may be a Reference or RegionReference. The file identifier or the identifier of any object in the file must also be provided. The return value is one of: - h5g.LINK - h5g.GROUP - h5g.DATASET - h5g.TYPE If the reference is zero-filled, returns None. """ if not ref: return None return H5Rget_obj_type(id.id, ref.typecode, &ref.ref) @with_phil def get_name(Reference ref not None, ObjectID loc not None): """(Reference ref, ObjectID loc) => STRING name Determine the name of the object pointed to by this reference. Requires the HDF5 1.8 API. """ cdef ssize_t namesize = 0 cdef char* namebuf = NULL namesize = H5Rget_name(loc.id, ref.typecode, &ref.ref, NULL, 0) if namesize > 0: namebuf = malloc(namesize+1) try: namesize = H5Rget_name(loc.id, ref.typecode, &ref.ref, namebuf, namesize+1) return namebuf finally: free(namebuf) cdef class Reference: """ Opaque representation of an HDF5 reference. Objects of this class are created exclusively by the library and cannot be modified. The read-only attribute "typecode" determines whether the reference is to an object in an HDF5 file (OBJECT) or a dataset region (DATASET_REGION). The object's truth value indicates whether it contains a nonzero reference. This does not guarantee that is valid, but is useful for rejecting "background" elements in a dataset. """ def __cinit__(self, *args, **kwds): self.typecode = H5R_OBJECT self.typesize = sizeof(hobj_ref_t) def __nonzero__(self): cdef int i for i from 0<=i&self.ref)[i] != 0: return True return False def __repr__(self): return "" % ("" if self else " (null)") cdef class RegionReference(Reference): """ Opaque representation of an HDF5 region reference. This is a subclass of Reference which exists mainly for programming convenience. """ def __cinit__(self, *args, **kwds): self.typecode = H5R_DATASET_REGION self.typesize = sizeof(hdset_reg_ref_t) def __repr__(self): return "" % ("" if self else " (null") h5py-2.7.1/h5py/ipy_completer.py0000644000175000017500000001147113114361073020201 0ustar tcaswelltcaswell00000000000000#+ # # This file is part of h5py, a low-level Python interface to the HDF5 library. # # Contributed by Darren Dale # # Copyright (C) 2009 Darren Dale # # http://h5py.alfven.org # License: BSD (See LICENSE.txt for full license) # #- # pylint: disable=eval-used,protected-access """ This is the h5py completer extension for ipython. It is loaded by calling the function h5py.enable_ipython_completer() from within an interactive IPython session. It will let you do things like:: f=File('foo.h5') f[' # or: f['ite which will do tab completion based on the subgroups of `f`. Also:: f['item1'].at will perform tab completion for the attributes in the usual way. This should also work:: a = b = f['item1'].attrs. as should:: f['item1/item2/it """ from __future__ import absolute_import import posixpath import re import readline from ._hl.attrs import AttributeManager from ._hl.base import HLObject try: # >=ipython-1.0 from IPython import get_ipython except ImportError: try: # support >=ipython-0.11, =ipython-0.11 from IPython.utils import generics except ImportError: # support ['|"])(?!.*(?P=s))(.*)$""") re_object_match = re.compile(r"(?:.*\=)?(.+?)(?:\[)") def _retrieve_obj(name, context): """ Filter function for completion. """ # we don't want to call any functions, but I couldn't find a robust regex # that filtered them without unintended side effects. So keys containing # "(" will not complete. if '(' in name: raise ValueError() try: # older versions of IPython: obj = eval(name, context.shell.user_ns) except AttributeError: # as of IPython-1.0: obj = eval(name, context.user_ns) return obj def h5py_item_completer(context, command): """Compute possible item matches for dict-like objects""" base, item = re_item_match.split(command)[1:4:2] try: obj = _retrieve_obj(base, context) except Exception: return [] path, _ = posixpath.split(item) try: if path: items = (posixpath.join(path, name) for name in obj[path].keys()) else: items = obj.keys() except AttributeError: return [] items = list(items) readline.set_completer_delims(' \t\n`!@#$^&*()=+[{]}\\|;:\'",<>?') return [i for i in items if i[:len(item)] == item] def h5py_attr_completer(context, command): """Compute possible attr matches for nested dict-like objects""" base, attr = re_attr_match.split(command)[1:3] base = base.strip() try: obj = _retrieve_obj(base, context) except Exception: return [] attrs = dir(obj) try: attrs = generics.complete_object(obj, attrs) except TryNext: pass omit__names = None try: # support >=ipython-0.12 omit__names = get_ipython().Completer.omit__names except AttributeError: pass if omit__names is None: try: # support ipython-0.11 omit__names = get_ipython().readline_omit__names except AttributeError: pass if omit__names is None: try: # support self.istr[0].type property rc: def __get__(self): return self.istr[0].rc def _hash(self): return hash((self.fileno, self.addr, self.type, self.rc)) cdef class ObjInfo(_ObjInfo): """ Represents the H5O_info_t structure """ cdef H5O_info_t infostruct cdef public _OHdr hdr def __init__(self): self.hdr = _OHdr() self.istr = &self.infostruct self.hdr.istr = &self.infostruct self.hdr.space.istr = &self.infostruct self.hdr.mesg.istr = &self.infostruct def __copy__(self): cdef ObjInfo newcopy newcopy = ObjInfo() newcopy.infostruct = self.infostruct return newcopy @with_phil def get_info(ObjectID loc not None, char* name=NULL, int index=-1, *, char* obj_name='.', int index_type=H5_INDEX_NAME, int order=H5_ITER_NATIVE, PropID lapl=None): """(ObjectID loc, STRING name=, INT index=, **kwds) => ObjInfo Get information describing an object in an HDF5 file. Provide the object itself, or the containing group and exactly one of "name" or "index". STRING obj_name (".") When "index" is specified, look in this subgroup instead. Otherwise ignored. PropID lapl (None) Link access property list INT index_type (h5.INDEX_NAME) INT order (h5.ITER_NATIVE) """ cdef ObjInfo info info = ObjInfo() if name != NULL and index >= 0: raise TypeError("At most one of name or index may be specified") elif name != NULL and index < 0: H5Oget_info_by_name(loc.id, name, &info.infostruct, pdefault(lapl)) elif name == NULL and index >= 0: H5Oget_info_by_idx(loc.id, obj_name, index_type, order, index, &info.infostruct, pdefault(lapl)) else: H5Oget_info(loc.id, &info.infostruct) return info IF HDF5_VERSION >= (1, 8, 5): @with_phil def exists_by_name(ObjectID loc not None, char *name, PropID lapl=None): """ (ObjectID loc, STRING name, PropID lapl=None) => BOOL exists Determines whether a link resolves to an actual object. """ return H5Oexists_by_name(loc.id, name, pdefault(lapl)) # === General object operations =============================================== @with_phil def open(ObjectID loc not None, char* name, PropID lapl=None): """(ObjectID loc, STRING name, PropID lapl=None) => ObjectID Open a group, dataset, or named datatype attached to an existing group. """ return wrap_identifier(H5Oopen(loc.id, name, pdefault(lapl))) @with_phil def link(ObjectID obj not None, GroupID loc not None, char* name, PropID lcpl=None, PropID lapl=None): """(ObjectID obj, GroupID loc, STRING name, PropID lcpl=None, PropID lapl=None) Create a new hard link to an object. Useful for objects created with h5g.create_anon() or h5d.create_anon(). """ H5Olink(obj.id, loc.id, name, pdefault(lcpl), pdefault(lapl)) @with_phil def copy(ObjectID src_loc not None, char* src_name, GroupID dst_loc not None, char* dst_name, PropID copypl=None, PropID lcpl=None): """(ObjectID src_loc, STRING src_name, GroupID dst_loc, STRING dst_name, PropID copypl=None, PropID lcpl=None) Copy a group, dataset or named datatype from one location to another. The source and destination need not be in the same file. The default behavior is a recursive copy of the object and all objects below it. This behavior is modified via the "copypl" property list. """ H5Ocopy(src_loc.id, src_name, dst_loc.id, dst_name, pdefault(copypl), pdefault(lcpl)) @with_phil def set_comment(ObjectID loc not None, char* comment, *, char* obj_name=".", PropID lapl=None): """(ObjectID loc, STRING comment, **kwds) Set the comment for any-file resident object. Keywords: STRING obj_name (".") Set comment on this group member instead PropID lapl (None) Link access property list """ H5Oset_comment_by_name(loc.id, obj_name, comment, pdefault(lapl)) @with_phil def get_comment(ObjectID loc not None, char* comment, *, char* obj_name=".", PropID lapl=None): """(ObjectID loc, STRING comment, **kwds) Get the comment for any-file resident object. Keywords: STRING obj_name (".") Set comment on this group member instead PropID lapl (None) Link access property list """ cdef ssize_t size cdef char* buf size = H5Oget_comment_by_name(loc.id, obj_name, NULL, 0, pdefault(lapl)) buf = emalloc(size+1) try: H5Oget_comment_by_name(loc.id, obj_name, buf, size+1, pdefault(lapl)) pstring = buf finally: efree(buf) return pstring # === Visit routines ========================================================== cdef class _ObjectVisitor: cdef object func cdef object retval cdef ObjInfo objinfo def __init__(self, func): self.func = func self.retval = None self.objinfo = ObjInfo() cdef herr_t cb_obj_iterate(hid_t obj, const char* name, const H5O_info_t *info, void* data) except 2: cdef _ObjectVisitor visit # HDF5 doesn't respect callback return for ".", so skip it if strcmp(name, ".") == 0: return 0 visit = <_ObjectVisitor>data visit.objinfo.infostruct = info[0] visit.retval = visit.func(name, visit.objinfo) if visit.retval is not None: return 1 return 0 cdef herr_t cb_obj_simple(hid_t obj, const char* name, const H5O_info_t *info, void* data) except 2: cdef _ObjectVisitor visit # Not all versions of HDF5 respect callback value for ".", so skip it if strcmp(name, ".") == 0: return 0 visit = <_ObjectVisitor>data visit.retval = visit.func(name) if visit.retval is not None: return 1 return 0 @with_phil def visit(ObjectID loc not None, object func, *, int idx_type=H5_INDEX_NAME, int order=H5_ITER_NATIVE, char* obj_name=".", PropID lapl=None, bint info=0): """(ObjectID loc, CALLABLE func, **kwds) => Iterate a function or callable object over all objects below the specified one. Your callable should conform to the signature:: func(STRING name) => Result or if the keyword argument "info" is True:: func(STRING name, ObjInfo info) => Result Returning None continues iteration; returning anything else aborts iteration and returns that value. Keywords: BOOL info (False) Callback is func(STRING, Objinfo) STRING obj_name (".") Visit a subgroup of "loc" instead PropLAID lapl (None) Control how "obj_name" is interpreted INT idx_type (h5.INDEX_NAME) What indexing strategy to use INT order (h5.ITER_NATIVE) Order in which iteration occurs Compatibility note: No callback is executed for the starting path ("."), as some versions of HDF5 don't correctly handle a return value for this case. This differs from the behavior of the native H5Ovisit, which provides a literal "." as the first value. """ cdef _ObjectVisitor visit = _ObjectVisitor(func) cdef H5O_iterate_t cfunc if info: cfunc = cb_obj_iterate else: cfunc = cb_obj_simple H5Ovisit_by_name(loc.id, obj_name, idx_type, order, cfunc, visit, pdefault(lapl)) return visit.retval h5py-2.7.1/h5py/h5f.pyx0000644000175000017500000003403613025343121016175 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. """ Low-level operations on HDF5 file objects. """ include "config.pxi" # Compile-time imports from _objects cimport pdefault from h5p cimport propwrap, PropFAID, PropFCID from h5t cimport typewrap from h5i cimport wrap_identifier from h5ac cimport CacheConfig from utils cimport emalloc, efree from h5py import _objects from ._objects import phil, with_phil import h5fd from cpython.bytes cimport PyBytes_FromStringAndSize, PyBytes_AsString # Initialization # === Public constants and data structures ==================================== ACC_TRUNC = H5F_ACC_TRUNC ACC_EXCL = H5F_ACC_EXCL ACC_RDWR = H5F_ACC_RDWR ACC_RDONLY = H5F_ACC_RDONLY IF HDF5_VERSION >= SWMR_MIN_HDF5_VERSION: ACC_SWMR_WRITE = H5F_ACC_SWMR_WRITE ACC_SWMR_READ = H5F_ACC_SWMR_READ SCOPE_LOCAL = H5F_SCOPE_LOCAL SCOPE_GLOBAL = H5F_SCOPE_GLOBAL CLOSE_WEAK = H5F_CLOSE_WEAK CLOSE_SEMI = H5F_CLOSE_SEMI CLOSE_STRONG = H5F_CLOSE_STRONG CLOSE_DEFAULT = H5F_CLOSE_DEFAULT OBJ_FILE = H5F_OBJ_FILE OBJ_DATASET = H5F_OBJ_DATASET OBJ_GROUP = H5F_OBJ_GROUP OBJ_DATATYPE = H5F_OBJ_DATATYPE OBJ_ATTR = H5F_OBJ_ATTR OBJ_ALL = H5F_OBJ_ALL OBJ_LOCAL = H5F_OBJ_LOCAL LIBVER_EARLIEST = H5F_LIBVER_EARLIEST LIBVER_LATEST = H5F_LIBVER_LATEST # === File operations ========================================================= @with_phil def open(char* name, unsigned int flags=H5F_ACC_RDWR, PropFAID fapl=None): """(STRING name, UINT flags=ACC_RDWR, PropFAID fapl=None) => FileID Open an existing HDF5 file. Keyword "flags" may be: ACC_RDWR Open in read-write mode ACC_RDONLY Open in readonly mode Keyword fapl may be a file access property list. """ return FileID(H5Fopen(name, flags, pdefault(fapl))) @with_phil def create(char* name, int flags=H5F_ACC_TRUNC, PropFCID fcpl=None, PropFAID fapl=None): """(STRING name, INT flags=ACC_TRUNC, PropFCID fcpl=None, PropFAID fapl=None) => FileID Create a new HDF5 file. Keyword "flags" may be: ACC_TRUNC Truncate an existing file, discarding its data ACC_EXCL Fail if a conflicting file exists To keep the behavior in line with that of Python's built-in functions, the default is ACC_TRUNC. Be careful! """ return FileID(H5Fcreate(name, flags, pdefault(fcpl), pdefault(fapl))) @with_phil def flush(ObjectID obj not None, int scope=H5F_SCOPE_LOCAL): """(ObjectID obj, INT scope=SCOPE_LOCAL) Tell the HDF5 library to flush file buffers to disk. "obj" may be the file identifier, or the identifier of any object residing in the file. Keyword "scope" may be: SCOPE_LOCAL Flush only the given file SCOPE_GLOBAL Flush the entire virtual file """ H5Fflush(obj.id, scope) @with_phil def is_hdf5(char* name): """(STRING name) => BOOL Determine if a given file is an HDF5 file. Note this raises an exception if the file doesn't exist. """ return (H5Fis_hdf5(name)) @with_phil def mount(ObjectID loc not None, char* name, FileID fid not None): """(ObjectID loc, STRING name, FileID fid) Mount an open file on the group "name" under group loc_id. Note that "name" must already exist. """ H5Fmount(loc.id, name, fid.id, H5P_DEFAULT) @with_phil def unmount(ObjectID loc not None, char* name): """(ObjectID loc, STRING name) Unmount a file, mounted at "name" under group loc_id. """ H5Funmount(loc.id, name) @with_phil def get_name(ObjectID obj not None): """(ObjectID obj) => STRING Determine the name of the file in which the specified object resides. """ cdef ssize_t size cdef char* name name = NULL size = H5Fget_name(obj.id, NULL, 0) assert size >= 0 name = emalloc(sizeof(char)*(size+1)) try: H5Fget_name(obj.id, name, size+1) pname = name return pname finally: efree(name) @with_phil def get_obj_count(object where=OBJ_ALL, int types=H5F_OBJ_ALL): """(OBJECT where=OBJ_ALL, types=OBJ_ALL) => INT Get the number of open objects. where Either a FileID instance representing an HDF5 file, or the special constant OBJ_ALL, to count objects in all files. type Specify what kinds of object to include. May be one of OBJ*, or any bitwise combination (e.g. OBJ_FILE | OBJ_ATTR). The special value OBJ_ALL matches all object types, and OBJ_LOCAL will only match objects opened through a specific identifier. """ cdef hid_t where_id if isinstance(where, FileID): where_id = where.id elif isinstance(where, int) or isinstance(where, long): where_id = where else: raise TypeError("Location must be a FileID or OBJ_ALL.") return H5Fget_obj_count(where_id, types) @with_phil def get_obj_ids(object where=OBJ_ALL, int types=H5F_OBJ_ALL): """(OBJECT where=OBJ_ALL, types=OBJ_ALL) => LIST Get a list of identifier instances for open objects. where Either a FileID instance representing an HDF5 file, or the special constant OBJ_ALL, to list objects in all files. type Specify what kinds of object to include. May be one of OBJ*, or any bitwise combination (e.g. OBJ_FILE | OBJ_ATTR). The special value OBJ_ALL matches all object types, and OBJ_LOCAL will only match objects opened through a specific identifier. """ cdef int count cdef int i cdef hid_t where_id cdef hid_t *obj_list = NULL cdef list py_obj_list = [] if isinstance(where, FileID): where_id = where.id else: try: where_id = int(where) except TypeError: raise TypeError("Location must be a FileID or OBJ_ALL.") try: count = H5Fget_obj_count(where_id, types) obj_list = emalloc(sizeof(hid_t)*count) if count > 0: # HDF5 complains that obj_list is NULL, even if count==0 H5Fget_obj_ids(where_id, types, count, obj_list) for i from 0<=i FileID Retrieve another identifier for a file (which must still be open). The new identifier is guaranteed to neither be mounted nor contain a mounted file. """ return FileID(H5Freopen(self.id)) @with_phil def get_filesize(self): """() => LONG size Determine the total size (in bytes) of the HDF5 file, including any user block. """ cdef hsize_t size H5Fget_filesize(self.id, &size) return size @with_phil def get_create_plist(self): """() => PropFCID Retrieve a copy of the file creation property list used to create this file. """ return propwrap(H5Fget_create_plist(self.id)) @with_phil def get_access_plist(self): """() => PropFAID Retrieve a copy of the file access property list which manages access to this file. """ return propwrap(H5Fget_access_plist(self.id)) @with_phil def get_freespace(self): """() => LONG freespace Determine the amount of free space in this file. Note that this only tracks free space until the file is closed. """ return H5Fget_freespace(self.id) @with_phil def get_intent(self): """ () => INT Determine the file's write intent, either of: - H5F_ACC_RDONLY - H5F_ACC_RDWR """ cdef unsigned int mode H5Fget_intent(self.id, &mode) return mode @with_phil def get_vfd_handle(self): """ () => INT Retrieve the file handle used by the virtual file driver. This method is only functional when the the SEC2 driver is used. """ if H5Pget_driver(H5Fget_access_plist(self.id)) != h5fd.SEC2: raise NotImplementedError cdef int *handle H5Fget_vfd_handle(self.id, H5Fget_access_plist(self.id), &handle) return handle[0] IF HDF5_VERSION >= (1, 8, 9): @with_phil def get_file_image(self): """ () => BYTES Retrieves a copy of the image of an existing, open file. Feature requries: 1.8.9 """ cdef ssize_t size size = H5Fget_file_image(self.id, NULL, 0) image = PyBytes_FromStringAndSize(NULL, size) H5Fget_file_image(self.id, PyBytes_AsString(image), size) return image IF MPI and HDF5_VERSION >= (1, 8, 9): @with_phil def set_mpi_atomicity(self, bint atomicity): """ (BOOL atomicity) For MPI-IO driver, set to atomic (True), which guarantees sequential I/O semantics, or non-atomic (False), which improves performance. Default is False. Feature requires: 1.8.9 and Parallel HDF5 """ H5Fset_mpi_atomicity(self.id, atomicity) @with_phil def get_mpi_atomicity(self): """ () => BOOL Return atomicity setting for MPI-IO driver. Feature requires: 1.8.9 and Parallel HDF5 """ cdef hbool_t atom H5Fget_mpi_atomicity(self.id, &atom) return atom @with_phil def get_mdc_hit_rate(self): """() => DOUBLE Retrieve the cache hit rate """ cdef double hit_rate H5Fget_mdc_hit_rate(self.id, &hit_rate) return hit_rate @with_phil def get_mdc_size(self): """() => (max_size, min_clean_size, cur_size, cur_num_entries) [SIZE_T, SIZE_T, SIZE_T, INT] Obtain current metadata cache size data for specified file. """ cdef size_t max_size cdef size_t min_clean_size cdef size_t cur_size cdef int cur_num_entries H5Fget_mdc_size(self.id, &max_size, &min_clean_size, &cur_size, &cur_num_entries) return (max_size, min_clean_size, cur_size, cur_num_entries) @with_phil def reset_mdc_hit_rate_stats(self): """no return rests the hit-rate statistics """ H5Freset_mdc_hit_rate_stats(self.id) @with_phil def get_mdc_config(self): """() => CacheConfig Returns an object that stores all the information about the meta-data cache configuration """ cdef CacheConfig config = CacheConfig() H5Fget_mdc_config(self.id, &config.cache_config) return config @with_phil def set_mdc_config(self, CacheConfig config not None): """(CacheConfig) => None Returns an object that stores all the information about the meta-data cache configuration """ # I feel this should have some sanity checking to make sure that H5Fset_mdc_config(self.id, &config.cache_config) IF HDF5_VERSION >= SWMR_MIN_HDF5_VERSION: @with_phil def start_swmr_write(self): """ no return Enables SWMR writing mode for a file. This function will activate SWMR writing mode for a file associated with file_id. This routine will prepare and ensure the file is safe for SWMR writing as follows: * Check that the file is opened with write access (H5F_ACC_RDWR). * Check that the file is opened with the latest library format to ensure data structures with check-summed metadata are used. * Check that the file is not already marked in SWMR writing mode. * Enable reading retries for check-summed metadata to remedy possible checksum failures from reading inconsistent metadata on a system that is not atomic. * Turn off usage of the library’s accumulator to avoid possible ordering problem on a system that is not atomic. * Perform a flush of the file’s data buffers and metadata to set a consistent state for starting SWMR write operations. Library objects are groups, datasets, and committed datatypes. For the current implementation, groups and datasets can remain open when activating SWMR writing mode, but not committed datatypes. Attributes attached to objects cannot remain open. Feature requires: 1.9.178 HDF5 """ H5Fstart_swmr_write(self.id) h5py-2.7.1/h5py/api_compat.h0000644000175000017500000000262713025343121017237 0ustar tcaswelltcaswell00000000000000/***** Preamble block ********************************************************* * * This file is part of h5py, a Python interface to the HDF5 library. * * http://www.h5py.org * * Copyright 2008-2013 Andrew Collette and contributors * * License: Standard 3-clause BSD; see "license.txt" for full license terms * and contributor agreement. * ****** End preamble block ****************************************************/ /* Contains compatibility macros and definitions for use by Cython code */ #ifndef H5PY_COMPAT #define H5PY_COMPAT #if defined(MPI_VERSION) && (MPI_VERSION < 3) && !defined(PyMPI_HAVE_MPI_Message) typedef void *PyMPI_MPI_Message; #define MPI_Message PyMPI_MPI_Message #endif #include #include "Python.h" #include "numpy/arrayobject.h" #include "hdf5.h" /* The HOFFSET macro can't be used from Cython. */ #define h5py_size_n64 (sizeof(npy_complex64)) #define h5py_size_n128 (sizeof(npy_complex128)) #ifdef NPY_COMPLEX256 #define h5py_size_n256 (sizeof(npy_complex256)) #endif #define h5py_offset_n64_real (HOFFSET(npy_complex64, real)) #define h5py_offset_n64_imag (HOFFSET(npy_complex64, imag)) #define h5py_offset_n128_real (HOFFSET(npy_complex128, real)) #define h5py_offset_n128_imag (HOFFSET(npy_complex128, imag)) #ifdef NPY_COMPLEX256 #define h5py_offset_n256_real (HOFFSET(npy_complex256, real)) #define h5py_offset_n256_imag (HOFFSET(npy_complex256, imag)) #endif #endif h5py-2.7.1/h5py/h5a.pyx0000644000175000017500000003070513025343121016167 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. """ Provides access to the low-level HDF5 "H5A" attribute interface. """ # Compile-time imports from _objects cimport pdefault from h5t cimport TypeID, typewrap, py_create from h5s cimport SpaceID from h5p cimport PropID from numpy cimport import_array, ndarray, PyArray_DATA from utils cimport check_numpy_read, check_numpy_write, emalloc, efree from _proxy cimport attr_rw from h5py import _objects from ._objects import phil, with_phil # Initialization import_array() # === General attribute operations ============================================ # --- create, create_by_name --- @with_phil def create(ObjectID loc not None, char* name, TypeID tid not None, SpaceID space not None, *, char* obj_name='.', PropID lapl=None): """(ObjectID loc, STRING name, TypeID tid, SpaceID space, **kwds) => AttrID Create a new attribute, attached to an existing object. STRING obj_name (".") Attach attribute to this group member instead PropID lapl Link access property list for obj_name """ return AttrID(H5Acreate_by_name(loc.id, obj_name, name, tid.id, space.id, H5P_DEFAULT, H5P_DEFAULT, pdefault(lapl))) # --- open, open_by_name, open_by_idx --- @with_phil def open(ObjectID loc not None, char* name=NULL, int index=-1, *, char* obj_name='.', int index_type=H5_INDEX_NAME, int order=H5_ITER_NATIVE, PropID lapl=None): """(ObjectID loc, STRING name=, INT index=, **kwds) => AttrID Open an attribute attached to an existing object. You must specify exactly one of either name or idx. Keywords are: STRING obj_name (".") Attribute is attached to this group member PropID lapl (None) Link access property list for obj_name INT index_type (h5.INDEX_NAME) INT order (h5.ITER_NATIVE) """ if (name == NULL and index < 0) or (name != NULL and index >= 0): raise TypeError("Exactly one of name or idx must be specified") if name != NULL: return AttrID(H5Aopen_by_name(loc.id, obj_name, name, H5P_DEFAULT, pdefault(lapl))) else: return AttrID(H5Aopen_by_idx(loc.id, obj_name, index_type, order, index, H5P_DEFAULT, pdefault(lapl))) # --- exists, exists_by_name --- @with_phil def exists(ObjectID loc not None, char* name, *, char* obj_name=".", PropID lapl=None): """(ObjectID loc, STRING name, **kwds) => BOOL Determine if an attribute is attached to this object. Keywords: STRING obj_name (".") Look for attributes attached to this group member PropID lapl (None): Link access property list for obj_name """ return H5Aexists_by_name(loc.id, obj_name, name, pdefault(lapl)) # --- rename, rename_by_name --- @with_phil def rename(ObjectID loc not None, char* name, char* new_name, *, char* obj_name='.', PropID lapl=None): """(ObjectID loc, STRING name, STRING new_name, **kwds) Rename an attribute. Keywords: STRING obj_name (".") Attribute is attached to this group member PropID lapl (None) Link access property list for obj_name """ H5Arename_by_name(loc.id, obj_name, name, new_name, pdefault(lapl)) @with_phil def delete(ObjectID loc not None, char* name=NULL, int index=-1, *, char* obj_name='.', int index_type=H5_INDEX_NAME, int order=H5_ITER_NATIVE, PropID lapl=None): """(ObjectID loc, STRING name=, INT index=, **kwds) Remove an attribute from an object. Specify exactly one of "name" or "index". Keyword-only arguments: STRING obj_name (".") Attribute is attached to this group member PropID lapl (None) Link access property list for obj_name INT index_type (h5.INDEX_NAME) INT order (h5.ITER_NATIVE) """ if name != NULL and index < 0: H5Adelete_by_name(loc.id, obj_name, name, pdefault(lapl)) elif name == NULL and index >= 0: H5Adelete_by_idx(loc.id, obj_name, index_type, order, index, pdefault(lapl)) else: raise TypeError("Exactly one of index or name must be specified.") @with_phil def get_num_attrs(ObjectID loc not None): """(ObjectID loc) => INT Determine the number of attributes attached to an HDF5 object. """ return H5Aget_num_attrs(loc.id) cdef class AttrInfo: cdef H5A_info_t info property corder_valid: """Indicates if the creation order is valid""" def __get__(self): return self.info.corder_valid property corder: """Creation order""" def __get__(self): return self.info.corder property cset: """Character set of attribute name (integer typecode from h5t)""" def __get__(self): return self.info.cset property data_size: """Size of raw data""" def __get__(self): return self.info.data_size def _hash(self): return hash((self.corder_valid, self.corder, self.cset, self.data_size)) @with_phil def get_info(ObjectID loc not None, char* name=NULL, int index=-1, *, char* obj_name='.', PropID lapl=None, int index_type=H5_INDEX_NAME, int order=H5_ITER_NATIVE): """(ObjectID loc, STRING name=, INT index=, **kwds) => AttrInfo Get information about an attribute, in one of two ways: 1. If you have the attribute identifier, just pass it in 2. If you have the parent object, supply it and exactly one of either name or index. STRING obj_name (".") Use this group member instead PropID lapl (None) Link access property list for obj_name INT index_type (h5.INDEX_NAME) Which index to use INT order (h5.ITER_NATIVE) What order the index is in """ cdef AttrInfo info = AttrInfo() if name == NULL and index < 0: H5Aget_info(loc.id, &info.info) elif name != NULL and index >= 0: raise TypeError("At most one of name and index may be specified") elif name != NULL: H5Aget_info_by_name(loc.id, obj_name, name, &info.info, pdefault(lapl)) elif index >= 0: H5Aget_info_by_idx(loc.id, obj_name, index_type, order, index, &info.info, pdefault(lapl)) return info # === Iteration routines ====================================================== cdef class _AttrVisitor: cdef object func cdef object retval def __init__(self, func): self.func = func self.retval = None cdef herr_t cb_attr_iter(hid_t loc_id, const char* attr_name, const H5A_info_t *ainfo, void* vis_in) except 2: cdef _AttrVisitor vis = <_AttrVisitor>vis_in cdef AttrInfo info = AttrInfo() info.info = ainfo[0] vis.retval = vis.func(attr_name, info) if vis.retval is not None: return 1 return 0 cdef herr_t cb_attr_simple(hid_t loc_id, const char* attr_name, const H5A_info_t *ainfo, void* vis_in) except 2: cdef _AttrVisitor vis = <_AttrVisitor>vis_in vis.retval = vis.func(attr_name) if vis.retval is not None: return 1 return 0 @with_phil def iterate(ObjectID loc not None, object func, int index=0, *, int index_type=H5_INDEX_NAME, int order=H5_ITER_NATIVE, bint info=0): """(ObjectID loc, CALLABLE func, INT index=0, **kwds) => Iterate a callable (function, method or callable object) over the attributes attached to this object. You callable should have the signature:: func(STRING name) => Result or if the keyword argument "info" is True:: func(STRING name, AttrInfo info) => Result Returning None continues iteration; returning anything else aborts iteration and returns that value. Keywords: BOOL info (False) Callback is func(STRING name, AttrInfo info), not func(STRING name) INT index_type (h5.INDEX_NAME) Which index to use INT order (h5.ITER_NATIVE) Index order to use """ if index < 0: raise ValueError("Starting index must be a non-negative integer.") cdef hsize_t i = index cdef _AttrVisitor vis = _AttrVisitor(func) cdef H5A_operator2_t cfunc if info: cfunc = cb_attr_iter else: cfunc = cb_attr_simple H5Aiterate2(loc.id, index_type, order, &i, cfunc, vis) return vis.retval # === Attribute class & methods =============================================== cdef class AttrID(ObjectID): """ Logical representation of an HDF5 attribute identifier. Objects of this class can be used in any HDF5 function call which expects an attribute identifier. Additionally, all ``H5A*`` functions which always take an attribute instance as the first argument are presented as methods of this class. * Hashable: No * Equality: Identifier comparison """ property name: """The attribute's name""" def __get__(self): with phil: return self.get_name() property shape: """A Numpy-style shape tuple representing the attribute's dataspace""" def __get__(self): cdef SpaceID space with phil: space = self.get_space() return space.get_simple_extent_dims() property dtype: """A Numpy-stype dtype object representing the attribute's datatype""" def __get__(self): cdef TypeID tid with phil: tid = self.get_type() return tid.py_dtype() @with_phil def read(self, ndarray arr not None, TypeID mtype=None): """(NDARRAY arr, TypeID mtype=None) Read the attribute data into the given Numpy array. Note that the Numpy array must have the same shape as the HDF5 attribute, and a conversion-compatible datatype. The Numpy array must be writable and C-contiguous. If this is not the case, the read will fail with an exception. If provided, the HDF5 TypeID mtype will override the array's dtype. """ cdef hid_t space_id space_id = 0 try: space_id = H5Aget_space(self.id) check_numpy_write(arr, space_id) if mtype is None: mtype = py_create(arr.dtype) attr_rw(self.id, mtype.id, PyArray_DATA(arr), 1) finally: if space_id: H5Sclose(space_id) @with_phil def write(self, ndarray arr not None, TypeID mtype=None): """(NDARRAY arr) Write the contents of a Numpy array too the attribute. Note that the Numpy array must have the same shape as the HDF5 attribute, and a conversion-compatible datatype. The Numpy array must be C-contiguous. If this is not the case, the write will fail with an exception. """ cdef hid_t space_id space_id = 0 try: space_id = H5Aget_space(self.id) check_numpy_read(arr, space_id) if mtype is None: mtype = py_create(arr.dtype) attr_rw(self.id, mtype.id, PyArray_DATA(arr), 0) finally: if space_id: H5Sclose(space_id) @with_phil def get_name(self): """() => STRING name Determine the name of this attribute. """ cdef int blen cdef char* buf buf = NULL try: blen = H5Aget_name(self.id, 0, NULL) assert blen >= 0 buf = emalloc(sizeof(char)*blen+1) blen = H5Aget_name(self.id, blen+1, buf) strout = buf finally: efree(buf) return strout @with_phil def get_space(self): """() => SpaceID Create and return a copy of the attribute's dataspace. """ return SpaceID(H5Aget_space(self.id)) @with_phil def get_type(self): """() => TypeID Create and return a copy of the attribute's datatype. """ return typewrap(H5Aget_type(self.id)) @with_phil def get_storage_size(self): """() => INT Get the amount of storage required for this attribute. """ return H5Aget_storage_size(self.id) h5py-2.7.1/h5py/h5i.pxd0000644000175000017500000000055513025343121016152 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. from defs cimport * from _objects cimport ObjectID cpdef ObjectID wrap_identifier(hid_t ident) h5py-2.7.1/h5py/h5d.pyx0000644000175000017500000003403113025343121016166 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. """ Provides access to the low-level HDF5 "H5D" dataset interface. """ include "config.pxi" # Compile-time imports from _objects cimport pdefault from numpy cimport ndarray, import_array, PyArray_DATA, NPY_WRITEABLE from utils cimport check_numpy_read, check_numpy_write, \ convert_tuple, emalloc, efree from h5t cimport TypeID, typewrap, py_create from h5s cimport SpaceID from h5p cimport PropID, propwrap from _proxy cimport dset_rw from h5py import _objects from ._objects import phil, with_phil # Initialization import_array() # === Public constants and data structures ==================================== COMPACT = H5D_COMPACT CONTIGUOUS = H5D_CONTIGUOUS CHUNKED = H5D_CHUNKED ALLOC_TIME_DEFAULT = H5D_ALLOC_TIME_DEFAULT ALLOC_TIME_LATE = H5D_ALLOC_TIME_LATE ALLOC_TIME_EARLY = H5D_ALLOC_TIME_EARLY ALLOC_TIME_INCR = H5D_ALLOC_TIME_INCR SPACE_STATUS_NOT_ALLOCATED = H5D_SPACE_STATUS_NOT_ALLOCATED SPACE_STATUS_PART_ALLOCATED = H5D_SPACE_STATUS_PART_ALLOCATED SPACE_STATUS_ALLOCATED = H5D_SPACE_STATUS_ALLOCATED FILL_TIME_ALLOC = H5D_FILL_TIME_ALLOC FILL_TIME_NEVER = H5D_FILL_TIME_NEVER FILL_TIME_IFSET = H5D_FILL_TIME_IFSET FILL_VALUE_UNDEFINED = H5D_FILL_VALUE_UNDEFINED FILL_VALUE_DEFAULT = H5D_FILL_VALUE_DEFAULT FILL_VALUE_USER_DEFINED = H5D_FILL_VALUE_USER_DEFINED IF HDF5_VERSION >= VDS_MIN_HDF5_VERSION: VIRTUAL = H5D_VIRTUAL VDS_FIRST_MISSING = H5D_VDS_FIRST_MISSING VDS_LAST_AVAILABLE = H5D_VDS_LAST_AVAILABLE # === Dataset operations ====================================================== @with_phil def create(ObjectID loc not None, object name, TypeID tid not None, SpaceID space not None, PropID dcpl=None, PropID lcpl=None, PropID dapl = None): """ (objectID loc, STRING name or None, TypeID tid, SpaceID space, PropDCID dcpl=None, PropID lcpl=None) => DatasetID Create a new dataset. If "name" is None, the dataset will be anonymous. """ cdef hid_t dsid cdef char* cname = NULL if name is not None: cname = name if cname != NULL: dsid = H5Dcreate2(loc.id, cname, tid.id, space.id, pdefault(lcpl), pdefault(dcpl), pdefault(dapl)) else: dsid = H5Dcreate_anon(loc.id, tid.id, space.id, pdefault(dcpl), pdefault(dapl)) return DatasetID(dsid) @with_phil def open(ObjectID loc not None, char* name, PropID dapl=None): """ (ObjectID loc, STRING name, PropID dapl=None) => DatasetID Open an existing dataset attached to a group or file object, by name. If specified, dapl may be a dataset access property list. """ return DatasetID(H5Dopen2(loc.id, name, pdefault(dapl))) # --- Proxy functions for safe(r) threading ----------------------------------- cdef class DatasetID(ObjectID): """ Represents an HDF5 dataset identifier. Objects of this class may be used in any HDF5 function which expects a dataset identifier. Also, all H5D* functions which take a dataset instance as their first argument are presented as methods of this class. Properties: dtype: Numpy dtype representing the dataset type shape: Numpy-style shape tuple representing the dataspace rank: Integer giving dataset rank * Hashable: Yes, unless anonymous * Equality: True HDF5 identity if unless anonymous """ property dtype: """ Numpy dtype object representing the dataset type """ def __get__(self): # Dataset type can't change cdef TypeID tid with phil: if self._dtype is None: tid = self.get_type() self._dtype = tid.dtype return self._dtype property shape: """ Numpy-style shape tuple representing the dataspace """ def __get__(self): # Shape can change (DatasetID.extend), so don't cache it cdef SpaceID sid with phil: sid = self.get_space() return sid.get_simple_extent_dims() property rank: """ Integer giving the dataset rank (0 = scalar) """ def __get__(self): cdef SpaceID sid with phil: sid = self.get_space() return sid.get_simple_extent_ndims() @with_phil def read(self, SpaceID mspace not None, SpaceID fspace not None, ndarray arr_obj not None, TypeID mtype=None, PropID dxpl=None): """ (SpaceID mspace, SpaceID fspace, NDARRAY arr_obj, TypeID mtype=None, PropDXID dxpl=None) Read data from an HDF5 dataset into a Numpy array. It is your responsibility to ensure that the memory dataspace provided is compatible with the shape of the Numpy array. Since a wide variety of dataspace configurations are possible, this is not checked. You can easily crash Python by reading in data from too large a dataspace. If a memory datatype is not specified, one will be auto-created based on the array's dtype. The provided Numpy array must be writable and C-contiguous. If this is not the case, ValueError will be raised and the read will fail. Keyword dxpl may be a dataset transfer property list. """ cdef hid_t self_id, mtype_id, mspace_id, fspace_id, plist_id cdef void* data cdef int oldflags if mtype is None: mtype = py_create(arr_obj.dtype) check_numpy_write(arr_obj, -1) self_id = self.id mtype_id = mtype.id mspace_id = mspace.id fspace_id = fspace.id plist_id = pdefault(dxpl) data = PyArray_DATA(arr_obj) dset_rw(self_id, mtype_id, mspace_id, fspace_id, plist_id, data, 1) @with_phil def write(self, SpaceID mspace not None, SpaceID fspace not None, ndarray arr_obj not None, TypeID mtype=None, PropID dxpl=None): """ (SpaceID mspace, SpaceID fspace, NDARRAY arr_obj, TypeID mtype=None, PropDXID dxpl=None) Write data from a Numpy array to an HDF5 dataset. Keyword dxpl may be a dataset transfer property list. It is your responsibility to ensure that the memory dataspace provided is compatible with the shape of the Numpy array. Since a wide variety of dataspace configurations are possible, this is not checked. You can easily crash Python by writing data from too large a dataspace. If a memory datatype is not specified, one will be auto-created based on the array's dtype. The provided Numpy array must be C-contiguous. If this is not the case, ValueError will be raised and the read will fail. """ cdef hid_t self_id, mtype_id, mspace_id, fspace_id, plist_id cdef void* data cdef int oldflags if mtype is None: mtype = py_create(arr_obj.dtype) check_numpy_read(arr_obj, -1) self_id = self.id mtype_id = mtype.id mspace_id = mspace.id fspace_id = fspace.id plist_id = pdefault(dxpl) data = PyArray_DATA(arr_obj) dset_rw(self_id, mtype_id, mspace_id, fspace_id, plist_id, data, 0) @with_phil def extend(self, tuple shape): """ (TUPLE shape) Extend the given dataset so it's at least as big as "shape". Note that a dataset may only be extended up to the maximum dimensions of its dataspace, which are fixed when the dataset is created. """ cdef int rank cdef hid_t space_id = 0 cdef hsize_t* dims = NULL try: space_id = H5Dget_space(self.id) rank = H5Sget_simple_extent_ndims(space_id) if len(shape) != rank: raise TypeError("New shape length (%d) must match dataset rank (%d)" % (len(shape), rank)) dims = emalloc(sizeof(hsize_t)*rank) convert_tuple(shape, dims, rank) H5Dextend(self.id, dims) finally: efree(dims) if space_id: H5Sclose(space_id) @with_phil def set_extent(self, tuple shape): """ (TUPLE shape) Set the size of the dataspace to match the given shape. If the new size is larger in any dimension, it must be compatible with the maximum dataspace size. """ cdef int rank cdef hid_t space_id = 0 cdef hsize_t* dims = NULL try: space_id = H5Dget_space(self.id) rank = H5Sget_simple_extent_ndims(space_id) if len(shape) != rank: raise TypeError("New shape length (%d) must match dataset rank (%d)" % (len(shape), rank)) dims = emalloc(sizeof(hsize_t)*rank) convert_tuple(shape, dims, rank) H5Dset_extent(self.id, dims) finally: efree(dims) if space_id: H5Sclose(space_id) @with_phil def get_space(self): """ () => SpaceID Create and return a new copy of the dataspace for this dataset. """ return SpaceID(H5Dget_space(self.id)) @with_phil def get_space_status(self): """ () => INT space_status_code Determine if space has been allocated for a dataset. Return value is one of: * SPACE_STATUS_NOT_ALLOCATED * SPACE_STATUS_PART_ALLOCATED * SPACE_STATUS_ALLOCATED """ cdef H5D_space_status_t status H5Dget_space_status(self.id, &status) return status @with_phil def get_type(self): """ () => TypeID Create and return a new copy of the datatype for this dataset. """ return typewrap(H5Dget_type(self.id)) @with_phil def get_create_plist(self): """ () => PropDCID Create an return a new copy of the dataset creation property list used when this dataset was created. """ return propwrap(H5Dget_create_plist(self.id)) @with_phil def get_access_plist(self): """ () => PropDAID Create an return a new copy of the dataset access property list. """ return propwrap(H5Dget_access_plist(self.id)) @with_phil def get_offset(self): """ () => LONG offset or None Get the offset of this dataset in the file, in bytes, or None if it doesn't have one. This is always the case for datasets which use chunked storage, compact datasets, and datasets for which space has not yet been allocated in the file. """ cdef haddr_t offset offset = H5Dget_offset(self.id) if offset == HADDR_UNDEF: return None return offset @with_phil def get_storage_size(self): """ () => LONG storage_size Determine the amount of file space required for a dataset. Note this only counts the space which has actually been allocated; it may even be zero. """ return H5Dget_storage_size(self.id) IF HDF5_VERSION >= SWMR_MIN_HDF5_VERSION: @with_phil def flush(self): """ no return Flushes all buffers associated with a dataset to disk. This function causes all buffers associated with a dataset to be immediately flushed to disk without removing the data from the cache. Use this in SWMR write mode to allow readers to be updated with the dataset changes. Feature requires: 1.9.178 HDF5 """ H5Dflush(self.id) @with_phil def refresh(self): """ no return Refreshes all buffers associated with a dataset. This function causes all buffers associated with a dataset to be cleared and immediately re-loaded with updated contents from disk. This function essentially closes the dataset, evicts all metadata associated with it from the cache, and then re-opens the dataset. The reopened dataset is automatically re-registered with the same ID. Use this in SWMR read mode to poll for dataset changes. Feature requires: 1.9.178 HDF5 """ H5Drefresh(self.id) IF HDF5_VERSION >= (1, 8, 11): def write_direct_chunk(self, offsets, bytes data, H5Z_filter_t filter_mask=H5Z_FILTER_NONE, PropID dxpl=None): """ (offsets, bytes data, H5Z_filter_t filter_mask=H5Z_FILTER_NONE, PropID dxpl=None) Writes data from a bytes array (as provided e.g. by struct.pack) directly to a chunk at position specified by the offsets argument. Feature requires: 1.8.11 HDF5 """ cdef hid_t dset_id cdef hid_t dxpl_id cdef hid_t space_id = 0 cdef hsize_t *offset = NULL cdef size_t data_size cdef int rank dset_id = self.id dxpl_id = pdefault(dxpl) space_id = H5Dget_space(self.id) rank = H5Sget_simple_extent_ndims(space_id) if len(offsets) != rank: raise TypeError("offset length (%d) must match dataset rank (%d)" % (len(offsets), rank)) try: offset = emalloc(sizeof(hsize_t)*rank) convert_tuple(offsets, offset, rank) H5DOwrite_chunk(dset_id, dxpl_id, filter_mask, offset, len(data), data) finally: efree(offset) if space_id: H5Sclose(space_id) h5py-2.7.1/h5py/h5f.pxd0000644000175000017500000000060113025343121016137 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. from defs cimport * from _objects cimport ObjectID from h5g cimport GroupID cdef class FileID(GroupID): pass h5py-2.7.1/h5py/h5o.pxd0000644000175000017500000000044113025343121016152 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. from defs cimport * h5py-2.7.1/h5py/_objects.pxd0000644000175000017500000000137013025343121017251 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. from defs cimport * cdef class ObjectID: cdef object __weakref__ cdef readonly hid_t id cdef public int locked # Cannot be closed, explicitly or auto cdef object _hash cdef size_t _pyid # Convenience functions cdef hid_t pdefault(ObjectID pid) cdef int is_h5py_obj_valid(ObjectID obj) # Inheritance scheme (for top-level cimport and import statements): # # _objects, _proxy, h5fd, h5z # h5i, h5r, utils # _conv, h5t, h5s # h5p # h5d, h5a, h5f, h5g # h5l h5py-2.7.1/h5py/utils.pyx0000644000175000017500000001243713025343121016654 0ustar tcaswelltcaswell00000000000000# cython: profile=False # This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. from numpy cimport ndarray, import_array, \ NPY_UINT16, NPY_UINT32, NPY_UINT64, npy_intp, \ PyArray_SimpleNew, PyArray_ContiguousFromAny, \ PyArray_FROM_OTF, PyArray_DIM, \ NPY_CONTIGUOUS, NPY_NOTSWAPPED, NPY_FORCECAST, \ NPY_C_CONTIGUOUS, NPY_WRITEABLE # Initialization import_array() # === Exception-aware memory allocation ======================================= cdef void* emalloc(size_t size) except? NULL: # Wrapper for malloc(size) with the following behavior: # 1. Always returns NULL for emalloc(0) # 2. Raises RuntimeError for emalloc(size<0) and returns NULL # 3. Raises RuntimeError if allocation fails and returns NULL cdef void *retval = NULL if size == 0: return NULL retval = malloc(size) if retval == NULL: errmsg = "Can't malloc %d bytes" % size PyErr_SetString(MemoryError, errmsg) return NULL return retval cdef void efree(void* what): free(what) def _test_emalloc(size_t size): """Stub to simplify unit tests""" cdef void* mem mem = emalloc(size) if size == 0: assert mem == NULL efree(mem) # === Testing of NumPy arrays ================================================= cdef int check_numpy(ndarray arr, hid_t space_id, int write): # -1 if exception, NOT AUTOMATICALLY CHECKED cdef int required_flags cdef hsize_t arr_rank cdef hsize_t space_rank cdef int i if arr is None: PyErr_SetString(TypeError, "Array is None") return -1 # Validate array flags if write: if not (arr.flags & NPY_C_CONTIGUOUS and arr.flags & NPY_WRITEABLE): PyErr_SetString(TypeError, "Array must be C-contiguous and writable") return -1 else: if not (arr.flags & NPY_C_CONTIGUOUS): PyErr_SetString(TypeError, "Array must be C-contiguous") return -1 return 1 cpdef int check_numpy_write(ndarray arr, hid_t space_id=-1) except -1: return check_numpy(arr, space_id, 1) cpdef int check_numpy_read(ndarray arr, hid_t space_id=-1) except -1: return check_numpy(arr, space_id, 0) # === Conversion between HDF5 buffers and tuples ============================== cdef int convert_tuple(object tpl, hsize_t *dims, hsize_t rank) except -1: # Convert a Python tuple to an hsize_t array. You must allocate # the array yourself and pass both it and the size to this function. # Returns 0 on success, -1 on failure and raises an exception. cdef int i if len(tpl) != rank: raise ValueError("Tuple length incompatible with array") try: for i from 0<=iemalloc(sizeof(npy_intp)*rank) try: for i from 0<=i= 0, also ensure that the length matches. # Otherwise raises ValueError if (tpl is None and none_allowed) or \ (isinstance(tpl, tuple) and (size < 0 or len(tpl) == size)): return 1 nmsg = "" if size < 0 else " of size %d" % size smsg = "" if not none_allowed else " or None" msg = "%s must be a tuple%s%s." % (name, smsg, nmsg) PyErr_SetString(ValueError, msg) return -1 h5py-2.7.1/h5py/h5i.pyx0000644000175000017500000000754413025343121016204 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. """ Identifier interface for object inspection. """ from ._objects import phil, with_phil # === Public constants and data structures ==================================== BADID = H5I_BADID FILE = H5I_FILE GROUP = H5I_GROUP DATASPACE = H5I_DATASPACE DATASET = H5I_DATASET ATTR = H5I_ATTR REFERENCE = H5I_REFERENCE GENPROP_CLS = H5I_GENPROP_CLS GENPROP_LST = H5I_GENPROP_LST DATATYPE = H5I_DATATYPE cpdef ObjectID wrap_identifier(hid_t ident): cdef H5I_type_t typecode cdef ObjectID obj typecode = H5Iget_type(ident) if typecode == H5I_FILE: import h5f obj = h5f.FileID(ident) elif typecode == H5I_DATASET: import h5d obj = h5d.DatasetID(ident) elif typecode == H5I_GROUP: import h5g obj = h5g.GroupID(ident) elif typecode == H5I_ATTR: import h5a obj = h5a.AttrID(ident) elif typecode == H5I_DATATYPE: import h5t obj = h5t.typewrap(ident) elif typecode == H5I_GENPROP_LST: import h5p obj = h5p.propwrap(ident) else: raise ValueError("Unrecognized type code %d" % typecode) return obj # === Identifier API ========================================================== @with_phil def get_type(ObjectID obj not None): """ (ObjectID obj) => INT type_code Determine the HDF5 typecode of an arbitrary HDF5 object. The return value is always one of the type constants defined in this module; if the ID is invalid, BADID is returned. """ return H5Iget_type(obj.id) @with_phil def get_name(ObjectID obj not None): """ (ObjectID obj) => STRING name, or None Determine (a) name of an HDF5 object. Because an object has as many names as there are hard links to it, this may not be unique. If the identifier is invalid or is not associated with a name (in the case of transient datatypes, dataspaces, etc), returns None. For some reason, this does not work on dereferenced objects. """ cdef int namelen cdef char* name try: namelen = H5Iget_name(obj.id, NULL, 0) except Exception: return None if namelen == 0: # 1.6.5 doesn't raise an exception return None assert namelen > 0 name = malloc(sizeof(char)*(namelen+1)) try: H5Iget_name(obj.id, name, namelen+1) pystring = name return pystring finally: free(name) @with_phil def get_file_id(ObjectID obj not None): """ (ObjectID obj) => FileID Obtain an identifier for the file in which this object resides. """ import h5f cdef hid_t fid fid = H5Iget_file_id(obj.id) return h5f.FileID(fid) @with_phil def inc_ref(ObjectID obj not None): """ (ObjectID obj) Increment the reference count for the given object. This function is provided for debugging only. Reference counting is automatically synchronized with Python, and you can easily break ObjectID instances by abusing this function. """ H5Iinc_ref(obj.id) @with_phil def get_ref(ObjectID obj not None): """ (ObjectID obj) => INT Retrieve the reference count for the given object. """ return H5Iget_ref(obj.id) @with_phil def dec_ref(ObjectID obj not None): """ (ObjectID obj) Decrement the reference count for the given object. This function is provided for debugging only. Reference counting is automatically synchronized with Python, and you can easily break ObjectID instances by abusing this function. """ H5Idec_ref(obj.id) h5py-2.7.1/h5py/_errors.pxd0000644000175000017500000002462313025343121017142 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. from api_types_hdf5 cimport * # Auto-set exception. Returns 1 if exception set, 0 if no HDF5 error found. cdef int set_exception() except -1 cdef extern from "hdf5.h": ctypedef enum H5E_major_t: H5E_NONE_MAJOR = 0, # special zero, no error H5E_ARGS, # invalid arguments to routine H5E_RESOURCE, # resource unavailable H5E_INTERNAL, # Internal error (too specific to document) H5E_FILE, # file Accessability H5E_IO, # Low-level I/O H5E_FUNC, # function Entry/Exit H5E_ATOM, # object Atom H5E_CACHE, # object Cache H5E_BTREE, # B-Tree Node H5E_SYM, # symbol Table H5E_HEAP, # Heap H5E_OHDR, # object Header H5E_DATATYPE, # Datatype H5E_DATASPACE, # Dataspace H5E_DATASET, # Dataset H5E_STORAGE, # data storage H5E_PLIST, # Property lists H5E_ATTR, # Attribute H5E_PLINE, # Data filters H5E_EFL, # External file list H5E_REFERENCE, # References H5E_VFL, # Virtual File Layer # H5E_TBBT, # Threaded, Balanced, Binary Trees (not in 1.8) H5E_TST, # Ternary Search Trees H5E_RS, # Reference Counted Strings H5E_ERROR, # Error API H5E_SLIST # Skip Lists ctypedef enum H5E_minor_t: # Generic low-level file I/O errors H5E_SEEKERROR # Seek failed H5E_READERROR # Read failed H5E_WRITEERROR # Write failed H5E_CLOSEERROR # Close failed H5E_OVERFLOW # Address overflowed H5E_FCNTL # File control (fcntl) failed # Resource errors H5E_NOSPACE # No space available for allocation H5E_CANTALLOC # Can't allocate space H5E_CANTCOPY # Unable to copy object H5E_CANTFREE # Unable to free object H5E_ALREADYEXISTS # Object already exists H5E_CANTLOCK # Unable to lock object H5E_CANTUNLOCK # Unable to unlock object H5E_CANTGC # Unable to garbage collect H5E_CANTGETSIZE # Unable to compute size H5E_OBJOPEN # Object is already open # Heap errors H5E_CANTRESTORE # Can't restore condition H5E_CANTCOMPUTE # Can't compute value H5E_CANTEXTEND # Can't extend heap's space H5E_CANTATTACH # Can't attach object H5E_CANTUPDATE # Can't update object H5E_CANTOPERATE # Can't operate on object # Function entry/exit interface errors H5E_CANTINIT # Unable to initialize object H5E_ALREADYINIT # Object already initialized H5E_CANTRELEASE # Unable to release object # Property list errors H5E_CANTGET # Can't get value H5E_CANTSET # Can't set value H5E_DUPCLASS # Duplicate class name in parent class # Free space errors H5E_CANTMERGE # Can't merge objects H5E_CANTREVIVE # Can't revive object H5E_CANTSHRINK # Can't shrink container # Object header related errors H5E_LINKCOUNT # Bad object header link H5E_VERSION # Wrong version number H5E_ALIGNMENT # Alignment error H5E_BADMESG # Unrecognized message H5E_CANTDELETE # Can't delete message H5E_BADITER # Iteration failed H5E_CANTPACK # Can't pack messages H5E_CANTRESET # Can't reset object count # System level errors H5E_SYSERRSTR # System error message # I/O pipeline errors H5E_NOFILTER # Requested filter is not available H5E_CALLBACK # Callback failed H5E_CANAPPLY # Error from filter 'can apply' callback H5E_SETLOCAL # Error from filter 'set local' callback H5E_NOENCODER # Filter present but encoding disabled H5E_CANTFILTER # Filter operation failed # Group related errors H5E_CANTOPENOBJ # Can't open object H5E_CANTCLOSEOBJ # Can't close object H5E_COMPLEN # Name component is too long H5E_PATH # Problem with path to object # No error H5E_NONE_MINOR # No error # File accessability errors H5E_FILEEXISTS # File already exists H5E_FILEOPEN # File already open H5E_CANTCREATE # Unable to create file H5E_CANTOPENFILE # Unable to open file H5E_CANTCLOSEFILE # Unable to close file H5E_NOTHDF5 # Not an HDF5 file H5E_BADFILE # Bad file ID accessed H5E_TRUNCATED # File has been truncated H5E_MOUNT # File mount error # Object atom related errors H5E_BADATOM # Unable to find atom information (already closed?) H5E_BADGROUP # Unable to find ID group information H5E_CANTREGISTER # Unable to register new atom H5E_CANTINC # Unable to increment reference count H5E_CANTDEC # Unable to decrement reference count H5E_NOIDS # Out of IDs for group # Cache related errors H5E_CANTFLUSH # Unable to flush data from cache H5E_CANTSERIALIZE # Unable to serialize data from cache H5E_CANTLOAD # Unable to load metadata into cache H5E_PROTECT # Protected metadata error H5E_NOTCACHED # Metadata not currently cached H5E_SYSTEM # Internal error detected H5E_CANTINS # Unable to insert metadata into cache H5E_CANTRENAME # Unable to rename metadata H5E_CANTPROTECT # Unable to protect metadata H5E_CANTUNPROTECT # Unable to unprotect metadata H5E_CANTPIN # Unable to pin cache entry H5E_CANTUNPIN # Unable to un-pin cache entry H5E_CANTMARKDIRTY # Unable to mark a pinned entry as dirty H5E_CANTDIRTY # Unable to mark metadata as dirty H5E_CANTEXPUNGE # Unable to expunge a metadata cache entry H5E_CANTRESIZE # Unable to resize a metadata cache entry # Link related errors H5E_TRAVERSE # Link traversal failure H5E_NLINKS # Too many soft links in path H5E_NOTREGISTERED # Link class not registered H5E_CANTMOVE # Move callback returned error H5E_CANTSORT # Can't sort objects # Parallel MPI errors H5E_MPI # Some MPI function failed H5E_MPIERRSTR # MPI Error String H5E_CANTRECV # Can't receive data # Dataspace errors H5E_CANTCLIP # Can't clip hyperslab region H5E_CANTCOUNT # Can't count elements H5E_CANTSELECT # Can't select hyperslab H5E_CANTNEXT # Can't move to next iterator location H5E_BADSELECT # Invalid selection H5E_CANTCOMPARE # Can't compare objects # Argument errors H5E_UNINITIALIZED # Information is uinitialized H5E_UNSUPPORTED # Feature is unsupported H5E_BADTYPE # Inappropriate type H5E_BADRANGE # Out of range H5E_BADVALUE # Bad value # B-tree related errors H5E_NOTFOUND # Object not found H5E_EXISTS # Object already exists H5E_CANTENCODE # Unable to encode value H5E_CANTDECODE # Unable to decode value H5E_CANTSPLIT # Unable to split node H5E_CANTREDISTRIBUTE # Unable to redistribute records H5E_CANTSWAP # Unable to swap records H5E_CANTINSERT # Unable to insert object H5E_CANTLIST # Unable to list node H5E_CANTMODIFY # Unable to modify record H5E_CANTREMOVE # Unable to remove object # Datatype conversion errors H5E_CANTCONVERT # Can't convert datatypes H5E_BADSIZE # Bad size for object cdef enum H5E_direction_t: H5E_WALK_UPWARD = 0 # begin deep, end at API function H5E_WALK_DOWNWARD = 1 # begin at API function, end deep ctypedef struct H5E_error_t: H5E_major_t maj_num # major error number H5E_minor_t min_num # minor error number char *func_name # function in which error occurred char *file_name # file in which error occurred unsigned line # line in file where error occurs char *desc # optional supplied description char *H5Eget_major(H5E_major_t n) char *H5Eget_minor(H5E_minor_t n) herr_t H5Eclear() except * ctypedef herr_t (*H5E_auto_t)(void *client_data) herr_t H5Eset_auto(H5E_auto_t func, void *client_data) herr_t H5Eget_auto(H5E_auto_t *func, void** client_data) herr_t H5Eprint(void *stream) ctypedef herr_t (*H5E_walk_t)(int n, H5E_error_t *err_desc, void* client_data) herr_t H5Ewalk(H5E_direction_t direction, H5E_walk_t func, void* client_data) # --- Functions for managing the HDF5 error callback mechanism --- ctypedef struct err_cookie: # Defines the error handler state (callback and callback data) H5E_auto_t func void *data # Set (via H5Eset_auto) the HDF5 error handler for this thread. Returns # the old (presently installed) handler. cdef err_cookie set_error_handler(err_cookie handler) h5py-2.7.1/h5py/api_types_hdf5.pxd0000644000175000017500000006240113025343121020366 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. from api_types_ext cimport * include "config.pxi" cdef extern from "hdf5.h": # Basic types ctypedef long int hid_t ctypedef int hbool_t ctypedef int herr_t ctypedef int htri_t ctypedef long long hsize_t ctypedef signed long long hssize_t ctypedef signed long long haddr_t ctypedef struct hvl_t: size_t len # Length of VL data (in base type units) void *p # Pointer to VL data int HADDR_UNDEF # New in 1.8.X ctypedef enum H5_iter_order_t: H5_ITER_UNKNOWN = -1, # Unknown order H5_ITER_INC, # Increasing order H5_ITER_DEC, # Decreasing order H5_ITER_NATIVE, # No particular order, whatever is fastest H5_ITER_N # Number of iteration orders ctypedef enum H5_index_t: H5_INDEX_UNKNOWN = -1, # Unknown index type H5_INDEX_NAME, # Index on names H5_INDEX_CRT_ORDER, # Index on creation order H5_INDEX_N # Number of indices defined # === H5D - Dataset API ======================================================= ctypedef enum H5D_layout_t: H5D_LAYOUT_ERROR = -1, H5D_COMPACT = 0, H5D_CONTIGUOUS = 1, H5D_CHUNKED = 2, H5D_VIRTUAL = 3, # New in 1.10 H5D_NLAYOUTS = 4 IF HDF5_VERSION >= VDS_MIN_HDF5_VERSION: ctypedef enum H5D_vds_view_t: H5D_VDS_ERROR = -1, H5D_VDS_FIRST_MISSING = 0, H5D_VDS_LAST_AVAILABLE = 1 ctypedef enum H5D_alloc_time_t: H5D_ALLOC_TIME_ERROR =-1, H5D_ALLOC_TIME_DEFAULT =0, H5D_ALLOC_TIME_EARLY =1, H5D_ALLOC_TIME_LATE =2, H5D_ALLOC_TIME_INCR =3 ctypedef enum H5D_space_status_t: H5D_SPACE_STATUS_ERROR =-1, H5D_SPACE_STATUS_NOT_ALLOCATED =0, H5D_SPACE_STATUS_PART_ALLOCATED =1, H5D_SPACE_STATUS_ALLOCATED =2 ctypedef enum H5D_fill_time_t: H5D_FILL_TIME_ERROR =-1, H5D_FILL_TIME_ALLOC =0, H5D_FILL_TIME_NEVER =1, H5D_FILL_TIME_IFSET =2 ctypedef enum H5D_fill_value_t: H5D_FILL_VALUE_ERROR =-1, H5D_FILL_VALUE_UNDEFINED =0, H5D_FILL_VALUE_DEFAULT =1, H5D_FILL_VALUE_USER_DEFINED =2 ctypedef herr_t (*H5D_operator_t)(void *elem, hid_t type_id, unsigned ndim, hsize_t *point, void *operator_data) except -1 # === H5F - File API ========================================================== # File constants cdef enum: H5F_ACC_TRUNC H5F_ACC_RDONLY H5F_ACC_RDWR H5F_ACC_EXCL H5F_ACC_DEBUG H5F_ACC_CREAT H5F_ACC_SWMR_WRITE H5F_ACC_SWMR_READ # The difference between a single file and a set of mounted files cdef enum H5F_scope_t: H5F_SCOPE_LOCAL = 0, # specified file handle only H5F_SCOPE_GLOBAL = 1, # entire virtual file H5F_SCOPE_DOWN = 2 # for internal use only cdef enum H5F_close_degree_t: H5F_CLOSE_WEAK = 0, H5F_CLOSE_SEMI = 1, H5F_CLOSE_STRONG = 2, H5F_CLOSE_DEFAULT = 3 int H5F_OBJ_FILE int H5F_OBJ_DATASET int H5F_OBJ_GROUP int H5F_OBJ_DATATYPE int H5F_OBJ_ATTR int H5F_OBJ_ALL int H5F_OBJ_LOCAL ctypedef enum H5F_libver_t: H5F_LIBVER_EARLIEST #/* Use the earliest possible format for storing objects */ H5F_LIBVER_LATEST #/* Use the latest possible format available for storing objects*/ # === H5FD - Low-level file descriptor API ==================================== ctypedef enum H5FD_mem_t: H5FD_MEM_NOLIST = -1, H5FD_MEM_DEFAULT = 0, H5FD_MEM_SUPER = 1, H5FD_MEM_BTREE = 2, H5FD_MEM_DRAW = 3, H5FD_MEM_GHEAP = 4, H5FD_MEM_LHEAP = 5, H5FD_MEM_OHDR = 6, H5FD_MEM_NTYPES # HDF5 uses a clever scheme wherein these are actually init() calls # Hopefully Cython won't have a problem with this. # Thankfully they are defined but -1 if unavailable hid_t H5FD_CORE hid_t H5FD_FAMILY # hid_t H5FD_GASS not in 1.8.X hid_t H5FD_LOG hid_t H5FD_MPIO hid_t H5FD_MULTI hid_t H5FD_SEC2 hid_t H5FD_STDIO int H5FD_LOG_LOC_READ # 0x0001 int H5FD_LOG_LOC_WRITE # 0x0002 int H5FD_LOG_LOC_SEEK # 0x0004 int H5FD_LOG_LOC_IO # (H5FD_LOG_LOC_READ|H5FD_LOG_LOC_WRITE|H5FD_LOG_LOC_SEEK) # Flags for tracking number of times each byte is read/written int H5FD_LOG_FILE_READ # 0x0008 int H5FD_LOG_FILE_WRITE # 0x0010 int H5FD_LOG_FILE_IO # (H5FD_LOG_FILE_READ|H5FD_LOG_FILE_WRITE) # Flag for tracking "flavor" (type) of information stored at each byte int H5FD_LOG_FLAVOR # 0x0020 # Flags for tracking total number of reads/writes/seeks int H5FD_LOG_NUM_READ # 0x0040 int H5FD_LOG_NUM_WRITE # 0x0080 int H5FD_LOG_NUM_SEEK # 0x0100 int H5FD_LOG_NUM_IO # (H5FD_LOG_NUM_READ|H5FD_LOG_NUM_WRITE|H5FD_LOG_NUM_SEEK) # Flags for tracking time spent in open/read/write/seek/close int H5FD_LOG_TIME_OPEN # 0x0200 # Not implemented yet int H5FD_LOG_TIME_READ # 0x0400 # Not implemented yet int H5FD_LOG_TIME_WRITE # 0x0800 # Partially implemented (need to track total time) int H5FD_LOG_TIME_SEEK # 0x1000 # Partially implemented (need to track total time & track time for seeks during reading) int H5FD_LOG_TIME_CLOSE # 0x2000 # Fully implemented int H5FD_LOG_TIME_IO # (H5FD_LOG_TIME_OPEN|H5FD_LOG_TIME_READ|H5FD_LOG_TIME_WRITE|H5FD_LOG_TIME_SEEK|H5FD_LOG_TIME_CLOSE) # Flag for tracking allocation of space in file int H5FD_LOG_ALLOC # 0x4000 int H5FD_LOG_ALL # (H5FD_LOG_ALLOC|H5FD_LOG_TIME_IO|H5FD_LOG_NUM_IO|H5FD_LOG_FLAVOR|H5FD_LOG_FILE_IO|H5FD_LOG_LOC_IO) IF MPI: ctypedef enum H5FD_mpio_xfer_t: H5FD_MPIO_INDEPENDENT = 0, H5FD_MPIO_COLLECTIVE # === H5G - Groups API ======================================================== ctypedef enum H5G_link_t: H5G_LINK_ERROR = -1, H5G_LINK_HARD = 0, H5G_LINK_SOFT = 1 cdef enum H5G_obj_t: H5G_UNKNOWN = -1, # Unknown object type H5G_LINK, # Object is a symbolic link H5G_GROUP, # Object is a group H5G_DATASET, # Object is a dataset H5G_TYPE, # Object is a named data type ctypedef struct H5G_stat_t: unsigned long fileno[2] unsigned long objno[2] unsigned int nlink H5G_obj_t type # new in HDF5 1.6 time_t mtime size_t linklen #H5O_stat_t ohdr # Object header information. New in HDF5 1.6 ctypedef herr_t (*H5G_iterate_t)(hid_t group, char *name, void* op_data) except 2 ctypedef enum H5G_storage_type_t: H5G_STORAGE_TYPE_UNKNOWN = -1, H5G_STORAGE_TYPE_SYMBOL_TABLE, H5G_STORAGE_TYPE_COMPACT, H5G_STORAGE_TYPE_DENSE ctypedef struct H5G_info_t: H5G_storage_type_t storage_type hsize_t nlinks int64_t max_corder # === H5I - Identifier and reflection interface =============================== ctypedef enum H5I_type_t: H5I_UNINIT = -2, # uninitialized Group H5I_BADID = -1, # invalid Group H5I_FILE = 1, # group ID for File objects H5I_GROUP, # group ID for Group objects H5I_DATATYPE, # group ID for Datatype objects H5I_DATASPACE, # group ID for Dataspace objects H5I_DATASET, # group ID for Dataset objects H5I_ATTR, # group ID for Attribute objects H5I_REFERENCE, # group ID for Reference objects H5I_VFL, # group ID for virtual file layer H5I_GENPROP_CLS, # group ID for generic property list classes H5I_GENPROP_LST, # group ID for generic property lists H5I_ERROR_CLASS, # group ID for error classes H5I_ERROR_MSG, # group ID for error messages H5I_ERROR_STACK, # group ID for error stacks H5I_NTYPES # number of valid groups, MUST BE LAST! # === H5L/H5O - Links interface (1.8.X only) ====================================== # TODO: put both versions in h5t.pxd ctypedef enum H5T_cset_t: H5T_CSET_ERROR = -1, # H5T_CSET_ASCII = 0, # US ASCII H5T_CSET_UTF8 = 1, # UTF-8 Unicode encoding unsigned int H5L_MAX_LINK_NAME_LEN # ((uint32_t) (-1)) (4GB - 1) # Link class types. # * Values less than 64 are reserved for the HDF5 library's internal use. # * Values 64 to 255 are for "user-defined" link class types; these types are # * defined by HDF5 but their behavior can be overridden by users. # * Users who want to create new classes of links should contact the HDF5 # * development team at hdfhelp@ncsa.uiuc.edu . # * These values can never change because they appear in HDF5 files. # ctypedef enum H5L_type_t: H5L_TYPE_ERROR = (-1), # Invalid link type id H5L_TYPE_HARD = 0, # Hard link id H5L_TYPE_SOFT = 1, # Soft link id H5L_TYPE_EXTERNAL = 64, # External link id H5L_TYPE_MAX = 255 # Maximum link type id # Information struct for link (for H5Lget_info/H5Lget_info_by_idx) cdef union _add_u: haddr_t address # Address hard link points to size_t val_size # Size of a soft link or UD link value ctypedef struct H5L_info_t: H5L_type_t type # Type of link hbool_t corder_valid # Indicate if creation order is valid int64_t corder # Creation order H5T_cset_t cset # Character set of link name _add_u u # Prototype for H5Literate/H5Literate_by_name() operator ctypedef herr_t (*H5L_iterate_t) (hid_t group, char *name, H5L_info_t *info, void *op_data) except 2 ctypedef uint32_t H5O_msg_crt_idx_t ctypedef enum H5O_type_t: H5O_TYPE_UNKNOWN = -1, # Unknown object type H5O_TYPE_GROUP, # Object is a group H5O_TYPE_DATASET, # Object is a dataset H5O_TYPE_NAMED_DATATYPE, # Object is a named data type H5O_TYPE_NTYPES # Number of different object types (must be last!) unsigned int H5O_COPY_SHALLOW_HIERARCHY_FLAG # (0x0001u) Copy only immediate members unsigned int H5O_COPY_EXPAND_SOFT_LINK_FLAG # (0x0002u) Expand soft links into new objects unsigned int H5O_COPY_EXPAND_EXT_LINK_FLAG # (0x0004u) Expand external links into new objects unsigned int H5O_COPY_EXPAND_REFERENCE_FLAG # (0x0008u) Copy objects that are pointed by references unsigned int H5O_COPY_WITHOUT_ATTR_FLAG # (0x0010u) Copy object without copying attributes unsigned int H5O_COPY_PRESERVE_NULL_FLAG # (0x0020u) Copy NULL messages (empty space) unsigned int H5O_COPY_ALL # (0x003Fu) All object copying flags (for internal checking) # --- Components for the H5O_info_t struct ---------------------------------- ctypedef struct space: hsize_t total # Total space for storing object header in file hsize_t meta # Space within header for object header metadata information hsize_t mesg # Space within header for actual message information hsize_t free # Free space within object header ctypedef struct mesg: unsigned long present # Flags to indicate presence of message type in header unsigned long shared # Flags to indicate message type is shared in header ctypedef struct hdr: unsigned version # Version number of header format in file unsigned nmesgs # Number of object header messages unsigned nchunks # Number of object header chunks unsigned flags # Object header status flags space space mesg mesg ctypedef struct H5_ih_info_t: hsize_t index_size, # btree and/or list hsize_t heap_size cdef struct meta_size: H5_ih_info_t obj, # v1/v2 B-tree & local/fractal heap for groups, B-tree for chunked datasets H5_ih_info_t attr # v2 B-tree & heap for attributes ctypedef struct H5O_info_t: unsigned long fileno # File number that object is located in haddr_t addr # Object address in file H5O_type_t type # Basic object type (group, dataset, etc.) unsigned rc # Reference count of object time_t atime # Access time time_t mtime # Modification time time_t ctime # Change time time_t btime # Birth time hsize_t num_attrs # # of attributes attached to object hdr hdr meta_size meta_size ctypedef herr_t (*H5O_iterate_t)(hid_t obj, char *name, H5O_info_t *info, void *op_data) except 2 # === H5P - Property list API ================================================= int H5P_DEFAULT ctypedef int H5Z_filter_t # HDF5 layouts ctypedef enum H5D_layout_t: H5D_LAYOUT_ERROR = -1, H5D_COMPACT = 0, # raw data is very small H5D_CONTIGUOUS = 1, # the default H5D_CHUNKED = 2, # slow and fancy H5D_NLAYOUTS = 3 # this one must be last! ctypedef enum H5D_alloc_time_t: H5D_ALLOC_TIME_ERROR =-1, H5D_ALLOC_TIME_DEFAULT =0, H5D_ALLOC_TIME_EARLY =1, H5D_ALLOC_TIME_LATE =2, H5D_ALLOC_TIME_INCR =3 ctypedef enum H5D_space_status_t: H5D_SPACE_STATUS_ERROR =-1, H5D_SPACE_STATUS_NOT_ALLOCATED =0, H5D_SPACE_STATUS_PART_ALLOCATED =1, H5D_SPACE_STATUS_ALLOCATED =2 ctypedef enum H5D_fill_time_t: H5D_FILL_TIME_ERROR =-1, H5D_FILL_TIME_ALLOC =0, H5D_FILL_TIME_NEVER =1, H5D_FILL_TIME_IFSET =2 ctypedef enum H5D_fill_value_t: H5D_FILL_VALUE_ERROR =-1, H5D_FILL_VALUE_UNDEFINED =0, H5D_FILL_VALUE_DEFAULT =1, H5D_FILL_VALUE_USER_DEFINED =2 cdef enum H5Z_EDC_t: H5Z_ERROR_EDC = -1, H5Z_DISABLE_EDC = 0, H5Z_ENABLE_EDC = 1, H5Z_NO_EDC = 2 cdef enum H5F_close_degree_t: H5F_CLOSE_WEAK = 0, H5F_CLOSE_SEMI = 1, H5F_CLOSE_STRONG = 2, H5F_CLOSE_DEFAULT = 3 ctypedef enum H5FD_mem_t: H5FD_MEM_NOLIST = -1, H5FD_MEM_DEFAULT = 0, H5FD_MEM_SUPER = 1, H5FD_MEM_BTREE = 2, H5FD_MEM_DRAW = 3, H5FD_MEM_GHEAP = 4, H5FD_MEM_LHEAP = 5, H5FD_MEM_OHDR = 6, H5FD_MEM_NTYPES # Property list classes hid_t H5P_NO_CLASS hid_t H5P_FILE_CREATE hid_t H5P_FILE_ACCESS hid_t H5P_DATASET_CREATE hid_t H5P_DATASET_ACCESS hid_t H5P_DATASET_XFER hid_t H5P_OBJECT_CREATE hid_t H5P_OBJECT_COPY hid_t H5P_LINK_CREATE hid_t H5P_LINK_ACCESS hid_t H5P_GROUP_CREATE hid_t H5P_CRT_ORDER_TRACKED hid_t H5P_CRT_ORDER_INDEXED # === H5R - Reference API ===================================================== size_t H5R_DSET_REG_REF_BUF_SIZE size_t H5R_OBJ_REF_BUF_SIZE ctypedef enum H5R_type_t: H5R_BADTYPE = (-1), H5R_OBJECT, H5R_DATASET_REGION, H5R_INTERNAL, H5R_MAXTYPE # === H5S - Dataspaces ======================================================== int H5S_ALL, H5S_MAX_RANK hsize_t H5S_UNLIMITED # Codes for defining selections ctypedef enum H5S_seloper_t: H5S_SELECT_NOOP = -1, H5S_SELECT_SET = 0, H5S_SELECT_OR, H5S_SELECT_AND, H5S_SELECT_XOR, H5S_SELECT_NOTB, H5S_SELECT_NOTA, H5S_SELECT_APPEND, H5S_SELECT_PREPEND, H5S_SELECT_INVALID # Must be the last one ctypedef enum H5S_class_t: H5S_NO_CLASS = -1, #/*error H5S_SCALAR = 0, #/*scalar variable H5S_SIMPLE = 1, #/*simple data space H5S_NULL = 2, # NULL data space # no longer defined in 1.8 #H5S_COMPLEX = 2 #/*complex data space ctypedef enum H5S_sel_type: H5S_SEL_ERROR = -1, #Error H5S_SEL_NONE = 0, #Nothing selected H5S_SEL_POINTS = 1, #Sequence of points selected H5S_SEL_HYPERSLABS = 2, #"New-style" hyperslab selection defined H5S_SEL_ALL = 3, #Entire extent selected H5S_SEL_N = 4 #/*THIS MUST BE LAST # === H5T - Datatypes ========================================================= # --- Enumerated constants -------------------------------------------------- # Byte orders ctypedef enum H5T_order_t: H5T_ORDER_ERROR = -1, # error H5T_ORDER_LE = 0, # little endian H5T_ORDER_BE = 1, # bit endian H5T_ORDER_VAX = 2, # VAX mixed endian H5T_ORDER_NONE = 3 # no particular order (strings, bits,..) # HDF5 signed enums ctypedef enum H5T_sign_t: H5T_SGN_ERROR = -1, # error H5T_SGN_NONE = 0, # this is an unsigned type H5T_SGN_2 = 1, # two's complement H5T_NSGN = 2 # this must be last! ctypedef enum H5T_norm_t: H5T_NORM_ERROR = -1, H5T_NORM_IMPLIED = 0, H5T_NORM_MSBSET = 1, H5T_NORM_NONE = 2 ctypedef enum H5T_cset_t: H5T_CSET_ERROR = -1, H5T_CSET_ASCII = 0 ctypedef enum H5T_str_t: H5T_STR_ERROR = -1, H5T_STR_NULLTERM = 0, H5T_STR_NULLPAD = 1, H5T_STR_SPACEPAD = 2 # Atomic datatype padding ctypedef enum H5T_pad_t: H5T_PAD_ZERO = 0, H5T_PAD_ONE = 1, H5T_PAD_BACKGROUND = 2 # HDF5 type classes cdef enum H5T_class_t: H5T_NO_CLASS = -1, # error H5T_INTEGER = 0, # integer types H5T_FLOAT = 1, # floating-point types H5T_TIME = 2, # date and time types H5T_STRING = 3, # character string types H5T_BITFIELD = 4, # bit field types H5T_OPAQUE = 5, # opaque types H5T_COMPOUND = 6, # compound types H5T_REFERENCE = 7, # reference types H5T_ENUM = 8, # enumeration types H5T_VLEN = 9, # variable-length types H5T_ARRAY = 10, # array types H5T_NCLASSES # this must be last # Native search direction cdef enum H5T_direction_t: H5T_DIR_DEFAULT, H5T_DIR_ASCEND, H5T_DIR_DESCEND # For vlen strings cdef size_t H5T_VARIABLE # --- Predefined datatypes -------------------------------------------------- cdef hid_t H5T_NATIVE_B8 cdef hid_t H5T_NATIVE_CHAR cdef hid_t H5T_NATIVE_SCHAR cdef hid_t H5T_NATIVE_UCHAR cdef hid_t H5T_NATIVE_SHORT cdef hid_t H5T_NATIVE_USHORT cdef hid_t H5T_NATIVE_INT cdef hid_t H5T_NATIVE_UINT cdef hid_t H5T_NATIVE_LONG cdef hid_t H5T_NATIVE_ULONG cdef hid_t H5T_NATIVE_LLONG cdef hid_t H5T_NATIVE_ULLONG cdef hid_t H5T_NATIVE_FLOAT cdef hid_t H5T_NATIVE_DOUBLE cdef hid_t H5T_NATIVE_LDOUBLE # "Standard" types cdef hid_t H5T_STD_I8LE cdef hid_t H5T_STD_I16LE cdef hid_t H5T_STD_I32LE cdef hid_t H5T_STD_I64LE cdef hid_t H5T_STD_U8LE cdef hid_t H5T_STD_U16LE cdef hid_t H5T_STD_U32LE cdef hid_t H5T_STD_U64LE cdef hid_t H5T_STD_B8LE cdef hid_t H5T_STD_B16LE cdef hid_t H5T_STD_B32LE cdef hid_t H5T_STD_B64LE cdef hid_t H5T_IEEE_F32LE cdef hid_t H5T_IEEE_F64LE cdef hid_t H5T_STD_I8BE cdef hid_t H5T_STD_I16BE cdef hid_t H5T_STD_I32BE cdef hid_t H5T_STD_I64BE cdef hid_t H5T_STD_U8BE cdef hid_t H5T_STD_U16BE cdef hid_t H5T_STD_U32BE cdef hid_t H5T_STD_U64BE cdef hid_t H5T_STD_B8BE cdef hid_t H5T_STD_B16BE cdef hid_t H5T_STD_B32BE cdef hid_t H5T_STD_B64BE cdef hid_t H5T_IEEE_F32BE cdef hid_t H5T_IEEE_F64BE cdef hid_t H5T_NATIVE_INT8 cdef hid_t H5T_NATIVE_UINT8 cdef hid_t H5T_NATIVE_INT16 cdef hid_t H5T_NATIVE_UINT16 cdef hid_t H5T_NATIVE_INT32 cdef hid_t H5T_NATIVE_UINT32 cdef hid_t H5T_NATIVE_INT64 cdef hid_t H5T_NATIVE_UINT64 # Unix time types cdef hid_t H5T_UNIX_D32LE cdef hid_t H5T_UNIX_D64LE cdef hid_t H5T_UNIX_D32BE cdef hid_t H5T_UNIX_D64BE # String types cdef hid_t H5T_FORTRAN_S1 cdef hid_t H5T_C_S1 # References cdef hid_t H5T_STD_REF_OBJ cdef hid_t H5T_STD_REF_DSETREG # Type-conversion infrastructure ctypedef enum H5T_pers_t: H5T_PERS_DONTCARE = -1, H5T_PERS_HARD = 0, # /*hard conversion function */ H5T_PERS_SOFT = 1 # /*soft conversion function */ ctypedef enum H5T_cmd_t: H5T_CONV_INIT = 0, #/*query and/or initialize private data */ H5T_CONV_CONV = 1, #/*convert data from source to dest datatype */ H5T_CONV_FREE = 2 #/*function is being removed from path */ ctypedef enum H5T_bkg_t: H5T_BKG_NO = 0, #/*background buffer is not needed, send NULL */ H5T_BKG_TEMP = 1, #/*bkg buffer used as temp storage only */ H5T_BKG_YES = 2 #/*init bkg buf with data before conversion */ ctypedef struct H5T_cdata_t: H5T_cmd_t command # /*what should the conversion function do? */ H5T_bkg_t need_bkg #/*is the background buffer needed? */ hbool_t recalc # /*recalculate private data */ void *priv # /*private data */ ctypedef struct hvl_t: size_t len # /* Length of VL data (in base type units) */ void *p #/* Pointer to VL data */ ctypedef herr_t (*H5T_conv_t)(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nelmts, size_t buf_stride, size_t bkg_stride, void *buf, void *bkg, hid_t dset_xfer_plist) except -1 # === H5Z - Filters =========================================================== ctypedef int H5Z_filter_t int H5Z_FILTER_ERROR int H5Z_FILTER_NONE int H5Z_FILTER_ALL int H5Z_FILTER_DEFLATE int H5Z_FILTER_SHUFFLE int H5Z_FILTER_FLETCHER32 int H5Z_FILTER_SZIP int H5Z_FILTER_NBIT int H5Z_FILTER_SCALEOFFSET int H5Z_FILTER_RESERVED int H5Z_FILTER_MAX int H5Z_MAX_NFILTERS int H5Z_FLAG_DEFMASK int H5Z_FLAG_MANDATORY int H5Z_FLAG_OPTIONAL int H5Z_FLAG_INVMASK int H5Z_FLAG_REVERSE int H5Z_FLAG_SKIP_EDC int H5_SZIP_ALLOW_K13_OPTION_MASK #1 int H5_SZIP_CHIP_OPTION_MASK #2 int H5_SZIP_EC_OPTION_MASK #4 int H5_SZIP_NN_OPTION_MASK #32 int H5_SZIP_MAX_PIXELS_PER_BLOCK #32 int H5Z_SO_INT_MINBITS_DEFAULT int H5Z_FILTER_CONFIG_ENCODE_ENABLED #(0x0001) int H5Z_FILTER_CONFIG_DECODE_ENABLED #(0x0002) cdef enum H5Z_EDC_t: H5Z_ERROR_EDC = -1, H5Z_DISABLE_EDC = 0, H5Z_ENABLE_EDC = 1, H5Z_NO_EDC = 2 cdef enum H5Z_SO_scale_type_t: H5Z_SO_FLOAT_DSCALE = 0, H5Z_SO_FLOAT_ESCALE = 1, H5Z_SO_INT = 2 # === H5A - Attributes API ==================================================== ctypedef herr_t (*H5A_operator_t)(hid_t loc_id, char *attr_name, void* operator_data) except 2 ctypedef struct H5A_info_t: hbool_t corder_valid # Indicate if creation order is valid H5O_msg_crt_idx_t corder # Creation order H5T_cset_t cset # Character set of attribute name hsize_t data_size # Size of raw data ctypedef herr_t (*H5A_operator2_t)(hid_t location_id, char *attr_name, H5A_info_t *ainfo, void *op_data) except 2 # === H5AC - Attribute Cache configuration API ================================ unsigned int H5AC__CURR_CACHE_CONFIG_VERSION # 1 # I don't really understand why this works, but # https://groups.google.com/forum/?fromgroups#!topic/cython-users/-fLG08E5lYM # suggests it and it _does_ work enum: H5AC__MAX_TRACE_FILE_NAME_LEN # 1024 unsigned int H5AC_METADATA_WRITE_STRATEGY__PROCESS_0_ONLY # 0 unsigned int H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED # 1 cdef extern from "H5Cpublic.h": # === H5C - Cache configuration API ================================ cdef enum H5C_cache_incr_mode: H5C_incr__off, H5C_incr__threshold, cdef enum H5C_cache_flash_incr_mode: H5C_flash_incr__off, H5C_flash_incr__add_space cdef enum H5C_cache_decr_mode: H5C_decr__off, H5C_decr__threshold, H5C_decr__age_out, H5C_decr__age_out_with_threshold ctypedef struct H5AC_cache_config_t: # /* general configuration fields: */ int version hbool_t rpt_fcn_enabled hbool_t evictions_enabled hbool_t set_initial_size size_t initial_size double min_clean_fraction size_t max_size size_t min_size long int epoch_length # /* size increase control fields: */ H5C_cache_incr_mode incr_mode double lower_hr_threshold double increment hbool_t apply_max_increment size_t max_increment H5C_cache_flash_incr_mode flash_incr_mode double flash_multiple double flash_threshold # /* size decrease control fields: */ H5C_cache_decr_mode decr_mode double upper_hr_threshold double decrement hbool_t apply_max_decrement size_t max_decrement int epochs_before_eviction hbool_t apply_empty_reserve double empty_reserve # /* parallel configuration fields: */ int dirty_bytes_threshold # int metadata_write_strategy # present in 1.8.6 and higher cdef extern from "hdf5_hl.h": # === H5DS - Dimension Scales API ============================================= ctypedef herr_t (*H5DS_iterate_t)(hid_t dset, unsigned dim, hid_t scale, void *visitor_data) except 2 h5py-2.7.1/h5py/_proxy.pxd0000644000175000017500000000077713025343121017013 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. from defs cimport * cdef herr_t attr_rw(hid_t attr, hid_t mtype, void *progbuf, int read) except -1 cdef herr_t dset_rw(hid_t dset, hid_t mtype, hid_t mspace, hid_t fspace, hid_t dxpl, void* progbuf, int read) except -1 h5py-2.7.1/h5py/h5r.pxd0000644000175000017500000000116713025343121016163 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. from defs cimport * cdef extern from "hdf5.h": ctypedef haddr_t hobj_ref_t ctypedef unsigned char hdset_reg_ref_t[12] cdef union ref_u: hobj_ref_t obj_ref hdset_reg_ref_t reg_ref cdef class Reference: cdef ref_u ref cdef readonly int typecode cdef readonly size_t typesize cdef class RegionReference(Reference): pass h5py-2.7.1/h5py/_locks.pxi0000644000175000017500000001073413025343121016744 0ustar tcaswelltcaswell00000000000000cdef class BogoLock: def __enter__(self): pass def __exit__(self, *args): pass ## {{{ http://code.activestate.com/recipes/577336/ (r3) from cpython cimport pythread from cpython.exc cimport PyErr_NoMemory cdef class FastRLock: """Fast, re-entrant locking. Under uncongested conditions, the lock is never acquired but only counted. Only when a second thread comes in and notices that the lock is needed, it acquires the lock and notifies the first thread to release it when it's done. This is all made possible by the wonderful GIL. """ cdef pythread.PyThread_type_lock _real_lock cdef long _owner # ID of thread owning the lock cdef int _count # re-entry count cdef int _pending_requests # number of pending requests for real lock cdef bint _is_locked # whether the real lock is acquired def __cinit__(self): self._owner = -1 self._count = 0 self._is_locked = False self._pending_requests = 0 self._real_lock = pythread.PyThread_allocate_lock() if self._real_lock is NULL: PyErr_NoMemory() def __dealloc__(self): if self._real_lock is not NULL: pythread.PyThread_free_lock(self._real_lock) self._real_lock = NULL def acquire(self, bint blocking=True): return lock_lock(self, pythread.PyThread_get_thread_ident(), blocking) def release(self): if self._owner != pythread.PyThread_get_thread_ident(): raise RuntimeError("cannot release un-acquired lock") unlock_lock(self) # compatibility with threading.RLock def __enter__(self): # self.acquire() return lock_lock(self, pythread.PyThread_get_thread_ident(), True) def __exit__(self, t, v, tb): # self.release() if self._owner != pythread.PyThread_get_thread_ident(): raise RuntimeError("cannot release un-acquired lock") unlock_lock(self) def _is_owned(self): return self._owner == pythread.PyThread_get_thread_ident() cdef inline bint lock_lock(FastRLock lock, long current_thread, bint blocking) nogil: # Note that this function *must* hold the GIL when being called. # We just use 'nogil' in the signature to make sure that no Python # code execution slips in that might free the GIL if lock._count: # locked! - by myself? if current_thread == lock._owner: lock._count += 1 return 1 elif not lock._pending_requests: # not locked, not requested - go! lock._owner = current_thread lock._count = 1 return 1 # need to get the real lock return _acquire_lock( lock, current_thread, pythread.WAIT_LOCK if blocking else pythread.NOWAIT_LOCK) cdef bint _acquire_lock(FastRLock lock, long current_thread, int wait) nogil: # Note that this function *must* hold the GIL when being called. # We just use 'nogil' in the signature to make sure that no Python # code execution slips in that might free the GIL if not lock._is_locked and not lock._pending_requests: # someone owns it but didn't acquire the real lock - do that # now and tell the owner to release it when done. Note that we # do not release the GIL here as we must absolutely be the one # who acquires the lock now. if not pythread.PyThread_acquire_lock(lock._real_lock, wait): return 0 #assert not lock._is_locked lock._is_locked = True lock._pending_requests += 1 with nogil: # wait for the lock owning thread to release it locked = pythread.PyThread_acquire_lock(lock._real_lock, wait) lock._pending_requests -= 1 #assert not lock._is_locked #assert lock._count == 0 if not locked: return 0 lock._is_locked = True lock._owner = current_thread lock._count = 1 return 1 cdef inline void unlock_lock(FastRLock lock) nogil: # Note that this function *must* hold the GIL when being called. # We just use 'nogil' in the signature to make sure that no Python # code execution slips in that might free the GIL #assert lock._owner == pythread.PyThread_get_thread_ident() #assert lock._count > 0 lock._count -= 1 if lock._count == 0: lock._owner = -1 if lock._is_locked: pythread.PyThread_release_lock(lock._real_lock) lock._is_locked = False ## end of http://code.activestate.com/recipes/577336/ }}} h5py-2.7.1/h5py/numpy.pxd0000644000175000017500000000552413025343121016636 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. # This file is based on code from the PyTables project. The complete PyTables # license is available at licenses/pytables.txt, in the distribution root # directory. # API for NumPy objects cdef extern from "numpy/arrayobject.h": # Platform independent types ctypedef int npy_intp ctypedef signed int npy_int8 ctypedef unsigned int npy_uint8 ctypedef signed int npy_int16 ctypedef unsigned int npy_uint16 ctypedef signed int npy_int32 ctypedef unsigned int npy_uint32 ctypedef signed long long npy_int64 ctypedef unsigned long long npy_uint64 ctypedef float npy_float32 ctypedef double npy_float64 cdef enum NPY_TYPES: NPY_BOOL NPY_BYTE NPY_UBYTE NPY_SHORT NPY_USHORT NPY_INT NPY_UINT NPY_LONG NPY_ULONG NPY_LONGLONG NPY_ULONGLONG NPY_FLOAT NPY_DOUBLE NPY_LONGDOUBLE NPY_CFLOAT NPY_CDOUBLE NPY_CLONGDOUBLE NPY_OBJECT NPY_STRING NPY_UNICODE NPY_VOID NPY_NTYPES NPY_NOTYPE # Platform independent types cdef enum: NPY_INT8, NPY_INT16, NPY_INT32, NPY_INT64, NPY_UINT8, NPY_UINT16, NPY_UINT32, NPY_UINT64, NPY_FLOAT32, NPY_FLOAT64, NPY_COMPLEX64, NPY_COMPLEX128 cdef enum: NPY_WRITEABLE, NPY_ALIGNED, NPY_C_CONTIGUOUS, NPY_CONTIGUOUS, NPY_FORCECAST, NPY_NOTSWAPPED, NPY_OWNDATA # Classes ctypedef extern class numpy.dtype [object PyArray_Descr]: cdef int type_num, elsize, alignment cdef char type, kind, byteorder, hasobject cdef object fields, typeobj ctypedef extern class numpy.ndarray [object PyArrayObject]: cdef char *data cdef int nd cdef npy_intp *dimensions cdef npy_intp *strides cdef object base cdef dtype descr cdef int flags ctypedef struct npy_cfloat: float real float imag ctypedef struct npy_cdouble: double real double imag # Functions int PyArray_DIM(ndarray arr, int i) object PyArray_FROM_OF(object arr, int requirements) object PyArray_GETITEM(object arr, void *itemptr) int PyArray_SETITEM(object arr, void *itemptr, object obj) dtype PyArray_DescrFromType(int type) object PyArray_Scalar(void *data, dtype descr, object base) long PyArray_NBYTES(object arr) int PyArray_CheckScalar(object sclr) void PyArray_ScalarAsCtype(object sclr, void* ptr) object PyArray_SimpleNew(int nd, npy_intp* dims, int typenum) object PyArray_ContiguousFromAny(object arr, int typenum, int min_depth, int max_depth) object PyArray_FROM_OTF(object arr, int typenum, int requirements) # The NumPy initialization function void import_array() void* PyArray_DATA(ndarray arr) h5py-2.7.1/h5py/h5a.pxd0000644000175000017500000000055013025343121016135 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. from defs cimport * from _objects cimport ObjectID cdef class AttrID(ObjectID): pass h5py-2.7.1/h5py/h5p.pyx0000644000175000017500000012236313143100476016215 0ustar tcaswelltcaswell00000000000000# This file is part of h5py, a Python interface to the HDF5 library. # # http://www.h5py.org # # Copyright 2008-2013 Andrew Collette and contributors # # License: Standard 3-clause BSD; see "license.txt" for full license terms # and contributor agreement. """ HDF5 property list interface. """ include "config.pxi" # Compile-time imports from cpython.buffer cimport PyObject_CheckBuffer, \ PyObject_GetBuffer, PyBuffer_Release, \ PyBUF_SIMPLE from utils cimport require_tuple, convert_dims, convert_tuple, \ emalloc, efree, \ check_numpy_write, check_numpy_read from numpy cimport ndarray, import_array from h5t cimport TypeID, py_create from h5s cimport SpaceID from h5ac cimport CacheConfig from h5py import _objects from ._objects import phil, with_phil if MPI: if MPI4PY_V2: from mpi4py.libmpi cimport MPI_Comm, MPI_Info, MPI_Comm_dup, MPI_Info_dup, \ MPI_Comm_free, MPI_Info_free else: from mpi4py.mpi_c cimport MPI_Comm, MPI_Info, MPI_Comm_dup, MPI_Info_dup, \ MPI_Comm_free, MPI_Info_free # Initialization import_array() # === C API =================================================================== cdef hid_t pdefault(PropID pid): if pid is None: return H5P_DEFAULT return pid.id cdef object propwrap(hid_t id_in): clsid = H5Pget_class(id_in) try: if H5Pequal(clsid, H5P_FILE_CREATE): pcls = PropFCID elif H5Pequal(clsid, H5P_FILE_ACCESS): pcls = PropFAID elif H5Pequal(clsid, H5P_DATASET_CREATE): pcls = PropDCID elif H5Pequal(clsid, H5P_DATASET_XFER): pcls = PropDXID elif H5Pequal(clsid, H5P_OBJECT_COPY): pcls = PropCopyID elif H5Pequal(clsid, H5P_LINK_CREATE): pcls = PropLCID elif H5Pequal(clsid, H5P_LINK_ACCESS): pcls = PropLAID elif H5Pequal(clsid, H5P_GROUP_CREATE): pcls = PropGCID elif H5Pequal(clsid, H5P_DATASET_ACCESS): pcls = PropDAID elif H5Pequal(clsid, H5P_OBJECT_CREATE): pcls = PropOCID else: raise ValueError("No class found for ID %d" % id_in) return pcls(id_in) finally: H5Pclose_class(clsid) cdef object lockcls(hid_t id_in): cdef PropClassID pid pid = PropClassID(id_in) pid.locked = 1 return pid # === Public constants and data structures ==================================== # Property list classes # These need to be locked, as the library won't let you close them. NO_CLASS = lockcls(H5P_NO_CLASS) FILE_CREATE = lockcls(H5P_FILE_CREATE) FILE_ACCESS = lockcls(H5P_FILE_ACCESS) DATASET_CREATE = lockcls(H5P_DATASET_CREATE) DATASET_XFER = lockcls(H5P_DATASET_XFER) DATASET_ACCESS = lockcls(H5P_DATASET_ACCESS) OBJECT_COPY = lockcls(H5P_OBJECT_COPY) LINK_CREATE = lockcls(H5P_LINK_CREATE) LINK_ACCESS = lockcls(H5P_LINK_ACCESS) GROUP_CREATE = lockcls(H5P_GROUP_CREATE) OBJECT_CREATE = lockcls(H5P_OBJECT_CREATE) CRT_ORDER_TRACKED = H5P_CRT_ORDER_TRACKED CRT_ORDER_INDEXED = H5P_CRT_ORDER_INDEXED DEFAULT = None # In the HDF5 header files this is actually 0, which is an # invalid identifier. The new strategy for default options # is to make them all None, to better match the Python style # for keyword arguments. # === Property list functional API ============================================ @with_phil def create(PropClassID cls not None): """(PropClassID cls) => PropID Create a new property list as an instance of a class; classes are: - FILE_CREATE - FILE_ACCESS - DATASET_CREATE - DATASET_XFER - DATASET_ACCESS - LINK_CREATE - LINK_ACCESS - GROUP_CREATE - OBJECT_COPY - OBJECT_CREATE """ cdef hid_t newid newid = H5Pcreate(cls.id) return propwrap(newid) # === Class API =============================================================== cdef class PropID(ObjectID): """ Base class for all property lists and classes """ @with_phil def equal(self, PropID plist not None): """(PropID plist) => BOOL Compare this property list (or class) to another for equality. """ return (H5Pequal(self.id, plist.id)) def __richcmp__(self, object other, int how): cdef bint truthval = 0 with phil: if how != 2 and how != 3: return NotImplemented if type(self) == type(other): truthval = self.equal(other) if how == 2: return truthval return not truthval def __hash__(self): raise TypeError("Property lists are unhashable") cdef class PropClassID(PropID): """ An HDF5 property list class. * Hashable: Yes, by identifier * Equality: Logical H5P comparison """ def __richcmp__(self, object other, int how): return PropID.__richcmp__(self, other, how) def __hash__(self): """ Since classes are library-created and immutable, they are uniquely identified by their HDF5 identifiers. """ return hash(self.id) cdef class PropInstanceID(PropID): """ Base class for property list instance objects. Provides methods which are common across all HDF5 property list classes. * Hashable: No * Equality: Logical H5P comparison """ @with_phil def copy(self): """() => PropList newid Create a new copy of an existing property list object. """ return type(self)(H5Pcopy(self.id)) def get_class(self): """() => PropClassID Determine the class of a property list object. """ return PropClassID(H5Pget_class(self.id)) cdef class PropCreateID(PropInstanceID): """ Generic object creation property list. """ pass cdef class PropCopyID(PropInstanceID): """ Generic object copy property list """ @with_phil def set_copy_object(self, unsigned int flags): """(UINT flags) Set flags for object copying process. Legal flags are from the h5o.COPY* family: h5o.COPY_SHALLOW_HIERARCHY_FLAG Copy only immediate members of a group. h5o.COPY_EXPAND_SOFT_LINK_FLAG Expand soft links into new objects. h5o.COPY_EXPAND_EXT_LINK_FLAG Expand external link into new objects. h5o.COPY_EXPAND_REFERENCE_FLAG Copy objects that are pointed to by references. h5o.COPY_WITHOUT_ATTR_FLAG Copy object without copying attributes. """ H5Pset_copy_object(self.id, flags) @with_phil def get_copy_object(self): """() => UINT flags Get copy process flags. Legal flags are h5o.COPY*. """ cdef unsigned int flags H5Pget_copy_object(self.id, &flags) return flags # === Concrete list implementations =========================================== # File creation cdef class PropFCID(PropCreateID): """ File creation property list. """ @with_phil def get_version(self): """() => TUPLE version_info Determine version information of various file attributes. Elements are: 0. UINT Super block version number 1. UINT Freelist version number 2. UINT Symbol table version number 3. UINT Shared object header version number """ cdef herr_t retval cdef unsigned int super_ cdef unsigned int freelist cdef unsigned int stab cdef unsigned int shhdr H5Pget_version(self.id, &super_, &freelist, &stab, &shhdr) return (super_, freelist, stab, shhdr) @with_phil def set_userblock(self, hsize_t size): """(INT/LONG size) Set the file user block size, in bytes. Must be a power of 2, and at least 512. """ H5Pset_userblock(self.id, size) @with_phil def get_userblock(self): """() => LONG size Determine the user block size, in bytes. """ cdef hsize_t size H5Pget_userblock(self.id, &size) return size @with_phil def set_sizes(self, size_t addr, size_t size): """(UINT addr, UINT size) Set the addressing offsets and lengths for objects in an HDF5 file, in bytes. """ H5Pset_sizes(self.id, addr, size) @with_phil def get_sizes(self): """() => TUPLE sizes Determine addressing offsets and lengths for objects in an HDF5 file, in bytes. Return value is a 2-tuple with values: 0. UINT Address offsets 1. UINT Lengths """ cdef size_t addr cdef size_t size H5Pget_sizes(self.id, &addr, &size) return (addr, size) @with_phil def set_link_creation_order(self, unsigned int flags): """ (UINT flags) Set tracking and indexing of creation order for links added to this group flags -- h5p.CRT_ORDER_TRACKED, h5p.CRT_ORDER_INDEXED """ H5Pset_link_creation_order(self.id, flags) @with_phil def get_link_creation_order(self): """ () -> UINT flags Get tracking and indexing of creation order for links added to this group """ cdef unsigned int flags H5Pget_link_creation_order(self.id, &flags) return flags # Dataset creation cdef class PropDCID(PropOCID): """ Dataset creation property list. """ @with_phil def set_layout(self, int layout_code): """(INT layout_code) Set dataset storage strategy; legal values are: - h5d.COMPACT - h5d.CONTIGUOUS - h5d.CHUNKED - h5d.VIRTUAL (If using HDF5 library version 1.10 or later) """ H5Pset_layout(self.id, layout_code) @with_phil def get_layout(self): """() => INT layout_code Determine the storage strategy of a dataset; legal values are: - h5d.COMPACT - h5d.CONTIGUOUS - h5d.CHUNKED - h5d.VIRTUAL (If using HDF5 library version 1.10 or later) """ return H5Pget_layout(self.id) @with_phil def set_chunk(self, object chunksize): """(TUPLE chunksize) Set the dataset chunk size. It's up to you to provide values which are compatible with your dataset. """ cdef int rank cdef hsize_t* dims dims = NULL require_tuple(chunksize, 0, -1, "chunksize") rank = len(chunksize) dims = emalloc(sizeof(hsize_t)*rank) try: convert_tuple(chunksize, dims, rank) H5Pset_chunk(self.id, rank, dims) finally: efree(dims) @with_phil def get_chunk(self): """() => TUPLE chunk_dimensions Obtain the dataset chunk size, as a tuple. """ cdef int rank cdef hsize_t *dims rank = H5Pget_chunk(self.id, 0, NULL) assert rank >= 0 dims = emalloc(sizeof(hsize_t)*rank) try: H5Pget_chunk(self.id, rank, dims) tpl = convert_dims(dims, rank) return tpl finally: efree(dims) @with_phil def set_fill_value(self, ndarray value not None): """(NDARRAY value) Set the dataset fill value. The object provided should be an 0-dimensional NumPy array; otherwise, the value will be read from the first element. """ cdef TypeID tid check_numpy_read(value, -1) tid = py_create(value.dtype) H5Pset_fill_value(self.id, tid.id, value.data) @with_phil def get_fill_value(self, ndarray value not None): """(NDARRAY value) Read the dataset fill value into a NumPy array. It will be converted to match the array dtype. If the array has nonzero rank, only the first element will contain the value. """ cdef TypeID tid check_numpy_write(value, -1) tid = py_create(value.dtype) H5Pget_fill_value(self.id, tid.id, value.data) @with_phil def fill_value_defined(self): """() => INT fill_status Determine the status of the dataset fill value. Return values are: - h5d.FILL_VALUE_UNDEFINED - h5d.FILL_VALUE_DEFAULT - h5d.FILL_VALUE_USER_DEFINED """ cdef H5D_fill_value_t val H5Pfill_value_defined(self.id, &val) return val @with_phil def set_fill_time(self, int fill_time): """(INT fill_time) Define when fill values are written to the dataset. Legal values (defined in module h5d) are: - h5d.FILL_TIME_ALLOC - h5d.FILL_TIME_NEVER - h5d.FILL_TIME_IFSET """ H5Pset_fill_time(self.id, fill_time) @with_phil def get_fill_time(self): """ () => INT Determine when fill values are written to the dataset. Legal values (defined in module h5d) are: - h5d.FILL_TIME_ALLOC - h5d.FILL_TIME_NEVER - h5d.FILL_TIME_IFSET """ cdef H5D_fill_time_t fill_time H5Pget_fill_time(self.id, &fill_time) return fill_time @with_phil def set_alloc_time(self, int alloc_time): """(INT alloc_time) Set the storage space allocation time. One of h5d.ALLOC_TIME*. """ H5Pset_alloc_time(self.id, alloc_time) @with_phil def get_alloc_time(self): """() => INT alloc_time Get the storage space allocation time. One of h5d.ALLOC_TIME*. """ cdef H5D_alloc_time_t alloc_time H5Pget_alloc_time(self.id, &alloc_time) return alloc_time # === Filter functions ==================================================== @with_phil def set_filter(self, int filter_code, unsigned int flags=0, object values=None): """(INT filter_code, UINT flags=0, TUPLE values=None) Set a filter in the pipeline. Params are: filter_code One of the following: - h5z.FILTER_DEFLATE - h5z.FILTER_SHUFFLE - h5z.FILTER_FLETCHER32 - h5z.FILTER_SZIP flags Bit flags (h5z.FLAG*) setting filter properties values TUPLE of UINTs giving auxiliary data for the filter """ cdef size_t nelements cdef unsigned int *cd_values cdef int i cd_values = NULL require_tuple(values, 1, -1, "values") try: if values is None or len(values) == 0: nelements = 0 cd_values = NULL else: nelements = len(values) cd_values = emalloc(sizeof(unsigned int)*nelements) for i from 0<=ifilter_code, flags, nelements, cd_values) finally: efree(cd_values) @with_phil def all_filters_avail(self): """() => BOOL Determine if all the filters in the pipelist are available to the library. """ return (H5Pall_filters_avail(self.id)) @with_phil def get_nfilters(self): """() => INT Determine the number of filters in the pipeline. """ return H5Pget_nfilters(self.id) @with_phil def get_filter(self, int filter_idx): """(UINT filter_idx) => TUPLE filter_info Get information about a filter, identified by its index. Tuple elements are: 0. INT filter code (h5z.FILTER*) 1. UINT flags (h5z.FLAG*) 2. TUPLE of UINT values; filter aux data (16 values max) 3. STRING name of filter (256 chars max) """ cdef list vlist cdef int filter_code cdef unsigned int flags cdef size_t nelements cdef unsigned int cd_values[16] cdef char name[257] cdef int i nelements = 16 # HDF5 library actually complains if this is too big. if filter_idx < 0: raise ValueError("Filter index must be a non-negative integer") filter_code = H5Pget_filter(self.id, filter_idx, &flags, &nelements, cd_values, 256, name) name[256] = c'\0' # in case it's > 256 chars vlist = [] for i from 0<=i TUPLE filter_info or None Get information about a filter, identified by its code (one of h5z.FILTER*). If the filter doesn't exist, returns None. Tuple elements are: 0. UINT flags (h5z.FLAG*) 1. TUPLE of UINT values; filter aux data (16 values max) 2. STRING name of filter (256 chars max) """ cdef list vlist cdef unsigned int flags cdef size_t nelements cdef unsigned int cd_values[16] cdef char name[257] cdef herr_t retval cdef int i nelements = 16 # HDF5 library actually complains if this is too big. if not self._has_filter(filter_code): # Avoid library segfault return None retval = H5Pget_filter_by_id(self.id, filter_code, &flags, &nelements, cd_values, 256, name) assert nelements <= 16 name[256] = c'\0' # In case HDF5 doesn't terminate it properly vlist = [] for i from 0<=ifilter_class) @with_phil def set_deflate(self, unsigned int level=5): """(UINT level=5) Enable deflate (gzip) compression, at the given level. Valid levels are 0-9, default is 5. """ H5Pset_deflate(self.id, level) @with_phil def set_fletcher32(self): """() Enable Fletcher32 error correction on this list. """ H5Pset_fletcher32(self.id) @with_phil def set_shuffle(self): """() Enable to use of the shuffle filter. Use this immediately before the deflate filter to increase the compression ratio. """ H5Pset_shuffle(self.id) @with_phil def set_szip(self, unsigned int options, unsigned int pixels_per_block): """(UINT options, UINT pixels_per_block) Enable SZIP compression. See the HDF5 docs for argument meanings, and general restrictions on use of the SZIP format. """ H5Pset_szip(self.id, options, pixels_per_block) @with_phil def set_scaleoffset(self, H5Z_SO_scale_type_t scale_type, int scale_factor): '''(H5Z_SO_scale_type_t scale_type, INT scale_factor) Enable scale/offset (usually lossy) compression; lossless (e.g. gzip) compression and other filters may be applied on top of this. Note that error detection (i.e. fletcher32) cannot precede this in the filter chain, or else all reads on lossily-compressed data will fail.''' H5Pset_scaleoffset(self.id, scale_type, scale_factor) # === Virtual dataset functions =========================================== IF HDF5_VERSION >= VDS_MIN_HDF5_VERSION: @with_phil def set_virtual(self, SpaceID vspace not None, char* src_file_name, char* src_dset_name, SpaceID src_space not None): """(SpaceID vspace, STR src_file_name, STR src_dset_name, SpaceID src_space) Set the mapping between virtual and source datasets. The virtual dataset is described by its virtual dataspace (vspace) to the elements. The source dataset is described by the name of the file where it is located (src_file_name), the name of the dataset (src_dset_name) and its dataspace (src_space). """ H5Pset_virtual(self.id, vspace.id, src_file_name, src_dset_name, src_space.id) @with_phil def get_virtual_count(self): """() => UINT Get the number of mappings for the virtual dataset. """ cdef size_t count H5Pget_virtual_count(self.id, &count) return count @with_phil def get_virtual_dsetname(self, size_t index=0): """(UINT index=0) => STR Get the name of a source dataset used in the mapping of the virtual dataset at the position index. """ cdef char* name = NULL cdef ssize_t size size = H5Pget_virtual_dsetname(self.id, index, NULL, 0) name = emalloc(size+1) try: H5Pget_virtual_dsetname(self.id, index, name, size+1) src_dset_name = name finally: efree(name) return src_dset_name @with_phil def get_virtual_filename(self, size_t index=0): """(UINT index=0) => STR Get the file name of a source dataset used in the mapping of the virtual dataset at the position index. """ cdef char* name = NULL cdef ssize_t size size = H5Pget_virtual_dsetname(self.id, index, NULL, 0) name = emalloc(size+1) try: H5Pget_virtual_filename(self.id, index, name, size+1) src_fname = name finally: efree(name) return src_fname @with_phil def get_virtual_vspace(self, size_t index=0): """(UINT index=0) => SpaceID Get a dataspace for the selection within the virtual dataset used in the mapping. """ return SpaceID(H5Pget_virtual_vspace(self.id, index)) @with_phil def get_virtual_srcspace(self, size_t index=0): """(UINT index=0) => SpaceID Get a dataspace for the selection within the source dataset used in the mapping. """ return SpaceID(H5Pget_virtual_srcspace(self.id, index)) # File access cdef class PropFAID(PropInstanceID): """ File access property list """ @with_phil def set_fclose_degree(self, int close_degree): """(INT close_degree) Set the file-close degree, which determines library behavior when a file is closed when objects are still open. Legal values: * h5f.CLOSE_WEAK * h5f.CLOSE_SEMI * h5f.CLOSE_STRONG * h5f.CLOSE_DEFAULT """ H5Pset_fclose_degree(self.id, close_degree) @with_phil def get_fclose_degree(self): """() => INT close_degree - h5fd. Get the file-close degree, which determines library behavior when a file is closed when objects are still open. Legal values: * h5f.CLOSE_WEAK * h5f.CLOSE_SEMI * h5f.CLOSE_STRONG * h5f.CLOSE_DEFAULT """ cdef H5F_close_degree_t deg H5Pget_fclose_degree(self.id, °) return deg @with_phil def set_fapl_core(self, size_t block_size=64*1024, hbool_t backing_store=1): """(UINT increment=64k, BOOL backing_store=True) Use the h5fd.CORE (memory-resident) file driver. increment Chunk size for new memory requests (default 1 meg) backing_store If True (default), memory contents are associated with an on-disk file, which is updated when the file is closed. Set to False for a purely in-memory file. """ H5Pset_fapl_core(self.id, block_size, backing_store) @with_phil def get_fapl_core(self): """() => TUPLE core_settings Determine settings for the h5fd.CORE (memory-resident) file driver. Tuple elements are: 0. UINT "increment": Chunk size for new memory requests 1. BOOL "backing_store": If True, write the memory contents to disk when the file is closed. """ cdef size_t increment cdef hbool_t backing_store H5Pget_fapl_core(self.id, &increment, &backing_store) return (increment, (backing_store)) @with_phil def set_fapl_family(self, hsize_t memb_size=2147483647, PropID memb_fapl=None): """(UINT memb_size=2**31-1, PropFAID memb_fapl=None) Set up the family driver. memb_size Member file size memb_fapl File access property list for each member access """ cdef hid_t plist_id plist_id = pdefault(memb_fapl) H5Pset_fapl_family(self.id, memb_size, plist_id) @with_phil def get_fapl_family(self): """() => TUPLE info Determine family driver settings. Tuple values are: 0. UINT memb_size 1. PropFAID memb_fapl or None """ cdef hid_t mfapl_id cdef hsize_t msize cdef PropFAID plist plist = None H5Pget_fapl_family(self.id, &msize, &mfapl_id) if mfapl_id > 0: plist = PropFAID(mfapl_id) return (msize, plist) @with_phil def set_fapl_log(self, char* logfile, unsigned int flags, size_t buf_size): """(STRING logfile, UINT flags, UINT buf_size) Enable the use of the logging driver. See the HDF5 documentation for details. Flag constants are stored in module h5fd. """ H5Pset_fapl_log(self.id, logfile, flags, buf_size) @with_phil def set_fapl_sec2(self): """() Select the "section-2" driver (h5fd.SEC2). """ H5Pset_fapl_sec2(self.id) @with_phil def set_fapl_stdio(self): """() Select the "stdio" driver (h5fd.STDIO) """ H5Pset_fapl_stdio(self.id) @with_phil def get_driver(self): """() => INT driver code Return an integer identifier for the driver used by this list. Although HDF5 implements these as full-fledged objects, they are treated as integers by Python. Built-in drivers identifiers are listed in module h5fd; they are: - h5fd.CORE - h5fd.FAMILY - h5fd.LOG - h5fd.MPIO - h5fd.MULTI - h5fd.SEC2 - h5fd.STDIO """ return H5Pget_driver(self.id) @with_phil def set_cache(self, int mdc, int rdcc, size_t rdcc_nbytes, double rdcc_w0): """(INT mdc, INT rdcc, UINT rdcc_nbytes, DOUBLE rdcc_w0) Set the metadata (mdc) and raw data chunk (rdcc) cache properties. See the HDF5 docs for a full explanation. """ H5Pset_cache(self.id, mdc, rdcc, rdcc_nbytes, rdcc_w0) @with_phil def get_cache(self): """() => TUPLE cache info Get the metadata and raw data chunk cache settings. See the HDF5 docs for element definitions. Return is a 4-tuple with entries: 1. INT mdc: Number of metadata objects 2. INT rdcc: Number of raw data chunks 3. UINT rdcc_nbytes: Size of raw data cache 4. DOUBLE rdcc_w0: Preemption policy for data cache. """ cdef int mdc cdef size_t rdcc, rdcc_nbytes cdef double w0 H5Pget_cache(self.id, &mdc, &rdcc, &rdcc_nbytes, &w0) return (mdc, rdcc, rdcc_nbytes, w0) @with_phil def set_sieve_buf_size(self, size_t size): """ (UINT size) Set the maximum size of the data sieve buffer (in bytes). This buffer can improve I/O performance for hyperslab I/O, by combining reads and writes into blocks of the given size. The default is 64k. """ H5Pset_sieve_buf_size(self.id, size) @with_phil def get_sieve_buf_size(self): """ () => UINT size Get the current maximum size of the data sieve buffer (in bytes). """ cdef size_t size H5Pget_sieve_buf_size(self.id, &size) return size @with_phil def set_libver_bounds(self, int low, int high): """ (INT low, INT high) Set the compatibility level for file format. Legal values are: - h5f.LIBVER_EARLIEST - h5f.LIBVER_LATEST """ H5Pset_libver_bounds(self.id, low, high) @with_phil def get_libver_bounds(self): """ () => (INT low, INT high) Get the compatibility level for file format. Returned values are from: - h5f.LIBVER_EARLIEST - h5f.LIBVER_LATEST """ cdef H5F_libver_t low cdef H5F_libver_t high H5Pget_libver_bounds(self.id, &low, &high) return (low, high) IF MPI: @with_phil def set_fapl_mpio(self, Comm comm not None, Info info not None): """ (Comm comm, Info info) Set MPI-I/O Parallel HDF5 driver. Comm: An mpi4py.MPI.Comm instance Info: An mpi4py.MPI.Info instance """ H5Pset_fapl_mpio(self.id, comm.ob_mpi, info.ob_mpi) @with_phil def get_fapl_mpio(self): """ () => (mpi4py.MPI.Comm, mpi4py.MPI.Info) Determine mpio driver MPI information. 0. The mpi4py.MPI.Comm Communicator 1. The mpi4py.MPI.Comm Info """ cdef MPI_Comm comm cdef MPI_Info info H5Pget_fapl_mpio(self.id, &comm, &info) pycomm = Comm() pyinfo = Info() MPI_Comm_dup(comm, &pycomm.ob_mpi) MPI_Info_dup(info, &pyinfo.ob_mpi) MPI_Comm_free(&comm) MPI_Info_free(&info) return (pycomm, pyinfo) @with_phil def set_fapl_mpiposix(self, Comm comm not None, bint use_gpfs_hints=0): """ Obsolete. """ raise RuntimeError("MPI-POSIX driver is broken; removed in h5py 2.3.1") @with_phil def get_mdc_config(self): """() => CacheConfig Returns an object that stores all the information about the meta-data cache configuration """ cdef CacheConfig config = CacheConfig() H5Pget_mdc_config(self.id, &config.cache_config) return config @with_phil def set_mdc_config(self, CacheConfig config not None): """(CacheConfig) => None Returns an object that stores all the information about the meta-data cache configuration """ H5Pset_mdc_config(self.id, &config.cache_config) def get_alignment(self): """ Retrieves the current settings for alignment properties from a file access property list. """ cdef hsize_t threshold, alignment H5Pget_alignment(self.id, &threshold, &alignment) return threshold, alignment def set_alignment(self, threshold, alignment): """ Sets alignment properties of a file access property list. """ H5Pset_alignment(self.id, threshold, alignment) IF HDF5_VERSION >= (1, 8, 9): @with_phil def set_file_image(self, image): """ Copy a file image into the property list. Passing None releases any image currently loaded. The parameter image must either be None or support the buffer protocol. """ cdef Py_buffer buf if image is None: H5Pset_file_image(self.id, NULL, 0) return if not PyObject_CheckBuffer(image): raise TypeError("image must support the buffer protocol") PyObject_GetBuffer(image, &buf, PyBUF_SIMPLE) try: H5Pset_file_image(self.id, buf.buf, buf.len) finally: PyBuffer_Release(&buf) # Link creation cdef class PropLCID(PropCreateID): """ Link creation property list """ @with_phil def set_char_encoding(self, int encoding): """ (INT encoding) Set the character encoding for link names. Legal values are: - h5t.CSET_ASCII - h5t.CSET_UTF8 """ H5Pset_char_encoding(self.id, encoding) @with_phil def get_char_encoding(self): """ () => INT encoding Get the character encoding for link names. Legal values are: - h5t.CSET_ASCII - h5t.CSET_UTF8 """ cdef H5T_cset_t encoding H5Pget_char_encoding(self.id, &encoding) return encoding @with_phil def set_create_intermediate_group(self, bint create): """(BOOL create) Set whether missing intermediate groups are automatically created. """ H5Pset_create_intermediate_group(self.id, create) @with_phil def get_create_intermediate_group(self): """() => BOOL Determine if missing intermediate groups are automatically created. """ cdef unsigned int create H5Pget_create_intermediate_group(self.id, &create) return create # Link access cdef class PropLAID(PropInstanceID): """ Link access property list """ def __cinit__(self, *args): self._buf = NULL def __dealloc__(self): efree(self._buf) @with_phil def set_nlinks(self, size_t nlinks): """(UINT nlinks) Set the maximum traversal depth for soft links """ H5Pset_nlinks(self.id, nlinks) @with_phil def get_nlinks(self): """() => UINT Get the maximum traversal depth for soft links """ cdef size_t nlinks H5Pget_nlinks(self.id, &nlinks) return nlinks @with_phil def set_elink_prefix(self, char* prefix): """(STRING prefix) Set the external link prefix. """ cdef size_t size # HDF5 requires that we hang on to this buffer efree(self._buf) size = strlen(prefix) self._buf = emalloc(size+1) strcpy(self._buf, prefix) H5Pset_elink_prefix(self.id, self._buf) @with_phil def get_elink_prefix(self): """() => STRING prefix Get the external link prefix """ cdef char* buf = NULL cdef ssize_t size size = H5Pget_elink_prefix(self.id, NULL, 0) buf = emalloc(size+1) try: H5Pget_elink_prefix(self.id, buf, size+1) pstr = buf finally: efree(buf) return pstr @with_phil def set_elink_fapl(self, PropID fapl not None): """ (PropFAID fapl) Set the file access property list used when opening external files. """ H5Pset_elink_fapl(self.id, fapl.id) @with_phil def get_elink_fapl(self): """ () => PropFAID fapl Get the file access property list used when opening external files. """ cdef hid_t fid fid = H5Pget_elink_fapl(self.id) if H5Iget_ref(fid) > 1: H5Idec_ref(fid) return propwrap(fid) # Group creation cdef class PropGCID(PropOCID): """ Group creation property list """ @with_phil def set_link_creation_order(self, unsigned int flags): """ (UINT flags) Set tracking and indexing of creation order for links added to this group flags -- h5p.CRT_ORDER_TRACKED, h5p.CRT_ORDER_INDEXED """ H5Pset_link_creation_order(self.id, flags) @with_phil def get_link_creation_order(self): """ () -> UINT flags Get tracking and indexing of creation order for links added to this group """ cdef unsigned int flags H5Pget_link_creation_order(self.id, &flags) return flags # Object creation property list cdef class PropOCID(PropCreateID): """ Object creation property list This seems to be a super class for dataset creation property list and group creation property list. The documentation is somewhat hazy """ @with_phil def set_obj_track_times(self,track_times): """Sets the recording of times associated with an object.""" H5Pset_obj_track_times(self.id,track_times) @with_phil def get_obj_track_times(self): """ Determines whether times associated with an object are being recorded. """ cdef hbool_t track_times H5Pget_obj_track_times(self.id,&track_times) return track_times # Dataset access cdef class PropDAID(PropInstanceID): """ Dataset access property list """ @with_phil def set_chunk_cache(self, size_t rdcc_nslots,size_t rdcc_nbytes, double rdcc_w0): """(size_t rdcc_nslots,size_t rdcc_nbytes, double rdcc_w0) Sets the raw data chunk cache parameters. """ H5Pset_chunk_cache(self.id,rdcc_nslots,rdcc_nbytes,rdcc_w0) @with_phil def get_chunk_cache(self): """() => TUPLE chunk cache info Get the metadata and raw data chunk cache settings. See the HDF5 docs for element definitions. Return is a 3-tuple with entries: 0. size_t rdcc_nslots: Number of chunk slots in the raw data chunk cache hash table. 1. size_t rdcc_nbytes: Total size of the raw data chunk cache, in bytes. 2. DOUBLE rdcc_w0: Preemption policy. """ cdef size_t rdcc_nslots cdef size_t rdcc_nbytes cdef double rdcc_w0 H5Pget_chunk_cache(self.id, &rdcc_nslots, &rdcc_nbytes, &rdcc_w0 ) return (rdcc_nslots,rdcc_nbytes,rdcc_w0) # === Virtual dataset functions =========================================== IF HDF5_VERSION >= VDS_MIN_HDF5_VERSION: @with_phil def set_virtual_view(self, unsigned int view): """(UINT view) Set the view of the virtual dataset (VDS) to include or exclude missing mapped elements. If view is set to h5d.VDS_FIRST_MISSING, the view includes all data before the first missing mapped data. This setting provides a view containing only the continuous data starting with the dataset’s first data element. Any break in continuity terminates the view. If view is set to h5d.VDS_LAST_AVAILABLE, the view includes all available mapped data. Missing mapped data is filled with the fill value set in the virtual dataset's creation property list. """ H5Pset_virtual_view(self.id, view) @with_phil def get_virtual_view(self): """() => UINT view Retrieve the view of the virtual dataset. Valid values are: - h5d.VDS_FIRST_MISSING - h5d.VDS_LAST_AVAILABLE """ cdef H5D_vds_view_t view H5Pget_virtual_view(self.id, &view) return view @with_phil def set_virtual_printf_gap(self, hsize_t gap_size=0): """(LONG gap_size=0) Set the maximum number of missing source files and/or datasets with the printf-style names when getting the extent of an unlimited virtual dataset. Instruct the library to stop looking for the mapped data stored in the files and/or datasets with the printf-style names after not finding gap_size files and/or datasets. The found source files and datasets will determine the extent of the unlimited virtual dataset with the printf-style mappings. Default value: 0. """ H5Pset_virtual_printf_gap(self.id, gap_size) @with_phil def get_virtual_printf_gap(self): """() => LONG gap_size Return the maximum number of missing source files and/or datasets with the printf-style names when getting the extent for an unlimited virtual dataset. """ cdef hsize_t gap_size H5Pget_virtual_printf_gap(self.id, &gap_size) return gap_size cdef class PropDXID(PropInstanceID): """ Data transfer property list """ IF MPI: def set_dxpl_mpio(self, int xfer_mode): """ Set the transfer mode for MPI I/O. Must be one of: - h5fd.MPIO_INDEPDENDENT (default) - h5fd.MPIO_COLLECTIVE """ H5Pset_dxpl_mpio(self.id, xfer_mode) def get_dxpl_mpio(self): """ Get the current transfer mode for MPI I/O. Will be one of: - h5fd.MPIO_INDEPDENDENT (default) - h5fd.MPIO_COLLECTIVE """ cdef H5FD_mpio_xfer_t mode H5Pget_dxpl_mpio(self.id, &mode) return mode h5py-2.7.1/licenses/0000755000175000017500000000000013152363123015670 5ustar tcaswelltcaswell00000000000000h5py-2.7.1/licenses/license.txt0000644000175000017500000000331513025343121020050 0ustar tcaswelltcaswell00000000000000Copyright Notice and Statement for the h5py Project =================================================== Copyright (c) 2008-2013 Andrew Collette and contributors http://www.h5py.org All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: a. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. b. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. c. Neither the name of the author nor the names of contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. h5py-2.7.1/licenses/stdint.txt0000644000175000017500000000256213025343033017740 0ustar tcaswelltcaswell00000000000000Copyright (c) 2006-2008 Alexander Chemeris Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. The name of the author may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.h5py-2.7.1/licenses/python.txt0000644000175000017500000000471013025343121017747 0ustar tcaswelltcaswell00000000000000Python license ============== #. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and the Individual or Organization ("Licensee") accessing and otherwise using Python Python 2.7.5 software in source or binary form and its associated documentation. #. Subject to the terms and conditions of this License Agreement, PSF hereby grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative works, distribute, and otherwise use Python Python 2.7.5 alone or in any derivative version, provided, however, that PSF's License Agreement and PSF's notice of copyright, i.e., "Copyright 2001-2013 Python Software Foundation; All Rights Reserved" are retained in Python Python 2.7.5 alone or in any derivative version prepared by Licensee. #. In the event Licensee prepares a derivative work that is based on or incorporates Python Python 2.7.5 or any part thereof, and wants to make the derivative work available to others as provided herein, then Licensee hereby agrees to include in any such work a brief summary of the changes made to Python Python 2.7.5. #. PSF is making Python Python 2.7.5 available to Licensee on an "AS IS" basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON Python 2.7.5 WILL NOT INFRINGE ANY THIRD PARTY RIGHTS. #. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON Python 2.7.5 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON Python 2.7.5, OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. #. This License Agreement will automatically terminate upon a material breach of its terms and conditions. #. Nothing in this License Agreement shall be deemed to create any relationship of agency, partnership, or joint venture between PSF and Licensee. This License Agreement does not grant permission to use PSF trademarks or trade name in a trademark sense to endorse or promote products or services of Licensee, or any third party. #. By copying, installing or otherwise using Python Python 2.7.5, Licensee agrees to be bound by the terms and conditions of this License Agreement. h5py-2.7.1/licenses/hdf5.txt0000644000175000017500000000726613025343033017267 0ustar tcaswelltcaswell00000000000000HDF5 (Hierarchical Data Format 5) Software Library and Utilities Copyright 2006-2007 by The HDF Group (THG). NCSA HDF5 (Hierarchical Data Format 5) Software Library and Utilities Copyright 1998-2006 by the Board of Trustees of the University of Illinois. All rights reserved. Contributors: National Center for Supercomputing Applications (NCSA) at the University of Illinois, Fortner Software, Unidata Program Center (netCDF), The Independent JPEG Group (JPEG), Jean-loup Gailly and Mark Adler (gzip), and Digital Equipment Corporation (DEC). Redistribution and use in source and binary forms, with or without modification, are permitted for any purpose (including commercial purposes) provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions, and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions, and the following disclaimer in the documentation and/or materials provided with the distribution. 3. In addition, redistributions of modified forms of the source or binary code must carry prominent notices stating that the original code was changed and the date of the change. 4. All publications or advertising materials mentioning features or use of this software are asked, but not required, to acknowledge that it was developed by The HDF Group and by the National Center for Supercomputing Applications at the University of Illinois at Urbana-Champaign and credit the contributors. 5. Neither the name of The HDF Group, the name of the University, nor the name of any Contributor may be used to endorse or promote products derived from this software without specific prior written permission from THG, the University, or the Contributor, respectively. DISCLAIMER: THIS SOFTWARE IS PROVIDED BY THE HDF GROUP (THG) AND THE CONTRIBUTORS "AS IS" WITH NO WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED. In no event shall THG or the Contributors be liable for any damages suffered by the users arising out of the use of this software, even if advised of the possibility of such damage. Portions of HDF5 were developed with support from the University of California, Lawrence Livermore National Laboratory (UC LLNL). The following statement applies to those portions of the product and must be retained in any redistribution of source code, binaries, documentation, and/or accompanying materials: This work was partially produced at the University of California, Lawrence Livermore National Laboratory (UC LLNL) under contract no. W-7405-ENG-48 (Contract 48) between the U.S. Department of Energy (DOE) and The Regents of the University of California (University) for the operation of UC LLNL. DISCLAIMER: This work was prepared as an account of work sponsored by an agency of the United States Government. Neither the United States Government nor the University of California nor any of their employees, makes any warranty, express or implied, or assumes any liability or responsibility for the accuracy, completeness, or usefulness of any information, apparatus, product, or process disclosed, or represents that its use would not infringe privately- owned rights. Reference herein to any specific commercial products, process, or service by trade name, trademark, manufacturer, or otherwise, does not necessarily constitute or imply its endorsement, recommendation, or favoring by the United States Government or the University of California. The views and opinions of authors expressed herein do not necessarily state or reflect those of the United States Government or the University of California, and shall not be used for advertising or product endorsement purposes. h5py-2.7.1/licenses/pytables.txt0000644000175000017500000000315013025343033020250 0ustar tcaswelltcaswell00000000000000Copyright Notice and Statement for PyTables Software Library and Utilities: Copyright (c) 2002, 2003, 2004 Francesc Altet Copyright (c) 2005, 2006, 2007 Carabos Coop. V. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: a. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. b. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. c. Neither the name of the Carabos Coop. V. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. h5py-2.7.1/h5py.egg-info/0000755000175000017500000000000013152363123016442 5ustar tcaswelltcaswell00000000000000h5py-2.7.1/h5py.egg-info/SOURCES.txt0000644000175000017500000000664013152363123020334 0ustar tcaswelltcaswell00000000000000ANN.rst MANIFEST.in README.rst api_gen.py pylintrc setup.py setup_build.py setup_configure.py tox.ini docs/Makefile docs/build.rst docs/conf.py docs/config.rst docs/contributing.rst docs/faq.rst docs/index.rst docs/licenses.rst docs/mpi.rst docs/quick.rst docs/refs.rst docs/special.rst docs/strings.rst docs/swmr.rst docs/high/attr.rst docs/high/dataset.rst docs/high/dims.rst docs/high/file.rst docs/high/group.rst docs/whatsnew/2.0.rst docs/whatsnew/2.1.rst docs/whatsnew/2.2.rst docs/whatsnew/2.3.rst docs/whatsnew/2.4.rst docs/whatsnew/2.5.rst docs/whatsnew/2.6.rst docs/whatsnew/2.7.1.rst docs/whatsnew/2.7.rst docs/whatsnew/index.rst docs_api/Makefile docs_api/automod.py docs_api/conf.py docs_api/h5.rst docs_api/h5a.rst docs_api/h5d.rst docs_api/h5f.rst docs_api/h5fd.rst docs_api/h5g.rst docs_api/h5i.rst docs_api/h5l.rst docs_api/h5o.rst docs_api/h5p.rst docs_api/h5r.rst docs_api/h5s.rst docs_api/h5t.rst docs_api/h5z.rst docs_api/index.rst docs_api/objects.rst examples/collective_io.py examples/multiprocessing_example.py examples/swmr_inotify_example.py examples/swmr_multiprocess.py examples/threading_example.py h5py/__init__.py h5py/_conv.pxd h5py/_conv.pyx h5py/_errors.pxd h5py/_errors.pyx h5py/_locks.pxi h5py/_objects.pxd h5py/_objects.pyx h5py/_proxy.pxd h5py/_proxy.pyx h5py/api_compat.h h5py/api_functions.txt h5py/api_types_ext.pxd h5py/api_types_hdf5.pxd h5py/h5.pxd h5py/h5.pyx h5py/h5a.pxd h5py/h5a.pyx h5py/h5ac.pxd h5py/h5ac.pyx h5py/h5d.pxd h5py/h5d.pyx h5py/h5ds.pxd h5py/h5ds.pyx h5py/h5f.pxd h5py/h5f.pyx h5py/h5fd.pxd h5py/h5fd.pyx h5py/h5g.pxd h5py/h5g.pyx h5py/h5i.pxd h5py/h5i.pyx h5py/h5l.pxd h5py/h5l.pyx h5py/h5o.pxd h5py/h5o.pyx h5py/h5p.pxd h5py/h5p.pyx h5py/h5r.pxd h5py/h5r.pyx h5py/h5s.pxd h5py/h5s.pyx h5py/h5t.pxd h5py/h5t.pyx h5py/h5z.pxd h5py/h5z.pyx h5py/highlevel.py h5py/ipy_completer.py h5py/numpy.pxd h5py/utils.pxd h5py/utils.pyx h5py/version.py h5py.egg-info/PKG-INFO h5py.egg-info/SOURCES.txt h5py.egg-info/dependency_links.txt h5py.egg-info/requires.txt h5py.egg-info/top_level.txt h5py/_hl/__init__.py h5py/_hl/attrs.py h5py/_hl/base.py h5py/_hl/compat.py h5py/_hl/dataset.py h5py/_hl/datatype.py h5py/_hl/dims.py h5py/_hl/files.py h5py/_hl/filters.py h5py/_hl/group.py h5py/_hl/selections.py h5py/_hl/selections2.py h5py/tests/__init__.py h5py/tests/common.py h5py/tests/hl/__init__.py h5py/tests/hl/test_attribute_create.py h5py/tests/hl/test_dataset_getitem.py h5py/tests/hl/test_dataset_swmr.py h5py/tests/hl/test_datatype.py h5py/tests/hl/test_dims_dimensionproxy.py h5py/tests/hl/test_file.py h5py/tests/hl/test_threads.py h5py/tests/old/__init__.py h5py/tests/old/test_attrs.py h5py/tests/old/test_attrs_data.py h5py/tests/old/test_base.py h5py/tests/old/test_dataset.py h5py/tests/old/test_datatype.py h5py/tests/old/test_dimension_scales.py h5py/tests/old/test_file.py h5py/tests/old/test_file_image.py h5py/tests/old/test_group.py h5py/tests/old/test_h5.py h5py/tests/old/test_h5d_direct_chunk_write.py h5py/tests/old/test_h5f.py h5py/tests/old/test_h5p.py h5py/tests/old/test_h5t.py h5py/tests/old/test_objects.py h5py/tests/old/test_selections.py h5py/tests/old/test_slicing.py licenses/hdf5.txt licenses/license.txt licenses/pytables.txt licenses/python.txt licenses/stdint.txt lzf/LICENSE.txt lzf/README.txt lzf/example.c lzf/lzf_filter.c lzf/lzf_filter.h lzf/lzf/lzf.h lzf/lzf/lzfP.h lzf/lzf/lzf_c.c lzf/lzf/lzf_d.c windows/README.txt windows/cacheinit.cmake windows/pavement.py windows/stdint.h windows/unistd.hh5py-2.7.1/h5py.egg-info/PKG-INFO0000644000175000017500000000410513152363123017537 0ustar tcaswelltcaswell00000000000000Metadata-Version: 1.1 Name: h5py Version: 2.7.1 Summary: Read and write HDF5 files from Python Home-page: http://www.h5py.org Author: Andrew Collette Author-email: andrew.collette@gmail.com License: UNKNOWN Download-URL: https://pypi.python.org/pypi/h5py Description: The h5py package provides both a high- and low-level interface to the HDF5 library from Python. The low-level interface is intended to be a complete wrapping of the HDF5 API, while the high-level component supports access to HDF5 files, datasets and groups using established Python and NumPy concepts. A strong emphasis on automatic conversion between Python (Numpy) datatypes and data structures and their HDF5 equivalents vastly simplifies the process of reading and writing data from Python. Supports HDF5 versions 1.8.4 and higher. On Windows, HDF5 is included with the installer. Platform: UNKNOWN Classifier: Development Status :: 5 - Production/Stable Classifier: Intended Audience :: Developers Classifier: Intended Audience :: Information Technology Classifier: Intended Audience :: Science/Research Classifier: License :: OSI Approved :: BSD License Classifier: Programming Language :: Cython Classifier: Programming Language :: Python Classifier: Programming Language :: Python :: 2 Classifier: Programming Language :: Python :: 2.6 Classifier: Programming Language :: Python :: 2.7 Classifier: Programming Language :: Python :: 3 Classifier: Programming Language :: Python :: 3.3 Classifier: Programming Language :: Python :: 3.4 Classifier: Programming Language :: Python :: 3.5 Classifier: Programming Language :: Python :: 3.6 Classifier: Programming Language :: Python :: Implementation :: CPython Classifier: Topic :: Scientific/Engineering Classifier: Topic :: Database Classifier: Topic :: Software Development :: Libraries :: Python Modules Classifier: Operating System :: Unix Classifier: Operating System :: POSIX :: Linux Classifier: Operating System :: MacOS :: MacOS X Classifier: Operating System :: Microsoft :: Windows h5py-2.7.1/h5py.egg-info/top_level.txt0000644000175000017500000000000513152363123021167 0ustar tcaswelltcaswell00000000000000h5py h5py-2.7.1/h5py.egg-info/requires.txt0000644000175000017500000000001713152363123021040 0ustar tcaswelltcaswell00000000000000numpy>=1.7 six h5py-2.7.1/h5py.egg-info/dependency_links.txt0000644000175000017500000000000113152363123022510 0ustar tcaswelltcaswell00000000000000 h5py-2.7.1/pylintrc0000644000175000017500000002703313025343121015652 0ustar tcaswelltcaswell00000000000000[MASTER] # Specify a configuration file. #rcfile= # Python code to execute, usually for sys.path manipulation such as # pygtk.require(). #init-hook= # Profiled execution. profile=no # Add files or directories to the blacklist. They should be base names, not # paths. ignore=tests # Pickle collected data for later comparisons. persistent=yes # List of plugins (as comma separated values of python modules names) to load, # usually to register additional checkers. load-plugins= # Use multiple processes to speed up Pylint. jobs=1 # Allow loading of arbitrary C extensions. Extensions are imported into the # active Python interpreter and may run arbitrary code. unsafe-load-any-extension=no # A comma-separated list of package or module names from where C extensions may # be loaded. Extensions are loading into the active Python interpreter and may # run arbitrary code extension-pkg-whitelist=numpy,h5py [MESSAGES CONTROL] # Only show warnings with the listed confidence levels. Leave empty to show # all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED confidence= # Disable the message, report, category or checker with the given id(s). You # can either give multiple identifiers separated by comma (,) or put this # option multiple times (only on the command line, not in the configuration # file where it should appear only once).You can also use "--disable=all" to # disable everything first and then reenable specific checks. For example, if # you want to run only the similarities checker, you can use "--disable=all # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use"--disable=all --enable=classes # --disable=W" # # | Checkers | Broken import checks | Other random garbage disable=format,design,similarities,cyclic-import,import-error,broad-except,no-self-use,no-name-in-module,invalid-name,abstract-method,star-args,import-self,no-init,locally-disabled,unidiomatic-typecheck # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option # multiple time. See also the "--disable" option for examples. # # | Some format checks which are OK enable=bad-indentation,mixed-indentation,unnecessary-semicolon,superfluous-parens [REPORTS] # Set the output format. Available formats are text, parseable, colorized, msvs # (visual studio) and html. You can also give a reporter class, eg # mypackage.mymodule.MyReporterClass. output-format=text # Put messages in a separate file for each module / package specified on the # command line instead of printing them on stdout. Reports (if any) will be # written in a file name "pylint_global.[txt|html]". files-output=no # Tells whether to display a full report or only the messages reports=no # Python expression which should return a note less than 10 (10 is the highest # note). You have access to the variables errors warning, statement which # respectively contain the number of errors / warnings messages and the total # number of statements analyzed. This is used by the global evaluation report # (RP0004). evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) # Add a comment according to your evaluation note. This is used by the global # evaluation report (RP0004). comment=no # Template used to display messages. This is a python new-style format string # used to format the message information. See doc for all details #msg-template= [BASIC] # Required attributes for module, separated by a comma required-attributes= # List of builtins function names that should not be used, separated by a comma bad-functions=map,filter # Good variable names which should always be accepted, separated by a comma good-names=i,j,k,ex,Run,_ # Bad variable names which should always be refused, separated by a comma bad-names=foo,bar,baz,toto,tutu,tata # Colon-delimited sets of names that determine each other's naming style when # the name regexes allow several styles. name-group= # Include a hint for the correct naming format with invalid-name include-naming-hint=no # Regular expression matching correct method names method-rgx=[a-z_][a-z0-9_]{2,30}$ # Naming hint for method names method-name-hint=[a-z_][a-z0-9_]{2,30}$ # Regular expression matching correct module names module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ # Naming hint for module names module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ # Regular expression matching correct inline iteration names inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ # Naming hint for inline iteration names inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ # Regular expression matching correct constant names const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ # Naming hint for constant names const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ # Regular expression matching correct variable names variable-rgx=[a-z_][a-z0-9_]{2,30}$ # Naming hint for variable names variable-name-hint=[a-z_][a-z0-9_]{2,30}$ # Regular expression matching correct argument names argument-rgx=[a-z_][a-z0-9_]{2,30}$ # Naming hint for argument names argument-name-hint=[a-z_][a-z0-9_]{2,30}$ # Regular expression matching correct class attribute names class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ # Naming hint for class attribute names class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ # Regular expression matching correct class names class-rgx=[A-Z_][a-zA-Z0-9]+$ # Naming hint for class names class-name-hint=[A-Z_][a-zA-Z0-9]+$ # Regular expression matching correct function names function-rgx=[a-z_][a-z0-9_]{2,30}$ # Naming hint for function names function-name-hint=[a-z_][a-z0-9_]{2,30}$ # Regular expression matching correct attribute names attr-rgx=[a-z_][a-z0-9_]{2,30}$ # Naming hint for attribute names attr-name-hint=[a-z_][a-z0-9_]{2,30}$ # Regular expression which should only match function or class names that do # not require a docstring. no-docstring-rgx=__.*__ # Minimum line length for functions/classes that require docstrings, shorter # ones are exempt. docstring-min-length=-1 [FORMAT] # Maximum number of characters on a single line. max-line-length=100 # Regexp for a line that is allowed to be longer than the limit. ignore-long-lines=^\s*(# )??$ # Allow the body of an if to be on the same line as the test if there is no # else. single-line-if-stmt=no # List of optional constructs for which whitespace checking is disabled no-space-check=trailing-comma,dict-separator # Maximum number of lines in a module max-module-lines=1000 # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 # tab). indent-string=' ' # Number of spaces of indent required inside a hanging or continued line. indent-after-paren=4 # Expected format of line ending, e.g. empty (any line ending), LF or CRLF. expected-line-ending-format= [LOGGING] # Logging modules to check that the string format arguments are in logging # function parameter format logging-modules=logging [MISCELLANEOUS] # List of note tags to take in consideration, separated by a comma. notes=FIXME,XXX,TODO [SIMILARITIES] # Minimum lines number of a similarity. min-similarity-lines=4 # Ignore comments when computing similarities. ignore-comments=yes # Ignore docstrings when computing similarities. ignore-docstrings=yes # Ignore imports when computing similarities. ignore-imports=no [SPELLING] # Spelling dictionary name. Available dictionaries: none. To make it working # install python-enchant package. spelling-dict= # List of comma separated words that should not be checked. spelling-ignore-words= # A path to a file that contains private dictionary; one word per line. spelling-private-dict-file= # Tells whether to store unknown words to indicated private dictionary in # --spelling-private-dict-file option instead of raising a message. spelling-store-unknown-words=no [TYPECHECK] # Tells whether missing members accessed in mixin class should be ignored. A # mixin class is detected if its name ends with "mixin" (case insensitive). ignore-mixin-members=yes # List of module names for which member attributes should not be checked # (useful for modules/projects where namespaces are manipulated during runtime # and thus existing member attributes cannot be deduced by static analysis ignored-modules= # List of classes names for which member attributes should not be checked # (useful for classes with attributes dynamically set). ignored-classes=SQLObject # When zope mode is activated, add a predefined set of Zope acquired attributes # to generated-members. zope=no # List of members which are set dynamically and missed by pylint inference # system, and so shouldn't trigger E0201 when accessed. Python regular # expressions are accepted. generated-members=REQUEST,acl_users,aq_parent [VARIABLES] # Tells whether we should check for unused import in __init__ files. init-import=no # A regular expression matching the name of dummy variables (i.e. expectedly # not used). dummy-variables-rgx=_$|dummy # List of additional names supposed to be defined in builtins. Remember that # you should avoid to define new builtins when possible. additional-builtins= # List of strings which can identify a callback function by name. A callback # name must start or end with one of those strings. callbacks=cb_,_cb [CLASSES] # List of interface methods to ignore, separated by a comma. This is used for # instance to not check methods defines in Zope's Interface base class. ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by # List of method names used to declare (i.e. assign) instance attributes. defining-attr-methods=__init__,__new__,setUp # List of valid names for the first argument in a class method. valid-classmethod-first-arg=cls # List of valid names for the first argument in a metaclass class method. valid-metaclass-classmethod-first-arg=mcs # List of member names, which should be excluded from the protected access # warning. exclude-protected=_asdict,_fields,_replace,_source,_make [DESIGN] # Maximum number of arguments for function / method max-args=5 # Argument names that match this expression will be ignored. Default to name # with leading underscore ignored-argument-names=_.* # Maximum number of locals for function / method body max-locals=15 # Maximum number of return / yield for function / method body max-returns=6 # Maximum number of branch for function / method body max-branches=12 # Maximum number of statements in function / method body max-statements=50 # Maximum number of parents for a class (see R0901). max-parents=7 # Maximum number of attributes for a class (see R0902). max-attributes=7 # Minimum number of public methods for a class (see R0903). min-public-methods=2 # Maximum number of public methods for a class (see R0904). max-public-methods=20 [IMPORTS] # Deprecated modules which should not be used, separated by a comma deprecated-modules=stringprep,optparse # Create a graph of every (i.e. internal and external) dependencies in the # given file (report RP0402 must not be disabled) import-graph= # Create a graph of external dependencies in the given file (report RP0402 must # not be disabled) ext-import-graph= # Create a graph of internal dependencies in the given file (report RP0402 must # not be disabled) int-import-graph= [EXCEPTIONS] # Exceptions that will emit a warning when being caught. Defaults to # "Exception" overgeneral-exceptions=Exception h5py-2.7.1/docs/0000755000175000017500000000000013152363123015013 5ustar tcaswelltcaswell00000000000000h5py-2.7.1/docs/faq.rst0000644000175000017500000002257113025343121016316 0ustar tcaswelltcaswell00000000000000.. _faq: FAQ === What datatypes are supported? ----------------------------- Below is a complete list of types for which h5py supports reading, writing and creating datasets. Each type is mapped to a native NumPy type. Fully supported types: ========================= ============================================ ====================== Type Precisions Notes ========================= ============================================ ====================== Integer 1, 2, 4 or 8 byte, BE/LE, signed/unsigned Float 2, 4, 8, 12, 16 byte, BE/LE Complex 8 or 16 byte, BE/LE Stored as HDF5 struct Compound Arbitrary names and offsets Strings (fixed-length) Any length Strings (variable-length) Any length, ASCII or Unicode Opaque (kind 'V') Any length Boolean NumPy 1-byte bool Stored as HDF5 enum Array Any supported type Enumeration Any NumPy integer type Read/write as integers References Region and object Variable length array Any supported type See :ref:`Special Types ` ========================= ============================================ ====================== Unsupported types: ========================= ============================================ Type Status ========================= ============================================ HDF5 "time" type NumPy "U" strings No HDF5 equivalent NumPy generic "O" Not planned ========================= ============================================ What compression/processing filters are supported? -------------------------------------------------- =================================== =========================================== ============================ Filter Function Availability =================================== =========================================== ============================ DEFLATE/GZIP Standard HDF5 compression All platforms SHUFFLE Increase compression ratio All platforms FLETCHER32 Error detection All platforms Scale-offset Integer/float scaling and truncation All platforms SZIP Fast, patented compression for int/float * UNIX: if supplied with HDF5. * Windows: read-only `LZF `_ Very fast compression, all types Ships with h5py, C source available =================================== =========================================== ============================ What file drivers are available? -------------------------------- A number of different HDF5 "drivers", which provide different modes of access to the filesystem, are accessible in h5py via the high-level interface. The currently supported drivers are: =================================== =========================================== ============================ Driver Purpose Notes =================================== =========================================== ============================ sec2 Standard optimized driver Default on UNIX/Windows stdio Buffered I/O using stdio.h core In-memory file (optionally backed to disk) family Multi-file driver mpio Parallel HDF5 file access =================================== =========================================== ============================ What's the difference between h5py and PyTables? ------------------------------------------------ The two projects have different design goals. PyTables presents a database-like approach to data storage, providing features like indexing and fast "in-kernel" queries on dataset contents. It also has a custom system to represent data types. In contrast, h5py is an attempt to map the HDF5 feature set to NumPy as closely as possible. For example, the high-level type system uses NumPy dtype objects exclusively, and method and attribute naming follows Python and NumPy conventions for dictionary and array access (i.e. ".dtype" and ".shape" attributes for datasets, ``group[name]`` indexing syntax for groups, etc). Underneath the "high-level" interface to h5py (i.e. NumPy-array-like objects; what you'll typically be using) is a large Cython layer which calls into C. This "low-level" interface provides access to nearly all of the HDF5 C API. This layer is object-oriented with respect to HDF5 identifiers, supports reference counting, automatic translation between NumPy and HDF5 type objects, translation between the HDF5 error stack and Python exceptions, and more. This greatly simplifies the design of the complicated high-level interface, by relying on the "Pythonicity" of the C API wrapping. There's also a PyTables perspective on this question at the `PyTables FAQ `_. Does h5py support Parallel HDF5? -------------------------------- Starting with version 2.2, h5py supports Parallel HDF5 on UNIX platforms. ``mpi4py`` is required, as well as an MPIO-enabled build of HDF5. Check out :ref:`parallel` for details. Variable-length (VLEN) data --------------------------- Starting with version 2.3, all supported types can be stored in variable-length arrays (previously only variable-length byte and unicode strings were supported) See :ref:`Special Types ` for use details. Please note that since strings in HDF5 are encoded as ASCII or UTF-8, NUL bytes are not allowed in strings. Enumerated types ---------------- HDF5 enumerated types are supported as. As NumPy has no native enum type, they are treated on the Python side as integers with a small amount of metadata attached to the dtype. NumPy object types ------------------ Storage of generic objects (NumPy dtype "O") is not implemented and not planned to be implemented, as the design goal for h5py is to expose the HDF5 feature set, not add to it. However, objects picked to the "plain-text" protocol (protocol 0) can be stored in HDF5 as strings. Appending data to a dataset --------------------------- The short response is that h5py is NumPy-like, not database-like. Unlike the HDF5 packet-table interface (and PyTables), there is no concept of appending rows. Rather, you can expand the shape of the dataset to fit your needs. For example, if I have a series of time traces 1024 points long, I can create an extendable dataset to store them: >>> dset = myfile.create_dataset("MyDataset", (10, 1024), maxshape=(None, 1024)) >>> dset.shape (10,1024) The keyword argument "maxshape" tells HDF5 that the first dimension of the dataset can be expanded to any size, while the second dimension is limited to a maximum size of 1024. We create the dataset with room for an initial ensemble of 10 time traces. If we later want to store 10 more time traces, the dataset can be expanded along the first axis: >>> dset.resize(20, axis=0) # or dset.resize((20,1024)) >>> dset.shape (20, 1024) Each axis can be resized up to the maximum values in "maxshape". Things to note: * Unlike NumPy arrays, when you resize a dataset the indices of existing data do not change; each axis grows or shrinks independently * The dataset rank (number of dimensions) is fixed when it is created Unicode ------- As of h5py 2.0.0, Unicode is supported for file names as well as for objects in the file. When object names are read, they are returned as Unicode by default. However, HDF5 has no predefined datatype to represent fixed-width UTF-16 or UTF-32 (NumPy format) strings. Therefore, the NumPy 'U' datatype is not supported. Development ----------- Building from Git ~~~~~~~~~~~~~~~~~ We moved to GitHub in December of 2012 (http://github.com/h5py/h5py). We use the following conventions for branches and tags: * master: integration branch for the next minor (or major) version * 2.0, 2.1, 2.2, etc: bugfix branches for released versions * tags 2.0.0, 2.0.1, etc: Released bugfix versions To build from a Git checkout: ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Clone the project:: $ git clone https://github.com/h5py/h5py.git $ cd h5py (Optional) Choose which branch to build from (e.g. a stable branch):: $ git checkout 2.1 Build the project. If given, /path/to/hdf5 should point to a directory containing a compiled, shared-library build of HDF5 (containing things like "include" and "lib"):: $ python setup.py build [--hdf5=/path/to/hdf5] (Optional) Run the unit tests:: $ python setup.py test Report any failing tests to the mailing list (h5py at googlegroups), or by filing a bug report at GitHub. h5py-2.7.1/docs/high/0000755000175000017500000000000013152363123015732 5ustar tcaswelltcaswell00000000000000h5py-2.7.1/docs/high/attr.rst0000644000175000017500000000732313025343121017436 0ustar tcaswelltcaswell00000000000000.. _attributes: Attributes ========== Attributes are a critical part of what makes HDF5 a "self-describing" format. They are small named pieces of data attached directly to :class:`Group` and :class:`Dataset` objects. This is the official way to store metadata in HDF5. Each Group or Dataset has a small proxy object attached to it, at ``.attrs``. Attributes have the following properties: - They may be created from any scalar or NumPy array - Each attribute should be small (generally < 64k) - There is no partial I/O (i.e. slicing); the entire attribute must be read. The ``.attrs`` proxy objects are of class :class:`AttributeManager`, below. This class supports a dictionary-style interface. Reference --------- .. class:: AttributeManager(parent) AttributeManager objects are created directly by h5py. You should access instances by ``group.attrs`` or ``dataset.attrs``, not by manually creating them. .. method:: __iter__() Get an iterator over attribute names. .. method:: __contains__(name) Determine if attribute `name` is attached to this object. .. method:: __getitem__(name) Retrieve an attribute. .. method:: __setitem__(name, val) Create an attribute, overwriting any existing attribute. The type and shape of the attribute are determined automatically by h5py. .. method:: __delitem__(name) Delete an attribute. KeyError if it doesn't exist. .. method:: keys() Get the names of all attributes attached to this object. On Py2, this is a list. On Py3, it's a set-like object. .. method:: values() Get the values of all attributes attached to this object. On Py2, this is a list. On Py3, it's a collection or bag-like object. .. method:: items() Get ``(name, value)`` tuples for all attributes attached to this object. On Py2, this is a list of tuples. On Py3, it's a collection or set-like object. .. method:: iterkeys() (Py2 only) Get an iterator over attribute names. .. method:: itervalues() (Py2 only) Get an iterator over attribute values. .. method:: iteritems() (Py2 only) Get an iterator over ``(name, value)`` pairs. .. method:: get(name, default=None) Retrieve `name`, or `default` if no such attribute exists. .. method:: create(name, data, shape=None, dtype=None) Create a new attribute, with control over the shape and type. Any existing attribute will be overwritten. :param name: Name of the new attribute :type name: String :param data: Value of the attribute; will be put through ``numpy.array(data)``. :param shape: Shape of the attribute. Overrides ``data.shape`` if both are given, in which case the total number of points must be unchanged. :type shape: Tuple :param dtype: Data type for the attribute. Overrides ``data.dtype`` if both are given. :type dtype: NumPy dtype .. method:: modify(name, value) Change the value of an attribute while preserving its type and shape. Unlike :meth:`AttributeManager.__setitem__`, if the attribute already exists, only its value will be changed. This can be useful for interacting with externally generated files, where the type and shape must not be altered. If the attribute doesn't exist, it will be created with a default shape and type. :param name: Name of attribute to modify. :type name: String :param value: New value. Will be put through ``numpy.array(value)``. h5py-2.7.1/docs/high/group.rst0000644000175000017500000003477313114352652017641 0ustar tcaswelltcaswell00000000000000.. _group: Groups ====== Groups are the container mechanism by which HDF5 files are organized. From a Python perspective, they operate somewhat like dictionaries. In this case the "keys" are the names of group members, and the "values" are the members themselves (:class:`Group` and :class:`Dataset`) objects. Group objects also contain most of the machinery which makes HDF5 useful. The :ref:`File object ` does double duty as the HDF5 *root group*, and serves as your entry point into the file: >>> f = h5py.File('foo.hdf5','w') >>> f.name u'/' >>> f.keys() [] Names of all objects in the file are all text strings (``unicode`` on Py2, ``str`` on Py3). These will be encoded with the HDF5-approved UTF-8 encoding before being passed to the HDF5 C library. Objects may also be retrieved using byte strings, which will be passed on to HDF5 as-is. .. _group_create: Creating groups --------------- New groups are easy to create:: >>> grp = f.create_group("bar") >>> grp.name '/bar' >>> subgrp = grp.create_group("baz") >>> subgrp.name '/bar/baz' Multiple intermediate groups can also be created implicitly:: >>> grp2 = f.create_group("/some/long/path") >>> grp2.name '/some/long/path' >>> grp3 = f['/some/long'] >>> grp3.name '/some/long' .. _group_links: Dict interface and links ------------------------ Groups implement a subset of the Python dictionary convention. They have methods like ``keys()``, ``values()`` and support iteration. Most importantly, they support the indexing syntax, and standard exceptions: >>> myds = subgrp["MyDS"] >>> missing = subgrp["missing"] KeyError: "Name doesn't exist (Symbol table: Object not found)" Objects can be deleted from the file using the standard syntax:: >>> del subgroup["MyDataset"] .. note:: When using h5py from Python 3, the keys(), values() and items() methods will return view-like objects instead of lists. These objects support containership testing and iteration, but can't be sliced like lists. .. _group_hardlinks: Hard links ~~~~~~~~~~ What happens when assigning an object to a name in the group? It depends on the type of object being assigned. For NumPy arrays or other data, the default is to create an :ref:`HDF5 datasets `:: >>> grp["name"] = 42 >>> out = grp["name"] >>> out When the object being stored is an existing Group or Dataset, a new link is made to the object:: >>> grp["other name"] = out >>> grp["other name"] Note that this is `not` a copy of the dataset! Like hard links in a UNIX file system, objects in an HDF5 file can be stored in multiple groups:: >>> f["other name"] == f["name"] True .. _group_softlinks: Soft links ~~~~~~~~~~ Also like a UNIX filesystem, HDF5 groups can contain "soft" or symbolic links, which contain a text path instead of a pointer to the object itself. You can easily create these in h5py by using ``h5py.SoftLink``:: >>> myfile = h5py.File('foo.hdf5','w') >>> group = myfile.create_group("somegroup") >>> myfile["alias"] = h5py.SoftLink('/somegroup') If the target is removed, they will "dangle": >>> del myfile['somegroup'] >>> print myfile['alias'] KeyError: 'Component not found (Symbol table: Object not found)' .. _group_extlinks: External links ~~~~~~~~~~~~~~ New in HDF5 1.8, external links are "soft links plus", which allow you to specify the name of the file as well as the path to the desired object. You can refer to objects in any file you wish. Use similar syntax as for soft links: >>> myfile = h5py.File('foo.hdf5','w') >>> myfile['ext link'] = h5py.ExternalLink("otherfile.hdf5", "/path/to/resource") When the link is accessed, the file "otherfile.hdf5" is opened, and object at "/path/to/resource" is returned. Since the object retrieved is in a different file, its ".file" and ".parent" properties will refer to objects in that file, *not* the file in which the link resides. .. note:: Currently, you can't access an external link if the file it points to is already open. This is related to how HDF5 manages file permissions internally. .. note:: How the filename is processed is operating system dependent, it is recommended to read :ref:`file_filenames` to understand potential limitations on filenames on your operating system. Note especially that Windows is particularly susceptible to problems with external links, due to possible encoding errors and how filenames are structured. Reference --------- .. class:: Group(identifier) Generally Group objects are created by opening objects in the file, or by the method :meth:`Group.create_group`. Call the constructor with a :class:`GroupID ` instance to create a new Group bound to an existing low-level identifier. .. method:: __iter__() Iterate over the names of objects directly attached to the group. Use :meth:`Group.visit` or :meth:`Group.visititems` for recursive access to group members. .. method:: __contains__(name) Dict-like containership testing. `name` may be a relative or absolute path. .. method:: __getitem__(name) Retrieve an object. `name` may be a relative or absolute path, or an :ref:`object or region reference `. See :ref:`group_links`. .. method:: __setitem__(name, value) Create a new link, or automatically create a dataset. See :ref:`group_links`. .. method:: keys() Get the names of directly attached group members. On Py2, this is a list. On Py3, it's a set-like object. Use :meth:`Group.visit` or :meth:`Group.visititems` for recursive access to group members. .. method:: values() Get the objects contained in the group (Group and Dataset instances). Broken soft or external links show up as None. On Py2, this is a list. On Py3, it's a collection or bag-like object. .. method:: items() Get ``(name, value)`` pairs for object directly attached to this group. Values for broken soft or external links show up as None. On Py2, this is a list. On Py3, it's a set-like object. .. method:: iterkeys() (Py2 only) Get an iterator over key names. Exactly equivalent to ``iter(group)``. Use :meth:`Group.visit` or :meth:`Group.visititems` for recursive access to group members. .. method:: itervalues() (Py2 only) Get an iterator over objects attached to the group. Broken soft and external links will show up as ``None``. .. method:: iteritems() (Py2 only) Get an iterator over ``(name, value)`` pairs for objects directly attached to the group. Broken soft and external link values show up as ``None``. .. method:: get(name, default=None, getclass=False, getlink=False) Retrieve an item, or information about an item. `name` and `default` work like the standard Python ``dict.get``. :param name: Name of the object to retrieve. May be a relative or absolute path. :param default: If the object isn't found, return this instead. :param getclass: If True, return the class of object instead; :class:`Group` or :class:`Dataset`. :param getlink: If true, return the type of link via a :class:`HardLink`, :class:`SoftLink` or :class:`ExternalLink` instance. If ``getclass`` is also True, returns the corresponding Link class without instantiating it. .. method:: visit(callable) Recursively visit all objects in this group and subgroups. You supply a callable with the signature:: callable(name) -> None or return value `name` will be the name of the object relative to the current group. Return None to continue visiting until all objects are exhausted. Returning anything else will immediately stop visiting and return that value from ``visit``:: >>> def find_foo(name): ... """ Find first object with 'foo' anywhere in the name """ ... if 'foo' in name: ... return name >>> group.visit(find_foo) u'some/subgroup/foo' .. method:: visititems(callable) Recursively visit all objects in this group and subgroups. Like :meth:`Group.visit`, except your callable should have the signature:: callable(name, object) -> None or return value In this case `object` will be a :class:`Group` or :class:`Dataset` instance. .. method:: move(source, dest) Move an object or link in the file. If `source` is a hard link, this effectively renames the object. If a soft or external link, the link itself is moved. :param source: Name of object or link to move. :type source: String :param dest: New location for object or link. :type dest: String .. method:: copy(source, dest, name=None, shallow=False, expand_soft=False, expand_external=False, expand_refs=False, without_attrs=False) Copy an object or group. The source and destination need not be in the same file. If the source is a Group object, by default all objects within that group will be copied recursively. :param source: What to copy. May be a path in the file or a Group/Dataset object. :param dest: Where to copy it. May be a path or Group object. :param name: If the destination is a Group object, use this for the name of the copied object (default is basename). :param shallow: Only copy immediate members of a group. :param expand_soft: Expand soft links into new objects. :param expand_external: Expand external links into new objects. :param expand_refs: Copy objects which are pointed to by references. :param without_attrs: Copy object(s) without copying HDF5 attributes. .. method:: create_group(name) Create and return a new group in the file. :param name: Name of group to create. May be an absolute or relative path. Provide None to create an anonymous group, to be linked into the file later. :type name: String or None :return: The new :class:`Group` object. .. method:: require_group(name) Open a group in the file, creating it if it doesn't exist. TypeError is raised if a conflicting object already exists. Parameters as in :meth:`Group.create_group`. .. method:: create_dataset(name, shape=None, dtype=None, data=None, **kwds) Create a new dataset. Options are explained in :ref:`dataset_create`. :param name: Name of dataset to create. May be an absolute or relative path. Provide None to create an anonymous dataset, to be linked into the file later. :param shape: Shape of new dataset (Tuple). :param dtype: Data type for new dataset :param data: Initialize dataset to this (NumPy array). :keyword chunks: Chunk shape, or True to enable auto-chunking. :keyword maxshape: Dataset will be resizable up to this shape (Tuple). Automatically enables chunking. Use None for the axes you want to be unlimited. :keyword compression: Compression strategy. See :ref:`dataset_compression`. :keyword compression_opts: Parameters for compression filter. :keyword scaleoffset: See :ref:`dataset_scaleoffset`. :keyword shuffle: Enable shuffle filter (T/**F**). See :ref:`dataset_shuffle`. :keyword fletcher32: Enable Fletcher32 checksum (T/**F**). See :ref:`dataset_fletcher32`. :keyword fillvalue: This value will be used when reading uninitialized parts of the dataset. :keyword track_times: Enable dataset creation timestamps (**T**/F). .. method:: require_dataset(name, shape=None, dtype=None, exact=None, **kwds) Open a dataset, creating it if it doesn't exist. If keyword "exact" is False (default), an existing dataset must have the same shape and a conversion-compatible dtype to be returned. If True, the shape and dtype must match exactly. Other dataset keywords (see create_dataset) may be provided, but are only used if a new dataset is to be created. Raises TypeError if an incompatible object already exists, or if the shape or dtype don't match according to the above rules. :keyword exact: Require shape and type to match exactly (T/**F**) .. attribute:: attrs :ref:`attributes` for this group. .. attribute:: id The groups's low-level identifer; an instance of :class:`GroupID `. .. attribute:: ref An HDF5 object reference pointing to this group. See :ref:`refs_object`. .. attribute:: regionref A proxy object allowing you to interrogate region references. See :ref:`refs_region`. .. attribute:: name String giving the full path to this group. .. attribute:: file :class:`File` instance in which this group resides. .. attribute:: parent :class:`Group` instance containing this group. Link classes ------------ .. class:: HardLink() Exists only to support :meth:`Group.get`. Has no state and provides no properties or methods. .. class:: SoftLink(path) Exists to allow creation of soft links in the file. See :ref:`group_softlinks`. These only serve as containers for a path; they are not related in any way to a particular file. :param path: Value of the soft link. :type path: String .. attribute:: path Value of the soft link .. class:: ExternalLink(filename, path) Like :class:`SoftLink`, only they specify a filename in addition to a path. See :ref:`group_extlinks`. :param filename: Name of the file to which the link points :type filename: String :param path: Path to the object in the external file. :type path: String .. attribute:: filename Name of the external file .. attribute:: path Path to the object in the external file h5py-2.7.1/docs/high/file.rst0000644000175000017500000002113313114352650017404 0ustar tcaswelltcaswell00000000000000.. _file: File Objects ============ File objects serve as your entry point into the world of HDF5. In addition to the File-specific capabilities listed here, every File instance is also an :ref:`HDF5 group ` representing the `root group` of the file. .. _file_open: Opening & creating files ------------------------ HDF5 files work generally like standard Python file objects. They support standard modes like r/w/a, and should be closed when they are no longer in use. However, there is obviously no concept of "text" vs "binary" mode. >>> f = h5py.File('myfile.hdf5','r') The file name may be a byte string or unicode string. Valid modes are: ======== ================================================ r Readonly, file must exist r+ Read/write, file must exist w Create file, truncate if exists w- or x Create file, fail if exists a Read/write if exists, create otherwise (default) ======== ================================================ .. _file_driver: File drivers ------------ HDF5 ships with a variety of different low-level drivers, which map the logical HDF5 address space to different storage mechanisms. You can specify which driver you want to use when the file is opened:: >>> f = h5py.File('myfile.hdf5', driver=, ) For example, the HDF5 "core" driver can be used to create a purely in-memory HDF5 file, optionally written out to disk when it is closed. Here's a list of supported drivers and their options: None **Strongly recommended.** Use the standard HDF5 driver appropriate for the current platform. On UNIX, this is the H5FD_SEC2 driver; on Windows, it is H5FD_WINDOWS. 'sec2' Unbuffered, optimized I/O using standard POSIX functions. 'stdio' Buffered I/O using functions from stdio.h. 'core' Memory-map the entire file; all operations are performed in memory and written back out when the file is closed. Keywords: backing_store: If True (default), save changes to a real file when closing. If False, the file exists purely in memory and is discarded when closed. block_size: Increment (in bytes) by which memory is extended. Default is 64k. 'family' Store the file on disk as a series of fixed-length chunks. Useful if the file system doesn't allow large files. Note: the filename you provide *must* contain a printf-style integer format code (e.g. %d"), which will be replaced by the file sequence number. Keywords: memb_size: Maximum file size (default is 2**31-1). .. _file_version: Version Bounding ---------------- HDF5 has been evolving for many years now. By default, the library will write objects in the most compatible fashion possible, so that older versions will still be able to read files generated by modern programs. However, there can be performance advantages if you are willing to forgo a certain level of backwards compatibility. By using the "libver" option to File, you can specify the minimum and maximum sophistication of these structures: >>> f = h5py.File('name.hdf5', libver='earliest') # most compatible >>> f = h5py.File('name.hdf5', libver='latest') # most modern Here "latest" means that HDF5 will always use the newest version of these structures without particular concern for backwards compatibility. The "earliest" option means that HDF5 will make a *best effort* to be backwards compatible. The default is "earliest". .. _file_userblock: User block ---------- HDF5 allows the user to insert arbitrary data at the beginning of the file, in a reserved space called the `user block`. The length of the user block must be specified when the file is created. It can be either zero (the default) or a power of two greater than or equal to 512. You can specify the size of the user block when creating a new file, via the ``userblock_size`` keyword to File; the userblock size of an open file can likewise be queried through the ``File.userblock_size`` property. Modifying the user block on an open file is not supported; this is a limitation of the HDF5 library. However, once the file is closed you are free to read and write data at the start of the file, provided your modifications don't leave the user block region. .. _file_filenames: Filenames on different systems ------------------------------ Different operating systems (and different file systems) store filenames with different encodings. Additionally, in Python there are at least two different representations of filenames, as encoded bytes (via str on Python 2, bytes on Python 3) or as a unicode string (via unicode on Python 2 and str on Python 3). The safest bet when creating a new file is to use unicode strings on all systems. macOS (OSX) ........... macOS is the simplest system to deal with, it only accepts UTF-8, so using unicode paths will just work (and should be preferred). Linux (and non-macOS Unix) .......................... Unix-like systems use locale settings to determine the correct encoding to use. These are set via a number of different environment variables, of which ``LANG`` and ``LC_ALL`` are the ones of most interest. Of special interest is the ``C`` locale, which Python will interpret as only allowing ASCII, meaning unicode paths should be preencoded. This will likely change in Python 3.7 with https://www.python.org/dev/peps/pep-0538/, but this will likely be backported by distributions to earlier versions. To summarise, use unicode strings where possible, but be aware that sometimes using encoded bytes may be necessary to read incorrectly encoded filenames. Windows ....... Windows systems have two different APIs to perform file-related operations, a ANSI (char, legacy) interface and a unicode (wchar) interface. HDF5 currently only supports the ANSI interface, which is limited in what it can encode. This means that it may not be possible to open certain files, and because :ref:`group_extlinks` do not specify their encoding, it is possible that opening an external link may not work. There is work being done to fix this (see https://github.com/h5py/h5py/issues/839), but it is likely there will need to be breaking changes make to allow Windows to have the same level of support for unicode filenames as other operating systems. The best suggestion is to use unicode strings, but to keep to ASCII for filenames to avoid possible breakage. Reference --------- .. note:: Unlike Python file objects, the attribute ``File.name`` gives the HDF5 name of the root group, "``/``". To access the on-disk name, use :attr:`File.filename`. .. class:: File(name, mode=None, driver=None, libver=None, userblock_size, **kwds) Open or create a new file. Note that in addition to the File-specific methods and properties listed below, File objects inherit the full interface of :class:`Group`. :param name: Name of file (`str` or `unicode`), or an instance of :class:`h5f.FileID` to bind to an existing file identifier. :param mode: Mode in which to open file; one of ("w", "r", "r+", "a", "w-"). See :ref:`file_open`. :param driver: File driver to use; see :ref:`file_driver`. :param libver: Compatibility bounds; see :ref:`file_version`. :param userblock_size: Size (in bytes) of the user block. If nonzero, must be a power of 2 and at least 512. See :ref:`file_userblock`. :param kwds: Driver-specific keywords; see :ref:`file_driver`. .. method:: close() Close this file. All open objects will become invalid. .. method:: flush() Request that the HDF5 library flush its buffers to disk. .. attribute:: id Low-level identifier (an instance of :class:`FileID `). .. attribute:: filename Name of this file on disk. Generally a Unicode string; a byte string will be used if HDF5 returns a non-UTF-8 encoded string. .. attribute:: mode String indicating if the file is open readonly ("r") or read-write ("r+"). Will always be one of these two values, regardless of the mode used to open the file. .. attribute:: driver String giving the driver used to open the file. Refer to :ref:`file_driver` for a list of drivers. .. attribute:: libver 2-tuple with library version settings. See :ref:`file_version`. .. attribute:: userblock_size Size of user block (in bytes). Generally 0. See :ref:`file_userblock`. h5py-2.7.1/docs/high/dims.rst0000644000175000017500000000654013025343121017420 0ustar tcaswelltcaswell00000000000000.. _dimension_scales: Dimension Scales ================ Datasets are multidimensional arrays. HDF5 provides support for labeling the dimensions and associating one or "dimension scales" with each dimension. A dimension scale is simply another HDF5 dataset. In principle, the length of the multidimensional array along the dimension of interest should be equal to the length of the dimension scale, but HDF5 does not enforce this property. The HDF5 library provides the H5DS API for working with dimension scales. H5py provides low-level bindings to this API in :mod:`h5py.h5ds`. These low-level bindings are in turn used to provide a high-level interface through the ``Dataset.dims`` property. Suppose we have the following data file:: f = File('foo.h5', 'w') f['data'] = np.ones((4, 3, 2), 'f') HDF5 allows the dimensions of ``data`` to be labeled, for example:: f['data'].dims[0].label = 'z' f['data'].dims[2].label = 'x' Note that the first dimension, which has a length of 4, has been labeled "z", the third dimension (in this case the fastest varying dimension), has been labeled "x", and the second dimension was given no label at all. We can also use HDF5 datasets as dimension scales. For example, if we have:: f['x1'] = [1, 2] f['x2'] = [1, 1.1] f['y1'] = [0, 1, 2] f['z1'] = [0, 1, 4, 9] We are going to treat the ``x1``, ``x2``, ``y1``, and ``z1`` datasets as dimension scales:: f['data'].dims.create_scale(f['x1']) f['data'].dims.create_scale(f['x2'], 'x2 name') f['data'].dims.create_scale(f['y1'], 'y1 name') f['data'].dims.create_scale(f['z1'], 'z1 name') When you create a dimension scale, you may provide a name for that scale. In this case, the ``x1`` scale was not given a name, but the others were. Now we can associate these dimension scales with the primary dataset:: f['data'].dims[0].attach_scale(f['z1']) f['data'].dims[1].attach_scale(f['y1']) f['data'].dims[2].attach_scale(f['x1']) f['data'].dims[2].attach_scale(f['x2']) Note that two dimension scales were associated with the third dimension of ``data``. You can also detach a dimension scale:: f['data'].dims[2].detach_scale(f['x2']) but for now, lets assume that we have both ``x1`` and ``x2`` still associated with the third dimension of ``data``. You can attach a dimension scale to any number of HDF5 datasets, you can even attach it to multiple dimensions of a single HDF5 dataset. Now that the dimensions of ``data`` have been labeled, and the dimension scales for the various axes have been specified, we have provided much more context with which ``data`` can be interpreted. For example, if you want to know the labels for the various dimensions of ``data``:: >>> [dim.label for dim in f['data'].dims] ['z', '', 'x'] If you want the names of the dimension scales associated with the "x" axis:: >>> f['data'].dims[2].keys() ['', 'x2 name'] :meth:`items` and :meth:`values` methods are also provided. The dimension scales themselves can also be accessed with:: f['data'].dims[2][1] or:: f['data'].dims[2]['x2 name'] such that:: >>> f['data'].dims[2][1] == f['x2'] True though, beware that if you attempt to index the dimension scales with a string, the first dimension scale whose name matches the string is the one that will be returned. There is no guarantee that the name of the dimension scale is unique. h5py-2.7.1/docs/high/dataset.rst0000644000175000017500000003774713114357361020140 0ustar tcaswelltcaswell00000000000000.. _dataset: Datasets ======== Datasets are very similar to NumPy arrays. They are homogenous collections of data elements, with an immutable datatype and (hyper)rectangular shape. Unlike NumPy arrays, they support a variety of transparent storage features such as compression, error-detection, and chunked I/O. They are represented in h5py by a thin proxy class which supports familiar NumPy operations like slicing, along with a variety of descriptive attributes: - **shape** attribute - **size** attribute - **dtype** attribute .. _dataset_create: Creating datasets ----------------- New datasets are created using either :meth:`Group.create_dataset` or :meth:`Group.require_dataset`. Existing datasets should be retrieved using the group indexing syntax (``dset = group["name"]``). To make an empty dataset, all you have to do is specify a name, shape, and optionally the data type (defaults to ``'f'``):: >>> dset = f.create_dataset("default", (100,)) >>> dset = f.create_dataset("ints", (100,), dtype='i8') You may initialize the dataset to an existing NumPy array:: >>> arr = np.arange(100) >>> dset = f.create_dataset("init", data=arr) Keywords ``shape`` and ``dtype`` may be specified along with ``data``; if so, they will override ``data.shape`` and ``data.dtype``. It's required that (1) the total number of points in ``shape`` match the total number of points in ``data.shape``, and that (2) it's possible to cast ``data.dtype`` to the requested ``dtype``. .. _dataset_chunks: Chunked storage --------------- An HDF5 dataset created with the default settings will be `contiguous`; in other words, laid out on disk in traditional C order. Datasets may also be created using HDF5's `chunked` storage layout. This means the dataset is divided up into regularly-sized pieces which are stored haphazardly on disk, and indexed using a B-tree. Chunked storage makes it possible to resize datasets, and because the data is stored in fixed-size chunks, to use compression filters. To enable chunked storage, set the keyword ``chunks`` to a tuple indicating the chunk shape:: >>> dset = f.create_dataset("chunked", (1000, 1000), chunks=(100, 100)) Data will be read and written in blocks with shape (100,100); for example, the data in ``dset[0:100,0:100]`` will be stored together in the file, as will the data points in range ``dset[400:500, 100:200]``. Chunking has performance implications. It's recommended to keep the total size of your chunks between 10 KiB and 1 MiB, larger for larger datasets. Also keep in mind that when any element in a chunk is accessed, the entire chunk is read from disk. Since picking a chunk shape can be confusing, you can have h5py guess a chunk shape for you:: >>> dset = f.create_dataset("autochunk", (1000, 1000), chunks=True) Auto-chunking is also enabled when using compression or ``maxshape``, etc., if a chunk shape is not manually specified. .. _dataset_resize: Resizable datasets ------------------ In HDF5, datasets can be resized once created up to a maximum size, by calling :meth:`Dataset.resize`. You specify this maximum size when creating the dataset, via the keyword ``maxshape``:: >>> dset = f.create_dataset("resizable", (10,10), maxshape=(500, 20)) Any (or all) axes may also be marked as "unlimited", in which case they may be increased up to the HDF5 per-axis limit of 2**64 elements. Indicate these axes using ``None``:: >>> dset = f.create_dataset("unlimited", (10, 10), maxshape=(None, 10)) .. note:: Resizing an array with existing data works differently than in NumPy; if any axis shrinks, the data in the missing region is discarded. Data does not "rearrange" itself as it does when resizing a NumPy array. .. _dataset_compression: Filter pipeline --------------- Chunked data may be transformed by the HDF5 `filter pipeline`. The most common use is applying transparent compression. Data is compressed on the way to disk, and automatically decompressed when read. Once the dataset is created with a particular compression filter applied, data may be read and written as normal with no special steps required. Enable compression with the ``compression`` keyword to :meth:`Group.create_dataset`:: >>> dset = f.create_dataset("zipped", (100, 100), compression="gzip") Options for each filter may be specified with ``compression_opts``:: >>> dset = f.create_dataset("zipped_max", (100, 100), compression="gzip", compression_opts=9) Lossless compression filters ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ GZIP filter (``"gzip"``) Available with every installation of HDF5, so it's best where portability is required. Good compression, moderate speed. ``compression_opts`` sets the compression level and may be an integer from 0 to 9, default is 4. LZF filter (``"lzf"``) Available with every installation of h5py (C source code also available). Low to moderate compression, very fast. No options. SZIP filter (``"szip"``) Patent-encumbered filter used in the NASA community. Not available with all installations of HDF5 due to legal reasons. Consult the HDF5 docs for filter options. Custom compression filters ~~~~~~~~~~~~~~~~~~~~~~~~~~ In addition to the compression filters listed above, compression filters can be dynamically loaded by the underlying HDF5 library. This is done by passing a filter number to :meth:`Group.create_dataset` as the ``compression`` parameter. The ``compression_opts`` parameter will then be passed to this filter. .. note:: The underlying implementation of the compression filter will have the ``H5Z_FLAG_OPTIONAL`` flag set. This indicates that if the compression filter doesn't compress a block while writing, no error will be thrown. The filter will then be skipped when subsequently reading the block. .. _dataset_scaleoffset: Scale-Offset filter ~~~~~~~~~~~~~~~~~~~ Filters enabled with the ``compression`` keywords are _lossless_; what comes out of the dataset is exactly what you put in. HDF5 also includes a lossy filter which trades precision for storage space. Works with integer and floating-point data only. Enable the scale-offset filter by setting :meth:`Group.create_dataset` keyword ``scaleoffset`` to an integer. For integer data, this specifies the number of bits to retain. Set to 0 to have HDF5 automatically compute the number of bits required for lossless compression of the chunk. For floating-point data, indicates the number of digits after the decimal point to retain. .. _dataset_shuffle: Shuffle filter ~~~~~~~~~~~~~~ Block-oriented compressors like GZIP or LZF work better when presented with runs of similar values. Enabling the shuffle filter rearranges the bytes in the chunk and may improve compression ratio. No significant speed penalty, lossless. Enable by setting :meth:`Group.create_dataset` keyword ``shuffle`` to True. .. _dataset_fletcher32: Fletcher32 filter ~~~~~~~~~~~~~~~~~ Adds a checksum to each chunk to detect data corruption. Attempts to read corrupted chunks will fail with an error. No significant speed penalty. Obviously shouldn't be used with lossy compression filters. Enable by setting :meth:`Group.create_dataset` keyword ``fletcher32`` to True. .. _dataset_slicing: Reading & writing data ---------------------- HDF5 datasets re-use the NumPy slicing syntax to read and write to the file. Slice specifications are translated directly to HDF5 "hyperslab" selections, and are a fast and efficient way to access data in the file. The following slicing arguments are recognized: * Indices: anything that can be converted to a Python long * Slices (i.e. ``[:]`` or ``[0:10]``) * Field names, in the case of compound data * At most one ``Ellipsis`` (``...``) object Here are a few examples (output omitted) >>> dset = f.create_dataset("MyDataset", (10,10,10), 'f') >>> dset[0,0,0] >>> dset[0,2:10,1:9:3] >>> dset[:,::2,5] >>> dset[0] >>> dset[1,5] >>> dset[0,...] >>> dset[...,6] For compound data, you can specify multiple field names alongside the numeric slices: >>> dset["FieldA"] >>> dset[0,:,4:5, "FieldA", "FieldB"] >>> dset[0, ..., "FieldC"] To retrieve the contents of a `scalar` dataset, you can use the same syntax as in NumPy: ``result = dset[()]``. In other words, index into the dataset using an empty tuple. For simple slicing, broadcasting is supported: >>> dset[0,:,:] = np.arange(10) # Broadcasts to (10,10) Broadcasting is implemented using repeated hyperslab selections, and is safe to use with very large target selections. It is supported for the above "simple" (integer, slice and ellipsis) slicing only. .. _dataset_fancy: Fancy indexing -------------- A subset of the NumPy fancy-indexing syntax is supported. Use this with caution, as the underlying HDF5 mechanisms may have different performance than you expect. For any axis, you can provide an explicit list of points you want; for a dataset with shape (10, 10):: >>> dset.shape (10, 10) >>> result = dset[0, [1,3,8]] >>> result.shape (3,) >>> result = dset[1:6, [5,8,9]] >>> result.shape (5, 3) The following restrictions exist: * List selections may not be empty * Selection coordinates must be given in increasing order * Duplicate selections are ignored * Very long lists (> 1000 elements) may produce poor performance NumPy boolean "mask" arrays can also be used to specify a selection. The result of this operation is a 1-D array with elements arranged in the standard NumPy (C-style) order. Behind the scenes, this generates a laundry list of points to select, so be careful when using it with large masks:: >>> arr = numpy.arange(100).reshape((10,10)) >>> dset = f.create_dataset("MyDataset", data=arr) >>> result = dset[arr > 50] >>> result.shape (49,) .. _dataset_iter: Length and iteration -------------------- As with NumPy arrays, the ``len()`` of a dataset is the length of the first axis, and iterating over a dataset iterates over the first axis. However, modifications to the yielded data are not recorded in the file. Resizing a dataset while iterating has undefined results. On 32-bit platforms, ``len(dataset)`` will fail if the first axis is bigger than 2**32. It's recommended to use :meth:`Dataset.len` for large datasets. Creating and Reading Empty (or Null) datasets and attributes ------------------------------------------------------------ HDF5 has the concept of Empty or Null datasets and attributes. These are not the same as an array with a shape of (), or a scalar dataspace in HDF5 terms. Instead, it is a dataset with an associated type, no data, and no shape. In h5py, we represent this as either a dataset with shape ``None``, or an instance of ``h5py.Empty``. Empty datasets and attributes cannot be sliced. To create an empty attribute, use ``h5py.Empty`` as per :ref:`attributes`:: >>> obj.attrs["EmptyAttr"] = h5py.Empty("f") Similarly, reading an empty attribute returns ``h5py.Empty``:: >>> obj.attrs["EmptyAttr"] h5py.Empty(dtype="f") Empty datasets can be created either by defining a ``dtype`` but no ``shape`` in ``create_dataset``:: >>> grp.create_dataset("EmptyDataset", dtype="f") or by ``data`` to an instance of ``h5py.Empty``:: >>> grp.create_dataset("EmptyDataset", data=h5py.Empty("f")) An empty dataset has shape defined as ``None``, which is the best way of determining whether a dataset is empty or not. An empty dataset can be "read" in a similar way to scalar datasets, i.e. if ``empty_dataset`` is an empty dataset,:: >>> empty_dataset[()] h5py.Empty(dtype="f") The dtype of the dataset can be accessed via ``.dtype`` as per normal. As empty datasets cannot be sliced, some methods of datasets such as ``read_direct`` will raise an exception if used on a empty dataset. Reference --------- .. class:: Dataset(identifier) Dataset objects are typically created via :meth:`Group.create_dataset`, or by retrieving existing datasets from a file. Call this constructor to create a new Dataset bound to an existing :class:`DatasetID ` identifier. .. method:: __getitem__(args) NumPy-style slicing to retrieve data. See :ref:`dataset_slicing`. .. method:: __setitem__(args) NumPy-style slicing to write data. See :ref:`dataset_slicing`. .. method:: read_direct(array, source_sel=None, dest_sel=None) Read from an HDF5 dataset directly into a NumPy array, which can avoid making an intermediate copy as happens with slicing. The destination array must be C-contiguous and writable, and must have a datatype to which the source data may be cast. Data type conversion will be carried out on the fly by HDF5. `source_sel` and `dest_sel` indicate the range of points in the dataset and destination array respectively. Use the output of ``numpy.s_[args]``:: >>> dset = f.create_dataset("dset", (100,), dtype='int64') >>> arr = np.zeros((100,), dtype='int32') >>> dset.read_direct(arr, np.s_[0:10], np.s_[50:60]) .. method:: astype(dtype) Return a context manager allowing you to read data as a particular type. Conversion is handled by HDF5 directly, on the fly:: >>> dset = f.create_dataset("bigint", (1000,), dtype='int64') >>> with dset.astype('int16'): ... out = dset[:] >>> out.dtype dtype('int16') .. method:: resize(size, axis=None) Change the shape of a dataset. `size` may be a tuple giving the new dataset shape, or an integer giving the new length of the specified `axis`. Datasets may be resized only up to :attr:`Dataset.maxshape`. .. method:: len() Return the size of the first axis. .. attribute:: shape NumPy-style shape tuple giving dataset dimensions. .. attribute:: dtype NumPy dtype object giving the dataset's type. .. attribute:: size Integer giving the total number of elements in the dataset. .. attribute:: maxshape NumPy-style shape tuple indicating the maxiumum dimensions up to which the dataset may be resized. Axes with ``None`` are unlimited. .. attribute:: chunks Tuple giving the chunk shape, or None if chunked storage is not used. See :ref:`dataset_chunks`. .. attribute:: compression String with the currently applied compression filter, or None if compression is not enabled for this dataset. See :ref:`dataset_compression`. .. attribute:: compression_opts Options for the compression filter. See :ref:`dataset_compression`. .. attribute:: scaleoffset Setting for the HDF5 scale-offset filter (integer), or None if scale-offset compression is not used for this dataset. See :ref:`dataset_scaleoffset`. .. attribute:: shuffle Whether the shuffle filter is applied (T/F). See :ref:`dataset_shuffle`. .. attribute:: fletcher32 Whether Fletcher32 checksumming is enabled (T/F). See :ref:`dataset_fletcher32`. .. attribute:: fillvalue Value used when reading uninitialized portions of the dataset, or None if no fill value has been defined, in which case HDF5 will use a type-appropriate default value. Can't be changed after the dataset is created. .. attribute:: dims Access to :ref:`dimension_scales`. .. attribute:: attrs :ref:`attributes` for this dataset. .. attribute:: id The dataset's low-level identifer; an instance of :class:`DatasetID `. .. attribute:: ref An HDF5 object reference pointing to this dataset. See :ref:`refs_object`. .. attribute:: regionref Proxy object for creating HDF5 region references. See :ref:`refs_region`. .. attribute:: name String giving the full path to this dataset. .. attribute:: file :class:`File` instance in which this dataset resides .. attribute:: parent :class:`Group` instance containing this dataset. h5py-2.7.1/docs/conf.py0000644000175000017500000002031313152363105016311 0ustar tcaswelltcaswell00000000000000# -*- coding: utf-8 -*- # # h5py documentation build configuration file, created by # sphinx-quickstart on Fri Jan 31 11:23:59 2014. # # This file is execfile()d with the current directory set to its # containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. # # All configuration values have a default; values that are commented out # serve to show the default. import sys import os # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. #sys.path.insert(0, os.path.abspath('.')) # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. #needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = ['sphinx.ext.intersphinx', 'sphinx.ext.extlinks'] intersphinx_mapping = {'low': ('http://api.h5py.org', None)} extlinks = {'issue': ('https://github.com/h5py/h5py/issues/%s', 'GH')} # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # The suffix of source filenames. source_suffix = '.rst' # The encoding of source files. #source_encoding = 'utf-8-sig' # The master toctree document. master_doc = 'index' # General information about the project. project = u'h5py' copyright = u'2014, Andrew Collette and contributors' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. version = '2.7' # The full version, including alpha/beta/rc tags. release = '2.7.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. #language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: #today = '' # Else, today_fmt is used as the format for a strftime call. #today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. exclude_patterns = ['_build'] # The reST default role (used for this markup: `text`) to use for all # documents. #default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. #add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). #add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. #show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. #modindex_common_prefix = [] # If true, keep warnings as "system message" paragraphs in the built documents. #keep_warnings = False # -- Options for HTML output ---------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. html_theme = 'default' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. #html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. #html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". #html_title = None # A shorter title for the navigation bar. Default is the same as html_title. #html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. #html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. #html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". # html_static_path = ['_static'] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. #html_extra_path = [] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. #html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. #html_use_smartypants = True # Custom sidebar templates, maps document names to template names. #html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. #html_additional_pages = {} # If false, no module index is generated. #html_domain_indices = True # If false, no index is generated. #html_use_index = True # If true, the index is split into individual pages for each letter. #html_split_index = False # If true, links to the reST sources are added to the pages. #html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. #html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. #html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. #html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). #html_file_suffix = None # Output file base name for HTML help builder. htmlhelp_basename = 'h5pydoc' # -- Options for LaTeX output --------------------------------------------- latex_elements = { # The paper size ('letterpaper' or 'a4paper'). #'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). #'pointsize': '10pt', # Additional stuff for the LaTeX preamble. #'preamble': '', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ ('index', 'h5py.tex', u'h5py Documentation', u'Andrew Collette and contributors', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of # the title page. #latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. #latex_use_parts = False # If true, show page references after internal links. #latex_show_pagerefs = False # If true, show URL addresses after external links. #latex_show_urls = False # Documents to append as an appendix to all manuals. #latex_appendices = [] # If false, no module index is generated. #latex_domain_indices = True # -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ ('index', 'h5py', u'h5py Documentation', [u'Andrew Collette and contributors'], 1) ] # If true, show URL addresses after external links. #man_show_urls = False # -- Options for Texinfo output ------------------------------------------- # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ ('index', 'h5py', u'h5py Documentation', u'Andrew Collette and contributors', 'h5py', 'One line description of project.', 'Miscellaneous'), ] # Documents to append as an appendix to all manuals. #texinfo_appendices = [] # If false, no module index is generated. #texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. #texinfo_show_urls = 'footnote' # If true, do not generate a @detailmenu in the "Top" node's menu. #texinfo_no_detailmenu = False h5py-2.7.1/docs/special.rst0000644000175000017500000001051113025343121017156 0ustar tcaswelltcaswell00000000000000Special types ============= HDF5 supports a few types which have no direct NumPy equivalent. Among the most useful and widely used are *variable-length* (VL) types, and enumerated types. As of version 2.3, h5py fully supports HDF5 enums and VL types. How special types are represented --------------------------------- Since there is no direct NumPy dtype for variable-length strings, enums or references, h5py extends the dtype system slightly to let HDF5 know how to store these types. Each type is represented by a native NumPy dtype, with a small amount of metadata attached. NumPy routines ignore the metadata, but h5py can use it to determine how to store the data. There are two functions for creating these "hinted" dtypes: .. function:: special_dtype(**kwds) Create a NumPy dtype object containing type hints. Only one keyword may be specified. :param vlen: Base type for HDF5 variable-length datatype. :param enum: 2-tuple ``(basetype, values_dict)``. ``basetype`` must be an integer dtype; ``values_dict`` is a dictionary mapping string names to integer values. :param ref: Provide class ``h5py.Reference`` or ``h5py.RegionReference`` to create a type representing object or region references respectively. .. function:: check_dtype(**kwds) Determine if the given dtype object is a special type. Example:: >>> out = h5py.check_dtype(vlen=mydtype) >>> if out is not None: ... print "Vlen of type %s" % out str :param vlen: Check for an HDF5 variable-length type; returns base class :param enum: Check for an enumerated type; returns 2-tuple ``(basetype, values_dict)``. :param ref: Check for an HDF5 object or region reference; returns either ``h5py.Reference`` or ``h5py.RegionReference``. Variable-length strings ----------------------- In HDF5, data in VL format is stored as arbitrary-length vectors of a base type. In particular, strings are stored C-style in null-terminated buffers. NumPy has no native mechanism to support this. Unfortunately, this is the de facto standard for representing strings in the HDF5 C API, and in many HDF5 applications. Thankfully, NumPy has a generic pointer type in the form of the "object" ("O") dtype. In h5py, variable-length strings are mapped to object arrays. A small amount of metadata attached to an "O" dtype tells h5py that its contents should be converted to VL strings when stored in the file. Existing VL strings can be read and written to with no additional effort; Python strings and fixed-length NumPy strings can be auto-converted to VL data and stored. Here's an example showing how to create a VL array of strings:: >>> f = h5py.File('foo.hdf5') >>> dt = h5py.special_dtype(vlen=str) >>> ds = f.create_dataset('VLDS', (100,100), dtype=dt) >>> ds.dtype.kind 'O' >>> h5py.check_dtype(vlen=ds.dtype) .. _vlen: Arbitrary vlen data ------------------- Starting with h5py 2.3, variable-length types are not restricted to strings. For example, you can create a "ragged" array of integers:: >>> dt = h5py.special_dtype(vlen=np.dtype('int32')) >>> dset = f.create_dataset('vlen_int', (100,), dtype=dt) >>> dset[0] = [1,2,3] >>> dset[1] = [1,2,3,4,5] Single elements are read as NumPy arrays:: >>> dset[0] array([1, 2, 3], dtype=int32) Multidimensional selections produce an object array whose members are integer arrays:: >>> dset[0:2] array([array([1, 2, 3], dtype=int32), array([1, 2, 3, 4, 5], dtype=int32)], dtype=object) Enumerated types ---------------- HDF5 has the concept of an *enumerated type*, which is an integer datatype with a restriction to certain named values. Since NumPy has no such datatype, HDF5 ENUM types are read and written as integers. Here's an example of creating an enumerated type:: >>> dt = h5py.special_dtype(enum=('i', {"RED": 0, "GREEN": 1, "BLUE": 42})) >>> h5py.check_dtype(enum=dt) {'BLUE': 42, 'GREEN': 1, 'RED': 0} >>> f = h5py.File('foo.hdf5','w') >>> ds = f.create_dataset("EnumDS", (100,100), dtype=dt) >>> ds.dtype.kind 'i' >>> ds[0,:] = 42 >>> ds[0,0] 42 >>> ds[1,0] 0 Object and region references ---------------------------- References have their :ref:`own section `. h5py-2.7.1/docs/config.rst0000644000175000017500000000451113025343121017006 0ustar tcaswelltcaswell00000000000000Configuring h5py ================ Library configuration --------------------- A few library options are available to change the behavior of the library. You can get a reference to the global library configuration object via the function ``h5py.get_config()``. This object supports the following attributes: **complex_names** Set to a 2-tuple of strings (real, imag) to control how complex numbers are saved. The default is ('r','i'). **bool_names** Booleans are saved as HDF5 enums. Set this to a 2-tuple of strings (false, true) to control the names used in the enum. The default is ("FALSE", "TRUE"). IPython ------- H5py ships with a custom ipython completer, which provides object introspection and tab completion for h5py objects in an ipython session. For example, if a file contains 3 groups, "foo", "bar", and "baz":: In [4]: f['b bar baz In [4]: f['f # Completes to: In [4]: f['foo' In [4]: f['foo']. f['foo'].attrs f['foo'].items f['foo'].ref f['foo'].copy f['foo'].iteritems f['foo'].require_dataset f['foo'].create_dataset f['foo'].iterkeys f['foo'].require_group f['foo'].create_group f['foo'].itervalues f['foo'].values f['foo'].file f['foo'].keys f['foo'].visit f['foo'].get f['foo'].name f['foo'].visititems f['foo'].id f['foo'].parent The easiest way to enable the custom completer is to do the following in an IPython session:: In [1]: import h5py In [2]: h5py.enable_ipython_completer() It is also possible to configure IPython to enable the completer every time you start a new session. For >=ipython-0.11, "h5py.ipy_completer" just needs to be added to the list of extensions in your ipython config file, for example :file:`~/.config/ipython/profile_default/ipython_config.py` (if this file does not exist, you can create it by invoking `ipython profile create`):: c = get_config() c.InteractiveShellApp.extensions = ['h5py.ipy_completer'] For `_. This is highly recommended reading for anyone intending to use the SWMR feature even through h5py. For production systems in particular pay attention to the file system requirements regarding POSIX I/O semantics. Using the SWMR feature from h5py -------------------------------- The following basic steps are typically required by writer and reader processes: - Writer process create the target file and all groups, datasets and attributes. - Writer process switch file into SWMR mode. - Reader process can open the file with swmr=True. - Writer writes and/or appends data to existing datasets (new groups and datasets *cannot* be created when in SWMR mode). - Writer regularly flushes the target dataset to make it visible to reader processes. - Reader refreshes target dataset before reading new meta-data and/or main data. - Writer eventually completes and close the file as normal. - Reader can finish and close file as normal whenever it is convenient. The following snippet demonstrate a SWMR writer appending to a single dataset:: f = h5py.File("swmr.h5", 'w', libver='latest') arr = np.array([1,2,3,4]) dset = f.create_dataset("data", chunks=(2,), maxshape=(None,), data=arr) f.swmr_mode = True # Now it is safe for the reader to open the swmr.h5 file for i in range(5): new_shape = ((i+1) * len(arr), ) dset.resize( new_shape ) dset[i*len(arr):] = arr dset.flush() # Notify the reader process that new data has been written The following snippet demonstrate how to monitor a dataset as a SWMR reader:: f = h5py.File("swmr.h5", 'r', libver='latest', swmr=True) dset = f["data"] while True: dset.id.refresh() shape = dset.shape print( shape ) Examples -------- In addition to the above example snippets, a few more complete examples can be found in the examples folder. These examples are described in the following sections Dataset monitor with inotify ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The inotify example demonstrate how to use SWMR in a reading application which monitors live progress as a dataset is being written by another process. This example uses the the linux inotify (`pyinotify `_ python bindings) to receive a signal each time the target file has been updated. .. literalinclude:: ../examples/swmr_inotify_example.py Multiprocess concurrent write and read ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The SWMR multiprocess example starts starts two concurrent child processes: a writer and a reader. The writer process first creates the target file and dataset. Then it switches the file into SWMR mode and the reader process is notified (with a multiprocessing.Event) that it is safe to open the file for reading. The writer process then continue to append chunks to the dataset. After each write it notifies the reader that new data has been written. Whether the new data is visible in the file at this point is subject to OS and file system latencies. The reader first waits for the initial "SWMR mode" notification from the writer, upon which it goes into a loop where it waits for further notifications from the writer. The reader may drop some notifications, but for each one received it will refresh the dataset and read the dimensions. After a time-out it will drop out of the loop and exit. .. literalinclude:: ../examples/swmr_multiprocess.py The example output below (from a virtual Ubuntu machine) illustrate some latency between the writer and reader:: python examples/swmr_multiprocess.py INFO 2015-02-26 18:05:03,195 root Starting reader INFO 2015-02-26 18:05:03,196 root Starting reader INFO 2015-02-26 18:05:03,197 reader Waiting for initial event INFO 2015-02-26 18:05:03,197 root Waiting for writer to finish INFO 2015-02-26 18:05:03,198 writer Creating file swmrmp.h5 INFO 2015-02-26 18:05:03,203 writer SWMR mode INFO 2015-02-26 18:05:03,205 reader Opening file swmrmp.h5 INFO 2015-02-26 18:05:03,210 writer Resizing dset shape: (4,) INFO 2015-02-26 18:05:03,212 writer Sending event INFO 2015-02-26 18:05:03,213 reader Read dset shape: (4,) INFO 2015-02-26 18:05:03,214 writer Resizing dset shape: (8,) INFO 2015-02-26 18:05:03,214 writer Sending event INFO 2015-02-26 18:05:03,215 writer Resizing dset shape: (12,) INFO 2015-02-26 18:05:03,215 writer Sending event INFO 2015-02-26 18:05:03,215 writer Resizing dset shape: (16,) INFO 2015-02-26 18:05:03,215 reader Read dset shape: (12,) INFO 2015-02-26 18:05:03,216 writer Sending event INFO 2015-02-26 18:05:03,216 writer Resizing dset shape: (20,) INFO 2015-02-26 18:05:03,216 reader Read dset shape: (16,) INFO 2015-02-26 18:05:03,217 writer Sending event INFO 2015-02-26 18:05:03,217 reader Read dset shape: (20,) INFO 2015-02-26 18:05:03,218 reader Read dset shape: (20,) INFO 2015-02-26 18:05:03,219 root Waiting for reader to finish h5py-2.7.1/docs/index.rst0000644000175000017500000000246513143100476016663 0ustar tcaswelltcaswell00000000000000HDF5 for Python =============== The h5py package is a Pythonic interface to the HDF5 binary data format. `HDF5 `_ lets you store huge amounts of numerical data, and easily manipulate that data from NumPy. For example, you can slice into multi-terabyte datasets stored on disk, as if they were real NumPy arrays. Thousands of datasets can be stored in a single file, categorized and tagged however you want. Where to start -------------- * :ref:`Quick-start guide ` * :ref:`Installation ` Other resources --------------- * `Python and HDF5 O'Reilly book `_ * `Ask questions on the mailing list at Google Groups `_ * `GitHub project `_ Introductory info ----------------- .. toctree:: :maxdepth: 1 quick build High-level API reference ------------------------ .. toctree:: :maxdepth: 1 high/file high/group high/dataset high/attr high/dims Advanced topics --------------- .. toctree:: :maxdepth: 1 config special strings refs mpi swmr Meta-info about the h5py project -------------------------------- .. toctree:: :maxdepth: 1 whatsnew/index contributing faq licenses h5py-2.7.1/docs/mpi.rst0000644000175000017500000001130313025343121016323 0ustar tcaswelltcaswell00000000000000.. _parallel: Parallel HDF5 ============= Starting with version 2.2.0, h5py includes support for Parallel HDF5. This is the "native" way to use HDF5 in a parallel computing environment. How does Parallel HDF5 work? ---------------------------- Parallel HDF5 is a configuration of the HDF5 library which lets you share open files across multiple parallel processes. It uses the MPI (Message Passing Interface) standard for interprocess communication. Consequently, when using Parallel HDF5 from Python, your application will also have to use the MPI library. This is accomplished through the `mpi4py `_ Python package, which provides excellent, complete Python bindings for MPI. Here's an example "Hello World" using ``mpi4py``:: >>> from mpi4py import MPI >>> print "Hello World (from process %d)" % MPI.COMM_WORLD.Get_rank() To run an MPI-based parallel program, use the ``mpiexec`` program to launch several parallel instances of Python:: $ mpiexec -n 4 python demo.py Hello World (from process 1) Hello World (from process 2) Hello World (from process 3) Hello World (from process 0) The ``mpi4py`` package includes all kinds of mechanisms to share data between processes, synchronize, etc. It's a different flavor of parallelism than, say, threads or ``multiprocessing``, but easy to get used to. Check out the `mpi4py web site `_ for more information and a great tutorial. Building against Parallel HDF5 ------------------------------ HDF5 must be built with at least the following options:: $./configure --enable-parallel --enable-shared Note that ``--enable-shared`` is required. Often, a "parallel" version of HDF5 will be available through your package manager. You can check to see what build options were used by using the program ``h5cc``:: $ h5cc -showconfig Once you've got a Parallel-enabled build of HDF5, h5py has to be compiled in "MPI mode". This is simple; set your default compiler to the ``mpicc`` wrapper and build h5py with the ``--mpi`` option:: $ export CC=mpicc $ python setup.py configure --mpi [--hdf5=/path/to/parallel/hdf5] $ python setup.py build Using Parallel HDF5 from h5py ----------------------------- The parallel features of HDF5 are mostly transparent. To open a file shared across multiple processes, use the ``mpio`` file driver. Here's an example program which opens a file, creates a single dataset and fills it with the process ID:: from mpi4py import MPI import h5py rank = MPI.COMM_WORLD.rank # The process ID (integer 0-3 for 4-process run) f = h5py.File('parallel_test.hdf5', 'w', driver='mpio', comm=MPI.COMM_WORLD) dset = f.create_dataset('test', (4,), dtype='i') dset[rank] = rank f.close() Run the program:: $ mpiexec -n 4 python demo2.py Looking at the file with ``h5dump``:: $ h5dump parallel_test.hdf5 HDF5 "parallel_test.hdf5" { GROUP "/" { DATASET "test" { DATATYPE H5T_STD_I32LE DATASPACE SIMPLE { ( 4 ) / ( 4 ) } DATA { (0): 0, 1, 2, 3 } } } } Collective versus independent operations ---------------------------------------- MPI-based programs work by launching many instances of the Python interpreter, each of which runs your script. There are certain requirements imposed on what each process can do. Certain operations in HDF5, for example, anything which modifies the file metadata, must be performed by all processes. Other operations, for example, writing data to a dataset, can be performed by some processes and not others. These two classes are called *collective* and *independent* operations. Anything which modifies the *structure* or metadata of a file must be done collectively. For example, when creating a group, each process must participate:: >>> grp = f.create_group('x') # right >>> if rank == 1: ... grp = f.create_group('x') # wrong; all processes must do this On the other hand, writing data to a dataset can be done independently:: >>> if rank > 2: ... dset[rank] = 42 # this is fine MPI atomic mode --------------- HDF5 versions 1.8.9+ support the MPI "atomic" file access mode, which trades speed for more stringent consistency requirements. Once you've opened a file with the ``mpio`` driver, you can place it in atomic mode using the settable ``atomic`` property:: >>> f = h5py.File('parallel_test.hdf5', 'w', driver='mpio', comm=MPI.COMM_WORLD) >>> f.atomic = True More information ---------------- Parallel HDF5 is a new feature in h5py. If you have any questions, feel free to ask on the mailing list (h5py at google groups). We welcome bug reports, enhancements and general inquiries. h5py-2.7.1/docs/licenses.rst0000644000175000017500000002671413025343121017357 0ustar tcaswelltcaswell00000000000000Licenses and legal info ======================= Copyright Notice and Statement for the h5py Project --------------------------------------------------- :: Copyright (c) 2008 Andrew Collette and contributors http://h5py.alfven.org All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: a. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. b. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. c. Neither the name of the author nor the names of contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. HDF5 Copyright Statement ------------------------ :: HDF5 (Hierarchical Data Format 5) Software Library and Utilities Copyright 2006-2007 by The HDF Group (THG). NCSA HDF5 (Hierarchical Data Format 5) Software Library and Utilities Copyright 1998-2006 by the Board of Trustees of the University of Illinois. All rights reserved. Contributors: National Center for Supercomputing Applications (NCSA) at the University of Illinois, Fortner Software, Unidata Program Center (netCDF), The Independent JPEG Group (JPEG), Jean-loup Gailly and Mark Adler (gzip), and Digital Equipment Corporation (DEC). Redistribution and use in source and binary forms, with or without modification, are permitted for any purpose (including commercial purposes) provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions, and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions, and the following disclaimer in the documentation and/or materials provided with the distribution. 3. In addition, redistributions of modified forms of the source or binary code must carry prominent notices stating that the original code was changed and the date of the change. 4. All publications or advertising materials mentioning features or use of this software are asked, but not required, to acknowledge that it was developed by The HDF Group and by the National Center for Supercomputing Applications at the University of Illinois at Urbana-Champaign and credit the contributors. 5. Neither the name of The HDF Group, the name of the University, nor the name of any Contributor may be used to endorse or promote products derived from this software without specific prior written permission from THG, the University, or the Contributor, respectively. DISCLAIMER: THIS SOFTWARE IS PROVIDED BY THE HDF GROUP (THG) AND THE CONTRIBUTORS "AS IS" WITH NO WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED. In no event shall THG or the Contributors be liable for any damages suffered by the users arising out of the use of this software, even if advised of the possibility of such damage. Portions of HDF5 were developed with support from the University of California, Lawrence Livermore National Laboratory (UC LLNL). The following statement applies to those portions of the product and must be retained in any redistribution of source code, binaries, documentation, and/or accompanying materials: This work was partially produced at the University of California, Lawrence Livermore National Laboratory (UC LLNL) under contract no. W-7405-ENG-48 (Contract 48) between the U.S. Department of Energy (DOE) and The Regents of the University of California (University) for the operation of UC LLNL. DISCLAIMER: This work was prepared as an account of work sponsored by an agency of the United States Government. Neither the United States Government nor the University of California nor any of their employees, makes any warranty, express or implied, or assumes any liability or responsibility for the accuracy, completeness, or usefulness of any information, apparatus, product, or process disclosed, or represents that its use would not infringe privately- owned rights. Reference herein to any specific commercial products, process, or service by trade name, trademark, manufacturer, or otherwise, does not necessarily constitute or imply its endorsement, recommendation, or favoring by the United States Government or the University of California. The views and opinions of authors expressed herein do not necessarily state or reflect those of the United States Government or the University of California, and shall not be used for advertising or product endorsement purposes. PyTables Copyright Statement ---------------------------- :: Copyright Notice and Statement for PyTables Software Library and Utilities: Copyright (c) 2002, 2003, 2004 Francesc Altet Copyright (c) 2005, 2006, 2007 Carabos Coop. V. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: a. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. b. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. c. Neither the name of the Carabos Coop. V. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. stdint.h (Windows version) License ---------------------------------- :: Copyright (c) 2006-2008 Alexander Chemeris Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. The name of the author may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. Python license -------------- #. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and the Individual or Organization ("Licensee") accessing and otherwise using Python Python 2.7.5 software in source or binary form and its associated documentation. #. Subject to the terms and conditions of this License Agreement, PSF hereby grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative works, distribute, and otherwise use Python Python 2.7.5 alone or in any derivative version, provided, however, that PSF's License Agreement and PSF's notice of copyright, i.e., "Copyright 2001-2013 Python Software Foundation; All Rights Reserved" are retained in Python Python 2.7.5 alone or in any derivative version prepared by Licensee. #. In the event Licensee prepares a derivative work that is based on or incorporates Python Python 2.7.5 or any part thereof, and wants to make the derivative work available to others as provided herein, then Licensee hereby agrees to include in any such work a brief summary of the changes made to Python Python 2.7.5. #. PSF is making Python Python 2.7.5 available to Licensee on an "AS IS" basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON Python 2.7.5 WILL NOT INFRINGE ANY THIRD PARTY RIGHTS. #. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON Python 2.7.5 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON Python 2.7.5, OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. #. This License Agreement will automatically terminate upon a material breach of its terms and conditions. #. Nothing in this License Agreement shall be deemed to create any relationship of agency, partnership, or joint venture between PSF and Licensee. This License Agreement does not grant permission to use PSF trademarks or trade name in a trademark sense to endorse or promote products or services of Licensee, or any third party. #. By copying, installing or otherwise using Python Python 2.7.5, Licensee agrees to be bound by the terms and conditions of this License Agreement. h5py-2.7.1/docs/strings.rst0000644000175000017500000001172613030556017017246 0ustar tcaswelltcaswell00000000000000.. _strings: Strings in HDF5 =============== The Most Important Thing ------------------------ If you remember nothing else, remember this: **All strings in HDF5 hold encoded text.** You *can't* store arbitrary binary data in HDF5 strings. Not only will this break, it will break in odd, hard-to-discover ways that will leave you confused and cursing. .. _str_binary: How to store raw binary data ---------------------------- If you have a non-text blob in a Python byte string (as opposed to ASCII or UTF-8 encoded text, which is fine), you should wrap it in a ``void`` type for storage. This will map to the HDF5 OPAQUE datatype, and will prevent your blob from getting mangled by the string machinery. Here's an example of how to store binary data in an attribute, and then recover it:: >>> binary_blob = b"Hello\x00Hello\x00" >>> dset.attrs["attribute_name"] = np.void(binary_blob) >>> out = dset.attrs["attribute_name"] >>> binary_blob = out.tostring() How to store text strings ------------------------- At the high-level interface, h5py exposes three kinds of strings. Each maps to a specific type within Python (but see :ref:`str_py3` below): * Fixed-length ASCII (NumPy ``S`` type) * Variable-length ASCII (Python 2 ``str``, Python 3 ``bytes``) * Variable-length UTF-8 (Python 2 ``unicode``, Python 3 ``str``) .. _str_py3: Compatibility ^^^^^^^^^^^^^ If you want to write maximally-compatible files and don't want to read the whole chapter: * Use ``numpy.string_`` for scalar attributes * Use the NumPy ``S`` dtype for datasets and array attributes Fixed-length ASCII ^^^^^^^^^^^^^^^^^^ These are created when you use ``numpy.string_``: >>> dset.attrs["name"] = numpy.string_("Hello") or the ``S`` dtype:: >>> dset = f.create_dataset("string_ds", (100,), dtype="S10") In the file, these map to fixed-width ASCII strings. One byte per character is used. The representation is "null-padded", which is the internal representation used by NumPy (and the only one which round-trips through HDF5). Technically, these strings are supposed to store `only` ASCII-encoded text, although in practice anything you can store in NumPy will round-trip. But for compatibility with other progams using HDF5 (IDL, MATLAB, etc.), you should use ASCII only. .. note:: This is the most-compatible way to store a string. Everything else can read it. Variable-length ASCII ^^^^^^^^^^^^^^^^^^^^^ These are created when you assign a byte string to an attribute:: >>> dset.attrs["attr"] = b"Hello" or when you create a dataset with an explicit "bytes" vlen type:: >>> dt = h5py.special_dtype(vlen=bytes) >>> dset = f.create_dataset("name", (100,), dtype=dt) Note that they're `not` fully identical to Python byte strings. You can only store ASCII-encoded text, without NULL bytes:: >>> dset.attrs["name"] = b"Hello\x00there" ValueError: VLEN strings do not support embedded NULLs In the file, these are created as variable-length strings with character set H5T_CSET_ASCII. Variable-length UTF-8 ^^^^^^^^^^^^^^^^^^^^^ These are created when you assign a ``unicode`` string to an attribute:: >>> dset.attrs["name"] = u"Hello" or if you create a dataset with an explicit ``unicode`` vlen type: >>> dt = h5py.special_dtype(vlen=unicode) >>> dset = f.create_dataset("name", (100,), dtype=dt) They can store any character a Python unicode string can store, with the exception of NULLs. In the file these are created as variable-length strings with character set H5T_CSET_UTF8. Exceptions for Python 3 ^^^^^^^^^^^^^^^^^^^^^^^ Most strings in the HDF5 world are stored in ASCII, which means they map to byte strings. But in Python 3, there's a strict separation between `data` and `text`, which intentionally makes it painful to handle encoded strings directly. So, when reading or writing scalar string attributes, on Python 3 they will `always` be returned as type ``str``, regardless of the underlying storage mechanism. The regular rules for writing apply; to get a fixed-width ASCII string, use ``numpy.string_``, and to get a variable-length ASCII string, use ``bytes``. What about NumPy's ``U`` type? ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ NumPy also has a Unicode type, a UTF-32 fixed-width format (4-byte characters). HDF5 has no support for wide characters. Rather than trying to hack around this and "pretend" to support it, h5py will raise an error when attempting to create datasets or attributes of this type. Object names ------------ Unicode strings are used exclusively for object names in the file:: >>> f.name u'/' You can supply either byte or unicode strings (on both Python 2 and Python 3) when creating or retrieving objects. If a byte string is supplied, it will be used as-is; Unicode strings will be encoded down to UTF-8. In the file, h5py uses the most-compatible representation; H5T_CSET_ASCII for characters in the ASCII range; H5T_CSET_UTF8 otherwise. >>> grp = f.create_dataset(b"name") >>> grp2 = f.create_dataset(u"name2") h5py-2.7.1/docs/refs.rst0000644000175000017500000001027313025343121016502 0ustar tcaswelltcaswell00000000000000.. _refs: Object and Region References ============================ In addition to soft and external links, HDF5 supplies one more mechanism to refer to objects and data in a file. HDF5 *references* are low-level pointers to other objects. The great advantage of references is that they can be stored and retrieved as data; you can create an attribute or an entire dataset of reference type. References come in two flavors, object references and region references. As the name suggests, object references point to a particular object in a file, either a dataset, group or named datatype. Region references always point to a dataset, and additionally contain information about a certain selection (*dataset region*) on that dataset. For example, if you have a dataset representing an image, you could specify a region of interest, and store it as an attribute on the dataset. .. _refs_object: Using object references ----------------------- It's trivial to create a new object reference; every high-level object in h5py has a read-only property "ref", which when accessed returns a new object reference: >>> myfile = h5py.File('myfile.hdf5') >>> mygroup = myfile['/some/group'] >>> ref = mygroup.ref >>> print ref "Dereferencing" these objects is straightforward; use the same syntax as when opening any other object: >>> mygroup2 = myfile[ref] >>> print mygroup2 .. _refs_region: Using region references ----------------------- Region references always contain a selection. You create them using the dataset property "regionref" and standard NumPy slicing syntax: >>> myds = myfile.create_dataset('dset', (200,200)) >>> regref = myds.regionref[0:10, 0:5] >>> print regref The reference itself can now be used in place of slicing arguments to the dataset: >>> subset = myds[regref] There is one complication; since HDF5 region references don't express shapes the same way as NumPy does, the data returned will be "flattened" into a 1-D array: >>> subset.shape (50,) This is similar to the behavior of NumPy's fancy indexing, which returns a 1D array for selections which don't conform to a regular grid. In addition to storing a selection, region references inherit from object references, and can be used anywhere an object reference is accepted. In this case the object they point to is the dataset used to create them. Storing references in a dataset ------------------------------- HDF5 treats object and region references as data. Consequently, there is a special HDF5 type to represent them. However, NumPy has no equivalent type. Rather than implement a special "reference type" for NumPy, references are handled at the Python layer as plain, ordinary python objects. To NumPy they are represented with the "object" dtype (kind 'O'). A small amount of metadata attached to the dtype tells h5py to interpret the data as containing reference objects. H5py contains a convenience function to create these "hinted dtypes" for you: >>> ref_dtype = h5py.special_dtype(ref=h5py.Reference) >>> type(ref_dtype) >>> ref_dtype.kind 'O' The types accepted by this "ref=" keyword argument are h5py.Reference (for object references) and h5py.RegionReference (for region references). To create an array of references, use this dtype as you normally would: >>> ref_dataset = myfile.create_dataset("MyRefs", (100,), dtype=ref_dtype) You can read from and write to the array as normal: >>> ref_dataset[0] = myfile.ref >>> print ref_dataset[0] Storing references in an attribute ---------------------------------- Simply assign the reference to a name; h5py will figure it out and store it with the correct type: >>> myref = myfile.ref >>> myfile.attrs["Root group reference"] = myref Null references --------------- When you create a dataset of reference type, the uninitialized elements are "null" references. H5py uses the truth value of a reference object to indicate whether or not it is null: >>> print bool(myfile.ref) True >>> nullref = ref_dataset[50] >>> print bool(nullref) False h5py-2.7.1/docs/contributing.rst0000644000175000017500000003136313030556017020263 0ustar tcaswelltcaswell00000000000000Bug Reports & Contributions =========================== Contributions and bug reports are welcome from anyone! Some of the best features in h5py, including thread support, dimension scales, and the scale-offset filter, came from user code contributions. Since we use GitHub, the workflow will be familiar to many people. If you have questions about the process or about the details of implementing your feature, always feel free to ask on the Google Groups list, either by emailing: h5py@googlegroups.com or via the web interface at: https://groups.google.com/forum/#!forum/h5py Anyone can post to this list. Your first message will be approved by a moderator, so don't worry if there's a brief delay. This guide is divided into three sections. The first describes how to file a bug report. The second describes the mechanics of how to submit a contribution to the h5py project; for example, how to create a pull request, which branch to base your work on, etc. We assume you're are familiar with Git, the version control system used by h5py. If not, `here's a great place to start `_. Finally, we describe the various subsystems inside h5py, and give technical guidance as to how to implement your changes. How to File a Bug Report ------------------------ Bug reports are always welcome! The issue tracker is at: http://github.com/h5py/h5py/issues If you're unsure whether you've found a bug ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Always feel free to ask on the mailing list (h5py at Google Groups). Discussions there are seen by lots of people and are archived by Google. Even if the issue you're having turns out not to be a bug in the end, other people can benefit from a record of the conversation. By the way, nobody will get mad if you file a bug and it turns out to be something else. That's just how software development goes. What to include ~~~~~~~~~~~~~~~ When filing a bug, there are two things you should include. The first is the output of ``h5py.version.info``:: >>> import h5py >>> print h5py.version.info The second is a detailed explanation of what went wrong. Unless the bug is really trivial, **include code if you can**, either via GitHub's inline markup:: ``` import h5py h5py.explode() # Destroyed my computer! ``` or by uploading a code sample to `Github Gist `_. How to Get Your Code into h5py ------------------------------ This section describes how to contribute changes to the h5py code base. Before you start, be sure to read the h5py license and contributor agreement in "license.txt". You can find this in the source distribution, or view it online at the main h5py repository at GitHub. The basic workflow is to clone h5py with git, make your changes in a topic branch, and then create a pull request at GitHub asking to merge the changes into the main h5py project. Here are some tips to getting your pull requests accepted: 1. Let people know you're working on something. This could mean posting a comment in an open issue, or sending an email to the mailing list. There's nothing wrong with just opening a pull request, but it might save you time if you ask for advice first. 2. Keep your changes focused. If you're fixing multiple issues, file multiple pull requests. Try to keep the amount of reformatting clutter small so the maintainers can easily see what you've changed in a diff. 3. Unit tests are mandatory for new features. This doesn't mean hundreds (or even dozens) of tests! Just enough to make sure the feature works as advertised. The maintainers will let you know if more are needed. .. _git_checkout: Clone the h5py repository ~~~~~~~~~~~~~~~~~~~~~~~~~ The best way to do this is by signing in to GitHub and cloning the h5py project directly. You'll end up with a new repository under your account; for example, if your username is ``yourname``, the repository would be at http://github.com/yourname/h5py. Then, clone your new copy of h5py to your local machine:: $ git clone http://github.com/yourname/h5py Create a topic branch for your feature ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If you're fixing a bug, you'll want to check out a branch against the appropriate stable branch. For example, to fix a bug you found in version 2.1.3, you'll want to check out against branch "2.1":: $ git checkout -b bugfix 2.1 If you're contributing a new feature, it's appropriate to develop against the "master" branch, so you would instead do:: $ git checkout -b newfeature master The exact name of the branch can be anything you want. For bug fixes, one approach is to put the issue number in the branch name. Implement the feature! ~~~~~~~~~~~~~~~~~~~~~~ You can implement the feature as a number of small changes, or as one big commit; there's no project policy. Double-check to make sure you've included all your files; run ``git status`` and check the output. Push your changes back and open a pull request ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Push your topic branch back up to your GitHub clone:: $ git push origin newfeature Then, `create a pull request `_ based on your topic branch. Work with the maintainers ~~~~~~~~~~~~~~~~~~~~~~~~~ Your pull request might be accepted right away. More commonly, the maintainers will post comments asking you to fix minor things, like add a few tests, clean up the style to be PEP-8 compliant, etc. The pull request page also shows whether the project builds correctly, using Travis CI. Check to see if the build succeeded (takes about 5 minutes), and if not, try to modify your changes to make it work. When making changes after creating your pull request, just add commits to your topic branch and push them to your GitHub repository. Don't try to rebase or open a new pull request! We don't mind having a few extra commits in the history, and it's helpful to keep all the history together in one place. How to Modify h5py ------------------ This section is a little more involved, and provides tips on how to modify h5py. The h5py package is built in layers. Starting from the bottom, they are: 1. The HDF5 C API (provided by libhdf5) 2. Auto-generated Cython wrappers for the C API (``api_gen.py``) 3. Low-level interface, written in Cython, using the wrappers from (2) 4. High-level interface, written in Python, with things like ``h5py.File``. 5. Unit test code Rather than talk about the layers in an abstract way, the parts below are guides to adding specific functionality to various parts of h5py. Most sections span at least two or three of these layers. Adding a function from the HDF5 C API ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This is one of the most common contributed changes. The example below shows how one would add the function ``H5Dget_storage_size``, which determines the space on disk used by an HDF5 dataset. This function is already partially wrapped in h5py, so you can see how it works. It's recommended that you follow along, if not by actually adding the feature then by at least opening the various files as we work through the example. First, get ahold of the function signature; the easiest place for this is at the `online HDF5 Reference Manual `_. Then, add the function's C signature to the file ``api_functions.txt``:: hsize_t H5Dget_storage_size(hid_t dset_id) This particular signature uses types (``hsize_t``, ``hid_t``) which are already defined elsewhere. But if the function you're adding needs a struct or enum definition, you can add it using Cython code to the file ``api_types_hdf5.pxd``. The next step is to add a Cython function or method which calls the function you added. The h5py modules follow the naming convention of the C API; functions starting with ``H5D`` are wrapped in ``h5d.pyx``. Opening ``h5d.pyx``, we notice that since this function takes a dataset identifier as the first argument, it belongs as a method on the DatasetID object. We write a wrapper method:: def get_storage_size(self): """ () => LONG storage_size Determine the amount of file space required for a dataset. Note this only counts the space which has actually been allocated; it may even be zero. """ return H5Dget_storage_size(self.id) The first line of the docstring gives the method signature. This is necessary because Cython will use a "generic" signature like ``method(*args, **kwds)`` when the file is compiled. The h5py documentation system will extract the first line and use it as the signature. Next, we decide whether we want to add access to this function to the high-level interface. That means users of the top-level ``h5py.Dataset`` object will be able to see how much space on disk their files use. The high-level interface is implemented in the subpackage ``h5py._hl``, and the Dataset object is in module ``dataset.py``. Opening it up, we add a property on the ``Dataset`` object:: @property def storagesize(self): """ Size (in bytes) of this dataset on disk. """ return self.id.get_storage_size() You'll see that the low-level ``DatasetID`` object is available on the high-level ``Dataset`` object as ``obj.id``. This is true of all the high-level objects, like ``File`` and ``Group`` as well. Finally (and don't skip this step), we write **unit tests** for this feature. Since the feature is ultimately exposed at the high-level interface, it's OK to write tests for the ``Dataset.storagesize`` property only. Unit tests for the high-level interface are located in the "tests" subfolder, right near ``dataset.py``. It looks like the right file is ``test_dataset.py``. Unit tests are implemented as methods on custom ``unittest.UnitTest`` subclasses; each new feature should be tested by its own new class. In the ``test_dataset`` module, we see there's already a subclass called ``BaseDataset``, which implements some simple set-up and cleanup methods and provides a ``h5py.File`` object as ``obj.f``. We'll base our test class on that:: class TestStorageSize(BaseDataset): """ Feature: Dataset.storagesize indicates how much space is used. """ def test_empty(self): """ Empty datasets take no space on disk """ dset = self.f.create_dataset("x", (100,100)) self.assertEqual(dset.storagesize, 0) def test_data(self): """ Storage size is correct for non-empty datasets """ dset = self.f.create_dataset("x", (100,), dtype='uint8') dset[...] = 42 self.assertEqual(dset.storagesize, 100) This set of tests would be adequate to get a pull request approved. We don't test every combination under the sun (different ranks, datasets with more than 2**32 elements, datasets with the string "kumquat" in the name...), but the basic, commonly encountered set of conditions. To build and test our changes, we have to do a few things. First of all, run the file ``api_gen.py`` to re-generate the Cython wrappers from ``api_functions.txt``:: $ python api_gen.py Then build the project, which recompiles ``h5d.pyx``:: $ python setup.py build Finally, run the test suite, which includes the two methods we just wrote:: $ python setup.py test If the tests pass, the feature is ready for a pull request. Adding a function only available in certain versions of HDF5 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ At the moment, h5py must be backwards-compatible all the way back to HDF5 1.8.4. Starting with h5py 2.2.0, it's possible to conditionally include functions which only appear in newer versions of HDF5. It's also possible to mark functions which requre Parallel HDF5. For example, the function ``H5Fset_mpi_atomicity`` was introduced in HDF5 1.8.9 and requires Parallel HDF5. Specifiers before the signature in ``api_functions.txt`` communicate this:: MPI 1.8.9 herr_t H5Fset_mpi_atomicity(hid_t file_id, hbool_t flag) You can specify either, both or none of "MPI" or a version number in "X.Y.Z" format. In the Cython code, these show up as "preprocessor" defines ``MPI`` and ``HDF5_VERSION``. So the low-level implementation (as a method on ``h5py.h5f.FileID``) looks like this:: IF MPI and HDF5_VERSION >= (1, 8, 9): def set_mpi_atomicity(self, bint atomicity): """ (BOOL atomicity) For MPI-IO driver, set to atomic (True), which guarantees sequential I/O semantics, or non-atomic (False), which improves performance. Default is False. Feature requires: 1.8.9 and Parallel HDF5 """ H5Fset_mpi_atomicity(self.id, atomicity) High-level code can check the version of the HDF5 library, or check to see if the method is present on ``FileID`` objects. h5py-2.7.1/docs/build.rst0000644000175000017500000001667213114357361016665 0ustar tcaswelltcaswell00000000000000.. _install: Installation ============ .. _install_recommends: It is highly recommended that you use a pre-built version of h5py, either from a Python Distribution, an OS-specific package manager, or a pre-built wheel from PyPI. Be aware however that most pre-built versions lack MPI support, and that they are built against a specific version of HDF5. If you require MPI support, or newer HDF5 features, you will need to build from source. After installing h5py, you should run the tests to be sure that everything was installed correctly. This can be done in the python interpreter via:: import h5py h5py.run_tests() On Python 2.6, unittest2 must be installed to run the tests. .. _prebuilt_install: Pre-built installation (recommended) ----------------------------------------- Pre-build h5py can be installed via many Python Distributions, OS-specific package managers, or via h5py wheels. Python Distributions .................... If you do not already use a Python Distribution, we recommend either `Anaconda `_/`Miniconda `_ or `Enthought Canopy `_, both of which support most versions of Microsoft Windows, OSX/MacOS, and a variety of Linux Distributions. Installation of h5py can be done on the command line via:: $ conda install h5py for Anaconda/MiniConda, and via:: $ enpkg h5py for Canopy. Wheels ...... If you have an existing Python installation (e.g. a python.org download, or one that comes with your OS), then on Windows, MacOS/OSX, and Linux on Intel computers, pre-built h5py wheels can be installed via pip from PyPI:: $ pip install h5py Additionally, for Windows users, `Chris Gohlke provides third-party wheels which use Intel's MKL `_. OS-Specific Package Managers ............................ On OSX/MacOS, h5py can be installed via `Homebrew `_, `Macports `_, or `Fink `_. The current state of h5py in various Linux Distributions can be seen at https://pkgs.org/download/python-h5py, and can be installed via the package manager. As far as the h5py developers know, none of the Windows package managers (e.g. `Chocolatey `_, `nuget `_) have h5py included, however they may assist in installing h5py's requirements when building from source. .. _source_install: Source installation ------------------- To install h5py from source, you need three things installed: * A supported Python version with development headers * HDF5 1.8.4 or newer with development headers * A C compiler OS-specific instructions for installing HDF5, Python and a C compiler are in the next few sections. Additional Python-level requirements should be installed automatically (which will require an internet connection). The actual installation of h5py should be done via:: $ pip install --no-binary=h5py h5py or, from a tarball or git :ref:`checkout ` :: $ pip install -v . or :: $ python setup.py install If you are working on a development version and the underlying cython files change it may be necessary to force a full rebuild. The easiest way to achieve this is :: $ git clean -xfd from the top of your clone and then rebuilding. Source installation on OSX/MacOS ................................ HDF5 and Python are most likely in your package manager (e.g. `Homebrew `_, `Macports `_, or `Fink `_). Be sure to install the development headers, as sometimes they are not included in the main package. XCode comes with a C compiler (clang), and your package manager will likely have other C compilers for you to install. Source installation on Linux/Other Unix ....................................... HDF5 and Python are most likely in your package manager. A C compiler almost definitely is, usually there is some kind of metapackage to install the default build tools, e.g. `build-essential`, which should be sufficient for our needs. Make sure that that you have the development headers, as they are usually not installed by default. They can usually be found in ``python-dev`` or similar and ``libhdf5-dev`` or similar. Source installation on Windows .............................. Installing from source on Windows is a much more difficult prospect than installing from source on other OSs, as not only are you likely to need to compile HDF5 from source, everything must be built with the correct version of Visual Studio. Additional patches are also needed to HDF5 to get HDF5 and Python to work together. We recommend examining the appveyor build scripts, and using those to build and install HDF5 and h5py. .. _custom_install: Custom installation ------------------- .. important:: Remember that pip installs wheels by default. To perform a custom installation with pip, you should use:: $ pip install --no-binary=h5py h5py or build from a git checkout or downloaded tarball to avoid getting a pre-built version of h5py. You can specify build options for h5py with the ``configure`` option to setup.py. Options may be given together or separately:: $ python setup.py configure --hdf5=/path/to/hdf5 $ python setup.py configure --hdf5-version=X.Y.Z $ python setup.py configure --mpi Note the ``--hdf5-version`` option is generally not needed, as h5py auto-detects the installed version of HDF5 (even for custom locations). Once set, build options apply to all future builds in the source directory. You can reset to the defaults with the ``--reset`` option:: $ python setup.py configure --reset You can also configure h5py using environment variables. This is handy when installing via ``pip``, as you don't have direct access to setup.py:: $ HDF5_DIR=/path/to/hdf5 pip install --no-binary=h5py h5py $ HDF5_VERSION=X.Y.Z pip install --no-binary=h5py h5py $ CC="mpicc" HDF5_MPI="ON" HDF5_DIR=/path/to/parallel-hdf5 pip install --no-binary=h5py h5py Here's a list of all the configure options currently supported: ======================= =========================== =========================== Option Via setup.py Via environment variable ======================= =========================== =========================== Custom path to HDF5 ``--hdf5=/path/to/hdf5`` ``HDF5_DIR=/path/to/hdf5`` Force HDF5 version ``--hdf5-version=X.Y.Z`` ``HDF5_VERSION=X.Y.Z`` Enable MPI mode ``--mpi`` ``HDF5_MPI=ON`` ======================= =========================== =========================== .. _build_mpi: Building against Parallel HDF5 ------------------------------ If you just want to build with ``mpicc``, and don't care about using Parallel HDF5 features in h5py itself:: $ export CC=mpicc $ pip install --no-binary=h5py h5py If you want access to the full Parallel HDF5 feature set in h5py (:ref:`parallel`), you will further have to build in MPI mode. This can either be done with command-line options from the h5py tarball or by:: $ export HDF5_MPI="ON" **You will need a shared-library build of Parallel HDF5 (i.e. built with ./configure --enable-shared --enable-parallel).** To build in MPI mode, use the ``--mpi`` option to ``setup.py configure`` or export ``HDF5_MPI="ON"`` beforehand:: $ export CC=mpicc $ export HDF5_MPI="ON" $ pip install --no-binary=h5py h5py See also :ref:`parallel`. h5py-2.7.1/docs/whatsnew/0000755000175000017500000000000013152363123016653 5ustar tcaswelltcaswell00000000000000h5py-2.7.1/docs/whatsnew/2.5.rst0000644000175000017500000000614313030556017017716 0ustar tcaswelltcaswell00000000000000What's new in h5py 2.5 ====================== Experimental support for Single Writer Multiple Reader (SWMR) ------------------------------------------------------------- This release introduces experimental support for the highly-anticipated "Single Writer Multiple Reader" (SWMR) feature in the upcoming HDF5 1.10 release. SWMR allows sharing of a single HDF5 file between multiple processes without the complexity of MPI or multiprocessing-based solutions. This is an experimental feature that should NOT be used in production code. We are interested in getting feedback from the broader community with respect to performance and the API design. For more details, check out the h5py user guide: http://docs.h5py.org/en/latest/swmr.html SWMR support was contributed by Ulrik Pedersen (`#551`_). Other changes ------------- * Use system Cython as a fallback if `cythonize()` fails (`#541`_ by Ulrik Pedersen). * Use pkg-config for builing/linking against hdf5 (`#505`_ by James Tocknell). * Disable building Cython on Travis (`#513`_ by Andrew Collette). * Improvements to release tarball (`#555`_, `#560`_ by Ghislain Antony Vaillant). * h5py now has one codebase for both Python 2 and 3; 2to3 removed from setup.py (`#508`_ by James Tocknell). * Add python 3.4 to tox (`#507`_ by James Tocknell). * Warn when importing from inside install dir (`#558`_ by Andrew Collette). * Tweak installation docs with reference to Anaconda and other Python package managers (`#546`_ by Andrew Collette). * Fix incompatible function pointer types (`#526`_, `#524`_ by Peter H. Li). * Add explicit `vlen is not None` check to work around https://github.com/numpy/numpy/issues/2190 (`#538` by Will Parkin). * Group and AttributeManager classes now inherit from the appropriate ABCs (`#527`_ by James Tocknell). * Don't strip metadata from special dtypes on read (`#512`_ by Antony Lee). * Add 'x' mode as an alias for 'w-' (`#510`_ by Antony Lee). * Support dynamical loading of LZF filter plugin (`#506`_ by Peter Colberg). * Fix accessing attributes with array type (`#501`_ by Andrew Collette). * Don't leak types in enum converter (`#503`_ by Andrew Collette). .. _`#551` : https://github.com/h5py/h5py/pull/551 .. _`#541` : https://github.com/h5py/h5py/pull/541 .. _`#505` : https://github.com/h5py/h5py/pull/505 .. _`#513` : https://github.com/h5py/h5py/pull/513 .. _`#555` : https://github.com/h5py/h5py/pull/555 .. _`#560` : https://github.com/h5py/h5py/pull/560 .. _`#508` : https://github.com/h5py/h5py/pull/508 .. _`#507` : https://github.com/h5py/h5py/pull/507 .. _`#558` : https://github.com/h5py/h5py/pull/558 .. _`#546` : https://github.com/h5py/h5py/pull/546 .. _`#526` : https://github.com/h5py/h5py/pull/526 .. _`#524` : https://github.com/h5py/h5py/pull/524 .. _`#538` : https://github.com/h5py/h5py/pull/538 .. _`#527` : https://github.com/h5py/h5py/pull/527 .. _`#512` : https://github.com/h5py/h5py/pull/512 .. _`#510` : https://github.com/h5py/h5py/pull/510 .. _`#506` : https://github.com/h5py/h5py/pull/506 .. _`#501` : https://github.com/h5py/h5py/pull/501 .. _`#503` : https://github.com/h5py/h5py/pull/503 Acknowlegements --------------- h5py-2.7.1/docs/whatsnew/2.4.rst0000644000175000017500000000264013025343121017705 0ustar tcaswelltcaswell00000000000000What's new in h5py 2.4 ====================== Build system changes -------------------- The setup.py-based build system has been reworked to be more maintainable, and to fix certain long-standing bugs. As a consequence, the options to setup.py have changed; a new top-level "configure" command handles options like ``--hdf5=/path/to/hdf5`` and ``--mpi``. Setup.py now works correctly under Python 3 when these options are used. Cython (0.17+) is now required when building from source on all platforms; the .c files are no longer shipped in the UNIX release. The minimum NumPy version is now 1.6.1. Files will now auto-close ------------------------- Files are now automatically closed when all objects within them are unreachable. Previously, if File.close() was not explicitly called, files would remain open and "leaks" were possible if the File object was lost. Thread safety improvements -------------------------- Access to all APIs, high- and low-level, are now protected by a global lock. The entire API is now believed to be thread-safe. Feedback and real-world testing is welcome. External link improvements -------------------------- External links now work if the target file is already open. Previously this was not possible because of a mismatch in the file close strengths. Thanks to --------- Many people, but especially: * Matthieu Brucher * Laurence Hole * John Tyree * Pierre de Buyl * Matthew Brett h5py-2.7.1/docs/whatsnew/2.6.rst0000644000175000017500000000773613030556017017730 0ustar tcaswelltcaswell00000000000000What's new in h5py 2.6 ====================== Support for HDF5 Virtual Dataset API ------------------------------------ Initial support for the HDF5 Virtual Dataset API, which was introduced in HDF5 1.10, was added to the low-level API. Ideas and input for how this should work as part of the high-level interface are welcome. This work was added in `#663`_ by Aleksandar Jelenak. Add MPI Collective I/O Support ------------------------------ Support for using MPI Collective I/O in both low-level and high-level code has been added. See the collective_io.py example for a simple demonstration of how to use MPI Collective I/O with the high level API. This work was added in `#648`_ by Jialin Liu. Numerous build/testing/CI improvements -------------------------------------- There were a number of improvements to the setup.py file, which should mean that `pip install h5py` should work in most places. Work was also done to clean up the current testing system, using tox is the recommended way of testing h5py across different Python versions. See `#576`_ by Jakob Lombacher, `#640`_ by Lawrence Mitchell, and `#650`_, `#651`_ and `#658`_ by James Tocknell. Cleanup of codebase based on pylint ----------------------------------- There was a large cleanup of pylint-identified problems by Andrew Collette (`#578`_, `#579`_). Fixes to low-level API ---------------------- Fixes to the typing of functions were added in `#597`_ by Ulrik Kofoed Pedersen, `#589`_ by Peter Chang, and `#625`_ by Spaghetti Sort. A fix for variable-length arrays was added in `#621`_ by Sam Mason. Fixes to compound types were added in `#639`_ by @nevion and `#606`_ by Yu Feng. Finally, a fix to type conversion was added in `#614`_ by Andrew Collette. Documentation improvements -------------------------- * Updates to FAQ by Dan Guest (`#608`_) and Peter Hill (`#607`_). * Updates MPI-releated documentation by Jens Timmerman (`#604`_) and Matthias König (`#572`_). * Fixes to documentation building by Ghislain Antony Vaillant (`#562`_, `#561`_). * Update PyTables link (`#574`_ by Dominik Kriegner) * Add File opening modes to docstring (`#563`_ by Antony Lee) Other changes ------------- * Add `Dataset.ndim` (`#649`_, `#660`_ by @jakirkham, `#661`_ by James Tocknell) * Fix import errors in IPython completer (`#605`_ by Niru Maheswaranathan) * Turn off error printing in new threads (`#583`_ by Andrew Collette) * Use item value in `KeyError` instead of error message (`#642`_ by Matthias Geier) .. _`#561` : https://github.com/h5py/h5py/pull/561 .. _`#562` : https://github.com/h5py/h5py/pull/562 .. _`#563` : https://github.com/h5py/h5py/pull/563 .. _`#572` : https://github.com/h5py/h5py/pull/572 .. _`#574` : https://github.com/h5py/h5py/pull/574 .. _`#576` : https://github.com/h5py/h5py/pull/576 .. _`#578` : https://github.com/h5py/h5py/pull/578 .. _`#579` : https://github.com/h5py/h5py/pull/579 .. _`#583` : https://github.com/h5py/h5py/pull/583 .. _`#589` : https://github.com/h5py/h5py/pull/589 .. _`#597` : https://github.com/h5py/h5py/pull/597 .. _`#604` : https://github.com/h5py/h5py/pull/604 .. _`#605` : https://github.com/h5py/h5py/pull/605 .. _`#606` : https://github.com/h5py/h5py/pull/606 .. _`#607` : https://github.com/h5py/h5py/pull/607 .. _`#608` : https://github.com/h5py/h5py/pull/608 .. _`#614` : https://github.com/h5py/h5py/pull/614 .. _`#621` : https://github.com/h5py/h5py/pull/621 .. _`#625` : https://github.com/h5py/h5py/pull/625 .. _`#639` : https://github.com/h5py/h5py/pull/639 .. _`#640` : https://github.com/h5py/h5py/pull/640 .. _`#642` : https://github.com/h5py/h5py/pull/642 .. _`#648` : https://github.com/h5py/h5py/pull/648 .. _`#649` : https://github.com/h5py/h5py/pull/649 .. _`#650` : https://github.com/h5py/h5py/pull/650 .. _`#651` : https://github.com/h5py/h5py/pull/651 .. _`#658` : https://github.com/h5py/h5py/pull/658 .. _`#660` : https://github.com/h5py/h5py/pull/660 .. _`#661` : https://github.com/h5py/h5py/pull/661 .. _`#663` : https://github.com/h5py/h5py/pull/663 Acknowlegements --------------- h5py-2.7.1/docs/whatsnew/index.rst0000644000175000017500000000036413152157047020525 0ustar tcaswelltcaswell00000000000000.. _whatsnew: ********************** "What's new" documents ********************** These document the changes between minor (or major) versions of h5py. .. toctree:: 2.7.1 2.7 2.6 2.5 2.4 2.3 2.2 2.1 2.0 h5py-2.7.1/docs/whatsnew/2.0.rst0000644000175000017500000001573313025343121017710 0ustar tcaswelltcaswell00000000000000What's new in h5py 2.0 ====================== HDF5 for Python (h5py) 2.0 represents the first major refactoring of the h5py codebase since the project's launch in 2008. Many of the most important changes are behind the scenes, and include changes to the way h5py interacts with the HDF5 library and Python. These changes have substantially improved h5py's stability, and make it possible to use more modern versions of HDF5 without compatibility concerns. It is now also possible to use h5py with Python 3. Enhancements unlikely to affect compatibility --------------------------------------------- * HDF5 1.8.3 through 1.8.7 now work correctly and are officially supported. * Python 3.2 is officially supported by h5py! Thanks especially to Darren Dale for getting this working. * Fill values can now be specified when creating a dataset. The fill time is H5D_FILL_TIME_IFSET for contiguous datasets, and H5D_FILL_TIME_ALLOC for chunked datasets. * On Python 3, dictionary-style methods like Group.keys() and Group.values() return view-like objects instead of lists. * Object and region references now work correctly in compound types. * Zero-length dimensions for extendible axes are now allowed. * H5py no longer attempts to auto-import ipython on startup. * File format bounds can now be given when opening a high-level File object (keyword "libver"). Changes which may break existing code ------------------------------------- Supported HDF5/Python versions ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ * HDF5 1.6.X is no longer supported on any platform; following the release of 1.6.10 some time ago, this branch is no longer maintained by The HDF Group. * Python 2.6 or later is now required to run h5py. This is a consequence of the numerous changes made to h5py for Python 3 compatibility. * On Python 2.6, unittest2 is now required to run the test suite. Group, Dataset and Datatype constructors have changed ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ In h5py 2.0, it is no longer possible to create new groups, datasets or named datatypes by passing names and settings to the constructors directly. Instead, you should use the standard Group methods create_group and create_dataset. The File constructor remains unchanged and is still the correct mechanism for opening and creating files. Code which manually creates Group, Dataset or Datatype objects will have to be modified to use create_group or create_dataset. File-resident datatypes can be created by assigning a NumPy dtype to a name (e.g. mygroup["name"] = numpy.dtype('S10')). Unicode is now used for object names ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Older versions of h5py used byte strings to represent names in the file. Starting with version 2.0, you may use either byte or unicode strings to create objects, but object names (obj.name, etc) will generally be returned as Unicode. Code which may be affected: * Anything which uses "isinstance" or explicit type checks on names, expecting "str" objects. Such checks should be removed, or changed to compare to "basestring" instead. * In Python 2.X, other parts of your application may complain if they are handed Unicode data which can't be encoded down to ascii. This is a general problem in Python 2. File objects must be manually closed ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ With h5py 1.3, when File objects (or low-level FileID) objects went out of scope, the corresponding HDF5 file was closed. This led to surprising behavior, especially when files were opened with the H5F_CLOSE_STRONG flag; "losing" the original File object meant that all open groups and datasets suddenly became invalid. Beginning with h5py 2.0, files must be manually closed, by calling the "close" method or by using the file object as a context manager. If you forget to close a file, the HDF5 library will try to close it for you when the application exits. Please note that opening the same file multiple times (i.e. without closing it first) continues to result in undefined behavior. Changes to scalar slicing code ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ When a scalar dataset was accessed with the syntax ``dataset[()]``, h5py incorrectly returned an ndarray. H5py now correctly returns an array scalar. Using ``dataset[...]`` on a scalar dataset still returns an ndarray. Array scalars now always returned when indexing a dataset ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ When using datasets of compound type, retrieving a single element incorrectly returned a tuple of values, rather than an instance of ``numpy.void_`` with the proper fields populated. Among other things, this meant you couldn't do things like ``dataset[index][field]``. H5py now always returns an array scalar, except in the case of object dtypes (references, vlen strings). Reading object-like data strips special type information ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ In the past, reading multiple data points from dataset with vlen or reference type returned a Numpy array with a "special dtype" (such as those created by ``h5py.special_dtype()``). In h5py 2.0, all such arrays now have a generic Numpy object dtype (``numpy.dtype('O')``). To get a copy of the dataset's dtype, always use the dataset's dtype property directly (``mydataset.dtype``). The selections module has been removed ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Only numpy-style slicing arguments remain supported in the high level interface. Existing code which uses the selections module should be refactored to use numpy slicing (and ``numpy.s_`` as appropriate), or the standard C-style HDF5 dataspace machinery. The H5Error exception class has been removed (along with h5py.h5e) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ All h5py exceptions are now native Python exceptions, no longer inheriting from H5Error. RuntimeError is raised if h5py can't figure out what exception is appropriate... every instance of this behavior is considered a bug. If you see h5py raising RuntimeError please report it so we can add the correct mapping! The old errors module (h5py.h5e) has also been removed. There is no public error-management API. File .mode property is now either 'r' or 'r+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Files can be opened using the same mode arguments as before, but now the property File.mode will always return 'r' (read-only) or 'r+' (read-write). Long-deprecated dict methods have been removed ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Certain ancient aliases for Group/AttributeManager methods (e.g. ``listnames``) have been removed. Please use the standard Python dict interface (Python 2 or Python 3 as appropriate) to interact with these objects. Known issues ------------ * Thread support has been improved in h5py 2.0. However, we still recommend that for your own sanity you use locking to serialize access to files. * There are reports of crashes related to storing object and region references. If this happens to you, please post on the mailing list or contact the h5py author directly. h5py-2.7.1/docs/whatsnew/2.3.rst0000644000175000017500000000506013025343121017703 0ustar tcaswelltcaswell00000000000000What's new in h5py 2.3 ====================== Support for arbitrary vlen data ------------------------------- Variable-length data is :ref:`no longer restricted to strings `. You can use this feature to produce "ragged" arrays, whose members are 1D arrays of variable length. The implementation of special types was changed to use the NumPy dtype "metadata" field. This change should be transparent, as access to special types is handled through ``h5py.special_dtype`` and ``h5py.check_dtype``. Improved exception messages --------------------------- H5py has historically suffered from low-detail exception messages generated automatically by HDF5. While the exception types in 2.3 remain identical to those in 2.2, the messages have been substantially improved to provide more information as to the source of the error. Examples:: ValueError: Unable to set extend dataset (Dimension cannot exceed the existing maximal size (new: 100 max: 1)) IOError: Unable to open file (Unable to open file: name = 'x3', errno = 2, error message = 'no such file or directory', flags = 0, o_flags = 0) KeyError: "Unable to open object (Object 'foo' doesn't exist)" Improved setuptools support --------------------------- ``setup.py`` now uses ``setup_requires`` to make installation via pip friendlier. Multiple low-level additions ---------------------------- Improved support for opening datasets via the low-level interface, by adding ``H5Dopen2`` and many new property-list functions. Improved support for MPI features --------------------------------- Added support for retrieving the MPI communicator and info objects from an open file. Added boilerplate code to allow compiling cleanly against newer versions of mpi4py. Readonly files can now be opened in default mode ------------------------------------------------ When opening a read-only file with no mode flags, now defaults to opening the file on RO mode rather than raising an exception. Single-step build for HDF5 on Windows ------------------------------------- Building h5py on windows has typically been hamstrung by the need to build a compatible version of HDF5 first. A new Paver-based system located in the "windows" distribution directory allows single-step compilation of HDF5 with settings that are known to work with h5py. For more, see: https://github.com/h5py/h5py/tree/master/windows Thanks to --------- * Martin Teichmann * Florian Rathgerber * Pierre de Buyl * Thomas Caswell * Andy Salnikov * Darren Dale * Robert David Grant * Toon Verstraelen * Many others who contributed bug reports h5py-2.7.1/docs/whatsnew/2.1.rst0000644000175000017500000000345013030556017017710 0ustar tcaswelltcaswell00000000000000What's new in h5py 2.1 ====================== Dimension scales ---------------- H5py now supports the Dimension Scales feature of HDF5! Thanks to Darren Dale for implementing this. You can find more information on using scales in the :ref:`dimension_scales` section of the docs. Unicode strings allowed in attributes ------------------------------------- Group, dataset and attribute names in h5py 2.X can all be given as unicode. Now, you can also store (scalar) unicode data in attribute values as well:: >>> myfile.attrs['x'] = u"I'm a Unicode string!" Storing Unicode strings in datasets or as members of compound types is not yet implemented. Dataset size property --------------------- Dataset objects now expose a ``.size`` property which provides the total number of elements in the dataspace. ``Dataset.value`` property is now deprecated. --------------------------------------------- The property ``Dataset.value``, which dates back to h5py 1.0, is deprecated and will be removed in a later release. This property dumps the entire dataset into a NumPy array. Code using ``.value`` should be updated to use NumPy indexing, using ``mydataset[...]`` or ``mydataset[()]`` as appropriate. Bug fixes --------- * Object and region references were sometimes incorrectly wrapped wrapped in a ``numpy.object_`` instance (issue 202) * H5py now ignores old versions of Cython (<0.13) when building (issue 221) * Link access property lists weren't being properly tracked in the high level interface (issue 212) * Race condition fixed in identifier tracking which led to Python crashes (issue 151) * Highlevel objects will now complain if you try to bind them to the wrong HDF5 object types (issue 191) * Unit tests can now be run after installation (issue 201) h5py-2.7.1/docs/whatsnew/2.7.rst0000644000175000017500000001504713063246304017724 0ustar tcaswelltcaswell00000000000000What's new in h5py 2.7 ====================== Python 3.2 is no longer supported --------------------------------- ``h5py`` 2.7 drops Python 3.2 support, and testing is not longer preformed on Python 3.2. The latest versions of ``pip``, ``virtualenv``, ``setuptools`` and ``numpy`` do not support Python 3.2, and dropping 3.2 allows both ``u`` and ``b`` prefixes to be used for strings. A clean up of some of the legacy code was done in `#675`_ by Andrew Collette. Additionally, support for Python 2.6 is soon to be dropped for ``pip`` (See https://github.com/pypa/pip/issues/3955) and ``setuptools`` (See https://github.com/pypa/setuptools/issues/878), and ``numpy`` has dropped Python 2.6 also in the latest release. While ``h5py`` has not dropped Python 2.6 this release, users are strongly encouraged to move to Python 2.7 where possible. Improved testing support ------------------------ There has been a major increase in the number of configurations ``h5py`` is automatically tested in, with Windows CI support added via Appveyor (`#795`_, `#798`_, `#799`_ and `#801`_ by James Tocknell) and testing of minimum requirements to ensure we still satisfy them (`#703`_ by James Tocknell). Additionally, ``tox`` was used to ensure that we don't run tests on Python versions which our dependencies have dropped or do not support (`#662`_, `#700`_ and `#733`_). Thanks to to the Appveyor support, unicode tests were made more robust (`#788`_, `#800`_ and `#804`_ by James Tocknell). Finally, other tests were improved or added where needed (`#724`_ by Matthew Brett, `#789`_, `#794`_ and `#802`_ by James Tocknell). Improved python compatibility ----------------------------- The ``ipython``/``jupyter`` completion support now has Python 3 support (`#715`_ by Joseph Kleinhenz). ``h5py`` now supports ``pathlib`` filenames (`#716`_ by James Tocknell). Documentation improvements -------------------------- An update to the installation instructions and some whitespace cleanup was done in `#808`_ by Thomas A Caswell, and mistake in the quickstart was fixed by Joydeep Bhattacharjee in `#708`_. setup.py improvements --------------------- Support for detecting the version of HDF5 via ``pkgconfig`` was added by Axel Huebl in `#734`_, and support for specifying the path to MPI-supported HDF5 was added by Axel Huebl in `#721`_. ``h5py's`` classifiers were updated to include supported python version and interpreters in `#811`_ by James Tocknell. Support for additional HDF5 features added ------------------------------------------ Low-level support for `HDF5 Direct Chunk Write`_ was added in `#691`_ by Simon Gregor Ebner. Minimal support for `HDF5 File Image Operations`_ was added by Andrea Bedini in `#680`_. Ideas and opinions for further support for both `HDF5 Direct Chunk Write`_ and `HDF5 File Image Operations`_ are welcome. High-level support for reading and writing null dataspaces was added in `#664`_ by James Tocknell. Improvements to type system --------------------------- Reading and writing of compound datatypes has improved, with support for different orderings and alignments (`#701`_ by Jonah Bernhard, `#702`_ by Caleb Morse `#738`_ by @smutch, `#765`_ by Nathan Goldbaum and `#793`_ by James Tocknell). Support for reading extended precision and non-standard floating point numbers has also been added (`#749`_, `#812`_ by Thomas A Caswell, `#787`_ by James Tocknell and `#781`_ by Martin Raspaud). Finally, compatibility improvements to ``Cython`` annotations of HDF5 types were added in `#692`_ and `#693`_ by Aleksandar Jelenak. Other changes ------------- * Fix deprecation of ``-`` for ``numpy`` boolean arrays (`#683`_ by James Tocknell) * Check for duplicates in fancy index validation (`#739`_ by Sam Toyer) * Avoid potential race condition (`#754`_ by James Tocknell) * Fix inconsistency when slicing with ``numpy.array`` of shape ``(1,)`` (`#772`_ by Artsiom) * Use ``size_t`` to store Python object id (`#773`_ by Christoph Gohlke) * Avoid errors when the Python GC runs during ``nonlocal_close()`` (`#776`_ by Antoine Pitrou) * Move from ``six.PY3`` to ``six.PY2`` (`#686`_ by James Tocknell) .. _`#662` : https://github.com/h5py/h5py/pull/662 .. _`#664` : https://github.com/h5py/h5py/pull/664 .. _`#675` : https://github.com/h5py/h5py/pull/675 .. _`#680` : https://github.com/h5py/h5py/pull/680 .. _`#683` : https://github.com/h5py/h5py/pull/683 .. _`#686` : https://github.com/h5py/h5py/pull/686 .. _`#691` : https://github.com/h5py/h5py/pull/691 .. _`#692` : https://github.com/h5py/h5py/pull/692 .. _`#693` : https://github.com/h5py/h5py/pull/693 .. _`#700` : https://github.com/h5py/h5py/pull/700 .. _`#701` : https://github.com/h5py/h5py/pull/701 .. _`#702` : https://github.com/h5py/h5py/pull/702 .. _`#703` : https://github.com/h5py/h5py/pull/703 .. _`#708` : https://github.com/h5py/h5py/pull/708 .. _`#715` : https://github.com/h5py/h5py/pull/715 .. _`#716` : https://github.com/h5py/h5py/pull/716 .. _`#721` : https://github.com/h5py/h5py/pull/721 .. _`#724` : https://github.com/h5py/h5py/pull/724 .. _`#733` : https://github.com/h5py/h5py/pull/733 .. _`#734` : https://github.com/h5py/h5py/pull/734 .. _`#738` : https://github.com/h5py/h5py/pull/738 .. _`#739` : https://github.com/h5py/h5py/pull/739 .. _`#749` : https://github.com/h5py/h5py/pull/749 .. _`#754` : https://github.com/h5py/h5py/pull/754 .. _`#765` : https://github.com/h5py/h5py/pull/765 .. _`#772` : https://github.com/h5py/h5py/pull/772 .. _`#773` : https://github.com/h5py/h5py/pull/773 .. _`#776` : https://github.com/h5py/h5py/pull/776 .. _`#781` : https://github.com/h5py/h5py/pull/781 .. _`#787` : https://github.com/h5py/h5py/pull/787 .. _`#788` : https://github.com/h5py/h5py/pull/788 .. _`#789` : https://github.com/h5py/h5py/pull/789 .. _`#793` : https://github.com/h5py/h5py/pull/793 .. _`#794` : https://github.com/h5py/h5py/pull/794 .. _`#795` : https://github.com/h5py/h5py/pull/795 .. _`#798` : https://github.com/h5py/h5py/pull/798 .. _`#799` : https://github.com/h5py/h5py/pull/799 .. _`#800` : https://github.com/h5py/h5py/pull/800 .. _`#801` : https://github.com/h5py/h5py/pull/801 .. _`#802` : https://github.com/h5py/h5py/pull/802 .. _`#804` : https://github.com/h5py/h5py/pull/804 .. _`#807` : https://github.com/h5py/h5py/pull/807 .. _`#808` : https://github.com/h5py/h5py/pull/808 .. _`#811` : https://github.com/h5py/h5py/pull/811 .. _`#812` : https://github.com/h5py/h5py/pull/812 .. _`HDF5 Direct Chunk Write` : https://support.hdfgroup.org/HDF5/doc/Advanced/DirectChunkWrite/ .. _`HDF5 File Image Operations` : http://www.hdfgroup.org/HDF5/doc/Advanced/FileImageOperations/HDF5FileImageOperations.pdf Acknowlegements --------------- h5py-2.7.1/docs/whatsnew/2.2.rst0000644000175000017500000000510413025343121017701 0ustar tcaswelltcaswell00000000000000What's new in h5py 2.2 ====================== Support for Parallel HDF5 ------------------------- On UNIX platforms, you can now take advantage of MPI and Parallel HDF5. Cython, ``mpi4py`` and an MPI-enabled build of HDF5 are required.. See :ref:`parallel` in the documentation for details. Support for Python 3.3 ---------------------- Python 3.3 is now officially supported. Mini float support (issue #141) ------------------------------- Two-byte floats (NumPy ``float16``) are supported. HDF5 scale/offset filter ------------------------ The Scale/Offset filter added in HDF5 1.8 is now available. Field indexing is now allowed when writing to a dataset (issue #42) ------------------------------------------------------------------- H5py has long supported reading only certain fields from a dataset:: >>> dset = f.create_dataset('x', (100,), dtype=np.dtype([('a', 'f'), ('b', 'i')])) >>> out = dset['a', 0:100:10] >>> out.dtype dtype('float32') Now, field names are also allowed when writing to a dataset: >>> dset['a', 20:50] = 1.0 Region references preserve shape (issue #295) --------------------------------------------- Previously, region references always resulted in a 1D selection, even when 2D slicing was used:: >>> dset = f.create_dataset('x', (10, 10)) >>> ref = dset.regionref[0:5,0:5] >>> out = dset[ref] >>> out.shape (25,) Shape is now preserved:: >>> out = dset[ref] >>> out.shape (5, 5) Additionally, the shape of both the target dataspace and the selection shape can be determined via new methods on the ``regionref`` proxy (now available on both datasets and groups):: >>> f.regionref.shape(ref) (10, 10) >>> f.regionref.selection(ref) (5, 5) Committed types can be linked to datasets and attributes -------------------------------------------------------- HDF5 supports "shared" named types stored in the file:: >>> f['name'] = np.dtype("int64") You can now use these types when creating a new dataset or attribute, and HDF5 will "link" the dataset type to the named type:: >>> dset = f.create_dataset('int dataset', (10,), dtype=f['name']) >>> f.attrs.create('int scalar attribute', shape=(), dtype=f['name']) ``move`` method on Group objects -------------------------------- It's no longer necessary to move objects in a file by manually re-linking them:: >>> f.create_group('a') >>> f['b'] = f['a'] >>> del f['a'] The method ``Group.move`` allows this to be performed in one step:: >>> f.move('a', 'b') Both the source and destination must be in the same file. h5py-2.7.1/docs/whatsnew/2.7.1.rst0000644000175000017500000000333113152157047020060 0ustar tcaswelltcaswell00000000000000What's new in h5py 2.7.1 ======================== 2.7.1 is the first bug-fix release in the 2.7.x series. Bug fixes --------- - :issue:`903` Fixed critical issue with cyclic gc which resulted in segfaults - :issue:`904` Avoid unaligned access fixing h5py on sparc64 - :issue:`883` Fixed compilation issues for some library locations - :issue:`868` Fix deadlock between phil and the import lock in py2 - :issue:`841` Improve windows handling if filenames - :issue:`874` Allow close to be called on file multiple times - :issue:`867`, :issue:`872` Warn on loaded vs complied hdf5 version issues - :issue:`902` Fix overflow computing size of dataset on windows - :issue:`912` Do not mangle capitalization of filenames in error messages - :issue:`842` Fix longdouble on ppc64le - :issue:`862`, :issue:`916` Fix compounds structs with variable-size members Fix h5py segfaulting on some Python 3 versions ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Through an intersection of `Python Issue 30484`_ and :issue:`888`, it was possible for the Python Garbage Collector to activate when closing ``h5py`` objects, which due to how dictionaries were iterated over in Python could cause a segfault. :issue:`903` fixes the Garbage Collector activating whilst closing, whilst `Python Issue 30484`_ had been fixed upstream (and backported to Python 3.3 onwards). Avoid unaligned memory access in conversion functions ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Some architectures (e.g. SPRAC64) do not allow unaligned memory access, which can come up when copying packed structs. :issue:`904` (by James Clarke) uses ``memcpy`` to avoid said unaligned memory access. .. _`Python Issue 30484`: https://bugs.python.org/issue30484 h5py-2.7.1/docs/Makefile0000644000175000017500000001514513030556017016462 0ustar tcaswelltcaswell00000000000000# Makefile for Sphinx documentation # # You can set these variables from the command line. SPHINXOPTS = -W SPHINXBUILD = sphinx-build PAPER = BUILDDIR = _build # User-friendly check for sphinx-build ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) endif # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext help: @echo "Please use \`make ' where is one of" @echo " html to make standalone HTML files" @echo " dirhtml to make HTML files named index.html in directories" @echo " singlehtml to make a single large HTML file" @echo " pickle to make pickle files" @echo " json to make JSON files" @echo " htmlhelp to make HTML files and a HTML help project" @echo " qthelp to make HTML files and a qthelp project" @echo " devhelp to make HTML files and a Devhelp project" @echo " epub to make an epub" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" @echo " latexpdf to make LaTeX files and run them through pdflatex" @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" @echo " text to make text files" @echo " man to make manual pages" @echo " texinfo to make Texinfo files" @echo " info to make Texinfo files and run them through makeinfo" @echo " gettext to make PO message catalogs" @echo " changes to make an overview of all changed/added/deprecated items" @echo " xml to make Docutils-native XML files" @echo " pseudoxml to make pseudoxml-XML files for display purposes" @echo " linkcheck to check all external links for integrity" @echo " doctest to run all doctests embedded in the documentation (if enabled)" clean: rm -rf $(BUILDDIR)/* html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." dirhtml: $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." singlehtml: $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml @echo @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." pickle: $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." json: $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." htmlhelp: $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." qthelp: $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ ".qhcp project file in $(BUILDDIR)/qthelp, like this:" @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/h5py.qhcp" @echo "To view the help file:" @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/h5py.qhc" devhelp: $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp @echo @echo "Build finished." @echo "To view the help file:" @echo "# mkdir -p $$HOME/.local/share/devhelp/h5py" @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/h5py" @echo "# devhelp" epub: $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub @echo @echo "Build finished. The epub file is in $(BUILDDIR)/epub." latex: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @echo "Run \`make' in that directory to run these through (pdf)latex" \ "(use \`make latexpdf' here to do that automatically)." latexpdf: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through pdflatex..." $(MAKE) -C $(BUILDDIR)/latex all-pdf @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." latexpdfja: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through platex and dvipdfmx..." $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." text: $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text @echo @echo "Build finished. The text files are in $(BUILDDIR)/text." man: $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man @echo @echo "Build finished. The manual pages are in $(BUILDDIR)/man." texinfo: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." @echo "Run \`make' in that directory to run these through makeinfo" \ "(use \`make info' here to do that automatically)." info: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo "Running Texinfo files through makeinfo..." make -C $(BUILDDIR)/texinfo info @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." gettext: $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale @echo @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." changes: $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." linkcheck: $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." doctest: $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." xml: $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml @echo @echo "Build finished. The XML files are in $(BUILDDIR)/xml." pseudoxml: $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml @echo @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." h5py-2.7.1/docs/quick.rst0000644000175000017500000001011713025343121016654 0ustar tcaswelltcaswell00000000000000.. _quick: Quick Start Guide ================= Install ------- With `Anaconda `_ or `Miniconda `_:: conda install h5py With `Enthought Canopy `_, use the GUI package manager or:: enpkg h5py With pip or setup.py, see :ref:`install`. Core concepts ------------- An HDF5 file is a container for two kinds of objects: `datasets`, which are array-like collections of data, and `groups`, which are folder-like containers that hold datasets and other groups. The most fundamental thing to remember when using h5py is: **Groups work like dictionaries, and datasets work like NumPy arrays** The very first thing you'll need to do is create a new file:: >>> import h5py >>> import numpy as np >>> >>> f = h5py.File("mytestfile.hdf5", "w") The :ref:`File object ` is your starting point. It has a couple of methods which look interesting. One of them is ``create_dataset``:: >>> dset = f.create_dataset("mydataset", (100,), dtype='i') The object we created isn't an array, but :ref:`an HDF5 dataset `. Like NumPy arrays, datasets have both a shape and a data type: >>> dset.shape (100,) >>> dset.dtype dtype('int32') They also support array-style slicing. This is how you read and write data from a dataset in the file: >>> dset[...] = np.arange(100) >>> dset[0] 0 >>> dset[10] 10 >>> dset[0:100:10] array([ 0, 10, 20, 30, 40, 50, 60, 70, 80, 90]) For more, see :ref:`file` and :ref:`dataset`. Groups and hierarchical organization ------------------------------------ "HDF" stands for "Hierarchical Data Format". Every object in an HDF5 file has a name, and they're arranged in a POSIX-style hierarchy with ``/``-separators:: >>> dset.name u'/mydataset' The "folders" in this system are called :ref:`groups `. The ``File`` object we created is itself a group, in this case the `root group`, named ``/``: >>> f.name u'/' Creating a subgroup is accomplished via the aptly-named ``create_group``:: >>> grp = f.create_group("subgroup") All ``Group`` objects also have the ``create_*`` methods like File:: >>> dset2 = grp.create_dataset("another_dataset", (50,), dtype='f') >>> dset2.name u'/subgroup/another_dataset' By the way, you don't have to create all the intermediate groups manually. Specifying a full path works just fine:: >>> dset3 = f.create_dataset('subgroup2/dataset_three', (10,), dtype='i') >>> dset3.name u'/subgroup2/dataset_three' Groups support most of the Python dictionary-style interface. You retrieve objects in the file using the item-retrieval syntax:: >>> dataset_three = f['subgroup2/dataset_three'] Iterating over a group provides the names of its members:: >>> for name in f: ... print name mydataset subgroup subgroup2 Containership testing also uses names: >>> "mydataset" in f True >>> "somethingelse" in f False You can even use full path names: >>> "subgroup/another_dataset" in f True There are also the familiar ``keys()``, ``values()``, ``items()`` and ``iter()`` methods, as well as ``get()``. Since iterating over a group only yields its directly-attached members, iterating over an entire file is accomplished with the ``Group`` methods ``visit()`` and ``visititems()``, which take a callable:: >>> def printname(name): ... print name >>> f.visit(printname) mydataset subgroup subgroup/another_dataset subgroup2 subgroup2/dataset_three For more, see :ref:`group`. Attributes ---------- One of the best features of HDF5 is that you can store metadata right next to the data it describes. All groups and datasets support attached named bits of data called `attributes`. Attributes are accessed through the ``attrs`` proxy object, which again implements the dictionary interface:: >>> dset.attrs['temperature'] = 99.5 >>> dset.attrs['temperature'] 99.5 >>> 'temperature' in dset.attrs True For more, see :ref:`attributes`. h5py-2.7.1/docs_api/0000755000175000017500000000000013152363123015644 5ustar tcaswelltcaswell00000000000000h5py-2.7.1/docs_api/h5l.rst0000644000175000017500000000033613025343121017063 0ustar tcaswelltcaswell00000000000000Module H5L ========== Linkproxy objects ----------------- .. automodule:: h5py.h5l :members: Module constants ---------------- Link types ~~~~~~~~~~ .. data:: TYPE_HARD .. data:: TYPE_SOFT .. data:: TYPE_EXTERNAL h5py-2.7.1/docs_api/h5d.rst0000644000175000017500000000133513025343121017053 0ustar tcaswelltcaswell00000000000000Module H5D ========== .. automodule:: h5py.h5d :members: Module constants ---------------- Storage strategies ~~~~~~~~~~~~~~~~~~ .. data:: COMPACT .. data:: CONTIGUOUS .. data:: CHUNKED .. _ref.h5d.ALLOC_TIME: Allocation times ~~~~~~~~~~~~~~~~ .. data:: ALLOC_TIME_DEFAULT .. data:: ALLOC_TIME_LATE .. data:: ALLOC_TIME_EARLY .. data:: ALLOC_TIME_INCR Allocation status ~~~~~~~~~~~~~~~~~ .. data:: SPACE_STATUS_NOT_ALLOCATED .. data:: SPACE_STATUS_PART_ALLOCATED .. data:: SPACE_STATUS_ALLOCATED Fill time ~~~~~~~~~ .. data:: FILL_TIME_ALLOC .. data:: FILL_TIME_NEVER .. data:: FILL_TIME_IFSET Fill values ~~~~~~~~~~~ .. data:: FILL_VALUE_UNDEFINED .. data:: FILL_VALUE_DEFAULT .. data:: FILL_VALUE_USER_DEFINED h5py-2.7.1/docs_api/objects.rst0000644000175000017500000000014013025343121020015 0ustar tcaswelltcaswell00000000000000Base object classes =================== .. automodule:: h5py._objects .. autoclass:: ObjectID h5py-2.7.1/docs_api/conf.py0000644000175000017500000002042213114361073017143 0ustar tcaswelltcaswell00000000000000# -*- coding: utf-8 -*- # # Low-level API for h5py documentation build configuration file, created by # sphinx-quickstart on Fri Jan 31 22:42:08 2014. # # This file is execfile()d with the current directory set to its # containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. # # All configuration values have a default; values that are commented out # serve to show the default. import sys import os import h5py # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. sys.path.insert(0, os.path.abspath('.')) # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. #needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = ['sphinx.ext.autodoc', 'automod'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # The suffix of source filenames. source_suffix = '.rst' # The encoding of source files. #source_encoding = 'utf-8-sig' # The master toctree document. master_doc = 'index' # General information about the project. project = u'Low-level API for h5py' copyright = u'2014, Andrew Collette and contributors' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. version = "%d.%d" % h5py.version.version_tuple[0:2] # The full version, including alpha/beta/rc tags. release = h5py.version.version # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. #language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: #today = '' # Else, today_fmt is used as the format for a strftime call. #today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. exclude_patterns = ['_build'] # The reST default role (used for this markup: `text`) to use for all # documents. #default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. #add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). #add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. #show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. #modindex_common_prefix = [] # If true, keep warnings as "system message" paragraphs in the built documents. #keep_warnings = False # -- Options for HTML output ---------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. html_theme = 'nature' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. #html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. #html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". html_title = "Low-level API for h5py" # A shorter title for the navigation bar. Default is the same as html_title. #html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. #html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. #html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. #html_extra_path = [] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. #html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. #html_use_smartypants = True # Custom sidebar templates, maps document names to template names. #html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. #html_additional_pages = {} # If false, no module index is generated. #html_domain_indices = True # If false, no index is generated. #html_use_index = True # If true, the index is split into individual pages for each letter. #html_split_index = False # If true, links to the reST sources are added to the pages. #html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. #html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. #html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. #html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). #html_file_suffix = None # Output file base name for HTML help builder. htmlhelp_basename = 'Low-levelAPIforh5pydoc' # -- Options for LaTeX output --------------------------------------------- latex_elements = { # The paper size ('letterpaper' or 'a4paper'). #'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). #'pointsize': '10pt', # Additional stuff for the LaTeX preamble. #'preamble': '', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ ('index', 'Low-levelAPIforh5py.tex', u'Low-level API for h5py Documentation', u'Andrew Collette and contributors', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of # the title page. #latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. #latex_use_parts = False # If true, show page references after internal links. #latex_show_pagerefs = False # If true, show URL addresses after external links. #latex_show_urls = False # Documents to append as an appendix to all manuals. #latex_appendices = [] # If false, no module index is generated. #latex_domain_indices = True # -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ ('index', 'low-levelapiforh5py', u'Low-level API for h5py Documentation', [u'Andrew Collette and contributors'], 1) ] # If true, show URL addresses after external links. #man_show_urls = False # -- Options for Texinfo output ------------------------------------------- # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ ('index', 'Low-levelAPIforh5py', u'Low-level API for h5py Documentation', u'Andrew Collette and contributors', 'Low-levelAPIforh5py', 'One line description of project.', 'Miscellaneous'), ] # Documents to append as an appendix to all manuals. #texinfo_appendices = [] # If false, no module index is generated. #texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. #texinfo_show_urls = 'footnote' # If true, do not generate a @detailmenu in the "Top" node's menu. #texinfo_no_detailmenu = False h5py-2.7.1/docs_api/h5.rst0000644000175000017500000000120213025343121016700 0ustar tcaswelltcaswell00000000000000Module H5 ========= .. automodule:: h5py.h5 Library API ----------- .. autofunction:: get_config .. autofunction:: get_libversion Configuration class ------------------- .. autoclass:: H5PYConfig Module constants ---------------- .. data:: INDEX_NAME Resolve indices in alphanumeric order .. data:: INDEX_CRT_ORDER Resolve indices in order of object creation. Not always available. .. data:: ITER_NATIVE Traverse index in the fastest possible order. No particular pattern is guaranteed. .. data:: ITER_INC Traverse index in increasing order .. data:: ITER_DEC Traverse index in decreasing order h5py-2.7.1/docs_api/automod.py0000644000175000017500000001714413114361073017675 0ustar tcaswelltcaswell00000000000000 """ Requires patched version of autodoc.py http://bugs.python.org/issue3422 """ import re from functools import partial # === Regexp replacement machinery ============================================ role_expr = re.compile(r"(:.+:(?:`.+`)?)") def safe_replace(istr, expr, rpl): """ Perform a role-safe replacement of all occurances of "expr", using the callable "rpl". """ outparts = [] for part in role_expr.split(istr): if not role_expr.search(part): part = expr.sub(rpl, part) outparts.append(part) return "".join(outparts) # === Replace literal class names ============================================= class_base = r""" (?P
  \W+
)
(?P%s)
(?P
  \W+
)
"""

class_exprs = { "ObjectID": "h5py.h5.ObjectID",
                "GroupID": "h5py.h5g.GroupID",
                "FileID": "h5py.h5f.FileID",
                "DatasetID": "h5py.h5d.DatasetID",
                "TypeID": "h5py.h5t.TypeID",
                "[Dd]ataset creation property list": "h5py.h5p.PropDCID",
                "[Dd]ataset transfer property list": "h5py.h5p.PropDXID",
                "[Ff]ile creation property list": "h5py.h5p.PropFCID",
                "[Ff]ile access property list": "h5py.h5p.PropFAID",
                "[Ll]ink access property list": "h5py.h5p.PropLAID",
                "[Ll]ink creation property list": "h5py.h5p.PropLCID",
                "[Gg]roup creation property list": "h5py.h5p.PropGCID"}


try:
    class_exprs = dict( 
        (re.compile(class_base % x.replace(" ",r"\s"), re.VERBOSE), y) \
        for x, y in class_exprs.iteritems() )
except AttributeError:
    class_exprs = dict( 
        (re.compile(class_base % x.replace(" ",r"\s"), re.VERBOSE), y) \
        for x, y in class_exprs.items() )


def replace_class(istr):

    def rpl(target, match):
        pre, name, post = match.group('pre', 'name', 'post')
        return '%s:class:`%s <%s>`%s' % (pre, name, target, post)

    for expr, target in class_exprs.iteritems():
        rpl2 = partial(rpl, target)
        istr = safe_replace(istr, expr, rpl2)

    return istr

# === Replace constant and category expressions ===============================

# e.g. h5f.OBJ_ALL -> :data:`h5f.OBJ_ALL `
# and  h5f.OBJ*    -> :ref:`h5f.OBJ* `

const_exclude = ['HDF5', 'API', 'H5', 'H5A', 'H5D', 'H5F', 'H5P', 'H5Z', 'INT',
                 'UINT', 'STRING', 'LONG', 'PHIL', 'GIL', 'TUPLE', 'LIST',
                 'FORTRAN', 'BOOL', 'NULL', 'NOT', 'SZIP']
const_exclude = ["%s(?:\W|$)" % x for x in const_exclude]
const_exclude = "|".join(const_exclude)

const_expr = re.compile(r"""
(?P
  (?:^|\s+)                   # Must be preceeded by whitespace or string start
  \W?                         # May have punctuation ( (CONST) or "CONST" )
  (?!%s)                      # Exclude known list of non-constant objects
)
(?Ph5[a-z]{0,2}\.)?   # Optional h5xx. prefix
(?P[A-Z_][A-Z0-9_]+)    # The constant name itself
(?P\*)?                 # Wildcard indicates this is a category
(?P
  \W?                         # May have trailing punctuation
  (?:$|\s+)                   # Must be followed by whitespace or end of string
)                      
""" % const_exclude, re.VERBOSE)

def replace_constant(istr, current_module):

    def rpl(match):
        mod, name, wild = match.group('module', 'name', 'wild')
        pre, post = match.group('pre', 'post')

        if mod is None:
            mod = current_module+'.'
            displayname = name
        else:
            displayname = mod+name

        if wild:
            target = 'ref.'+mod+name
            role = ':ref:'
            displayname += '*'
        else:
            target = 'h5py.'+mod+name
            role = ':data:'

        return '%s%s`%s <%s>`%s' % (pre, role, displayname, target, post)

    return safe_replace(istr, const_expr, rpl)


# === Replace literal references to modules ===================================

mod_expr = re.compile(r"""
(?P
  (?:^|\s+)                 # Must be preceeded by whitespace
  \W?                       # Optional opening paren/quote/whatever
)
(?!h5py)                    # Don't match the package name
(?Ph5[a-z]{0,2})      # Names of the form h5, h5a, h5fd
(?P
  \W?                       # Optional closing paren/quote/whatever
  (?:$|\s+)                 # Must be followed by whitespace
)
""", re.VERBOSE)

def replace_module(istr):

    def rpl(match):
        pre, name, post = match.group('pre', 'name', 'post')
        return '%s:mod:`%s `%s' % (pre, name, name, post)

    return safe_replace(istr, mod_expr, rpl)


# === Replace parameter lists =================================================

# e.g. "    + STRING path ('/default')" -> ":param STRING path: ('/default')"

param_expr = re.compile(r"""
^
\s*
\+
\s+
(?P
  [^\s\(]
  .*
  [^\s\)]
)
(?:
  \s+
  \(
  (?P
    [^\s\(]
    .*
    [^\s\)]
  )
  \)
)?
$
""", re.VERBOSE)

def replace_param(istr):
    """ Replace parameter lists.  Not role-safe. """

    def rpl(match):
        desc, default = match.group('desc', 'default')
        default = ' (%s) ' % default if default is not None else ''
        return ':param %s:%s' % (desc, default)

    return param_expr.sub(rpl, istr)



# === Begin Sphinx extension code =============================================

def is_callable(docstring):
    return str(docstring).strip().startswith('(')

def setup(spx):

    def proc_doc(app, what, name, obj, options, lines):
        """ Process docstrings for modules and routines """

        final_lines = lines[:]

        # Remove the signature lines from the docstring
        if is_callable(obj.__doc__):
            doclines = []
            arglines = []
            final_lines = arglines
            for line in lines:
                if len(line.strip()) == 0:
                    final_lines = doclines
                final_lines.append(line)

        # Resolve class names, constants and modules
        if hasattr(obj, 'im_class'):
            mod = obj.im_class.__module__
        elif hasattr(obj, '__module__'):
            mod = obj.__module__
        else:
            mod = ".".join(name.split('.')[0:2])  # i.e. "h5py.h5z"
        mod = mod.split('.')[1]  # i.e. 'h5z'

        del lines[:]
        for line in final_lines:
            #line = replace_param(line)
            line = replace_constant(line, mod)
            line = replace_module(line)
            line = replace_class(line)
            line = line.replace('**kwds', '\*\*kwds').replace('*args','\*args')
            lines.append(line)




    def proc_sig(app, what, name, obj, options, signature, return_annotation):
        """ Auto-generate function signatures from docstrings """

        def getsig(docstring):
            """ Get (sig, return) from a docstring, or None. """
            if not is_callable(docstring):
                return None

            lines = []
            for line in docstring.split("\n"):
                if len(line.strip()) == 0:
                    break
                lines.append(line)
            rawsig = " ".join(x.strip() for x in lines)

            if '=>' in rawsig:
                sig, ret = tuple(x.strip() for x in rawsig.split('=>'))
            elif '->' in rawsig:
                sig, ret = tuple(x.strip() for x in rawsig.split('->'))
            else:
                sig = rawsig
                ret = None

            if sig == "()":
                sig = "( )" # Why? Ask autodoc.

            return (sig, ret)

        sigtuple = getsig(obj.__doc__)

        return sigtuple

    spx.connect('autodoc-process-signature', proc_sig)
    spx.connect('autodoc-process-docstring', proc_doc)

h5py-2.7.1/docs_api/h5s.rst0000644000175000017500000000165413025343121017076 0ustar  tcaswelltcaswell00000000000000Module H5S
==========

.. automodule:: h5py.h5s

Functional API
--------------

.. autofunction:: create
.. autofunction:: create_simple
.. autofunction:: decode

Dataspace objects
-----------------

.. autoclass:: SpaceID
    :show-inheritance:
    :members:

Module constants
----------------

.. data:: ALL

    Accepted in place of an actual datapace; means "every point"

.. data:: UNLIMITED
    
    Indicates an unlimited maximum dimension

Dataspace class codes
~~~~~~~~~~~~~~~~~~~~~

.. data:: NO_CLASS
.. data:: SCALAR
.. data:: SIMPLE

Selection codes
~~~~~~~~~~~~~~~

.. data:: SELECT_NOOP
.. data:: SELECT_SET
.. data:: SELECT_OR
.. data:: SELECT_AND
.. data:: SELECT_XOR
.. data:: SELECT_NOTB
.. data:: SELECT_NOTA
.. data:: SELECT_APPEND
.. data:: SELECT_PREPEND
.. data:: SELECT_INVALID

Existing selection type
~~~~~~~~~~~~~~~~~~~~~~~

.. data:: SEL_NONE
.. data:: SEL_POINTS
.. data:: SEL_HYPERSLABS
.. data:: SEL_ALL




h5py-2.7.1/docs_api/h5fd.rst0000644000175000017500000000173013025343121017220 0ustar  tcaswelltcaswell00000000000000Module H5FD
===========

.. automodule:: h5py.h5fd

Module constants
----------------

.. data:: MEM_DEFAULT
.. data:: MEM_SUPER
.. data:: MEM_BTREE
.. data:: MEM_DRAW
.. data:: MEM_GHEAP
.. data:: MEM_LHEAP
.. data:: MEM_OHDR
.. data:: MEM_NTYPES

File drivers
~~~~~~~~~~~~

.. data:: CORE
.. data:: FAMILY
.. data:: LOG
.. data:: MPIO
.. data:: MULTI
.. data:: SEC2
.. data:: STDIO
.. data:: WINDOWS

Logging driver settings
~~~~~~~~~~~~~~~~~~~~~~~

.. note:: Not all logging flags are currently implemented by HDF5.

.. data:: LOG_LOC_READ
.. data:: LOG_LOC_WRITE
.. data:: LOG_LOC_SEEK
.. data:: LOG_LOC_IO

.. data:: LOG_FILE_READ 
.. data:: LOG_FILE_WRITE
.. data:: LOG_FILE_IO

.. data:: LOG_FLAVOR

.. data:: LOG_NUM_READ
.. data:: LOG_NUM_WRITE
.. data:: LOG_NUM_SEEK
.. data:: LOG_NUM_IO

.. data:: LOG_TIME_OPEN
.. data:: LOG_TIME_READ
.. data:: LOG_TIME_WRITE
.. data:: LOG_TIME_SEEK
.. data:: LOG_TIME_CLOSE
.. data:: LOG_TIME_IO

.. data:: LOG_ALLOC
.. data:: LOG_ALL


h5py-2.7.1/docs_api/index.rst0000644000175000017500000000152613025343121017504 0ustar  tcaswelltcaswell00000000000000
Low-Level API Reference
=======================


This documentation contains the auto-generated API information for the
HDF5 for Python "low-level" interface, a collection of Cython modules
which form the interface to the HDF5 C library.  It's hosted separately from
our main documentation as it requires autodoc.

These docs are updated less frequently than the spiffy ReadTheDocs-hosted
main documentation; this means roughly once per minor (X.Y) release.

This may not be what you're looking for!
----------------------------------------

**The main docs for h5py are at** http://docs.h5py.org.  **These are
the docs specifically for the h5py low-level interface.**

Contents
--------

.. toctree::
    :maxdepth: 2

    objects
    h5
    h5a
    h5ac
    h5d
    h5f
    h5fd
    h5g
    h5i
    h5l
    h5o
    h5p
    h5r
    h5s
    h5t
    h5zh5py-2.7.1/docs_api/h5z.rst0000644000175000017500000000211513025343121017076 0ustar  tcaswelltcaswell00000000000000Module H5Z
==========

.. automodule:: h5py.h5z
    :members:

Module constants
----------------

.. _ref.h5z.FILTER:

Predefined filters
~~~~~~~~~~~~~~~~~~

.. data:: FILTER_NONE
.. data:: FILTER_ALL
.. data:: FILTER_DEFLATE
.. data:: FILTER_SHUFFLE
.. data:: FILTER_FLETCHER32
.. data:: FILTER_SZIP
.. data:: FILTER_SCALEOFFSET
.. data:: FILTER_LZF

.. _ref.h5z.FLAG:

Filter flags
~~~~~~~~~~~~

.. data:: FLAG_DEFMASK
.. data:: FLAG_MANDATORY
.. data:: FLAG_OPTIONAL
.. data:: FLAG_INVMASK
.. data:: FLAG_REVERSE
.. data:: FLAG_SKIP_EDC

.. _ref.h5z.SZIP:

SZIP-specific options
~~~~~~~~~~~~~~~~~~~~~

.. data:: SZIP_ALLOW_K13_OPTION_MASK
.. data:: SZIP_CHIP_OPTION_MASK
.. data:: SZIP_EC_OPTION_MASK
.. data:: SZIP_NN_OPTION_MASK
.. data:: SZIP_MAX_PIXELS_PER_BLOCK

Scale/offset-specific options
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

.. data:: SO_FLOAT_DSCALE
.. data:: SO_FLOAT_ESCALE
.. data:: SO_INT
.. data:: SO_INT_MINBITS_DEFAULT

Other flags
~~~~~~~~~~~

.. data:: FILTER_CONFIG_ENCODE_ENABLED
.. data:: FILTER_CONFIG_DECODE_ENABLED

.. data:: DISABLE_EDC
.. data:: ENABLE_EDC
.. data:: NO_EDC


h5py-2.7.1/docs_api/h5t.rst0000644000175000017500000000704613025343121017100 0ustar  tcaswelltcaswell00000000000000Module H5T
==========

.. automodule:: h5py.h5t

Functions specific to h5py
--------------------------

.. autofunction:: py_create
.. autofunction:: special_dtype
.. autofunction:: check_dtype

Functional API
--------------
.. autofunction:: create
.. autofunction:: open
.. autofunction:: array_create
.. autofunction:: enum_create
.. autofunction:: vlen_create
.. autofunction:: decode
.. autofunction:: convert
.. autofunction:: find

Type classes
------------

.. autoclass:: TypeID
    :members:

Atomic classes
~~~~~~~~~~~~~~

Atomic types are integers and floats.  Much of the functionality for each is
inherited from the base class :class:`TypeAtomicID`.

.. autoclass:: TypeAtomicID
    :show-inheritance:
    :members:
    
.. autoclass:: TypeIntegerID
    :show-inheritance:
    :members:

.. autoclass:: TypeFloatID
    :show-inheritance:
    :members:

Strings
~~~~~~~

.. autoclass:: TypeStringID
    :show-inheritance:
    :members:

Compound Types
~~~~~~~~~~~~~~

Traditional compound type (like NumPy record type) and enumerated types share
a base class, :class:`TypeCompositeID`.

.. autoclass:: TypeCompositeID
    :show-inheritance:
    :members:

.. autoclass:: TypeCompoundID
    :show-inheritance:
    :members:

.. autoclass:: TypeEnumID
    :show-inheritance:
    :members:

Other types
~~~~~~~~~~~

.. autoclass:: TypeArrayID
    :show-inheritance:
    :members:

.. autoclass:: TypeOpaqueID
    :show-inheritance:
    :members:

.. autoclass:: TypeVlenID
    :show-inheritance:
    :members:

.. autoclass:: TypeBitfieldID
    :show-inheritance:
    :members:

.. autoclass:: TypeReferenceID
    :show-inheritance:
    :members:

Predefined Datatypes
--------------------

These locked types are pre-allocated by the library.

Floating-point
~~~~~~~~~~~~~~

.. data:: IEEE_F32LE
.. data:: IEEE_F32BE
.. data:: IEEE_F64LE
.. data:: IEEE_F64BE

Integer types
~~~~~~~~~~~~~

.. data:: STD_I8LE
.. data:: STD_I16LE
.. data:: STD_I32LE
.. data:: STD_I64LE

.. data:: STD_I8BE
.. data:: STD_I16BE
.. data:: STD_I32BE
.. data:: STD_I64BE

.. data:: STD_U8LE
.. data:: STD_U16LE
.. data:: STD_U32LE
.. data:: STD_U64LE

.. data:: STD_U8BE
.. data:: STD_U16BE
.. data:: STD_U32BE
.. data:: STD_U64BE

.. data:: NATIVE_INT8
.. data:: NATIVE_UINT8
.. data:: NATIVE_INT16
.. data:: NATIVE_UINT16
.. data:: NATIVE_INT32
.. data:: NATIVE_UINT32
.. data:: NATIVE_INT64
.. data:: NATIVE_UINT64
.. data:: NATIVE_FLOAT
.. data:: NATIVE_DOUBLE 

Reference types
~~~~~~~~~~~~~~~

.. data:: STD_REF_OBJ
.. data:: STD_REF_DSETREG

String types
~~~~~~~~~~~~

.. data:: C_S1

    Null-terminated fixed-length string

.. data:: FORTRAN_S1

    Zero-padded fixed-length string
    
.. data:: VARIABLE

    Variable-length string

Python object type
~~~~~~~~~~~~~~~~~~

.. data:: PYTHON_OBJECT

Module constants
----------------

Datatype class codes
~~~~~~~~~~~~~~~~~~~~

.. data:: NO_CLASS
.. data:: INTEGER
.. data:: FLOAT
.. data:: TIME
.. data:: STRING
.. data:: BITFIELD
.. data:: OPAQUE
.. data:: COMPOUND
.. data:: REFERENCE
.. data:: ENUM
.. data:: VLEN
.. data:: ARRAY

API Constants
~~~~~~~~~~~~~

.. data:: SGN_NONE
.. data:: SGN_2

.. data:: ORDER_LE
.. data:: ORDER_BE
.. data:: ORDER_VAX
.. data:: ORDER_NONE
.. data:: ORDER_NATIVE

.. data:: DIR_DEFAULT
.. data:: DIR_ASCEND
.. data:: DIR_DESCEND

.. data:: STR_NULLTERM
.. data:: STR_NULLPAD
.. data:: STR_SPACEPAD

.. data:: NORM_IMPLIED
.. data:: NORM_MSBSET
.. data:: NORM_NONE

.. data:: CSET_ASCII
.. DATA:: CSET_UTF8

.. data:: PAD_ZERO
.. data:: PAD_ONE
.. data:: PAD_BACKGROUND

.. data:: BKG_NO
.. data:: BKG_TEMP
.. data:: BKG_YES













h5py-2.7.1/docs_api/h5a.rst0000644000175000017500000000065613025343121017055 0ustar  tcaswelltcaswell00000000000000Module H5A
==========

.. automodule:: h5py.h5a

Functional API
--------------

.. autofunction:: create
.. autofunction:: open
.. autofunction:: exists
.. autofunction:: rename
.. autofunction:: delete
.. autofunction:: get_num_attrs
.. autofunction:: get_info
.. autofunction:: iterate

Info objects
------------

.. autoclass:: AttrInfo
    :members:

Attribute objects
-----------------

.. autoclass:: AttrID
    :members:


h5py-2.7.1/docs_api/h5p.rst0000644000175000017500000000240013025343121017061 0ustar  tcaswelltcaswell00000000000000Module H5P
==========

.. automodule:: h5py.h5p

Functional API
--------------

.. autofunction:: create

Base classes
------------

.. autoclass:: PropID
    :show-inheritance:
    :members:

.. autoclass:: PropClassID
    :show-inheritance:
    :members:

.. autoclass:: PropInstanceID
    :show-inheritance:
    :members:

.. autoclass:: PropCreateID
    :show-inheritance:
    :members:

.. autoclass:: PropCopyID
    :show-inheritance:
    :members:

File creation
-------------

.. autoclass:: PropFCID
    :show-inheritance:
    :members:

File access
-----------

.. autoclass:: PropFAID
    :show-inheritance:
    :members:

Dataset creation
----------------

.. autoclass:: PropDCID
    :show-inheritance:
    :members:


Link creation
-------------

.. autoclass:: PropLCID
    :show-inheritance:
    :members:


Link access
-----------

.. autoclass:: PropLAID
    :show-inheritance:
    :members:


Group creation
--------------

.. autoclass:: PropGCID
    :show-inheritance:
    :members:


Module constants
----------------

Predefined classes
~~~~~~~~~~~~~~~~~~

.. data:: DEFAULT
.. data:: FILE_CREATE
.. data:: FILE_ACCESS
.. data:: DATASET_CREATE
.. data:: DATASET_XFER
.. data:: OBJECT_COPY
.. data:: LINK_CREATE
.. data:: LINK_ACCESS
.. data:: GROUP_CREATE

h5py-2.7.1/docs_api/h5f.rst0000644000175000017500000000230613025343121017054 0ustar  tcaswelltcaswell00000000000000Module H5F
==========

.. automodule:: h5py.h5f

Functional API
--------------

.. autofunction:: open
.. autofunction:: create
.. autofunction:: flush
.. autofunction:: is_hdf5
.. autofunction:: mount
.. autofunction:: unmount
.. autofunction:: get_name
.. autofunction:: get_obj_count
.. autofunction:: get_obj_ids

File objects
------------

.. autoclass:: FileID
    :members:

Module constants
----------------

.. _ref.h5f.ACC:

File access flags
~~~~~~~~~~~~~~~~~

.. data:: ACC_TRUNC

    Create/truncate file

.. data:: ACC_EXCL

    Create file if it doesn't exist; fail otherwise

.. data:: ACC_RDWR

    Open in read/write mode

.. data:: ACC_RDONLY

    Open in read-only mode


.. _ref.h5f.CLOSE:

File close strength
~~~~~~~~~~~~~~~~~~~

.. data:: CLOSE_WEAK
.. data:: CLOSE_SEMI
.. data:: CLOSE_STRONG
.. data:: CLOSE_DEFAULT

.. _ref.h5f.SCOPE:

File scope
~~~~~~~~~~

.. data:: SCOPE_LOCAL
.. data:: SCOPE_GLOBAL

.. _ref.h5f.OBJ:

Object types
~~~~~~~~~~~~

.. data:: OBJ_FILE
.. data:: OBJ_DATASET
.. data:: OBJ_GROUP
.. data:: OBJ_DATATYPE
.. data:: OBJ_ATTR
.. data:: OBJ_ALL
.. data:: OBJ_LOCAL

Library version bounding
~~~~~~~~~~~~~~~~~~~~~~~~

.. data:: LIBVER_EARLIEST
.. data:: LIBVER_LATEST


h5py-2.7.1/docs_api/h5o.rst0000644000175000017500000000162713025343121017072 0ustar  tcaswelltcaswell00000000000000Module H5O
==========

.. automodule:: h5py.h5o

Functional API
--------------

.. autofunction:: open
.. autofunction:: link
.. autofunction:: copy
.. autofunction:: set_comment
.. autofunction:: get_comment
.. autofunction:: visit
.. autofunction:: get_info

Info classes
------------

.. autoclass:: ObjInfo
    :members:

Module constants
----------------

Object types
~~~~~~~~~~~~

.. data:: TYPE_GROUP
.. data:: TYPE_DATASET
.. data:: TYPE_NAMED_DATATYPE

.. _ref.h5o.COPY:

Copy flags
~~~~~~~~~~

.. data:: COPY_SHALLOW_HIERARCHY_FLAG

    Copy only immediate members of a group.

.. data:: COPY_EXPAND_SOFT_LINK_FLAG

    Expand soft links into new objects.

.. data:: COPY_EXPAND_EXT_LINK_FLAG

    Expand external link into new objects.

.. data:: COPY_EXPAND_REFERENCE_FLAG

    Copy objects that are pointed to by references.

.. data:: COPY_WITHOUT_ATTR_FLAG

    Copy object without copying attributes.

h5py-2.7.1/docs_api/Makefile0000644000175000017500000001523613025343121017306 0ustar  tcaswelltcaswell00000000000000# Makefile for Sphinx documentation
#

# You can set these variables from the command line.
SPHINXOPTS    =
SPHINXBUILD   = sphinx-build
PAPER         =
BUILDDIR      = _build

# User-friendly check for sphinx-build
ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/)
endif

# Internal variables.
PAPEROPT_a4     = -D latex_paper_size=a4
PAPEROPT_letter = -D latex_paper_size=letter
ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
# the i18n builder cannot share the environment and doctrees with the others
I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .

.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext

help:
	@echo "Please use \`make ' where  is one of"
	@echo "  html       to make standalone HTML files"
	@echo "  dirhtml    to make HTML files named index.html in directories"
	@echo "  singlehtml to make a single large HTML file"
	@echo "  pickle     to make pickle files"
	@echo "  json       to make JSON files"
	@echo "  htmlhelp   to make HTML files and a HTML help project"
	@echo "  qthelp     to make HTML files and a qthelp project"
	@echo "  devhelp    to make HTML files and a Devhelp project"
	@echo "  epub       to make an epub"
	@echo "  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
	@echo "  latexpdf   to make LaTeX files and run them through pdflatex"
	@echo "  latexpdfja to make LaTeX files and run them through platex/dvipdfmx"
	@echo "  text       to make text files"
	@echo "  man        to make manual pages"
	@echo "  texinfo    to make Texinfo files"
	@echo "  info       to make Texinfo files and run them through makeinfo"
	@echo "  gettext    to make PO message catalogs"
	@echo "  changes    to make an overview of all changed/added/deprecated items"
	@echo "  xml        to make Docutils-native XML files"
	@echo "  pseudoxml  to make pseudoxml-XML files for display purposes"
	@echo "  linkcheck  to check all external links for integrity"
	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"

clean:
	rm -rf $(BUILDDIR)/*

html:
	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
	@echo
	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."

dirhtml:
	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
	@echo
	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."

singlehtml:
	$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
	@echo
	@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."

pickle:
	$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
	@echo
	@echo "Build finished; now you can process the pickle files."

json:
	$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
	@echo
	@echo "Build finished; now you can process the JSON files."

htmlhelp:
	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
	@echo
	@echo "Build finished; now you can run HTML Help Workshop with the" \
	      ".hhp project file in $(BUILDDIR)/htmlhelp."

qthelp:
	$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
	@echo
	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
	      ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
	@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Low-levelAPIforh5py.qhcp"
	@echo "To view the help file:"
	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Low-levelAPIforh5py.qhc"

devhelp:
	$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
	@echo
	@echo "Build finished."
	@echo "To view the help file:"
	@echo "# mkdir -p $$HOME/.local/share/devhelp/Low-levelAPIforh5py"
	@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Low-levelAPIforh5py"
	@echo "# devhelp"

epub:
	$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
	@echo
	@echo "Build finished. The epub file is in $(BUILDDIR)/epub."

latex:
	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
	@echo
	@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
	@echo "Run \`make' in that directory to run these through (pdf)latex" \
	      "(use \`make latexpdf' here to do that automatically)."

latexpdf:
	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
	@echo "Running LaTeX files through pdflatex..."
	$(MAKE) -C $(BUILDDIR)/latex all-pdf
	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."

latexpdfja:
	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
	@echo "Running LaTeX files through platex and dvipdfmx..."
	$(MAKE) -C $(BUILDDIR)/latex all-pdf-ja
	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."

text:
	$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
	@echo
	@echo "Build finished. The text files are in $(BUILDDIR)/text."

man:
	$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
	@echo
	@echo "Build finished. The manual pages are in $(BUILDDIR)/man."

texinfo:
	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
	@echo
	@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
	@echo "Run \`make' in that directory to run these through makeinfo" \
	      "(use \`make info' here to do that automatically)."

info:
	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
	@echo "Running Texinfo files through makeinfo..."
	make -C $(BUILDDIR)/texinfo info
	@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."

gettext:
	$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
	@echo
	@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."

changes:
	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
	@echo
	@echo "The overview file is in $(BUILDDIR)/changes."

linkcheck:
	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
	@echo
	@echo "Link check complete; look for any errors in the above output " \
	      "or in $(BUILDDIR)/linkcheck/output.txt."

doctest:
	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
	@echo "Testing of doctests in the sources finished, look at the " \
	      "results in $(BUILDDIR)/doctest/output.txt."

xml:
	$(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml
	@echo
	@echo "Build finished. The XML files are in $(BUILDDIR)/xml."

pseudoxml:
	$(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml
	@echo
	@echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml."
h5py-2.7.1/docs_api/h5r.rst0000644000175000017500000000101313025343121017062 0ustar  tcaswelltcaswell00000000000000Module H5R
==========

.. automodule:: h5py.h5r

Functional API
--------------

.. autofunction:: create
.. autofunction:: dereference
.. autofunction:: get_region
.. autofunction:: get_obj_type
.. autofunction:: get_name

Reference classes
-----------------

.. autoclass:: Reference
    :members:

.. autoclass:: RegionReference
    :show-inheritance:
    :members:

API constants
-------------

.. data:: OBJECT

    Typecode for object references

.. data:: DATASET_REGION

    Typecode for dataset region references


h5py-2.7.1/docs_api/h5g.rst0000644000175000017500000000117013025343121017053 0ustar  tcaswelltcaswell00000000000000Module H5G
==========

.. automodule:: h5py.h5g

Functional API
--------------

.. autofunction:: open
.. autofunction:: create
.. autofunction:: iterate
.. autofunction:: get_objinfo

Info objects
------------

.. autoclass:: GroupStat
    :members:

Group objects
-------------

.. autoclass:: GroupID
    :members:

Module constants
----------------

Object type codes
~~~~~~~~~~~~~~~~~

.. data:: LINK

    Symbolic link

.. data:: GROUP

    HDF5 group

.. data:: DATASET

    HDF5 dataset

.. data:: TYPE

    Named (file-resident) datatype

Link type codes
~~~~~~~~~~~~~~~

.. data:: LINK_HARD
.. data:: LINK_SOFT











h5py-2.7.1/docs_api/h5i.rst0000644000175000017500000000054013025343121017055 0ustar  tcaswelltcaswell00000000000000Module H5I
==========

Functional API
--------------

.. automodule:: h5py.h5i
    :members:

Module constants
----------------

Identifier classes
~~~~~~~~~~~~~~~~~~

.. data:: BADID
.. data:: FILE
.. data:: GROUP
.. data:: DATASPACE
.. data:: DATASET
.. data:: ATTR
.. data:: REFERENCE
.. data:: GENPROP_CLS
.. data:: GENPROP_LST
.. data:: DATATYPE

h5py-2.7.1/tox.ini0000644000175000017500000000345113146376232015411 0ustar  tcaswelltcaswell00000000000000[tox]
envlist = {py26,py27,py33,py34,py35,py36,pypy}-{test}-{deps,mindeps},docs,check-manifest,checkreadme

[testenv]
deps =
    deps: numpy>=1.7
    deps: cython>=0.19
    mindeps: numpy==1.7
    mindeps: cython==0.19
commands =
    test: python {toxinidir}/ci/fix_paths.py {envsitepackagesdir}
    test: python -c "from sys import exit; import h5py; exit(0) if h5py.run_tests().wasSuccessful() else exit(1)"
changedir =
    test: {toxworkdir}
passenv =
    HDF5_DIR
    TOXPYTHON
basepython =
    pypy: {env:TOXPYTHON:pypy}
    py26: {env:TOXPYTHON:python2.6}
    py27: {env:TOXPYTHON:python2.7}
    py33: {env:TOXPYTHON:python3.3}
    py34: {env:TOXPYTHON:python3.4}
    py35: {env:TOXPYTHON:python3.5}
    py36: {env:TOXPYTHON:python3.6}

[testenv:py26-test-deps]
deps =
    unittest2
    numpy>=1.7,<1.11
    cython>=0.19

[testenv:py33-test-deps]
deps =
    numpy>=1.7,<1.12
    cython>=0.19

[testenv:py26-test-mindeps]
deps =
    unittest2
    numpy==1.7
    cython==0.19

[testenv:py33-test-mindeps]
deps =
    numpy==1.7
    cython==0.19

[testenv:py34-test-mindeps]
deps =
    numpy==1.9
    cython==0.19

[testenv:py35-test-mindeps]
deps =
    numpy==1.10.0.post2
    cython==0.19

[testenv:py36-test-mindeps]
deps =
    numpy==1.12
    cython==0.19

[testenv:py34-test-mpi4py]
deps =
    numpy==1.9
    cython==0.19
    mpi4py>=1.3.1

[testenv:docs]
skip_install=True
basepython = {env:TOXPYTHON:python}
changedir=docs
deps=
    sphinx
commands=
    sphinx-build -W -b html -d {envtmpdir}/doctrees .  {envtmpdir}/html

[testenv:check-manifest]
skip_install=True
basepython = {env:TOXPYTHON:python}
deps=check-manifest
setenv =
    CHECK_MANIFEST=true
commands=
    check-manifest

[testenv:checkreadme]
skip_install=True
basepython = {env:TOXPYTHON:python}
deps=readme_renderer
commands=
    python setup.py check -s -r
h5py-2.7.1/README.rst0000644000175000017500000000275413114357361015567 0ustar  tcaswelltcaswell00000000000000.. image:: https://travis-ci.org/h5py/h5py.png
   :target: https://travis-ci.org/h5py/h5py
.. image:: https://ci.appveyor.com/api/projects/status/h3iajp4d1myotprc/branch/master?svg=true
   :target: https://ci.appveyor.com/project/h5py/h5py/branch/master

HDF5 for Python
===============
`h5py` is a thin, pythonic wrapper around the `HDF5 `_, which runs on Python 2 (2.6-2.7), and Python 3 (3.3-3.6).

Websites
--------

* Main website: http://www.h5py.org
* Source code: http://github.com/h5py/h5py
* Mailing list: https://groups.google.com/d/forum/h5py

Installation
------------

Pre-build `h5py` can either be installed via your Python Distribution (e.g.
`Continuum Anaconda`_, `Enthought Canopy`_) or from `PyPI`_ via `pip`_.
`h5py` is also distributed in many Linux Distributions (e.g. Ubuntu, Fedora),
and in the MacOS package managers `Homebrew `_,
`Macports `_, or `Fink `_.

More detailed installation instructions, including how to install `h5py` with
MPI support, can be found at: http://docs.h5py.org/en/latest/build.html.


Reporting bugs
--------------

Open a bug at http://github.com/h5py/h5py/issues.  For general questions, ask
on the list (https://groups.google.com/d/forum/h5py).

.. _`Continuum Anaconda`: http://continuum.io/downloads
.. _`Enthought Canopy`: https://www.enthought.com/products/canopy/
.. _`PyPI`: https://pypi.org/project/h5py/
.. _`pip`: https://pip.pypa.io/en/stable/
h5py-2.7.1/MANIFEST.in0000644000175000017500000000122313146376232015627 0ustar  tcaswelltcaswell00000000000000include ANN.rst
include api_gen.py
include MANIFEST.in
include pylintrc
include README.rst
include setup_build.py
include setup_configure.py
include tox.ini

recursive-include docs *
prune docs/_build
recursive-include docs_api *
prune docs_api/_build
recursive-include examples *.py

recursive-include h5py *.h *.pyx *.pxd *.pxi *.py *.txt
exclude h5py/config.pxi
exclude h5py/defs.pxd
exclude h5py/defs.pyx

recursive-include licenses *
recursive-include lzf *
recursive-include windows *

recursive-exclude * .DS_Store

exclude ci other .github
recursive-exclude ci *
recursive-exclude other *
recursive-exclude .github *
exclude pavement.py
exclude *.yml
h5py-2.7.1/api_gen.py0000644000175000017500000001766613114361073016057 0ustar  tcaswelltcaswell00000000000000
"""
    Generate the lowest-level Cython bindings to HDF5.
    
    In order to translate HDF5 errors to exceptions, the raw HDF5 API is
    wrapped with Cython "error wrappers".  These are cdef functions with
    the same names and signatures as their HDF5 equivalents, but implemented
    in the h5py.defs extension module.
    
    The h5py.defs files (defs.pyx and defs.pxd), along with the "real" HDF5
    function definitions (_hdf5.pxd), are auto-generated by this script from
    api_functions.txt.  This file also contains annotations which indicate
    whether a function requires a certain minimum version of HDF5, an
    MPI-aware build of h5py, or special error handling.
    
    This script is called automatically by the h5py build system when the
    output files are missing, or api_functions.txt has been updated.
    
    See the Line class in this module for documentation of the format for
    api_functions.txt.
        
    h5py/_hdf5.pxd:     Cython "extern" definitions for HDF5 functions
    h5py/defs.pxd:      Cython definitions for error wrappers
    h5py/defs.pyx:      Cython implementations of error wrappers
"""

import re
import os.path as op


class Line(object):

    """
        Represents one line from the api_functions.txt file.
        
        Exists to provide the following attributes:
        
        mpi:        Bool indicating if MPI required
        error:      Bool indicating if special error handling required
        version:    None or a minimum-version tuple
        code:       String with function return type
        fname:      String with function name
        sig:        String with raw function signature
        args:       String with sequence of arguments to call function
        
        Example:    MPI ERROR 1.8.12 int foo(char* a, size_t b)
        
        .mpi:       True
        .error:     True
        .version:   (1, 8, 12)
        .code:      "int"
        .fname:     "foo"
        .sig:       "char* a, size_t b"
        .args:      "a, b"
    """
        
    PATTERN = re.compile("""(?P(MPI)[ ]+)?
                            (?P(ERROR)[ ]+)?
                            (?P([0-9]+\.[0-9]+\.[0-9]+))?
                            ([ ]+)?
                            (?P(unsigned[ ]+)?[a-zA-Z_]+[a-zA-Z0-9_]*\**)[ ]+
                            (?P[a-zA-Z_]+[a-zA-Z0-9_]*)[ ]*
                            \((?P[a-zA-Z0-9_,* ]*)\)
                            """, re.VERBOSE)

    SIG_PATTERN = re.compile("""
                             (?:unsigned[ ]+)?
                             (?:[a-zA-Z_]+[a-zA-Z0-9_]*\**)
                             [ ]+[ *]*
                             (?P[a-zA-Z_]+[a-zA-Z0-9_]*)
                             """, re.VERBOSE)
                            
    def __init__(self, text):
        """ Break the line into pieces and populate object attributes.
        
        text: A valid function line, with leading/trailing whitespace stripped.
        """
        
        m = self.PATTERN.match(text)
        if m is None:
            raise ValueError("Invalid line encountered: {0}".format(text))
            
        parts = m.groupdict()
        
        self.mpi = parts['mpi'] is not None
        self.error = parts['error'] is not None
        self.version = parts['version']
        if self.version is not None:
            self.version = tuple(int(x) for x in self.version.split('.'))
        self.code = parts['code']
        self.fname = parts['fname']
        self.sig = parts['sig']

        sig_const_stripped = self.sig.replace('const', '')
        self.args = self.SIG_PATTERN.findall(sig_const_stripped)
        if self.args is None:
            raise ValueError("Invalid function signature: {0}".format(self.sig))
        self.args = ", ".join(self.args)


raw_preamble = """\
include "config.pxi"
from api_types_hdf5 cimport *
from api_types_ext cimport *

"""

def_preamble = """\
include "config.pxi"

from api_types_hdf5 cimport *
from api_types_ext cimport *

"""

imp_preamble = """\
include "config.pxi"
from api_types_ext cimport *
from api_types_hdf5 cimport *

cimport _hdf5

from _errors cimport set_exception
"""


class LineProcessor(object):

    def run(self):

        # Function definitions file
        self.functions = open(op.join('h5py', 'api_functions.txt'), 'r')

        # Create output files
        self.raw_defs =     open(op.join('h5py', '_hdf5.pxd'), 'w')
        self.cython_defs =  open(op.join('h5py', 'defs.pxd'), 'w')
        self.cython_imp =   open(op.join('h5py', 'defs.pyx'), 'w')

        self.raw_defs.write(raw_preamble)
        self.cython_defs.write(def_preamble)
        self.cython_imp.write(imp_preamble)

        for text in self.functions:
        
            # Directive specifying a header file
            if not text.startswith(' ') and not text.startswith('#') and \
            len(text.strip()) > 0:
                inc = text.split(':')[0]
                self.raw_defs.write('cdef extern from "%s.h":\n' % inc)
                continue
            
            text = text.strip()

            # Whitespace or comment line
            if len(text) == 0 or text[0] == '#':
                continue

            # Valid function line
            self.line = Line(text)
            self.write_raw_sig()
            self.write_cython_sig()
            self.write_cython_imp()
    
        self.functions.close()
        self.cython_imp.close()
        self.cython_defs.close()
        self.raw_defs.close()

    def add_cython_if(self, block):
        """ Wrap a block of code in the required "IF" checks """
        
        def wrapif(condition, code):
            code = code.replace('\n', '\n    ', code.count('\n')-1) # Yes, -1.
            code = "IF {0}:\n    {1}".format(condition, code)
            return code

        if self.line.mpi:
            block = wrapif('MPI', block)
        if self.line.version is not None:
            block = wrapif('HDF5_VERSION >= {0.version}'.format(self.line), block)

        return block

    def write_raw_sig(self):
        """ Write out "cdef extern"-style definition for an HDF5 function """

        raw_sig = "{0.code} {0.fname}({0.sig}) except *\n".format(self.line)
        raw_sig = self.add_cython_if(raw_sig)
        raw_sig = "\n".join(("  "+x if x.strip() else x) for x in raw_sig.split("\n"))
        self.raw_defs.write(raw_sig)

    def write_cython_sig(self):
        """ Write out Cython signature for wrapper function """

        cython_sig = "cdef {0.code} {0.fname}({0.sig}) except *\n".format(self.line)
        cython_sig = self.add_cython_if(cython_sig)
        self.cython_defs.write(cython_sig)

    def write_cython_imp(self):
        """ Write out Cython wrapper implementation """

        # Figure out what test and return value to use with error reporting
        if '*' in self.line.code or self.line.code in ('H5T_conv_t',):
            condition = "==NULL"
            retval = "NULL"
        elif self.line.code in ('int', 'herr_t', 'htri_t', 'hid_t','hssize_t','ssize_t') \
          or re.match(r'H5[A-Z]+_[a-zA-Z_]+_t', self.line.code):
            condition = "<0"
            retval = "-1"
        elif self.line.code in ('unsigned int','haddr_t','hsize_t','size_t'):
            condition = "==0"
            retval = 0
        else:
            raise ValueError("Return code <<%s>> unknown" % self.line.code)

        # Have to use except * because Cython can't handle special types here
        imp = """\
cdef {0.code} {0.fname}({0.sig}) except *:
    cdef {0.code} r
    _hdf5.H5Eset_auto(NULL, NULL)
    r = _hdf5.{0.fname}({0.args})
    if r{condition}:
        if set_exception():
            return <{0.code}>{retval}
        elif {0.error}:
            raise RuntimeError("Unspecified error in {0.fname} (return value {condition})")
    return r

"""
        imp = imp.format(self.line, condition=condition, retval=retval)
        imp = self.add_cython_if(imp)
        self.cython_imp.write(imp)


def run():
    lp = LineProcessor()
    lp.run()


if __name__ == '__main__':
    run()
h5py-2.7.1/ANN.rst0000644000175000017500000000343613025343121015232 0ustar  tcaswelltcaswell00000000000000Announcing HDF5 for Python (h5py) 2.5.0
========================================

The h5py team is happy to announce the availability of h5py 2.5.0.

This release introduces experimental support for the highly-anticipated
"Single Writer Multiple Reader" (SWMR) feature in the upcoming HDF5 1.10
release.  SWMR allows sharing of a single HDF5 file between multiple processes
without the complexity of MPI or multiprocessing-based solutions.  

This is an experimental feature that should NOT be used in production code.
We are interested in getting feedback from the broader community with respect
to performance and the API design.

For more details, check out the h5py user guide:
http://docs.h5py.org/en/latest/swmr.html

SWMR support was contributed by Ulrik Pedersen.


What's h5py?
------------

The h5py package is a Pythonic interface to the HDF5 binary data format.

It lets you store huge amounts of numerical data, and easily manipulate
that data from NumPy. For example, you can slice into multi-terabyte
datasets stored on disk, as if they were real NumPy arrays. Thousands of
datasets can be stored in a single file, categorized and tagged however
you want.

Documentation is at:

http://docs.h5py.org


Changes
-------

* Experimental SWMR support
* Group and AttributeManager classes now inherit from the appropriate ABCs
* Fixed an issue with 64-bit float VLENS
* Cython warning cleanups related to "const"
* Entire code base ported to "six"; 2to3 removed from setup.py
  

Acknowlegements
---------------

This release incorporates changes from, among others:

* Ulrik Pedersen
* James Tocknell
* Will Parkin
* Antony Lee
* Peter H. Li
* Peter Colberg
* Ghislain Antony Vaillant


Where to get it
---------------

Downloads, documentation, and more are available at the h5py website:

http://www.h5py.org