pyzolib-0.3.4/0000775000175000017500000000000012573243411013424 5ustar almaralmar00000000000000pyzolib-0.3.4/qt/0000775000175000017500000000000012573243411014050 5ustar almaralmar00000000000000pyzolib-0.3.4/qt/__init__.py0000664000175000017500000003175612245716760016205 0ustar almaralmar00000000000000# -*- coding: utf-8 -*- # Copyright (c) 2013, Almar Klein # # This file is distributed under the terms of the (new) BSD License. """ Module that serves as a proxy for loading Qt libraries. This module has several goals: * Import QtCore, QtGui, etc. from PySide or PyQt4 (whichever is available). * Fix some incompatibilities between the two. * For applications that bring their own Qt libs, avoid clashes. * Allow using the PySide or PyQt4 libraries of the system (the ones in /usr/lib/...), so that frozen applications can look good. To use do ``from pyzolib.qt import QtCore, QtGui``. Note that this proxy package is designed to be portable; it should be possible to use it in your own application or library. To use in frozen applications, create a qt.conf next to the executable, that has text as specified in DEFAULT_QT_CONF_TEXT. By modifying the text, preferences can be changed. Notes ----- To prevent colliding of Qt libs when an app brings its own libs, in particular on KDE, the plugins of Qt should be disabled. This needs to be done in two places: * via qt.conf "Plugins = ''" * set the QT_PLUGIN_PATH variable to empty string The latter is equivalent to QtGui.QApplication.setLibraryPaths([]), but has the advantage that it can be set beforehand. A downside of the plugins being disabled is that the native style (GTK+, Oxygen) cannot be used and other features (Unity integrated toolbar) are not available. This module allows a work-around by loading the native libs. """ import sys import os import imp import importlib VERBOSE = False def qt_name(): """ Return the name of the Qt lib in use: 'PySide', 'PyQt4' or None. """ try: importer_instance._import_qt() except ImportError: pass else: return importer_instance._qtPackage.__name__ def loadWidget(filename, parent=None): """ Load a widget from a .ui file. Returns a QWidget object. """ # Note that PyQt4 has PyQt4.uic.loadUi(filename, basewidget) # allowing the newly created widget to inherit from a given widget # instance. This is not supported in PySide and therefore not # suported by this function. # Check if not os.path.isfile(filename): raise ValueError('Filename in loadWidget() is not a valid file.') if qt_name().lower() == 'pyside': # Import (from PySide import QtCore, QtUiTools) QtCore = importer_instance.load_module('QtCore') QtUiTools = importer_instance.load_module('QtUiTools') # Create loader and load widget loader = QtUiTools.QUiLoader() uifile = QtCore.QFile(filename) uifile.open(QtCore.QFile.ReadOnly) w = loader.load(uifile, parent) uifile.close() return w else: # Import (from PyQt4 import QtCore, uic) QtCore = importer_instance.load_module('QtCore') uic = importer_instance.load_module('uic') # Load widget w = uic.loadUi(filename) # We set the parent explicitly if parent is not None: w.setParent(parent) return w class QtProxyImporter: """ Importer to import Qt modules, either from PySide or from PyQt, and either from this Python's version, or the system ones (if available and matching). """ def __init__(self): self._qtPackage = None self._enabled = True self._import_path = None # None for 'normal' (non-system) import def find_module(self, fullname, path=None): """ This is called by Python's import mechanism. We return ourself only if this really looks like a Qt import, and when its imported as a submodule from this stub package. """ # Only proceed if we are enabled if not self._enabled: return None # Get different parts of the module name nameparts = fullname.split('.') # sip is required by PyQt4 if fullname == 'sip': self._import_qt() return self # If the import is relative to this package, we will try to # import relative to the selected qtPackage if '.'.join(nameparts[:-1]) == __name__: self._import_qt() return self def load_module(self, fullname): """ This method is called by Python's import mechanism after this instance has been returned from find_module. Here we actually import the module and do some furher processing. """ # Get different parts of the module name nameparts = fullname.split('.') modulename = nameparts[-1] # We can only proceed if qtPackage was loaded if self._qtPackage is None: raise ImportError() # Get qt dir or dummy if self._import_path: qtdir = os.path.dirname(self._qtPackage.__file__) else: qtdir = '/nonexisting/dir/with/subdirs/dummy' # Get real name and path to load it from if fullname == self._qtPackage.__name__: return self._qtPackage elif fullname == 'sip': realmodulename = 'sip' searchdir = os.path.dirname(qtdir) elif modulename.startswith('Qt') or modulename == 'uic': realmodulename = '%s.%s' % (self._qtPackage.__name__, modulename) searchdir = qtdir else: raise ImportError() # Import. We also need to modify sys.path in case this is a system package if os.path.isdir(qtdir): if VERBOSE: print('load_module explicitly: %s' % fullname) sys.path.insert(0, os.path.dirname(qtdir)) try: for entry in os.listdir(searchdir): if entry.startswith(modulename+'.'): m = imp.load_dynamic( realmodulename, os.path.join(searchdir, entry)) break else: raise ImportError('Could not import %s' % realmodulename) finally: sys.path.pop(0) else: # Module can be inside a zip-file when frozen # Import normally, and disable ourselves so we do not recurse if VERBOSE: print('load_module normally: %s' % realmodulename) self._enabled = False try: p = __import__(realmodulename) finally: self._enabled = True # Get the actual modele if '.' in realmodulename: m = getattr(p, modulename) else: m = p # Also register in sys.modules under the name as it was imported sys.modules[realmodulename] = m sys.modules[fullname] = m # Fix some compatibility issues self._fix_compat(m) # Done return m def _determine_preference(self): """ Determine preference by reading from qt.conf. """ # Get dirs to look for qt.conf dirs = [os.path.dirname(sys.executable)] script_dir = '' if sys.path: script_dir = sys.path[0] if getattr(sys, 'frozen', None): script_dir = os.path.dirname(script_dir) dirs.append(script_dir) # Read qt.conf for dir in dirs: qt_conf = os.path.join(dir, 'qt.conf') if os.path.isfile(qt_conf): text = open(qt_conf, 'rb').read().decode('utf-8', 'ignore') break else: text = '' # Parse qt.conf prefer_system = False prefer_toolkit = '' # for line in text.splitlines(): line = line.split('#',1)[0].strip() if '=' not in line: continue key, val = [i.strip() for i in line.split('=', 1)] if key == 'PreferSystem' and val.lower() in ('yes', 'true', '1'): prefer_system = True if key == 'PreferToolkit': prefer_toolkit = val return prefer_system, prefer_toolkit def _import_qt(self, toolkit=None): """ This is where we import either PySide or PyQt4. This is done only once. """ # Make qtPackage global and only proceed if its not set yet if self._qtPackage is not None: return # Establish preference prefer_system, prefer_toolkit = self._determine_preference() # Check toolkit, use pyside by default prefer_toolkit = toolkit or prefer_toolkit or 'pyside' if prefer_toolkit.lower() not in ('pyside', 'pyqt4'): prefer_toolkit = 'pyside' print('Invalid Qt toolit preference given: "%s"' % prefer_toolkit) # Really import self._qtPackage = self._import_qt_for_real(prefer_system, prefer_toolkit) # Disable plugins if necessary if self._qtPackage and sys.platform.startswith('linux'): if not self._qtPackage.__file__.startswith('/usr'): os.environ['QT_PLUGIN_PATH'] = '' def _import_qt_for_real(self, prefer_system, prefer_toolkit): """ The actual importing. """ # Perhaps it is already loaded if 'PySide' in sys.modules: return sys.modules['PySide'] elif 'PyQt4' in sys.modules: return sys.modules['PyQt4'] # Init potential imports pyside_imports = [('PySide', None)] pyqt4_imports = [('PyQt4', None)] pyside_system_imports = [] pyqt4_system_imports = [] # Get possible paths, but only on Linux if sys.platform.startswith('linux'): # Determine where PySide or PyQt4 can be ver = sys.version[:3] possible_paths = ['/usr/local/lib/python%s/dist-packages' % ver, os.path.expanduser('~/.local/lib/python%s/site-packages' % ver)] if os.path.isdir('/usr/lib/python%s' % ver): possible_paths.append('/usr/lib/python%s/dist-packages' % ver[0]) # Trty if it is there for path in possible_paths: if os.path.isdir(os.path.join(path, 'PySide')): pyside_system_imports.append(('PySide', path)) if os.path.isdir(os.path.join(path, 'PyQt4')): pyqt4_system_imports.append(('PyQt4', path)) # Combine imports in right order if prefer_system: if 'pyside' == prefer_toolkit.lower(): imports = pyside_system_imports + pyqt4_system_imports + \ pyside_imports + pyqt4_imports else: imports = pyqt4_system_imports + pyside_system_imports + \ pyqt4_imports + pyside_imports else: if 'pyside' == prefer_toolkit.lower(): imports = pyside_imports + pyqt4_imports #+ \ #pyside_system_imports + pyqt4_system_imports else: imports = pyqt4_imports + pyside_imports #+ \ #pyqt4_system_imports + pyside_system_imports # Try importing package = None for package_name, path in imports: if path: sys.path.insert(0, path) if VERBOSE: print('Attempting to import %s (system=%i)' % (package_name, bool(path))) self._import_path = path try: return __import__(package_name, level=0) except ImportError as err: if VERBOSE: print('Import failed') finally: if path: sys.path.pop(0) else: raise ImportError('Could not import PySide nor PyQt4.') def _fix_compat(self, m): """ Fix incompatibilities between PySide and PyQt4. """ if self._qtPackage.__name__ == 'PySide': pass else: if m.__name__.endswith('QtCore'): m.Signal = m.pyqtSignal # todo: more compat, like uic loading importer_instance = QtProxyImporter() sys.meta_path.insert(0, importer_instance) DEFAULT_QT_CONF_TEXT = """## This file contains configuration options for Qt. ## It disables plugins so that an application that brings its own ## Qt libraries do not clashs with the native Qt. It also has options ## that the pyzolib.qt proxy uses to allow you to use your system ## PySide/PyQt4 libraries. [Py] ## Preferred toolkit: PySide or PyQt4 PreferToolkit = PySide ## Uncomment if pyzolib.qt should try to use the system libraries ## Note that you version of Python must be ABI compatible with the ## version on your system for this to work #PreferSystem = yes [Paths] ## This disables plugins, avoiding Qt library clashes Plugins = '' ## On Ubuntu Unity, if PreferSystem is enabled, uncomment this to ## enable the fancy menu bar. #Plugins = /usr/lib/x86_64-linux-gnu/qt4/plugins """ pyzolib-0.3.4/dllutils.py0000664000175000017500000002151212227572037015640 0ustar almaralmar00000000000000# -*- coding: utf-8 -*- # Copyright (C) 2012 Almar Klein # This module is distributed under the terms of the (new) BSD License. """ Various utilities to modify Dynamic Link libraries. Needed to build the Pyzo distro, and it's possible that this functionality is needed to fix extension modules after installation in a Pyzo distro. This is a mix of utilities for Windows, Mac and Linux. """ import os import stat import sys import subprocess import time import re _COMMAND_TO_SEARCH_PATH = [] def get_command_to_set_search_path(): """ Get the command to change the RPATH of executables and dynamic libraries. Returns None if there is no such command or if it cannot be found. """ # Check if already computed if _COMMAND_TO_SEARCH_PATH: return _COMMAND_TO_SEARCH_PATH[0] # Get name of the utility # In Pyzo it should be present in 'shared'. utilCommand = None if sys.platform.startswith('win'): return if sys.platform.startswith('linux'): utilname = 'patchelf' if sys.platform.startswith('darwin'): utilname = 'install_name_tool' if True: # Try old Pyzo utilCommand = os.path.join(sys.prefix, 'shared', utilname) if not os.path.isfile(utilCommand): utilCommand = utilname # Try new Pyzo / anaconda utilCommand = os.path.join(sys.prefix, 'bin', utilname) if not os.path.isfile(utilCommand): utilCommand = utilname # Test whether it exists try: subprocess.check_output(['which', utilCommand]) except Exception: raise RuntimeError('Could not get command (%s) to set search path.' % utilCommand) # Store and return _COMMAND_TO_SEARCH_PATH.append(utilCommand) return utilCommand def set_search_path(fname, *args): """ set_search_path(fname, *args) For the given library/executable, set the search path to the relative paths specified in args. For Linux: The RPATH is the path to search for its dependencies. http://enchildfone.wordpress.com/2010/03/23/a-description-of-rpath-origin-ld_library_path-and-portable-linux-binaries/ For Mac: We use the @rpath identifier to get similar behavior to Linux. But each dependency must be specified. To realize this, we need to check for each dependency whether it is on one of te given search paths. For Windows: not supported in any way. Windows searches next to the library and then in system paths and PATH. """ # Prepare args = [arg.lstrip('/') for arg in args if arg] args = [arg for arg in args if arg != '.'] # Because we add empty dir anyway args.append('') # make libs search next to themselves command = get_command_to_set_search_path() if sys.platform.startswith('linux'): # Create search path value rpath = ':'.join( ['$ORIGIN/'+arg for arg in args] ) # Modify rpath using a call to patchelf utility cmd = [command, '--set-rpath', rpath, fname] subprocess.check_call(cmd) print('Set RPATH for %r' % os.path.basename(fname)) #print('Set RPATH for %r: %r' % (os.path.basename(fname), rpath)) elif sys.platform.startswith('darwin'): # ensure write permissions mode = os.stat(fname).st_mode if not (mode & stat.S_IWUSR): os.chmod(fname, mode | stat.S_IWUSR) # let the file itself know its place (simpyl on rpath) name = os.path.basename(fname) subprocess.call(('install_name_tool', '-id', '@rpath/'+name, fname)) # find the references: call otool -L on the file otool = subprocess.Popen(('otool', '-L', fname), stdout = subprocess.PIPE) references = otool.stdout.readlines()[1:] # Replace each reference rereferencedlibs = [] for reference in references: # find the actual referenced file name referencedFile = reference.decode().strip().split()[0] if referencedFile.startswith('@'): continue # the referencedFile is already a relative path # Get lib name _, name = os.path.split(referencedFile) if name.lower() == 'python': name = 'libpython' # Rename Python lib on Mac # see if we provided the referenced file potentiallibs = [os.path.join(os.path.dirname(fname), arg, name) for arg in args] # if so, change the reference and rpath if any([os.path.isfile(p) for p in potentiallibs]): subprocess.call(('install_name_tool', '-change', referencedFile, '@rpath/'+name, fname)) for arg in args: mac_add_rpath(fname, '@loader_path/' + arg) mac_add_rpath(fname, '@executable_path/') # use libpython next to exe rereferencedlibs.append(name) if rereferencedlibs: print('Replaced refs for "%s": %s' % (os.path.basename(fname), ', '.join(rereferencedlibs)) ) elif sys.platform.startswith('win'): raise RuntimeError('Windows has no way of setting the search path on a library or exe.') else: raise RuntimeError('Do not know how to set search path of library or exe on %s' % sys.platform) def mac_add_rpath(fname, rpath): """ mac_add_rpath(fname, rpath) Set the rpath for a Mac library or executble. If the rpath is already registered, it is ignored. """ cmd = ['install_name_tool', '-add_rpath', rpath, fname] p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) while p.poll() is None: time.sleep(0.01) if p.returncode: msg = p.stdout.read().decode('utf-8') if 'would duplicate path' in msg: pass # Ignore t else: raise RuntimeError('Could not set rpath: ' + msg) def remove_CRT_dependencies(dirname, recurse=True): """ remove_CRT_dependencies(path, recurse=True) Check all .dll and .pyd files in the given directory (and its subdirectories if recurse is True), removing the dependency on the Windows C runtime from the embedded manifest. """ dllExt = ['.dll', '.pyd'] for entry in os.listdir(dirname): p = os.path.join(dirname, entry) if recurse and os.path.isdir(p): remove_CRT_dependencies(p, recurse) elif os.path.isfile(p) and os.path.splitext(p)[1].lower() in dllExt: remove_CRT_dependency(p) def remove_CRT_dependency(filename): """ remove_CRT_dependency(filename) Modify the embedded manifest of a Windows dll (or pyd file), such that it no longer depends on the Windows C runtime. In effect, the dll will fall back to using the C runtime that the executable depends on (and has loaded in memory). This function is not necessary for dll's and pyd's that come with Python, because these are build without the CRT dependencies for a while. However, some third party packages (e.g. PySide) do have these dependencies, and they need to be removed in order to work on a system that does not have the C-runtime installed. Based on this diff by C. Gohlke: http://bugs.python.org/file15113/msvc9compiler_stripruntimes_regexp2.diff See discussion at: http://bugs.python.org/issue4120 """ if 'QtCore' in filename: 1/0 # Read the whole file with open(filename, 'rb') as f: try: bb = f.read() except IOError: #raise IOError('Could not read %s'%filename) print('Warning: could not read %s'%filename) return # Remove assemblyIdentity tag # This code is different from that in python's distutils/msvc9compiler.py # by removing re.DOTALL and replaceing the second DOT with "(.|\n|\r)", # which means that the first DOT cannot contain newlines. Would we not do # this, the match is too greedy (and causes tk85.dll to break). pattern = r"""|)""" pattern = re.compile(pattern.encode('ascii')) bb, hasMatch = _replacePatternWithSpaces(pattern, bb) if hasMatch: # Remove dependentAssembly tag if it's empty pattern = "\s*".encode('ascii') bb, hasMatch = _replacePatternWithSpaces(pattern, bb) # Write back with open(filename, "wb") as f: f.write(bb) print('Removed embedded MSVCR dependency for: %s' % filename) def _replacePatternWithSpaces(pattern, bb): match = re.search(pattern, bb) if match is not None: L = match.end() - match.start() bb = re.sub(pattern, b" "*L, bb) return bb, True else: return bb, False pyzolib-0.3.4/path.py0000664000175000017500000001434412325713106014736 0ustar almaralmar00000000000000# -*- coding: utf-8 -*- # Copyright (c) 2012, The Pyzo team # # This file is distributed under the terms of the (new) BSD License. """ Definition of a Path class (that inherits from string) for object oriented path processing. Inspired by http://pypi.python.org/pypi/path.py, but modfied to also work on Python 3 and a bit more compact (a simpler API). """ import os import sys import fnmatch print('Note about pyzolib.path - better use pathlib (part of py34)') # Python 2/3 compatibility (from six.py) if sys.version_info[0] == 3: text_type = str string_types = str, else: text_type = unicode string_types = basestring, class Path(text_type): """ Path(path, clean=True) Object oriented approach to path handling. If clean is True, applies expandvars, expanduser, normpath and realpath to clean the path. Concatenate paths using the "/" operator. """ def __new__(cls, path, clean=True): # Clean path if clean: path = os.path.expandvars(path) path = os.path.expanduser(path) path = os.path.normpath(path) #path = os.path.realpath(path) # this kills relative paths # Instantiate obj = text_type.__new__(cls, path) return obj ## Magic functions def __repr__(self): return 'Path(%s)' % text_type.__repr__(self) # Adding a path and a string yields a path. def __add__(self, other): if isinstance(other, string_types): return Path(text_type.__add__(self, other), False) else: return NotImplemented() def __radd__(self, other): if isinstance(other, string_types): return Path(other.__add__(self), False) else: return NotImplemented() # The / operator joins paths. def __div__(self, rel): """ fp.__div__(rel) == fp / rel == fp.joinpath(rel) Join two path components, adding a separator character if needed. """ return Path(os.path.join(self, rel)) # Clean=True, so '..' is converted # Make the / operator work even when true division is enabled. __truediv__ = __div__ def __eq__(self, other): return str.__eq__(self.normcase(), os.path.normcase(other)) def __neq__(self, other): return not Path.__eq__(self, other) def __hash__(self): # The __eq__ and __neq__ make it unhashable, thus we need __has__ too return str.__hash__(self) ## Identity @property def isfile(self): return os.path.isfile(self) @property def isdir(self): # Add os.sep, because trailing spaces seem to be ignored on Windows return os.path.isdir(self+os.sep) @property def stat(self): return os.stat(self) ## Getting parts @property def dirname(self): return Path(os.path.dirname(self), False) @property def basename(self): return Path(os.path.basename(self), False) @property def ext(self): return os.path.splitext(self)[1] @property def drive(self): drive, r = os.path.splitdrive(self) return Path(drive, False) ## Listing def listdir(self, pattern=None): """ Return the list of entries contained in this directory. """ names = os.listdir(self) if pattern is not None: names = fnmatch.filter(names, pattern) return [self / child for child in names] def dirs(self, pattern=None): """ Return the list of directories contained in this directory. """ return [p for p in self.listdir(pattern) if p.isdir] def files(self, pattern=None): """ Return the list of file contained in this directory. """ return [p for p in self.listdir(pattern) if p.isfile] ## Transforming def normcase(self): """ Makes the path lowercase on case-insensitive file systems (like Windows), otherwise (e.g. Linux) leaves the path unchanged. """ return Path(os.path.normcase(self), False) def realpath(self): """ Return the absolute version of a path, follow symlinks. """ return Path(os.path.abspath(self), False) def abspath(self): """ Return the absolute version of a path. """ return Path(os.path.abspath(self), False) def relpath(self, start=None): """ Return a relative filepath either from the current directory or from an optional start point. """ return Path(os.path.relpath(self, reference), False) # todo: a walk function ## Actions def makedir(self, mode=0o777, tolerant=False): """ Make dir. If tolerant is True, will only attempt if the dir does not yet exist. """ if not tolerant or not os.path.isdir(self): os.mkdir(self, mode) def makedirs(self, mode=0o777, tolerant=False): """ Make dir (and parent dirs). If tolerant is True, will only attempt if the dir does not yet exist. """ if not tolerant or not os.path.isdir(self): os.makedirs(self, mode) def removedir(self, tolerant=False): """ Remove directory. If tolerant is True, will only attempt if the dir exists. """ if not tolerant or os.path.isdir(self): os.rmdir(self) def removedirs(self, tolerant=False): """ Remove directory and all empty intermediate ones. If tolerant is True, will only attempt if the dir exists. """ if not tolerant or os.path.isdir(self): os.removedirs(self) def remove(self, tolerant=False): """ Remove file. If tolerant is True, will only attempt if the file exists. """ if not tolerant or not os.path.isfile(self): os.remove(self) if __name__ == '__main__': p = Path('c:/almar') s = set() s.add(p) s.add(p) pyzolib-0.3.4/gccutils.py0000664000175000017500000001020512211112473015601 0ustar almaralmar00000000000000# -*- coding: utf-8 -*- # Copyright (c) 2012, The Pyzo team # # This file is distributed under the terms of the (new) BSD License. """ Module gccutils Utilities to make sure there is a working gcc compiler. Particularly for on Windows (MinGW and MinGW-W64) and also on Mac. """ import os import sys import struct import subprocess from pyzolib.paths import ISWIN, ISMAC, ISLINUX # todo: test on Mac and Linux (Windows ok) def has_gcc(): """ has_gcc() Returns True if the gcc command invokes the gcc compiler. """ # Use subprocess to see if the gcc executable is there try: p = subprocess.Popen('gcc --version', stdout=subprocess.PIPE, shell=True) text = p.stdout.readline().decode('utf-8').rstrip() except OSError: text = '' # Check if text.startswith('gcc'): return text else: return False def gcc_dir(): """ gcc_dir() Get the path to the directory containing the GCC compiler. Search in different places, including the Pyzo directory. On Windows searches the default installation dirs (c:/mingw*). """ # Get number of bits of this process NBITS = 8 * struct.calcsize("P") # Define gcc names on different OS's if ISWIN: dirName, exeName = 'mingw%i' %NBITS, 'gcc.exe' else: dirName, exeName = 'gcc%i' %NBITS, 'gcc' # Init possible_paths = [] # Is Pyzo there, and is there a mingw install? If so, prefer that one path = None#pyzo_dir() if path: path = os.path.join(path, 'ext', dirName) if path and os.path.isdir(path): possible_paths.append(path) # Init possible paths with default mingw directories if ISWIN: possible_paths.append(os.path.join(sys.prefix, 'mingw')) if NBITS == 32: possible_paths.extend(['c:/mingw32', 'c:/mingw']) possible_paths.extend(['c:/mingw-w64', 'c:/mingw64']) elif NBITS == 64: possible_paths.extend(['c:/mingw-w64', 'c:/mingw64']) possible_paths.extend(['c:/mingw', 'c:/mingw32']) elif ISMAC: pass # todo: what to do on Mac, or would that be /usr/bin etc # Check possible directories for path in possible_paths: path_gcc = os.path.join(path, 'bin', exeName) if os.path.isfile(path_gcc): return path else: return None def _insert_gcc_dir_in_pythonpath(): """ Try to find the gcc directory and add it to the PATH env. variable. This function can safely be run multiple times. """ # Can we find a gcc installation? # This tries to find one that matches with the number of bits of the # Python installation path = gcc_dir() if not path: return False # Let's hope it is installed in another way else: pathToAdd = os.path.join(path, 'bin') #print('Found gcc compiler in "%s"' % path) # Get all directories in PATH, excluding the one we want to add s = os.path.pathsep paths1 = os.environ['PATH'].split(s) paths1 = [p for p in paths1 if (p and p!=pathToAdd)] # Add to the front of PATH to make it override the default gcc paths1.insert(0, pathToAdd) os.environ['PATH'] = s.join(paths1) return True def prepare_gcc(): """ prepare_gcc() Make sure that the bin directory of the gcc compiler is in the PATH. Modifies os.environ['PATH']. Call this before compiling a Cython module, and also when freezing, because compiled Cython modules may dynamically link to gcc libraries. """ # Try inserting gcc dir of Pyzo (or MinGW on Windows) in os.environ['PATH'] inserted = _insert_gcc_dir_in_pythonpath() # Check if we now have a working gcc working = has_gcc() # Let the user know if it's not working if not working: if inserted: print('Warning: gcc directory detected, but gcc not working.') else: print('Warning: gcc not available; please install.') pyzolib-0.3.4/pyximport.py0000664000175000017500000005270512216614061016057 0ustar almaralmar00000000000000# -*- coding: utf-8 -*- # Copyright (c) 2012, The Pyzo team # # This file is distributed under the terms of the (new) BSD License. """ Module pyximport Provides functionality to compile Cython files on the fly. Before installing a cython module, call pyximport.install(). The module will be compiled if necessary, and then imported. Extra keyword arguments can be supplied to install() to control the compiling stage. In this manner even complex code can be compiled without the need of a setup script. See the docs of the install function for more information. Example1 -------- from pyzo import pyximport pyximport.install() import your_cython_module Example2 -------- from pyzo import pyximport pyximport.install( language='c++', compiler='native', include_dirs=['include'], library_dirs=['lib'], libraries=['1394camera'] ) import your_cython_module """ Remarks_related_to_pyzo = """ We need a clear distinction between developer and end-user. The latter is the person who uses apps that are created using Pyzo. The end-user does not have a Pyzo directory, and should not need a gcc compiler; cx_freeze will make sure that all the libraries are packed. But we have to help cx_freeze do that. For one thing, we need normal "import a_cython_module" statements. Notes for Windows (Pyzo related): * MinGW or Microsoft studio. Not sure if latter also works with free version. * MinGW is relatively portable, we could send it along with the rest of Pyzo. * But MinGW has no 64 bit support, The MinGW-w64 project does, but its less stable. * To get MinGW or MinGW-w64 relatively easy: http://tdm-gcc.tdragon.net * Users with 32bit should get mingw, others 64bit. Cross compiling, nah. * Both the compile-args and link-args need '-m32' and '-DMS_WIN32' (or 64) * The TDM version compiles but is very unstable, getting invalid accesss errors * The normal version (sezero build) asks for msvcr90.dll, so ship that along!! * libpython32.a needs to be placed in [pythondir]libs directory. Otherwise distutils tries to build it, which fails on Py3k. * the mingw directory must be in os.environ['PATH'], also when freezing! Troubleshooting: * Make sure c:/mingw/bin is in the PATH * Make sure to include the numpy libs Known problems on Windows: * "No module named msvccompiler in numpy.distutils" "Unable to find vcvarsall.bat" --> there is no MS compiler installed. Install visual studio 2008. Not 2010! * "'gcc' is not recognized as an internal or external command, operable program or batch file." "Could not locate executable gcc" --> Mingw is used, but is not installed * "fatal error LNK1181: cannot open input file 'files.obj'" There are probably spaces in the paths to the libraries. """ # todo: test when frozen # todo: test when used from a deployed site_packages with no admin rights # todo: in that case, can we not use a temp directory? import os, sys import struct import traceback import shutil import imp from pyzolib import paths from pyzolib.gccutils import prepare_gcc # Get number of bits of this process NBITS = 8 * struct.calcsize("P") def getFileNames(moduleName, path='', sourceExt='.pyx'): """ Get source and binary file names: * sourceName: the filename of the .pyx source file * binName: the filename of the corresponding .pyd/.os binary file * binName2: like binName but with mangled name using Python version+bits """ # Source file name sourceName = os.path.join(path, moduleName + sourceExt) # Binary name (plain) if sys.platform.startswith('win'): binName = os.path.join(path, moduleName + '.pyd') else: binName = os.path.join(path, moduleName + '.so') # Binary name with extension to mark python version and nbits binName2, ext = os.path.splitext(binName) binName2 = binName2 + '_py%i%i_%i' + ext binName2 = binName2 % (sys.version_info[0], sys.version_info[1], NBITS) return sourceName, binName, binName2 def getFileModificationTimes(*args): """ Given filenames as arguments, get the modification times of each file, or 0 of the file does not exist. """ # ms accuracy, on Linux shutil.copy still results in a tiny winy difference T = [] for arg in args: if os.path.isfile(arg): t = os.stat(arg).st_mtime t = round(t, 3) else: t = 0 T.append(t) # Done return tuple(T) def locate_cython_module(moduleName): """ Locate the .pyx or .pyd/.os file corresponding with the given name. """ # Get paths to search module in paths1 = [p for p in sys.path] paths1.insert(0, '') # Search for path in paths1: fnames = getFileNames(moduleName, path) # source, bin, bin2 if any([os.path.isfile(fname) for fname in fnames]): return path else: return None # Note that empty string is valid path (current directory) def install(**kwargs): """ install(**kwargs) Install the Cython importer. Call this every time before importing a Cython module. The code is recompiled if necessary. For simple code, one just needs to call install(). For more advanced code, use the keyword arguments to specify language, include dirs etc. This makes setup scripts unnecessary in many cases. Keyword arguments ----------------- compiler : {'gcc', 'native'} On Windows, use this to specify the compiler. Default gcc. language : {'c', 'c++'} Language to compile the Cython code to. Default c. include_dirs : list List of directories to find header files. The list is extended with numpy header files. library_dirs : list List of directories to find libraries. libraries : list Names of libraries to link to. extra_compile_args : list Extra compile arguments. '-O2' is added by default, and when using gcc, some flags to distinguish 32bit/64bit code. extra_link_args : list Extra link arguments. When using gcc, some flags to distinguish 32bit/64bit code. """ # NOTE: The kwargs are passed to all stages of the compile tree. # Every function can pick from it what it needs. # Never try to compile if we are in a frozen app if paths.is_frozen(): return # Prevent double installation -> uninstall any current importers for importer in [i for i in sys.meta_path]: if isinstance(importer, Importer): importer.uninstall() # Install new importer sys.meta_path.insert(0, Importer(kwargs)) class Importer(object): """ Importer Is registered at sys.meta_path when calling install(). It will try to find a cython module and return a Loader object for it. """ def __init__(self, user_kwargs): self._user_kwargs = user_kwargs def uninstall(self): """ Remove all Importer instances from sys.meta_path. """ # Collect instances of this class at sys.meta_path importersToRemove = [] for importer in sys.meta_path: if isinstance(importer, Importer): importersToRemove.append(importer) # Remove these for importer in importersToRemove: try: sys.meta_path.remove(importer) except Exception: pass def find_module(self, fullname, packagePath=None): """ The method to implement to be a real Importer. Try finding the module (if nor already imported) and uninstall. """ # Check if we can exit early if fullname in sys.modules: return None if fullname.startswith('Cython.'): return None # print('Finding module', fullname, packagePath) # Uninstall now self.uninstall() # Find the module. On Python 2.7 the underscore may be removed from # a module name. I have no idea why they do that, but this fixes it. for fullname2 in [fullname, fullname+'_']: res = self._find_module(fullname2, packagePath) if res: return res def _find_module(self, fullname, packagePath=None): """ Try to find the module. If found, return Loader instance. """ # Init moduleName = fullname.split('.')[-1] path = None # Get path where the module is located if packagePath: fnames = getFileNames(moduleName, packagePath[0]) # source, bin, bin2 if any([os.path.isfile(fname) for fname in fnames]): path = packagePath[0] else: path = locate_cython_module(moduleName) # Success? if path is None: return None else: path = os.path.abspath(path) return Loader(fullname, moduleName, path, self._user_kwargs) class Loader(object): """ Loader When a Cython module is found, an instance of this class is created. It is used to import the module using the filename that was mangled using the Python version. But first, the module is (re)compiled if necessary. """ def __init__(self, fullname, moduleName, path, user_kwargs): self._fullname = fullname self._moduleName = moduleName self._path = path self._user_kwargs = user_kwargs def load_module(self, fullname): """ This is the method that we should implement to be a real Loader. It gets the binary to load (which may involve some compiling) and then loads it. """ # Test if not (self._fullname == fullname or self._fullname == fullname+'_'): raise RuntimeError("invalid module, expected %s, got %s" % (self._fullname, fullname) ) # Prepare GCC. Also if not compiling, because it may need gcc libs prepare_gcc() # Import module moduleNameToLoad = self.get_binary_to_load(self._fullname) return imp.load_dynamic(self._fullname, moduleNameToLoad) def get_binary_to_load(self, fullname): """ Get the binary to load. We might have to (re)create it. We have a source file sourceName. From that, we compile a binary called binName2, which has its name mangled with the Python version. This binary is copied for freezing (and normal import) compatibility to binName. Step 1: compile binName2 (if binName2 out of date) Step 2: copy binName2 to binName (if not currently the same file) Step 3: return binName2 (prefered) or binName. Different things can happen: * If step 1 fails raise error or show a warning (stuff might still work) * If step 2 fails we will show a warning (freezing will not work) * If step 3 fails it is an error (we can check after step 1) """ # Get names sourceName, binName, binName2 = getFileNames(self._moduleName, self._path) # Get modification times of these files. sourceTime, binTime, binTime2 = getFileModificationTimes( sourceName, binName, binName2) # Step 1: create binary (compile and copy), update times if sourceTime > binTime2: self.create_binary(sourceName, binName, binName2) sourceTime, binTime, binTime2 = getFileModificationTimes( sourceName, binName, binName2) # Test if ok. Compiling may have failed, but if we have the binaries # it will still work if not (binTime or binTime2): raise RuntimeError("No binary available for Cython module %s." % self._moduleName ) # Step 2: copy binName2 to binName # Copy the special name to a file that has the name of the module # (binName). This is what cx_freeze will detect and collect. # This action is done every time so that the binary is up to date # with the latest Python version with which the module was imported. if binTime2 and (binTime == 0 or binTime2 != binTime): try: shutil.copy2(binName2, binName) print('Copied %s to match with current Python version.' % binName) except Exception: print('Could not copy %s to match it with the current Python '+ 'version, therefore not ready for freezing.' % binName) # Step 3: return name of binary to import # Prefer the one with the mangled name. In this way, we do not lock # the binary with the moduleName, so it can be overwritten (updated). # Note that above we already check if either binary exists. if binTime2: return binName2 # ok else: return binName # not so good, a warning should have been shown def create_binary(self, sourceName, binName, binName2): """ Creates the binary. This is just a small wrapper around the function that compiles the Cython code. Here we also copy the resulting library to the right location and clean up the build dir. """ # Init build_dir = 'build_py%i%i_%i' % (sys.version_info[0], sys.version_info[1], NBITS) base_dir, name = os.path.split(binName) binName3 = os.path.join(base_dir, build_dir, name) if not os.path.isdir(os.path.dirname(binName3)): os.makedirs(os.path.dirname(binName3)) try: # Try to compile compile_error = self.compile_cython(sourceName, build_dir) # Try to copy the file if not os.path.isfile(binName3): try: import sysconfig abitag = sysconfig.get_config_var('SOABI') binName3 = binName3.replace('.so', '.'+abitag+'.so') except Exception: pass if not os.path.isfile(binName3): print(compile_error) raise RuntimeError('Could not find %s, was it not compiled?' % binName3) if not compile_error: try: shutil.copy2(binName3, binName2) except Exception: print(compile_error) raise RuntimeError('Could not write %s, is a process holding it?' % binName2) finally: self.cleanup(os.path.join(base_dir, build_dir)) # Make sure its a string compile_error = compile_error or '' # Decide if we can proceed: either binary should be available binExists, binExists2 = os.path.isfile(binName), os.path.isfile(binName2) if binExists or binExists2: if compile_error: print(compile_error) else: raise RuntimeError("No binary available for Cython module %s. %s" % (self._moduleName, compile_error) ) def compile_cython(self, sourceName, build_dir='cython_build'): """ compile_cython(sourceName, **kwargs) Do the actual compiling. Raises an error if there is a critical problem. Or returns an error string if compiling fails but things might still work (e.g. Cython is not installed but user has the binaries). """ # Get extension args given during install() user_kwargs = self._user_kwargs # Get compiler # todo: Explicitly use selected compiler downstream compiler = user_kwargs.get('compiler', 'gcc').lower() if compiler not in ['gcc', 'mingw', 'native']: raise RuntimeError('Unkown compiler %s.' % compiler) if compiler in ['mingw']: compiler = 'gcc' # Try importing Cython, if not available, return gracefully try: from Cython.Distutils import build_ext as build_pyx except ImportError: return "Could not compile: require Cython (www.cython.org)." # Store interpreter state old_argv = sys.argv old_dir = os.getcwd() # Prepare state for distutils sys.argv = [sourceName] # Is like the script that was "called" sys.argv.append('build_ext') #sys.argv.append('--inplace') sys.argv.append('--build-lib=%s'%build_dir) sys.argv.append('--build-temp=%s'%build_dir) if sys.platform.startswith('win') and compiler=='gcc': # Force using mingw (is gcc compiler fow Windows) sys.argv.append('-cmingw32') if 'DISTUTILS_USE_SDK' in os.environ: del os.environ['DISTUTILS_USE_SDK'] # Goto the right directory os.chdir(os.path.dirname(sourceName)) # Get modulename modNamePlus = os.path.split(sourceName)[1] modName = os.path.splitext(modNamePlus)[0] # Set language language = user_kwargs.get('language', 'c') # Init extension args include_dirs = ['.'] + user_kwargs.get('include_dirs',[]) library_dirs = ['.'] + user_kwargs.get('library_dirs',[]) libraries = [] + user_kwargs.get('libraries',[]) extra_compile_args = ['-O2'] + user_kwargs.get('extra_compile_args',[]) extra_link_args = [] + user_kwargs.get('extra_link_args',[]) # Set number of bits # Includes fix for http://bugs.python.org/issue4709 # See also http://projects.scipy.org/numpy/wiki/MicrosoftToolchainSupport if compiler=='gcc': for L in [extra_compile_args, extra_link_args]: L.append('-m%i' % NBITS) L.extend(['-static-libgcc', '-static-libstdc++']) # At least on Windows, wont hurt in Linux either I think if sys.platform.startswith('win'): L.append('-DMS_WIN%i' % NBITS) # Means "#define MS_WINxx" # Try compiling try: # Imports from distutils.core import setup from distutils.extension import Extension from numpy.distutils.misc_util import get_numpy_include_dirs # Get numpy headers include_dirs.extend(get_numpy_include_dirs()) # # Extra libs (for msvcr90.dll for exampe) # if sys.platform.startswith('win') and paths.pyzo_lib_dir(): # library_dirs.append(str(paths.pyzo_lib_dir())) # Extra extension kwargs? extension_kwargs = user_kwargs.get('extension_kwargs',{}) # Create extension module object ext1 = Extension(modName, [modNamePlus], language=language, include_dirs=include_dirs, library_dirs=library_dirs, libraries=libraries, extra_compile_args=extra_compile_args, extra_link_args=extra_link_args, **extension_kwargs ) # Compile ext_modules = [ext1] setup( cmdclass = {'build_ext': build_pyx}, ext_modules = ext_modules, ) except BaseException as err: # also catch system exit #msg = traceback.format_exception_only(*sys.exc_info()[:2]) #raise RuntimeError("Building module %s failed: %s" % (modName, msg)) print("Building module %s failed: " % modName) raise else: print('Successfully compiled cython file: %s' % modName) finally: # Put back the state sys.argv = old_argv os.chdir(old_dir) def cleanup(self, path): """ Try to remove the build directory and its contents. """ # Get directory if not os.path.isdir(path): return def _clearDir(path): # Remove contents for fname in os.listdir(path): fname = (os.path.join(path, fname)) if os.path.isdir(fname): _clearDir(fname) elif os.path.isfile(fname): try: os.remove(fname) except Exception: pass # Remove dir itself try: os.rmdir(path) except Exception: pass # Clean _clearDir(path) if os.path.isdir(path): print('Could not remove build directory.') pyzolib-0.3.4/shebang.py0000664000175000017500000000526312352042701015405 0ustar almaralmar00000000000000""" pyzolib.shebang.py Fix shebangs for Pyzo distro. """ import os import sys def fix(prefix=None, exe=None, verbose=True): """ Try to fix the shebangs of all scripts in Pyzo's bin folder. The given prefix must be the prefix of a Pyzo distro. If not given, sys.prefix is used. The given exe must be the path to the Python interpreter to put in the shebang. If not given, "prefix/bin/python3" is used. Returns the number of fixed shebangs. Note that updating a package using conda will also update (and fix) the shebang. """ # Init prefix = prefix or sys.prefix exe = exe or os.path.join(prefix, 'bin', 'python3') bin_folder = os.path.join(prefix, 'bin') count = 0 # Check if sys.platform.startswith('win'): print('Windows has no shebangs.') return else: exename1, exename2 = 'pyzo', 'pyzo.app' # .app directory on OS X if not (os.path.isfile(os.path.join(prefix, exename1)) or os.path.exists(os.path.join(prefix, exename2))): raise RuntimeError('Can only fix shebangs of a Pyzo distro.') return # Process all files for fname in os.listdir(bin_folder): filename = os.path.join(bin_folder, fname) # Skip links and binaries if os.path.islink(filename): #print('SKIP %s: skip link' % fname) continue stat = os.stat(filename) if stat.st_size > 10*1024: if verbose: print('SKIP %s: > 10kB (probably binary)' % fname) continue # Open the file try: text = open(filename, 'rb').read().decode('utf-8') except UnicodeDecodeError: if verbose: print('SKIP %s: cannot decode (probably binary)' % fname) continue lines = text.split('\n') line0 = lines[0] # Only modify if it has a python shebang if not (line0.startswith('#!') and 'python' in line0): if verbose: print('SKIP %s: no Python shebang to replace' % fname) continue # Replace line0 = '#!%s' % exe lines[0] = line0 newtext = '\n'.join(lines) # Try writing back try: open(filename, 'wb').write(newtext.encode('utf-8')) except IOError: if verbose: print('SKIP: %s: cannot write (need sudo?)' % fname) continue # If we get here ... success! count += 1 if verbose: print('FIXED %s' % fname) # Report if verbose: print('Modified %i shebangs' % count) return count if __name__ == '__main__': fix() pyzolib-0.3.4/PKG-INFO0000664000175000017500000000314612573243411014525 0ustar almaralmar00000000000000Metadata-Version: 1.1 Name: pyzolib Version: 0.3.4 Summary: Utilities for the Pyzo environment. Home-page: http://bitbucket.org/pyzo/pyzolib Author: Almar Klein Author-email: almar.klein@gmail.com License: (new) BSD Description: Package pyzolib The pyzolib package provides basic functionality for the Pyzo environment. It contains a collection of modules and small packages that should be imported as "from pyzolib import xxx" The packages currently are: * path - object oriented path processing (no more os.path.x) * paths - Get paths to useful directories in a cross platform manner. * qt - Proxy for importing QtCore et al. from PySide or PyQt4 * ssdf - the Simple Structured Data Format (for config files and scientific databases) * insertdocs - a sphynx pre-processor to include docstrings in the text, allowing readthedocs.org to host the docs without requiring importing code. * pyximport - for easy on the fly compilation of Cython, using the Pyzo environment to establish the location of a gcc compiler. * gccutils - used by the above to manage the gcc compiler. * interprerers - list the Python interpreters available on this system. * dllutils - utilities to set the RPATH in dynamic libararies and remove depndencies on the MSVCR from the embedded manifest. * shebang - for making shebangs in pyzo distro absolute. Keywords: Pyzo cython gcc path paths interpreters shebang Platform: any Provides: pyzolib pyzolib-0.3.4/insertdocs.py0000664000175000017500000007232512156025455016167 0ustar almaralmar00000000000000# -*- coding: utf-8 -*- # Copyright (C) 2012 Almar Klein # This module is distributed under the terms of the (new) BSD License. """ insertdocs.py Module to insert documentation from docstrings in RST files, which can then be used by Sphinx to generate beautiful html or pdf documentation. Sphinx has a very nice autodoc extension, but that approach has shortcomings: * The sphinx that is used to create the docs must be able to import all modules that you want to use docstrings of. * This is specifically a problem when using for instance readthedocs.org. The approach advocated by this module is to insert the docstrings in the .rst files before building the documentation. This has the advantage that the subsequent build process can be done by about any RST builder; you only need the doc directory with the .rst files to build the docs. How to use ---------- Step 1: in the .rst documentation files, you use the directive ``.. insertdocs:: objectName`` to specify that a certain object should be documented there. insertdocs can document modules, classes, functions (and their members). Step 2: before building, run the parse_rst_files function that is defined in this module. It will inject all the docstrings and also automatically make references for the object names it knows. Notes: when the docstrings are inserted, the directives are replaced by comments: ``.. insertdocs start::`` another comment is placed at the end of the inserted text: ``.. insertdocs end::``. To remove the inserted docs for easier editing, use the clear_rst_files function. Options ------- insertdocs tries to be a bit compatible with Sphynx's autodoc. In currently supports the ``:members:`` and the ``:inherited-members:`` options. Note than when the docs are inserted the options look like: ``.. insertdocs :members:``. Example script (Windows) ------------------------ # docsinserter.py - put this next to conf.py # Imports to fill the global namespace import yoton # Insert docs from pyzolib import insertdocs insertdocs.parse_rst_files(globals(), '.') # Tell Sphinx to build the docs import subprocess # p = subprocess.check_call(['make.bat', 'html']) """ import os import sys import re # Version dependent defs V2 = sys.version_info[0] == 2 if V2: D = __builtins__ if not isinstance(D, dict): D = D.__dict__ bytes = D['str'] str = D['unicode'] basestring = basestring else: basestring = str # to check if instance is string bytes, str = bytes, str DIRECTIVES_AUTO = ['.. autoclass::', '.. autofunction::', '.. automodule::'] DIRECTIVE_BASE = '.. insertdocs::' DIRECTIVE_START = '.. insertdocs start::' # is really a comment DIRECTIVE_END = '.. insertdocs end::' # is really a comment PREFIX_OPTION = '.. insertdocs ' # todo: options for directive (members) ## Functions to parse the files def clear_rst_files(path='.'): """ clear_rst_files(path='.') Remove the inserted docstrings from the documentation and put the insertdocs directives. Also removes any auto-generated references. """ return parse_rst_files(None, path, True) def parse_rst_files(NS, path='.', clear=False): """ parse_rst_files(NS, path='.') Parse all RST files in the given directory and insert the docstrings where the insertdocs directive has been placed. Also automatically creates references to objects that are known. """ printPrefix = ['Inserted ', 'Cleared'][clear] # Get list of files path = os.path.abspath(path) files = [] for fname in os.listdir(path): if fname.endswith('.rst'): files.append(fname) # Insert docstrings changedCount = 0 knownNames = [] for fname in files: # Get text and parse the docs fname2 = os.path.join(path, fname) text = open(fname2, 'rb').read().decode('utf-8') text, changed, names = _parse_rst_file_docs(text, NS, clear) knownNames.extend(names) # If it was modified, save the file. if changed: changedCount += 1 print(printPrefix + 'docstrings in %s' % fname) f = open(fname2, 'wb') f.write(text.encode('utf-8')) f.close() # Message done print(printPrefix + 'docstrings in %i/%i files.' % (changedCount, len(files))) # Clear? if clear: knownNames = [] # Handle cross-refences changedCount = 0 for fname in files: # Get text and parse the references fname2 = os.path.join(path, fname) text = open(fname2, 'rb').read().decode('utf-8') text, changed = _parse_rst_file_refs(text, knownNames) # If it was modified, save the file. if changed: changedCount += 1 print(printPrefix + 'auto-refs in %s' % fname) f = open(fname2, 'wb') f.write(text.encode('utf-8')) f.close() # Message done print(printPrefix + 'auto-references in %i/%i files.' % (changedCount, len(files))) class LineByLine: """ Class to walk over the lines looking for our derective and changing the code. """ def __init__(self, text, NS, cleanMode=False): # Store lines and init new version self._lines1 = [line for line in text.splitlines()] if text.endswith('\n') or True: # If endswith newline, splitlines has popped one, # otherwise, make it end with a newline! self._lines1.append('') # Current index, self._i = -1 # New lines and flag to signal whether anything has changed self._lines2 = [] self._changed = False # Namespace to look for docstrings self._NS = NS # Whether to clean up (default is False) self._cleanMode = cleanMode # To store a "piece" self._objectName = None self._options = {} # List of known names self._knownNames = [] def get(self): line = self.pop() self._lines2.append(line) return line def pop(self): self._i += 1 # Get line or raise stop try: line = self._lines1[self._i] except IndexError: raise StopIteration # Return (do not add line) return line def set(self, line): if self._changed or line != self._lines2[-1]: self._lines2[-1] = line self._changed = True def append(self, line): self._lines2.append(line) self._changed = True def add_option(self, name, value): name = name.replace('-', '_') if not value: value = True self._options[name] = value def search_start(self): """ Search for the start. If found, search for options. Then search for end if necessary, then insert text. """ # Prepare self._objectName = None self._options = {} while True: line = self.get() if line.startswith(DIRECTIVE_BASE): search_for_end = False break elif line.startswith(DIRECTIVE_START): search_for_end = True break # Store object name self._objectName = line.split('::',1)[1].strip() # Change syntax to how we want it if self._cleanMode: self.set(DIRECTIVE_BASE + ' ' + self._objectName) else: self.set(DIRECTIVE_START + ' ' + self._objectName) # Next step, search options self.search_options() # Search for end if we need to if search_for_end: self.search_end() # Insert text if we need to if not self._cleanMode: self.insert_text() # Done return self._objectName def search_options(self): """ Process lines until we have one that is not an option. """ while True: line = self.get() # Detect options m1 = re.search(r'^\s+?:(.+?):', line) m2 = re.search(r'^.. insertdocs\s+?:(.+?):', line) m = m1 or m2 if m is not None: # Option found optionName = m.group(1) optionValue = line[m.end():].strip() self.add_option(optionName, optionValue) # Change syntax to how we need it if self._cleanMode: self.set(' :%s: %s' % (optionName, optionValue)) else: self.set('.. insertdocs :%s: %s' % (optionName, optionValue)) else: # No more options break def insert_text(self): """ Insert text generated from docstrings. Then search for the end. """ # Get text to insert extraText = '' if self._NS is not None: extraText = get_docs(self._objectName, NS=self._NS, **self._options) # Insert it if extraText: self._knownNames.append(self._objectName) self.append('') for extraline in extraText.splitlines(): self.append(extraline) # Always insert end self.append(DIRECTIVE_END) self.append('') # extra blanc line def search_end(self): """ Keep getting lines until we reach the end. """ while True: line = self.pop() if line.startswith(DIRECTIVE_END): self.pop() # pop one extra, because we added an extra blanc line break def _parse_rst_file_docs(allText, NS=None, clean=False): # Instantiate lines object lines = LineByLine(allText, NS, clean) # Let it process until we are out of lines count = 0 knownNames = [] try: while True: objectName = lines.search_start() # Returns if a "piece" is processed count += 1 knownNames.append(objectName) except StopIteration: pass # Done (if changed, rebould text from all the lines) if lines._changed: allText = '\n'.join(lines._lines2) return allText, lines._changed, knownNames def _parse_rst_file_refs(allText, knownNames): # Remove all insertdocs :ref: instances r = re.compile(':ref:`(.+?)`') allText, n1 = r.subn(r'\1', allText) # Check all lines lines1 = [] lines2 = [] nChanged = 0 for line in allText.splitlines(): lines1.append(line) # No refs in headings # if line.startswith('===') or line.startswith('---') or line.startswith('^^^'): # if lines2: # lines2[-1] = lines1[-2] # lines2.append(line) # continue # No refs in directives or comments if line.startswith('..'): lines2.append(line) continue # Try! for name in knownNames: i0 = 0 while i0 >= 0: i0 = line.find(name, i0) if i0 < 0: break # Found something i1 = i0 + len(name) pre, post = line[:i0], line[i1:] nquotes = pre.count('``') # Not in a quote? if nquotes%2 == 1: i0 = i1 continue # Check if ending is ok endingok= True for ending in [' ', ',', ':', ';', '. ']: # no braces: `( will format wrong by sphinx! if post.startswith(ending): break else: endingok = False # If all is well... modify line if endingok or not post or post=='.': nChanged += 1 line = pre + make_xref(name) + post i1 += 4 # Next! i0 = i1 else: lines2.append(line) # Done if nChanged: # Add newline (splitlines removes one if there is at least one) lines2.append('') allText = '\n'.join(lines2) return allText, (n1+nChanged) > 0 ## Functions to get the RST docs for an object def get_docs(objectName, NS, **kwargs): """ get_docs(objectName, NS, **kwargs) Get the docs for the given string, class, function, or list with any of the above items. """ # Make object try: object = eval(objectName, {}, NS) except Exception: print('Warning: do not know object "%s".' % objectName) return '' if isinstance(object, basestring): return smart_format(object, **kwargs) elif isinstance(object, list): tmp = [get_docs(ob, NS, **kwargs) for ob in object] return '\n\n'.join(tmp) elif isclass(object): return get_class_docs(object, objectName, **kwargs) elif 'function' in str(type(object)): return get_function_docs(object, objectName, **kwargs) elif 'method' in str(type(object)): return get_function_docs(object, objectName, **kwargs) elif 'module' in str(type(object)): return get_module_docs(object, objectName, **kwargs) else: print('Cannot determine how to generate docs from object "%s".' % objectName) def get_property_docs(prop, fullName, **kwargs): """ get_property_docs(prop, fullName) Get RST content for the specified property. Makes a "header" from the property name (with a label) and indents the body of the documentation. (Need the name, since we cannot obtain it from the property object. ) """ # Get docs header, docs = split_docs (prop, fullName) # Return with markup result = '%s\n\n' % make_label(fullName) result += '.. py:attribute:: %s\n\n' % header result += indent(docs, 2) return result def get_function_docs(fun, fullName=None, isMethod=False, **kwargs): """ get_function_docs(fun, fullName=None, isMethod=False, **kwargs) Get RST content for the specified function or method. Makes a "header" from the property name (with a label) and indents the body of the documentation. """ # Get docs if fullName is None: fullName = fun.__name__ header, docs = split_docs(fun, fullName) # Return with markup result = '%s\n\n' % make_label(fullName) if isMethod: result += '.. py:method:: %s\n\n' % header else: result += '.. py:function:: %s\n\n' % header result += indent(docs, 2) return result def get_class_docs(cls, fullName='', members=None, inherited_members=None, **kwargs): """ get_class_docs(cls, fullName='', inherited_members=False) Get RST content for the specified class. Writes the formatted docstring of the class, lists all methods and properties and gives the docs for all methods and properties (as given by get_function_docs() and get_property_docs()). If inherited_members is True, also include inherited properties and methods. """ # Get name if not fullName: fullName = get_class_name(cls) # Init the variable to hold the docs total_docs = '' # Produce label and title total_docs += '%s\n\n' % make_label(fullName) header, docs = split_docs(cls, fullName) total_docs += '.. py:class:: %s\n\n' % header #total_docs += '%s\n%s\n\n' % (header, '^'*len(fullName)) # Show inheritance bases = [] for base in cls.__bases__: tmp = get_class_name(base) # todo: auto-crossref bases.append( tmp ) total_docs += ' *Inherits from %s*\n\n' % ', '.join(bases) # Insert docs itself (add indentation) total_docs += indent(docs, 2) + '\n\n' # Stop here? if not members: return total_docs elif isinstance(members, str): memberSelection = [m.strip() for m in members.split(',')] else: memberSelection = None # containers for attributes methods = {} properties = {} # Collect attributes atts = {} def collect_attributes(cls): for att, val in cls.__dict__.items(): atts[att] = val if inherited_members: for c in cls.__bases__: collect_attributes(c) collect_attributes(cls) # Collect docs for methods and properties for att in atts.keys(): if att.startswith('_'): continue if memberSelection and att not in memberSelection: continue # Get value val = atts[att] # Skip if attribute does not have a docstring if not val.__doc__: print('Skipping %s.%s: no docstring' % (fullName, att)) continue # Get info if 'function' in str(type(val)): methods[att] = get_function_docs(val, fullName+'.'+att, isMethod=True) elif 'property' in str(type(val)): properties[att] = get_property_docs(val, fullName + '.' + att) # todo: if class summary: need table # # Insert summary of properties with links # if properties: # propList = [] # for key in sorted( properties.keys() ): # propList.append( '[#%s %s]' % (key, key) ) # tmp = '*The %s class implements the following properties:*\n' # total_docs += tmp % fullName # #propList = [' * '+m for m in propList] # #total_docs += '\n'.join(propList) + '\n\n' # total_docs += create_table_from_list(propList) + '\n' # # # Insert summary of methods with links # if methods: # method_list = [] # for key in sorted( methods.keys() ): # method_list.append( '[#%s %s]' % (key, key) ) # tmp = '*The %s class implements the following methods:*
\n' # total_docs += tmp % fullName # #method_list = [' * '+m for m in method_list] # #total_docs += '\n'.join(method_list) + '\n\n' # total_docs += create_table_from_list(method_list) + '\n' # Insert properties if properties: total_docs += ' *PROPERTIES*\n\n' for key in sorted( properties.keys() ): total_docs += indent(properties[key],2) + '\n\n' # Insert methods if methods: total_docs += ' *METHODS*\n\n' for key in sorted( methods.keys() ): total_docs += indent(methods[key], 2) + '\n\n' # Done total_docs += '\n\n' return total_docs def get_module_docs(module, fullName='', members=None, **kwargs): """ get_module_docs(module, fullName='') Get RST documentation for a module. """ # Get name if not fullName: fullName = get_class_name(cls) # Get our docs header, docs = split_docs(module, fullName) # Produce label and title total_docs = '' total_docs += '%s\n\n' % make_label(fullName) total_docs += '.. py:module:: %s\n\n' % header #total_docs += 'Module %s\n%s\n\n' % ( header, '='*((len(header)+30)) ) # -> User should use :mod:`modulename` -- description in the rst file # Insert our own docs total_docs += docs + '\n\n' # Stop here? if not members: return total_docs elif isinstance(members, str): memberSelection = [m.strip() for m in members.split(',')] else: memberSelection = None # Collect children classes, functions = {}, {} # Collect docs for classes and functions for att in dir(module): if att.startswith('_'): continue if memberSelection and att not in memberSelection: continue # Get value, val = getattr(module, att) # Skip if not defined in module if not hasattr(val, '__module__'): continue if val.__module__.split('.')[-1] != fullName.split('.')[-1]: continue # Skip if attribute does not have a docstring if not val.__doc__: print('Skipping %s.%s: no docstring' % (fullName, att)) continue # Get info if isclass(val): classes[att] = get_class_docs(val, fullName+'.'+att) elif 'function' in str(type(val)): functions[att] = get_function_docs(val, fullName+'.'+att) # Insert functions if functions: total_docs += 'Functions\n----------\n\n' for key in sorted( functions.keys() ): total_docs += functions[key] + '\n\n' # Insert methods if classes: total_docs += 'Classes\n----------\n\n' for key in sorted( classes.keys() ): total_docs += classes[key] + '\n\n' total_docs += '\n\n' return total_docs ## Functions to parse the docstrings to RST def smart_format(text): """ smart_format(text) Smart formats text -> changing headers to bold text, handling code examples, etc. This is where most of the smarty (and hard-to-maintain) bits are. """ class Line: def __init__(self, text): self.text = text self.sText = text.lstrip() self.indent = len(self.text) - len(self.sText) self.needNL = False self.isParameter = False # Get lines lines = text.splitlines() # Test minimal indentation minIndent = 9999 for line in lines[1:]: tmp = line.lstrip() indent = len(line) - len(tmp) if tmp: minIndent = min(minIndent, indent) # Remove minimal indentation lines2 = [ Line(lines[0].lstrip()) ] for line in lines[1:]: lines2.append( Line(line[minIndent:]) ) # Prepare state variables prevLine = Line('') inExample = False inCode = False # Format line by line lines3 = [] for line in lines2: # Detect special cases if line.indent == prevLine.indent and ( "---" in line.text or "===" in line.text): underCount = line.text.count('-') + line.text.count('=') len1, len2 = len(line.text.strip()), len(prevLine.text.strip()) if underCount == len1 and len2 and len1 >= len2: # Header if True: lines3[-1] = '**%s**\n' % (prevLine.sText) line.text = line.sText = '' line.needNL = True # Start example? inExample = False if prevLine.sText.lower().startswith('example'): line.text = '.. code-block:: python\n' inExample = True elif ' : ' in line.text: # Parameter (numpy style) pass elif line.sText[:3] in ['{{{', '}}}']: # Code block if line.sText[0] == '{': inCode = True prevline.text = prevlineline.text + '::' else: inCode = False line.text = '' elif inExample or inCode: line.text = ' ' + line.text else: line.text = line.text # Done with line prevLine = line lines3.append(line.text) # Done line by line formatting lines3.append('') docs = '\n'.join(lines3) # # "Pack" underscores and asterix that are not intended as markup # # Mark all asterixes that surround a word or bullet # docs = re.sub('(\s)\*(\w+)\*(\s)', '\g<1>\0\g<2>\0\g<3>', docs) # docs = re.sub( re.compile('^(\s+)\* ',re.MULTILINE), '\g<1>\0 ', docs) # # Pack all remaining asterixes # docs = docs.replace('*',"`*`").replace('\0','*') # # Do the same for underscores (but no need to look for bullets) # # Underscores within a word do not need esacping. # docs = re.sub('(\s)_(\w+)_(\s)', '\g<1>\0\g<2>\0\g<3>', docs) # docs = docs.replace(' _'," `_`").replace('_ ', '`_` ').replace('\0','_') # # Pack square brackets # docs = docs.replace("[[","").replace("]]","") # docs = docs.replace("[","`[`").replace("]","`]`") # docs = docs.replace("", "[").replace("", "]") return docs def split_docs(ob, fullName): """ split_docs(ob, fullName) Get the docstring of the given object as a two element tuple: (header, body) The header contains the name and signature (if available) of the class or function. Uses smart_format() internally. """ # Get name and base name if '.' in fullName: tmp = fullName.rsplit('.', 1) baseName, name = tmp[0], tmp[1] else: baseName, name = '', fullName def searchEndBrace(text, i0): """ Start on opening brace. """ i = i0 level = int(text[i0]=='(') while level and (i `' def isclass(object): return isinstance(object, type) or type(object).__name__ == 'classobj' def indent(text, n): lines = text.splitlines() for i in range(len(lines)): lines[i] = " "*n + lines[i] return '\n'.join(lines) # def create_table_from_list(elements, columns=3): # """ create_table_from_list(elements, columns=3) # # Create a table from a list, consisting of a specified number # of columns. # """ # # import math # # # Check how many elements in each column # n = len(elements) # tmp = n / float(columns) # rows = int( math.ceil(tmp) ) # # # Correct rows in a smart way, so that each column has at least # # three items # ok = False # while not ok: # cn = [] # for i in range(columns): # tmp = n - rows*i # if tmp <= 0: # tmp = 9999999999 # cn.append( min(rows, tmp) ) # #print cn # if rows >= n: # ok = True # elif min(cn) <= 3: # rows += 1 # else: # ok = True # # # # Open table # text = "\n" # # # Insert columns # for col in range(columns): # text += '\n' # # # Close table and return # text += '
\n' # for row in range(rows): # i = col*rows + row # if i < len(elements): # text += elements[i] + '
' # text += '
\n' # return text pyzolib-0.3.4/setup.py0000664000175000017500000000330712337603240015137 0ustar almaralmar00000000000000# -*- coding: utf-8 -*- # Copyright (c) 2012, The Pyzo team # # This file is distributed under the terms of the (new) BSD License. import os import sys from distutils.core import setup name = 'pyzolib' description = 'Utilities for the Pyzo environment.' # Get version and docstring __version__ = None __doc__ = '' docStatus = 0 # Not started, in progress, done initFile = os.path.join(os.path.dirname(__file__), '__init__.py') for line in open(initFile).readlines(): if (line.startswith('__version__')): exec(line.strip()) elif line.startswith('"""'): if docStatus == 0: docStatus = 1 line = line.lstrip('"') elif docStatus == 1: docStatus = 2 if docStatus == 1: __doc__ += line setup( name = name, version = __version__, author = 'Almar Klein', author_email = 'almar.klein@gmail.com', license = '(new) BSD', url = 'http://bitbucket.org/pyzo/pyzolib', keywords = "Pyzo cython gcc path paths interpreters shebang", description = description, long_description = __doc__, platforms = 'any', provides = ['pyzolib'], requires = [], # No requirements packages = [ 'pyzolib', 'pyzolib.ssdf', 'pyzolib.interpreters', 'pyzolib.qt', ], py_modules = [ 'pyzolib.path', 'pyzolib.paths', 'pyzolib.pyximport', 'pyzolib.gccutils', 'pyzolib.insertdocs', 'pyzolib.dllutils', 'pyzolib.shebang', ], package_dir = {'pyzolib': '.'}, # must be a dot, not an empty string ) pyzolib-0.3.4/interpreters/0000775000175000017500000000000012573243411016152 5ustar almaralmar00000000000000pyzolib-0.3.4/interpreters/inwinreg.py0000664000175000017500000002063412326157623020360 0ustar almaralmar00000000000000# -*- coding: utf-8 -*- # Copyright (c) 2012, Almar Klein """ This module implements functionality to obtain registered Python interpreters and to register a Python interpreter in the Windows registry. See interpreters.py for higher level functionality to also detect Python interpreters in common locations, discarting invalid paths and obtaining version numbers. """ import sys import os try: import winreg except ImportError: winreg = None PYTHON_KEY = 'SOFTWARE\\Python\\PythonCore' PYTHON_KEY_WOW64 = 'SOFTWARE\\Wow6432Node\\Python\\PythonCore' INSTALL_KEY = "InstallPath" PATH_KEY = "PythonPath" class PythonInReg: """ Class to represent a specific version of the Python interpreter registered (or being registered in the registry). This is a helper class for the functions defined in this module; it should not be instantiated directly. """ USER_ONE = 1 USER_ALL = 2 def __init__(self, user, version, wow64=False): self._user = user self._key = (wow64 and PYTHON_KEY_WOW64 or PYTHON_KEY) + '\\' + version def __repr__(self): userstr = [None, 'USER_ONE', 'USER_ALL'][self._user] installPath = self.installPath() reg = self._reg() if not reg: return '' % (self.version(), userstr) elif installPath: return '' % (self.version(), userstr, installPath) else: return '' % (self.version(), userstr) def _root(self): if self._user == PythonInReg.USER_ONE: return winreg.HKEY_CURRENT_USER else: return winreg.HKEY_LOCAL_MACHINE def _reg(self): # Get key for this version try: return winreg.OpenKey(self._root(), self._key, 0, winreg.KEY_READ) except Exception: return None def create(self): """ Create key. If already exists, does nothing. """ # Get key for this version reg = self._reg() if reg: winreg.CloseKey(reg) #print('Unable to create Python version %s: already exists.' % self.version()) else: # Try to create try: reg = winreg.CreateKey(self._root(), self._key) winreg.CloseKey(reg) except Exception: raise RuntimeError('Unable to create python version %s.' % self.version()) print('Created %s.' % str(self)) def delete(self): # Get key for this version reg = self._reg() if not reg: print('Unable to delete Python version %s: does not exist.') # Delete attributes try: winreg.DeleteKey(reg, INSTALL_KEY) except Exception: pass try: winreg.DeleteKey(reg, PATH_KEY) except Exception: pass # Delete main key for this version, or show warning try: winreg.DeleteKey(self._root(), self._key) except Exception: print('Could not delete %s.' % str(self)) return print('Deleted %s.' % str(self)) def setInstallPath(self, installPath): # Get key for this version reg = self._reg() if not reg: raise RuntimeError('Could not set installPath for version %s: version does not exist.' % self.version()) # Set value or raise error try: winreg.SetValue(reg, INSTALL_KEY, winreg.REG_SZ, installPath) winreg.CloseKey(reg) except Exception: winreg.CloseKey(reg) raise RuntimeError('Could not set installPath for %s.' % str(self)) def installPath(self): # Get key for this version reg = self._reg() if not reg: return None # Get value or return None try: installPath = winreg.QueryValue(reg, INSTALL_KEY) winreg.CloseKey(reg) return installPath except Exception: winreg.CloseKey(reg) return None def setPythonPath(self, pythonPath): # Get key for this version reg = self._reg() if not reg: raise RuntimeError('Could not set pythonPath for version %s: version does not exist.' % self.version()) # Set value or raise error try: winreg.SetValue(reg, PATH_KEY, winreg.REG_SZ, pythonPath) winreg.CloseKey(reg) except Exception: winreg.CloseKey(reg) raise RuntimeError('Could not set pythonPath for %s.' % str(self)) def pythonPath(self): # Get key for this version reg = self._reg() if not reg: return None # Get value or return None try: pythonPath = winreg.QueryValue(reg, PATH_KEY) winreg.CloseKey(reg) return pythonPath except Exception: winreg.CloseKey(reg) return None def version(self): """ Get the Python version. """ return self._key[-3:] def get_interpreters_in_reg(): """ get_interpreters_in_reg() Get a list of PythonInReg instances: one for each interpreter in the registry. This function checks both LOCAL_MACHINE and CURRENT_USER. """ versions = [] for user in [1, 2]: for wow64 in [False, True]: versions.extend( _get_interpreter_in_reg(user, wow64) ) return versions def _get_interpreter_in_reg(user, wow64=False): # Get base key if user == PythonInReg.USER_ONE: HKEY = winreg.HKEY_CURRENT_USER else: HKEY = winreg.HKEY_LOCAL_MACHINE # Get Python key if wow64: PYKEY = PYTHON_KEY_WOW64 else: PYKEY = PYTHON_KEY # Try to open Python key try: reg = winreg.OpenKey(HKEY, PYKEY, 0, winreg.KEY_READ) except Exception: return [] # Get info about subkeys nsub, nval, modified = winreg.QueryInfoKey(reg) # Query all versions = [] for i in range(nsub): # Get name and subkey version = winreg.EnumKey(reg, i) versions.append( PythonInReg(user, version, wow64) ) # Done winreg.CloseKey(reg) return versions def register_interpreter(version=None, installPath=None, user=None, wow64=False): """ register_interpreter(version=None, installPath=None, user=None, wow64=False) Register a certain Python version. If version and installPath are not given, the current Python process is registered. if user is not given, tries LOCAL_MACHINE first but uses CURRENT_USER if that fails. """ if version is None: version = sys.version[:3] if installPath is None: installPath = sys.prefix # Get existing versions existingVersions = get_interpreters_in_reg() # Determine what users to try if user is None: users = [2, 1] else: users = [user] success = False for user in users: # Create new PythonInReg instance v = PythonInReg(user, version, wow64) # Check if already exists ok = True for ev in existingVersions: if ev._key != v._key or ev._user != v._user: continue # Different key; no problem if (not ev.installPath()) or (not os.path.isdir(ev.installPath())): continue # Key the same, but existing entry is invalid if ev.installPath() == installPath: # Exactly the same, no action required, return now! return ev # Ok, there's a problem ok = False print('Warning: version %s is already installed in "%s".' % (version, ev.installPath())) if not ok: continue # Try to create the key try: v.create() v.setInstallPath(installPath) success = True break except RuntimeError: continue if success: return v else: raise RuntimeError('Could not register Python version %s at %s.' % (version, installPath)) if __name__ == '__main__': for v in get_interpreters_in_reg(): print(v) pyzolib-0.3.4/interpreters/pythoninterpreter.py0000664000175000017500000000743212350771642022344 0ustar almaralmar00000000000000import os import sys import subprocess from pyzolib.interpreters.inwinreg import register_interpreter class PythonInterpreter: """ Clas to represent a Python interpreter. It has properties to get the path and version. Upon creation the version number is acquired by calling the interpreter in a subprocess. If this fails, the version becomes ''. """ def __init__(self, path): if not isinstance(path, str): raise ValueError('Path for PythonInterpreter is not a string: %r' % path) if not os.path.isfile(path): raise ValueError('Path for PythonInterpreter is invalid: %r' % path) self._path = os.path.normpath(os.path.abspath(path)) self._normpath = os.path.normcase(self._path) self._is_pyzo = False self._problem = '' self._version = None def __repr__(self): cls_name = self.__class__.__name__ return '<%s version %s at %s>' % (cls_name, self.version, self.path) def __hash__(self): return hash(self._normpath) def __eq__(self, other): return self._normpath == other._normpath @property def path(self): """ The full path to the executable of the Python interpreter. """ return self._path @property def is_pyzo(self): """ Whether this is a Pyzo interpreter. """ return self._is_pyzo @property def version(self): """ The version number as a string, usually 3 numbers. """ if self._version is None: self._version = self._getversion() return self._version @property def version_info(self): """ The version number as a tuple of integers. For comparing. """ return _versionStringToTuple(self.version) def register(self): """ Register this Python intepreter. On Windows this modifies the CURRENT_USER. On All other OS's this is a no-op. """ if sys.platform.startswith('win'): path = os.path.split(self.path)[0] # Remove "python.exe" register_interpreter(self.version[:3], path) def _getversion(self): # Check if path is even a file if not os.path.isfile(self._path): self._problem = '%s is not a valid file.' return '' # Poll Python executable (--version does not work on 2.4) # shell=True prevents loads of command windows popping up on Windows, # but if used on Linux it would enter interpreter mode cmd = [self._path, '-V'] try: v = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=sys.platform.startswith('win')) except (OSError, IOError, subprocess.CalledProcessError) as e: self._problem = str(e) return '' # Extract the version, apply some defensive programming v = v.decode('ascii','ignore').strip().lower() if v.startswith('python'): v = v.split(' ')[1] v = v.split(' ')[0] # Try turning it into version_info try: _versionStringToTuple(v) except ValueError: return '' # Done return v class PyzoInterpreter(PythonInterpreter): """ A Pyzo interpreter. """ def __init__(self, path): PythonInterpreter.__init__(self, path) self._is_pyzo = True def _versionStringToTuple(version): # Truncate version number to first occurance of non-numeric character tversion = '' for c in version: if c in '0123456789.': tversion += c # Split by dots, make each number an integer tversion = tversion.strip('.') return tuple( [int(a) for a in tversion.split('.') if a] ) pyzolib-0.3.4/interpreters/__init__.py0000664000175000017500000001244512573243205020272 0ustar almaralmar00000000000000 # -*- coding: utf-8 -*- # Copyright (c) 2012, Almar Klein """ This module implements functionality to list installed Python interpreters, as well as Pyzo interpreters. This list is compiled by looking at common location, and on Windows the registry is searched as well. """ import sys import os from pyzolib import paths from pyzolib import ssdf from pyzolib.interpreters.pythoninterpreter import PythonInterpreter, PyzoInterpreter, _versionStringToTuple from pyzolib.interpreters.inwinreg import get_interpreters_in_reg def get_interpreters(minimumVersion=None): """ get_interpreters(minimumVersion=None) Returns a list of PythonInterpreter instances. If minimumVersion is given, return only the interprers with at least that version, and also sort the result by version number. """ # Get Python interpreters if sys.platform.startswith('win'): pythons = _get_interpreters_win() else: pythons = _get_interpreters_posix() pythons = set([PythonInterpreter(p) for p in pythons]) # Get conda paths condas = set([PythonInterpreter(p) for p in _get_interpreters_conda()]) # Get Pyzo paths pyzos = set([PyzoInterpreter(p) for p in _get_interpreters_pyzo()]) # Remove Pyzo interpreters from pythons pythons = pythons.difference(condas) pythons = pythons.difference(pyzos) # Almost done interpreters = list(condas) + list(pyzos) + list(pythons) minimumVersion = minimumVersion or '0' return _select_interpreters(interpreters, minimumVersion) def _select_interpreters(interpreters, minimumVersion): """ Given a list of PythonInterpreter instances, return a list with the interpreters selected that are valid and have their version equal or larger than the given minimimVersion. The returned list is sorted by version number. """ if not isinstance(minimumVersion, str): raise ValueError('minimumVersion in get_interpreters must be a string.') # Remove invalid interpreters interpreters = [i for i in interpreters if i.version] # Remove the ones below the reference version if minimumVersion is not None: refTuple = _versionStringToTuple(minimumVersion) interpreters = [i for i in interpreters if (i.version_info >= refTuple)] # Return, sorted by version return sorted(interpreters, key=lambda x:x.version_info) def _get_interpreters_win(): found = [] # Query from registry for v in get_interpreters_in_reg(): found.append(v.installPath() ) # Check common locations for rootname in ['c:/', 'C:/program files/', 'C:/program files (x86)/']: if not os.path.isdir(rootname): continue for dname in os.listdir(rootname): if dname.lower().startswith('python'): try: version = float(dname[len('python'):]) except ValueError: continue else: found.append(os.path.join(rootname, dname)) # Normalize all paths, and remove trailing backslashes found = [os.path.normcase(os.path.abspath(v)).strip('\\') for v in found] # Append "python.exe" and check if that file exists found2 = [] for dname in found: exename = os.path.join(dname, 'python.exe') if os.path.isfile(exename): found2.append(exename) # Returnas set (remove duplicates) return set(found2) def _get_interpreters_posix(): found=[] for searchpath in ['/usr/bin','/usr/local/bin','/opt/local/bin']: # Get files try: files = os.listdir(searchpath) except Exception: continue # Search for python executables for fname in files: if fname.startswith('python') and not fname.count('config'): if len(fname) < 16: # Get filename and resolve symlink filename = os.path.join(searchpath, fname) filename = os.path.realpath(filename) # Seen on OS X that was not a valid file if os.path.isfile(filename): found.append(filename) # Return as set (remove duplicates) return set(found) def _get_interpreters_pyzo(): """ Get a list of known Pyzo interpreters. """ pythonname = 'python' + '.exe' * sys.platform.startswith('win') exes = [] for d in paths.pyzo_dirs(): for fname in [ os.path.join(d, 'bin', pythonname + '3'), os.path.join(d, pythonname), ]: if os.path.isfile(fname): exes.append(fname) break return exes def _get_interpreters_conda(): """ Get known conda environments """ if sys.platform.startswith('win'): pythonname = 'python' + '.exe' else: pythonname = 'bin/python' exes = [] filename = os.path.expanduser('~/.conda/environments.txt') if os.path.isfile(filename): for line in open(filename, 'rt').readlines(): line = line.strip() exe_filename = os.path.join(line, pythonname) if line and os.path.isfile(exe_filename): exes.append(exe_filename) return exes if __name__ == '__main__': for pi in get_interpreters(): print(pi) pyzolib-0.3.4/ssdf/0000775000175000017500000000000012573243411014363 5ustar almaralmar00000000000000pyzolib-0.3.4/ssdf/classmanager.py0000664000175000017500000000724312156025455017406 0ustar almaralmar00000000000000# -*- coding: utf-8 -*- # Copyright (C) 2012, Almar Klein # # SSDF is distributed under the terms of the (new) BSD License. # See http://www.opensource.org/licenses/bsd-license.php """ ssdf.classmaneger.py Implements the code to register classes at ssdf for automatic conversion (kinda like pickling). """ import sys from . import __version__ class _ClassManager: """ _ClassManager This static class enables registering classes, which can then be stored to ssdf and loaded from ssdf. On module loading, the sys module is given a reference to the ClassManager called '_ssdf_class_manager'. Other classes can thereby register themselves without knowing how to import ssdf (provided that ssdf is already imported). """ # Global dict with registered classes _registered_classes = {} @classmethod def _register_at_sys(manager): """ _register_at_sys() Register the manager at the sys module if there is not already one of a higher version. """ if hasattr(sys, '_ssdf_class_manager'): other = sys._ssdf_class_manager if manager.__version__() >= other.__version__(): sys._ssdf_class_manager = manager manager._registered_classes.update(other._registered_classes) return manager else: return other else: sys._ssdf_class_manager = manager return manager @classmethod def __version__(manager): return __version__ @classmethod def is_compatible_class(manager, cls): """ is_compatible_class(cls) Returns True if the given class is SSDF-compatible. """ return not manager.is_incompatible_class(cls) @classmethod def is_incompatible_class(manager, cls): """ is_incompatible_class(cls) Returns a string giving the reason why the given class if not SSDF-compatible. If the class is compatible, this function returns None. """ if not hasattr(cls, '__to_ssdf__'): return "class does not have '__to_ssdf__' method" if not hasattr(cls, '__from_ssdf__'): return "class does not have '__from_ssdf__' classmethod" if not isinstance(cls, type): return "class is not a type (does not inherit object on Python 2.x)" @classmethod def register_class(manager, *args): """ register_class(class1, class2, class3, ...) Register one or more classes. Registered classes can be saved and restored from ssdf. A class needs to implement two methods to qualify for registration: * A method __to_ssdf__() that returns an ssdf.Struct * A classmethod __from_ssdf__(s) that accepts an ssdf.Struct and creates an instance of that class. """ for cls in args: incomp = manager.is_incompatible_class(cls) if incomp: raise ValueError('Cannot register class %s: %s.' % (cls.__name__, incomp) ) else: manager._registered_classes[cls.__name__] = cls @classmethod def is_registered_class(manager, cls): """ is_registered_class(cls) Returns True if the given class is registered. """ return cls in manager._registered_classes.values() # Put in this module namespace and in sys module namespace # The ClassManager is the latest version ClassManager = _ClassManager._register_at_sys() pyzolib-0.3.4/ssdf/ssdf_text.py0000664000175000017500000003775112346147723016764 0ustar almaralmar00000000000000# -*- coding: utf-8 -*- # Copyright (C) 2012, Almar Klein # # SSDF is distributed under the terms of the (new) BSD License. # See http://www.opensource.org/licenses/bsd-license.php """ ssdf.ssdf_text.py Implements functionality to read/write text ssdf files. """ import sys import struct import base64 import zlib import re from . import ClassManager from .ssdf_base import Struct, VirtualArray, SSDFReader, SSDFWriter, Block, _CLASS_NAME from .ssdf_base import _shapeString, _FLOAT_TYPES, _INT_TYPES, _TYPE_DICT from .ssdf_base import np, string_types, binary_type, ascii_type, reduce # Get base64 encode/decode functions PY3 = sys.version_info[0] == 3 if PY3: base64encode = base64.encodebytes base64decode = base64.decodebytes else: base64encode = base64.encodestring base64decode = base64.decodestring # The data types for arrays and how the struct (un)pack formatters. _DTYPES = { 'uint8':' 0 ) or ( word4.endswith(',') ) # Get data if size==0: # Empty array data = binary_type() elif asAscii: # Stored in ascii dataparts = [] fmt = _DTYPES[dtypestr] for val in word4.split(','): if not val.strip(): continue try: if 'int' in dtypestr: val = int(val) else: val = float(val) dataparts.append(struct.pack(fmt, val)) except Exception: if 'int' in dtypestr: dataparts.append(struct.pack(fmt, 0)) else: dataparts.append(struct.pack(fmt, float('nan'))) data = binary_type().join(dataparts) else: # Stored binary # Get data: decode and decompress in blocks dataparts = [] for blockt in word4.split(';'): blockc = base64decode(blockt.encode('utf-8')) block = zlib.decompress(blockc) dataparts.append(block) data = binary_type().join(dataparts) # Almost done ... np.need() # Really try importing Numpy now if not np: # Make virtual array to allow saving it again return VirtualArray(shape, dtypestr, data) elif data: # Convert to numpy array value = np.frombuffer(data, dtype=dtypestr ) # Set and check shape if size == value.size: value.shape = tuple(shape) else: print("SSDF: prod(shape)!=size on line %i."%self._blocknr) return value else: # Empty numpy array return np.zeros(shape, dtype=dtypestr) pyzolib-0.3.4/ssdf/ssdf_bin.py0000664000175000017500000003660312346147655016547 0ustar almaralmar00000000000000# -*- coding: utf-8 -*- # Copyright (C) 2012, Almar Klein # # SSDF is distributed under the terms of the (new) BSD License. # See http://www.opensource.org/licenses/bsd-license.php """ ssdf.ssdf_bin.py Implements functionality to read/write binary ssdf (.bsdf) files. """ import struct import zlib from . import ClassManager from .ssdf_base import Struct, VirtualArray, SSDFReader, SSDFWriter, Block, _CLASS_NAME from .ssdf_base import np, binary_type, ascii_type # Formatters for struct (un)packing _SMALL_NUMBER_FMT = ' 0: # Add portion to buffer, store remainder self._parts.append(data[:i]) data = data[i:] # Write the buffer away and reset buffer self._write_new_partition() # Add data to buffer self._parts.append(data) self._pp += len(data) def flush(self): """ flush() After the last write, use this to compress and write the last partition. """ self._write_new_partition() pyzolib-0.3.4/ssdf/setup.py0000664000175000017500000000370012156025455016100 0ustar almaralmar00000000000000# -*- coding: utf-8 -*- # Copyright (c) 2011, Almar Klein description = 'Simple Structured Data Format.' long_description = """ Simple Structured Data Format ----------------------------- Ssdf is a simple format for stroring structured data. It supports seven data types, two of which are container elements: None, int, float, (Unicode) string, numpy array, list/tuple, dict/Struct. One spec, two formats --------------------- Ssdf is actually two formats: the original text format is human readable, and thus enables inspecting databases and data structures using a simple text editor. The binary format is more efficient and uses compression on the whole file (the text format only compresses arrays). It is more suited for storing really large databases or structures containing large arrays. Both formats are fully compatible. Notes ----- SSDF comes as a single module. While it's a bit big (approaching 2k lines), it enables easier deployment inside a package in a way that works for Python 2 as well as Python 3. """ from distutils.core import setup # Get version for line in file('ssdf.py').readlines(): if (line.startswith('__version__')): exec(line.strip()) setup( name = 'ssdf', version = __version__, author = 'Almar Klein', author_email = 'almar.klein at gmail', license = 'BSD', url = 'https://bitbucket.org/almarklein/ssdf', keywords = "simple structured data fileformat", description = description, long_description = long_description, platforms = 'any', provides = ['ssdf'], requires = [], py_modules = ['ssdf'], zip_safe = False, # I want examples to work ) # Note that the dir in package_dir must NOT be an empty string! This # would work for distutils, but not for setuptools... # I found this on-line doc very helpful for creating a setup script: # http://docs.python.org/distutils/examples.html pyzolib-0.3.4/ssdf/__init__.py0000664000175000017500000002325612346145751016512 0ustar almaralmar00000000000000# -*- coding: utf-8 -*- # Copyright (C) 2012, Almar Klein # # SSDF is distributed under the terms of the (new) BSD License. # See http://www.opensource.org/licenses/bsd-license.php """ Package ssdf Simple Structured Data Format ----------------------------- Ssdf is a simple format for stroring structured data. It supports seven data types, two of which are container elements: None, int, float, (Unicode) string, numpy array, list/tuple, dict/Struct. One spec, two formats --------------------- Ssdf is actually two formats: the original text format is human readable, and thus enables inspecting databases and data structures using a simple text editor. The binary format is more efficient and uses compression on the whole file (the text format only compresses arrays). It is more suited for storing really large databases or structures containing large arrays. Both formats are fully compatible. Functions of interest --------------------- * save - save a struct to a file * saves - serialize a struct to a string * saveb - serialize a struct to bytes * load - load a struct from file * loads - load a struct from a string * loadb - load a struct from bytes * update - update a struct on file with a given struct * copy - create a deep copy of a struct * new - create a new empty struct * clear - clear a struct, removing all elements * count - count the number of elements in a struct Notes ----- From version 2.1, ssdf is a package (instead of a single module). The package uses relative imports and the repository is flat, which makes it easy to include ssdf as a subrepository in an application or library. Pyzolib includes the ssdf package in this manner. """ __version__ = '2.1.1' import time # Create/import claassManager and insert two of its functions in this namespace from .classmanager import ClassManager register_class = ClassManager.register_class is_compatible_class = ClassManager.is_compatible_class # Import the rest from . import ssdf_base from .ssdf_base import Struct, isstruct, VirtualArray, binary_type, string_types from . import ssdf_text, ssdf_bin def _get_mode(filename, mode): """ _get_mode(filename, mode) Given filename and mode returns the mode (as 1 or 2). """ # Determine mode from extension mode_ext = 0 if filename.lower().endswith('.ssdf'): mode_ext = 1 elif filename.lower().endswith('.bsdf'): mode_ext = 2 # Determine given mode if isinstance(mode, string_types): mode = mode.lower() if mode in [1, 'text', 'string', 'str']: mode = 1 elif mode in [2, 'binary', 'bin', 'bytes']: mode = 2 elif mode: ValueError("Unknown mode given '%s'." % repr(mode)) # Determine mode if not mode: mode = mode_ext elif mode_ext and mode_ext != mode: raise ValueError('Given mode does not correspond with extension.') if not mode: raise ValueError('No mode specified (and no known extension used).') # Done return mode def save(filename, struct, mode=None): """ save(filename, struct, mode=None) Save the given struct or dict to the filesystem using the given filename. Two modes are supported: text mode stores in a human readable format, and binary mode stores in a more efficient (compressed) binary format. Parameters ---------- filename : str The location in the filesystem to save the file. Files with extension '.ssdf' are stored in text mode, and '.bsdf' files are stored in binary mode. If another extension is used, the mode should be specified explicitly. struct : {Struct, dict} The object to save. mode : optional {'text', 'str', 1, 'bin', 'bytes', 2} This parameter can be used to explicitly specify the mode. Note that it is an error to use binary mode on a '.ssdf' file or text mode on a '.bsdf' file. """ # Check if not (isstruct(struct) or isinstance(struct, dict)): raise ValueError('ssdf.save() expects the second argument to be a struct.') # Open file f = open(filename, 'wb') try: # Get mode mode = _get_mode(filename, mode) # Write if mode==1: writer = ssdf_text.TextSSDFWriter() # Write code directive and header header = '# This Simple Structured Data Format (SSDF) file was ' header += 'created from Python on %s.\n' % time.asctime() f.write('# -*- coding: utf-8 -*-\n'.encode('utf-8')) f.write(header.encode('utf-8')) # Write lines writer.write(struct, f) elif mode==2: writer = ssdf_bin.BinarySSDFWriter() writer.write(struct, f) finally: f.close() def saves(struct): """ saves(struct) Serialize the given struct or dict to a (Unicode) string. Parameters ---------- struct : {Struct, dict} The object to save. """ # Check if not (isstruct(struct) or isinstance(struct, dict)): raise ValueError('ssdf.saves() expects a struct.') # Write writer = ssdf_text.TextSSDFWriter() return writer.write(struct) def saveb(struct): """ saveb(struct) Serialize the given struct or dict to (compressed) bytes. Parameters ---------- struct : {Struct, dict} The object to save. """ # Check if not (isstruct(struct) or isinstance(struct, dict)): raise ValueError('ssdf.saveb() expects a struct.') # Write writer = ssdf_bin.BinarySSDFWriter() return writer.write(struct) def load(filename): """ load(filename) Load a struct from the filesystem using the given filename. Two modes are supported: text mode stores in a human readable format, and binary mode stores in a more efficient (compressed) binary format. Parameters ---------- filename : str The location in the filesystem of the file to load. """ # Open file f = open(filename, 'rb') try: # Get mode try: firstfour = f.read(4).decode('utf-8') except Exception: raise ValueError('Not a valid ssdf file.') if firstfour == 'BSDF': mode = 2 else: mode = 1 # This is an assumption. # Read f.seek(0) if mode==1: reader = ssdf_text.TextSSDFReader() return reader.read(f) elif mode==2: reader = ssdf_bin.BinarySSDFReader() return reader.read(f) finally: f.close() def loadb(bb): """ loadb(bb) Load a struct from the given bytes. Parameters ---------- bb : bytes A serialized struct (obtained using ssdf.saveb()). """ # Check if not isinstance(bb, binary_type): raise ValueError('ssdf.loadb() expects bytes.') # Read reader = ssdf_bin.BinarySSDFReader() return reader.read(bb) def loads(ss): """ loads(ss) Load a struct from the given string. Parameters ---------- ss : (Unicode) string A serialized struct (obtained using ssdf.saves()). """ # Check if not isinstance(ss, string_types): raise ValueError('ssdf.loads() expects a string.') # Read reader = ssdf_text.TextSSDFReader() return reader.read(ss) def update(filename, struct): """ update(filename, struct) Update an existing ssdf file with the given struct. For every dict in the data tree, the elements are updated. Note that any lists occuring in both data trees are simply replaced. """ # Load existing struct s = load(filename) # Insert stuff def insert(ob1, ob2): for name in ob2: if ( name in ob1 and isstruct(ob1[name]) and isstruct(ob2[name]) ): insert(ob1[name], ob2[name]) else: ob1[name] = ob2[name] insert(s, struct) # Save save(filename, s) def new(): """ new() Create a new Struct object. The same as "Struct()". """ return Struct() def clear(struct): """ clear(struct) Clear all elements of the given struct object. """ for key in [key for key in struct]: del(struct.__dict__[key]) def count(object): """ count(object): Count the number of elements in the given object. An element is defined as one of the 7 datatypes supported by ssdf (dict/struct, tuple/list, array, string, int, float, None). """ n = 1 if isstruct(object) or isinstance(object, dict): for key in object: val = object[key] n += count(val) elif isinstance(object, (tuple, list)): for val in object: n += count(val) return n def copy(object): """ copy(objec) Return a deep copy the given object. The object and its children should be ssdf-compatible data types. Note that dicts are converted to structs and tuples to lists. """ if isstruct(object) or isinstance(object, dict): newObject = Struct() for key in object: val = object[key] newObject[key] = copy(val) return newObject elif isinstance(object, (tuple, list)): return [copy(ob) for ob in object] elif isinstance(object, VirtualArray): return VirtualArray(object.shape, object.dtype, object.data) elif ssdf_base.np and isinstance(object, ssdf_base.np.ndarray): return object.copy() else: # immutable return object pyzolib-0.3.4/ssdf/ssdf_base.py0000664000175000017500000003632312346150044016672 0ustar almaralmar00000000000000# -*- coding: utf-8 -*- # Copyright (C) 2012, Almar Klein # # SSDF is distributed under the terms of the (new) BSD License. # See http://www.opensource.org/licenses/bsd-license.php """ ssdf.ssdf_base.py Implements the base functionality for ssdf: * The Struct class * Some constants and small functions * The VirtualArray class * The base classes for SSDFReader, SSDFWriter anf Block """ import sys from . import ClassManager # Proxy class for numpy module. We try to delay importing numpy if possible. class _NumpyProxy(object): def __init__(self): self._np = None self._tried_hard = False self._try_soft_import() def __bool__(self): if self._np is None: self._try_soft_import() return bool(self._np) def need(self): if not self._tried_hard: self._try_hard_import() def _try_soft_import(self): self._np = sys.modules.get('numpy', None) if self._np: self._inject() self._set_types() def _try_hard_import(self): self._tried_hard = True try: import numpy as np except ImportError: np = None self._np = np if self._np: self._inject() self._set_types() def _inject(self): for name in dir(self._np): if not name.startswith('_'): setattr(self, name, getattr(self._np, name)) def _set_types(self): global _FLOAT_TYPES global _INT_TYPES np = self._np _FLOAT_TYPES, _INT_TYPES = set(_FLOAT_TYPES), set(_INT_TYPES) _FLOAT_TYPES.update([np.float32, np.float64]) _INT_TYPES.update([ np.int8, np.int16, np.int32, np.int64, np.uint8, np.uint16, np.uint32, np.uint64 ]) _FLOAT_TYPES, _INT_TYPES = tuple(_FLOAT_TYPES), tuple(_INT_TYPES) # From six.py PY3 = sys.version_info[0] == 3 if PY3: string_types = str, integer_types = int, text_type = str binary_type = bytes ascii_type = str # Simple string from functools import reduce else: string_types = basestring, integer_types = (int, long) text_type = unicode binary_type = str ascii_type = str # Simple string reduce = reduce # To store other classes _CLASS_NAME = '_CLASS_NAME_' # Same as in ssdf_bin (we only need the dict type id. _TYPE_DICT = ord('D') # Determine float and int types _FLOAT_TYPES = tuple(set([float])) _INT_TYPES = tuple(set(integer_types)) # The numpy proxy np = _NumpyProxy() def isstruct(ob): """ isstruct(ob) Returns whether the given object is an SSDF struct. """ if hasattr(ob, '__is_ssdf_struct__'): return bool(ob.__is_ssdf_struct__) else: return False def _not_equal(ob1, ob2): """ _not_equal(ob1, ob2) Returns None if the objects are equal. Otherwise returns a string indicating how the objects are inequal. """ if isstruct(ob1) or isinstance(ob1, dict): if not ( isstruct(ob2) or isinstance(ob2, dict) ): return '' # Test number of elements keys1 = [key for key in ob1] keys2 = [key for key in ob2] if len(keys1) != len(keys2): return '' # Test all elements for key in keys1: if key not in keys2: return '' not_equal = _not_equal(ob1[key], ob2[key]) if not_equal: return '.' + key + not_equal elif isinstance(ob1, (tuple, list)): if not isinstance(ob2, (tuple, list)): return '' if len(ob1) != len(ob2): return '' # Test all elements for i in range(len(ob1)): not_equal = _not_equal(ob1[i], ob2[i]) if not_equal: return ('[%i]' % i) + not_equal elif isinstance(ob1, VirtualArray): if not isinstance(ob2, VirtualArray): return '' # Test properties if not ( ob1.shape==ob2.shape and ob1.dtype==ob2.dtype and ob1.data==ob2.data ): return '' elif np and isinstance(ob1, np.ndarray): if not isinstance(ob2, np.ndarray): return '' # Test properties if not ( ob1.shape==ob2.shape and ob1.dtype==ob2.dtype and (ob1==ob2).sum()==ob1.size ): return '' else: # Use default equality operator if not (ob1 == ob2): return '' def _isvalidname(name): """ _isvalidname(name) Returns attribute name, or None, if not valid """ # Is it a string? if not ( name and isinstance(name, string_types) ): return None # Check name namechars = str('abcdefghijklmnopqrstuvwxyz_0123456789') name2 = name.lower() if name2[0] not in namechars[0:-10]: return None tmp = list(map(lambda x: x not in namechars, name2[2:])) # Return if sum(tmp)==0: return name def _shapeString(ob): """ _shapeString(ob) Returns a string that represents the shape of the given array. """ ss = str() for n in ob.shape: ss += '%ix' % n return ss[:-1] class Struct(object): """ Struct(dictionary=None) Object to holds named data (syntactic sugar for a dictionary). Attributes can be any of the seven SSDF supported types: struct/dict, tuple/list, numpy array, (Unicode) string, int, float, None. Elements can be added in two ways: * s.foo = 'bar' # the object way * s['foo'] = 'bar' # the dictionary way Supported features ------------------ * Iteration - yields the keys/names in the struct * len() - returns the number of elements in the struct * del statement can be used to remove elements * two structs can be added, yielding a new struct with combined elements * testing for equality with other structs Notes ----- * The keys in the given dict should be valid names (invalid keys are ignoired). * On saving, names starting with two underscores are ignored. * This class does not inherit from dict to keep its namespace clean, avoid nameclashes, and to enable autocompletion of its items in most IDE's. * To get the underlying dict, simply use s.__dict__. """ # Indentifier __is_ssdf_struct__ = True def __init__(self, a_dict=None): # Plain struct? if a_dict is None: return if not isinstance(a_dict, dict) and not isstruct(a_dict): tmp = "Struct can only be initialized with a Struct or a dict." raise ValueError(tmp) else: # Try loading from object def _getValue(val): """ Get the value, as suitable for Struct. """ if isinstance(val, (string_types,) + _FLOAT_TYPES + _INT_TYPES ): return val if np and isinstance(val, np.ndarray): return val elif isinstance(val,(tuple,list)): L = list() for element in val: L.append( _getValue(element) ) return L elif isinstance(val, dict): return Struct(val) else: pass # leave it # Copy all keys in the dict that are not methods for key in a_dict: if not _isvalidname(key): print("Ignoring invalid key-name '%s'." % key) continue val = a_dict[key] self[key] = _getValue(val) def __getitem__(self, key): # Name ok? key2 = _isvalidname(key) if not key2: raise KeyError("Trying to get invalid name '%s'." % key) # Name exists? if not key in self.__dict__: raise KeyError(str(key)) # Return return self.__dict__[key] def __setitem__(self, key, value): # Name ok? key2 = _isvalidname(key) if not key2: raise KeyError("Trying to set invalid name '%s'." % key) # Set self.__dict__[key] = value def __iter__(self): """ Returns iterator over keys. """ return self.__dict__.__iter__() def __delitem__(self, key): return self.__dict__.__delitem__(key) def __len__(self): """ Return amount of fields in the Struct object. """ return len(self.__dict__) def __add__(self, other): """ Enable adding two structs by combining their elemens. """ s = Struct() s.__dict__.update(self.__dict__) s.__dict__.update(other.__dict__) return s def __eq__(self, other): return not _not_equal(self, other) def __ne__(self, other): return _not_equal(self, other) def __repr__(self): """ Short string representation. """ return "" % len(self) def __str__(self): """ Long string representation. """ # Get alignment value c = 0 for key in self: c = max(c, len(key)) # How many chars left (to display on less than 80 lines) charsLeft = 79 - (c+4) # 2 spaces and ': ' s = 'Elements in SSDF struct:\n' for key in self: if key.startswith("__"): continue tmp = "%s" % (key) value = self[key] valuestr = repr(value) if len(valuestr)>charsLeft or '\n' in valuestr: typestr = str(type(value))[7:-2] if np and isinstance(value,np.ndarray): shapestr = _shapeString(value) valuestr = "" %(shapestr,str(value.dtype)) elif isinstance(value, string_types): valuestr = valuestr[:charsLeft-3] + '...' #valuestr = "" % (typestr, len(value)) else: valuestr = "<%s with length %i>" % (typestr, len(value)) s += tmp.rjust(c+2) + ": %s\n" % (valuestr) return s class VirtualArray(object): """ VirtualArray A VirtualArray represents an array when numpy is not available. This enables preserving the array when saving back a loaded dataset. """ def __init__(self, shape, dtype, data): self.shape = tuple(shape) self.dtype = dtype self.data = data def tostring(self): return self.data @property def size(self): if self.shape: return reduce( lambda a,b:a*b, self.shape) else: return 1 class SSDFReader: def build_tree(self, root, blocks): """ build_tree(root, blocks) Build up the tree using the indentation information in the blocks. The tree is build up from the given root. """ tree = [root] for block in blocks: # Select leaf in tree while block._indent <= tree[-1]._indent: tree.pop() # Append (to object and to simple tree structure) tree[-1]._children.append(block) tree.append(block) def serialize_struct(self, object, f=None): raise NotImplementedError() def read(self, file_or_string): raise NotImplementedError() class SSDFWriter: def flatten_tree(self, block, sort=False): """ flatten_tree(block, sort=False) Returns a flat list containing the given block and all its children. If sort is True, packs blocks such that the data structures consisting of less blocks appear first. """ # Get list of strings for each child listOfLists = [] for child in block._children: childList = self.flatten_tree(child, sort) listOfLists.append( childList ) # Sort by length if sort and listOfLists and block._type == _TYPE_DICT: listOfLists.sort(key=len) # Produce flat list flatList = [block] for childList in listOfLists: flatList.extend(childList) # Done return flatList def write(self, object, f=None): raise NotImplementedError() class Block: """ Block A block represents a data element. This is where the conversion from Python objects to text/bytes and vice versa occurs. A block is a line in a text file or a piece of data in a binary file. A block contains all information about indentation, name, and value of the data element that it represents. The raw representation of its value is refered to as 'data'. """ def __init__(self, indent, blocknr, name=None, type=None, data=None): self._indent = indent self._blocknr = blocknr # for producing usefull read error messages self._name = name self._type = type # used by binary only (and text-dict) self._data = data # the raw data, bytes or string self._children = [] # used only by dicts and lists @classmethod def from_object(cls, indent, name, value): # Instantiate a block self = cls(indent, -1, name) # Set object's data if value is None: self._from_none() elif ClassManager.is_registered_class(value.__class__): s = value.__to_ssdf__() s[_CLASS_NAME] = value.__class__.__name__ self._from_dict(s) elif isinstance(value, _INT_TYPES): self._from_int(value) elif isinstance(value, _FLOAT_TYPES): self._from_float(value) elif isinstance(value, bool): self._from_int(int(value)) elif isinstance(value, string_types): self._from_unicode(value) elif np and isinstance(value, np.ndarray): self._from_array(value) elif isinstance(value, VirtualArray): self._from_array(value) elif isinstance(value, dict) or isstruct(value): self._from_dict(value) elif isinstance(value, (list, tuple)): self._from_list(value) else: # We do not know self._from_none() tmp = repr(value) if len(tmp) > 64: tmp = tmp[:64] + '...' if name is not None: print("SSDF: %s is unknown object: %s %s" % (name, tmp, repr(type(value)) )) else: print("SSDF: unknown object: %s %s" % (tmp, repr(type(value)) )) # Done return self pyzolib-0.3.4/__init__.py0000664000175000017500000000243012573243242015536 0ustar almaralmar00000000000000# -*- coding: utf-8 -*- # Copyright (c) 2012, The Pyzo team # # This file is distributed under the terms of the (new) BSD License. """ Package pyzolib The pyzolib package provides basic functionality for the Pyzo environment. It contains a collection of modules and small packages that should be imported as "from pyzolib import xxx" The packages currently are: * path - object oriented path processing (no more os.path.x) * paths - Get paths to useful directories in a cross platform manner. * qt - Proxy for importing QtCore et al. from PySide or PyQt4 * ssdf - the Simple Structured Data Format (for config files and scientific databases) * insertdocs - a sphynx pre-processor to include docstrings in the text, allowing readthedocs.org to host the docs without requiring importing code. * pyximport - for easy on the fly compilation of Cython, using the Pyzo environment to establish the location of a gcc compiler. * gccutils - used by the above to manage the gcc compiler. * interprerers - list the Python interpreters available on this system. * dllutils - utilities to set the RPATH in dynamic libararies and remove depndencies on the MSVCR from the embedded manifest. * shebang - for making shebangs in pyzo distro absolute. """ __version__ = '0.3.4' pyzolib-0.3.4/paths.py0000664000175000017500000002415312312555701015121 0ustar almaralmar00000000000000# -*- coding: utf-8 -*- # Copyright (c) 2012, Almar Klein, Rob Reilink # # This file is distributed under the terms of the (new) BSD License. """ Module paths Get paths to useful directories in a cross platform manner. The functions in this module are designed to be stand-alone, so that they can easily be copied and used in code that does not want pyzolib as a dependency. """ # Notes: # * site.getusersitepackages() returns a dir in roaming userspace on Windows # so better avoid that. # * site.getuserbase() returns appdata_dir('Python', True) # * See docstring: that's why the functions tend to not re-use each-other import sys ISWIN = sys.platform.startswith('win') ISMAC = sys.platform.startswith('darwin') ISLINUX = sys.platform.startswith('linux') PY2 = sys.version_info[0] == 2 PY3 = sys.version_info[0] == 3 # From pyzolib/paths.py (https://bitbucket.org/pyzo/pyzolib/src/tip/paths.py) import sys def is_frozen(): """ is_frozen() Return whether this app is a frozen application (using e.g. cx_freeze). """ return bool( getattr(sys, 'frozen', None) ) # From pyzolib/paths.py (https://bitbucket.org/pyzo/pyzolib/src/tip/paths.py) import os, sys, tempfile def temp_dir(appname=None, nospaces=False): """ temp_dir(appname=None, nospaces=False) Get path to a temporary directory with write access. If appname is given, a subdir is appended (and created if necessary). If nospaces, will ensure that the path has no spaces. """ # Do it the Python way path = tempfile.gettempdir() # Try harder if we have to if nospaces and ' ' in path: if sys.platform.startswith('win'): for path in ['c:\\TEMP', 'c:\\TMP']: if os.path.isdir(path): break if not os.path.isdir(path): os.mkdir(path) else: for path in ['/tmp', '/var/tmp']: # http://www.tuxfiles.org/linuxhelp/linuxdir.html if os.path.isdir(path): break else: raise RuntimeError('Could not locate temporary directory.') # Get path specific for this app if appname: path = os.path.join(path, appname) if not os.path.isdir(path): os.mkdir(path) # Done return path # From pyzolib/paths.py (https://bitbucket.org/pyzo/pyzolib/src/tip/paths.py) import os def user_dir(): """ user_dir() Get the path to the user directory. (e.g. "/home/jack", "c:/Users/jack") """ return os.path.expanduser('~') # From pyzolib/paths.py (https://bitbucket.org/pyzo/pyzolib/src/tip/paths.py) import os, sys def appdata_dir(appname=None, roaming=False, macAsLinux=False): """ appdata_dir(appname=None, roaming=False, macAsLinux=False) Get the path to the application directory, where applications are allowed to write user specific files (e.g. configurations). For non-user specific data, consider using common_appdata_dir(). If appname is given, a subdir is appended (and created if necessary). If roaming is True, will prefer a roaming directory (Windows Vista/7). If macAsLinux is True, will return the Linux-like location on Mac. """ # Define default user directory userDir = os.path.expanduser('~') # Get system app data dir path = None if sys.platform.startswith('win'): path1, path2 = os.getenv('LOCALAPPDATA'), os.getenv('APPDATA') path = (path2 or path1) if roaming else (path1 or path2) elif sys.platform.startswith('darwin') and not macAsLinux: path = os.path.join(userDir, 'Library', 'Application Support') # On Linux and as fallback if not (path and os.path.isdir(path)): path = userDir # Maybe we should store things local to the executable (in case of a # portable distro or a frozen application that wants to be portable) prefix = sys.prefix if getattr(sys, 'frozen', None): # See application_dir() function prefix = os.path.abspath(os.path.dirname(sys.path[0])) for reldir in ('settings', '../settings'): localpath = os.path.abspath(os.path.join(prefix, reldir)) if os.path.isdir(localpath): try: open(os.path.join(localpath, 'test.write'), 'wb').close() os.remove(os.path.join(localpath, 'test.write')) except IOError: pass # We cannot write in this directory else: path = localpath break # Get path specific for this app if appname: if path == userDir: appname = '.' + appname.lstrip('.') # Make it a hidden directory path = os.path.join(path, appname) if not os.path.isdir(path): os.mkdir(path) # Done return path # From pyzolib/paths.py (https://bitbucket.org/pyzo/pyzolib/src/tip/paths.py) import os, sys def common_appdata_dir(appname=None): """ common_appdata_dir(appname=None) Get the path to the common application directory. Applications are allowed to write files here. For user specific data, consider using appdata_dir(). If appname is given, a subdir is appended (and created if necessary). """ # Try to get path path = None if sys.platform.startswith('win'): path = os.getenv('ALLUSERSPROFILE', os.getenv('PROGRAMDATA')) elif sys.platform.startswith('darwin'): path = '/Library/Application Support' else: # Not sure what to use. Apps are only allowed to write to the home # dir and tmp dir, right? pass # If no success, use appdata_dir() instead if not (path and os.path.isdir(path)): path = appdata_dir() # Get path specific for this app if appname: path = os.path.join(path, appname) if not os.path.isdir(path): os.mkdir(path) # Done return path # Other approaches that we considered, but which did not work for links, # or are less reliable for other reasons are: # * sys.executable: does not work for links # * sys.prefix: dito # * sys.exec_prefix: dito # * os.__file__: does not work when frozen # * __file__: only accessable from main module namespace, does not work when frozen # todo: get this included in Python sys or os module! # From pyzolib/paths.py (https://bitbucket.org/pyzo/pyzolib/src/tip/paths.py) import os, sys def application_dir(): """ application_dir() Get the directory in which the current application is located. The "application" can be a Python script or a frozen application. This function raises a RuntimeError if in interpreter mode. """ # Test if the current process can be considered an "application" if not sys.path or not sys.path[0]: raise RuntimeError('Cannot determine app path because sys.path[0] is empty!') # Get the path. If frozen, sys.path[0] is the name of the executable, # otherwise it is the path to the directory that contains the script. thepath = sys.path[0] if getattr(sys, 'frozen', None): thepath = os.path.dirname(thepath) # Return absolute version, or symlinks may not work return os.path.abspath(thepath) ## Pyzo specific # # A Pyzo distribution maintains a file in the appdata dir that lists # the directory where it is intalled. Pyzo can in principle be installed # multiple times. In that case the file contains multiple entries. # This file is checked each time the pyzo executable is run. Therefore # a user can move the Pyzo directory and simply run the Pyzo executable # to update the registration. def pyzo_dirs(newdir=None, makelast=False): """ pyzo_dirs(newdir=None, makelast=False) Compatibility function. Like pyzo_dirs2, but returns a list of directories and does not allow setting the version. """ return [p[0] for p in pyzo_dirs2(newdir, makelast=makelast)] # From pyzolib/paths.py (https://bitbucket.org/pyzo/pyzolib/src/tip/paths.py) import os, sys def pyzo_dirs2(path=None, version='0', **kwargs): """ pyzo_dirs2(dir=None, version='0', makelast=False) Get the locations of installed Pyzo directories. Returns a list of tuples: (dirname, version). In future versions more information may be added to the file, so please take larger tuples into account. If path is a dir containing a python exe, it is added it to the list. If the keyword arg 'makelast' is given and True, will ensure that the given path is the last in the list (i.e. the default). """ defaultPyzo = '', '0' # To fill in values for shorter items newPyzo = (str(path), str(version)) if path else None # Get application dir userDir = os.path.expanduser('~') path = None if sys.platform.startswith('win'): path = os.getenv('LOCALAPPDATA', os.getenv('APPDATA')) elif sys.platform.startswith('darwin'): path = os.path.join(userDir, 'Library', 'Application Support') # Get application dir for Pyzo if path and os.path.isdir(path): path = os.path.join(path, 'pyzo') else: path = os.path.join(userDir, '.pyzo') # On Linux and as fallback if not os.path.isdir(path): os.mkdir(path) # Open file and parse fname = os.path.join(path, 'pyzodirs') pyzos, npyzos = [], 0 if os.path.isfile(fname): lines = open(fname, 'rb').read().decode('utf-8').split('\n') pyzos = [tuple(d.split(':::')) for d in [d.strip() for d in lines] if d] npyzos = len(pyzos) # Add dir if necessary if newPyzo and os.path.isdir(newPyzo[0]): if kwargs.get('makelast', False) or newPyzo not in pyzos: npyzos = 0 # force save pyzos = [p for p in pyzos if p[0] != newPyzo[0]] # rm based on dir pyzos.append(newPyzo) # Check validity of all pyzos, write back if necessary, and return pythonname = 'python' + '.exe' * sys.platform.startswith('win') pyzos = [p for p in pyzos if os.path.isfile(os.path.join(p[0], pythonname))] if len(pyzos) != npyzos: lines = [':::'.join(p) for p in pyzos] open(fname, 'wb').write( ('\n'.join(lines)).encode('utf-8') ) return [p+defaultPyzo[len(p):] for p in pyzos] ## Windows specific # Maybe for directory of programs, pictures etc.