python-cpl-0.6.2/0000755000175000017500000000000012312010046013527 5ustar olesoles00000000000000python-cpl-0.6.2/PKG-INFO0000644000175000017500000000427212312010046014631 0ustar olesoles00000000000000Metadata-Version: 1.1 Name: python-cpl Version: 0.6.2 Summary: Python interface for the Common Pipeline Library. Home-page: https://pypi.python.org/pypi/python-cpl/0.6.2 Author: Ole Streicher Author-email: python-cpl@liska.ath.cx License: GPL Download-URL: http://pypi.python.org/packages/source/p/python-cpl/python-cpl-0.6.2.tar.gz Description: This module can list, configure and execute CPL-based recipes from Python. The input, calibration and output data can be specified as FITS files or as astropy.io.fits objects in memory. The Common Pipeline Library (CPL) comprises a set of ISO-C libraries that provide a comprehensive, efficient and robust software toolkit. It forms a basis for the creation of automated astronomical data-reduction tasks. One of the features provided by the CPL is the ability to create data-reduction algorithms that run as plugins (dynamic libraries). These are called "recipes" and are one of the main aspects of the CPL data-reduction development environment. More information about the CPL can be found here: http://www.eso.org/sci/software/cpl/ The interface may be used to run ESO pipeline recipes linked to CPL versions 4.0 to 6.3.1. Build instructions ------------------ python-cpl requires: - Python 2.6 or later - astropy or pyfits python-cpl uses the standard Python distutils system to build and install itself. From the command line run:: python setup.py install to install python-cpl. Platform: UNKNOWN Classifier: Development Status :: 4 - Beta Classifier: Intended Audience :: Science/Research Classifier: License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+) Classifier: Operating System :: MacOS :: MacOS X Classifier: Operating System :: POSIX Classifier: Operating System :: Unix Classifier: Programming Language :: C Classifier: Programming Language :: Python Classifier: Programming Language :: Python :: 2 Classifier: Topic :: Scientific/Engineering :: Astronomy Requires: astropy Provides: cpl python-cpl-0.6.2/cpl/0000755000175000017500000000000012312010046014305 5ustar olesoles00000000000000python-cpl-0.6.2/cpl/CPL_library.c0000664000175000017500000002400212311576522016632 0ustar olesoles00000000000000#include #include #include "CPL_library.h" unsigned long supported_versions[] = { CPL_VERSION(6,4,1), CPL_VERSION(6,4,0), CPL_VERSION(6,3,1), CPL_VERSION(6,3,0), CPL_VERSION(6,2,0), CPL_VERSION(6,1,1), CPL_VERSION(6,0,1), CPL_VERSION(6,0,0), CPL_VERSION(5,3,1), CPL_VERSION(5,2,0), CPL_VERSION(5,1,0), CPL_VERSION(5,0,1), CPL_VERSION(5,0,0), CPL_VERSION(4,2,0), CPL_VERSION(4,1,0), CPL_VERSION(4,0,1), CPL_VERSION(4,0,0), 0 }; static cpl_library_t **libraries = NULL; /* This module provides all needed functions to run a recipe from the framework. These functions are extracted from the recipe shared lib (resp. the CPL linked to that) by their names. I checked that the API didn't change from CPL 4.0 which is now the minimal supported version here. Since some constants changed over the time, all used constants are also included in the structure. The constants are set directly depending on the CPL version number. Note that beta releases have to be taken very cautiously here since they may contain incompatible changes here. */ cpl_library_t *create_library(const char *fname) { void *handle = dlopen(fname, RTLD_LAZY); if (handle == NULL) { return NULL; } char *error = dlerror(); typeof(cpl_init) *init = dlsym(handle, "cpl_init"); error = dlerror(); if (error != NULL) { dlclose(handle); return NULL; } if (libraries == NULL) { libraries = malloc(sizeof(cpl_library_t *)); libraries[0] = NULL; } int i; for (i = 0; libraries[i] != NULL; i++) { if (init == libraries[i]->init) { dlclose(handle); return libraries[i]; } } cpl_library_t *cpl = malloc(sizeof(cpl_library_t)); cpl->init = init; cpl->init(CPL_INIT_DEFAULT); typeof(cpl_version_get_major) *get_major = dlsym(handle, "cpl_version_get_major"); typeof(cpl_version_get_minor) *get_minor = dlsym(handle, "cpl_version_get_minor"); typeof(cpl_version_get_micro) *get_micro = dlsym(handle, "cpl_version_get_micro"); cpl->version = CPL_VERSION(get_major(), get_minor(), get_micro()); cpl->end = dlsym(handle, "cpl_end"); cpl->version_get_version = dlsym(handle, "cpl_version_get_version"); cpl->get_description = dlsym(handle, "cpl_get_description"); cpl->memory_dump = dlsym(handle, "cpl_memory_dump"); cpl->memory_is_empty = dlsym(handle, "cpl_memory_is_empty"); cpl->free = dlsym(handle, "cpl_free"); cpl->plugin_get_author = dlsym(handle, "cpl_plugin_get_author"); cpl->plugin_get_copyright = dlsym(handle, "cpl_plugin_get_copyright"); cpl->plugin_get_deinit = dlsym(handle, "cpl_plugin_get_deinit"); cpl->plugin_get_description = dlsym(handle, "cpl_plugin_get_description"); cpl->plugin_get_email = dlsym(handle, "cpl_plugin_get_email"); cpl->plugin_get_exec = dlsym(handle, "cpl_plugin_get_exec"); cpl->plugin_get_init = dlsym(handle, "cpl_plugin_get_init"); cpl->plugin_get_name = dlsym(handle, "cpl_plugin_get_name"); cpl->plugin_get_synopsis = dlsym(handle, "cpl_plugin_get_synopsis"); cpl->plugin_get_version = dlsym(handle, "cpl_plugin_get_version"); cpl->plugin_get_version_string = dlsym(handle, "cpl_plugin_get_version_string"); cpl->pluginlist_delete = dlsym(handle, "cpl_pluginlist_delete"); cpl->pluginlist_find = dlsym(handle, "cpl_pluginlist_find"); cpl->pluginlist_get_first = dlsym(handle, "cpl_pluginlist_get_first"); cpl->pluginlist_get_next = dlsym(handle, "cpl_pluginlist_get_next"); cpl->pluginlist_new = dlsym(handle, "cpl_pluginlist_new"); cpl->dfs_update_product_header = dlsym(handle, "cpl_dfs_update_product_header"); cpl->error_get_code = dlsym(handle, "cpl_error_get_code"); cpl->error_get_file = dlsym(handle, "cpl_error_get_file"); cpl->error_get_function = dlsym(handle, "cpl_error_get_function"); cpl->error_get_line = dlsym(handle, "cpl_error_get_line"); cpl->error_get_message = dlsym(handle, "cpl_error_get_message"); cpl->error_reset = dlsym(handle, "cpl_error_reset"); cpl->error_set_message_macro = dlsym(handle, "cpl_error_set_message_macro"); cpl->errorstate_dump = dlsym(handle, "cpl_errorstate_dump"); cpl->errorstate_get = dlsym(handle, "cpl_errorstate_get"); cpl->frame_get_filename = dlsym(handle, "cpl_frame_get_filename"); cpl->frame_get_group = dlsym(handle, "cpl_frame_get_group"); cpl->frame_get_tag = dlsym(handle, "cpl_frame_get_tag"); cpl->frame_new = dlsym(handle, "cpl_frame_new"); cpl->frame_set_filename = dlsym(handle, "cpl_frame_set_filename"); cpl->frame_set_tag = dlsym(handle, "cpl_frame_set_tag"); cpl->frameset_delete = dlsym(handle, "cpl_frameset_delete"); if (cpl->version >= CPL_VERSION(6,3,0)) { cpl->frameset_get_position = dlsym(handle, "cpl_frameset_get_position"); } else { // fallback variant: not threadsafe, deprecated after 6.2 cpl->frameset_get_position = dlsym(handle, "cpl_frameset_get_frame"); } cpl->frameset_get_size = dlsym(handle, "cpl_frameset_get_size"); cpl->frameset_insert = dlsym(handle, "cpl_frameset_insert"); cpl->frameset_new = dlsym(handle, "cpl_frameset_new"); cpl->msg_error = dlsym(handle, "cpl_msg_error"); cpl->msg_set_level = dlsym(handle, "cpl_msg_set_level"); cpl->msg_set_log_level = dlsym(handle, "cpl_msg_set_log_level"); cpl->msg_set_log_name = dlsym(handle, "cpl_msg_set_log_name"); cpl->msg_stop_log = dlsym(handle, "cpl_msg_stop_log"); cpl->parameter_get_alias = dlsym(handle, "cpl_parameter_get_alias"); cpl->parameter_get_class = dlsym(handle, "cpl_parameter_get_class"); cpl->parameter_get_context = dlsym(handle, "cpl_parameter_get_context"); cpl->parameter_get_default_bool = dlsym(handle, "cpl_parameter_get_default_bool"); cpl->parameter_get_default_double = dlsym(handle, "cpl_parameter_get_default_double"); cpl->parameter_get_default_int = dlsym(handle, "cpl_parameter_get_default_int"); cpl->parameter_get_default_string = dlsym(handle, "cpl_parameter_get_default_string"); cpl->parameter_get_enum_double = dlsym(handle, "cpl_parameter_get_enum_double"); cpl->parameter_get_enum_int = dlsym(handle, "cpl_parameter_get_enum_int"); cpl->parameter_get_enum_size = dlsym(handle, "cpl_parameter_get_enum_size"); cpl->parameter_get_enum_string = dlsym(handle, "cpl_parameter_get_enum_string"); cpl->parameter_get_help = dlsym(handle, "cpl_parameter_get_help"); cpl->parameter_get_name = dlsym(handle, "cpl_parameter_get_name"); cpl->parameter_get_range_max_double = dlsym(handle, "cpl_parameter_get_range_max_double"); cpl->parameter_get_range_max_int = dlsym(handle, "cpl_parameter_get_range_max_int"); cpl->parameter_get_range_min_double = dlsym(handle, "cpl_parameter_get_range_min_double"); cpl->parameter_get_range_min_int = dlsym(handle, "cpl_parameter_get_range_min_int"); cpl->parameter_get_type = dlsym(handle, "cpl_parameter_get_type"); cpl->parameter_set_bool = dlsym(handle, "cpl_parameter_set_bool"); cpl->parameter_set_double = dlsym(handle, "cpl_parameter_set_double"); cpl->parameter_set_int = dlsym(handle, "cpl_parameter_set_int"); cpl->parameter_set_string = dlsym(handle, "cpl_parameter_set_string"); cpl->parameter_is_enabled = dlsym(handle, "cpl_parameter_is_enabled"); cpl->parameterlist_delete = dlsym(handle, "cpl_parameterlist_delete"); cpl->parameterlist_find = dlsym(handle, "cpl_parameterlist_find"); cpl->parameterlist_get_first = dlsym(handle, "cpl_parameterlist_get_first"); cpl->parameterlist_get_next = dlsym(handle, "cpl_parameterlist_get_next"); cpl->parameterlist_get_size = dlsym(handle, "cpl_parameterlist_get_size"); cpl->recipeconfig_delete = dlsym(handle, "cpl_recipeconfig_delete"); cpl->recipeconfig_get_inputs = dlsym(handle, "cpl_recipeconfig_get_inputs"); cpl->recipeconfig_get_max_count = dlsym(handle, "cpl_recipeconfig_get_max_count"); cpl->recipeconfig_get_min_count = dlsym(handle, "cpl_recipeconfig_get_min_count"); cpl->recipeconfig_get_outputs = dlsym(handle, "cpl_recipeconfig_get_outputs"); cpl->recipeconfig_get_tags = dlsym(handle, "cpl_recipeconfig_get_tags"); error = dlerror(); if (error != NULL) { dlclose(handle); free(cpl); return NULL; } cpl->get_recipeconfig = dlsym(handle, "muse_processing_get_recipeconfig"); dlerror(); cpl->TYPE_BOOL = CPL_TYPE_BOOL; cpl->TYPE_INT = CPL_TYPE_INT; cpl->TYPE_DOUBLE = CPL_TYPE_DOUBLE; cpl->TYPE_STRING = CPL_TYPE_STRING; cpl->is_supported = UNKNOWN_VERSION; for (i = 0; supported_versions[i] != 0; i++) { if (cpl->version == supported_versions[i]) { cpl->is_supported = KNOWN_VERSION; break; } if (CPL_VERSION_MAJOR_CODE(cpl->version) == CPL_VERSION_MAJOR_CODE(supported_versions[i])) { cpl->is_supported = KNOWN_MAJOR; } } /* Between 5.3.1 and 6.0, the cpl_type enum changed. http://upstream-tracker.org/compat_reports/cpl/5.3.1_to_6.0/abi_compat_report.html#Medium_Risk_Problems for these changes; the numbers were taken from there. According to upstream-tracker, this seems to be the only relevant API change between 4.0.0 and 6.2.0. Also the cpl_size is newly introduced (former it was int), in cpl_frame *cpl_frameset_get_frame(cpl_frameset *self, cpl_size position); cpl_size cpl_frameset_get_size(const cpl_frameset *self); cpl_size cpl_parameterlist_get_size(const cpl_parameterlist *self); cpl_size cpl_recipeconfig_get_min_count(const cpl_recipeconfig* self, const char* tag, const char* input); cpl_size cpl_recipeconfig_get_max_count(const cpl_recipeconfig* self, const char* tag, const char* input); Currently, we just ignore this :-) */ if (cpl->version < CPL_VERSION(6,0,0)) { cpl->TYPE_INT = (1 << 8); cpl->TYPE_DOUBLE = (1 << 13); } libraries = realloc(libraries, sizeof(cpl_library_t *) * (i+2)); libraries[i] = cpl; libraries[i+1] = NULL; return cpl; } python-cpl-0.6.2/cpl/frames.py0000644000175000017500000001575312233701327016162 0ustar olesoles00000000000000from __future__ import absolute_import import os import tempfile try: from astropy.io import fits except: import pyfits as fits from . import md5sum class FrameConfig(object): '''Frame configuration. Each :class:`FrameConfig` object stores information about one the data type a recipe can process. They are used for defining the calibration files. However, since this information is not generally provided by CPL recipes, it contains only dummy information, except for the MUSE recipes. The objects stores a frame tag, a unique identifier for a certain kind of frame, the minimum and maximum number of frames needed. Attributes: .. attribute:: tag Category tag name. The tag name is used to distinguish between different types of files. An examples of tag names is 'MASTER_BIAS' which specifies the master bias calibration file(s). .. attribute:: min Minimal number of frames, or :obj:`None` if not specified. A frame is required if the :attr:`min` is set to a value greater than 0. .. attribute:: max Maximal number of frames, or :obj:`None` if not specified .. attribute:: frames List of frames (file names or :class:`astropy.io.fits.HDUList` objects) that are assigned to this frame type. ''' def __init__(self, tag, min_frames = 0, max_frames = 0, frames = None): self.tag = tag self.min = min_frames if min_frames > 0 else None self.max = max_frames if max_frames > 0 else None self.frames = frames self.__doc__ = self._doc() def extend_range(self, min_frames, max_frames): if self.min is not None: self.min = min(self.min, min_frames) if min_frames is not None \ else None if self.max is not None: self.max = max(self.max, max_frames) if max_frames is not None \ else None def set_range(self, min_frames, max_frames): self.min = min_frames self.max = max_frames def __str__(self): return str(self.frames) def __repr__(self): return 'FrameDef(%s, frames=%s)' % (repr(self.tag), repr(self.frames)) def _doc(self): if self.max is None or self.min is None: r = ' one frame or list of frames' elif self.max == 1: r = ' one frame' elif self.min > 1 and self.max > self.min: r = ' list of %i-%i frames' % (self.min, self.max) elif self.max > 1: r = ' one frame or list of max. %i frames' % self.max elif self.min > 1: r = ' list of min. %i frames' % self.max else: r = ' one frame or list of frames' if not self.min: r += ' (optional)' return r def __getitem__(self, i): return (self.tag, self.frames)[i] class FrameList(object): def __init__(self, recipe, other = None): self._recipe = recipe self._values = dict() if isinstance(other, self.__class__): self._set_items((o.tag, o.frames) for o in other) elif isinstance(other, dict): self._set_items(other.items()) elif other: self._set_items(other) def _set_items(self, l): for o in l: self[o[0]] = o[1] @property def _cpl_dict(self): cpl_frameconfigs = self._recipe._recipe.frameConfig() if cpl_frameconfigs is None: return None s = dict() for configs in cpl_frameconfigs: c_cfg = configs[1] for f in c_cfg: if f[0] in s: s[f[0]].extend_range(f[1], f[2]) elif f[0] in self._values: s[f[0]] = self._values[f[0]] s[f[0]].set_range(f[1], f[2]) else: s[f[0]] = FrameConfig(f[0], f[1], f[2]) self._values[f[0]] = s[f[0]] return s @property def _dict(self): return self._cpl_dict or self._values def __iter__(self): return iter(self._dict.values()) def __getitem__(self, key): return self._dict[key] def __setitem__(self, key, value): d = self._cpl_dict if d is not None: d[key].frames = value else: self._values.setdefault(key, FrameConfig(key)).frames = value def __delitem__(self, key): self._dict[key].frames = None def __contains__(self, key): return key in self._dict def __len__(self): return len(self._dict) def __getattr__(self, key): return self[key] def __setattr__(self, key, value): if key.startswith('_'): super(FrameList, self).__setattr__(key, value) else: self[key] = value def __delattr__(self, key): del self[key] def __dir__(self): return self._dict.keys() def __repr__(self): return repr(dict(self)) def __str__(self): return str(dict(self)) def __eq__(self, other): return dict(self) == other @property def __doc__(self): r = 'Frames for recipe %s.\n\nAttributes:\n' % ( self._recipe.name) for s in self: r += '%s: %s\n' % (self._key(s), s.__doc__) return r def _aslist(self, frames): flist = FrameList(self._recipe, self) if frames is not None: flist._set_items(frames.items()) return [(f.tag, f.frames) for f in flist] def mkabspath(frames, tmpdir): '''Convert all filenames in the frames list into absolute paths. :class:`astropy.io.fits.HDUList`s will be converted to temporary files located in the temporary directory tmpdir. The replacement is done in-place. The function will return the list of temporary files. param frames: :class:`list` of (tag, frame) tuples with frame being either a file name or a HDU list. param tmpdir: directory where the temporary files are being created. ''' tmpfiles = list() for i, frame in enumerate(frames): if isinstance(frame[1], fits.HDUList): md5 = md5sum.update_md5(frame[1]) filename = os.path.abspath(os.path.join(tmpdir, '%s_%s.fits' % (frame[0], md5[:8]))) try: os.remove(filename) except: pass frames[i] = ( frame[0], filename ) tmpfiles.append(filename) frame[1].writeto(filename) else: frames[i] = ( frame[0], os.path.abspath(frame[1]) ) return tmpfiles def expandframelist(frames): '''Convert a dictionary with frames into a frame list where each frame gets its own entry in the form (tag, frame) ''' framelist = list() for tag, f in frames: if isinstance(f, list) and not isinstance(f, fits.HDUList): framelist += [ (tag, frame) for frame in f ] elif f is not None: framelist.append((tag, f)) return framelist python-cpl-0.6.2/cpl/run.py0000664000175000017500000001031312131520164015471 0ustar olesoles00000000000000import cpl import sys import logging from optparse import OptionParser, OptionGroup cpl.esorex.init() for i in range(1, len(sys.argv)): if not sys.argv[i].startswith('--'): recipe = cpl.Recipe(sys.argv[i]) recipe.output_dir = '.' del sys.argv[i] break else: recipe = None parser = OptionParser(prog='%s %s' % (sys.argv[0], recipe.__name__), version = recipe.__version__, description=recipe.description[0], epilog=recipe.description[1] + '\n') if recipe else \ OptionParser(prog=sys.argv[0], version=cpl.__version__) def output_dir_callback(option, opt, value, parser): recipe.output_dir = value parser.add_option('--output-dir', action = 'callback', type='str', help = 'The directory where the product files will be written', callback = output_dir_callback) log = logging.getLogger() log.setLevel(logging.INFO) cpl.esorex.msg.level = logging.ERROR def logfile_callback(option, opt, value, parser): ch = logging.FileHandler(value) ch.setLevel(logging.DEBUG) ch.setFormatter(logging.Formatter( "%(asctime)s [%(levelname)s] %(name)s: %(message)s")) log.addHandler(ch) parser.add_option('--log-file', action = 'callback', type='str', help = 'Filename of logfile', callback = logfile_callback) def loglevel_callback(option, opt, value, parser): levels = {'debug':logging.DEBUG, 'info':logging.INFO, 'warning':logging.warning, 'error':logging.ERROR } log.setLevel(levels[value]) parser.add_option('--log-level', action = 'callback', type='str', help = 'Controls the severity level of ' 'messages that will be printed to the logfile.', callback = loglevel_callback) def msglevel_callback(option, opt, value, parser): levels = {'debug':logging.DEBUG, 'info':logging.INFO, 'warning':logging.warning, 'error':logging.ERROR } cpl.esorex.msg.level = levels[value] parser.add_option('--msg-level', action = 'callback', type='str', help = 'Controls the severity level of ' 'messages that will be printed to the terminal.', callback = msglevel_callback) def tag_callback(option, opt, value, parser): recipe.tag = value def param_callback(option, opt, value, parser, p): p.value = value def calib_callback(option, opt, value, parser, c): p.frame = value types = { bool:'string', str:'string', int:'int', float:'float' } if recipe: parser.add_option('--tag', action = 'callback', type='str', help = 'Input file tag %s' % repr(recipe.tags), callback = tag_callback) group = OptionGroup(parser, 'Recipe parameters') for p in recipe.param: group.add_option('--%s' % p.name, action = 'callback', type=types.get(p.default.__class__), help = '%s [%s]' % (p.__doc__, p.default), callback = param_callback, callback_args = (p,)) parser.add_option_group(group) if len(recipe.calib) > 0: group = OptionGroup(parser, 'Calibration frames') for f in recipe.calib: help = '' if f.min < 1: help = 'optional' elif f.min > 1: help = 'min %i' % f.min else: help = 'required' if f.max != 1: if help: help += ', ' help += 'may be repeated' group.add_option('--%s' % f.tag, action='callback', type = 'string', help = help, callback = calib_callback, callback_args = (f,)) parser.add_option_group(group) (option, args) = parser.parse_args(sys.argv) if not args[1:]: parser.print_help() sys.exit() try: recipe(args[1:]) except cpl.result.RecipeCrash as ex: log.exception(ex) print(repr(ex)) except cpl.result.CplError as ex: log.exception(ex) print(repr(ex)) python-cpl-0.6.2/cpl/cpl_api.h0000644000175000017500000002055312233701327016105 0ustar olesoles00000000000000/* This header file is compiled from the original CPL header files to contain just the functions and macros that we need in the framework. Since it is mainly copied and pasted, here is the original license statement from /usr/include/cpl.h: * Id: cpl.h,v 1.31 2009/12/02 10:29:45 lbilbao Exp * * This file is part of the ESO Common Pipeline Library * Copyright (C) 2001-2008 European Southern Observatory * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #ifndef CPL_API_H #define CPL_API_H #define CPL_VERSION(major, minor, micro) \ (((major) * 65536) + ((minor) * 256) + (micro)) #define CPL_VERSION_MAJOR_CODE(code) (((code) >> 16) & 0xff) #define CPL_VERSION_MINOR_CODE(code) (((code) >> 8) & 0xff) #define CPL_VERSION_MICRO_CODE(code) ((code) & 0xff) #define CPL_VERSION_CODE CPL_VERSION(6,3,0) typedef int cpl_error_code, cpl_errorstate, cpl_boolean, cpl_frame_group, cpl_parameter_mode, cpl_parameter_class, cpl_type, cpl_msg_severity; typedef long long cpl_size; typedef void cpl_pluginlist, cpl_frameset, cpl_frame, cpl_parameter, cpl_parameterlist, cpl_recipeconfig; typedef struct _cpl_plugin_ cpl_plugin; typedef int (*cpl_plugin_func)(cpl_plugin *); struct _cpl_plugin_ { unsigned int api; unsigned long version; unsigned long type; const char *name; const char *synopsis; const char *description; const char *author; const char *email; const char *copyright; cpl_plugin_func initialize; cpl_plugin_func execute; cpl_plugin_func deinitialize; }; struct _cpl_recipe_ { cpl_plugin interface; cpl_parameterlist *parameters; cpl_frameset *frames; }; typedef struct _cpl_recipe_ cpl_recipe; unsigned int cpl_version_get_major(void); unsigned int cpl_version_get_minor(void); unsigned int cpl_version_get_micro(void); void cpl_init(unsigned); void cpl_end(void); const char * cpl_get_description(unsigned); int cpl_memory_is_empty(void); void cpl_memory_dump(void); void cpl_free(void *); const char *cpl_plugin_get_author(const cpl_plugin *self); const char *cpl_plugin_get_copyright(const cpl_plugin *self); cpl_plugin_func cpl_plugin_get_deinit(const cpl_plugin *self); const char *cpl_plugin_get_description(const cpl_plugin *self); const char *cpl_plugin_get_email(const cpl_plugin *self); cpl_plugin_func cpl_plugin_get_exec(const cpl_plugin *self); cpl_plugin_func cpl_plugin_get_init(const cpl_plugin *self); const char *cpl_plugin_get_name(const cpl_plugin *self); const char *cpl_plugin_get_synopsis(const cpl_plugin *self); unsigned long cpl_plugin_get_version(const cpl_plugin *self); char *cpl_plugin_get_version_string(const cpl_plugin *self); void cpl_pluginlist_delete(cpl_pluginlist *); cpl_plugin *cpl_pluginlist_find(cpl_pluginlist *, const char *); cpl_plugin *cpl_pluginlist_get_first(cpl_pluginlist *); cpl_plugin *cpl_pluginlist_get_next(cpl_pluginlist *); cpl_pluginlist *cpl_pluginlist_new(void); cpl_error_code cpl_dfs_update_product_header(cpl_frameset *); void cpl_msg_error(const char *, const char *, ...); cpl_error_code cpl_error_get_code(void); const char *cpl_error_get_file(void); const char *cpl_error_get_function(void); unsigned cpl_error_get_line(void); const char *cpl_error_get_message(void); void cpl_error_reset(void); cpl_error_code cpl_error_set_message_macro(const char *, cpl_error_code, const char *, unsigned, const char *, ...); void cpl_errorstate_dump(cpl_errorstate, cpl_boolean, void (*)(unsigned, unsigned, unsigned)); cpl_errorstate cpl_errorstate_get(void); const char *cpl_frame_get_filename(const cpl_frame *self); cpl_frame_group cpl_frame_get_group(const cpl_frame *self); const char *cpl_frame_get_tag(const cpl_frame *self); cpl_frame *cpl_frame_new(void); cpl_error_code cpl_frame_set_filename(cpl_frame *self, const char *filename); cpl_error_code cpl_frame_set_tag(cpl_frame *self, const char *tag); void cpl_frameset_delete(cpl_frameset *self); cpl_frame *cpl_frameset_get_position(cpl_frameset *self, cpl_size position); cpl_size cpl_frameset_get_size(const cpl_frameset *self); cpl_error_code cpl_frameset_insert(cpl_frameset *self, cpl_frame *frame); cpl_frameset *cpl_frameset_new(void); void cpl_msg_set_level(cpl_msg_severity); cpl_error_code cpl_msg_set_log_level(cpl_msg_severity); cpl_error_code cpl_msg_set_log_name(const char *); cpl_error_code cpl_msg_stop_log(void); const char *cpl_parameter_get_alias(const cpl_parameter *self, cpl_parameter_mode mode); cpl_parameter_class cpl_parameter_get_class(const cpl_parameter *self); const char *cpl_parameter_get_context(const cpl_parameter *self); int cpl_parameter_get_default_bool(const cpl_parameter *self); int cpl_parameter_get_default_int(const cpl_parameter *self); double cpl_parameter_get_default_double(const cpl_parameter *self); const char *cpl_parameter_get_default_string(const cpl_parameter *self); int cpl_parameter_get_enum_size(const cpl_parameter *self); int cpl_parameter_get_enum_int(const cpl_parameter *self, int position); double cpl_parameter_get_enum_double(const cpl_parameter *self, int position); const char *cpl_parameter_get_enum_string(const cpl_parameter *self, int position); const char *cpl_parameter_get_help(const cpl_parameter *self); const char *cpl_parameter_get_name(const cpl_parameter *self); int cpl_parameter_get_range_min_int(const cpl_parameter *self); double cpl_parameter_get_range_min_double(const cpl_parameter *self); int cpl_parameter_get_range_max_int(const cpl_parameter *self); double cpl_parameter_get_range_max_double(const cpl_parameter *self); cpl_type cpl_parameter_get_type(const cpl_parameter *self); cpl_error_code cpl_parameter_set_bool(cpl_parameter *self, int value); cpl_error_code cpl_parameter_set_int(cpl_parameter *self, int value); cpl_error_code cpl_parameter_set_double(cpl_parameter *self, double value); cpl_error_code cpl_parameter_set_string(cpl_parameter *self, const char *value); int cpl_parameter_is_enabled(const cpl_parameter *self, cpl_parameter_mode mode); void cpl_parameterlist_delete(cpl_parameterlist *self); cpl_parameter *cpl_parameterlist_find(cpl_parameterlist *self, const char *name); cpl_parameter *cpl_parameterlist_get_first(cpl_parameterlist *self); cpl_parameter *cpl_parameterlist_get_next(cpl_parameterlist *self); cpl_size cpl_parameterlist_get_size(const cpl_parameterlist *self); void cpl_recipeconfig_delete(const cpl_recipeconfig* self); char** cpl_recipeconfig_get_inputs(const cpl_recipeconfig* self, const char* tag); cpl_size cpl_recipeconfig_get_min_count(const cpl_recipeconfig* self, const char* tag, const char* input); cpl_size cpl_recipeconfig_get_max_count(const cpl_recipeconfig* self, const char* tag, const char* input); char** cpl_recipeconfig_get_outputs(const cpl_recipeconfig* self, const char* tag); char** cpl_recipeconfig_get_tags(const cpl_recipeconfig* self); const char *cpl_version_get_version(void); #define CPL_INIT_DEFAULT 0 #define CPL_DESCRIPTION_DEFAULT 0 #define CPL_MSG_OFF 4 #define CPL_FALSE 0 #define CPL_ERROR_NONE 0 #define CPL_ERROR_FILE_NOT_CREATED 8 #define CPL_FRAME_GROUP_PRODUCT 3 #define CPL_PARAMETER_CLASS_ENUM (1 << 3) #define CPL_PARAMETER_CLASS_RANGE (1 << 2) #define CPL_PARAMETER_MODE_CLI (1 << 0) #define CPL_PARAMETER_MODE_ENV (1 << 1) #define CPL_PARAMETER_MODE_CFG (1 << 2) #define CPL_TYPE_BOOL (1 << 7) #define CPL_TYPE_DOUBLE (1 << 17) #define CPL_TYPE_INT (1 << 10) #define CPL_TYPE_STRING ((1 << 5)|(1 << 0)) #endif /* CPL_API_H */ python-cpl-0.6.2/cpl/md5sum.py0000660000175000017500000000110112131520164016066 0ustar olesoles00000000000000import os import hashlib def datamd5(hdulist): '''Calculate the MD5SUM of all data regions of a HDUList. ''' sum = hashlib.md5() for hdu in hdulist: if hdu.data is not None: sum.update(bytes(hdu.data.data)) pad = b'\0' * (2880 - hdu.data.nbytes % 2880) sum.update(pad) return sum.hexdigest() def verify_md5(hdulist): return hdulist[0].header.get('DATAMD5') == datamd5(hdulist) def update_md5(hdulist): sum = datamd5(hdulist) hdulist[0].header.update('DATAMD5', sum, 'MD5 checksum') return sum python-cpl-0.6.2/cpl/CPL_recipe.c0000664000175000017500000007777712311576522016470 0ustar olesoles00000000000000#include #include #include #include #include #include #include #include #include #ifdef linux #include #define HAVE_PRCTL #include #define HAVE_MCHECK #define HAVE_MTRACE #include #define HAVE_MALLOPT #endif /* Define PY_Type for Python <= 2.6 */ #ifndef Py_TYPE #define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) #endif #ifndef PyVarObject_HEAD_INIT #define PyVarObject_HEAD_INIT(type, size) \ PyObject_HEAD_INIT(type) size, #endif #include "CPL_library.h" #define CPL_list_doc \ "List all CPL recipe names contained in a shared library." static PyObject * CPL_list(PyObject *self, PyObject *args) { const char *file; if (!PyArg_ParseTuple(args, "s", &file)) return NULL; void *handle = dlopen(file, RTLD_LAZY); if (handle == NULL) { Py_INCREF(Py_None); return Py_None; } char *error =dlerror(); int (*cpl_plugin_get_info)(cpl_pluginlist *) = dlsym(handle, "cpl_plugin_get_info"); error = dlerror(); if (error != NULL) { dlclose(handle); Py_INCREF(Py_None); return Py_None; } cpl_library_t *cpl = create_library(file); PyObject *res = PyList_New(0); Py_INCREF(res); cpl_pluginlist *list = cpl->pluginlist_new(); (*cpl_plugin_get_info)(list); cpl_plugin *plugin; for (plugin = cpl->pluginlist_get_first(list); plugin != NULL; plugin = cpl->pluginlist_get_next(list)) { cpl->error_reset(); cpl->plugin_get_init(plugin)(plugin); char *version = cpl->plugin_get_version_string(plugin); PyList_Append(res, Py_BuildValue("sis", cpl->plugin_get_name(plugin), cpl->plugin_get_version(plugin), version)); cpl->free(version); cpl->plugin_get_deinit(plugin)(plugin); } cpl->pluginlist_delete(list); cpl->error_reset(); dlclose(handle); return res; } #define CPL_supported_versions_doc \ "List all supported CPL versions." static PyObject * CPL_supported_versions(PyObject *self, PyObject *args) { PyObject *res = PyList_New(0); Py_INCREF(res); int i; for (i = 0; supported_versions[i] != 0; i++) { PyList_Append(res, Py_BuildValue( "iii", CPL_VERSION_MAJOR_CODE(supported_versions[i]), CPL_VERSION_MINOR_CODE(supported_versions[i]), CPL_VERSION_MICRO_CODE(supported_versions[i]))); } return res; } static PyMethodDef CPL_methods[] = { {"list", CPL_list, METH_VARARGS, CPL_list_doc}, {"cpl_versions", CPL_supported_versions, METH_NOARGS, CPL_supported_versions_doc}, {NULL, NULL, 0, NULL} /* Sentinel */ }; typedef struct { PyObject_HEAD cpl_plugin *plugin; cpl_pluginlist *pluginlist; void *handle; cpl_recipeconfig *recipeconfig; cpl_library_t *cpl; } CPL_recipe; static void CPL_recipe_dealloc(CPL_recipe* self) { if (self->plugin != NULL) { self->cpl->plugin_get_deinit(self->plugin)(self->plugin); } if (self->pluginlist != NULL) { self->cpl->pluginlist_delete(self->pluginlist); } if (self->handle != NULL) { dlclose(self->handle); } Py_TYPE(self)->tp_free((PyObject*)self); } static PyObject * CPL_recipe_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { CPL_recipe *self = (CPL_recipe *)type->tp_alloc(type, 0); if (self != NULL) { self->plugin = NULL; self->pluginlist = NULL; self->handle = NULL; self->recipeconfig = NULL; self->cpl = NULL; } return (PyObject *)self; } #define CPL_recipe_doc \ "Raw CPL recipe object.\n\n" \ "Constructor parameters:\n" \ " - shared library file name\n" \ " - recipe name\n" static int CPL_recipe_init(CPL_recipe *self, PyObject *args, PyObject *kwds) { const char *file; const char *recipe; if (!PyArg_ParseTuple(args, "ss", &file, &recipe)) return -1; self->handle = dlopen(file, RTLD_LAZY); if (self->handle == NULL) { PyErr_SetString(PyExc_IOError, "cannot open shared library"); return -1; } dlerror(); int (*cpl_plugin_get_info)(cpl_pluginlist *) = dlsym(self->handle, "cpl_plugin_get_info"); char *error = dlerror(); if (error != NULL) { PyErr_SetString(PyExc_IOError, error); return -1; } self->cpl = create_library(file); self->cpl->error_reset(); self->pluginlist = self->cpl->pluginlist_new(); (*cpl_plugin_get_info)(self->pluginlist); self->plugin = self->cpl->pluginlist_find(self->pluginlist, recipe); if (self->plugin == NULL) { PyErr_SetString(PyExc_IOError, "cannot find recipe in shared library"); return -1; } else { self->cpl->plugin_get_init(self->plugin)(self->plugin); } if (self->cpl->get_recipeconfig != NULL) { self->recipeconfig = self->cpl->get_recipeconfig((cpl_recipe *)self->plugin); } else { self->recipeconfig = NULL; } return 0; } #define CPL_is_supported_doc \ "Check whether the CPL version is supported by python-cpl." static PyObject * CPL_is_supported(CPL_recipe *self) { return (self->cpl->is_supported == UNKNOWN_VERSION)?Py_False:Py_True; } #define CPL_version_doc \ "Get the CPL version string." static PyObject * CPL_version(CPL_recipe *self) { return Py_BuildValue("s", self->cpl->version_get_version()); } #define CPL_description_doc \ "Get the string of version numbers of CPL and its libraries." static PyObject * CPL_description(CPL_recipe *self) { return Py_BuildValue("s", self->cpl->get_description(CPL_DESCRIPTION_DEFAULT)); } static PyObject * getParameter(CPL_recipe *self, cpl_parameter *param) { cpl_type type = self->cpl->parameter_get_type(param); cpl_parameter_class class = self->cpl->parameter_get_class(param); const char *name = self->cpl->parameter_get_alias(param, CPL_PARAMETER_MODE_CLI); const char *fullname = self->cpl->parameter_get_name(param); const char *context = self->cpl->parameter_get_context(param); const char *help = self->cpl->parameter_get_help(param); PyObject *range = Py_None; if (class == CPL_PARAMETER_CLASS_RANGE) { if (type == self->cpl->TYPE_INT) { range = Py_BuildValue("ii", self->cpl->parameter_get_range_min_int(param), self->cpl->parameter_get_range_max_int(param)); } else if (type == self->cpl->TYPE_DOUBLE) { range = Py_BuildValue("dd", self->cpl->parameter_get_range_min_double(param), self->cpl->parameter_get_range_max_double(param)); } } Py_INCREF(range); PyObject *sequence = Py_None; if (class == CPL_PARAMETER_CLASS_ENUM) { sequence = PyList_New(0); int n_enum = self->cpl->parameter_get_enum_size(param); int i; for (i = 0; i < n_enum; i++) { if (type == self->cpl->TYPE_INT) { PyList_Append( sequence, Py_BuildValue("i", self->cpl->parameter_get_enum_int(param, i))); } else if (type == self->cpl->TYPE_DOUBLE) { PyList_Append( sequence, Py_BuildValue("d", self->cpl->parameter_get_enum_double(param, i))); } else if (type == self->cpl->TYPE_STRING) { PyList_Append( sequence, Py_BuildValue("s", self->cpl->parameter_get_enum_string(param, i))); } } } Py_INCREF(sequence); PyObject *deflt = Py_None; PyObject *ptype = Py_None; if (type == self->cpl->TYPE_BOOL) { ptype = (PyObject *)&PyBool_Type; deflt = (self->cpl->parameter_get_default_bool(param))?Py_True:Py_False; } else if (type == self->cpl->TYPE_INT) { ptype = (PyObject *)&PyLong_Type; deflt = Py_BuildValue("i", self->cpl->parameter_get_default_int(param)); } else if (type == self->cpl->TYPE_DOUBLE) { ptype = (PyObject *)&PyFloat_Type; deflt = Py_BuildValue("d", self->cpl->parameter_get_default_double(param)); } else if (type == self->cpl->TYPE_STRING) { #if PY_MAJOR_VERSION < 3 ptype = (PyObject *)&PyString_Type; #else ptype = (PyObject *)&PyUnicode_Type; #endif deflt = Py_BuildValue("s", self->cpl->parameter_get_default_string(param)); } Py_INCREF(deflt); Py_INCREF(ptype); PyObject *enabled = Py_BuildValue( "OOO", self->cpl->parameter_is_enabled(param, CPL_PARAMETER_MODE_CLI)?Py_True:Py_False, self->cpl->parameter_is_enabled(param, CPL_PARAMETER_MODE_ENV)?Py_True:Py_False, self->cpl->parameter_is_enabled(param, CPL_PARAMETER_MODE_CFG)?Py_True:Py_False); Py_INCREF(enabled); PyObject *par = Py_BuildValue("ssssNNNNN", name, context, fullname, help, range, sequence, deflt, ptype, enabled); Py_INCREF(par); return par; } #define CPL_recipe_get_params_doc \ "Get the possible parameters.\n\n" \ "Returns a list of tuples where each tuple defines one parameter:\n" \ " - parameter name\n" \ " - parameter context\n" \ " - description\n" \ " - range (min, max), if valid range is limited, or None\n" \ " - allowed values, if only certain values are allowed, or None\n" \ " - default value\n" \ " - triple (cli, env, cfg) with enabled-values for param modes" static PyObject * CPL_recipe_get_params(CPL_recipe *self) { if (self->plugin == NULL) { PyErr_SetString(PyExc_IOError, "NULL recipe"); return NULL; } cpl_parameterlist *pars = ((cpl_recipe *)self->plugin)->parameters; PyObject *res = PyList_New(0); if (pars && self->cpl->parameterlist_get_size(pars)) { cpl_parameter *param; for (param = self->cpl->parameterlist_get_first(pars); param != NULL; param = self->cpl->parameterlist_get_next(pars)) { PyList_Append(res, getParameter(self, param)); } } Py_INCREF(res); return res; } #define CPL_recipe_get_author_doc \ "Get the author and his email.\n\n" \ "Returns a pair where the first field is the author name and the\n" \ "second field is the E-mail address." static PyObject * CPL_recipe_get_author(CPL_recipe *self) { if (self->plugin == NULL) { PyErr_SetString(PyExc_IOError, "NULL recipe"); return NULL; } return Py_BuildValue("ss", self->cpl->plugin_get_author(self->plugin), self->cpl->plugin_get_email(self->plugin)); } #define CPL_recipe_get_description_doc \ "Get the synopsis and description.\n\n" \ "Returns a pair where the first field is the synopsis string and the\n" \ "second field is the description string." static PyObject * CPL_recipe_get_description(CPL_recipe *self) { if (self->plugin == NULL) { PyErr_SetString(PyExc_IOError, "NULL recipe"); return NULL; } return Py_BuildValue("ss", self->cpl->plugin_get_synopsis(self->plugin), self->cpl->plugin_get_description(self->plugin)); } #define CPL_recipe_get_version_doc \ "Get the version as integer and string.\n\n" \ "Returns a pair where the first entry is the version number as integer\n" \ "and the second entry is the version string.\n" static PyObject * CPL_recipe_get_version(CPL_recipe *self) { if (self->plugin == NULL) { PyErr_SetString(PyExc_IOError, "NULL recipe"); return NULL; } return Py_BuildValue("is", self->cpl->plugin_get_version(self->plugin), self->cpl->plugin_get_version_string(self->plugin)); } #define CPL_recipe_get_copyright_doc \ "Get the license and copyright information." static PyObject * CPL_recipe_get_copyright(CPL_recipe *self) { if (self->plugin == NULL) { PyErr_SetString(PyExc_IOError, "NULL recipe"); return NULL; } return Py_BuildValue("s", self->cpl->plugin_get_copyright(self->plugin)); } #define CPL_recipe_get_frameconfig_doc \ "Get the possible frame configurations.\n\n" \ "Returns a list of tuples. Each tupel is the frame configuration of one\n"\ "input frame tag. It consists of\n" \ " - input frame configuration (tupel with tag, minimal and maximal\n" \ " number of frames\n" \ " - list of configuration frames (each is a tupel with tag, minimal and\n"\ " maximal number of frames)\n" \ " - list of output tags\n" \ "Unset minimum/maximum values are indicated by -1" static PyObject * CPL_recipe_get_frameconfig(CPL_recipe *self) { if (self->plugin == NULL) { PyErr_SetString(PyExc_IOError, "NULL recipe"); return NULL; } if (self->recipeconfig == NULL) { Py_INCREF(Py_None); return Py_None; } PyObject *res = PyList_New(0); char **tags = self->cpl->recipeconfig_get_tags(self->recipeconfig); int i_tag; for (i_tag = 0; tags[i_tag] != NULL; i_tag++) { int min = self->cpl->recipeconfig_get_min_count(self->recipeconfig, tags[i_tag], tags[i_tag]); int max = self->cpl->recipeconfig_get_max_count(self->recipeconfig, tags[i_tag], tags[i_tag]); PyObject *raw = Py_BuildValue("sii", tags[i_tag], min, max); PyObject *calib = PyList_New(0); char **inputs = self->cpl->recipeconfig_get_inputs(self->recipeconfig, tags[i_tag]); int i_input; for (i_input = 0; inputs[i_input] != NULL; i_input++) { int min = self->cpl->recipeconfig_get_min_count(self->recipeconfig, tags[i_tag], inputs[i_input]); int max = self->cpl->recipeconfig_get_max_count(self->recipeconfig, tags[i_tag], inputs[i_input]); PyList_Append(calib, Py_BuildValue("sii", inputs[i_input], min, max)); self->cpl->free(inputs[i_input]); } self->cpl->free(inputs); PyObject *output = PyList_New(0); char **outputs = self->cpl->recipeconfig_get_outputs(self->recipeconfig, tags[i_tag]); int i_output; for (i_output = 0; outputs[i_output] != NULL; i_output++) { PyList_Append(output, Py_BuildValue("s", outputs[i_output])); self->cpl->free(outputs[i_output]); } self->cpl->free(outputs); PyList_Append(res, Py_BuildValue("OOO", raw, calib, output)); self->cpl->free(tags[i_tag]); } self->cpl->free(tags); return res; } static cpl_frameset * get_frames(CPL_recipe *self, PyObject *framelist) { cpl_frameset *frames = self->cpl->frameset_new(); PyObject *iter = PyObject_GetIter(framelist); PyObject *item; while ((item = PyIter_Next(iter))) { const char *tag; const char* file; PyArg_ParseTuple(item, "ss", &tag, &file); cpl_frame *frame = self->cpl->frame_new(); self->cpl->frame_set_filename(frame, file); self->cpl->frame_set_tag(frame, tag); self->cpl->frameset_insert(frames, frame); Py_DECREF(item); } Py_DECREF(iter); return frames; } static void clear_parameters(CPL_recipe *self, cpl_parameterlist *parameters) { cpl_parameter *par = self->cpl->parameterlist_get_first(parameters); while (par != NULL) { cpl_type type = self->cpl->parameter_get_type(par); if (type == self->cpl->TYPE_STRING) { const char *default_value = self->cpl->parameter_get_default_string(par); if (default_value == NULL) { default_value = ""; } self->cpl->parameter_set_string(par, default_value); } else if (type == self->cpl->TYPE_INT) { self->cpl->parameter_set_int(par, self->cpl->parameter_get_default_int(par)); } else if (type == self->cpl->TYPE_DOUBLE) { self->cpl->parameter_set_double(par, self->cpl->parameter_get_default_double(par)); } else if (type == self->cpl->TYPE_BOOL) { self->cpl->parameter_set_bool(par, self->cpl->parameter_get_default_bool(par)); } par = self->cpl->parameterlist_get_next(parameters); } } static void set_parameters(CPL_recipe *self, cpl_parameterlist *parameters, PyObject *parlist) { PyObject *iter = PyObject_GetIter(parlist); PyObject *item; while ((item = PyIter_Next(iter))) { const char *name; PyObject *value; PyArg_ParseTuple(item, "sO", &name, &value); cpl_parameter *par = self->cpl->parameterlist_find(parameters, name); if (par == NULL) { continue; } cpl_type type = self->cpl->parameter_get_type(par); if (type == self->cpl->TYPE_STRING) { #if PY_MAJOR_VERSION < 3 if (PyString_Check(value)) { self->cpl->parameter_set_string(par, PyString_AsString(value)); } #else if (PyBytes_Check(value)) { self->cpl->parameter_set_string(par, PyBytes_AsString(value)); } #endif } else if (type == self->cpl->TYPE_INT) { if (PyLong_Check(value)) { self->cpl->parameter_set_int(par, PyLong_AsLong(value)); } } else if (type == self->cpl->TYPE_DOUBLE) { if (PyFloat_Check(value)) { self->cpl->parameter_set_double(par, PyFloat_AsDouble(value)); } } else if (type == self->cpl->TYPE_BOOL) { self->cpl->parameter_set_bool(par, PyObject_IsTrue(value)); } Py_DECREF(item); } Py_DECREF(iter); } static void set_environment(PyObject *runenv) { PyObject *iter = PyObject_GetIter(runenv); PyObject *item; while ((item = PyIter_Next(iter))) { const char *name; PyObject *value; PyArg_ParseTuple(item, "sO", &name, &value); if ((name == NULL) || (value == NULL)) { continue; } #if PY_MAJOR_VERSION < 3 if (PyString_Check(value)) { setenv(name, PyString_AsString(value), 1); } #else if (PyBytes_Check(value)) { setenv(name, PyBytes_AsString(value), 1); } #endif if (value == Py_None) { unsetenv(name); } Py_DECREF(item); } Py_DECREF(iter); } static PyObject * exec_build_retval(void *ptr) { long ret_code = ((long *)ptr)[1]; double user_time = ((long *)ptr)[2] * 1e-6; double sys_time = ((long *)ptr)[3] * 1e-6; int memcheck = ((long *)ptr)[4]; PyObject *stats = Py_BuildValue("iffi", ret_code, user_time, sys_time, memcheck); long n_errors = ((long *)ptr)[5]; long index = 6 * sizeof(long); PyObject *errors = PyList_New(0); for (; n_errors > 0; n_errors--) { long error_code = *((long *)(ptr + index)); index += sizeof(long); long error_line = *((long *)(ptr + index)); index += sizeof(long); const char *error_msg = ptr + index; index += strlen(error_msg) + 1; const char *error_file = ptr + index; index += strlen(error_file) + 1; const char *error_func = ptr + index; index += strlen(error_func) + 1; PyList_Append(errors, Py_BuildValue("issis", error_code, error_msg, error_file, error_line, error_func)); } PyObject *frames = PyList_New(0); while (index < ((long *)ptr)[0]) { const char *tag = ptr + index; index += strlen(tag) + 1; const char *file = ptr + index; index += strlen(file) + 1; PyList_Append(frames, Py_BuildValue("ss", tag, file)); } return Py_BuildValue("OOO", frames, errors, stats); } static void *sbuffer_append_string(void *buf, const char *str) { buf = realloc(buf, ((long *)buf)[0] + strlen(str) + 1); strcpy(buf + *((long *)buf), str); *((long *)buf) += strlen(str) + 1; return buf; } static void *sbuffer_append_bytes(void *buf, const void *src, size_t nbytes) { buf = realloc(buf, ((long *)buf)[0] + nbytes); memcpy(buf + *((long *)buf), src, nbytes); *((long *)buf) += nbytes; return buf; } static void *sbuffer_append_long(void *buf, long val) { buf = realloc(buf, *((long *)buf) + sizeof(long)); *((long *)(buf + ((long *)buf)[0])) = val; *((long *)buf) += sizeof(long); return buf; } static void *serialized_error_ptr = NULL; static cpl_library_t *serialized_cpl = NULL; static void exec_serialize_one_error(unsigned self, unsigned first, unsigned last) { if (serialized_error_ptr == NULL) { serialized_error_ptr = malloc(sizeof(long)); ((long *)serialized_error_ptr)[0] = sizeof(long); serialized_error_ptr = sbuffer_append_long(serialized_error_ptr, 0); } if (serialized_cpl->error_get_code() == CPL_ERROR_NONE) { return; } ((long *)serialized_error_ptr)[1]++; // number of errors serialized_error_ptr = sbuffer_append_long(serialized_error_ptr, serialized_cpl->error_get_code()); serialized_error_ptr = sbuffer_append_long(serialized_error_ptr, serialized_cpl->error_get_line()); serialized_error_ptr = sbuffer_append_string(serialized_error_ptr, serialized_cpl->error_get_message()); serialized_error_ptr = sbuffer_append_string(serialized_error_ptr, serialized_cpl->error_get_file()); serialized_error_ptr = sbuffer_append_string(serialized_error_ptr, serialized_cpl->error_get_function()); } static void * exec_serialize_retval(CPL_recipe *self, cpl_frameset *frames, cpl_errorstate prestate, int retval, const struct tms *tms_clock) { int n_frames = self->cpl->frameset_get_size(frames); int i_frame; void *ptr = malloc(sizeof(long)); ((long *)ptr)[0] = sizeof(long); ptr = sbuffer_append_long(ptr, retval); ptr = sbuffer_append_long(ptr, 1000000L * (tms_clock->tms_utime + tms_clock->tms_cutime) / sysconf(_SC_CLK_TCK)); ptr = sbuffer_append_long(ptr, 1000000L * (tms_clock->tms_stime + tms_clock->tms_cstime) / sysconf(_SC_CLK_TCK)); ptr = sbuffer_append_long(ptr, self->cpl->memory_is_empty()); serialized_cpl = self->cpl; self->cpl->errorstate_dump(prestate, CPL_FALSE, exec_serialize_one_error); ptr = sbuffer_append_bytes(ptr, serialized_error_ptr + sizeof(long), ((long *)serialized_error_ptr)[0] - sizeof(long)); free(serialized_error_ptr); serialized_error_ptr = NULL; serialized_cpl = NULL; for (i_frame = 0; i_frame < n_frames; i_frame++) { cpl_frame *f = self->cpl->frameset_get_position(frames, i_frame); if (self->cpl->frame_get_group(f) != CPL_FRAME_GROUP_PRODUCT) { continue; } ptr = sbuffer_append_string(ptr, self->cpl->frame_get_tag(f)); ptr = sbuffer_append_string(ptr, self->cpl->frame_get_filename(f)); } return ptr; } static int do_backtrace(void) { char cmd[300]; snprintf(cmd, sizeof(cmd), "cat >> gdb_commands << EOF\n" "set height 0\nset width 0\nbt full\ninfo sources\ninfo files\n" "EOF"); int retval = system(cmd); snprintf(cmd, sizeof(cmd), "gdb -batch -x gdb_commands --pid %i --readnow >> recipe.backtrace-unprocessed 2> /dev/null", (int)getpid()); retval |= system(cmd); unlink("gdb_commands"); return retval; } #ifdef HAVE_MCHECK static void mcheck_handler(enum mcheck_status status) { char cmd[100]; snprintf(cmd, sizeof(cmd), "echo Memory corruption > recipe.backtrace-unprocessed"); int retval = system(cmd); if (retval == 0) { do_backtrace(); } abort(); } #endif static int segv_handler(int sig) { char cmd[100]; snprintf(cmd, sizeof(cmd), "echo Received signal: %i > recipe.backtrace-unprocessed", sig); int retval = system(cmd); do_backtrace(); signal(sig, SIG_DFL); return retval; } static void setup_tracing(CPL_recipe *self, int memory_trace) { #ifdef HAVE_PRCTL #ifdef PR_SET_PTRACER /* Sets the top of the process tree that is allowed to use PTRACE on the calling process */ prctl(PR_SET_PTRACER, getpid(), 0, 0, 0); #endif #ifdef PR_SET_NAME /* Set the process name for the calling process */ prctl(PR_SET_NAME, self->cpl->plugin_get_name(self->plugin), 0, 0, 0); #endif #endif #ifdef HAVE_MCHECK mcheck(mcheck_handler); #endif #ifdef HAVE_MALLOPT mallopt(M_CHECK_ACTION, 0); #endif #ifdef HAVE_MTRACE if (memory_trace) { setenv("MALLOC_TRACE", "recipe.mtrace", 1); mtrace(); } #endif typedef void (*sighandler_t)(int); signal(SIGSEGV, (sighandler_t) segv_handler); signal(SIGINT, (sighandler_t) segv_handler); signal(SIGHUP, (sighandler_t) segv_handler); signal(SIGFPE, (sighandler_t) segv_handler); signal(SIGQUIT, (sighandler_t) segv_handler); signal(SIGBUS, (sighandler_t) segv_handler); signal(SIGTERM, (sighandler_t) segv_handler); signal(SIGABRT, (sighandler_t) segv_handler); signal(SIGTERM, (sighandler_t) segv_handler); } #define CPL_recipe_exec_doc \ "Execute with parameters and frames.\n\n" \ "The parameters shall contain an iterable of (name, value) pairs\n" \ "where the values have the correct type for the parameter.\n" \ "The frames shall contain an iterable of (name, tag) pairs." static PyObject * CPL_recipe_exec(CPL_recipe *self, PyObject *args) { PyObject *parlist; PyObject *soflist; PyObject *runenv; const char *dirname; const char *logfile; int loglevel; int memory_dump; int memory_trace; if (!PyArg_ParseTuple(args, "sOOOsiii", &dirname, &parlist, &soflist, &runenv, &logfile, &loglevel, &memory_dump, &memory_trace)) return NULL; if (!PySequence_Check(parlist)) { PyErr_SetString(PyExc_TypeError, "Second parameter not a list"); return NULL; } if (!PySequence_Check(soflist)) { PyErr_SetString(PyExc_TypeError, "Third parameter not a list"); return NULL; } if (!PySequence_Check(runenv)) { PyErr_SetString(PyExc_TypeError, "Fourth parameter not a list"); return NULL; } if (self->plugin == NULL) { PyErr_SetString(PyExc_IOError, "NULL recipe"); return NULL; } self->cpl->error_reset(); cpl_recipe *recipe = (cpl_recipe *)self->plugin; self->cpl->frameset_delete(recipe->frames); recipe->frames = get_frames(self, soflist); clear_parameters(self, recipe->parameters); set_parameters(self, recipe->parameters, parlist); if (self->cpl->error_get_code() != CPL_ERROR_NONE) { PyErr_SetString(PyExc_IOError, "CPL error on inititalization"); return NULL; } int fd[2]; if (pipe(fd) == -1) { PyErr_SetString(PyExc_IOError, "Cannot pipe()"); return NULL; } pid_t childpid = fork(); if (childpid == -1) { PyErr_SetString(PyExc_IOError, "Cannot fork()"); return NULL; } if (childpid == 0) { close(fd[0]); int retval; struct tms clock_end; set_environment(runenv); self->cpl->msg_set_log_name(logfile); self->cpl->msg_set_log_level(loglevel); self->cpl->msg_set_level(CPL_MSG_OFF); cpl_errorstate prestate = self->cpl->errorstate_get(); if (chdir(dirname) == 0) { struct tms clock_start; times(&clock_start); setup_tracing(self, memory_trace); retval = self->cpl->plugin_get_exec(self->plugin)(self->plugin); int reto = self->cpl->dfs_update_product_header(recipe->frames); if (reto != CPL_ERROR_NONE) { self->cpl->msg_error (__func__, "could not update the product header"); } times(&clock_end); clock_end.tms_utime -= clock_start.tms_utime; clock_end.tms_stime -= clock_start.tms_stime; clock_end.tms_cutime -= clock_start.tms_cutime; clock_end.tms_cstime -= clock_start.tms_cstime; self->cpl->msg_stop_log(); } else { retval = CPL_ERROR_FILE_NOT_CREATED; self->cpl->error_set_message_macro(__func__, retval, __FILE__, __LINE__, " "); } void *ptr = exec_serialize_retval(self, recipe->frames, prestate, retval, &clock_end); long n_bytes = write(fd[1], ptr, ((long *)ptr)[0]); close(fd[1]); retval = (n_bytes != ((long *)ptr)[0]); free(ptr); self->cpl->frameset_delete(recipe->frames); self->cpl->parameterlist_delete(recipe->parameters); recipe->parameters = NULL; recipe->frames = NULL; self->cpl->plugin_get_deinit(self->plugin)(self->plugin); self->cpl->pluginlist_delete(self->pluginlist); Py_TYPE(self)->tp_free((PyObject*)self); if ((memory_dump > 1) || ((memory_dump > 0) && (!self->cpl->memory_is_empty()))) { self->cpl->memory_dump(); } self->cpl->end(); muntrace(); _exit(retval); } close(fd[1]); long nbytes; void *ptr = malloc(2 * sizeof(long)); Py_BEGIN_ALLOW_THREADS nbytes = read(fd[0], ptr, 2 * sizeof(long)); if (nbytes == 2 * sizeof(long)) { ptr = realloc(ptr, ((long *)ptr)[0]); nbytes += read(fd[0], ptr + 2 * sizeof(long), ((long *)ptr)[0] - 2 * sizeof(long)); } else { // broken pipe while reading first two bytes ((long *)ptr)[0] = 2 * sizeof(long); } close(fd[0]); waitpid(childpid, NULL, 0); Py_END_ALLOW_THREADS if (nbytes != ((long *)ptr)[0]) { PyErr_SetString(PyExc_IOError, "Recipe crashed"); return NULL; } PyObject *retval = exec_build_retval(ptr); free(ptr); return retval; } static PyMethodDef CPL_recipe_methods[] = { {"params", (PyCFunction)CPL_recipe_get_params, METH_NOARGS, CPL_recipe_get_params_doc}, {"author", (PyCFunction)CPL_recipe_get_author, METH_NOARGS, CPL_recipe_get_author_doc}, {"version", (PyCFunction)CPL_recipe_get_version, METH_NOARGS, CPL_recipe_get_version_doc}, {"description", (PyCFunction)CPL_recipe_get_description, METH_NOARGS, CPL_recipe_get_description_doc}, {"copyright", (PyCFunction)CPL_recipe_get_copyright, METH_NOARGS, CPL_recipe_get_copyright_doc}, {"frameConfig", (PyCFunction)CPL_recipe_get_frameconfig, METH_NOARGS, CPL_recipe_get_frameconfig_doc}, {"run", (PyCFunction)CPL_recipe_exec, METH_VARARGS, CPL_recipe_exec_doc}, {"cpl_is_supported", (PyCFunction)CPL_is_supported, METH_NOARGS, CPL_is_supported_doc}, {"cpl_version", (PyCFunction)CPL_version, METH_NOARGS, CPL_version_doc}, {"cpl_description", (PyCFunction)CPL_description, METH_NOARGS, CPL_version_doc}, {NULL, NULL, 0, NULL} /* Sentinel */ }; static PyTypeObject CPL_recipeType = { PyVarObject_HEAD_INIT(NULL, 0) "CPL_recipe.recipe", /*tp_name*/ sizeof(CPL_recipe), /*tp_basicsize*/ 0, /*tp_itemsize*/ (destructor)CPL_recipe_dealloc, /*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ 0, /*tp_compare*/ 0, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ 0, /*tp_as_mapping*/ 0, /*tp_hash */ 0, /*tp_call*/ 0, /*tp_str*/ 0, /*tp_getattro*/ 0, /*tp_setattro*/ 0, /*tp_as_buffer*/ Py_TPFLAGS_DEFAULT, /*tp_flags*/ CPL_recipe_doc, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ CPL_recipe_methods, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ (initproc)CPL_recipe_init, /* tp_init */ 0, /* tp_alloc */ CPL_recipe_new, /* tp_new */ }; #if PY_MAJOR_VERSION >= 3 PyMODINIT_FUNC PyInit_CPL_recipe(void) { static struct PyModuleDef moduledef = { PyModuleDef_HEAD_INIT, "CPL_recipe", /* m_name */ NULL, /* m_doc */ -1, /* m_size */ CPL_methods, /* m_methods */ NULL, /* m_reload */ NULL, /* m_traverse */ NULL, /* m_clear */ NULL, /* m_free */ }; CPL_recipeType.tp_new = PyType_GenericNew; if (PyType_Ready(&CPL_recipeType) < 0) { return NULL; } PyObject *m = PyModule_Create(&moduledef); Py_INCREF(&CPL_recipeType); PyModule_AddObject(m, "recipe", (PyObject *)&CPL_recipeType); return m; } #else PyMODINIT_FUNC initCPL_recipe(void) { CPL_recipeType.tp_new = PyType_GenericNew; if (PyType_Ready(&CPL_recipeType) < 0) { return; } PyObject *m = Py_InitModule3("CPL_recipe", CPL_methods, NULL); Py_INCREF(&CPL_recipeType); PyModule_AddObject(m, "recipe", (PyObject *)&CPL_recipeType); } #endif python-cpl-0.6.2/cpl/dfs.py0000640000175000017500000002373312233701327015452 0ustar olesoles00000000000000import os import sys try: from astropy.io import fits except: import pyfits as fits import cpl class ProcessingInfo(object): '''This class contains support for reading input files and parameters from the FITS header of a CPL processed file. This is done through the FITS headers that were written by the DFS function called within the processing recipe. .. attribute:: name Recipe name .. attribute:: version Recipe version string .. attribute:: pipeline Pipeline name .. attribute:: cpl_version CPL version string .. attribute:: tag Tag name .. attribute:: calib Calibration frames from a FITS file processed with CPL. The result of this function may directly set as :attr:`cpl.Recipe.calib` attribute:: import cpl myrecipe = cpl.Recipe('muse_bias') myrecipe.calib = cpl.dfs.ProcessingInfo('MASTER_BIAS_0.fits').calib .. note:: This will not work properly for files that had :class:`astropy.io.fits.HDUList` inputs since they have assigned a temporary file name only. .. attribute:: raw Raw (input) frames .. note:: This will not work properly for files that had :class:`astropy.io.fits.HDUList` inputs since they have assigned a temporary file name only. .. attribute:: param Processing parameters. The result of this function may directly set as :attr:`cpl.Recipe.param` attribute:: import cpl myrecipe = cpl.Recipe('muse_bias') myrecipe.param = cpl.dfs.ProcessingInfo('MASTER_BIAS_0.fits').param .. attribute:: md5sum MD5 sum of the data portions of the output file (header keyword 'DATAMD5'). .. attribute:: md5sums MD5 sums of the input and calibration files. :class:`dict` with the file name as key and the corresponding MD5 sum as value. .. note:: Due to a design decision in CPL, the raw input files are not accompanied with the MD5 sum. ''' def __init__(self, source, datapaths = None): ''' :param source: Object pointing to the result file header :type source: :class:`str` or :class:`astropy.io.fits.HDUList` or :class:`astropy.io.fits.PrimaryHDU` or :class:`astropy.io.fits.Header` :param datapaths: Dictionary with frame tags as keys and directory paths as values to provide a full path for the raw and calibration frames. Optional. :type datapaths: :class:`dict` ''' if isinstance(source, str): header = fits.open(source)[0].header elif isinstance(source, fits.HDUList): header = source[0].header elif isinstance(source, fits.PrimaryHDU): header = source.header elif isinstance(source, (fits.Header, dict)): header = source else: raise ValueError('Cannot assign type %s to header' % source.__class__.__name__) self.name = header['HIERARCH ESO PRO REC1 ID'] self.product = header['HIERARCH ESO PRO CATG'] self.orig_filename = header['PIPEFILE'] if datapaths and self.product in datapaths: self.orig_filename = os.path.join(datapaths[self.product], self.orig_filename) pipe_id = header.get('HIERARCH ESO PRO REC1 PIPE ID') if pipe_id: self.pipeline, version = pipe_id.split('/') num_version = 0 for i in version.split('.'): num_version = num_version * 100 + int(i) self.version = (num_version, version) else: self.pipeline = None self.version = None self.cpl_version = header.get('HIERARCH ESO PRO REC1 DRS ID') self.md5sum = header.get('DATAMD5') self.md5sums = {} self.calib = _get_rec_keys(header, 'CAL', 'CATG', 'NAME', datapaths) for cat, md5 in _get_rec_keys(header, 'CAL', 'CATG', 'DATAMD5').items(): if isinstance(md5, list): for m, f in zip(md5, self.calib[cat]): if m is not None: self.md5sums[f] = m elif md5 is not None: self.md5sums[self.calib[cat]] = md5 raw = _get_rec_keys(header, 'RAW', 'CATG', 'NAME', datapaths) if raw: self.tag = list(raw.keys())[0] self.raw = raw[self.tag] md5 = _get_rec_keys(header, 'RAW', 'CATG', 'DATAMD5')[self.tag] if isinstance(md5, list): for m, f in zip(md5, self.raw): if m is not None: self.md5sums[f] = m elif md5 is not None: self.md5sums[self.raw] = md5 else: self.tag = None self.input = None param = _get_rec_keys(header, 'PARAM', 'NAME', 'VALUE') self.param = dict() for k,v in param.items(): self.param[k] = _best_type(v) def create_recipe(self): recipe = cpl.Recipe(self.name) recipe.param = self.param recipe.calib = self.calib recipe.tag = self.tag return recipe def create_script(self, scriptfile = sys.stdout): if isinstance(scriptfile, str): scriptfile = file(scriptfile, mode='w') scriptfile.write('import cpl\n\n') scriptfile.write('# Recipe: %s.%s, Version %s, CPL version %s\n' % (self.pipeline, self.name, self.version[1], self.cpl_version)) scriptfile.write('%s = cpl.Recipe(%s, version = %s)\n' % (self.name, repr(self.name), repr(self.version[0]))) scriptfile.write('\n# Parameters:\n') for k,v in self.param.items(): scriptfile.write('%s.param.%s = %s\n' % (self.name, k, repr(v))) if self.calib: scriptfile.write('\n# Calibration frames:\n') for k,v in self.calib.items(): scriptfile.write('%s.calib.%s = %s\n' % (self.name, k, repr(v))) scriptfile.write('\n# Process input frames:\n') scriptfile.write('%s.tag = %s\n' % (self.name, repr(self.tag))) scriptfile.write('res = %s(%s)\n' % (self.name, repr(self.raw))) scriptfile.write('%s = res.%s\n' % (self.product.lower(), self.product)) scriptfile.write('%s.writeto(%s)\n' % (self.product.lower(), repr(self.orig_filename))) def printinfo(self): print('Recipe: %s, Version %s, CPL version %s ' % ( self.name, self.version, self.cpl_version)) print('Parameters:') for k,v in self.param.items(): print(' %s.%s.%s = %s' % (self.pipeline, self.name, k, v)) if self.calib: print('Calibration frames:') for k,v in self.calib.items(): if isinstance(v, str): print(' %s %s' % (v,k)) else: for n in v: print(' %s %s' % (n,k)) print('Input frames:') if isinstance(self.raw, str): print(' %s %s' % (self.raw, self.tag)) else: for n in self.raw: print(' %s %s' % (n, self.tag)) def _get_rec_keys(header, key, name, value, datapaths = None): '''Get a dictionary of key/value pairs from the DFS section of the header. :param key: Common keyword for the value. Usually 'PARAM' for parameters, 'RAW' for raw frames, and 'CAL' for calibration frames. :type key: :class:`str` :param name: Header keyword (last part) for the name of each key :type name: :class:`str` :param value: Header keyword (last part) for the value of each key :type name: :class:`str` :param datapaths: Dictionary with frame tags as keys and directory paths as values to provide a full path for the raw and calibration frames. Optional. :type datapaths: :class:`dict` When the header HIERARCH ESO PRO REC1 PARAM1 NAME = 'nifu' HIERARCH ESO PRO REC1 PARAM1 VALUE = '1' HIERARCH ESO PRO REC1 PARAM2 NAME = 'combine' HIERARCH ESO PRO REC1 PARAM2 VALUE = 'median' is called with _get_rec_keys('PARAM', 'NAME', 'VALUE') the returned dictionary will contain the keys res['nifu'] = '1' res['combine'] = 'median' ''' res = dict() for i in range(1, 2**16): try: prefix = 'HIERARCH ESO PRO REC1 %s%i' % (key, i) k = header['%s %s' % (prefix, name)] fn = header.get('%s %s' % (prefix, value)) if datapaths and k in datapaths: fn = os.path.join(datapaths[k], fn) if k not in res: res[k] = fn elif isinstance(res[k], list): res[k].append(fn) else: res[k] = [ res[k], fn ] except KeyError: break return res def _best_type(value): '''Convert the value to the best applicable type: :class:`int`, :class:`float`, :class:`bool` or :class`str`. :param value: Value to convert. :type value: :class:`str` ''' for t in int, float: try: return t(value) except ValueError: pass return {'true':True, 'false':False}.get(value, value) if __name__ == '__main__': import sys datapaths = { 'BIAS':'raw', 'DARK':'raw', 'FLAT':'raw', 'ARC':'raw', 'OBJECT':'raw', 'LINE_CATALOG':'aux', 'TRACE_TABLE':'aux', 'GEOMETRY_TABLE':'aux', 'MASTER_BIAS':'result', 'MASTER_DARK':'result', 'MASTER_FLAT':'result', 'WAVECAL_TABLE':'result', 'PIXTABLE_OBJECT':'result', } for arg in sys.argv[1:]: print('---------------------') print('file: %s' % arg) pi = ProcessingInfo(arg, datapaths = datapaths) pi.printinfo() python-cpl-0.6.2/cpl/logger.py0000660000175000017500000001174212131520164016147 0ustar olesoles00000000000000from __future__ import absolute_import import datetime import logging import os import re import sys import tempfile import threading from . import CPL_recipe class NullHandler(logging.Handler): def emit(self, record): pass logging.getLogger('cpl').addHandler(NullHandler()) level = { "DEBUG":logging.DEBUG, "INFO":logging.INFO, "WARNING":logging.WARN, "ERROR":logging.ERROR, "OFF":(logging.CRITICAL + 1)} cpl_verbosity = [ logging.DEBUG, logging.INFO, logging.WARN, logging.ERROR, logging.CRITICAL + 1 ] class LogServer(threading.Thread): def __init__(self, name, level = None): threading.Thread.__init__(self) self.name = name self.logger = logging.getLogger(name) self.level = cpl_verbosity.index(level) if level is not None else 0 self.entries = LogList() self.regexp = re.compile('(\\d\\d):(\\d\\d):(\\d\\d)' + '\\s\\[\\s*(\\w+)\\s*\\]' + '\\s(\\w+):' + '(\\s\\[tid=(\\d+)\\])?' + '\\s(.+)') tmphdl, self.logfile = tempfile.mkstemp(prefix = 'cpl', suffix='.log') os.close(tmphdl) os.remove(self.logfile) os.mkfifo(self.logfile) self.start() def run(self): try: with open(self.logfile, 'rb', buffering = 0) as logfile: line = logfile.readline() os.remove(self.logfile) while line: self.log(line) line = logfile.readline() except: pass def log(self, s): '''Convert CPL log messages into python log records. A typical CPL log message looks like 10:35:25 [WARNING] rtest: [tid=000] No file tagged with FLAT ''' try: m = self.regexp.match(s) if m is not None: g = m.groups() creation_date = datetime.datetime.combine( datetime.date.today(), datetime.time(int(g[0]),int(g[1]),int(g[2]))) lvl = level.get(g[3], logging.NOTSET) func = g[4] log = logging.getLogger('%s.%s' % (self.logger.name, func)) threadid = int(g[6]) if g[6] else None msg = g[-1] record = logging.LogRecord(log.name, lvl, None, None, msg, None, None, func) created = float(creation_date.strftime('%s')) if record.created < created: created -= 86400 record.relativeCreated -= record.msecs record.relativeCreated += 1000*(created - record.created + 1) record.created = created record.msecs = 0.0 record.threadid = threadid record.threadName = ('Cpl-%03i' % threadid) if threadid \ else 'CplThread' elif self.entries: r0 = self.entries[-1] msg = s.rstrip() lvl = r0.levelno log = logging.getLogger(r0.name) record = logging.LogRecord(r0.name, lvl, None, None, msg, None, None, r0.funcName) record.relativeCreated = r0.relativeCreated record.created = r0.created record.msecs = r0.msecs record.threadid = r0.threadid record.threadName = r0.threadName else: return self.entries.append(record) if log.isEnabledFor(lvl) and log.filter(record): log.handle(record) except: pass class LogList(list): '''List of log messages. Accessing this :class:`list` directly will return the :class:`logging.LogRecord` instances. Example:: res = muse_bias(bias_frames) for logrecord in res.log: print '%s: %s' % (entry.funcname, entry.msg) To get them formatted as string, use the :attr:`error`, :attr:`warning`, :attr:`info` or :attr:`debug` attributes:: res = muse_bias(bias_frames) for line in res.log.info: print line ''' def filter(self, level): return [ '%s: %s' % (entry.funcName, entry.msg) for entry in self if entry.levelno >= level ] @property def error(self): '''Error messages as list of :class:`str` ''' return self.filter(logging.ERROR) @property def warning(self): '''Warnings and error messages as list of :class:`str` ''' return self.filter(logging.WARN) @property def info(self): '''Info, warning and error messages as list of :class:`str` ''' return self.filter(logging.INFO) @property def debug(self): '''Debug, info, warning, and error messages as list of :class:`str` ''' return self.filter(logging.DEBUG) python-cpl-0.6.2/cpl/__init__.py0000644000175000017500000000117012233701327016430 0ustar olesoles00000000000000from __future__ import absolute_import from .version import version as __version__ from .version import author as __author__ from .version import email as __email__ from .version import license_ as __license__ from .version import doc as __doc__ from .recipe import Recipe from .param import Parameter from .frames import FrameConfig from .result import Result, CplError, RecipeCrash from . import dfs from . import esorex from . import CPL_recipe Recipe.dir = '.' cpl_versions = [ '%i.%i.%i' % ver for ver in CPL_recipe.cpl_versions() ] del CPL_recipe del absolute_import del recipe, version, param, frames, result, ver, md5sum python-cpl-0.6.2/cpl/CPL_library.h0000664000175000017500000001325712311576522016651 0ustar olesoles00000000000000 #ifndef CPL_LIBRARY_H #define CPL_LIBRARY_H /* For the header, either the CPL one can be used, or the header that was extracted from the 6.3 release. For API safety, it is better to include the one provided with python-cpl. The other option is just for the adoption to a new CPL version. */ #ifdef USE_INSTALLED_CPL_HEADER #include #else #include "cpl_api.h" #endif #if CPL_VERSION_CODE < CPL_VERSION(6,3,0) #error CPL version too old. Minimum required version is 6.3.0. #endif #if CPL_VERSION_CODE > CPL_VERSION(6,3,0) #warning Newer CPL version: check API compability with 6.3.0 at http://upstream-tracker.org/versions/cpl.html #endif extern unsigned long supported_versions[]; #define UNKNOWN_VERSION 0 #define KNOWN_MAJOR 1 #define KNOWN_VERSION 2 typedef struct { unsigned long version; int is_supported; typeof(cpl_init) *init; typeof(cpl_end) *end; typeof(cpl_get_description) *get_description; typeof(cpl_memory_dump) *memory_dump; typeof(cpl_memory_is_empty) *memory_is_empty; typeof(cpl_free) *free; typeof(cpl_plugin_get_author) *plugin_get_author; typeof(cpl_plugin_get_copyright) *plugin_get_copyright; typeof(cpl_plugin_get_deinit) *plugin_get_deinit; typeof(cpl_plugin_get_description) *plugin_get_description; typeof(cpl_plugin_get_email) *plugin_get_email; typeof(cpl_plugin_get_exec) *plugin_get_exec; typeof(cpl_plugin_get_init) *plugin_get_init; typeof(cpl_plugin_get_name) *plugin_get_name; typeof(cpl_plugin_get_synopsis) *plugin_get_synopsis; typeof(cpl_plugin_get_version) *plugin_get_version; typeof(cpl_plugin_get_version_string) *plugin_get_version_string; typeof(cpl_pluginlist_delete) *pluginlist_delete; typeof(cpl_pluginlist_find) *pluginlist_find; typeof(cpl_pluginlist_get_first) *pluginlist_get_first; typeof(cpl_pluginlist_get_next) *pluginlist_get_next; typeof(cpl_pluginlist_new) *pluginlist_new; typeof(cpl_dfs_update_product_header) *dfs_update_product_header; typeof(cpl_error_get_code) *error_get_code; typeof(cpl_error_get_file) *error_get_file; typeof(cpl_error_get_function) *error_get_function; typeof(cpl_error_get_line) *error_get_line; typeof(cpl_error_get_message) *error_get_message; typeof(cpl_error_reset) *error_reset; typeof(cpl_error_set_message_macro) *error_set_message_macro; typeof(cpl_errorstate_dump) *errorstate_dump; typeof(cpl_errorstate_get) *errorstate_get; typeof(cpl_frame_get_filename) *frame_get_filename; typeof(cpl_frame_get_group) *frame_get_group; typeof(cpl_frame_get_tag) *frame_get_tag; typeof(cpl_frame_new) *frame_new; typeof(cpl_frame_set_filename) *frame_set_filename; typeof(cpl_frame_set_tag) *frame_set_tag; typeof(cpl_frameset_delete) *frameset_delete; typeof(cpl_frameset_get_position) *frameset_get_position; typeof(cpl_frameset_get_size) *frameset_get_size; typeof(cpl_frameset_insert) *frameset_insert; typeof(cpl_frameset_new) *frameset_new; typeof(cpl_msg_error) *msg_error; typeof(cpl_msg_set_level) *msg_set_level; typeof(cpl_msg_set_log_level) *msg_set_log_level; typeof(cpl_msg_set_log_name) *msg_set_log_name; typeof(cpl_msg_stop_log) *msg_stop_log; typeof(cpl_parameter_get_alias) *parameter_get_alias; typeof(cpl_parameter_get_class) *parameter_get_class; typeof(cpl_parameter_get_context) *parameter_get_context; typeof(cpl_parameter_get_default_bool) *parameter_get_default_bool; typeof(cpl_parameter_get_default_double) *parameter_get_default_double; typeof(cpl_parameter_get_default_int) *parameter_get_default_int; typeof(cpl_parameter_get_default_string) *parameter_get_default_string; typeof(cpl_parameter_get_enum_double) *parameter_get_enum_double; typeof(cpl_parameter_get_enum_int) *parameter_get_enum_int; typeof(cpl_parameter_get_enum_size) *parameter_get_enum_size; typeof(cpl_parameter_get_enum_string) *parameter_get_enum_string; typeof(cpl_parameter_get_help) *parameter_get_help; typeof(cpl_parameter_get_name) *parameter_get_name; typeof(cpl_parameter_get_range_max_double) *parameter_get_range_max_double; typeof(cpl_parameter_get_range_max_int) *parameter_get_range_max_int; typeof(cpl_parameter_get_range_min_double) *parameter_get_range_min_double; typeof(cpl_parameter_get_range_min_int) *parameter_get_range_min_int; typeof(cpl_parameter_get_type) *parameter_get_type; typeof(cpl_parameter_set_bool) *parameter_set_bool; typeof(cpl_parameter_set_double) *parameter_set_double; typeof(cpl_parameter_set_int) *parameter_set_int; typeof(cpl_parameter_set_string) *parameter_set_string; typeof(cpl_parameter_is_enabled) *parameter_is_enabled; typeof(cpl_parameterlist_delete) *parameterlist_delete; typeof(cpl_parameterlist_find) *parameterlist_find; typeof(cpl_parameterlist_get_first) *parameterlist_get_first; typeof(cpl_parameterlist_get_next) *parameterlist_get_next; typeof(cpl_parameterlist_get_size) *parameterlist_get_size; typeof(cpl_recipeconfig_delete) *recipeconfig_delete; typeof(cpl_recipeconfig_get_inputs) *recipeconfig_get_inputs; typeof(cpl_recipeconfig_get_max_count) *recipeconfig_get_max_count; typeof(cpl_recipeconfig_get_min_count) *recipeconfig_get_min_count; typeof(cpl_recipeconfig_get_outputs) *recipeconfig_get_outputs; typeof(cpl_recipeconfig_get_tags) *recipeconfig_get_tags; typeof(cpl_version_get_version) *version_get_version; cpl_recipeconfig *(*get_recipeconfig)(cpl_recipe *); cpl_type TYPE_BOOL; cpl_type TYPE_INT; cpl_type TYPE_DOUBLE; cpl_type TYPE_STRING; } cpl_library_t; cpl_library_t *create_library(const char *fname); #endif /* CPL_LIBRARY_H */ python-cpl-0.6.2/cpl/recipe.py0000640000175000017500000005711412233701351016142 0ustar olesoles00000000000000from __future__ import absolute_import import os import shutil import tempfile import threading import collections import warnings import textwrap try: from astropy.io import fits except: import pyfits as fits from . import CPL_recipe from .frames import FrameList, mkabspath, expandframelist from .result import Result, RecipeCrash from .param import ParameterList from .logger import LogServer class Recipe(object): '''Pluggable Data Reduction Module (PDRM) from a ESO pipeline. Recipes are loaded from shared libraries that are provided with the pipeline library of the instrument. The module does not need to be linked to the same library version as the one used for the compilation of python-cpl. Currently, recipes compiled with CPL versions from 4.0 are supported. The list of supported versions is stored as :attr:`cpl.cpl_versions`. The libraries are searched in the directories specified by the class attribute :attr:`Recipe.path` or its subdirectories. The search path is automatically set to the esorex path when :func:`cpl.esorex.init()` is called. ''' path = [ '.' ] '''Search path for the recipes. It may be set to either a string, or to a list of strings. All shared libraries in the search path and their subdirectories are searched for CPL recipes. On default, the path is set to the current directory. The search path is automatically set to the esorex path when :func:`cpl.esorex.init()` is called. ''' memory_mode = 0 '''CPL memory management mode. The valid values are 0 Use the default system functions for memory handling 1 Exit if a memory-allocation fails, provide checking for memory leaks, limited reporting of memory allocation and limited protection on deallocation of invalid pointers. 2 Exit if a memory-allocation fails, provide checking for memory leaks, extended reporting of memory allocation and protection on deallocation of invalid pointers. .. note:: This variable is only effective before the CPL library was initialized. Even :func:`cpl.Recipe.list()` initializes the library. Therefore it is highly recommended to set this as the first action after importing :mod:`cpl`. ''' def __init__(self, name, filename = None, version = None, threaded = False): '''Try to load a recipe with the specified name in the directory specified by the class attribute :attr:`Recipe.path` or its subdirectories. :param name: Name of the recipe. Required. Use :func:`cpl.Recipe.list()` to get a list of available recipes. :type name: :class:`str` :param filename: Name of the shared library. Optional. If not set, :attr:`Recipe.path` is searched for the library file. :type filename: :class:`str` :param version: Version number. Optional. If not set, the newest version is loaded. :type version: :class:`int` or :class:`str` :param threaded: Run the recipe in the background, returning immediately after calling it. Default is :attr:`False`. This may be also set as an attribute or specified as a parameter when calling the recipe. :type threaded: :class:`bool` ''' os.putenv('CPL_MEMORY_MODE', str(Recipe.memory_mode)); self._recipe = None self.__name__ = name '''Recipe name.''' if not filename: filename = Recipe.get_recipefilename(name, version) if not filename: raise IOError('Recipe %s not found at path %s' % (repr(name), repr(Recipe.path))) self.__file__ = filename '''Shared library file name.''' self._recipe = CPL_recipe.recipe(filename, name) if version and version not in self.version: raise IOError('wrong version %s (requested %s) for %s in %s' % (str(self.version), str(version), name, filename)) if not self._recipe.cpl_is_supported(): warnings.warn("Unsupported CPL version %s linked to %s" % (self.cpl_version, filename)) self._param = ParameterList(self) self._calib = FrameList(self) self.env = dict() '''Bla''' self._tags = None self.tag = self.tags[0] if self.tags else None '''Default tag when the recipe is called. This is set automatically only if the recipe provided the information about input tags. Otherwise this tag has to be set manually. ''' self.output_dir = None self.temp_dir = '.' '''Base directory for temporary directories where the recipe is executed. The working dir is created as a subdir with a random file name. If set to :obj:`None`, the system temp dir is used. Defaults to :literal:`'.'`. ''' self.memory_dump = 0 self.threaded = threaded self.mtrace = False self.__doc__ = self._doc() @property def __author__(self): '''Author name''' return self._recipe.author()[0] @property def __email__(self): '''Author email''' return self._recipe.author()[1] @property def description(self): '''Pair (synopsis, description) of two strings.''' return self._recipe.description() @property def version(self): '''Pair (versionnumber, versionstring) of an integer and a string. The integer will be increased on development progress.''' return self._recipe.version() @property def __version__(self): return self._recipe.version()[1] @property def __copyright__(self): '''Copyright string of the recipe''' return self._recipe.copyright() @property def cpl_version(self): '''Version of the CPL library that is linked to the recipe, as a string''' return self._recipe.cpl_version() @property def cpl_description(self): '''Version numbers of CPL and its libraries that were linked to the recipe, as a string.''' return self._recipe.cpl_description() @property def tags(self): '''Possible tags for the raw input frames, or ':obj:`None` if this information is not provided by the recipe.''' frameconfig = self._recipe.frameConfig() return [ c[0][0] for c in frameconfig ] if frameconfig else self._tags @tags.setter def tags(self, t): if self._recipe.frameConfig(): raise AttributeError('Tags are immutable') else: self._tags = t @property def tag(self): '''Default raw input frame tag. After creation, it is set to the first tag from the "tags" property and may be changed to any of these tags. If the recipe does not provide the tag information, it must be set manually, or the tag name has to be provided when calling the recipe.''' return self._tag @tag.setter def tag(self, t): if self.tags is None or t in self.tags: self._tag = t else: raise KeyError("Tag %s not in %s" % (repr(t), str(self.tags))) @property def calib(self): '''This attribute contains the calibration frames for the recipe. It is iterable and then returns all calibration frames: >>> for f in muse_scibasic.calib: ... print f.tag, f.min, f.max, f.frames TRACE_TABLE 1 1 None WAVECAL_TABLE 1 1 None MASTER_BIAS 1 1 master_bias_0.fits MASTER_DARK None 1 None GEOMETRY_TABLE 1 1 None BADPIX_TABLE None None ['badpix_1.fits', 'badpix_2.fits'] MASTER_FLAT None 1 None .. note:: Only MUSE recipes are able to provide the full list of calibration frames and the minimal/maximal number of calibration frames. For other recipes, only frames that were set by the users are returned here. Their minimum and maximum value will be set to :obj:`None`. In order to assing a FITS file to a tag, the file name or the :class:`astropy.io.fits.HDUList` is assigned to the calibration attribute: >>> muse_scibasic.calib.MASTER_BIAS = 'MASTER_BIAS_0.fits' Using :class:`astropy.io.fits.HDUList` is useful when it needs to be patched before fed into the recipe. >>> master_bias = astropy.io.fits.open('MASTER_BIAS_0.fits') >>> master_bias[0].header['HIERARCH ESO DET CHIP1 OUT1 GAIN'] = 2.5 >>> muse_scibasic.calib.MASTER_BIAS = master_bias Note that :class:`astropy.io.fits.HDUList` objects are stored in temporary files before the recipe is called which may produce some overhead. Also, the CPL then assigns the random temporary file names to the FITS keywords ``HIERARCH ESO PRO RECm RAWn NAME`` which should be corrected afterwards if needed. To assign more than one frame, put them into a list: >>> muse_scibasic.calib.BADPIX_TABLE = [ 'badpix1.fits', 'badpix2.fits' ] All calibration frames can be set in one step by assigning a :class:`dict` to the parameters. In this case, frame that are not in the map are set are removed from the list, and unknown frame tags are silently ignored. The key of the map is the tag name; the values are either a string, or a list of strings, containing the file name(s) or the :class:`astropy.io.fits.HDUList` objects. >>> muse_scibasic.calib = { 'MASTER_BIAS':'master_bias_0.fits', ... 'BADPIX_TABLE':[ 'badpix_1.fits', 'badpix_2.fits' ] } In a recipe call, the calibration frame lists may be overwritten by specifying them in a :class:`dict`: >>> res = muse_scibasic( ..., calib = {'MASTER_BIAS':'master_bias_1.fits'}) ''' return self._calib @calib.setter def calib(self, source = None): if isinstance(source, str) or hasattr(source, 'read'): source = esorex.load_sof(source) self._calib = FrameList(self, source) @calib.deleter def calib(self): self._calib = FrameList(self, None) @property def param(self): '''This attribute contains all recipe parameters. It is iteratable and then returns all individual parameters: >>> for p in muse_scibasic.param: ... print p.name, p.value, p.default ... nifu None 99 cr None dcr xbox None 15 ybox None 40 passes None 2 thres None 4.5 sample None False dlambda None 1.2 On interactive sessions, all parameter settings can be easily printed by printing the :attr:`param` attribute of the recipe: >>> print muse_scibasic.param [Parameter('nifu', default=99), Parameter('cr', default=dcr), Parameter('xbox', default=15), Parameter('ybox', default=40), Parameter('passes', default=2), Parameter('thres', default=4.5), Parameter('sample', default=False), Parameter('dlambda', default=1.2)] To set the value of a recipe parameter, the value can be assigned to the according attribute: >>> muse_scibasic.param.nifu = 1 The new value is checked against parameter type, and possible value limitations provided by the recipe. Hyphens in parameter names are converted to underscores. In a recipe call, the same parameter can be specified as :class:`dict`: >>> res = muse_scibasic( ..., param = {'nifu':1}) To reset a value to its default, it is either deleted, or set to :obj:`None`. The following two lines: >>> muse_scibasic.param.nifu = None >>> del muse_scibasic.param.nifu will both reset the parameter to its default value. All parameters can be set in one step by assigning a :class:`dict` to the parameters. In this case, all values that are not in the map are reset to default, and unknown parameter names are ignored. The keys of the map may contain contain the name or the fullname with context: >>> muse_scibasic.param = { 'nifu':1, 'xbox':11, 'resample':True } ''' return self._param @param.setter def param(self, source = None): if isinstance(source, str) or hasattr(source, 'read'): source = esorex.load_rc(source) self._param = ParameterList(self, source) @param.deleter def param(self): self._param = ParameterList(self, None) @property def output(self): '''Return a dictionary of output frame tags. Keys are the tag names, values are the corresponding list of output tags. If the recipe does not provide this information, an exception is raised. ''' return dict((c[0][0], c[2]) for c in self._recipe.frameConfig()) def __call__(self, *data, **ndata): '''Call the recipes execution with a certain input frame. :param raw: Data input frames. :type raw: :class:`astropy.io.fits.HDUlist` or :class:`str` or a :class:`list` of them, or :class:`dict` :param tag: Overwrite the :attr:`tag` attribute (optional). :type tag: :class:`str` :param threaded: overwrite the :attr:`threaded` attribute (optional). :type threaded: :class:`bool` :param loglevel: set the log level for python :mod:`logging` (optional). :type loglevel: :class:`int` :param logname: set the log name for the python :class:`logging.Logger` (optional, default is 'cpl.' + recipename). :type logname: :class:`str` :param output_dir: Set or overwrite the :attr:`output_dir` attribute. (optional) :type output_dir: :class:`str` :param param: overwrite the CPL parameters of the recipe specified as keys with their dictionary values (optional). :type param: :class:`dict` :param calib: Overwrite the calibration frame lists for the tags specified as keys with their dictionary values (optional). :type calib: :class:`dict` :param env: overwrite environment variables for the recipe call (optional). :type env: :class:`dict` :return: The object with the return frames as :class:`astropy.io.fits.HDUList` objects :rtype: :class:`cpl.Result` :raise: :exc:`exceptions.ValueError` If the invocation parameters are incorrect. :raise: :exc:`exceptions.IOError` If the temporary directory could not be built, the recipe could not start or the files could not be read/written. :raise: :exc:`cpl.CplError` If the recipe returns an error. :raise: :exc:`cpl.RecipeCrash` If the CPL recipe crashes with a SIGSEV or a SIGBUS .. note:: If the recipe is executed in the background (``threaded = True``) and an exception occurs, this exception is raised whenever result fields are accessed. ''' tmpfiles = [] threaded = ndata.get('threaded', self.threaded) mtrace = ndata.get('mtrace', self.mtrace) loglevel = ndata.get('loglevel') logname = ndata.get('logname', 'cpl.%s' % self.__name__) output_dir = ndata.get('output_dir', self.output_dir) output_format = str if output_dir else fits.HDUList if output_dir is None: output_dir = tempfile.mkdtemp(dir = self.temp_dir, prefix = self.__name__ + "-") parlist = self.param._aslist(ndata.get('param')) raw_frames = self._get_raw_frames(*data, **ndata) if len(raw_frames) < 1: raise ValueError('No raw frames specified.') input_len = -1 if isinstance(raw_frames[0][1], fits.HDUList) else \ len(raw_frames[0][1]) if isinstance(raw_frames[0][1], list) else -1 calib_frames = self.calib._aslist(ndata.get('calib')) framelist = expandframelist(raw_frames + calib_frames) runenv = dict(self.env) runenv.update(ndata.get('env', dict())) logger = None delete = output_format == fits.HDUList try: if (not os.access(output_dir, os.F_OK)): os.makedirs(output_dir) mkabspath(framelist, output_dir) logger = LogServer(logname, loglevel) except: try: self._cleanup(output_dir, logger, delete) except: pass raise if not threaded: return self._exec(output_dir, parlist, framelist, runenv, input_len, logger, output_format, delete, mtrace) else: return Threaded( self._exec, output_dir, parlist, framelist, runenv, input_len, logger, output_format, delete, mtrace) def _exec(self, output_dir, parlist, framelist, runenv, input_len, logger, output_format, delete, mtrace): try: return Result(self._recipe.frameConfig(), output_dir, self._recipe.run(output_dir, parlist, framelist, list(runenv.items()), logger.logfile, logger.level, self.memory_dump, mtrace), input_len, logger, output_format) finally: self._cleanup(output_dir, logger, delete) def _get_raw_frames(self, *data, **ndata): '''Return the input frames. Returns a :class:`list` with (tag, the input frame(s)) pairs. Note that more than one input tag is not allowed here. ''' data = list(data) if 'raw' in ndata: data.append(ndata['raw']) tag = ndata.get('tag', self.tag) m = { } for f in data: if isinstance(f, dict): m.update(f) elif tag is None: raise ValueError('No raw input tag') elif tag not in m: m[tag] = f elif isinstance(m[tag], list) \ and not isinstance(m[tag], fits.HDUList): m[tag].append(f) else: m[tag] = [ m[tag], f ] return list(m.items()) def _cleanup(self, output_dir, logger, delete): try: bt = os.path.join(output_dir, 'recipe.backtrace-unprocessed') if os.path.exists(bt): with open(bt) as bt_file: os.rename(bt, os.path.join(output_dir, 'recipe.backtrace')) ex = RecipeCrash(bt_file) ex.log(logger.logger) raise ex finally: if delete: shutil.rmtree(output_dir) def _doc(self): s = '%s\n\n%s\n\n' % (textwrap.fill(self.description[0]), textwrap.fill(self.description[1])) if len(self.param) > 0: r = 'Parameters:\n%s\n' % self._param.__doc__ else: r = 'No parameters\n' if self._recipe.frameConfig() is not None: c = textwrap.fill(repr([f.tag for f in self.calib]), initial_indent = 'Calibration frames: ', subsequent_indent = ' ' * 21) + '\n\n' else: c = '' if self.tags is not None: t = 'Raw and product frames:\n' maxlen = max(len(f) for f in self.tags) for f in self.tags: t += textwrap.fill(repr(self.output[f]), initial_indent = ' %s --> ' % f.rjust(maxlen), subsequent_indent = ' ' * (maxlen + 7)) + '\n' else: t = '' return s + r + c + t + '\n\n' def __repr__(self): return 'Recipe(%s, version = %s)' % (repr(self.__name__), repr(self.version[0])) @staticmethod def list(): '''Return a list of recipes. Searches for all recipes in in the directory specified by the class attribute :attr:`Recipe.path` or its subdirectories. ''' os.putenv('CPL_MEMORY_MODE', str(Recipe.memory_mode)); plugins = collections.defaultdict(list) for f in Recipe.get_libs(): plugin_f = CPL_recipe.list(f) if plugin_f: for p in plugin_f: plugins[p[0]].append(p[2]) return list(plugins.items()) @staticmethod def get_recipefilename(name, version = None): os.putenv('CPL_MEMORY_MODE', str(Recipe.memory_mode)); filename = None rversion = -1 for f in Recipe.get_libs(): plugin_f = CPL_recipe.list(f) if plugin_f: for p in plugin_f: if p[0] != name: continue if version in p[1:3]: return f if rversion < p[1]: rversion = p[1] filename = f return filename @staticmethod def get_libs(): libs = [ ] path = Recipe.path.split(':') if isinstance(Recipe.path, str) else Recipe.path for p in path: for root, dir, files in os.walk(p): libs += [ os.path.join(root, f) for f in files if f.endswith('.so') ] return libs @staticmethod def set_maxthreads(n): '''Set the maximal number of threads to be executed in parallel. .. note:: This affects only threads that are started afterwards with the ``threaded = True`` flag. .. seealso:: :ref:`parallel` ''' Threaded.set_maxthreads(n) class Threaded(threading.Thread): '''Simple threading interface. Creating this object will start the execution of func(*args, **nargs). It returns an object that has the same attribute as the function return value, if the function execution was completed. Accessing any of the attributes will cause a wait until the function execution is ready. Note that the attribute delegation will work only for attributes (not for methods), and it will not work for attributes defined by the threading.Thread interface. If the function returns an exception, this exception is thrown by any attempt to access an attribute. ''' pool_sema = threading.BoundedSemaphore(65536) def __init__(self, func, *args, **nargs): threading.Thread.__init__(self) self._func = func self._args = args self._nargs = nargs self._res = None self._exception = None self.start() def run(self): with Threaded.pool_sema: try: self._res = self._func(*self._args, **self._nargs) except Exception as exception: self._exception = exception @property def _result(self): self.join() if self._exception is None: return self._res else: raise self._exception def __getitem__(self, key): return self._result[key] def __contains__(self, key): return key in self._result.tags def __len__(self): return len(self._result.tags) def __iter__(self): return self._result.__iter__() def __getattr__(self, name): return self._result.__dict__[name] @staticmethod def set_maxthreads(n): with Threaded.pool_sema: Threaded.pool_sema = threading.BoundedSemaphore(n) python-cpl-0.6.2/cpl/esorex.py0000644000175000017500000002141212263770313016203 0ustar olesoles00000000000000'''`EsoRex `_ is a standard execution environment for CPL recipes provided by `ESO `_. ''' import os import logging from .recipe import Recipe from . import logger def load_sof(source): '''Read an :program:`EsoRex` SOF file. :param source: SOF ("Set Of Files") file object or string with SOF file content. :type source: :class:`str` or :class:`file` These files contain the raw and calibration files for a recipe. The content of the file is returned as a map with the tag as key and the list of file names as value. The result of this function may directly set as :attr:`cpl.Recipe.calib` attribute:: import cpl myrecipe = cpl.Recipe('muse_bias') myrecipe.calib = cpl.esorex.read_sof(open('muse_bias.sof')) .. note:: The raw data frame is silently ignored wenn setting :attr:`cpl.Recipe.calib` for MUSE recipes. Other recipes ignore the raw data frame only if it was set manually as :attr:`cpl.Recipe.tag` or in :attr:`cpl.Recipe.tags` since there is no way to automatically distinguish between them. ''' if isinstance(source, str): return load_sof(open(source) if os.path.exists(source) else source.split('\n')) else: res = dict() for line in source: if not line or line.startswith('#'): continue ls = line.split() fn = ls[0] key = ls[1] if key not in res: res[key] = fn elif isinstance(res[key], list): res[key].append(fn) else: res[key] = [ res[key], fn ] return res def load_rc(source = None): '''Read an :program:`EsoRex` configuration file. :param source: Configuration file object, or string with file content. If not set, the :program:`EsoRex` config file :file:`~/.esorex/esorex.rc` is used. :type source: :class:`str` or :class:`file` These files contain configuration parameters for :program:`EsoRex` or recipes. The content of the file is returned as a map with the (full) parameter name as key and its setting as string value. The result of this function may directly set as :attr:`cpl.Recipe.param` attribute:: import cpl myrecipe = cpl.Recipe('muse_bias') myrecipe.param = cpl.esorex.load_rc('muse_bias.rc') ''' if source is None: source = open(os.path.expanduser('~/.esorex/esorex.rc')) if isinstance(source, str): return load_rc(open(source) if os.path.exists(source) else source.split('\n')) else: res = dict() for line in source: if not line or not line.strip() or line.startswith('#'): continue name = line.split('=', 1)[0] value = line.split('=', 1)[1] if name and value: res[name.strip()] = value.strip() return res def init(source = None): '''Set up the logging and the recipe search path from the :file:`esorex.rc` file. :param source: Configuration file object, or string with file content. If not set, the esorex config file :file:`~/.esorex/esorex.rc` is used. :type source: :class:`str` or :class:`file` ''' rc = load_rc(source) if 'esorex.caller.recipe-dir' in rc: Recipe.path = rc['esorex.caller.recipe-dir'].split(':') if 'esorex.caller.msg-level' in rc: msg.level = rc['esorex.caller.msg-level'] if 'esorex.caller.log-level' in rc: log.level = rc['esorex.caller.log-level'] if 'esorex.caller.log-dir' in rc: log.dir = rc['esorex.caller.log-dir'] if 'esorex.caller.log-file' in rc: log.filename = rc['esorex.caller.log-file'] class CplLogger(object): DEBUG = logging.DEBUG INFO = logging.INFO WARN = logging.WARN ERROR = logging.ERROR OFF = logging.CRITICAL + 1 def __init__(self): self.handler = None self._component = False self._time = False self._threadid = False self.format = None self.dir = None self._level = CplLogger.OFF def _init_handler(self): if not self.handler: self.handler = logging.StreamHandler() logging.getLogger().addHandler(self.handler) self.handler.setLevel(self._level) self.handler.setFormatter(logging.Formatter(self.format, '%H:%M:%S')) def _shutdown_handler(self): if self.handler: logging.getLogger().removeHandler(self.handler) self.handler.close() self.handler = None @property def level(self): '''Log level for output to the terminal. Any of [ DEBUG, INFO, WARN, ERROR, OFF ]. ''' return self._level @level.setter def level(self, level): if isinstance(level, (str)): level = logger.level[level.upper()] if level == CplLogger.OFF: self._shutdown_handler() else: self._init_handler() logging.getLogger().setLevel(logging.DEBUG) if self.handler: self.handler.setLevel(level) self._level = level @property def format(self): '''Output format. .. seealso :: `logging.LogRecord attributes `_ Key mappings in the logging output.''' return self._format @format.setter def format(self, fmt): if fmt == None: fmt = '%(asctime)s ' if self._time else '' fmt += '[%(levelname)7s]' fmt += '[%(threadName)s] ' if self._threadid else ' ' fmt += '%(name)s: ' if self._component else '' fmt += '%(message)s' if self.handler: self.handler.setFormatter(logging.Formatter(fmt, '%H:%M:%S')) self._format = fmt @property def component(self): '''If :obj:`True`, attach the component name to output messages. ''' return self._component @component.setter def component(self, enable): self._component = enable self.format = None @property def time(self): '''If :obj:`True`, attach a time tag to output messages. ''' return self._time @time.setter def time(self, enable): self._time = enable self.format = None @property def threadid(self): '''If :obj:`True`, attach a thread tag to output messages. ''' return self._threadid @threadid.setter def threadid(self, enable): self._threadid = enable self.format = None class CplFileLogger(CplLogger): def __init__(self): CplLogger.__init__(self) self._filename = None self.threadid = True self.component = True self.time = True self.level = CplLogger.INFO def _init_handler(self): if not self.handler: if self._filename: if self.dir: fname = os.path.join(self.dir, self._filename) self.handler = logging.FileHandler(fname) else: self.handler = logging.FileHandler(self._filename) else: self.handler = None if self.handler: logging.getLogger().addHandler(self.handler) self.handler.setLevel(self._level) self.handler.setFormatter(logging.Formatter(self.format, '%H:%M:%S')) @property def filename(self): '''Log file name. ''' return self._filename @filename.setter def filename(self, name): if self._filename != name: self._shutdown_handler() self._filename = name self._init_handler() msg = CplLogger() '''This variable is a :class:`CplLogger` instance that provides a convienience stream handler similar to the terminal logging functionality of the CPL. It basically does the same as:: import logging log = logging.getLogger() log.setLevel(logging.INFO) ch = logging.StreamHandler() ch.setLevel(logging.OFF) ch.setFormatter(logging.Formatter('[%(levelname)7s] %(message)s')) log.addHandler(ch) ''' log = CplFileLogger() '''This variable is a :class:`CplFileLogger` instance that provides a convienience file handler similar to the file logging functionality of the CPL. It basically does the same as:: import logging log = logging.getLogger() log.setLevel(logging.INFO) ch = logging.FileHandler(filename) ch.setLevel(logging.INFO) ch.setFormatter(logging.Formatter('%(asctime)s [%(levelname)7s] %(funcName)s: %(message)s')) log.addHandler(ch) ''' python-cpl-0.6.2/cpl/param.py0000664000175000017500000002065012236171243016000 0ustar olesoles00000000000000import textwrap class Parameter(object): '''Runtime configuration parameter of a recipe. Parameters are designed to handle monitor/control data and they provide a standard way to pass information to the recipe. The CPL implementation supports three classes of parameters: a plain value, a value within a given range, or a value as part of an enumeration. When a parameter is created it is created for a particular value type. In the latter two cases, validation is performed whenever the value is set. Attributes: .. attribute:: Parameter.value The value of the parameter, or :obj:`None` if set to default .. attribute:: Parameter.default The default value of the parameter (readonly). .. attribute:: Parameter.name The parameter name (readonly). Parameter names are unique. They define the identity of a given parameter. .. attribute:: Parameter.context The parameter context (readonly). The context usually consists of the instrument name and the recipe name, separated by a dot. The context is used to associate parameters together. .. attribute: Parameter.fullname The parameter name including the context (readonly). The fullname usually consists of the parameter context and the parameter name, separated by a dot. .. attribute:: Parameter.range The numeric range of a parameter, or :obj:`None` if the parameter range is unlimited (readonly). .. attribute:: Parameter.sequence A :class:`list` of possible values for the parameter if the parameter are limited to an enumeration of possible values (readonly). The following example prints the attributes of one parameter: >>> print 'name: ', muse_scibasic.param.cr.name name: cr >>> print 'fullname:', muse_scibasic.param.cr.fullname fullname: muse.muse_scibasic.cr >>> print 'context: ', muse_scibasic.param.cr.context context: muse.muse_scibasic >>> print 'sequence:', muse_scibasic.param.cr.sequence sequence: ['dcr', 'none'] >>> print 'range: ', muse_scibasic.param.cr.range range: None >>> print 'default: ', muse_scibasic.param.cr.default default: dcr >>> print 'value: ', muse_scibasic.param.cr.value value: None ''' def __init__(self, name): self.name = name self._value = None def _set_attributes(self, context = None, fullname = None, default = None, desc = None, range_ = None, sequence = None, ptype = None, enabled = None): self.context = context self.range = range_ self.sequence = sequence self.default = default self.fullname = fullname self.type = ptype or default.__class__ self.enabled = enabled self.__doc__ = textwrap.fill("%s (%s; default: %s)" % (desc, self.type.__name__, repr(self.default))) @property def value(self): return self._value @value.setter def value(self, value): if value is not None and self.type is not None.__class__: if self.type is bool and isinstance(value, str): d = {'true':True, 'false':False, 'yes':True, 'no':False} value = d.get(value.lower(), value) value = self.type(value) if self.sequence and value not in self.sequence: raise ValueError("'%s' is not in %s" % (value, self.sequence)) if self.range and not (self.range[0] <= value <= self.range[-1]): raise ValueError("'%s' is not in range %s" % (value, self.range)) self._value = value @value.deleter def value(self): self._value = None def __str__(self): return '%s%s' % ( repr(self.value if self.value is not None else self.default), ' (default)' if self.value is None else '') def __repr__(self): return 'Parameter(%s, %s=%s)' % ( repr(self.name), "value" if self.value is not None else "default", repr(self.value if self.value is not None else self.default)) def __getitem__(self,i): return (self.name, self.value or self.default)[i] class ParameterList(object): def __init__(self, recipe, other = None, prefix = None): self._recipe = recipe self._dict = dict() self._pars = list() self._prefix = prefix childs = set() for name, context, fullname, desc, prange, \ sequence, deflt, ptype, enabled in recipe._recipe.params(): if prefix: if name.startswith(prefix + '.'): aname = name[len(prefix)+1:] else: continue else: aname = name if '.' in aname: aname = aname.split('.', 1)[0] if prefix: aname = prefix + '.' + aname childs.add(aname) else: par = Parameter(name) par._set_attributes(context, fullname, deflt, desc, prange, sequence, ptype, enabled) self._dict[name] = par self._dict[fullname] = par self._dict[self._paramname(aname)] = par self._pars.append(par) for name in childs: clist = ParameterList(recipe, prefix = name) self._dict[name] = clist for par in clist._pars: self._dict[par.name] = par self._dict[par.fullname] = par self._pars.append(par) aname = self._paramname(name) self._dict[aname] = clist if other: self._set_items(other) self.__doc__ = self._doc() def _set_items(self, other): if isinstance(other, self.__class__): l = ((o.name, o.value) for o in other) elif isinstance(other, dict): l = other.items() else: l = other for o in l: self[o[0]] = o[1] def _del_items(self): for p in self: del p.value @staticmethod def _paramname(s): for c in [ '-', ' ' ]: if isinstance(c, tuple): s = s.replace(c[0], c[1]) else: s = s.replace(c, '_') return s def __iter__(self): return self._pars.__iter__() def __getitem__(self, key): return self._dict[key] def __setitem__(self, key, value): p = self[key] if isinstance(p, self.__class__): if value is not None: p._set_items(value) else: p._del_items() else: p.value = value def __delitem__(self, key): p = self[key] if isinstance(p, self.__class__): p._del_items() else: del p.value def __str__(self): return dict(self).__str__() def __contains__(self, key): return key in self._dict def __len__(self): return len(self._pars) def __getattr__(self, key): return self[key] def __setattr__(self, key, value): if key.startswith('_'): super(ParameterList, self).__setattr__(key, value) else: self[key] = value def __delattr__(self, key): del self[key] def __dir__(self): return list(set(self._paramname(d) for d in self._dict.keys() if '.' not in d)) def __repr__(self): return repr(dict(self)) def __eq__(self, other): return dict(self) == other def _doc(self): if len(self) == 0: return 'No parameters' r = '' maxlen = max(len(p.name) for p in self) for p in self: r += textwrap.fill( p.__doc__, subsequent_indent = ' ' * (maxlen + 3), initial_indent = ' %s: ' % p.name.rjust(maxlen)) + '\n' return r def _aslist(self, par): parlist = ParameterList(self._recipe, self) if par is not None: parlist._set_items(par.items()) l = list() for param in parlist: if isinstance(param, Parameter): if param.value is not None: l.append((param.fullname, param.value)) else: l += param._aslist(par) return l python-cpl-0.6.2/cpl/result.py0000644000175000017500000003275412233701327016223 0ustar olesoles00000000000000import collections import os import signal import logging try: from astropy.io import fits except: import pyfits as fits class Result(object): def __init__(self, recipedefs, dir, res, input_len = 0, logger = None, output_format = fits.HDUList): '''Build an object containing all result frames. Calling :meth:`cpl.Recipe.__call__` returns an object that contains all result ('production') frames in attributes. All results for one tag are summarized in one attribute of the same name. If the argument `output_format` is :class:`astropy.io.fits.HDUList` (default), then the attribute content is either a :class:`astropy.io.fits.HDUList` or a class:`list` of HDU lists, depending on the recipe and the call: If the recipe produces one out put frame of a tag per input file, the attribute contains a list if the recipe was called with a list, and if the recipe was called with a single input frame, the result attribute will also contain a single input frame. If the recipe combines all input frames to one output frame, a single :class:`astropy.io.fits.HDUList` es returned, independent of the input parameters. Similarly, if the argument `output_format` is set to :class:`str`, the attribute content is either a :class:`str` or a class:`list` of :class:`str`, containing the paths of output files. In this case, removing the output files is suppressed. .. todo:: This behaviour is made on some heuristics based on the number and type of the input frames. The heuristics will go wrong if there is only one input frame, specified as a list, but the recipe tries to summarize the input. In this case, the attribute will contain a list where a single :class:`astropy.io.fits.HDUList` was expected. To solve this problem, the "MASTER" flag has to be forwarded from the (MUSE) recipe which means that it should be exported by the recipe -- this is a major change since it probably leads into a replacement of CPLs recipeconfig module by something more sophisticated. And this is not usable for non-MUSE recipes anyway. So, we will skip this to probably some distant future. ''' self.dir = os.path.abspath(dir) logger.join() if res[2][0]: raise CplError(res[2][0], res[1], logger) self.tags = set() for tag, frame in res[0]: if (output_format == fits.HDUList): # Move the file to the base dir to avoid NFS problems outframe = os.path.join( os.path.dirname(self.dir), '%s.%s' % (os.path.basename(self.dir), frame)) os.rename(os.path.join(self.dir, frame), outframe) else: outframe = os.path.join(self.dir, frame) if output_format == fits.HDUList: hdulist = fits.open(outframe, memmap = True, mode = 'update') hdulist.readall() os.remove(outframe) outframe = hdulist tag = tag if tag not in self.__dict__: self.__dict__[tag] = outframe if input_len != 1 \ else [ outframe ] self.tags.add(tag) elif isinstance(self.__dict__[tag], (fits.HDUList, str)): self.__dict__[tag] = [ self.__dict__[tag], outframe ] else: self.__dict__[tag].append(outframe) mtracefname = os.path.join(self.dir, 'recipe.mtrace') mtrace = None if os.path.exists(mtracefname): try: mtrace = os.popen("mtrace %s" % mtracefname).read(); except: mtrace = None self.stat = Stat(res[2], mtrace) self.error = CplError(res[2][0], res[1], logger) if res[1] else None self.log = logger.entries if logger else None def __getitem__(self, key): if key in self.tags: return self.__dict__[key] else: raise KeyError(key) def __contains__(self, key): return key in self.tags def __len__(self): return len(self.tags) def __iter__(self): return iter((key, self.__dict__[key]) for key in self.tags) class Stat(object): def __init__(self, stat, mtrace): self.return_code = stat[0] self.user_time = stat[1] self.sys_time = stat[2] self.memory_is_empty = { -1:None, 0:False, 1:True }[stat[3]] self.mtrace = mtrace; class CplError(Exception): '''Error message from the recipe. If the CPL recipe invocation returns an error, it is converted into a :class:`cpl.CplError` exception and no frames are returned. Also, the error is notified in the log file. The exception is raised on recipe invocation, or when accessing the result frames if the recipe was started in background (:attr:`cpl.Recipe.threaded` set to :obj:`True`). Attributes: .. attribute:: code The CPL error code returned from the recipe. .. attribute:: msg The supplied error message. .. attribute:: filename The source file name where the error occurred. .. attribute:: line The line number where the error occurred. .. attribute:: log Log lines of the recipe that lead to this exception. .. seealso:: :class:`cpl.logger.LogList` .. attribute:: next Next error, or :obj:`None`. ''' def __init__(self, retval, res, logger = None): self.retval = retval self.log = logger.entries if logger else None self.next = None if not res: self.code, self.msg, self.file, self.line, self.function = ( None, None, None, None, None) else: self.code, self.msg, self.file, self.line, self.function = res[0] o = self for r in res[1:]: o.next = CplError(retval, [ r ], logger) o = o.next def __iter__(self): class Iter: current = self def next(self): if Iter.current is None: raise StopIteration s = Iter.current Iter.current = Iter.current.next return s return Iter() def __str__(self): if self.code is None: s = 'Unspecified' else: s = "%s (%i) in %s() (%s:%i)" % (self.msg, self.code, self.function, self.file, self.line) if self.next: for e in self.next: s += "\n %s (%i) in %s() (%s:%i)" % (e.msg, e.code, e.function, e.file, e.line) return s class RecipeCrash(Exception): '''Recipe crash exception If the CPL recipe crashes with a SIGSEV or a SIGBUS, the C stack trace is tried to conserved in this exception. The stack trace is obtained with the GNU debugger gdb. If the debugger is not available, or if the debugger cannot be attached to the crashed recipe, the Exception remains empty. When converted to a string, the Exception will return a stack trace similar to the Python stack trace. The exception is raised on recipe invocation, or when accessing the result frames if the recipe was started in background (:attr:`cpl.Recipe.threaded` set to :obj:`True`). Attributes: .. attribute:: elements List of stack elements, with the most recent element (the one that caused the crash) at the end. Each stack element is a :func:`collections.namedtuple` with the following attributes: .. attribute:: filename Source file name, including full path, if available. .. attribute:: line Line number, if available .. attribute:: func Function name, if available .. attribute:: params Dictionary parameters the function was called with. The key here is the parameter name, the value is a string describing the value set. .. attribute:: localvars Dictionary of local variables of the function, if available. The key here is the parameter name, the value is a string describing the value set. .. attribute:: signal Signal that caused the crash. ''' StackElement = collections.namedtuple('StackElement', 'filename line func params localvars') signals = {signal.SIGSEGV:'SIGSEV: Segmentation Fault', signal.SIGBUS:'SIGBUS: Bus Error', signal.SIGHUP:'SIGHUP: Hangup', signal.SIGABRT:'SIGABRT: Abnormal process termination', signal.SIGTERM:'SIGTERM: Terminated by user', signal.SIGQUIT:'SIGQUIT: Quit', signal.SIGFPE:'SIGFPE: Arithmetic Exception', signal.SIGINT:'SIGINT: Interrupt (Ctrl-C)', None:'Memory inconsistency detected'} def __init__(self, bt_file): self.elements = [] current_element = None parse_functions = True parse_sourcelist = False sourcefiles = dict() self.signal = None self.lines = [] for line in bt_file: self.lines.append(line) if line.startswith('Received signal:'): self.signal = int(line.split(':')[1]) if line.startswith('Memory corruption'): self.signal = None elif line.find('signal handler called') >= 0: del self.elements[:] elif parse_functions: if line.startswith('#'): try: current_element = self._parse_function_line(line) except StopIteration: parse_functions = False elif current_element is not None: self._add_variable(current_element.localvars, line) if line.startswith('Source files'): parse_sourcelist = True parse_functions = False elif parse_sourcelist: sourcefiles.update(dict((os.path.basename(s.strip()), s.strip()) for s in line.split(',') if s.rfind('/') > 0 )) self.elements = [ RecipeCrash.StackElement(sourcefiles.get(e.filename, e.filename), e.line, e.func, e.params, e.localvars) for e in self.elements ] Exception.__init__(self, str(self)) def _add_variable(self, vars, line): s = line.strip().split('=', 1) if len(s) > 1: vars[s[0].strip()] = s[1].strip() def _parse_function_line(self, line): s = line.split() funcname = s[3] if s[1].startswith('0x') else s[1] if funcname.startswith('Py'): raise StopIteration() pars = {} for fp in line[line.find('(')+1:line.rfind(')')].split(','): self._add_variable(pars, fp) l = line[line.rfind(')')+1:].split() if not l: return None source = l[-1].split(':') filename = source[0] lineno = int(source[1]) if len(source) > 1 else None current_element = RecipeCrash.StackElement(filename, lineno, funcname, pars, {}) self.elements.insert(0, current_element) return current_element def log(self, logger): '''Put the content of the crash into the log. ''' log = logging.getLogger('%s' % logger.name) log.error('Recipe crashed. Traceback (most recent call last):') for e in self.elements: logc = logging.getLogger('%s.%s' % (logger.name, e.func)) logc.error(' File "%s", %sin %s\n' % ( e.filename, 'line %i, ' % e.line if e.line else '', e.func)) if os.path.exists(e.filename) and e.line: logc.error(' %s\n' % file(e.filename).readlines()[e.line-1].strip()) if e.params: logc.error(' Parameters:') for p, v in e.params.items(): logc.error(' %s = %s' % (p, v)) if e.localvars: logc.error(' Local variables:') for p, v in e.localvars.items(): logc.error(' %s = %s' % (p, v)) log.error(RecipeCrash.signals.get(self.signal, '%s: Unknown' % str(self.signal))) def __repr__(self): return 'RecipeCrash()' def __str__(self): s = 'Recipe Traceback (most recent call last):\n' for e in self.elements: s += ' File "%s", %sin %s\n' % ((e.filename), 'line %i, ' % e.line if e.line else '', e.func) if os.path.exists(e.filename) and e.line: s += ' %s\n' % file(e.filename).readlines()[e.line-1].strip() s += RecipeCrash.signals.get(self.signal, '%s: Unknown' % str(self.signal)) return s python-cpl-0.6.2/cpl/version.py0000664000175000017500000000175412312010042016351 0ustar olesoles00000000000000version = '0.6.2' author = 'Ole Streicher' email = 'python-cpl@liska.ath.cx' license_ = 'GPL' doc = 'Python interface for the Common Pipeline Library.\nThis module can list, configure and execute CPL-based recipes from Python.\nThe input, calibration and output data can be specified as FITS files\nor as astropy.io.fits objects in memory.\n\nThe Common Pipeline Library (CPL) comprises a set of ISO-C libraries that\nprovide a comprehensive, efficient and robust software toolkit. It forms a\nbasis for the creation of automated astronomical data-reduction tasks. One of\nthe features provided by the CPL is the ability to create data-reduction\nalgorithms that run as plugins (dynamic libraries). These are called "recipes"\nand are one of the main aspects of the CPL data-reduction development\nenvironment. More information about the CPL can be found here:\n\n http://www.eso.org/sci/software/cpl/\n\nThe interface may be used to run ESO pipeline recipes linked to CPL \nversions 4.0 to 6.3.1.\n\n' python-cpl-0.6.2/README0000664000175000017500000000220612235657603014435 0ustar olesoles00000000000000Python interface for the Common Pipeline Library. This module can list, configure and execute CPL-based recipes from Python. The input, calibration and output data can be specified as FITS files or as astropy.io.fits objects in memory. The Common Pipeline Library (CPL) comprises a set of ISO-C libraries that provide a comprehensive, efficient and robust software toolkit. It forms a basis for the creation of automated astronomical data-reduction tasks. One of the features provided by the CPL is the ability to create data-reduction algorithms that run as plugins (dynamic libraries). These are called "recipes" and are one of the main aspects of the CPL data-reduction development environment. More information about the CPL can be found here: http://www.eso.org/sci/software/cpl/ The interface may be used to run ESO pipeline recipes linked to CPL versions 4.0 to 6.3.1. Build instructions ------------------ python-cpl requires: - Python 2.6 or later - astropy or pyfits python-cpl uses the standard Python distutils system to build and install itself. From the command line run:: python setup.py install to install python-cpl. python-cpl-0.6.2/setup.py0000664000175000017500000000404712312007724015261 0ustar olesoles00000000000000import os from distutils.core import setup, Extension from pkg_resources import require, DistributionNotFound author = 'Ole Streicher' email = 'python-cpl@liska.ath.cx' license_ = 'GPL' cpl_version = '0.6.2' with open('README') as readme: description = readme.read().splitlines() long_description = "\n".join(description[2:]) description = description[0] doc = '%s\n%s' % (description, long_description[:long_description.find('Build instructions')]) pkgname = 'python-cpl' baseurl = 'http://pypi.python.org/packages/source/%s/%s' % (pkgname[0], pkgname) classifiers = '''Development Status :: 4 - Beta Intended Audience :: Science/Research License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+) Operating System :: MacOS :: MacOS X Operating System :: POSIX Operating System :: Unix Programming Language :: C Programming Language :: Python Programming Language :: Python :: 2 Topic :: Scientific/Engineering :: Astronomy '''.splitlines() def create_version_file(cpl_version = cpl_version): vfile = open(os.path.join('cpl', 'version.py'), 'w') vfile.write("version = %s\n" % repr(cpl_version)) vfile.write("author = %s\n" % repr(author)) vfile.write("email = %s\n" % repr(email)) vfile.write("license_ = %s\n" % repr(license_)) vfile.write("doc = %s\n" % repr(doc)) vfile.close() create_version_file() module1 = Extension('cpl.CPL_recipe', sources = ['cpl/CPL_recipe.c', 'cpl/CPL_library.c']) try: require('astropy') required='astropy' except DistributionNotFound: required = 'pyfits' setup( name = pkgname, version = cpl_version, author = author, author_email = email, description = description, long_description = long_description, license = license_, url = 'https://pypi.python.org/pypi/%s/%s' % (pkgname, cpl_version), download_url = '%s/%s-%s.tar.gz' % (baseurl, pkgname, cpl_version), classifiers = classifiers, requires = [ required ], provides = ['cpl'], packages = ['cpl'], ext_modules = [module1] )