python-cpl-0.7.4/0000755000175000017500000000000013370121236013175 5ustar oleole00000000000000python-cpl-0.7.4/PKG-INFO0000644000175000017500000000336513370121236014301 0ustar oleole00000000000000Metadata-Version: 1.2 Name: python-cpl Version: 0.7.4 Summary: Python interface for the ESO Common Pipeline Library Home-page: https://pypi.org/project/python-cpl/0.7.4 Author: Ole Streicher Author-email: python-cpl@liska.ath.cx License: GPL Download-URL: https://files.pythonhosted.org/packages/source/p/python-cpl/python-cpl-0.7.4.tar.gz/python-cpl-0.7.4.tar.gz Description: This module can list, configure and execute CPL-based recipes from Python (python2 and python3). The input, calibration and output data can be specified as FITS files or as ``astropy.io.fits`` objects in memory. The ESO `Common Pipeline Library `_ (CPL) comprises a set of ISO-C libraries that provide a comprehensive, efficient and robust software toolkit. It forms a basis for the creation of automated astronomical data-reduction tasks. One of the features provided by the CPL is the ability to create data-reduction algorithms that run as plugins (dynamic libraries). These are called "recipes" and are one of the main aspects of the CPL data-reduction development environment. Platform: UNKNOWN Classifier: Development Status :: 4 - Beta Classifier: Intended Audience :: Science/Research Classifier: License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+) Classifier: Operating System :: MacOS :: MacOS X Classifier: Operating System :: POSIX Classifier: Operating System :: Unix Classifier: Programming Language :: C Classifier: Programming Language :: Python Classifier: Programming Language :: Python :: 2 Classifier: Programming Language :: Python :: 3 Classifier: Topic :: Scientific/Engineering :: Astronomy Provides: cpl Requires-Python: >=2.7 python-cpl-0.7.4/test/0000755000175000017500000000000013370121236014154 5ustar oleole00000000000000python-cpl-0.7.4/test/TestRecipe.py0000644000175000017500000012316713370120012016576 0ustar oleole00000000000000import logging import os import shutil import tempfile import unittest import numpy from astropy.io import fits import cpl cpl.Recipe.memory_mode = 0 recipe_name = 'rtest' raw_tag = 'RRRECIPE_DOCATG_RAW' rtest_esorexrc = '''# example RC file iiinstrument.rtest.string_option = more iiinstrument.rtest.bool_option = False iiinstrument.rtest.float_option = 1.125 iiinstrument.rtest.int_option = -2 iiinstrument.rtest.enum_option = second iiinstrument.rtest.range_option = 0.5 iiinstrument.rtest.dotted.opt = 1 ''' def create_recipe(name, builddir): var = { 'CC': os.getenv("CC", "gcc"), 'CPPFLAGS': os.getenv("CPPFLAGS", ""), 'CFLAGS': os.getenv("CFLAGS", ""), 'LDFLAGS': os.getenv("LDFLAGS", ""), 'LIBS': "-lcplcore -lcpldfs", 'cname': os.path.join(os.path.dirname(__file__), name + ".c"), 'oname': os.path.join(builddir, name + '.o'), 'soname': os.path.join(builddir, name + '.so'), } os.system("{CC} {CPPFLAGS} {CFLAGS} -fPIC -c -o {oname} {cname}".format(**var)) os.system("{CC} {LDFLAGS} -shared -o {soname} {oname} {LIBS}".format(**var)) os.remove(var['oname']) class CplTestCase(unittest.TestCase): def setUp(self): unittest.TestCase.setUp(self) self.temp_dir = tempfile.mkdtemp() create_recipe(recipe_name, self.temp_dir) cpl.Recipe.path = self.temp_dir def tearDown(self): unittest.TestCase.tearDown(self) shutil.rmtree(self.temp_dir) class RecipeTestCase(CplTestCase): def setUp(self): CplTestCase.setUp(self) self.recipe = cpl.Recipe(recipe_name) self.recipe.temp_dir = self.temp_dir self.recipe.tag = raw_tag self.image_size = (16, 16) self.raw_frame = fits.HDUList([ fits.PrimaryHDU(numpy.random.randint(0, 65001, self.image_size))]) self.raw_frame[0].header['HIERARCH ESO DET DIT'] = 0.0 self.raw_frame[0].header['HIERARCH ESO PRO CATG'] = raw_tag class RecipeStatic(CplTestCase): def test_list(self): '''List available recipes''' l = cpl.Recipe.list() self.assertTrue(isinstance(l, list)) self.assertEqual(len(l), 1) self.assertEqual(l[0], (recipe_name, ['0.0.1'])) def test_create_recipe(self): '''Create a recipe specified by its name''' recipe = cpl.Recipe(recipe_name) self.assertTrue(isinstance(recipe, cpl.Recipe)) def test_create_recipe_version(self): '''Create a recipe specified by its name and version''' recipe = cpl.Recipe(recipe_name, version = '0.0.1') self.assertTrue(isinstance(recipe, cpl.Recipe)) def test_create_recipe_wrong_name(self): '''Create a recipe specified by a wrong name''' self.assertRaises(IOError, cpl.Recipe, 'wrongname') def test_create_recipe_wrong_version(self): '''Create a recipe specified by a wrong version''' self.assertRaises(IOError, cpl.Recipe, recipe_name, version='0.0.10') def test_create_recipe_filename(self): '''Create a recipe specified by a the name and the filename''' recipe = cpl.Recipe(recipe_name, filename = os.path.join(self.temp_dir, 'rtest.so')) self.assertTrue(isinstance(recipe, cpl.Recipe)) def test_create_recipe_wrong_filename(self): '''Create a recipe specified by a wrong filename''' self.assertRaises(IOError, cpl.Recipe, recipe_name, filename = os.path.join( os.path.dirname(os.path.abspath(__file__)), 'iiinstrumentp', 'recipes', '.libs', 'rtest.o')) class RecipeCommon(RecipeTestCase): def test_name(self): '''Recipe name''' self.assertEqual(self.recipe.__name__, recipe_name) def test_author(self): '''Author attribute''' self.assertEqual(self.recipe.__author__, 'Ole Streicher') def test_email(self): '''Author attribute''' self.assertEqual(self.recipe.__email__, 'python-cpl@liska.ath.cx') def test_description(self): '''Synopsis and description''' self.assertTrue(isinstance(self.recipe.description[0], str)) self.assertTrue(len(self.recipe.description[0]) > 0) self.assertTrue(isinstance(self.recipe.description[1], str)) self.assertTrue(len(self.recipe.description[1]) > 0) def test_copyright(self): '''Copyright''' self.assertTrue(isinstance(self.recipe.__copyright__, str)) self.assertTrue(len(self.recipe.__copyright__) > 0) class RecipeParams(RecipeTestCase): def test_string_parameter(self): '''String parameter''' self.assertTrue(isinstance(self.recipe.param.stropt, cpl.Parameter)) self.assertEqual(self.recipe.param.stropt.name, 'stropt') self.assertEqual(self.recipe.param.stropt.context, 'iiinstrument.rtest') self.assertEqual(self.recipe.param.stropt.default, None) self.assertEqual(self.recipe.param.stropt.value, None) self.assertEqual(self.recipe.param.stropt.range, None) self.assertEqual(self.recipe.param.stropt.sequence, None) self.recipe.param.stropt = 'more' self.assertEqual(self.recipe.param.stropt.value, 'more') del self.recipe.param.stropt self.assertEqual(self.recipe.param.stropt.value, None) def test_boolean_parameter(self): '''Boolean parameter''' self.assertTrue(isinstance(self.recipe.param.boolopt, cpl.Parameter)) self.assertEqual(self.recipe.param.boolopt.name, 'boolopt') self.assertEqual(self.recipe.param.boolopt.default, True) self.assertEqual(self.recipe.param.boolopt.value, None) self.recipe.param.boolopt = False self.assertEqual(self.recipe.param.boolopt.value, False) del self.recipe.param.boolopt self.assertEqual(self.recipe.param.boolopt.value, None) def test_float_parameter(self): '''Float parameter''' self.assertTrue(isinstance(self.recipe.param.floatopt, cpl.Parameter)) self.assertEqual(self.recipe.param.floatopt.name, 'floatopt') self.assertEqual(self.recipe.param.floatopt.default, 0.1) self.assertEqual(self.recipe.param.floatopt.value, None) self.recipe.param.floatopt = 1.1 self.assertEqual(self.recipe.param.floatopt.value, 1.1) del self.recipe.param.floatopt self.assertEqual(self.recipe.param.floatopt.value, None) def test_int_parameter(self): '''Integer parameter''' self.assertTrue(isinstance(self.recipe.param.intopt, cpl.Parameter)) self.assertEqual(self.recipe.param.intopt.name, 'intopt') self.assertEqual(self.recipe.param.intopt.default, 2) self.assertEqual(self.recipe.param.intopt.value, None) self.recipe.param.intopt = -1 self.assertEqual(self.recipe.param.intopt.value, -1) del self.recipe.param.intopt self.assertEqual(self.recipe.param.intopt.value, None) def test_enum_parameter(self): '''Enumeration (string) parameter''' self.assertTrue(isinstance(self.recipe.param.enumopt, cpl.Parameter)) self.assertEqual(self.recipe.param.enumopt.name, 'enumopt') self.assertEqual(self.recipe.param.enumopt.default, 'first') self.assertEqual(self.recipe.param.enumopt.value, None) self.recipe.param.enumopt = 'second' self.assertEqual(self.recipe.param.enumopt.value, 'second') del self.recipe.param.enumopt self.assertEqual(self.recipe.param.enumopt.value, None) def setenumoptinvalid(): self.recipe.param.enumopt = 'invalid' self.assertRaises(ValueError, setenumoptinvalid) def test_range_parameter(self): '''Range (float) parameter''' self.assertTrue(isinstance(self.recipe.param.rangeopt, cpl.Parameter)) self.assertEqual(self.recipe.param.rangeopt.name, 'rangeopt') self.assertEqual(self.recipe.param.rangeopt.default, 0.1) self.assertEqual(self.recipe.param.rangeopt.value, None) self.recipe.param.rangeopt = 0.4 self.assertEqual(self.recipe.param.rangeopt.value, 0.4) del self.recipe.param.rangeopt self.assertEqual(self.recipe.param.rangeopt.value, None) def setrangeoptinvalid(): self.recipe.param.rangeopt = 1.5 self.assertRaises(ValueError, setrangeoptinvalid) def test_as_dict(self): '''Use the parameter list as a dictionary''' self.assertEqual(self.recipe.param.boolopt, self.recipe.param['boolopt']) self.assertEqual(self.recipe.param.boolopt, self.recipe.param['iiinstrument.rtest.bool_option']) def test_dotted_par(self): '''Use a parameter that has a dot in its alias''' self.assertEqual(self.recipe.param.dot.opt, self.recipe.param.dot['opt']) self.assertEqual(self.recipe.param.dot.opt, self.recipe.param['dot.opt']) self.assertEqual(self.recipe.param.dot.opt, self.recipe.param['iiinstrument.rtest.dotted.opt']) def test_iterate(self): '''Iteration over all parameters''' for p in self.recipe.param: self.assertTrue(isinstance(p, cpl.Parameter)) pars = [p.name for p in self.recipe.param] self.assertEqual(len(pars), len(self.recipe.param)) self.assertTrue('stropt' in pars) self.assertTrue('boolopt' in pars) def test_set_dict(self): '''Assign a dictionary to the parameter list''' self.recipe.param = { 'stropt':'dmore', 'boolopt':True } self.assertEqual(self.recipe.param.boolopt.value, True) self.assertEqual(self.recipe.param.stropt.value, 'dmore') # Check that we can assign a dictionary with the short names and string self.recipe.param = { 'stropt':'dmore', 'boolopt':'False' } self.assertEqual(self.recipe.param.boolopt.value, False) # Check that we can assign a dictionary with the long names self.recipe.param = { 'iiinstrument.rtest.string_option':'dless', 'iiinstrument.rtest.float_option':1.5, 'iiinstrument.rtest.bool_option':True } self.assertEqual(self.recipe.param.stropt.value, 'dless') self.assertEqual(self.recipe.param.floatopt.value, 1.5) self.assertEqual(self.recipe.param.boolopt.value, True) def test_param_load_from_esorexrc(self): '''Load parameters from an Esorex .rc file''' self.recipe.param = rtest_esorexrc self.assertEqual(self.recipe.param.stropt.value, 'more') self.assertEqual(self.recipe.param.boolopt.value, False) self.assertEqual(self.recipe.param.floatopt.value, 1.125) self.assertEqual(self.recipe.param.intopt.value, -2) self.assertEqual(self.recipe.param.enumopt.value, 'second') self.assertEqual(self.recipe.param.rangeopt.value, 0.5) self.assertEqual(self.recipe.param.dot.opt.value, 1) def test_delete(self): '''Delete all parameter values to reset to default''' self.recipe.param.boolopt.value = True self.recipe.param.stropt.value = 'something' del self.recipe.param self.assertEqual(self.recipe.param.stropt.value, None) self.assertEqual(self.recipe.param.boolopt.value, None) def test_dir(self): '''[TAB] completition. This requires to have the __dir__() method working. ''' self.assertEqual(set(self.recipe.param.__dir__()), set(p.name if '.' not in p.name else p.name.split('.', 1)[0] for p in self.recipe.param )) def test_str(self): '''Parameter string representation. Since the string depends on the order of parameter, we check that it evaluates to the correct dict. ''' self.assertEqual(eval(str(self.recipe.param)), { 'stropt': None, 'boolopt': True, 'floatopt': 0.1, 'intopt': 2, 'enumopt': 'first', 'rangeopt': 0.1, 'crashing': 'no', 'memleak': False, 'sleep': 0.1, 'disabled': -0.1, 'dot.opt': 0}) def test_repr(self): '''Canonical parameter string representation. Since the string depends on the order of parameter, we check that it evaluates to the correct dict. ''' self.assertEqual(eval(repr(self.recipe.param)), { 'stropt': None, 'boolopt': True, 'floatopt': 0.1, 'intopt': 2, 'enumopt': 'first', 'rangeopt': 0.1, 'crashing': 'no', 'memleak': False, 'sleep': 0.1, 'disabled': -0.1, 'dot.opt': 0}) def test_eq(self): '''Trivial equality test''' self.assertTrue(self.recipe.param == self.recipe.param) class RecipeCalib(RecipeTestCase): def test_set(self): '''Set a calibration frame''' self.recipe.calib.FLAT = 'flat.fits' self.assertEqual(self.recipe.calib.FLAT.frames, 'flat.fits') def test_set_dict(self): '''Assign a dictionary to the calibration frame list''' self.recipe.calib = { 'FLAT':'flat2.fits' } self.assertEqual(self.recipe.calib.FLAT.frames, 'flat2.fits') def test_param_load_from_esorexsof(self): '''Load calibration from an Esorex SOF file''' self.recipe.calib = 'flat.fits FLAT\n' self.assertEqual(self.recipe.calib.FLAT.frames, 'flat.fits') def test_del(self): '''Delete a calibration frame set''' self.recipe.calib.FLAT = 'flat.fits' del self.recipe.calib.FLAT f = self.recipe.calib.FLAT.frames self.assertEqual(f, None) def test_del_all(self): '''Delete all calibration frame sets''' self.recipe.calib.FLAT = 'flat.fits' del self.recipe.calib try: f = self.recipe.calib.FLAT.frames except: f = None self.assertEqual(f, None) def test_dir(self): '''[TAB] completition. This requires to have the __dir__() method working. ''' self.assertEqual(set(self.recipe.calib.__dir__()), set(f.tag for f in self.recipe.calib)) def test_str(self): '''Calibration frames string representation''' self.assertEqual(str(self.recipe.calib), "{}") def test_repr(self): '''Canonical calibration frames string representation''' self.assertEqual(repr(self.recipe.calib), "{}") def test_eq(self): '''Trivial equality test''' self.assertTrue(self.recipe.calib == self.recipe.calib) class RecipeExec(RecipeTestCase): def setUp(self): RecipeTestCase.setUp(self) self.flat_frame = fits.HDUList([ fits.PrimaryHDU(numpy.random.randint(0, 65001, self.image_size))]) def test_frames_keyword_dict(self): '''Raw and calibration frames specified as keyword dict''' self.recipe.tag = None res = self.recipe(raw = {'RRRECIPE_DOCATG_RAW': self.raw_frame }, calib = { 'FLAT':self.flat_frame }) self.assertTrue(isinstance(res, cpl.Result)) self.assertTrue(isinstance(res.THE_PRO_CATG_VALUE, fits.HDUList)) self.assertTrue(abs(self.raw_frame[0].data - res.THE_PRO_CATG_VALUE[0].data).max() == 0) try: res.THE_PRO_CATG_VALUE.close() except: pass def test_frames_keyword_calib(self): '''Raw frame specified as keyword, calibration frame set in recipe''' self.recipe.tag = None self.recipe.calib.FLAT = self.flat_frame res = self.recipe({'RRRECIPE_DOCATG_RAW':self.raw_frame}) self.assertTrue(isinstance(res, cpl.Result)) self.assertTrue(isinstance(res.THE_PRO_CATG_VALUE, fits.HDUList)) try: res.THE_PRO_CATG_VALUE.close() except: pass def test_frames_tag_keyword(self): '''The 'tag' parameter''' self.recipe.tag = None self.recipe.calib.FLAT = self.flat_frame res = self.recipe(self.raw_frame, tag = raw_tag) self.assertTrue(isinstance(res, cpl.Result)) self.assertTrue(isinstance(res.THE_PRO_CATG_VALUE, fits.HDUList)) try: res.THE_PRO_CATG_VALUE.close() except: pass def test_frames_tag_attribute(self): '''The 'tag' attribute''' self.recipe.tag = raw_tag res = self.recipe(self.raw_frame) self.assertTrue(isinstance(res, cpl.Result)) self.assertTrue(isinstance(res.THE_PRO_CATG_VALUE, fits.HDUList)) try: res.THE_PRO_CATG_VALUE.close() except: pass def test_frames_one_element_input_list(self): '''Use 1-element list as input''' # --> we want a list back''' res = self.recipe([self.raw_frame]) self.assertTrue(isinstance(res, cpl.Result)) self.assertFalse(isinstance(res.THE_PRO_CATG_VALUE, fits.HDUList)) self.assertTrue(isinstance(res.THE_PRO_CATG_VALUE, list)) try: res.THE_PRO_CATG_VALUE[0].close() except: pass def test_frames_many_element_input_list(self): '''Use multiple files as input''' # --> since we only get back one image, it is # assumed to be a 'master', and we get back a plain frame''' res = self.recipe([self.raw_frame, self.raw_frame]) self.assertTrue(isinstance(res, cpl.Result)) self.assertTrue(isinstance(res.THE_PRO_CATG_VALUE, fits.HDUList)) try: res.THE_PRO_CATG_VALUE.close() except: pass def test_output_dir_attribute(self): '''Write an output dir specified as attribute''' output_dir = os.path.join(self.temp_dir, 'out') self.recipe.output_dir = output_dir res = self.recipe(self.raw_frame) self.assertTrue(isinstance(res, cpl.Result)) self.assertTrue(isinstance(res.THE_PRO_CATG_VALUE, str)) self.assertEqual(os.path.basename(res.THE_PRO_CATG_VALUE), 'rtest.fits') self.assertTrue(os.path.isdir(output_dir)) self.assertTrue(os.path.isfile(res.THE_PRO_CATG_VALUE)) hdu = fits.open(res.THE_PRO_CATG_VALUE) self.assertTrue(isinstance(hdu, fits.HDUList)) try: hdu.close() except: pass def test_output_dir_keyword(self): '''Write an output dir specified as call keyword arg''' output_dir = os.path.join(self.temp_dir, 'out') res = self.recipe(self.raw_frame, output_dir = output_dir) self.recipe.output_dir = output_dir res = self.recipe(self.raw_frame) self.assertTrue(os.path.isdir(output_dir)) self.assertTrue(isinstance(res, cpl.Result)) self.assertTrue(isinstance(res.THE_PRO_CATG_VALUE, str)) self.assertEqual(os.path.basename(res.THE_PRO_CATG_VALUE), 'rtest.fits') self.assertTrue(os.path.isfile(res.THE_PRO_CATG_VALUE)) hdu = fits.open(res.THE_PRO_CATG_VALUE) self.assertTrue(isinstance(hdu, fits.HDUList)) try: hdu.close() except: pass def test_param_default(self): '''Test default parameter settings''' res = self.recipe(self.raw_frame).THE_PRO_CATG_VALUE self.assertEqual(res[0].header['HIERARCH ESO QC STROPT'].strip(), self.recipe.param.stropt.default or '') self.assertEqual(res[0].header['HIERARCH ESO QC BOOLOPT'], self.recipe.param.boolopt.default) self.assertEqual(res[0].header['HIERARCH ESO QC INTOPT'], self.recipe.param.intopt.default) self.assertEqual(res[0].header['HIERARCH ESO QC FLOATOPT'], self.recipe.param.floatopt.default) self.assertEqual(res[0].header['HIERARCH ESO QC ENUMOPT'], self.recipe.param.enumopt.default) self.assertEqual(res[0].header['HIERARCH ESO QC RANGEOPT'], self.recipe.param.rangeopt.default) try: res.close() except: pass def test_param_keyword_dict(self): '''Parameter handling via keyword dict''' res = self.recipe(self.raw_frame, param = { 'stropt':'more' }).THE_PRO_CATG_VALUE self.assertEqual(res[0].header['HIERARCH ESO QC STROPT'], 'more') try: res.close() except: pass def test_param_keyword_dict_wrong(self): '''Parameter handling via keyword dict''' self.assertRaises(KeyError, self.recipe, self.raw_frame, param = { 'wrong':True }) def test_param_setting(self): '''Parameter handling via recipe setting''' self.recipe.param.stropt = 'more' with self.recipe(self.raw_frame).THE_PRO_CATG_VALUE as res: self.assertEqual(res[0].header['HIERARCH ESO QC STROPT'], 'more') def test_param_delete(self): '''Delete a parameter in a second run after setting it''' self.recipe.param.intopt = 123 with self.recipe(self.raw_frame).THE_PRO_CATG_VALUE as res: pass del self.recipe.param.intopt with self.recipe(self.raw_frame).THE_PRO_CATG_VALUE as res: self.assertEqual(res[0].header['HIERARCH ESO QC INTOPT'], 2) def test_param_overwrite(self): '''Overwrite the recipe setting param via via keyword arg''' self.recipe.param.stropt = 'more' res = self.recipe(self.raw_frame, param = {'stropt':'less'}).THE_PRO_CATG_VALUE self.assertEqual(res[0].header['HIERARCH ESO QC STROPT'], 'less') def test_param_types(self): '''Parameter types''' self.recipe.param.stropt = 'more' self.recipe.param.boolopt = False self.recipe.param.intopt = 123 self.recipe.param.floatopt = -0.25 self.recipe.param.enumopt = 'third' self.recipe.param.rangeopt = 0.125 with self.recipe(self.raw_frame).THE_PRO_CATG_VALUE as res: self.assertEqual(res[0].header['HIERARCH ESO QC STROPT'], 'more') self.assertEqual(res[0].header['HIERARCH ESO QC BOOLOPT'], False) self.assertEqual(res[0].header['HIERARCH ESO QC INTOPT'], 123) self.assertEqual(res[0].header['HIERARCH ESO QC FLOATOPT'], -0.25) self.assertEqual(res[0].header['HIERARCH ESO QC ENUMOPT'], 'third') self.assertEqual(res[0].header['HIERARCH ESO QC RANGEOPT'], 0.125) def test_disabled(self): '''Parameter with CLI disabled''' self.assertFalse(self.recipe.param.disabled.enabled[0]) self.assertTrue(self.recipe.param.intopt.enabled[0]) # self.recipe.param.disabled = 0.2 # res = self.recipe(self.raw_frame) # self.assertEqual(res[0].header['HIERARCH ESO QC DISABLED'], 0.2) # try: # res.close() # except: # pass def test_environment_setting(self): '''Additional environment parameter via recipe setting''' self.recipe.env['TESTENV'] = 'unkk' with self.recipe(self.raw_frame).THE_PRO_CATG_VALUE as res: self.assertEqual(res[0].header['HIERARCH ESO QC TESTENV'], 'unkk') def test_environment_keyword(self): '''Additional environment parameter via recipe call keyword''' with self.recipe(self.raw_frame, env = {'TESTENV':'kknu'}).THE_PRO_CATG_VALUE as res: self.assertEqual(res[0].header['HIERARCH ESO QC TESTENV'], 'kknu') def test_error(self): '''Error handling''' self.recipe.tag = 'some_unknown_tag' self.assertRaises(cpl.CplError, self.recipe, self.raw_frame) def test_parallel(self): '''Parallel execution''' results = list() for i in range(20): # mark each frame so that we can see their order self.raw_frame[0].header['HIERARCH ESO RAW1 NR'] = i results.append(self.recipe(self.raw_frame, param = {'intopt':i}, env = {'TESTENV':('knu%02i' % i)}, threaded = True)) for i, res in enumerate(results): # check if we got the correct type self.assertTrue(isinstance(res.THE_PRO_CATG_VALUE, fits.HDUList)) # check if we have the correct parameter self.assertEqual(res.THE_PRO_CATG_VALUE[0].header[ 'HIERARCH ESO QC INTOPT'], i) # check if we have the correct environment self.assertEqual(res.THE_PRO_CATG_VALUE[0].header[ 'HIERARCH ESO QC TESTENV'], ('knu%02i' % i)) # check if we have the correct input frame self.assertEqual(res.THE_PRO_CATG_VALUE[0].header[ 'HIERARCH ESO RAW1 NR'], i) # check that the data were moved correctly self.assertTrue(abs(self.raw_frame[0].data - res.THE_PRO_CATG_VALUE[0].data).max() < 1e-6) try: res.THE_PRO_CATG_VALUE.close() except: pass def test_error_parallel(self): '''Error handling in parallel execution''' self.recipe.tag = 'some_unknown_tag' res = self.recipe(self.raw_frame, threaded = True) def get(x): return x.THE_PRO_CATG_VALUE self.assertRaises(cpl.CplError, get, res) def test_md5sum_result(self): '''MD5sum of the result file''' self.recipe.tag = raw_tag res = self.recipe(self.raw_frame) key = 'DATAMD5' md5sum = res.THE_PRO_CATG_VALUE[0].header[key] try: res.THE_PRO_CATG_VALUE.close() except: pass self.assertNotEqual(md5sum, 'Not computed') self.assertEqual(len(md5sum), len('9d123996fa9a7bda315d07e063043454')) def test_md5sum_calib(self): '''Created MD5sum for a HDUList calib file''' self.recipe.tag = raw_tag self.recipe.calib.FLAT = self.flat_frame res = self.recipe(self.raw_frame) key = 'HIERARCH ESO PRO REC1 CAL1 DATAMD5' md5sum = res.THE_PRO_CATG_VALUE[0].header[key] try: res.THE_PRO_CATG_VALUE.close() except: pass self.assertNotEqual(md5sum, 'Not computed') self.assertEqual(len(md5sum), len('9d123996fa9a7bda315d07e063043454')) class RecipeCrashing(RecipeTestCase): def _test_corrupted(self): '''Handling of recipe crashes because of corrupted memory''' self.recipe.param.crashing = 'free' self.assertRaises(cpl.RecipeCrash, self.recipe, self.raw_frame) def _test_segfault(self): '''Handling of recipe crashes because of segmentation fault''' self.recipe.param.crashing = 'segfault' self.assertRaises(cpl.RecipeCrash, self.recipe, self.raw_frame) def _test_cleanup_after_crash(self): '''Test that a second run after a crash will succeed''' output_dir = os.path.join(self.temp_dir, 'out') self.recipe.output_dir = output_dir self.recipe.param.crashing = 'segfault' self.assertRaises(cpl.RecipeCrash, self.recipe, self.raw_frame) del self.recipe.param.crashing self.recipe(self.raw_frame) class RecipeRes(RecipeTestCase): def setUp(self): RecipeTestCase.setUp(self) self.res = self.recipe(self.raw_frame) def tearDown(self): RecipeTestCase.tearDown(self) try: res.THE_PRO_CATG_VALUE.close() except: pass def test_attribute(self): '''The result as an attribute''' self.assertTrue(isinstance(self.res.THE_PRO_CATG_VALUE, fits.HDUList)) def test_dict(self): '''The result as an attribute''' self.assertTrue(isinstance(self.res['THE_PRO_CATG_VALUE'], fits.HDUList)) def test_in(self): '''Check whether a tag is part of the result''' self.assertTrue('THE_PRO_CATG_VALUE' in self.res) self.assertFalse('Anothervalue' in self.res) def test_keyerror(self): '''Accessing an inexisting value''' self.assertRaises(KeyError, lambda val: self.res[val], 'Anothervalue') def test_len(self): '''Length of the result''' self.assertEqual(len(self.res), 1) def test_iter(self): '''Iterate over the result''' for tag, hdu in self.res: self.assertEqual(tag, 'THE_PRO_CATG_VALUE') self.assertTrue(isinstance(hdu, fits.HDUList)) class RecipeEsorex(CplTestCase): def tearDown(self): CplTestCase.tearDown(self) cpl.esorex.msg.level = cpl.esorex.msg.OFF cpl.esorex.log.level = cpl.esorex.msg.OFF def test_read_sof(self): '''Read a SOF file''' soffile = 'geometry_table1.fits GEOMETRY_TABLE\n' \ 'geometry_table2.fits GEOMETRY_TABLE\n' \ 'MASTER_BIAS-01.fits MASTER_BIAS\n' \ 'MASTER_FLAT-01.fits MASTER_FLAT\n' \ '#sky_fullmoon_1.fits SKY\n' \ 'sky_fullmoon_2.fits SKY\n' self.assertEqual(cpl.esorex.load_sof(soffile), { 'GEOMETRY_TABLE': ['geometry_table1.fits', 'geometry_table2.fits' ], 'MASTER_BIAS': 'MASTER_BIAS-01.fits', 'MASTER_FLAT': 'MASTER_FLAT-01.fits', 'SKY': 'sky_fullmoon_2.fits' }) def test_read_rc(self): '''Read an EsoRec .rc file''' self.assertEqual(cpl.esorex.load_rc(rtest_esorexrc), { 'iiinstrument.rtest.string_option': 'more', 'iiinstrument.rtest.bool_option': 'False', 'iiinstrument.rtest.float_option': '1.125', 'iiinstrument.rtest.int_option': '-2', 'iiinstrument.rtest.enum_option': 'second', 'iiinstrument.rtest.range_option': '0.5', 'iiinstrument.rtest.dotted.opt': '1', }) def test_esorex_init(self): '''Init CPL from an esorex.rc file''' rcfile = '''esorex.caller.recipe-dir=/some/dir esorex.caller.msg-level=debug esorex.caller.log-level=info esorex.caller.log-dir=%s esorex.caller.log-file=some.log''' % self.temp_dir cpl.esorex.init(rcfile) self.assertEqual(cpl.esorex.msg.level, cpl.esorex.msg.DEBUG) self.assertEqual(cpl.esorex.log.level, cpl.esorex.msg.INFO) self.assertEqual(cpl.esorex.log.dir, self.temp_dir) self.assertEqual(cpl.esorex.log.filename, 'some.log') self.assertEqual(cpl.Recipe.path, ['/some/dir']) def test_esorex_log(self): '''Write a logfile controlled by the convienence logger''' dirname = os.path.join(self.temp_dir, 'log') filename = 'python-cpl.log' log_msg = 'Esorex convienence log' os.mkdir(dirname) cpl.esorex.log.dir = dirname cpl.esorex.log.filename = filename cpl.esorex.log.level = cpl.esorex.log.INFO filename = os.path.join(dirname, filename) logging.getLogger('cpl').info(log_msg) self.assertTrue(os.path.exists(filename)) logfile = open(filename) log_content = logfile.read() logfile.close() self.assertTrue(log_msg in log_content) self.assertTrue('INFO' in log_content) def test_esorex_log_off(self): '''Switch the logfile off after writing something''' dirname = os.path.join(self.temp_dir, 'log') filename = 'python-cpl_off.log' log_msg = 'Esorex convienence log' os.mkdir(dirname) cpl.esorex.log.dir = dirname cpl.esorex.log.filename = 'python-cpl_debug.log' cpl.esorex.log.level = 'debug' logging.getLogger('cpl').debug(log_msg) cpl.esorex.log.filename = filename cpl.esorex.log.level = 'off' logging.getLogger('cpl').debug(log_msg) filename = os.path.join(dirname, filename) logfile = open(filename) log_content = logfile.read() logfile.close() self.assertEqual(len(log_content), 0) class RecipeLog(RecipeTestCase): def setUp(self): RecipeTestCase.setUp(self) self.handler = RecipeLog.THandler() logging.getLogger('cpl.rtest').addHandler(self.handler) self.other_handler = RecipeLog.THandler() logging.getLogger('othername').addHandler(self.other_handler) def tearDown(self): RecipeTestCase.tearDown(self) logging.getLogger('cpl.rtest').removeHandler(self.handler) logging.getLogger('othername').removeHandler(self.other_handler) class THandler(logging.Handler): def __init__(self): logging.Handler.__init__(self) self.logs = list() def emit(self, record): self.logs.append(record) def clear(self): self.logs = list() def test_logging_DEBUG(self): '''Injection of CPL messages into the python logging system''' self.handler.clear() logging.getLogger().setLevel(logging.DEBUG) res = self.recipe(self.raw_frame) try: res.THE_PRO_CATG_VALUE.close() except: pass # check that the logs are not empty self.assertNotEqual(len(self.handler.logs), 0) funcnames = set() lognames = set() for r in self.handler.logs: # Check that we saved the right class self.assertTrue(isinstance(r, logging.LogRecord)) # Check that a message was provided self.assertNotEqual(r.msg, None) # Check that a function name was provided self.assertNotEqual(r.funcName, None) funcnames.add(r.funcName) lognames.add(r.name) # Check that we had at least one expected entry self.assertTrue('cpl_dfs_product_save' in funcnames) self.assertTrue('cpl.rtest.cpl_dfs_product_save' in lognames) def test_logging_INFO(self): '''Filtering INFO messages''' self.handler.clear() logging.getLogger('cpl.rtest').setLevel(logging.INFO) res = self.recipe(self.raw_frame) try: res.THE_PRO_CATG_VALUE.close() except: pass # check that the logs are not empty self.assertNotEqual(len(self.handler.logs), 0) def test_logging_WARN(self): '''Filtering WARN messages''' self.handler.clear() logging.getLogger('cpl.rtest').setLevel(logging.WARN) res = self.recipe(self.raw_frame) try: res.THE_PRO_CATG_VALUE.close() except: pass # check that the logs are not empty self.assertNotEqual(len(self.handler.logs), 0) def test_logging_ERROR(self): '''Filtering of error messages''' # There is no error msg written by the recipe, so it should be empty. self.handler.clear() logging.getLogger('cpl.rtest').setLevel(logging.ERROR) res = self.recipe(self.raw_frame) try: res.THE_PRO_CATG_VALUE.close() except: pass self.assertEqual(len(self.handler.logs), 0) def test_logging_common(self): '''Log name specification on recipe call''' self.handler.clear() self.other_handler.clear() res = self.recipe(self.raw_frame, logname = 'othername') try: res.THE_PRO_CATG_VALUE.close() except: pass self.assertNotEqual(len(self.other_handler.logs), 0) def test_logging_multiline(self): '''Multiple lines in messages''' self.handler.clear() logging.getLogger('cpl.rtest').setLevel(logging.INFO) res = self.recipe(self.raw_frame) try: res.THE_PRO_CATG_VALUE.close() except: pass # check that the multi line log sequence appears multiline = 0 tag = 'multiline#' for l in self.handler.logs: if tag not in l.msg: continue i = int(l.msg[l.msg.index(tag)+len(tag):].split()[0]) self.assertEqual(multiline + 1, i) multiline = i self.assertEqual(multiline, 3) def test_result(self): '''"log" attribute of the result object''' res = self.recipe(self.raw_frame) # Check that we get a not-empty list back self.assertTrue(isinstance(res.log, list)) self.assertNotEqual(len(res.log), 0) self.assertTrue(isinstance(res.log[0], logging.LogRecord)) # Check that we can read debug messages self.assertNotEqual(len(res.log.debug), 0) self.assertTrue(isinstance(res.log.debug[0], str)) # Check that we can read info messages self.assertNotEqual(len(res.log.info), 0) self.assertTrue(isinstance(res.log.info[0], str)) # Check that we can read warning messages self.assertNotEqual(len(res.log.warning), 0) self.assertTrue(isinstance(res.log.warning[0], str)) # Check that there were no error messages self.assertEqual(len(res.log.error), 0) try: res.THE_PRO_CATG_VALUE.close() except: pass def test_error(self): '''"log" attribute of the CplError object''' try: self.recipe('test.fits') except cpl.CplError as r: res = r # Check that we get a not-empty list back self.assertTrue(isinstance(res.log, list)) self.assertNotEqual(len(res.log), 0) self.assertTrue(isinstance(res.log[0], logging.LogRecord)) # Check that we can read debug messages self.assertNotEqual(len(res.log.debug), 0) self.assertTrue(isinstance(res.log.debug[0], str)) # Check that we can read info messages self.assertNotEqual(len(res.log.info), 0) self.assertTrue(isinstance(res.log.info[0], str)) # Check that we can read warning messages self.assertNotEqual(len(res.log.warning), 0) self.assertTrue(isinstance(res.log.warning[0], str)) # Check that we can read error messages self.assertNotEqual(len(res.log.error), 0) self.assertTrue(isinstance(res.log.error[0], str)) # Check that we can convert the error to a string self.assertTrue(isinstance(res.__str__(), str)) # Check that we can iterate over error messages for r in res: self.assertTrue(isinstance(res, cpl.CplError)) class ProcessingInfo(RecipeTestCase): def setUp(self): RecipeTestCase.setUp(self) '''Parameter storage in the result''' self.recipe.param.stropt = 'more' self.recipe.param.boolopt = False self.recipe.param.intopt = 123 self.recipe.param.floatopt = -0.25 self.recipe.param.enumopt = 'third' self.recipe.param.rangeopt = 0.125 self.recipe.calib.FLAT = fits.HDUList([ fits.PrimaryHDU(numpy.random.randint(0, 65001, self.image_size))]) self.res = self.recipe(self.raw_frame).THE_PRO_CATG_VALUE self.pinfo = cpl.dfs.ProcessingInfo(self.res) def tearDown(self): RecipeTestCase.tearDown(self) try: self.res.close() except: pass def test_list(self): '''All processing infos as a list''' pi = cpl.dfs.ProcessingInfo.list(self.res[0]) self.assertTrue(len(pi), 1) self.assertTrue(pi[0], self.pinfo) def test_param(self): '''Parameter information''' self.assertEqual(len(self.pinfo.param), len(self.recipe.param)) for p in self.recipe.param: self.assertEqual(self.pinfo.param[p.name], p.value if p.value is not None else p.default) def test_calib(self): '''Calibration frame information''' self.assertEqual(len(self.pinfo.calib), 1) self.assertEqual(self.pinfo.calib['FLAT'][-5:], '.fits') def test_tag(self): '''Input tag information''' self.assertEqual(self.pinfo.tag, self.recipe.tag) def test_raw(self): '''Raw file information''' self.assertEqual(self.pinfo.raw[-5:], '.fits') def test_name(self): '''Recipe and pipeline name information''' self.assertEqual(self.pinfo.name, self.recipe.__name__) self.assertEqual(self.pinfo.pipeline, 'iiinstrument') def test_version(self): '''Version information''' self.assertEqual(self.pinfo.version[0], self.recipe.version[0]) self.assertEqual(self.pinfo.cpl_version, 'cpl-%s' % self.recipe.cpl_version) def test_md5(self): '''MD5 checksums''' md5sum = self.res[0].header.get('DATAMD5') self.assertEqual(md5sum, self.pinfo.md5sum) md5sum = self.res[0].header.get('HIERARCH ESO PRO REC1 CAL1 DATAMD5') self.assertEqual(md5sum, self.pinfo.md5sums[self.pinfo.calib['FLAT']]) def test_recipe(self): '''Recreate and configure the recipe''' recipe = self.pinfo.create_recipe() self.assertTrue(isinstance(recipe, cpl.Recipe)) self.assertEqual(len(recipe.calib), 1) self.assertEqual(recipe.calib.FLAT.frames[-5:], '.fits') self.assertEqual(recipe.__name__, self.recipe.__name__) self.assertEqual(recipe.version[0], self.recipe.version[0]) self.assertEqual(len(recipe.param), len(self.recipe.param)) if __name__ == '__main__': unittest.main() python-cpl-0.7.4/test/rtest.c0000644000175000017500000004035613147022346015475 0ustar oleole00000000000000/* * Copyright (C) 2002,2003 European Southern Observatory * Copyright (C) 2011-2015 Ole Streicher * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #include #if ( CPL_VERSION_CODE < CPL_VERSION(6,3,0)) #define cpl_frameset_get_position cpl_frameset_get_frame #endif #define RRRECIPE_RAW "RRRECIPE_DOCATG_RAW" #define IIINSTRUMENT_CALIB_FLAT "FLAT" static int rtest_create(cpl_plugin *); static int rtest_exec(cpl_plugin *); static int rtest_destroy(cpl_plugin *); static int rtest(cpl_frameset *, const cpl_parameterlist *); static char rtest_description[] = "Recipe to test CPL frameworks like esorex or python-cpl.\n"; static char *license = "GPL"; /** @brief Set the group as RAW or CALIB in a frameset @param set the input frameset @return CPL_ERROR_NONE iff OK */ cpl_error_code dfs_set_groups(cpl_frameset * set) { cpl_errorstate prestate = cpl_errorstate_get(); cpl_frame * frame = NULL; int i = 0; int n = cpl_frameset_get_size(set); /* Loop on frames */ for (i = 0; i < n; i++) { frame = cpl_frameset_get_position(set, i); const char * tag = cpl_frame_get_tag(frame); if (tag == NULL) { cpl_msg_warning(cpl_func, "Frame %d has no tag", i); } else if (!strcmp(tag, RRRECIPE_RAW)) { /* RAW frames */ cpl_frame_set_group(frame, CPL_FRAME_GROUP_RAW); } else if (!strcmp(tag, IIINSTRUMENT_CALIB_FLAT)) { /* CALIB frames */ cpl_frame_set_group(frame, CPL_FRAME_GROUP_CALIB); } } if (!cpl_errorstate_is_equal(prestate)) { return cpl_error_set_message(cpl_func, cpl_error_get_code(), "Could not identify RAW and CALIB " "frames"); } return CPL_ERROR_NONE; } /** @brief find out the DIT value @param plist property list to read from @return the requested value */ static double pfits_get_dit(const cpl_propertylist * plist) { cpl_errorstate prestate = cpl_errorstate_get(); const double value = cpl_propertylist_get_double(plist, "ESO DET DIT"); /* Check for a change in the CPL error state */ /* - if it did change then propagate the error and return */ cpl_ensure(cpl_errorstate_is_equal(prestate), cpl_error_get_code(), 0.0); return value; } /** @brief Build the list of available plugins, for this module. @param list the plugin list @return 0 if everything is ok, 1 otherwise @note Only this function is exported Create the recipe instance and make it available to the application using the interface. */ int cpl_plugin_get_info(cpl_pluginlist * list) { cpl_recipe * recipe = cpl_calloc(1, sizeof *recipe ); cpl_plugin * plugin = &recipe->interface; if (cpl_plugin_init(plugin, CPL_PLUGIN_API, 1, CPL_PLUGIN_TYPE_RECIPE, "rtest", "Framework test recipe", rtest_description, "Ole Streicher", "python-cpl@liska.ath.cx", license, rtest_create, rtest_exec, rtest_destroy)) { cpl_msg_error(cpl_func, "Plugin initialization failed"); (void)cpl_error_set_where(cpl_func); return 1; } if (cpl_pluginlist_append(list, plugin)) { cpl_msg_error(cpl_func, "Error adding plugin to list"); (void)cpl_error_set_where(cpl_func); return 1; } return 0; } /** @brief Setup the recipe options @param plugin the plugin @return 0 if everything is ok Defining the command-line/configuration parameters for the recipe. */ static int rtest_create(cpl_plugin * plugin) { cpl_ensure_code((plugin != NULL), CPL_ERROR_NULL_INPUT); cpl_ensure_code((cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE), CPL_ERROR_TYPE_MISMATCH); cpl_recipe *recipe = (cpl_recipe *)plugin; /* Create the parameters list in the cpl_recipe object */ recipe->parameters = cpl_parameterlist_new(); if (recipe->parameters == NULL) { cpl_msg_error(cpl_func, "Parameter list allocation failed"); cpl_ensure_code(0, (int)CPL_ERROR_ILLEGAL_OUTPUT); } /* Fill the parameters list */ cpl_parameter * p; /* --stropt */ p = cpl_parameter_new_value("iiinstrument.rtest.string_option", CPL_TYPE_STRING, "A string option; saved as ESO QC STROPT", "iiinstrument.rtest",NULL); cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "stropt"); cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV); cpl_parameterlist_append(recipe->parameters, p); /* --boolopt */ p = cpl_parameter_new_value("iiinstrument.rtest.bool_option", CPL_TYPE_BOOL, "A flag; saved as ESO QC BOOLOPT", "iiinstrument.rtest", TRUE); cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "boolopt"); cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV); cpl_parameterlist_append(recipe->parameters, p); /* --floatopt */ p = cpl_parameter_new_value("iiinstrument.rtest.float_option", CPL_TYPE_DOUBLE, "A double option; saved as ESO QC FLOATOPT", "iiinstrument.rtest", 0.1); cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "floatopt"); cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV); cpl_parameterlist_append(recipe->parameters, p); /* --inttopt */ p = cpl_parameter_new_value("iiinstrument.rtest.int_option", CPL_TYPE_INT, "An interger; saved as ESO QC INTOPT", "iiinstrument.rtest", 2); cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "intopt"); cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV); cpl_parameterlist_append(recipe->parameters, p); /* --enumopt */ p = cpl_parameter_new_enum("iiinstrument.rtest.enum_option", CPL_TYPE_STRING, "An enumeration option, saved as ESO QC ENUMOPT", "iiinstrument.rtest", "first", 3, "first", "second", "third"); cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "enumopt"); cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV); cpl_parameterlist_append(recipe->parameters, p); /* --rangeopt */ p = cpl_parameter_new_range("iiinstrument.rtest.range_option", CPL_TYPE_DOUBLE, "A double option with a range, saved as ESO QC RANGEOPT", "iiinstrument.rtest", 0.1, -0.5, 0.5); cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "rangeopt"); cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV); cpl_parameterlist_append(recipe->parameters, p); /* --dot.opt */ p = cpl_parameter_new_value("iiinstrument.rtest.dotted.opt", CPL_TYPE_INT, "An (integer) option with a dot in its name", "iiinstrument.rtest", 0); cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "dot.opt"); cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV); cpl_parameterlist_append(recipe->parameters, p); /* --crashing */ p = cpl_parameter_new_enum("iiinstrument.rtest.crashing", CPL_TYPE_STRING, "Crash the recipe?", "iiinstrument.rtest", "no", 3, "no", "free", "segfault"); cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "crashing"); cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV); cpl_parameterlist_append(recipe->parameters, p); /* --memleak */ p = cpl_parameter_new_value("iiinstrument.rtest.memleak", CPL_TYPE_BOOL, "If yes, dont deallocate some memory", "iiinstrument.rtest", FALSE); cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "memleak"); cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV); cpl_parameterlist_append(recipe->parameters, p); /* --sleep */ p = cpl_parameter_new_value("iiinstrument.rtest.sleep", CPL_TYPE_DOUBLE, "Simulate some computing by sleeping for specified time [seconds]", "iiinstrument.rtest", 0.1); cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "sleep"); cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV); cpl_parameterlist_append(recipe->parameters, p); /* --disabled */ p = cpl_parameter_new_value("iiinstrument.rtest.disabled", CPL_TYPE_DOUBLE, "Dummy disabled parameter", "iiinstrument.rtest", -0.1); cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "disabled"); cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV); cpl_parameter_disable(p, CPL_PARAMETER_MODE_CLI); cpl_parameterlist_append(recipe->parameters, p); return 0; } /** @brief Execute the plugin instance given by the interface @param plugin the plugin @return 0 if everything is ok */ static int rtest_exec(cpl_plugin * plugin) { cpl_ensure_code((plugin != NULL), CPL_ERROR_NULL_INPUT); cpl_ensure_code((cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE), CPL_ERROR_TYPE_MISMATCH); cpl_recipe *recipe = (cpl_recipe *)plugin; cpl_ensure_code((recipe->parameters != NULL), (int)CPL_ERROR_NULL_INPUT); cpl_ensure_code((recipe->frames != NULL), (int)CPL_ERROR_NULL_INPUT); int recipe_status = rtest(recipe->frames, recipe->parameters); /* Ensure DFS-compliance of the products */ if (cpl_dfs_update_product_header(recipe->frames)) { if (!recipe_status) recipe_status = (int)cpl_error_get_code(); } return recipe_status; } /** @brief Destroy what has been created by the 'create' function @param plugin the plugin @return 0 if everything is ok */ static int rtest_destroy(cpl_plugin * plugin) { cpl_ensure_code((plugin != NULL), CPL_ERROR_NULL_INPUT); cpl_ensure_code((cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE), CPL_ERROR_TYPE_MISMATCH); cpl_recipe *recipe = (cpl_recipe *)plugin; cpl_parameterlist_delete(recipe->parameters); return 0; } /** @brief Interpret the command line options and execute the data processing @param frameset the frames list @param parlist the parameters list @return 0 if everything is ok */ static int rtest(cpl_frameset * frameset, const cpl_parameterlist * parlist) { /* Use the errorstate to detect an error in a function that does not return an error code. */ cpl_errorstate prestate = cpl_errorstate_get(); const cpl_parameter *param; /* --stropt */ param = cpl_parameterlist_find_const(parlist, "iiinstrument.rtest.string_option"); const char *str_option = cpl_parameter_get_string(param); cpl_ensure_code(str_option != NULL, CPL_ERROR_NULL_INPUT); /* --boolopt */ param = cpl_parameterlist_find_const(parlist, "iiinstrument.rtest.bool_option"); int bool_option = cpl_parameter_get_bool(param); /* --floatopt */ param = cpl_parameterlist_find_const(parlist, "iiinstrument.rtest.float_option"); double float_option = cpl_parameter_get_double(param); /* --intopt */ param = cpl_parameterlist_find_const(parlist, "iiinstrument.rtest.int_option"); int int_option = cpl_parameter_get_int(param); /* --enumopt */ param = cpl_parameterlist_find_const(parlist, "iiinstrument.rtest.enum_option"); const char *enum_option = cpl_parameter_get_string(param); /* --rangeopt */ param = cpl_parameterlist_find_const(parlist, "iiinstrument.rtest.range_option"); double range_option = cpl_parameter_get_double(param); /* --crashing */ param = cpl_parameterlist_find_const(parlist, "iiinstrument.rtest.crashing"); const char *crashing = cpl_parameter_get_string(param); /* --memleak */ param = cpl_parameterlist_find_const(parlist, "iiinstrument.rtest.memleak"); int memleak = cpl_parameter_get_bool(param); /* --sleep */ param = cpl_parameterlist_find_const(parlist, "iiinstrument.rtest.sleep"); double sleep_secs = cpl_parameter_get_double(param); param = cpl_parameterlist_find_const(parlist, "iiinstrument.rtest.disabled"); double disabled_option = cpl_parameter_get_double(param); if (!cpl_errorstate_is_equal(prestate)) { return (int)cpl_error_set_message(cpl_func, cpl_error_get_code(), "Could not retrieve the input " "parameters"); } /* Identify the RAW and CALIB frames in the input frameset */ cpl_ensure_code(dfs_set_groups(frameset) == CPL_ERROR_NONE, cpl_error_get_code()); /* - raw input file */ const cpl_frame *rawframe = cpl_frameset_find_const(frameset, RRRECIPE_RAW); if (rawframe == NULL) { /* cpl_frameset_find_const() does not set an error code, when a frame is not found, so we will set one here. */ return (int)cpl_error_set_message(cpl_func, CPL_ERROR_DATA_NOT_FOUND, "No file tagged with %s", RRRECIPE_RAW); } cpl_propertylist *plist = cpl_propertylist_load_regexp(cpl_frame_get_filename(rawframe), 0, "ESO DET ", 0); if (plist == NULL) { /* In this case an error message is added to the error propagation */ cpl_msg_error(cpl_func, "Could not read plist from %s", cpl_frame_get_filename(rawframe)); return (int)cpl_error_set_message(cpl_func, cpl_error_get_code(), "Could not read the FITS header"); } double qc_param = pfits_get_dit(plist); cpl_errorstate_set(prestate); cpl_propertylist_delete(plist); /* - calibration input file */ const cpl_frame *flat = cpl_frameset_find(frameset,IIINSTRUMENT_CALIB_FLAT); if (flat == NULL) { cpl_msg_warning(cpl_func, "No file tagged with %s", IIINSTRUMENT_CALIB_FLAT); } /* Check for a change in the CPL error state */ /* - if it did change then propagate the error and return */ cpl_ensure_code(cpl_errorstate_is_equal(prestate), cpl_error_get_code()); /* Load raw image */ cpl_image *image = cpl_image_load(cpl_frame_get_filename(rawframe), CPL_TYPE_FLOAT, 0, 0); /* A multiline debug message */ cpl_msg_info(cpl_func, "multiline#1\nmultiline#2\nmultiline#3"); /* Do some fake processing */ usleep((unsigned int)(1e6*sleep_secs)); /* Add QC parameters */ cpl_propertylist *qclist = cpl_propertylist_new(); cpl_propertylist_append_double(qclist, "ESO QC QCPARAM", qc_param); cpl_propertylist_append_string(qclist, "ESO PRO CATG","THE_PRO_CATG_VALUE"); if (str_option != NULL) { cpl_propertylist_append_string(qclist, "ESO QC STROPT", str_option); } else { cpl_propertylist_append_string(qclist, "ESO QC STROPT", "(null)"); } cpl_propertylist_append_bool(qclist, "ESO QC BOOLOPT", bool_option); cpl_propertylist_append_double(qclist, "ESO QC FLOATOPT", float_option); cpl_propertylist_append_int(qclist, "ESO QC INTOPT", int_option); if (enum_option != NULL) { cpl_propertylist_append_string(qclist, "ESO QC ENUMOPT", enum_option); } else { cpl_propertylist_append_string(qclist, "ESO QC ENUMOPT", "(null)"); } cpl_propertylist_append_double(qclist, "ESO QC RANGEOPT", range_option); const char *testenv = getenv("TESTENV"); if (testenv != NULL) { cpl_propertylist_append_string(qclist, "ESO QC TESTENV", testenv); } else { cpl_propertylist_append_string(qclist, "ESO QC TESTENV", "(null)"); } cpl_propertylist_append_double(qclist, "ESO QC DISABLEDOPT", disabled_option); prestate = cpl_errorstate_get(); if (cpl_dfs_save_image(frameset, NULL, parlist, frameset, NULL, image, CPL_BPP_IEEE_FLOAT, "rtest", qclist, NULL, "iiinstrument/0.0.1", "rtest.fits")) { /* Propagate the error */ (void)cpl_error_set_where(cpl_func); } if (!cpl_errorstate_is_equal(prestate)) { cpl_msg_error(__func__, "in cpl_dfs_save_image()"); } cpl_image_delete(image); cpl_propertylist_delete(qclist); /* Let's see if we can crash the machine by some random code */ if (strcmp(crashing, "free") == 0) { cpl_image_delete(image); cpl_propertylist_delete(qclist); } if (strcmp(crashing, "segfault") == 0) { double *crashvar = NULL; *crashvar = 1.99; } if (memleak) { __attribute__((unused)) void * r = cpl_malloc(16); } return (int)cpl_error_get_code(); } python-cpl-0.7.4/COPYING0000644000175000017500000004325413147022356014245 0ustar oleole00000000000000 GNU GENERAL PUBLIC LICENSE Version 2, June 1991 Copyright (C) 1989, 1991 Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Preamble The licenses for most software are designed to take away your freedom to share and change it. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change free software--to make sure the software is free for all its users. This General Public License applies to most of the Free Software Foundation's software and to any other program whose authors commit to using it. (Some other Free Software Foundation software is covered by the GNU Lesser General Public License instead.) You can apply it to your programs, too. When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for this service if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs; and that you know you can do these things. To protect your rights, we need to make restrictions that forbid anyone to deny you these rights or to ask you to surrender the rights. These restrictions translate to certain responsibilities for you if you distribute copies of the software, or if you modify it. For example, if you distribute copies of such a program, whether gratis or for a fee, you must give the recipients all the rights that you have. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights. We protect your rights with two steps: (1) copyright the software, and (2) offer you this license which gives you legal permission to copy, distribute and/or modify the software. Also, for each author's protection and ours, we want to make certain that everyone understands that there is no warranty for this free software. If the software is modified by someone else and passed on, we want its recipients to know that what they have is not the original, so that any problems introduced by others will not reflect on the original authors' reputations. Finally, any free program is threatened constantly by software patents. We wish to avoid the danger that redistributors of a free program will individually obtain patent licenses, in effect making the program proprietary. To prevent this, we have made it clear that any patent must be licensed for everyone's free use or not licensed at all. The precise terms and conditions for copying, distribution and modification follow. GNU GENERAL PUBLIC LICENSE TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 0. This License applies to any program or other work which contains a notice placed by the copyright holder saying it may be distributed under the terms of this General Public License. The "Program", below, refers to any such program or work, and a "work based on the Program" means either the Program or any derivative work under copyright law: that is to say, a work containing the Program or a portion of it, either verbatim or with modifications and/or translated into another language. (Hereinafter, translation is included without limitation in the term "modification".) Each licensee is addressed as "you". Activities other than copying, distribution and modification are not covered by this License; they are outside its scope. The act of running the Program is not restricted, and the output from the Program is covered only if its contents constitute a work based on the Program (independent of having been made by running the Program). Whether that is true depends on what the Program does. 1. You may copy and distribute verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice and disclaimer of warranty; keep intact all the notices that refer to this License and to the absence of any warranty; and give any other recipients of the Program a copy of this License along with the Program. You may charge a fee for the physical act of transferring a copy, and you may at your option offer warranty protection in exchange for a fee. 2. You may modify your copy or copies of the Program or any portion of it, thus forming a work based on the Program, and copy and distribute such modifications or work under the terms of Section 1 above, provided that you also meet all of these conditions: a) You must cause the modified files to carry prominent notices stating that you changed the files and the date of any change. b) You must cause any work that you distribute or publish, that in whole or in part contains or is derived from the Program or any part thereof, to be licensed as a whole at no charge to all third parties under the terms of this License. c) If the modified program normally reads commands interactively when run, you must cause it, when started running for such interactive use in the most ordinary way, to print or display an announcement including an appropriate copyright notice and a notice that there is no warranty (or else, saying that you provide a warranty) and that users may redistribute the program under these conditions, and telling the user how to view a copy of this License. (Exception: if the Program itself is interactive but does not normally print such an announcement, your work based on the Program is not required to print an announcement.) These requirements apply to the modified work as a whole. If identifiable sections of that work are not derived from the Program, and can be reasonably considered independent and separate works in themselves, then this License, and its terms, do not apply to those sections when you distribute them as separate works. But when you distribute the same sections as part of a whole which is a work based on the Program, the distribution of the whole must be on the terms of this License, whose permissions for other licensees extend to the entire whole, and thus to each and every part regardless of who wrote it. Thus, it is not the intent of this section to claim rights or contest your rights to work written entirely by you; rather, the intent is to exercise the right to control the distribution of derivative or collective works based on the Program. In addition, mere aggregation of another work not based on the Program with the Program (or with a work based on the Program) on a volume of a storage or distribution medium does not bring the other work under the scope of this License. 3. You may copy and distribute the Program (or a work based on it, under Section 2) in object code or executable form under the terms of Sections 1 and 2 above provided that you also do one of the following: a) Accompany it with the complete corresponding machine-readable source code, which must be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, b) Accompany it with a written offer, valid for at least three years, to give any third party, for a charge no more than your cost of physically performing source distribution, a complete machine-readable copy of the corresponding source code, to be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, c) Accompany it with the information you received as to the offer to distribute corresponding source code. (This alternative is allowed only for noncommercial distribution and only if you received the program in object code or executable form with such an offer, in accord with Subsection b above.) The source code for a work means the preferred form of the work for making modifications to it. For an executable work, complete source code means all the source code for all modules it contains, plus any associated interface definition files, plus the scripts used to control compilation and installation of the executable. However, as a special exception, the source code distributed need not include anything that is normally distributed (in either source or binary form) with the major components (compiler, kernel, and so on) of the operating system on which the executable runs, unless that component itself accompanies the executable. If distribution of executable or object code is made by offering access to copy from a designated place, then offering equivalent access to copy the source code from the same place counts as distribution of the source code, even though third parties are not compelled to copy the source along with the object code. 4. You may not copy, modify, sublicense, or distribute the Program except as expressly provided under this License. Any attempt otherwise to copy, modify, sublicense or distribute the Program is void, and will automatically terminate your rights under this License. However, parties who have received copies, or rights, from you under this License will not have their licenses terminated so long as such parties remain in full compliance. 5. You are not required to accept this License, since you have not signed it. However, nothing else grants you permission to modify or distribute the Program or its derivative works. These actions are prohibited by law if you do not accept this License. Therefore, by modifying or distributing the Program (or any work based on the Program), you indicate your acceptance of this License to do so, and all its terms and conditions for copying, distributing or modifying the Program or works based on it. 6. Each time you redistribute the Program (or any work based on the Program), the recipient automatically receives a license from the original licensor to copy, distribute or modify the Program subject to these terms and conditions. You may not impose any further restrictions on the recipients' exercise of the rights granted herein. You are not responsible for enforcing compliance by third parties to this License. 7. If, as a consequence of a court judgment or allegation of patent infringement or for any other reason (not limited to patent issues), conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot distribute so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not distribute the Program at all. For example, if a patent license would not permit royalty-free redistribution of the Program by all those who receive copies directly or indirectly through you, then the only way you could satisfy both it and this License would be to refrain entirely from distribution of the Program. If any portion of this section is held invalid or unenforceable under any particular circumstance, the balance of the section is intended to apply and the section as a whole is intended to apply in other circumstances. It is not the purpose of this section to induce you to infringe any patents or other property right claims or to contest validity of any such claims; this section has the sole purpose of protecting the integrity of the free software distribution system, which is implemented by public license practices. Many people have made generous contributions to the wide range of software distributed through that system in reliance on consistent application of that system; it is up to the author/donor to decide if he or she is willing to distribute software through any other system and a licensee cannot impose that choice. This section is intended to make thoroughly clear what is believed to be a consequence of the rest of this License. 8. If the distribution and/or use of the Program is restricted in certain countries either by patents or by copyrighted interfaces, the original copyright holder who places the Program under this License may add an explicit geographical distribution limitation excluding those countries, so that distribution is permitted only in or among countries not thus excluded. In such case, this License incorporates the limitation as if written in the body of this License. 9. The Free Software Foundation may publish revised and/or new versions of the General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Program specifies a version number of this License which applies to it and "any later version", you have the option of following the terms and conditions either of that version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of this License, you may choose any version ever published by the Free Software Foundation. 10. If you wish to incorporate parts of the Program into other free programs whose distribution conditions are different, write to the author to ask for permission. For software which is copyrighted by the Free Software Foundation, write to the Free Software Foundation; we sometimes make exceptions for this. Our decision will be guided by the two goals of preserving the free status of all derivatives of our free software and of promoting the sharing and reuse of software generally. NO WARRANTY 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. END OF TERMS AND CONDITIONS How to Apply These Terms to Your New Programs If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms. To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively convey the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. Copyright (C) This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. Also add information on how to contact you by electronic and paper mail. If the program is interactive, make it output a short notice like this when it starts in an interactive mode: Gnomovision version 69, Copyright (C) year name of author Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details. The hypothetical commands `show w' and `show c' should show the appropriate parts of the General Public License. Of course, the commands you use may be called something other than `show w' and `show c'; they could even be mouse-clicks or menu items--whatever suits your program. You should also get your employer (if you work as a programmer) or your school, if any, to sign a "copyright disclaimer" for the program, if necessary. Here is a sample; alter the names: Yoyodyne, Inc., hereby disclaims all copyright interest in the program `Gnomovision' (which makes passes at compilers) written by James Hacker. , 1 April 1989 Ty Coon, President of Vice This General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Lesser General Public License instead of this License. python-cpl-0.7.4/README.rst0000644000175000017500000000375713370120012014667 0ustar oleole00000000000000python-cpl ========== *Python interface for the Common Pipeline Library* .. image:: https://img.shields.io/pypi/v/python-cpl.svg :target: https://pypi.python.org/pypi/python-cpl Python-cpl is an `Astropy Affiliated Package `_ that can list, configure and execute recipes from `ESO data reduction pipelines `_ from Python (python2 and python3). The input, calibration and output data can be specified as FITS files or as ``astropy.io.fits`` objects in memory. The ESO `Common Pipeline Library `_ (CPL) comprises a set of ISO-C libraries that provide a comprehensive, efficient and robust software toolkit. It forms a basis for the creation of automated astronomical data-reduction tasks. One of the features provided by the CPL is the ability to create data-reduction algorithms that run as plugins (dynamic libraries). These are called "recipes" and are one of the main aspects of the CPL data-reduction development environment. Python-CPL releases are `registered on PyPI `_, and development is occurring at the `project's github page `_. For installation instructions, see the `online documentation `_. The travis test status, coverage, and documentation build status of the github repository is: .. image:: https://travis-ci.org/olebole/python-cpl.png :target: https://travis-ci.org/olebole/python-cpl .. image:: https://coveralls.io/repos/olebole/python-cpl/badge.svg?branch=master :target: https://coveralls.io/r/olebole/python-cpl?branch=master .. image:: https://landscape.io/github/olebole/python-cpl/master/landscape.svg?style=flat :target: https://landscape.io/github/olebole/python-cpl/master .. image:: https://readthedocs.org/projects/python-cpl/badge/?version=latest :target: https://readthedocs.org/projects/python-cpl/?badge=latest python-cpl-0.7.4/python_cpl.egg-info/0000755000175000017500000000000013370121236017046 5ustar oleole00000000000000python-cpl-0.7.4/python_cpl.egg-info/SOURCES.txt0000644000175000017500000000121613370121236020732 0ustar oleole00000000000000COPYING MANIFEST.in README.rst setup.py cpl/CPL_library.c cpl/CPL_library.h cpl/CPL_recipe.c cpl/__init__.py cpl/cpl_api.h cpl/dfs.py cpl/esorex.py cpl/frames.py cpl/logger.py cpl/md5sum.py cpl/param.py cpl/recipe.py cpl/result.py cpl/version.py doc/Makefile doc/conf.py doc/dfs.rst doc/esorex.rst doc/frames.rst doc/index.rst doc/install.rst doc/msg.rst doc/parallel.rst doc/param.rst doc/recipe.rst doc/restrictions.rst doc/result.rst doc/tutorial.rst python_cpl.egg-info/PKG-INFO python_cpl.egg-info/SOURCES.txt python_cpl.egg-info/dependency_links.txt python_cpl.egg-info/requires.txt python_cpl.egg-info/top_level.txt test/TestRecipe.py test/rtest.cpython-cpl-0.7.4/python_cpl.egg-info/PKG-INFO0000644000175000017500000000336513370121236020152 0ustar oleole00000000000000Metadata-Version: 1.2 Name: python-cpl Version: 0.7.4 Summary: Python interface for the ESO Common Pipeline Library Home-page: https://pypi.org/project/python-cpl/0.7.4 Author: Ole Streicher Author-email: python-cpl@liska.ath.cx License: GPL Download-URL: https://files.pythonhosted.org/packages/source/p/python-cpl/python-cpl-0.7.4.tar.gz/python-cpl-0.7.4.tar.gz Description: This module can list, configure and execute CPL-based recipes from Python (python2 and python3). The input, calibration and output data can be specified as FITS files or as ``astropy.io.fits`` objects in memory. The ESO `Common Pipeline Library `_ (CPL) comprises a set of ISO-C libraries that provide a comprehensive, efficient and robust software toolkit. It forms a basis for the creation of automated astronomical data-reduction tasks. One of the features provided by the CPL is the ability to create data-reduction algorithms that run as plugins (dynamic libraries). These are called "recipes" and are one of the main aspects of the CPL data-reduction development environment. Platform: UNKNOWN Classifier: Development Status :: 4 - Beta Classifier: Intended Audience :: Science/Research Classifier: License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+) Classifier: Operating System :: MacOS :: MacOS X Classifier: Operating System :: POSIX Classifier: Operating System :: Unix Classifier: Programming Language :: C Classifier: Programming Language :: Python Classifier: Programming Language :: Python :: 2 Classifier: Programming Language :: Python :: 3 Classifier: Topic :: Scientific/Engineering :: Astronomy Provides: cpl Requires-Python: >=2.7 python-cpl-0.7.4/python_cpl.egg-info/requires.txt0000644000175000017500000000001013370121236021435 0ustar oleole00000000000000astropy python-cpl-0.7.4/python_cpl.egg-info/dependency_links.txt0000644000175000017500000000000113370121236023114 0ustar oleole00000000000000 python-cpl-0.7.4/python_cpl.egg-info/top_level.txt0000644000175000017500000000000413370121236021572 0ustar oleole00000000000000cpl python-cpl-0.7.4/cpl/0000755000175000017500000000000013370121236013753 5ustar oleole00000000000000python-cpl-0.7.4/cpl/recipe.py0000644000175000017500000005706013370120012015573 0ustar oleole00000000000000from __future__ import absolute_import import os import shutil import tempfile import threading import collections import warnings import textwrap from astropy.io import fits from . import CPL_recipe from .frames import FrameList, mkabspath, expandframelist from .result import Result, RecipeCrash from .param import ParameterList from .logger import LogServer class Recipe(object): '''Pluggable Data Reduction Module (PDRM) from a ESO pipeline. Recipes are loaded from shared libraries that are provided with the pipeline library of the instrument. The module does not need to be linked to the same library version as the one used for the compilation of python-cpl. Currently, recipes compiled with CPL versions from 4.0 are supported. The list of supported versions is stored as :attr:`cpl.cpl_versions`. The libraries are searched in the directories specified by the class attribute :attr:`Recipe.path` or its subdirectories. The search path is automatically set to the esorex path when :func:`cpl.esorex.init()` is called. ''' path = [ '.' ] '''Search path for the recipes. It may be set to either a string, or to a list of strings. All shared libraries in the search path and their subdirectories are searched for CPL recipes. On default, the path is set to the current directory. The search path is automatically set to the esorex path when :func:`cpl.esorex.init()` is called. ''' memory_mode = 0 '''CPL memory management mode. The valid values are 0 Use the default system functions for memory handling 1 Exit if a memory-allocation fails, provide checking for memory leaks, limited reporting of memory allocation and limited protection on deallocation of invalid pointers. 2 Exit if a memory-allocation fails, provide checking for memory leaks, extended reporting of memory allocation and protection on deallocation of invalid pointers. .. note:: This variable is only effective before the CPL library was initialized. Even :func:`cpl.Recipe.list()` initializes the library. Therefore it is highly recommended to set this as the first action after importing :mod:`cpl`. ''' def __init__(self, name, filename = None, version = None, threaded = False): '''Try to load a recipe with the specified name in the directory specified by the class attribute :attr:`Recipe.path` or its subdirectories. :param name: Name of the recipe. Required. Use :func:`cpl.Recipe.list()` to get a list of available recipes. :type name: :class:`str` :param filename: Name of the shared library. Optional. If not set, :attr:`Recipe.path` is searched for the library file. :type filename: :class:`str` :param version: Version number. Optional. If not set, the newest version is loaded. :type version: :class:`int` or :class:`str` :param threaded: Run the recipe in the background, returning immediately after calling it. Default is :attr:`False`. This may be also set as an attribute or specified as a parameter when calling the recipe. :type threaded: :class:`bool` ''' os.putenv('CPL_MEMORY_MODE', str(Recipe.memory_mode)); self._recipe = None self.__name__ = name '''Recipe name.''' if not filename: filename = Recipe.get_recipefilename(name, version) if not filename: raise IOError('Recipe %s not found at path %s' % (repr(name), repr(Recipe.path))) self.__file__ = filename '''Shared library file name.''' self._recipe = CPL_recipe.recipe(filename, name) if version and version not in self.version: raise IOError('wrong version %s (requested %s) for %s in %s' % (str(self.version), str(version), name, filename)) if not self._recipe.cpl_is_supported(): warnings.warn("Unsupported CPL version %s linked to %s" % (self.cpl_version, filename)) self._param = ParameterList(self) self._calib = FrameList(self) self.env = dict() '''Bla''' self._tags = None self.tag = self.tags[0] if self.tags else None '''Default tag when the recipe is called. This is set automatically only if the recipe provided the information about input tags. Otherwise this tag has to be set manually. ''' self.output_dir = None self.temp_dir = '.' '''Base directory for temporary directories where the recipe is executed. The working dir is created as a subdir with a random file name. If set to :obj:`None`, the system temp dir is used. Defaults to :literal:`'.'`. ''' self.memory_dump = 0 self.threaded = threaded self.mtrace = False self.__doc__ = self._doc() @property def __author__(self): '''Author name''' return self._recipe.author()[0] @property def __email__(self): '''Author email''' return self._recipe.author()[1] @property def description(self): '''Pair (synopsis, description) of two strings.''' return self._recipe.description() @property def version(self): '''Pair (versionnumber, versionstring) of an integer and a string. The integer will be increased on development progress.''' return self._recipe.version() @property def __version__(self): return self._recipe.version()[1] @property def __copyright__(self): '''Copyright string of the recipe''' return self._recipe.copyright() @property def cpl_version(self): '''Version of the CPL library that is linked to the recipe, as a string''' return self._recipe.cpl_version() @property def cpl_description(self): '''Version numbers of CPL and its libraries that were linked to the recipe, as a string.''' return self._recipe.cpl_description() @property def tags(self): '''Possible tags for the raw input frames, or ':obj:`None` if this information is not provided by the recipe.''' frameconfig = self._recipe.frameConfig() return [ c[0][0] for c in frameconfig ] if frameconfig else self._tags @tags.setter def tags(self, t): if self._recipe.frameConfig(): raise AttributeError('Tags are immutable') else: self._tags = t @property def tag(self): '''Default raw input frame tag. After creation, it is set to the first tag from the "tags" property and may be changed to any of these tags. If the recipe does not provide the tag information, it must be set manually, or the tag name has to be provided when calling the recipe.''' return self._tag @tag.setter def tag(self, t): if self.tags is None or t in self.tags: self._tag = t else: raise KeyError("Tag %s not in %s" % (repr(t), str(self.tags))) @property def calib(self): '''This attribute contains the calibration frames for the recipe. It is iterable and then returns all calibration frames: >>> for f in muse_scibasic.calib: ... print f.tag, f.min, f.max, f.frames TRACE_TABLE 1 1 None WAVECAL_TABLE 1 1 None MASTER_BIAS 1 1 master_bias_0.fits MASTER_DARK None 1 None GEOMETRY_TABLE 1 1 None BADPIX_TABLE None None ['badpix_1.fits', 'badpix_2.fits'] MASTER_FLAT None 1 None .. note:: Only MUSE recipes are able to provide the full list of calibration frames and the minimal/maximal number of calibration frames. For other recipes, only frames that were set by the users are returned here. Their minimum and maximum value will be set to :obj:`None`. In order to assing a FITS file to a tag, the file name or the :class:`astropy.io.fits.HDUList` is assigned to the calibration attribute: >>> muse_scibasic.calib.MASTER_BIAS = 'MASTER_BIAS_0.fits' Using :class:`astropy.io.fits.HDUList` is useful when it needs to be patched before fed into the recipe. >>> master_bias = astropy.io.fits.open('MASTER_BIAS_0.fits') >>> master_bias[0].header['HIERARCH ESO DET CHIP1 OUT1 GAIN'] = 2.5 >>> muse_scibasic.calib.MASTER_BIAS = master_bias Note that :class:`astropy.io.fits.HDUList` objects are stored in temporary files before the recipe is called which may produce some overhead. Also, the CPL then assigns the random temporary file names to the FITS keywords ``HIERARCH ESO PRO RECm RAWn NAME`` which should be corrected afterwards if needed. To assign more than one frame, put them into a list: >>> muse_scibasic.calib.BADPIX_TABLE = [ 'badpix1.fits', 'badpix2.fits' ] All calibration frames can be set in one step by assigning a :class:`dict` to the parameters. In this case, frame that are not in the map are set are removed from the list, and unknown frame tags are silently ignored. The key of the map is the tag name; the values are either a string, or a list of strings, containing the file name(s) or the :class:`astropy.io.fits.HDUList` objects. >>> muse_scibasic.calib = { 'MASTER_BIAS':'master_bias_0.fits', ... 'BADPIX_TABLE':[ 'badpix_1.fits', 'badpix_2.fits' ] } In a recipe call, the calibration frame lists may be overwritten by specifying them in a :class:`dict`: >>> res = muse_scibasic( ..., calib = {'MASTER_BIAS':'master_bias_1.fits'}) ''' return self._calib @calib.setter def calib(self, source = None): if isinstance(source, str) or hasattr(source, 'read'): from .esorex import load_sof source = load_sof(source) self._calib = FrameList(self, source) @calib.deleter def calib(self): self._calib = FrameList(self, None) @property def param(self): '''This attribute contains all recipe parameters. It is iteratable and then returns all individual parameters: >>> for p in muse_scibasic.param: ... print p.name, p.value, p.default ... nifu None 99 cr None dcr xbox None 15 ybox None 40 passes None 2 thres None 4.5 sample None False dlambda None 1.2 On interactive sessions, all parameter settings can be easily printed by printing the :attr:`param` attribute of the recipe: >>> print muse_scibasic.param [Parameter('nifu', default=99), Parameter('cr', default=dcr), Parameter('xbox', default=15), Parameter('ybox', default=40), Parameter('passes', default=2), Parameter('thres', default=4.5), Parameter('sample', default=False), Parameter('dlambda', default=1.2)] To set the value of a recipe parameter, the value can be assigned to the according attribute: >>> muse_scibasic.param.nifu = 1 The new value is checked against parameter type, and possible value limitations provided by the recipe. Hyphens in parameter names are converted to underscores. In a recipe call, the same parameter can be specified as :class:`dict`: >>> res = muse_scibasic( ..., param = {'nifu':1}) To reset a value to its default, it is either deleted, or set to :obj:`None`. The following two lines: >>> muse_scibasic.param.nifu = None >>> del muse_scibasic.param.nifu will both reset the parameter to its default value. All parameters can be set in one step by assigning a :class:`dict` to the parameters. In this case, all values that are not in the map are reset to default, and unknown parameter names are ignored. The keys of the map may contain contain the name or the fullname with context: >>> muse_scibasic.param = { 'nifu':1, 'xbox':11, 'resample':True } ''' return self._param @param.setter def param(self, source = None): if isinstance(source, str) or hasattr(source, 'read'): from .esorex import load_rc source = load_rc(source) self._param = ParameterList(self, source) @param.deleter def param(self): self._param = ParameterList(self, None) @property def output(self): '''Return a dictionary of output frame tags. Keys are the tag names, values are the corresponding list of output tags. If the recipe does not provide this information, an exception is raised. ''' return dict((c[0][0], c[2]) for c in self._recipe.frameConfig()) def __call__(self, *data, **ndata): '''Call the recipes execution with a certain input frame. :param raw: Data input frames. :type raw: :class:`astropy.io.fits.HDUlist` or :class:`str` or a :class:`list` of them, or :class:`dict` :param tag: Overwrite the :attr:`tag` attribute (optional). :type tag: :class:`str` :param threaded: overwrite the :attr:`threaded` attribute (optional). :type threaded: :class:`bool` :param loglevel: set the log level for python :mod:`logging` (optional). :type loglevel: :class:`int` :param logname: set the log name for the python :class:`logging.Logger` (optional, default is 'cpl.' + recipename). :type logname: :class:`str` :param output_dir: Set or overwrite the :attr:`output_dir` attribute. (optional) :type output_dir: :class:`str` :param param: overwrite the CPL parameters of the recipe specified as keys with their dictionary values (optional). :type param: :class:`dict` :param calib: Overwrite the calibration frame lists for the tags specified as keys with their dictionary values (optional). :type calib: :class:`dict` :param env: overwrite environment variables for the recipe call (optional). :type env: :class:`dict` :return: The object with the return frames as :class:`astropy.io.fits.HDUList` objects :rtype: :class:`cpl.Result` :raise: :exc:`exceptions.ValueError` If the invocation parameters are incorrect. :raise: :exc:`exceptions.IOError` If the temporary directory could not be built, the recipe could not start or the files could not be read/written. :raise: :exc:`cpl.CplError` If the recipe returns an error. :raise: :exc:`cpl.RecipeCrash` If the CPL recipe crashes with a SIGSEV or a SIGBUS .. note:: If the recipe is executed in the background (``threaded = True``) and an exception occurs, this exception is raised whenever result fields are accessed. ''' threaded = ndata.get('threaded', self.threaded) mtrace = ndata.get('mtrace', self.mtrace) loglevel = ndata.get('loglevel') logname = ndata.get('logname', 'cpl.%s' % self.__name__) output_dir = ndata.get('output_dir', self.output_dir) output_format = str if output_dir else fits.HDUList if output_dir is None: output_dir = tempfile.mkdtemp(dir = self.temp_dir, prefix = self.__name__ + "-") parlist = self.param._aslist(ndata.get('param')) raw_frames = self._get_raw_frames(*data, **ndata) if len(raw_frames) < 1: raise ValueError('No raw frames specified.') input_len = -1 if isinstance(raw_frames[0][1], fits.HDUList) else \ len(raw_frames[0][1]) if isinstance(raw_frames[0][1], list) else -1 calib_frames = self.calib._aslist(ndata.get('calib')) framelist = expandframelist(raw_frames + calib_frames) runenv = dict(self.env) runenv.update(ndata.get('env', dict())) logger = None delete = output_format == fits.HDUList try: if (not os.access(output_dir, os.F_OK)): os.makedirs(output_dir) mkabspath(framelist, output_dir) logger = LogServer(logname, loglevel) except: try: self._cleanup(output_dir, logger, delete) except: pass raise if not threaded: return self._exec(output_dir, parlist, framelist, runenv, input_len, logger, output_format, delete, mtrace) else: return Threaded( self._exec, output_dir, parlist, framelist, runenv, input_len, logger, output_format, delete, mtrace) def _exec(self, output_dir, parlist, framelist, runenv, input_len, logger, output_format, delete, mtrace): try: return Result(output_dir, self._recipe.run(output_dir, parlist, framelist, list(runenv.items()), logger.logfile, logger.level, self.memory_dump, mtrace), input_len, logger, output_format) finally: self._cleanup(output_dir, logger, delete) def _get_raw_frames(self, *data, **ndata): '''Return the input frames. Returns a :class:`list` with (tag, the input frame(s)) pairs. Note that more than one input tag is not allowed here. ''' data = list(data) if 'raw' in ndata: data.append(ndata['raw']) tag = ndata.get('tag', self.tag) m = { } for f in data: if isinstance(f, dict): m.update(f) elif tag is None: raise ValueError('No raw input tag') elif tag not in m: m[tag] = f elif isinstance(m[tag], list) \ and not isinstance(m[tag], fits.HDUList): m[tag].append(f) else: m[tag] = [ m[tag], f ] return list(m.items()) def _cleanup(self, output_dir, logger, delete): try: bt = os.path.join(output_dir, 'recipe.backtrace-unprocessed') if os.path.exists(bt): with open(bt) as bt_file: os.rename(bt, os.path.join(output_dir, 'recipe.backtrace')) ex = RecipeCrash(bt_file) ex.log(logger.logger) raise ex finally: if delete: shutil.rmtree(output_dir) def _doc(self): s = '%s\n\n%s\n\n' % (textwrap.fill(self.description[0]), textwrap.fill(self.description[1])) if len(self.param) > 0: r = 'Parameters:\n%s\n' % self._param.__doc__ else: r = 'No parameters\n' if self._recipe.frameConfig() is not None: c = textwrap.fill(repr([f.tag for f in self.calib]), initial_indent = 'Calibration frames: ', subsequent_indent = ' ' * 21) + '\n\n' else: c = '' if self.tags is not None: t = 'Raw and product frames:\n' maxlen = max(len(f) for f in self.tags) for f in self.tags: t += textwrap.fill(repr(self.output[f]), initial_indent = ' %s --> ' % f.rjust(maxlen), subsequent_indent = ' ' * (maxlen + 7)) + '\n' else: t = '' return s + r + c + t + '\n\n' def __repr__(self): return 'Recipe(%s, version = %s)' % (repr(self.__name__), repr(self.version[0])) @staticmethod def list(): '''Return a list of recipes. Searches for all recipes in in the directory specified by the class attribute :attr:`Recipe.path` or its subdirectories. ''' os.putenv('CPL_MEMORY_MODE', str(Recipe.memory_mode)); plugins = collections.defaultdict(list) for f in Recipe.get_libs(): plugin_f = CPL_recipe.list(f) if plugin_f: for p in plugin_f: plugins[p[0]].append(p[2]) return list(plugins.items()) @staticmethod def get_recipefilename(name, version = None): os.putenv('CPL_MEMORY_MODE', str(Recipe.memory_mode)); filename = None rversion = -1 for f in Recipe.get_libs(): plugin_f = CPL_recipe.list(f) if plugin_f: for p in plugin_f: if p[0] != name: continue if version in p[1:3]: return f if rversion < p[1]: rversion = p[1] filename = f return filename @staticmethod def get_libs(): libs = [ ] path = Recipe.path.split(':') if isinstance(Recipe.path, str) else Recipe.path for p in path: for root, d, files in os.walk(p): libs += [ os.path.join(root, f) for f in files if f.endswith('.so') ] return libs @staticmethod def set_maxthreads(n): '''Set the maximal number of threads to be executed in parallel. .. note:: This affects only threads that are started afterwards with the ``threaded = True`` flag. .. seealso:: :ref:`parallel` ''' Threaded.set_maxthreads(n) class Threaded(threading.Thread): '''Simple threading interface. Creating this object will start the execution of func(*args, **nargs). It returns an object that has the same attribute as the function return value, if the function execution was completed. Accessing any of the attributes will cause a wait until the function execution is ready. Note that the attribute delegation will work only for attributes (not for methods), and it will not work for attributes defined by the threading.Thread interface. If the function returns an exception, this exception is thrown by any attempt to access an attribute. ''' pool_sema = threading.BoundedSemaphore(65536) def __init__(self, func, *args, **nargs): threading.Thread.__init__(self) self._func = func self._args = args self._nargs = nargs self._res = None self._exception = None self.start() def run(self): with Threaded.pool_sema: try: self._res = self._func(*self._args, **self._nargs) except Exception as exception: self._exception = exception @property def _result(self): self.join() if self._exception is None: return self._res else: raise self._exception def __getitem__(self, key): return self._result[key] def __contains__(self, key): return key in self._result.tags def __len__(self): return len(self._result.tags) def __iter__(self): return self._result.__iter__() def __getattr__(self, name): return self._result.__dict__[name] @staticmethod def set_maxthreads(n): with Threaded.pool_sema: Threaded.pool_sema = threading.BoundedSemaphore(n) python-cpl-0.7.4/cpl/md5sum.py0000644000175000017500000000111013147022346015534 0ustar oleole00000000000000import hashlib def datamd5(hdulist): '''Calculate the MD5SUM of all data regions of a HDUList. ''' md5sum = hashlib.md5() for hdu in hdulist: if hdu.data is not None: md5sum.update(bytes(hdu.data.data)) pad = b'\0' * (2880 - hdu.data.nbytes % 2880) md5sum.update(pad) return md5sum.hexdigest() def verify_md5(hdulist): return hdulist[0].header.get('DATAMD5') == datamd5(hdulist) def update_md5(hdulist): md5sum = datamd5(hdulist) hdulist[0].header['DATAMD5'] = (md5sum, 'MD5 checksum') return md5sum python-cpl-0.7.4/cpl/__init__.py0000644000175000017500000000260712515646002016074 0ustar oleole00000000000000'''Python interface for the Common Pipeline Library This module can list, configure and execute CPL-based recipes from Python (python2 and python3). The input, calibration and output data can be specified as FITS files or as ``astropy.io.fits`` objects in memory. The ESO `Common Pipeline Library `_ (CPL) comprises a set of ISO-C libraries that provide a comprehensive, efficient and robust software toolkit. It forms a basis for the creation of automated astronomical data-reduction tasks. One of the features provided by the CPL is the ability to create data-reduction algorithms that run as plugins (dynamic libraries). These are called "recipes" and are one of the main aspects of the CPL data-reduction development environment. ''' from __future__ import absolute_import from .version import version as __version__ from .version import author as __author__ from .version import email as __email__ from .version import license_ as __license__ from .recipe import Recipe from .param import Parameter from .frames import FrameConfig from .result import Result, CplError, RecipeCrash from . import dfs from . import esorex from . import CPL_recipe Recipe.dir = '.' cpl_versions = [ '%i.%i.%i' % ver for ver in CPL_recipe.cpl_versions() ] del CPL_recipe del absolute_import del recipe, version, param, frames, result, md5sum try: del ver except NameError: pass python-cpl-0.7.4/cpl/esorex.py0000644000175000017500000002141213147022346015636 0ustar oleole00000000000000'''`EsoRex `_ is a standard execution environment for CPL recipes provided by `ESO `_. ''' import os import logging from .recipe import Recipe from . import logger def load_sof(source): '''Read an :program:`EsoRex` SOF file. :param source: SOF ("Set Of Files") file object or string with SOF file content. :type source: :class:`str` or :class:`file` These files contain the raw and calibration files for a recipe. The content of the file is returned as a map with the tag as key and the list of file names as value. The result of this function may directly set as :attr:`cpl.Recipe.calib` attribute:: import cpl myrecipe = cpl.Recipe('muse_bias') myrecipe.calib = cpl.esorex.read_sof(open('muse_bias.sof')) .. note:: The raw data frame is silently ignored wenn setting :attr:`cpl.Recipe.calib` for MUSE recipes. Other recipes ignore the raw data frame only if it was set manually as :attr:`cpl.Recipe.tag` or in :attr:`cpl.Recipe.tags` since there is no way to automatically distinguish between them. ''' if isinstance(source, str): return load_sof(open(source) if os.path.exists(source) else source.split('\n')) else: res = dict() for line in source: if not line or line.startswith('#'): continue ls = line.split() fn = ls[0] key = ls[1] if key not in res: res[key] = fn elif isinstance(res[key], list): res[key].append(fn) else: res[key] = [ res[key], fn ] return res def load_rc(source = None): '''Read an :program:`EsoRex` configuration file. :param source: Configuration file object, or string with file content. If not set, the :program:`EsoRex` config file :file:`~/.esorex/esorex.rc` is used. :type source: :class:`str` or :class:`file` These files contain configuration parameters for :program:`EsoRex` or recipes. The content of the file is returned as a map with the (full) parameter name as key and its setting as string value. The result of this function may directly set as :attr:`cpl.Recipe.param` attribute:: import cpl myrecipe = cpl.Recipe('muse_bias') myrecipe.param = cpl.esorex.load_rc('muse_bias.rc') ''' if source is None: source = open(os.path.expanduser('~/.esorex/esorex.rc')) if isinstance(source, str): return load_rc(open(source) if os.path.exists(source) else source.split('\n')) else: res = dict() for line in source: if not line or not line.strip() or line.startswith('#'): continue name = line.split('=', 1)[0] value = line.split('=', 1)[1] if name and value: res[name.strip()] = value.strip() return res def init(source = None): '''Set up the logging and the recipe search path from the :file:`esorex.rc` file. :param source: Configuration file object, or string with file content. If not set, the esorex config file :file:`~/.esorex/esorex.rc` is used. :type source: :class:`str` or :class:`file` ''' rc = load_rc(source) if 'esorex.caller.recipe-dir' in rc: Recipe.path = rc['esorex.caller.recipe-dir'].split(':') if 'esorex.caller.msg-level' in rc: msg.level = rc['esorex.caller.msg-level'] if 'esorex.caller.log-level' in rc: log.level = rc['esorex.caller.log-level'] if 'esorex.caller.log-dir' in rc: log.dir = rc['esorex.caller.log-dir'] if 'esorex.caller.log-file' in rc: log.filename = rc['esorex.caller.log-file'] class CplLogger(object): DEBUG = logging.DEBUG INFO = logging.INFO WARN = logging.WARN ERROR = logging.ERROR OFF = logging.CRITICAL + 1 def __init__(self): self.handler = None self._component = False self._time = False self._threadid = False self.format = None self.dir = None self._level = CplLogger.OFF def _init_handler(self): if not self.handler: self.handler = logging.StreamHandler() logging.getLogger().addHandler(self.handler) self.handler.setLevel(self._level) self.handler.setFormatter(logging.Formatter(self.format, '%H:%M:%S')) def _shutdown_handler(self): if self.handler: logging.getLogger().removeHandler(self.handler) self.handler.close() self.handler = None @property def level(self): '''Log level for output to the terminal. Any of [ DEBUG, INFO, WARN, ERROR, OFF ]. ''' return self._level @level.setter def level(self, level): if isinstance(level, (str)): level = logger.level[level.upper()] if level == CplLogger.OFF: self._shutdown_handler() else: self._init_handler() logging.getLogger().setLevel(logging.DEBUG) if self.handler: self.handler.setLevel(level) self._level = level @property def format(self): '''Output format. .. seealso :: `logging.LogRecord attributes `_ Key mappings in the logging output.''' return self._format @format.setter def format(self, fmt): if fmt is None: fmt = '%(asctime)s ' if self._time else '' fmt += '[%(levelname)7s]' fmt += '[%(threadName)s] ' if self._threadid else ' ' fmt += '%(name)s: ' if self._component else '' fmt += '%(message)s' if self.handler: self.handler.setFormatter(logging.Formatter(fmt, '%H:%M:%S')) self._format = fmt @property def component(self): '''If :obj:`True`, attach the component name to output messages. ''' return self._component @component.setter def component(self, enable): self._component = enable self.format = None @property def time(self): '''If :obj:`True`, attach a time tag to output messages. ''' return self._time @time.setter def time(self, enable): self._time = enable self.format = None @property def threadid(self): '''If :obj:`True`, attach a thread tag to output messages. ''' return self._threadid @threadid.setter def threadid(self, enable): self._threadid = enable self.format = None class CplFileLogger(CplLogger): def __init__(self): CplLogger.__init__(self) self._filename = None self.threadid = True self.component = True self.time = True self.level = CplLogger.INFO def _init_handler(self): if not self.handler: if self._filename: if self.dir: fname = os.path.join(self.dir, self._filename) self.handler = logging.FileHandler(fname) else: self.handler = logging.FileHandler(self._filename) else: self.handler = None if self.handler: logging.getLogger().addHandler(self.handler) self.handler.setLevel(self._level) self.handler.setFormatter(logging.Formatter(self.format, '%H:%M:%S')) @property def filename(self): '''Log file name. ''' return self._filename @filename.setter def filename(self, name): if self._filename != name: self._shutdown_handler() self._filename = name self._init_handler() msg = CplLogger() '''This variable is a :class:`CplLogger` instance that provides a convienience stream handler similar to the terminal logging functionality of the CPL. It basically does the same as:: import logging log = logging.getLogger() log.setLevel(logging.INFO) ch = logging.StreamHandler() ch.setLevel(logging.OFF) ch.setFormatter(logging.Formatter('[%(levelname)7s] %(message)s')) log.addHandler(ch) ''' log = CplFileLogger() '''This variable is a :class:`CplFileLogger` instance that provides a convienience file handler similar to the file logging functionality of the CPL. It basically does the same as:: import logging log = logging.getLogger() log.setLevel(logging.INFO) ch = logging.FileHandler(filename) ch.setLevel(logging.INFO) ch.setFormatter(logging.Formatter('%(asctime)s [%(levelname)7s] %(funcName)s: %(message)s')) log.addHandler(ch) ''' python-cpl-0.7.4/cpl/dfs.py0000644000175000017500000002613613370120012015100 0ustar oleole00000000000000import sys from astropy.io import fits import cpl class ProcessingInfo(object): '''Support for reading input files and parameters from the FITS header of a CPL processed file. This is done through the FITS headers that were written by the DFS function called within the processing recipe. .. attribute:: name Recipe name .. attribute:: version Recipe version string .. attribute:: pipeline Pipeline name .. attribute:: cpl_version CPL version string .. attribute:: tag Tag name .. attribute:: calib Calibration frames from a FITS file processed with CPL. The result of this function may directly set as :attr:`cpl.Recipe.calib` attribute:: import cpl myrecipe = cpl.Recipe('muse_bias') myrecipe.calib = cpl.dfs.ProcessingInfo('MASTER_BIAS_0.fits').calib .. note:: This will not work properly for files that had :class:`astropy.io.fits.HDUList` inputs since they have assigned a temporary file name only. .. attribute:: raw Raw (input) frames .. note:: This will not work properly for files that had :class:`astropy.io.fits.HDUList` inputs since they have assigned a temporary file name only. .. attribute:: param Processing parameters. The result of this function may directly set as :attr:`cpl.Recipe.param` attribute:: import cpl myrecipe = cpl.Recipe('muse_bias') myrecipe.param = cpl.dfs.ProcessingInfo('MASTER_BIAS_0.fits').param .. attribute:: md5sum MD5 sum of the data portions of the output file (header keyword 'DATAMD5'). .. attribute:: md5sums MD5 sums of the input and calibration files. :class:`dict` with the file name as key and the corresponding MD5 sum as value. .. note:: Due to a design decision in CPL, the raw input files are not accompanied with the MD5 sum. ''' def __init__(self, source, recno = -1): ''' :param source: Object pointing to the result file header :type source: :class:`str` or :class:`astropy.io.fits.HDUList` or :class:`astropy.io.fits.PrimaryHDU` or :class:`astropy.io.fits.Header` :param recno: Record number. Optional. If not given, the last record (with the highest record number) is used. :type recno: :class:`int` ''' if isinstance(source, str): header = fits.open(source)[0].header elif isinstance(source, (fits.HDUList, list)): header = source[0].header elif isinstance(source, fits.PrimaryHDU): header = source.header elif isinstance(source, (fits.Header, dict)): header = source else: raise ValueError('Cannot assign type {0} to header'.format( source.__class__.__name__)) if recno < 0: for reccnt in range(1, 2**16): if 'HIERARCH ESO PRO REC{0} ID'.format(reccnt) not in header: break recno += reccnt self.name = header['HIERARCH ESO PRO REC{0} ID'.format(recno)] self.product = header['HIERARCH ESO PRO CATG'] self.orig_filename = header['PIPEFILE'] pipe_id = header.get('HIERARCH ESO PRO REC{0} PIPE ID'.format(recno)) if pipe_id: self.pipeline, version = pipe_id.split('/') num_version = 0 for i in version.split('.'): num_version = num_version * 100 + int(i) self.version = (num_version, version) else: self.pipeline = None self.version = None self.cpl_version = header.get('HIERARCH ESO PRO REC{0} DRS ID'.format(recno)) self.md5sum = header.get('DATAMD5') self.md5sums = {} self.calib = ProcessingInfo._get_rec_keys(header, recno, 'CAL', 'CATG', 'NAME') for cat, md5 in ProcessingInfo._get_rec_keys(header, recno, 'CAL', 'CATG', 'DATAMD5').items(): if isinstance(md5, list): for i, m in enumerate(md5): if m is not None: self.md5sums[self.calib[cat][i]] = m elif md5 is not None: self.md5sums[self.calib[cat]] = md5 raw = ProcessingInfo._get_rec_keys(header, recno, 'RAW', 'CATG', 'NAME') if raw: self.tag = list(raw.keys())[0] self.raw = raw[self.tag] md5 = ProcessingInfo._get_rec_keys(header, recno, 'RAW', 'CATG', 'DATAMD5')[self.tag] if isinstance(md5, list): for i, m in enumerate(md5): if m is not None: self.md5sums[self.raw[i]] = m elif md5 is not None: self.md5sums[self.raw] = md5 else: self.tag = None self.raw = None self.input = None param = ProcessingInfo._get_rec_keys(header, recno, 'PARAM', 'NAME', 'VALUE') self.param = dict() for k,v in param.items(): self.param[k] = ProcessingInfo._best_type(v) def create_recipe(self): '''Create a recipe and configure it with the parameters, calibration frames, and the input tag. The recipe version will be the latest available one. ''' recipe = cpl.Recipe(self.name) recipe.param = self.param recipe.calib = self.calib recipe.tag = self.tag return recipe def create_script(self, scriptfile = sys.stdout): '''Create a sample script that creates the recipe, configures it with the parameters, calibration frames and input tags, and finally starts the recipe. ''' if isinstance(scriptfile, str): scriptfile = file(scriptfile, mode='w') scriptfile.write('import cpl\n\n') scriptfile.write('# Recipe: {0}.{1}, Version {2}, CPL version {3}\n'.format( self.pipeline, self.name, self.version[1], self.cpl_version)) scriptfile.write('{0} = cpl.Recipe({1}, version = {2})\n'.format( self.name, repr(self.name), repr(self.version[0]))) scriptfile.write('\n# Parameters:\n') for k,v in self.param.items(): scriptfile.write('{0}.param.{1} = {2}\n'.format(self.name, k, repr(v))) if self.calib: scriptfile.write('\n# Calibration frames:\n') for k,v in self.calib.items(): scriptfile.write('{0}.calib.{1} = {2}\n'.format(self.name, k, repr(v))) scriptfile.write('\n# Process input frames:\n') scriptfile.write('{0}.tag = {1}\n'.format(self.name, repr(self.tag))) scriptfile.write('res = {0}({1})\n'.format(self.name, repr(self.raw))) # scriptfile.write('{0} = res.{1}\n'.format(self.product.lower(), self.product)) # scriptfile.write('{0}.writeto({1})\n'.format(self.product.lower(), # repr(self.orig_filename))) def __str__(self): s = 'Recipe: {0}, Version {1}, CPL version {2}\n'.format( self.name, self.version, self.cpl_version) s += 'Parameters:\n' for k,v in self.param.items(): s += ' {0}.{1}.{2} = {3}\n'.format(self.pipeline, self.name, k, v) if self.calib: s += 'Calibration frames:\n' for k,v in self.calib.items(): if isinstance(v, (str, unicode)): s += ' {0} {1}\n'.format(v,k) else: m = max(len(n) for n in v) for n in v: s += ' {0:<{width}} {1}\n'.format(n, m, width = m) if self.raw is not None: s += 'Input frames:\n' if isinstance(self.raw, (str, unicode)): s += ' {0} {1}\n'.format(self.raw, self.tag) else: m = max(len(n) for n in self.raw) for n in self.raw: s += ' {0:<{width}} {1}\n'.format(n, self.tag, width = m) return s def printinfo(self): '''Print the recipe information to standard output. ''' print(str(self)) @staticmethod def _get_rec_keys(header, recno, key, name, value): '''Get a dictionary of key/value pairs from the DFS section of the header. :param key: Common keyword for the value. Usually 'PARAM' for parameters, 'RAW' for raw frames, and 'CAL' for calibration frames. :type key: :class:`str` :param recno: Record number. :type recno: :class:`int` :param name: Header keyword (last part) for the name of each key :type name: :class:`str` :param value: Header keyword (last part) for the value of each key :type name: :class:`str` When the header HIERARCH ESO PRO REC1 PARAM1 NAME = 'nifu' HIERARCH ESO PRO REC1 PARAM1 VALUE = '1' HIERARCH ESO PRO REC1 PARAM2 NAME = 'combine' HIERARCH ESO PRO REC1 PARAM2 VALUE = 'median' is called with ProcessingInfo._get_rec_keys(1, 'PARAM', 'NAME', 'VALUE') the returned dictionary will contain the keys res['nifu'] = '1' res['combine'] = 'median' ''' res = dict() for i in range(1, 2**16): try: prefix = 'HIERARCH ESO PRO REC{0} {1}{2}'.format(recno, key, i) k = header['{0} {1}'.format(prefix, name)] fn = header.get('{0} {1}'.format(prefix, value)) if k not in res: res[k] = fn elif isinstance(res[k], list): res[k].append(fn) else: res[k] = [ res[k], fn ] except KeyError: break return res @staticmethod def _best_type(value): '''Convert the value to the best applicable type: :class:`int`, :class:`float`, :class:`bool` or :class`str`. :param value: Value to convert. :type value: :class:`str` ''' for t in int, float: try: return t(value) except ValueError: pass return {'true':True, 'false':False}.get(value, value) @staticmethod def list(source): '''Get a list of all `ProcessingInfo` objects in the FITS header. The list is sorted by the execution order. :param source: Object pointing to the result file header :type source: :class:`str` or :class:`astropy.io.fits.HDUList` or :class:`astropy.io.fits.PrimaryHDU` or :class:`astropy.io.fits.Header` ''' pi = [] for i in range(1, 2**16): try: pi.append(ProcessingInfo(source, i)) except KeyError: break return pi if __name__ == '__main__': for arg in sys.argv[1:]: print('{0}\nfile: {1}'.format('-' * 72, arg)) pi = cpl.dfs.ProcessingInfo(arg) pi.printinfo() python-cpl-0.7.4/cpl/CPL_library.h0000644000175000017500000001334513370120012016263 0ustar oleole00000000000000 #ifndef CPL_LIBRARY_H #define CPL_LIBRARY_H /* For the header, either the CPL one can be used, or the header that was extracted from the 6.3 release. For API safety, it is better to include the one provided with python-cpl. The other option is just for the adoption to a new CPL version. */ #ifdef USE_INSTALLED_CPL_HEADER #include #else #include "cpl_api.h" #endif #if CPL_VERSION_CODE < CPL_VERSION(6,3,0) #error CPL version too old. Minimum required version is 6.3.0. #endif #if CPL_VERSION_CODE > CPL_VERSION(7,0,0) #warning Newer CPL version: check API compability with 7.0.0 at http://upstream-tracker.org/versions/cpl.html #endif extern unsigned long supported_versions[]; #define UNKNOWN_VERSION 0 #define KNOWN_MAJOR 1 #define KNOWN_VERSION 2 typedef struct { unsigned long version; int is_supported; typeof(cpl_init) *init; typeof(cpl_end) *end; typeof(cpl_get_description) *get_description; typeof(cpl_memory_dump) *memory_dump; typeof(cpl_memory_is_empty) *memory_is_empty; typeof(cpl_free) *free; typeof(cpl_plugin_get_author) *plugin_get_author; typeof(cpl_plugin_get_copyright) *plugin_get_copyright; typeof(cpl_plugin_get_deinit) *plugin_get_deinit; typeof(cpl_plugin_get_description) *plugin_get_description; typeof(cpl_plugin_get_email) *plugin_get_email; typeof(cpl_plugin_get_exec) *plugin_get_exec; typeof(cpl_plugin_get_init) *plugin_get_init; typeof(cpl_plugin_get_name) *plugin_get_name; typeof(cpl_plugin_get_synopsis) *plugin_get_synopsis; typeof(cpl_plugin_get_version) *plugin_get_version; typeof(cpl_plugin_get_version_string) *plugin_get_version_string; typeof(cpl_pluginlist_delete) *pluginlist_delete; typeof(cpl_pluginlist_find) *pluginlist_find; typeof(cpl_pluginlist_get_first) *pluginlist_get_first; typeof(cpl_pluginlist_get_next) *pluginlist_get_next; typeof(cpl_pluginlist_new) *pluginlist_new; typeof(cpl_dfs_update_product_header) *dfs_update_product_header; typeof(cpl_dfs_sign_products) *dfs_sign_products; typeof(cpl_error_get_code) *error_get_code; typeof(cpl_error_get_file) *error_get_file; typeof(cpl_error_get_function) *error_get_function; typeof(cpl_error_get_line) *error_get_line; typeof(cpl_error_get_message) *error_get_message; typeof(cpl_error_reset) *error_reset; typeof(cpl_error_set_message_macro) *error_set_message_macro; typeof(cpl_errorstate_dump) *errorstate_dump; typeof(cpl_errorstate_get) *errorstate_get; typeof(cpl_frame_get_filename) *frame_get_filename; typeof(cpl_frame_get_group) *frame_get_group; typeof(cpl_frame_get_tag) *frame_get_tag; typeof(cpl_frame_new) *frame_new; typeof(cpl_frame_set_filename) *frame_set_filename; typeof(cpl_frame_set_tag) *frame_set_tag; typeof(cpl_frameset_delete) *frameset_delete; typeof(cpl_frameset_get_position) *frameset_get_position; typeof(cpl_frameset_get_size) *frameset_get_size; typeof(cpl_frameset_insert) *frameset_insert; typeof(cpl_frameset_new) *frameset_new; typeof(cpl_msg_error) *msg_error; typeof(cpl_msg_set_level) *msg_set_level; typeof(cpl_msg_set_log_level) *msg_set_log_level; typeof(cpl_msg_set_log_name) *msg_set_log_name; typeof(cpl_msg_stop_log) *msg_stop_log; typeof(cpl_parameter_get_alias) *parameter_get_alias; typeof(cpl_parameter_get_class) *parameter_get_class; typeof(cpl_parameter_get_context) *parameter_get_context; typeof(cpl_parameter_get_default_bool) *parameter_get_default_bool; typeof(cpl_parameter_get_default_double) *parameter_get_default_double; typeof(cpl_parameter_get_default_int) *parameter_get_default_int; typeof(cpl_parameter_get_default_string) *parameter_get_default_string; typeof(cpl_parameter_get_enum_double) *parameter_get_enum_double; typeof(cpl_parameter_get_enum_int) *parameter_get_enum_int; typeof(cpl_parameter_get_enum_size) *parameter_get_enum_size; typeof(cpl_parameter_get_enum_string) *parameter_get_enum_string; typeof(cpl_parameter_get_help) *parameter_get_help; typeof(cpl_parameter_get_name) *parameter_get_name; typeof(cpl_parameter_get_range_max_double) *parameter_get_range_max_double; typeof(cpl_parameter_get_range_max_int) *parameter_get_range_max_int; typeof(cpl_parameter_get_range_min_double) *parameter_get_range_min_double; typeof(cpl_parameter_get_range_min_int) *parameter_get_range_min_int; typeof(cpl_parameter_get_type) *parameter_get_type; typeof(cpl_parameter_set_bool) *parameter_set_bool; typeof(cpl_parameter_set_double) *parameter_set_double; typeof(cpl_parameter_set_int) *parameter_set_int; typeof(cpl_parameter_set_string) *parameter_set_string; typeof(cpl_parameter_is_enabled) *parameter_is_enabled; typeof(cpl_parameterlist_delete) *parameterlist_delete; typeof(cpl_parameterlist_find) *parameterlist_find; typeof(cpl_parameterlist_get_first) *parameterlist_get_first; typeof(cpl_parameterlist_get_next) *parameterlist_get_next; typeof(cpl_parameterlist_get_size) *parameterlist_get_size; typeof(cpl_recipeconfig_delete) *recipeconfig_delete; typeof(cpl_recipeconfig_get_inputs) *recipeconfig_get_inputs; typeof(cpl_recipeconfig_get_max_count) *recipeconfig_get_max_count; typeof(cpl_recipeconfig_get_min_count) *recipeconfig_get_min_count; typeof(cpl_recipeconfig_get_outputs) *recipeconfig_get_outputs; typeof(cpl_recipeconfig_get_tags) *recipeconfig_get_tags; typeof(cpl_version_get_version) *version_get_version; cpl_recipeconfig *(*get_recipeconfig)(cpl_recipe *); cpl_type TYPE_BOOL; cpl_type TYPE_INT; cpl_type TYPE_DOUBLE; cpl_type TYPE_STRING; } cpl_library_t; cpl_library_t *create_library(const char *fname); #endif /* CPL_LIBRARY_H */ python-cpl-0.7.4/cpl/result.py0000644000175000017500000003305113370120012015634 0ustar oleole00000000000000import collections import os import signal import logging from astropy.io import fits class Result(object): def __init__(self, directory, res, input_len = 0, logger = None, output_format = fits.HDUList): '''Build an object containing all result frames. Calling :meth:`cpl.Recipe.__call__` returns an object that contains all result ('production') frames in attributes. All results for one tag are summarized in one attribute of the same name. If the argument `output_format` is :class:`astropy.io.fits.HDUList` (default), then the attribute content is either a :class:`astropy.io.fits.HDUList` or a class:`list` of HDU lists, depending on the recipe and the call: If the recipe produces one out put frame of a tag per input file, the attribute contains a list if the recipe was called with a list, and if the recipe was called with a single input frame, the result attribute will also contain a single input frame. If the recipe combines all input frames to one output frame, a single :class:`astropy.io.fits.HDUList` es returned, independent of the input parameters. Similarly, if the argument `output_format` is set to :class:`str`, the attribute content is either a :class:`str` or a class:`list` of :class:`str`, containing the paths of output files. In this case, removing the output files is suppressed. .. todo:: This behaviour is made on some heuristics based on the number and type of the input frames. The heuristics will go wrong if there is only one input frame, specified as a list, but the recipe tries to summarize the input. In this case, the attribute will contain a list where a single :class:`astropy.io.fits.HDUList` was expected. To solve this problem, the "MASTER" flag has to be forwarded from the (MUSE) recipe which means that it should be exported by the recipe -- this is a major change since it probably leads into a replacement of CPLs recipeconfig module by something more sophisticated. And this is not usable for non-MUSE recipes anyway. So, we will skip this to probably some distant future. ''' self.dir = os.path.abspath(directory) logger.join() if res[2][0]: raise CplError(res[2][0], res[1], logger) self.tags = set() for tag, frame in res[0]: if (output_format == fits.HDUList): # Move the file to the base dir to avoid NFS problems outframe = os.path.join( os.path.dirname(self.dir), '%s.%s' % (os.path.basename(self.dir), frame)) os.rename(os.path.join(self.dir, frame), outframe) else: outframe = os.path.join(self.dir, frame) if output_format == fits.HDUList: hdulist = fits.open(outframe, memmap = True, mode = 'update') hdulist.readall() os.remove(outframe) outframe = hdulist tag = tag if tag not in self.__dict__: self.__dict__[tag] = outframe if input_len != 1 \ else [ outframe ] self.tags.add(tag) elif isinstance(self.__dict__[tag], (fits.HDUList, str)): self.__dict__[tag] = [ self.__dict__[tag], outframe ] else: self.__dict__[tag].append(outframe) mtracefname = os.path.join(self.dir, 'recipe.mtrace') mtrace = None if os.path.exists(mtracefname): try: mtrace = os.popen("mtrace %s" % mtracefname).read(); except: mtrace = None self.stat = Stat(res[2], mtrace) self.error = CplError(res[2][0], res[1], logger) if res[1] else None self.log = logger.entries if logger else None def __getitem__(self, key): if key in self.tags: return self.__dict__[key] else: raise KeyError(key) def __contains__(self, key): return key in self.tags def __len__(self): return len(self.tags) def __iter__(self): return iter((key, self.__dict__[key]) for key in self.tags) class Stat(object): def __init__(self, stat, mtrace): self.return_code = stat[0] self.user_time = stat[1] self.sys_time = stat[2] self.memory_is_empty = { -1:None, 0:False, 1:True }[stat[3]] self.mtrace = mtrace; class CplError(Exception): '''Error message from the recipe. If the CPL recipe invocation returns an error, it is converted into a :class:`cpl.CplError` exception and no frames are returned. Also, the error is notified in the log file. The exception is raised on recipe invocation, or when accessing the result frames if the recipe was started in background (:attr:`cpl.Recipe.threaded` set to :obj:`True`). Attributes: .. attribute:: code The CPL error code returned from the recipe. .. attribute:: msg The supplied error message. .. attribute:: filename The source file name where the error occurred. .. attribute:: line The line number where the error occurred. .. attribute:: log Log lines of the recipe that lead to this exception. .. seealso:: :class:`cpl.logger.LogList` .. attribute:: next_error Next error, or :obj:`None`. ''' def __init__(self, retval, res, logger = None): self.retval = retval self.log = logger.entries if logger else None self.next_error = None if not res: self.code, self.msg, self.file, self.line, self.function = ( None, None, None, None, None) else: self.code, self.msg, self.file, self.line, self.function = res[0] o = self for r in res[1:]: o.next_error = CplError(retval, [ r ], logger) o = o.next_error def __iter__(self): class Iter(object): current = self def __next__(self): if Iter.current is None: raise StopIteration s = Iter.current Iter.current = Iter.current.next_error return s def next(self): return self.__next__() return Iter() def __str__(self): if self.code is None: s = 'Unspecified' else: s = "%s (%i) in %s() (%s:%i)" % (self.msg, self.code, self.function, self.file, self.line) if self.next_error: for e in self.next_error: s += "\n %s (%i) in %s() (%s:%i)" % (e.msg, e.code, e.function, e.file, e.line) return s class RecipeCrash(Exception): '''Recipe crash exception If the CPL recipe crashes with a SIGSEV or a SIGBUS, the C stack trace is tried to conserved in this exception. The stack trace is obtained with the GNU debugger gdb. If the debugger is not available, or if the debugger cannot be attached to the crashed recipe, the Exception remains empty. When converted to a string, the Exception will return a stack trace similar to the Python stack trace. The exception is raised on recipe invocation, or when accessing the result frames if the recipe was started in background (:attr:`cpl.Recipe.threaded` set to :obj:`True`). Attributes: .. attribute:: elements List of stack elements, with the most recent element (the one that caused the crash) at the end. Each stack element is a :func:`collections.namedtuple` with the following attributes: .. attribute:: filename Source file name, including full path, if available. .. attribute:: line Line number, if available .. attribute:: func Function name, if available .. attribute:: params Dictionary parameters the function was called with. The key here is the parameter name, the value is a string describing the value set. .. attribute:: localvars Dictionary of local variables of the function, if available. The key here is the parameter name, the value is a string describing the value set. .. attribute:: signal Signal that caused the crash. ''' StackElement = collections.namedtuple('StackElement', 'filename line func params localvars') signals = {signal.SIGSEGV:'SIGSEV: Segmentation Fault', signal.SIGBUS:'SIGBUS: Bus Error', signal.SIGHUP:'SIGHUP: Hangup', signal.SIGABRT:'SIGABRT: Abnormal process termination', signal.SIGTERM:'SIGTERM: Terminated by user', signal.SIGQUIT:'SIGQUIT: Quit', signal.SIGFPE:'SIGFPE: Arithmetic Exception', signal.SIGINT:'SIGINT: Interrupt (Ctrl-C)', None:'Memory inconsistency detected'} def __init__(self, bt_file): self.elements = [] current_element = None parse_functions = True parse_sourcelist = False sourcefiles = dict() self.signal = None self.lines = [] for line in bt_file: self.lines.append(line) if line.startswith('Received signal:'): self.signal = int(line.split(':')[1]) if line.startswith('Memory corruption'): self.signal = None elif line.find('signal handler called') >= 0: del self.elements[:] elif parse_functions: if line.startswith('#'): try: current_element = self._parse_function_line(line) except StopIteration: parse_functions = False elif current_element is not None: self._add_variable(current_element.localvars, line) if line.startswith('Source files'): parse_sourcelist = True parse_functions = False elif parse_sourcelist: sourcefiles.update(dict((os.path.basename(s.strip()), s.strip()) for s in line.split(',') if s.rfind('/') > 0 )) self.elements = [ RecipeCrash.StackElement(sourcefiles.get(e.filename, e.filename), e.line, e.func, e.params, e.localvars) for e in self.elements ] Exception.__init__(self, str(self)) def _add_variable(self, variables, line): s = line.strip().split('=', 1) if len(s) > 1: variables[s[0].strip()] = s[1].strip() def _parse_function_line(self, line): s = line.split() funcname = s[3] if s[1].startswith('0x') else s[1] if funcname.startswith('Py'): raise StopIteration() pars = {} for fp in line[line.find('(')+1:line.rfind(')')].split(','): self._add_variable(pars, fp) l = line[line.rfind(')')+1:].split() if not l: return None source = l[-1].split(':') filename = source[0] lineno = int(source[1]) if len(source) > 1 else None current_element = RecipeCrash.StackElement(filename, lineno, funcname, pars, {}) self.elements.insert(0, current_element) return current_element def log(self, logger): '''Put the content of the crash into the log. ''' log = logging.getLogger('%s' % logger.name) log.error('Recipe crashed. Traceback (most recent call last):') for e in self.elements: logc = logging.getLogger('%s.%s' % (logger.name, e.func)) logc.error(' File "%s", %sin %s\n',e.filename, 'line %i, ' % e.line if e.line else '', e.func) if os.path.exists(e.filename) and e.line: logc.error(' %s\n', open(e.filename).readlines()[e.line-1].strip()) if e.params: logc.error(' Parameters:') for p, v in e.params.items(): logc.error(' %s = %s', p, v) if e.localvars: logc.error(' Local variables:') for p, v in e.localvars.items(): logc.error(' %s = %s', p, v) log.error(RecipeCrash.signals.get(self.signal, '%s: Unknown' % str(self.signal))) def __repr__(self): return 'RecipeCrash()' def __str__(self): s = 'Recipe Traceback (most recent call last):\n' for e in self.elements: s += ' File "%s", %sin %s\n' % ((e.filename), 'line %i, ' % e.line if e.line else '', e.func) if os.path.exists(e.filename) and e.line: s += ' %s\n' % open(e.filename).readlines()[e.line-1].strip() s += RecipeCrash.signals.get(self.signal, '%s: Unknown' % str(self.signal)) return s python-cpl-0.7.4/cpl/version.py0000644000175000017500000000013613370121236016012 0ustar oleole00000000000000version = '0.7.4' author = 'Ole Streicher' email = 'python-cpl@liska.ath.cx' license_ = 'GPL' python-cpl-0.7.4/cpl/cpl_api.h0000644000175000017500000002101613370120012015522 0ustar oleole00000000000000/* This header file is compiled from the original CPL header files to contain just the functions and macros that we need in the framework. Since it is mainly copied and pasted, here is the original license statement from /usr/include/cpl.h: * Id: cpl.h,v 1.31 2009/12/02 10:29:45 lbilbao Exp * * This file is part of the ESO Common Pipeline Library * Copyright (C) 2001-2008 European Southern Observatory * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #ifndef CPL_API_H #define CPL_API_H #define CPL_VERSION(major, minor, micro) \ (((major) * 65536) + ((minor) * 256) + (micro)) #define CPL_VERSION_MAJOR_CODE(code) (((code) >> 16) & 0xff) #define CPL_VERSION_MINOR_CODE(code) (((code) >> 8) & 0xff) #define CPL_VERSION_MICRO_CODE(code) ((code) & 0xff) #define CPL_VERSION_CODE CPL_VERSION(7,0,0) typedef int cpl_error_code, cpl_errorstate, cpl_boolean, cpl_frame_group, cpl_parameter_mode, cpl_parameter_class, cpl_type, cpl_msg_severity; typedef long long cpl_size; typedef void cpl_pluginlist, cpl_frameset, cpl_frame, cpl_parameter, cpl_parameterlist, cpl_recipeconfig; typedef struct _cpl_plugin_ cpl_plugin; typedef int (*cpl_plugin_func)(cpl_plugin *); struct _cpl_plugin_ { unsigned int api; unsigned long version; unsigned long type; const char *name; const char *synopsis; const char *description; const char *author; const char *email; const char *copyright; cpl_plugin_func initialize; cpl_plugin_func execute; cpl_plugin_func deinitialize; }; struct _cpl_recipe_ { cpl_plugin interface; cpl_parameterlist *parameters; cpl_frameset *frames; }; typedef struct _cpl_recipe_ cpl_recipe; unsigned int cpl_version_get_major(void); unsigned int cpl_version_get_minor(void); unsigned int cpl_version_get_micro(void); void cpl_init(unsigned); void cpl_end(void); const char * cpl_get_description(unsigned); int cpl_memory_is_empty(void); void cpl_memory_dump(void); void cpl_free(void *); const char *cpl_plugin_get_author(const cpl_plugin *self); const char *cpl_plugin_get_copyright(const cpl_plugin *self); cpl_plugin_func cpl_plugin_get_deinit(const cpl_plugin *self); const char *cpl_plugin_get_description(const cpl_plugin *self); const char *cpl_plugin_get_email(const cpl_plugin *self); cpl_plugin_func cpl_plugin_get_exec(const cpl_plugin *self); cpl_plugin_func cpl_plugin_get_init(const cpl_plugin *self); const char *cpl_plugin_get_name(const cpl_plugin *self); const char *cpl_plugin_get_synopsis(const cpl_plugin *self); unsigned long cpl_plugin_get_version(const cpl_plugin *self); char *cpl_plugin_get_version_string(const cpl_plugin *self); void cpl_pluginlist_delete(cpl_pluginlist *); cpl_plugin *cpl_pluginlist_find(cpl_pluginlist *, const char *); cpl_plugin *cpl_pluginlist_get_first(cpl_pluginlist *); cpl_plugin *cpl_pluginlist_get_next(cpl_pluginlist *); cpl_pluginlist *cpl_pluginlist_new(void); cpl_error_code cpl_dfs_update_product_header(cpl_frameset *); cpl_error_code cpl_dfs_sign_products(const cpl_frameset *, unsigned int); void cpl_msg_error(const char *, const char *, ...); cpl_error_code cpl_error_get_code(void); const char *cpl_error_get_file(void); const char *cpl_error_get_function(void); unsigned cpl_error_get_line(void); const char *cpl_error_get_message(void); void cpl_error_reset(void); cpl_error_code cpl_error_set_message_macro(const char *, cpl_error_code, const char *, unsigned, const char *, ...); void cpl_errorstate_dump(cpl_errorstate, cpl_boolean, void (*)(unsigned, unsigned, unsigned)); cpl_errorstate cpl_errorstate_get(void); const char *cpl_frame_get_filename(const cpl_frame *self); cpl_frame_group cpl_frame_get_group(const cpl_frame *self); const char *cpl_frame_get_tag(const cpl_frame *self); cpl_frame *cpl_frame_new(void); cpl_error_code cpl_frame_set_filename(cpl_frame *self, const char *filename); cpl_error_code cpl_frame_set_tag(cpl_frame *self, const char *tag); void cpl_frameset_delete(cpl_frameset *self); cpl_frame *cpl_frameset_get_position(cpl_frameset *self, cpl_size position); cpl_size cpl_frameset_get_size(const cpl_frameset *self); cpl_error_code cpl_frameset_insert(cpl_frameset *self, cpl_frame *frame); cpl_frameset *cpl_frameset_new(void); void cpl_msg_set_level(cpl_msg_severity); cpl_error_code cpl_msg_set_log_level(cpl_msg_severity); cpl_error_code cpl_msg_set_log_name(const char *); cpl_error_code cpl_msg_stop_log(void); const char *cpl_parameter_get_alias(const cpl_parameter *self, cpl_parameter_mode mode); cpl_parameter_class cpl_parameter_get_class(const cpl_parameter *self); const char *cpl_parameter_get_context(const cpl_parameter *self); int cpl_parameter_get_default_bool(const cpl_parameter *self); int cpl_parameter_get_default_int(const cpl_parameter *self); double cpl_parameter_get_default_double(const cpl_parameter *self); const char *cpl_parameter_get_default_string(const cpl_parameter *self); int cpl_parameter_get_enum_size(const cpl_parameter *self); int cpl_parameter_get_enum_int(const cpl_parameter *self, int position); double cpl_parameter_get_enum_double(const cpl_parameter *self, int position); const char *cpl_parameter_get_enum_string(const cpl_parameter *self, int position); const char *cpl_parameter_get_help(const cpl_parameter *self); const char *cpl_parameter_get_name(const cpl_parameter *self); int cpl_parameter_get_range_min_int(const cpl_parameter *self); double cpl_parameter_get_range_min_double(const cpl_parameter *self); int cpl_parameter_get_range_max_int(const cpl_parameter *self); double cpl_parameter_get_range_max_double(const cpl_parameter *self); cpl_type cpl_parameter_get_type(const cpl_parameter *self); cpl_error_code cpl_parameter_set_bool(cpl_parameter *self, int value); cpl_error_code cpl_parameter_set_int(cpl_parameter *self, int value); cpl_error_code cpl_parameter_set_double(cpl_parameter *self, double value); cpl_error_code cpl_parameter_set_string(cpl_parameter *self, const char *value); int cpl_parameter_is_enabled(const cpl_parameter *self, cpl_parameter_mode mode); void cpl_parameterlist_delete(cpl_parameterlist *self); cpl_parameter *cpl_parameterlist_find(cpl_parameterlist *self, const char *name); cpl_parameter *cpl_parameterlist_get_first(cpl_parameterlist *self); cpl_parameter *cpl_parameterlist_get_next(cpl_parameterlist *self); cpl_size cpl_parameterlist_get_size(const cpl_parameterlist *self); void cpl_recipeconfig_delete(const cpl_recipeconfig* self); char** cpl_recipeconfig_get_inputs(const cpl_recipeconfig* self, const char* tag); cpl_size cpl_recipeconfig_get_min_count(const cpl_recipeconfig* self, const char* tag, const char* input); cpl_size cpl_recipeconfig_get_max_count(const cpl_recipeconfig* self, const char* tag, const char* input); char** cpl_recipeconfig_get_outputs(const cpl_recipeconfig* self, const char* tag); char** cpl_recipeconfig_get_tags(const cpl_recipeconfig* self); const char *cpl_version_get_version(void); #define CPL_INIT_DEFAULT 0 #define CPL_DESCRIPTION_DEFAULT 0 #define CPL_MSG_OFF 4 #define CPL_FALSE 0 #define CPL_ERROR_NONE 0 #define CPL_ERROR_FILE_NOT_CREATED 8 #define CPL_FRAME_GROUP_PRODUCT 3 #define CPL_PARAMETER_CLASS_ENUM (1 << 3) #define CPL_PARAMETER_CLASS_RANGE (1 << 2) #define CPL_PARAMETER_MODE_CLI (1 << 0) #define CPL_PARAMETER_MODE_ENV (1 << 1) #define CPL_PARAMETER_MODE_CFG (1 << 2) #define CPL_TYPE_BOOL (1 << 7) #define CPL_TYPE_DOUBLE (1 << 17) #define CPL_TYPE_INT (1 << 10) #define CPL_TYPE_STRING ((1 << 5)|(1 << 0)) #define CPL_DFS_SIGNATURE_DATAMD5 (1 << 0) #define CPL_DFS_SIGNATURE_CHECKSUM (1 << 1) #endif /* CPL_API_H */ python-cpl-0.7.4/cpl/param.py0000644000175000017500000002067213370120012015423 0ustar oleole00000000000000import textwrap class Parameter(object): '''Runtime configuration parameter of a recipe. Parameters are designed to handle monitor/control data and they provide a standard way to pass information to the recipe. The CPL implementation supports three classes of parameters: a plain value, a value within a given range, or a value as part of an enumeration. When a parameter is created it is created for a particular value type. In the latter two cases, validation is performed whenever the value is set. Attributes: .. attribute:: Parameter.value The value of the parameter, or :obj:`None` if set to default .. attribute:: Parameter.default The default value of the parameter (readonly). .. attribute:: Parameter.name The parameter name (readonly). Parameter names are unique. They define the identity of a given parameter. .. attribute:: Parameter.context The parameter context (readonly). The context usually consists of the instrument name and the recipe name, separated by a dot. The context is used to associate parameters together. .. attribute: Parameter.fullname The parameter name including the context (readonly). The fullname usually consists of the parameter context and the parameter name, separated by a dot. .. attribute:: Parameter.range The numeric range of a parameter, or :obj:`None` if the parameter range is unlimited (readonly). .. attribute:: Parameter.sequence A :class:`list` of possible values for the parameter if the parameter are limited to an enumeration of possible values (readonly). The following example prints the attributes of one parameter: >>> print 'name: ', muse_scibasic.param.cr.name name: cr >>> print 'fullname:', muse_scibasic.param.cr.fullname fullname: muse.muse_scibasic.cr >>> print 'context: ', muse_scibasic.param.cr.context context: muse.muse_scibasic >>> print 'sequence:', muse_scibasic.param.cr.sequence sequence: ['dcr', 'none'] >>> print 'range: ', muse_scibasic.param.cr.range range: None >>> print 'default: ', muse_scibasic.param.cr.default default: dcr >>> print 'value: ', muse_scibasic.param.cr.value value: None ''' def __init__(self, name): self.name = name self._value = None def _set_attributes(self, context = None, fullname = None, default = None, desc = None, range_ = None, sequence = None, ptype = None, enabled = None): self.context = context self.range = range_ self.sequence = sequence self.default = default self.fullname = fullname self.type = ptype or default.__class__ self.enabled = enabled self.__doc__ = textwrap.fill("%s (%s; default: %s)" % (desc, self.type.__name__, repr(self.default))) @property def value(self): return self._value @value.setter def value(self, value): if value is not None and self.type is not None.__class__: if self.type is bool and isinstance(value, str): d = {'true':True, 'false':False, 'yes':True, 'no':False} value = d.get(value.lower(), value) value = self.type(value) if self.sequence and value not in self.sequence: raise ValueError("'%s' is not in %s" % (value, self.sequence)) if self.range and not (self.range[0] <= value <= self.range[-1]): raise ValueError("'%s' is not in range %s" % (value, self.range)) self._value = value @value.deleter def value(self): self._value = None def __str__(self): return '%s%s' % ( repr(self.value if self.value is not None else self.default), ' (default)' if self.value is None else '') def __repr__(self): return 'Parameter(%s, %s=%s)' % ( repr(self.name), "value" if self.value is not None else "default", repr(self.value if self.value is not None else self.default)) def __getitem__(self,i): return (self.name, self.value or self.default)[i] class ParameterList(object): def __init__(self, recipe, other = None, prefix = None): self._recipe = recipe self._dict = dict() self._pars = list() self._prefix = prefix childs = set() for name, context, fullname, desc, prange, \ sequence, deflt, ptype, enabled in recipe._recipe.params(): if prefix: if name.startswith(prefix + '.'): aname = name[len(prefix)+1:] else: continue else: aname = name if '.' in aname: aname = aname.split('.', 1)[0] if prefix: aname = prefix + '.' + aname childs.add(aname) else: par = Parameter(name) par._set_attributes(context, fullname, deflt, desc, prange, sequence, ptype, enabled) self._dict[name] = par self._dict[fullname] = par self._dict[self._paramname(aname)] = par self._pars.append(par) for name in childs: clist = ParameterList(recipe, prefix = name) self._dict[name] = clist for par in clist._pars: self._dict[par.name] = par self._dict[par.fullname] = par self._pars.append(par) aname = self._paramname(name) self._dict[aname] = clist if other: self._set_items(other) self.__doc__ = self._doc() def _set_items(self, other): if isinstance(other, self.__class__): l = ((o.name, o.value) for o in other) elif isinstance(other, dict): l = other.items() else: l = other for o in l: self[o[0]] = o[1] def _del_items(self): for p in self: del p.value @staticmethod def _paramname(s): for c in [ '-', ' ' ]: if isinstance(c, tuple): s = s.replace(c[0], c[1]) else: s = s.replace(c, '_') return s def __iter__(self): return self._pars.__iter__() def __getitem__(self, key): return self._dict[key] def __setitem__(self, key, value): p = self[key] if isinstance(p, self.__class__): if value is not None: p._set_items(value) else: p._del_items() else: p.value = value def __delitem__(self, key): p = self[key] if isinstance(p, self.__class__): p._del_items() else: del p.value def __str__(self): return dict(iter(self)).__str__() def __contains__(self, key): return key in self._dict def __len__(self): return len(self._pars) def __getattr__(self, key): return self[key] def __setattr__(self, key, value): if key.startswith('_'): super(ParameterList, self).__setattr__(key, value) else: self[key] = value def __delattr__(self, key): del self[key] def __dir__(self): return list(set(self._paramname(d) for d in self._dict.keys() if '.' not in d)) def __repr__(self): return repr(dict(iter(self))) def __eq__(self, other): return dict(iter(self)) == other def _doc(self): if len(self) == 0: return 'No parameters' r = '' maxlen = max(len(p.name) for p in self) for p in self: r += textwrap.fill( p.__doc__, subsequent_indent = ' ' * (maxlen + 3), initial_indent = ' %s: ' % p.name.rjust(maxlen)) + '\n' return r def _aslist(self, par): parlist = ParameterList(self._recipe, self) if par is not None: parlist._set_items(par.items()) l = list() for param in parlist: if isinstance(param, Parameter): if param.value is not None: l.append((param.fullname, param.value)) else: l += param._aslist(par) return l python-cpl-0.7.4/cpl/CPL_library.c0000644000175000017500000002450313370120012016254 0ustar oleole00000000000000#include #include #include "CPL_library.h" unsigned long supported_versions[] = { CPL_VERSION(7,1,0), CPL_VERSION(7,0,0), CPL_VERSION(6,6,1), CPL_VERSION(6,6,0), CPL_VERSION(6,5,1), CPL_VERSION(6,5,0), CPL_VERSION(6,4,1), CPL_VERSION(6,4,0), CPL_VERSION(6,3,1), CPL_VERSION(6,3,0), CPL_VERSION(6,2,0), CPL_VERSION(6,1,1), CPL_VERSION(6,0,1), CPL_VERSION(6,0,0), CPL_VERSION(5,3,1), CPL_VERSION(5,2,0), CPL_VERSION(5,1,0), CPL_VERSION(5,0,1), CPL_VERSION(5,0,0), CPL_VERSION(4,2,0), CPL_VERSION(4,1,0), CPL_VERSION(4,0,1), CPL_VERSION(4,0,0), 0 }; static cpl_library_t **libraries = NULL; /* This module provides all needed functions to run a recipe from the framework. These functions are extracted from the recipe shared lib (resp. the CPL linked to that) by their names. I checked that the API didn't change from CPL 4.0 which is now the minimal supported version here. Since some constants changed over the time, all used constants are also included in the structure. The constants are set directly depending on the CPL version number. Note that beta releases have to be taken very cautiously here since they may contain incompatible changes here. */ cpl_library_t *create_library(const char *fname) { void *handle = dlopen(fname, RTLD_LAZY); if (handle == NULL) { return NULL; } char *error = dlerror(); typeof(cpl_init) *init = dlsym(handle, "cpl_init"); error = dlerror(); if (error != NULL) { dlclose(handle); return NULL; } if (libraries == NULL) { libraries = malloc(sizeof(cpl_library_t *)); libraries[0] = NULL; } int i; for (i = 0; libraries[i] != NULL; i++) { if (init == libraries[i]->init) { dlclose(handle); return libraries[i]; } } cpl_library_t *cpl = malloc(sizeof(cpl_library_t)); cpl->init = init; cpl->init(CPL_INIT_DEFAULT); typeof(cpl_version_get_major) *get_major = dlsym(handle, "cpl_version_get_major"); typeof(cpl_version_get_minor) *get_minor = dlsym(handle, "cpl_version_get_minor"); typeof(cpl_version_get_micro) *get_micro = dlsym(handle, "cpl_version_get_micro"); cpl->version = CPL_VERSION(get_major(), get_minor(), get_micro()); cpl->end = dlsym(handle, "cpl_end"); cpl->version_get_version = dlsym(handle, "cpl_version_get_version"); cpl->get_description = dlsym(handle, "cpl_get_description"); cpl->memory_dump = dlsym(handle, "cpl_memory_dump"); cpl->memory_is_empty = dlsym(handle, "cpl_memory_is_empty"); cpl->free = dlsym(handle, "cpl_free"); cpl->plugin_get_author = dlsym(handle, "cpl_plugin_get_author"); cpl->plugin_get_copyright = dlsym(handle, "cpl_plugin_get_copyright"); cpl->plugin_get_deinit = dlsym(handle, "cpl_plugin_get_deinit"); cpl->plugin_get_description = dlsym(handle, "cpl_plugin_get_description"); cpl->plugin_get_email = dlsym(handle, "cpl_plugin_get_email"); cpl->plugin_get_exec = dlsym(handle, "cpl_plugin_get_exec"); cpl->plugin_get_init = dlsym(handle, "cpl_plugin_get_init"); cpl->plugin_get_name = dlsym(handle, "cpl_plugin_get_name"); cpl->plugin_get_synopsis = dlsym(handle, "cpl_plugin_get_synopsis"); cpl->plugin_get_version = dlsym(handle, "cpl_plugin_get_version"); cpl->plugin_get_version_string = dlsym(handle, "cpl_plugin_get_version_string"); cpl->pluginlist_delete = dlsym(handle, "cpl_pluginlist_delete"); cpl->pluginlist_find = dlsym(handle, "cpl_pluginlist_find"); cpl->pluginlist_get_first = dlsym(handle, "cpl_pluginlist_get_first"); cpl->pluginlist_get_next = dlsym(handle, "cpl_pluginlist_get_next"); cpl->pluginlist_new = dlsym(handle, "cpl_pluginlist_new"); cpl->dfs_update_product_header = dlsym(handle, "cpl_dfs_update_product_header"); if (cpl->version >= CPL_VERSION(6,5,0)) { cpl->dfs_sign_products = dlsym(handle, "cpl_dfs_sign_products"); } else { cpl->dfs_sign_products = NULL; } cpl->error_get_code = dlsym(handle, "cpl_error_get_code"); cpl->error_get_file = dlsym(handle, "cpl_error_get_file"); cpl->error_get_function = dlsym(handle, "cpl_error_get_function"); cpl->error_get_line = dlsym(handle, "cpl_error_get_line"); cpl->error_get_message = dlsym(handle, "cpl_error_get_message"); cpl->error_reset = dlsym(handle, "cpl_error_reset"); cpl->error_set_message_macro = dlsym(handle, "cpl_error_set_message_macro"); cpl->errorstate_dump = dlsym(handle, "cpl_errorstate_dump"); cpl->errorstate_get = dlsym(handle, "cpl_errorstate_get"); cpl->frame_get_filename = dlsym(handle, "cpl_frame_get_filename"); cpl->frame_get_group = dlsym(handle, "cpl_frame_get_group"); cpl->frame_get_tag = dlsym(handle, "cpl_frame_get_tag"); cpl->frame_new = dlsym(handle, "cpl_frame_new"); cpl->frame_set_filename = dlsym(handle, "cpl_frame_set_filename"); cpl->frame_set_tag = dlsym(handle, "cpl_frame_set_tag"); cpl->frameset_delete = dlsym(handle, "cpl_frameset_delete"); if (cpl->version >= CPL_VERSION(6,3,0)) { cpl->frameset_get_position = dlsym(handle, "cpl_frameset_get_position"); } else { // fallback variant: not threadsafe, deprecated after 6.2 cpl->frameset_get_position = dlsym(handle, "cpl_frameset_get_frame"); } cpl->frameset_get_size = dlsym(handle, "cpl_frameset_get_size"); cpl->frameset_insert = dlsym(handle, "cpl_frameset_insert"); cpl->frameset_new = dlsym(handle, "cpl_frameset_new"); cpl->msg_error = dlsym(handle, "cpl_msg_error"); cpl->msg_set_level = dlsym(handle, "cpl_msg_set_level"); cpl->msg_set_log_level = dlsym(handle, "cpl_msg_set_log_level"); cpl->msg_set_log_name = dlsym(handle, "cpl_msg_set_log_name"); cpl->msg_stop_log = dlsym(handle, "cpl_msg_stop_log"); cpl->parameter_get_alias = dlsym(handle, "cpl_parameter_get_alias"); cpl->parameter_get_class = dlsym(handle, "cpl_parameter_get_class"); cpl->parameter_get_context = dlsym(handle, "cpl_parameter_get_context"); cpl->parameter_get_default_bool = dlsym(handle, "cpl_parameter_get_default_bool"); cpl->parameter_get_default_double = dlsym(handle, "cpl_parameter_get_default_double"); cpl->parameter_get_default_int = dlsym(handle, "cpl_parameter_get_default_int"); cpl->parameter_get_default_string = dlsym(handle, "cpl_parameter_get_default_string"); cpl->parameter_get_enum_double = dlsym(handle, "cpl_parameter_get_enum_double"); cpl->parameter_get_enum_int = dlsym(handle, "cpl_parameter_get_enum_int"); cpl->parameter_get_enum_size = dlsym(handle, "cpl_parameter_get_enum_size"); cpl->parameter_get_enum_string = dlsym(handle, "cpl_parameter_get_enum_string"); cpl->parameter_get_help = dlsym(handle, "cpl_parameter_get_help"); cpl->parameter_get_name = dlsym(handle, "cpl_parameter_get_name"); cpl->parameter_get_range_max_double = dlsym(handle, "cpl_parameter_get_range_max_double"); cpl->parameter_get_range_max_int = dlsym(handle, "cpl_parameter_get_range_max_int"); cpl->parameter_get_range_min_double = dlsym(handle, "cpl_parameter_get_range_min_double"); cpl->parameter_get_range_min_int = dlsym(handle, "cpl_parameter_get_range_min_int"); cpl->parameter_get_type = dlsym(handle, "cpl_parameter_get_type"); cpl->parameter_set_bool = dlsym(handle, "cpl_parameter_set_bool"); cpl->parameter_set_double = dlsym(handle, "cpl_parameter_set_double"); cpl->parameter_set_int = dlsym(handle, "cpl_parameter_set_int"); cpl->parameter_set_string = dlsym(handle, "cpl_parameter_set_string"); cpl->parameter_is_enabled = dlsym(handle, "cpl_parameter_is_enabled"); cpl->parameterlist_delete = dlsym(handle, "cpl_parameterlist_delete"); cpl->parameterlist_find = dlsym(handle, "cpl_parameterlist_find"); cpl->parameterlist_get_first = dlsym(handle, "cpl_parameterlist_get_first"); cpl->parameterlist_get_next = dlsym(handle, "cpl_parameterlist_get_next"); cpl->parameterlist_get_size = dlsym(handle, "cpl_parameterlist_get_size"); cpl->recipeconfig_delete = dlsym(handle, "cpl_recipeconfig_delete"); cpl->recipeconfig_get_inputs = dlsym(handle, "cpl_recipeconfig_get_inputs"); cpl->recipeconfig_get_max_count = dlsym(handle, "cpl_recipeconfig_get_max_count"); cpl->recipeconfig_get_min_count = dlsym(handle, "cpl_recipeconfig_get_min_count"); cpl->recipeconfig_get_outputs = dlsym(handle, "cpl_recipeconfig_get_outputs"); cpl->recipeconfig_get_tags = dlsym(handle, "cpl_recipeconfig_get_tags"); error = dlerror(); if (error != NULL) { dlclose(handle); free(cpl); return NULL; } cpl->get_recipeconfig = dlsym(handle, "muse_processing_get_recipeconfig"); dlerror(); cpl->TYPE_BOOL = CPL_TYPE_BOOL; cpl->TYPE_INT = CPL_TYPE_INT; cpl->TYPE_DOUBLE = CPL_TYPE_DOUBLE; cpl->TYPE_STRING = CPL_TYPE_STRING; cpl->is_supported = UNKNOWN_VERSION; for (i = 0; supported_versions[i] != 0; i++) { if (cpl->version == supported_versions[i]) { cpl->is_supported = KNOWN_VERSION; break; } if (CPL_VERSION_MAJOR_CODE(cpl->version) == CPL_VERSION_MAJOR_CODE(supported_versions[i])) { cpl->is_supported = KNOWN_MAJOR; } } /* Between 5.3.1 and 6.0, the cpl_type enum changed. http://upstream-tracker.org/compat_reports/cpl/5.3.1_to_6.0/abi_compat_report.html#Medium_Risk_Problems for these changes; the numbers were taken from there. According to upstream-tracker, this seems to be the only relevant API change between 4.0.0 and 6.2.0. Also the cpl_size is newly introduced (former it was int), in cpl_frame *cpl_frameset_get_frame(cpl_frameset *self, cpl_size position); cpl_size cpl_frameset_get_size(const cpl_frameset *self); cpl_size cpl_parameterlist_get_size(const cpl_parameterlist *self); cpl_size cpl_recipeconfig_get_min_count(const cpl_recipeconfig* self, const char* tag, const char* input); cpl_size cpl_recipeconfig_get_max_count(const cpl_recipeconfig* self, const char* tag, const char* input); Currently, we just ignore this :-) */ if (cpl->version < CPL_VERSION(6,0,0)) { cpl->TYPE_INT = (1 << 8); cpl->TYPE_DOUBLE = (1 << 13); } libraries = realloc(libraries, sizeof(cpl_library_t *) * (i+2)); libraries[i] = cpl; libraries[i+1] = NULL; return cpl; } python-cpl-0.7.4/cpl/logger.py0000644000175000017500000001172213147022346015613 0ustar oleole00000000000000from __future__ import absolute_import import datetime import logging import os import re import tempfile import threading class NullHandler(logging.Handler): def emit(self, record): pass logging.getLogger('cpl').addHandler(NullHandler()) level = { "DEBUG":logging.DEBUG, "INFO":logging.INFO, "WARNING":logging.WARN, "ERROR":logging.ERROR, "OFF":(logging.CRITICAL + 1)} cpl_verbosity = [ logging.DEBUG, logging.INFO, logging.WARN, logging.ERROR, logging.CRITICAL + 1 ] class LogServer(threading.Thread): def __init__(self, name, level = None): threading.Thread.__init__(self) self.name = name self.logger = logging.getLogger(name) self.level = cpl_verbosity.index(level) if level is not None else 0 self.entries = LogList() self.regexp = re.compile('(\\d\\d):(\\d\\d):(\\d\\d)' + '\\s\\[\\s*(\\w+)\\s*\\]' + '\\s(\\w+):' + '(\\s\\[tid=(\\d+)\\])?' + '\\s(.+)') tmphdl, self.logfile = tempfile.mkstemp(prefix = 'cpl', suffix='.log') os.close(tmphdl) os.remove(self.logfile) os.mkfifo(self.logfile) self.start() def run(self): try: with open(self.logfile, 'rb', buffering = 0) as logfile: line = logfile.readline() os.remove(self.logfile) while line: self.log(str(line.decode('ascii'))) line = logfile.readline() except: pass def log(self, s): '''Convert CPL log messages into python log records. A typical CPL log message looks like 10:35:25 [WARNING] rtest: [tid=000] No file tagged with FLAT ''' try: m = self.regexp.match(s) if m is not None: g = m.groups() creation_date = datetime.datetime.combine( datetime.date.today(), datetime.time(int(g[0]),int(g[1]),int(g[2]))) lvl = level.get(g[3], logging.NOTSET) func = g[4] log = logging.getLogger('%s.%s' % (self.logger.name, func)) threadid = int(g[6]) if g[6] else None msg = g[-1] record = logging.LogRecord(log.name, lvl, None, None, msg, None, None, func) created = float(creation_date.strftime('%s')) if record.created < created: created -= 86400 record.relativeCreated -= record.msecs record.relativeCreated += 1000*(created - record.created + 1) record.created = created record.msecs = 0.0 record.threadid = threadid record.threadName = ('Cpl-%03i' % threadid) if threadid \ else 'CplThread' elif self.entries: r0 = self.entries[-1] msg = s.rstrip() lvl = r0.levelno log = logging.getLogger(r0.name) record = logging.LogRecord(r0.name, lvl, None, None, msg, None, None, r0.funcName) record.relativeCreated = r0.relativeCreated record.created = r0.created record.msecs = r0.msecs record.threadid = r0.threadid record.threadName = r0.threadName else: return self.entries.append(record) if log.isEnabledFor(lvl) and log.filter(record): log.handle(record) except: pass class LogList(list): '''List of log messages. Accessing this :class:`list` directly will return the :class:`logging.LogRecord` instances. Example:: res = muse_bias(bias_frames) for logrecord in res.log: print '%s: %s' % (entry.funcname, entry.msg) To get them formatted as string, use the :attr:`error`, :attr:`warning`, :attr:`info` or :attr:`debug` attributes:: res = muse_bias(bias_frames) for line in res.log.info: print line ''' def filter(self, level): return [ '%s: %s' % (entry.funcName, entry.msg) for entry in self if entry.levelno >= level ] @property def error(self): '''Error messages as list of :class:`str` ''' return self.filter(logging.ERROR) @property def warning(self): '''Warnings and error messages as list of :class:`str` ''' return self.filter(logging.WARN) @property def info(self): '''Info, warning and error messages as list of :class:`str` ''' return self.filter(logging.INFO) @property def debug(self): '''Debug, info, warning, and error messages as list of :class:`str` ''' return self.filter(logging.DEBUG) python-cpl-0.7.4/cpl/frames.py0000644000175000017500000001570213370120012015576 0ustar oleole00000000000000from __future__ import absolute_import import os from astropy.io import fits from . import md5sum class FrameConfig(object): '''Frame configuration. Each :class:`FrameConfig` object stores information about one the data type a recipe can process. They are used for defining the calibration files. However, since this information is not generally provided by CPL recipes, it contains only dummy information, except for the MUSE recipes. The objects stores a frame tag, a unique identifier for a certain kind of frame, the minimum and maximum number of frames needed. Attributes: .. attribute:: tag Category tag name. The tag name is used to distinguish between different types of files. An examples of tag names is 'MASTER_BIAS' which specifies the master bias calibration file(s). .. attribute:: min Minimal number of frames, or :obj:`None` if not specified. A frame is required if the :attr:`min` is set to a value greater than 0. .. attribute:: max Maximal number of frames, or :obj:`None` if not specified .. attribute:: frames List of frames (file names or :class:`astropy.io.fits.HDUList` objects) that are assigned to this frame type. ''' def __init__(self, tag, min_frames = 0, max_frames = 0, frames = None): self.tag = tag self.min = min_frames if min_frames > 0 else None self.max = max_frames if max_frames > 0 else None self.frames = frames self.__doc__ = self._doc() def extend_range(self, min_frames, max_frames): if self.min is not None: self.min = min(self.min, min_frames) if min_frames is not None \ else None if self.max is not None: self.max = max(self.max, max_frames) if max_frames is not None \ else None def set_range(self, min_frames, max_frames): self.min = min_frames self.max = max_frames def __str__(self): return str(self.frames) def __repr__(self): return 'FrameDef(%s, frames=%s)' % (repr(self.tag), repr(self.frames)) def _doc(self): if self.max is None or self.min is None: r = ' one frame or list of frames' elif self.max == 1: r = ' one frame' elif self.min > 1 and self.max > self.min: r = ' list of %i-%i frames' % (self.min, self.max) elif self.max > 1: r = ' one frame or list of max. %i frames' % self.max elif self.min > 1: r = ' list of min. %i frames' % self.max else: r = ' one frame or list of frames' if not self.min: r += ' (optional)' return r def __getitem__(self, i): return (self.tag, self.frames)[i] class FrameList(object): def __init__(self, recipe, other = None): self._recipe = recipe self._values = dict() if isinstance(other, self.__class__): self._set_items((o.tag, o.frames) for o in other) elif isinstance(other, dict): self._set_items(other.items()) elif other: self._set_items(other) def _set_items(self, l): for o in l: self[o[0]] = o[1] @property def _cpl_dict(self): cpl_frameconfigs = self._recipe._recipe.frameConfig() if cpl_frameconfigs is None: return None s = dict() for configs in cpl_frameconfigs: c_cfg = configs[1] for f in c_cfg: if f[0] in s: s[f[0]].extend_range(f[1], f[2]) elif f[0] in self._values: s[f[0]] = self._values[f[0]] s[f[0]].set_range(f[1], f[2]) else: s[f[0]] = FrameConfig(f[0], f[1], f[2]) self._values[f[0]] = s[f[0]] return s @property def _dict(self): return self._cpl_dict or self._values def __iter__(self): return iter(self._dict.values()) def __getitem__(self, key): return self._dict[key] def __setitem__(self, key, value): d = self._cpl_dict if d is not None: d[key].frames = value else: self._values.setdefault(key, FrameConfig(key)).frames = value def __delitem__(self, key): self._dict[key].frames = None def __contains__(self, key): return key in self._dict def __len__(self): return len(self._dict) def __getattr__(self, key): return self[key] def __setattr__(self, key, value): if key.startswith('_'): super(FrameList, self).__setattr__(key, value) else: self[key] = value def __delattr__(self, key): del self[key] def __dir__(self): return self._dict.keys() def __repr__(self): return repr(dict(iter(self))) def __str__(self): return str(dict(iter(self))) def __eq__(self, other): return dict(iter(self)) == other @property def __doc__(self): r = 'Frames for recipe %s.\n\nAttributes:\n' % ( self._recipe.name) for s in self: r += '%s: %s\n' % (self._key(s), s.__doc__) return r def _aslist(self, frames): flist = FrameList(self._recipe, self) if frames is not None: flist._set_items(frames.items()) return [(f.tag, f.frames) for f in flist] def mkabspath(frames, tmpdir): '''Convert all filenames in the frames list into absolute paths. :class:`astropy.io.fits.HDUList`s will be converted to temporary files located in the temporary directory tmpdir. The replacement is done in-place. The function will return the list of temporary files. param frames: :class:`list` of (tag, frame) tuples with frame being either a file name or a HDU list. param tmpdir: directory where the temporary files are being created. ''' tmpfiles = list() for i, frame in enumerate(frames): if isinstance(frame[1], fits.HDUList): md5 = md5sum.update_md5(frame[1]) filename = os.path.abspath(os.path.join(tmpdir, '%s_%s.fits' % (frame[0], md5[:8]))) try: os.remove(filename) except: pass frames[i] = ( frame[0], filename ) tmpfiles.append(filename) frame[1].writeto(filename) else: frames[i] = ( frame[0], os.path.abspath(frame[1]) ) return tmpfiles def expandframelist(frames): '''Convert a dictionary with frames into a frame list where each frame gets its own entry in the form (tag, frame) ''' framelist = list() for tag, f in frames: if isinstance(f, list) and not isinstance(f, fits.HDUList): framelist += [ (tag, frame) for frame in f ] elif f is not None: framelist.append((tag, f)) return framelist python-cpl-0.7.4/cpl/CPL_recipe.c0000644000175000017500000010167113370120012016061 0ustar oleole00000000000000#include #include #include #include #include #include #include #include #include #ifdef linux #include #define HAVE_PRCTL #include #define HAVE_MCHECK #define HAVE_MTRACE #include #define HAVE_MALLOPT #endif /* Define PY_Type for Python <= 2.6 */ #ifndef Py_TYPE #define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) #endif #ifndef PyVarObject_HEAD_INIT #define PyVarObject_HEAD_INIT(type, size) \ PyObject_HEAD_INIT(type) size, #endif #include "CPL_library.h" #define CPL_list_doc \ "List all CPL recipe names contained in a shared library." static PyObject * CPL_list(PyObject *self, PyObject *args) { const char *file; if (!PyArg_ParseTuple(args, "s", &file)) return NULL; void *handle = dlopen(file, RTLD_LAZY); if (handle == NULL) { Py_INCREF(Py_None); return Py_None; } char *error =dlerror(); int (*cpl_plugin_get_info)(cpl_pluginlist *) = dlsym(handle, "cpl_plugin_get_info"); error = dlerror(); if (error != NULL) { dlclose(handle); Py_INCREF(Py_None); return Py_None; } cpl_library_t *cpl = create_library(file); PyObject *res = PyList_New(0); Py_INCREF(res); cpl_pluginlist *list = cpl->pluginlist_new(); (*cpl_plugin_get_info)(list); cpl_plugin *plugin; for (plugin = cpl->pluginlist_get_first(list); plugin != NULL; plugin = cpl->pluginlist_get_next(list)) { cpl->error_reset(); cpl->plugin_get_init(plugin)(plugin); char *version = cpl->plugin_get_version_string(plugin); PyList_Append(res, Py_BuildValue("sis", cpl->plugin_get_name(plugin), cpl->plugin_get_version(plugin), version)); cpl->free(version); cpl->plugin_get_deinit(plugin)(plugin); } cpl->pluginlist_delete(list); cpl->error_reset(); dlclose(handle); return res; } #define CPL_supported_versions_doc \ "List all supported CPL versions." static PyObject * CPL_supported_versions(PyObject *self, PyObject *args) { PyObject *res = PyList_New(0); Py_INCREF(res); int i; for (i = 0; supported_versions[i] != 0; i++) { PyList_Append(res, Py_BuildValue( "iii", CPL_VERSION_MAJOR_CODE(supported_versions[i]), CPL_VERSION_MINOR_CODE(supported_versions[i]), CPL_VERSION_MICRO_CODE(supported_versions[i]))); } return res; } static PyMethodDef CPL_methods[] = { {"list", CPL_list, METH_VARARGS, CPL_list_doc}, {"cpl_versions", CPL_supported_versions, METH_NOARGS, CPL_supported_versions_doc}, {NULL, NULL, 0, NULL} /* Sentinel */ }; typedef struct { PyObject_HEAD cpl_plugin *plugin; cpl_pluginlist *pluginlist; void *handle; cpl_recipeconfig *recipeconfig; cpl_library_t *cpl; } CPL_recipe; static void CPL_recipe_dealloc(CPL_recipe* self) { if (self->plugin != NULL) { self->cpl->plugin_get_deinit(self->plugin)(self->plugin); } if (self->pluginlist != NULL) { self->cpl->pluginlist_delete(self->pluginlist); } if (self->handle != NULL) { dlclose(self->handle); } Py_TYPE(self)->tp_free((PyObject*)self); } static PyObject * CPL_recipe_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { CPL_recipe *self = (CPL_recipe *)type->tp_alloc(type, 0); if (self != NULL) { self->plugin = NULL; self->pluginlist = NULL; self->handle = NULL; self->recipeconfig = NULL; self->cpl = NULL; } return (PyObject *)self; } #define CPL_recipe_doc \ "Raw CPL recipe object.\n\n" \ "Constructor parameters:\n" \ " - shared library file name\n" \ " - recipe name\n" static int CPL_recipe_init(CPL_recipe *self, PyObject *args, PyObject *kwds) { const char *file; const char *recipe; if (!PyArg_ParseTuple(args, "ss", &file, &recipe)) return -1; self->handle = dlopen(file, RTLD_LAZY); if (self->handle == NULL) { PyErr_SetString(PyExc_IOError, "cannot open shared library"); return -1; } dlerror(); int (*cpl_plugin_get_info)(cpl_pluginlist *) = dlsym(self->handle, "cpl_plugin_get_info"); char *error = dlerror(); if (error != NULL) { PyErr_SetString(PyExc_IOError, error); return -1; } self->cpl = create_library(file); self->cpl->error_reset(); self->pluginlist = self->cpl->pluginlist_new(); (*cpl_plugin_get_info)(self->pluginlist); self->plugin = self->cpl->pluginlist_find(self->pluginlist, recipe); if (self->plugin == NULL) { PyErr_SetString(PyExc_IOError, "cannot find recipe in shared library"); return -1; } else { self->cpl->plugin_get_init(self->plugin)(self->plugin); } if (self->cpl->get_recipeconfig != NULL) { self->recipeconfig = self->cpl->get_recipeconfig((cpl_recipe *)self->plugin); } else { self->recipeconfig = NULL; } return 0; } #define CPL_is_supported_doc \ "Check whether the CPL version is supported by python-cpl." static PyObject * CPL_is_supported(CPL_recipe *self) { return (self->cpl->is_supported == UNKNOWN_VERSION)?Py_False:Py_True; } #define CPL_version_doc \ "Get the CPL version string." static PyObject * CPL_version(CPL_recipe *self) { return Py_BuildValue("s", self->cpl->version_get_version()); } #define CPL_description_doc \ "Get the string of version numbers of CPL and its libraries." static PyObject * CPL_description(CPL_recipe *self) { return Py_BuildValue("s", self->cpl->get_description(CPL_DESCRIPTION_DEFAULT)); } static PyObject * getParameter(CPL_recipe *self, cpl_parameter *param) { cpl_type type = self->cpl->parameter_get_type(param); cpl_parameter_class class = self->cpl->parameter_get_class(param); const char *name = self->cpl->parameter_get_alias(param, CPL_PARAMETER_MODE_CLI); const char *fullname = self->cpl->parameter_get_name(param); const char *context = self->cpl->parameter_get_context(param); const char *help = self->cpl->parameter_get_help(param); PyObject *range = Py_None; if (class == CPL_PARAMETER_CLASS_RANGE) { if (type == self->cpl->TYPE_INT) { range = Py_BuildValue("ii", self->cpl->parameter_get_range_min_int(param), self->cpl->parameter_get_range_max_int(param)); } else if (type == self->cpl->TYPE_DOUBLE) { range = Py_BuildValue("dd", self->cpl->parameter_get_range_min_double(param), self->cpl->parameter_get_range_max_double(param)); } } Py_INCREF(range); PyObject *sequence = Py_None; if (class == CPL_PARAMETER_CLASS_ENUM) { sequence = PyList_New(0); int n_enum = self->cpl->parameter_get_enum_size(param); int i; for (i = 0; i < n_enum; i++) { if (type == self->cpl->TYPE_INT) { PyList_Append( sequence, Py_BuildValue("i", self->cpl->parameter_get_enum_int(param, i))); } else if (type == self->cpl->TYPE_DOUBLE) { PyList_Append( sequence, Py_BuildValue("d", self->cpl->parameter_get_enum_double(param, i))); } else if (type == self->cpl->TYPE_STRING) { PyList_Append( sequence, Py_BuildValue("s", self->cpl->parameter_get_enum_string(param, i))); } } } Py_INCREF(sequence); PyObject *deflt = Py_None; PyObject *ptype = Py_None; if (type == self->cpl->TYPE_BOOL) { ptype = (PyObject *)&PyBool_Type; deflt = (self->cpl->parameter_get_default_bool(param))?Py_True:Py_False; } else if (type == self->cpl->TYPE_INT) { ptype = (PyObject *)&PyLong_Type; deflt = Py_BuildValue("i", self->cpl->parameter_get_default_int(param)); } else if (type == self->cpl->TYPE_DOUBLE) { ptype = (PyObject *)&PyFloat_Type; deflt = Py_BuildValue("d", self->cpl->parameter_get_default_double(param)); } else if (type == self->cpl->TYPE_STRING) { #if PY_MAJOR_VERSION < 3 ptype = (PyObject *)&PyString_Type; #else ptype = (PyObject *)&PyUnicode_Type; #endif deflt = Py_BuildValue("s", self->cpl->parameter_get_default_string(param)); } Py_INCREF(deflt); Py_INCREF(ptype); PyObject *enabled = Py_BuildValue( "OOO", self->cpl->parameter_is_enabled(param, CPL_PARAMETER_MODE_CLI)?Py_True:Py_False, self->cpl->parameter_is_enabled(param, CPL_PARAMETER_MODE_ENV)?Py_True:Py_False, self->cpl->parameter_is_enabled(param, CPL_PARAMETER_MODE_CFG)?Py_True:Py_False); Py_INCREF(enabled); PyObject *par = Py_BuildValue("ssssNNNNN", name, context, fullname, help, range, sequence, deflt, ptype, enabled); Py_INCREF(par); return par; } #define CPL_recipe_get_params_doc \ "Get the possible parameters.\n\n" \ "Returns a list of tuples where each tuple defines one parameter:\n" \ " - parameter name\n" \ " - parameter context\n" \ " - description\n" \ " - range (min, max), if valid range is limited, or None\n" \ " - allowed values, if only certain values are allowed, or None\n" \ " - default value\n" \ " - triple (cli, env, cfg) with enabled-values for param modes" static PyObject * CPL_recipe_get_params(CPL_recipe *self) { if (self->plugin == NULL) { PyErr_SetString(PyExc_IOError, "NULL recipe"); return NULL; } cpl_parameterlist *pars = ((cpl_recipe *)self->plugin)->parameters; PyObject *res = PyList_New(0); if (pars && self->cpl->parameterlist_get_size(pars)) { cpl_parameter *param; for (param = self->cpl->parameterlist_get_first(pars); param != NULL; param = self->cpl->parameterlist_get_next(pars)) { PyList_Append(res, getParameter(self, param)); } } Py_INCREF(res); return res; } #define CPL_recipe_get_author_doc \ "Get the author and his email.\n\n" \ "Returns a pair where the first field is the author name and the\n" \ "second field is the E-mail address." static PyObject * CPL_recipe_get_author(CPL_recipe *self) { if (self->plugin == NULL) { PyErr_SetString(PyExc_IOError, "NULL recipe"); return NULL; } return Py_BuildValue("ss", self->cpl->plugin_get_author(self->plugin), self->cpl->plugin_get_email(self->plugin)); } #define CPL_recipe_get_description_doc \ "Get the synopsis and description.\n\n" \ "Returns a pair where the first field is the synopsis string and the\n" \ "second field is the description string." static PyObject * CPL_recipe_get_description(CPL_recipe *self) { if (self->plugin == NULL) { PyErr_SetString(PyExc_IOError, "NULL recipe"); return NULL; } return Py_BuildValue("ss", self->cpl->plugin_get_synopsis(self->plugin), self->cpl->plugin_get_description(self->plugin)); } #define CPL_recipe_get_version_doc \ "Get the version as integer and string.\n\n" \ "Returns a pair where the first entry is the version number as integer\n" \ "and the second entry is the version string.\n" static PyObject * CPL_recipe_get_version(CPL_recipe *self) { if (self->plugin == NULL) { PyErr_SetString(PyExc_IOError, "NULL recipe"); return NULL; } return Py_BuildValue("is", self->cpl->plugin_get_version(self->plugin), self->cpl->plugin_get_version_string(self->plugin)); } #define CPL_recipe_get_copyright_doc \ "Get the license and copyright information." static PyObject * CPL_recipe_get_copyright(CPL_recipe *self) { if (self->plugin == NULL) { PyErr_SetString(PyExc_IOError, "NULL recipe"); return NULL; } return Py_BuildValue("s", self->cpl->plugin_get_copyright(self->plugin)); } #define CPL_recipe_get_frameconfig_doc \ "Get the possible frame configurations.\n\n" \ "Returns a list of tuples. Each tupel is the frame configuration of one\n"\ "input frame tag. It consists of\n" \ " - input frame configuration (tupel with tag, minimal and maximal\n" \ " number of frames\n" \ " - list of configuration frames (each is a tupel with tag, minimal and\n"\ " maximal number of frames)\n" \ " - list of output tags\n" \ "Unset minimum/maximum values are indicated by -1" static PyObject * CPL_recipe_get_frameconfig(CPL_recipe *self) { if (self->plugin == NULL) { PyErr_SetString(PyExc_IOError, "NULL recipe"); return NULL; } if (self->recipeconfig == NULL) { Py_INCREF(Py_None); return Py_None; } PyObject *res = PyList_New(0); char **tags = self->cpl->recipeconfig_get_tags(self->recipeconfig); int i_tag; for (i_tag = 0; tags[i_tag] != NULL; i_tag++) { int min = self->cpl->recipeconfig_get_min_count(self->recipeconfig, tags[i_tag], tags[i_tag]); int max = self->cpl->recipeconfig_get_max_count(self->recipeconfig, tags[i_tag], tags[i_tag]); PyObject *raw = Py_BuildValue("sii", tags[i_tag], min, max); PyObject *calib = PyList_New(0); char **inputs = self->cpl->recipeconfig_get_inputs(self->recipeconfig, tags[i_tag]); int i_input; for (i_input = 0; inputs[i_input] != NULL; i_input++) { int min = self->cpl->recipeconfig_get_min_count(self->recipeconfig, tags[i_tag], inputs[i_input]); int max = self->cpl->recipeconfig_get_max_count(self->recipeconfig, tags[i_tag], inputs[i_input]); PyList_Append(calib, Py_BuildValue("sii", inputs[i_input], min, max)); self->cpl->free(inputs[i_input]); } self->cpl->free(inputs); PyObject *output = PyList_New(0); char **outputs = self->cpl->recipeconfig_get_outputs(self->recipeconfig, tags[i_tag]); int i_output; for (i_output = 0; outputs[i_output] != NULL; i_output++) { PyList_Append(output, Py_BuildValue("s", outputs[i_output])); self->cpl->free(outputs[i_output]); } self->cpl->free(outputs); PyList_Append(res, Py_BuildValue("OOO", raw, calib, output)); self->cpl->free(tags[i_tag]); } self->cpl->free(tags); return res; } static cpl_frameset * get_frames(CPL_recipe *self, PyObject *framelist) { cpl_frameset *frames = self->cpl->frameset_new(); PyObject *iter = PyObject_GetIter(framelist); PyObject *item; while ((item = PyIter_Next(iter))) { const char *tag; const char* file; PyArg_ParseTuple(item, "ss", &tag, &file); cpl_frame *frame = self->cpl->frame_new(); self->cpl->frame_set_filename(frame, file); self->cpl->frame_set_tag(frame, tag); self->cpl->frameset_insert(frames, frame); Py_DECREF(item); } Py_DECREF(iter); return frames; } static void clear_parameters(CPL_recipe *self, cpl_parameterlist *parameters) { cpl_parameter *par = self->cpl->parameterlist_get_first(parameters); while (par != NULL) { cpl_type type = self->cpl->parameter_get_type(par); if (type == self->cpl->TYPE_STRING) { const char *default_value = self->cpl->parameter_get_default_string(par); if (default_value == NULL) { default_value = ""; } self->cpl->parameter_set_string(par, default_value); } else if (type == self->cpl->TYPE_INT) { self->cpl->parameter_set_int(par, self->cpl->parameter_get_default_int(par)); } else if (type == self->cpl->TYPE_DOUBLE) { self->cpl->parameter_set_double(par, self->cpl->parameter_get_default_double(par)); } else if (type == self->cpl->TYPE_BOOL) { self->cpl->parameter_set_bool(par, self->cpl->parameter_get_default_bool(par)); } par = self->cpl->parameterlist_get_next(parameters); } } static void set_parameters(CPL_recipe *self, cpl_parameterlist *parameters, PyObject *parlist) { PyObject *iter = PyObject_GetIter(parlist); PyObject *item; while ((item = PyIter_Next(iter))) { const char *name; PyObject *value; PyArg_ParseTuple(item, "sO", &name, &value); cpl_parameter *par = self->cpl->parameterlist_find(parameters, name); if (par == NULL) { continue; } cpl_type type = self->cpl->parameter_get_type(par); if (type == self->cpl->TYPE_STRING) { #if PY_MAJOR_VERSION < 3 if (PyString_Check(value)) { self->cpl->parameter_set_string(par, PyString_AsString(value)); } #else if (PyUnicode_Check(value)) { PyObject* temp = PyUnicode_AsASCIIString(value); if (temp != NULL) { self->cpl->parameter_set_string(par, PyBytes_AsString(temp)); Py_XDECREF(temp); } } #endif } else if (type == self->cpl->TYPE_INT) { if (PyLong_Check(value)) { self->cpl->parameter_set_int(par, PyLong_AsLong(value)); } } else if (type == self->cpl->TYPE_DOUBLE) { if (PyFloat_Check(value)) { self->cpl->parameter_set_double(par, PyFloat_AsDouble(value)); } } else if (type == self->cpl->TYPE_BOOL) { self->cpl->parameter_set_bool(par, PyObject_IsTrue(value)); } Py_DECREF(item); } Py_DECREF(iter); } static void set_environment(PyObject *runenv) { PyObject *iter = PyObject_GetIter(runenv); PyObject *item; while ((item = PyIter_Next(iter))) { const char *name; PyObject *value; PyArg_ParseTuple(item, "sO", &name, &value); if ((name == NULL) || (value == NULL)) { continue; } #if PY_MAJOR_VERSION < 3 if (PyString_Check(value)) { setenv(name, PyString_AsString(value), 1); } #else if (PyUnicode_Check(value)) { PyObject* temp = PyUnicode_AsASCIIString(value); if (temp != NULL) { setenv(name, PyBytes_AsString(temp), 1); Py_XDECREF(temp); } } #endif if (value == Py_None) { unsetenv(name); } Py_DECREF(item); } Py_DECREF(iter); } static PyObject * exec_build_retval(void *ptr) { long ret_code = ((long *)ptr)[1]; double user_time = ((long *)ptr)[2] * 1e-6; double sys_time = ((long *)ptr)[3] * 1e-6; int memcheck = ((long *)ptr)[4]; PyObject *stats = Py_BuildValue("iffi", ret_code, user_time, sys_time, memcheck); long n_errors = ((long *)ptr)[5]; long index = 6 * sizeof(long); PyObject *errors = PyList_New(0); for (; n_errors > 0; n_errors--) { long error_code = *((long *)(ptr + index)); index += sizeof(long); long error_line = *((long *)(ptr + index)); index += sizeof(long); const char *error_msg = ptr + index; index += strlen(error_msg) + 1; const char *error_file = ptr + index; index += strlen(error_file) + 1; const char *error_func = ptr + index; index += strlen(error_func) + 1; PyList_Append(errors, Py_BuildValue("issis", error_code, error_msg, error_file, error_line, error_func)); } PyObject *frames = PyList_New(0); while (index < ((long *)ptr)[0]) { const char *tag = ptr + index; index += strlen(tag) + 1; const char *file = ptr + index; index += strlen(file) + 1; PyList_Append(frames, Py_BuildValue("ss", tag, file)); } return Py_BuildValue("OOO", frames, errors, stats); } static void *sbuffer_append_string(void *buf, const char *str) { buf = realloc(buf, ((long *)buf)[0] + strlen(str) + 1); strcpy(buf + *((long *)buf), str); *((long *)buf) += strlen(str) + 1; return buf; } static void *sbuffer_append_bytes(void *buf, const void *src, size_t nbytes) { buf = realloc(buf, ((long *)buf)[0] + nbytes); memcpy(buf + *((long *)buf), src, nbytes); *((long *)buf) += nbytes; return buf; } static void *sbuffer_append_long(void *buf, long val) { buf = realloc(buf, *((long *)buf) + sizeof(long)); *((long *)(buf + ((long *)buf)[0])) = val; *((long *)buf) += sizeof(long); return buf; } static void *serialized_error_ptr = NULL; static cpl_library_t *serialized_cpl = NULL; static void exec_serialize_one_error(unsigned self, unsigned first, unsigned last) { if (serialized_error_ptr == NULL) { serialized_error_ptr = malloc(sizeof(long)); ((long *)serialized_error_ptr)[0] = sizeof(long); serialized_error_ptr = sbuffer_append_long(serialized_error_ptr, 0); } if (serialized_cpl->error_get_code() == CPL_ERROR_NONE) { return; } ((long *)serialized_error_ptr)[1]++; // number of errors serialized_error_ptr = sbuffer_append_long(serialized_error_ptr, serialized_cpl->error_get_code()); serialized_error_ptr = sbuffer_append_long(serialized_error_ptr, serialized_cpl->error_get_line()); serialized_error_ptr = sbuffer_append_string(serialized_error_ptr, serialized_cpl->error_get_message()); serialized_error_ptr = sbuffer_append_string(serialized_error_ptr, serialized_cpl->error_get_file()); serialized_error_ptr = sbuffer_append_string(serialized_error_ptr, serialized_cpl->error_get_function()); } static void * exec_serialize_retval(CPL_recipe *self, cpl_frameset *frames, cpl_errorstate prestate, int retval, const struct tms *tms_clock) { int n_frames = self->cpl->frameset_get_size(frames); int i_frame; void *ptr = malloc(sizeof(long)); ((long *)ptr)[0] = sizeof(long); ptr = sbuffer_append_long(ptr, retval); ptr = sbuffer_append_long(ptr, 1000000L * (tms_clock->tms_utime + tms_clock->tms_cutime) / sysconf(_SC_CLK_TCK)); ptr = sbuffer_append_long(ptr, 1000000L * (tms_clock->tms_stime + tms_clock->tms_cstime) / sysconf(_SC_CLK_TCK)); ptr = sbuffer_append_long(ptr, self->cpl->memory_is_empty()); serialized_cpl = self->cpl; self->cpl->errorstate_dump(prestate, CPL_FALSE, exec_serialize_one_error); ptr = sbuffer_append_bytes(ptr, serialized_error_ptr + sizeof(long), ((long *)serialized_error_ptr)[0] - sizeof(long)); free(serialized_error_ptr); serialized_error_ptr = NULL; serialized_cpl = NULL; for (i_frame = 0; i_frame < n_frames; i_frame++) { cpl_frame *f = self->cpl->frameset_get_position(frames, i_frame); if (self->cpl->frame_get_group(f) != CPL_FRAME_GROUP_PRODUCT) { continue; } ptr = sbuffer_append_string(ptr, self->cpl->frame_get_tag(f)); ptr = sbuffer_append_string(ptr, self->cpl->frame_get_filename(f)); } return ptr; } static int do_backtrace(void) { char cmd[300]; snprintf(cmd, sizeof(cmd), "cat >> gdb_commands << EOF\n" "set height 0\nset width 0\nbt full\ninfo sources\ninfo files\n" "EOF"); int retval = system(cmd); snprintf(cmd, sizeof(cmd), "gdb -batch -x gdb_commands --pid %i --readnow >> recipe.backtrace-unprocessed 2> /dev/null", (int)getpid()); retval |= system(cmd); unlink("gdb_commands"); return retval; } #ifdef HAVE_MCHECK static void mcheck_handler(enum mcheck_status status) { char cmd[100]; snprintf(cmd, sizeof(cmd), "echo Memory corruption > recipe.backtrace-unprocessed"); int retval = system(cmd); if (retval == 0) { do_backtrace(); } abort(); } #endif static int segv_handler(int sig) { char cmd[100]; snprintf(cmd, sizeof(cmd), "echo Received signal: %i > recipe.backtrace-unprocessed", sig); int retval = system(cmd); do_backtrace(); signal(sig, SIG_DFL); return retval; } static void setup_tracing(CPL_recipe *self, int memory_trace) { #ifdef HAVE_PRCTL #ifdef PR_SET_PTRACER /* Sets the top of the process tree that is allowed to use PTRACE on the calling process */ prctl(PR_SET_PTRACER, getpid(), 0, 0, 0); #endif #ifdef PR_SET_NAME /* Set the process name for the calling process */ prctl(PR_SET_NAME, self->cpl->plugin_get_name(self->plugin), 0, 0, 0); #endif #endif #ifdef HAVE_MCHECK mcheck(mcheck_handler); #endif #ifdef HAVE_MALLOPT mallopt(M_CHECK_ACTION, 0); #endif #ifdef HAVE_MTRACE if (memory_trace) { setenv("MALLOC_TRACE", "recipe.mtrace", 1); mtrace(); } #endif typedef void (*sighandler_t)(int); signal(SIGSEGV, (sighandler_t) segv_handler); signal(SIGINT, (sighandler_t) segv_handler); signal(SIGHUP, (sighandler_t) segv_handler); signal(SIGFPE, (sighandler_t) segv_handler); signal(SIGQUIT, (sighandler_t) segv_handler); signal(SIGBUS, (sighandler_t) segv_handler); signal(SIGTERM, (sighandler_t) segv_handler); signal(SIGABRT, (sighandler_t) segv_handler); signal(SIGTERM, (sighandler_t) segv_handler); } #define CPL_recipe_exec_doc \ "Execute with parameters and frames.\n\n" \ "The parameters shall contain an iterable of (name, value) pairs\n" \ "where the values have the correct type for the parameter.\n" \ "The frames shall contain an iterable of (name, tag) pairs." static PyObject * CPL_recipe_exec(CPL_recipe *self, PyObject *args) { PyObject *parlist; PyObject *soflist; PyObject *runenv; const char *dirname; const char *logfile; int loglevel; int memory_dump; int memory_trace; if (!PyArg_ParseTuple(args, "sOOOsiii", &dirname, &parlist, &soflist, &runenv, &logfile, &loglevel, &memory_dump, &memory_trace)) return NULL; if (!PySequence_Check(parlist)) { PyErr_SetString(PyExc_TypeError, "Second parameter not a list"); return NULL; } if (!PySequence_Check(soflist)) { PyErr_SetString(PyExc_TypeError, "Third parameter not a list"); return NULL; } if (!PySequence_Check(runenv)) { PyErr_SetString(PyExc_TypeError, "Fourth parameter not a list"); return NULL; } if (self->plugin == NULL) { PyErr_SetString(PyExc_IOError, "NULL recipe"); return NULL; } self->cpl->error_reset(); cpl_recipe *recipe = (cpl_recipe *)self->plugin; self->cpl->frameset_delete(recipe->frames); recipe->frames = get_frames(self, soflist); clear_parameters(self, recipe->parameters); set_parameters(self, recipe->parameters, parlist); if (self->cpl->error_get_code() != CPL_ERROR_NONE) { PyErr_SetString(PyExc_IOError, "CPL error on inititalization"); return NULL; } int fd[2]; if (pipe(fd) == -1) { PyErr_SetString(PyExc_IOError, "Cannot pipe()"); return NULL; } pid_t childpid = fork(); if (childpid == -1) { PyErr_SetString(PyExc_IOError, "Cannot fork()"); return NULL; } if (childpid == 0) { close(fd[0]); int retval; struct tms clock_end; set_environment(runenv); self->cpl->msg_set_log_name(logfile); self->cpl->msg_set_log_level(loglevel); self->cpl->msg_set_level(CPL_MSG_OFF); cpl_errorstate prestate = self->cpl->errorstate_get(); if (chdir(dirname) == 0) { struct tms clock_start; times(&clock_start); setup_tracing(self, memory_trace); retval = self->cpl->plugin_get_exec(self->plugin)(self->plugin); int reto; if (self->cpl->dfs_sign_products != NULL) { reto = self->cpl->dfs_sign_products(recipe->frames, CPL_DFS_SIGNATURE_DATAMD5 | CPL_DFS_SIGNATURE_CHECKSUM); } else { reto = self->cpl->dfs_update_product_header(recipe->frames); } if (reto != CPL_ERROR_NONE) { self->cpl->msg_error (__func__, "could not update the product header." " %s (%s:%s:%u)", self->cpl->error_get_message(), self->cpl->error_get_function(), self->cpl->error_get_file(), self->cpl->error_get_line()); } times(&clock_end); clock_end.tms_utime -= clock_start.tms_utime; clock_end.tms_stime -= clock_start.tms_stime; clock_end.tms_cutime -= clock_start.tms_cutime; clock_end.tms_cstime -= clock_start.tms_cstime; self->cpl->msg_stop_log(); } else { retval = CPL_ERROR_FILE_NOT_CREATED; self->cpl->error_set_message_macro(__func__, retval, __FILE__, __LINE__, " "); } void *ptr = exec_serialize_retval(self, recipe->frames, prestate, retval, &clock_end); long n_bytes = write(fd[1], ptr, ((long *)ptr)[0]); close(fd[1]); retval = (n_bytes != ((long *)ptr)[0]); free(ptr); self->cpl->frameset_delete(recipe->frames); self->cpl->parameterlist_delete(recipe->parameters); recipe->parameters = NULL; recipe->frames = NULL; self->cpl->plugin_get_deinit(self->plugin)(self->plugin); self->cpl->pluginlist_delete(self->pluginlist); Py_TYPE(self)->tp_free((PyObject*)self); if ((memory_dump > 1) || ((memory_dump > 0) && (!self->cpl->memory_is_empty()))) { self->cpl->memory_dump(); } self->cpl->end(); #ifdef HAVE_MTRACE muntrace(); #endif _exit(retval); } close(fd[1]); long nbytes; long nbytes2; void *ptr = malloc(2 * sizeof(long)); Py_BEGIN_ALLOW_THREADS do { nbytes = read(fd[0], ptr, 2 * sizeof(long)); if (nbytes >= 0 || errno != EINTR) break; } while (1); if (nbytes == 2 * sizeof(long)) { ptr = realloc(ptr, ((long *)ptr)[0]); do { nbytes2 = read(fd[0], ptr + 2 * sizeof(long), ((long *)ptr)[0] - 2 * sizeof(long)); if (nbytes2 >= 0 || errno != EINTR) break; } while (1); nbytes += nbytes2; } else { // broken pipe while reading first two bytes ((long *)ptr)[0] = 2 * sizeof(long); } close(fd[0]); waitpid(childpid, NULL, 0); Py_END_ALLOW_THREADS if (nbytes != ((long *)ptr)[0]) { PyErr_SetString(PyExc_IOError, "Recipe crashed"); return NULL; } PyObject *retval = exec_build_retval(ptr); free(ptr); return retval; } static PyMethodDef CPL_recipe_methods[] = { {"params", (PyCFunction)CPL_recipe_get_params, METH_NOARGS, CPL_recipe_get_params_doc}, {"author", (PyCFunction)CPL_recipe_get_author, METH_NOARGS, CPL_recipe_get_author_doc}, {"version", (PyCFunction)CPL_recipe_get_version, METH_NOARGS, CPL_recipe_get_version_doc}, {"description", (PyCFunction)CPL_recipe_get_description, METH_NOARGS, CPL_recipe_get_description_doc}, {"copyright", (PyCFunction)CPL_recipe_get_copyright, METH_NOARGS, CPL_recipe_get_copyright_doc}, {"frameConfig", (PyCFunction)CPL_recipe_get_frameconfig, METH_NOARGS, CPL_recipe_get_frameconfig_doc}, {"run", (PyCFunction)CPL_recipe_exec, METH_VARARGS, CPL_recipe_exec_doc}, {"cpl_is_supported", (PyCFunction)CPL_is_supported, METH_NOARGS, CPL_is_supported_doc}, {"cpl_version", (PyCFunction)CPL_version, METH_NOARGS, CPL_version_doc}, {"cpl_description", (PyCFunction)CPL_description, METH_NOARGS, CPL_version_doc}, {NULL, NULL, 0, NULL} /* Sentinel */ }; static PyTypeObject CPL_recipeType = { PyVarObject_HEAD_INIT(NULL, 0) "CPL_recipe.recipe", /*tp_name*/ sizeof(CPL_recipe), /*tp_basicsize*/ 0, /*tp_itemsize*/ (destructor)CPL_recipe_dealloc, /*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ 0, /*tp_compare*/ 0, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ 0, /*tp_as_mapping*/ 0, /*tp_hash */ 0, /*tp_call*/ 0, /*tp_str*/ 0, /*tp_getattro*/ 0, /*tp_setattro*/ 0, /*tp_as_buffer*/ Py_TPFLAGS_DEFAULT, /*tp_flags*/ CPL_recipe_doc, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ CPL_recipe_methods, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ (initproc)CPL_recipe_init, /* tp_init */ 0, /* tp_alloc */ CPL_recipe_new, /* tp_new */ }; #if PY_MAJOR_VERSION >= 3 PyMODINIT_FUNC PyInit_CPL_recipe(void) { static struct PyModuleDef moduledef = { PyModuleDef_HEAD_INIT, "CPL_recipe", /* m_name */ NULL, /* m_doc */ -1, /* m_size */ CPL_methods, /* m_methods */ NULL, /* m_reload */ NULL, /* m_traverse */ NULL, /* m_clear */ NULL, /* m_free */ }; CPL_recipeType.tp_new = PyType_GenericNew; if (PyType_Ready(&CPL_recipeType) < 0) { return NULL; } PyObject *m = PyModule_Create(&moduledef); Py_INCREF(&CPL_recipeType); PyModule_AddObject(m, "recipe", (PyObject *)&CPL_recipeType); return m; } #else PyMODINIT_FUNC initCPL_recipe(void) { CPL_recipeType.tp_new = PyType_GenericNew; if (PyType_Ready(&CPL_recipeType) < 0) { return; } PyObject *m = Py_InitModule3("CPL_recipe", CPL_methods, NULL); Py_INCREF(&CPL_recipeType); PyModule_AddObject(m, "recipe", (PyObject *)&CPL_recipeType); } #endif python-cpl-0.7.4/MANIFEST.in0000644000175000017500000000013413147022356014736 0ustar oleole00000000000000include cpl/*.h include doc/* include test/*.py test/*.c include README.rst include COPYING python-cpl-0.7.4/setup.cfg0000644000175000017500000000004613370121236015016 0ustar oleole00000000000000[egg_info] tag_build = tag_date = 0 python-cpl-0.7.4/doc/0000755000175000017500000000000013370121236013742 5ustar oleole00000000000000python-cpl-0.7.4/doc/parallel.rst0000644000175000017500000000624612235465526016314 0ustar oleole00000000000000.. _parallel: Parallel execution ================== The library allows a simple parallelization of recipe processing. The parallelization is done using independent processes and thus does not depend on parallelization features in the CPL or the recipe implementation. To specify that a recipe should be executed in the background, the :attr:`threaded` attribute needs to be set to :obj:`True`. This may be done either in the recipe constructor, as a recipe attribute or as a parameter of the execution call. Each of the following three recipes will start a background process for the BIAS calculation:: # Create a threaded recipe r1 = cpl.Recipe('muse_bias', threaded = True) result1 = r1([ 'bias1.fits', 'bias2.fits', 'bias3.fits']) # Prepare a recipe for background execution r2 = cpl.Recipe('muse_bias') r2.threaded = True result2 = r2([ 'bias1.fits', 'bias2.fits', 'bias3.fits']) # Execute a recipe in background r3 = cpl.Recipe('muse_bias') result3 = r3([ 'bias1.fits', 'bias2.fits', 'bias3.fits'], threaded = True) If the :attr:`threaded` attribute is set to :obj:`True`, the execution call of the recipe immediately returns while the recipe is executed in the background. The current thread is stopped only if any of the results of the recipe is accessed and the recipe is still not finished. The result frame of a background recipe is a subclass of :class:`threading.Thread`. This interface may be used to control the thread execution. The simples way to use parallel processing is to create a list where the members are created by the execution of the recipe. The following example shows the parallel execution of the 'muse_focus' recipe:: muse_focus = cpl.Recipe('muse_focus', threaded = True) muse_focus.calib.MASTER_BIAS = 'master_bias.fits' # Create a list of input files files = [ 'MUSE_CUNGC%02i.fits' % i for i in range(20, 30) ] # Create a list of recipe results. Note that for each entry, a background # process is started. results = [ muse_focus(f) for f in files ] # Save the results. The current thread is stopped until the according # recipe is finished. for i, res in enumerate(results): res.FOCUS_TABLE.writeto('FOCUS_TABLE_%02i.fits' % (i+1)) When using parallel processing note that the number of parallel processes is not limited by default, so this feature may produce a high load when called with a large number of processes. Parallelization in the recipe itself or in the CPL may also result in additional load. To limit the maximal number of parallel processes, the function :func:`cpl.Recipe.set_maxthreads()` can be called with the maximal number of parallel processes. Note that this function controls only the threads that are started afterwards. If the recipe execution fails, the according exception will be raised whenever one of the results is accessed. .. note :: Recipes may contain an internal parallelization using the `openMP `_ interface. Although it is recommended to leave them untouched, they may be changed via environment variable settungs in the :attr:`cpl.Recipe.env` attribute. See http://gcc.gnu.org/onlinedocs/libgomp/Environment-Variables.html for a list of environment variables. python-cpl-0.7.4/doc/install.rst0000644000175000017500000000450613370120012016136 0ustar oleole00000000000000Installation ============ Prequisites ----------- * `Python `_ 2.6 or higher, * `Astropy `_ Binary packages --------------- On Debian and debian-based systems (Ubuntu, Mint), python-cpl can be installed with the command .. code-block:: sh apt-get install python-cpl Python CPL comes with the `Ubuntu `_ distribution since 12.04. Debian packages are in `Wheezy (Debian 7) `_, `Squeeze (Debian 8) `_, and `Testing `_ Source code ----------- * `Python Package Index `_ * `Git repository `_. To access, do a:: git clone git://github.com/olebole/python-cpl.git This gives you the current version in the subdirectory :file:`python-cpl`. To update to the current version of an existing repository, do a ``git pull`` in the :file:`python-cpl` directory. For more detailed information, check the manual page of :manpage:`git(1)` and the `github `_ page of the project. Compilation ----------- For compilation, a C compiler is needed additionally to the software mentioned above. The installation follows the standard procedure used in python. On default, the installation path :file:`/usr/local`. If using a non-standard installation path, add the directory :file:`{PREFIX}/lib/python2.7/site-packages/` (:file:`lib64/python2.7/site-packages/` on 64 bit systems) to your environment variable :envvar:`PYTHONPATH` where where :file:`{PREFIX}` is the installation path for the package. In the source directory of python-cpl, run .. code-block:: sh python setup.py install --prefix=PREFIX There are other options available as well; use the :option:`--help` option to list them. Test suite ---------- There are a number of tests defined in :file:`test/TestRecipe.py`: .. code-block:: sh python TestRecipe.py The test recipe needs an installed CPL development environment. The tests may print a memory corruption detection by glibc. This is normal, since the tests also check this behaviour in the recipe. Tests are also automatically buils by `Travis CI `_. python-cpl-0.7.4/doc/param.rst0000644000175000017500000000022011460377136015600 0ustar oleole00000000000000.. module:: cpl The :class:`cpl.Parameter` class ================================ .. autoclass:: Parameter .. seealso:: :attr:`Recipe.param` python-cpl-0.7.4/doc/tutorial.rst0000644000175000017500000001231512235465526016355 0ustar oleole00000000000000Tutorial ======== Simple example -------------- The following code takes BIAS input file names from the command line and writes the MASTER BIAS to the file name provided with the :option:`-o` option:: from optparse import OptionParser import sys import cpl parser = OptionParser(usage='%prog files') parser.add_option('-o', '--output', help='Output file', default='master_bias.fits') parser.add_option('-b', '--badpix-table', help='Bad pixel table') (opt, filenames) = parser.parse_args() if not filenames: parser.print_help() sys.exit() cpl.esorex.init() muse_bias = cpl.Recipe('muse_bias') muse_bias.param.nifu = 1 muse_bias.calib.BADPIX_TABLE = opt.badpix_table res = muse_bias(filenames) res.MASTER_BIAS.writeto(opt.output) Quick guide ----------- Input lines are indicated with ">>>" (the python prompt). The package can be imported with >>> import cpl If you migrate from `Esorex `_, you may just init the search path for CPL recipes from the esorex startup: >>> cpl.esorex.init() Otherwise, you will need to explicitely set the recipe search path: >>> cpl.Recipe.path = '/store/01/MUSE/recipes' List available recipes: >>> cpl.Recipe.list() [('muse_quick_image', ['0.2.0', '0.3.0']), ('muse_scipost', ['0.2.0', '0.3.0']), ('muse_scibasic', ['0.2.0', '0.3.0']), ('muse_flat', ['0.2.0', '0.3.0']), ('muse_subtract_sky', ['0.2.0', '0.3.0']), ('muse_bias', ['0.2.0', '0.3.0']), ('muse_ronbias', ['0.2.0', '0.3.0']), ('muse_fluxcal', ['0.2.0', '0.3.0']), ('muse_focus', ['0.2.0', '0.3.0']), ('muse_lingain', ['0.2.0', '0.3.0']), ('muse_dark', ['0.2.0', '0.3.0']), ('muse_combine_pixtables', ['0.2.0', '0.3.0']), ('muse_astrometry', ['0.2.0', '0.3.0']), ('muse_wavecal', ['0.2.0', '0.3.0']), ('muse_exp_combine', ['0.2.0', '0.3.0']), ('muse_dar_correct', ['0.2.0', '0.3.0']), ('muse_standard', ['0.2.0', '0.3.0']), ('muse_create_sky', ['0.2.0', '0.3.0']), ('muse_apply_astrometry', ['0.2.0', '0.3.0']), ('muse_rebin', ['0.2.0', '0.3.0'])] Create a recipe specified by name: >>> muse_scibasic = cpl.Recipe('muse_scibasic') By default, it loads the recipe with the highest version number. You may also explicitely specify the version number: >>> muse_scibasic = cpl.Recipe('muse_scibasic', version = '0.2.0') List all parameters: >>> print muse_scibasic.param {'ybox': 40, 'passes': 2, 'resample': False, 'xbox': 15, 'dlambda': 1.25, 'cr': 'none', 'thres': 5.8, 'nifu': 0, 'saveimage': True} Set a parameter: >>> muse_scibasic.param.nifu = 1 Print the value of a parameter (:obj:`None` if the parameter is set to default) >>> print muse_scibasic.param.nifu.value 1 List all calibration frames: >>> print muse_scibasic.calib {'TRACE_TABLE': None, 'MASTER_SKYFLAT': None, 'WAVECAL_TABLE': None, 'MASTER_BIAS': None, 'MASTER_DARK': None, 'GEOMETRY_TABLE': None, 'BADPIX_TABLE': None, 'MASTER_FLAT': None, 'GAINRON_STAT': None} Set calibration frames with files: >>> muse_scibasic.calib.MASTER_BIAS = 'MASTER_BIAS-01.fits' >>> muse_scibasic.calib.MASTER_FLAT = 'MASTER_FLAT-01.fits' >>> muse_scibasic.calib.TRACE_TABLE = 'TRACE_TABLE-01.fits' >>> muse_scibasic.calib.GEOMETRY_TABLE = 'geometry_table.fits' You may also set calibration frames with :class:`astropy.io.fits.HDUList` objects. This is especially useful if you want to change the file on the fly: >>> import astropy.io.fits >>> wavecal = astropy.io.fits.open('WAVECAL_TABLE-01_flat.fits') >>> wavecal[1].data.field('wlcc00')[:] *= 1.01 >>> muse_scibasic.calib.WAVECAL_TABLE = wavecal To set more than one file for a tag, put the file names and/or :class:`astropy.io.fits.HDUList` objects into a list: >>> muse_scibasic.calib.MASTER_BIAS = [ 'MASTER_BIAS-%02i.fits' % (i+1) ... for i in range(24) ] To run the recipe, call it with the input file names as arguments. The product frames are returned in the return value of the call. If you don't specify an input frame tag, the default (first) one of the recipe is used. >>> res = muse_scibasic('Scene_fusion_1.fits') Run the recipe with a nondefault tag (use raw data tag as argument name): >>> res = muse_scibasic(raw = {'SKY':'sky_newmoon_no_noise_1.fits'}) Parameters and calibration frames may be changed for a specific call by specifying them as arguments: >>> res = muse_scibasic('Scene_fusion_1.fits', param = {'nifu': 2}, ... calib = {'MASTER_FLAT': None, ... 'WAVECAL_TABLE': 'WAVECAL_TABLE_noflat.fits'}) The results of a calibration run are :class:`astropy.io.fits.HDUList` objects. To save them (use output tags as attributes): >>> res.PIXTABLE_OBJECT.writeto('Scene_fusion_pixtable.fits') They can also be used directly as input of other recipes. >>> muse_sky = cpl.Recipe('muse_sky') ... >>> res_sky = muse_sky(res.PIXTABLE_OBJECT) If not saved, the output is usually lost! During recipe run, a temporary directory is created where the :class:`astropy.io.fits.HDUList` input objects and the output files are put into. This directory is cleaned up afterwards. To control message verbosity on terminal (use :literal:`'debug'`, :literal:`'info'`, :literal:`'warn'`, :literal:`'error'` or :literal:`'off'`): >>> cpl.msg.esorex.level = 'debug' python-cpl-0.7.4/doc/index.rst0000664000175000017500000000313112515637757015627 0ustar oleole00000000000000.. Python bindings for CPL recipes documentation master file, created by sphinx-quickstart on Sat Aug 28 10:52:22 2010. You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. .. title:: The CPL recipe python interface The CPL recipe python interface ############################### .. module:: cpl This is a non-official python module to access CPL recipes. It is not meant as part of the CPL or the MUSE pipeline software, but may be useful for testing and analysis. .. seealso:: http://www.eso.org/sci/software/cpl "The Common Pipeline Library (CPL) consists of a set of C libraries, which have been developed to standardise the way VLT instrument pipelines are built, to shorten their development cycle and to ease their maintenance. The Common Pipeline Library was not designed as a general purpose image processing library, but rather to address two primary requirements. The first of these was to provide an interface to the VLT pipeline runtime- environment. The second was to provide a software kit of medium-level tools, which allows astronomical data-reduction tasks to be built rapidly." [`ESO `_] .. toctree:: :numbered: install tutorial recipe parallel param frames result msg esorex dfs restrictions Feedback ======== Bug reports should be made on the `developer web page `_. Send python specific questions to python-cpl@liska.ath.cx. Questions regading CPL should be mailed to cpl-help@eso.org. python-cpl-0.7.4/doc/esorex.rst0000664000175000017500000000226112055720772016015 0ustar oleole00000000000000.. module:: cpl :mod:`cpl.esorex` :program:`EsoRex` legacy support ================================================== .. automodule:: cpl.esorex Support for configuration and SOF files --------------------------------------- .. autofunction:: init .. autofunction:: load_rc .. autofunction:: load_sof Convienence logging control --------------------------- .. autodata:: msg The following attributes control the format of the terminal messages: .. autoattribute:: cpl.esorex.CplLogger.level .. autoattribute:: cpl.esorex.CplLogger.format .. autoattribute:: cpl.esorex.CplLogger.time .. autoattribute:: cpl.esorex.CplLogger.component .. autoattribute:: cpl.esorex.CplLogger.threadid .. autodata:: log The following attributes control the format of the log file messages: .. autoattribute:: cpl.esorex.CplLogger.filename .. attribute:: CplLogger.dir Directory name that is prepended to the log file name. .. autoattribute:: cpl.esorex.CplLogger.level .. autoattribute:: cpl.esorex.CplLogger.format .. autoattribute:: cpl.esorex.CplLogger.time .. autoattribute:: cpl.esorex.CplLogger.component .. autoattribute:: cpl.esorex.CplLogger.threadid python-cpl-0.7.4/doc/dfs.rst0000664000175000017500000000024411650463667015272 0ustar oleole00000000000000:mod:`cpl.dfs` DFS header parsing ================================= .. automodule:: cpl.dfs .. autoclass:: ProcessingInfo .. automethod:: ProcessingInfo.__init__ python-cpl-0.7.4/doc/recipe.rst0000644000175000017500000001017612235465526015764 0ustar oleole00000000000000.. module:: cpl The Recipe interface ==================== .. autoclass:: Recipe Static members -------------- .. autoattribute:: Recipe.path .. autoattribute:: Recipe.memory_mode .. automethod:: Recipe.list() .. automethod:: Recipe.set_maxthreads(n) Constructor ----------- .. automethod:: Recipe.__init__ Common attributes and methods ----------------------------- These attributes and methods are available for all recipes. .. attribute:: Recipe.__name__ Recipe name. .. autoinstanceattribute:: Recipe.__file__ Shared library file name. .. autoattribute:: Recipe.__author__ .. autoattribute:: Recipe.__email__ .. autoattribute:: Recipe.__copyright__ .. autoattribute:: Recipe.description .. autoattribute:: Recipe.version .. autoattribute:: Recipe.cpl_version .. autoattribute:: Recipe.cpl_description .. attribute:: Recipe.output_dir Output directory if specified, or :obj:`None`. The recipe will write the output files into this directory and return their file names. If the directory does not exist, it will be created before the recipe is executed. Output files within the output directory will be silently overwritten. If no output directory is set, the recipe call will result in :class:`astropy.io.fits.HDUList` result objects. The output directory may be also set as parameter in the recipe call. .. attribute:: Recipe.temp_dir Base directory for temporary directories where the recipe is executed. The working dir is created as a subdir with a random file name. If set to :obj:`None`, the system temp dir is used. Defaults to :literal:`'.'`. .. attribute:: Recipe.threaded Specify whether the recipe should be executed synchroniously or as an extra process in the background. .. seealso:: :ref:`parallel` .. autoattribute:: Recipe.tag .. autoattribute:: Recipe.tags .. autoattribute:: Recipe.output .. attribute:: Recipe.memory_dump If set to 1, a memory dump is issued to stdout if the memory was not totally freed after the execution. If set to 2, the dump is always issued. Standard is 0: nothing dumped. Recipe parameters ----------------- Recipe parameters may be set either via the :attr:`Recipe.param` attribute or as named keywords on the run execution. A value set in the recipe call will overwrite any value that was set previously in the :attr:`Recipe.param` attribute for that specific call. .. autoattribute:: Recipe.param .. seealso:: :class:`cpl.Parameter` Recipe frames ------------- There are three groups of frames: calibration ("calib") frames, input ("raw") frames, and result ("product") frames. Calibration frames may be set either via the :attr:`Recipe.calib` attribute or as named keywords on the run execution. A value set in the recipe call will overwrite any value that was set previously in the :attr:`Recipe.calib` attribute for that specific call. Input frames are always set in the recipe call. If their tag name was not given, the tag name from :attr:`Recipe.tag` is used if the recipe provides it. .. autoattribute:: Recipe.calib .. seealso:: :class:`cpl.FrameConfig` Runtime environment ------------------- For debugging purposes, the runtime environment of the recipe may be changed. The change may be either done by specifying the :attr:`Recipe.env` attribute of as a parameter on the recipe invocation. The change will have no influence on the environment of the framework itself. .. note:: Some variables are only read on startup (like :envvar:`MALLOC_CHECK_`), changing or deleting them will have no effect. .. autoinstanceattribute:: Recipe.env Environment changes for the recipe. This is a :class:`dict` with the name of the environment variable as the key and the content as the value. It is possible to overwrite a specific environment variable. Specifying :obj:`None` as value will remove the variable:: >>> muse_flat.env['MUSE_RESAMPLE_LAMBDA_LOG'] = '1' >>> muse_flat.env['MUSE_TIMA_FILENAME'] = 'tima.fits' In a recipe call, the runtime environment may be overwritten as well:: >>> res = muse_flat( ..., env = {'MUSE_PLOT_TRACE':'true'}) Recipe invocation ----------------- .. automethod:: Recipe.__call__ .. seealso:: :ref:`parallel` python-cpl-0.7.4/doc/Makefile0000664000175000017500000000610012051443224015400 0ustar oleole00000000000000# Makefile for Sphinx documentation # # You can set these variables from the command line. SPHINXOPTS = SPHINXBUILD = PYTHONPATH=.. sphinx-build PAPER = BUILDDIR = _build # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . .PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest help: @echo "Please use \`make ' where is one of" @echo " html to make standalone HTML files" @echo " dirhtml to make HTML files named index.html in directories" @echo " pickle to make pickle files" @echo " json to make JSON files" @echo " htmlhelp to make HTML files and a HTML help project" @echo " qthelp to make HTML files and a qthelp project" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" @echo " changes to make an overview of all changed/added/deprecated items" @echo " linkcheck to check all external links for integrity" @echo " doctest to run all doctests embedded in the documentation (if enabled)" clean: -rm -rf $(BUILDDIR)/* html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." dirhtml: $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." pickle: $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." json: $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." htmlhelp: $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." qthelp: $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ ".qhcp project file in $(BUILDDIR)/qthelp, like this:" @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/None.qhcp" @echo "To view the help file:" @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/None.qhc" latex: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ "run these through (pdf)latex." changes: $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." linkcheck: $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." doctest: $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." python-cpl-0.7.4/doc/conf.py0000664000175000017500000001500712515646551015262 0ustar oleole00000000000000# -*- coding: utf-8 -*- # # Python bindings for CPL recipes documentation build configuration file, created by # sphinx-quickstart on Sat Aug 28 10:52:22 2010. # # This file is execfile()d with the current directory set to its containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. # # All configuration values have a default; values that are commented out # serve to show the default. import sys, os import cpl # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. #sys.path.append(os.path.abspath('.')) # -- General configuration ----------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx' ] # , 'cpl.cplsphinx' # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # The suffix of source filenames. source_suffix = '.rst' # The encoding of source files. source_encoding = 'utf-8' # The master toctree document. master_doc = 'index' # General information about the project. project = u'Python bindings for CPL recipes' copyright = u'2010-2015, Ole Streicher' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The full version, including alpha/beta/rc tags. release = cpl.__version__ # The short X.Y version. #version = '0.7' version=release # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. #language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: #today = '' # Else, today_fmt is used as the format for a strftime call. #today_fmt = '%B %d, %Y' # List of documents that shouldn't be included in the build. #unused_docs = [] # List of directories, relative to source directory, that shouldn't be searched # for source files. exclude_trees = ['_build'] # The reST default role (used for this markup: `text`) to use for all documents. #default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. #add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). #add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. #show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. #modindex_common_prefix = [] # -- Options for HTML output --------------------------------------------------- # The theme to use for HTML and HTML Help pages. Major themes that come with # Sphinx are currently 'default' and 'sphinxdoc'. html_theme = 'default' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. #html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. #html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". html_title = "Python bindings for CPL" # A shorter title for the navigation bar. Default is the same as html_title. #html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. #html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. #html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". #html_static_path = ['_static'] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. #html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. #html_use_smartypants = True # Custom sidebar templates, maps document names to template names. #html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. #html_additional_pages = {} # If false, no module index is generated. #html_use_modindex = True # If false, no index is generated. #html_use_index = True # If true, the index is split into individual pages for each letter. #html_split_index = False # If true, links to the reST sources are added to the pages. html_show_sourcelink = False # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. #html_use_opensearch = '' # If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). #html_file_suffix = '' # Output file base name for HTML help builder. htmlhelp_basename = 'python-cpl' # -- Options for LaTeX output -------------------------------------------------- # The paper size ('letter' or 'a4'). latex_paper_size = 'a4' # The font size ('10pt', '11pt' or '12pt'). #latex_font_size = '11pt' # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ ('index', 'python-cpl-%s.tex' % version, u'Python bindings for CPL recipes', u'Ole Streicher', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of # the title page. #latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. #latex_use_parts = False # Additional stuff for the LaTeX preamble. #latex_preamble = '' # Documents to append as an appendix to all manuals. #latex_appendices = [] # If false, no module index is generated. latex_use_modindex = False # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = {'http://docs.python.org/': None, 'http://docs.astropy.org/':None} python-cpl-0.7.4/doc/restrictions.rst0000664000175000017500000000416512055720772017245 0ustar oleole00000000000000.. _restrictions: .. module:: cpl Restrictions for CPL recipes ============================ Not every information can be retrieved from recipes with the standard CPL functions. Only MUSE recipes provide additional interfaces that allow the definition of input, calibration and output frames. All other interfaces will have the following restrictions: #. The :attr:`Recipe.calib` attribute is not filled with templates for calibration frames. After recipe creation, this attribute is empty. Also, no check on the required calibration frames may be done before calling the recipe. Anything that is set here will be forwarded to the recipe. #. In the :mod:`cpl.esorex` support, directly assigning the recipe calibration files from the SOF file with :literal:`recipe.calib = cpl.esorex.read_sof('file')` will also put the raw input file into :attr:`Recipe.calib` unless :attr:`Recipe.tags` and/or :attr:`Recipe.tag` are set manually. The standard recipe interface does not provide a way to distinguish between raw input and calibration files. #. The :attr:`Recipe.tags` attribute is set to :obj:`None`. #. The :attr:`Recipe.tag` attribute is not initially set. If this attribute is not set manually, the tag is required when executing the attribute. #. Accessing the attribute :meth:`Recipe.output` raises an exception. Technical Background -------------------- CPL recipes register all their parameter definitions with the CPL function :func:`cpl_parameterlist_append()`. All registered parameters may be retrieved from the recipe structure as a structure which contains all defined parameters. For frames, such a mechanism does not exist, although components of the infrastructure are implemented. The CPL modules :mod:`cpl_recipeconfig` allows the definition of input, raw, and output frames for a recipe. However, this module is only half-way done, has no connection to the recipe definition and is not mandantory for CPL recipes. The MUSE pipeline recipes (with the exception of those contributed by ESO) implement a central frameconfig registry which allows to access this meta information from the Python interface. python-cpl-0.7.4/doc/frames.rst0000644000175000017500000000022511460377136015762 0ustar oleole00000000000000.. module:: cpl The :class:`cpl.FrameConfig` class ================================== .. autoclass:: FrameConfig .. seealso:: :attr:`Recipe.calib` python-cpl-0.7.4/doc/msg.rst0000664000175000017500000000556212055720772015305 0ustar oleole00000000000000Log messages ============ We provide CPL log messages in two different ways: via Python logging and as a list of messages in the :class:`cpl.Result` object. For convienience, simple terminal messages and predefined log file output in a style similar to the original CPL messages. .. currentmodule:: cpl.logger Python style logging -------------------- The preferred and most flexible way to do logging is the use of the :mod:`logging` module of Python. A basic setup (similar to the style used in `esorex `_) is:: import logging log = logging.getLogger() log.setLevel(logging.INFO) ch = logging.FileHandler('cpl_recipe.log') ch.setLevel(logging.INFO) fr = logging.Formatter('%(created)s [%(levelname)s] %(name)s: %(message)s', '%H:%M:%S') ch.setFormatter(fr) log.addHandler(ch) The default basic log name for CPL log messages in the recipes is :file:`cpl.{recipename}`. The log name can be changed with the ``logname`` parameter of :class:`cpl.Recipe.__call__()` to follow own naming rules, or to separate the output of recipes that are executed in parallel:: res = [ muse_focus(f, logname = 'cpl.muse_focus%02i' % (i+1), threading = True) for i, f in enumerate(inputfiles) ] To the basic log name the function name is appended to allow selective logging of a certain function. The following sample line:: logging.getLogger('cpl.muse_sky.muse_sky_create_skymask').setLevel(logging.DEBUG) will log the debug messages from :func:`muse_sky_create_skymask()` additionally to the other messages. .. note:: Since the log messages are cached in CPL, they may occur with some delay in the python log module. Also, log messages from different recipes running in parallel may be mixed in their chronological order. The resolution of the log time stamp is one second. The fields :attr:`logging.LogRecord.args`, :attr:`logging.LogRecord.exc_info` and :attr:`logging.LogRecord.lineno` are not set. Also, due to limitations in the CPL logging module, level filtering is done only after the creation of the log entries. This may cause performance problems if extensive debug logging is done and filtered out by :class:`logging.Logger.setLevel()`. In this case the :class:`cpl.Recipe.__call__()` parameter ``loglevel`` may be used. .. seealso :: :data:`cpl.esorex.msg` and :data:`cpl.esorex.log` EsoRex like convienience logging. Log message lists ----------------- The :class:`cpl.Result` object as well as a :class:`cpl.CplError` have an attribute :attr:`cpl.Result.log` resp. :attr:`cpl.CplError.log` that contains the :class:`list` of all log messages. .. autoclass:: cpl.logger.LogList .. autoattribute:: cpl.logger.LogList.error .. autoattribute:: cpl.logger.LogList.warning .. autoattribute:: cpl.logger.LogList.info .. autoattribute:: cpl.logger.LogList.debug python-cpl-0.7.4/doc/result.rst0000644000175000017500000001246012235465526016031 0ustar oleole00000000000000.. module:: cpl Execution results ================= Result frames ------------- .. class:: cpl.Result Calling :meth:`cpl.Recipe.__call__` returns an object that contains all result ('production') frames in attributes. All results for one tag are summarized in one attribute of the same name. So, the ``muse_bias`` recipe returns a frame with the tag MASTER_BIAS in the according attribute:: res = muse_bias(...) res.MASTER_BIAS.writeto('master_bias') The attribute content is either a :class:`astropy.io.fits.HDUList` or a :func:`list` of HDU lists, depending on the recipe and the call: If the recipe produces one out put frame of a tag per input file, the attribute contains a list if the recipe was called with a list, and if the recipe was called with a single input frame, the result attribute will also contain a single input frame. If the recipe combines all input frames to one output frame, a single :class:`astropy.io.fits.HDUList` es returned, independent of the input parameters. The following examples will illustrate this:: muse_scibasic = cpl.Recipe('muse_scibasic') ... # Only single input frame, so we get one output frame res = muse_scibasic('raw.fits') res.PIXTABLE_OBJ.writeto('pixtable.fits') # List of input frames results in a list of output frames res = muse_scibasic([ 'raw1.fits', 'raw2.fits', 'raw3.fits' ]) for i, h in res.PIXTABLE_OBJ: h.writeto('pixtable%i.fits' % (i+1)) # If we call the recipe with a list containing a single frame, we get a list # with a single frame back res = muse_scibasic([ 'raw1.fits' ]) res.PIXTABLE_OBJ[0].writeto('pixtable1.fits') # The bias recipe always returns one MASTER BIAS, regardless of number of # input frames. So we always get a single frame back. muse_bias = cpl.Recipe('muse_bias') ... res = muse_bias([ 'bias1.fits', 'bias2.fits', 'bias3.fits' ]) res.MASTER_BIAS.writeto('master_bias.fits') .. note:: This works well only for MUSE recipes. Other recipes dont provide the necessary information about the recipe. Run statistics -------------- In Addition to the result frames the :class:`cpl.Result` object provides the attribute :attr:`cpl.Result.stat` which contains several statistics of the recipe execution: .. attribute:: cpl.Result.return_code The return code of the recipe. Since an exception is thrown if the return code indicates an error, this attribute is always set to 0. .. attribute:: cpl.Result.stat.user_time CPU time in user mode, in seconds. .. attribute:: cpl.Result.stat.sys_time CPU time in system mode, in seconds. .. attribute:: cpl.Result.stat.memory_is_empty Flag whether the recipe terminated with freeing all available Memory. This information is only available if the CPL internal memory allocation functions are used. If this information is not available, this flag ist set to :obj:`None`. .. seealso:: :attr:`Recipe.memory_mode` Execution log ------------- .. attribute:: cpl.Result.log List of log messages for the recipe. .. seealso:: :class:`cpl.logger.LogList` .. attribute:: cpl.Result.error If one or more error was set during the recipe run, the first error is stored in this attribute. The following errors are chained and can be accessed with the :attr:`cpl.CplError.next` attribute. .. note:: An error here does not indicate a failed recipe execution, since a failed execution would result in a non-zero return code, and an exception would be thrown. .. seealso:: :class:`cpl.CplError` Thread control -------------- If the recipe was called in the background (see :ref:`parallel`), the result object is returned immediately and is dervived from :class:`threading.Thread`. Its interface can be used to control the thread execution: .. method:: cpl.Result.isAlive() Returns whether the recipe is still running .. method:: cpl.Result.join(timeout = None) Wait until the recipe terminates. This blocks the calling thread until the recipe terminates – either normally or through an unhandled exception – or until the optional timeout occurs. When the timeout argument is present and not :obj:`None`, it should be a floating point number specifying a timeout for the operation in seconds (or fractions thereof). As :meth:`join` always returns :obj:`None`, you must call :meth:`isAlive` after :meth:`join` to decide whether a timeout happened – if the recipe is still running, the :meth:`join` call timed out. When the timeout argument is not present or :obj:`None`, the operation will block until the recipe terminates. A thread can be :meth:`cpl.Result.join` ed many times. Like in the foreground execution, the output frames may be retrieved as attributes of the :class:`cpl.Result` frame. If any of the attributes is accessed, the calling thread will block until the recipe is terminated. If the recipe execution raised an exception, this exception will be raised whenever an attribute is accessed. CPL Exceptions -------------- .. autoexception:: CplError .. autoexception:: RecipeCrash python-cpl-0.7.4/setup.py0000644000175000017500000000477413370120526014724 0ustar oleole00000000000000import os from setuptools import setup, Extension author = 'Ole Streicher' email = 'python-cpl@liska.ath.cx' license_ = 'GPL' cpl_version = '0.7.4' description = "Python interface for the ESO Common Pipeline Library" long_description = '''\ This module can list, configure and execute CPL-based recipes from Python (python2 and python3). The input, calibration and output data can be specified as FITS files or as ``astropy.io.fits`` objects in memory. The ESO `Common Pipeline Library `_ (CPL) comprises a set of ISO-C libraries that provide a comprehensive, efficient and robust software toolkit. It forms a basis for the creation of automated astronomical data-reduction tasks. One of the features provided by the CPL is the ability to create data-reduction algorithms that run as plugins (dynamic libraries). These are called "recipes" and are one of the main aspects of the CPL data-reduction development environment. ''' pkgname = 'python-cpl' baseurl = ('https://files.pythonhosted.org/packages/source/' '{0}/{1}/{1}-{2}.tar.gz'.format(pkgname[0], pkgname, cpl_version)) classifiers = '''Development Status :: 4 - Beta Intended Audience :: Science/Research License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+) Operating System :: MacOS :: MacOS X Operating System :: POSIX Operating System :: Unix Programming Language :: C Programming Language :: Python Programming Language :: Python :: 2 Programming Language :: Python :: 3 Topic :: Scientific/Engineering :: Astronomy '''.splitlines() def create_version_file(cpl_version=cpl_version): with open(os.path.join('cpl', 'version.py'), 'w') as vfile: vfile.write("version = %s\n" % repr(cpl_version)) vfile.write("author = %s\n" % repr(author)) vfile.write("email = %s\n" % repr(email)) vfile.write("license_ = %s\n" % repr(license_)) try: create_version_file() except IOError: pass module1 = Extension('cpl.CPL_recipe', sources=['cpl/CPL_recipe.c', 'cpl/CPL_library.c']) setup( name=pkgname, version=cpl_version, author=author, author_email=email, description=description, long_description=long_description, license=license_, url='https://pypi.org/project/%s/%s' % (pkgname, cpl_version), download_url='%s/%s-%s.tar.gz' % (baseurl, pkgname, cpl_version), classifiers=classifiers, python_requires='>=2.7', install_requires=['astropy'], provides=['cpl'], packages=['cpl'], ext_modules=[module1] )