yorick-hdf5-0.8.0/0000755000076500001440000000000011254102034013220 5ustar frigautusersyorick-hdf5-0.8.0/check.i0000644000076500001440000001164411254101475014465 0ustar frigautusers// $Id: check.i,v 1.2 2008/11/21 16:19:17 frigaut Exp $ require,"hdf5.i"; require,"fits.i"; //plug_dir,"."; dfname = "data.h5"; func checkhdf5(notiming=,keep=) /* DOCUMENT checkhdf5(notiming=,keep=) keep=1 does not delete h5 files. SEE ALSO: */ { if (notiming==[]) notiming=1; d1 = span(0,60,100); d2 = random_n([2,3,5]); d3 = 5.678; d4 = ["history list","written on 2005nov8"]; a1 = 20.6; a2 = "minutes"; a3 = [1,3,1,4,2,2]; c=["blue","red","cyan"]; // create file with data: write,"Creating data.h5 with data"; f = h5open(dfname,"w"); h5write,f,"/2005mar04/time",d1; h5write,f,"/2005mar04/data",d2; h5write,f,"/2005mar04/dewpoint",d3; h5write,f,"/2005mar04/firsthalf/21hr/dewpoint",d3; h5close,f; f = h5open(dfname,"a"); // check append h5write,f,"/2005mar04/comments",d4; h5write,f,"/2005mar05/name with blanks",8.97; write,"\nNow adding comment attributes"; h5awrite,f,"/2005mar04","Temp on this date",a1; h5awrite,f,"/2005mar04/time","Units",a2; h5awrite,f,"/2005mar04","cups of coffee",a3; h5awrite,f,"/2005mar04","colors",c; h5close,f; h5awrite,dfname,"/2005mar04/time","Units2",a2; write,"Reading attribute color"; c1=h5aread(dfname,"/2005mar04","colors"); if (!allof(c==c1)) error,"Read attribute data != written data"; write,"Deleting attribute color"; h5adelete,dfname,"/2005mar04","colors"; write,"\nRunning h5info"; h5info,dfname,att=1; write,"\nAdding soft and hard links"; h5link,dfname,"/bestdata","/2005mar04"; h5link,dfname,"/multiple/path/bestdata","/2005mar04"; h5link,dfname,"/21hr","/2005mar04/firsthalf/21hr"; h5link,dfname,"/HL","/2005mar04",0; h5info,dfname,att=1; write,"\nTesting reads"; d = h5read(dfname,"/2005mar04/time"); if (!allof(d==d1)) error,"Read data != written data"; d = h5read(dfname,"/2005mar04/data"); if (!allof(d==d2)) error,"Read data != written data"; d = h5read(dfname,"/2005mar04/dewpoint"); if (!allof(d==d3)) error,"Read data != written data"; d = h5read(dfname,"/2005mar04/comments"); if (!allof(d==d4)) error,"Read data != written data"; a = h5aread(dfname,"/2005mar04","Temp on this date"); if (!allof(a==a1)) error,"Read attribute != written attribute"; a = h5aread(dfname,"/2005mar04/time","Units"); if (!allof(a==a2)) error,"Read attribute != written attribute"; a = h5aread(dfname,"/2005mar04","cups of coffee"); if (!allof(a==a3)) error,"Read attribute != written attribute"; d = h5read(dfname,"/HL/time"); if (!allof(d==d1)) error,"Read data != written data for hard link"; d = h5read(dfname,"/bestdata/time"); if (!allof(d==d1)) error,"Read data != written data for soft link"; write,"all Read OK and identical to written data"; write,"\nTesting compression"; dim = 256; x= (indgen(dim)-dim)*array(1.0f,dim)(-,); y = transpose(x); ar=float(exp(-(sqrt(x^2.+y^2.)/12.))*20.); h5write,"zip.h5","/image",ar,zip=3; ar1=h5read("zip.h5","/image"); if (!allof(d==d1)) error,"Read data != written data for compression"; write,"Read Compressed OK and identical to written data"; h5write,"nozip.h5","/image",ar; write,"\nTesting writing strings"; s=["A fight is a contract that takes two people to honor.",\ "A combative stance means that you've accepted the contract.",\ "In which case, you deserve what you get.",\ " -- Professor Cheng Man-ch'ing"]; h5write,"strings.h5","/StringsEx",s; h5info,"strings.h5"; sr = h5read("strings.h5","/StringsEx"); if (!allof(s==sr)) error,"Read data != written data for string"; write,"Read OK and identical to written string data"; if (!notiming) { write,"\nTiming"; dim = 1024; x= (indgen(dim)-dim)*array(1.0f,dim)(-,); y = transpose(x); ar=float(exp(-(sqrt(x^2.+y^2.)/12.))*20.); write,"Writing..."; pause,100; tic;h5write,"nozip.h5","/image",ar;t=tac(); tic;h5write,"nozip.h5","/image",ar;t=tac(); write,format="hdf5 %d^2 float write: %fs\n",dim,t; pause,100; tic;fits_write,"nozip.fits",ar,overwrite=1;t=tac(); tic;fits_write,"nozip.fits",ar,overwrite=1;t=tac(); write,format="stock fits %d^2 float write: %fs\n",dim,t; write,"Reading..."; pause,100; tic;d=h5read("nozip.h5","/image");t=tac(); tic;d=h5read("nozip.h5","/image");t=tac(); write,format="hdf5 %d^2 float read: %fs\n",dim,t; pause,100; tic;d=fits_read("nozip.fits");t=tac(); tic;d=fits_read("nozip.fits");t=tac(); write,format="stock fits dim^2 float read: %fs\n",dim,t; } if (!keep) { remove,"data.h5"; remove,"strings.h5"; remove,"nozip.h5"; remove,"zip.h5"; } } checkhdf5; //if (!keep) system,"rm "+dfname+" zip.h5 nozip.h5 strings.h5"; func checkmemleaks { for (i=1;i<=10;i++) { checkhdf5,notiming=1; if (i==2) { ystats = yorick_stats(); ystats = yorick_stats(); } } if (nallof((ystats2=yorick_stats())==ystats)) { write,"memory leak?"; write,ystats; write,ystats2; } else write,"\nNo memory leaks detected"; } yorick-hdf5-0.8.0/h5convert_fromshell.i0000644000076500001440000000235611111554161017374 0ustar frigautusers// to be called by the shell script h5convert // usefull for mass conversion and easy access from // the shell require,"util_fr.i"; require,"hdf5.i"; func h5convert_help(void) { write,"h5convert [-v# -f -p -h] file1 file2 ..."; write," -v#: verbose level (0,1,2)"; write," -p: in place (output name = input name)"; write," -f: force processing file, even though it should not"; write," -h: this help"; write,"files can use widlcards"; quit; } // progress args a = get_argv(); w = (a == "h5convert_fromshell.i"); if (where(w)(1)==numberof(a)) h5convert_help; // keep only flags and files a = a (where(w)(1)+1:); // find flags w = strglob("-*",a); if (anyof(w)) { flags = a(where(w)); if (anyof(flags=="-v0")) verbose=0; if (anyof(flags=="-v1")) verbose=1; if (anyof(flags=="-v2")) verbose=2; if (anyof(flags=="-f")) force=1; if (anyof(flags=="-p")) inplace=1; if (anyof(flags=="-h")) h5convert_help; } // files to process (can have wildcards) w = where(w==0); if (numberof(w)==0) h5convert_help; files = a(w); filesv = []; // build string vector with all files for (i=1;i<=numberof(files);i++) { grow,filesv,findfiles(files(i)); } // do it h5old2new,filesv,inplace=inplace,verbose=verbose,force=force; quit yorick-hdf5-0.8.0/h5scan_fromshell.i0000644000076500001440000000010511111540601016620 0ustar frigautusersa = get_argv(); if (anyof(a=="-a")) att=1; h5info,a(0),att=att; quit yorick-hdf5-0.8.0/h5info0000755000076500001440000000005511111552310014334 0ustar frigautusers#!/bin/sh yorick -q -i h5scan_fromshell.i $* yorick-hdf5-0.8.0/check2.i0000644000076500001440000000202211254101511014524 0ustar frigautusers#include "hdf5.i" g = [[1,2,3],[4,5,6],[7,8,9],[10,11,12]]; gt=transpose(g); f = h5open("h5yorout.h5","w"); h5write, f,"base/g",g; h5write, f,"base/gt",gt; h5write, f,"base/dummy",1.; // attributes h5awrite, f,"base/dummy","g as attribute",g; h5awrite, f,"base/dummy","attribute#2",[1,2,3]; h5awrite, f,"base/dummy","attribute#3","this is a string"; h5close, f; "g";info,g; nprint,g*1.; g(*); "gt";info,gt; nprint,gt*1.; system,"h5dump h5yorout.h5"; "g read";gr=h5read("h5yorout.h5","base/g"); info,gr; nprint,gr*1.; "gt read";gtr=h5read("h5yorout.h5","base/gt"); info,gtr; nprint,gtr*1.; write,"\n\n\n###### Dataset ########"; write,"g:\nInput"; nprint,g*1.; write,"\nh5dump of g:"; system,"h5dump -d /base/g h5yorout.h5"; write,"\ng as read back:"; nprint,gr*1.; write,"\n\n\n###### Attribute #######"; write,"g:\nInput"; nprint,g*1.; write,"\nh5dump of g:"; system,"h5dump -a \"/base/dummy/g as attribute\" h5yorout.h5"; write,"\ng as read back:"; gar=h5aread("h5yorout.h5","base/dummy","g as attribute"); info,gar; nprint,gr*1.; quit yorick-hdf5-0.8.0/hdf5.i0000644000076500001440000015017111254101642014231 0ustar frigautusersHDF5_VERSION = "0.8.0"; /* HDF5 Yorick plugin * * $Id: hdf5.i,v 1.5 2008/11/21 19:00:54 frigaut Exp $ * * Francois Rigaut, February 2005 * Created: 24 February 2005 * last revision/addition: * * $Log: hdf5.i,v $ * 2009/09/16: * - Version 0.8.0 * - I just realized today there was an issue with the new HDF5 APIs. * added dynamic declaration of H5 static variable (from their value * in the HDF5 include files). * - Also, writting attributes with string value was to be broken (see * note in h5awrite. Fixed by setting last param of H5Acreate to 0. * * Revision 1.5 2008/11/21 19:00:54 frigaut * - added some text to man page * - added warning mechanism through h5v062bug_warning() function. * - version 0.7.1 * * Revision 1.4 2008/11/21 17:29:47 frigaut * - added some support for reading 64bits longs in a 32 bits OS * * Revision 1.3 2008/11/21 16:19:17 frigaut * - added h5old2new to convert pre-v0.6.2 files to post v0.6.2 file formats * - added h5convert and h5info shell wrappers * - added h5convert_fromshell.i and h5scan_fromshell.i to go with it. * - added h5convert.1 man page (but not yet h5info.1) * - updated Makefile to include these new files for the install and * make package * - bumped version to 0.7.0 * * Revision 1.2 2008/11/13 21:19:44 frigaut * - Fixed swapped array dimension and messed up array shape * as reported by David Strozzi. * * Revision 1.1.1.1 2007/12/27 15:10:25 frigaut * Initial Import - yorick-hdf5 * * 11 dec 2005, v0.6 * many hours of work on this plugin. * - the whole h5open/h5close is now much more robust. * - fixed many occurences where things were left open * - added 2 new functions: h5version, h5list_open * * 8 nov 2005 * fixed issue when reading large array of strings in hdf5.c * * 25 may 2005 * fixed a bug in creating multiple levels groups * * 24 february 2005 * initial revision * * * Copyright (c) 2005, Francois RIGAUT (frigaut@gemini.edu, Gemini * Observatory, 670 N A'Ohoku Place, HILO HI-96720). * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the * Free Software Foundation; either version 2 of the License, or (at your * option) any later version. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details (to receive a copy of the GNU * General Public License, write to the Free Software Foundation, Inc., 675 * Mass Ave, Cambridge, MA 02139, USA). */ plug_in, "hdf5"; local hdf5; /* DOCUMENT HDF5 plugin: Simple HIERARCHICAL DATA FORMAT 5 wrappers. DATA I/O: h5read(fname,target) return data h5write,fname,fullpath,data,zip=,mode= write data h5open(filename,mode) return file handle h5close(filename) close file INFO, GROUP LINKING: h5info,fname,target,att= print out file info/structure h5link(fname,group,group2link2,linktype) link datasets h5delete,fname,object delete data ATTRIBUTE I/O: h5awrite(fname,object,aname,attdata) write an object attribute h5aread(fname,object,aname) read an object attribute h5adelete,fname,object,aname delete an object attribute MISC: h5list_open list open files h5version return linhdf5 version ERROR RECOVERY h5off close all reference to the h5 library (clean up). + Many of the atomic HDF5 functions are available (e.g. H5Fopen, H5Dopen) with mostly the same APIs. Many simple tasks can be done with the provided high level wrappers. Beware that programming an HDF5 custom wrapper is not trivial. Make sure you close every objects opened! This implementation supports reading of most of the HDF5 supported datatype. Only yorick datatype can be used for writes. Compression, soft and hard links, as well as support for attribute read/writes is provided. There is no support for hyperslabs, compound, enum or opaque datatype. Generally, there is very little support for datatype related functionalities. EXAMPLES: 1. simplest: h5write,"sinus.h5","/sin2t","sin(2*indgen(100)); 2. ex2: save 2 vectors fd1=h5open("sinus.h5","w"); // "w" start from scratch t = span(0.,2*pi,100); h5write,fd1,"/data/t",t; h5write,fd1,"/data/sin2t",sin(2*t); h5close,fd1; 3. ex3: append another vector and attribute h5write,"sinus.h5","/data/damped sin",sin(t)*exp(-0.7*t),mode="a"; h5awrite,"sinus.h5","/data/damped sin", \ "functional form","sin(t)*exp(-0.7t)"; 4. examine the content of the file h5info,"sinus.h5"; NOTE: HDF5_SAFE is a global flag that, if set, will prevent to open an existing file in mode "w". In this mode, if the file has indeed to be overwritten, it needs to be deleted prior to the call to h5open. SEE ALSO: */ //:::::::::::::::::::::::::::::::::::: HDF5_SAFE=0; // if set to 1, will prevent opening file that // already exist in mode "w". This can prevent // accidental deletion of important data, however // it can be a pain. //:::::::::::::::::::::::::::::::::::: struct _hdf5info { string name, objectType; pointer aname; string target; }; struct _hdf5file_struct { string fname; long fd; long nref; string curmode; }; //:::::::::::::::::::::::::::::::::::: hdf5_varname_maxcar = 36; // length of variable name printout. can be changed. // DO NOT change this: plugin_version_string="Generated by yorick HDF5 plugin version"; func h5v062bug_warning(force=) { // check if warning has been done already: if ((open(Y_USER+".hdf5_convert_warning_done","r",1)) && (!force)) return; // if not, do it: text = \ ["\n========================================================================", "========== IMPORTANT COMPATIBILITY NOTE ABOUT HDF5 FILES ===============", "=========== GENERATED WITH YORICK HDF5 PLUGIN V < 0.6.2 ===============", "========================================================================", "\nPlease read if you have hdf5 files generated with the yorick hdf5 plugin", "version 0.6.1 or earlier. These version were affected by a bug that ", "affected the compatibility of yorick generated data files with external", "I/O programs (HDF utilities and other external resources using the HDF5", "general I/O librairies). Namely, multi-dimensional arrays generated", "with the yorick HDF5 plugin appeared to external hdf5 resources with", "swapped dimensions (but not swapped elements). For instance, a 2x3 array", "as:", " 1 2", " 3 4", " 5 6"]; write,format="%s\n",text; rdline,,1,prompt="Type return to continue "; text = \ ["\nwould appear with h5dump (HDF group utility):", " 1 2 3", " 4 5 6", "","which is obviously messed up (not even a transpose)", "This did not affect yorick-to-yorick I/O (the same bug was present at", "input and output, thus cancelling out). However, I had to fix this bug.", "","IMPORTANT: This means however that the files generated while this bug", "was present have to be converted. I have no way to know if a given file", "was generated with the yorick plugin or some external facility, so I can", "not convert files automatically. Thus you, the user, has to convert your", "own files. I apologize for this. To facilitate the conversion, I have", "written (and debugged) a small function: h5old2new (see help page).", "In the hdf5 tarball, you should also find a small shell script, h5convert", "that wraps h5old2new and provide a convenient way to rename bunch of ", "files. See the man file (man ./h5convert.1 in the hdf5 directory, or", "help,h5convert within yorick.", "","Note that this only concerns multi-dimensional arrays. Other data types", "are not affected.", "This message will not be re-displayed.", "Call h5v062bug_warning,force=1 to display this message again\n"]; write,format="%s\n",text; rdline,,1,prompt="Type return to continue "; f=open(Y_USER+".hdf5_convert_warning_done","w"); close,f; } h5v062bug_warning; local h5convert; /* DOCUMENT This is a shell command. The help is provided here for convenience. NAME h5convert - Conversion of files written with hdf5 yorick plugin pre- v0.6.2 to "fixed" format of v0.6.2 and higher SYNOPSIS h5convert [-p -v# -f -h] files DESCRIPTION h5convert is a shell utility to convert files that were generated with the HDF5 yorick plugin v<0.6.2 to the new, fixed format used in v>=0.6.2. -p perform in-place conversion (output file name = input file name) -f force conversion of files generated with version>0.6.2 -v# verbose: -v0:silent -v1:files -v2:files and contents -h Print out short help AUTHOR Francois Rigaut, Gemini Observatory BUGS This is a patch for a previous bug. Beware: the flag "-p" will rename the converted file to the original name when done. Beware! This should be safe, but I haven't tested all possible scenario (i.e. objects type). This will erase your original data. I encourage you to check this conversion function on a few files before using this option. In particular, do not try to convert files generated by external HDF resources, as they will be converted while they should not! Also, these files may include data type and object type not yet implemented in the plugin, in which case it will trigger an error and the file may result corrupted. You have been warned. SEE ALSO: */ func h5info(file,target,att=,silent=,_recur=) /* DOCUMENT h5info,filename,target_group,att= Prints out the content of a HDF5 file (groups, datasets and links) filename: file name (string) or file descriptor (long) target_group: the root group at which to start scanning (default "/") att=1: print out objects attributes (to group and datasets only) SEE ALSO: h5read, h5write, h5link, h5delete */ { extern objnovec,objnoname,objectInfo; // initialize some variable at top of recursion if (!_recur) { objnovec=[]; objnoname=[]; objectInfo = []; } if (!target) target="/"; //default start group // Open file if (structof(file)==string) { has2bclose=1; file = h5open(file,"r"); } // Open group gid = _H5Gopen(file,target); if (gid<0) { if (has2bclose) h5close,file; error,"Unable to find group (h5info target has to be a group)"; } if (!_recur) { // process the root target group grow,objectInfo,_hdf5info(); objectInfo(0).name = target; objectInfo(0).objectType =" GROUP"; if (!silent) \ write,format="%-"+swrite(format="%d",hdf5_varname_maxcar)+"s",target; if (!silent) write,format=" %-8s\n","GROUP"; if (att) { natt = _H5Aget_num_attrs(gid); tmp=[]; for (na=0;nahdf5_varname_maxcar) dispname="..."+ strpart(fullname,-(hdf5_varname_maxcar-4):0); if (!silent) \ write,format="%-"+swrite(format="%d",hdf5_varname_maxcar)+"s",dispname; objno=[0,0]; otype = _H5Gget_objtype_by_name(gid,fullname,objno); objectInfo(0).name=fullname; if (otype==H5G_GROUP) { // object is other group. we will enter recursion. if ((objnovec!=[])&&anyof((objno==objnovec)(sum,)==2)) { w = where((objno==objnovec)(sum,)==2)(1); link2 = objnoname(w); if (!silent) write,format=" %s -> %s\n","HARDLINK",link2; objectInfo(0).objectType="HARDLINK"; objectInfo(0).target=link2; } else { if (!silent) write,format=" %-8s\n","GROUP"; objectInfo(0).objectType="GROUP"; if (att) { gid2 = _H5Gopen(file,fullname); natt = _H5Aget_num_attrs(gid2); tmp=[]; for (na=0;na1) { path = sum(tok(1:-1)+"/"); if (strpart(fname,1:1)=="/") path = "/"+path; path = strpart(path,1:-1); } else path="."; name=tok(0); // in case mode="r" or "a", file should already exist. check it. ctn = lsdir(path); if (anyof(mode==["a","r"])) { if (noneof(ctn==name)) error,swrite(format="File does not exist (mode=\"%s\")",mode); } else { // mode "w" if ((HDF5_SAFE)&&(anyof(ctn==name))) error,swrite(format="File already exist (mode=\"%s\") and HDF5_SAFE=1", mode); } // Open file if (mode=="r") { file=_H5Fopen(fname,H5F_ACC_RDONLY,0); if (file<0) { error,"Unable to open file (already open?)"; } } else { // does not exist, have to create: if (noneof(ctn==name)) mode="w"; if (mode=="a") { // open existing file = _H5Fopen(fname,H5F_ACC_RDWR,0); if (file<0) { error,"Unable to open file (already open?)"; } } else { // create: flags = H5F_ACC_TRUNC; file=_H5Fcreate(fname,H5F_ACC_TRUNC,0,0); if (file<0) { error,"Unable to create file (already open?)"; } } } // success. we can add the fname to the open file list: grow,_hdf5file,_hdf5file_struct(fname=fname,fd=file); w = where(_hdf5file.fname==fname)(1); _hdf5file(w).nref=1; _hdf5file(w).curmode=mode; return file; } //:::::::::::::::::::::::::::::::::::: func h5close(files) /* DOCUMENT func h5close(files) Close h5 access to file(s) (after a h5open) files: scalar or vector of file descriptor or file names Called without argument, this function closes all opened h5 files. */ { extern _hdf5file; if (files==[]) { // close all open files if (_hdf5file==[]) { write,format="%s\n","No file open"; return 0; } // select all files fd: files = _hdf5file.fd; // force nref to 1 so that close will happen _hdf5file.nref = _hdf5file.nref*0+1; } for (i=1;i<=numberof(files);i++) { // loop on input files if (!files(i)) continue; // could be 0 if (_hdf5file==[]) { write,format="%s\n","No file open"; return 0; } if (structof(files(i))==string) { // user passed file name, not file descriptor w = where(_hdf5file.fname==files(i)); wn = where(_hdf5file.fname!=files(i)); if (numberof(w)==0) error,swrite(format="%s does not exist. Can not close.",files(i)); fd = _hdf5file(w(1)).fd; } else { // user passed FD w = where(_hdf5file.fd==files(i)); wn = where(_hdf5file.fd!=files(i)); if (numberof(w)==0) error,swrite(format="File descriptor (%d) does not correspond"+ " to any open file. Can not close",files(i)); fd = files(i); } // see comment in h5open. If files has been opened several times, // we don't close it until the # opened reference is == 1 w=where(_hdf5file.fd==files(i))(1); if ((_hdf5file(w).nref)(1)>1) { (_hdf5file(w).nref)--; continue; } status = _H5Fclose(fd); if (status) error,swrite(format="Error closing file %s\n",_hdf5file(w).fname); // now suppress entry in _hdf5file: if (numberof(wn)==0) _hdf5file=[]; else _hdf5file=_hdf5file(wn); } return 0; } //:::::::::::::::::::::::::::::::::::: func h5list_open(void) /* DOCUMENT func h5list_open(void) Print (or return) a lit of open h5 files. SEE ALSO: h5open, h5close */ { extern _hdf5file; if (_hdf5file==[]) { write,format="%s\n","No file open"; return; } if (am_subroutine()) { write,format="%-20s mode \"%s\" (%d ref)\n",_hdf5file.fname, _hdf5file.curmode,_hdf5file.nref; } else { return _hdf5file.fname; } } //:::::::::::::::::::::::::::::::::::: func h5read(file,target,pre062=) /* DOCUMENT data=h5read(file,target,pre062=) Read content of one dataset in a HDF5 file and return the data file: h5 file name (string) or h5 file id (output from h5open) target: dataset name (string) pre062: set this keyword to read files generated by this plugin version pre-0.6.2 (hdf5 libraries array dimensions issue) This will read all data type but datatype will be casted to one of the yorick datatype (char,short,int,long,float,double,string). SEE ALSO: H5write, h5info, h5link, h5delete */ { if (structof(file)==string) { has2bclose=1; file = h5open(file,"r"); } // Open dataset dataset=_H5Dopen(file,target); if (dataset<0) { if (has2bclose) h5close,file; return; //error,"Unable to find Dataset"; } // get dataspace id: dspid=_H5Dget_space(dataset); //get rank: rank = _H5Sget_simple_extent_ndims(dspid); if (rank) { dims = maxdims = array(long,rank); status = _H5Sget_simple_extent_dims(dspid,dims,maxdims); if (!pre062) dims = dims(::-1); // dim bug fix } else {dims=0;} ytype=yotype(_H5Dget_type(dataset),h5type); if (ytype==-1) error,"Unknown Datatype"; data = array(ytype,_(rank,dims)); if (structof(ytype)==string) { nelem = numberof(data); data=_H5Dreads(dataset,data,nelem); } else { status=_H5Dread(dataset,h5type,0,0,0,&data); } status=_H5Dclose(dataset); if (has2bclose) {h5close,file;} return data; } //:::::::::::::::::::::::::::::::::::: func h5write(file,fullpath,data,zip=,mode=,noheader=) /* DOCUMENT h5write,file,fullpath,data,zip=,mode=,noheader= Write data in a HDF5 file. fname: file name or file handle (from h5open) fullpath: full path to the dataset (string), including hierarchy (ex: "/g1/data"). If parent group(s) do not exist, they will be created. This function will refuse to overwrite an existing dataset. To do so, use delete the dataset with h5delete prior to the h5write. data: data. Any yorick valid data. Scalar or arrays. Any yorick type is accepted (from char to double, and strings). Strings are stored in H5T_VARIABLE, which is a variable length type in the HDF5 specification. zip=N will use compression (N=0-9). Larger N will compress more (but take longer). mode= "w"=write-only (erase previous content, default) or "a"=append Warning: If file is a string, then mode="w" is used, which means an existing file will be overwritten. To update an existing file, use a file handle with the h5open/h5write/h5close combination. SEE ALSO: h5read, h5info, h5link, h5delete */ { if (!mode) mode="w"; tmp = strpart(fullpath,strword(fullpath,"/",20)); tmp = tmp(where(tmp)); dataname = tmp(0); if (numberof(tmp)>1) { group = ("/"+tmp(1:-1))(sum); } else group="/"; // Open/Create FILE if (structof(file)==string) { has2bclose=1; // default to overwrite when using the simple // filename option file = h5open(file,mode); } if ((group!="/")&&(strpart(group,0:0)=="/")) group=strpart(group,1:-1); // Open/Create GROUP gid = _H5Gopen(file,group); if (gid<0) { //group does not exist, create: g = strpart(group,strword(group,"/",20)); g = g(where(g)); g = "/"+g; for (i=1;i<=numberof(g);i++) { gid = _H5Gopen(file,g(1:i)(sum)); if (gid<0) { gid = _H5Gcreate(file,g(1:i)(sum),0); status=_H5Gclose(gid); } if (gid<0) { if (has2bclose) h5close,file; error,"Unable to create group"; } status=_H5Gclose(gid); } gid = _H5Gopen(file,group); } if (data==[]) { // just create a group gid = _H5Gopen(file,fullpath); if (gid<0) { gid = _H5Gcreate(file,fullpath,0); status=_H5Gclose(gid); } if (gid<0) { if (has2bclose) h5close,file; error,"Unable to create group"; } swrite=0; } else { // create dataset and write data // Determine dimsof() data rank = dimsof(data)(1); if (rank>0) { dims = dimsof(data)(2:); dims = dims(::-1); // dim bug fix } if (rank==0) { dataspace = _H5Screate(H5S_SCALAR); } else { dataspace=_H5Screate_simple(rank,dims); if (dataspace<0) { status=_H5Gclose(gid); if (has2bclose) h5close,file; error,"Unable to create dataspace"; } } // Handle compression if ((zip!=[])&&(rank)) { plist = _H5Pcreate(H5P_DATASET_CREATE); rank = dimsof(data)(1); cdims = min(dimsof(data)(2:),20); status = _H5Pset_chunk(plist, 2, cdims); status = _H5Pset_deflate( plist, zip); } else plist=H5P_DEFAULT; if (structof(data)==char) htype=H5T_NATIVE_CHAR; if (structof(data)==short) htype=H5T_NATIVE_SHORT; if (structof(data)==int) htype=H5T_NATIVE_INT; if (structof(data)==long) htype=H5T_NATIVE_LONG; if (structof(data)==float) htype=H5T_NATIVE_FLOAT; if (structof(data)==double) htype=H5T_NATIVE_DOUBLE; if (structof(data)==string) { /* create a datatype for the text */ htype = _H5Tcopy(H5T_C_S1); /* set the total size for the datatype */ status = _H5Tset_size (htype,H5T_VARIABLE); plist = _H5Pcreate(H5P_DATASET_CREATE); } // Create dataset dataset=_H5Dcreate(file,group+"/"+dataname,htype,dataspace,plist); if (dataset<0) { status=_H5Sclose(dataspace); status=_H5Gclose(gid); if (has2bclose) h5close,file; error,"Unable to create dataset (already exist?)"; } // and finally, write data... swrite=_H5Dwrite(dataset,htype,0,0,0,&data); status=_H5Sclose(dataspace); status=_H5Dclose(dataset); } status=_H5Gclose(gid); if (has2bclose) h5close,file; if (swrite<0) error,"Unable to write data"; // write the yorick plugin version as an attribute of "/" h5awrite,file,"/",plugin_version_string,HDF5_VERSION,try=1; } //:::::::::::::::::::::::::::::::::::: func h5adelete(file,object,aname) /* DOCUMENT h5adelete,file,object,aname Delete attribute "aname" in object "object" in HDF5 file "file" SEE ALSO: h5aread, h5awrite */ { // Open FILE if (structof(file)==string) { has2bclose=1; file = h5open(file,"a"); } oid=_H5Dopen(file,object); if (oid<0) { // may be a group? oid=_H5Gopen(file,object); isgroup=1; if (oid<0) { if (has2bclose) h5close,file; error,swrite(format="Unable to find object %s\n",object); } } if (structof(aname)==string) { // attribute name passed as a string // check attribute exist: attid = _H5Aopen_name(oid,aname); if (attid<0) { if (isgroup) status=_H5Gclose(oid); else status=_H5Dclose(oid); if (has2bclose) h5close,file; error,swrite(format="No such attribute \"%s\" in %s\n",aname,object); } // we can (have to) close it status = _H5Aclose(attid); } else if ((structof(aname)==int)||(structof(aname)==long)) { // attribute name passed as an index attid = _H5Aopen_idx(oid,aname); if (attid<0) { if (isgroup) status=_H5Gclose(oid); else status=_H5Dclose(oid); if (has2bclose) h5close,file; error,swrite(format="No such attribute #%d in %s\n",aname,object); } // if it exist, then get name: aname = array(" ",129)(sum); status= _H5Aget_name(attid,128,aname); } else { if (isgroup) status=_H5Gclose(oid); else status=_H5Dclose(oid); if (has2bclose) h5close,file; error,"Unknow attribute name type"; } status = _H5Adelete(oid,aname); if (isgroup) status=_H5Gclose(oid); else status=_H5Dclose(oid); if (has2bclose) h5close,file; } //:::::::::::::::::::::::::::::::::::: func h5aread(file,object,aname,pre062=) /* DOCUMENT attrib = h5aread(file,object,aname,pre062=) Read the value of an object attribute in a HDF5 file file: file name (string) or file handle (from h5open) object: object name (type GROUP or DATASET) (string) aname: attribute name (string) or id (int). pre062: set this keyword to read files generated by this plugin version pre-0.6.2 (hdf5 libraries array dimensions issue) If aname=[], then all attributes of the given object are read out and returned. SEE ALSO: h5awrite, h5adelete */ { // Open FILE if (structof(file)==string) { has2bclose=1; file = h5open(file,"r"); } // Open object oid=_H5Dopen(file,object); if (oid<0) { // may be a group? oid=_H5Gopen(file,object); isgroup=1; if (oid<0) { if (has2bclose) h5close,file; error,swrite(format="Unable to find object %s\n",object); } } if (structof(aname)==string) { attid = _H5Aopen_name(oid,aname); if (attid<0) { if (isgroup) status=_H5Gclose(oid); else status=_H5Dclose(oid); if (has2bclose) h5close,file; error,swrite(format="No such attribute \"%s\" in %s\n",aname,object); } } else if ((structof(aname)==int)||(structof(aname)==long)) { attid = _H5Aopen_idx(oid,aname); if (attid<0) { if (isgroup) status=_H5Gclose(oid); else status=_H5Dclose(oid); if (has2bclose) h5close,file; error,swrite(format="No such attribute #%d in %s\n",aname,object); } str = array(" ",128)(sum); status= _H5Aget_name(attid,128,str); //write,format="Retrieving Attribute %s\n",str; } else { if (isgroup) status=_H5Gclose(oid); else status=_H5Dclose(oid); if (has2bclose) h5close,file; error,"Unknow attribute name type"; } dspid = _H5Aget_space(attid); rank = _H5Sget_simple_extent_ndims(dspid); if (rank) { dims = maxdims = array(long,rank); status = _H5Sget_simple_extent_dims(dspid,dims,maxdims); if (!pre062) dims = dims(::-1); // dim bug fix } else {dims=0;} ytype=yotype(_H5Aget_type(attid),h5type); if (ytype==-1) error,"Unknow attribute data type"; data = array(ytype,_(rank,dims)); if (structof(ytype)==string) { nelem = numberof(data); data=_H5Areads(attid,data,nelem); } else { status=_H5Aread(attid,h5type,&data); } status=_H5Aclose(attid); if (isgroup) status=_H5Gclose(oid); else status=_H5Dclose(oid); if (has2bclose) h5close,file; if (numberof(data)==1) data=data(1); return data; } //:::::::::::::::::::::::::::::::::::: func h5awrite(file,object,aname,attdata,try=) /* DOCUMENT h5awrite,file,object,aname,attdata Write an object attribute in a HDF5 file, attached to a group or a dataset. file: file name (string) or file handle (from h5open) object: object name (type GROUP or DATASET) (string) aname: attribute name (string) attdata: attribute value (data). Any yorick type. Can be scalar or an array. HDF5 limits length to about 1000 elements. try: if set, no errors are triggered, but the return status is set with the error number: 0: no error 1: Unable to find object 2: Unable to create attribute dataspace 3: Unable to create attribute (already exist?) SEE ALSO: h5aread, h5adelete */ { // Open FILE if (structof(file)==string) { has2bclose=1; file = h5open(file,"a"); } // open object oid=_H5Dopen(file,object); if (oid<0) { // may be a group? oid=_H5Gopen(file,object); isgroup=1; if (oid<0) { if (has2bclose) h5close,file; if (try) return 1; else error,swrite(format="Unable to find object %s\n",object); } } props=H5P_DEFAULT; if (structof(attdata)==char) htype=H5T_NATIVE_CHAR; if (structof(attdata)==short) htype=H5T_NATIVE_SHORT; if (structof(attdata)==int) htype=H5T_NATIVE_INT; if (structof(attdata)==long) htype=H5T_NATIVE_LONG; if (structof(attdata)==float) htype=H5T_NATIVE_FLOAT; if (structof(attdata)==double) htype=H5T_NATIVE_DOUBLE; if (structof(attdata)==string) { /* create a datatype for the text */ htype = _H5Tcopy(H5T_C_S1); /* set the total size for the datatype */ status = _H5Tset_size (htype,H5T_VARIABLE); props = _H5Pcreate(H5P_DATASET_CREATE); } rank = dimsof(attdata)(1); if (rank>0) { dims = dimsof(attdata)(2:); dims = dims(::-1); // dim bug fix } if (rank==0) { dataspace = _H5Screate(H5S_SCALAR); } else { dataspace=_H5Screate_simple(rank,dims); if (dataspace<0) { if (isgroup) status=_H5Gclose(oid); else status=_H5Dclose(oid); if (has2bclose) h5close,file; if (try) return 2; else error,"Unable to create attribute dataspace"; } } // 20090916: from the new hdf5 doc (H5Acreate1): The attribute creation // property list, acpl_id (the last one), is currently unused; it may be // used in the future for optional attribute properties. At this time, // H5P_DEFAULT is the only accepted value. // hence, forcing props to zero (and that made it works, broken if props // != 0, i.e. for strings -see above-): props = 0; /* Create a dataset attribute. */ attid = _H5Acreate(oid, aname, htype, dataspace, props); if (attid<0) { if (has2bclose) h5close,file; if (try) return 3; else error,"Unable to create attribute (already exist?)"; } /* Write the attribute data. */ status = _H5Awrite(attid, htype, &attdata); /* Close the attribute. */ status = _H5Aclose(attid); status=_H5Sclose(dataspace); if (isgroup) status=_H5Gclose(oid); else status=_H5Dclose(oid); if (has2bclose) h5close,file; } //:::::::::::::::::::::::::::::::::::: func h5link(file,group,group2link2,linktype) /* DOCUMENT h5link,file,group,group2link2,linktype Create a SOFT or HARD link to a group in a HDF5 file file: file name (string) or file handle (from h5open) group: group to create that link to group2link2 Must not already exist. group2link2: group to link to. Must exist. Has to be a group, Can NOT be a dataset (error). linktype: link type. valid are: H5G_LINK_HARD or H5G_LINK_SOFT SEE ALSO: h5read, h5write, h5info, h5delete */ { if (linktype==[]) linktype=H5G_LINK_SOFT; if (noneof(linktype==[H5G_LINK_HARD,H5G_LINK_SOFT])) { error,"Unknow link type"; } // Open FILE if (structof(file)==string) { has2bclose=1; file = h5open(file,"a"); } if ((group!="/")&&(strpart(group,0:0)=="/")) group=strpart(group,1:-1); // Open/Create GROUP gid = _H5Gopen(file,group); if (gid>0) { // group already exist. it should not status=_H5Gclose(gid); if (has2bclose) h5close,file; error,"Group already exist"; } // we create up to group parent level, as final link group will be // created by the H5Glink2 call g = strpart(group,strword(group,"/",20)); g = g(where(g)); if (numberof(g)==1) { groupParent="/"; } else { groupParent = ("/"+g(1:-1))(sum); } gid = _H5Gopen(file,groupParent); if (gid<0) { //groupParent does not exist, create: g = strpart(groupParent,strword(groupParent,"/",20)); g = g(where(g)); g = "/"+g; gid2 = array(long,numberof(g)); for (i=1;i<=numberof(g);i++) { gid2(i) = _H5Gopen(file,g(1:i)(sum)); if (gid2(i)<0) gid2(i) = _H5Gcreate(file,g(1:i)(sum),0); if (gid2(i)<0) { for (j=1;j<=(i-1);j++) status = _H5Gclose(gid2(j)); if (has2bclose) h5close,file; error,"Unable to create groupParent (already exist?)"; } } gid = gid2(0); } // check that group2link2 exists gid2link2 = _H5Gopen(file,group2link2); if (gid2link2<0) { if (anyof(gid2)) { for (j=1;j<=numberof(gid2);j++) status = _H5Gclose(gid2(j)); } else status=_H5Gclose(gid); if (has2bclose) h5close,file; error,"Unable to open destination group (has to be a group, not dataset)"; } // now we can link slink = _H5Glink2(gid2link2, group2link2, linktype, gid, group); if (anyof(gid2)) { for (j=1;j<=numberof(gid2);j++) status = _H5Gclose(gid2(j)); } else status=_H5Gclose(gid); status=_H5Gclose(gid2link2); if (has2bclose) h5close,file; if (slink<0) error,"Unable to link groups"; } //:::::::::::::::::::::::::::::::::::: func h5delete(file,object) /* DOCUMENT h5delete,file,object Unlink or delete an object in a HDF5 file. file: file name (string) or file handle (from h5open) object: object name (string) SEE ALSO: h5write, h5read, h5link, h5info */ { // Open FILE if (structof(file)==string) { has2bclose=1; file = h5open(file,"a"); } status=_H5Gunlink(file, object); if (has2bclose) h5close,file; if (status<0) error,"Unable to link groups"; } h5unlink=h5delete; //:::::::::::::::::::::::::::::::::::: func h5version(void) { res = _H5version(); if (am_subroutine()) { write,format="libhdf5 version %d.%d.%d\n",res(1),res(2),res(3); } else return res; } //:::::::::::::::::::::::::::::::::::: func h5old2new(files,inplace=,verbose=,force=) /* DOCUMENT h5old2new(file,inplace=,verbose=,force=) Convert hdf5 files produced with hdf5 version <= 0.6.1 to hdf5 file compatible with version 0.6.2 and later. Files produced with yorick-hdf5 v<=0.6.1 suffer from an issue in the dimension that are reported to external HDF5 utilities. The dimension vector is swapped, i.e. an array [10,20] will appear as an array [20,10] to an external HDF5 reader (e.g. h5dump), but the elements themselves are not swapped, resulting in a completely different looking array. The "old" format files were perfectly fine within yorick as the same bug was present in the input and output. Only compatibility with external hdf5 tools was affected. As there is no way to know with which version of the plugin a file has been written, I can not correct this automatically when reading old files with the new plugin version. However, I provide a utility function (h5old2new()) that will convert "old file" (written with v<=0.5.1) to the new, more correct format. files: scalar or vector string inplace: will rename the converted file to the original name when done. Beware! This should be safe, but I haven't tested all possible scenario. This will erase your original data. I encourage you to check this conversion function on a few files before using this option verbose: 0: no output 1: only prints out filename 2: print out file and object names while processing force: Normally, if this function encounters the attribute "Generated by yorick hdf5 plugin version" (plugin_version_string) and this is set to something higher than "0.6.2", this function will refuse to process the file (because it is not supposed to suffer from the swapped dims issue). Use force=1 to force processing. SEE ALSO: */ { if (verbose==[]) verbose=1; // default at least print out filename for (n=1;n<=numberof(files);n++) { file = files(n); fin = h5open(file); all = h5info(fin,silent=1,att=1); // check if this file has been written with a version // of the plugin for which this problem was fixed (>=0.6.2) // if so, skip the file (unless force is set) if ((*all(1).aname!=[]) && // all(1) is always root ("/") (anyof(*all(1).aname==plugin_version_string))) { file_plugin_version = h5aread(fin,"/",plugin_version_string); ymajor = yminor = ymicro = 0; sread,file_plugin_version,format="%d.%d.%d",ymajor, yminor, ymicro; version = ymajor*10000+yminor*100+ymicro; if (version>=602) { write,format="WARNING: %s should not need conversion,",file; if (force) { write,format="%s\n","but you have set force=1"; // write,format="%s\n"," I assume you know what you're doing"; } else { write,format="%s\n"," skipping."; continue; } } } fileout = streplace(file,strfind(".h5",file), ".new.h5"); remove,fileout; // shamelessly fout = h5open(fileout,"w"); natt=0; for (i=1;i<=numberof(all);i++) { if ((*all(i).aname)!=[]) natt+=numberof(*(all(i).aname)); } inplacestr = (inplace?" inplace ":""); if (verbose==1) write,format="---> Converting %s%s...",file,inplacestr; else if (verbose>1) { write,format="---> Converting %s%s(found %d elements + %d attr.):\n", \ file,inplacestr,numberof(all),natt; } // convert: loop on all items: for (i=1;i<=numberof(all);i++) { if (verbose>1) write,format="%-9s: %s\n",all(i).objectType,all(i).name; if (all(i).objectType=="GROUP") { // let's go ahead and create the group, so that // we can write possible attributes h5write,fout,all(i).name+"/",[]; // create group } else if (all(i).objectType=="DATASET") { data = h5read(fin,all(i).name,pre062=1); h5write,fout,all(i).name,data; } else if (all(i).objectType=="SOFTLINK") { // it should be safe to write it now, as // in the initial file creation, the link // can only have been created after the target // object (is that sure?) h5link,fout,all(i).name,all(i).target,H5G_LINK_SOFT; } else if (all(i).objectType=="HARDLINK") { // same h5link,fout,all(i).name,all(i).target,H5G_LINK_HARD; } // process possible attributes if ((*all(i).aname)!=[]) { // there are some for (j=1;j<=numberof(*all(i).aname);j++) { if (verbose>1) \ write,format=" ATTRIBUTE: %s\n",(*all(i).aname)(j); // read the attribute adata = h5aread(fin,all(i).name,(*all(i).aname)(j),pre062=1); // write h5awrite,fout,all(i).name,(*all(i).aname)(j),adata; } } } if (verbose>0) { if (inplace) write,""; else write,format="done, results in %s\n",fileout; } h5close,fin; h5close,fout; if (inplace) rename,fileout,file; } // end loop on files } //:::::::::::::::::::::::::::::::::::: //::::::UTILITY FUNCTIONS::::::::::::: //:::::::::::::::::::::::::::::::::::: func yotype(tid,&h5type) { tclass = _H5Tget_class(tid); if (tclass < 0) { error,"Invalid datatype"; } else { tsize=_H5Tget_size(tid); if (hdf5_debug) write,format="tclass=%d tsize=%d\n",tclass,tsize; if (tclass == H5T_INTEGER) { if (tsize==sizeof(long) ) {h5type=H5T_NATIVE_LONG; return 0l;} if (tsize==sizeof(int) ) {h5type=H5T_NATIVE_INT; return 0n;} if (tsize==sizeof(short)) {h5type=H5T_NATIVE_SHORT; return 0s;} if (tsize==sizeof(char) ) {h5type=H5T_NATIVE_CHAR; return '0';} // read 64bits long on a 32 bits OS: if (tsize==8 ) {h5type=H5T_NATIVE_LONG; return 0l;} } else if (tclass == H5T_FLOAT) { if (tsize==sizeof(float)) {h5type=H5T_NATIVE_FLOAT; return 0.0f;} if (tsize==sizeof(double)) {h5type=H5T_NATIVE_DOUBLE; return 0.0;} } else if (tclass == H5T_STRING) { h5type=H5T_C_S1; return ""; } else { // print,"Unknown Datatype"; return -1; } error,"Unknown or invalid datatype"; } } func h5off(void) /* DOCUMENT h5ooff Flushes all data to disk, closes file identifiers, and cleans up memory. SEE ALSO: */ { extern _hdf5file; _H5close; _hdf5file=[]; } /**********************************************/ /* DEFINE BUILTINS AND EXTERNS H5 DEFINITIONS */ /**********************************************/ extern _H5Eon; extern _H5Eoff; extern _H5open; extern _H5close; extern _H5version; // FILE APIs: extern _H5Fcreate; extern _H5Fopen; extern _H5Fclose; // GROUP APIs: extern _H5Gget_linkval; extern _H5Gopen; extern _H5Gclose; extern _H5Gcreate; extern _H5Gget_num_objs; // numobj = _H5Gget_num_objs(gid) extern _H5Gget_objname_by_idx; extern _H5Gget_objtype_by_idx; extern _H5Gget_objtype_by_name; extern _H5Glink2; extern _H5Gunlink; // PROPERTY LIST APIs: extern _H5Pcreate; //int H5Pcreate(int cls_id ) extern _H5Pset_deflate; //int H5Pset_deflate(int plist, int level) extern _H5Pset_chunk; // ATTRIBUTE APIs: extern _H5Acreate; extern _H5Adelete; extern _H5Aget_num_attrs; extern _H5Aget_type; extern _H5Aget_space; extern _H5Aget_name; extern _H5Aopen_idx; extern _H5Aopen_name; extern _H5Aread; extern _H5Awrite; extern _H5Aclose; // DATASET API: extern _H5Dclose; extern _H5Dcreate; extern _H5Dopen; extern _H5Dget_space; extern _H5Dget_type; extern _H5Dwrite; extern _H5Dread; // DATASPACE APIs: extern _H5Sclose extern _H5Screate extern _H5Sget_simple_extent_ndims; extern _H5Sget_simple_extent_type; extern _H5Screate_simple; extern _H5Sget_simple_extent_dims; // Datatype APIs //************** extern _H5Tcopy; extern _H5Tget_class; extern _H5Tget_size; extern _H5Tset_cset; extern _H5Tset_size; extern _H5Tset_strpad; // special read function for Attributes and Dataset string reads: extern _H5Areads; extern _H5Dreads; // The following types are dynamically assigned at the hdf5 library // compile time, so I need to resort to function calls here. extern _H5T_C_S1; H5T_C_S1=_H5T_C_S1(); extern _H5T_NATIVE_CHAR; H5T_NATIVE_CHAR=_H5T_NATIVE_CHAR(); extern _H5T_NATIVE_SHORT; H5T_NATIVE_SHORT=_H5T_NATIVE_SHORT(); extern _H5T_NATIVE_INT; H5T_NATIVE_INT=_H5T_NATIVE_INT(); extern _H5T_NATIVE_LONG; H5T_NATIVE_LONG=_H5T_NATIVE_LONG(); extern _H5T_NATIVE_FLOAT; H5T_NATIVE_FLOAT=_H5T_NATIVE_FLOAT(); extern _H5T_NATIVE_DOUBLE; H5T_NATIVE_DOUBLE=_H5T_NATIVE_DOUBLE(); extern _H5T_IEEE_F32BE; H5T_IEEE_F32BE=_H5T_IEEE_F32BE(); extern _H5T_IEEE_F32LE; H5T_IEEE_F32LE=_H5T_IEEE_F32LE(); extern _H5T_IEEE_F64BE; H5T_IEEE_F64BE=_H5T_IEEE_F64BE(); extern _H5T_IEEE_F64LE; H5T_IEEE_F64LE=_H5T_IEEE_F64LE(); extern _H5T_STD_I8BE; H5T_STD_I8BE=_H5T_STD_I8BE(); extern _H5T_STD_I8LE; H5T_STD_I8LE=_H5T_STD_I8LE(); extern _H5T_STD_I16BE; H5T_STD_I16BE=_H5T_STD_I16BE(); extern _H5T_STD_I16LE; H5T_STD_I16LE=_H5T_STD_I16LE(); extern _H5T_STD_I32BE; H5T_STD_I32BE=_H5T_STD_I32BE(); extern _H5T_STD_I32LE; H5T_STD_I32LE=_H5T_STD_I32LE(); extern _H5T_STD_I64BE; H5T_STD_I64BE=_H5T_STD_I64BE(); extern _H5T_STD_I64LE; H5T_STD_I64LE=_H5T_STD_I64LE(); extern _H5T_STD_U8BE; H5T_STD_U8BE=_H5T_STD_U8BE(); extern _H5T_STD_U8LE; H5T_STD_U8LE=_H5T_STD_U8LE(); extern _H5T_STD_U16BE; H5T_STD_U16BE=_H5T_STD_U16BE(); extern _H5T_STD_U16LE; H5T_STD_U16LE=_H5T_STD_U16LE(); extern _H5T_STD_U32BE; H5T_STD_U32BE=_H5T_STD_U32BE(); extern _H5T_STD_U32LE; H5T_STD_U32LE=_H5T_STD_U32LE(); extern _H5T_STD_U64BE; H5T_STD_U64BE=_H5T_STD_U64BE(); extern _H5T_STD_U64LE; H5T_STD_U64LE=_H5T_STD_U64LE(); extern _H5T_STD_B8BE; H5T_STD_B8BE=_H5T_STD_B8BE(); extern _H5T_STD_B8LE; H5T_STD_B8LE=_H5T_STD_B8LE(); extern _H5T_STD_B16BE; H5T_STD_B16BE=_H5T_STD_B16BE(); extern _H5T_STD_B16LE; H5T_STD_B16LE=_H5T_STD_B16LE(); extern _H5T_STD_B32BE; H5T_STD_B32BE=_H5T_STD_B32BE(); extern _H5T_STD_B32LE; H5T_STD_B32LE=_H5T_STD_B32LE(); extern _H5T_STD_B64BE; H5T_STD_B64BE=_H5T_STD_B64BE(); extern _H5T_STD_B64LE; H5T_STD_B64LE=_H5T_STD_B64LE(); extern _H5T_STD_REF_OBJ; H5T_STD_REF_OBJ=_H5T_STD_REF_OBJ(); extern _H5T_UNIX_D32BE; H5T_UNIX_D32BE=_H5T_UNIX_D32BE(); extern _H5T_UNIX_D32LE; H5T_UNIX_D32LE=_H5T_UNIX_D32LE(); extern _H5T_UNIX_D64BE; H5T_UNIX_D64BE=_H5T_UNIX_D64BE(); extern _H5T_UNIX_D64LE; H5T_UNIX_D64LE=_H5T_UNIX_D64LE(); extern _H5P_DATASET_CREATE; H5P_DATASET_CREATE=_H5P_DATASET_CREATE(); H5F_ACC_RDONLY = 0x0000; /* absence of rdwr => rd-only */ H5F_ACC_RDWR = 0x0001; /* open for read and write */ H5F_ACC_TRUNC = 0x0002; /* overwrite existing files */ H5F_ACC_EXCL = 0x0004; /* fail if file already exists*/ H5F_ACC_DEBUG = 0x0008; /* print debug info */ H5F_ACC_CREAT = 0x0010; /* create non-existing files */ extern _H5P_DEFAULT; H5P_DEFAULT=_H5P_DEFAULT(); //extern _H5T_DEFAULT; H5T_DEFAULT=_H5T_DEFAULT(); //H5P_DEFAULT = 0; H5T_DEFAULT = 0; extern _H5S_NO_CLASS; H5S_NO_CLASS=_H5S_NO_CLASS(); /* error */ extern _H5S_SCALAR; H5S_SCALAR=_H5S_SCALAR(); /* scalar variable */ extern _H5S_SIMPLE; H5S_SIMPLE=_H5S_SIMPLE(); /* simple data space */ //H5S_NO_CLASS = -1; /* error */ //H5S_SCALAR = 0; /* scalar variable */ //H5S_SIMPLE = 1; /* simple data space */ extern _H5T_NO_CLASS; H5T_NO_CLASS=_H5T_NO_CLASS(); /* error */ extern _H5T_INTEGER; H5T_INTEGER=_H5T_INTEGER(); /* integer types */ extern _H5T_FLOAT; H5T_FLOAT=_H5T_FLOAT(); /* floating-point types */ extern _H5T_TIME; H5T_TIME=_H5T_TIME(); /* date and time types */ extern _H5T_STRING; H5T_STRING=_H5T_STRING(); /* character string types */ extern _H5T_BITFIELD; H5T_BITFIELD=_H5T_BITFIELD(); /* bit field types */ extern _H5T_OPAQUE; H5T_OPAQUE=_H5T_OPAQUE(); /* opaque types */ extern _H5T_COMPOUND; H5T_COMPOUND=_H5T_COMPOUND(); /* compound types */ extern _H5T_REFERENCE; H5T_REFERENCE=_H5T_REFERENCE();/* reference types */ extern _H5T_ENUM; H5T_ENUM=_H5T_ENUM(); /* enumeration types */ extern _H5T_VLEN; H5T_VLEN=_H5T_VLEN(); /* Variable-Length types */ extern _H5T_ARRAY; H5T_ARRAY=_H5T_ARRAY(); /* Array types */ H5T_VARIABLE = -1; /* Indicate that a string is variable length (null- */ /* terminated in C, instead of fixed length) */ /* 2009sep16: HDF5 was broken. This is because the typedef for the following const was changed in the last hdf5 APIs, and I stupidly declared it in here, instead of getting them from hdf5 at build time. Thus I have rewritten this (below). Here is how it appeared in previous version (<=0.7.1) of this plugin: H5G_UNKNOWN = -1; H5G_LINK = 0; // now 3 H5G_GROUP = 1; // now 0 H5G_DATASET = 2; // now 1 H5G_TYPE = 3; // now 2 and here is how I coded it now: */ extern _H5G_UNKNOWN; H5G_UNKNOWN=_H5G_UNKNOWN(); extern _H5G_GROUP; H5G_GROUP=_H5G_GROUP(); extern _H5G_DATASET; H5G_DATASET=_H5G_DATASET(); extern _H5G_TYPE; H5G_TYPE=_H5G_TYPE(); extern _H5G_LINK; H5G_LINK=_H5G_LINK(); H5G_LINK_ERROR = -1; /* link types used by H5Glink2 */ H5G_LINK_HARD = 0; H5G_LINK_SOFT = 1; H5T_STR_NULLTERM = 0; H5F_SCOPE_LOCAL = 0; /* specified file handle only */ H5F_SCOPE_GLOBAL = 1; /* entire virtual file */ _H5Eoff; /* Error reporting OFF by default */ yorick-hdf5-0.8.0/h5convert0000755000076500001440000000006011111552351015062 0ustar frigautusers#!/bin/sh yorick -q -i h5convert_fromshell.i $* yorick-hdf5-0.8.0/hdf5_start.i0000644000076500001440000000020210734740141015440 0ustar frigautusersautoload,"hdf5.i", h5info, h5read, h5write; autoload,"hdf5.i", h5link, h5delete; autoload,"hdf5.i", h5adelete, h5aread, h5awrite; yorick-hdf5-0.8.0/LICENSE0000644000076500001440000004313110734740140014237 0ustar frigautusers GNU GENERAL PUBLIC LICENSE Version 2, June 1991 Copyright (C) 1989, 1991 Free Software Foundation, Inc. 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Preamble The licenses for most software are designed to take away your freedom to share and change it. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change free software--to make sure the software is free for all its users. This General Public License applies to most of the Free Software Foundation's software and to any other program whose authors commit to using it. (Some other Free Software Foundation software is covered by the GNU Library General Public License instead.) You can apply it to your programs, too. When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for this service if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs; and that you know you can do these things. To protect your rights, we need to make restrictions that forbid anyone to deny you these rights or to ask you to surrender the rights. These restrictions translate to certain responsibilities for you if you distribute copies of the software, or if you modify it. For example, if you distribute copies of such a program, whether gratis or for a fee, you must give the recipients all the rights that you have. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights. We protect your rights with two steps: (1) copyright the software, and (2) offer you this license which gives you legal permission to copy, distribute and/or modify the software. Also, for each author's protection and ours, we want to make certain that everyone understands that there is no warranty for this free software. If the software is modified by someone else and passed on, we want its recipients to know that what they have is not the original, so that any problems introduced by others will not reflect on the original authors' reputations. Finally, any free program is threatened constantly by software patents. We wish to avoid the danger that redistributors of a free program will individually obtain patent licenses, in effect making the program proprietary. To prevent this, we have made it clear that any patent must be licensed for everyone's free use or not licensed at all. The precise terms and conditions for copying, distribution and modification follow. GNU GENERAL PUBLIC LICENSE TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 0. This License applies to any program or other work which contains a notice placed by the copyright holder saying it may be distributed under the terms of this General Public License. The "Program", below, refers to any such program or work, and a "work based on the Program" means either the Program or any derivative work under copyright law: that is to say, a work containing the Program or a portion of it, either verbatim or with modifications and/or translated into another language. (Hereinafter, translation is included without limitation in the term "modification".) Each licensee is addressed as "you". Activities other than copying, distribution and modification are not covered by this License; they are outside its scope. The act of running the Program is not restricted, and the output from the Program is covered only if its contents constitute a work based on the Program (independent of having been made by running the Program). Whether that is true depends on what the Program does. 1. You may copy and distribute verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice and disclaimer of warranty; keep intact all the notices that refer to this License and to the absence of any warranty; and give any other recipients of the Program a copy of this License along with the Program. You may charge a fee for the physical act of transferring a copy, and you may at your option offer warranty protection in exchange for a fee. 2. You may modify your copy or copies of the Program or any portion of it, thus forming a work based on the Program, and copy and distribute such modifications or work under the terms of Section 1 above, provided that you also meet all of these conditions: a) You must cause the modified files to carry prominent notices stating that you changed the files and the date of any change. b) You must cause any work that you distribute or publish, that in whole or in part contains or is derived from the Program or any part thereof, to be licensed as a whole at no charge to all third parties under the terms of this License. c) If the modified program normally reads commands interactively when run, you must cause it, when started running for such interactive use in the most ordinary way, to print or display an announcement including an appropriate copyright notice and a notice that there is no warranty (or else, saying that you provide a warranty) and that users may redistribute the program under these conditions, and telling the user how to view a copy of this License. (Exception: if the Program itself is interactive but does not normally print such an announcement, your work based on the Program is not required to print an announcement.) These requirements apply to the modified work as a whole. If identifiable sections of that work are not derived from the Program, and can be reasonably considered independent and separate works in themselves, then this License, and its terms, do not apply to those sections when you distribute them as separate works. But when you distribute the same sections as part of a whole which is a work based on the Program, the distribution of the whole must be on the terms of this License, whose permissions for other licensees extend to the entire whole, and thus to each and every part regardless of who wrote it. Thus, it is not the intent of this section to claim rights or contest your rights to work written entirely by you; rather, the intent is to exercise the right to control the distribution of derivative or collective works based on the Program. In addition, mere aggregation of another work not based on the Program with the Program (or with a work based on the Program) on a volume of a storage or distribution medium does not bring the other work under the scope of this License. 3. You may copy and distribute the Program (or a work based on it, under Section 2) in object code or executable form under the terms of Sections 1 and 2 above provided that you also do one of the following: a) Accompany it with the complete corresponding machine-readable source code, which must be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, b) Accompany it with a written offer, valid for at least three years, to give any third party, for a charge no more than your cost of physically performing source distribution, a complete machine-readable copy of the corresponding source code, to be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, c) Accompany it with the information you received as to the offer to distribute corresponding source code. (This alternative is allowed only for noncommercial distribution and only if you received the program in object code or executable form with such an offer, in accord with Subsection b above.) The source code for a work means the preferred form of the work for making modifications to it. For an executable work, complete source code means all the source code for all modules it contains, plus any associated interface definition files, plus the scripts used to control compilation and installation of the executable. However, as a special exception, the source code distributed need not include anything that is normally distributed (in either source or binary form) with the major components (compiler, kernel, and so on) of the operating system on which the executable runs, unless that component itself accompanies the executable. If distribution of executable or object code is made by offering access to copy from a designated place, then offering equivalent access to copy the source code from the same place counts as distribution of the source code, even though third parties are not compelled to copy the source along with the object code. 4. You may not copy, modify, sublicense, or distribute the Program except as expressly provided under this License. Any attempt otherwise to copy, modify, sublicense or distribute the Program is void, and will automatically terminate your rights under this License. However, parties who have received copies, or rights, from you under this License will not have their licenses terminated so long as such parties remain in full compliance. 5. You are not required to accept this License, since you have not signed it. However, nothing else grants you permission to modify or distribute the Program or its derivative works. These actions are prohibited by law if you do not accept this License. Therefore, by modifying or distributing the Program (or any work based on the Program), you indicate your acceptance of this License to do so, and all its terms and conditions for copying, distributing or modifying the Program or works based on it. 6. Each time you redistribute the Program (or any work based on the Program), the recipient automatically receives a license from the original licensor to copy, distribute or modify the Program subject to these terms and conditions. You may not impose any further restrictions on the recipients' exercise of the rights granted herein. You are not responsible for enforcing compliance by third parties to this License. 7. If, as a consequence of a court judgment or allegation of patent infringement or for any other reason (not limited to patent issues), conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot distribute so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not distribute the Program at all. For example, if a patent license would not permit royalty-free redistribution of the Program by all those who receive copies directly or indirectly through you, then the only way you could satisfy both it and this License would be to refrain entirely from distribution of the Program. If any portion of this section is held invalid or unenforceable under any particular circumstance, the balance of the section is intended to apply and the section as a whole is intended to apply in other circumstances. It is not the purpose of this section to induce you to infringe any patents or other property right claims or to contest validity of any such claims; this section has the sole purpose of protecting the integrity of the free software distribution system, which is implemented by public license practices. Many people have made generous contributions to the wide range of software distributed through that system in reliance on consistent application of that system; it is up to the author/donor to decide if he or she is willing to distribute software through any other system and a licensee cannot impose that choice. This section is intended to make thoroughly clear what is believed to be a consequence of the rest of this License. 8. If the distribution and/or use of the Program is restricted in certain countries either by patents or by copyrighted interfaces, the original copyright holder who places the Program under this License may add an explicit geographical distribution limitation excluding those countries, so that distribution is permitted only in or among countries not thus excluded. In such case, this License incorporates the limitation as if written in the body of this License. 9. The Free Software Foundation may publish revised and/or new versions of the General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Program specifies a version number of this License which applies to it and "any later version", you have the option of following the terms and conditions either of that version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of this License, you may choose any version ever published by the Free Software Foundation. 10. If you wish to incorporate parts of the Program into other free programs whose distribution conditions are different, write to the author to ask for permission. For software which is copyrighted by the Free Software Foundation, write to the Free Software Foundation; we sometimes make exceptions for this. Our decision will be guided by the two goals of preserving the free status of all derivatives of our free software and of promoting the sharing and reuse of software generally. NO WARRANTY 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. END OF TERMS AND CONDITIONS How to Apply These Terms to Your New Programs If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms. To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively convey the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. Copyright (C) This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA Also add information on how to contact you by electronic and paper mail. If the program is interactive, make it output a short notice like this when it starts in an interactive mode: Gnomovision version 69, Copyright (C) year name of author Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details. The hypothetical commands `show w' and `show c' should show the appropriate parts of the General Public License. Of course, the commands you use may be called something other than `show w' and `show c'; they could even be mouse-clicks or menu items--whatever suits your program. You should also get your employer (if you work as a programmer) or your school, if any, to sign a "copyright disclaimer" for the program, if necessary. Here is a sample; alter the names: Yoyodyne, Inc., hereby disclaims all copyright interest in the program `Gnomovision' (which makes passes at compilers) written by James Hacker. , 1 April 1989 Ty Coon, President of Vice This General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Library General Public License instead of this License. yorick-hdf5-0.8.0/hdf5.info0000644000076500001440000000477511254101701014740 0ustar frigautusersPackage: hdf5 Kind: plugin Version: 0.8.0 Revision: 1 Description: Hierarchical Data Format 5 interface License: GPL Maintainer: Francois Rigaut OS: Depends: yorick(>=1.6.02) Source: http://www.maumae.net/yorick/packages/%o/tarballs/hdf5-%v-%o.tgz Source-MD5: Source-Directory: contrib/hdf5 DocFiles: README Homepage: http://www.maumae.net/yorick/doc/plugins.php DescDetail: << HDF5 is the yorick interface plugin to the NCSA Hierarchical Data Format version 5. It includes function for reading, writing, updating, getting information on HDF5 files. man page: DATA I/O: h5read(fname,target) return data h5write,fname,fullpath,data,zip= write data h5open(filename,mode) return file handle h5close(filename) close file INFO, GROUP LINKING: h5info,fname,target,att= print out file info/structure h5link(fname,group,group2link2,linktype) link datasets h5delete,fname,object delete data ATTRIBUTE I/O: h5awrite(fname,object,aname,attdata) write an object attribute h5aread(fname,object,aname) read an object attribute h5adelete,fname,object,aname delete an object attribute MISC: h5list_open list open files h5version return linhdf5 version ERROR RECOVERY h5off close all reference to the h5 library. + Many of the atomic HDF5 functions are available (e.g. H5Fopen, H5Dopen) with mostly the same APIs. Many simple tasks can be done with the provided high level wrappers. Beware that programming an HDF5 custom wrapper is not trivial. Make sure you close every objects opened! This implementation supports reading of most of the HDF5 supported datatype. Only yorick datatype can be used for writes. Compression, soft and hard links, as well as support for attribute read/writes is provided. There is no support for hyperslabs, compound, enum or opaque datatype. Generally, there is very little support for datatype related functionalities. << DescUsage: << See packages/tarballs/hdf5/check.i for a test suite. Type "yorick -batch check.i" in a terminal to run it. << DescPort: << This package will compile Yorick only on MacOSX 10.3.4 or later, because of a bug in the system math library libm (part of /usr/lib/LibSystem.dylib) in earlier versions of MacOSX 10.3. << yorick-hdf5-0.8.0/hdf5doc.txt0000644000076500001440000001366010734740141015314 0ustar frigautusers/* * hdf5doc.txt * documentation for the yorick hdf5 plugin * * $Id: hdf5doc.txt,v 1.1.1.1 2007/12/27 15:10:25 frigaut Exp $ * * Author: Francois Rigaut. * Written 2004 * last revision/addition: 2007dec26 * * Copyright (c) 2003, Francois RIGAUT (frigaut@gemini.edu, Gemini * Observatory, 670 N A'Ohoku Place, HILO HI-96720). * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the * Free Software Foundation; either version 2 of the License, or (at your * option) any later version. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details (to receive a copy of the GNU * General Public License, write to the Free Software Foundation, Inc., 675 * Mass Ave, Cambridge, MA 02139, USA). * * $Log: hdf5doc.txt,v $ * Revision 1.1.1.1 2007/12/27 15:10:25 frigaut * Initial Import - yorick-hdf5 * */ HDF5 yorick plugin documentation ================================ ******************************** ******* DRAFT 2005dec11 ******** ******************************** 1. Overview This plugin uses libhdf5, version 1.6.4 or greater. It will probably work with slightly lesser version, but I haven't tested it. The current version avilable from the NSCA site is (dec2005) is 1.6.5. This plugin provide a minimal interface to HDF5 (Hierarchical Data Format 5). I am not going to expand on HDF5 here. Suffice to say that this format provides a way to store heterogeneous data in a single file entity. The data are stored under a hierarchical directory structure (hence the name), like: /header /header/date /header/time /header/author /2005mar25/data/raw/time_vector /2005mar25/data/raw/entropy etc... A directory (e.g. /header) is called a "Group". A scalar/vector/array of numbers/strings is called "Dataset". One can also set and store attribute to groups or datasets. Here is an example of a file as dumped by the h5info function: > h5info,"data.h5",att=1 /2005mar04 GROUP ATTRIB(0): Temp on this date ATTRIB(1): cups of coffee /2005mar04/comments DATASET STRING DIMSOF()=[1,2] /2005mar04/data DATASET DOUBLE DIMSOF()=[1,100] /2005mar04/dewpoint DATASET DOUBLE SCALAR /2005mar04/time DATASET DOUBLE DIMSOF()=[1,100] ATTRIB(0): Units ATTRIB(1): Units2 /2005mar05 GROUP /2005mar05/name with blanks DATASET DOUBLE SCALAR /HL HARDLINK -> /2005mar04 /bestdata SOFTLINK -> /2005mar04 > which is, I believe, pretty much self-explanatory. Note the hardlink and softlink references at the end (which are what they say they are: links to other groups in this file). 2. FUNCTION API As of v0.6, there are only a handfull of functions, but they should provide most of the functionality to save regular datas (with the exception of structure and pointers). These functions are: DATA I/O: h5read(fname,target) return data h5write,fname,fullpath,data,zip= write data h5open(filename,mode) return file handle h5close(filename) close file INFO, GROUP LINKING: h5info,fname,target,att= print out file info/structure h5link(fname,group,group2link2,linktype) link datasets h5delete,fname,object delete data ATTRIBUTE I/O: h5awrite(fname,object,aname,attdata) write an object attribute h5aread(fname,object,aname) read an object attribute h5adelete,fname,object,aname delete an object attribute MISC: h5list_open list open files h5version return linhdf5 version ERROR RECOVERY h5off close all reference to the h5 library (clean up). Each function is detailled below: 2.1 Data I/O: 2.1.1 h5open fd = h5open(filename,mode); Opens file "filename" in mode "mode". mode can be "r" (read-only), "w" (write-only) or "a" (append). This function returns a file descriptor fd that can be used until released by a call to h5close. fd can be used with h5read and h5open (see below). Warning: when a file is opened with h5open, it will have to be explicitely closed with h5close. If a file is open in read-only and one try to reopen it in write-only, it will cause a error (and vice-versa) 2.1.2 h5write h5write,filename,fullpath_to_dataset,data,zip=,mode= write "data" as object "fullpath" in file "fname". fname can be either a string (filename) or a file descriptor returned by h5open. The scalar string fullpath contains the full path to the dataset (/full/path/dataname). "data" are the data. Any yorick type is valid and will be saved as such. zip is a long 0-9. if set, data will be compressed (the larger the number, the more compression, and the larger the compression time). mode can be set here (when fname is a string). Use "a" it to append data to an existing file. Default is to overwrite ("w"). 2.1.3 h5read result = h5read(filename,dataset) return the data pointed to by the scalar string "dataset" in file "filename". 2.1.4 h5close h5close,file Close all references to the file pointed at by "file". File can either be a scalar string or a file descriptor (as the one returned by h5open()). to be continued... yorick-hdf5-0.8.0/h5convert.10000644000076500001440000000536311111600264015226 0ustar frigautusers.TH H5CONVERT 1 "2008 November 21" .UC 4 .SH NAME h5convert \- Conversion of files written with hdf5 yorick plugin pre v0.6.2 to "fixed" format of v0.6.2 and higher .SH SYNOPSIS .TP 3 h5convert [-p -v# -f -h] files .SH DESCRIPTION .I h5convert is a shell utility to convert files that were generated with the HDF5 yorick plugin v<0.6.2 to the new, fixed format used in v>=0.6.2. -p perform in-place conversion (output file name = input file name) -f force conversion of files generated with version>0.6.2 -v# verbose: -v0:silent -v1:files -v2:files and contents -h Print out short help Files generated with v<0.6.2, if read with the hdf5 plugin v>=0.6.2, will lead to messed up multi-dimensional arrays. To avoid that, you should convert your files with h5convert. The plugin version >= 0.6.2 now stamp the file with a "generated by" string ("/" group attribute) which identifies the yorick plugin version. As a sefaguard, h5convert will refuse to process (convert) these files, so you can batch process directories, or even apply h5convert to files if you are not sure they have been converted (important exceptions, see below). WARNING: do not try to convert files generated by external HDF resources, as they will be converted while they should not! Also, these files may include data type and object type not yet implemented in the plugin, in which case it will trigger an error and the file may result corrupted. You have been warned. The pre-v0.6.2 bug consisted in an inversion of the dimension vector of multi-dimensional arrays, such as, for instance, a 2x3 array appeared to hdf5 utilities (frm HDF group, e.g. h5dump) as a 3x2 array, but with the elements untransposed, e.g.: Written by yorick: 1 2 3 4 5 6 Read by h5dump: 1 2 3 4 5 6 If you are not using HDF group utilities, or more generally, you are only using hdf5 files within yorick (read/write), thus bug did not affect you. However, you should still process all your h5 files if you upgrade to the plugin version v>=0.6.2 (recommended). .PP .SH AUTHOR .PP Francois Rigaut, Gemini Observatory .PP .SH BUGS This is a patch for a previous bug. Beware: the flag "-p" will rename the converted file to the original name when done. Beware! This should be safe, but I haven't tested all possible scenario (i.e. objects type). This will erase your original data. I encourage you to check this conversion function on a few files before using this option. In particular, do not try to convert files generated by external HDF resources, as they will be converted while they should not! Also, these files may include data type and object type not yet implemented in the plugin, in which case it will trigger an error and the file may result corrupted. You have been warned. .SH SEE ALSO yorick(1) yorick-hdf5-0.8.0/Makefile0000644000076500001440000001107111254071505014670 0ustar frigautusers# $Id: Makefile,v 1.3 2008/11/21 16:19:17 frigaut Exp $ Y_MAKEDIR=/home/frigaut/yorick-2.1.05x-arch Y_EXE=/home/frigaut/yorick-2.1.05x-arch/bin/yorick Y_EXE_PKGS= Y_EXE_HOME=/home/frigaut/yorick-2.1.05x-arch Y_EXE_SITE=/home/frigaut/yorick-2.1.05x-arch # ----------------------------------------------------- optimization flags COPT=$(COPT_DEFAULT) TGT=$(DEFAULT_TGT) # ------------------------------------------------ macros for this package PKG_NAME=hdf5 PKG_I=hdf5.i h5scan_fromshell.i h5convert_fromshell.i OBJS=hdf5.o # change to give the executable a name other than yorick PKG_EXENAME=yorick # PKG_DEPLIBS=-Lsomedir -lsomelib for dependencies of this package PKG_DEPLIBS=-lhdf5 -lz # set compiler or loader (rare) flags specific to this package # yorick-hdf5 was written for hdf5 v1.6. Some distro (e.g. f9) have # moved to v1.8, which has incompatible APIs. Fortunately, hdf5 # developers have provided a way to expose the v1.6 APIs: PKG_CFLAGS=-D H5_USE_16_API PKG_LDFLAGS=-D H5_USE_16_API # another solution is to build hdf5 v1.6 (e.g. in /usr/local) and # use this: # PKG_CFLAGS=-I/usr/local/include # PKG_LDFLAGS=-L/usr/local/lib -lhdf5 -Wl,--rpath=/usr/local/lib # list of additional package names you want in PKG_EXENAME # (typically Y_EXE_PKGS should be first here) EXTRA_PKGS=$(Y_EXE_PKGS) # list of additional files for clean PKG_CLEAN= # autoload file for this package, if any PKG_I_START=hdf5_start.i # -------------------------------- standard targets and rules (in Makepkg) # set macros Makepkg uses in target and dependency names # DLL_TARGETS, LIB_TARGETS, EXE_TARGETS # are any additional targets (defined below) prerequisite to # the plugin library, archive library, and executable, respectively PKG_I_DEPS=$(PKG_I) include $(Y_MAKEDIR)/Make.cfg include $(Y_MAKEDIR)/Makepkg include $(Y_MAKEDIR)/Make$(TGT) # override macros Makepkg sets for rules and other macros # Y_HOME and Y_SITE in Make.cfg may not be correct (e.g.- relocatable) Y_HOME=$(Y_EXE_HOME) Y_SITE=$(Y_EXE_SITE) # reduce chance of yorick-1.5 corrupting this Makefile MAKE_TEMPLATE = protect-against-1.5 # ------------------------------------- targets and rules for this package # simple example: #myfunc.o: myapi.h # more complex example (also consider using PKG_CFLAGS above): #myfunc.o: myapi.h myfunc.c # $(CC) $(CPPFLAGS) $(CFLAGS) -DMY_SWITCH -o $@ -c myfunc.c clean:: -rm -rf binaries # -------------------------------------------------------- end of Makefile # for the binary package production (add full path to lib*.a below): # macosx: PKG_DEPLIBS_STATIC=-lm -lz /usr/lib/libhdf5.a PKG_ARCH = $(OSTYPE)-$(MACHTYPE) # or linux or windows PKG_VERSION = $(shell (awk '{if ($$1=="Version:") print $$2}' $(PKG_NAME).info)) # .info might not exist, in which case he line above will exit in error. # packages or devel_pkgs: PKG_DEST_URL = packages package: $(MAKE) $(LD_DLL) -o $(PKG_NAME).so $(OBJS) ywrap.o $(PKG_DEPLIBS_STATIC) $(DLL_DEF) mkdir -p binaries/$(PKG_NAME)/dist/y_home/lib mkdir -p binaries/$(PKG_NAME)/dist/y_home/i-start mkdir -p binaries/$(PKG_NAME)/dist/y_site/i0 mkdir -p binaries/$(PKG_NAME)/dist/y_home/bin cp -p $(PKG_I) binaries/$(PKG_NAME)/dist/y_site/i0/ cp -p $(PKG_NAME).so binaries/$(PKG_NAME)/dist/y_home/lib/ cp -p h5info binaries/$(PKG_NAME)/dist/y_home/bin/. cp -p h5convert binaries/$(PKG_NAME)/dist/y_home/bin/. if test -f "check.i"; then cp -p check.i binaries/$(PKG_NAME)/.; fi if test -n "$(PKG_I_START)"; then cp -p $(PKG_I_START) \ binaries/$(PKG_NAME)/dist/y_home/i-start/; fi cat $(PKG_NAME).info | sed -e 's/OS:/OS: $(PKG_ARCH)/' > tmp.info mv tmp.info binaries/$(PKG_NAME)/$(PKG_NAME).info cd binaries; tar zcvf $(PKG_NAME)-$(PKG_VERSION)-$(PKG_ARCH).tgz $(PKG_NAME) distbin: package if test -f "binaries/$(PKG_NAME)-$(PKG_VERSION)-$(PKG_ARCH).tgz" ; then \ ncftpput -f $(HOME)/.ncftp/maumae www/yorick/$(PKG_DEST_URL)/$(PKG_ARCH)/tarballs/ \ binaries/$(PKG_NAME)-$(PKG_VERSION)-$(PKG_ARCH).tgz; fi if test -f "binaries/$(PKG_NAME)/$(PKG_NAME).info" ; then \ ncftpput -f $(HOME)/.ncftp/maumae www/yorick/$(PKG_DEST_URL)/$(PKG_ARCH)/info/ \ binaries/$(PKG_NAME)/$(PKG_NAME).info; fi distsrc: make clean; rm -rf binaries cd ..; tar --exclude binaries --exclude .svn --exclude CVS --exclude *.spec -zcvf \ $(PKG_NAME)-$(PKG_VERSION)-src.tgz yorick-$(PKG_NAME)-$(PKG_VERSION);\ ncftpput -f $(HOME)/.ncftp/maumae www/yorick/$(PKG_DEST_URL)/src/ \ $(PKG_NAME)-$(PKG_VERSION)-src.tgz cd ..; ncftpput -f $(HOME)/.ncftp/maumae www/yorick/contrib/ \ $(PKG_NAME)-$(PKG_VERSION)-src.tgz # -------------------------------------------------------- end of Makefile yorick-hdf5-0.8.0/hdf5.c0000644000076500001440000006015111254075574014237 0ustar frigautusers/* * hdf5.c * wrapper routines for the hdf5 c library * * $Id: hdf5.c,v 1.1.1.1 2007/12/27 15:10:25 frigaut Exp $ * * Author: Francois Rigaut. * Written 2004 * last revision/addition: 2007dec26 * * Copyright (c) 2003, Francois RIGAUT (frigaut@gemini.edu, Gemini * Observatory, 670 N A'Ohoku Place, HILO HI-96720). * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the * Free Software Foundation; either version 2 of the License, or (at your * option) any later version. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details (to receive a copy of the GNU * General Public License, write to the Free Software Foundation, Inc., 675 * Mass Ave, Cambridge, MA 02139, USA). * * $Log: hdf5.c,v $ * Revision 1.1.1.1 2007/12/27 15:10:25 frigaut * Initial Import - yorick-hdf5 * */ #include #include /* For strcmp() */ #include /* For EXIT_FAILURE, EXIT_SUCCESS */ #include "hdf5.h" #include "ydata.h" #include "pstdlib.h" void Y__H5Eon() { H5Eset_auto((H5E_auto_t)H5Eprint,stderr); } void Y__H5Eoff() { H5Eset_auto(NULL,NULL); } void Y__H5close() { H5close(); } void Y__H5open() { H5open(); } void Y__H5version() { unsigned majnum,minnum,relnum; H5get_libversion ( &majnum, &minnum, &relnum); Array *a= PushDataBlock(NewArray(&longStruct, ynew_dim(3L, 0))); a->value.l[0]=(long)majnum; a->value.l[1]=(long)minnum; a->value.l[2]=(long)relnum; } void Y__H5Fcreate(int nArgs) { char *filename = YGetString(sp-nArgs+1); long mode = YGetInteger(sp-nArgs+2); long create_id = YGetInteger(sp-nArgs+3); long access_id = YGetInteger(sp-nArgs+4); int status; status=H5Fcreate(filename, (uint) mode, (hid_t) create_id, (hid_t) access_id); PushIntValue(status); } void Y__H5Fopen(int nArgs) { char *filename = YGetString(sp-nArgs+1); long flags = YGetInteger(sp-nArgs+2); long access_id = YGetInteger(sp-nArgs+3); int status; status=H5Fopen(filename, (uint) flags, (hid_t) access_id); PushIntValue(status); } void Y__H5Fclose(int nArgs) { long file_id = YGetInteger(sp-nArgs+1); int status; H5Fflush((hid_t) file_id, H5F_SCOPE_LOCAL); status=H5Fclose((hid_t) file_id); PushIntValue(status); } void Y__H5Areads(int nArgs) { long attid = YGetInteger(sp-nArgs+1); Dimension *strdims = 0; char **data= YGet_Q(sp-nArgs+2,0,&strdims); long nelem = YGetInteger(sp-nArgs+3); hid_t atype; void **buf[nelem]; int i; atype = H5Tcopy(H5T_C_S1); H5Tset_size(atype,H5T_VARIABLE); H5Tset_strpad(atype,H5T_STR_NULLTERM); H5Tset_cset(atype,H5T_CSET_ASCII); H5Aread(attid,atype,&buf); Array *a= PushDataBlock(NewArray(&stringStruct,strdims)); // below and 134: added (char *) on 2009/07/16 for (i=0;ivalue.q[i] = p_strcpy((char *)buf[i]); //free(buf); PopTo(sp-nArgs-1); Drop(nArgs); } void Y__H5Dreads(int nArgs) { long did = YGetInteger(sp-nArgs+1); Dimension *strdims = 0; char **data= YGet_Q(sp-nArgs+2,0,&strdims); long nelem = YGetInteger(sp-nArgs+3); hid_t atype; void **buf[nelem]; int i; atype = H5Tcopy(H5T_C_S1); H5Tset_size(atype,H5T_VARIABLE); H5Tset_strpad(atype,H5T_STR_NULLTERM); H5Tset_cset(atype,H5T_CSET_ASCII); H5Dread(did,atype,0,0,0,&buf); Array *a= PushDataBlock(NewArray(&stringStruct,strdims)); for (i=0;ivalue.q[i] = p_strcpy((char *)buf[i]); PopTo(sp-nArgs-1); Drop(nArgs); } void Y__H5Gget_linkval(int nArgs) { long loc_id = YGetInteger(sp-nArgs+1); char *gname = YGetString(sp-nArgs+2); long size = YGetInteger(sp-nArgs+3); char *value = YGetString(sp-nArgs+4); PushIntValue((long)H5Gget_linkval((hid_t) loc_id, gname, (size_t)size, value)); PopTo(sp-nArgs-1); Drop(nArgs); } void Y__H5Gopen(nArgs) { long loc_id = YGetInteger(sp-nArgs+1); char *gname = YGetString(sp-nArgs+2); PushIntValue((long)H5Gopen((hid_t)loc_id, gname)); PopTo(sp-nArgs-1); Drop(nArgs); } void Y__H5Gclose(nArgs) { long gid = YGetInteger(sp-nArgs+1); PushIntValue((long)H5Gclose((hid_t)gid)); PopTo(sp-nArgs-1); Drop(nArgs); } void Y__H5Gcreate(nArgs) { long loc_id = YGetInteger(sp-nArgs+1); char *gname = YGetString(sp-nArgs+2); long size_hint = YGetInteger(sp-nArgs+3); PushIntValue((long)H5Gcreate((hid_t)loc_id, gname, (size_t)size_hint)); PopTo(sp-nArgs-1); Drop(nArgs); } void Y__H5Gget_num_objs(nArgs) { long gid = YGetInteger(sp-nArgs+1); hsize_t num_obj=0; H5Gget_num_objs((hid_t)gid, &num_obj); PushIntValue((long)num_obj); PopTo(sp-nArgs-1); Drop(nArgs); } void Y__H5Gget_objname_by_idx(int nArgs) { long loc_id = YGetInteger(sp-nArgs+1); long idx = YGetInteger(sp-nArgs+2); char *name = YGetString(sp-nArgs+3); long size = YGetInteger(sp-nArgs+4); H5Gget_objname_by_idx((hid_t)loc_id, (hsize_t)idx,name,(size_t)size); Drop(nArgs); } void Y__H5Gget_objtype_by_idx(int nArgs) { long loc_id = YGetInteger(sp-nArgs+1); long idx = YGetInteger(sp-nArgs+2); PushIntValue((long)H5Gget_objtype_by_idx((hid_t)loc_id, (hsize_t)idx)); PopTo(sp-nArgs-1); Drop(nArgs); } void Y__H5Gget_objtype_by_name(int nArgs) { long loc_id = YGetInteger(sp-nArgs+1); char *name = YGetString(sp-nArgs+2); Dimension *dims = 0; long *objnum = YGet_L(sp-nArgs+3,0, &dims); H5G_stat_t statbuf; hbool_t followlink=0; H5Gget_objinfo((hid_t)loc_id,name,followlink,&statbuf); objnum[0] = statbuf.objno[0]; objnum[1] = statbuf.objno[1]; PushIntValue((long)statbuf.type); PopTo(sp-nArgs-1); Drop(nArgs); } void Y__H5Glink2(int nArgs) { long curr_loc_id = YGetInteger(sp-nArgs+1); char *curname = YGetString(sp-nArgs+2); long link_type = YGetInteger(sp-nArgs+3); long new_loc_id = YGetInteger(sp-nArgs+4); char *newname = YGetString(sp-nArgs+5); PushIntValue((long)H5Glink2((hid_t)curr_loc_id, curname, (H5G_link_t)link_type, (hid_t)new_loc_id, newname)); PopTo(sp-nArgs-1); Drop(nArgs); } void Y__H5Gunlink(int nArgs) { long loc_id = YGetInteger(sp-nArgs+1); char *name = YGetString(sp-nArgs+2); PushIntValue((long)H5Gunlink((hid_t)loc_id, name)); PopTo(sp-nArgs-1); Drop(nArgs); } void Y__H5Pcreate(int nArgs) { long cls_id = YGetInteger(sp-nArgs+1); PushIntValue((long)H5Pcreate((hid_t)cls_id)); PopTo(sp-nArgs-1); Drop(nArgs); } void Y__H5Pset_deflate(int nArgs) { long plist = YGetInteger(sp-nArgs+1); long level = YGetInteger(sp-nArgs+2); PushIntValue((long)H5Pset_deflate((hid_t)plist,(int)level)); PopTo(sp-nArgs-1); Drop(nArgs); } //herr_t H5Pset_chunk(hid_t plist, int ndims, const hsize_t * dim) void Y__H5Pset_chunk(int nArgs) { long plist=YGetInteger(sp-nArgs+1); long ndims=YGetInteger(sp-nArgs+2); Dimension *tmpdims = 0; long *dim=YGet_L(sp-nArgs+3,0,&tmpdims); hsize_t hdim[5]; long status,i; for (i=0;i