pax_global_header00006660000000000000000000000064145102340250014506gustar00rootroot0000000000000052 comment=a3fb72adf526be4ab3cbd20fef2e050e4cea7aaf pytsk-20231007/000077500000000000000000000000001451023402500131205ustar00rootroot00000000000000pytsk-20231007/.github/000077500000000000000000000000001451023402500144605ustar00rootroot00000000000000pytsk-20231007/.github/workflows/000077500000000000000000000000001451023402500165155ustar00rootroot00000000000000pytsk-20231007/.github/workflows/build.yml000066400000000000000000000051231451023402500203400ustar00rootroot00000000000000name: build on: [push, pull_request] permissions: read-all jobs: build_macos: runs-on: macos-latest strategy: matrix: include: - python-version: '3.11' toxenv: 'py311' steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - name: Install build dependencies run: | brew update -q brew install -q autoconf automake gettext gnu-sed pkgconfig python@${{ matrix.python-version }} tox brew link --force gettext ln -s /usr/local/bin/glibtoolize /usr/local/bin/libtoolize - name: Build and test Python module run: | python setup.py update export CXXFLAGS="-std=c++14" python setup.py build tox -e${{ matrix.toxenv }} build_ubuntu: runs-on: ubuntu-22.04 strategy: matrix: include: - python-version: '3.7' toxenv: 'py37' - python-version: '3.8' toxenv: 'py38' - python-version: '3.9' toxenv: 'py39' - python-version: '3.10' toxenv: 'py310' - python-version: '3.11' toxenv: 'py311' - python-version: '3.12' toxenv: 'py312' steps: - uses: actions/checkout@v3 - name: Install build dependencies run: | sudo add-apt-repository universe && sudo add-apt-repository -y ppa:deadsnakes/ppa && sudo add-apt-repository -y ppa:gift/dev && sudo apt-get update && sudo apt-get install -y autoconf automake autopoint autotools-dev build-essential git libtool pkg-config python${{ matrix.python-version }} python${{ matrix.python-version }}-dev python${{ matrix.python-version }}-venv python3-distutils python3-pip python3-setuptools - name: Install tox run: | python3 -m pip install tox - name: Build and test Python module run: | python setup.py update python setup.py build tox -e${{ matrix.toxenv }} build_windows: runs-on: windows-latest strategy: matrix: python-version: ['3.11'] architecture: ['x86', 'x64'] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - name: Build Python module run: | python setup.py update python setup.py build python setup.py install - name: Test Python module run: | python run_tests.py pytsk-20231007/.gitignore000066400000000000000000000004271451023402500151130ustar00rootroot00000000000000# Files to ignore by git # Back-up files *~ *.so *.swp # Generic auto-generated build files *.pyc *.pyo # Specific auto-generated build files /.tox /a.out /__pycache__ /build /dist /MANIFEST /pytsk3.egg-info /tmp # Project specific auto-generated files /pytsk3.c /pytsk3.cpp pytsk-20231007/.gitmodules000066400000000000000000000001341451023402500152730ustar00rootroot00000000000000[submodule "sleuthkit"] path = sleuthkit url = https://github.com/sleuthkit/sleuthkit.git pytsk-20231007/AUTHORS000066400000000000000000000011011451023402500141610ustar00rootroot00000000000000Copyright 2010, Michael Cohen . Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. pytsk-20231007/LICENSE000066400000000000000000000261361451023402500141350ustar00rootroot00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. pytsk-20231007/MANIFEST.in000066400000000000000000000017071451023402500146630ustar00rootroot00000000000000include AUTHORS LICENSE version.txt include *.c include *.h include *.py exclude *.pyc exclude .git .gitignore .gitmodules exclude .travis.yml exclude API-CHANGES.txt config.log ruleset.xml setupDevRepos.py travis_build.sh recursive-include dpkg * recursive-include patches *.patch recursive-include sleuthkit * recursive-exclude sleuthkit/autom4te.cache * recursive-exclude sleuthkit/bindings * recursive-exclude sleuthkit/debian * recursive-exclude sleuthkit/docs * recursive-exclude sleuthkit/framework * recursive-exclude sleuthkit/man * recursive-exclude sleuthkit/packages * recursive-exclude sleuthkit/rejistry++ * recursive-exclude sleuthkit/release * recursive-exclude sleuthkit/samples * recursive-exclude sleuthkit/tests * recursive-exclude sleuthkit/tools * recursive-exclude sleuthkit/unit_tests * recursive-exclude sleuthkit/win32 * recursive-exclude sleuthkit/xcode * recursive-include talloc * recursive-include test_data * recursive-include travis * pytsk-20231007/README000066400000000000000000000024371451023402500140060ustar00rootroot00000000000000pytsk is a Python binding for the SleuthKit. This is a Python binding against the libtsk (SleuthKit library). The aim is to make the binding reflect the TSK API as much as possible in capabilities, while at the same time having a nice Pythonic OO interface: 4.11.1: http://www.sleuthkit.org/sleuthkit/docs/api-docs/4.11.1/ WARNING: use pytsk at your own risk. libtsk is known to have many defects. For processing data from untrusted sources it is highly recommended to add additional security measures, such as a security sandbox. If downloaded pytsk using git you'll have to first run: python setup.py update If you want to use the latest version of Sleuthkit that is checked into git (also known as HEAD), instead of the currently supported version, you can run: python setup.py update --use-head To build the bindings just use the standard Python setuptools: python setup.py build python setup.py install At the top level of the source tree. The Python binding is autogenerated from the libtsk header files using a small OO C shim. This means that most of the fields in many of the structs are already available. We aim to provide most of the functionality using this shim (e.g. traversing and iterating over lists etc). The authoritative source of documentation is the library API linked above. pytsk-20231007/aff4_errors.h000066400000000000000000000051301451023402500155040ustar00rootroot00000000000000/* AFF4 error functions. * * Copyright 2010, Michael Cohen . * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef AFF4_ERRORS_H_ #define AFF4_ERRORS_H_ #include "class.h" #ifdef __cplusplus extern "C" { #endif // Some helpful little things #define ERROR_BUFFER_SIZE 1024 /** This is used for error reporting. This is similar to the way python does it, i.e. we set the error flag and return NULL. */ #define EZero 0 #define EGeneric 1 #define EOverflow 2 #define EWarning 3 #define EUnderflow 4 #define EIOError 5 #define ENoMemory 6 #define EInvalidParameter 7 #define ERuntimeError 8 #define EKeyError 9 // Reserved for impossible conditions #define EProgrammingError 10 DLL_PUBLIC void *aff4_raise_errors(int t, const char *string, ...); /** We only set the error state if its not already set */ #define RaiseError(t, message, ...) \ aff4_raise_errors(t, "%s: (%s:%d) " message, __FUNCTION__, __FILE__, __LINE__, ## __VA_ARGS__); #define LogWarnings(format, ...) \ do { \ RaiseError(EWarning, format, ## __VA_ARGS__); \ PrintError(); \ } while(0); #define ClearError() \ do {*aff4_get_current_error(NULL) = EZero;} while(0); #define PrintError() \ do {char *error_str; if(*aff4_get_current_error(&error_str)) fprintf(stdout, "%s", error_str); fflush(stdout); ClearError(); }while(0); #define CheckError(error) \ (*aff4_get_current_error(NULL) == error) /** The current error state is returned by this function. This is done in a thread safe manner. */ DLL_PUBLIC int *aff4_get_current_error(char **error_str); // These macros are used when we need to do something which might // change the error state on the error path of a function. #define PUSH_ERROR_STATE { int *tmp_error_p = aff4_get_current_error(NULL); int tmp_error = *tmp_error_p; int exception __attribute__((unused)); #define POP_ERROR_STATE *tmp_error_p = tmp_error;}; #ifdef __cplusplus } /* closing brace for extern "C" */ #endif #endif /* !AFF4_ERRORS_H_ */ pytsk-20231007/class.cpp000066400000000000000000000030011451023402500147230ustar00rootroot00000000000000/* C class and object types functions. * * Copyright 2013, Michael Cohen . * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "misc.h" #include "class.h" #define BUFF_SIZE 1024 // Noone should instantiate Object directly. this should be already // allocated therefore: DLL_PUBLIC void Object_init(Object cthis) { cthis->__class__ = &__Object; cthis->__super__ = NULL; }; struct Object_t __Object = { &__Object, //.__class__ &__Object, //.__super__ "Object", //.__name__ "", //.__doc__ sizeof(struct Object_t), //.__size NULL //.__extension }; int issubclass(Object obj, Object cclass) { obj = obj->__class__; while(1) { if(obj == cclass->__class__) return 1; obj=obj->__super__; if(obj == &__Object || obj==NULL) return 0; }; }; void unimplemented(Object self) { printf("%s contains unimplemented functions.. is it an abstract class?\n", NAMEOF(self)); abort(); }; pytsk-20231007/class.h000066400000000000000000000361121451023402500144010ustar00rootroot00000000000000/* C class and object types functions. * * Copyright 2013, Michael Cohen . * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef __PYTSK_CCLASS_H__ #define __PYTSK_CCLASS_H__ /* Classes and objects in C This file makes it easy to implement classes and objects in C. To define a class we need to perform three steps: Define the class prototype. This is suitable to go in a .h file for general use by other code. Note all classes extend Object. Example:: CCLASS(Foo, Object) int x; int y; //This declares a method of a class Foo, called Con returning a //Foo object. In other words it is a constructor. Foo METHOD(Foo, Con, int x, int y); int METHOD(Foo, add); END_CCLASS Now we need to define some functions for the constructor and methods. Note that the constuctor is using ALLOCATE_CCLASS to allocate space for the class structures. Callers may call with self==NULL to force allocation of a new class. Note that we do not call the constructor of our superclass implicitly here. (Calling the sperclass constructor is optional, but ALLOCATE_CCLASS is not.). Foo Foo_Con(Foo self,int x,int y) { self->x = x; self->y = y; return self; }; int Foo_add(Foo cthis) { return (cthis->x + cthis->y); }; Now we need to define the Virtual function table - These are those functions and attributes which are defined in the class (over its superclass). Basically these are all those things in the class definition above, with real function names binding them. (Note that by convention we preceed the name of the method with the name of the class): VIRTUAL(Foo,Object) VMETHOD(Con) = Foo_Con; VMETHOD(add) = Foo_add; END_VIRTUAL We can use inheritance too: CCLASS(Bar, Foo) Bar METHOD(Bar, Con, char *something) END_CCLASS Here Bar extends Foo and defines a new constructor with a different prototype: VIRTUAL(Bar,Foo) VMETHOD(Con) = Bar_Con END_VIRTUAL If there is a function which expects a Foo, we will need to over ride the Foo constructor in the Bar, so the function will not see the difference between the Foo and Bar: CCLASS(Bar,Foo) int bar_attr; END_CCLASS Foo Bar_Con(Foo self, int x, int y) { ... } VIRTUAL(Bar, Foo) VMETHOD(super.Con) = Bar_Con END_VIRTUAL Note that in this case we are over riding the Con method defined in Foo while creating derived Bar classes. The notation in the VIRTUAL table is to use super.Con, because Foo's Con method (the one we are over riding), can be located by using super.Con inside a Bar object. Imagine now that in Bar_Con we wish to use methods and attributes defined in Bar. Since Bar_Con over rides Bar's base class (Foo) it must have the prototype described above. Since self is of type Foo its impossible to use self->bar_attr (There is no bar_attr in Foo - its in Bar). In this case, we need to make a type cast to convice C that self is actually a Bar not a Foo: Foo Bar_Con(Foo self, int x, int y) { Bar cthis = (Bar)self; cthis->bar_attr=1 }; This allows us to access bars attributes. This is a general oddity with C style classes, which C++ and Java hide. In C we must always know which class defines which method and attribute and reference the right class's method. So for example if we want to call a Bar's add method: Bar a; a->super.add() because add is defined in Bar's super class (Foo). Constract this with C++ or Java which hide where methods are defined and simply make all methods appear like they were defined inside the derived class. This takes a while to get used to but the compiler will ensure that the references are correct - otherwise things will generally not compile properly. This difference can be used for good and bad. It is possible in C to call the base class's version of the method at any time (despite the fact it was over ridden). For example: CCLASS(Derived, Foo) int METHOD(Derived, add); END_CCLASS VIRTUAL(Derived, Foo) VMETHOD(add) = Derived_add END_VIRTUAL If d is a Derived object, we can call Foo's version like this: d->super.add() But Derived's version is accessed by: d->add() Sometimes a derived class may want to over ride the base class's methods as well, in this case the VIRTUAL section should over ride super.add as well. */ #ifdef __cplusplus extern "C" { #endif #include "misc.h" #include #define CCLASS(cclass,super_cclass) \ typedef struct cclass ## _t *cclass; \ DLL_PUBLIC cclass alloc_ ## cclass(void); /* Allocates object memory */ \ DLL_PUBLIC int cclass ## _init(Object self); /* Class initializer */ \ DLL_PUBLIC extern struct cclass ## _t __ ## cclass; /* Public C class template */ \ struct cclass ## _t { \ struct super_cclass ## _t super; /* Super C class Fields we inherit */ \ cclass __class__; /* Pointer to our own C class */ \ super_cclass __super__; /* Pointer to our super C class */ #define METHOD(cls, name, ... ) \ (* name)(cls self, ## __VA_ARGS__ ) // Class methods are attached to the C class but are not called with // an instance. This is similar to the Python class method or java // static methods. #define CCLASS_METHOD(name, ... ) \ (*name)(__VA_ARGS__) /* This is a convenience macro which may be used if x if really large */ #define CALL(x, method, ... ) \ (x)->method((x), ## __VA_ARGS__) #define END_CCLASS }; /* This is used to set the C classes up for use: * * cclass_init = checks the C class template (__class) to see if it has * been allocated. otherwise allocates it in the global context. * * cclass_Alloc = Allocates new memory for an instance of the C class * This is a recursive function calling each super C class in turn * and setting the currently over ridden defaults. So for eample * suppose the C class (foo) derives from bar, we first fill the * template with bars methods, and attributes. Then we over write * those with foos methods and attributes. */ #define VIRTUAL(cclass,supercclass) \ struct cclass ## _t __ ## cclass; \ \ DLL_PUBLIC cclass alloc_ ## cclass(void) { \ cclass result = (cclass) talloc_memdup(NULL, (const void *) &__## cclass, sizeof(__## cclass)); \ return result; \ }; \ \ DLL_PUBLIC int cclass ## _init(Object cthis) { \ cclass self = (cclass)cthis; \ if(self->__super__) return 1; \ supercclass ##_init(cthis); \ cthis->__class__ = (Object)&__ ## cclass; \ self->__class__ = (cclass)&__ ## cclass; \ cthis->__super__ = (Object)&__ ## supercclass; \ self->__super__ = (supercclass)&__ ## supercclass; \ cthis->__size = sizeof(struct cclass ## _t); \ cthis->__name__ = #cclass; #define SET_DOCSTRING(string) \ ((Object)self)->__doc__ = string #define END_VIRTUAL return 1; }; #define VMETHOD(method) \ (self)->method #define VMETHOD_BASE(base, method) \ (((base)self)->method) #define CCLASS_ATTR(self, base, method) \ (((base)self)->method) #define VATTR(attribute) \ (self)->attribute #define NAMEOF(obj) \ ((Object)obj)->__name__ #define SIZEOF(obj) \ ((Object)obj)->__size #define DOCSTRING(obj) \ ((Object)obj)->__doc__ #define INIT_CCLASS(cclass) \ cclass ## _init((Object)&__ ## cclass) /* This MACRO is used to construct a new Class using a constructor. * * This is done to try and hide the bare (unbound) method names in * order to prevent name space pollution. (Bare methods may be * defined as static within the implementation file). This macro * ensures that C class structures are initialised properly before * calling their constructors. * * We require the following args: * cclass - the type of C class to make * virt_cclass - The C class where the method was defined * constructors - The constructor method to use * context - a talloc context to use. * * Note that the cclass and virt_cclass do not have to be the same if * the method was not defined in the current C class. For example * suppose Foo extends Bar, but method is defined in Bar but * inherited in Foo: * * CONSTRUCT(Foo, Bar, super.method, context) * * virt_cclass is Bar because thats where method was defined. */ // The following only initialises the C class if the __super__ element // is NULL. This is fast as it wont call the initaliser unnecessaily // This requires the C class initializers to have been called // previously. Therefore they are not exported. #define CONSTRUCT(cclass, virt_cclass, constructor, context, ...) \ (cclass)(((virt_cclass) (&__ ## cclass))->constructor( \ (virt_cclass) _talloc_memdup( \ context, &__ ## cclass, \ sizeof(struct cclass ## _t), \ __location__ "(" #cclass ")"), \ ## __VA_ARGS__) ) /* _talloc_memdup version #define CONSTRUCT_CREATE(cclass, virt_cclass, context) \ (virt_cclass) _talloc_memdup(context, &__ ## cclass, sizeof(struct cclass ## _t), __location__ "(" #cclass ")") */ #define CONSTRUCT_CREATE(cclass, virt_cclass, context) \ (virt_cclass) talloc_memdup(context, &__ ## cclass, sizeof(struct cclass ## _t)) #define CONSTRUCT_INITIALIZE(cclass, virt_cclass, constructor, object, ...) \ (cclass)(((virt_cclass) (&__ ## cclass))->constructor(object, ## __VA_ARGS__)) /* This variant is useful when all we have is a C class reference * (GETCCLASS(Foo)) or &__Foo */ #define CONSTRUCT_FROM_REFERENCE(cclass, constructor, context, ... ) \ ( (cclass)->constructor( \ (void *)_talloc_memdup(context, ((Object)cclass), ((Object)cclass)->__size, __location__ "(" #cclass "." #constructor ")"), \ ## __VA_ARGS__) ) /* Finds the size of the C class in x */ #define CCLASS_SIZE(cclass) \ ((Object)cclass)->__size typedef struct Object_t *Object; struct Object_t { //A reference to a C class instance - this is useful to be able to //tell which C class an object really belongs to: Object __class__; //And its super C class: Object __super__; const char *__name__; /** Objects may have a doc string associated with them. */ const char *__doc__; //How large the C class is: int __size; /* A pointer to an extension - An extension is some other arbitrary object which may be linked with this one. */ void *extension; }; #define SUPER(base, imp, method, ...) \ ((base)&__ ## imp)->method((base)self, ## __VA_ARGS__) #define GETCCLASS(cclass) \ (Object)&__ ## cclass // Returns true if the obj belongs to the C class #define ISINSTANCE(obj,cclass) \ (((Object)obj)->__class__ == GETCCLASS(cclass)) // This is a string comparison version of ISINSTANCE which works // across different shared objects. #define ISNAMEINSTANCE(obj, cclass) \ (obj && !strcmp(cclass, NAMEOF(obj))) // We need to ensure that C class was properly initialised: #define ISSUBCCLASS(obj,cclass) \ issubcclass((Object)obj, (Object)&__ ## cclass) #define CCLASSOF(obj) \ ((Object)obj)->__class__ DLL_PUBLIC void Object_init(Object); DLL_PUBLIC extern struct Object_t __Object; /** Find out if obj is an instance of cls or a derived C class. Use like this: if(issubcclass(obj, (Object)&__FileLikeObject)) { ... }; You can also do this in a faster way if you already know the C class hierarchy (but it could break if the hierarchy changes): { Object cls = ((Object)obj)->__class__; if(cls == (Object)&__Image || \ cls == (Object)&__FileLikeObject || \ cls == (Object)&__AFFObject || ....) { ... }; }; */ int issubcclass(Object obj, Object cclass); DLL_PUBLIC extern void unimplemented(Object self); #define UNIMPLEMENTED(cclass, method) \ ((cclass)self)->method = (void *)unimplemented; #define ZSTRING_NO_NULL(str) str , (strlen(str)) #define ZSTRING(str) str , (strlen(str)+1) // These dont do anything but are useful to indicate when a function // parameter is used purely to return a value. They are now used to // assist the python binding generator in generating the right sort // of code #define OUT #define IN // This modifier before a C class means that the C class is abstract and // does not have an implementation - we do not generate bindings for // that C class then. #define ABSTRACT // This modifier indicates that the following pointer is pointing to // a borrowed reference - callers must not free the memory after use. #define BORROWED // This tells the autobinder to generated bindings to this struct #define BOUND // This tells the autobinder to ignore this C class as it should be // private to the implementation - external callers should not // access this. #define PRIVATE // This attribute of a method means that this method is a // desctructor - the object is no longer valid after this method is // run #define DESTRUCTOR // including this after an argument definition will cause the // autogenerator to assign default values to that parameter and make // it optional #define DEFAULT(x) // This explicitely denote that the type is a null terminated char // ptr as opposed to a pointer to char and length. typedef char * ZString; /* The following is a direction for the autogenerator to proxy the given C class. This is done in the following way: 1) a new python type is created called Proxy_cclass_name() with a constructor which takes a surrogate object. 2) The proxy C class contains a member "base" of the type of the proxied C class. 3) The returned python object may be passed to any C functions which expect the proxied C class, and internal C calls will be converted to python method calls on the proxied object. */ #define PROXY_CCLASS(name) /* This signals the autogenerator to bind the named struct */ #define BIND_STRUCT(name) // This means that the memory owned by this pointer is managed // externally (not using talloc). It is dangerous to use this // keyword too much because we are unable to manage its memory // appropriately and it can be free'd from under us. #define FOREIGN #ifdef __cplusplus } /* closing brace for extern "C" */ #endif #endif /* ifndef __PYTSK_CCLASS_H__ */ pytsk-20231007/class_parser.py000066400000000000000000004507671451023402500161760ustar00rootroot00000000000000#!/usr/bin/python # # Copyright 2010, Michael Cohen . # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Documentation regarding the Python bounded code. This code originally released as part of the AFF4 project (http://code.google.com/p/aff4/). Memory Management ================= AFF4 uses a reference count system for memory management similar in many ways to the native Python system. The basic idea is that memory returned by the library always carries a new reference. When the caller is done with the memory, they must call aff4_free() on the memory, afterwhich the memory is considered invalid. The memory may still not be freed at this point depending on its total reference count. New references may be taken to the same memory at any time using the aff4_incref() function. This increases the reference count of the object, and prevents it from being really freed until the correct number of aff4_free() calls are made to it. This idea is important for example in the following sequence: FileLikeObject fd = resolver->create(resolver, "w"); RDFURN uri = fd->urn; Now uri hold a reference to the urn attribute of fd, but that attribute is actually owned by fd. If fd is freed in future, e.g. (the close method actually frees the fd implicitely): fd->close(fd); Now the uri object is dangling. To prevent fd->urn from disappearing when fd is freed, we need to take another reference to it: FileLikeObject fd = resolver->create(resolver, "w"); RDFURN uri = fd->urn; aff4_incref(uri); fd->close(fd); Now uri is valid (but fd is no longer valid). When we are finished with uri we just call: aff4_free(uri); Python Integration ------------------ For every AFF4 object, we create a Python wrapper object of the corresponding type. The wrapper object contains Python wrapper methods to allow access to the AFF4 object methods, as well as getattr methods for attributes. It is very important to allow Python to inherit from C classes directly - this requires every internal C method call to be diverted to the Python object. The C object looks like this normally: struct obj { __class__ pointer to static struct initialised with C method pointers ... Some private members ... Attributes; /* Following are the methods */ int (*method)(struct obj *self, ....); }; I.e. when the method is called the struct.method member is dereferenced to find the location of the function handling it, the object is stuffed into the first arg, and the parameters are stuffed into following args. Directing Python calls ---------------------- The Python object which is created is a proxy for the c object. When Python methods are called in the Python object, they need to be directed into the C structure and a C call must be made, then the return value must be reconverted into Python objects and returned into Python. This occurs automatically by the wrapper: struct PythonWrapper { PyObject_HEAD void *base; }; When a Python method is called on this new Python type this is what happens: 1) The method name is looked up in the PyMethodDef struct as per normal. 2) If the method is recognised as a valid method the Python wrapper function is called (pyCLASSNAME_method) 3) This method is broken into the general steps: PyObject *pyCLASSNAME_method(PythonWrapper self, PyObject *args, PyObject *kwds) { set up c declerations for all args - call .definition() on all the args and return type parse argument using PyArg_ParseTupleAndKeywords Precall preparations Make the C call Post call processing of the returned value (check for errors etc) Convert the return value to a Python object using: return_type.to_Python_object() return the Python object or raise an exception }; So the aim of the wrapper function is to convert Python args to C args, find the C method corresponding to the method name by dereferencing the c object and then call it. The problem now is what happens when a C method internally calls another method. This is a problem because the C method has no idea its running within Python and so will just call the regular C method that was there already. This makes it impossible to subclass the C class and update the C method with a Python method. What we really want is when a C method is called internally, we want to end up calling the Python object instead to allow a purely Python implementation to override the C method. This happens by way of a ProxiedMethod - A proxied method is in a sense the reverse of the wrapper method: return_type ProxyCLASSNAME_method(CCLASSNAME self, ....) { Take all C args and create Python objects from them Dereference the object extension ((Object) self)->extension to obtain the Python object which wraps this C class. If an extension does not exist, just call the method as normal, otherwise make a Python call on the wrapper object. Convert the returned Python object to a C type and return it. }; To make all this work we have the following structures: struct PythonWrapper { PyObject_HEAD struct CCLASSNAME *base - This is a copy of the item, with all function pointer pointing at proxy functions. We can always get the original C function pointers through base->__class__ - We also set the base object extension to be the Python object: ((Object) base)->extension = PythonWrapper. This allows us to get back the Python object from base. }; When a Python method is invoked, we use cbase to find the C method pointer, but we pass to it base: self->base->__class__->method(self->base, ....) base is a proper C object which had its methods dynamically replaced with proxies. Now if an internal C method is called, the method will dereference base and retrieve the proxied method. Calling the proxied method will retreive the original Python object from the object extension and make a Python call. In the case where a method is not overridden by Python, internal C method calls will generate an unnecessary conversion from C to Python and then back to C. Memory management in Python extension ------------------------------------- When calling a method which returns a new reference, we just store the reference in the "base" member of the Python object. When Python garbage collects our Python object, we call aff4_free() on it. The getattr method creates a new Python wrapper object of the correct type, and sets its base attribute to point at the target AFF4 object. We then aff4_incref() the target to ensure that it does not get freed until we are finished with it. Python Object ----- | P1 | C Object | Base|-->+------+ | | | C1 | | | | | ----- |Member|--------------+-->+----+ +------+ | | C2 | | | | Getattr ------- | | | Member | P2 | | +----+ | Base |--+ New reference ------- Python Object Figure 1: Python object 1 owns C1's memory (when P1 is GC'ed C1 is freed). A reference to a member of C1 is made via P1's getattr method. The getattr method creates P2 to provide access to C2 by setting base to C2's address. We need to guarantee however, that C2 will not be freed suddenly (e.g. if C1 is freed). We therefore increase C2's reference count using aff4_incref(); """ import io import os import pdb import re import sys import lexer DEBUG = 0 # The pytsk3 version. VERSION = "20231007" # These functions are used to manage library memory. FREE = "aff4_free" INCREF = "aff4_incref" CURRENT_ERROR_FUNCTION = "aff4_get_current_error" CONSTANTS_BLACKLIST = ["TSK3_H_"] # Some constants. DOCSTRING_RE = re.compile("[ ]*\n[ \t]+[*][ ]?") def dispatch(name, type, *args, **kwargs): if not type: return PVoid(name) m = re.match("struct ([a-zA-Z0-9]+)_t *", type) if m: type = m.group(1) type_components = type.split() attributes = set() if type_components[0] in method_attributes: attributes.add(type_components.pop(0)) type = " ".join(type_components) result = type_dispatcher[type](name, type, *args, **kwargs) result.attributes = attributes return result def log(msg): if DEBUG > 0: sys.stderr.write("{0:s}\n".format(msg)) def format_as_docstring(string): # Remove C/C++ comment code statements. string = DOCSTRING_RE.sub("\n", string) byte_string = string.encode("unicode-escape") # Escapes double quoted string. We need to run this after unicode-escape to # prevent this operation to escape the escape character (\). In Python 3 # the replace method requires the arguments to be byte strings. byte_string = byte_string.replace(b"\"", b"\\\"") # Make sure to return the string a Unicode otherwise in Python 3 the string # is prefixed with b when written or printed. return byte_string.decode("utf-8") class Module(object): public_api = None public_header = None def __init__(self, name): self.name = name self.constants = set() self.constants_blacklist = CONSTANTS_BLACKLIST self.classes = {} self.headers = "" self.files = [] self.active_structs = set() self.function_definitions = set() init_string = "" def initialization(self): result = self.init_string + ( "\n" "talloc_set_log_fn((void (*)(const char *)) printf);\n" "// DEBUG: talloc_enable_leak_report();\n" "// DEBUG: talloc_enable_leak_report_full();\n") for cls in self.classes.values(): if cls.is_active(): result += cls.initialise() return result def add_constant(self, constant, type="numeric"): """This will be called to add #define constant macros.""" self.constants.add((constant, type)) def add_class(self, cls, handler): self.classes[cls.class_name] = cls # Make a wrapper in the type dispatcher so we can handle # passing this class from/to Python type_dispatcher[cls.class_name] = handler def get_string(self): """Retrieves a string representation.""" result = "Module {0:s}\n".format(self.name) classes_list = list(self.classes.values()) classes_list.sort(key=lambda cls: cls.class_name) for cls in classes_list: if cls.is_active(): result += " {0:s}\n".format(cls.get_string()) constants_list = list(self.constants) constants_list.sort() result += "Constants:\n" for name, _ in constants_list: result += " {0:s}\n".format(name) return result def private_functions(self): """Emits hard coded private functions for doing various things""" values_dict = { "classes_length": len(self.classes) + 1, "get_current_error": CURRENT_ERROR_FUNCTION} return """ /* The following is a static array mapping CCLASS() pointers to their * Python wrappers. This is used to allow the correct wrapper to be * chosen depending on the object type found - regardless of the * prototype. * * This is basically a safer way for us to cast the correct Python type * depending on context rather than assuming a type based on the .h * definition. For example consider the function * * AFFObject Resolver.open(uri, mode) * * The .h file implies that an AFFObject object is returned, but this is * not true as most of the time an object of a derived C class will be * returned. In C we cast the returned value to the correct type. In the * Python wrapper we just instantiate the correct Python object wrapper * at runtime depending on the actual returned type. We use this lookup * table to do so. */ static int TOTAL_CCLASSES=0; /* This is a global reference to this module so classes can call each * other. */ static PyObject *g_module = NULL; #define CONSTRUCT_INITIALIZE(cclass, virt_cclass, constructor, object, ...) \\ (cclass)(((virt_cclass) (&__ ## cclass))->constructor(object, ## __VA_ARGS__)) #undef BUFF_SIZE #define BUFF_SIZE 10240 /* Python compatibility macros */ #if !defined( PyMODINIT_FUNC ) #if PY_MAJOR_VERSION >= 3 #define PyMODINIT_FUNC PyObject * #else #define PyMODINIT_FUNC void #endif #endif /* !defined( PyMODINIT_FUNC ) */ #if !defined( PyVarObject_HEAD_INIT ) #define PyVarObject_HEAD_INIT( type, size ) \\ PyObject_HEAD_INIT( type ) \\ size, #endif /* !defined( PyVarObject_HEAD_INIT ) */ #if PY_MAJOR_VERSION >= 3 #define Py_TPFLAGS_HAVE_ITER 0 #endif #if !defined( Py_TYPE ) #define Py_TYPE( object ) \\ ( ( (PyObject *) object )->ob_type ) #endif /* !defined( Py_TYPE ) */ /* Generic wrapper type */ typedef struct Gen_wrapper_t *Gen_wrapper; struct Gen_wrapper_t {{ PyObject_HEAD void *base; /* Value to indicate the base is a Python object. */ int base_is_python_object; /* Value to indicate the base is managed internal. */ int base_is_internal; PyObject *python_object1; PyObject *python_object2; }}; static struct python_wrapper_map_t {{ Object class_ref; PyTypeObject *python_type; void (*initialize_proxies)(Gen_wrapper self, void *item); }} python_wrappers[{classes_length:d}]; /* Create the relevant wrapper from the item based on the lookup table. */ Gen_wrapper new_class_wrapper(Object item, int item_is_python_object) {{ Gen_wrapper result = NULL; Object cls = NULL; struct python_wrapper_map_t *python_wrapper = NULL; int cls_index = 0; // Return a Py_None object for a NULL pointer if(item == NULL) {{ Py_IncRef((PyObject *) Py_None); return (Gen_wrapper) Py_None; }} // Search for subclasses for(cls = (Object) item->__class__; cls != cls->__super__; cls = cls->__super__) {{ for(cls_index = 0; cls_index < TOTAL_CCLASSES; cls_index++) {{ python_wrapper = &(python_wrappers[cls_index]); if(python_wrapper->class_ref == cls) {{ PyErr_Clear(); result = (Gen_wrapper) _PyObject_New(python_wrapper->python_type); result->base = item; result->base_is_python_object = item_is_python_object; result->base_is_internal = 1; result->python_object1 = NULL; result->python_object2 = NULL; python_wrapper->initialize_proxies(result, (void *) item); return result; }} }} }} PyErr_Format(PyExc_RuntimeError, "Unable to find a wrapper for object %s", NAMEOF(item)); return NULL; }} typedef void (*function_initialize_Gen_wrapper_t)(Gen_wrapper, void*); static PyObject *resolve_exception(char **error_buff) {{ int *type = (int *){get_current_error:s}(error_buff); switch(*type) {{ case EProgrammingError: return PyExc_SystemError; case EKeyError: return PyExc_KeyError; case ERuntimeError: return PyExc_RuntimeError; case EInvalidParameter: return PyExc_TypeError; case EWarning: return PyExc_AssertionError; case EIOError: return PyExc_IOError; default: return PyExc_RuntimeError; }} }} static int type_check(PyObject *obj, PyTypeObject *type) {{ PyTypeObject *tmp = NULL; // Recurse through the inheritance tree and check if the types are expected if(obj) {{ for(tmp = Py_TYPE(obj); tmp && tmp != &PyBaseObject_Type; tmp = tmp->tp_base) {{ if(tmp == type) return 1; }} }} return 0; }} static int check_error() {{ char *buffer = NULL; int *error_type = (int *)aff4_get_current_error(&buffer); if(*error_type != EZero) {{ PyObject *exception = resolve_exception(&buffer); if(buffer != NULL) {{ PyErr_Format(exception, "%s", buffer); }} else {{ PyErr_Format(exception, "Unable to retrieve exception reason."); }} ClearError(); return 1; }} return 0; }} /* This function checks if a method was overridden in self over a * method defined in type. This is used to determine if a Python class is * extending this C type. If not, a proxy function is not written and C * calls are made directly. * * This is an optimization to eliminate the need for a call into Python * in the case where Python objects do not actually extend any methods. * * We basically just iterate over the MRO and determine if a method is * defined in each level until we reach the base class. */ static int check_method_override(PyObject *self, PyTypeObject *type, const char *method) {{ struct _typeobject *ob_type = NULL; PyObject *mro = NULL; PyObject *py_method = NULL; PyObject *item_object = NULL; PyObject *dict = NULL; Py_ssize_t item_index = 0; Py_ssize_t number_of_items = 0; int found = 0; ob_type = Py_TYPE(self); if(ob_type == NULL ) {{ return 0; }} mro = ob_type->tp_mro; #if PY_MAJOR_VERSION >= 3 py_method = PyUnicode_FromString(method); #else py_method = PyString_FromString(method); #endif number_of_items = PySequence_Size(mro); for(item_index = 0; item_index < number_of_items; item_index++) {{ item_object = PySequence_GetItem(mro, item_index); // Ok - we got to the base class - finish up if(item_object == (PyObject *) type) {{ Py_DecRef(item_object); break; }} /* Extract the dict and check if it contains the method (the * dict is not a real dictionary so we can not use * PyDict_Contains). */ dict = PyObject_GetAttrString(item_object, "__dict__"); if(dict != NULL && PySequence_Contains(dict, py_method)) {{ found = 1; }} Py_DecRef(dict); Py_DecRef(item_object); if(found != 0) {{ break; }} }} Py_DecRef(py_method); PyErr_Clear(); return found; }} /* Fetches the Python error (exception) */ void pytsk_fetch_error(void) {{ PyObject *exception_traceback = NULL; PyObject *exception_type = NULL; PyObject *exception_value = NULL; PyObject *string_object = NULL; char *str_c = NULL; char *error_str = NULL; int *error_type = (int *) {get_current_error:s}(&error_str); #if PY_MAJOR_VERSION >= 3 PyObject *utf8_string_object = NULL; #endif // Fetch the exception state and convert it to a string: PyErr_Fetch(&exception_type, &exception_value, &exception_traceback); string_object = PyObject_Repr(exception_value); #if PY_MAJOR_VERSION >= 3 utf8_string_object = PyUnicode_AsUTF8String(string_object); if(utf8_string_object != NULL) {{ str_c = PyBytes_AsString(utf8_string_object); }} #else str_c = PyString_AsString(string_object); #endif if(str_c != NULL) {{ strncpy(error_str, str_c, BUFF_SIZE-1); error_str[BUFF_SIZE - 1] = 0; *error_type = ERuntimeError; }} PyErr_Restore(exception_type, exception_value, exception_traceback); #if PY_MAJOR_VERSION >= 3 if( utf8_string_object != NULL ) {{ Py_DecRef(utf8_string_object); }} #endif Py_DecRef(string_object); return; }} /* Copies a Python int or long object to an unsigned 64-bit value */ uint64_t integer_object_copy_to_uint64(PyObject *integer_object) {{ #if defined( HAVE_LONG_LONG ) PY_LONG_LONG long_value = 0; #else long long_value = 0; #endif int result = 0; if(integer_object == NULL) {{ PyErr_Format(PyExc_ValueError, "Missing integer object"); return (uint64_t) -1; }} PyErr_Clear(); result = PyObject_IsInstance(integer_object, (PyObject *) &PyLong_Type); if(result == -1) {{ pytsk_fetch_error(); return (uint64_t) -1; }} else if(result != 0) {{ PyErr_Clear(); #if defined( HAVE_LONG_LONG ) long_value = PyLong_AsUnsignedLongLong(integer_object); #else long_value = PyLong_AsUnsignedLong(integer_object); #endif }} #if PY_MAJOR_VERSION < 3 if(result == 0) {{ PyErr_Clear(); result = PyObject_IsInstance(integer_object, (PyObject *) &PyInt_Type); if(result == -1) {{ pytsk_fetch_error(); return (uint64_t) -1; }} else if(result != 0) {{ PyErr_Clear(); #if defined( HAVE_LONG_LONG ) long_value = PyInt_AsUnsignedLongLongMask(integer_object); #else long_value = PyInt_AsUnsignedLongMask(integer_object); #endif }} }} #endif /* PY_MAJOR_VERSION < 3 */ if(result == 0) {{ if(PyErr_Occurred()) {{ pytsk_fetch_error(); return (uint64_t) -1; }} }} #if defined( HAVE_LONG_LONG ) #if ( SIZEOF_LONG_LONG > 8 ) if((long_value < (PY_LONG_LONG) 0) || (long_value > (PY_LONG_LONG) UINT64_MAX)) {{ #else if(long_value < (PY_LONG_LONG) 0) {{ #endif PyErr_Format(PyExc_ValueError, "Integer object value out of bounds"); return (uint64_t) -1; }} #else #if ( SIZEOF_LONG > 8 ) if((long_value < (long) 0) || (long_value > (long) UINT64_MAX)) {{ #else if(long_value < (PY_LONG_LONG) 0) {{ #endif PyErr_Format(PyExc_ValueError, "Integer object value out of bounds"); return (uint64_t) -1; }} #endif return (uint64_t) long_value; }} """.format(**values_dict) def initialise_class(self, class_name, out, done=None): if done and class_name in done: return done.add(class_name) cls = self.classes[class_name] """Write out class initialisation code into the main init function.""" if cls.is_active(): base_class = self.classes.get(cls.base_class_name) if base_class and base_class.is_active(): # We have a base class - ensure it gets written out # first: self.initialise_class(cls.base_class_name, out, done) # Now assign ourselves as derived from them out.write( " {0:s}_Type.tp_base = &{1:s}_Type;".format( cls.class_name, cls.base_class_name)) values_dict = { "name": cls.class_name} out.write(( "\n" " /* Initialize: {name:s} */\n" " {name:s}_Type.tp_new = PyType_GenericNew;\n").format( **values_dict)) if isinstance(cls, Enum): out.write(( " if ({name:s}_init_type(&{name:s}_Type) != 1) {{\n" " goto on_error;\n" " }}\n").format(**values_dict)) out.write(( " if (PyType_Ready(&{name:s}_Type) < 0) {{\n" " goto on_error;\n" " }}\n" " Py_IncRef((PyObject *)&{name:s}_Type);\n" " PyModule_AddObject(module, \"{name:s}\", (PyObject *)&{name:s}_Type);\n").format( **values_dict)) def write(self, out): # Write the headers if self.public_api: self.public_api.write( "#ifdef BUILDING_DLL\n" "#include \"misc.h\"\n" "#else\n" "#include \"aff4_public.h\"\n" "#endif\n") # Prepare all classes for cls in self.classes.values(): cls.prepare() out.write(( "/*************************************************************\n" " * Autogenerated module {0:s}\n" " *\n" " * This module was autogenerated from the following files:\n").format( self.name)) for filename in self.files: out.write(" * {0:s}\n".format(filename)) out.write( " *\n" " * This module implements the following classes:\n") out.write(self.get_string()) out.write( " ************************************************************/\n" "\n") out.write(self.headers) out.write( "\n" "#ifdef __cplusplus\n" "extern \"C\" {\n" "#endif\n" "\n" "#include \n") out.write(self.private_functions()) for cls in self.classes.values(): if cls.is_active(): out.write( "/******************** {0:s} ***********************/".format( cls.class_name)) cls.struct(out) cls.prototypes(out) out.write( "/*****************************************************\n" " * Implementation\n" " ****************************************************/\n" "\n") for cls in self.classes.values(): if cls.is_active(): cls.PyMethodDef(out) cls.PyGetSetDef(out) cls.PyTypeObject(out) for cls in self.classes.values(): if cls.is_active(): cls.code(out) # Write the module initializer values_dict = { "module": self.name, "version": VERSION, "version_length": len(VERSION)} out.write(( "/* Retrieves the {module:s} version\n" " * Returns a Python object if successful or NULL on error\n" " */\n" "PyObject *{module:s}_get_version(PyObject *self, PyObject *arguments) {{\n" " const char *errors = NULL;\n" " return(PyUnicode_DecodeUTF8(\"{version:s}\", (Py_ssize_t) {version_length:d}, errors));\n" "}}\n" "\n" "static PyMethodDef {module:s}_module_methods[] = {{\n" " {{ \"get_version\",\n" " (PyCFunction) {module:s}_get_version,\n" " METH_NOARGS,\n" " \"get_version() -> String\\n\"\n" " \"\\n\"\n" " \"Retrieves the version.\" }},\n" "\n" " {{NULL, NULL, 0, NULL}} /* Sentinel */\n" "}};\n" "\n" "#if PY_MAJOR_VERSION >= 3\n" "\n" "/* The {module:s} module definition\n" " */\n" "PyModuleDef {module:s}_module_definition = {{\n" " PyModuleDef_HEAD_INIT,\n" "\n" " /* m_name */\n" " \"{module:s}\",\n" " /* m_doc */\n" " \"Python {module:s} module.\",\n" " /* m_size */\n" " -1,\n" " /* m_methods */\n" " {module:s}_module_methods,\n" " /* m_reload */\n" " NULL,\n" " /* m_traverse */\n" " NULL,\n" " /* m_clear */\n" " NULL,\n" " /* m_free */\n" " NULL,\n" "}};\n" "\n" "#endif /* PY_MAJOR_VERSION >= 3 */\n" "\n" "/* Initializes the {module:s} module\n" " */\n" "#if PY_MAJOR_VERSION >= 3\n" "PyMODINIT_FUNC PyInit_{module:s}(void) {{\n" "#else\n" "PyMODINIT_FUNC init{module:s}(void) {{\n" "#endif\n" " PyGILState_STATE gil_state;\n" "\n" " PyObject *module = NULL;\n" " PyObject *d = NULL;\n" " PyObject *tmp = NULL;\n" "\n" " /* Create the module\n" " * This function must be called before grabbing the GIL\n" " * otherwise the module will segfault on a version mismatch\n" " */\n" "#if PY_MAJOR_VERSION >= 3\n" " module = PyModule_Create(\n" " &{module:s}_module_definition );\n" "#else\n" " module = Py_InitModule3(\n" " \"{module:s}\",\n" " {module:s}_module_methods,\n" " \"Python {module:s} module.\" );\n" "#endif\n" " if (module == NULL) {{\n" "#if PY_MAJOR_VERSION >= 3\n" " return(NULL);\n" "#else\n" " return;\n" "#endif\n" " }}\n" " d = PyModule_GetDict(module);\n" "\n" " /* Make sure threads are enabled */\n" "#if PY_VERSION_HEX < 0x03070000\n" " PyEval_InitThreads();\n" "#endif\n" " gil_state = PyGILState_Ensure();\n" "\n" " g_module = module;\n").format(**values_dict)) # The trick is to initialise the classes in order of their # inheritance. The following code will order initializations # according to their inheritance tree done = set() for class_name in self.classes.keys(): self.initialise_class(class_name, out, done) # Add the constants here. Make sure they are sorted so builds # of pytsk3.c are reproducible. for constant, type in sorted(self.constants): if type == "integer": out.write( " tmp = PyLong_FromUnsignedLongLong((uint64_t) {0:s});\n".format(constant)) elif type == "string": if constant == "TSK_VERSION_STR": out.write(( "#if PY_MAJOR_VERSION >= 3\n" " tmp = PyUnicode_FromString((char *){0:s});\n" "#else\n" " tmp = PyString_FromString((char *){0:s});\n" "#endif\n").format(constant)) else: out.write(( "#if PY_MAJOR_VERSION >= 3\n" " tmp = PyBytes_FromString((char *){0:s});\n" "#else\n" " tmp = PyString_FromString((char *){0:s});\n" "#endif\n").format(constant)) else: out.write( " /* I dont know how to convert {0:s} type {1:s} */\n".format( constant, type)) continue out.write(( " PyDict_SetItemString(d, \"{0:s}\", tmp);\n" " Py_DecRef(tmp);\n").format(constant)) out.write(self.initialization()) out.write( " PyGILState_Release(gil_state);\n" "\n" "#if PY_MAJOR_VERSION >= 3\n" " return module;\n" "#else\n" " return;\n" "#endif\n" "\n" "on_error:\n" " PyGILState_Release(gil_state);\n" "\n" "#if PY_MAJOR_VERSION >= 3\n" " return NULL;\n" "#else\n" " return;\n" "#endif\n" "}\n" "\n" "#ifdef __cplusplus\n" "}\n" "#endif\n") class Type(object): interface = None buildstr = "O" sense = "IN" error_value = "return 0;" active = True def __init__(self, name, type, *args, **kwargs): super(Type, self).__init__() self.name = name self.type = type self.attributes = set() self.additional_args = kwargs def comment(self): return "{0:s} {1:s} ".format(self.type, self.name) def get_string(self): """Retrieves a string representation.""" if self.name == "func_return": return self.type if "void" in self.type: return "" return "{0:s} : {1:s}".format(self.type, self.name) def python_name(self): return self.name def python_proxy_post_call(self): """This is called after a proxy call""" return "" def returned_python_definition(self, *arg, **kwargs): return self.definition(*arg, **kwargs) def definition(self, default=None, **kwargs): if default: return " {0:s} {1:s} = {2:s};\n".format( self.type, self.name, default) elif "array_size" in self.additional_args: return ( " int array_index = 0;\n" " {0:s} UNUSED *{1:s};\n").format( self.type, self.name) else: return " {0:s} UNUSED {1:s};\n".format( self.type, self.name) def local_definition(self, default=None, **kwargs): return "" def byref(self): return "&{0:s}".format(self.name) def call_arg(self): return self.name def passthru_call(self): """Returns how we should call the function when simply passing args directly""" return self.call_arg() def pre_call(self, method, **kwargs): return "" def assign(self, call, method, target=None, **kwargs): return ( "Py_BEGIN_ALLOW_THREADS\n" "{0:s} = {1:s};\n" "Py_END_ALLOW_THREADS\n").format( target or self.name, call) def post_call(self, method): # Check for errors result = ( "if(check_error()) {\n" " goto on_error;\n" "}\n") if "DESTRUCTOR" in self.attributes: result += "self->base = NULL; //DESTRUCTOR - C object no longer valid\n" return result def from_python_object(self, source, destination, method, **kwargs): return "" def return_value(self, value): return "return {0!s};".format(value) class String(Type): interface = "string" buildstr = "s" error_value = "return NULL;" def __init__(self, name, type, *args, **kwargs): super(String, self).__init__(name, type, *args, **kwargs) self.length = "strlen({0:s})".format(name) def byref(self): return "&{0:s}".format(self.name) def to_python_object(self, name=None, result="Py_result", **kwargs): values_dict = { "length": self.length, "name": name or self.name, "result": result} result = ( " PyErr_Clear();\n" "\n" " if(!{name:s}) {{\n" " Py_IncRef(Py_None);\n" " {result:s} = Py_None;\n" " }} else {{\n" "#if PY_MAJOR_VERSION >= 3\n" " {result:s} = PyBytes_FromStringAndSize((char *){name:s}, {length:s});\n" "#else\n" " {result:s} = PyString_FromStringAndSize((char *){name:s}, {length:s});\n" "#endif\n" " if(!{result:s}) {{\n" " goto on_error;\n" " }}\n" " }}\n").format(**values_dict) if "BORROWED" not in self.attributes and "BORROWED" not in kwargs: result += "talloc_unlink(NULL, {0:s});\n".format(name) return result def from_python_object(self, source, destination, method, context="NULL"): method.error_set = True values_dict = { "context": context, "destination": destination, "source": source} return ( "{{\n" " char *buff = NULL;\n" " Py_ssize_t length = 0;\n" "\n" " PyErr_Clear();\n" "\n" "#if PY_MAJOR_VERSION >= 3\n" " if(PyBytes_AsStringAndSize({source:s}, &buff, &length) == -1) {{\n" "#else\n" " if(PyString_AsStringAndSize({source:s}, &buff, &length) == -1) {{\n" "#endif\n" " goto on_error;\n" " }}\n" " {destination:s} = talloc_size({context:s}, length + 1);\n" " memcpy({destination:s}, buff, length);\n" " {destination:s}[length] = 0;\n" "}};\n").format(**values_dict) class ZString(String): interface = "null_terminated_string" def definition(self, default=None, **kwargs): if default == "\"\"": default = "(char *) \"\"" return super(ZString, self).definition(default=default, **kwargs) class BorrowedString(String): def to_python_object(self, name=None, result="Py_result", **kwargs): values_dict = { "length": self.length, "name": name or self.name, "result": result} return ( " PyErr_Clear();\n" "#if PY_MAJOR_VERSION >= 3\n" " {result:s} = PyBytes_FromStringAndSize((char *){name:s}, {length:s});\n" "#else\n" " {result:s} = PyString_FromStringAndSize((char *){name:s}, {length:s});\n" "#endif\n").format(**values_dict) class Char_and_Length(Type): interface = "char_and_length" buildstr = "s#" error_value = "return NULL;" def __init__(self, data, data_type, length, length_type, *args, **kwargs): super(Char_and_Length, self).__init__(data, data_type, *args, **kwargs) self.name = data self.data_type = data_type self.length = length self.length_type = length_type def comment(self): return "{0:s} {1:s}, {2:s} {3:s}".format( self.data_type, self.name, self.length_type, self.length) def definition(self, default="\"\"", **kwargs): return ( " char *{0:s} = {1:s};\n" " Py_ssize_t {2:s} = strlen({3:s});\n").format( self.name, default, self.length, default) def byref(self): return "&{0:s}, &{1:s}".format(self.name, self.length) def call_arg(self): return "({0:s}){1:s}, ({2:s}){3:s}".format( self.data_type, self.name, self.length_type, self.length) def to_python_object(self, name=None, result="Py_result", **kwargs): values_dict = { "length": self.length, "name": self.name, "result": result} return ( " PyErr_Clear();\n" "#if PY_MAJOR_VERSION >= 3\n" " {result:s} = PyBytes_FromStringAndSize((char *){name:s}, {length:s});\n" "#else\n" " {result:s} = PyString_FromStringAndSize((char *){name:s}, {length:s});\n" "#endif\n" "\n" " if(!{result:s}) {{\n" " goto on_error;\n" " }}\n").format(**values_dict) class Integer(Type): interface = "integer" buildstr = "i" int_type = "int" def __init__(self, name, type, *args, **kwargs): super(Integer, self).__init__(name, type, *args, **kwargs) self.type = self.int_type self.original_type = type def to_python_object(self, name=None, result="Py_result", **kwargs): values_dict = { "name": name or self.name, "result": result} return ( " PyErr_Clear();\n" "#if PY_MAJOR_VERSION >= 3\n" " {result:s} = PyLong_FromLong({name:s});\n" "#else\n" " {result:s} = PyInt_FromLong({name:s});\n" "#endif\n").format(**values_dict) def from_python_object(self, source, destination, method, **kwargs): values_dict = { "destination": destination, "source": source} return ( " PyErr_Clear();\n" "#if PY_MAJOR_VERSION >= 3\n" " {destination:s} = PyLong_AsLongMask({source:s});\n" "#else\n" " {destination:s} = PyInt_AsLongMask({source:s});\n" "#endif\n").format(**values_dict) def comment(self): return "{0:s} {1:s} ".format(self.original_type, self.name) class IntegerUnsigned(Integer): buildstr = "I" int_type = "unsigned int" def to_python_object(self, name=None, result="Py_result", **kwargs): if "array_size" in self.additional_args: values_dict = { "name": name or self.name, "result": result, "array_size": self.additional_args["array_size"] } return ( " PyErr_Clear();\n" " {result:s} = PyList_New(0);\n" " for(array_index = 0; array_index < {array_size:s}; array_index++) {{\n" "#if PY_MAJOR_VERSION >= 3\n" " PyList_Append({result:s}, PyLong_FromLong((long) {name:s}[array_index]));\n" "#else\n" " PyList_Append({result:s}, PyInt_FromLong((long) {name:s}[array_index]));\n" "#endif\n" " }}\n" ).format(**values_dict) else: values_dict = { "name": name or self.name, "result": result} return ( " PyErr_Clear();\n" "#if PY_MAJOR_VERSION >= 3\n" " {result:s} = PyLong_FromLong((long) {name:s});\n" "#else\n" " {result:s} = PyInt_FromLong((long) {name:s});\n" "#endif\n").format(**values_dict) def from_python_object(self, source, destination, method, **kwargs): values_dict = { "destination": destination, "source": source} return ( " PyErr_Clear();\n" "#if PY_MAJOR_VERSION >= 3\n" " {destination:s} = PyLong_AsUnsignedLongMask({source:s});\n" "#else\n" " {destination:s} = PyInt_AsUnsignedLongMask({source:s});\n" "#endif\n").format(**values_dict) class Integer8(Integer): int_type = "int8_t" class Integer8Unsigned(IntegerUnsigned): int_type = "uint8_t" class Integer16(Integer): int_type = "int16_t" class Integer16Unsigned(IntegerUnsigned): int_type = "uint16_t" class Integer32(Integer): int_type = "int32_t" class Integer32Unsigned(IntegerUnsigned): int_type = "uint32_t" class Integer64(Integer): buildstr = "L" int_type = "int64_t" def to_python_object(self, name=None, result="Py_result", **kwargs): values_dict = { "name": name or self.name, "result": result} return ( " PyErr_Clear();\n" "#if defined( HAVE_LONG_LONG )\n" " {result:s} = PyLong_FromLongLong({name:s});\n" "#else\n" " {result:s} = PyLong_FromLong({name:s});\n" "#endif\n").format(**values_dict) def from_python_object(self, source, destination, method, **kwargs): values_dict = { "destination": destination, "source": source} return ( " PyErr_Clear();\n" "#if PY_MAJOR_VERSION >= 3\n" "#if defined( HAVE_LONG_LONG )\n" " {destination:s} = PyLong_AsLongLongMask({source:s});\n" "#else\n" " {destination:s} = PyLong_AsLongMask({source:s});\n" "#endif\n" "#else\n" "#if defined( HAVE_LONG_LONG )\n" " {destination:s} = PyInt_AsLongLongMask({source:s});\n" "#else\n" " {destination:s} = PyInt_AsLongMask({source:s});\n" "#endif\n" "#endif /* PY_MAJOR_VERSION >= 3 */\n").format(**values_dict) class Integer64Unsigned(Integer): buildstr = "K" int_type = "uint64_t" def to_python_object(self, name=None, result="Py_result", **kwargs): values_dict = { "name": name or self.name, "result": result} return ( " PyErr_Clear();\n" "#if defined( HAVE_LONG_LONG )\n" " {result:s} = PyLong_FromUnsignedLongLong({name:s});\n" "#else\n" " {result:s} = PyLong_FromUnsignedLong({name:s});\n" "#endif\n").format(**values_dict) def from_python_object(self, source, destination, method, **kwargs): values_dict = { "destination": destination, "source": source} # TODO: use integer_object_copy_to_uint64 instead to support both # long and int objects. return ( " PyErr_Clear();\n" "#if PY_MAJOR_VERSION >= 3\n" "#if defined( HAVE_LONG_LONG )\n" " {destination:s} = PyLong_AsUnsignedLongLongMask({source:s});\n" "#else\n" " {destination:s} = PyLong_AsUnsignedLongMask({source:s});\n" "#endif\n" "#else\n" "#if defined( HAVE_LONG_LONG )\n" " {destination:s} = PyInt_AsUnsignedLongLongMask({source:s});\n" "#else\n" " {destination:s} = PyInt_AsUnsignedLongMask({source:s});\n" "#endif\n" "#endif /* PY_MAJOR_VERSION >= 3 */\n").format(**values_dict) class Long(Integer): buildstr = "l" int_type = "long" def to_python_object(self, name=None, result="Py_result", **kwargs): values_dict = { "name": name or self.name, "result": result} return ( "PyErr_Clear();\n" "{result:s} = PyLong_FromLongLong({name:s});\n").format( **values_dict) def from_python_object(self, source, destination, method, **kwargs): values_dict = { "destination": destination, "source": source} return ( "PyErr_Clear();\n" "{destination:s} = PyLong_AsLongMask({source:s});\n").format( **values_dict) class LongUnsigned(Integer): buildstr = "k" int_type = "unsigned long" def to_python_object(self, name=None, result="Py_result", **kwargs): values_dict = { "name": name or self.name, "result": result} return ( "PyErr_Clear();\n" "{result:s} = PyLong_FromUnsignedLong({name:s});\n").format( **values_dict) def from_python_object(self, source, destination, method, **kwargs): values_dict = { "destination": destination, "source": source} return ( "PyErr_Clear();\n" "{destination:s} = PyLong_AsUnsignedLongMask({source:s});\n").format( **values_dict) class Char(Integer): buildstr = "s" interface = "small_integer" def to_python_object(self, name=None, result="Py_result", **kwargs): # We really want to return a string here values_dict = { "name": name or self.name, "result": result} return ( "{{\n" " char *str_{name:s} = &{name:s};\n" "\n" " PyErr_Clear();\n" "#if PY_MAJOR_VERSION >= 3\n" " {result:s} = PyBytes_FromStringAndSize(str_{name:s}, 1);\n" "#else\n" " {result:s} = PyString_FromStringAndSize(str_{name:s}, 1);\n" "#endif\n" "\n" " if(!{result:s}) {{\n" " goto on_error;\n" "}}\n").format(**values_dict) def definition(self, default="\"\\x0\"", **kwargs): # Shut up unused warnings return ( "char {0:s} UNUSED = 0;\n" "char *str_{0:s} UNUSED = {1:s};\n").format( self.name, default) def byref(self): return "&str_{0:s}".format(self.name) def pre_call(self, method, **kwargs): method.error_set = True values_dict = { "name": self.name} return ( " if(strlen(str_{name:s}) != 1) {\n" " PyErr_Format(PyExc_RuntimeError, \"You must only provide a single character for arg {name:s}\");\n" " goto on_error;\n" " }\n" "\n" " {name:s} = str_{name:s}[0];\n").format( **values_dict) class StringOut(String): sense = "OUT" class IntegerOut(Integer): """Handle Integers pushed out through OUT int *result.""" sense = "OUT_DONE" buildstr = "" int_type = "int *" def definition(self, default=0, **kwargs): # We need to make static storage for the pointers storage = "storage_{0:s}".format(self.name) bare_type = self.type.split()[0] type_definition = Type.definition( self, default="&{0:s}".format(storage)) return ( "{0:s} {1:s} = 0;\n" "{2:s}\n").format( bare_type, storage, type_definition) def to_python_object(self, name=None, result="Py_result", **kwargs): values_dict = { "name": name or self.name, "result": result} return ( "PyErr_Clear();\n" "{result:s} = PyLong_FromLongLong(*{name:s});\n").format( **values_dict) def python_name(self): return None def byref(self): return self.name def call_arg(self): return "{0:s}".format(self.name) def passthru_call(self): return self.name class PInteger32UnsignedOut(IntegerOut): buildstr = "" int_type = "uint32_t *" class PInteger64UnsignedOut(IntegerOut): buildstr = "" int_type = "uint64_t *" class Char_and_Length_OUT(Char_and_Length): sense = "OUT_DONE" buildstr = "l" def definition(self, default=0, **kwargs): values_dict = { "default": default, "length": self.length, "name": self.name} return ( " char *{name:s} = NULL;\n" " Py_ssize_t {length:s} = {default:d};\n" " PyObject *tmp_{name:s} = NULL;\n").format( **values_dict) def error_cleanup(self): values_dict = { "name": self.name} return ( " if(tmp_{name:s} != NULL) {{\n" " Py_DecRef(tmp_{name:s});\n" " }}\n").format(**values_dict) def python_name(self): return self.length def byref(self): return "&{0:s}".format(self.length) def pre_call(self, method, **kwargs): values_dict = { "length": self.length, "name": self.name} return ( " PyErr_Clear();\n" "\n" "#if PY_MAJOR_VERSION >= 3\n" " tmp_{name:s} = PyBytes_FromStringAndSize(NULL, {length:s});\n" "#else\n" " tmp_{name:s} = PyString_FromStringAndSize(NULL, {length:s});\n" "#endif\n" " if(!tmp_{name:s}) {{\n" " goto on_error;\n" " }}\n" "\n" "#if PY_MAJOR_VERSION >= 3\n" " PyBytes_AsStringAndSize(tmp_{name:s}, &{name:s}, (Py_ssize_t *)&{length:s});\n" "#else\n" " PyString_AsStringAndSize(tmp_{name:s}, &{name:s}, (Py_ssize_t *)&{length:s});\n" "#endif\n").format(**values_dict) def to_python_object(self, name=None, result="Py_result", sense="in", **kwargs): if "results" in kwargs: kwargs["results"].pop(0) if sense == "proxied": return "py_{0:s} = PyLong_FromLong({1:s});\n".format( self.name, self.length) values_dict = { "length": self.length, "name": name or self.name, "result": result} return ( " /* NOTE - this should never happen\n" " * it might indicate an overflow condition.\n" " */\n" " if(func_return > (uint64_t) {length:s}) {{\n" " printf(\"Programming Error - possible overflow!!\\n\");\n" " abort();\n" "\n" " // Do we need to truncate the buffer for a short read?\n" " }} else if(func_return < (uint64_t) {length:s}) {{\n" "#if PY_MAJOR_VERSION >= 3\n" " _PyBytes_Resize(&tmp_{name:s}, (Py_ssize_t) func_return);\n" "#else\n" " _PyString_Resize(&tmp_{name:s}, (Py_ssize_t) func_return);\n" "#endif\n" " }}\n" "\n" " {result:s} = tmp_{name:s};\n").format(**values_dict) def python_proxy_post_call(self, result="Py_result"): values_dict = { "name": self.name, "result": result} return ( "{{\n" " char *tmp_buff = NULL;\n" " Py_ssize_t tmp_len = 0;\n" "\n" "#if PY_MAJOR_VERSION >= 3\n" " if(PyBytes_AsStringAndSize({result:s}, &tmp_buff, &tmp_len) == -1) {{\n" "#else\n" " if(PyString_AsStringAndSize({result:s}, &tmp_buff, &tmp_len) == -1) {{\n" "#endif\n" " goto on_error;\n" " }}\n" " memcpy({name:s}, tmp_buff, tmp_len);\n" " Py_DecRef({result:s});\n" " {result:s} = PyLong_FromLong(tmp_len);\n" "}}\n").format(**values_dict) class TDB_DATA_P(Char_and_Length_OUT): bare_type = "TDB_DATA" def __init__(self, name, type, *args, **kwargs): super(TDB_DATA_P, self).__init__(name, type, *args, **kwargs) def definition(self, default=None, **kwargs): return Type.definition(self) def byref(self): return "{0:s}.dptr, &{0:s}.dsize".format(self.name) def pre_call(self, method, **kwargs): return "" def call_arg(self): return Type.call_arg(self) def to_python_object(self, name=None, result="Py_result", **kwargs): values_dict = { "name": name or self.name, "result": result} return ( " PyErr_Clear();\n" "#if PY_MAJOR_VERSION >= 3\n" " {result:s} = PyBytes_FromStringAndSize((char *){name:s}->dptr, {name:s}->dsize);\n" "#else\n" " {result:s} = PyString_FromStringAndSize((char *){name:s}->dptr, {name:s}->dsize);\n" "#endif\n" " talloc_free({name:s});\n").format(**values_dict) def from_python_object(self, source, destination, method, **kwargs): method.error_set = True values_dict = { "bare_type": self.bare_type, "destination": destination, "source": source} return ( "{destination:s} = talloc_zero(self, {bare_type:s});\n" "{{\n" " char *buf = NULL;\n" " Py_ssize_t tmp = 0;\n" "\n" " PyErr_Clear();\n" "\n" "#if PY_MAJOR_VERSION >= 3\n" " if(PyBytes_AsStringAndSize({source:s}, &buf, &tmp) == -1) {{\n" "#else\n" " if(PyString_AsStringAndSize({source:s}, &buf, &tmp) == -1) {{\n" "#endif\n" " goto on_error;\n" " }}\n" "\n" " // Take a copy of the Python string\n" " {destination:s}->dptr = talloc_memdup({destination:s}, buf, tmp);\n" " {destination:s}->dsize = tmp;\n" "}}\n" "// We no longer need the Python object\n" "Py_DecRef({source:s});\n").format(**values_dict) class TDB_DATA(TDB_DATA_P): error_value = ( "{result:s}.dptr = NULL;\n" "return {result:s};") def from_python_object(self, source, destination, method, **kwargs): method.error_set = True values_dict = { "destination": destination, "source": source} return ( "{{\n" " char *buf = NULL;\n" " Py_ssize_t tmp = 0;\n" "\n" " PyErr_Clear();\n" "\n" "#if PY_MAJOR_VERSION >= 3\n" " if(PyBytes_AsStringAndSize({source:s}, &buf, &tmp) == -1) {{\n" "#else\n" " if(PyString_AsStringAndSize({source:s}, &buf, &tmp) == -1) {{\n" "#endif\n" " goto on_error;\n" " }}\n" " // Take a copy of the Python string - This leaks - how to fix it?\n" " {destination:s}.dptr = talloc_memdup(NULL, buf, tmp);\n" " {destination:s}.dsize = tmp;\n" "}}\n" "// We no longer need the Python object\n" "Py_DecRef({source:s});\n").format(**values_dict) def to_python_object(self, name=None, result="Py_result", **kwargs): values_dict = { "name": name or self.name, "result": result} return ( " PyErr_Clear();\n" "#if PY_MAJOR_VERSION >= 3\n" " {result:s} = PyBytes_FromStringAndSize((char *){name:s}.dptr, {name:s}.dsize);\n" "#else\n" " {result:s} = PyString_FromStringAndSize((char *){name:s}.dptr, {name:s}.dsize);\n" "#endif\n").format(**values_dict) class Void(Type): buildstr = "" error_value = "return;" original_type = "" def __init__(self, name, type="void", *args, **kwargs): super(Void, self).__init__(name, type, *args, **kwargs) def comment(self): return "void *ctx" def definition(self, default=None, **kwargs): return "" def to_python_object(self, name=None, result="Py_result", **kwargs): return ( "Py_IncRef(Py_None);\n" "Py_result = Py_None;\n") def call_arg(self): return "NULL" def byref(self): return None def assign(self, call, method, target=None, **kwargs): # We don't assign the result to anything. return ( " Py_BEGIN_ALLOW_THREADS\n" " (void) {0:s};\n" " Py_END_ALLOW_THREADS\n").format(call) def return_value(self, value): return "return;" class PVoid(Void): def __init__(self, name, type="void *", *args, **kwargs): super(PVoid, self).__init__(name, type, *args, **kwargs) class StringArray(String): interface = "array" buildstr = "O" def definition(self, default="\"\"", **kwargs): return ( " char **{0:s} = NULL;\n" " PyObject *py_{0:s} = NULL;\n").format(self.name) def byref(self): return "&py_{0:s}".format(self.name) def from_python_object(self, source, destination, method, context="NULL"): method.error_set = True values_dict = { "destination": destination, "source": source} return ( "{{\n" " Py_ssize_t i = 0;\n" " Py_ssize_t size = 0;\n" "\n" " if({source:s}) {{\n" " if(!PySequence_Check({source:s})) {{\n" " PyErr_Format(PyExc_ValueError, \"{destination:s} must be a sequence\");\n" " goto on_error;\n" " }}\n" " size = PySequence_Size({source:s});\n" " }}\n" " {destination:s} = talloc_zero_array(NULL, char *, size + 1);\n" "\n" " for(i = 0; i < size; i++) {{\n" " PyObject *tmp = PySequence_GetItem({source:s}, i);\n" " if(!tmp) {{\n" " goto on_error;\n" " }}\n" "#if PY_MAJOR_VERSION >= 3\n" " {destination:s}[i] = PyBytes_AsString(tmp);\n" "#else\n" " {destination:s}[i] = PyString_AsString(tmp);\n" "#endif\n" "\n" " if(!{destination:s}[i]) {{\n" " Py_DecRef(tmp);\n" " goto on_error;\n" " }}\n" " Py_DecRef(tmp);\n" " }}\n" "}}\n").format(**values_dict) def pre_call(self, method, **kwargs): return self.from_python_object( "py_{0:s}".format(self.name), self.name, method) def error_condition(self): return ( " if({0:s}) {{\n" " talloc_free({0:s});\n" " }}\n").format(self.name) class Wrapper(Type): """This class represents a wrapped C type """ sense = "IN" error_value = "return NULL;" def from_python_object(self, source, destination, method, **kwargs): values_dict = { "destination": destination, "source": source, "type": self.type} return ( " /* First check that the returned value is in fact a Wrapper */\n" " if(!type_check({source:s}, &{type:s}_Type)) {{\n" " PyErr_Format(PyExc_RuntimeError, \"function must return an {type:s} instance\");\n" " goto on_error;\n" " }}\n" "\n" " {destination:s} = ({type:s}) ((Gen_wrapper) {source:s})->base;\n" "\n" " if(!{destination:s}) {{\n" " PyErr_Format(PyExc_RuntimeError, \"{type:s} instance is no longer valid (was it gc'ed?)\");\n" " goto on_error;\n" "}}\n" "\n").format(**values_dict) def to_python_object(self, **kwargs): return "" def returned_python_definition(self, default="NULL", sense="in", **kwargs): return "{0:s} {1:s} = {2:s};\n".format( self.type, self.name, default) def byref(self): return "&wrapped_{0:s}".format(self.name) def definition(self, default="NULL", sense="in", **kwargs): result = " Gen_wrapper wrapped_{0:s} UNUSED = {1:s};\n".format( self.name, default) if sense == "in" and not "OUT" in self.attributes: result += " {0:s} UNUSED {1:s};\n".format( self.type, self.name) return result def call_arg(self): return "{0:s}".format(self.name) def pre_call(self, method, python_object_index=1, **kwargs): if "OUT" in self.attributes or self.sense == "OUT": return "" self.original_type = self.type.split()[0] values_dict = { "name": self.name, "original_type": self.original_type, "python_object_index": python_object_index, "type": self.type} return ( " if(wrapped_{name:s} == NULL || (PyObject *)wrapped_{name:s} == Py_None) {{\n" " {name:s} = NULL;\n" " }} else if(!type_check((PyObject *)wrapped_{name:s},&{original_type:s}_Type)) {{\n" " PyErr_Format(PyExc_RuntimeError, \"{name:s} must be derived from type {original_type:s}\");\n" " goto on_error;\n" " }} else if(wrapped_{name:s}->base == NULL) {{\n" " PyErr_Format(PyExc_RuntimeError, \"{original_type:s} instance is no longer valid (was it gc'ed?)\");\n" " goto on_error;\n" " }} else {{\n" " {name:s} = ({type:s}) wrapped_{name:s}->base;\n" " if(self->python_object{python_object_index:d} == NULL) {{\n" " self->python_object{python_object_index:d} = (PyObject *) wrapped_{name:s};\n" " Py_IncRef(self->python_object{python_object_index:d});\n" " }}\n" " }}\n").format(**values_dict) def assign(self, call, method, target=None, **kwargs): method.error_set = True; values_dict = { "call": call.strip(), "incref": INCREF, "name": target or self.name, "type": self.type} result = ( " {{\n" " Object returned_object = NULL;\n" "\n" " ClearError();\n" "\n" " Py_BEGIN_ALLOW_THREADS\n" " // This call will return a Python object if the base is a proxied Python object\n" " // or a talloc managed object otherwise.\n" " returned_object = (Object) {call:s};\n" " Py_END_ALLOW_THREADS\n" "\n" " if(check_error()) {{\n" " if(returned_object != NULL) {{\n" " if(self->base_is_python_object != 0) {{\n" " Py_DecRef((PyObject *) returned_object);\n" " }} else if(self->base_is_internal != 0) {{\n" " talloc_free(returned_object);\n" " }}\n" " }}\n" " goto on_error;\n" " }}\n").format(**values_dict) # Is NULL an acceptable return type? In some Python code NULL # can be returned (e.g. in iterators) but usually it should # be converted to Py_None. if "NULL_OK" in self.attributes: result += ( " if(returned_object == NULL) {\n" " goto on_error;\n" " }\n") result += ( " wrapped_{name:s} = new_class_wrapper(returned_object, self->base_is_python_object);\n" "\n" " if(wrapped_{name:s} == NULL) {{\n" " if(returned_object != NULL) {{\n" " if(self->base_is_python_object != 0) {{\n" " Py_DecRef((PyObject *) returned_object);\n" " }} else if(self->base_is_internal != 0) {{\n" " talloc_free(returned_object);\n" " }}\n" " }}\n" " goto on_error;\n" " }}\n").format(**values_dict) if "BORROWED" in self.attributes: result += ( " #error unchecked BORROWED code segment\n" " {incref:s}(wrapped_{name:s}->base);\n" " if(((Object) wrapped_{name:s}->base)->extension) {{\n" " Py_IncRef((PyObject *) ((Object) wrapped_{name:s}->base)->extension);\n" " }}\n").format(**values_dict) result += ( " }\n") return result def to_python_object( self, name=None, result="Py_result", sense="in", **kwargs): values_dict = { "name": name or self.name, "result": result} if sense == "proxied": return ( "{result:s} = (PyObject *) new_class_wrapper((Object){name:s}, 0);\n").format( **values_dict) return "{result:s} = (PyObject *) wrapped_{name:s};\n".format( **values_dict) class PointerWrapper(Wrapper): """A pointer to a wrapped class """ def __init__(self, name, type, *args, **kwargs): type = type.split()[0] super(PointerWrapper, self).__init__(name, type, *args, **kwargs) def comment(self): return "{0:s} *{1:s}".format(self.type, self.name) def definition(self, default="NULL", sense="in", **kwargs): result = " Gen_wrapper wrapped_{0:s} = {1:s};\n".format( self.name, default) if sense == "in" and not "OUT" in self.attributes: result += " {0:s} *{1:s};\n".format(self.type, self.name) return result def byref(self): return "&wrapped_{0:s}".format(self.name) def pre_call(self, method, **kwargs): if "OUT" in self.attributes or self.sense == "OUT": return "" self.original_type = self.type.split()[0] values_dict = { "name": self.name, "original_type": self.original_type} return ( "if(!wrapped_{name:s} || (PyObject *)wrapped_{name:s}==Py_None) {{\n" " {name:s} = NULL;\n" "}} else if(!type_check((PyObject *)wrapped_{name:s},&{original_type:s}_Type)) {{\n" " PyErr_Format(PyExc_RuntimeError, \"{name:s} must be derived from type {original_type:s}\");\n" " goto on_error;\n" "}} else {{\n" " {name:s} = ({original_type:s} *)&wrapped_{name:s}->base;\n" "}};\n").format(**values_dict) class StructWrapper(Wrapper): """A wrapper for struct classes """ active = False def __init__(self, name, type, *args, **kwargs): super(StructWrapper, self).__init__(name, type, *args, **kwargs) self.original_type = type.split()[0] def assign(self, call, method, target=None, borrowed=True, **kwargs): self.original_type = self.type.split()[0] values_dict = { "call": call.strip(), "name": target or self.name, "type": self.original_type} result = ( "\n" " PyErr_Clear();\n" "\n" " wrapped_{name:s} = (Gen_wrapper) PyObject_New(py{type:s}, &{type:s}_Type);\n" "\n").format(**values_dict) if borrowed: result += ( " // Base is borrowed from another object.\n" " wrapped_{name:s}->base = {call:s};\n" " wrapped_{name:s}->base_is_python_object = 0;\n" " wrapped_{name:s}->base_is_internal = 0;\n" " wrapped_{name:s}->python_object1 = NULL;\n" " wrapped_{name:s}->python_object2 = NULL;\n" "\n").format(**values_dict) else: result += ( " wrapped_{name:s}->base = {call:s};\n" " wrapped_{name:s}->base_is_python_object = 0;\n" " wrapped_{name:s}->base_is_internal = 1;\n" " wrapped_{name:s}->python_object1 = NULL;\n" " wrapped_{name:s}->python_object2 = NULL;\n" "\n").format(**values_dict) if "NULL_OK" in self.attributes: result += ( " if(wrapped_{name:s}->base == NULL) {{\n" " Py_DecRef((PyObject *) wrapped_{name:s});\n" " return NULL;\n" " }}\n").format(**values_dict) result += ( " // A NULL object gets translated to a None\n" " if(wrapped_{name:s}->base == NULL) {{\n" " Py_DecRef((PyObject *) wrapped_{name:s});\n" " Py_IncRef(Py_None);\n" " wrapped_{name:s} = (Gen_wrapper) Py_None;\n" " }}\n").format(**values_dict) # TODO: with the following code commented out is makes no sense to have the else clause here. # " }} else {{\n").format(**values_dict) # if "FOREIGN" in self.attributes: # result += "// Not taking references to foreign memory\n" # elif "BORROWED" in self.attributes: # result += "talloc_reference({name:s}->ctx, {name:s}->base);\n".format(**values_dict) # else: # result += "talloc_steal({name:s}->ctx, {name:s}->base);\n".format(**values_dict) # result += "}}\n" return result def byref(self): return "&{0:s}".format(self.name) def definition(self, default="NULL", sense="in", **kwargs): result = " Gen_wrapper wrapped_{0:s} = {1:s};\n".format( self.name, default) if sense == "in" and not "OUT" in self.attributes: result += " {0:s} *{1:s} = NULL;\n".format( self.original_type, self.name) return result; class PointerStructWrapper(StructWrapper): def from_python_object(self, source, destination, method, **kwargs): return "{0:s} = ({1:s} *) ((Gen_wrapper) {2:s})->base;\n".format( destination, self.original_type, source) def byref(self): return "&wrapped_{0:s}".format(self.name) class Timeval(Type): """Handle struct timeval values.""" interface = "numeric" buildstr = "f" def definition(self, default=None, **kwargs): return ( "struct timeval {0:s};\n".format(self.name) + self.local_definition(default=default, **kwargs)) def local_definition(self, default=None, **kwargs): return "float {0:s}_flt;\n".format(self.name) def byref(self): return "&{0:s}_flt".format(self.name) def pre_call(self, method, **kwargs): return ( "{0:s}.tv_sec = (int){0:s}_flt;\n" "{0:s}.tv_usec = ({0:s}_flt - {0:s}.tv_sec) * 1e6;\n").format( self.name) def to_python_object(self, name=None, result="Py_result", **kwargs): values_dict = { "name": name or self.name, "result": result} return ( "{name:s}_flt = (double)({name:s}.tv_sec) + {name:s}.tv_usec;\n" "{result:s} = PyFloat_FromDouble({name:s}_flt);\n").format( **values_dict) class PyObject(Type): """Accept an opaque Python object.""" interface = "opaque" buildstr = "O" def definition(self, default="NULL", **kwargs): self.default = default values_dict = { "default": self.default, "name": self.name} return ( "PyObject *{name:s} = {default:s};\n").format( **values_dict) def byref(self): return "&{0:s}".format(self.name) type_dispatcher = { "IN unsigned char *": String, "IN char *": String, "unsigned char *": String, "char *": String, "ZString": ZString, "OUT unsigned char *": StringOut, "OUT char *": StringOut, "OUT uint64_t *": PInteger64UnsignedOut, "OUT uint32_t *": PInteger32UnsignedOut, "void *": PVoid, "void": Void, "TDB_DATA *": TDB_DATA_P, "TDB_DATA": TDB_DATA, "TSK_INUM_T": Integer, "off_t": Integer64, "size_t": Integer64Unsigned, "ssize_t": Integer64, "time_t": Integer64, "unsigned long": LongUnsigned, "long": Long, "unsigned long int": LongUnsigned, "long int": Integer, "unsigned int": Integer, "int": Integer, "uint64_t": Integer64Unsigned, "uint32_t": Integer32Unsigned, "uint16_t": Integer16Unsigned, "uint8_t": Integer8Unsigned, "int64_t": Integer64, "int32_t": Integer32, "int16_t": Integer16, "int8_t": Integer8, "char": Char, "struct timeval": Timeval, "char **": StringArray, "PyObject *": PyObject, } method_attributes = ["BORROWED", "DESTRUCTOR", "IGNORE"] class ResultException(object): value = 0 exception = "PyExc_IOError" def __init__(self, check, exception, message): self.check = check self.exception = exception self.message = message def write(self, out): out.write(( "\n" "/* Handle exceptions */\n" "if({0:s}) {{\n" " PyErr_Format(PyExc_{1:s}, {2:s});\n" " goto on_error;\n" "}}\n" "\n").format(self.check, self.exception, self.message)) class Method(object): default_re = re.compile("DEFAULT\(([A-Z_a-z0-9]+)\) =(.+);") exception_re = re.compile("RAISES\(([^,]+),\s*([^\)]+)\) =(.+);") typedefed_re = re.compile(r"struct (.+)_t \*") def __init__( self, class_name, base_class_name, name, args, return_type, myclass=None): if not isinstance(myclass, ClassGenerator): raise RuntimeError("myclass must be a class generator") self.args = [] self.base_class_name = base_class_name self.class_name = class_name self.defaults = {} self.definition_class_name = class_name self.docstring = "" self.error_set = False self.exception = None self.name = name self.myclass = myclass for type, name in args: self.add_arg(type, name) try: self.return_type = dispatch("func_return", return_type) self.return_type.attributes.add("OUT") self.return_type.original_type = return_type except KeyError: # Is it a wrapped type? if return_type: log("Unable to handle return type {0:s}.{1:s} {2:s}".format( self.class_name, self.name, return_type)) # pdb.set_trace() self.return_type = PVoid("func_return") def get_string(self): """Retrieves a string representation.""" return "def {0:s} {1:s}({2:s}):".format( self.return_type.get_string(), self.name, " , ".join([a.get_string() for a in self.args])) def clone(self, new_class_name): self.find_optional_vars() result = self.__class__( new_class_name, self.base_class_name, self.name, [], "void *", myclass=self.myclass) result.args = self.args result.return_type = self.return_type result.definition_class_name = self.definition_class_name result.defaults = self.defaults result.exception = self.exception return result def find_optional_vars(self): for line in self.docstring.splitlines(): m = self.default_re.search(line) if m: name = m.group(1) value = m.group(2) log("Setting default value for {0:s} of {1:s}".format( m.group(1), m.group(2))) self.defaults[name] = value.strip() m = self.exception_re.search(line) if m: self.exception = ResultException( m.group(1), m.group(2), m.group(3)) def write_local_vars(self, out): self.find_optional_vars() # We do it in two passes - first mandatory then optional kwlist = " const char *kwlist[] = {" # Mandatory for type in self.args: python_name = type.python_name() if python_name and python_name not in self.defaults: kwlist += "\"{0:s}\", ".format(python_name) for type in self.args: python_name = type.python_name() if python_name and python_name in self.defaults: kwlist += "\"{0:s}\", ".format(python_name) kwlist += " NULL};\n" for type in self.args: out.write( " // DEBUG: local arg type: {0:s}\n".format( type.__class__.__name__)) python_name = type.python_name() try: out.write(type.definition(default=self.defaults[python_name])) except KeyError: out.write(type.definition()) # Make up the format string for the parse args in two pases parse_line = "" for type in self.args: python_name = type.python_name() if type.buildstr and python_name not in self.defaults: parse_line += type.buildstr optional_args = "" for type in self.args: python_name = type.python_name() if type.buildstr and python_name in self.defaults: optional_args += type.buildstr if optional_args: parse_line += "|" + optional_args # Iterators have a different prototype and do not need to # unpack any args if not "iternext" in self.name: # Now parse the args from Python objects out.write("\n") out.write(kwlist) out.write(( "\n" " if(!PyArg_ParseTupleAndKeywords(args, kwds, \"{0:s}\", ").format( parse_line)) tmp = ["(char **) kwlist"] for type in self.args: ref = type.byref() if ref: tmp.append(ref) out.write(", ".join(tmp)) self.error_set = True out.write( ")) {\n" " goto on_error;\n" " }\n") def error_condition(self): result = "" if "DESTRUCTOR" in self.return_type.attributes: result += "self->base = NULL;\n" if hasattr(self, "args"): for type in self.args: if hasattr(type, "error_cleanup"): result += type.error_cleanup() result += " return NULL;\n"; return result def write_definition(self, out): out.write( "\n" "/********************************************************\n" "Autogenerated wrapper for function:\n") out.write(self.comment()) out.write("********************************************************/\n") self._prototype(out) out.write(( "{{\n" " PyObject *returned_result = NULL;\n" " PyObject *Py_result = NULL;\n" "\n" " // DEBUG: return type: {0:s}\n" " ").format( self.return_type.__class__.__name__)) out.write(self.return_type.definition()) self.write_local_vars(out) values_dict = { "class_name": self.class_name, "method": self.name} out.write(( "\n" " // Make sure that we have something valid to wrap\n" " if(self->base == NULL) {{\n" " return PyErr_Format(PyExc_RuntimeError, \"{class_name:s} object no longer valid\");\n" " }}\n" "\n").format(**values_dict)) # Precall preparations out.write(" // Precall preparations\n") out.write(self.return_type.pre_call(self)) for type in self.args: out.write(type.pre_call(self)) values_dict = { "class_name": self.class_name, "def_class_name": self.definition_class_name, "method": self.name} out.write(( " // Check the function is implemented\n" " {{\n" " void *method = (void *) (({def_class_name:s}) self->base)->{method:s};\n" "\n" " if(method == NULL || (void *) unimplemented == (void *) method) {{\n" " PyErr_Format(PyExc_RuntimeError, \"{class_name:s}.{method:s} is not implemented\");\n" " goto on_error;\n" " }}\n" "\n" " // Make the call\n" " ClearError();\n").format(**values_dict)) base = "(({0:s}) self->base)".format(self.definition_class_name) call = " {0:s}->{1:s}({2:s}".format(base, self.name, base) tmp = "" for argument in self.args: call_arg = argument.call_arg() if isinstance(argument, EnumType): tmp += ", ({0:s}) {1:s}".format(argument.type, call_arg) else: tmp += ", {0:s}".format(call_arg) call += "{0:s})".format(tmp) # Now call the wrapped function out.write(self.return_type.assign(call, self, borrowed=False)) if self.exception: self.exception.write(out) self.error_set = True out.write( " };\n" "\n" " // Postcall preparations\n") # Postcall preparations post_calls = [] post_call = self.return_type.post_call(self) post_calls.append(post_call) out.write(" {0:s}".format(post_call)) for type in self.args: post_call = type.post_call(self) if post_call not in post_calls: post_calls.append(post_call) out.write(" {0:s}".format(post_call)) # Now assemble the results results = [self.return_type.to_python_object()] for type in self.args: if type.sense == "OUT_DONE": results.append(type.to_python_object(results=results)) # If all the results are returned by reference we dont need # to prepend the void return value at all. if isinstance(self.return_type, Void) and len(results) > 1: results.pop(0) out.write( "\n" " // prepare results\n") # Make a tuple of results and pass them back if len(results) > 1: out.write("returned_result = PyList_New(0);\n") for result in results: out.write(result) out.write( "PyList_Append(returned_result, Py_result);\n" "Py_DecRef(Py_result);\n") out.write("return returned_result;\n") else: out.write(results[0]) # This useless code removes compiler warnings out.write( " returned_result = Py_result;\n" " return returned_result;\n") # Write the error part of the function if self.error_set: out.write(( "\n" "on_error:\n" "{0:s}").format(self.error_condition())) out.write("};\n\n") def add_arg(self, type, name): try: t = type_dispatcher[type](name, type) except KeyError: # Sometimes types must be typedefed in advance try: m = self.typedefed_re.match(type) type = m.group(1) log("Trying {0:s} for {1:s}".format(type, m.group(0))) t = type_dispatcher[type](name, type) except (KeyError, AttributeError): log("Unable to handle type {0:s}.{1:s} {2:s}".format( self.class_name, self.name, type)) return # Here we collapse char * + int type interfaces into a # coherent string like interface. try: previous = self.args[-1] if t.interface == "integer" and previous.interface == "string": # We make a distinction between IN variables and OUT # variables if previous.sense == "OUT": cls = Char_and_Length_OUT else: cls = Char_and_Length cls = cls( previous.name, previous.type, name, type) self.args[-1] = cls return except IndexError: pass self.args.append(t) def comment(self): args = [] for type in self.args: args.append(type.comment()) return "{0:s} {1:s}.{2:s}({3:s});\n".format( self.return_type.original_type, self.class_name, self.name, ", ".join(args)) def prototype(self, out): self._prototype(out) out.write(";\n") def _prototype(self, out): values_dict = { "class_name": self.class_name, "method": self.name} out.write( "static PyObject *py{class_name:s}_{method:s}(py{class_name:s} *self, PyObject *args, PyObject *kwds)".format( **values_dict)) def PyMethodDef(self, out): docstring = self.comment() + "\n\n" + self.docstring.strip() values_dict = { "class_name": self.class_name, "docstring": format_as_docstring(docstring), "name": self.name} out.write(( " {{ \"{name:s}\",\n" " (PyCFunction) py{class_name:s}_{name:s},\n" " METH_VARARGS|METH_KEYWORDS,\n" " \"{docstring:s}\" }},\n" "\n").format(**values_dict)) class IteratorMethod(Method): """A method which implements an iterator.""" def __init__(self, *args, **kwargs): super(IteratorMethod, self).__init__(*args, **kwargs) # Tell the return type that a NULL Python return is ok self.return_type.attributes.add("NULL_OK") def get_string(self): """Retrieves a string representation.""" return "Iterator returning {0:s}.".format(self.return_type.get_string()) def _prototype(self, out): values_dict = { "class_name": self.class_name, "method": self.name} out.write( "static PyObject *py{class_name:s}_{method:s}(py{class_name:s} *self)".format( **values_dict)) def PyMethodDef(self, out): # This method should not go in the method table as its linked # in directly. pass class SelfIteratorMethod(IteratorMethod): def write_definition(self, out): out.write( "\n" "/********************************************************\n" " * Autogenerated wrapper for function:\n") out.write(self.comment()) out.write( "********************************************************/\n") self._prototype(out) values_dict = { "class_name": self.class_name, "method": self.name} out.write(( "{{\n" " (({class_name:s}) self->base)->{method:s}(({class_name:s}) self->base);\n" " return PyObject_SelfIter((PyObject *) self);\n" "}}\n").format(**values_dict)) class ConstructorMethod(Method): # Python constructors are a bit different than regular methods def _prototype(self, out): values_dict = { "class_name": self.class_name, "method": self.name} out.write( "static int py{class_name:s}_init(py{class_name:s} *self, PyObject *args, PyObject *kwds)\n".format( **values_dict)) def prototype(self, out): self._prototype(out) values_dict = { "class_name": self.class_name} out.write(( ";\n" "static void py{class_name:s}_initialize_proxies(py{class_name:s} *self, void *item);\n").format( **values_dict)) def write_destructor(self, out): values_dict = { "class_name": self.class_name, "free": FREE} out.write(( "static void {class_name:s}_dealloc(py{class_name:s} *self) {{\n" " struct _typeobject *ob_type = NULL;\n" "\n" " if(self != NULL) {{\n" " if(self->base != NULL) {{\n" " if(self->base_is_python_object != 0) {{\n" " Py_DecRef((PyObject*) self->base);\n" " }} else if(self->base_is_internal != 0) {{\n" " {free:s}(self->base);\n" " }}\n" " self->base = NULL;\n" " }}\n" " if(self->python_object2 != NULL) {{\n" " Py_DecRef(self->python_object2);\n" " self->python_object2 = NULL;\n" " }}\n" " if(self->python_object1 != NULL) {{\n" " Py_DecRef(self->python_object1);\n" " self->python_object1 = NULL;\n" " }}\n" " ob_type = Py_TYPE(self);\n" " if(ob_type != NULL && ob_type->tp_free != NULL) {{\n" " ob_type->tp_free((PyObject*) self);\n" " }}\n" " }}\n" "}}\n" "\n").format(**values_dict)) def error_condition(self): return " return -1;"; def initialise_proxies(self, out): self.myclass.module.function_definitions.add( "py{0:s}_initialize_proxies".format(self.class_name)) values_dict = { "class_name": self.class_name} out.write(( "static void py{class_name:s}_initialize_proxies(py{class_name:s} *self, void *item) {{\n" " {class_name:s} target = ({class_name:s}) item;\n" "\n" " /* Maintain a reference to the Python object\n" " * in the C object extension\n" " */\n" " ((Object) item)->extension = self;\n" "\n").format(**values_dict)) # Install proxies for all the method in the current class. for method in self.myclass.module.classes[self.class_name].methods: if method.name.startswith("_"): continue # Since the SleuthKit uses close method also for freeing it needs # to be handled separately to prevent the C/C++ code calling back # into a garbage collected Python object. For close we keep the # default implementation and have its destructor deal with # correctly closing the SleuthKit object. if method.name != "close": values_dict = { "class_name": method.class_name, "definition_class_name": method.definition_class_name, "name": method.name, "proxied_name": method.proxied.get_name()} out.write(( " if(check_method_override((PyObject *) self, &{class_name:s}_Type, \"{name:s}\")) {{\n" " // Proxy the {name:s} method\n" " (({definition_class_name:s}) target)->{name:s} = {proxied_name:s};\n" " }}\n").format(**values_dict)) out.write("}\n\n") def write_definition(self, out): self.initialise_proxies(out) self._prototype(out) values_dict = { "class_name": self.class_name, "definition_class_name": self.definition_class_name} out.write(( "{{\n" " {class_name:s} result_constructor = NULL;\n").format( **values_dict)) # pdb.set_trace() self.write_local_vars(out) # Assign the initialise_proxies handler out.write(( " self->python_object1 = NULL;\n" " self->python_object2 = NULL;\n" "\n" " /* Initialise is used to keep a reference on the object?\n" " * If not called no longer valid warnings have been seen\n" " * on Windows.\n" " */\n" " self->initialise = (function_initialize_Gen_wrapper_t) py{class_name:s}_initialize_proxies;\n" "\n").format(**values_dict)) # Precall preparations python_object_index = 1 for type in self.args: out.write(type.pre_call( self, python_object_index=python_object_index)) python_object_index += 1 # Now call the wrapped function out.write(( " ClearError();\n" "\n" " /* Allocate a new instance */\n" " self->base = ({class_name:s}) alloc_{class_name:s}();\n" " self->base_is_python_object = 0;\n" " self->base_is_internal = 1;\n" " self->object_is_proxied = 0;\n" "\n" " /* Update the target by replacing its methods with proxies\n" " * to call back into Python\n" " */\n" " py{class_name:s}_initialize_proxies(self, self->base);\n" "\n" " /* Now call the constructor */\n" " Py_BEGIN_ALLOW_THREADS\n" " result_constructor = CONSTRUCT_INITIALIZE({class_name:s}, {definition_class_name:s}, Con, self->base").format( **values_dict)) tmp = "" for argument in self.args: call_arg = argument.call_arg() if isinstance(argument, EnumType): tmp += ", ({0:s}) {1:s}".format(argument.type, call_arg) else: tmp += ", {0:s}".format(call_arg) self.error_set = True out.write(tmp) out.write(( ");\n" " Py_END_ALLOW_THREADS\n" "\n" " if(!CheckError(EZero)) {{\n" " char *buffer = NULL;\n" " PyObject *exception = resolve_exception(&buffer);\n" "\n" " PyErr_Format(exception, \"%s\", buffer);\n" " ClearError();\n" " goto on_error;\n" " }}\n" " if(result_constructor == NULL) {{\n" " PyErr_Format(PyExc_IOError, \"Unable to construct class {class_name:s}\");\n" " goto on_error;\n" " }}\n" "\n" " return 0;\n").format(**values_dict)) # Write the error part of the function. if self.error_set: out.write(( "\n" "on_error:\n" " if(self->python_object2 != NULL) {{\n" " Py_DecRef(self->python_object2);\n" " self->python_object2 = NULL;\n" " }}\n" " if(self->python_object1 != NULL) {{\n" " Py_DecRef(self->python_object1);\n" " self->python_object1 = NULL;\n" " }}\n" " if(self->base != NULL) {{\n" " talloc_free(self->base);\n" " self->base = NULL;\n" " }}\n" "{0:s}\n").format(self.error_condition())) out.write("}\n\n") class GetattrMethod(Method): def __init__(self, class_name, base_class_name, myclass): # Cannot use super here due to certain logic in Method.__init__(). self._attributes = [] self.base_class_name = base_class_name self.class_name = class_name self.error_set = True self.myclass = myclass self.name = "" self.return_type = Void("") self.rename_class_name(class_name) def get_string(self): """Retrieves a string representation.""" result = "" for class_name, attr in self.get_attributes(): result += " {0:s}\n".format(attr.get_string()) return result def add_attribute(self, attr): if attr.name: self._attributes.append([self.class_name, attr]) def rename_class_name(self, new_name): """This allows us to rename the class_name at a later stage. Required for late initialization of Structs whose name is not know until much later on. """ # TODO fix this behavior, new_name can be None but it is unclear what # the behavious should be. Python 3 requires the values to be set to # string types. if not new_name: self.class_name = "" self.name = "" else: self.class_name = new_name self.name = "py{0:s}_getattr".format(new_name) for attribure in self._attributes: attribure[0] = new_name def get_attributes(self): for class_name, attr in self._attributes: try: # If its not an active struct, skip it if (not type_dispatcher[attr.type].active and not attr.type in self.myclass.module.active_structs): continue except KeyError: pass yield class_name, attr def clone(self, class_name): result = self.__class__(class_name, self.base_class_name, self.myclass) result._attributes = self._attributes[:] return result def prototype(self, out): if not self.name: return values_dict = { "class_name": self.class_name, "name": self.name} # Define getattr. out.write( "static PyObject *{name:s}(py{class_name:s} *self, PyObject *name);\n".format( **values_dict)) # Define getters. for _, attr in self.get_attributes(): values_dict = { "class_name": self.class_name, "name": attr.name} out.write( "PyObject *py{class_name:s}_{name:s}_getter(py{class_name:s} *self, PyObject *arguments);\n".format( **values_dict)) def built_ins(self, out): """Check for some built in attributes we need to support.""" out.write( " if(strcmp(name, \"__members__\") == 0) {\n" " PyMethodDef *i = NULL;\n" " PyObject *list_object = NULL;\n" " PyObject *string_object = NULL;\n" "\n" " list_object = PyList_New(0);\n" " if(list_object == NULL) {\n" " goto on_error;\n" " }\n" "\n") # Add attributes for class_name, attr in self.get_attributes(): values_dict = { "name": attr.name} out.write(( "#if PY_MAJOR_VERSION >= 3\n" " string_object = PyUnicode_FromString(\"{name:s}\");\n" "#else\n" " string_object = PyString_FromString(\"{name:s}\");\n" "#endif\n" " PyList_Append(list_object, string_object);\n" " Py_DecRef(string_object);\n" "\n").format(**values_dict)) # Add methods out.write(( "\n" " for(i = {0:s}_methods; i->ml_name; i++) {{\n" "#if PY_MAJOR_VERSION >= 3\n" " string_object = PyUnicode_FromString(i->ml_name);\n" "#else\n" " string_object = PyString_FromString(i->ml_name);\n" "#endif\n" " PyList_Append(list_object, string_object);\n" " Py_DecRef(string_object);\n" " }}\n" "#if PY_MAJOR_VERSION >= 3\n" " if( utf8_string_object != NULL ) {{\n" " Py_DecRef(utf8_string_object);\n" " }}\n" "#endif\n" " return list_object;\n" " }}\n").format(self.class_name)) def write_definition(self, out): if not self.name: return values_dict = { "class_name": self.class_name, "name": self.name} out.write(( "static PyObject *py{class_name:s}_getattr(py{class_name:s} *self, PyObject *pyname) {{\n" " PyObject *result = NULL;\n" " char *name = NULL;\n" "\n" "#if PY_MAJOR_VERSION >= 3\n" " PyObject *utf8_string_object = NULL;\n" "#endif\n" "\n" " // Try to hand it off to the Python native handler first\n" " result = PyObject_GenericGetAttr((PyObject*) self, pyname);\n" "\n" " if(result) {{\n" " return result;\n" " }}\n" "\n" " PyErr_Clear();\n" " // No - nothing interesting was found by python\n" "#if PY_MAJOR_VERSION >= 3\n" " utf8_string_object = PyUnicode_AsUTF8String(pyname);\n" "\n" " if(utf8_string_object != NULL) {{\n" " name = PyBytes_AsString(utf8_string_object);\n" " }}\n" "#else\n" " name = PyString_AsString(pyname);\n" "#endif\n" "\n" " if(!self->base) {{\n" "#if PY_MAJOR_VERSION >= 3\n" " if( utf8_string_object != NULL ) {{\n" " Py_DecRef(utf8_string_object);\n" " }}\n" "#endif\n" " return PyErr_Format(PyExc_RuntimeError, \"Wrapped object ({class_name:s}.{name:s}) no longer valid\");\n" " }}\n" " if(!name) {{\n" " goto on_error;\n" " }}\n").format(**values_dict)) self.built_ins(out) out.write( "\n" "#if PY_MAJOR_VERSION >= 3\n" " if( utf8_string_object != NULL ) {{\n" " Py_DecRef(utf8_string_object);\n" " }}\n" "#endif\n" " return PyObject_GenericGetAttr((PyObject *) self, pyname);\n") # Write the error part of the function. if self.error_set: out.write( "on_error:\n" "#if PY_MAJOR_VERSION >= 3\n" " if( utf8_string_object != NULL ) {{\n" " Py_DecRef(utf8_string_object);\n" " }}\n" "#endif\n" + self.error_condition()) out.write("}\n\n") self.write_definition_getters(out) def write_definition_getters(self, out): for _, attr in self.get_attributes(): if self.base_class_name: call = "((({0:s}) self->base)->{1:s})".format( self.class_name, attr.name) else: call = "(self->base->{0:s})".format(attr.name) values_dict = { "class_name": self.class_name, "name": attr.name, "python_obj": attr.to_python_object(), "python_assign": attr.assign(call, self, borrowed=True), "python_def": attr.definition(sense="out")} out.write(( "PyObject *py{class_name:s}_{name:s}_getter(py{class_name:s} *self, PyObject *arguments) {{\n" " PyObject *Py_result = NULL;\n" "{python_def:s}\n" "\n" "{python_assign:s}\n" "{python_obj:s}\n" "\n" " return Py_result;\n" "\n").format(**values_dict)) # Work-around for the String class that generates code that contains "goto on_error". if isinstance(attr, String): out.write(( "on_error:\n" " {0:s}\n").format(attr.error_value)) out.write("}\n\n") def PyGetSetDef(self, out): for _, attr in self.get_attributes(): # TODO: improve docstring. docstring = "{0:s}.".format(attr.name) values_dict = { "class_name": self.class_name, "docstring": format_as_docstring(docstring), "name": attr.name} out.write(( " {{ \"{name:s}\",\n" " (getter) py{class_name:s}_{name:s}_getter,\n" " (setter) 0,\n" " \"{docstring:s}\",\n" " NULL }},\n" "\n").format(**values_dict)) class ProxiedMethod(Method): def __init__(self, method, myclass): # Cannot use super here due to certain logic in Method.__init__(). self.args = method.args self.base_class_name = method.base_class_name self.class_name = method.class_name self.defaults = {} self.definition_class_name = method.definition_class_name self.docstring = "Proxy for {0:s}".format(method.name) self.error_set = False self.exception = None self.method = method self.myclass = myclass self.name = method.name self.return_type = method.return_type def get_name(self): return "Proxied{0:s}_{1:s}".format( self.myclass.class_name, self.name) def _prototype(self, out): out.write("static {0:s} {1:s}({2:s} self".format( self.return_type.type.strip(), self.get_name(), self.definition_class_name)) for arg in self.args: tmp = arg.comment().strip() if tmp: out.write(", {0:s}".format(tmp)) out.write(")") def prototype(self, out): self._prototype(out) out.write(";\n") def write_definition(self, out): name = self.get_name() if name in self.myclass.module.function_definitions: return else: self.myclass.module.function_definitions.add(name) self._prototype(out) self._write_definition(out) def _write_definition(self, out): out.write( " {\n" " PyGILState_STATE gil_state;\n" " PyObject *Py_result = NULL;\n" " PyObject *method_name = NULL;\n") out.write(self.return_type.returned_python_definition()) for arg in self.args: out.write(arg.local_definition()) out.write("PyObject *py_{0:s} = NULL;\n".format(arg.name)) out.write(( "\n" " // Grab the GIL so we can do Python stuff\n" " gil_state = PyGILState_Ensure();\n" "\n" "#if PY_MAJOR_VERSION >= 3\n" " method_name = PyUnicode_FromString(\"{0:s}\");\n" "#else\n" " method_name = PyString_FromString(\"{0:s}\");\n" "#endif\n").format(self.name)) out.write("\n// Obtain Python objects for all the args:\n") for arg in self.args: out.write(arg.to_python_object( result=("py_{0:s}".format(arg.name)), sense="proxied", BORROWED=True)) out.write(( " if(((Object) self)->extension == NULL) {{\n" " RaiseError(ERuntimeError, \"No proxied object in {0:s}\");\n" " goto on_error;\n" " }}\n").format(self.myclass.class_name)) out.write( "\n" " // Now call the method\n" " PyErr_Clear();\n" " Py_result = PyObject_CallMethodObjArgs((PyObject *) ((Object) self)->extension, method_name, ") for arg in self.args: out.write("py_{0:s},".format(arg.name)) # Sentinal out.write( "NULL);\n" "\n") self.error_set = True out.write(( " /* Check for Python errors */\n" " if(PyErr_Occurred()) {{\n" " pytsk_fetch_error();\n" "\n" " goto on_error;\n" " }}\n" "\n").format(CURRENT_ERROR_FUNCTION)) for arg in self.args: out.write(arg.python_proxy_post_call()) # Now convert the Python value back to a value return_type = self.return_type.from_python_object( "Py_result", self.return_type.name, self, context="self") out.write(" {0:s}".format(return_type)) out.write( " if(Py_result != NULL) {\n" " Py_DecRef(Py_result);\n" " }\n" " Py_DecRef(method_name);\n" "\n") # Decref all our Python objects: for arg in self.args: out.write(( " if(py_{0:s} != NULL) {{\n" " Py_DecRef(py_{0:s});\n" " }}\n").format(arg.name)) out.write(( " PyGILState_Release(gil_state);\n" "\n" " {0:s}\n").format( self.return_type.return_value("func_return"))) if self.error_set: out.write( "\n" "on_error:\n" " if(Py_result != NULL) {\n" " Py_DecRef(Py_result);\n" " }\n" " Py_DecRef(method_name);\n" "\n") # Decref all our Python objects: for arg in self.args: out.write(( " if(py_{0:s} != NULL) {{\n" " Py_DecRef(py_{0:s});\n" " }}\n").format(arg.name)) out.write(( " PyGILState_Release(gil_state);\n" "\n" " {0:s}\n").format( self.error_condition())) out.write( "}\n" "\n") def error_condition(self): values_dict = { "result": "func_return"} return self.return_type.error_value.format(**values_dict) class StructConstructor(ConstructorMethod): """A constructor for struct wrappers - basically just allocate memory for the struct. """ def prototype(self, out): return Method.prototype(self, out) def write_destructor(self, out): """We do not deallocate memory from structs. This is a real problem since struct memory is usually allocated in some proprietary way and we cant just call free on it when done. """ values_dict = { "class_name": self.class_name} out.write(( "static void {class_name:s}_dealloc(py{class_name:s} *self) {{\n" " struct _typeobject *ob_type = NULL;\n" "\n" " if(self != NULL) {{\n" " if(self->base != NULL) {{\n" " self->base = NULL;\n" " }}\n" " ob_type = Py_TYPE(self);\n" " if(ob_type != NULL && ob_type->tp_free != NULL) {{\n" " ob_type->tp_free((PyObject*) self);\n" " }}\n" " }}\n" "}}\n" "\n").format(**values_dict)) def write_definition(self, out): values_dict = { "class_name": self.class_name} out.write(( "static int py{class_name:s}_init(py{class_name:s} *self, PyObject *args, PyObject *kwds) {{\n" " // Base is borrowed from another object.\n" " self->base = NULL;\n" " return 0;\n" "}}\n" "\n").format(**values_dict)) class EmptyConstructor(ConstructorMethod): def prototype(self, out): return Method.prototype(self, out) def write_definition(self, out): values_dict = { "class_name": self.class_name} out.write( "static int py{class_name:s}_init(py{class_name:s} *self, PyObject *args, PyObject *kwds) {{\n" " return 0;\n" "}}\n" "\n".format(**values_dict)) class ClassGenerator(object): docstring = "" def __init__(self, class_name, base_class_name, module): self.class_name = class_name self.methods = [] # self.methods = [DefinitionMethod( # class_name, base_class_name, "_definition", [], "", # myclass=self)] self.module = module self.constructor = EmptyConstructor( class_name, base_class_name, "Con", [], "", myclass=self) self.base_class_name = base_class_name self.attributes = GetattrMethod( self.class_name, self.base_class_name, self) self.modifier = set() self.active = True self.iterator = None def get_string(self): """Retrieves a string representation.""" result = ( "#{0:s}\n" "Class {1:s}({2:s}):\n" " Constructor:{3:s}\n" " Attributes:\n{4:s}\n" " Methods:\n").format( self.docstring, self.class_name, self.base_class_name, self.constructor.get_string(), self.attributes.get_string()) for method in self.methods: result += " {0:s}\n".format(method.get_string()) return result def prepare(self): """This method is called just before we need to write the output and allows us to do any last minute fixups. """ pass def is_active(self): """Returns true if this class is active and should be generated""" if self.class_name in self.module.active_structs: return True if (not self.active or self.modifier and ("PRIVATE" in self.modifier or "ABSTRACT" in self.modifier)): log("{0:s} is not active {1!s}".format( self.class_name, self.modifier)) return False return True def clone(self, new_class_name): """Creates a clone of this class - usefull when implementing class extensions. """ result = ClassGenerator(new_class_name, self.class_name, self.module) result.constructor = self.constructor.clone(new_class_name) result.methods = [ method.clone(new_class_name) for method in self.methods] result.attributes = self.attributes.clone(new_class_name) return result def add_attribute(self, attr_name, attr_type, modifier, *args, **kwargs): try: if not self.module.classes[attr_type].is_active(): return except KeyError: pass try: # All attribute references are always borrowed - that # means we dont want to free them after accessing them type_class = dispatch( attr_name, "BORROWED {0:s}".format(attr_type), *args, **kwargs) except KeyError: # TODO: fix that self.class_name is None. log("Unknown attribute type {0:s} for {1!s}.{2:s}".format( attr_type, self.class_name, attr_name)) return type_class.attributes.add(modifier) self.attributes.add_attribute(type_class) def add_constructor(self, method_name, args, return_type, docstring): if method_name.startswith("Con"): self.constructor = ConstructorMethod( self.class_name, self.base_class_name, method_name, args, return_type, myclass=self) self.constructor.docstring = docstring def struct(self, out): values_dict = { "class_name": self.class_name} out.write(( "\n" "typedef struct {{\n" " PyObject_HEAD\n" " {class_name:s} base;\n" " int base_is_python_object;\n" " int base_is_internal;\n" " PyObject *python_object1;\n" " PyObject *python_object2;\n" " int object_is_proxied;\n" "\n" " void (*initialise)(Gen_wrapper self, void *item);\n" "}} py{class_name:s};\n").format(**values_dict)) def code(self, out): if not self.constructor: raise RuntimeError( "No constructor found for class {0:s}".format(self.class_name)) self.constructor.write_destructor(out) self.constructor.write_definition(out) if self.attributes: self.attributes.write_definition(out) for method in self.methods: method.write_definition(out) if hasattr(method, "proxied"): method.proxied.write_definition(out) def initialise(self): values_dict = { "class_name": self.class_name} result = ( "python_wrappers[TOTAL_CCLASSES].class_ref = (Object)&__{class_name:s};\n" "python_wrappers[TOTAL_CCLASSES].python_type = &{class_name:s}_Type;\n").format(**values_dict) func_name = "py{class_name:s}_initialize_proxies".format(**values_dict) if func_name in self.module.function_definitions: result += ( "python_wrappers[TOTAL_CCLASSES].initialize_proxies = (void (*)(Gen_wrapper, void *)) &{0:s};\n").format( func_name) result += "TOTAL_CCLASSES++;\n" return result def PyGetSetDef(self, out): out.write( "static PyGetSetDef {0:s}_get_set_definitions[] = {{\n".format( self.class_name)) if self.attributes: self.attributes.PyGetSetDef(out) out.write( " {NULL, NULL, NULL, NULL, NULL} /* Sentinel */\n" "};\n" "\n") def PyMethodDef(self, out): out.write("static PyMethodDef {0:s}_methods[] = {{\n".format( self.class_name)) for method in self.methods: method.PyMethodDef(out) out.write( " {NULL, NULL, 0, NULL} /* Sentinel */\n" "};\n" "\n") def prototypes(self, out): """Write prototype suitable for .h file""" out.write("/* static PyTypeObject {0:s}_Type; */\n".format( self.class_name)) self.constructor.prototype(out) out.write("static void {0:s}_dealloc(py{0:s} *self);\n".format( self.class_name)) if self.attributes: self.attributes.prototype(out) for method in self.methods: method.prototype(out) # Each method, except for close, needs a proxy method that # is called when the object is sub typed. if method.name == "close": continue method.proxied = ProxiedMethod(method, method.myclass) method.proxied.prototype(out) def numeric_protocol_int(self): pass def numeric_protocol_nonzero(self): values_dict = { "class_name": self.class_name} return ( "static int {class_name:s}_nonzero(py{class_name:s} *v) {{\n" " return v->base != 0;\n" "}}\n").format(**values_dict) def numeric_protocol(self, out): args = { "class": self.class_name} for type, func in [ ("nonzero", self.numeric_protocol_nonzero), ("int", self.numeric_protocol_int)]: definition = func() if definition: out.write(definition) args[type] = "{0:s}_{1:s}".format(self.class_name, type) else: args[type] = "0" out.write(( "#if PY_MAJOR_VERSION >= 3\n" "static PyNumberMethods {class:s}_as_number = {{\n" " (binaryfunc) 0, /* nb_add */\n" " (binaryfunc) 0, /* nb_subtract */\n" " (binaryfunc) 0, /* nb_multiply */\n" " (binaryfunc) 0, /* nb_remainder */\n" " (binaryfunc) 0, /* nb_divmod */\n" " (ternaryfunc) 0, /* nb_power */\n" " (unaryfunc) 0, /* nb_negative */\n" " (unaryfunc) 0, /* nb_positive */\n" " (unaryfunc) 0, /* nb_absolute */\n" " (inquiry) {nonzero:s}, /* nb_bool */\n" " (unaryfunc) 0, /* nb_invert */\n" " (binaryfunc) 0, /* nb_lshift */\n" " (binaryfunc) 0, /* nb_rshift */\n" " (binaryfunc) 0, /* nb_and */\n" " (binaryfunc) 0, /* nb_xor */\n" " (binaryfunc) 0, /* nb_or */\n" " (unaryfunc) {int:s}, /* nb_int */\n" " (void *) NULL, /* nb_reserved */\n" " (unaryfunc) 0, /* nb_float */\n" "\n" " (binaryfunc) 0, /* nb_inplace_add */\n" " (binaryfunc) 0, /* nb_inplace_subtract */\n" " (binaryfunc) 0, /* nb_inplace_multiply */\n" " (binaryfunc) 0, /* nb_inplace_remainder */\n" " (ternaryfunc) 0, /* nb_inplace_power */\n" " (binaryfunc) 0, /* nb_inplace_lshift */\n" " (binaryfunc) 0, /* nb_inplace_rshift */\n" " (binaryfunc) 0, /* nb_inplace_and */\n" " (binaryfunc) 0, /* nb_inplace_xor */\n" " (binaryfunc) 0, /* nb_inplace_or */\n" "\n" " (binaryfunc) 0, /* nb_floor_divide */\n" " (binaryfunc) 0, /* nb_true_divide */\n" " (binaryfunc) 0, /* nb_inplace_floor_divide */\n" " (binaryfunc) 0, /* nb_inplace_true_divide */\n" "\n" " (unaryfunc) 0, /* nb_index */\n" "}};\n" "#else\n" "static PyNumberMethods {class:s}_as_number = {{\n" " (binaryfunc) 0, /* nb_add */\n" " (binaryfunc) 0, /* nb_subtract */\n" " (binaryfunc) 0, /* nb_multiply */\n" " (binaryfunc) 0, /* nb_divide */\n" " (binaryfunc) 0, /* nb_remainder */\n" " (binaryfunc) 0, /* nb_divmod */\n" " (ternaryfunc) 0, /* nb_power */\n" " (unaryfunc) 0, /* nb_negative */\n" " (unaryfunc) 0, /* nb_positive */\n" " (unaryfunc) 0, /* nb_absolute */\n" " (inquiry) {nonzero:s}, /* nb_nonzero */\n" " (unaryfunc) 0, /* nb_invert */\n" " (binaryfunc) 0, /* nb_lshift */\n" " (binaryfunc) 0, /* nb_rshift */\n" " (binaryfunc) 0, /* nb_and */\n" " (binaryfunc) 0, /* nb_xor */\n" " (binaryfunc) 0, /* nb_or */\n" " (coercion) 0, /* nb_coerce */\n" " (unaryfunc) {int:s}, /* nb_int */\n" " (unaryfunc) 0, /* nb_long */\n" " (unaryfunc) 0, /* nb_float */\n" " (unaryfunc) 0, /* nb_oct */\n" " (unaryfunc) 0, /* nb_hex */\n" "\n" " (binaryfunc) 0, /* nb_inplace_add */\n" " (binaryfunc) 0, /* nb_inplace_subtract */\n" " (binaryfunc) 0, /* nb_inplace_multiply */\n" " (binaryfunc) 0, /* nb_inplace_divide */\n" " (binaryfunc) 0, /* nb_inplace_remainder */\n" " (ternaryfunc) 0, /* nb_inplace_power */\n" " (binaryfunc) 0, /* nb_inplace_lshift */\n" " (binaryfunc) 0, /* nb_inplace_rshift */\n" " (binaryfunc) 0, /* nb_inplace_and */\n" " (binaryfunc) 0, /* nb_inplace_xor */\n" " (binaryfunc) 0, /* nb_inplace_or */\n" "\n" " (binaryfunc) 0, /* nb_floor_divide */\n" " (binaryfunc) 0, /* nb_true_divide */\n" " (binaryfunc) 0, /* nb_inplace_floor_divide */\n" " (binaryfunc) 0, /* nb_inplace_true_divide */\n" "\n" " (unaryfunc) 0, /* nb_index */\n" "}};\n" "#endif /* PY_MAJOR_VERSION >= 3 */\n" "\n").format(**args)) return "&{class:s}_as_number".format(**args) def PyTypeObject(self, out): docstring = "{0:s}: {1:s}".format( self.class_name, format_as_docstring(self.docstring)) args = { "class": self.class_name, "module": self.module.name, "iterator": 0, "iternext": 0, "tp_str": 0, "tp_eq": 0, "getattr_func": 0, "docstring": docstring} if self.attributes: args["getattr_func"] = self.attributes.name args["numeric_protocol"] = self.numeric_protocol(out) if "ITERATOR" in self.modifier: args["iterator"] = "PyObject_SelfIter" args["iternext"] = "py{0:s}_iternext".format(self.class_name) if "SELF_ITER" in self.modifier: args["iterator"] = "py{0:s}___iter__".format(self.class_name) if "TP_STR" in self.modifier: args["tp_str"] = "py{0:s}___str__".format(self.class_name) if "TP_EQUAL" in self.modifier: args["tp_eq"] = "{0:s}_eq".format(self.class_name) out.write(( "static PyTypeObject {class:s}_Type = {{\n" " PyVarObject_HEAD_INIT(NULL, 0)\n" " /* tp_name */\n" " \"{module:s}.{class:s}\",\n" " /* tp_basicsize */\n" " sizeof(py{class:s}),\n" " /* tp_itemsize */\n" " 0,\n" " /* tp_dealloc */\n" " (destructor) {class:s}_dealloc,\n" " /* tp_print */\n" " 0,\n" " /* tp_getattr */\n" " 0,\n" " /* tp_setattr */\n" " 0,\n" " /* tp_compare */\n" " 0,\n" " /* tp_repr */\n" " 0,\n" " /* tp_as_number */\n" " {numeric_protocol:s},\n" " /* tp_as_sequence */\n" " 0,\n" " /* tp_as_mapping */\n" " 0,\n" " /* tp_hash */\n" " 0,\n" " /* tp_call */\n" " 0,\n" " /* tp_str */\n" " (reprfunc) {tp_str!s},\n" " /* tp_getattro */\n" " (getattrofunc) {getattr_func!s},\n" " /* tp_setattro */\n" " 0,\n" " /* tp_as_buffer */\n" " 0,\n" " /* tp_flags */\n" " Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,\n" " /* tp_doc */\n" " \"{docstring:s}\",\n" " /* tp_traverse */\n" " 0,\n" " /* tp_clear */\n" " 0,\n" " /* tp_richcompare */\n" " {tp_eq!s},\n" " /* tp_weaklistoffset */\n" " 0,\n" " /* tp_iter */\n" " (getiterfunc) {iterator!s},\n" " /* tp_iternext */\n" " (iternextfunc) {iternext!s},\n" " /* tp_methods */\n" " {class:s}_methods,\n" " /* tp_members */\n" " 0,\n" " /* tp_getset */\n" " {class:s}_get_set_definitions,\n" " /* tp_base */\n" " 0,\n" " /* tp_dict */\n" " 0,\n" " /* tp_descr_get */\n" " 0,\n" " /* tp_descr_set */\n" " 0,\n" " /* tp_dictoffset */\n" " 0,\n" " /* tp_init */\n" " (initproc) py{class:s}_init,\n" " /* tp_alloc */\n" " 0,\n" " /* tp_new */\n" " 0,\n" "}};\n" "\n").format(**args)) class StructGenerator(ClassGenerator): """A wrapper generator for structs.""" def __init__(self, class_name, module): self.class_name = class_name self.methods = [] self.module = module self.base_class_name = None self.active = False self.modifier = set() self.constructor = None self.attributes = GetattrMethod( self.class_name, self.base_class_name, self) def get_string(self): """Retrieves a string representation.""" return ( "# {0:s}\n" "Struct {1:s}:\n" "{2:s}\n").format( self.docstring, self.class_name, self.attributes.get_string()) def prepare(self): # This is needed for late stage initialization - sometimes # our class_name is not know until now. if not self.constructor: self.constructor = StructConstructor( self.class_name, self.base_class_name, "Con", [], "void", myclass=self) self.attributes.rename_class_name(self.class_name) for x in self.attributes._attributes: x[1].attributes.add("FOREIGN") def struct(self, out): values_dict = { "class_name": self.class_name} out.write(( "\n" "typedef struct {{\n" " PyObject_HEAD\n" " {class_name:s} *base;\n" " int base_is_python_object;\n" " int base_is_internal;\n" " PyObject *python_object1;\n" " PyObject *python_object2;\n" " int object_is_proxied;\n" " {class_name:s} *cbase;\n" "}} py{class_name:s};\n").format( **values_dict)) def initialise(self): return "" class EnumConstructor(ConstructorMethod): def prototype(self, out): return Method.prototype(self, out) def write_destructor(self, out): values_dict = { "class_name": self.class_name} out.write(( "static void {class_name:s}_dealloc(py{class_name:s} *self) {{\n" " struct _typeobject *ob_type = NULL;\n" "\n" " if(self != NULL) {{\n" " Py_DecRef(self->value);\n" " ob_type = Py_TYPE(self);\n" " if(ob_type != NULL && ob_type->tp_free != NULL) {{\n" " ob_type->tp_free((PyObject*) self);\n" " }}\n" " }}\n" "}}\n").format(**values_dict)) def write_definition(self, out): self.myclass.modifier.add("TP_STR") self.myclass.modifier.add("TP_EQUAL") self._prototype(out) values_dict = { "class_name": self.class_name} out.write(( "{{\n" " const char *kwlist[] = {{\"value\", NULL}};\n" "\n" " if(!PyArg_ParseTupleAndKeywords(args, kwds, \"O\", (char **) kwlist, &self->value)) {{\n" " goto on_error;\n" " }}\n" "\n" " Py_IncRef(self->value);\n" "\n" " return 0;\n" "\n" "on_error:\n" " return -1;\n" "}}\n" "\n").format(**values_dict)) class Enum(StructGenerator): def __init__(self, name, module): super(Enum, self).__init__(name, module) self.values = [] self.name = name self.attributes = None self.active = True def get_string(self): """Retrieves a string representation.""" result = "Enum {0:s}:\n".format(self.name) for attr in self.values: result += " {0:s}\n".format(attr) return result def prepare(self): self.constructor = EnumConstructor( self.class_name, self.base_class_name, "Con", [], "void", myclass=self) StructGenerator.prepare(self) def struct(self, out): values_dict = { "class_name": self.class_name} out.write(( "\n" "typedef struct {{\n" " PyObject_HEAD\n" " PyObject *value;\n" "}} py{class_name:s};\n" "\n" "int {class_name:s}_init_type(\n" " PyTypeObject *type_object )\n" "{{\n" " type_object->tp_dict = PyDict_New();\n").format( **values_dict)) if self.values: out.write(" PyObject *integer_object = NULL;\n") for attr in self.values: values_dict = { "class_name": self.class_name, "value": attr} out.write(( " integer_object = PyLong_FromLong({value:s});\n" "\n" " PyDict_SetItemString(type_object->tp_dict, \"{value:s}\", integer_object);\n" "\n" " Py_DecRef(integer_object);\n" "\n").format(**values_dict)) out.write(( " return( 1 );\n" "}\n" "\n")) def PyGetSetDef(self, out): out.write(( "static PyGetSetDef {0:s}_get_set_definitions[] = {{\n" " {{NULL, NULL, NULL, NULL, NULL}} /* Sentinel */\n" "}};\n" "\n").format(self.class_name)) def PyMethodDef(self, out): out.write(( "static PyMethodDef {0:s}_methods[] = {{\n" " {{NULL, NULL, 0, NULL}} /* Sentinel */\n" "}};\n" "\n").format(self.class_name)) def numeric_protocol_nonzero(self): pass def numeric_protocol_int(self): values_dict = { "class_name": self.class_name} return ( "static PyObject *{class_name:s}_int(py{class_name:s} *self) {{\n" " Py_IncRef(self->value);\n" " return self->value;\n" "}}\n").format(**values_dict) def initialise(self): return "\n" class EnumType(Integer): buildstr = "i" def __init__(self, name, type, *args, **kwargs): super(EnumType, self).__init__(name, type, *args, **kwargs) self.type = type def definition(self, default=None, **kwargs): # Force the enum to be an int just in case the compiler chooses # a random size. if default: return " int {0:s} = {1:s};\n".format(self.name, default) else: return " int UNUSED {0:s} = 0;\n".format(self.name) def to_python_object(self, name=None, result="Py_result", **kwargs): values_dict = { "name": name or self.name, "result": result} return ( " PyErr_Clear();\n" "#if PY_MAJOR_VERSION >= 3\n" " {result:s} = PyLong_FromLong({name:s});\n" "#else\n" " {result:s} = PyInt_FromLong({name:s});\n" "#endif\n").format(**values_dict) def pre_call(self, method, **kwargs): method.error_set = True values_dict = { "name": self.name, "type": self.type} return "" class HeaderParser(lexer.SelfFeederMixIn): tokens = [ ["INITIAL", r"#define\s+", "PUSH_STATE", "DEFINE"], ["DEFINE", r"([A-Za-z_0-9]+)\s+[^\n]+", "DEFINE,POP_STATE", None], ["DEFINE", r"\n", "POP_STATE", None], # Ignore macros with args ["DEFINE", r"\([^\n]+", "POP_STATE", None], # Recognize ansi c comments [".", r"/\*(.)", "PUSH_STATE", "COMMENT"], ["COMMENT", r"(.+?)\*/\s+", "COMMENT_END,POP_STATE", None], ["COMMENT", r"(.+)", "COMMENT", None], # And c++ comments [".", r"//([^\n]+)", "COMMENT", None], # An empty line clears the current comment [".", r"\r?\n\r?\n", "CLEAR_COMMENT", None], # Ignore whitespace [".", r"\s+", "SPACE", None], [".", r"\\\n", "SPACE", None], # Recognize CCLASS() definitions ["INITIAL", r"^([A-Z]+)?\s*CCLASS\(([A-Z_a-z0-9]+)\s*,\s*([A-Z_a-z0-9]+)\)", "PUSH_STATE,CCLASS_START", "CCLASS"], ["CCLASS", r"^\s*(FOREIGN|ABSTRACT|PRIVATE)?([0-9A-Z_a-z ]+( |\*))METHOD\(([A-Z_a-z0-9]+),\s*([A-Z_a-z0-9]+),?", "PUSH_STATE,METHOD_START", "METHOD"], ["METHOD", r"\s*([0-9A-Z a-z_]+\s+\*?\*?)([0-9A-Za-z_]+),?", "METHOD_ARG", None], ["METHOD", r"\);", "POP_STATE,METHOD_END", None], ["CCLASS", r"^\s*(FOREIGN|ABSTRACT)?([0-9A-Z_a-z ]+\s+\*?)\s*([A-Z_a-z0-9]+)\s*;", "CCLASS_ATTRIBUTE", None], ["CCLASS", "END_CCLASS", "END_CCLASS,POP_STATE", None], # Recognize struct definitions (With name) ["INITIAL", "([A-Z_a-z0-9 ]+)?struct\s+([A-Z_a-z0-9]+)\s+{", "PUSH_STATE,STRUCT_START", "STRUCT"], # Without name (using typedef) ["INITIAL", "typedef\s+struct\s+{", "PUSH_STATE,TYPEDEF_STRUCT_START", "STRUCT"], ["STRUCT", r"^\s*([0-9A-Z_a-z ]+\s+\*?)\s*([A-Z_a-z0-9]+)(?:\[([A-Z_a-z0-9]+)\])?\s*;", "STRUCT_ATTRIBUTE", None], ["STRUCT", r"^\s*([0-9A-Z_a-z ]+)\*\s+([A-Z_a-z0-9]+)\s*;", "STRUCT_ATTRIBUTE_PTR", None], # Struct ended with typedef ["STRUCT", "}\s+([0-9A-Za-z_]+);", "POP_STATE,TYPEDEF_STRUCT_END", None], ["STRUCT", "}", "POP_STATE,STRUCT_END", None], # Handle recursive struct or union definition (At the moment # we cant handle them at all) ["(RECURSIVE_)?STRUCT", "(struct|union)\s+([_A-Za-z0-9]+)?\s*{", "PUSH_STATE", "RECURSIVE_STRUCT"], ["RECURSIVE_STRUCT", "}\s+[0-9A-Za-z]+", "POP_STATE", None], ["RECURSIVE_STRUCT", "};", "POP_STATE", None], # Process enums (2 forms - named and typedefed) ["INITIAL", r"enum\s+([0-9A-Za-z_]+)\s+{", "PUSH_STATE,ENUM_START", "ENUM"], # Unnamed ["INITIAL", r"typedef\s+enum\s+{", "PUSH_STATE,TYPEDEF_ENUM_START", "ENUM"], ["ENUM", r"([0-9A-Za-z_]+)\s+=[^\n]+", "ENUM_VALUE", None], # Typedefed ending ["ENUM", r"}\s+([0-9A-Za-z_]+);", "POP_STATE,TYPEDEFED_ENUM_END", None], ["ENUM", r"}", "POP_STATE,ENUM_END", None], ["INITIAL", r"BIND_STRUCT\(([0-9A-Za-z_ \*]+)\)", "BIND_STRUCT", None], # A simple typedef of one type for another type: ["INITIAL", r"typedef ([A-Za-z_0-9]+) +([^;]+);", "SIMPLE_TYPEDEF", None], # Handle proxied directives ["INITIAL", r"PXXROXY_CCLASS\(([A-Za-z0-9_]+)\)", "PROXY_CCLASS", None], ] def __init__(self, name, verbose=0, base=""): if DEBUG > 0: verbose = 1 self.module = Module(name) self.base = base super(HeaderParser, self).__init__(verbose=verbose) file_object = io.BytesIO( b"// Base object\n" b"CCLASS(Object, Obj)\n" b"END_CCLASS\n") self.parse_fd(file_object) current_comment = "" def COMMENT(self, t, m): self.current_comment += m.group(1) + "\n" def COMMENT_END(self, t, m): self.current_comment += m.group(1) def CLEAR_COMMENT(self, t, m): self.current_comment = "" def DEFINE(self, t, m): line = m.group(0) line = line.split("/*")[0] if "\"" in line: type = "string" else: type = "integer" name = m.group(1).strip() if (len(name) > 3 and name[0] != "_" and name == name.upper() and name not in self.module.constants_blacklist): self.module.add_constant(name, type) current_class = None def CCLASS_START(self, t, m): class_name = m.group(2).strip() base_class_name = m.group(3).strip() try: self.current_class = self.module.classes[base_class_name].clone(class_name) except (KeyError, AttributeError): log("Base class {0:s} is not defined !!!!".format(base_class_name)) self.current_class = ClassGenerator(class_name, base_class_name, self.module) self.current_class.docstring = self.current_comment self.current_class.modifier.add(m.group(1)) self.module.add_class(self.current_class, Wrapper) identifier = "{0:s} *".format(class_name) type_dispatcher[identifier] = PointerWrapper current_method = None def METHOD_START(self, t, m): return_type = m.group(2).strip() method_name = m.group(5).strip() modifier = m.group(1) or "" if "PRIVATE" in modifier: return # Is it a regular method or a constructor? self.current_method = Method if (return_type == self.current_class.class_name and method_name.startswith("Con")): self.current_method = ConstructorMethod elif method_name == "iternext": self.current_method = IteratorMethod self.current_class.modifier.add("ITERATOR") elif method_name == "__iter__": self.current_method = SelfIteratorMethod self.current_class.modifier.add("SELF_ITER") elif method_name == "__str__": self.current_class.modifier.add("TP_STR") self.current_method = self.current_method( self.current_class.class_name, self.current_class.base_class_name, method_name, [], return_type, myclass=self.current_class) self.current_method.docstring = self.current_comment self.current_method.modifier = modifier def METHOD_ARG(self, t, m): name = m.group(2).strip() type = m.group(1).strip() if self.current_method: self.current_method.add_arg(type, name) def METHOD_END(self, t, m): if not self.current_method: return if isinstance(self.current_method, ConstructorMethod): self.current_class.constructor = self.current_method else: found = False for i in range(len(self.current_class.methods)): # Try to replace existing methods with this new method method = self.current_class.methods[i] if method.name == self.current_method.name: self.current_class.methods[i] = self.current_method self.current_method = None return # Method does not exist, just add to the end self.current_class.methods.append(self.current_method) self.current_method = None def CCLASS_ATTRIBUTE(self, t, m): modifier = m.group(1) or "" type = m.group(2).strip() name = m.group(3).strip() self.current_class.add_attribute(name, type, modifier) def END_CCLASS(self, t, m): self.current_class = None current_struct = None def STRUCT_START(self, t, m): self.current_struct = StructGenerator(m.group(2).strip(), self.module) self.current_struct.docstring = self.current_comment self.current_struct.modifier.add(m.group(1)) def TYPEDEF_STRUCT_START(self, t, m): self.current_struct = StructGenerator(None, self.module) self.current_struct.docstring = self.current_comment def STRUCT_ATTRIBUTE(self, t, m): name = m.group(2).strip() type = m.group(1).strip() array_size = m.group(3) if array_size is not None: array_size = array_size.strip() self.current_struct.add_attribute(name, type, "", array_size=array_size) else: self.current_struct.add_attribute(name, type, "") def STRUCT_ATTRIBUTE_PTR(self, t, m): type = "{0:s} *".format(m.group(1).strip()) name = m.group(2).strip() self.current_struct.add_attribute(name, type, "") def STRUCT_END(self, t, m): self.module.add_class(self.current_struct, StructWrapper) identifier = "{0:s} *".format(self.current_struct.class_name) type_dispatcher[identifier] = PointerStructWrapper self.current_struct = None def TYPEDEF_STRUCT_END(self, t, m): self.current_struct.class_name = m.group(1).strip() self.STRUCT_END(t, m) current_enum = None def ENUM_START(self, t, m): self.current_enum = Enum(m.group(1).strip(), self.module) def TYPEDEF_ENUM_START(self, t, m): self.current_enum = Enum(None, self.module) def ENUM_VALUE(self, t, m): self.current_enum.values.append(m.group(1).strip()) def ENUM_END(self, t, m): self.module.classes[self.current_enum.name] = self.current_enum # For now we just treat enums as an integer, and also add # them to the constant table. In future it would be nice to # have them as a proper Python object so we can override # __unicode__, __str__ and __int__. for attr in self.current_enum.values: self.module.add_constant(attr, "integer") # type_dispatcher[self.current_enum.name] = Integer type_dispatcher[self.current_enum.name] = EnumType self.current_enum = None def TYPEDEFED_ENUM_END(self, t, m): self.current_enum.name = self.current_enum.class_name = m.group(1) self.ENUM_END(t, m) def BIND_STRUCT(self, t, m): self.module.active_structs.add(m.group(1)) self.module.active_structs.add("{0:s} *".format(m.group(1))) def SIMPLE_TYPEDEF(self, t, m): # We basically add a new type as a copy of the old # type old, new = m.group(1).strip(), m.group(2).strip() if old in type_dispatcher: type_dispatcher[new] = type_dispatcher[old] def PROXY_CCLASS(self, t, m): base_class_name = m.group(1).strip() class_name = "Proxied{0:s}".format(base_class_name) try: proxied_class = self.module.classes[base_class_name] except KeyError: raise RuntimeError(( "Need to create a proxy for {0:s} but it has not been " "defined (yet). You must place the PROXIED_CCLASS() " "instruction after the class definition").format( base_class_name)) current_class = ProxyClassGenerator(class_name, base_class_name, self.module) # self.current_class.constructor.args += proxied_class.constructor.args current_class.docstring = self.current_comment # Create proxies for all these methods for method in proxied_class.methods: if method.name[0] != "_": current_class.methods.append(ProxiedMethod(method, current_class)) self.module.add_class(current_class, Wrapper) def parse_filenames(self, filenames): for f in filenames: self._parse(f) # Second pass for f in filenames: self._parse(f) def _parse(self, filename): file_object = open(filename, "rb") self.parse_fd(file_object) file_object.close() if filename not in self.module.files: if filename.startswith(self.base): filename = filename[len(self.base):] self.module.headers += "#include \"{0:s}\"\n".format(filename) self.module.files.append(filename) def write(self, out): try: self.module.write(out) except: # pdb.post_mortem() raise def write_headers(self): pass # pdb.set_trace() if __name__ == "__main__": p = HeaderParser("pytsk3", verbose=1) for arg in sys.argv[1:]: p.parse_fd(open(arg, "rb")) log("second parse") for arg in sys.argv[1:]: p.parse_fd(open(arg, "rb")) p.write(sys.stdout) p.write_headers() pytsk-20231007/dpkg/000077500000000000000000000000001451023402500140455ustar00rootroot00000000000000pytsk-20231007/dpkg/changelog000066400000000000000000000002121451023402500157120ustar00rootroot00000000000000pytsk3 (20231007-1) unstable; urgency=low * Auto-generated -- Joachim Metz Sat, 07 Oct 2023 12:22:28 -0100 pytsk-20231007/dpkg/compat000066400000000000000000000000021451023402500152430ustar00rootroot000000000000009 pytsk-20231007/dpkg/control000066400000000000000000000013701451023402500154510ustar00rootroot00000000000000Source: pytsk3 Section: python Priority: extra Maintainer: Joachim Metz Build-Depends: debhelper (>= 9), dh-autoreconf, dh-python, python3-all (>= 3.2~), python3-all-dev, python3-setuptools Standards-Version: 3.9.5 X-Python3-Version: >= 3.5 Homepage: https://github.com/py4n6/pytsk/ Package: python3-pytsk3 Architecture: any Depends: ${shlibs:Depends}, ${misc:Depends} Conflicts: python3-tsk Replaces: python3-tsk Description: Python 3 bindings for the SleuthKit (libtsk) Python 3 bindings for the SleuthKit (libtsk). Package: python3-pytsk3-dbg Architecture: any Section: debug Depends: python3-pytsk3 (= ${binary:Version}), ${misc:Depends} Description: Debugging symbols for python3-pytsk3 Debugging symbols for python3-pytsk3. pytsk-20231007/dpkg/copyright000066400000000000000000000031401451023402500157760ustar00rootroot00000000000000This work was packaged for Debian by: Joachim Metz on Wed, 26 Sep 2012 17:00:00 +0200 It was downloaded from: https://github.com/py4n6/pytsk/ Upstream Author(s): Michael Cohen Copyright: Copyright 2010 Michael Cohen License: Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. On Debian systems, the complete text of the Apache-2.0 License can be found in `/usr/share/common-licenses/Apache-2.0'. The Debian packaging is: Copyright (C) 2012 Joachim Metz License: Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. pytsk-20231007/dpkg/python-pytsk3.docs000066400000000000000000000000171451023402500174710ustar00rootroot00000000000000LICENSE README pytsk-20231007/dpkg/python3-pytsk3.docs000066400000000000000000000000171451023402500175540ustar00rootroot00000000000000LICENSE README pytsk-20231007/dpkg/rules000077500000000000000000000034071451023402500151310ustar00rootroot00000000000000#!/usr/bin/make -f # debian/rules that uses debhelper >= 9. # Uncomment this to turn on verbose mode. #export DH_VERBOSE=1 # This has to be exported to make some magic below work. export DH_OPTIONS %: dh $@ --with python3 .PHONY: override_dh_auto_clean override_dh_auto_clean: set -ex; for python in $(shell py3versions -r); do \ $$python setup.py clean -a; \ done; rm -rf __pycache__ build pytsk3.egg-info/SOURCES.txt pytsk3.egg-info/PKG-INFO .PHONY: override_dh_auto_build override_dh_auto_build: (cd sleuthkit && autoreconf -fiv) set -ex; for python in $(shell py3versions -r); do \ $$python setup.py build; \ done; .PHONY: override_dh_auto_install override_dh_auto_install: set -ex; for python in $(shell py3versions -r); do \ $$python setup.py install --root=$(CURDIR)/debian/python3-pytsk3 --install-layout=deb; \ done; .PHONY: override_dh_auto_test override_dh_auto_test: .PHONY: override_dh_installmenu override_dh_installmenu: .PHONY: override_dh_installmime override_dh_installmime: .PHONY: override_dh_installmodules override_dh_installmodules: .PHONY: override_dh_installlogcheck override_dh_installlogcheck: .PHONY: override_dh_installlogrotate override_dh_installlogrotate: .PHONY: override_dh_installpam override_dh_installpam: .PHONY: override_dh_installppp override_dh_installppp: .PHONY: override_dh_installudev override_dh_installudev: .PHONY: override_dh_installwm override_dh_installwm: .PHONY: override_dh_installxfonts override_dh_installxfonts: .PHONY: override_dh_gconf override_dh_gconf: .PHONY: override_dh_icons override_dh_icons: .PHONY: override_dh_perl override_dh_perl: .PHONY: override_dh_strip override_dh_strip: ifeq (,$(filter nostrip,$(DEB_BUILD_OPTIONS))) dh_strip -ppython3-pytsk3 --dbg-package=python3-pytsk3-dbg endif pytsk-20231007/dpkg/source/000077500000000000000000000000001451023402500153455ustar00rootroot00000000000000pytsk-20231007/dpkg/source/format000066400000000000000000000000041451023402500165520ustar00rootroot000000000000001.0 pytsk-20231007/dpkg/source/options000066400000000000000000000000521451023402500167600ustar00rootroot00000000000000tar-ignore = "a.out" tar-ignore = "tmp/*" pytsk-20231007/error.cpp000066400000000000000000000060551451023402500147630ustar00rootroot00000000000000/* Error functions. * * Copyright 2010, Michael Cohen . * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #if !defined( WIN32 ) #include #endif #include "aff4_errors.h" #include "class.h" #define ERROR_BUFF_SIZE 10240 // Windows version not truely threadsafe for now #if defined( WIN32 ) static char global_error_buffer[ERROR_BUFF_SIZE]; static int global_error_type = 0; #else /** These slots carry the TLS error keys */ static pthread_key_t error_str_slot; static pthread_once_t error_once = PTHREAD_ONCE_INIT; static pthread_key_t error_value_slot; #endif #if defined( WIN32 ) static void error_init(void) { memset(global_error_buffer, 0, sizeof(global_error_buffer)); }; #else static void error_init(void); void error_dest(void *slot) { if(slot) talloc_free(slot); }; void error_init(void) { // We create the error buffer slots if(pthread_key_create(&error_str_slot, error_dest) || pthread_key_create(&error_value_slot, error_dest)) { printf("Unable to set up TLS variables\n"); abort(); }; }; #endif DLL_PUBLIC void *aff4_raise_errors(int t, const char *reason, ...) { char *error_buffer; char tmp[ERROR_BUFF_SIZE]; // This has to succeed: int *type = aff4_get_current_error(&error_buffer); if(reason) { va_list ap; va_start(ap, reason); vsnprintf(tmp, ERROR_BUFF_SIZE-1, reason,ap); tmp[ERROR_BUFF_SIZE-1]=0; va_end(ap); }; if(*type == EZero) { *error_buffer = 0; //update the error type *type = t; } else { strncat(error_buffer, "\n", ERROR_BUFF_SIZE -1 ); }; strncat(error_buffer, tmp, ERROR_BUFF_SIZE-1); return NULL; }; #if defined( WIN32 ) DLL_PUBLIC int *aff4_get_current_error(char **error_buffer) { if(error_buffer != NULL) { *error_buffer = global_error_buffer; }; return &global_error_type; }; #else DLL_PUBLIC int *aff4_get_current_error(char **error_buffer) { int *type; (void) pthread_once(&error_once, error_init); type = (int *) pthread_getspecific(error_value_slot); // This is optional if(error_buffer != NULL) { *error_buffer = (char *) pthread_getspecific(error_str_slot); // If TLS buffers are not set we need to create them // TODO: the TLS buffers need to be freed on exit. if(*error_buffer == NULL) { *error_buffer = (char *) talloc_size(NULL, ERROR_BUFF_SIZE); pthread_setspecific(error_str_slot, *error_buffer); }; }; if(!type) { type = (int *) talloc_size(NULL, ERROR_BUFF_SIZE); pthread_setspecific(error_value_slot, type); }; return type; }; #endif pytsk-20231007/generate_bindings.py000077500000000000000000000044101451023402500171430ustar00rootroot00000000000000#!/usr/bin/python # # Script to generate the Python bindings. # # Copyright 2012, Joachim Metz . # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import sys import class_parser def generate_bindings(target, source_files, env=None, initialization="", free="talloc_free"): """ Generated the Python bindings """ module_name = os.path.splitext(os.path.basename(target))[0] print("Generating Python bindings for module %s from %s" % ( module_name, source_files)) env = env or dict(V=0) # Sets the free function class_parser.FREE = free p = class_parser.HeaderParser(module_name, verbose=env["V"]) p.module.init_string = initialization p.parse_filenames(source_files) fd = open(target, "w") p.write(fd) fd.close() if __name__ == "__main__": if len(sys.argv) != 2: print("Usage: ./generate_bindings.py path_to_source") sys.exit(1) tsk_source_path = sys.argv[1] include_base = "tsk3" if not os.path.exists(os.path.join(tsk_source_path, include_base)): # sleuthkit 4.1 changed the names of the include headers. include_base = "tsk" if not os.path.exists(os.path.join(tsk_source_path, include_base)): print("Unable to find sleuthkit include headers.") sys.exit(1) sources = [ os.path.join(tsk_source_path, include_base, "libtsk.h"), os.path.join(tsk_source_path, include_base, "base", "tsk_base.h"), os.path.join(tsk_source_path, include_base, "fs", "tsk_fs.h"), os.path.join(tsk_source_path, include_base, "img", "tsk_img.h"), os.path.join(tsk_source_path, include_base, "vs", "tsk_vs.h"), "tsk3.h", ] generate_bindings("pytsk3.c", sources, initialization="tsk_init();") pytsk-20231007/lexer.py000066400000000000000000000145631451023402500146220ustar00rootroot00000000000000#!/usr/bin/python # # Copyright 2013, Michael Cohen . # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """A simple feed lexer.""" import re import sys class Lexer(object): """A generic feed lexer.""" ## The following is a description of the states we have and the ## way we move through them: format is an array of ## [ state_re, re, token/action, next state ] tokens = [] state = "INITIAL" buffer = "" error = 0 verbose = 0 state_stack = [] processed = 0 processed_buffer = "" saved_state = None flags = 0 def __init__(self, verbose=0, fd=None): super(Lexer, self).__init__() self.encoding = "utf-8" if not self.verbose: self.verbose = verbose if len(self.tokens[0]) == 4: for row in self.tokens: row.append(re.compile(row[0], re.DOTALL)) row.append(re.compile(row[1], re.DOTALL | re.M | re.S | self.flags)) self.fd = fd def save_state(self, dummy_t=None, m=None): """Returns a dict which represents the current state of the lexer. When provided to restore_state, the lexer is guaranteed to be in the same state as when the save_state was called. Note that derived classes may need to extend this. """ ## Unable to save our state if we have errors. We need to guarantee ## that we rewind to a good part of the file. if self.error: return try: end = m.end() except: end = 0 self.saved_state = dict( state_stack = self.state_stack[:], processed = self.processed - end, processed_buffer = self.processed_buffer, readptr = self.fd.tell() - len(self.buffer) - end, state = self.state, objects = self.objects[:], error = self.error, ) if self.verbose > 1: sys.stderr.write("Saving state {0:s}\n".format(self.processed)) def restore_state(self): state = self.saved_state if not state: return self.state_stack = state["state_stack"] self.processed = state["processed"] self.processed_buffer = state["processed_buffer"] self.buffer = "" self.fd.seek(state["readptr"]) self.state = state["state"] self.objects = state["objects"] self.error = state["error"] if self.verbose > 1: sys.stderr.write("Restoring state to offset {0:s}\n".format(self.processed)) def next_token(self, end=True): ## Now try to match any of the regexes in order: current_state = self.state for _, re_str, token, next_state, state, regex in self.tokens: ## Does the rule apply for us now? if state.match(current_state): if self.verbose > 2: sys.stderr.write("{0:s}: Trying to match {1:s} with {2:s}\n".format( self.state, repr(self.buffer[:10]), repr(re_str))) match = regex.match(self.buffer) if match: if self.verbose > 3: sys.stderr.write("{0:s} matched {1:s}\n".format( re_str, match.group(0).encode("utf8"))) ## The match consumes the data off the buffer (the ## handler can put it back if it likes) self.processed_buffer += self.buffer[:match.end()] self.buffer = self.buffer[match.end():] self.processed += match.end() ## Try to iterate over all the callbacks specified: for t in token.split(","): try: if self.verbose > 0: sys.stderr.write("0x{0:X}: Calling {1:s} {2:s}\n".format( self.processed, t, repr(match.group(0)))) cb = getattr(self, t, self.default_handler) except AttributeError: continue ## Is there a callback to handle this action? callback_state = cb(t, match) if callback_state == "CONTINUE": continue elif callback_state: next_state = callback_state self.state = next_state if next_state: self.state = next_state return token ## Check that we are making progress - if we are too full, we ## assume we are stuck: if end and len(self.buffer) > 0 or len(self.buffer) > 1024: self.processed_buffer += self.buffer[:1] self.buffer = self.buffer[1:] self.ERROR( "Lexer Stuck, discarding 1 byte ({0:s}) - state {1:s}".format( repr(self.buffer[:10]), self.state)) return "ERROR" ## No token were found return def feed(self, data): """Feeds the lexer. Args: data: binary string containing the data (instance of bytes). """ self.buffer += data.decode(self.encoding) def empty(self): return not len(self.buffer) def default_handler(self, token, match): if self.verbose > 2: sys.stderr.write("Default handler: {0:s} with {1:s}\n".format( token, repr(match.group(0)))) def ERROR(self, message=None, weight=1): if self.verbose > 0 and message: sys.stderr.write("Error({0:d}): {1!s}\n".format(weight, message)) self.error += weight def PUSH_STATE(self, dummy_token=None, dummy_match=None): if self.verbose > 1: sys.stderr.write("Storing state {0:s}\n".format(self.state)) self.state_stack.append(self.state) def POP_STATE(self, dummy_token=None, dummy_match=None): try: state = self.state_stack.pop() if self.verbose > 1: sys.stderr.write("Returned state to {0:s}\n".format(state)) except IndexError: sys.stderr.write("Tried to pop the state but failed - possible recursion error\n") state = None return state def close(self): """Just a conveniece function to force us to parse all the data.""" while self.next_token(): pass class SelfFeederMixIn(Lexer): """This mixin is used to make a lexer which feeds itself one sector at the time. Note that self.fd must be the fd we read from. """ def parse_fd(self, fd): self.feed(fd.read()) while self.next_token(): pass pytsk-20231007/make_dist.sh000077500000000000000000000021421451023402500154160ustar00rootroot00000000000000#!/bin/sh # Script to package pytsk VERSION=`grep -e "^VERSION = " class_parser.py | sed 's/^.*"\([0-9]*\)"$/\1/'`; rm -f pytsk-*.tgz PYTSK_SOURCE_FILES="\ ../pytsk/aff4_errors.h \ ../pytsk/class.c \ ../pytsk/class.h \ ../pytsk/error.c \ ../pytsk/misc.h \ ../pytsk/pytsk3.h \ ../pytsk/tsk3.c \ ../pytsk/tsk3.h" TALLOC_SOURCE_FILES="\ ../pytsk/talloc/LICENSE \ ../pytsk/talloc/README \ ../pytsk/talloc/replace.h \ ../pytsk/talloc/talloc.c \ ../pytsk/talloc/talloc.h" SCRIPTS="\ ../pytsk/class_parser.py \ ../pytsk/generate_bindings.py \ ../pytsk/lexer.py \ ../pytsk/make_dist.sh \ ../pytsk/run_tests.py \ ../pytsk/setup.py \ ../pytsk/tests/*.py" DATA_FILES="\ ../pytsk/LICENSE \ ../pytsk/MANIFEST.in \ ../pytsk/README \ ../pytsk/dpkg \ ../pytsk/msvscpp \ ../pytsk/samples \ ../pytsk/test_data" FILES="\ ${PYTSK_SOURCE_FILES} \ ${TALLOC_SOURCE_FILES} \ ${SCRIPTS} \ ${DATA_FILES}" echo "Creating: pytsk-${VERSION}.tgz" tar zcf pytsk-${VERSION}.tgz --exclude __pycache__ ${FILES} 2>/dev/null pytsk-20231007/misc.h000066400000000000000000000054671451023402500142400ustar00rootroot00000000000000/* Miscellaneous definitions. * * Copyright 2010, Michael Cohen . * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef _PYTSK_MISC_H #define _PYTSK_MISC_H #include #if defined( HAVE_INTTYPES_H ) #include #elif !defined( _MSC_VER ) #include #endif #if defined( WIN32 ) #include #include #include #include #else /* sys/types.h needs to be included before sys/socket.h on * some platforms like FreeBSD. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #endif /* defined( WIN32 ) */ #ifdef __cplusplus extern "C" { #endif #if defined( _MSC_VER ) #define DLL_PUBLIC __declspec(dllexport) #elif !defined( HEADERS_ONLY ) #define DLL_PUBLIC __attribute__ ((visibility("default"))) #else #define DLL_PUBLIC #endif /* Used by class parser */ #if defined( _MSC_VER ) #define UNUSED #else #define UNUSED __attribute__((unused)) #endif #if !defined( PYTSK3_ATTRIBUTE_UNUSED ) #if defined( __GNUC__ ) && __GNUC__ >= 3 #define PYTSK3_ATTRIBUTE_UNUSED __attribute__ ((__unused__)) #else #define PYTSK3_ATTRIBUTE_UNUSED #endif #endif #if defined( _MSC_VER ) #define PYTSK3_UNREFERENCED_PARAMETER( parameter ) \ UNREFERENCED_PARAMETER( parameter ); #else #define PYTSK3_UNREFERENCED_PARAMETER( parameter ) \ /* parameter */ #endif #if !defined( _MSC_VER ) #ifdef min #undef min #endif #define min(X, Y) ((X) < (Y) ? (X) : (Y)) #ifdef max #undef max #endif #define max(X, Y) ((X) > (Y) ? (X) : (Y)) #endif /* if !defined( _MSC_VER ) */ #ifndef MIN #define MIN(a,b) ((a)<(b)?(a):(b)) #endif #ifndef MAX #define MAX(a,b) ((a)>(b)?(a):(b)) #endif #if defined( _MSC_VER ) #if !defined( HAVE_SSIZE_T ) #define HAVE_SSIZE_T #if defined( MS_WIN64 ) typedef __int64 ssize_t; #else typedef _W64 int ssize_t; #endif #endif /* !defined( HAVE_SSIZE_T ) */ #endif /* defined( _MSC_VER ) */ #if defined( WIN32 ) #define MSG_NOSIGNAL 0 typedef unsigned long int in_addr_t; #else #define O_BINARY 0 #endif #define true 1 #define false 0 #ifdef __cplusplus } #endif #endif pytsk-20231007/patches/000077500000000000000000000000001451023402500145475ustar00rootroot00000000000000pytsk-20231007/patches/sleuthkit-4.12.1-configure.ac000066400000000000000000000016371451023402500215770ustar00rootroot00000000000000diff --git a/configure.ac b/configure.ac index 94c0c3b7..354e666e 100644 --- a/configure.ac +++ b/configure.ac @@ -407,31 +407,8 @@ AC_CONFIG_FILES([ tsk/img/Makefile tsk/vs/Makefile tsk/fs/Makefile - tsk/hashdb/Makefile - tsk/auto/Makefile tsk/pool/Makefile - tsk/util/Makefile - tools/Makefile - tools/imgtools/Makefile - tools/vstools/Makefile - tools/fstools/Makefile - tools/hashtools/Makefile - tools/srchtools/Makefile - tools/autotools/Makefile - tools/pooltools/Makefile - tools/sorter/Makefile - tools/timeline/Makefile - tools/fiwalk/Makefile - tools/fiwalk/src/Makefile - tools/fiwalk/plugins/Makefile - tests/Makefile - samples/Makefile - man/Makefile - bindings/java/Makefile - bindings/java/jni/Makefile - case-uco/java/Makefile - unit_tests/Makefile - unit_tests/base/Makefile]) + tsk/util/Makefile]) AC_OUTPUT pytsk-20231007/pylintrc000066400000000000000000000200471451023402500147120ustar00rootroot00000000000000# Original file copied from: # http://src.chromium.org/chrome/trunk/tools/depot_tools/pylintrc [MASTER] # Specify a configuration file. #rcfile= # Python code to execute, usually for sys.path manipulation such as # pygtk.require(). #init-hook= # Add files or directories to the blacklist. They should be base names, not # paths. ignore=CVS # Pickle collected data for later comparisons. persistent=yes # List of plugins (as comma separated values of python modules names) to load, # usually to register additional checkers. load-plugins= [MESSAGES CONTROL] # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option # multiple time. #enable= # Disable the message, report, category or checker with the given id(s). You # can either give multiple identifier separated by comma (,) or put this option # multiple time (only on the command line, not in the configuration file where # it should appear only once). # CHANGED: # # C0103: Invalid name "" # C0302: Too many lines in module (N) # # I0010: Unable to consider inline option '' # I0011: Locally disabling WNNNN # # R0201: Method could be a function # R0801: Similar lines in N files # R0901: Too many ancestors (N/7) # R0902: Too many instance attributes (N/7) # R0903: Too few public methods (N/2) # R0904: Too many public methods (N/20) # R0911: Too many return statements (N/6) # R0912: Too many branches (N/12) # R0913: Too many arguments (N/5) # R0914: Too many local variables (N/15) # R0915: Too many statements (N/50) # R0921: Abstract class not referenced # R0922: Abstract class is only referenced 1 times # # W0141: Used builtin function '' # W0142: Used * or ** magic # W0402: Uses of a deprecated module 'string' # W0404: 41: Reimport 'XX' (imported line NN) # W0511: TODO # W1201: Specify string format arguments as logging function parameters # # Disabled: # deprecated-lambda # locally-enabled # logging-format-interpolation # no-member # redefined-variable-type # relative-import # simplifiable-if-statement # too-many-boolean-expressions (N/5) # too-many-nested-blocks (N/5) # ungrouped-imports disable=C0103,C0302,I0010,I0011,R0201,R0801,R0901,R0902,R0903,R0904,R0911,R0912,R0913,R0914,R0915,R0921,R0922,W0141,W0142,W0402,W0404,W0511,W1201,deprecated-lambda,locally-enabled,logging-format-interpolation,no-member,redefined-variable-type,relative-import,simplifiable-if-statement,too-many-boolean-expressions,too-many-nested-blocks,ungrouped-imports [REPORTS] # Set the output format. Available formats are text, parseable, colorized, msvs # (visual studio) and html output-format=text # Put messages in a separate file for each module / package specified on the # command line instead of printing them on stdout. Reports (if any) will be # written in a file name "pylint_global.[txt|html]". files-output=no # Tells whether to display a full report or only the messages # CHANGED: reports=no # Python expression which should return a note less than 10 (10 is the highest # note). You have access to the variables errors warning, statement which # respectively contain the number of errors / warnings messages and the total # number of statements analyzed. This is used by the global evaluation report # (RP0004). evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) [VARIABLES] # Tells whether we should check for unused import in __init__ files. init-import=no # A regular expression matching the beginning of the name of unused variables. # By default this is _ and dummy but we prefer _ and unused. dummy-variables-rgx=_|unused # List of additional names supposed to be defined in builtins. Remember that # you should avoid to define new builtins when possible. additional-builtins= [TYPECHECK] # Tells whether missing members accessed in mixin class should be ignored. A # mixin class is detected if its name ends with "mixin" (case insensitive). ignore-mixin-members=yes # List of classes names for which member attributes should not be checked # (useful for classes with attributes dynamically set). ignored-classes=SQLObject,twisted.internet.reactor,hashlib,google.appengine.api.memcache # List of members which are set dynamically and missed by pylint inference # system, and so shouldn't trigger E0201 when accessed. Python regular # expressions are accepted. generated-members=REQUEST,acl_users,aq_parent,multiprocessing.managers.SyncManager [MISCELLANEOUS] # List of note tags to take in consideration, separated by a comma. notes=FIXME,XXX,TODO [SIMILARITIES] # Minimum lines number of a similarity. min-similarity-lines=4 # Ignore comments when computing similarities. ignore-comments=yes # Ignore docstrings when computing similarities. ignore-docstrings=yes [FORMAT] # Maximum number of characters on a single line. max-line-length=80 # Maximum number of lines in a module max-module-lines=1000 # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 # tab). # CHANGED: indent-string=' ' [BASIC] # List of builtins function names that should not be used, separated by a comma bad-functions=map,filter,apply,input # Regular expression which should only match correct module names module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ # Regular expression which should only match correct module level names const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ # Regular expression which should only match correct class names class-rgx=[A-Z_][a-zA-Z0-9]+$ # Regular expression which should only match correct function names function-rgx=[a-z_][a-z0-9_]{2,30}$ # Regular expression which should only match correct method names method-rgx=[a-z_][a-z0-9_]{2,30}$ # Regular expression which should only match correct instance attribute names attr-rgx=[a-z_][a-z0-9_]{2,30}$ # Regular expression which should only match correct argument names argument-rgx=[a-z_][a-z0-9_]{2,30}$ # Regular expression which should only match correct variable names variable-rgx=[a-z_][a-z0-9_]{2,30}$ # Regular expression which should only match correct list comprehension / # generator expression variable names inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ # Good variable names which should always be accepted, separated by a comma good-names=i,j,k,ex,Run,_ # Bad variable names which should always be refused, separated by a comma bad-names=foo,bar,baz,toto,tutu,tata # Regular expression which should only match functions or classes name which do # not require a docstring no-docstring-rgx=__.*__ [DESIGN] # Maximum number of arguments for function / method max-args=5 # Argument names that match this expression will be ignored. Default to name # with leading underscore ignored-argument-names=_.* # Maximum number of locals for function / method body max-locals=15 # Maximum number of return / yield for function / method body max-returns=6 # Maximum number of branch for function / method body max-branchs=12 # Maximum number of statements in function / method body max-statements=50 # Maximum number of parents for a class (see R0901). max-parents=7 # Maximum number of attributes for a class (see R0902). max-attributes=7 # Minimum number of public methods for a class (see R0903). min-public-methods=2 # Maximum number of public methods for a class (see R0904). max-public-methods=20 [CLASSES] # List of method names used to declare (i.e. assign) instance attributes. defining-attr-methods=__init__,__new__,setUp # List of valid names for the first argument in a class method. valid-classmethod-first-arg=cls [IMPORTS] # Deprecated modules which should not be used, separated by a comma deprecated-modules=regsub,string,TERMIOS,Bastion,rexec # Create a graph of every (i.e. internal and external) dependencies in the # given file (report RP0402 must not be disabled) import-graph= # Create a graph of external dependencies in the given file (report RP0402 must # not be disabled) ext-import-graph= # Create a graph of internal dependencies in the given file (report RP0402 must # not be disabled) int-import-graph= [EXCEPTIONS] # Exceptions that will emit a warning when being caught. Defaults to # "Exception" overgeneral-exceptions=Exception pytsk-20231007/pyproject.toml000066400000000000000000000001331451023402500160310ustar00rootroot00000000000000[build-system] requires = ["setuptools", "wheel"] build-backend = "setuptools.build_meta" pytsk-20231007/pytsk3.h000066400000000000000000000006111451023402500145240ustar00rootroot00000000000000/* ** pytsk3.h ** ** Made by mic ** Login ** ** Started on Sat Apr 17 20:48:58 2010 mic ** Last update Sat Apr 17 20:48:58 2010 mic this is a shadow file: Do not directly include it - we redefine some of TSK specific structs so we can bind them here. */ #ifndef PYTSK3_H_ # define PYTSK3_H_ #include #include "class.h" #endif /* !PYTSK3_H_ */ pytsk-20231007/run_tests.py000077500000000000000000000017411451023402500155260ustar00rootroot00000000000000#!/usr/bin/python # # Script to run tests. # # Copyright 2012, Kristinn Gudjonsson . # Copyright 2013, Joachim Metz . # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Script to run the tests.""" import unittest import sys if __name__ == "__main__": test_suite = unittest.TestLoader().discover("tests", pattern="*.py") test_results = unittest.TextTestRunner(verbosity=2).run(test_suite) if not test_results.wasSuccessful(): sys.exit(1) pytsk-20231007/setup.cfg000066400000000000000000000013471451023402500147460ustar00rootroot00000000000000[metadata] name = pytsk3 version = 20231007 description = Python bindings for the SleuthKit long_description = Python bindings for the SleuthKit author = Michael Cohen author_email = scudette@gmail.com maintainer = Joachim Metz maintainer_email = joachim.metz@gmail.com url = https://github.com/py4n6/pytsk license = Apache License, Version 2.0 license_files = LICENSE sleuthkit/licenses/cpl1.0.txt sleuthkit/licenses/IBM-LICENSE classifiers = Development Status :: 3 - Alpha Programming Language :: Python [options] python_requires = >=3.7 [bdist_rpm] release = 1 packager = Joachim Metz doc_files = LICENSE sleuthkit/licenses/cpl1.0.txt sleuthkit/licenses/IBM-LICENSE README build_requires = python-setuptools pytsk-20231007/setup.py000077500000000000000000000324461451023402500146460ustar00rootroot00000000000000#!/usr/bin/python # # Copyright 2010, Michael Cohen . # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Install the pytsk python module. You can control the installation process using the following environment variables: SLEUTHKIT_SOURCE: The path to the locally downloaded tarball of the sleuthkit. If not specified we download from the internet. SLEUTHKIT_PATH: A path to the locally build sleuthkit source tree. If not specified we use SLEUTHKIT_SOURCE environment variable (above). """ from __future__ import print_function import copy import glob import re import os import subprocess import sys import time from setuptools import setup, Command, Extension from setuptools.command.build_ext import build_ext from setuptools.command.sdist import sdist import distutils.ccompiler from distutils import log from distutils.ccompiler import new_compiler from distutils.dep_util import newer_group # Change PYTHONPATH. sys.path.insert(0, '.') import generate_bindings version_tuple = (sys.version_info[0], sys.version_info[1]) if version_tuple < (3, 7): print(( 'Unsupported Python version: {0:s}, version 3.7 or higher ' 'required.').format(sys.version)) sys.exit(1) class BuildExtCommand(build_ext): """Custom handler for the build_ext command.""" def build_extension(self, extension): """Builds the extension. Args: extension: distutils extension object. """ if (extension.sources is None or not isinstance(extension.sources, (list, tuple))): raise errors.DistutilsSetupError(( 'in \'ext_modules\' option (extension \'{0:s}\'), ' '\'sources\' must be present and must be ' 'a list of source filenames').format(extension.name)) extension_path = self.get_ext_fullpath(extension.name) depends = extension.sources + extension.depends if not (self.force or newer_group(depends, extension_path, 'newer')): log.debug('skipping \'%s\' extension (up-to-date)', extension.name) return log.info('building \'%s\' extension', extension.name) # C and C++ source files need to be compiled seperately otherwise # the extension will not build on Mac OS. c_sources = [] cxx_sources = [] for source in extension.sources: if source.endswith('.c'): c_sources.append(source) else: cxx_sources.append(source) objects = [] for lang, sources in (('c', c_sources), ('c++', cxx_sources)): extra_args = extension.extra_compile_args or [] if lang == 'c++': if self.compiler.compiler_type == 'msvc': extra_args.append('/EHsc') else: extra_args.append('-std=c++14') macros = extension.define_macros[:] for undef in extension.undef_macros: macros.append((undef,)) compiled_objects = self.compiler.compile( sources, output_dir=self.build_temp, macros=macros, include_dirs=extension.include_dirs, debug=self.debug, extra_postargs=extra_args, depends=extension.depends) objects.extend(compiled_objects) self._built_objects = objects[:] if extension.extra_objects: objects.extend(extension.extra_objects) extra_args = extension.extra_link_args or [] # When MinGW32 is used statically link libgcc and libstdc++. if self.compiler.compiler_type == 'mingw32': extra_args.extend(['-static-libgcc', '-static-libstdc++']) # Now link the object files together into a "shared object" -- # of course, first we have to figure out all the other things # that go into the mix. if extension.extra_objects: objects.extend(extension.extra_objects) extra_args = extension.extra_link_args or [] # Detect target language, if not provided language = extension.language or self.compiler.detect_language(sources) self.compiler.link_shared_object( objects, extension_path, libraries=self.get_libraries(extension), library_dirs=extension.library_dirs, runtime_library_dirs=extension.runtime_library_dirs, extra_postargs=extra_args, export_symbols=self.get_export_symbols(extension), debug=self.debug, build_temp=self.build_temp, target_lang=language) def configure_source(self, compiler): """Configures the source. Args: compiler: distutils compiler object. """ define_macros = [("HAVE_TSK_LIBTSK_H", "")] if compiler.compiler_type == "msvc": define_macros.extend([ ("WIN32", "1"), ("UNICODE", "1"), ("NOMINMAX", "1"), ("_CRT_SECURE_NO_WARNINGS", "1")]) # TODO: ("GUID_WINDOWS", "1"), else: # We want to build as much as possible self contained Python # binding. command = [ "sh", "configure", "--disable-java", "--disable-multithreading", "--without-afflib", "--without-libbfio", "--without-libewf", "--without-libvhdi", "--without-libvmdk", "--without-libvslvm", "--without-zlib"] output = subprocess.check_output(command, cwd="sleuthkit") print_line = False for line in output.split(b"\n"): line = line.rstrip() if line == b"configure:": print_line = True if print_line: if sys.version_info[0] >= 3: line = line.decode("ascii") print(line) define_macros.extend([ ("HAVE_CONFIG_H", "1"), ("LOCALEDIR", "\"/usr/share/locale\"")]) self.libraries = ["stdc++"] self.define = define_macros def run(self): compiler = new_compiler(compiler=self.compiler) # pylint: disable=attribute-defined-outside-init self.configure_source(compiler) libtsk_path = os.path.join("sleuthkit", "tsk") if not os.access("pytsk3.cpp", os.R_OK): # Generate the Python binding code (pytsk3.cpp). libtsk_header_files = [ os.path.join(libtsk_path, "libtsk.h"), os.path.join(libtsk_path, "base", "tsk_base.h"), os.path.join(libtsk_path, "fs", "tsk_fs.h"), os.path.join(libtsk_path, "img", "tsk_img.h"), os.path.join(libtsk_path, "vs", "tsk_vs.h"), "tsk3.h"] print("Generating bindings...") generate_bindings.generate_bindings( "pytsk3.cpp", libtsk_header_files, initialization="tsk_init();") build_ext.run(self) class SDistCommand(sdist): """Custom handler for generating source dist.""" def run(self): libtsk_path = os.path.join("sleuthkit", "tsk") # sleuthkit submodule is not there, probably because this has been # freshly checked out. if not os.access(libtsk_path, os.R_OK): subprocess.check_call(["git", "submodule", "init"]) subprocess.check_call(["git", "submodule", "update"]) if not os.path.exists(os.path.join("sleuthkit", "configure")): raise RuntimeError( "Missing: sleuthkit/configure run 'setup.py build' first.") sdist.run(self) class UpdateCommand(Command): """Update sleuthkit source. This is normally only run by packagers to make a new release. """ _SLEUTHKIT_GIT_TAG = "4.12.1" version = time.strftime("%Y%m%d") timezone_minutes, _ = divmod(time.timezone, 60) timezone_hours, timezone_minutes = divmod(timezone_minutes, 60) # If timezone_hours is -1 %02d will format as -1 instead of -01 # hence we detect the sign and force a leading zero. if timezone_hours < 0: timezone_string = "-%02d%02d" % (-timezone_hours, timezone_minutes) else: timezone_string = "+%02d%02d" % (timezone_hours, timezone_minutes) version_pkg = "%s %s" % ( time.strftime("%a, %d %b %Y %H:%M:%S"), timezone_string) user_options = [("use-head", None, ( "Use the latest version of Sleuthkit checked into git (HEAD) instead of " "tag: {0:s}".format(_SLEUTHKIT_GIT_TAG)))] def initialize_options(self): self.use_head = False def finalize_options(self): self.use_head = bool(self.use_head) files = { "sleuthkit/Makefile.am": [ ("SUBDIRS = .+", "SUBDIRS = tsk"), ], "class_parser.py": [ ('VERSION = "[^"]+"', 'VERSION = "%s"' % version), ], "dpkg/changelog": [ (r"pytsk3 \([^\)]+\)", "pytsk3 (%s-1)" % version), ("(<[^>]+>).+", r"\1 %s" % version_pkg), ], } def patch_sleuthkit(self): """Applies patches to the SleuthKit source code.""" for filename, rules in iter(self.files.items()): filename = os.path.join(*filename.split("/")) with open(filename, "r") as file_object: data = file_object.read() for search, replace in rules: data = re.sub(search, replace, data) with open(filename, "w") as fd: fd.write(data) patch_files = [ "sleuthkit-{0:s}-configure.ac".format(self._SLEUTHKIT_GIT_TAG)] for patch_file in patch_files: patch_file = os.path.join("patches", patch_file) if not os.path.exists(patch_file): print("No such patch file: {0:s}".format(patch_file)) continue patch_file = os.path.join("..", patch_file) subprocess.check_call(["git", "apply", patch_file], cwd="sleuthkit") def run(self): subprocess.check_call(["git", "stash"], cwd="sleuthkit") subprocess.check_call(["git", "submodule", "init"]) subprocess.check_call(["git", "submodule", "update"]) print("Updating sleuthkit") subprocess.check_call(["git", "reset", "--hard"], cwd="sleuthkit") subprocess.check_call(["git", "clean", "-x", "-f", "-d"], cwd="sleuthkit") subprocess.check_call(["git", "checkout", "master"], cwd="sleuthkit") subprocess.check_call(["git", "pull"], cwd="sleuthkit") if self.use_head: print("Pulling from HEAD") else: print("Pulling from tag: {0:s}".format(self._SLEUTHKIT_GIT_TAG)) subprocess.check_call(["git", "fetch", "--force", "--tags"], cwd="sleuthkit") git_tag_path = "tags/sleuthkit-{0:s}".format(self._SLEUTHKIT_GIT_TAG) subprocess.check_call(["git", "checkout", git_tag_path], cwd="sleuthkit") self.patch_sleuthkit() compiler_type = distutils.ccompiler.get_default_compiler() if compiler_type != "msvc": subprocess.check_call(["./bootstrap"], cwd="sleuthkit") # Now derive the version based on the date. with open("setup.cfg", "r", encoding="utf-8") as file_object: setup_cfg_lines = file_object.readlines() with open("setup.cfg", "w", encoding="utf-8") as file_object: for line in setup_cfg_lines: if line.startswith("version = "): line = "version = {0:s}\n".format(self.version) file_object.write(line) libtsk_path = os.path.join("sleuthkit", "tsk") # Generate the Python binding code (pytsk3.cpp). libtsk_header_files = [ os.path.join(libtsk_path, "libtsk.h"), os.path.join(libtsk_path, "base", "tsk_base.h"), os.path.join(libtsk_path, "fs", "tsk_fs.h"), os.path.join(libtsk_path, "img", "tsk_img.h"), os.path.join(libtsk_path, "vs", "tsk_vs.h"), "tsk3.h"] print("Generating bindings...") generate_bindings.generate_bindings( "pytsk3.cpp", libtsk_header_files, initialization="tsk_init();") class ProjectBuilder(object): """Class to help build the project.""" def __init__(self, argv): """Initializes a project builder object.""" self._argv = argv # The path to the sleuthkit/tsk directory. self._libtsk_path = os.path.join("sleuthkit", "tsk") # Paths under the sleuthkit/tsk directory which contain files we need # to compile. self._sub_library_names = ["base", "docs", "fs", "img", "pool", "util", "vs"] # The args for the extension builder. self.extension_args = { "include_dirs": ["talloc", self._libtsk_path, "sleuthkit", "."], "library_dirs": []} # The sources to build. self._source_files = [ "class.cpp", "error.cpp", "tsk3.cpp", "pytsk3.cpp", "talloc/talloc.c"] # Path to the top of the unpacked sleuthkit sources. self._sleuthkit_path = "sleuthkit" def build(self): """Build everything.""" # Fetch all c and cpp files from the subdirs to compile. extension_file = os.path.join( self._libtsk_path, "auto", "guid.cpp") self._source_files.append(extension_file) for library_name in self._sub_library_names: for extension in ("*.c", "*.cpp"): extension_glob = os.path.join( self._libtsk_path, library_name, extension) self._source_files.extend(glob.glob(extension_glob)) # Sort the soure files to make sure they are in consistent order when # building. source_files = sorted(self._source_files) ext_modules = [Extension("pytsk3", source_files, **self.extension_args)] setup_args = dict( cmdclass={ "build_ext": BuildExtCommand, "sdist": SDistCommand, "update": UpdateCommand}, ext_modules=ext_modules) setup(**setup_args) if __name__ == "__main__": ProjectBuilder(sys.argv).build() pytsk-20231007/sleuthkit/000077500000000000000000000000001451023402500151345ustar00rootroot00000000000000pytsk-20231007/talloc/000077500000000000000000000000001451023402500143765ustar00rootroot00000000000000pytsk-20231007/talloc/LICENSE000066400000000000000000000167301451023402500154120ustar00rootroot00000000000000 GNU LESSER GENERAL PUBLIC LICENSE Version 3, 29 June 2007 Copyright (C) 2007 Free Software Foundation, Inc. Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. This version of the GNU Lesser General Public License incorporates the terms and conditions of version 3 of the GNU General Public License, supplemented by the additional permissions listed below. 0. Additional Definitions. As used herein, "this License" refers to version 3 of the GNU Lesser General Public License, and the "GNU GPL" refers to version 3 of the GNU General Public License. "The Library" refers to a covered work governed by this License, other than an Application or a Combined Work as defined below. An "Application" is any work that makes use of an interface provided by the Library, but which is not otherwise based on the Library. Defining a subclass of a class defined by the Library is deemed a mode of using an interface provided by the Library. A "Combined Work" is a work produced by combining or linking an Application with the Library. The particular version of the Library with which the Combined Work was made is also called the "Linked Version". The "Minimal Corresponding Source" for a Combined Work means the Corresponding Source for the Combined Work, excluding any source code for portions of the Combined Work that, considered in isolation, are based on the Application, and not on the Linked Version. The "Corresponding Application Code" for a Combined Work means the object code and/or source code for the Application, including any data and utility programs needed for reproducing the Combined Work from the Application, but excluding the System Libraries of the Combined Work. 1. Exception to Section 3 of the GNU GPL. You may convey a covered work under sections 3 and 4 of this License without being bound by section 3 of the GNU GPL. 2. Conveying Modified Versions. If you modify a copy of the Library, and, in your modifications, a facility refers to a function or data to be supplied by an Application that uses the facility (other than as an argument passed when the facility is invoked), then you may convey a copy of the modified version: a) under this License, provided that you make a good faith effort to ensure that, in the event an Application does not supply the function or data, the facility still operates, and performs whatever part of its purpose remains meaningful, or b) under the GNU GPL, with none of the additional permissions of this License applicable to that copy. 3. Object Code Incorporating Material from Library Header Files. The object code form of an Application may incorporate material from a header file that is part of the Library. You may convey such object code under terms of your choice, provided that, if the incorporated material is not limited to numerical parameters, data structure layouts and accessors, or small macros, inline functions and templates (ten or fewer lines in length), you do both of the following: a) Give prominent notice with each copy of the object code that the Library is used in it and that the Library and its use are covered by this License. b) Accompany the object code with a copy of the GNU GPL and this license document. 4. Combined Works. You may convey a Combined Work under terms of your choice that, taken together, effectively do not restrict modification of the portions of the Library contained in the Combined Work and reverse engineering for debugging such modifications, if you also do each of the following: a) Give prominent notice with each copy of the Combined Work that the Library is used in it and that the Library and its use are covered by this License. b) Accompany the Combined Work with a copy of the GNU GPL and this license document. c) For a Combined Work that displays copyright notices during execution, include the copyright notice for the Library among these notices, as well as a reference directing the user to the copies of the GNU GPL and this license document. d) Do one of the following: 0) Convey the Minimal Corresponding Source under the terms of this License, and the Corresponding Application Code in a form suitable for, and under terms that permit, the user to recombine or relink the Application with a modified version of the Linked Version to produce a modified Combined Work, in the manner specified by section 6 of the GNU GPL for conveying Corresponding Source. 1) Use a suitable shared library mechanism for linking with the Library. A suitable mechanism is one that (a) uses at run time a copy of the Library already present on the user's computer system, and (b) will operate properly with a modified version of the Library that is interface-compatible with the Linked Version. e) Provide Installation Information, but only if you would otherwise be required to provide such information under section 6 of the GNU GPL, and only to the extent that such information is necessary to install and execute a modified version of the Combined Work produced by recombining or relinking the Application with a modified version of the Linked Version. (If you use option 4d0, the Installation Information must accompany the Minimal Corresponding Source and Corresponding Application Code. If you use option 4d1, you must provide the Installation Information in the manner specified by section 6 of the GNU GPL for conveying Corresponding Source.) 5. Combined Libraries. You may place library facilities that are a work based on the Library side by side in a single library together with other library facilities that are not Applications and are not covered by this License, and convey such a combined library under terms of your choice, if you do both of the following: a) Accompany the combined library with a copy of the same work based on the Library, uncombined with any other library facilities, conveyed under the terms of this License. b) Give prominent notice with the combined library that part of it is a work based on the Library, and explaining where to find the accompanying uncombined form of the same work. 6. Revised Versions of the GNU Lesser General Public License. The Free Software Foundation may publish revised and/or new versions of the GNU Lesser General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Library as you received it specifies that a certain numbered version of the GNU Lesser General Public License "or any later version" applies to it, you have the option of following the terms and conditions either of that published version or of any later version published by the Free Software Foundation. If the Library as you received it does not specify a version number of the GNU Lesser General Public License, you may choose any version of the GNU Lesser General Public License ever published by the Free Software Foundation. If the Library as you received it specifies that a proxy can decide whether future versions of the GNU Lesser General Public License shall apply, that proxy's public statement of acceptance of any version is permanent authorization for you to choose that version for the Library. pytsk-20231007/talloc/README000066400000000000000000000006521451023402500152610ustar00rootroot00000000000000Talloc is part of the Samba project and can be found at: http://talloc.samba.org/talloc/doc/html/index.html It is licensed under the: GNU Lesser General Public License. See the corresponding LICENSE file or http://www.gnu.org/licenses/ The files talloc.c and talloc.h are unaltered copies taken from: http://www.samba.org/ftp/talloc/talloc-2.1.0.tar.gz replace.h was added to force talloc.c to compile on various systems. pytsk-20231007/talloc/replace.h000066400000000000000000000010321451023402500161560ustar00rootroot00000000000000#ifndef _REPLACE_H_ #define _REPLACE_H_ #include #include #if !defined( UINT_MAX ) #include #endif #define _PUBLIC_ extern typedef int bool; #define true 1 #define false 0 typedef unsigned char uint8_t; #if !defined( MIN ) #define MIN(a,b) ((a)<(b)?(a):(b)) #endif #if defined( _MSC_VER ) #define inline /* inline */ #if defined( MS_WIN64 ) typedef __int64 ssize_t; #else typedef _W64 int ssize_t; #endif #else #define HAVE_VA_COPY #endif /* defined( _MSC_VER ) */ #endif /* _REPLACE_H_ */ pytsk-20231007/talloc/talloc.c000066400000000000000000002010531451023402500160210ustar00rootroot00000000000000/* Samba Unix SMB/CIFS implementation. Samba trivial allocation library - new interface NOTE: Please read talloc_guide.txt for full documentation Copyright (C) Andrew Tridgell 2004 Copyright (C) Stefan Metzmacher 2006 ** NOTE! The following LGPL license applies to the talloc ** library. This does NOT imply that all of Samba is released ** under the LGPL This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation; either version 3 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. You should have received a copy of the GNU Lesser General Public License along with this library; if not, see . */ /* inspired by http://swapped.cc/halloc/ */ #include "replace.h" #include "talloc.h" #include #ifdef TALLOC_BUILD_VERSION_MAJOR #if (TALLOC_VERSION_MAJOR != TALLOC_BUILD_VERSION_MAJOR) #error "TALLOC_VERSION_MAJOR != TALLOC_BUILD_VERSION_MAJOR" #endif #endif #ifdef TALLOC_BUILD_VERSION_MINOR #if (TALLOC_VERSION_MINOR != TALLOC_BUILD_VERSION_MINOR) #error "TALLOC_VERSION_MINOR != TALLOC_BUILD_VERSION_MINOR" #endif #endif /* Special macros that are no-ops except when run under Valgrind on * x86. They've moved a little bit from valgrind 1.0.4 to 1.9.4 */ #ifdef HAVE_VALGRIND_MEMCHECK_H /* memcheck.h includes valgrind.h */ #include #elif defined(HAVE_VALGRIND_H) #include #endif /* use this to force every realloc to change the pointer, to stress test code that might not cope */ #define ALWAYS_REALLOC 0 #define MAX_TALLOC_SIZE 0x10000000 #define TALLOC_MAGIC_BASE 0xe814ec70 #define TALLOC_MAGIC ( \ TALLOC_MAGIC_BASE + \ (TALLOC_VERSION_MAJOR << 12) + \ (TALLOC_VERSION_MINOR << 4) \ ) #define TALLOC_FLAG_FREE 0x01 #define TALLOC_FLAG_LOOP 0x02 #define TALLOC_FLAG_POOL 0x04 /* This is a talloc pool */ #define TALLOC_FLAG_POOLMEM 0x08 /* This is allocated in a pool */ #define TALLOC_MAGIC_REFERENCE ((const char *)1) /* by default we abort when given a bad pointer (such as when talloc_free() is called on a pointer that came from malloc() */ #ifndef TALLOC_ABORT #define TALLOC_ABORT(reason) abort() #endif #ifndef discard_const_p #if defined(__intptr_t_defined) || defined(HAVE_INTPTR_T) # define discard_const_p(type, ptr) ((type *)((intptr_t)(ptr))) #else # define discard_const_p(type, ptr) ((type *)(ptr)) #endif #endif /* these macros gain us a few percent of speed on gcc */ #if (__GNUC__ >= 3) /* the strange !! is to ensure that __builtin_expect() takes either 0 or 1 as its first argument */ #ifndef likely #define likely(x) __builtin_expect(!!(x), 1) #endif #ifndef unlikely #define unlikely(x) __builtin_expect(!!(x), 0) #endif #else #ifndef likely #define likely(x) (x) #endif #ifndef unlikely #define unlikely(x) (x) #endif #endif /* this null_context is only used if talloc_enable_leak_report() or talloc_enable_leak_report_full() is called, otherwise it remains NULL */ static void *null_context; static void *autofree_context; /* used to enable fill of memory on free, which can be useful for * catching use after free errors when valgrind is too slow */ static struct { bool initialised; bool enabled; uint8_t fill_value; } talloc_fill; #define TALLOC_FILL_ENV "TALLOC_FREE_FILL" /* * do not wipe the header, to allow the * double-free logic to still work */ #define TC_INVALIDATE_FULL_FILL_CHUNK(_tc) do { \ if (unlikely(talloc_fill.enabled)) { \ size_t _flen = (_tc)->size; \ char *_fptr = (char *)TC_PTR_FROM_CHUNK(_tc); \ memset(_fptr, talloc_fill.fill_value, _flen); \ } \ } while (0) #if defined(DEVELOPER) && defined(VALGRIND_MAKE_MEM_NOACCESS) /* Mark the whole chunk as not accessable */ #define TC_INVALIDATE_FULL_VALGRIND_CHUNK(_tc) do { \ size_t _flen = TC_HDR_SIZE + (_tc)->size; \ char *_fptr = (char *)(_tc); \ VALGRIND_MAKE_MEM_NOACCESS(_fptr, _flen); \ } while(0) #else #define TC_INVALIDATE_FULL_VALGRIND_CHUNK(_tc) do { } while (0) #endif #define TC_INVALIDATE_FULL_CHUNK(_tc) do { \ TC_INVALIDATE_FULL_FILL_CHUNK(_tc); \ TC_INVALIDATE_FULL_VALGRIND_CHUNK(_tc); \ } while (0) #define TC_INVALIDATE_SHRINK_FILL_CHUNK(_tc, _new_size) do { \ if (unlikely(talloc_fill.enabled)) { \ size_t _flen = (_tc)->size - (_new_size); \ char *_fptr = (char *)TC_PTR_FROM_CHUNK(_tc); \ _fptr += (_new_size); \ memset(_fptr, talloc_fill.fill_value, _flen); \ } \ } while (0) #if defined(DEVELOPER) && defined(VALGRIND_MAKE_MEM_NOACCESS) /* Mark the unused bytes not accessable */ #define TC_INVALIDATE_SHRINK_VALGRIND_CHUNK(_tc, _new_size) do { \ size_t _flen = (_tc)->size - (_new_size); \ char *_fptr = (char *)TC_PTR_FROM_CHUNK(_tc); \ _fptr += (_new_size); \ VALGRIND_MAKE_MEM_NOACCESS(_fptr, _flen); \ } while (0) #else #define TC_INVALIDATE_SHRINK_VALGRIND_CHUNK(_tc, _new_size) do { } while (0) #endif #define TC_INVALIDATE_SHRINK_CHUNK(_tc, _new_size) do { \ TC_INVALIDATE_SHRINK_FILL_CHUNK(_tc, _new_size); \ TC_INVALIDATE_SHRINK_VALGRIND_CHUNK(_tc, _new_size); \ } while (0) #define TC_UNDEFINE_SHRINK_FILL_CHUNK(_tc, _new_size) do { \ if (unlikely(talloc_fill.enabled)) { \ size_t _flen = (_tc)->size - (_new_size); \ char *_fptr = (char *)TC_PTR_FROM_CHUNK(_tc); \ _fptr += (_new_size); \ memset(_fptr, talloc_fill.fill_value, _flen); \ } \ } while (0) #if defined(DEVELOPER) && defined(VALGRIND_MAKE_MEM_UNDEFINED) /* Mark the unused bytes as undefined */ #define TC_UNDEFINE_SHRINK_VALGRIND_CHUNK(_tc, _new_size) do { \ size_t _flen = (_tc)->size - (_new_size); \ char *_fptr = (char *)TC_PTR_FROM_CHUNK(_tc); \ _fptr += (_new_size); \ VALGRIND_MAKE_MEM_UNDEFINED(_fptr, _flen); \ } while (0) #else #define TC_UNDEFINE_SHRINK_VALGRIND_CHUNK(_tc, _new_size) do { } while (0) #endif #define TC_UNDEFINE_SHRINK_CHUNK(_tc, _new_size) do { \ TC_UNDEFINE_SHRINK_FILL_CHUNK(_tc, _new_size); \ TC_UNDEFINE_SHRINK_VALGRIND_CHUNK(_tc, _new_size); \ } while (0) #if defined(DEVELOPER) && defined(VALGRIND_MAKE_MEM_UNDEFINED) /* Mark the new bytes as undefined */ #define TC_UNDEFINE_GROW_VALGRIND_CHUNK(_tc, _new_size) do { \ size_t _old_used = TC_HDR_SIZE + (_tc)->size; \ size_t _new_used = TC_HDR_SIZE + (_new_size); \ size_t _flen = _new_used - _old_used; \ char *_fptr = _old_used + (char *)(_tc); \ VALGRIND_MAKE_MEM_UNDEFINED(_fptr, _flen); \ } while (0) #else #define TC_UNDEFINE_GROW_VALGRIND_CHUNK(_tc, _new_size) do { } while (0) #endif #define TC_UNDEFINE_GROW_CHUNK(_tc, _new_size) do { \ TC_UNDEFINE_GROW_VALGRIND_CHUNK(_tc, _new_size); \ } while (0) struct talloc_reference_handle { struct talloc_reference_handle *next, *prev; void *ptr; const char *location; }; struct talloc_memlimit { struct talloc_chunk *parent; struct talloc_memlimit *upper; size_t max_size; size_t cur_size; }; static bool talloc_memlimit_check(struct talloc_memlimit *limit, size_t size); static void talloc_memlimit_grow(struct talloc_memlimit *limit, size_t size); static void talloc_memlimit_shrink(struct talloc_memlimit *limit, size_t size); static void talloc_memlimit_update_on_free(struct talloc_chunk *tc); typedef int (*talloc_destructor_t)(void *); struct talloc_pool_hdr; struct talloc_chunk { struct talloc_chunk *next, *prev; struct talloc_chunk *parent, *child; struct talloc_reference_handle *refs; talloc_destructor_t destructor; const char *name; size_t size; unsigned flags; /* * limit semantics: * if 'limit' is set it means all *new* children of the context will * be limited to a total aggregate size ox max_size for memory * allocations. * cur_size is used to keep track of the current use */ struct talloc_memlimit *limit; /* * For members of a pool (i.e. TALLOC_FLAG_POOLMEM is set), "pool" * is a pointer to the struct talloc_chunk of the pool that it was * allocated from. This way children can quickly find the pool to chew * from. */ struct talloc_pool_hdr *pool; }; /* 16 byte alignment seems to keep everyone happy */ #define TC_ALIGN16(s) (((s)+15)&~15) #define TC_HDR_SIZE TC_ALIGN16(sizeof(struct talloc_chunk)) #define TC_PTR_FROM_CHUNK(tc) ((void *)(TC_HDR_SIZE + (char*)tc)) _PUBLIC_ int talloc_version_major(void) { return TALLOC_VERSION_MAJOR; } _PUBLIC_ int talloc_version_minor(void) { return TALLOC_VERSION_MINOR; } static void (*talloc_log_fn)(const char *message); _PUBLIC_ void talloc_set_log_fn(void (*log_fn)(const char *message)) { talloc_log_fn = log_fn; } static void talloc_log(const char *fmt, ...) PRINTF_ATTRIBUTE(1,2); static void talloc_log(const char *fmt, ...) { va_list ap; char *message; if (!talloc_log_fn) { return; } va_start(ap, fmt); message = talloc_vasprintf(NULL, fmt, ap); va_end(ap); talloc_log_fn(message); talloc_free(message); } static void talloc_log_stderr(const char *message) { fprintf(stderr, "%s", message); } _PUBLIC_ void talloc_set_log_stderr(void) { talloc_set_log_fn(talloc_log_stderr); } static void (*talloc_abort_fn)(const char *reason); _PUBLIC_ void talloc_set_abort_fn(void (*abort_fn)(const char *reason)) { talloc_abort_fn = abort_fn; } static void talloc_abort(const char *reason) { talloc_log("%s\n", reason); if (!talloc_abort_fn) { TALLOC_ABORT(reason); } talloc_abort_fn(reason); } static void talloc_abort_magic(unsigned magic) { unsigned striped = magic - TALLOC_MAGIC_BASE; unsigned major = (striped & 0xFFFFF000) >> 12; unsigned minor = (striped & 0x00000FF0) >> 4; talloc_log("Bad talloc magic[0x%08X/%u/%u] expected[0x%08X/%u/%u]\n", magic, major, minor, TALLOC_MAGIC, TALLOC_VERSION_MAJOR, TALLOC_VERSION_MINOR); talloc_abort("Bad talloc magic value - wrong talloc version used/mixed"); } static void talloc_abort_access_after_free(void) { talloc_abort("Bad talloc magic value - access after free"); } static void talloc_abort_unknown_value(void) { talloc_abort("Bad talloc magic value - unknown value"); } /* panic if we get a bad magic value */ static inline struct talloc_chunk *talloc_chunk_from_ptr(const void *ptr) { const char *pp = (const char *)ptr; struct talloc_chunk *tc = discard_const_p(struct talloc_chunk, pp - TC_HDR_SIZE); if (unlikely((tc->flags & (TALLOC_FLAG_FREE | ~0xF)) != TALLOC_MAGIC)) { if ((tc->flags & (~0xFFF)) == TALLOC_MAGIC_BASE) { talloc_abort_magic(tc->flags & (~0xF)); return NULL; } if (tc->flags & TALLOC_FLAG_FREE) { talloc_log("talloc: access after free error - first free may be at %s\n", tc->name); talloc_abort_access_after_free(); return NULL; } else { talloc_abort_unknown_value(); return NULL; } } return tc; } /* hook into the front of the list */ #define _TLIST_ADD(list, p) \ do { \ if (!(list)) { \ (list) = (p); \ (p)->next = (p)->prev = NULL; \ } else { \ (list)->prev = (p); \ (p)->next = (list); \ (p)->prev = NULL; \ (list) = (p); \ }\ } while (0) /* remove an element from a list - element doesn't have to be in list. */ #define _TLIST_REMOVE(list, p) \ do { \ if ((p) == (list)) { \ (list) = (p)->next; \ if (list) (list)->prev = NULL; \ } else { \ if ((p)->prev) (p)->prev->next = (p)->next; \ if ((p)->next) (p)->next->prev = (p)->prev; \ } \ if ((p) && ((p) != (list))) (p)->next = (p)->prev = NULL; \ } while (0) /* return the parent chunk of a pointer */ static inline struct talloc_chunk *talloc_parent_chunk(const void *ptr) { struct talloc_chunk *tc; if (unlikely(ptr == NULL)) { return NULL; } tc = talloc_chunk_from_ptr(ptr); while (tc->prev) tc=tc->prev; return tc->parent; } _PUBLIC_ void *talloc_parent(const void *ptr) { struct talloc_chunk *tc = talloc_parent_chunk(ptr); return tc? TC_PTR_FROM_CHUNK(tc) : NULL; } /* find parents name */ _PUBLIC_ const char *talloc_parent_name(const void *ptr) { struct talloc_chunk *tc = talloc_parent_chunk(ptr); return tc? tc->name : NULL; } /* A pool carries an in-pool object count count in the first 16 bytes. bytes. This is done to support talloc_steal() to a parent outside of the pool. The count includes the pool itself, so a talloc_free() on a pool will only destroy the pool if the count has dropped to zero. A talloc_free() of a pool member will reduce the count, and eventually also call free(3) on the pool memory. The object count is not put into "struct talloc_chunk" because it is only relevant for talloc pools and the alignment to 16 bytes would increase the memory footprint of each talloc chunk by those 16 bytes. */ struct talloc_pool_hdr { void *end; unsigned int object_count; size_t poolsize; }; #define TP_HDR_SIZE TC_ALIGN16(sizeof(struct talloc_pool_hdr)) static struct talloc_pool_hdr *talloc_pool_from_chunk(struct talloc_chunk *c) { return (struct talloc_pool_hdr *)((char *)c - TP_HDR_SIZE); } static struct talloc_chunk *talloc_chunk_from_pool(struct talloc_pool_hdr *h) { return (struct talloc_chunk *)((char *)h + TP_HDR_SIZE); } static void *tc_pool_end(struct talloc_pool_hdr *pool_hdr) { struct talloc_chunk *tc = talloc_chunk_from_pool(pool_hdr); return (char *)tc + TC_HDR_SIZE + pool_hdr->poolsize; } static size_t tc_pool_space_left(struct talloc_pool_hdr *pool_hdr) { return (char *)tc_pool_end(pool_hdr) - (char *)pool_hdr->end; } /* If tc is inside a pool, this gives the next neighbour. */ static void *tc_next_chunk(struct talloc_chunk *tc) { return (char *)tc + TC_ALIGN16(TC_HDR_SIZE + tc->size); } static void *tc_pool_first_chunk(struct talloc_pool_hdr *pool_hdr) { struct talloc_chunk *tc = talloc_chunk_from_pool(pool_hdr); return tc_next_chunk(tc); } /* Mark the whole remaining pool as not accessable */ static void tc_invalidate_pool(struct talloc_pool_hdr *pool_hdr) { size_t flen = tc_pool_space_left(pool_hdr); if (unlikely(talloc_fill.enabled)) { memset(pool_hdr->end, talloc_fill.fill_value, flen); } #if defined(DEVELOPER) && defined(VALGRIND_MAKE_MEM_NOACCESS) VALGRIND_MAKE_MEM_NOACCESS(pool_hdr->end, flen); #endif } /* Allocate from a pool */ static struct talloc_chunk *talloc_alloc_pool(struct talloc_chunk *parent, size_t size, size_t prefix_len) { struct talloc_pool_hdr *pool_hdr = NULL; size_t space_left; struct talloc_chunk *result; size_t chunk_size; if (parent == NULL) { return NULL; } if (parent->flags & TALLOC_FLAG_POOL) { pool_hdr = talloc_pool_from_chunk(parent); } else if (parent->flags & TALLOC_FLAG_POOLMEM) { pool_hdr = parent->pool; } if (pool_hdr == NULL) { return NULL; } space_left = tc_pool_space_left(pool_hdr); /* * Align size to 16 bytes */ chunk_size = TC_ALIGN16(size + prefix_len); if (space_left < chunk_size) { return NULL; } result = (struct talloc_chunk *)((char *)pool_hdr->end + prefix_len); #if defined(DEVELOPER) && defined(VALGRIND_MAKE_MEM_UNDEFINED) VALGRIND_MAKE_MEM_UNDEFINED(pool_hdr->end, chunk_size); #endif pool_hdr->end = (void *)((char *)pool_hdr->end + chunk_size); result->flags = TALLOC_MAGIC | TALLOC_FLAG_POOLMEM; result->pool = pool_hdr; pool_hdr->object_count++; return result; } /* Allocate a bit of memory as a child of an existing pointer */ static inline void *__talloc_with_prefix(const void *context, size_t size, size_t prefix_len) { struct talloc_chunk *tc = NULL; struct talloc_memlimit *limit = NULL; size_t total_len = TC_HDR_SIZE + size + prefix_len; if (unlikely(context == NULL)) { context = null_context; } if (unlikely(size >= MAX_TALLOC_SIZE)) { return NULL; } if (unlikely(total_len < TC_HDR_SIZE)) { return NULL; } if (context != NULL) { struct talloc_chunk *ptc = talloc_chunk_from_ptr(context); if (ptc->limit != NULL) { limit = ptc->limit; } tc = talloc_alloc_pool(ptc, TC_HDR_SIZE+size, prefix_len); } if (tc == NULL) { char *ptr; /* * Only do the memlimit check/update on actual allocation. */ if (!talloc_memlimit_check(limit, total_len)) { errno = ENOMEM; return NULL; } ptr = malloc(total_len); if (unlikely(ptr == NULL)) { return NULL; } tc = (struct talloc_chunk *)(ptr + prefix_len); tc->flags = TALLOC_MAGIC; tc->pool = NULL; talloc_memlimit_grow(limit, total_len); } tc->limit = limit; tc->size = size; tc->destructor = NULL; tc->child = NULL; tc->name = NULL; tc->refs = NULL; if (likely(context)) { struct talloc_chunk *parent = talloc_chunk_from_ptr(context); if (parent->child) { parent->child->parent = NULL; tc->next = parent->child; tc->next->prev = tc; } else { tc->next = NULL; } tc->parent = parent; tc->prev = NULL; parent->child = tc; } else { tc->next = tc->prev = tc->parent = NULL; } return TC_PTR_FROM_CHUNK(tc); } static inline void *__talloc(const void *context, size_t size) { return __talloc_with_prefix(context, size, 0); } /* * Create a talloc pool */ _PUBLIC_ void *talloc_pool(const void *context, size_t size) { struct talloc_chunk *tc; struct talloc_pool_hdr *pool_hdr; void *result; result = __talloc_with_prefix(context, size, TP_HDR_SIZE); if (unlikely(result == NULL)) { return NULL; } tc = talloc_chunk_from_ptr(result); pool_hdr = talloc_pool_from_chunk(tc); tc->flags |= TALLOC_FLAG_POOL; tc->size = 0; pool_hdr->object_count = 1; pool_hdr->end = result; pool_hdr->poolsize = size; tc_invalidate_pool(pool_hdr); return result; } /* * Create a talloc pool correctly sized for a basic size plus * a number of subobjects whose total size is given. Essentially * a custom allocator for talloc to reduce fragmentation. */ _PUBLIC_ void *_talloc_pooled_object(const void *ctx, size_t type_size, const char *type_name, unsigned num_subobjects, size_t total_subobjects_size) { size_t poolsize, subobjects_slack, tmp; struct talloc_chunk *tc; struct talloc_pool_hdr *pool_hdr; void *ret; poolsize = type_size + total_subobjects_size; if ((poolsize < type_size) || (poolsize < total_subobjects_size)) { goto overflow; } if (num_subobjects == UINT_MAX) { goto overflow; } num_subobjects += 1; /* the object body itself */ /* * Alignment can increase the pool size by at most 15 bytes per object * plus alignment for the object itself */ subobjects_slack = (TC_HDR_SIZE + TP_HDR_SIZE + 15) * num_subobjects; if (subobjects_slack < num_subobjects) { goto overflow; } tmp = poolsize + subobjects_slack; if ((tmp < poolsize) || (tmp < subobjects_slack)) { goto overflow; } poolsize = tmp; ret = talloc_pool(ctx, poolsize); if (ret == NULL) { return NULL; } tc = talloc_chunk_from_ptr(ret); tc->size = type_size; pool_hdr = talloc_pool_from_chunk(tc); #if defined(DEVELOPER) && defined(VALGRIND_MAKE_MEM_UNDEFINED) VALGRIND_MAKE_MEM_UNDEFINED(pool_hdr->end, type_size); #endif pool_hdr->end = ((char *)pool_hdr->end + TC_ALIGN16(type_size)); talloc_set_name_const(ret, type_name); return ret; overflow: return NULL; } /* setup a destructor to be called on free of a pointer the destructor should return 0 on success, or -1 on failure. if the destructor fails then the free is failed, and the memory can be continued to be used */ _PUBLIC_ void _talloc_set_destructor(const void *ptr, int (*destructor)(void *)) { struct talloc_chunk *tc = talloc_chunk_from_ptr(ptr); tc->destructor = destructor; } /* increase the reference count on a piece of memory. */ _PUBLIC_ int talloc_increase_ref_count(const void *ptr) { if (unlikely(!talloc_reference(null_context, ptr))) { return -1; } return 0; } /* helper for talloc_reference() this is referenced by a function pointer and should not be inline */ static int talloc_reference_destructor(struct talloc_reference_handle *handle) { struct talloc_chunk *ptr_tc = talloc_chunk_from_ptr(handle->ptr); _TLIST_REMOVE(ptr_tc->refs, handle); return 0; } /* more efficient way to add a name to a pointer - the name must point to a true string constant */ static inline void _talloc_set_name_const(const void *ptr, const char *name) { struct talloc_chunk *tc = talloc_chunk_from_ptr(ptr); tc->name = name; } /* internal talloc_named_const() */ static inline void *_talloc_named_const(const void *context, size_t size, const char *name) { void *ptr; ptr = __talloc(context, size); if (unlikely(ptr == NULL)) { return NULL; } _talloc_set_name_const(ptr, name); return ptr; } /* make a secondary reference to a pointer, hanging off the given context. the pointer remains valid until both the original caller and this given context are freed. the major use for this is when two different structures need to reference the same underlying data, and you want to be able to free the two instances separately, and in either order */ _PUBLIC_ void *_talloc_reference_loc(const void *context, const void *ptr, const char *location) { struct talloc_chunk *tc; struct talloc_reference_handle *handle; if (unlikely(ptr == NULL)) return NULL; tc = talloc_chunk_from_ptr(ptr); handle = (struct talloc_reference_handle *)_talloc_named_const(context, sizeof(struct talloc_reference_handle), TALLOC_MAGIC_REFERENCE); if (unlikely(handle == NULL)) return NULL; /* note that we hang the destructor off the handle, not the main context as that allows the caller to still setup their own destructor on the context if they want to */ talloc_set_destructor(handle, talloc_reference_destructor); handle->ptr = discard_const_p(void, ptr); handle->location = location; _TLIST_ADD(tc->refs, handle); return handle->ptr; } static void *_talloc_steal_internal(const void *new_ctx, const void *ptr); static inline void _talloc_free_poolmem(struct talloc_chunk *tc, const char *location) { struct talloc_pool_hdr *pool; struct talloc_chunk *pool_tc; void *next_tc; pool = tc->pool; pool_tc = talloc_chunk_from_pool(pool); next_tc = tc_next_chunk(tc); tc->flags |= TALLOC_FLAG_FREE; /* we mark the freed memory with where we called the free * from. This means on a double free error we can report where * the first free came from */ tc->name = location; TC_INVALIDATE_FULL_CHUNK(tc); if (unlikely(pool->object_count == 0)) { talloc_abort("Pool object count zero!"); return; } pool->object_count--; if (unlikely(pool->object_count == 1 && !(pool_tc->flags & TALLOC_FLAG_FREE))) { /* * if there is just one object left in the pool * and pool->flags does not have TALLOC_FLAG_FREE, * it means this is the pool itself and * the rest is available for new objects * again. */ pool->end = tc_pool_first_chunk(pool); tc_invalidate_pool(pool); return; } if (unlikely(pool->object_count == 0)) { /* * we mark the freed memory with where we called the free * from. This means on a double free error we can report where * the first free came from */ pool_tc->name = location; if (pool_tc->flags & TALLOC_FLAG_POOLMEM) { _talloc_free_poolmem(pool_tc, location); } else { /* * The talloc_memlimit_update_on_free() * call takes into account the * prefix TP_HDR_SIZE allocated before * the pool talloc_chunk. */ talloc_memlimit_update_on_free(pool_tc); TC_INVALIDATE_FULL_CHUNK(pool_tc); free(pool); } return; } if (pool->end == next_tc) { /* * if pool->pool still points to end of * 'tc' (which is stored in the 'next_tc' variable), * we can reclaim the memory of 'tc'. */ pool->end = tc; return; } /* * Do nothing. The memory is just "wasted", waiting for the pool * itself to be freed. */ } static inline void _talloc_free_children_internal(struct talloc_chunk *tc, void *ptr, const char *location); /* internal talloc_free call */ static inline int _talloc_free_internal(void *ptr, const char *location) { struct talloc_chunk *tc; void *ptr_to_free; if (unlikely(ptr == NULL)) { return -1; } /* possibly initialised the talloc fill value */ if (unlikely(!talloc_fill.initialised)) { const char *fill = getenv(TALLOC_FILL_ENV); if (fill != NULL) { talloc_fill.enabled = true; talloc_fill.fill_value = strtoul(fill, NULL, 0); } talloc_fill.initialised = true; } tc = talloc_chunk_from_ptr(ptr); if (unlikely(tc->refs)) { int is_child; /* check if this is a reference from a child or * grandchild back to it's parent or grandparent * * in that case we need to remove the reference and * call another instance of talloc_free() on the current * pointer. */ is_child = talloc_is_parent(tc->refs, ptr); _talloc_free_internal(tc->refs, location); if (is_child) { return _talloc_free_internal(ptr, location); } return -1; } if (unlikely(tc->flags & TALLOC_FLAG_LOOP)) { /* we have a free loop - stop looping */ return 0; } if (unlikely(tc->destructor)) { talloc_destructor_t d = tc->destructor; if (d == (talloc_destructor_t)-1) { return -1; } tc->destructor = (talloc_destructor_t)-1; if (d(ptr) == -1) { tc->destructor = d; return -1; } tc->destructor = NULL; } if (tc->parent) { _TLIST_REMOVE(tc->parent->child, tc); if (tc->parent->child) { tc->parent->child->parent = tc->parent; } } else { if (tc->prev) tc->prev->next = tc->next; if (tc->next) tc->next->prev = tc->prev; tc->prev = tc->next = NULL; } tc->flags |= TALLOC_FLAG_LOOP; _talloc_free_children_internal(tc, ptr, location); tc->flags |= TALLOC_FLAG_FREE; /* we mark the freed memory with where we called the free * from. This means on a double free error we can report where * the first free came from */ tc->name = location; if (tc->flags & TALLOC_FLAG_POOL) { struct talloc_pool_hdr *pool; pool = talloc_pool_from_chunk(tc); if (unlikely(pool->object_count == 0)) { talloc_abort("Pool object count zero!"); return 0; } pool->object_count--; if (likely(pool->object_count != 0)) { return 0; } /* * With object_count==0, a pool becomes a normal piece of * memory to free. If it's allocated inside a pool, it needs * to be freed as poolmem, else it needs to be just freed. */ ptr_to_free = pool; } else { ptr_to_free = tc; } if (tc->flags & TALLOC_FLAG_POOLMEM) { _talloc_free_poolmem(tc, location); return 0; } talloc_memlimit_update_on_free(tc); TC_INVALIDATE_FULL_CHUNK(tc); free(ptr_to_free); return 0; } static size_t _talloc_total_limit_size(const void *ptr, struct talloc_memlimit *old_limit, struct talloc_memlimit *new_limit); /* move a lump of memory from one talloc context to another return the ptr on success, or NULL if it could not be transferred. passing NULL as ptr will always return NULL with no side effects. */ static void *_talloc_steal_internal(const void *new_ctx, const void *ptr) { struct talloc_chunk *tc, *new_tc; size_t ctx_size = 0; if (unlikely(!ptr)) { return NULL; } if (unlikely(new_ctx == NULL)) { new_ctx = null_context; } tc = talloc_chunk_from_ptr(ptr); if (tc->limit != NULL) { ctx_size = _talloc_total_limit_size(ptr, NULL, NULL); /* Decrement the memory limit from the source .. */ talloc_memlimit_shrink(tc->limit->upper, ctx_size); if (tc->limit->parent == tc) { tc->limit->upper = NULL; } else { tc->limit = NULL; } } if (unlikely(new_ctx == NULL)) { if (tc->parent) { _TLIST_REMOVE(tc->parent->child, tc); if (tc->parent->child) { tc->parent->child->parent = tc->parent; } } else { if (tc->prev) tc->prev->next = tc->next; if (tc->next) tc->next->prev = tc->prev; } tc->parent = tc->next = tc->prev = NULL; return discard_const_p(void, ptr); } new_tc = talloc_chunk_from_ptr(new_ctx); if (unlikely(tc == new_tc || tc->parent == new_tc)) { return discard_const_p(void, ptr); } if (tc->parent) { _TLIST_REMOVE(tc->parent->child, tc); if (tc->parent->child) { tc->parent->child->parent = tc->parent; } } else { if (tc->prev) tc->prev->next = tc->next; if (tc->next) tc->next->prev = tc->prev; tc->prev = tc->next = NULL; } tc->parent = new_tc; if (new_tc->child) new_tc->child->parent = NULL; _TLIST_ADD(new_tc->child, tc); if (tc->limit || new_tc->limit) { ctx_size = _talloc_total_limit_size(ptr, tc->limit, new_tc->limit); /* .. and increment it in the destination. */ if (new_tc->limit) { talloc_memlimit_grow(new_tc->limit, ctx_size); } } return discard_const_p(void, ptr); } /* move a lump of memory from one talloc context to another return the ptr on success, or NULL if it could not be transferred. passing NULL as ptr will always return NULL with no side effects. */ _PUBLIC_ void *_talloc_steal_loc(const void *new_ctx, const void *ptr, const char *location) { struct talloc_chunk *tc; if (unlikely(ptr == NULL)) { return NULL; } tc = talloc_chunk_from_ptr(ptr); if (unlikely(tc->refs != NULL) && talloc_parent(ptr) != new_ctx) { struct talloc_reference_handle *h; talloc_log("WARNING: talloc_steal with references at %s\n", location); for (h=tc->refs; h; h=h->next) { talloc_log("\treference at %s\n", h->location); } } #if 0 /* this test is probably too expensive to have on in the normal build, but it useful for debugging */ if (talloc_is_parent(new_ctx, ptr)) { talloc_log("WARNING: stealing into talloc child at %s\n", location); } #endif return _talloc_steal_internal(new_ctx, ptr); } /* this is like a talloc_steal(), but you must supply the old parent. This resolves the ambiguity in a talloc_steal() which is called on a context that has more than one parent (via references) The old parent can be either a reference or a parent */ _PUBLIC_ void *talloc_reparent(const void *old_parent, const void *new_parent, const void *ptr) { struct talloc_chunk *tc; struct talloc_reference_handle *h; if (unlikely(ptr == NULL)) { return NULL; } if (old_parent == talloc_parent(ptr)) { return _talloc_steal_internal(new_parent, ptr); } tc = talloc_chunk_from_ptr(ptr); for (h=tc->refs;h;h=h->next) { if (talloc_parent(h) == old_parent) { if (_talloc_steal_internal(new_parent, h) != h) { return NULL; } return discard_const_p(void, ptr); } } /* it wasn't a parent */ return NULL; } /* remove a secondary reference to a pointer. This undo's what talloc_reference() has done. The context and pointer arguments must match those given to a talloc_reference() */ static inline int talloc_unreference(const void *context, const void *ptr) { struct talloc_chunk *tc = talloc_chunk_from_ptr(ptr); struct talloc_reference_handle *h; if (unlikely(context == NULL)) { context = null_context; } for (h=tc->refs;h;h=h->next) { struct talloc_chunk *p = talloc_parent_chunk(h); if (p == NULL) { if (context == NULL) break; } else if (TC_PTR_FROM_CHUNK(p) == context) { break; } } if (h == NULL) { return -1; } return _talloc_free_internal(h, __location__); } /* remove a specific parent context from a pointer. This is a more controlled variant of talloc_free() */ _PUBLIC_ int talloc_unlink(const void *context, void *ptr) { struct talloc_chunk *tc_p, *new_p, *tc_c; void *new_parent; if (ptr == NULL) { return -1; } if (context == NULL) { context = null_context; } if (talloc_unreference(context, ptr) == 0) { return 0; } if (context != NULL) { tc_c = talloc_chunk_from_ptr(context); } else { tc_c = NULL; } if (tc_c != talloc_parent_chunk(ptr)) { return -1; } tc_p = talloc_chunk_from_ptr(ptr); if (tc_p->refs == NULL) { return _talloc_free_internal(ptr, __location__); } new_p = talloc_parent_chunk(tc_p->refs); if (new_p) { new_parent = TC_PTR_FROM_CHUNK(new_p); } else { new_parent = NULL; } if (talloc_unreference(new_parent, ptr) != 0) { return -1; } _talloc_steal_internal(new_parent, ptr); return 0; } /* add a name to an existing pointer - va_list version */ static inline const char *talloc_set_name_v(const void *ptr, const char *fmt, va_list ap) PRINTF_ATTRIBUTE(2,0); static inline const char *talloc_set_name_v(const void *ptr, const char *fmt, va_list ap) { struct talloc_chunk *tc = talloc_chunk_from_ptr(ptr); tc->name = talloc_vasprintf(ptr, fmt, ap); if (likely(tc->name)) { _talloc_set_name_const(tc->name, ".name"); } return tc->name; } /* add a name to an existing pointer */ _PUBLIC_ const char *talloc_set_name(const void *ptr, const char *fmt, ...) { const char *name; va_list ap; va_start(ap, fmt); name = talloc_set_name_v(ptr, fmt, ap); va_end(ap); return name; } /* create a named talloc pointer. Any talloc pointer can be named, and talloc_named() operates just like talloc() except that it allows you to name the pointer. */ _PUBLIC_ void *talloc_named(const void *context, size_t size, const char *fmt, ...) { va_list ap; void *ptr; const char *name; ptr = __talloc(context, size); if (unlikely(ptr == NULL)) return NULL; va_start(ap, fmt); name = talloc_set_name_v(ptr, fmt, ap); va_end(ap); if (unlikely(name == NULL)) { _talloc_free_internal(ptr, __location__); return NULL; } return ptr; } /* return the name of a talloc ptr, or "UNNAMED" */ _PUBLIC_ const char *talloc_get_name(const void *ptr) { struct talloc_chunk *tc = talloc_chunk_from_ptr(ptr); if (unlikely(tc->name == TALLOC_MAGIC_REFERENCE)) { return ".reference"; } if (likely(tc->name)) { return tc->name; } return "UNNAMED"; } /* check if a pointer has the given name. If it does, return the pointer, otherwise return NULL */ _PUBLIC_ void *talloc_check_name(const void *ptr, const char *name) { const char *pname; if (unlikely(ptr == NULL)) return NULL; pname = talloc_get_name(ptr); if (likely(pname == name || strcmp(pname, name) == 0)) { return discard_const_p(void, ptr); } return NULL; } static void talloc_abort_type_mismatch(const char *location, const char *name, const char *expected) { const char *reason; reason = talloc_asprintf(NULL, "%s: Type mismatch: name[%s] expected[%s]", location, name?name:"NULL", expected); if (!reason) { reason = "Type mismatch"; } talloc_abort(reason); } _PUBLIC_ void *_talloc_get_type_abort(const void *ptr, const char *name, const char *location) { const char *pname; if (unlikely(ptr == NULL)) { talloc_abort_type_mismatch(location, NULL, name); return NULL; } pname = talloc_get_name(ptr); if (likely(pname == name || strcmp(pname, name) == 0)) { return discard_const_p(void, ptr); } talloc_abort_type_mismatch(location, pname, name); return NULL; } /* this is for compatibility with older versions of talloc */ _PUBLIC_ void *talloc_init(const char *fmt, ...) { va_list ap; void *ptr; const char *name; ptr = __talloc(NULL, 0); if (unlikely(ptr == NULL)) return NULL; va_start(ap, fmt); name = talloc_set_name_v(ptr, fmt, ap); va_end(ap); if (unlikely(name == NULL)) { _talloc_free_internal(ptr, __location__); return NULL; } return ptr; } static inline void _talloc_free_children_internal(struct talloc_chunk *tc, void *ptr, const char *location) { while (tc->child) { /* we need to work out who will own an abandoned child if it cannot be freed. In priority order, the first choice is owner of any remaining reference to this pointer, the second choice is our parent, and the final choice is the null context. */ void *child = TC_PTR_FROM_CHUNK(tc->child); const void *new_parent = null_context; if (unlikely(tc->child->refs)) { struct talloc_chunk *p = talloc_parent_chunk(tc->child->refs); if (p) new_parent = TC_PTR_FROM_CHUNK(p); } if (unlikely(_talloc_free_internal(child, location) == -1)) { if (new_parent == null_context) { struct talloc_chunk *p = talloc_parent_chunk(ptr); if (p) new_parent = TC_PTR_FROM_CHUNK(p); } _talloc_steal_internal(new_parent, child); } } } /* this is a replacement for the Samba3 talloc_destroy_pool functionality. It should probably not be used in new code. It's in here to keep the talloc code consistent across Samba 3 and 4. */ _PUBLIC_ void talloc_free_children(void *ptr) { struct talloc_chunk *tc_name = NULL; struct talloc_chunk *tc; if (unlikely(ptr == NULL)) { return; } tc = talloc_chunk_from_ptr(ptr); /* we do not want to free the context name if it is a child .. */ if (likely(tc->child)) { for (tc_name = tc->child; tc_name; tc_name = tc_name->next) { if (tc->name == TC_PTR_FROM_CHUNK(tc_name)) break; } if (tc_name) { _TLIST_REMOVE(tc->child, tc_name); if (tc->child) { tc->child->parent = tc; } } } _talloc_free_children_internal(tc, ptr, __location__); /* .. so we put it back after all other children have been freed */ if (tc_name) { if (tc->child) { tc->child->parent = NULL; } tc_name->parent = tc; _TLIST_ADD(tc->child, tc_name); } } /* Allocate a bit of memory as a child of an existing pointer */ _PUBLIC_ void *_talloc(const void *context, size_t size) { return __talloc(context, size); } /* externally callable talloc_set_name_const() */ _PUBLIC_ void talloc_set_name_const(const void *ptr, const char *name) { _talloc_set_name_const(ptr, name); } /* create a named talloc pointer. Any talloc pointer can be named, and talloc_named() operates just like talloc() except that it allows you to name the pointer. */ _PUBLIC_ void *talloc_named_const(const void *context, size_t size, const char *name) { return _talloc_named_const(context, size, name); } /* free a talloc pointer. This also frees all child pointers of this pointer recursively return 0 if the memory is actually freed, otherwise -1. The memory will not be freed if the ref_count is > 1 or the destructor (if any) returns non-zero */ _PUBLIC_ int _talloc_free(void *ptr, const char *location) { struct talloc_chunk *tc; if (unlikely(ptr == NULL)) { return -1; } tc = talloc_chunk_from_ptr(ptr); if (unlikely(tc->refs != NULL)) { struct talloc_reference_handle *h; if (talloc_parent(ptr) == null_context && tc->refs->next == NULL) { /* in this case we do know which parent should get this pointer, as there is really only one parent */ return talloc_unlink(null_context, ptr); } talloc_log("ERROR: talloc_free with references at %s\n", location); for (h=tc->refs; h; h=h->next) { talloc_log("\treference at %s\n", h->location); } return -1; } return _talloc_free_internal(ptr, location); } /* A talloc version of realloc. The context argument is only used if ptr is NULL */ _PUBLIC_ void *_talloc_realloc(const void *context, void *ptr, size_t size, const char *name) { struct talloc_chunk *tc; void *new_ptr; bool malloced = false; struct talloc_pool_hdr *pool_hdr = NULL; size_t old_size = 0; size_t new_size = 0; /* size zero is equivalent to free() */ if (unlikely(size == 0)) { talloc_unlink(context, ptr); return NULL; } if (unlikely(size >= MAX_TALLOC_SIZE)) { return NULL; } /* realloc(NULL) is equivalent to malloc() */ if (ptr == NULL) { return _talloc_named_const(context, size, name); } tc = talloc_chunk_from_ptr(ptr); /* don't allow realloc on referenced pointers */ if (unlikely(tc->refs)) { return NULL; } /* don't let anybody try to realloc a talloc_pool */ if (unlikely(tc->flags & TALLOC_FLAG_POOL)) { return NULL; } if (tc->limit && (size > tc->size)) { if (!talloc_memlimit_check(tc->limit, (size - tc->size))) { errno = ENOMEM; return NULL; } } /* handle realloc inside a talloc_pool */ if (unlikely(tc->flags & TALLOC_FLAG_POOLMEM)) { pool_hdr = tc->pool; } #if (ALWAYS_REALLOC == 0) /* don't shrink if we have less than 1k to gain */ if (size < tc->size && tc->limit == NULL) { if (pool_hdr) { void *next_tc = tc_next_chunk(tc); TC_INVALIDATE_SHRINK_CHUNK(tc, size); tc->size = size; if (next_tc == pool_hdr->end) { /* note: tc->size has changed, so this works */ pool_hdr->end = tc_next_chunk(tc); } return ptr; } else if ((tc->size - size) < 1024) { /* * if we call TC_INVALIDATE_SHRINK_CHUNK() here * we would need to call TC_UNDEFINE_GROW_CHUNK() * after each realloc call, which slows down * testing a lot :-(. * * That is why we only mark memory as undefined here. */ TC_UNDEFINE_SHRINK_CHUNK(tc, size); /* do not shrink if we have less than 1k to gain */ tc->size = size; return ptr; } } else if (tc->size == size) { /* * do not change the pointer if it is exactly * the same size. */ return ptr; } #endif /* by resetting magic we catch users of the old memory */ tc->flags |= TALLOC_FLAG_FREE; #if ALWAYS_REALLOC if (pool_hdr) { new_ptr = talloc_alloc_pool(tc, size + TC_HDR_SIZE, 0); pool_hdr->object_count--; if (new_ptr == NULL) { new_ptr = malloc(TC_HDR_SIZE+size); malloced = true; new_size = size; } if (new_ptr) { memcpy(new_ptr, tc, MIN(tc->size,size) + TC_HDR_SIZE); TC_INVALIDATE_FULL_CHUNK(tc); } } else { /* We're doing malloc then free here, so record the difference. */ old_size = tc->size; new_size = size; new_ptr = malloc(size + TC_HDR_SIZE); if (new_ptr) { memcpy(new_ptr, tc, MIN(tc->size, size) + TC_HDR_SIZE); free(tc); } } #else if (pool_hdr) { struct talloc_chunk *pool_tc; void *next_tc = tc_next_chunk(tc); size_t old_chunk_size = TC_ALIGN16(TC_HDR_SIZE + tc->size); size_t new_chunk_size = TC_ALIGN16(TC_HDR_SIZE + size); size_t space_needed; size_t space_left; unsigned int chunk_count = pool_hdr->object_count; pool_tc = talloc_chunk_from_pool(pool_hdr); if (!(pool_tc->flags & TALLOC_FLAG_FREE)) { chunk_count -= 1; } if (chunk_count == 1) { /* * optimize for the case where 'tc' is the only * chunk in the pool. */ char *start = tc_pool_first_chunk(pool_hdr); space_needed = new_chunk_size; space_left = (char *)tc_pool_end(pool_hdr) - start; if (space_left >= space_needed) { size_t old_used = TC_HDR_SIZE + tc->size; size_t new_used = TC_HDR_SIZE + size; new_ptr = start; #if defined(DEVELOPER) && defined(VALGRIND_MAKE_MEM_UNDEFINED) { /* * The area from * start -> tc may have * been freed and thus been marked as * VALGRIND_MEM_NOACCESS. Set it to * VALGRIND_MEM_UNDEFINED so we can * copy into it without valgrind errors. * We can't just mark * new_ptr -> new_ptr + old_used * as this may overlap on top of tc, * (which is why we use memmove, not * memcpy below) hence the MIN. */ size_t undef_len = MIN((((char *)tc) - ((char *)new_ptr)),old_used); VALGRIND_MAKE_MEM_UNDEFINED(new_ptr, undef_len); } #endif memmove(new_ptr, tc, old_used); tc = (struct talloc_chunk *)new_ptr; TC_UNDEFINE_GROW_CHUNK(tc, size); /* * first we do not align the pool pointer * because we want to invalidate the padding * too. */ pool_hdr->end = new_used + (char *)new_ptr; tc_invalidate_pool(pool_hdr); /* now the aligned pointer */ pool_hdr->end = new_chunk_size + (char *)new_ptr; goto got_new_ptr; } next_tc = NULL; } if (new_chunk_size == old_chunk_size) { TC_UNDEFINE_GROW_CHUNK(tc, size); tc->flags &= ~TALLOC_FLAG_FREE; tc->size = size; return ptr; } if (next_tc == pool_hdr->end) { /* * optimize for the case where 'tc' is the last * chunk in the pool. */ space_needed = new_chunk_size - old_chunk_size; space_left = tc_pool_space_left(pool_hdr); if (space_left >= space_needed) { TC_UNDEFINE_GROW_CHUNK(tc, size); tc->flags &= ~TALLOC_FLAG_FREE; tc->size = size; pool_hdr->end = tc_next_chunk(tc); return ptr; } } new_ptr = talloc_alloc_pool(tc, size + TC_HDR_SIZE, 0); if (new_ptr == NULL) { new_ptr = malloc(TC_HDR_SIZE+size); malloced = true; new_size = size; } if (new_ptr) { memcpy(new_ptr, tc, MIN(tc->size,size) + TC_HDR_SIZE); _talloc_free_poolmem(tc, __location__ "_talloc_realloc"); } } else { /* We're doing realloc here, so record the difference. */ old_size = tc->size; new_size = size; new_ptr = realloc(tc, size + TC_HDR_SIZE); } got_new_ptr: #endif if (unlikely(!new_ptr)) { tc->flags &= ~TALLOC_FLAG_FREE; return NULL; } tc = (struct talloc_chunk *)new_ptr; tc->flags &= ~TALLOC_FLAG_FREE; if (malloced) { tc->flags &= ~TALLOC_FLAG_POOLMEM; } if (tc->parent) { tc->parent->child = tc; } if (tc->child) { tc->child->parent = tc; } if (tc->prev) { tc->prev->next = tc; } if (tc->next) { tc->next->prev = tc; } if (new_size > old_size) { talloc_memlimit_grow(tc->limit, new_size - old_size); } else if (new_size < old_size) { talloc_memlimit_shrink(tc->limit, old_size - new_size); } tc->size = size; _talloc_set_name_const(TC_PTR_FROM_CHUNK(tc), name); return TC_PTR_FROM_CHUNK(tc); } /* a wrapper around talloc_steal() for situations where you are moving a pointer between two structures, and want the old pointer to be set to NULL */ _PUBLIC_ void *_talloc_move(const void *new_ctx, const void *_pptr) { const void **pptr = discard_const_p(const void *,_pptr); void *ret = talloc_steal(new_ctx, discard_const_p(void, *pptr)); (*pptr) = NULL; return ret; } enum talloc_mem_count_type { TOTAL_MEM_SIZE, TOTAL_MEM_BLOCKS, TOTAL_MEM_LIMIT, }; static size_t _talloc_total_mem_internal(const void *ptr, enum talloc_mem_count_type type, struct talloc_memlimit *old_limit, struct talloc_memlimit *new_limit) { size_t total = 0; struct talloc_chunk *c, *tc; if (ptr == NULL) { ptr = null_context; } if (ptr == NULL) { return 0; } tc = talloc_chunk_from_ptr(ptr); if (old_limit || new_limit) { if (tc->limit && tc->limit->upper == old_limit) { tc->limit->upper = new_limit; } } /* optimize in the memlimits case */ if (type == TOTAL_MEM_LIMIT && tc->limit != NULL && tc->limit != old_limit && tc->limit->parent == tc) { return tc->limit->cur_size; } if (tc->flags & TALLOC_FLAG_LOOP) { return 0; } tc->flags |= TALLOC_FLAG_LOOP; if (old_limit || new_limit) { if (old_limit == tc->limit) { tc->limit = new_limit; } } switch (type) { case TOTAL_MEM_SIZE: if (likely(tc->name != TALLOC_MAGIC_REFERENCE)) { total = tc->size; } break; case TOTAL_MEM_BLOCKS: total++; break; case TOTAL_MEM_LIMIT: if (likely(tc->name != TALLOC_MAGIC_REFERENCE)) { /* * Don't count memory allocated from a pool * when calculating limits. Only count the * pool itself. */ if (!(tc->flags & TALLOC_FLAG_POOLMEM)) { if (tc->flags & TALLOC_FLAG_POOL) { /* * If this is a pool, the allocated * size is in the pool header, and * remember to add in the prefix * length. */ struct talloc_pool_hdr *pool_hdr = talloc_pool_from_chunk(tc); total = pool_hdr->poolsize + TC_HDR_SIZE + TP_HDR_SIZE; } else { total = tc->size + TC_HDR_SIZE; } } } break; } for (c = tc->child; c; c = c->next) { total += _talloc_total_mem_internal(TC_PTR_FROM_CHUNK(c), type, old_limit, new_limit); } tc->flags &= ~TALLOC_FLAG_LOOP; return total; } /* return the total size of a talloc pool (subtree) */ _PUBLIC_ size_t talloc_total_size(const void *ptr) { return _talloc_total_mem_internal(ptr, TOTAL_MEM_SIZE, NULL, NULL); } /* return the total number of blocks in a talloc pool (subtree) */ _PUBLIC_ size_t talloc_total_blocks(const void *ptr) { return _talloc_total_mem_internal(ptr, TOTAL_MEM_BLOCKS, NULL, NULL); } /* return the number of external references to a pointer */ _PUBLIC_ size_t talloc_reference_count(const void *ptr) { struct talloc_chunk *tc = talloc_chunk_from_ptr(ptr); struct talloc_reference_handle *h; size_t ret = 0; for (h=tc->refs;h;h=h->next) { ret++; } return ret; } /* report on memory usage by all children of a pointer, giving a full tree view */ _PUBLIC_ void talloc_report_depth_cb(const void *ptr, int depth, int max_depth, void (*callback)(const void *ptr, int depth, int max_depth, int is_ref, void *private_data), void *private_data) { struct talloc_chunk *c, *tc; if (ptr == NULL) { ptr = null_context; } if (ptr == NULL) return; tc = talloc_chunk_from_ptr(ptr); if (tc->flags & TALLOC_FLAG_LOOP) { return; } callback(ptr, depth, max_depth, 0, private_data); if (max_depth >= 0 && depth >= max_depth) { return; } tc->flags |= TALLOC_FLAG_LOOP; for (c=tc->child;c;c=c->next) { if (c->name == TALLOC_MAGIC_REFERENCE) { struct talloc_reference_handle *h = (struct talloc_reference_handle *)TC_PTR_FROM_CHUNK(c); callback(h->ptr, depth + 1, max_depth, 1, private_data); } else { talloc_report_depth_cb(TC_PTR_FROM_CHUNK(c), depth + 1, max_depth, callback, private_data); } } tc->flags &= ~TALLOC_FLAG_LOOP; } static void talloc_report_depth_FILE_helper(const void *ptr, int depth, int max_depth, int is_ref, void *_f) { const char *name = talloc_get_name(ptr); struct talloc_chunk *tc; FILE *f = (FILE *)_f; if (is_ref) { fprintf(f, "%*sreference to: %s\n", depth*4, "", name); return; } tc = talloc_chunk_from_ptr(ptr); if (tc->limit && tc->limit->parent == tc) { fprintf(f, "%*s%-30s is a memlimit context" " (max_size = %lu bytes, cur_size = %lu bytes)\n", depth*4, "", name, (unsigned long)tc->limit->max_size, (unsigned long)tc->limit->cur_size); } if (depth == 0) { fprintf(f,"%stalloc report on '%s' (total %6lu bytes in %3lu blocks)\n", (max_depth < 0 ? "full " :""), name, (unsigned long)talloc_total_size(ptr), (unsigned long)talloc_total_blocks(ptr)); return; } fprintf(f, "%*s%-30s contains %6lu bytes in %3lu blocks (ref %d) %p\n", depth*4, "", name, (unsigned long)talloc_total_size(ptr), (unsigned long)talloc_total_blocks(ptr), (int)talloc_reference_count(ptr), ptr); #if 0 fprintf(f, "content: "); if (talloc_total_size(ptr)) { int tot = talloc_total_size(ptr); int i; for (i = 0; i < tot; i++) { if ((((char *)ptr)[i] > 31) && (((char *)ptr)[i] < 126)) { fprintf(f, "%c", ((char *)ptr)[i]); } else { fprintf(f, "~%02x", ((char *)ptr)[i]); } } } fprintf(f, "\n"); #endif } /* report on memory usage by all children of a pointer, giving a full tree view */ _PUBLIC_ void talloc_report_depth_file(const void *ptr, int depth, int max_depth, FILE *f) { if (f) { talloc_report_depth_cb(ptr, depth, max_depth, talloc_report_depth_FILE_helper, f); fflush(f); } } /* report on memory usage by all children of a pointer, giving a full tree view */ _PUBLIC_ void talloc_report_full(const void *ptr, FILE *f) { talloc_report_depth_file(ptr, 0, -1, f); } /* report on memory usage by all children of a pointer */ _PUBLIC_ void talloc_report(const void *ptr, FILE *f) { talloc_report_depth_file(ptr, 0, 1, f); } /* report on any memory hanging off the null context */ static void talloc_report_null(void) { if (talloc_total_size(null_context) != 0) { talloc_report(null_context, stderr); } } /* report on any memory hanging off the null context */ static void talloc_report_null_full(void) { if (talloc_total_size(null_context) != 0) { talloc_report_full(null_context, stderr); } } /* enable tracking of the NULL context */ _PUBLIC_ void talloc_enable_null_tracking(void) { if (null_context == NULL) { null_context = _talloc_named_const(NULL, 0, "null_context"); if (autofree_context != NULL) { talloc_reparent(NULL, null_context, autofree_context); } } } /* enable tracking of the NULL context, not moving the autofree context into the NULL context. This is needed for the talloc testsuite */ _PUBLIC_ void talloc_enable_null_tracking_no_autofree(void) { if (null_context == NULL) { null_context = _talloc_named_const(NULL, 0, "null_context"); } } /* disable tracking of the NULL context */ _PUBLIC_ void talloc_disable_null_tracking(void) { if (null_context != NULL) { /* we have to move any children onto the real NULL context */ struct talloc_chunk *tc, *tc2; tc = talloc_chunk_from_ptr(null_context); for (tc2 = tc->child; tc2; tc2=tc2->next) { if (tc2->parent == tc) tc2->parent = NULL; if (tc2->prev == tc) tc2->prev = NULL; } for (tc2 = tc->next; tc2; tc2=tc2->next) { if (tc2->parent == tc) tc2->parent = NULL; if (tc2->prev == tc) tc2->prev = NULL; } tc->child = NULL; tc->next = NULL; } talloc_free(null_context); null_context = NULL; } /* enable leak reporting on exit */ _PUBLIC_ void talloc_enable_leak_report(void) { talloc_enable_null_tracking(); atexit(talloc_report_null); } /* enable full leak reporting on exit */ _PUBLIC_ void talloc_enable_leak_report_full(void) { talloc_enable_null_tracking(); atexit(talloc_report_null_full); } /* talloc and zero memory. */ _PUBLIC_ void *_talloc_zero(const void *ctx, size_t size, const char *name) { void *p = _talloc_named_const(ctx, size, name); if (p) { memset(p, '\0', size); } return p; } /* memdup with a talloc. */ _PUBLIC_ void *_talloc_memdup(const void *t, const void *p, size_t size, const char *name) { void *newp = _talloc_named_const(t, size, name); if (likely(newp)) { memcpy(newp, p, size); } return newp; } static inline char *__talloc_strlendup(const void *t, const char *p, size_t len) { char *ret; ret = (char *)__talloc(t, len + 1); if (unlikely(!ret)) return NULL; memcpy(ret, p, len); ret[len] = 0; _talloc_set_name_const(ret, ret); return ret; } /* strdup with a talloc */ _PUBLIC_ char *talloc_strdup(const void *t, const char *p) { if (unlikely(!p)) return NULL; return __talloc_strlendup(t, p, strlen(p)); } /* strndup with a talloc */ _PUBLIC_ char *talloc_strndup(const void *t, const char *p, size_t n) { if (unlikely(!p)) return NULL; return __talloc_strlendup(t, p, strnlen(p, n)); } static inline char *__talloc_strlendup_append(char *s, size_t slen, const char *a, size_t alen) { char *ret; ret = talloc_realloc(NULL, s, char, slen + alen + 1); if (unlikely(!ret)) return NULL; /* append the string and the trailing \0 */ memcpy(&ret[slen], a, alen); ret[slen+alen] = 0; _talloc_set_name_const(ret, ret); return ret; } /* * Appends at the end of the string. */ _PUBLIC_ char *talloc_strdup_append(char *s, const char *a) { if (unlikely(!s)) { return talloc_strdup(NULL, a); } if (unlikely(!a)) { return s; } return __talloc_strlendup_append(s, strlen(s), a, strlen(a)); } /* * Appends at the end of the talloc'ed buffer, * not the end of the string. */ _PUBLIC_ char *talloc_strdup_append_buffer(char *s, const char *a) { size_t slen; if (unlikely(!s)) { return talloc_strdup(NULL, a); } if (unlikely(!a)) { return s; } slen = talloc_get_size(s); if (likely(slen > 0)) { slen--; } return __talloc_strlendup_append(s, slen, a, strlen(a)); } /* * Appends at the end of the string. */ _PUBLIC_ char *talloc_strndup_append(char *s, const char *a, size_t n) { if (unlikely(!s)) { return talloc_strndup(NULL, a, n); } if (unlikely(!a)) { return s; } return __talloc_strlendup_append(s, strlen(s), a, strnlen(a, n)); } /* * Appends at the end of the talloc'ed buffer, * not the end of the string. */ _PUBLIC_ char *talloc_strndup_append_buffer(char *s, const char *a, size_t n) { size_t slen; if (unlikely(!s)) { return talloc_strndup(NULL, a, n); } if (unlikely(!a)) { return s; } slen = talloc_get_size(s); if (likely(slen > 0)) { slen--; } return __talloc_strlendup_append(s, slen, a, strnlen(a, n)); } #ifndef HAVE_VA_COPY #ifdef HAVE___VA_COPY #define va_copy(dest, src) __va_copy(dest, src) #else #define va_copy(dest, src) (dest) = (src) #endif #endif _PUBLIC_ char *talloc_vasprintf(const void *t, const char *fmt, va_list ap) { int len; char *ret; va_list ap2; char c; /* this call looks strange, but it makes it work on older solaris boxes */ va_copy(ap2, ap); len = vsnprintf(&c, 1, fmt, ap2); va_end(ap2); if (unlikely(len < 0)) { return NULL; } ret = (char *)__talloc(t, len+1); if (unlikely(!ret)) return NULL; va_copy(ap2, ap); vsnprintf(ret, len+1, fmt, ap2); va_end(ap2); _talloc_set_name_const(ret, ret); return ret; } /* Perform string formatting, and return a pointer to newly allocated memory holding the result, inside a memory pool. */ _PUBLIC_ char *talloc_asprintf(const void *t, const char *fmt, ...) { va_list ap; char *ret; va_start(ap, fmt); ret = talloc_vasprintf(t, fmt, ap); va_end(ap); return ret; } static inline char *__talloc_vaslenprintf_append(char *s, size_t slen, const char *fmt, va_list ap) PRINTF_ATTRIBUTE(3,0); static inline char *__talloc_vaslenprintf_append(char *s, size_t slen, const char *fmt, va_list ap) { ssize_t alen; va_list ap2; char c; va_copy(ap2, ap); alen = vsnprintf(&c, 1, fmt, ap2); va_end(ap2); if (alen <= 0) { /* Either the vsnprintf failed or the format resulted in * no characters being formatted. In the former case, we * ought to return NULL, in the latter we ought to return * the original string. Most current callers of this * function expect it to never return NULL. */ return s; } s = talloc_realloc(NULL, s, char, slen + alen + 1); if (!s) return NULL; va_copy(ap2, ap); vsnprintf(s + slen, alen + 1, fmt, ap2); va_end(ap2); _talloc_set_name_const(s, s); return s; } /** * Realloc @p s to append the formatted result of @p fmt and @p ap, * and return @p s, which may have moved. Good for gradually * accumulating output into a string buffer. Appends at the end * of the string. **/ _PUBLIC_ char *talloc_vasprintf_append(char *s, const char *fmt, va_list ap) { if (unlikely(!s)) { return talloc_vasprintf(NULL, fmt, ap); } return __talloc_vaslenprintf_append(s, strlen(s), fmt, ap); } /** * Realloc @p s to append the formatted result of @p fmt and @p ap, * and return @p s, which may have moved. Always appends at the * end of the talloc'ed buffer, not the end of the string. **/ _PUBLIC_ char *talloc_vasprintf_append_buffer(char *s, const char *fmt, va_list ap) { size_t slen; if (unlikely(!s)) { return talloc_vasprintf(NULL, fmt, ap); } slen = talloc_get_size(s); if (likely(slen > 0)) { slen--; } return __talloc_vaslenprintf_append(s, slen, fmt, ap); } /* Realloc @p s to append the formatted result of @p fmt and return @p s, which may have moved. Good for gradually accumulating output into a string buffer. */ _PUBLIC_ char *talloc_asprintf_append(char *s, const char *fmt, ...) { va_list ap; va_start(ap, fmt); s = talloc_vasprintf_append(s, fmt, ap); va_end(ap); return s; } /* Realloc @p s to append the formatted result of @p fmt and return @p s, which may have moved. Good for gradually accumulating output into a buffer. */ _PUBLIC_ char *talloc_asprintf_append_buffer(char *s, const char *fmt, ...) { va_list ap; va_start(ap, fmt); s = talloc_vasprintf_append_buffer(s, fmt, ap); va_end(ap); return s; } /* alloc an array, checking for integer overflow in the array size */ _PUBLIC_ void *_talloc_array(const void *ctx, size_t el_size, unsigned count, const char *name) { if (count >= MAX_TALLOC_SIZE/el_size) { return NULL; } return _talloc_named_const(ctx, el_size * count, name); } /* alloc an zero array, checking for integer overflow in the array size */ _PUBLIC_ void *_talloc_zero_array(const void *ctx, size_t el_size, unsigned count, const char *name) { if (count >= MAX_TALLOC_SIZE/el_size) { return NULL; } return _talloc_zero(ctx, el_size * count, name); } /* realloc an array, checking for integer overflow in the array size */ _PUBLIC_ void *_talloc_realloc_array(const void *ctx, void *ptr, size_t el_size, unsigned count, const char *name) { if (count >= MAX_TALLOC_SIZE/el_size) { return NULL; } return _talloc_realloc(ctx, ptr, el_size * count, name); } /* a function version of talloc_realloc(), so it can be passed as a function pointer to libraries that want a realloc function (a realloc function encapsulates all the basic capabilities of an allocation library, which is why this is useful) */ _PUBLIC_ void *talloc_realloc_fn(const void *context, void *ptr, size_t size) { return _talloc_realloc(context, ptr, size, NULL); } static int talloc_autofree_destructor(void *ptr) { autofree_context = NULL; return 0; } static void talloc_autofree(void) { talloc_free(autofree_context); } /* return a context which will be auto-freed on exit this is useful for reducing the noise in leak reports */ _PUBLIC_ void *talloc_autofree_context(void) { if (autofree_context == NULL) { autofree_context = _talloc_named_const(NULL, 0, "autofree_context"); talloc_set_destructor(autofree_context, talloc_autofree_destructor); atexit(talloc_autofree); } return autofree_context; } _PUBLIC_ size_t talloc_get_size(const void *context) { struct talloc_chunk *tc; if (context == NULL) { context = null_context; } if (context == NULL) { return 0; } tc = talloc_chunk_from_ptr(context); return tc->size; } /* find a parent of this context that has the given name, if any */ _PUBLIC_ void *talloc_find_parent_byname(const void *context, const char *name) { struct talloc_chunk *tc; if (context == NULL) { return NULL; } tc = talloc_chunk_from_ptr(context); while (tc) { if (tc->name && strcmp(tc->name, name) == 0) { return TC_PTR_FROM_CHUNK(tc); } while (tc && tc->prev) tc = tc->prev; if (tc) { tc = tc->parent; } } return NULL; } /* show the parentage of a context */ _PUBLIC_ void talloc_show_parents(const void *context, FILE *file) { struct talloc_chunk *tc; if (context == NULL) { fprintf(file, "talloc no parents for NULL\n"); return; } tc = talloc_chunk_from_ptr(context); fprintf(file, "talloc parents of '%s'\n", talloc_get_name(context)); while (tc) { fprintf(file, "\t'%s'\n", talloc_get_name(TC_PTR_FROM_CHUNK(tc))); while (tc && tc->prev) tc = tc->prev; if (tc) { tc = tc->parent; } } fflush(file); } /* return 1 if ptr is a parent of context */ static int _talloc_is_parent(const void *context, const void *ptr, int depth) { struct talloc_chunk *tc; if (context == NULL) { return 0; } tc = talloc_chunk_from_ptr(context); while (tc && depth > 0) { if (TC_PTR_FROM_CHUNK(tc) == ptr) return 1; while (tc && tc->prev) tc = tc->prev; if (tc) { tc = tc->parent; depth--; } } return 0; } /* return 1 if ptr is a parent of context */ _PUBLIC_ int talloc_is_parent(const void *context, const void *ptr) { return _talloc_is_parent(context, ptr, TALLOC_MAX_DEPTH); } /* return the total size of memory used by this context and all children */ static size_t _talloc_total_limit_size(const void *ptr, struct talloc_memlimit *old_limit, struct talloc_memlimit *new_limit) { return _talloc_total_mem_internal(ptr, TOTAL_MEM_LIMIT, old_limit, new_limit); } static bool talloc_memlimit_check(struct talloc_memlimit *limit, size_t size) { struct talloc_memlimit *l; for (l = limit; l != NULL; l = l->upper) { if (l->max_size != 0 && ((l->max_size <= l->cur_size) || (l->max_size - l->cur_size < size))) { return false; } } return true; } /* Update memory limits when freeing a talloc_chunk. */ static void talloc_memlimit_update_on_free(struct talloc_chunk *tc) { size_t limit_shrink_size; if (!tc->limit) { return; } /* * Pool entries don't count. Only the pools * themselves are counted as part of the memory * limits. Note that this also takes care of * nested pools which have both flags * TALLOC_FLAG_POOLMEM|TALLOC_FLAG_POOL set. */ if (tc->flags & TALLOC_FLAG_POOLMEM) { return; } /* * If we are part of a memory limited context hierarchy * we need to subtract the memory used from the counters */ limit_shrink_size = tc->size+TC_HDR_SIZE; /* * If we're deallocating a pool, take into * account the prefix size added for the pool. */ if (tc->flags & TALLOC_FLAG_POOL) { limit_shrink_size += TP_HDR_SIZE; } talloc_memlimit_shrink(tc->limit, limit_shrink_size); if (tc->limit->parent == tc) { free(tc->limit); } tc->limit = NULL; } /* Increase memory limit accounting after a malloc/realloc. */ static void talloc_memlimit_grow(struct talloc_memlimit *limit, size_t size) { struct talloc_memlimit *l; for (l = limit; l != NULL; l = l->upper) { size_t new_cur_size = l->cur_size + size; if (new_cur_size < l->cur_size) { talloc_abort("logic error in talloc_memlimit_grow\n"); return; } l->cur_size = new_cur_size; } } /* Decrease memory limit accounting after a free/realloc. */ static void talloc_memlimit_shrink(struct talloc_memlimit *limit, size_t size) { struct talloc_memlimit *l; for (l = limit; l != NULL; l = l->upper) { if (l->cur_size < size) { talloc_abort("logic error in talloc_memlimit_shrink\n"); return; } l->cur_size = l->cur_size - size; } } _PUBLIC_ int talloc_set_memlimit(const void *ctx, size_t max_size) { struct talloc_chunk *tc = talloc_chunk_from_ptr(ctx); struct talloc_memlimit *orig_limit; struct talloc_memlimit *limit = NULL; if (tc->limit && tc->limit->parent == tc) { tc->limit->max_size = max_size; return 0; } orig_limit = tc->limit; limit = malloc(sizeof(struct talloc_memlimit)); if (limit == NULL) { return 1; } limit->parent = tc; limit->max_size = max_size; limit->cur_size = _talloc_total_limit_size(ctx, tc->limit, limit); if (orig_limit) { limit->upper = orig_limit; } else { limit->upper = NULL; } return 0; } pytsk-20231007/talloc/talloc.h000066400000000000000000001730271451023402500160370ustar00rootroot00000000000000#ifndef _TALLOC_H_ #define _TALLOC_H_ /* Unix SMB/CIFS implementation. Samba temporary memory allocation functions Copyright (C) Andrew Tridgell 2004-2005 Copyright (C) Stefan Metzmacher 2006 ** NOTE! The following LGPL license applies to the talloc ** library. This does NOT imply that all of Samba is released ** under the LGPL This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation; either version 3 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. You should have received a copy of the GNU Lesser General Public License along with this library; if not, see . */ #include #include #include #ifdef __cplusplus extern "C" { #endif /** * @defgroup talloc The talloc API * * talloc is a hierarchical, reference counted memory pool system with * destructors. It is the core memory allocator used in Samba. * * @{ */ #define TALLOC_VERSION_MAJOR 2 #define TALLOC_VERSION_MINOR 0 int talloc_version_major(void); int talloc_version_minor(void); /** * @brief Define a talloc parent type * * As talloc is a hierarchial memory allocator, every talloc chunk is a * potential parent to other talloc chunks. So defining a separate type for a * talloc chunk is not strictly necessary. TALLOC_CTX is defined nevertheless, * as it provides an indicator for function arguments. You will frequently * write code like * * @code * struct foo *foo_create(TALLOC_CTX *mem_ctx) * { * struct foo *result; * result = talloc(mem_ctx, struct foo); * if (result == NULL) return NULL; * ... initialize foo ... * return result; * } * @endcode * * In this type of allocating functions it is handy to have a general * TALLOC_CTX type to indicate which parent to put allocated structures on. */ typedef void TALLOC_CTX; /* this uses a little trick to allow __LINE__ to be stringified */ #ifndef __location__ #define __TALLOC_STRING_LINE1__(s) #s #define __TALLOC_STRING_LINE2__(s) __TALLOC_STRING_LINE1__(s) #define __TALLOC_STRING_LINE3__ __TALLOC_STRING_LINE2__(__LINE__) #define __location__ __FILE__ ":" __TALLOC_STRING_LINE3__ #endif #ifndef TALLOC_DEPRECATED #define TALLOC_DEPRECATED 0 #endif #ifndef PRINTF_ATTRIBUTE #if (__GNUC__ >= 3) /** Use gcc attribute to check printf fns. a1 is the 1-based index of * the parameter containing the format, and a2 the index of the first * argument. Note that some gcc 2.x versions don't handle this * properly **/ #define PRINTF_ATTRIBUTE(a1, a2) __attribute__ ((format (__printf__, a1, a2))) #else #define PRINTF_ATTRIBUTE(a1, a2) #endif #endif #ifdef DOXYGEN /** * @brief Create a new talloc context. * * The talloc() macro is the core of the talloc library. It takes a memory * context and a type, and returns a pointer to a new area of memory of the * given type. * * The returned pointer is itself a talloc context, so you can use it as the * context argument to more calls to talloc if you wish. * * The returned pointer is a "child" of the supplied context. This means that if * you talloc_free() the context then the new child disappears as well. * Alternatively you can free just the child. * * @param[in] ctx A talloc context to create a new reference on or NULL to * create a new top level context. * * @param[in] type The type of memory to allocate. * * @return A type casted talloc context or NULL on error. * * @code * unsigned int *a, *b; * * a = talloc(NULL, unsigned int); * b = talloc(a, unsigned int); * @endcode * * @see talloc_zero * @see talloc_array * @see talloc_steal * @see talloc_free */ void *talloc(const void *ctx, #type); #else #define talloc(ctx, type) (type *)talloc_named_const(ctx, sizeof(type), #type) void *_talloc(const void *context, size_t size); #endif /** * @brief Create a new top level talloc context. * * This function creates a zero length named talloc context as a top level * context. It is equivalent to: * * @code * talloc_named(NULL, 0, fmt, ...); * @endcode * @param[in] fmt Format string for the name. * * @param[in] ... Additional printf-style arguments. * * @return The allocated memory chunk, NULL on error. * * @see talloc_named() */ void *talloc_init(const char *fmt, ...) PRINTF_ATTRIBUTE(1,2); #ifdef DOXYGEN /** * @brief Free a chunk of talloc memory. * * The talloc_free() function frees a piece of talloc memory, and all its * children. You can call talloc_free() on any pointer returned by * talloc(). * * The return value of talloc_free() indicates success or failure, with 0 * returned for success and -1 for failure. A possible failure condition * is if the pointer had a destructor attached to it and the destructor * returned -1. See talloc_set_destructor() for details on * destructors. Likewise, if "ptr" is NULL, then the function will make * no modifications and return -1. * * From version 2.0 and onwards, as a special case, talloc_free() is * refused on pointers that have more than one parent associated, as talloc * would have no way of knowing which parent should be removed. This is * different from older versions in the sense that always the reference to * the most recently established parent has been destroyed. Hence to free a * pointer that has more than one parent please use talloc_unlink(). * * To help you find problems in your code caused by this behaviour, if * you do try and free a pointer with more than one parent then the * talloc logging function will be called to give output like this: * * @code * ERROR: talloc_free with references at some_dir/source/foo.c:123 * reference at some_dir/source/other.c:325 * reference at some_dir/source/third.c:121 * @endcode * * Please see the documentation for talloc_set_log_fn() and * talloc_set_log_stderr() for more information on talloc logging * functions. * * If TALLOC_FREE_FILL environment variable is set, * the memory occupied by the context is filled with the value of this variable. * The value should be a numeric representation of the character you want to * use. * * talloc_free() operates recursively on its children. * * @param[in] ptr The chunk to be freed. * * @return Returns 0 on success and -1 on error. A possible * failure condition is if the pointer had a destructor * attached to it and the destructor returned -1. Likewise, * if "ptr" is NULL, then the function will make no * modifications and returns -1. * * Example: * @code * unsigned int *a, *b; * a = talloc(NULL, unsigned int); * b = talloc(a, unsigned int); * * talloc_free(a); // Frees a and b * @endcode * * @see talloc_set_destructor() * @see talloc_unlink() */ int talloc_free(void *ptr); #else #define talloc_free(ctx) _talloc_free(ctx, __location__) int _talloc_free(void *ptr, const char *location); #endif /** * @brief Free a talloc chunk's children. * * The function walks along the list of all children of a talloc context and * talloc_free()s only the children, not the context itself. * * A NULL argument is handled as no-op. * * @param[in] ptr The chunk that you want to free the children of * (NULL is allowed too) */ void talloc_free_children(void *ptr); #ifdef DOXYGEN /** * @brief Assign a destructor function to be called when a chunk is freed. * * The function talloc_set_destructor() sets the "destructor" for the pointer * "ptr". A destructor is a function that is called when the memory used by a * pointer is about to be released. The destructor receives the pointer as an * argument, and should return 0 for success and -1 for failure. * * The destructor can do anything it wants to, including freeing other pieces * of memory. A common use for destructors is to clean up operating system * resources (such as open file descriptors) contained in the structure the * destructor is placed on. * * You can only place one destructor on a pointer. If you need more than one * destructor then you can create a zero-length child of the pointer and place * an additional destructor on that. * * To remove a destructor call talloc_set_destructor() with NULL for the * destructor. * * If your destructor attempts to talloc_free() the pointer that it is the * destructor for then talloc_free() will return -1 and the free will be * ignored. This would be a pointless operation anyway, as the destructor is * only called when the memory is just about to go away. * * @param[in] ptr The talloc chunk to add a destructor to. * * @param[in] destructor The destructor function to be called. NULL to remove * it. * * Example: * @code * static int destroy_fd(int *fd) { * close(*fd); * return 0; * } * * int *open_file(const char *filename) { * int *fd = talloc(NULL, int); * *fd = open(filename, O_RDONLY); * if (*fd < 0) { * talloc_free(fd); * return NULL; * } * // Whenever they free this, we close the file. * talloc_set_destructor(fd, destroy_fd); * return fd; * } * @endcode * * @see talloc() * @see talloc_free() */ void talloc_set_destructor(const void *ptr, int (*destructor)(void *)); /** * @brief Change a talloc chunk's parent. * * The talloc_steal() function changes the parent context of a talloc * pointer. It is typically used when the context that the pointer is * currently a child of is going to be freed and you wish to keep the * memory for a longer time. * * To make the changed hierarchy less error-prone, you might consider to use * talloc_move(). * * If you try and call talloc_steal() on a pointer that has more than one * parent then the result is ambiguous. Talloc will choose to remove the * parent that is currently indicated by talloc_parent() and replace it with * the chosen parent. You will also get a message like this via the talloc * logging functions: * * @code * WARNING: talloc_steal with references at some_dir/source/foo.c:123 * reference at some_dir/source/other.c:325 * reference at some_dir/source/third.c:121 * @endcode * * To unambiguously change the parent of a pointer please see the function * talloc_reparent(). See the talloc_set_log_fn() documentation for more * information on talloc logging. * * @param[in] new_ctx The new parent context. * * @param[in] ptr The talloc chunk to move. * * @return Returns the pointer that you pass it. It does not have * any failure modes. * * @note It is possible to produce loops in the parent/child relationship * if you are not careful with talloc_steal(). No guarantees are provided * as to your sanity or the safety of your data if you do this. */ void *talloc_steal(const void *new_ctx, const void *ptr); #else /* DOXYGEN */ /* try to make talloc_set_destructor() and talloc_steal() type safe, if we have a recent gcc */ #if (__GNUC__ >= 3) #define _TALLOC_TYPEOF(ptr) __typeof__(ptr) #define talloc_set_destructor(ptr, function) \ do { \ int (*_talloc_destructor_fn)(_TALLOC_TYPEOF(ptr)) = (function); \ _talloc_set_destructor((ptr), (int (*)(void *))_talloc_destructor_fn); \ } while(0) /* this extremely strange macro is to avoid some braindamaged warning stupidity in gcc 4.1.x */ #define talloc_steal(ctx, ptr) ({ _TALLOC_TYPEOF(ptr) __talloc_steal_ret = (_TALLOC_TYPEOF(ptr))_talloc_steal_loc((ctx),(ptr), __location__); __talloc_steal_ret; }) #else /* __GNUC__ >= 3 */ #define talloc_set_destructor(ptr, function) \ _talloc_set_destructor((ptr), (int (*)(void *))(function)) #define _TALLOC_TYPEOF(ptr) void * #define talloc_steal(ctx, ptr) (_TALLOC_TYPEOF(ptr))_talloc_steal_loc((ctx),(ptr), __location__) #endif /* __GNUC__ >= 3 */ void _talloc_set_destructor(const void *ptr, int (*_destructor)(void *)); void *_talloc_steal_loc(const void *new_ctx, const void *ptr, const char *location); #endif /* DOXYGEN */ /** * @brief Assign a name to a talloc chunk. * * Each talloc pointer has a "name". The name is used principally for * debugging purposes, although it is also possible to set and get the name on * a pointer in as a way of "marking" pointers in your code. * * The main use for names on pointer is for "talloc reports". See * talloc_report() and talloc_report_full() for details. Also see * talloc_enable_leak_report() and talloc_enable_leak_report_full(). * * The talloc_set_name() function allocates memory as a child of the * pointer. It is logically equivalent to: * * @code * talloc_set_name_const(ptr, talloc_asprintf(ptr, fmt, ...)); * @endcode * * @param[in] ptr The talloc chunk to assign a name to. * * @param[in] fmt Format string for the name. * * @param[in] ... Add printf-style additional arguments. * * @return The assigned name, NULL on error. * * @note Multiple calls to talloc_set_name() will allocate more memory without * releasing the name. All of the memory is released when the ptr is freed * using talloc_free(). */ const char *talloc_set_name(const void *ptr, const char *fmt, ...) PRINTF_ATTRIBUTE(2,3); #ifdef DOXYGEN /** * @brief Change a talloc chunk's parent. * * This function has the same effect as talloc_steal(), and additionally sets * the source pointer to NULL. You would use it like this: * * @code * struct foo *X = talloc(tmp_ctx, struct foo); * struct foo *Y; * Y = talloc_move(new_ctx, &X); * @endcode * * @param[in] new_ctx The new parent context. * * @param[in] pptr Pointer to the talloc chunk to move. * * @return The pointer of the talloc chunk it has been moved to, * NULL on error. */ void *talloc_move(const void *new_ctx, void **pptr); #else #define talloc_move(ctx, pptr) (_TALLOC_TYPEOF(*(pptr)))_talloc_move((ctx),(void *)(pptr)) void *_talloc_move(const void *new_ctx, const void *pptr); #endif /** * @brief Assign a name to a talloc chunk. * * The function is just like talloc_set_name(), but it takes a string constant, * and is much faster. It is extensively used by the "auto naming" macros, such * as talloc_p(). * * This function does not allocate any memory. It just copies the supplied * pointer into the internal representation of the talloc ptr. This means you * must not pass a name pointer to memory that will disappear before the ptr * is freed with talloc_free(). * * @param[in] ptr The talloc chunk to assign a name to. * * @param[in] name Format string for the name. */ void talloc_set_name_const(const void *ptr, const char *name); /** * @brief Create a named talloc chunk. * * The talloc_named() function creates a named talloc pointer. It is * equivalent to: * * @code * ptr = talloc_size(context, size); * talloc_set_name(ptr, fmt, ....); * @endcode * * @param[in] context The talloc context to hang the result off. * * @param[in] size Number of char's that you want to allocate. * * @param[in] fmt Format string for the name. * * @param[in] ... Additional printf-style arguments. * * @return The allocated memory chunk, NULL on error. * * @see talloc_set_name() */ void *talloc_named(const void *context, size_t size, const char *fmt, ...) PRINTF_ATTRIBUTE(3,4); /** * @brief Basic routine to allocate a chunk of memory. * * This is equivalent to: * * @code * ptr = talloc_size(context, size); * talloc_set_name_const(ptr, name); * @endcode * * @param[in] context The parent context. * * @param[in] size The number of char's that we want to allocate. * * @param[in] name The name the talloc block has. * * @return The allocated memory chunk, NULL on error. */ void *talloc_named_const(const void *context, size_t size, const char *name); #ifdef DOXYGEN /** * @brief Untyped allocation. * * The function should be used when you don't have a convenient type to pass to * talloc(). Unlike talloc(), it is not type safe (as it returns a void *), so * you are on your own for type checking. * * Best to use talloc() or talloc_array() instead. * * @param[in] ctx The talloc context to hang the result off. * * @param[in] size Number of char's that you want to allocate. * * @return The allocated memory chunk, NULL on error. * * Example: * @code * void *mem = talloc_size(NULL, 100); * @endcode */ void *talloc_size(const void *ctx, size_t size); #else #define talloc_size(ctx, size) talloc_named_const(ctx, size, __location__) #endif #ifdef DOXYGEN /** * @brief Allocate into a typed pointer. * * The talloc_ptrtype() macro should be used when you have a pointer and want * to allocate memory to point at with this pointer. When compiling with * gcc >= 3 it is typesafe. Note this is a wrapper of talloc_size() and * talloc_get_name() will return the current location in the source file and * not the type. * * @param[in] ctx The talloc context to hang the result off. * * @param[in] type The pointer you want to assign the result to. * * @return The properly casted allocated memory chunk, NULL on * error. * * Example: * @code * unsigned int *a = talloc_ptrtype(NULL, a); * @endcode */ void *talloc_ptrtype(const void *ctx, #type); #else #define talloc_ptrtype(ctx, ptr) (_TALLOC_TYPEOF(ptr))talloc_size(ctx, sizeof(*(ptr))) #endif #ifdef DOXYGEN /** * @brief Allocate a new 0-sized talloc chunk. * * This is a utility macro that creates a new memory context hanging off an * existing context, automatically naming it "talloc_new: __location__" where * __location__ is the source line it is called from. It is particularly * useful for creating a new temporary working context. * * @param[in] ctx The talloc parent context. * * @return A new talloc chunk, NULL on error. */ void *talloc_new(const void *ctx); #else #define talloc_new(ctx) talloc_named_const(ctx, 0, "talloc_new: " __location__) #endif #ifdef DOXYGEN /** * @brief Allocate a 0-initizialized structure. * * The macro is equivalent to: * * @code * ptr = talloc(ctx, type); * if (ptr) memset(ptr, 0, sizeof(type)); * @endcode * * @param[in] ctx The talloc context to hang the result off. * * @param[in] type The type that we want to allocate. * * @return Pointer to a piece of memory, properly cast to 'type *', * NULL on error. * * Example: * @code * unsigned int *a, *b; * a = talloc_zero(NULL, unsigned int); * b = talloc_zero(a, unsigned int); * @endcode * * @see talloc() * @see talloc_zero_size() * @see talloc_zero_array() */ void *talloc_zero(const void *ctx, #type); /** * @brief Allocate untyped, 0-initialized memory. * * @param[in] ctx The talloc context to hang the result off. * * @param[in] size Number of char's that you want to allocate. * * @return The allocated memory chunk. */ void *talloc_zero_size(const void *ctx, size_t size); #else #define talloc_zero(ctx, type) (type *)_talloc_zero(ctx, sizeof(type), #type) #define talloc_zero_size(ctx, size) _talloc_zero(ctx, size, __location__) void *_talloc_zero(const void *ctx, size_t size, const char *name); #endif /** * @brief Return the name of a talloc chunk. * * @param[in] ptr The talloc chunk. * * @return The current name for the given talloc pointer. * * @see talloc_set_name() */ const char *talloc_get_name(const void *ptr); /** * @brief Verify that a talloc chunk carries a specified name. * * This function checks if a pointer has the specified name. If it does * then the pointer is returned. * * @param[in] ptr The talloc chunk to check. * * @param[in] name The name to check against. * * @return The pointer if the name matches, NULL if it doesn't. */ void *talloc_check_name(const void *ptr, const char *name); /** * @brief Get the parent chunk of a pointer. * * @param[in] ptr The talloc pointer to inspect. * * @return The talloc parent of ptr, NULL on error. */ void *talloc_parent(const void *ptr); /** * @brief Get a talloc chunk's parent name. * * @param[in] ptr The talloc pointer to inspect. * * @return The name of ptr's parent chunk. */ const char *talloc_parent_name(const void *ptr); /** * @brief Get the total size of a talloc chunk including its children. * * The function returns the total size in bytes used by this pointer and all * child pointers. Mostly useful for debugging. * * Passing NULL is allowed, but it will only give a meaningful result if * talloc_enable_leak_report() or talloc_enable_leak_report_full() has * been called. * * @param[in] ptr The talloc chunk. * * @return The total size. */ size_t talloc_total_size(const void *ptr); /** * @brief Get the number of talloc chunks hanging off a chunk. * * The talloc_total_blocks() function returns the total memory block * count used by this pointer and all child pointers. Mostly useful for * debugging. * * Passing NULL is allowed, but it will only give a meaningful result if * talloc_enable_leak_report() or talloc_enable_leak_report_full() has * been called. * * @param[in] ptr The talloc chunk. * * @return The total size. */ size_t talloc_total_blocks(const void *ptr); #ifdef DOXYGEN /** * @brief Duplicate a memory area into a talloc chunk. * * The function is equivalent to: * * @code * ptr = talloc_size(ctx, size); * if (ptr) memcpy(ptr, p, size); * @endcode * * @param[in] t The talloc context to hang the result off. * * @param[in] p The memory chunk you want to duplicate. * * @param[in] size Number of char's that you want copy. * * @return The allocated memory chunk. * * @see talloc_size() */ void *talloc_memdup(const void *t, const void *p, size_t size); #else #define talloc_memdup(t, p, size) _talloc_memdup(t, p, size, __location__) void *_talloc_memdup(const void *t, const void *p, size_t size, const char *name); #endif #ifdef DOXYGEN /** * @brief Assign a type to a talloc chunk. * * This macro allows you to force the name of a pointer to be of a particular * type. This can be used in conjunction with talloc_get_type() to do type * checking on void* pointers. * * It is equivalent to this: * * @code * talloc_set_name_const(ptr, #type) * @endcode * * @param[in] ptr The talloc chunk to assign the type to. * * @param[in] type The type to assign. */ void talloc_set_type(const char *ptr, #type); /** * @brief Get a typed pointer out of a talloc pointer. * * This macro allows you to do type checking on talloc pointers. It is * particularly useful for void* private pointers. It is equivalent to * this: * * @code * (type *)talloc_check_name(ptr, #type) * @endcode * * @param[in] ptr The talloc pointer to check. * * @param[in] type The type to check against. * * @return The properly casted pointer given by ptr, NULL on error. */ type *talloc_get_type(const void *ptr, #type); #else #define talloc_set_type(ptr, type) talloc_set_name_const(ptr, #type) #define talloc_get_type(ptr, type) (type *)talloc_check_name(ptr, #type) #endif #ifdef DOXYGEN /** * @brief Safely turn a void pointer into a typed pointer. * * This macro is used together with talloc(mem_ctx, struct foo). If you had to * assing the talloc chunk pointer to some void pointer variable, * talloc_get_type_abort() is the recommended way to get the convert the void * pointer back to a typed pointer. * * @param[in] ptr The void pointer to convert. * * @param[in] type The type that this chunk contains * * @return The same value as ptr, type-checked and properly cast. */ void *talloc_get_type_abort(const void *ptr, #type); #else #define talloc_get_type_abort(ptr, type) (type *)_talloc_get_type_abort(ptr, #type, __location__) void *_talloc_get_type_abort(const void *ptr, const char *name, const char *location); #endif /** * @brief Find a parent context by name. * * Find a parent memory context of the current context that has the given * name. This can be very useful in complex programs where it may be * difficult to pass all information down to the level you need, but you * know the structure you want is a parent of another context. * * @param[in] ctx The talloc chunk to start from. * * @param[in] name The name of the parent we look for. * * @return The memory context we are looking for, NULL if not * found. */ void *talloc_find_parent_byname(const void *ctx, const char *name); #ifdef DOXYGEN /** * @brief Find a parent context by type. * * Find a parent memory context of the current context that has the given * name. This can be very useful in complex programs where it may be * difficult to pass all information down to the level you need, but you * know the structure you want is a parent of another context. * * Like talloc_find_parent_byname() but takes a type, making it typesafe. * * @param[in] ptr The talloc chunk to start from. * * @param[in] type The type of the parent to look for. * * @return The memory context we are looking for, NULL if not * found. */ void *talloc_find_parent_bytype(const void *ptr, #type); #else #define talloc_find_parent_bytype(ptr, type) (type *)talloc_find_parent_byname(ptr, #type) #endif /** * @brief Allocate a talloc pool. * * A talloc pool is a pure optimization for specific situations. In the * release process for Samba 3.2 we found out that we had become considerably * slower than Samba 3.0 was. Profiling showed that malloc(3) was a large CPU * consumer in benchmarks. For Samba 3.2 we have internally converted many * static buffers to dynamically allocated ones, so malloc(3) being beaten * more was no surprise. But it made us slower. * * talloc_pool() is an optimization to call malloc(3) a lot less for the use * pattern Samba has: The SMB protocol is mainly a request/response protocol * where we have to allocate a certain amount of memory per request and free * that after the SMB reply is sent to the client. * * talloc_pool() creates a talloc chunk that you can use as a talloc parent * exactly as you would use any other ::TALLOC_CTX. The difference is that * when you talloc a child of this pool, no malloc(3) is done. Instead, talloc * just increments a pointer inside the talloc_pool. This also works * recursively. If you use the child of the talloc pool as a parent for * grand-children, their memory is also taken from the talloc pool. * * If there is not enough memory in the pool to allocate the new child, * it will create a new talloc chunk as if the parent was a normal talloc * context. * * If you talloc_free() children of a talloc pool, the memory is not given * back to the system. Instead, free(3) is only called if the talloc_pool() * itself is released with talloc_free(). * * The downside of a talloc pool is that if you talloc_move() a child of a * talloc pool to a talloc parent outside the pool, the whole pool memory is * not free(3)'ed until that moved chunk is also talloc_free()ed. * * @param[in] context The talloc context to hang the result off. * * @param[in] size Size of the talloc pool. * * @return The allocated talloc pool, NULL on error. */ void *talloc_pool(const void *context, size_t size); #ifdef DOXYGEN /** * @brief Allocate a talloc object as/with an additional pool. * * This is like talloc_pool(), but's it's more flexible * and allows an object to be a pool for its children. * * @param[in] ctx The talloc context to hang the result off. * * @param[in] type The type that we want to allocate. * * @param[in] num_subobjects The expected number of subobjects, which will * be allocated within the pool. This allocates * space for talloc_chunk headers. * * @param[in] total_subobjects_size The size that all subobjects can use in total. * * * @return The allocated talloc object, NULL on error. */ void *talloc_pooled_object(const void *ctx, #type, unsigned num_subobjects, size_t total_subobjects_size); #else #define talloc_pooled_object(_ctx, _type, \ _num_subobjects, \ _total_subobjects_size) \ (_type *)_talloc_pooled_object((_ctx), sizeof(_type), #_type, \ (_num_subobjects), \ (_total_subobjects_size)) void *_talloc_pooled_object(const void *ctx, size_t type_size, const char *type_name, unsigned num_subobjects, size_t total_subobjects_size); #endif /** * @brief Free a talloc chunk and NULL out the pointer. * * TALLOC_FREE() frees a pointer and sets it to NULL. Use this if you want * immediate feedback (i.e. crash) if you use a pointer after having free'ed * it. * * @param[in] ctx The chunk to be freed. */ #define TALLOC_FREE(ctx) do { talloc_free(ctx); ctx=NULL; } while(0) /* @} ******************************************************************/ /** * \defgroup talloc_ref The talloc reference function. * @ingroup talloc * * This module contains the definitions around talloc references * * @{ */ /** * @brief Increase the reference count of a talloc chunk. * * The talloc_increase_ref_count(ptr) function is exactly equivalent to: * * @code * talloc_reference(NULL, ptr); * @endcode * * You can use either syntax, depending on which you think is clearer in * your code. * * @param[in] ptr The pointer to increase the reference count. * * @return 0 on success, -1 on error. */ int talloc_increase_ref_count(const void *ptr); /** * @brief Get the number of references to a talloc chunk. * * @param[in] ptr The pointer to retrieve the reference count from. * * @return The number of references. */ size_t talloc_reference_count(const void *ptr); #ifdef DOXYGEN /** * @brief Create an additional talloc parent to a pointer. * * The talloc_reference() function makes "context" an additional parent of * ptr. Each additional reference consumes around 48 bytes of memory on intel * x86 platforms. * * If ptr is NULL, then the function is a no-op, and simply returns NULL. * * After creating a reference you can free it in one of the following ways: * * - you can talloc_free() any parent of the original pointer. That * will reduce the number of parents of this pointer by 1, and will * cause this pointer to be freed if it runs out of parents. * * - you can talloc_free() the pointer itself if it has at maximum one * parent. This behaviour has been changed since the release of version * 2.0. Further informations in the description of "talloc_free". * * For more control on which parent to remove, see talloc_unlink() * @param[in] ctx The additional parent. * * @param[in] ptr The pointer you want to create an additional parent for. * * @return The original pointer 'ptr', NULL if talloc ran out of * memory in creating the reference. * * Example: * @code * unsigned int *a, *b, *c; * a = talloc(NULL, unsigned int); * b = talloc(NULL, unsigned int); * c = talloc(a, unsigned int); * // b also serves as a parent of c. * talloc_reference(b, c); * @endcode * * @see talloc_unlink() */ void *talloc_reference(const void *ctx, const void *ptr); #else #define talloc_reference(ctx, ptr) (_TALLOC_TYPEOF(ptr))_talloc_reference_loc((ctx),(ptr), __location__) void *_talloc_reference_loc(const void *context, const void *ptr, const char *location); #endif /** * @brief Remove a specific parent from a talloc chunk. * * The function removes a specific parent from ptr. The context passed must * either be a context used in talloc_reference() with this pointer, or must be * a direct parent of ptr. * * You can just use talloc_free() instead of talloc_unlink() if there * is at maximum one parent. This behaviour has been changed since the * release of version 2.0. Further informations in the description of * "talloc_free". * * @param[in] context The talloc parent to remove. * * @param[in] ptr The talloc ptr you want to remove the parent from. * * @return 0 on success, -1 on error. * * @note If the parent has already been removed using talloc_free() then * this function will fail and will return -1. Likewise, if ptr is NULL, * then the function will make no modifications and return -1. * * Example: * @code * unsigned int *a, *b, *c; * a = talloc(NULL, unsigned int); * b = talloc(NULL, unsigned int); * c = talloc(a, unsigned int); * // b also serves as a parent of c. * talloc_reference(b, c); * talloc_unlink(b, c); * @endcode */ int talloc_unlink(const void *context, void *ptr); /** * @brief Provide a talloc context that is freed at program exit. * * This is a handy utility function that returns a talloc context * which will be automatically freed on program exit. This can be used * to reduce the noise in memory leak reports. * * Never use this in code that might be used in objects loaded with * dlopen and unloaded with dlclose. talloc_autofree_context() * internally uses atexit(3). Some platforms like modern Linux handles * this fine, but for example FreeBSD does not deal well with dlopen() * and atexit() used simultaneously: dlclose() does not clean up the * list of atexit-handlers, so when the program exits the code that * was registered from within talloc_autofree_context() is gone, the * program crashes at exit. * * @return A talloc context, NULL on error. */ void *talloc_autofree_context(void); /** * @brief Get the size of a talloc chunk. * * This function lets you know the amount of memory allocated so far by * this context. It does NOT account for subcontext memory. * This can be used to calculate the size of an array. * * @param[in] ctx The talloc chunk. * * @return The size of the talloc chunk. */ size_t talloc_get_size(const void *ctx); /** * @brief Show the parentage of a context. * * @param[in] context The talloc context to look at. * * @param[in] file The output to use, a file, stdout or stderr. */ void talloc_show_parents(const void *context, FILE *file); /** * @brief Check if a context is parent of a talloc chunk. * * This checks if context is referenced in the talloc hierarchy above ptr. * * @param[in] context The assumed talloc context. * * @param[in] ptr The talloc chunk to check. * * @return Return 1 if this is the case, 0 if not. */ int talloc_is_parent(const void *context, const void *ptr); /** * @brief Change the parent context of a talloc pointer. * * The function changes the parent context of a talloc pointer. It is typically * used when the context that the pointer is currently a child of is going to be * freed and you wish to keep the memory for a longer time. * * The difference between talloc_reparent() and talloc_steal() is that * talloc_reparent() can specify which parent you wish to change. This is * useful when a pointer has multiple parents via references. * * @param[in] old_parent * @param[in] new_parent * @param[in] ptr * * @return Return the pointer you passed. It does not have any * failure modes. */ void *talloc_reparent(const void *old_parent, const void *new_parent, const void *ptr); /* @} ******************************************************************/ /** * @defgroup talloc_array The talloc array functions * @ingroup talloc * * Talloc contains some handy helpers for handling Arrays conveniently * * @{ */ #ifdef DOXYGEN /** * @brief Allocate an array. * * The macro is equivalent to: * * @code * (type *)talloc_size(ctx, sizeof(type) * count); * @endcode * * except that it provides integer overflow protection for the multiply, * returning NULL if the multiply overflows. * * @param[in] ctx The talloc context to hang the result off. * * @param[in] type The type that we want to allocate. * * @param[in] count The number of 'type' elements you want to allocate. * * @return The allocated result, properly cast to 'type *', NULL on * error. * * Example: * @code * unsigned int *a, *b; * a = talloc_zero(NULL, unsigned int); * b = talloc_array(a, unsigned int, 100); * @endcode * * @see talloc() * @see talloc_zero_array() */ void *talloc_array(const void *ctx, #type, unsigned count); #else #define talloc_array(ctx, type, count) (type *)_talloc_array(ctx, sizeof(type), count, #type) void *_talloc_array(const void *ctx, size_t el_size, unsigned count, const char *name); #endif #ifdef DOXYGEN /** * @brief Allocate an array. * * @param[in] ctx The talloc context to hang the result off. * * @param[in] size The size of an array element. * * @param[in] count The number of elements you want to allocate. * * @return The allocated result, NULL on error. */ void *talloc_array_size(const void *ctx, size_t size, unsigned count); #else #define talloc_array_size(ctx, size, count) _talloc_array(ctx, size, count, __location__) #endif #ifdef DOXYGEN /** * @brief Allocate an array into a typed pointer. * * The macro should be used when you have a pointer to an array and want to * allocate memory of an array to point at with this pointer. When compiling * with gcc >= 3 it is typesafe. Note this is a wrapper of talloc_array_size() * and talloc_get_name() will return the current location in the source file * and not the type. * * @param[in] ctx The talloc context to hang the result off. * * @param[in] ptr The pointer you want to assign the result to. * * @param[in] count The number of elements you want to allocate. * * @return The allocated memory chunk, properly casted. NULL on * error. */ void *talloc_array_ptrtype(const void *ctx, const void *ptr, unsigned count); #else #define talloc_array_ptrtype(ctx, ptr, count) (_TALLOC_TYPEOF(ptr))talloc_array_size(ctx, sizeof(*(ptr)), count) #endif #ifdef DOXYGEN /** * @brief Get the number of elements in a talloc'ed array. * * A talloc chunk carries its own size, so for talloc'ed arrays it is not * necessary to store the number of elements explicitly. * * @param[in] ctx The allocated array. * * @return The number of elements in ctx. */ size_t talloc_array_length(const void *ctx); #else #define talloc_array_length(ctx) (talloc_get_size(ctx)/sizeof(*ctx)) #endif #ifdef DOXYGEN /** * @brief Allocate a zero-initialized array * * @param[in] ctx The talloc context to hang the result off. * * @param[in] type The type that we want to allocate. * * @param[in] count The number of "type" elements you want to allocate. * * @return The allocated result casted to "type *", NULL on error. * * The talloc_zero_array() macro is equivalent to: * * @code * ptr = talloc_array(ctx, type, count); * if (ptr) memset(ptr, sizeof(type) * count); * @endcode */ void *talloc_zero_array(const void *ctx, #type, unsigned count); #else #define talloc_zero_array(ctx, type, count) (type *)_talloc_zero_array(ctx, sizeof(type), count, #type) void *_talloc_zero_array(const void *ctx, size_t el_size, unsigned count, const char *name); #endif #ifdef DOXYGEN /** * @brief Change the size of a talloc array. * * The macro changes the size of a talloc pointer. The 'count' argument is the * number of elements of type 'type' that you want the resulting pointer to * hold. * * talloc_realloc() has the following equivalences: * * @code * talloc_realloc(ctx, NULL, type, 1) ==> talloc(ctx, type); * talloc_realloc(ctx, NULL, type, N) ==> talloc_array(ctx, type, N); * talloc_realloc(ctx, ptr, type, 0) ==> talloc_free(ptr); * @endcode * * The "context" argument is only used if "ptr" is NULL, otherwise it is * ignored. * * @param[in] ctx The parent context used if ptr is NULL. * * @param[in] ptr The chunk to be resized. * * @param[in] type The type of the array element inside ptr. * * @param[in] count The intended number of array elements. * * @return The new array, NULL on error. The call will fail either * due to a lack of memory, or because the pointer has more * than one parent (see talloc_reference()). */ void *talloc_realloc(const void *ctx, void *ptr, #type, size_t count); #else #define talloc_realloc(ctx, p, type, count) (type *)_talloc_realloc_array(ctx, p, sizeof(type), count, #type) void *_talloc_realloc_array(const void *ctx, void *ptr, size_t el_size, unsigned count, const char *name); #endif #ifdef DOXYGEN /** * @brief Untyped realloc to change the size of a talloc array. * * The macro is useful when the type is not known so the typesafe * talloc_realloc() cannot be used. * * @param[in] ctx The parent context used if 'ptr' is NULL. * * @param[in] ptr The chunk to be resized. * * @param[in] size The new chunk size. * * @return The new array, NULL on error. */ void *talloc_realloc_size(const void *ctx, void *ptr, size_t size); #else #define talloc_realloc_size(ctx, ptr, size) _talloc_realloc(ctx, ptr, size, __location__) void *_talloc_realloc(const void *context, void *ptr, size_t size, const char *name); #endif /** * @brief Provide a function version of talloc_realloc_size. * * This is a non-macro version of talloc_realloc(), which is useful as * libraries sometimes want a ralloc function pointer. A realloc() * implementation encapsulates the functionality of malloc(), free() and * realloc() in one call, which is why it is useful to be able to pass around * a single function pointer. * * @param[in] context The parent context used if ptr is NULL. * * @param[in] ptr The chunk to be resized. * * @param[in] size The new chunk size. * * @return The new chunk, NULL on error. */ void *talloc_realloc_fn(const void *context, void *ptr, size_t size); /* @} ******************************************************************/ /** * @defgroup talloc_string The talloc string functions. * @ingroup talloc * * talloc string allocation and manipulation functions. * @{ */ /** * @brief Duplicate a string into a talloc chunk. * * This function is equivalent to: * * @code * ptr = talloc_size(ctx, strlen(p)+1); * if (ptr) memcpy(ptr, p, strlen(p)+1); * @endcode * * This functions sets the name of the new pointer to the passed * string. This is equivalent to: * * @code * talloc_set_name_const(ptr, ptr) * @endcode * * @param[in] t The talloc context to hang the result off. * * @param[in] p The string you want to duplicate. * * @return The duplicated string, NULL on error. */ char *talloc_strdup(const void *t, const char *p); /** * @brief Append a string to given string. * * The destination string is reallocated to take * strlen(s) + strlen(a) + 1 characters. * * This functions sets the name of the new pointer to the new * string. This is equivalent to: * * @code * talloc_set_name_const(ptr, ptr) * @endcode * * If s == NULL then new context is created. * * @param[in] s The destination to append to. * * @param[in] a The string you want to append. * * @return The concatenated strings, NULL on error. * * @see talloc_strdup() * @see talloc_strdup_append_buffer() */ char *talloc_strdup_append(char *s, const char *a); /** * @brief Append a string to a given buffer. * * This is a more efficient version of talloc_strdup_append(). It determines the * length of the destination string by the size of the talloc context. * * Use this very carefully as it produces a different result than * talloc_strdup_append() when a zero character is in the middle of the * destination string. * * @code * char *str_a = talloc_strdup(NULL, "hello world"); * char *str_b = talloc_strdup(NULL, "hello world"); * str_a[5] = str_b[5] = '\0' * * char *app = talloc_strdup_append(str_a, ", hello"); * char *buf = talloc_strdup_append_buffer(str_b, ", hello"); * * printf("%s\n", app); // hello, hello (app = "hello, hello") * printf("%s\n", buf); // hello (buf = "hello\0world, hello") * @endcode * * If s == NULL then new context is created. * * @param[in] s The destination buffer to append to. * * @param[in] a The string you want to append. * * @return The concatenated strings, NULL on error. * * @see talloc_strdup() * @see talloc_strdup_append() * @see talloc_array_length() */ char *talloc_strdup_append_buffer(char *s, const char *a); /** * @brief Duplicate a length-limited string into a talloc chunk. * * This function is the talloc equivalent of the C library function strndup(3). * * This functions sets the name of the new pointer to the passed string. This is * equivalent to: * * @code * talloc_set_name_const(ptr, ptr) * @endcode * * @param[in] t The talloc context to hang the result off. * * @param[in] p The string you want to duplicate. * * @param[in] n The maximum string length to duplicate. * * @return The duplicated string, NULL on error. */ char *talloc_strndup(const void *t, const char *p, size_t n); /** * @brief Append at most n characters of a string to given string. * * The destination string is reallocated to take * strlen(s) + strnlen(a, n) + 1 characters. * * This functions sets the name of the new pointer to the new * string. This is equivalent to: * * @code * talloc_set_name_const(ptr, ptr) * @endcode * * If s == NULL then new context is created. * * @param[in] s The destination string to append to. * * @param[in] a The source string you want to append. * * @param[in] n The number of characters you want to append from the * string. * * @return The concatenated strings, NULL on error. * * @see talloc_strndup() * @see talloc_strndup_append_buffer() */ char *talloc_strndup_append(char *s, const char *a, size_t n); /** * @brief Append at most n characters of a string to given buffer * * This is a more efficient version of talloc_strndup_append(). It determines * the length of the destination string by the size of the talloc context. * * Use this very carefully as it produces a different result than * talloc_strndup_append() when a zero character is in the middle of the * destination string. * * @code * char *str_a = talloc_strdup(NULL, "hello world"); * char *str_b = talloc_strdup(NULL, "hello world"); * str_a[5] = str_b[5] = '\0' * * char *app = talloc_strndup_append(str_a, ", hello", 7); * char *buf = talloc_strndup_append_buffer(str_b, ", hello", 7); * * printf("%s\n", app); // hello, hello (app = "hello, hello") * printf("%s\n", buf); // hello (buf = "hello\0world, hello") * @endcode * * If s == NULL then new context is created. * * @param[in] s The destination buffer to append to. * * @param[in] a The source string you want to append. * * @param[in] n The number of characters you want to append from the * string. * * @return The concatenated strings, NULL on error. * * @see talloc_strndup() * @see talloc_strndup_append() * @see talloc_array_length() */ char *talloc_strndup_append_buffer(char *s, const char *a, size_t n); /** * @brief Format a string given a va_list. * * This function is the talloc equivalent of the C library function * vasprintf(3). * * This functions sets the name of the new pointer to the new string. This is * equivalent to: * * @code * talloc_set_name_const(ptr, ptr) * @endcode * * @param[in] t The talloc context to hang the result off. * * @param[in] fmt The format string. * * @param[in] ap The parameters used to fill fmt. * * @return The formatted string, NULL on error. */ char *talloc_vasprintf(const void *t, const char *fmt, va_list ap) PRINTF_ATTRIBUTE(2,0); /** * @brief Format a string given a va_list and append it to the given destination * string. * * @param[in] s The destination string to append to. * * @param[in] fmt The format string. * * @param[in] ap The parameters used to fill fmt. * * @return The formatted string, NULL on error. * * @see talloc_vasprintf() */ char *talloc_vasprintf_append(char *s, const char *fmt, va_list ap) PRINTF_ATTRIBUTE(2,0); /** * @brief Format a string given a va_list and append it to the given destination * buffer. * * @param[in] s The destination buffer to append to. * * @param[in] fmt The format string. * * @param[in] ap The parameters used to fill fmt. * * @return The formatted string, NULL on error. * * @see talloc_vasprintf() */ char *talloc_vasprintf_append_buffer(char *s, const char *fmt, va_list ap) PRINTF_ATTRIBUTE(2,0); /** * @brief Format a string. * * This function is the talloc equivalent of the C library function asprintf(3). * * This functions sets the name of the new pointer to the new string. This is * equivalent to: * * @code * talloc_set_name_const(ptr, ptr) * @endcode * * @param[in] t The talloc context to hang the result off. * * @param[in] fmt The format string. * * @param[in] ... The parameters used to fill fmt. * * @return The formatted string, NULL on error. */ char *talloc_asprintf(const void *t, const char *fmt, ...) PRINTF_ATTRIBUTE(2,3); /** * @brief Append a formatted string to another string. * * This function appends the given formatted string to the given string. Use * this variant when the string in the current talloc buffer may have been * truncated in length. * * This functions sets the name of the new pointer to the new * string. This is equivalent to: * * @code * talloc_set_name_const(ptr, ptr) * @endcode * * If s == NULL then new context is created. * * @param[in] s The string to append to. * * @param[in] fmt The format string. * * @param[in] ... The parameters used to fill fmt. * * @return The formatted string, NULL on error. */ char *talloc_asprintf_append(char *s, const char *fmt, ...) PRINTF_ATTRIBUTE(2,3); /** * @brief Append a formatted string to another string. * * This is a more efficient version of talloc_asprintf_append(). It determines * the length of the destination string by the size of the talloc context. * * Use this very carefully as it produces a different result than * talloc_asprintf_append() when a zero character is in the middle of the * destination string. * * @code * char *str_a = talloc_strdup(NULL, "hello world"); * char *str_b = talloc_strdup(NULL, "hello world"); * str_a[5] = str_b[5] = '\0' * * char *app = talloc_asprintf_append(str_a, "%s", ", hello"); * char *buf = talloc_strdup_append_buffer(str_b, "%s", ", hello"); * * printf("%s\n", app); // hello, hello (app = "hello, hello") * printf("%s\n", buf); // hello (buf = "hello\0world, hello") * @endcode * * If s == NULL then new context is created. * * @param[in] s The string to append to * * @param[in] fmt The format string. * * @param[in] ... The parameters used to fill fmt. * * @return The formatted string, NULL on error. * * @see talloc_asprintf() * @see talloc_asprintf_append() */ char *talloc_asprintf_append_buffer(char *s, const char *fmt, ...) PRINTF_ATTRIBUTE(2,3); /* @} ******************************************************************/ /** * @defgroup talloc_debug The talloc debugging support functions * @ingroup talloc * * To aid memory debugging, talloc contains routines to inspect the currently * allocated memory hierarchy. * * @{ */ /** * @brief Walk a complete talloc hierarchy. * * This provides a more flexible reports than talloc_report(). It * will recursively call the callback for the entire tree of memory * referenced by the pointer. References in the tree are passed with * is_ref = 1 and the pointer that is referenced. * * You can pass NULL for the pointer, in which case a report is * printed for the top level memory context, but only if * talloc_enable_leak_report() or talloc_enable_leak_report_full() * has been called. * * The recursion is stopped when depth >= max_depth. * max_depth = -1 means only stop at leaf nodes. * * @param[in] ptr The talloc chunk. * * @param[in] depth Internal parameter to control recursion. Call with 0. * * @param[in] max_depth Maximum recursion level. * * @param[in] callback Function to be called on every chunk. * * @param[in] private_data Private pointer passed to callback. */ void talloc_report_depth_cb(const void *ptr, int depth, int max_depth, void (*callback)(const void *ptr, int depth, int max_depth, int is_ref, void *private_data), void *private_data); /** * @brief Print a talloc hierarchy. * * This provides a more flexible reports than talloc_report(). It * will let you specify the depth and max_depth. * * @param[in] ptr The talloc chunk. * * @param[in] depth Internal parameter to control recursion. Call with 0. * * @param[in] max_depth Maximum recursion level. * * @param[in] f The file handle to print to. */ void talloc_report_depth_file(const void *ptr, int depth, int max_depth, FILE *f); /** * @brief Print a summary report of all memory used by ptr. * * This provides a more detailed report than talloc_report(). It will * recursively print the entire tree of memory referenced by the * pointer. References in the tree are shown by giving the name of the * pointer that is referenced. * * You can pass NULL for the pointer, in which case a report is printed * for the top level memory context, but only if * talloc_enable_leak_report() or talloc_enable_leak_report_full() has * been called. * * @param[in] ptr The talloc chunk. * * @param[in] f The file handle to print to. * * Example: * @code * unsigned int *a, *b; * a = talloc(NULL, unsigned int); * b = talloc(a, unsigned int); * fprintf(stderr, "Dumping memory tree for a:\n"); * talloc_report_full(a, stderr); * @endcode * * @see talloc_report() */ void talloc_report_full(const void *ptr, FILE *f); /** * @brief Print a summary report of all memory used by ptr. * * This function prints a summary report of all memory used by ptr. One line of * report is printed for each immediate child of ptr, showing the total memory * and number of blocks used by that child. * * You can pass NULL for the pointer, in which case a report is printed * for the top level memory context, but only if talloc_enable_leak_report() * or talloc_enable_leak_report_full() has been called. * * @param[in] ptr The talloc chunk. * * @param[in] f The file handle to print to. * * Example: * @code * unsigned int *a, *b; * a = talloc(NULL, unsigned int); * b = talloc(a, unsigned int); * fprintf(stderr, "Summary of memory tree for a:\n"); * talloc_report(a, stderr); * @endcode * * @see talloc_report_full() */ void talloc_report(const void *ptr, FILE *f); /** * @brief Enable tracking the use of NULL memory contexts. * * This enables tracking of the NULL memory context without enabling leak * reporting on exit. Useful for when you want to do your own leak * reporting call via talloc_report_null_full(); */ void talloc_enable_null_tracking(void); /** * @brief Enable tracking the use of NULL memory contexts. * * This enables tracking of the NULL memory context without enabling leak * reporting on exit. Useful for when you want to do your own leak * reporting call via talloc_report_null_full(); */ void talloc_enable_null_tracking_no_autofree(void); /** * @brief Disable tracking of the NULL memory context. * * This disables tracking of the NULL memory context. */ void talloc_disable_null_tracking(void); /** * @brief Enable leak report when a program exits. * * This enables calling of talloc_report(NULL, stderr) when the program * exits. In Samba4 this is enabled by using the --leak-report command * line option. * * For it to be useful, this function must be called before any other * talloc function as it establishes a "null context" that acts as the * top of the tree. If you don't call this function first then passing * NULL to talloc_report() or talloc_report_full() won't give you the * full tree printout. * * Here is a typical talloc report: * * @code * talloc report on 'null_context' (total 267 bytes in 15 blocks) * libcli/auth/spnego_parse.c:55 contains 31 bytes in 2 blocks * libcli/auth/spnego_parse.c:55 contains 31 bytes in 2 blocks * iconv(UTF8,CP850) contains 42 bytes in 2 blocks * libcli/auth/spnego_parse.c:55 contains 31 bytes in 2 blocks * iconv(CP850,UTF8) contains 42 bytes in 2 blocks * iconv(UTF8,UTF-16LE) contains 45 bytes in 2 blocks * iconv(UTF-16LE,UTF8) contains 45 bytes in 2 blocks * @endcode */ void talloc_enable_leak_report(void); /** * @brief Enable full leak report when a program exits. * * This enables calling of talloc_report_full(NULL, stderr) when the * program exits. In Samba4 this is enabled by using the * --leak-report-full command line option. * * For it to be useful, this function must be called before any other * talloc function as it establishes a "null context" that acts as the * top of the tree. If you don't call this function first then passing * NULL to talloc_report() or talloc_report_full() won't give you the * full tree printout. * * Here is a typical full report: * * @code * full talloc report on 'root' (total 18 bytes in 8 blocks) * p1 contains 18 bytes in 7 blocks (ref 0) * r1 contains 13 bytes in 2 blocks (ref 0) * reference to: p2 * p2 contains 1 bytes in 1 blocks (ref 1) * x3 contains 1 bytes in 1 blocks (ref 0) * x2 contains 1 bytes in 1 blocks (ref 0) * x1 contains 1 bytes in 1 blocks (ref 0) * @endcode */ void talloc_enable_leak_report_full(void); /** * @brief Set a custom "abort" function that is called on serious error. * * The default "abort" function is abort(). * * The "abort" function is called when: * *
    *
  • talloc_get_type_abort() fails
    • *
  • the provided pointer is not a valid talloc context
    • *
  • when the context meta data are invalid
    • *
  • when access after free is detected
    • *
* * Example: * * @code * void my_abort(const char *reason) * { * fprintf(stderr, "talloc abort: %s\n", reason); * abort(); * } * * talloc_set_abort_fn(my_abort); * @endcode * * @param[in] abort_fn The new "abort" function. * * @see talloc_set_log_fn() * @see talloc_get_type() */ void talloc_set_abort_fn(void (*abort_fn)(const char *reason)); /** * @brief Set a logging function. * * @param[in] log_fn The logging function. * * @see talloc_set_log_stderr() * @see talloc_set_abort_fn() */ void talloc_set_log_fn(void (*log_fn)(const char *message)); /** * @brief Set stderr as the output for logs. * * @see talloc_set_log_fn() * @see talloc_set_abort_fn() */ void talloc_set_log_stderr(void); /** * @brief Set a max memory limit for the current context hierarchy * This affects all children of this context and constrain any * allocation in the hierarchy to never exceed the limit set. * The limit can be removed by setting 0 (unlimited) as the * max_size by calling the funciton again on the sam context. * Memory limits can also be nested, meaning a hild can have * a stricter memory limit than a parent. * Memory limits are enforced only at memory allocation time. * Stealing a context into a 'limited' hierarchy properly * updates memory usage but does *not* cause failure if the * move causes the new parent to exceed its limits. However * any further allocation on that hierarchy will then fail. * * @param[in] ctx The talloc context to set the limit on * @param[in] max_size The (new) max_size */ int talloc_set_memlimit(const void *ctx, size_t max_size); /* @} ******************************************************************/ #if TALLOC_DEPRECATED #define talloc_zero_p(ctx, type) talloc_zero(ctx, type) #define talloc_p(ctx, type) talloc(ctx, type) #define talloc_array_p(ctx, type, count) talloc_array(ctx, type, count) #define talloc_realloc_p(ctx, p, type, count) talloc_realloc(ctx, p, type, count) #define talloc_destroy(ctx) talloc_free(ctx) #define talloc_append_string(c, s, a) (s?talloc_strdup_append(s,a):talloc_strdup(c, a)) #endif #ifndef TALLOC_MAX_DEPTH #define TALLOC_MAX_DEPTH 10000 #endif #ifdef __cplusplus } /* end of extern "C" */ #endif #endif pytsk-20231007/test_data/000077500000000000000000000000001451023402500150705ustar00rootroot00000000000000pytsk-20231007/test_data/bogus.raw000066400000000000000000000000141451023402500167150ustar00rootroot00000000000000BOGUS DATA. pytsk-20231007/test_data/image.raw000066400000000000000000003100001451023402500166570ustar00rootroot00000000000000dK Ĭ¿OS¿ONí €8ñøÚ¸WïIÛœõ´ W6íŃ’ÌÓÐD™®ú QÓUÿΆ«¿OKÿÿ?`øÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿïÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ†«¿O†«¿O†«¿OíACN·¬¿Oµ¬¿Oµ¬¿Oˆ€0†«¿O†«¿O†«¿OÀA0†«¿O†«¿O†«¿O íACN\¬¿O[¬¿O[¬¿Oˆ?ü ¡€CNh¬¿Oµ¬¿Of¬¿Oµ¬¿OˆJü ¡€CN5R¬¿OR¬¿OR¬¿OˆCü ¡CNtµ¬¿O¿¬¿Oµ¬¿OˆNü ¡€CN[¬¿O[¬¿O[¬¿OˆIü ¡ . ..  lost+found  a_directoryÀ passwords.txt¨passwords.txt~.swp .ô.. . .. another_fileÔa_filee.Ä.another_file.swp¨ another_file~.swpxplace,user,password bank,joesmith,superrich alarm system,-,1234 treasure chest,-,1111 uber secret laire,admin,admin This is a text file. We should be able to parse it. This is another file. pytsk-20231007/test_data/tsk_volume_system.raw000066400000000000000000055000001451023402500213770ustar00rootroot00000000000000î¦õSƒ$^%-_á Uª&ƒ-à Uª ð?Å•  ÈŒ¹RÿÿSïÈŒ¹R €8å´Yú½1N…–>ª™¿ü2_­­‘$ˆA¦Ÿõ­‹_d£s ÈŒ¹R Å•ÿÿÿÿÿ€ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÈŒ¹RÈŒ¹RÈŒ¹RíAÈŒ¹RÈŒ¹RÈŒ¹R€0ÈŒ¹RÈŒ¹RÈŒ¹R *ÀA0ÈŒ¹RÈŒ¹RÈŒ¹R !"#$%&'() . .. è lost+found .ô..pytsk-20231007/tests/000077500000000000000000000000001451023402500142625ustar00rootroot00000000000000pytsk-20231007/tests/__init__.py000066400000000000000000000011571451023402500163770ustar00rootroot00000000000000#!/usr/bin/python # # Copyright 2013, Joachim Metz . # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. pytsk-20231007/tests/fs_info.py000066400000000000000000000115031451023402500162570ustar00rootroot00000000000000#!/usr/bin/python # -*- coding: utf-8 -*- """Tests for FS_Info.""" import os import unittest import pytsk3 import test_lib # fls -l ./test_data/image.raw # d/d 11: lost+found 2012-05-25 17:55:50 (CEST) # 2012-05-25 17:55:50 (CEST) 2012-05-25 17:55:50 (CEST) # 0000-00-00 00:00:00 (UTC) 12288 0 0 # d/d 12: a_directory 2012-05-25 17:59:23 (CEST) # 2012-05-25 17:59:24 (CEST) 2012-05-25 17:59:23 (CEST) # 0000-00-00 00:00:00 (UTC) 1024 5000 151107 # r/r 15: passwords.txt 2012-05-25 18:00:53 (CEST) # 2012-05-25 18:00:53 (CEST) 2012-05-25 18:01:03 (CEST) # 0000-00-00 00:00:00 (UTC) 116 5000 151107 # r/- * 0: passwords.txt~ 0000-00-00 00:00:00 (UTC) # 0000-00-00 00:00:00 (UTC) 0000-00-00 00:00:00 (UTC) # 0000-00-00 00:00:00 (UTC) 0 0 0 # d/d 17: $OrphanFiles 0000-00-00 00:00:00 (UTC) # 0000-00-00 00:00:00 (UTC) 0000-00-00 00:00:00 (UTC) # 0000-00-00 00:00:00 (UTC) 0 0 0 class TSKFsInfoTestCase(unittest.TestCase): """FS_Info test case.""" def _testInitialize(self, fs_info): """Test the initialize functionality. Args: fs_info: the FS_Info object. """ self.assertNotEqual(fs_info, None) def _testOpenMeta(self, fs_info): """Test the open meta functionality. Args: fs_info: the FS_Info object. """ self.assertNotEqual(fs_info, None) file_object = fs_info.open_meta(15) self.assertNotEqual(file_object, None) with self.assertRaises(IOError): file_object = fs_info.open_meta(19) class TSKFsInfoTest(TSKFsInfoTestCase): """FS_Info for testing.""" def setUp(self): """Sets up the needed objects used throughout the test.""" test_file = os.path.join('test_data', 'image.raw') self._img_info = pytsk3.Img_Info(test_file) def testInitialize(self): """Test the initialize functionality.""" fs_info = pytsk3.FS_Info(self._img_info, offset=0) self._testInitialize(fs_info) def testOpenMeta(self): """Test the open meta functionality.""" fs_info = pytsk3.FS_Info(self._img_info, offset=0) self._testOpenMeta(fs_info) class TSKFsInfoBogusTest(TSKFsInfoTestCase): """FS_Info for testing that fails.""" def setUp(self): """Sets up the needed objects used throughout the test.""" test_file = os.path.join('test_data', 'bogus.raw') self._img_info = pytsk3.Img_Info(test_file) def testInitialize(self): """Test the initialize functionality.""" with self.assertRaises(IOError): pytsk3.FS_Info(self._img_info, offset=0) class TSKFsInfoFileObjectTest(TSKFsInfoTestCase): """Tests the FS_Info object using an Img_Info file-like object.""" def setUp(self): """Sets up the needed objects used throughout the test.""" test_file = os.path.join('test_data', 'image.raw') self._file_object = open(test_file, 'rb') stat_info = os.stat(test_file) self._file_size = stat_info.st_size self._img_info = test_lib.FileObjectImageInfo( self._file_object, self._file_size) def testInitialize(self): """Test the initialize functionality.""" fs_info = pytsk3.FS_Info(self._img_info, offset=0) self._testInitialize(fs_info) def testOpenMeta(self): """Test the open meta functionality.""" fs_info = pytsk3.FS_Info(self._img_info, offset=0) self._testOpenMeta(fs_info) class TSKFsInfoFileObjectWithDetectTest(TSKFsInfoTestCase): """Tests the FS_Info object with auto-detect Img_Info.""" def setUp(self): """Sets up the needed objects used throughout the test.""" test_file = os.path.join('test_data', 'image.raw') self._file_object = open(test_file, 'rb') stat_info = os.stat(test_file) self._file_size = stat_info.st_size self._img_info = test_lib.FileObjectImageInfo( self._file_object, self._file_size, image_type=pytsk3.TSK_IMG_TYPE_DETECT) def testInitialize(self): """Test the initialize functionality.""" fs_info = pytsk3.FS_Info(self._img_info, offset=0) self._testInitialize(fs_info) def testOpenMeta(self): """Test the open meta functionality.""" fs_info = pytsk3.FS_Info(self._img_info, offset=0) self._testOpenMeta(fs_info) class TSKFsInfoFileObjectWithLargeSize(TSKFsInfoTestCase): """Tests the FS_Info object with a large size Img_Info.""" def setUp(self): """Sets up the needed objects used throughout the test.""" test_file = os.path.join('test_data', 'image.raw') self._file_object = open(test_file, 'rb') self._file_size = 1024 * 1024 * 1024 * 1024 self._img_info = test_lib.FileObjectImageInfo( self._file_object, self._file_size) def testInitialize(self): """Test the initialize functionality.""" fs_info = pytsk3.FS_Info(self._img_info, offset=0) self._testInitialize(fs_info) def testOpenMeta(self): """Test the open meta functionality.""" fs_info = pytsk3.FS_Info(self._img_info, offset=0) self._testOpenMeta(fs_info) if __name__ == '__main__': unittest.main() pytsk-20231007/tests/img_info.py000066400000000000000000000131221451023402500164220ustar00rootroot00000000000000#!/usr/bin/python # -*- coding: utf-8 -*- """Tests for Img_Info.""" import os import unittest import pytsk3 import test_lib class TSKImgInfoTestCase(unittest.TestCase): """Img_Info test case.""" def _testInitialize(self, img_info): """Test the initialize functionality. Args: img_info: the Img_Info object. """ self.assertNotEqual(img_info, None) def _testGetSize(self, img_info): """Test the get size functionality. Args: img_info: the Img_Info object. """ self.assertNotEqual(img_info, None) self.assertEqual(img_info.get_size(), self._file_size) def _testRead(self, img_info): """Test the read functionality. Args: img_info: the Img_Info object. """ self.assertNotEqual(img_info, None) self.assertEqual(img_info.read(0x5800, 16), b'place,user,passw') self.assertEqual(img_info.read(0x7c00, 16), b'This is another ') # Conforming to the POSIX seek the offset can exceed the file size # but reading will result in no data being returned. self.assertEqual(img_info.read(0x19000, 16), b'') with self.assertRaises(IOError): img_info.read(-1, 16) class TSKImgInfoTest(TSKImgInfoTestCase): """The unit test for the Img_Info object.""" def setUp(self): """Sets up the needed objects used throughout the test.""" self._test_file = os.path.join('test_data', 'image.raw') self._file_size = 102400 def testInitialize(self): """Test the initialize functionality.""" img_info = pytsk3.Img_Info(url=self._test_file) self._testInitialize(img_info) img_info.close() def testGetSize(self): """Test the get size functionality.""" img_info = pytsk3.Img_Info(url=self._test_file) self._testGetSize(img_info) img_info.close() def testRead(self): """Test the read functionality.""" img_info = pytsk3.Img_Info(url=self._test_file) self.assertNotEqual(img_info, None) self.assertEqual(img_info.read(0x5800, 16), b'place,user,passw') self.assertEqual(img_info.read(0x7c00, 16), b'This is another ') # Conforming to the POSIX seek the offset can exceed the file size # but reading will result in no data being returned. Note that the SleuthKit # does not conform to the posix standard and will raise and IO error. with self.assertRaises(IOError): img_info.read(0x19000, 16) with self.assertRaises(IOError): img_info.read(-1, 16) img_info.close() class TSKImgInfoFileObjectTest(TSKImgInfoTestCase): """The unit test for the Img_Info object using a file-like object.""" def setUp(self): """Sets up the needed objects used throughout the test.""" test_file = os.path.join('test_data', 'image.raw') self._file_object = open(test_file, 'rb') stat_info = os.stat(test_file) self._file_size = stat_info.st_size def testInitialize(self): """Test the initialize functionality.""" img_info = test_lib.FileObjectImageInfo(self._file_object, self._file_size) self._testInitialize(img_info) img_info.close() def testGetSize(self): """Test the get size functionality.""" img_info = test_lib.FileObjectImageInfo(self._file_object, self._file_size) self._testGetSize(img_info) img_info.close() def testRead(self): """Test the read functionality.""" img_info = test_lib.FileObjectImageInfo(self._file_object, self._file_size) self._testRead(img_info) img_info.close() class TSKImgInfoFileObjectWithDetectTest(TSKImgInfoTestCase): """The unit test for the Img_Info object using a file-like object with image type: pytsk3.TSK_IMG_TYPE_DETECT.""" def setUp(self): """Sets up the needed objects used throughout the test.""" test_file = os.path.join('test_data', 'image.raw') self._file_object = open(test_file, 'rb') stat_info = os.stat(test_file) self._file_size = stat_info.st_size def testInitialize(self): """Test the initialize functionality.""" img_info = test_lib.FileObjectImageInfo( self._file_object, self._file_size, image_type=pytsk3.TSK_IMG_TYPE_DETECT) self._testInitialize(img_info) img_info.close() def testGetSize(self): """Test the get size functionality.""" img_info = test_lib.FileObjectImageInfo( self._file_object, self._file_size, image_type=pytsk3.TSK_IMG_TYPE_DETECT) self._testGetSize(img_info) img_info.close() def testRead(self): """Test the read functionality.""" img_info = test_lib.FileObjectImageInfo( self._file_object, self._file_size, image_type=pytsk3.TSK_IMG_TYPE_DETECT) self._testRead(img_info) img_info.close() class TSKImgInfoFileObjectLargeSizeTest(TSKImgInfoTestCase): """The unit test for the Img_Info object using a file-like object with a large size.""" def setUp(self): """Sets up the needed objects used throughout the test.""" test_file = os.path.join('test_data', 'image.raw') self._file_object = open(test_file, 'rb') self._file_size = 1024 * 1024 * 1024 * 1024 def testInitialize(self): """Test the initialize functionality.""" img_info = test_lib.FileObjectImageInfo(self._file_object, self._file_size) self._testInitialize(img_info) img_info.close() def testGetSize(self): """Test the get size functionality.""" img_info = test_lib.FileObjectImageInfo(self._file_object, self._file_size) self._testGetSize(img_info) img_info.close() def testRead(self): """Test the read functionality.""" img_info = test_lib.FileObjectImageInfo(self._file_object, self._file_size) self._testRead(img_info) img_info.close() if __name__ == '__main__': unittest.main() pytsk-20231007/tests/test_lib.py000066400000000000000000000031511451023402500164410ustar00rootroot00000000000000"""Shared test case.""" import os import pytsk3 class FileObjectImageInfo(pytsk3.Img_Info): """Img_Info that uses a file-like object.""" def __init__( self, file_object, file_size, image_type=pytsk3.TSK_IMG_TYPE_RAW): """Initializes the image object. Args: file_object: the file-like object (instance of io.FileIO). file_size: the file size. image_type: optional SleuthKit image type. The default is RAW (pytsk3.TSK_IMG_TYPE_RAW). Raises: ValueError: if the file-like object is invalid. """ if not file_object: raise ValueError(u'Missing file-like object.') # pytsk3.Img_Info does not let you set attributes after initialization. self._file_object = file_object self._file_size = file_size # Using the old parent class invocation style otherwise some versions # of pylint complain also setting type to RAW to make sure Img_Info # does not do detection. pytsk3.Img_Info.__init__(self, url='', type=image_type) # Note: that the following functions are part of the pytsk3.Img_Info object # interface. def close(self): """Closes the volume IO object.""" self._file_object = None def read(self, offset, size): """Reads a byte string from the image object at the specified offset. Args: offset: offset where to start reading. size: number of bytes to read. Returns: A byte string containing the data read. """ self._file_object.seek(offset, os.SEEK_SET) return self._file_object.read(size) def get_size(self): """Retrieves the size.""" return self._file_size pytsk-20231007/tests/volume_info.py000066400000000000000000000160271451023402500171640ustar00rootroot00000000000000#!/usr/bin/python # -*- coding: utf-8 -*- """Tests for Volume_Info.""" import os import unittest import pytsk3 import test_lib # mmls ../test_data/tsk_volume_system.raw # DOS Partition Table # Offset Sector: 0 # Units are in 512-byte sectors # # Slot Start End Length Description # 00: Meta 0000000000 0000000000 0000000001 Primary Table (#0) # 01: ----- 0000000000 0000000000 0000000001 Unallocated # 02: 00:00 0000000001 0000000350 0000000350 Linux (0x83) # 03: Meta 0000000351 0000002879 0000002529 DOS Extended (0x05) # 04: Meta 0000000351 0000000351 0000000001 Extended Table (#1) # 05: ----- 0000000351 0000000351 0000000001 Unallocated # 06: 01:00 0000000352 0000002879 0000002528 Linux (0x83) class TSKVolumeInfoTestCase(unittest.TestCase): """Volume_Info test case.""" maxDiff = None def _testInitialize(self, volume_info): """Test the initialize functionality. Args: volume_info: the Volume_Info object. """ self.assertNotEqual(volume_info, None) def _testIterate(self, volume_info): """Test the iterate functionality. Args: volume_info: the Volume_Info object. """ self.assertNotEqual(volume_info, None) self.assertNotEqual(getattr(volume_info, 'info', None), None) self.assertEqual(volume_info.info.vstype, pytsk3.TSK_VS_TYPE_DOS) parts = [] for part in volume_info: part_string = ( u'{0:02d}: {1:010d} {2:010d} {3:010d} {4:s}\n').format( part.addr, part.start, part.start + part.len - 1, part.len, part.desc.decode('utf-8')) parts.append(part_string) self.assertEqual(len(parts), 7) expected_parts_string = ( u'00: 0000000000 0000000000 0000000001 Primary Table (#0)\n' u'01: 0000000000 0000000000 0000000001 Unallocated\n' u'02: 0000000001 0000000350 0000000350 Linux (0x83)\n' u'03: 0000000351 0000002879 0000002529 DOS Extended (0x05)\n' u'04: 0000000351 0000000351 0000000001 Extended Table (#1)\n' u'05: 0000000351 0000000351 0000000001 Unallocated\n' u'06: 0000000352 0000002879 0000002528 Linux (0x83)\n') self.assertEqual(u''.join(parts), expected_parts_string) class TSKVolumeInfoTest(TSKVolumeInfoTestCase): """Volume_Info for testing.""" def setUp(self): """Sets up the needed objects used throughout the test.""" test_file = os.path.join('test_data', 'tsk_volume_system.raw') self._img_info = pytsk3.Img_Info(test_file) def testInitialize(self): """Test the initialize functionality.""" volume_info = pytsk3.Volume_Info(self._img_info) self._testInitialize(volume_info) def testIterate(self): """Test the iterate functionality.""" volume_info = pytsk3.Volume_Info(self._img_info) self._testIterate(volume_info) class TSKVolumeInfoBogusTest(TSKVolumeInfoTestCase): """Volume_Info for testing that fails.""" def setUp(self): """Sets up the needed objects used throughout the test.""" test_file = os.path.join('test_data', 'bogus.raw') self._img_info = pytsk3.Img_Info(test_file) def testInitialize(self): """Test the initialize functionality.""" with self.assertRaises(IOError): pytsk3.Volume_Info(self._img_info) class TSKVolumeInfoFileObjectTest(TSKVolumeInfoTestCase): """Tests the Volume_Info object using an Img_Info file-like object.""" def setUp(self): """Sets up the needed objects used throughout the test.""" test_file = os.path.join('test_data', 'tsk_volume_system.raw') self._file_object = open(test_file, 'rb') stat_info = os.stat(test_file) self._file_size = stat_info.st_size self._img_info = test_lib.FileObjectImageInfo( self._file_object, self._file_size) def testInitialize(self): """Test the initialize functionality.""" volume_info = pytsk3.Volume_Info(self._img_info) self._testInitialize(volume_info) def testIterate(self): """Test the iterate functionality.""" volume_info = pytsk3.Volume_Info(self._img_info) self._testIterate(volume_info) class TSKVolumeInfoFileObjectWithDetectTest(TSKVolumeInfoTestCase): """Tests the Volume_Info object with auto-detect Img_Info.""" def setUp(self): """Sets up the needed objects used throughout the test.""" test_file = os.path.join('test_data', 'tsk_volume_system.raw') self._file_object = open(test_file, 'rb') stat_info = os.stat(test_file) self._file_size = stat_info.st_size self._img_info = test_lib.FileObjectImageInfo( self._file_object, self._file_size, image_type=pytsk3.TSK_IMG_TYPE_DETECT) def testInitialize(self): """Test the initialize functionality.""" volume_info = pytsk3.Volume_Info(self._img_info) self._testInitialize(volume_info) def testIterate(self): """Test the iterate functionality.""" volume_info = pytsk3.Volume_Info(self._img_info) self._testIterate(volume_info) class TSKVolumeInfoFileObjectWithLargeSize(TSKVolumeInfoTestCase): """Tests the Volume_Info object with a large size Img_Info.""" def setUp(self): """Sets up the needed objects used throughout the test.""" test_file = os.path.join('test_data', 'tsk_volume_system.raw') self._file_object = open(test_file, 'rb') self._file_size = 1024 * 1024 * 1024 * 1024 self._img_info = test_lib.FileObjectImageInfo( self._file_object, self._file_size) def testInitialize(self): """Test the initialize functionality.""" volume_info = pytsk3.Volume_Info(self._img_info) self._testInitialize(volume_info) def testIterate(self): """Test the iterate functionality.""" volume_info = pytsk3.Volume_Info(self._img_info) self.assertNotEqual(volume_info, None) self.assertNotEqual(getattr(volume_info, 'info', None), None) self.assertEqual(volume_info.info.vstype, pytsk3.TSK_VS_TYPE_DOS) parts = [] for part in volume_info: part_string = ( u'{0:02d}: {1:010d} {2:010d} {3:010d} {4:s}\n').format( part.addr, part.start, part.start + part.len - 1, part.len, part.desc.decode('utf-8')) parts.append(part_string) # Note that due to the size the SleuthKit will add a non-existing part: # 07: 0000002880 2147483647 2147480768 Unallocated self.assertEqual(len(parts), 8) expected_parts_string = ( u'00: 0000000000 0000000000 0000000001 Primary Table (#0)\n' u'01: 0000000000 0000000000 0000000001 Unallocated\n' u'02: 0000000001 0000000350 0000000350 Linux (0x83)\n' u'03: 0000000351 0000002879 0000002529 DOS Extended (0x05)\n' u'04: 0000000351 0000000351 0000000001 Extended Table (#1)\n' u'05: 0000000351 0000000351 0000000001 Unallocated\n' u'06: 0000000352 0000002879 0000002528 Linux (0x83)\n' u'07: 0000002880 2147483647 2147480768 Unallocated\n') self.assertEqual(u''.join(parts), expected_parts_string) if __name__ == '__main__': unittest.main() pytsk-20231007/tox.ini000066400000000000000000000004561451023402500144400ustar00rootroot00000000000000[tox] envlist = py3{7,8,9,10,11,12} [testenv] pip_pre = True passenv = CFLAGS CPPFLAGS INCLUDE LDFLAGS LIB deps = build setuptools wheel commands = python -m build --no-isolation --outdir=dist --wheel python -m pip install --no-index --find-links=dist pytsk3 python run_tests.py pytsk-20231007/tsk3.cpp000066400000000000000000000464751451023402500145300ustar00rootroot00000000000000/* SleuthKit functions. * * Copyright 2010, Michael Cohen . * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "tsk3.h" #include #if defined( TSK_MULTITHREAD_LIB ) extern "C" { extern void tsk_init_lock(tsk_lock_t * lock); extern void tsk_deinit_lock(tsk_lock_t * lock); } #endif /* defined( TSK_MULTITHREAD_LIB ) */ /* Prototypes for IMG_INFO hooks * Note that IMG_INFO_read is called by the SleuthKit the Img_Info_read * is its equivalent called by the pytsk3 when no proxy object is defined. */ ssize_t IMG_INFO_read(TSK_IMG_INFO *self, TSK_OFF_T off, char *buf, size_t len); void IMG_INFO_close(TSK_IMG_INFO *self); /* This macro is used to receive the object reference from a member of the type. */ #define GET_Object_from_member(type, object, member) \ (type)(((char *)object) - (unsigned long)(&((type)0)->member)) /* Img_Info destructor */ static int Img_Info_dest(Img_Info self) { if(self == NULL) { return -1; } tsk_img_close((TSK_IMG_INFO *) self->img); if(self->img_is_internal != 0) { #if defined( TSK_MULTITHREAD_LIB ) tsk_deinit_lock(&(self->img->base.cache_lock)); #endif // If img is internal talloc will free it. } self->img = NULL; return 0; } /* Img_Info constructor */ static Img_Info Img_Info_Con(Img_Info self, char *urn, TSK_IMG_TYPE_ENUM type) { if(self == NULL) { RaiseError(EInvalidParameter, "Invalid parameter: self."); return NULL; } if(urn != NULL && urn[0] != 0) { #ifdef TSK_VERSION_NUM self->img = (Extended_TSK_IMG_INFO *) tsk_img_open_utf8(1, (const char **) &urn, type, 0); #else self->img = (Extended_TSK_IMG_INFO *) tsk_img_open_utf8(1, (const char **) &urn, type); #endif self->img_is_internal = 0; } else { // Initialise the img struct with the correct callbacks: self->img = talloc_zero(self, Extended_TSK_IMG_INFO); self->img_is_internal = 1; self->img->container = self; #if defined( TSK_MULTITHREAD_LIB ) tsk_init_lock(&(self->img->base.cache_lock)); #endif self->img->base.read = IMG_INFO_read; self->img->base.close = IMG_INFO_close; self->img->base.size = CALL(self, get_size); #ifdef TSK_VERSION_NUM self->img->base.sector_size = 512; #endif #if defined( TSK_VERSION_NUM ) && ( TSK_VERSION_NUM >= 0x040103ff ) self->img->base.itype = TSK_IMG_TYPE_EXTERNAL; #else self->img->base.itype = TSK_IMG_TYPE_RAW_SING; #endif } if(self->img == NULL) { RaiseError(EIOError, "Unable to open image: %s", tsk_error_get()); tsk_error_reset(); return NULL; } self->img_is_open = 1; talloc_set_destructor((void *) self, (int(*)(void *)) &Img_Info_dest); return self; } uint64_t Img_Info_read(Img_Info self, TSK_OFF_T off, OUT char *buf, size_t len) { ssize_t read_count = 0; if(self == NULL) { RaiseError(EInvalidParameter, "Invalid parameter: self."); return 0; } if(self->img_is_open == 0) { RaiseError(EIOError, "Invalid Img_Info not opened."); return 0; } if(off < 0) { RaiseError(EIOError, "Invalid offset value out of bounds."); return 0; } if(buf == NULL) { RaiseError(EInvalidParameter, "Invalid parameter: buf."); return 0; } read_count = CALL((TSK_IMG_INFO *) self->img, read, off, buf, len); if(read_count < 0) { RaiseError(EIOError, "Unable to read image: %s", tsk_error_get()); tsk_error_reset(); return 0; } return read_count; } void Img_Info_close(Img_Info self) { if(self != NULL) { self->img_is_open = 0; } } uint64_t Img_Info_get_size(Img_Info self) { if(self == NULL) { RaiseError(EInvalidParameter, "Invalid parameter: self."); return 0; } if(self->img != NULL) { return ((TSK_IMG_INFO *) self->img)->size; } return (uint64_t) -1; } VIRTUAL(Img_Info, Object) { VMETHOD(Con) = Img_Info_Con; VMETHOD(read) = Img_Info_read; VMETHOD(close) = Img_Info_close; VMETHOD(get_size) = Img_Info_get_size; } END_VIRTUAL void IMG_INFO_close(TSK_IMG_INFO *img) { Extended_TSK_IMG_INFO *self = (Extended_TSK_IMG_INFO *) img; CALL(self->container, close); }; ssize_t IMG_INFO_read(TSK_IMG_INFO *img, TSK_OFF_T off, char *buf, size_t len) { Extended_TSK_IMG_INFO *self = (Extended_TSK_IMG_INFO *) img; if(len == 0) { return 0; } return (ssize_t) CALL(self->container, read, (uint64_t) off, buf, len); } /* FS_Info destructor */ int FS_Info_dest(FS_Info self) { if(self == NULL) { return -1; } tsk_fs_close(self->info); self->info = NULL; self->extended_img_info = NULL; return 0; } /* FS_Info constructor */ static FS_Info FS_Info_Con(FS_Info self, Img_Info img, TSK_OFF_T offset, TSK_FS_TYPE_ENUM type) { if(self == NULL) { RaiseError(EInvalidParameter, "Invalid parameter: self."); return NULL; } if(img == NULL) { RaiseError(EInvalidParameter, "Invalid parameter: img."); return NULL; } self->extended_img_info = img->img; self->info = tsk_fs_open_img((TSK_IMG_INFO *) self->extended_img_info, offset, type); if(!self->info) { RaiseError(EIOError, "Unable to open the image as a filesystem at offset: 0x%08" PRIxOFF " with error: %s", offset, tsk_error_get()); tsk_error_reset(); return NULL; } // Make sure that the filesystem is properly closed when we get freed talloc_set_destructor((void *) self, (int(*)(void *)) &FS_Info_dest); return self; } static Directory FS_Info_open_dir(FS_Info self, ZString path, TSK_INUM_T inode) { Directory object = NULL; if(self == NULL) { RaiseError(EInvalidParameter, "Invalid parameter: self."); return NULL; } // CONSTRUCT_CREATE calls _talloc_memdup to allocate memory for the object. object = CONSTRUCT_CREATE(Directory, Directory, NULL); if(object != NULL) { // CONSTRUCT_INITIALIZE calls the constructor function on the object. if(CONSTRUCT_INITIALIZE(Directory, Directory, Con, object, self, path, inode) == NULL) { goto on_error; } } return object; on_error: if(object != NULL) { talloc_free(object); } return NULL; }; static File FS_Info_open(FS_Info self, ZString path) { TSK_FS_FILE *info = NULL; File object = NULL; if(self == NULL) { RaiseError(EInvalidParameter, "Invalid parameter: self."); return NULL; } info = tsk_fs_file_open(self->info, NULL, path); if(info == NULL) { RaiseError(EIOError, "Unable to open file: %s", tsk_error_get()); tsk_error_reset(); goto on_error; } // CONSTRUCT_CREATE calls _talloc_memdup to allocate memory for the object. object = CONSTRUCT_CREATE(File, File, NULL); if(object != NULL) { // CONSTRUCT_INITIALIZE calls the constructor function on the object. if(CONSTRUCT_INITIALIZE(File, File, Con, object, self, info) == NULL) { goto on_error; } // Tell the File object to manage info. object->info_is_internal = 1; } return object; on_error: if(object != NULL) { talloc_free(object); } if(info != NULL) { tsk_fs_file_close(info); } return NULL; }; static File FS_Info_open_meta(FS_Info self, TSK_INUM_T inode) { TSK_FS_FILE *info = NULL; File object = NULL; if(self == NULL) { RaiseError(EInvalidParameter, "Invalid parameter: self."); return NULL; } info = tsk_fs_file_open_meta(self->info, NULL, inode); if(info == NULL) { RaiseError(EIOError, "Unable to open file: %s", tsk_error_get()); tsk_error_reset(); goto on_error; } // CONSTRUCT_CREATE calls _talloc_memdup to allocate memory for the object. object = CONSTRUCT_CREATE(File, File, NULL); if(object != NULL) { // CONSTRUCT_INITIALIZE calls the constructor function on the object. if(CONSTRUCT_INITIALIZE(File, File, Con, object, self, info) == NULL) { goto on_error; } // Tell the File object to manage info. object->info_is_internal = 1; } return object; on_error: if(object != NULL) { talloc_free(object); } if(info != NULL) { tsk_fs_file_close(info); } return NULL; } static void FS_Info_exit(FS_Info self PYTSK3_ATTRIBUTE_UNUSED) { PYTSK3_UNREFERENCED_PARAMETER(self) exit(0); }; VIRTUAL(FS_Info, Object) { VMETHOD(Con) = FS_Info_Con; VMETHOD(open_dir) = FS_Info_open_dir; VMETHOD(open) = FS_Info_open; VMETHOD(open_meta) = FS_Info_open_meta; VMETHOD(exit) = FS_Info_exit; } END_VIRTUAL /* Directory destructor */ static int Directory_dest(Directory self) { if(self == NULL) { return -1; } tsk_fs_dir_close(self->info); self->info = NULL; return 0; } /* Directory constructor */ static Directory Directory_Con(Directory self, FS_Info fs, ZString path, TSK_INUM_T inode) { if(self == NULL) { RaiseError(EInvalidParameter, "Invalid parameter: self."); return NULL; } if(fs == NULL) { RaiseError(EInvalidParameter, "Invalid parameter: fs."); return NULL; } if(path == NULL) { self->info = tsk_fs_dir_open_meta(fs->info, inode); } else { self->info = tsk_fs_dir_open(fs->info, path); } if(self->info == NULL) { RaiseError(EIOError, "Unable to open directory: %s", tsk_error_get()); tsk_error_reset(); return NULL; } self->current = 0; self->size = tsk_fs_dir_getsize(self->info); self->fs = fs; // TODO: is this still applicable? // Add a reference to them to ensure they dont get freed until we do. // talloc_reference(self, fs); talloc_set_destructor((void *) self, (int(*)(void *)) &Directory_dest); return self; } static File Directory_next(Directory self) { TSK_FS_FILE *info = NULL; File object = NULL; if(self == NULL) { RaiseError(EInvalidParameter, "Invalid parameter: self."); return NULL; } if((self->current < 0) || ((uint64_t) self->current > (uint64_t) self->size)) { RaiseError(EInvalidParameter, "Invalid parameter: current."); return NULL; } if((uint64_t) self->current == (uint64_t) self->size) { return NULL; } info = tsk_fs_dir_get(self->info, self->current); if(info == NULL) { RaiseError(EIOError, "Error opening File: %s", tsk_error_get()); tsk_error_reset(); goto on_error; } // CONSTRUCT_CREATE calls _talloc_memdup to allocate memory for the object. object = CONSTRUCT_CREATE(File, File, NULL); if(object != NULL) { // CONSTRUCT_INITIALIZE calls the constructor function on the object. if(CONSTRUCT_INITIALIZE(File, File, Con, object, self->fs, info) == NULL) { goto on_error; } // Tell the File object to manage info. object->info_is_internal = 1; } self->current++; return object; on_error: if(object != NULL) { talloc_free(object); } if(info != NULL) { tsk_fs_file_close(info); } return NULL; }; static void Directory_iter(Directory self) { self->current = 0; }; VIRTUAL(Directory, Object) { VMETHOD(Con) = Directory_Con; VMETHOD(iternext) = Directory_next; VMETHOD(__iter__) = Directory_iter; } END_VIRTUAL /* File destructor */ static int File_dest(File self) { if(self == NULL) { return -1; } if(self->info_is_internal != 0) { // Here internal refers to the File object managing info // not that info was allocated by talloc. tsk_fs_file_close(self->info); } self->info = NULL; return 0; } /* File constructor */ static File File_Con(File self, FS_Info fs, TSK_FS_FILE *info) { if(self == NULL) { RaiseError(EInvalidParameter, "Invalid parameter: self."); return NULL; } if(fs == NULL) { RaiseError(EInvalidParameter, "Invalid parameter: fs."); return NULL; } if(info == NULL) { RaiseError(EInvalidParameter, "Invalid parameter: info."); return NULL; } self->fs = fs; self->info = info; // Get the total number of attributes. self->max_attr = tsk_fs_file_attr_getsize(info); talloc_set_destructor((void *) self, (int(*)(void *)) &File_dest); return self; }; static uint64_t File_read_random(File self, TSK_OFF_T offset, OUT char *buff, int len, TSK_FS_ATTR_TYPE_ENUM type, int id, TSK_FS_FILE_READ_FLAG_ENUM flags) { ssize_t result; if((id < -1) || (id > 0xffff)) { RaiseError(EInvalidParameter, "id parameter is invalid."); return 0; }; if(id == -1) { result = tsk_fs_file_read(self->info, offset, buff, len, flags); } else { result = tsk_fs_file_read_type(self->info, type, (uint16_t) id, offset, buff, len, flags); }; if(result < 0) { RaiseError(EIOError, "Read error: %s", tsk_error_get()); tsk_error_reset(); return 0; }; return result; }; static Directory File_as_directory(File self) { Directory object = NULL; if(self == NULL) { RaiseError(EInvalidParameter, "Invalid parameter: self."); return NULL; } if(self->info == NULL) { RaiseError(EInvalidParameter, "Invalid parameter: self->info."); return NULL; } #if defined( TSK_VERSION_NUM ) && ( TSK_VERSION_NUM >= 0x040402ff ) if(self->info->meta == NULL || !(TSK_FS_IS_DIR_META(self->info->meta->type))) { #else if(self->info->meta == NULL || self->info->meta->type != TSK_FS_META_TYPE_DIR) { #endif RaiseError(EIOError, "Not a directory"); return NULL; } // CONSTRUCT_CREATE calls _talloc_memdup to allocate memory for the object. object = CONSTRUCT_CREATE(Directory, Directory, NULL); if(object != NULL) { // CONSTRUCT_INITIALIZE calls the constructor function on the object. if(CONSTRUCT_INITIALIZE(Directory, Directory, Con, object, self->fs, NULL, self->info->meta->addr) == NULL) { goto on_error; } } return object; on_error: if(object != NULL) { talloc_free(object); } return NULL; }; static Attribute File_iternext(File self) { TSK_FS_ATTR *attribute = NULL; Attribute object = NULL; if(self == NULL) { RaiseError(EInvalidParameter, "Invalid parameter: self."); return NULL; } if(self->current_attr < 0 || self->current_attr > self->max_attr) { RaiseError(EInvalidParameter, "Invalid parameter: self->current_attr."); return NULL; } if(self->current_attr == self->max_attr) { return NULL; } // It looks like attribute is managed by the SleuthKit. attribute = (TSK_FS_ATTR *) tsk_fs_file_attr_get_idx(self->info, self->current_attr); if(!attribute) { RaiseError(EIOError, "Error opening File: %s", tsk_error_get()); tsk_error_reset(); return NULL; } // CONSTRUCT_CREATE calls _talloc_memdup to allocate memory for the object. object = CONSTRUCT_CREATE(Attribute, Attribute, NULL); if(object != NULL) { // CONSTRUCT_INITIALIZE calls the constructor function on the object. if(CONSTRUCT_INITIALIZE(Attribute, Attribute, Con, object, attribute) == NULL) { goto on_error; } } self->current_attr++; return object; on_error: if(object != NULL) { talloc_free(object); } return NULL; }; static void File_iter__(File self) { self->current_attr = 0; }; VIRTUAL(File, Object) { VMETHOD(Con) = File_Con; VMETHOD(read_random) = File_read_random; VMETHOD(as_directory) = File_as_directory; VMETHOD(iternext) = File_iternext; VMETHOD(__iter__) = File_iter__; } END_VIRTUAL /* Attribute constructor */ static Attribute Attribute_Con(Attribute self, TSK_FS_ATTR *info) { if(self == NULL) { RaiseError(EInvalidParameter, "Invalid parameter: self."); return NULL; } if(info == NULL) { RaiseError(EInvalidParameter, "Invalid parameter: info."); return NULL; } self->info = info; return self; } static void Attribute_iter(Attribute self) { self->current = self->info->nrd.run; }; static TSK_FS_ATTR_RUN *Attribute_iternext(Attribute self) { TSK_FS_ATTR_RUN *result = NULL; if(self->current == NULL) { return NULL; } result = self->current; self->current = self->current->next; if(self->current == self->info->nrd.run) { self->current = NULL; } return (TSK_FS_ATTR_RUN *) talloc_memdup(NULL, result, sizeof(*result)); } VIRTUAL(Attribute, Object) { VMETHOD(Con) = Attribute_Con; VMETHOD(iternext) = Attribute_iternext; VMETHOD(__iter__) = Attribute_iter; } END_VIRTUAL /* The following implement the volume system. */ /* Volume_Info destructor */ static int Volume_Info_dest(Volume_Info self) { if(self == NULL) { return -1; } tsk_vs_close(self->info); self->info = NULL; return 0; } /* Volume_Info constructor */ static Volume_Info Volume_Info_Con(Volume_Info self, Img_Info img, TSK_VS_TYPE_ENUM type, TSK_OFF_T offset) { if(self == NULL) { RaiseError(EInvalidParameter, "Invalid parameter: self."); return NULL; } if(img == NULL) { RaiseError(EInvalidParameter, "Invalid parameter: img."); return NULL; } self->info = tsk_vs_open((TSK_IMG_INFO *) img->img, offset, type); if(self->info == NULL) { RaiseError(EIOError, "Error opening Volume_Info: %s", tsk_error_get()); tsk_error_reset(); return NULL; } talloc_set_destructor((void *) self, (int(*)(void *)) &Volume_Info_dest); return self; } static void Volume_Info_iter(Volume_Info self) { self->current = 0; }; static TSK_VS_PART_INFO *Volume_Info_iternext(Volume_Info self) { return (TSK_VS_PART_INFO *)tsk_vs_part_get(self->info, self->current++); }; VIRTUAL(Volume_Info, Object) { VMETHOD(Con) = Volume_Info_Con; VMETHOD(__iter__) = Volume_Info_iter; VMETHOD(iternext) = Volume_Info_iternext; } END_VIRTUAL void tsk_init() { // libtsk uses mktime and localtime that rely on the TZ environment variable // however that leads to inconsistent behavior with different TZ values. // Hence that we force TZ to be UTC, when possible. #if defined( _MSC_VER ) _putenv_s("TZ", "UTC"); _tzset(); // Some installations of MinGW do not support setenv #elif !defined( __MINGW32__ ) setenv("TZ", "UTC", 1); tzset(); #endif //tsk_verbose++; Img_Info_init((Object)&__Img_Info); FS_Info_init((Object)&__FS_Info); Directory_init((Object)&__Directory); File_init((Object)&__File); Attribute_init((Object)&__Attribute); Volume_Info_init((Object)&__Volume_Info); }; pytsk-20231007/tsk3.h000066400000000000000000000152261451023402500141630ustar00rootroot00000000000000/* SleuthKit functions. * * Copyright 2010, Michael Cohen . * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #if !defined( TSK3_H_ ) #define TSK3_H_ #if defined( HAVE_TSK3_LIBTSK_H ) #include #elif defined( HAVE_TSK_LIBTSK_H ) #include #else #error Missing libtsk header #endif #include "aff4_errors.h" #include "class.h" typedef struct { TSK_IMG_INFO base; struct Img_Info_t *container; } Extended_TSK_IMG_INFO; BIND_STRUCT(Extended_TSK_IMG_INFO); /** Bind the following structs */ BIND_STRUCT(TSK_FS_INFO); BIND_STRUCT(TSK_FS_NAME); BIND_STRUCT(TSK_FS_META); BIND_STRUCT(TSK_FS_DIR); BIND_STRUCT(TSK_FS_FILE); BIND_STRUCT(TSK_FS_BLOCK); BIND_STRUCT(TSK_FS_ATTR); BIND_STRUCT(TSK_FS_ATTR_RUN); BIND_STRUCT(TSK_VS_PART_INFO); BIND_STRUCT(TSK_VS_INFO); /** This is a normal IMG_INFO which takes a filename and passes it to TSK. It just uses the standard TSK image handling code to support EWF, AFF etc. This is usually the first object you would instantiate in order to use the TSK library: img = Img_Info(filename) you would then pass it to an FS_Info object: fs = FS_Info(img) Then open an inode or path f = fs.open_dir(inode = 2) */ CCLASS(Img_Info, Object) PRIVATE Extended_TSK_IMG_INFO *img; /* Value to indicate if img is managed internally */ PRIVATE int img_is_internal; /* Value to indicate if img is open */ PRIVATE int img_is_open; /* Open an image using the Sleuthkit. * * DEFAULT(type) = TSK_IMG_TYPE_DETECT; * DEFAULT(url) = ""; */ Img_Info METHOD(Img_Info, Con, ZString url, TSK_IMG_TYPE_ENUM type); /* Read a random buffer from the image */ uint64_t METHOD(Img_Info, read, TSK_OFF_T off, OUT char *buf, size_t len); /* Retrieve the size of the image */ uint64_t METHOD(Img_Info, get_size); /* Closes the image */ void METHOD(Img_Info, close); END_CCLASS /** This object handles volumes. */ CCLASS(Volume_Info, Object) FOREIGN TSK_VS_INFO *info; int current; /** Open a volume using the Sleuthkit. DEFAULT(offset) = 0; DEFAULT(type) = TSK_VS_TYPE_DETECT; */ Volume_Info METHOD(Volume_Info, Con, Img_Info img, TSK_VS_TYPE_ENUM type, TSK_OFF_T offset); void METHOD(Volume_Info, __iter__); TSK_VS_PART_INFO *METHOD(Volume_Info, iternext); END_CCLASS // Forward declerations struct FS_Info_t; struct Directory_t; /** An attribute is associated with a file. In some filesystem (e.g. NTFS) a file may contain many attributes. Attributes can be iterated over to obtain the attribute runs (e.g. to recover block allocation information). */ CCLASS(Attribute, Object) FOREIGN TSK_FS_ATTR *info; FOREIGN TSK_FS_ATTR_RUN *current; Attribute METHOD(Attribute, Con, TSK_FS_ATTR *info); void METHOD(Attribute, __iter__); TSK_FS_ATTR_RUN *METHOD(Attribute, iternext); END_CCLASS /** This represents a file object. A file has both metadata and data streams. Its usually not useful to instantiate this C class by itself - you need to call FS_Info.open() or iterate over a Directory() object. This object may be used to read the content of the file using read_random(). Iterating over this object will return all the attributes for this file. */ CCLASS(File, Object) FOREIGN TSK_FS_FILE *info; /* Value to indicate if info is managed internally */ PRIVATE int info_is_internal; PRIVATE struct FS_Info_t *fs; int max_attr; int current_attr; File METHOD(File, Con, struct FS_Info_t *fs, TSK_FS_FILE *info); /** Read a buffer from a random location in the file. DEFAULT(flags) = 0; DEFAULT(type) = TSK_FS_ATTR_TYPE_DEFAULT; DEFAULT(id) = -1; */ uint64_t METHOD(File, read_random, TSK_OFF_T offset, OUT char *buff, int len, TSK_FS_ATTR_TYPE_ENUM type, int id, TSK_FS_FILE_READ_FLAG_ENUM flags); /* Obtain a directory object that represents this inode. This may be useful if the file is actually a directory and we want to iterate over its contents. */ struct Directory_t *METHOD(File, as_directory); void METHOD(File, __iter__); Attribute METHOD(File, iternext); END_CCLASS /** This represents a Directory within the filesystem. You can iterate over this object to obtain all the File objects contained within this directory: for f in d: print f.info.name.name */ CCLASS(Directory, Object) TSK_FS_DIR *info; PRIVATE struct FS_Info_t *fs; /* Total number of files in this directory */ size_t size; /* Current file returned in the next iteration */ int current; /* We can open the directory using a path, its inode number. DEFAULT(path) = NULL; DEFAULT(inode) = 0; */ Directory METHOD(Directory, Con, struct FS_Info_t *fs, \ ZString path, TSK_INUM_T inode); /** An iterator of all files in the present directory. */ void METHOD(Directory, __iter__); File METHOD(Directory, iternext); END_CCLASS /** This is used to obtain a filesystem object from an Img_Info object. From this FS_Info we can open files or directories by inode, or path. */ CCLASS(FS_Info, Object) FOREIGN TSK_FS_INFO *info; PRIVATE Extended_TSK_IMG_INFO *extended_img_info; /** Open the filesystem stored on image. DEFAULT(type) = TSK_FS_TYPE_DETECT; DEFAULT(offset) = 0; */ FS_Info METHOD(FS_Info, Con, Img_Info img, TSK_OFF_T offset, TSK_FS_TYPE_ENUM type); /** A convenience function to open a directory in this image. DEFAULT(path) = NULL; DEFAULT(inode) = 2; */ Directory METHOD(FS_Info, open_dir, ZString path, TSK_INUM_T inode); /** A convenience function to open a file in this image. */ File METHOD(FS_Info, open, ZString path); // Open a file by inode number File METHOD(FS_Info, open_meta, TSK_INUM_T inode); void METHOD(FS_Info, exit); END_CCLASS int *tsk_get_current_error(char **buff); void tsk_init(void); #endif /* !TSK3_H_ */