pax_global_header00006660000000000000000000000064125704044360014517gustar00rootroot0000000000000052 comment=914b041a48da9594d1bdac997bae378bdd6b6f58 pymtbl-0.4.0/000077500000000000000000000000001257040443600130275ustar00rootroot00000000000000pymtbl-0.4.0/.gitignore000066400000000000000000000001331257040443600150140ustar00rootroot00000000000000.*swp /MANIFEST /build/ /dist/ mtbl.c examples/*.mtbl examples/wf/*.mtbl examples/wf/*.txt pymtbl-0.4.0/COPYRIGHT000066400000000000000000000010771257040443600143270ustar00rootroot00000000000000Copyright (c) 2012-2014 by Farsight Security, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. pymtbl-0.4.0/LICENSE000066400000000000000000000261361257040443600140440ustar00rootroot00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. pymtbl-0.4.0/MANIFEST.in000066400000000000000000000001261257040443600145640ustar00rootroot00000000000000include COPYRIGHT LICENSE README.md examples/* examples/wf/* mtbl.c mtbl.pxi mtbl.pyx pymtbl-0.4.0/README.md000066400000000000000000000024451257040443600143130ustar00rootroot00000000000000pymtbl: Python bindings for the mtbl sorted string table library ---------------------------------------------------------------- `pymtbl` provides a simple Pythonic wrapper for [mtbl](https://github.com/farsightsec/mtbl)'s reader, writer, sorter, and merger interfaces. The `examples/` directory contains scripts demonstrating each of these interfaces. The following transcript shows the basic reader and writer interfaces: >>> import mtbl >>> w = mtbl.writer('example.mtbl', compression=mtbl.COMPRESSION_SNAPPY) >>> w['key1'] = 'val1' >>> w['key17'] = 'val17' >>> w['key2'] = 'val2' >>> w['key23'] = 'val23' >>> w['key3'] = 'val3' >>> w['key4'] = 'val4' >>> w['key05'] = 'val05' Traceback (most recent call last): File "", line 1, in File "mtbl.pyx", line 361, in mtbl.writer.__setitem__ (mtbl.c:4108) mtbl.KeyOrderError >>> w.close() >>> r = mtbl.reader('example.mtbl', verify_checksums=True) >>> for k,v in r.get_range('key19', 'key23'): print k,v ... key2 val2 key23 val23 >>> for k,v in r.get_prefix('key2'): print k,v ... key2 val2 key23 val23 >>> for k,v in r.iteritems(): print k, v ... key1 val1 key17 val17 key2 val2 key23 val23 key3 val3 key4 val4 >>> pymtbl-0.4.0/debian/000077500000000000000000000000001257040443600142515ustar00rootroot00000000000000pymtbl-0.4.0/debian/changelog000066400000000000000000000011321257040443600161200ustar00rootroot00000000000000pymtbl (0.4.0-1) unstable; urgency=medium * New upstream release. -- Robert Edmonds Sat, 29 Aug 2015 15:33:45 -0400 pymtbl (0.2-1) unstable; urgency=medium * New upstream release. -- Robert S. Edmonds Tue, 21 Jan 2014 18:30:53 -0500 pymtbl (0.1-2) unstable; urgency=low * Add missing build-dependency on cython. -- Robert S. Edmonds Sun, 18 Mar 2012 14:07:33 -0400 pymtbl (0.1-1) unstable; urgency=low * Initial release; closes: #661209. -- Robert S. Edmonds Sun, 04 Mar 2012 17:51:56 -0500 pymtbl-0.4.0/debian/compat000066400000000000000000000000021257040443600154470ustar00rootroot000000000000009 pymtbl-0.4.0/debian/control000066400000000000000000000023761257040443600156640ustar00rootroot00000000000000Source: pymtbl Section: python Priority: optional Maintainer: Robert Edmonds Build-Depends: cython, debhelper (>= 9~), dh-python, libmtbl-dev (>= 0.8.0~), pkg-config, python-all-dev (>= 2.6.6-3~), Standards-Version: 3.9.6 Package: python-mtbl Architecture: any Depends: ${misc:Depends}, ${python:Depends}, ${shlibs:Depends} Description: immutable sorted string table library (Python bindings) mtbl is a C library implementation of the Sorted String Table (SSTable) data structure, based on the SSTable implementation in the open source Google LevelDB library. An SSTable is a file containing an immutable mapping of keys to values. Keys are stored in sorted order, with an index at the end of the file allowing keys to be located quickly. . mtbl is not a database library. It does not provide an updateable key-value data store, but rather exposes primitives for creating, searching and merging SSTable files. Unlike databases which use the SSTable data structure internally as part of their data store, management of SSTable files -- creation, merging, deletion, combining of search results from multiple SSTables -- is left to the discretion of the mtbl library user. . This package contains the Python extension module for libmtbl. pymtbl-0.4.0/debian/copyright000066400000000000000000000021021257040443600161770ustar00rootroot00000000000000Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ Files: * Copyright: 2012-2014 Farsight Security, Inc. License: Apache-2.0 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at . http://www.apache.org/licenses/LICENSE-2.0 . Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. . On Debian systems, the full text of the Apache License version 2 can be found in the file `/usr/share/common-licenses/Apache-2.0'. Files: debian/* Copyright: 2012 Robert S. Edmonds License: permissive Copying and distribution of this package, with or without modification, are permitted in any medium without royalty provided the copyright notice and this notice are preserved. pymtbl-0.4.0/debian/docs000066400000000000000000000000121257040443600151150ustar00rootroot00000000000000README.md pymtbl-0.4.0/debian/gbp.conf000066400000000000000000000002541257040443600156710ustar00rootroot00000000000000[DEFAULT] debian-branch = next upstream-branch = next upstream-tag = tags/v%(version)s pristine-tar = False [dch] id-length = 8 commit-msg = debian/changelog: %(version)s pymtbl-0.4.0/debian/rules000077500000000000000000000000551257040443600153310ustar00rootroot00000000000000#!/usr/bin/make -f %: dh $@ --with python2 pymtbl-0.4.0/debian/source/000077500000000000000000000000001257040443600155515ustar00rootroot00000000000000pymtbl-0.4.0/debian/source/format000066400000000000000000000000141257040443600167570ustar00rootroot000000000000003.0 (quilt) pymtbl-0.4.0/debian/source/local-options000066400000000000000000000000241257040443600202530ustar00rootroot00000000000000single-debian-patch pymtbl-0.4.0/examples/000077500000000000000000000000001257040443600146455ustar00rootroot00000000000000pymtbl-0.4.0/examples/pymtbl_make_random_table.py000077500000000000000000000035541257040443600222440ustar00rootroot00000000000000#!/usr/bin/env python import locale import random import string import sys import time import mtbl report_interval = 10000000 megabyte = 1048576.0 def main(fname, num_keys): writer = mtbl.writer(fname, compression=mtbl.COMPRESSION_SNAPPY) a = time.time() last = a total_bytes = 0 count = 0 total = 0 while count < num_keys: count += 1 if random.random() >= 0.5: total += 1 key = '%010d' % count val = random.choice(string.ascii_lowercase) * random.randint(1, 50) writer[key] = val total_bytes += len(key) + len(val) if (count % report_interval) == 0: b = time.time() last_secs = b - last last = b sys.stderr.write('wrote %s entries (%s MB) in %s seconds, %s entries/second\n' % ( locale.format('%d', total, grouping=True), locale.format('%d', total_bytes / megabyte, grouping=True), locale.format('%f', last_secs, grouping=True), locale.format('%d', report_interval / last_secs, grouping=True) ) ) b = time.time() total_secs = b - a sys.stderr.write('wrote %s total entries (%s MB) in %s seconds, %s entries/second\n' % ( locale.format('%d', total, grouping=True), locale.format('%d', total_bytes / megabyte, grouping=True), locale.format('%f', total_secs, grouping=True), locale.format('%d', total / total_secs, grouping=True) ) ) def usage(): sys.stderr.write('Usage: %s \n' % sys.argv[0]) sys.exit(1) if __name__ == '__main__': locale.setlocale(locale.LC_ALL, '') if len(sys.argv) != 3: usage() try: fname = sys.argv[1] num_keys = int(sys.argv[2]) except: usage() main(fname, num_keys) pymtbl-0.4.0/examples/pymtbl_make_random_table_2.py000077500000000000000000000037351257040443600224660ustar00rootroot00000000000000#!/usr/bin/env python import locale import random import string import sys import time import mtbl report_interval = 100000 megabyte = 1048576.0 def merge_func(key, val0, val1): return val0 + val1 def main(fname, num_keys): sorter = mtbl.sorter(merge_func) writer = mtbl.writer(fname, compression=mtbl.COMPRESSION_SNAPPY) a = time.time() last = a total_bytes = 0 count = 0 while count < num_keys: count += 1 key = '%020d' % random.randint(0, sys.maxint) val = random.choice(string.ascii_lowercase) * random.randint(1, 50) sorter[key] = val total_bytes += len(key) + len(val) if (count % report_interval) == 0: b = time.time() last_secs = b - last last = b sys.stderr.write('generated %s entries (%s MB) in %s seconds, %s entries/second\n' % ( locale.format('%d', count, grouping=True), locale.format('%d', total_bytes / megabyte, grouping=True), locale.format('%f', last_secs, grouping=True), locale.format('%d', report_interval / last_secs, grouping=True) ) ) sys.stderr.write('writing to output file %s\n' % fname) sorter.write(writer) b = time.time() total_secs = b - a sys.stderr.write('wrote %s total entries (%s MB) in %s seconds, %s entries/second\n' % ( locale.format('%d', count, grouping=True), locale.format('%d', total_bytes / megabyte, grouping=True), locale.format('%f', total_secs, grouping=True), locale.format('%d', count / total_secs, grouping=True) ) ) def usage(): sys.stderr.write('Usage: %s \n' % sys.argv[0]) sys.exit(1) if __name__ == '__main__': locale.setlocale(locale.LC_ALL, '') if len(sys.argv) != 3: usage() try: fname = sys.argv[1] num_keys = int(sys.argv[2]) except: usage() main(fname, num_keys) pymtbl-0.4.0/examples/pymtbl_merge_tables.py000077500000000000000000000013751257040443600212500ustar00rootroot00000000000000#!/usr/bin/env python import sys import mtbl def merge_func(key, val0, val1): return val0 + ' ' + val1 def main(output_fname, input_fnames): merger = mtbl.merger(merge_func) writer = mtbl.writer(output_fname, compression=mtbl.COMPRESSION_SNAPPY) for fname in input_fnames: reader = mtbl.reader(fname) merger.add_reader(reader) for k, v in merger.iteritems(): writer[k] = v writer.close() def usage(): sys.stderr.write('Usage: %s [...]\n' % sys.argv[0]) sys.exit(1) if __name__ == '__main__': if len(sys.argv) < 3: usage() try: output_fname = sys.argv[1] input_fnames = sys.argv[2:] except: usage() main(output_fname, input_fnames) pymtbl-0.4.0/examples/pymtbl_merge_tables_2.py000077500000000000000000000013341257040443600214640ustar00rootroot00000000000000#!/usr/bin/env python import sys import mtbl def merge_func(key, val0, val1): return val0 + ' ' + val1 def main(output_fname, input_fnames): merger = mtbl.merger(merge_func) writer = mtbl.writer(output_fname, compression=mtbl.COMPRESSION_SNAPPY) for fname in input_fnames: reader = mtbl.reader(fname) merger.add_reader(reader) merger.write(writer) writer.close() def usage(): sys.stderr.write('Usage: %s [...]\n' % sys.argv[0]) sys.exit(1) if __name__ == '__main__': if len(sys.argv) < 3: usage() try: output_fname = sys.argv[1] input_fnames = sys.argv[2:] except: usage() main(output_fname, input_fnames) pymtbl-0.4.0/examples/pymtbl_search_random_table.py000077500000000000000000000033441257040443600225710ustar00rootroot00000000000000#!/usr/bin/env python import locale import random import sys import time import mtbl report_interval = 10000 def main(fname, num_keys, num_iters): reader = mtbl.reader(fname) a = time.time() last = a num_found = 0 count = 0 while count < num_iters: key = '%010d' % random.randint(0, num_keys) if reader.has_key(key): val = reader[key] num_found += 1 count += 1 if (count % report_interval) == 0: b = time.time() last_secs = b - last last = b sys.stderr.write('%s lookups, %s keys found in %s seconds, %s lookups/second\n' % ( locale.format('%d', count, grouping=True), locale.format('%d', num_found, grouping=True), locale.format('%f', last_secs, grouping=True), locale.format('%d', report_interval / last_secs, grouping=True) ) ) b = time.time() total_secs = b - a sys.stderr.write('%s total lookups, %s keys found in %s seconds, %s lookups/second\n' % ( locale.format('%d', count, grouping=True), locale.format('%d', num_found, grouping=True), locale.format('%f', total_secs, grouping=True), locale.format('%d', count / total_secs, grouping=True) ) ) def usage(): sys.stderr.write('Usage: %s \n' % sys.argv[0]) sys.exit(1) if __name__ == '__main__': locale.setlocale(locale.LC_ALL, '') if len(sys.argv) != 4: usage() try: fname = sys.argv[1] num_keys = int(sys.argv[2]) num_iters = int(sys.argv[3]) except: usage() main(fname, num_keys, num_iters) pymtbl-0.4.0/examples/wf/000077500000000000000000000000001257040443600152615ustar00rootroot00000000000000pymtbl-0.4.0/examples/wf/Makefile000066400000000000000000000007701257040443600167250ustar00rootroot00000000000000EBOOKS = 2814 4217 4300 BOOK_FILES = $(foreach ebook,$(EBOOKS),$(ebook).txt) MTBL_FILES = $(foreach ebook,$(EBOOKS),$(ebook).mtbl) %.txt: @wget -O $@ http://www.gutenberg.org/files/$(shell basename $@ .txt)/$@ %.mtbl: %.txt rm -f $@ ./pymtbl_wf_analyze.py $< $@ wf.txt: wf.mtbl $(BOOK_FILES) ./pymtbl_wf_dump.py wf.mtbl | sort -rn > wf.txt wf.mtbl: $(MTBL_FILES) $(BOOK_FILES) rm -f wf.mtbl ./pymtbl_wf_merge.py $(MTBL_FILES) wf.mtbl clean: rm -f wf.mtbl wf.txt $(MTBL_FILES) .PHONY: clean pymtbl-0.4.0/examples/wf/README000066400000000000000000000032611257040443600161430ustar00rootroot00000000000000Simple word frequency example using pymtbl ------------------------------------------ This directory shows a simple "word frequency" application of the pymtbl module. It is driven by a Makefile which downloads several English language texts and invokes Python scripts to analyze, merge, and decode data. Each text is analyzed individually and the results are combined to generate an overall word frequency table. Running "make" in this directory should generate an output file "wf.txt". It will also generate a file "wf.mtbl" containing the combined word frequencies from all of the downloaded texts, as well as several intermediate ".mtbl" files containing word frequencies for the individual texts. This example stores the frequency for each word in MTBL SSTable files where each key is a word and its associated value is a varint-encoded integer indicating the number of times the word occurred. Scripts: * pymtbl_wf_analyze.py: reads an ASCII formatted Project Gutenberg etext (.txt) and generates an MTBL SSTable file (.mtbl) containing words and their frequencies. * pymtbl_wf_merge.py: reads one or more MTBL SSTable files and merges the key-value entries in the input files into a single output. For instance, merging the following two input tables: Key Value "and" 1 "or" 2 "the" 3 (Input table #1) Key Value "and" 2 "the" 4 (Input table #2) Will result in the following merged output table: Key Value "and" 3 "or" 2 "the" 7 (Output table) * pymtbl_wf_dump.py: decodes the word frequency key-value entries in an MTBL file and prints them to stdout. pymtbl-0.4.0/examples/wf/pymtbl_wf_analyze.py000077500000000000000000000021431257040443600213640ustar00rootroot00000000000000#!/usr/bin/env python import string import sys import mtbl def merge_func(key, val0, val1): i0 = mtbl.varint_decode(val0) i1 = mtbl.varint_decode(val1) return mtbl.varint_encode(i0 + i1) def main(txt_fname, mtbl_fname): txt = open(txt_fname) sorter = mtbl.sorter(merge_func) writer = mtbl.writer(mtbl_fname, compression=mtbl.COMPRESSION_SNAPPY) # trim header while True: line = txt.readline() if line.startswith('*** START OF THIS PROJECT GUTENBERG EBOOK'): break for x in range(0, 5): txt.readline() for line in txt: if line.startswith('End of the Project Gutenberg EBook') or \ line.startswith('*** END OF THIS PROJECT GUTENBERG EBOOK'): break for tok in line.strip().split(): word = tok.strip(string.punctuation).lower() sorter[word] = mtbl.varint_encode(1) sorter.write(writer) if __name__ == '__main__': if not len(sys.argv) == 3: sys.stderr.write('Usage: %s \n' % sys.argv[0]) sys.exit(1) main(sys.argv[1], sys.argv[2]) pymtbl-0.4.0/examples/wf/pymtbl_wf_dump.py000077500000000000000000000006361257040443600206730ustar00rootroot00000000000000#!/usr/bin/env python import string import sys import mtbl def main(mtbl_fname): reader = mtbl.reader(mtbl_fname) for k, v in reader.items(): word = k count = mtbl.varint_decode(v) print '%s\t%s' % (count, word) if __name__ == '__main__': if not len(sys.argv) == 2: sys.stderr.write('Usage: %s \n' % sys.argv[0]) sys.exit(1) main(sys.argv[1]) pymtbl-0.4.0/examples/wf/pymtbl_wf_merge.py000077500000000000000000000013211257040443600210150ustar00rootroot00000000000000#!/usr/bin/env python import string import struct import sys import mtbl def merge_func(key, val0, val1): i0 = mtbl.varint_decode(val0) i1 = mtbl.varint_decode(val1) return mtbl.varint_encode(i0 + i1) def main(input_fnames, output_fname): merger = mtbl.merger(merge_func) writer = mtbl.writer(output_fname, compression=mtbl.COMPRESSION_SNAPPY) for fname in input_fnames: reader = mtbl.reader(fname) merger.add_reader(reader) merger.write(writer) if __name__ == '__main__': if len(sys.argv) < 3: sys.stderr.write('Usage: %s [...] \n' % sys.argv[0]) sys.exit(1) main(sys.argv[1:-1], sys.argv[-1]) pymtbl-0.4.0/mtbl.pxi000066400000000000000000000100421257040443600145040ustar00rootroot00000000000000cimport cython from cpython cimport bool from cpython.string cimport * from libc.stddef cimport * from libc.stdint cimport * from libc.stdlib cimport * from libc.string cimport * cdef extern from "mtbl.h": ctypedef enum mtbl_res: mtbl_res_failure mtbl_res_success ctypedef enum mtbl_compression_type: MTBL_COMPRESSION_NONE MTBL_COMPRESSION_SNAPPY MTBL_COMPRESSION_ZLIB MTBL_COMPRESSION_LZ4 MTBL_COMPRESSION_LZ4HC struct mtbl_iter: pass struct mtbl_source: pass struct mtbl_reader: pass struct mtbl_reader_options: pass struct mtbl_writer: pass struct mtbl_writer_options: pass struct mtbl_merger: pass struct mtbl_merger_options: pass struct mtbl_sorter: pass struct mtbl_sorter_options: pass ctypedef void (*mtbl_merge_func)(void *clos, uint8_t *, size_t, uint8_t *, size_t, uint8_t *, size_t, uint8_t **, size_t *) # iter void mtbl_iter_destroy(mtbl_iter **) nogil mtbl_res mtbl_iter_next(mtbl_iter *, uint8_t **, size_t *, uint8_t **, size_t *) nogil # source mtbl_iter *mtbl_source_iter(mtbl_source *) nogil mtbl_iter *mtbl_source_get(mtbl_source *, uint8_t *, size_t) nogil mtbl_iter *mtbl_source_get_range(mtbl_source *, uint8_t *, size_t, uint8_t *, size_t) nogil mtbl_iter *mtbl_source_get_prefix(mtbl_source *, uint8_t *, size_t) nogil mtbl_res mtbl_source_write(mtbl_source *, mtbl_writer *) nogil # reader mtbl_reader *mtbl_reader_init(char *, mtbl_reader_options *) nogil void mtbl_reader_destroy(mtbl_reader **) nogil mtbl_source *mtbl_reader_source(mtbl_reader *) nogil mtbl_reader_options *mtbl_reader_options_init() nogil void mtbl_reader_options_destroy(mtbl_reader_options **) nogil void mtbl_reader_options_set_verify_checksums(mtbl_reader_options *, bool) nogil # writer mtbl_writer *mtbl_writer_init(char *, mtbl_writer_options *) nogil void mtbl_writer_destroy(mtbl_writer **) nogil mtbl_res mtbl_writer_add(mtbl_writer *, uint8_t *, size_t, uint8_t *, size_t) nogil mtbl_writer_options *mtbl_writer_options_init() nogil void mtbl_writer_options_destroy(mtbl_writer_options **) nogil void mtbl_writer_options_set_compression(mtbl_writer_options *, mtbl_compression_type) nogil void mtbl_writer_options_set_block_size(mtbl_writer_options *, size_t) nogil void mtbl_writer_options_set_block_restart_interval(mtbl_writer_options *, size_t) nogil # merger mtbl_merger *mtbl_merger_init(mtbl_merger_options *) nogil void mtbl_merger_destroy(mtbl_merger **) nogil void mtbl_merger_add_source(mtbl_merger *, mtbl_source *) nogil mtbl_source *mtbl_merger_source(mtbl_merger *) nogil mtbl_merger_options *mtbl_merger_options_init() nogil void mtbl_merger_options_destroy(mtbl_merger_options **) nogil void mtbl_merger_options_set_merge_func(mtbl_merger_options *, mtbl_merge_func, void *) nogil # sorter mtbl_sorter *mtbl_sorter_init(mtbl_sorter_options *) nogil void mtbl_sorter_destroy(mtbl_sorter **) nogil mtbl_res mtbl_sorter_add(mtbl_sorter *, uint8_t *, size_t, uint8_t *, size_t) nogil mtbl_res mtbl_sorter_write(mtbl_sorter *, mtbl_writer *) nogil mtbl_iter *mtbl_sorter_iter(mtbl_sorter *) nogil mtbl_sorter_options *mtbl_sorter_options_init() nogil void mtbl_sorter_options_destroy(mtbl_sorter_options **) nogil void mtbl_sorter_options_set_merge_func(mtbl_sorter_options *, mtbl_merge_func, void *) nogil void mtbl_sorter_options_set_temp_dir(mtbl_sorter_options *, char *) nogil void mtbl_sorter_options_set_max_memory(mtbl_sorter_options *, size_t) nogil # varint unsigned mtbl_varint_length(uint64_t) nogil unsigned mtbl_varint_length_packed(uint8_t *, size_t) nogil size_t mtbl_varint_encode32(uint8_t *, uint32_t) nogil size_t mtbl_varint_encode64(uint8_t *, uint64_t) nogil size_t mtbl_varint_decode32(uint8_t *, uint32_t *) nogil size_t mtbl_varint_decode64(uint8_t *, uint64_t *) nogil pymtbl-0.4.0/mtbl.pyx000066400000000000000000000507011257040443600145320ustar00rootroot00000000000000include "mtbl.pxi" import threading DEFAULT_SORTER_TEMP_DIR = '/var/tmp' DEFAULT_SORTER_MEMORY = 1073741824 COMPRESSION_NONE = MTBL_COMPRESSION_NONE COMPRESSION_SNAPPY = MTBL_COMPRESSION_SNAPPY COMPRESSION_ZLIB = MTBL_COMPRESSION_ZLIB COMPRESSION_LZ4 = MTBL_COMPRESSION_LZ4 COMPRESSION_LZ4HC = MTBL_COMPRESSION_LZ4HC class KeyOrderError(Exception): pass class TableClosedException(Exception): pass class UnknownCompressionTypeException(Exception): pass class UninitializedException(Exception): pass class VarintDecodingError(Exception): pass ImmutableError = TypeError('object does not support mutation') def varint_length(uint64_t value): """varint_length(v) -> number of bytes the integer v would require in varint encoding.""" return mtbl_varint_length(value) def varint_length_packed(bytes py_buf): """varint_length_packed(b) -> number of varint-packed bytes at the start of b.""" cdef uint8_t *buf cdef Py_ssize_t len_buf cdef size_t sz PyString_AsStringAndSize(py_buf, &buf, &len_buf) with nogil: sz = mtbl_varint_length_packed(buf, len_buf) if sz == 0: raise VarintDecodingError return sz def varint_encode(long v): """varint_encode(v) -> encode integer v using packed variable-width encoding.""" cdef uint8_t buf[10] cdef size_t sz with nogil: sz = mtbl_varint_encode64(buf, v) return PyString_FromStringAndSize( buf, sz) def varint_decode(bytes py_buf): """varint_decode(b) -> decode variable-width packed integer from b""" cdef uint64_t val cdef uint8_t *buf cdef Py_ssize_t len_buf cdef size_t bytes_read PyString_AsStringAndSize(py_buf, &buf, &len_buf) if mtbl_varint_length_packed(buf, len_buf) == 0: raise VarintDecodingError with nogil: mtbl_varint_decode64(buf, &val) return val @cython.internal cdef class iterkeys(object): cdef mtbl_iter *_instance cdef object _parent def __cinit__(self): self._instance = NULL def __init__(self, object _parent): self._parent = _parent def __dealloc__(self): with nogil: mtbl_iter_destroy(&self._instance) def __iter__(self): return self def __next__(self): cdef mtbl_res res cdef uint8_t *key cdef uint8_t *val cdef size_t len_key cdef size_t len_val if self._instance == NULL: raise StopIteration with nogil: res = mtbl_iter_next(self._instance, &key, &len_key, &val, &len_val) if res == mtbl_res_failure: raise StopIteration return PyString_FromStringAndSize( key, len_key) @cython.internal cdef class itervalues(object): cdef mtbl_iter *_instance cdef object _parent def __cinit__(self): self._instance = NULL def __init__(self, object _parent): self._parent = _parent def __dealloc__(self): with nogil: mtbl_iter_destroy(&self._instance) def __iter__(self): return self def __next__(self): cdef mtbl_res res cdef uint8_t *key cdef uint8_t *val cdef size_t len_key cdef size_t len_val if self._instance == NULL: raise StopIteration with nogil: res = mtbl_iter_next(self._instance, &key, &len_key, &val, &len_val) if res == mtbl_res_failure: raise StopIteration return PyString_FromStringAndSize( val, len_val) @cython.internal cdef class iteritems(object): cdef mtbl_iter *_instance cdef object _parent def __cinit__(self): self._instance = NULL def __init__(self, object _parent): self._parent = _parent def __dealloc__(self): with nogil: mtbl_iter_destroy(&self._instance) def __iter__(self): return self def __next__(self): cdef mtbl_res res cdef uint8_t *key cdef uint8_t *val cdef size_t len_key cdef size_t len_val if self._instance == NULL: raise StopIteration with nogil: res = mtbl_iter_next(self._instance, &key, &len_key, &val, &len_val) if res == mtbl_res_failure: raise StopIteration return (PyString_FromStringAndSize( key, len_key), PyString_FromStringAndSize( val, len_val)) cdef get_iterkeys(parent, mtbl_iter *instance): it = iterkeys(parent) it._instance = instance return it cdef get_itervalues(parent, mtbl_iter *instance): it = itervalues(parent) it._instance = instance return it cdef get_iteritems(parent, mtbl_iter *instance): it = iteritems(parent) it._instance = instance return it @cython.internal cdef class DictMixin(object): def __iter__(self): return self.iterkeys() def iter(self): return self.iterkeys() def items(self): """D.items() -> list of D's (key, value) pairs, as 2-tuples""" return [ (k, v) for k, v in self.iteritems() ] def keys(self): """D.keys() -> list of D's keys""" return [ k for k in self.iterkeys() ] def values(self): """D.values() -> list of D's values""" return [ v for v in self.itervalues() ] def __delitem__(self, key): """will raise ImmutableError""" raise ImmutableError def __setitem__(self, key, value): """will raise ImmutableError""" raise ImmutableError def pop(self, *a, **b): """will raise ImmutableError""" raise ImmutableError def popitem(self): """will raise ImmutableError""" raise ImmutableError def update(self, *a, **b): """will raise ImmutableError""" raise ImmutableError cdef class reader(DictMixin): """ reader(fname) -> new MTBL reader initialized from file fname Keyword arguments: verify_checksums -- whether to verify data block checksums (default False) """ cdef mtbl_reader *_instance def __cinit__(self): self._instance = NULL def __dealloc__(self): with nogil: mtbl_reader_destroy(&self._instance) def __init__(self, char * fname, bool verify_checksums=False): cdef mtbl_reader_options *opt with nogil: opt = mtbl_reader_options_init() mtbl_reader_options_set_verify_checksums(opt, verify_checksums) self._instance = mtbl_reader_init(fname, opt) mtbl_reader_options_destroy(&opt) if (self._instance == NULL): raise IOError("unable to open file: '%s'" % fname) cdef check_initialized(self): if self._instance == NULL: raise UninitializedException def iterkeys(self): """R.iterkeys() -> an iterator over the keys of R.""" self.check_initialized() return get_iterkeys(self, mtbl_source_iter(mtbl_reader_source(self._instance))) def itervalues(self): """R.itervalues() -> an iterator over the values of R.""" self.check_initialized() return get_itervalues(self, mtbl_source_iter(mtbl_reader_source(self._instance))) def iteritems(self): """R.iteritems() -> an iterator over the (key, value) items of R.""" self.check_initialized() return get_iteritems(self, mtbl_source_iter(mtbl_reader_source(self._instance))) def __contains__(self, bytes py_key): """R.__contains__(k) -> True if R has a key k, else False""" try: self.__getitem__(py_key) return True except KeyError: pass return False def has_key(self, bytes py_key): """R.has_key(k) -> True if R has a key k, else False.""" return self.__contains__(py_key) def get(self, bytes py_key, default=None): """R.get(k[,d]) -> R[k] if k in R, else d. d defaults to None.""" try: return self.__getitem__(py_key) except KeyError: pass return default def get_range(self, bytes py_key0, bytes py_key1): """ R.get_range(key0, key1) -> an iterator over all (key, value) items in R where key is between key0 and key1 inclusive. """ cdef mtbl_res res cdef uint8_t *key0 cdef uint8_t *key1 cdef size_t len_key0 cdef size_t len_key1 self.check_initialized() key0 = PyString_AsString(py_key0) key1 = PyString_AsString(py_key1) len_key0 = PyString_Size(py_key0) len_key1 = PyString_Size(py_key1) return get_iteritems(self, mtbl_source_get_range( mtbl_reader_source(self._instance), key0, len_key0, key1, len_key1)) def get_prefix(self, bytes py_key): """ R.get_prefix(key_prefix) -> an iterator over all (key, value) items in R where key begins with key_prefix. """ cdef mtbl_res res cdef uint8_t *key cdef size_t len_key self.check_initialized() key = PyString_AsString(py_key) len_key = PyString_Size(py_key) return get_iteritems(self, mtbl_source_get_prefix( mtbl_reader_source(self._instance), key, len_key)) def __getitem__(self, bytes py_key): cdef mtbl_iter *it cdef mtbl_res res cdef uint8_t *key cdef uint8_t *val cdef size_t len_key cdef size_t len_val self.check_initialized() key = PyString_AsString(py_key) len_key = PyString_Size(py_key) items = [] with nogil: it = mtbl_source_get(mtbl_reader_source(self._instance), key, len_key) if it == NULL: raise KeyError(py_key) while True: with nogil: res = mtbl_iter_next(it, &key, &len_key, &val, &len_val) if res == mtbl_res_failure: break items.append(PyString_FromStringAndSize( val, len_val)) with nogil: mtbl_iter_destroy(&it) if not items: raise KeyError(py_key) return items cdef class writer(object): """ writer(fname) -> new MTBL writer, output to file fname Keyword arguments: compression -- compression type (default COMPRESSION_NONE) block_size -- maximum data block size in bytes (default 8192) block_restart_interval -- how frequently to restart key prefix compression (default 16) """ cdef mtbl_writer *_instance cdef _lock def __cinit__(self): self._instance = NULL def __dealloc__(self): with nogil: mtbl_writer_destroy(&self._instance) def __init__(self, char * fname, mtbl_compression_type compression=COMPRESSION_NONE, size_t block_size=8192, size_t block_restart_interval=16): if not (compression == COMPRESSION_NONE or compression == COMPRESSION_SNAPPY or compression == COMPRESSION_ZLIB or compression == COMPRESSION_LZ4 or compression == COMPRESSION_LZ4HC): raise UnknownCompressionTypeException self._lock = threading.Semaphore() cdef mtbl_writer_options *opt with nogil: opt = mtbl_writer_options_init() mtbl_writer_options_set_compression(opt, compression) mtbl_writer_options_set_block_size(opt, block_size) mtbl_writer_options_set_block_restart_interval(opt, block_restart_interval) self._instance = mtbl_writer_init(fname, opt) mtbl_writer_options_destroy(&opt) if self._instance == NULL: raise IOError("unable to initialize file: '%s'" % fname) def close(self): """W.close() -- finalize and close the writer""" with self._lock: with nogil: mtbl_writer_destroy(&self._instance) def __setitem__(self, bytes py_key, bytes py_val): """ W.__setitem__(key, value) <==> W[key] = value Adds a new (key, value) entry to the writer. key and value must be byte strings, and key must be lexicographically greater than any previously written key. """ cdef mtbl_res res cdef uint8_t *key cdef uint8_t *val cdef size_t len_key cdef size_t len_val if self._instance == NULL: raise TableClosedException key = PyString_AsString(py_key) val = PyString_AsString(py_val) len_key = PyString_Size(py_key) len_val = PyString_Size(py_val) with self._lock: with nogil: res = mtbl_writer_add(self._instance, key, len_key, val, len_val) if res == mtbl_res_failure: raise KeyOrderError def __delitem__(self, key): """will raise ImmutableError""" raise ImmutableError cdef void merge_func_wrapper(void *clos, uint8_t *key, size_t len_key, uint8_t *val0, size_t len_val0, uint8_t *val1, size_t len_val1, uint8_t **merged_val, size_t *len_merged_val) with gil: cdef str py_key cdef str py_val0 cdef str py_val1 cdef str py_merged_val py_key = PyString_FromStringAndSize( key, len_key) py_val0 = PyString_FromStringAndSize( val0, len_val0) py_val1 = PyString_FromStringAndSize( val1, len_val1) py_merged_val = ( clos)(py_key, py_val0, py_val1) len_merged_val[0] = PyString_Size(py_merged_val) merged_val[0] = malloc(len_merged_val[0]) memcpy(merged_val[0], PyString_AsString(py_merged_val), len_merged_val[0]) cdef class merger(object): """ merger(merge_func) -> new MTBL merger merge_func is the user-supplied value merging function: merge_func(key, val0, val1) -> merged_val all parameters are byte strings, and the return value must be a byte string. """ cdef mtbl_merger *_instance cdef set _references cdef _lock def __cinit__(self): self._instance = NULL def __dealloc__(self): with nogil: mtbl_merger_destroy(&self._instance) def __init__(self, object merge_func): cdef mtbl_merger_options *opt opt = mtbl_merger_options_init() mtbl_merger_options_set_merge_func(opt, merge_func_wrapper, merge_func) self._instance = mtbl_merger_init(opt) mtbl_merger_options_destroy(&opt) self._references = set() self._lock = threading.Semaphore() def add_reader(self, reader r): """M.add_reader(mtbl.reader) -- add a reader object as a merge input""" with self._lock: with nogil: mtbl_merger_add_source(self._instance, mtbl_reader_source(r._instance)) self._references.add(r) def write(self, writer w): """M.write(mtbl.writer) -- dump merged output to writer""" cdef mtbl_res res with w._lock: with nogil: res = mtbl_source_write(mtbl_merger_source(self._instance), w._instance) if res != mtbl_res_success: raise RuntimeError def __iter__(self): return self.iterkeys() def iterkeys(self): """M.iterkeys() -> an iterator over the merged keys of M.""" return get_iterkeys(self, mtbl_source_iter(mtbl_merger_source(self._instance))) def itervalues(self): """M.itervalues() -> an iterator over the merged values of M.""" return get_itervalues(self, mtbl_source_iter(mtbl_merger_source(self._instance))) def iteritems(self): """M.iteritems() -> an iterator over the merged (key, value) items of M.""" return get_iteritems(self, mtbl_source_iter(mtbl_merger_source(self._instance))) def get(self, bytes py_key): """ M.get(key) -> an iterator over all (key, value) items in M which match key. """ cdef mtbl_res res cdef uint8_t *key cdef size_t len_key key = PyString_AsString(py_key) len_key = PyString_Size(py_key) return get_iteritems(self, mtbl_source_get(mtbl_merger_source(self._instance), key, len_key)) def get_range(self, bytes py_key0, bytes py_key1): """ M.get_range(key0, key1) -> an iterator over all (key, value) items in M where key is between key0 and key1 inclusive. """ cdef mtbl_res res cdef uint8_t *key0 cdef uint8_t *key1 cdef size_t len_key0 cdef size_t len_key1 key0 = PyString_AsString(py_key0) key1 = PyString_AsString(py_key1) len_key0 = PyString_Size(py_key0) len_key1 = PyString_Size(py_key1) return get_iteritems(self, mtbl_source_get_range( mtbl_merger_source(self._instance), key0, len_key0, key1, len_key1)) def get_prefix(self, bytes py_key): """ M.get_prefix(key_prefix) -> an iterator over all (key, value) items in M where key begins with key_prefix. """ cdef mtbl_res res cdef uint8_t *key cdef size_t len_key key = PyString_AsString(py_key) len_key = PyString_Size(py_key) return get_iteritems(self, mtbl_source_get_prefix( mtbl_merger_source(self._instance), key, len_key)) cdef class sorter(object): """ sorter(merge_func) -> new MTBL sorter merge_func is the user-supplied value merging function: merge_func(key, val0, val1) -> merged_val all parameters are byte strings, and the return value must be a byte string. Keyword arguments: temp_dir -- temporary directory (default "/var/tmp") max_memory -- maxmimum amount of memory for in-memory sorting in bytes (default 1 GB) """ cdef mtbl_sorter *_instance cdef _lock def __cinit__(self): self._instance = NULL def __dealloc__(self): with nogil: mtbl_sorter_destroy(&self._instance) def __init__(self, object merge_func, bytes temp_dir=DEFAULT_SORTER_TEMP_DIR, size_t max_memory=DEFAULT_SORTER_MEMORY): cdef mtbl_sorter_options *opt self._lock = threading.Semaphore() with nogil: opt = mtbl_sorter_options_init() mtbl_sorter_options_set_merge_func(opt, merge_func_wrapper, merge_func) mtbl_sorter_options_set_temp_dir(opt, temp_dir) mtbl_sorter_options_set_max_memory(opt, max_memory) with nogil: self._instance = mtbl_sorter_init(opt) mtbl_sorter_options_destroy(&opt) def write(self, writer w): """S.write(mtbl.writer) -- dump sorted output to writer""" cdef mtbl_res res with w._lock: with nogil: res = mtbl_sorter_write(self._instance, w._instance) if res != mtbl_res_success: raise RuntimeError def __setitem__(self, bytes py_key, bytes py_val): """ S.__setitem__(key, value) <==> S[key] = value Adds a new (key, value) item to the sorter. If the key already exists, the user-supplied merge function will be called to merge the conflicting values. """ cdef mtbl_res res cdef uint8_t *key cdef uint8_t *val cdef size_t len_key cdef size_t len_val if self._instance == NULL: raise RuntimeError key = PyString_AsString(py_key) val = PyString_AsString(py_val) len_key = PyString_Size(py_key) len_val = PyString_Size(py_val) with self._lock: with nogil: res = mtbl_sorter_add(self._instance, key, len_key, val, len_val) if res == mtbl_res_failure: raise KeyOrderError def __delitem__(self, key): """will raise ImmutableError""" raise ImmutableError def __iter__(self): return self.iterkeys() def iterkeys(self): """S.iterkeys() -> an iterator over the sorted keys of R.""" return get_iterkeys(self, mtbl_sorter_iter(self._instance)) def itervalues(self): """S.itervalues() -> an iterator over the sorted values of R.""" return get_itervalues(self, mtbl_sorter_iter(self._instance)) def iteritems(self): """S.iteritems() -> an iterator over the sorted (key, value) items of R.""" return get_iteritems(self, mtbl_sorter_iter(self._instance)) pymtbl-0.4.0/setup.py000077500000000000000000000021521257040443600145440ustar00rootroot00000000000000#!/usr/bin/env python NAME = 'pymtbl' VERSION = '0.4.0' from distutils.core import setup from distutils.extension import Extension def pkgconfig(*packages, **kw): import subprocess flag_map = { '-I': 'include_dirs', '-L': 'library_dirs', '-l': 'libraries' } pkg_config_cmd = 'pkg-config --cflags --libs "%s"' % ' '.join(packages) for token in subprocess.check_output(pkg_config_cmd, shell=True).split(): flag = token[:2] arg = token[2:] if flag in flag_map: kw.setdefault(flag_map[flag], []).append(arg) return kw try: from Cython.Distutils import build_ext setup( name = NAME, version = VERSION, ext_modules = [ Extension('mtbl', ['mtbl.pyx'], **pkgconfig('libmtbl >= 0.8.0')) ], cmdclass = {'build_ext': build_ext}, ) except ImportError: import os if os.path.isfile('mtbl.c'): setup( name = NAME, version = VERSION, ext_modules = [ Extension('mtbl', ['mtbl.c'], **pkgconfig('libmtbl >= 0.8.0')) ], ) else: raise