pax_global_header00006660000000000000000000000064122676001150014512gustar00rootroot0000000000000052 comment=9f8835ca5c9698294df20612f8a0a44fcc2412c9 pymtbl-0.2/000077500000000000000000000000001226760011500126625ustar00rootroot00000000000000pymtbl-0.2/.gitignore000066400000000000000000000001331226760011500146470ustar00rootroot00000000000000.*swp /MANIFEST /build/ /dist/ mtbl.c examples/*.mtbl examples/wf/*.mtbl examples/wf/*.txt pymtbl-0.2/COPYRIGHT000066400000000000000000000010771226760011500141620ustar00rootroot00000000000000Copyright (c) 2012-2014 by Farsight Security, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. pymtbl-0.2/LICENSE000066400000000000000000000261361226760011500136770ustar00rootroot00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. pymtbl-0.2/MANIFEST.in000066400000000000000000000001261226760011500144170ustar00rootroot00000000000000include COPYRIGHT LICENSE README.md examples/* examples/wf/* mtbl.c mtbl.pxi mtbl.pyx pymtbl-0.2/README.md000066400000000000000000000024451226760011500141460ustar00rootroot00000000000000pymtbl: Python bindings for the mtbl sorted string table library ---------------------------------------------------------------- `pymtbl` provides a simple Pythonic wrapper for [mtbl](https://github.com/farsightsec/mtbl)'s reader, writer, sorter, and merger interfaces. The `examples/` directory contains scripts demonstrating each of these interfaces. The following transcript shows the basic reader and writer interfaces: >>> import mtbl >>> w = mtbl.writer('example.mtbl', compression=mtbl.COMPRESSION_SNAPPY) >>> w['key1'] = 'val1' >>> w['key17'] = 'val17' >>> w['key2'] = 'val2' >>> w['key23'] = 'val23' >>> w['key3'] = 'val3' >>> w['key4'] = 'val4' >>> w['key05'] = 'val05' Traceback (most recent call last): File "", line 1, in File "mtbl.pyx", line 361, in mtbl.writer.__setitem__ (mtbl.c:4108) mtbl.KeyOrderError >>> w.close() >>> r = mtbl.reader('example.mtbl', verify_checksums=True) >>> for k,v in r.get_range('key19', 'key23'): print k,v ... key2 val2 key23 val23 >>> for k,v in r.get_prefix('key2'): print k,v ... key2 val2 key23 val23 >>> for k,v in r.iteritems(): print k, v ... key1 val1 key17 val17 key2 val2 key23 val23 key3 val3 key4 val4 >>> pymtbl-0.2/examples/000077500000000000000000000000001226760011500145005ustar00rootroot00000000000000pymtbl-0.2/examples/pymtbl_make_random_table.py000077500000000000000000000035541226760011500220770ustar00rootroot00000000000000#!/usr/bin/env python import locale import random import string import sys import time import mtbl report_interval = 10000000 megabyte = 1048576.0 def main(fname, num_keys): writer = mtbl.writer(fname, compression=mtbl.COMPRESSION_SNAPPY) a = time.time() last = a total_bytes = 0 count = 0 total = 0 while count < num_keys: count += 1 if random.random() >= 0.5: total += 1 key = '%010d' % count val = random.choice(string.ascii_lowercase) * random.randint(1, 50) writer[key] = val total_bytes += len(key) + len(val) if (count % report_interval) == 0: b = time.time() last_secs = b - last last = b sys.stderr.write('wrote %s entries (%s MB) in %s seconds, %s entries/second\n' % ( locale.format('%d', total, grouping=True), locale.format('%d', total_bytes / megabyte, grouping=True), locale.format('%f', last_secs, grouping=True), locale.format('%d', report_interval / last_secs, grouping=True) ) ) b = time.time() total_secs = b - a sys.stderr.write('wrote %s total entries (%s MB) in %s seconds, %s entries/second\n' % ( locale.format('%d', total, grouping=True), locale.format('%d', total_bytes / megabyte, grouping=True), locale.format('%f', total_secs, grouping=True), locale.format('%d', total / total_secs, grouping=True) ) ) def usage(): sys.stderr.write('Usage: %s \n' % sys.argv[0]) sys.exit(1) if __name__ == '__main__': locale.setlocale(locale.LC_ALL, '') if len(sys.argv) != 3: usage() try: fname = sys.argv[1] num_keys = int(sys.argv[2]) except: usage() main(fname, num_keys) pymtbl-0.2/examples/pymtbl_make_random_table_2.py000077500000000000000000000037351226760011500223210ustar00rootroot00000000000000#!/usr/bin/env python import locale import random import string import sys import time import mtbl report_interval = 100000 megabyte = 1048576.0 def merge_func(key, val0, val1): return val0 + val1 def main(fname, num_keys): sorter = mtbl.sorter(merge_func) writer = mtbl.writer(fname, compression=mtbl.COMPRESSION_SNAPPY) a = time.time() last = a total_bytes = 0 count = 0 while count < num_keys: count += 1 key = '%020d' % random.randint(0, sys.maxint) val = random.choice(string.ascii_lowercase) * random.randint(1, 50) sorter[key] = val total_bytes += len(key) + len(val) if (count % report_interval) == 0: b = time.time() last_secs = b - last last = b sys.stderr.write('generated %s entries (%s MB) in %s seconds, %s entries/second\n' % ( locale.format('%d', count, grouping=True), locale.format('%d', total_bytes / megabyte, grouping=True), locale.format('%f', last_secs, grouping=True), locale.format('%d', report_interval / last_secs, grouping=True) ) ) sys.stderr.write('writing to output file %s\n' % fname) sorter.write(writer) b = time.time() total_secs = b - a sys.stderr.write('wrote %s total entries (%s MB) in %s seconds, %s entries/second\n' % ( locale.format('%d', count, grouping=True), locale.format('%d', total_bytes / megabyte, grouping=True), locale.format('%f', total_secs, grouping=True), locale.format('%d', count / total_secs, grouping=True) ) ) def usage(): sys.stderr.write('Usage: %s \n' % sys.argv[0]) sys.exit(1) if __name__ == '__main__': locale.setlocale(locale.LC_ALL, '') if len(sys.argv) != 3: usage() try: fname = sys.argv[1] num_keys = int(sys.argv[2]) except: usage() main(fname, num_keys) pymtbl-0.2/examples/pymtbl_merge_tables.py000077500000000000000000000013751226760011500211030ustar00rootroot00000000000000#!/usr/bin/env python import sys import mtbl def merge_func(key, val0, val1): return val0 + ' ' + val1 def main(output_fname, input_fnames): merger = mtbl.merger(merge_func) writer = mtbl.writer(output_fname, compression=mtbl.COMPRESSION_SNAPPY) for fname in input_fnames: reader = mtbl.reader(fname) merger.add_reader(reader) for k, v in merger.iteritems(): writer[k] = v writer.close() def usage(): sys.stderr.write('Usage: %s [...]\n' % sys.argv[0]) sys.exit(1) if __name__ == '__main__': if len(sys.argv) < 3: usage() try: output_fname = sys.argv[1] input_fnames = sys.argv[2:] except: usage() main(output_fname, input_fnames) pymtbl-0.2/examples/pymtbl_merge_tables_2.py000077500000000000000000000013341226760011500213170ustar00rootroot00000000000000#!/usr/bin/env python import sys import mtbl def merge_func(key, val0, val1): return val0 + ' ' + val1 def main(output_fname, input_fnames): merger = mtbl.merger(merge_func) writer = mtbl.writer(output_fname, compression=mtbl.COMPRESSION_SNAPPY) for fname in input_fnames: reader = mtbl.reader(fname) merger.add_reader(reader) merger.write(writer) writer.close() def usage(): sys.stderr.write('Usage: %s [...]\n' % sys.argv[0]) sys.exit(1) if __name__ == '__main__': if len(sys.argv) < 3: usage() try: output_fname = sys.argv[1] input_fnames = sys.argv[2:] except: usage() main(output_fname, input_fnames) pymtbl-0.2/examples/pymtbl_search_random_table.py000077500000000000000000000033441226760011500224240ustar00rootroot00000000000000#!/usr/bin/env python import locale import random import sys import time import mtbl report_interval = 10000 def main(fname, num_keys, num_iters): reader = mtbl.reader(fname) a = time.time() last = a num_found = 0 count = 0 while count < num_iters: key = '%010d' % random.randint(0, num_keys) if reader.has_key(key): val = reader[key] num_found += 1 count += 1 if (count % report_interval) == 0: b = time.time() last_secs = b - last last = b sys.stderr.write('%s lookups, %s keys found in %s seconds, %s lookups/second\n' % ( locale.format('%d', count, grouping=True), locale.format('%d', num_found, grouping=True), locale.format('%f', last_secs, grouping=True), locale.format('%d', report_interval / last_secs, grouping=True) ) ) b = time.time() total_secs = b - a sys.stderr.write('%s total lookups, %s keys found in %s seconds, %s lookups/second\n' % ( locale.format('%d', count, grouping=True), locale.format('%d', num_found, grouping=True), locale.format('%f', total_secs, grouping=True), locale.format('%d', count / total_secs, grouping=True) ) ) def usage(): sys.stderr.write('Usage: %s \n' % sys.argv[0]) sys.exit(1) if __name__ == '__main__': locale.setlocale(locale.LC_ALL, '') if len(sys.argv) != 4: usage() try: fname = sys.argv[1] num_keys = int(sys.argv[2]) num_iters = int(sys.argv[3]) except: usage() main(fname, num_keys, num_iters) pymtbl-0.2/examples/wf/000077500000000000000000000000001226760011500151145ustar00rootroot00000000000000pymtbl-0.2/examples/wf/Makefile000066400000000000000000000007701226760011500165600ustar00rootroot00000000000000EBOOKS = 2814 4217 4300 BOOK_FILES = $(foreach ebook,$(EBOOKS),$(ebook).txt) MTBL_FILES = $(foreach ebook,$(EBOOKS),$(ebook).mtbl) %.txt: @wget -O $@ http://www.gutenberg.org/files/$(shell basename $@ .txt)/$@ %.mtbl: %.txt rm -f $@ ./pymtbl_wf_analyze.py $< $@ wf.txt: wf.mtbl $(BOOK_FILES) ./pymtbl_wf_dump.py wf.mtbl | sort -rn > wf.txt wf.mtbl: $(MTBL_FILES) $(BOOK_FILES) rm -f wf.mtbl ./pymtbl_wf_merge.py $(MTBL_FILES) wf.mtbl clean: rm -f wf.mtbl wf.txt $(MTBL_FILES) .PHONY: clean pymtbl-0.2/examples/wf/README000066400000000000000000000032611226760011500157760ustar00rootroot00000000000000Simple word frequency example using pymtbl ------------------------------------------ This directory shows a simple "word frequency" application of the pymtbl module. It is driven by a Makefile which downloads several English language texts and invokes Python scripts to analyze, merge, and decode data. Each text is analyzed individually and the results are combined to generate an overall word frequency table. Running "make" in this directory should generate an output file "wf.txt". It will also generate a file "wf.mtbl" containing the combined word frequencies from all of the downloaded texts, as well as several intermediate ".mtbl" files containing word frequencies for the individual texts. This example stores the frequency for each word in MTBL SSTable files where each key is a word and its associated value is a varint-encoded integer indicating the number of times the word occurred. Scripts: * pymtbl_wf_analyze.py: reads an ASCII formatted Project Gutenberg etext (.txt) and generates an MTBL SSTable file (.mtbl) containing words and their frequencies. * pymtbl_wf_merge.py: reads one or more MTBL SSTable files and merges the key-value entries in the input files into a single output. For instance, merging the following two input tables: Key Value "and" 1 "or" 2 "the" 3 (Input table #1) Key Value "and" 2 "the" 4 (Input table #2) Will result in the following merged output table: Key Value "and" 3 "or" 2 "the" 7 (Output table) * pymtbl_wf_dump.py: decodes the word frequency key-value entries in an MTBL file and prints them to stdout. pymtbl-0.2/examples/wf/pymtbl_wf_analyze.py000077500000000000000000000021431226760011500212170ustar00rootroot00000000000000#!/usr/bin/env python import string import sys import mtbl def merge_func(key, val0, val1): i0 = mtbl.varint_decode(val0) i1 = mtbl.varint_decode(val1) return mtbl.varint_encode(i0 + i1) def main(txt_fname, mtbl_fname): txt = open(txt_fname) sorter = mtbl.sorter(merge_func) writer = mtbl.writer(mtbl_fname, compression=mtbl.COMPRESSION_SNAPPY) # trim header while True: line = txt.readline() if line.startswith('*** START OF THIS PROJECT GUTENBERG EBOOK'): break for x in range(0, 5): txt.readline() for line in txt: if line.startswith('End of the Project Gutenberg EBook') or \ line.startswith('*** END OF THIS PROJECT GUTENBERG EBOOK'): break for tok in line.strip().split(): word = tok.strip(string.punctuation).lower() sorter[word] = mtbl.varint_encode(1) sorter.write(writer) if __name__ == '__main__': if not len(sys.argv) == 3: sys.stderr.write('Usage: %s \n' % sys.argv[0]) sys.exit(1) main(sys.argv[1], sys.argv[2]) pymtbl-0.2/examples/wf/pymtbl_wf_dump.py000077500000000000000000000006361226760011500205260ustar00rootroot00000000000000#!/usr/bin/env python import string import sys import mtbl def main(mtbl_fname): reader = mtbl.reader(mtbl_fname) for k, v in reader.items(): word = k count = mtbl.varint_decode(v) print '%s\t%s' % (count, word) if __name__ == '__main__': if not len(sys.argv) == 2: sys.stderr.write('Usage: %s \n' % sys.argv[0]) sys.exit(1) main(sys.argv[1]) pymtbl-0.2/examples/wf/pymtbl_wf_merge.py000077500000000000000000000013211226760011500206500ustar00rootroot00000000000000#!/usr/bin/env python import string import struct import sys import mtbl def merge_func(key, val0, val1): i0 = mtbl.varint_decode(val0) i1 = mtbl.varint_decode(val1) return mtbl.varint_encode(i0 + i1) def main(input_fnames, output_fname): merger = mtbl.merger(merge_func) writer = mtbl.writer(output_fname, compression=mtbl.COMPRESSION_SNAPPY) for fname in input_fnames: reader = mtbl.reader(fname) merger.add_reader(reader) merger.write(writer) if __name__ == '__main__': if len(sys.argv) < 3: sys.stderr.write('Usage: %s [...] \n' % sys.argv[0]) sys.exit(1) main(sys.argv[1:-1], sys.argv[-1]) pymtbl-0.2/mtbl.pxi000066400000000000000000000073361226760011500143530ustar00rootroot00000000000000cimport cython from cpython cimport bool from cpython.string cimport * from libc.stddef cimport * from libc.stdint cimport * from libc.stdlib cimport * from libc.string cimport * cdef extern from "mtbl.h": ctypedef enum mtbl_res: mtbl_res_failure mtbl_res_success ctypedef enum mtbl_compression_type: MTBL_COMPRESSION_NONE MTBL_COMPRESSION_SNAPPY MTBL_COMPRESSION_ZLIB struct mtbl_iter: pass struct mtbl_source: pass struct mtbl_reader: pass struct mtbl_reader_options: pass struct mtbl_writer: pass struct mtbl_writer_options: pass struct mtbl_merger: pass struct mtbl_merger_options: pass struct mtbl_sorter: pass struct mtbl_sorter_options: pass ctypedef void (*mtbl_merge_func)(void *clos, uint8_t *, size_t, uint8_t *, size_t, uint8_t *, size_t, uint8_t **, size_t *) # iter void mtbl_iter_destroy(mtbl_iter **) mtbl_res mtbl_iter_next(mtbl_iter *, uint8_t **, size_t *, uint8_t **, size_t *) # source mtbl_iter *mtbl_source_iter(mtbl_source *) mtbl_iter *mtbl_source_get(mtbl_source *, uint8_t *, size_t) mtbl_iter *mtbl_source_get_range(mtbl_source *, uint8_t *, size_t, uint8_t *, size_t) mtbl_iter *mtbl_source_get_prefix(mtbl_source *, uint8_t *, size_t) mtbl_res mtbl_source_write(mtbl_source *, mtbl_writer *) # reader mtbl_reader *mtbl_reader_init(char *, mtbl_reader_options *) void mtbl_reader_destroy(mtbl_reader **) mtbl_source *mtbl_reader_source(mtbl_reader *) mtbl_reader_options *mtbl_reader_options_init() void mtbl_reader_options_destroy(mtbl_reader_options **) void mtbl_reader_options_set_verify_checksums(mtbl_reader_options *, bool) # writer mtbl_writer *mtbl_writer_init(char *, mtbl_writer_options *) void mtbl_writer_destroy(mtbl_writer **) mtbl_res mtbl_writer_add(mtbl_writer *, uint8_t *, size_t, uint8_t *, size_t) mtbl_writer_options *mtbl_writer_options_init() void mtbl_writer_options_destroy(mtbl_writer_options **) void mtbl_writer_options_set_compression(mtbl_writer_options *, mtbl_compression_type) void mtbl_writer_options_set_block_size(mtbl_writer_options *, size_t) void mtbl_writer_options_set_block_restart_interval(mtbl_writer_options *, size_t) # merger mtbl_merger *mtbl_merger_init(mtbl_merger_options *) void mtbl_merger_destroy(mtbl_merger **) void mtbl_merger_add_source(mtbl_merger *, mtbl_source *) mtbl_source *mtbl_merger_source(mtbl_merger *) mtbl_merger_options *mtbl_merger_options_init() void mtbl_merger_options_destroy(mtbl_merger_options **) void mtbl_merger_options_set_merge_func(mtbl_merger_options *, mtbl_merge_func, void *) # sorter mtbl_sorter *mtbl_sorter_init(mtbl_sorter_options *) void mtbl_sorter_destroy(mtbl_sorter **) mtbl_res mtbl_sorter_add(mtbl_sorter *, uint8_t *, size_t, uint8_t *, size_t) mtbl_res mtbl_sorter_write(mtbl_sorter *, mtbl_writer *) mtbl_iter *mtbl_sorter_iter(mtbl_sorter *) mtbl_sorter_options *mtbl_sorter_options_init() void mtbl_sorter_options_destroy(mtbl_sorter_options **) void mtbl_sorter_options_set_merge_func(mtbl_sorter_options *, mtbl_merge_func, void *) void mtbl_sorter_options_set_temp_dir(mtbl_sorter_options *, char *) void mtbl_sorter_options_set_max_memory(mtbl_sorter_options *, size_t) # varint unsigned mtbl_varint_length(uint64_t) unsigned mtbl_varint_length_packed(uint8_t *, size_t) size_t mtbl_varint_encode32(uint8_t *, uint32_t) size_t mtbl_varint_encode64(uint8_t *, uint64_t) size_t mtbl_varint_decode32(uint8_t *, uint32_t *) size_t mtbl_varint_decode64(uint8_t *, uint64_t *) pymtbl-0.2/mtbl.pyx000066400000000000000000000463651226760011500144000ustar00rootroot00000000000000include "mtbl.pxi" DEFAULT_SORTER_TEMP_DIR = '/var/tmp' DEFAULT_SORTER_MEMORY = 1073741824 COMPRESSION_NONE = MTBL_COMPRESSION_NONE COMPRESSION_SNAPPY = MTBL_COMPRESSION_SNAPPY COMPRESSION_ZLIB = MTBL_COMPRESSION_ZLIB class KeyOrderError(Exception): pass class TableClosedException(Exception): pass class UnknownCompressionTypeException(Exception): pass class UninitializedException(Exception): pass class VarintDecodingError(Exception): pass ImmutableError = TypeError('object does not support mutation') def varint_length(uint64_t value): """varint_length(v) -> number of bytes the integer v would require in varint encoding.""" return mtbl_varint_length(value) def varint_length_packed(bytes py_buf): """varint_length_packed(b) -> number of varint-packed bytes at the start of b.""" cdef uint8_t *buf cdef Py_ssize_t len_buf cdef size_t sz PyString_AsStringAndSize(py_buf, &buf, &len_buf) sz = mtbl_varint_length_packed(buf, len_buf) if sz == 0: raise VarintDecodingError return sz def varint_encode(long v): """varint_encode(v) -> encode integer v using packed variable-width encoding.""" cdef uint8_t buf[10] cdef size_t sz sz = mtbl_varint_encode64(buf, v) return PyString_FromStringAndSize( buf, sz) def varint_decode(bytes py_buf): """varint_decode(b) -> decode variable-width packed integer from b""" cdef uint64_t val cdef uint8_t *buf cdef Py_ssize_t len_buf cdef size_t bytes_read PyString_AsStringAndSize(py_buf, &buf, &len_buf) if mtbl_varint_length_packed(buf, len_buf) == 0: raise VarintDecodingError mtbl_varint_decode64(buf, &val) return val @cython.internal cdef class iterkeys(object): cdef mtbl_iter *_instance cdef object _parent def __cinit__(self): self._instance = NULL def __init__(self, object _parent): self._parent = _parent def __dealloc__(self): mtbl_iter_destroy(&self._instance) def __iter__(self): return self def __next__(self): cdef mtbl_res res cdef uint8_t *key cdef uint8_t *val cdef size_t len_key cdef size_t len_val if self._instance == NULL: raise StopIteration res = mtbl_iter_next(self._instance, &key, &len_key, &val, &len_val) if res == mtbl_res_failure: raise StopIteration return PyString_FromStringAndSize( key, len_key) @cython.internal cdef class itervalues(object): cdef mtbl_iter *_instance cdef object _parent def __cinit__(self): self._instance = NULL def __init__(self, object _parent): self._parent = _parent def __dealloc__(self): mtbl_iter_destroy(&self._instance) def __iter__(self): return self def __next__(self): cdef mtbl_res res cdef uint8_t *key cdef uint8_t *val cdef size_t len_key cdef size_t len_val if self._instance == NULL: raise StopIteration res = mtbl_iter_next(self._instance, &key, &len_key, &val, &len_val) if res == mtbl_res_failure: raise StopIteration return PyString_FromStringAndSize( val, len_val) @cython.internal cdef class iteritems(object): cdef mtbl_iter *_instance cdef object _parent def __cinit__(self): self._instance = NULL def __init__(self, object _parent): self._parent = _parent def __dealloc__(self): mtbl_iter_destroy(&self._instance) def __iter__(self): return self def __next__(self): cdef mtbl_res res cdef uint8_t *key cdef uint8_t *val cdef size_t len_key cdef size_t len_val if self._instance == NULL: raise StopIteration res = mtbl_iter_next(self._instance, &key, &len_key, &val, &len_val) if res == mtbl_res_failure: raise StopIteration return (PyString_FromStringAndSize( key, len_key), PyString_FromStringAndSize( val, len_val)) cdef get_iterkeys(parent, mtbl_iter *instance): it = iterkeys(parent) it._instance = instance return it cdef get_itervalues(parent, mtbl_iter *instance): it = itervalues(parent) it._instance = instance return it cdef get_iteritems(parent, mtbl_iter *instance): it = iteritems(parent) it._instance = instance return it @cython.internal cdef class DictMixin(object): def __iter__(self): return self.iterkeys() def iter(self): return self.iterkeys() def items(self): """D.items() -> list of D's (key, value) pairs, as 2-tuples""" return [ (k, v) for k, v in self.iteritems() ] def keys(self): """D.keys() -> list of D's keys""" return [ k for k in self.iterkeys() ] def values(self): """D.values() -> list of D's values""" return [ v for v in self.itervalues() ] def __delitem__(self, key): """will raise ImmutableError""" raise ImmutableError def __setitem__(self, key, value): """will raise ImmutableError""" raise ImmutableError def pop(self, *a, **b): """will raise ImmutableError""" raise ImmutableError def popitem(self): """will raise ImmutableError""" raise ImmutableError def update(self, *a, **b): """will raise ImmutableError""" raise ImmutableError cdef class reader(DictMixin): """ reader(fname) -> new MTBL reader initialized from file fname Keyword arguments: verify_checksums -- whether to verify data block checksums (default False) """ cdef mtbl_reader *_instance def __cinit__(self): self._instance = NULL def __dealloc__(self): mtbl_reader_destroy(&self._instance) def __init__(self, bytes fname, bool verify_checksums=False): cdef mtbl_reader_options *opt opt = mtbl_reader_options_init() mtbl_reader_options_set_verify_checksums(opt, verify_checksums) self._instance = mtbl_reader_init(fname, opt) mtbl_reader_options_destroy(&opt) if (self._instance == NULL): raise IOError("unable to open file: '%s'" % fname) cdef check_initialized(self): if self._instance == NULL: raise UninitializedException def iterkeys(self): """R.iterkeys() -> an iterator over the keys of R.""" self.check_initialized() return get_iterkeys(self, mtbl_source_iter(mtbl_reader_source(self._instance))) def itervalues(self): """R.itervalues() -> an iterator over the values of R.""" self.check_initialized() return get_itervalues(self, mtbl_source_iter(mtbl_reader_source(self._instance))) def iteritems(self): """R.iteritems() -> an iterator over the (key, value) items of R.""" self.check_initialized() return get_iteritems(self, mtbl_source_iter(mtbl_reader_source(self._instance))) def __contains__(self, bytes py_key): """R.__contains__(k) -> True if R has a key k, else False""" try: self.__getitem__(py_key) return True except KeyError: pass return False def has_key(self, bytes py_key): """R.has_key(k) -> True if R has a key k, else False.""" return self.__contains__(py_key) def get(self, bytes py_key, default=None): """R.get(k[,d]) -> R[k] if k in R, else d. d defaults to None.""" try: return self.__getitem__(py_key) except KeyError: pass return default def get_range(self, bytes py_key0, bytes py_key1): """ R.get_range(key0, key1) -> an iterator over all (key, value) items in R where key is between key0 and key1 inclusive. """ cdef mtbl_res res cdef uint8_t *key0 cdef uint8_t *key1 cdef size_t len_key0 cdef size_t len_key1 self.check_initialized() key0 = PyString_AsString(py_key0) key1 = PyString_AsString(py_key1) len_key0 = PyString_Size(py_key0) len_key1 = PyString_Size(py_key1) return get_iteritems(self, mtbl_source_get_range( mtbl_reader_source(self._instance), key0, len_key0, key1, len_key1)) def get_prefix(self, bytes py_key): """ R.get_prefix(key_prefix) -> an iterator over all (key, value) items in R where key begins with key_prefix. """ cdef mtbl_res res cdef uint8_t *key cdef size_t len_key self.check_initialized() key = PyString_AsString(py_key) len_key = PyString_Size(py_key) return get_iteritems(self, mtbl_source_get_prefix( mtbl_reader_source(self._instance), key, len_key)) def __getitem__(self, bytes py_key): cdef mtbl_iter *it cdef mtbl_res res cdef uint8_t *key cdef uint8_t *val cdef size_t len_key cdef size_t len_val self.check_initialized() key = PyString_AsString(py_key) len_key = PyString_Size(py_key) items = [] it = mtbl_source_get(mtbl_reader_source(self._instance), key, len_key) if it == NULL: raise KeyError(py_key) while True: res = mtbl_iter_next(it, &key, &len_key, &val, &len_val) if res == mtbl_res_failure: break items.append(PyString_FromStringAndSize( val, len_val)) mtbl_iter_destroy(&it) if not items: raise KeyError(py_key) return items cdef class writer(object): """ writer(fname) -> new MTBL writer, output to file fname Keyword arguments: compression -- compression type (default COMPRESSION_NONE) block_size -- maximum data block size in bytes (default 8192) block_restart_interval -- how frequently to restart key prefix compression (default 16) """ cdef mtbl_writer *_instance def __cinit__(self): self._instance = NULL def __dealloc__(self): mtbl_writer_destroy(&self._instance) def __init__(self, bytes fname, mtbl_compression_type compression=COMPRESSION_NONE, size_t block_size=8192, size_t block_restart_interval=16): if not (compression == COMPRESSION_NONE or compression == COMPRESSION_SNAPPY or compression == COMPRESSION_ZLIB): raise UnknownCompressionTypeException cdef mtbl_writer_options *opt opt = mtbl_writer_options_init() mtbl_writer_options_set_compression(opt, compression) mtbl_writer_options_set_block_size(opt, block_size) mtbl_writer_options_set_block_restart_interval(opt, block_restart_interval) self._instance = mtbl_writer_init(fname, opt) mtbl_writer_options_destroy(&opt) if self._instance == NULL: raise IOError("unable to initialize file: '%s'" % fname) def close(self): """W.close() -- finalize and close the writer""" mtbl_writer_destroy(&self._instance) def __setitem__(self, bytes py_key, bytes py_val): """ W.__setitem__(key, value) <==> W[key] = value Adds a new (key, value) entry to the writer. key and value must be byte strings, and key must be lexicographically greater than any previously written key. """ cdef mtbl_res res cdef uint8_t *key cdef uint8_t *val cdef size_t len_key cdef size_t len_val if self._instance == NULL: raise TableClosedException key = PyString_AsString(py_key) val = PyString_AsString(py_val) len_key = PyString_Size(py_key) len_val = PyString_Size(py_val) res = mtbl_writer_add(self._instance, key, len_key, val, len_val) if res == mtbl_res_failure: raise KeyOrderError def __delitem__(self, key): """will raise ImmutableError""" raise ImmutableError cdef void merge_func_wrapper(void *clos, uint8_t *key, size_t len_key, uint8_t *val0, uint8_t len_val0, uint8_t *val1, uint8_t len_val1, uint8_t **merged_val, size_t *len_merged_val) with gil: cdef str py_key cdef str py_val0 cdef str py_val1 cdef str py_merged_val py_key = PyString_FromStringAndSize( key, len_key) py_val0 = PyString_FromStringAndSize( val0, len_val0) py_val1 = PyString_FromStringAndSize( val1, len_val1) py_merged_val = ( clos)(py_key, py_val0, py_val1) len_merged_val[0] = PyString_Size(py_merged_val) merged_val[0] = malloc(len_merged_val[0]) memcpy(merged_val[0], PyString_AsString(py_merged_val), len_merged_val[0]) cdef class merger(object): """ merger(merge_func) -> new MTBL merger merge_func is the user-supplied value merging function: merge_func(key, val0, val1) -> merged_val all parameters are byte strings, and the return value must be a byte string. """ cdef mtbl_merger *_instance cdef set _references def __cinit__(self): self._instance = NULL def __dealloc__(self): mtbl_merger_destroy(&self._instance) def __init__(self, object merge_func): cdef mtbl_merger_options *opt opt = mtbl_merger_options_init() mtbl_merger_options_set_merge_func(opt, merge_func_wrapper, merge_func) self._instance = mtbl_merger_init(opt) mtbl_merger_options_destroy(&opt) self._references = set() def add_reader(self, reader r): """M.add_reader(mtbl.reader) -- add a reader object as a merge input""" mtbl_merger_add_source(self._instance, mtbl_reader_source(r._instance)) self._references.add(r) def write(self, writer w): """M.write(mtbl.writer) -- dump merged output to writer""" cdef mtbl_res res res = mtbl_source_write(mtbl_merger_source(self._instance), w._instance) if res != mtbl_res_success: raise RuntimeError def __iter__(self): return self.iterkeys() def iterkeys(self): """M.iterkeys() -> an iterator over the merged keys of M.""" return get_iterkeys(self, mtbl_source_iter(mtbl_merger_source(self._instance))) def itervalues(self): """M.itervalues() -> an iterator over the merged values of M.""" return get_itervalues(self, mtbl_source_iter(mtbl_merger_source(self._instance))) def iteritems(self): """M.iteritems() -> an iterator over the merged (key, value) items of M.""" return get_iteritems(self, mtbl_source_iter(mtbl_merger_source(self._instance))) def get(self, bytes py_key): """ M.get(key) -> an iterator over all (key, value) items in M which match key. """ cdef mtbl_res res cdef uint8_t *key cdef size_t len_key key = PyString_AsString(py_key) len_key = PyString_Size(py_key) return get_iteritems(self, mtbl_source_get(mtbl_merger_source(self._instance), key, len_key)) def get_range(self, bytes py_key0, bytes py_key1): """ M.get_range(key0, key1) -> an iterator over all (key, value) items in M where key is between key0 and key1 inclusive. """ cdef mtbl_res res cdef uint8_t *key0 cdef uint8_t *key1 cdef size_t len_key0 cdef size_t len_key1 key0 = PyString_AsString(py_key0) key1 = PyString_AsString(py_key1) len_key0 = PyString_Size(py_key0) len_key1 = PyString_Size(py_key1) return get_iteritems(self, mtbl_source_get_range( mtbl_merger_source(self._instance), key0, len_key0, key1, len_key1)) def get_prefix(self, bytes py_key): """ M.get_prefix(key_prefix) -> an iterator over all (key, value) items in M where key begins with key_prefix. """ cdef mtbl_res res cdef uint8_t *key cdef size_t len_key key = PyString_AsString(py_key) len_key = PyString_Size(py_key) return get_iteritems(self, mtbl_source_get_prefix( mtbl_merger_source(self._instance), key, len_key)) cdef class sorter(object): """ sorter(merge_func) -> new MTBL sorter merge_func is the user-supplied value merging function: merge_func(key, val0, val1) -> merged_val all parameters are byte strings, and the return value must be a byte string. Keyword arguments: temp_dir -- temporary directory (default "/var/tmp") max_memory -- maxmimum amount of memory for in-memory sorting in bytes (default 1 GB) """ cdef mtbl_sorter *_instance def __cinit__(self): self._instance = NULL def __dealloc__(self): mtbl_sorter_destroy(&self._instance) def __init__(self, object merge_func, bytes temp_dir=DEFAULT_SORTER_TEMP_DIR, size_t max_memory=DEFAULT_SORTER_MEMORY): cdef mtbl_sorter_options *opt opt = mtbl_sorter_options_init() mtbl_sorter_options_set_merge_func(opt, merge_func_wrapper, merge_func) mtbl_sorter_options_set_temp_dir(opt, temp_dir) mtbl_sorter_options_set_max_memory(opt, max_memory) self._instance = mtbl_sorter_init(opt) mtbl_sorter_options_destroy(&opt) def write(self, writer w): """S.write(mtbl.writer) -- dump sorted output to writer""" cdef mtbl_res res res = mtbl_sorter_write(self._instance, w._instance) if res != mtbl_res_success: raise RuntimeError def __setitem__(self, bytes py_key, bytes py_val): """ S.__setitem__(key, value) <==> S[key] = value Adds a new (key, value) item to the sorter. If the key already exists, the user-supplied merge function will be called to merge the conflicting values. """ cdef mtbl_res res cdef uint8_t *key cdef uint8_t *val cdef size_t len_key cdef size_t len_val if self._instance == NULL: raise RuntimeError key = PyString_AsString(py_key) val = PyString_AsString(py_val) len_key = PyString_Size(py_key) len_val = PyString_Size(py_val) res = mtbl_sorter_add(self._instance, key, len_key, val, len_val) if res == mtbl_res_failure: raise KeyOrderError def __delitem__(self, key): """will raise ImmutableError""" raise ImmutableError def __iter__(self): return self.iterkeys() def iterkeys(self): """S.iterkeys() -> an iterator over the sorted keys of R.""" return get_iterkeys(self, mtbl_sorter_iter(self._instance)) def itervalues(self): """S.itervalues() -> an iterator over the sorted values of R.""" return get_itervalues(self, mtbl_sorter_iter(self._instance)) def iteritems(self): """S.iteritems() -> an iterator over the sorted (key, value) items of R.""" return get_iteritems(self, mtbl_sorter_iter(self._instance)) pymtbl-0.2/setup.py000077500000000000000000000011561226760011500144020ustar00rootroot00000000000000#!/usr/bin/env python NAME = 'pymtbl' VERSION = '0.2' from distutils.core import setup from distutils.extension import Extension import os try: from Cython.Distutils import build_ext setup( name = NAME, version = VERSION, ext_modules = [ Extension('mtbl', ['mtbl.pyx'], libraries = ['mtbl']) ], cmdclass = {'build_ext': build_ext}, ) except ImportError: if os.path.isfile('mtbl.c'): setup( name = NAME, version = VERSION, ext_modules = [ Extension('mtbl', ['mtbl.c'], libraries = ['mtbl']) ], ) else: raise