pax_global_header00006660000000000000000000000064122730276040014515gustar00rootroot0000000000000052 comment=ac49d30baa29e0ee0445528356894c83a8645ebe mtbl-0.5/000077500000000000000000000000001227302760400123175ustar00rootroot00000000000000mtbl-0.5/.gitignore000066400000000000000000000002521227302760400143060ustar00rootroot00000000000000.*swp *.la *.lo *.log *.o *.tar.gz *.trs .deps/ .dirstamp .libs/ /aclocal.m4 /autom4te.cache /build-aux /config.* /configure /libtool /stamp-h1 Makefile Makefile.in TAGS mtbl-0.5/.travis.yml000066400000000000000000000003101227302760400144220ustar00rootroot00000000000000language: c before_install: - sudo apt-get update -qq install: - sudo apt-get -q install libsnappy-dev zlib1g-dev script: - ./autogen.sh - ./configure - make - make distcheck VERBOSE=1 mtbl-0.5/COPYRIGHT000066400000000000000000000050701227302760400136140ustar00rootroot00000000000000Copyright (c) 2012-2014 by Farsight Security, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Copyright (c) 2011 The LevelDB Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. Copyright 2008, Dave Benson. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. mtbl-0.5/ChangeLog000066400000000000000000000042261227302760400140750ustar00rootroot00000000000000mtbl (0.5) * The COPYRIGHT, LICENSE, and README.md files are now distributed in the tarball. These files were inadvertently not included in the previous release. * The build system now properly detects big endian architectures. The previous release inadvertently omitted this check, causing a test suite failure. * Verify that the length of the MTBL input is long enough to read the MTBL header block. This prevents invalid reads in mtbl_reader_init_fd(). See Debian bug #716628 for details. (http://bugs.debian.org/716628). * Make sure to install the manpages if they are available even if the manpages are not being rebuilt. -- Robert Edmonds Fri, 31 Jan 2014 18:01:13 -0500 mtbl (0.4) * Update copyright and license statements as a result of the transition from Internet Systems Consortium to Farsight Security. * Replace the "librsf" submodule with the "libmy" subtree. This includes a faster CRC32C implementation on supported x86-64 CPUs. * mtbl_merge(1): rename the internal 'timespec_get' function, since this function name is used by ISO C11 and causes build failures on glibc >= 2.17. * mtbl_reader(3), mtbl_fileset(3): add missing assertions. * Install the pkg-config .pc file into the correct location on certain systems. -- Robert Edmonds Tue, 21 Jan 2014 15:46:00 -0500 mtbl (0.3) * mtbl_dump(1): print error messages instead of assertion failures. * mtbl_merger(3): don't assert on NULL iterator. * mtbl_merger(3): avoid incorrectly outputing empty entries. * mtbl_merger(3): buffer concatenation optimization. -- Robert Edmonds Wed Dec 26 17:19:53 2012 -0500 mtbl (0.2) * Add mtbl_fileset(3) interface. * mtbl_merger(3): handle corner case where iterator is NULL after initial entry fill. * mtbl_merge(1): set MTBL block size of output via MTBL_MERGE_BLOCK_SIZE environment variable. * Refactor using common functions in librsf submodule. -- Robert Edmonds Wed, 23 May 2012 16:03:16 -0400 mtbl (0.1) * Initial release. -- Robert Edmonds Fri, 24 Feb 2012 19:05:54 -0500 mtbl-0.5/LICENSE000066400000000000000000000261361227302760400133340ustar00rootroot00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. mtbl-0.5/Makefile.am000066400000000000000000000071431227302760400143600ustar00rootroot00000000000000AUTOMAKE_OPTIONS = parallel-tests check_PROGRAMS = bin_PROGRAMS = TESTS = EXTRA_DIST = COPYRIGHT LICENSE README.md CLEANFILES = ACLOCAL_AMFLAGS = -I m4 ${ACLOCAL_FLAGS} AM_CPPFLAGS = \ -include $(top_builddir)/config.h \ -I${top_srcdir}/mtbl AM_CFLAGS = ${my_CFLAGS} AM_LDFLAGS = -Wl,--as-needed # ## ### library ## # LIBMTBL_VERSION_INFO=0:0:0 include_HEADERS = mtbl/mtbl.h lib_LTLIBRARIES = mtbl/libmtbl.la mtbl_libmtbl_la_SOURCES = \ libmy/crc32c.c libmy/crc32c.h libmy/crc32c-slicing.c libmy/crc32c-sse42.c \ libmy/heap.c libmy/heap.h \ libmy/my_alloc.h \ libmy/my_fileset.c libmy/my_fileset.h \ libmy/ubuf.h \ libmy/vector.h \ mtbl/block.c \ mtbl/block_builder.c \ mtbl/bytes.h \ mtbl/crc32c_wrap.c \ mtbl/fileset.c \ mtbl/fixed.c \ mtbl/iter.c \ mtbl/merger.c \ mtbl/mtbl.h \ mtbl/mtbl-private.h \ mtbl/reader.c \ mtbl/sorter.c \ mtbl/source.c \ mtbl/trailer.c \ mtbl/varint.c \ mtbl/writer.c mtbl_libmtbl_la_LIBADD = -lsnappy -lz mtbl_libmtbl_la_LDFLAGS = $(AM_LDFLAGS) \ -version-info $(LIBMTBL_VERSION_INFO) \ -export-symbols-regex "^(mtbl_[a-z].*)" pkgconfig_DATA = mtbl/libmtbl.pc EXTRA_DIST += mtbl/libmtbl.pc.in CLEANFILES += mtbl/libmtbl.pc # ## ### programs ## # bin_PROGRAMS += src/mtbl_dump src_mtbl_dump_SOURCES = src/mtbl_dump.c libmy/print_string.h src_mtbl_dump_LDADD = mtbl/libmtbl.la bin_PROGRAMS += src/mtbl_info src_mtbl_info_SOURCES = src/mtbl_info.c src_mtbl_info_LDADD = mtbl/libmtbl.la bin_PROGRAMS += src/mtbl_merge src_mtbl_merge_SOURCES = src/mtbl_merge.c libmy/getenv_int.h src_mtbl_merge_LDADD = mtbl/libmtbl.la # ## ### tests ## # AM_TESTS_ENVIRONMENT = top_builddir='$(top_builddir)'; top_srcdir='$(top_srcdir)'; export top_builddir top_srcdir; TESTS_ENVIRONMENT = $(AM_TESTS_ENVIRONMENT) TESTS += t/test-block_builder check_PROGRAMS += t/test-block_builder t_test_block_builder_SOURCES = t/test-block_builder.c t_test_block_builder_LDADD = mtbl/libmtbl.la TESTS += t/test-crc32c check_PROGRAMS += t/test-crc32c t_test_crc32c_SOURCES = t/test-crc32c.c t_test_crc32c_LDADD = mtbl/libmtbl.la TESTS += t/test-fixed check_PROGRAMS += t/test-fixed t_test_fixed_SOURCES = t/test-fixed.c t_test_fixed_LDADD = mtbl/libmtbl.la TESTS += t/test-trailer check_PROGRAMS += t/test-trailer t_test_trailer_SOURCES = t/test-trailer.c t_test_trailer_LDADD = mtbl/libmtbl.la TESTS += t/test-varint check_PROGRAMS += t/test-varint t_test_varint_SOURCES = t/test-varint.c t_test_varint_LDADD = mtbl/libmtbl.la TESTS += t/test-vector check_PROGRAMS += t/test-vector t_test_vector_SOURCES = t/test-vector.c t_test_vector_LDADD = mtbl/libmtbl.la TESTS += t/test-deb716628.sh EXTRA_DIST += t/test-deb716628.sh EXTRA_DIST += t/test-deb716628.data t/test-deb716628.sh: src/mtbl_dump # ## ### documentation ## # if BUILD_MAN SUFFIXES = .1.txt .3.txt .7.txt .1 .3 .7 ASCIIDOC_PROCESS = $(AM_V_GEN) $(ASCIIDOC) -f manpage --no-xmllint --asciidoc-opt="-f man/asciidoc.conf" $< .1.txt.1: $(ASCIIDOC_PROCESS) .3.txt.3: $(ASCIIDOC_PROCESS) .7.txt.7: $(ASCIIDOC_PROCESS) endif dist_man_MANS = \ man/mtbl_dump.1 \ man/mtbl_info.1 \ man/mtbl_merge.1 \ man/mtbl_iter.3 \ man/mtbl_source.3 \ man/mtbl_reader.3 \ man/mtbl_writer.3 \ man/mtbl_merger.3 \ man/mtbl_sorter.3 \ man/mtbl_fileset.3 \ man/mtbl_crc32c.3 \ man/mtbl_fixed.3 \ man/mtbl_varint.3 \ man/mtbl.7 EXTRA_DIST += \ man/mtbl_dump.1.txt \ man/mtbl_info.1.txt \ man/mtbl_merge.1.txt \ man/mtbl_iter.3.txt \ man/mtbl_source.3.txt \ man/mtbl_reader.3.txt \ man/mtbl_writer.3.txt \ man/mtbl_merger.3.txt \ man/mtbl_sorter.3.txt \ man/mtbl_fileset.3.txt \ man/mtbl_crc32c.3.txt \ man/mtbl_fixed.3.txt \ man/mtbl_varint.3.txt \ man/mtbl.7.txt mtbl-0.5/README.md000066400000000000000000000053551227302760400136060ustar00rootroot00000000000000[![Build Status](https://travis-ci.org/farsightsec/mtbl.png?branch=master)](https://travis-ci.org/farsightsec/mtbl) `mtbl`: immutable sorted string table library ============================================= Introduction ------------ `mtbl` is a C library implementation of the Sorted String Table (SSTable) data structure, based on the SSTable implementation in the open source [Google LevelDB library](http://code.google.com/p/leveldb/). An SSTable is a file containing an immutable mapping of keys to values. Keys are stored in sorted order, with an index at the end of the file allowing keys to be located quickly. `mtbl` is not a database library. It does not provide an updateable key-value data store, but rather exposes primitives for creating, searching and merging SSTable files. Unlike databases which use the SSTable data structure internally as part of their data store, management of SSTable files -- creation, merging, deletion, combining of search results from multiple SSTables -- is left to the discretion of the `mtbl` library user. `mtbl` SSTable files consist of a sequence of data blocks containing sorted key-value pairs, where keys and values are arbitrary byte arrays. Data blocks are optionally compressed using `zlib` or the [Snappy library](http://code.google.com/p/snappy/). The data blocks are followed by an index block, allowing for fast searches over the keyspace. The basic `mtbl` interface is the writer, which receives a sequence of key-value pairs in sorted order with no duplicate keys, and writes them to data blocks in the SSTable output file. An index containing offsets to data blocks and the last key in each data block is buffered in memory until the writer object is closed, at which point the index is written to the end of the SSTable file. This allows SSTable files to be written in a single pass with sequential I/O operations only. Once written, SSTable files can be searched using the `mtbl` reader interface. Searches can retrieve key-value pairs based on an exact key match, a key prefix match, or a key range. Results are retrieved using a simple iterator interface. The `mtbl` library also provides two utility interfaces which facilitate a sort-and-merge workflow for bulk data loading. The sorter interface receives arbitrarily ordered key-value pairs and provides them in sorted order, buffering to disk as needed. The merger interface reads from multiple SSTables simultaneously and provides the key-value pairs from the combined inputs in sorted order. Since `mtbl` does not allow duplicate keys in an SSTable file, both the sorter and merger interfaces require a caller-provided merge function which will be called to merge multiple values for the same key. These interfaces also make use of sequential I/O operations only. mtbl-0.5/autogen.sh000077500000000000000000000000371227302760400143200ustar00rootroot00000000000000#!/bin/sh exec autoreconf -fvi mtbl-0.5/configure.ac000066400000000000000000000044151227302760400146110ustar00rootroot00000000000000AC_PREREQ(2.60) AC_INIT([mtbl], [0.5]) AC_CONFIG_SRCDIR([mtbl/mtbl.h]) AC_CONFIG_AUX_DIR([build-aux]) AM_INIT_AUTOMAKE([foreign 1.11 -Wall -Wno-portability silent-rules subdir-objects color-tests]) AC_PROG_CC_STDC AC_USE_SYSTEM_EXTENSIONS AC_SYS_LARGEFILE AC_CONFIG_MACRO_DIR([m4]) AM_SILENT_RULES([yes]) LT_INIT my_CFLAGS="-Wall \ -Wmissing-declarations -Wmissing-prototypes \ -Wnested-externs -Wpointer-arith \ -Wpointer-arith -Wsign-compare -Wchar-subscripts \ -Wstrict-prototypes -Wshadow \ -Wformat-security" AC_SUBST([my_CFLAGS]) AC_CONFIG_HEADERS(config.h) AC_CONFIG_FILES([Makefile mtbl/libmtbl.pc]) PKG_PROG_PKG_CONFIG if test -n "$PKG_CONFIG"; then # Horrible hack for systems where the pkg-config install directory is simply wrong! if $PKG_CONFIG --variable=pc_path pkg-config 2>/dev/null | grep -q /libdata/; then PKG_INSTALLDIR(['${prefix}/libdata/pkgconfig']) else PKG_INSTALLDIR fi else AC_MSG_ERROR([pkg-config is required!]) fi AC_C_BIGENDIAN AC_CHECK_FUNC([mkstemp], [], [ AC_MSG_ERROR([required system function not found]) ]) AC_CHECK_HEADERS([sys/endian.h endian.h]) AC_CHECK_HEADER([snappy-c.h], [], [ AC_MSG_ERROR([required header file not found]) ]) AC_CHECK_LIB([snappy], [snappy_compress], [], [ AC_MSG_ERROR([required library not found]) ]) AC_CHECK_HEADER([zlib.h], [], [ AC_MSG_ERROR([required header file not found]) ]) AC_CHECK_LIB([z], [deflate], [], [ AC_MSG_ERROR([required library not found]) ]) AC_SEARCH_LIBS([dlopen], [dl]) AC_SEARCH_LIBS([clock_gettime], [rt]) AC_PATH_PROG([ASCIIDOC], [a2x]) AM_CONDITIONAL([BUILD_MAN], [test -n "$ASCIIDOC"]) if test -n "$ASCIIDOC"; then DOC_MAN_MSG="yes (asciidoc available)" else DOC_MAN_MSG="no (asciidoc not available)" fi AC_OUTPUT AC_MSG_RESULT([ $PACKAGE $VERSION compiler: ${CC} cflags: ${CFLAGS} ldflags: ${LDFLAGS} libs: ${LIBS} prefix: ${prefix} sysconfdir: ${sysconfdir} libdir: ${libdir} includedir: ${includedir} pkgconfigdir: ${pkgconfigdir} bigendian: ${ac_cv_c_bigendian} building manpage docs: ${DOC_MAN_MSG} ]) mtbl-0.5/libmy/000077500000000000000000000000001227302760400134335ustar00rootroot00000000000000mtbl-0.5/libmy/.gitignore000066400000000000000000000000471227302760400154240ustar00rootroot00000000000000*.lo *.o .*swp .deps/ .dirstamp .libs/ mtbl-0.5/libmy/COPYRIGHT000066400000000000000000000010771227302760400147330ustar00rootroot00000000000000Copyright (c) 2012-2014 by Farsight Security, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. mtbl-0.5/libmy/LICENSE000066400000000000000000000261361227302760400144500ustar00rootroot00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. mtbl-0.5/libmy/argv.c000066400000000000000000002515251227302760400145500ustar00rootroot00000000000000/* * Generic argv processor... * * Copyright 2000 by Gray Watson * * This file is part of the argv library. * * Permission to use, copy, modify, and distribute this software for * any purpose and without fee is hereby granted, provided that the * above copyright notice and this permission notice appear in all * copies, and that the name of Gray Watson not be used in advertising * or publicity pertaining to distribution of the document or software * without specific, written prior permission. * * Gray Watson makes no representations about the suitability of the * software described herein for any purpose. It is provided "as is" * without express or implied warranty. * * The author may be contacted via http://256.com/gray/ * * $Id: argv.c,v 1.111 2010/02/15 13:34:25 gray Exp $ */ #include #include #include #include #include "argv.h" #include "argv_loc.h" /* internal routines -- we need this because of recusion */ static void do_list(argv_t *grid, const int arg_c, char **argv, argv_t **queue_list, int *queue_head_p, int *queue_tail_p, int *okay_bp); #define LOC_SNPRINTF (void)snprintf #define SNP_ARG(buf, buf_size) buf, buf_size /* * exported variables */ /* This is a processed version of argv[0], pre-path removed: /bin/ls -> ls */ char argv_program[PROGRAM_NAME + 1] = "Unknown"; /* A global value of argv from main after argv_process has been called */ char **argv_argv = NULL; /* A global value of argc from main after argv_process has been called */ int argv_argc = 0; /* This should be set externally to provide general program help to user */ char *argv_help_string = NULL; /* This should be set externally to provide version information to the user */ char *argv_version_string = NULL; /* * Are we running interactively? This will exit on errors. Set to * false to return error codes instead. */ int argv_interactive = ARGV_TRUE; /* * The FILE stream that argv out_puts all its errors. Set to NULL to * not dump any error messages. Default is stderr. */ FILE *argv_error_stream = ERROR_STREAM_INIT; /* * This is the error code to exit with when we have a usage error and * we are in interactive mode. */ int argv_error_code = EXIT_CODE; /* * global settings */ /* * Set to 1 (the default) to enable the handling of -l=foo or * --logfile=foo type of arguments. Set to 0 to disable. This allows * you to specifically assign a value to an argument. */ int argv_close_enable_b = 1; /* * If the library sees a "--" argument, it will turn off further * argument process. Set to 1 to enable the ability of specifying * additional "--" arguments to reenable (basically toggle on then * off) argument processing. Set to 0 (the default) to disable this * behavior. */ int argv_last_toggle_b = 0; /* * Set to 1 (the default) to have the library accept multiple usage of * the same argument. Set to 0 to have the library generate an error * if you use an argument twice. */ int argv_multi_accept_b = 1; /* * Set to one of the ARGV_USAGE_ defines in the argv.h file. This * tell the library what usage information to display when --usage is * specified by the user. Default is ARGV_USAGE_LONG. */ int argv_usage_type = ARGV_USAGE_LONG; /* * Set to one of the ARGV_USAGE_ defines in the argv.h file. This * tell the library what usage information to display when an error is * encountered. The usage information accompanies the error message. * Default is ARGV_USAGE_SEE. */ int argv_error_type = ARGV_USAGE_LONG; /* * Set to 1 (the default) if you want the library look for associated * arguments from the associated program's environmental variable. If * set the 0 then no environmental variable will be used. If you are * running program foo then the library will look for the * environmental variable ARGV_foo and will add those to the argument * list specified on the command line. By default they will be * inserted in front of those on the command line unless the * argv_env_after_b is set to 1. * * NOTE: this is set by argv_process automatically. If you do not * want this behavior, you should use argv_process_no_env. */ int argv_process_env_b = 1; /* * Set to 1 if you want the library to append the arguments from the * program's environmental variable after those specified on the * command line. If set the 0 (the default) then they will be * inserted before those specified on the command line. See * argv_process_env_b for more information. */ int argv_env_after_b = 0; /* * local variables */ /* empty argument array */ static argv_t empty[] = {{ ARGV_LAST, NULL, 0, NULL, NULL, NULL }}; static int enabled_b = ARGV_FALSE; /* are the lights on? */ /****************************** startup routine ******************************/ /* * static void argv_startup * * DESCRIPTION: * * Turn on the lights. * * RETURNS: * * None. * * ARGUMENTS: * * None. */ static void argv_startup(void) { if (enabled_b) { return; } enabled_b = ARGV_TRUE; /* ANSI says we cannot predefine this above */ if (argv_error_stream == ERROR_STREAM_INIT) { argv_error_stream = stderr; } } /***************************** general utilities *****************************/ /* * static int btoi * * DESCRIPTION: * * Binary string to integer translation. * * RETURNS: * * Integer converted from the string. * * ARGUMENTS: * * str - String of binary 0s and 1s that we are converting. */ static int btoi(const char *str) { int ret = 0; /* strip off spaces */ for (; isspace((unsigned char) *str); str++) { } for (; *str == '0' || *str == '1'; str++) { ret *= 2; ret += *str - '0'; } return ret; } /* * static int otoi * * DESCRIPTION: * * Octal string to integer translation. * * RETURNS: * * Integer converted from the string. * * ARGUMENTS: * * str - String of octal digits that we are converting. */ static int otoi(const char *str) { int ret = 0; /* strip off spaces */ for (; isspace((unsigned char) *str); str++) { } for (; *str >= '0' && *str <= '7'; str++) { ret *= 8; ret += *str - '0'; } return ret; } /* * static int htoi * * DESCRIPTION: * * Hexadecimal string to integer translation. * * RETURNS: * * Integer converted from the string. * * ARGUMENTS: * * str - String of hexadecimal characters and digits that we are * converting. */ static int htoi(const char *str) { int ret = 0; /* strip off spaces */ for (; isspace((unsigned char) *str); str++) { } /* skip a leading 0[xX] */ if (*str == '0' && (*(str + 1) == 'x' || *(str + 1) == 'X')) { str += 2; } for (; isdigit((unsigned char) *str) || (*str >= 'a' && *str <= 'f') || (*str >= 'A' && *str <= 'F'); str++) { ret *= 16; if (*str >= 'a' && *str <= 'f') { ret += *str - 'a' + 10; } else if (*str >= 'A' && *str <= 'F') { ret += *str - 'A' + 10; } else { ret += *str - '0'; } } return ret; } /* * static char *string_copy * * DESCRIPTION: * * Basically a strdup for compatibility sake. * * RETURNS: * * Character pointer that must be freed later. * * ARGUMENTS: * * str - String we are copying. */ static char *string_copy(const char *str) { const char *str_p; char *copy, *copy_p; int len; len = strlen(str); copy = (char *)malloc(len + 1); if (copy == NULL) { if (argv_error_stream != NULL) { (void)fprintf(argv_error_stream, "%s: memory error during argument processing\n", argv_program); } if (argv_interactive) { (void)exit(argv_error_code); } return NULL; } for (str_p = str, copy_p = copy; *str_p != '\0';) { *copy_p++ = *str_p++; } *copy_p = '\0'; return copy; } /* * static char **vectorize * * DESCRIPTION: * * Break a string up into its arguments separated by one of the * characters in a token string and return an array of char pointers. * * NOTE: the string argument should stay around until that time. * * RETURNS: * * Success - Allocated list of character poiners into the string * argument which must be freed later. * * Failure - NULL * * ARGUMENTS: * * str - String we are tokenizing. * * tok - List of token characters to look for in the string. * * num_tok_p - Pointer to an integer which will be set to the number * of tokens found in the string. */ static char **vectorize(char *str, const char *tok, int *num_tok_p) { char **vect_p; char *tmp, *str_p, *tok_p; int tok_c, tok_n; /* count the tokens */ tmp = string_copy(str); if (tmp == NULL) { return NULL; } str_p = tmp; tok_c = 0; while (1) { tok_p = strsep(&str_p, tok); if (tok_p == NULL) { break; } /* skip empty tokens */ if (*tok_p != '\0') { tok_c++; } } tok_n = tok_c; free(tmp); *num_tok_p = tok_n; if (tok_c == 0) { return NULL; } /* allocate the pointer grid */ vect_p = (char **)malloc(sizeof(char *) * tok_n); if (vect_p == NULL) { if (argv_error_stream != NULL) { (void)fprintf(argv_error_stream, "%s: memory error during argument processing\n", argv_program); } if (argv_interactive) { (void)exit(argv_error_code); } return NULL; } /* load the tokens into the list */ str_p = str; for (tok_c = 0; tok_c < tok_n;) { tok_p = strsep(&str_p, tok); if (tok_p == NULL) { break; } /* skip empty tokens */ if (*tok_p != '\0') { vect_p[tok_c] = tok_p; tok_c++; } } return vect_p; } /* * static int expand_buf * * DESCRIPTION: * * Translates a buffer of bytes into its printable version. * * NOTE: it does _not_ add a \0 at the end of OUT. * * RETURNS: * * Number of characters written in to the output buffer. * * ARGUMENTS: * * buf - Input buffer of bytes. * * buf_size - Size of the input buffer. If < 0 then the routing will * translate up to the first \0. * * out - Output buffer for the translated characters. * * out_size - Maximum size of the output buffer. */ static int expand_buf(const void *buf, const int buf_size, char *out, const int out_size) { int buf_c; const unsigned char *buf_p, *spec_p; char *max_p, *out_p = out; /* setup our max pointer */ max_p = out + out_size; /* run through the input buffer, counting the characters as we go */ for (buf_c = 0, buf_p = (const unsigned char *)buf;; buf_c++, buf_p++) { /* did we reach the end of the buffer? */ if (buf_size < 0) { if (*buf_p == '\0') { break; } } else { if (buf_c >= buf_size) { break; } } /* search for special characters */ for (spec_p = (unsigned char *)SPECIAL_CHARS + 1; *(spec_p - 1) != '\0'; spec_p += 2) { if (*spec_p == *buf_p) { break; } } /* did we find one? */ if (*(spec_p - 1) != '\0') { if (out_p + 2 >= max_p) { break; } LOC_SNPRINTF(SNP_ARG(out_p, 2), "\\%c", *(spec_p - 1)); out_p += 2; continue; } /* print out any 7-bit printable characters */ if (*buf_p < 128 && isprint(*buf_p)) { if (out_p + 1 >= max_p) { break; } *out_p = *(char *)buf_p; out_p += 1; } else { if (out_p + 4 >= max_p) { break; } LOC_SNPRINTF(SNP_ARG(out_p, 4), "\\%03o", *buf_p); out_p += 4; } } return out_p - out; } /****************************** usage routines *******************************/ /* * static void usage_short * * DESCRIPTION: * * Print a short-format usage message. * * RETURNS: * * None. * * ARGUMENTS: * * args - Array of argv_t structions whose usage messages you print. * * flags - User flags. */ static void usage_short(const argv_t *args, const int flag) { const argv_t *arg_p; int len, col_c = 0; int mark_b = ARGV_FALSE; const char *prefix; if (argv_error_stream == NULL) { return; } /* print the usage message header */ (void)fprintf(argv_error_stream, "%s%s", USAGE_LABEL, argv_program); col_c += USAGE_LABEL_LENGTH + strlen(argv_program); /* * print all of the boolean arguments first. * NOTE: we assume they all fit on the line */ for (arg_p = args; arg_p->ar_short_arg != ARGV_LAST; arg_p++) { /* skip or-specifiers */ if (arg_p->ar_short_arg == ARGV_OR || arg_p->ar_short_arg == ARGV_XOR) { continue; } /* skip non booleans */ if (HAS_ARG(arg_p->ar_type)) { continue; } /* skip args with no short component */ if (arg_p->ar_short_arg == '\0') { continue; } if (! mark_b) { len = 2 + SHORT_PREFIX_LENGTH; prefix = " ["; /* we check for -2 here because we should have 1 arg and ] on line */ if (col_c + len > SCREEN_WIDTH - 2) { (void)fprintf(argv_error_stream, "\n%*.*s", (int)USAGE_LABEL_LENGTH, (int)USAGE_LABEL_LENGTH, ""); col_c = USAGE_LABEL_LENGTH; /* if we are the start of a line, skip any starting spaces */ if (*prefix == ' ') { prefix++; len--; } } (void)fprintf(argv_error_stream, "%s%s", prefix, SHORT_PREFIX); col_c += len; mark_b = ARGV_TRUE; } len = 1; /* we check for -1 here because we should need ] */ if (col_c + len > SCREEN_WIDTH - 1) { (void)fprintf(argv_error_stream, "]\n%*.*s", (int)USAGE_LABEL_LENGTH, (int)USAGE_LABEL_LENGTH, ""); col_c = USAGE_LABEL_LENGTH; /* restart the short option list */ (void)fprintf(argv_error_stream, "[%s", SHORT_PREFIX); col_c += 1 + SHORT_PREFIX_LENGTH; } (void)fprintf(argv_error_stream, "%c", arg_p->ar_short_arg); col_c++; } if (mark_b) { (void)fprintf(argv_error_stream, "]"); col_c++; } /* print remaining (non-boolean) arguments */ for (arg_p = args; arg_p->ar_short_arg != ARGV_LAST; arg_p++) { int var_len; const char *var_str, *postfix; /* skip or-specifiers */ if (arg_p->ar_short_arg == ARGV_OR || arg_p->ar_short_arg == ARGV_XOR) { continue; } /* skip booleans types */ if (! HAS_ARG(arg_p->ar_type)) { continue; } if (arg_p->ar_var_label == NULL) { if (ARGV_TYPE(arg_p->ar_type) == ARGV_BOOL_ARG || ARGV_TYPE(arg_p->ar_type) == ARGV_BOOL_INT_ARG) { var_str = BOOL_ARG_LABEL; var_len = BOOL_ARG_LENGTH; } else { var_len = UNKNOWN_ARG_LENGTH; var_str = UNKNOWN_ARG; } } else { var_len = strlen(arg_p->ar_var_label); var_str = arg_p->ar_var_label; } if (arg_p->ar_short_arg == ARGV_MAND) { /* print the mandatory argument desc */ len = 1 + var_len; prefix = " "; postfix = ""; } else if (arg_p->ar_short_arg == ARGV_MAYBE) { /* print the maybe argument desc */ len = 2 + var_len + 1; prefix = " ["; postfix = "]"; } else { /* handle options with arguments */ /* " [" + short_prefix + char */ len = 2 + SHORT_PREFIX_LENGTH + 1; prefix = " ["; /* do we need to wrap */ if (col_c + len > SCREEN_WIDTH) { (void)fprintf(argv_error_stream, "\n%*.*s", (int)USAGE_LABEL_LENGTH, (int)USAGE_LABEL_LENGTH, ""); col_c = USAGE_LABEL_LENGTH; /* if we are the start of a line, skip any starting spaces */ if (*prefix == ' ') { prefix++; len--; } } (void)fprintf(argv_error_stream, "%s%s%c", prefix, SHORT_PREFIX, arg_p->ar_short_arg); col_c += len; len = 1 + var_len + 1; prefix = " "; postfix = "]"; } if (col_c + len > SCREEN_WIDTH) { (void)fprintf(argv_error_stream, "\n%*.*s", (int)USAGE_LABEL_LENGTH, (int)USAGE_LABEL_LENGTH, ""); col_c = USAGE_LABEL_LENGTH; /* if we are the start of a line, skip any starting spaces */ if (*prefix == ' ') { prefix++; len--; } } (void)fprintf(argv_error_stream, "%s%s%s", prefix, var_str, postfix); col_c += len; } (void)fprintf(argv_error_stream, "\n"); if (flag == ARGV_USAGE_SHORT_REM) { (void)fprintf(argv_error_stream, "%*.*sUse the '%s%s' argument for more assistance.\n", (int)USAGE_LABEL_LENGTH, (int)USAGE_LABEL_LENGTH, "", LONG_PREFIX, USAGE_ARG); } } /* * static void display_arg * * DESCRIPTION: * * Display an argument type while keeping track of the column we are * in. * * RETURNS: * * None. * * ARGUMENTS: * * stream - Output stream we are writing to. * * arg_p - Argument that we are displaying. * * max - Maximum column position to write to. * * col_cp - Pointer to an integer to record the column position. */ static void display_arg(FILE *stream, const argv_t *arg_p, const int max, int *col_cp) { int var_len, len; if (arg_p->ar_var_label == NULL) { var_len = 0; } else { var_len = strlen(arg_p->ar_var_label); } switch (ARGV_TYPE(arg_p->ar_type)) { case ARGV_BOOL: case ARGV_BOOL_NEG: case ARGV_INCR: case ARGV_BOOL_INT: case ARGV_BOOL_INT_NEG: break; case ARGV_BOOL_ARG: case ARGV_BOOL_INT_ARG: (void)fprintf(stream, "%s", BOOL_ARG_LABEL); (*col_cp) += BOOL_ARG_LENGTH; break; case ARGV_CHAR: case ARGV_CHAR_P: case ARGV_SHORT: case ARGV_U_SHORT: case ARGV_INT: case ARGV_U_INT: case ARGV_LONG: case ARGV_U_LONG: case ARGV_FLOAT: case ARGV_DOUBLE: case ARGV_BIN: case ARGV_OCT: case ARGV_HEX: case ARGV_SIZE: case ARGV_U_SIZE: if (arg_p->ar_var_label == NULL) { len = max - *col_cp; (void)fprintf(stream, "%-.*s", len, UNKNOWN_ARG); *col_cp += MIN(len, (int)UNKNOWN_ARG_LENGTH); } else { len = max - *col_cp; (void)fprintf(stream, "%-.*s", len, arg_p->ar_var_label); *col_cp += MIN(len, var_len); } break; } } /* * static void display_option * * DESCRIPTION: * * Display an option entry while while keeping track of the column we * are in. * * RETURNS: * * None. * * ARGUMENTS: * * stream - Output stream we are writing to. * * arg_p - Argument that we are displaying. * * max - Maximum column position to write to. * * col_cp - Pointer to an integer to record the column position. */ static void display_option(FILE *stream, const argv_t *arg_p, int *col_cp) { if (stream == NULL) { return; } (void)fputc('[', stream); (*col_cp)++; /* arg maybe does not have a -? preface */ if (arg_p->ar_short_arg != ARGV_MAYBE) { (void)fprintf(stream, "%s%c", SHORT_PREFIX, arg_p->ar_short_arg); *col_cp += SHORT_PREFIX_LENGTH + 1; if (HAS_ARG(arg_p->ar_type)) { /* display optional argument */ (void)fputc(' ', stream); (*col_cp)++; } } display_arg(stream, arg_p, LONG_COLUMN - 1, col_cp); (void)fputc(']', stream); (*col_cp)++; } /* * static void usage_long * * DESCRIPTION: * * Print a long-format usage message. * * RETURNS: * * None. * * ars - Array of argv_t structures whose usage we are printing. */ static void usage_long(const argv_t *args) { const argv_t *arg_p; int col_c, len; if (argv_error_stream == NULL) { return; } /* print the usage message header */ (void)fprintf(argv_error_stream, "%s%s\n", USAGE_LABEL, argv_program); /* run through the argument structure */ for (arg_p = args; arg_p->ar_short_arg != ARGV_LAST; arg_p++) { /* skip or specifiers */ if (arg_p->ar_short_arg == ARGV_OR || arg_p->ar_short_arg == ARGV_XOR) { continue; } /* indent to the short-option col_c */ (void)fprintf(argv_error_stream, "%*.*s", SHORT_COLUMN, SHORT_COLUMN, ""); /* start column counter */ col_c = SHORT_COLUMN; /* print the short-arg stuff if there */ if (arg_p->ar_short_arg == '\0') { (void)fputc('[', argv_error_stream); col_c++; } else { if (arg_p->ar_short_arg == '\0') { ; } else if (arg_p->ar_short_arg == ARGV_MAND) { display_arg(argv_error_stream, arg_p, COMMENT_COLUMN, &col_c); } else { /* ARGV_MAYBE handled here */ display_option(argv_error_stream, arg_p, &col_c); } /* put the long-option message on the correct column */ if (col_c < LONG_COLUMN) { (void)fprintf(argv_error_stream, "%*.*s", LONG_COLUMN - col_c, LONG_COLUMN - col_c, ""); col_c = LONG_COLUMN; } } /* print the long-option message */ if (arg_p->ar_long_arg != NULL) { len = COMMENT_COLUMN - col_c - (LONG_PREFIX_LENGTH + 1); if (arg_p->ar_short_arg != '\0') { (void)fprintf(argv_error_stream, "%s", LONG_LABEL); col_c += LONG_LABEL_LENGTH; len -= LONG_LABEL_LENGTH; } (void)fprintf(argv_error_stream, "%s%-.*s", LONG_PREFIX, len, arg_p->ar_long_arg); col_c += LONG_PREFIX_LENGTH + MIN(len, (int)strlen(arg_p->ar_long_arg)); } /* add the optional argument if no short-arg */ if (arg_p->ar_short_arg == '\0') { if (HAS_ARG(arg_p->ar_type)) { (void)fputc(' ', argv_error_stream); col_c++; } /* display any optional arguments */ display_arg(argv_error_stream, arg_p, COMMENT_COLUMN - 1, &col_c); (void)fputc(']', argv_error_stream); col_c++; } /* print the comment */ if (arg_p->ar_comment != NULL) { /* put the comment message on the correct column */ if (col_c < COMMENT_COLUMN) { (void)fprintf(argv_error_stream, "%*.*s", COMMENT_COLUMN - col_c, COMMENT_COLUMN - col_c, ""); col_c = COMMENT_COLUMN; } len = SCREEN_WIDTH - col_c - COMMENT_LABEL_LENGTH; (void)fprintf(argv_error_stream, "%s%-.*s", COMMENT_LABEL, len, arg_p->ar_comment); } (void)fprintf(argv_error_stream, "\n"); } } /* * static void do_usage * * DESCRIPTION: * * Print the usage messages. * * RETURNS: * * None. * * ARGUMENTS: * * args - Array of argv_t structures. * * flag - Users flags which will tell us whether to display short or * long usage messages. */ static void do_usage(const argv_t *args, const int flag) { if (argv_error_stream == NULL) { return; } if (flag == ARGV_USAGE_SEE) { (void)fprintf(argv_error_stream, "%*.*sUse the '%s%s' argument for assistance.\n", (int)USAGE_LABEL_LENGTH, (int)USAGE_LABEL_LENGTH, "", LONG_PREFIX, USAGE_ARG); } else if (flag == ARGV_USAGE_SHORT || flag == ARGV_USAGE_SHORT_REM) { usage_short(args, flag); } else if (flag == ARGV_USAGE_LONG || flag == ARGV_USAGE_ALL) { usage_long(args); } if (flag == ARGV_USAGE_ALL) { (void)fprintf(argv_error_stream, "\n"); (void)fprintf(argv_error_stream, "%*.*sUse '%s%s' for default usage information.\n", SHORT_COLUMN, SHORT_COLUMN, "", LONG_PREFIX, USAGE_ARG); (void)fprintf(argv_error_stream, "%*.*sUse '%s%s' for short usage information.\n", SHORT_COLUMN, SHORT_COLUMN, "", LONG_PREFIX, USAGE_SHORT_ARG); (void)fprintf(argv_error_stream, "%*.*sUse '%s%s' for long usage information.\n", SHORT_COLUMN, SHORT_COLUMN, "", LONG_PREFIX, USAGE_LONG_ARG); (void)fprintf(argv_error_stream, "%*.*sUse '%s%s' for all usage information.\n", SHORT_COLUMN, SHORT_COLUMN, "", LONG_PREFIX, USAGE_ALL_ARG); (void)fprintf(argv_error_stream, "%*.*sUse '%s%s' to display the help message.\n", SHORT_COLUMN, SHORT_COLUMN, "", LONG_PREFIX, HELP_ARG); (void)fprintf(argv_error_stream, "%*.*sUse '%s%s' to display the version message.\n", SHORT_COLUMN, SHORT_COLUMN, "", LONG_PREFIX, VERSION_ARG); (void)fprintf(argv_error_stream, "%*.*sUse '%s%s' to display the options and their values.\n", SHORT_COLUMN, SHORT_COLUMN, "", LONG_PREFIX, DISPLAY_ARG); } } /******************************* preprocessing *******************************/ /* * static int preprocess_array * * DESCRIPTION: * * Preprocess argument array entries and set the mandatory and maybe * flags. * * RETURNS: * * Success - 0 * * Faulure - -1 * * ARGUMENTS: * * args - Array of argv_t structures. * * arg_n - Number of entries in the argv_t array. We need this for a * couple of reasons. */ static int preprocess_array(argv_t *args, const int arg_n) { argv_t *arg_p; int mand_array_b = ARGV_FALSE, maybe_field_b = ARGV_FALSE; /* count the args and find the first mandatory */ for (arg_p = args; arg_p < args + arg_n; arg_p++) { /* clear internal flags */ arg_p->ar_type &= ~ARGV_FLAG_USED; /* do we have a mandatory-array? */ if (arg_p->ar_short_arg == ARGV_MAND) { if (mand_array_b) { if (argv_error_stream != NULL) { (void)fprintf(argv_error_stream, "%s: %s, no ARGV_MAND's can follow a MAND or MAYBE array\n", argv_program, INTERNAL_ERROR_NAME); } if (argv_interactive) { (void)exit(argv_error_code); } return ERROR; } if (maybe_field_b) { if (argv_error_stream != NULL) { (void)fprintf(argv_error_stream, "%s: %s, no ARGV_MAND's can follow a ARGV_MAYBE\n", argv_program, INTERNAL_ERROR_NAME); } if (argv_interactive) { (void)exit(argv_error_code); } return ERROR; } if (arg_p->ar_type & ARGV_FLAG_ARRAY) { mand_array_b = ARGV_TRUE; } } /* do we have a maybe field? */ if (arg_p->ar_short_arg == ARGV_MAYBE) { if (mand_array_b) { if (argv_error_stream != NULL) { (void)fprintf(argv_error_stream, "%s: %s, no ARGV_MAYBE's can follow a MAND or MAYBE array\n", argv_program, INTERNAL_ERROR_NAME); } if (argv_interactive) { (void)exit(argv_error_code); } return ERROR; } maybe_field_b = ARGV_TRUE; if (arg_p->ar_type & ARGV_FLAG_ARRAY) { mand_array_b = ARGV_TRUE; } } /* handle initializing the argument array */ if (arg_p->ar_type & ARGV_FLAG_ARRAY) { argv_array_t *arrp = (argv_array_t *)arg_p->ar_variable; if (! HAS_ARG(arg_p->ar_type)) { if (argv_error_stream != NULL) { (void)fprintf(argv_error_stream, "%s: %s, cannot have an array of boolean values\n", argv_program, INTERNAL_ERROR_NAME); } if (argv_interactive) { (void)exit(argv_error_code); } return ERROR; } if (ARGV_TYPE(arg_p->ar_type) == ARGV_INCR) { if (argv_error_stream != NULL) { (void)fprintf(argv_error_stream, "%s: %s, cannot have an array of incremental values\n", argv_program, INTERNAL_ERROR_NAME); } if (argv_interactive) { (void)exit(argv_error_code); } return ERROR; } arrp->aa_entry_n = 0; } /* verify variable pointer */ if (arg_p->ar_variable == NULL && arg_p->ar_short_arg != ARGV_OR && arg_p->ar_short_arg != ARGV_XOR) { if (argv_error_stream != NULL) { (void)fprintf(argv_error_stream, "%s: %s, NULL variable specified in arg array\n", argv_program, INTERNAL_ERROR_NAME); } if (argv_interactive) { (void)exit(argv_error_code); } return ERROR; } /* verify [X]OR's */ if (arg_p->ar_short_arg == ARGV_OR || arg_p->ar_short_arg == ARGV_XOR) { /* that they are not at the start or end of list */ if (arg_p == args || arg_p >= (args + arg_n - 1)) { if (argv_error_stream != NULL) { (void)fprintf(argv_error_stream, "%s: %s, ARGV_[X]OR entries cannot be at start or end of array\n", argv_program, INTERNAL_ERROR_NAME); } if (argv_interactive) { (void)exit(argv_error_code); } return ERROR; } /* that two aren't next to each other */ if ((arg_p - 1)->ar_short_arg == ARGV_OR || (arg_p - 1)->ar_short_arg == ARGV_XOR) { if (argv_error_stream != NULL) { (void)fprintf(argv_error_stream, "%s: %s, two ARGV_[X]OR entries cannot be next to each other\n", argv_program, INTERNAL_ERROR_NAME); } if (argv_interactive) { (void)exit(argv_error_code); } return ERROR; } } } return NOERROR; } /* * static int string_to_value * * DESCRIPTION: * * Translate string value argument into a variable value depending on * its type. * * RETURNS: * * Success - 0 * * Faulure - -1 * * ARGUMENTS: * * arg - Argument string. * * var - Pointer to our variable. * * type - Type of the variable. */ static int string_to_value(const char *arg, ARGV_PNT var, const unsigned int type) { argv_array_t *arr_p; argv_type_t *type_p; unsigned int val_type = ARGV_TYPE(type), size = 0; /* find the type and the size for array */ for (type_p = argv_types; type_p->at_value != 0; type_p++) { if (type_p->at_value == val_type) { size = type_p->at_size; break; } } if (type_p->at_value == 0) { if (argv_error_stream != NULL) { (void)fprintf(argv_error_stream, "%s: illegal variable type %d\n", __FILE__, val_type); } return ERROR; } if (type & ARGV_FLAG_ARRAY) { arr_p = (argv_array_t *)var; if (arr_p->aa_entry_n == 0) { arr_p->aa_entries = (char *)malloc(ARRAY_INCR *size); } else if (arr_p->aa_entry_n % ARRAY_INCR == 0) { arr_p->aa_entries = (char *)realloc(arr_p->aa_entries, (arr_p->aa_entry_n + ARRAY_INCR) * size); } if (arr_p->aa_entries == NULL) { if (argv_error_stream != NULL) { (void)fprintf(argv_error_stream, "%s: memory error during argument processing\n", argv_program); } if (argv_interactive) { (void)exit(argv_error_code); } return ERROR; } var = (char *)(arr_p->aa_entries) + arr_p->aa_entry_n * size; arr_p->aa_entry_n++; } /* translate depending on type */ switch (val_type) { case ARGV_BOOL: /* if no close argument, set to true */ if (arg == NULL) { *(char *)var = ARGV_TRUE; } else if (*(char *)arg == 't' || *(char *)arg == 'T' || *(char *)arg == 'y' || *(char *)arg == 'Y' || *(char *)arg == '1') { *(char *)var = ARGV_TRUE; } else { *(char *)var = ARGV_FALSE; } break; case ARGV_BOOL_NEG: /* if no close argument, set to false */ if (arg == NULL) { *(char *)var = ARGV_FALSE; } else if (*(char *)arg == 't' || *(char *)arg == 'T' || *(char *)arg == 'y' || *(char *)arg == 'Y' || *(char *)arg == '1') { *(char *)var = ARGV_TRUE; } else { *(char *)var = ARGV_FALSE; } break; case ARGV_BOOL_ARG: if (*(char *)arg == 't' || *(char *)arg == 'T' || *(char *)arg == 'y' || *(char *)arg == 'Y' || *(char *)arg == '1') { *(char *)var = ARGV_TRUE; } else { *(char *)var = ARGV_FALSE; } break; case ARGV_CHAR: *(char *)var = *(char *)arg; break; case ARGV_CHAR_P: *(char **)var = string_copy((char *)arg); if (*(char **)var == NULL) { return ERROR; } break; case ARGV_SHORT: *(short *)var = (short)atoi(arg); break; case ARGV_U_SHORT: *(unsigned short *)var = (unsigned short)atoi(arg); break; case ARGV_INT: *(int *)var = atoi(arg); break; case ARGV_U_INT: *(unsigned int *)var = atoi(arg); break; case ARGV_LONG: *(long *)var = atol(arg); break; case ARGV_U_LONG: *(unsigned long *)var = atol(arg); break; case ARGV_FLOAT: (void)sscanf(arg, "%f", (float *)var); break; case ARGV_DOUBLE: (void)sscanf(arg, "%lf", (double *)var); break; case ARGV_BIN: *(int *)var = btoi(arg); break; case ARGV_OCT: *(int *)var = otoi(arg); break; case ARGV_HEX: *(int *)var = htoi(arg); break; case ARGV_INCR: /* if no close argument then increment else set the value */ if (arg == NULL) { (*(int *)var)++; } else { *(int *)var = atoi(arg); } break; case ARGV_SIZE: { const char *arg_p; long val; /* take initial integer point */ val = atol(arg); for (arg_p = arg; *arg_p == ' ' || *arg_p == '-' || *arg_p == '+' || (*arg_p >= '0' && *arg_p <= '9'); arg_p++) { } if (*arg_p == 'b' || *arg_p == 'B') { val *= 1; } else if (*arg_p == 'k' || *arg_p == 'K') { val *= 1024; } else if (*arg_p == 'm' || *arg_p == 'M') { val *= 1024 * 1024; } else if (*arg_p == 'g' || *arg_p == 'G') { val *= 1024 * 1024 * 1024; } *(long *)var = val; } break; case ARGV_U_SIZE: { const char *arg_p; unsigned long val; /* take initial integer point */ val = (unsigned long)atol(arg); for (arg_p = arg; *arg_p == ' ' || *arg_p == '-' || *arg_p == '+' || (*arg_p >= '0' && *arg_p <= '9'); arg_p++) { } if (*arg_p == 'b' || *arg_p == 'B') { val *= 1; } else if (*arg_p == 'k' || *arg_p == 'K') { val *= 1024; } else if (*arg_p == 'm' || *arg_p == 'M') { val *= 1024 * 1024; } else if (*arg_p == 'g' || *arg_p == 'G') { val *= 1024 * 1024 * 1024; } *(unsigned long *)var = val; } break; case ARGV_BOOL_INT: /* if no close argument, set to true */ if (arg == NULL) { *(int *)var = ARGV_TRUE; } else if (*(char *)arg == 't' || *(char *)arg == 'T' || *(char *)arg == 'y' || *(char *)arg == 'Y' || *(char *)arg == '1') { *(int *)var = ARGV_TRUE; } else { *(int *)var = ARGV_FALSE; } break; case ARGV_BOOL_INT_NEG: /* if no close argument, set to false */ if (arg == NULL) { *(int *)var = ARGV_FALSE; } else if (*(char *)arg == 't' || *(char *)arg == 'T' || *(char *)arg == 'y' || *(char *)arg == 'Y' || *(char *)arg == '1') { *(int *)var = ARGV_TRUE; } else { *(int *)var = ARGV_FALSE; } break; case ARGV_BOOL_INT_ARG: if (*(char *)arg == 't' || *(char *)arg == 'T' || *(char *)arg == 'y' || *(char *)arg == 'Y' || *(char *)arg == '1') { *(int *)var = ARGV_TRUE; } else { *(int *)var = ARGV_FALSE; } break; } return NOERROR; } /* * static int value_to_string * * DESCRIPTION: * * Translate value from variable depending on its type intoits string * represetnation in buffer. * * RETURNS: * * Number of characters added to the buffer. * * ARGUMENTS: * * var - Variable pointer. * * type - Type of variable. * * buf - User buffer to convert into. * * buf_size - Size of the user buffer. */ static int value_to_string(const ARGV_PNT var, const unsigned int type, char *buf, const int buf_size) { int len = 0; /* * NOTE: without a snprintf, we have to hope that buf_size > integer * and the string repesentations of the numbers. */ /* translate depending on type */ switch (ARGV_TYPE(type)) { case ARGV_BOOL: case ARGV_BOOL_NEG: case ARGV_BOOL_ARG: if (*(char *)var) { strncpy(buf, "true (! 0)", buf_size); } else { strncpy(buf, "false (0)", buf_size); } buf[buf_size - 1] = '\0'; len = strlen(buf); break; case ARGV_CHAR: len = expand_buf((char *)var, 1, buf, buf_size); break; case ARGV_CHAR_P: if (*(char **)var == NULL) { strncpy(buf, "(null)", buf_size); buf[buf_size - 1] = '\0'; len = strlen(buf); } else { len = expand_buf(*(char **)var, -1, buf, buf_size); } break; case ARGV_SHORT: LOC_SNPRINTF(SNP_ARG(buf, buf_size), "%d", *(short *)var); len = strlen(buf); break; case ARGV_U_SHORT: LOC_SNPRINTF(SNP_ARG(buf, buf_size), "%d", *(unsigned short *)var); len = strlen(buf); break; case ARGV_INT: LOC_SNPRINTF(SNP_ARG(buf, buf_size), "%d", *(int *)var); len = strlen(buf); break; case ARGV_U_INT: LOC_SNPRINTF(SNP_ARG(buf, buf_size), "%u", *(unsigned int *)var); len = strlen(buf); break; case ARGV_LONG: LOC_SNPRINTF(SNP_ARG(buf, buf_size), "%ld", *(long *)var); len = strlen(buf); break; case ARGV_U_LONG: LOC_SNPRINTF(SNP_ARG(buf, buf_size), "%lu", *(unsigned long *)var); len = strlen(buf); break; case ARGV_FLOAT: LOC_SNPRINTF(SNP_ARG(buf, buf_size), "%f", *(float *)var); len = strlen(buf); break; case ARGV_DOUBLE: LOC_SNPRINTF(SNP_ARG(buf, buf_size), "%f", *(double *)var); len = strlen(buf); break; /* this should be a routine */ case ARGV_BIN: { int bit_c, bit, first_b = ARGV_FALSE; char binary[2 + 128 + 1], *bin_bounds_p, *bin_p = binary; if (*(int *)var == 0) { strncpy(buf, "0", buf_size); } else { bin_bounds_p = binary + sizeof(binary); /* initially write binary number into tmp buffer, then copy into out */ *bin_p++ = '0'; *bin_p++ = 'b'; for (bit_c = sizeof(int) * BITS_IN_BYTE - 1; bit_c >= 0; bit_c--) { bit = *(int *)var & (1 << bit_c); if (bit == 0) { if (first_b) { *bin_p++ = '0'; } } else { *bin_p++ = '1'; first_b = ARGV_TRUE; } } /* add on the decimal equivalent */ LOC_SNPRINTF(SNP_ARG(bin_p, bin_bounds_p - bin_p)," (%d)", *(int *)var); /* find the \0 at end */ for (; *bin_p != '\0'; bin_p++) { } /* now we copy from the binary buffer to the output */ strncpy(buf, binary, buf_size); } buf[buf_size - 1] = '\0'; len = strlen(buf); } break; case ARGV_OCT: if (*(int *)var == 0) { (void)strncpy(buf, "0", buf_size); buf[buf_size - 1] = '\0'; } else { LOC_SNPRINTF(SNP_ARG(buf, buf_size), "%#o (%d)", *(int *)var, *(int *)var); } len = strlen(buf); break; case ARGV_HEX: if (*(int *)var == 0) { (void)strcpy(buf, "0"); } else { LOC_SNPRINTF(SNP_ARG(buf, buf_size), "%#x (%d)", *(int *)var, *(int *)var); } len = strlen(buf); break; case ARGV_INCR: LOC_SNPRINTF(SNP_ARG(buf, buf_size), "%d", *(int *)var); len = strlen(buf); break; case ARGV_SIZE: { long morf, val = *(long *)var; if (val == 0) { (void)strcpy(buf, "0"); } else if (val % (1024 * 1024 * 1024) == 0) { morf = val / (1024 * 1024 * 1024); LOC_SNPRINTF(SNP_ARG(buf, buf_size), "%ldg (%ld)", morf, val); } else if (val % (1024 * 1024) == 0) { morf = val / (1024 * 1024); LOC_SNPRINTF(SNP_ARG(buf, buf_size), "%ldm (%ld)", morf, val); } else if (val % 1024 == 0) { morf = val / 1024; LOC_SNPRINTF(SNP_ARG(buf, buf_size), "%ldk (%ld)", morf, val); } else { LOC_SNPRINTF(SNP_ARG(buf, buf_size), "%ld", val); } len = strlen(buf); } break; case ARGV_U_SIZE: { unsigned long morf, val = *(unsigned long *)var; if (val == 0) { (void)strcpy(buf, "0"); } else if (val % (1024 * 1024 * 1024) == 0) { morf = val / (1024 * 1024 * 1024); LOC_SNPRINTF(SNP_ARG(buf, buf_size), "%ldg (%ld)", morf, val); } else if (val % (1024 * 1024) == 0) { morf = val / (1024 * 1024); LOC_SNPRINTF(SNP_ARG(buf, buf_size), "%ldm (%ld)", morf, val); } else if (val % 1024 == 0) { morf = val / 1024; LOC_SNPRINTF(SNP_ARG(buf, buf_size), "%ldk (%ld)", morf, val); } else { LOC_SNPRINTF(SNP_ARG(buf, buf_size), "%ld", val); } len = strlen(buf); } break; case ARGV_BOOL_INT: case ARGV_BOOL_INT_NEG: case ARGV_BOOL_INT_ARG: if (*(int *)var) { strncpy(buf, "true (! 0)", buf_size); } else { strncpy(buf, "false (0)", buf_size); } buf[buf_size - 1] = '\0'; len = strlen(buf); break; default: strncpy(buf, "(unknown)", buf_size); buf[buf_size - 1] = '\0'; len = strlen(buf); break; } return len; } /* * static void display_variables * * DESCRIPTION: * * Display all of the variable values from our array. * * RETURNS: * * None. * * ARGUMENTS: * * args - Array of argv_t structures whose variables we are * displaying. */ static void display_variables(const argv_t *args) { const argv_t *arg_p; argv_type_t *type_p; char buf[256]; int len, col_c; unsigned int val_type; /* run through the argument structure */ for (arg_p = args; arg_p->ar_short_arg != ARGV_LAST; arg_p++) { val_type = ARGV_TYPE(arg_p->ar_type); /* skip or specifiers */ if (arg_p->ar_short_arg == ARGV_OR || arg_p->ar_short_arg == ARGV_XOR) { continue; } col_c = 0; if (arg_p->ar_short_arg == '\0') { if (arg_p->ar_long_arg != NULL) { len = COMMENT_COLUMN - col_c - (LONG_PREFIX_LENGTH + 1); if (arg_p->ar_short_arg != '\0') { (void)fprintf(argv_error_stream, "%s", LONG_LABEL); col_c += LONG_LABEL_LENGTH; len -= LONG_LABEL_LENGTH; } (void)fprintf(argv_error_stream, "%s%-.*s", LONG_PREFIX, len, arg_p->ar_long_arg); col_c += LONG_PREFIX_LENGTH + MIN(len, (int)strlen(arg_p->ar_long_arg)); } } else if (arg_p->ar_short_arg == ARGV_MAND) { display_arg(argv_error_stream, arg_p, COMMENT_COLUMN, &col_c); } else { /* ARGV_MAYBE handled here */ display_option(argv_error_stream, arg_p, &col_c); } /* put the type in the correct column */ if (col_c < LONG_COLUMN) { (void)fprintf(argv_error_stream, "%*.*s", LONG_COLUMN - col_c, LONG_COLUMN - col_c, ""); col_c = LONG_COLUMN; } /* find the type */ type_p = NULL; for (type_p = argv_types; type_p->at_value != 0; type_p++) { if (type_p->at_value == ARGV_TYPE(arg_p->ar_type)) { int tlen; len = COMMENT_COLUMN - col_c - 1; tlen = strlen(type_p->at_name); (void)fprintf(argv_error_stream, " %-.*s", len, type_p->at_name); col_c += MIN(len, tlen); if (arg_p->ar_type & ARGV_FLAG_ARRAY) { (void)fprintf(argv_error_stream, "%s", ARRAY_LABEL); col_c += sizeof(ARRAY_LABEL) - 1; } break; } } if (col_c < COMMENT_COLUMN) { (void)fprintf(argv_error_stream, "%*.*s", COMMENT_COLUMN - col_c, COMMENT_COLUMN - col_c, ""); col_c = COMMENT_COLUMN; } if (arg_p->ar_type & ARGV_FLAG_ARRAY) { argv_array_t *arr_p; int entry_c, size = 0; /* find the type and the size for array */ if (type_p == NULL) { (void)fprintf(argv_error_stream, "%s: illegal variable type %d\n", __FILE__, val_type); continue; } size = type_p->at_size; arr_p = (argv_array_t *)arg_p->ar_variable; if (arr_p->aa_entry_n == 0) { (void)fprintf(argv_error_stream, "no entries"); } else { for (entry_c = 0; entry_c < arr_p->aa_entry_n; entry_c++) { ARGV_PNT var; if (entry_c > 0) { (void)fputc(',', argv_error_stream); } var = (char *)(arr_p->aa_entries) + entry_c * size; len = value_to_string(var, val_type, buf, sizeof(buf)); (void)fwrite(buf, sizeof(char), len, argv_error_stream); } } } else { len = value_to_string(arg_p->ar_variable, val_type, buf, sizeof(buf)); (void)fwrite(buf, sizeof(char), len, argv_error_stream); } (void)fputc('\n', argv_error_stream); } } /************************** checking used arguments **************************/ /* * static int check_or * * DESCRIPTION: * * Check out if an argument has an ARGV_OR attached to it and both * variables have not been set. * * RETURNS: * * Success - 0 * * Faulure - -1 * * ARGUMENTS: * * args - Array of argv_t structures that we are checking. * * which_p - Pointer to the specific argument that we are checking for * the ARGV_OR. */ static int check_or(const argv_t *args, const argv_t *which_p) { const argv_t *arg_p, *match_p = NULL; /* check ORs below */ for (arg_p = which_p - 2; arg_p >= args; arg_p -= 2) { if ((arg_p + 1)->ar_short_arg != ARGV_OR && (arg_p + 1)->ar_short_arg != ARGV_XOR) { break; } if (arg_p->ar_type & ARGV_FLAG_USED) { match_p = arg_p; break; } } /* check ORs above */ if (match_p == NULL) { /* NOTE: we assume that which_p is not pointing now to ARGV_LAST */ for (arg_p = which_p + 2; arg_p->ar_short_arg != ARGV_LAST && (arg_p - 1)->ar_short_arg != ARGV_LAST; arg_p += 2) { if ((arg_p - 1)->ar_short_arg != ARGV_OR && (arg_p - 1)->ar_short_arg != ARGV_XOR) { break; } if (arg_p->ar_type & ARGV_FLAG_USED) { match_p = arg_p; break; } } } /* did we not find a problem? */ if (match_p == NULL) { return NOERROR; } if (argv_error_stream == NULL) { return ERROR; } (void)fprintf(argv_error_stream, "%s: %s, specify only one of the following:\n", argv_program, USAGE_ERROR_NAME); /* little hack to print the one that matched and the one we were checking */ for (;;) { if (match_p->ar_long_arg == NULL) { (void)fprintf(argv_error_stream, "%*.*s%s%c\n", (int)USAGE_LABEL_LENGTH, (int)USAGE_LABEL_LENGTH, "", SHORT_PREFIX, match_p->ar_short_arg); } else { (void)fprintf(argv_error_stream, "%*.*s%s%c (%s%s)\n", (int)USAGE_LABEL_LENGTH, (int)USAGE_LABEL_LENGTH, "", SHORT_PREFIX, match_p->ar_short_arg, LONG_PREFIX, match_p->ar_long_arg); } if (match_p == which_p) { break; } match_p = which_p; } return ERROR; } /* * static int check_xor * * DESCRIPTION: * * Check out if an argument has an ARGV_XOR attached to it and that at * least one but not both variables have been set. * * RETURNS: * * Success - 0 * * Faulure - -1 * * ARGUMENTS: * * args - Array of argv_t structures that we are checking. * * which_p - Pointer to the specific argument that we are checking for * the ARGV_XOR. */ static int check_xor(const argv_t *args) { const argv_t *start_p = NULL, *arg_p; /* run through the list of arguments */ for (arg_p = args; arg_p->ar_short_arg != ARGV_LAST; arg_p++) { /* only check the XORs */ if (arg_p->ar_short_arg != ARGV_XOR) { continue; } start_p = arg_p; /* * NOTE: we are guaranteed that we are on a XOR so there is * something below and above... */ if ((arg_p - 1)->ar_type & ARGV_FLAG_USED) { start_p = NULL; } /* run through all XORs */ for (;;) { arg_p++; if (arg_p->ar_type & ARGV_FLAG_USED) { start_p = NULL; } if ((arg_p + 1)->ar_short_arg != ARGV_XOR) { break; } arg_p++; } /* were none of the xor's filled? */ if (start_p != NULL) { break; } } /* did we not find a problem? */ if (start_p == NULL) { return NOERROR; } /* arg_p points to the first XOR which failed */ if (argv_error_stream == NULL) { return ERROR; } (void)fprintf(argv_error_stream, "%s: %s, must specify one of:\n", argv_program, USAGE_ERROR_NAME); for (arg_p = start_p;; arg_p += 2) { /* * NOTE: we are guaranteed that we are on a XOR so there is * something below and above... */ (void)fprintf(argv_error_stream, "%*.*s%s%c", (int)USAGE_LABEL_LENGTH, (int)USAGE_LABEL_LENGTH, "", SHORT_PREFIX, (arg_p - 1)->ar_short_arg); if ((arg_p - 1)->ar_long_arg != NULL) { (void)fprintf(argv_error_stream, " (%s%s)", LONG_PREFIX, (arg_p - 1)->ar_long_arg); } (void)fprintf(argv_error_stream, "\n"); if (arg_p->ar_short_arg != ARGV_XOR) { break; } } return ERROR; } /* * static int check_mand * * DESCRIPTION: * * Verify that all of the mandatory arguments in our array have been * specified. * * RETURNS: * * Success - 0 * * Faulure - -1 * * ARGUMENTS: * * args - Array of argv_t structures that we are checking. */ static int check_mand(const argv_t *args) { const argv_t *arg_p; int mand_c = 0, flag_c = 0; /* see if there are any mandatory args left */ for (arg_p = args; arg_p->ar_short_arg != ARGV_LAST; arg_p++) { if (arg_p->ar_short_arg == ARGV_MAND && (! (arg_p->ar_type & ARGV_FLAG_USED))) { mand_c++; } if (arg_p->ar_type & ARGV_FLAG_MAND && (! (arg_p->ar_type & ARGV_FLAG_USED))) { flag_c++; if (argv_error_stream != NULL) { if (flag_c == 1) { (void)fprintf(argv_error_stream, "%s: %s, these mandatory flags must be specified:\n", argv_program, USAGE_ERROR_NAME); } (void)fprintf(argv_error_stream, "%*.*s%s%c", (int)USAGE_LABEL_LENGTH, (int)USAGE_LABEL_LENGTH, "", SHORT_PREFIX, arg_p->ar_short_arg); if (arg_p->ar_long_arg != NULL) { (void)fprintf(argv_error_stream, " (%s%s)", LONG_PREFIX, arg_p->ar_long_arg); } (void)fprintf(argv_error_stream, "\n"); } } } if (mand_c > 0 && argv_error_stream != NULL) { (void)fprintf(argv_error_stream, "%s: %s, %d more mandatory argument%s must be specified\n", argv_program, USAGE_ERROR_NAME, mand_c, (mand_c == 1 ? "" : "s")); } if (mand_c > 0 || flag_c > 0) { return ERROR; } else { return NOERROR; } } /* * static int check_opt * * DESCRIPTION: * * Check for any missing argument options. * * RETURNS: * * Success - 0 * * Faulure - -1 * * ARGUMENTS: * * queue_head - Head of the option queue. * * queue_tail - Tail of the option queue. */ static int check_opt(const int queue_head, const int queue_tail) { int queue_c; queue_c = queue_head - queue_tail; if (queue_c > 0) { if (argv_error_stream != NULL) { (void)fprintf(argv_error_stream, "%s: %s, %d more option-argument%s must be specified\n", argv_program, USAGE_ERROR_NAME, queue_c, (queue_c == 1 ? "" : "s")); } return ERROR; } return NOERROR; } /**************************** argument processing ****************************/ /* * static void file_args * * DESCRIPTION: * * Read in arguments from a file and process them like they were * specified on the command line. * * RETURNS: * * Success - 0 * * Faulure - -1 * * ARGUMENTS: * * path -> File of the arguments we are reading in. * * grid -> Array of argv_t structures we are using. * * queue_list <-> Our option queue for storing options to arguments. * * queue_head_p <-> Pointer to integer which will be updated with the * head position in our option queue. * * queue_tail_p <-> Pointer to integer which will be updated with the * tail position in our option queue. * * okay_bp <- Pointer to an integer which is set with 0 if the * arguments specified in the env variable are somehow invalid. */ static void file_args(const char *path, argv_t *grid, argv_t **queue_list, int *queue_head_p, int *queue_tail_p, int *okay_bp) { char **argv, **argv_p; int arg_c, max; FILE *infile; char line[FILE_LINE_SIZE], *line_p; /* open the input file */ if (strcmp(path, "-") == 0) { infile = stdin; } else { infile = fopen(path, "r"); } if (infile == NULL) { *okay_bp = ARGV_FALSE; if (argv_error_stream != NULL) { (void)fprintf(argv_error_stream, "%s: could not load command-line arguments from: %s\n", argv_program, path); } if (argv_interactive) { (void)exit(argv_error_code); } return; } /* get an array of char * */ arg_c = 0; max = ARRAY_INCR; argv = malloc(sizeof(char *) * max); if (argv == NULL) { *okay_bp = ARGV_FALSE; if (infile != stdin) { (void)fclose(infile); } if (argv_error_stream != NULL) { (void)fprintf(argv_error_stream, "%s: memory error during argument processing\n", argv_program); } if (argv_interactive) { (void)exit(argv_error_code); } return; } argv_p = argv; /* read in the file lines */ while (fgets(line, sizeof(line), infile) != NULL) { /* punch the \n at end of line */ for (line_p = line; *line_p != '\n' && *line_p != '\0'; line_p++) { } *line_p = '\0'; /* skip blank lines */ if (line_p == line) { continue; } *argv_p = string_copy(line); if (*argv_p == NULL) { *okay_bp = ARGV_FALSE; return; } argv_p++; arg_c++; /* do we need to grow the array of pointers? */ if (arg_c == max) { max += ARRAY_INCR; argv = realloc(argv, sizeof(char *) * max); if (argv == NULL) { *okay_bp = ARGV_FALSE; if (infile != stdin) { (void)fclose(infile); } if (argv_error_stream != NULL) { (void)fprintf(argv_error_stream, "%s: memory error during argument processing\n", argv_program); } if (argv_interactive) { (void)exit(argv_error_code); } return; } argv_p = argv + arg_c; } } /* now do the list */ do_list(grid, arg_c, argv, queue_list, queue_head_p, queue_tail_p, okay_bp); /* now free up the list */ for (argv_p = argv; argv_p < argv + arg_c; argv_p++) { free(*argv_p); } free(argv); if (infile != stdin) { (void)fclose(infile); } } /* * static void do_arg * * DESCRIPTION: * * Process an argument in MATCH_P which looking at GRID. sets okay_p * to FALSE if the argument was not okay. * * RETURNS: * * None. * * ARGUMENTS: * * grid -> Our array of argv_t structures. * * match_p -> Entry in our argv_t structure array that matches the * specified argument. * * close_p -> Pointer to the value closely associated (with an '=') * with this option or NULL if none. * * queue_list <-> Our option queue for storing options to arguments. * * queue_head_p <-> Pointer to integer which will be updated with the * head position in our option queue. * * okay_bp <- Pointer to an integer which is set with 0 if the * arguments specified in the env variable are somehow invalid. */ static void do_arg(argv_t *grid, argv_t *match_p, const char *close_p, argv_t **queue_list, int *queue_head_p, int *okay_bp) { if (! argv_multi_accept_b) { /* * have we used this one before? * NOTE: should this be a warning or a non-error altogether? */ if (match_p->ar_type & ARGV_FLAG_USED && (! (match_p->ar_type & ARGV_FLAG_ARRAY)) && ARGV_TYPE(match_p->ar_type) != ARGV_INCR) { if (argv_error_stream != NULL) { (void)fprintf(argv_error_stream, "%s: %s, you've already specified the '%c' argument\n", argv_program, USAGE_ERROR_NAME, match_p->ar_short_arg); } *okay_bp = ARGV_FALSE; } } /* we used this argument */ match_p->ar_type |= ARGV_FLAG_USED; /* check arguments that must be OR'd */ if (check_or(grid, match_p) != NOERROR) { /* * don't return here else we might generate an XOR error * because the argument wasn't specified */ *okay_bp = ARGV_FALSE; } /* * If we have a close argument, pass to translate. If it is a * boolean or increment variable, then pass in a value of null * else queue it for needing a value argument. */ if (argv_close_enable_b && close_p != NULL) { if (string_to_value(close_p, match_p->ar_variable, match_p->ar_type) != NOERROR) { *okay_bp = ARGV_FALSE; } } else if (! HAS_ARG(match_p->ar_type)) { if (string_to_value(NULL, match_p->ar_variable, match_p->ar_type) != NOERROR) { *okay_bp = ARGV_FALSE; } } else if (argv_close_enable_b && close_p != NULL) { if (string_to_value(close_p, match_p->ar_variable, match_p->ar_type) != NOERROR) { *okay_bp = ARGV_FALSE; } } else { queue_list[*queue_head_p] = match_p; (*queue_head_p)++; } } /* * static int is_number * * DESCRIPTION: * * Examine an argument string to see if it really is a negative number * being passed into a previously specified argument. * * Thanks much to Nick Kisseberth for pointing out this oversight. * * RETURNS: * * 1 if a number otherwise 0. * * ARGUMENTS: * * str - String which may be a number. */ static int is_number(const char *str) { const char *str_p; /* empty strings are not numbers */ if (str[0] == '\0') { return 0; } /* * All chars in the string should be number chars for it to be a * number. Yes this will return yes if the argument is "00-" but * we'll chalk this up to user error. */ for (str_p = str; *str_p != '\0'; str_p++) { if (strchr(NUMBER_ARG_CHARS, *str_p) == NULL) { return 0; } } return 1; } /* * static void do_list * * DESCRIPTION: * * Process a list of arguments with our array of argv_t structures * * RETURNS: * * None. * * ARGUMENTS: * * grid - Our array of argv_t structures. * * arg_c - Number of arguments in argv. * * argv - User argument array of character pointers. * * queue_list <-> Our option queue for storing options to arguments. * * queue_head_p <-> Pointer to integer which will be updated with the * head position in our option queue. * * queue_tail_p <-> Pointer to integer which will be updated with the * tail position in our option queue. * * okay_bp - Pointer to an integer which is set with 0 if the * arguments specified in the env variable are somehow invalid. */ static void do_list(argv_t *grid, const int arg_c, char **argv, argv_t **queue_list, int *queue_head_p, int *queue_tail_p, int *okay_bp) { argv_t *grid_p, *match_p; int len, char_c, unwant_c = 0; int last_arg_b = ARGV_FALSE; char *close_p = NULL, **arg_p; /* run throught rest of arguments */ for (arg_p = argv; arg_p < argv + arg_c; arg_p++) { /* have we reached the LAST_ARG marker? */ if (strcmp(LAST_ARG, *arg_p) == 0) { if (last_arg_b) { if (argv_last_toggle_b) { last_arg_b = ARGV_FALSE; continue; } } else { last_arg_b = ARGV_TRUE; continue; } } /* are we processing a long option? */ if ((! last_arg_b) && strncmp(LONG_PREFIX, *arg_p, LONG_PREFIX_LENGTH) == 0) { /* * check for close equals marker * * NOTE: duplicated in the short prefix section below. In here otherwise * we process normal args with x=5 instead of just -x=5. */ if (argv_close_enable_b) { close_p = strchr(*arg_p, ARG_EQUALS); /* if we found the special char then punch the null and set pointer */ if (close_p != NULL) { *close_p = '\0'; close_p++; } } /* get length of rest of argument */ len = strlen(*arg_p) - LONG_PREFIX_LENGTH; /* we need more than the prefix */ if (len <= 0) { if (argv_error_stream != NULL) { (void)fprintf(argv_error_stream, "%s: %s, empty long-option prefix '%s'\n", argv_program, USAGE_ERROR_NAME, *arg_p); } *okay_bp = ARGV_FALSE; continue; } match_p = NULL; /* run though long options looking for a match */ for (grid_p = grid; grid_p->ar_short_arg != ARGV_LAST; grid_p++) { if (grid_p->ar_long_arg == NULL) { continue; } if (strncmp(*arg_p + LONG_PREFIX_LENGTH, grid_p->ar_long_arg, len) == 0) { if (match_p != NULL) { if (argv_error_stream != NULL) { (void)fprintf(argv_error_stream, "%s: %s, '%s' might be '%s' or '%s'\n", argv_program, USAGE_ERROR_NAME, *arg_p, grid_p->ar_long_arg, match_p->ar_long_arg); } *okay_bp = ARGV_FALSE; break; } /* record a possible match */ match_p = grid_p; /* don't break, need to see if another one matches */ } } /* if we found a match but quit then we must have found two matches */ if (match_p != NULL && grid_p->ar_short_arg != ARGV_LAST) { continue; } if (match_p != NULL) { (void)do_arg(grid, match_p, close_p, queue_list, queue_head_p, okay_bp); continue; } /* we did not find long-option match */ /* check for special file value */ if (strncmp(FILE_ARG, *arg_p + LONG_PREFIX_LENGTH, len) == 0) { if (argv_close_enable_b && close_p != NULL) { /* open the file and read in the args */ file_args(close_p, grid, queue_list, queue_head_p, queue_tail_p, okay_bp); } else { /* HACK: we enqueue null for the file argument */ queue_list[*queue_head_p] = NULL; (*queue_head_p)++; } continue; } /* check for special usage value */ if (strncmp(USAGE_ARG, *arg_p + LONG_PREFIX_LENGTH, len) == 0 || strncmp(HELP_ARG, *arg_p + LONG_PREFIX_LENGTH, len) == 0) { if (argv_interactive) { do_usage(grid, argv_usage_type); (void)exit(0); } continue; } /* check for special short-usage value */ if (strncmp(USAGE_SHORT_ARG, *arg_p + LONG_PREFIX_LENGTH, len) == 0) { if (argv_interactive) { do_usage(grid, ARGV_USAGE_SHORT); (void)exit(0); } continue; } /* check for special long-usage value */ if (strncmp(USAGE_LONG_ARG, *arg_p + LONG_PREFIX_LENGTH, len) == 0) { if (argv_interactive) { do_usage(grid, ARGV_USAGE_LONG); (void)exit(0); } continue; } /* check for special long-usage value */ if (strncmp(USAGE_ALL_ARG, *arg_p + LONG_PREFIX_LENGTH, len) == 0) { if (argv_interactive) { do_usage(grid, ARGV_USAGE_ALL); (void)exit(0); } continue; } /* check for special help value */ if (strncmp(HELP_ARG, *arg_p + LONG_PREFIX_LENGTH, len) == 0) { if (argv_interactive) { if (argv_error_stream != NULL) { if (argv_help_string == NULL) { (void)fprintf(argv_error_stream, "%s: I'm sorry, no help is available.\n", argv_program); } else { (void)fprintf(argv_error_stream, "%s: %s\n", argv_program, argv_help_string); } } (void)exit(0); } continue; } /* check for special version value */ if (strncmp(VERSION_ARG, *arg_p + LONG_PREFIX_LENGTH, len) == 0) { if (argv_interactive) { if (argv_error_stream != NULL) { if (argv_version_string == NULL) { (void)fprintf(argv_error_stream, "%s: no version information is available.\n", argv_program); } else { (void)fprintf(argv_error_stream, "%s: %s\n", argv_program, argv_version_string); } } (void)exit(0); } continue; } /* check for display arguments value */ if (strncmp(DISPLAY_ARG, *arg_p + LONG_PREFIX_LENGTH, len) == 0) { if (argv_interactive) { if (argv_error_stream != NULL) { display_variables(grid); } (void)exit(0); } continue; } if (argv_error_stream != NULL) { (void)fprintf(argv_error_stream, "%s: %s, unknown long option '%s'.\n", argv_program, USAGE_ERROR_NAME, *arg_p); } *okay_bp = ARGV_FALSE; continue; } /* are we processing a short option? */ if ((! last_arg_b) && strncmp(SHORT_PREFIX, *arg_p, SHORT_PREFIX_LENGTH) == 0 && !(strlen(*arg_p) == 1)) { /* * check for close equals marker * * NOTE: duplicated in the long prefix section above. In here otherwise * we process normal args with x=5 instead of just -x=5. */ if (argv_close_enable_b) { close_p = strchr(*arg_p, ARG_EQUALS); /* if we found the special char then punch the null and set pointer */ if (close_p != NULL) { *close_p = '\0'; close_p++; } } /* get length of rest of argument */ len = strlen(*arg_p) - SHORT_PREFIX_LENGTH; /* we need more than the prefix */ if (len <= 0) { if (argv_error_stream != NULL) { (void)fprintf(argv_error_stream, "%s: %s, empty short-option prefix '%s'\n", argv_program, USAGE_ERROR_NAME, *arg_p); } *okay_bp = ARGV_FALSE; continue; } /* run through the chars in this option */ for (char_c = 0; char_c < len; char_c++) { /* run through the arg list looking for a match */ for (match_p = grid; match_p->ar_short_arg != ARGV_LAST; match_p++) { if (match_p->ar_short_arg == (*arg_p)[SHORT_PREFIX_LENGTH + char_c]) { break; } } /* did we not find argument? */ if (match_p->ar_short_arg == ARGV_LAST) { /* check for special usage value */ if ((*arg_p)[SHORT_PREFIX_LENGTH + char_c] == USAGE_CHAR_ARG) { if (argv_interactive) { do_usage(grid, argv_usage_type); (void)exit(0); } continue; } /* * allow values with negative signs if we are at the start * of an argument list, and if the argument is a number, and * we already have a variable looking for a value. Thanks * to Nick Kisseberth for pointing out this oversight. */ if (char_c == 0 && is_number(*arg_p) && *queue_head_p > *queue_tail_p) { match_p = queue_list[*queue_tail_p]; /* * NOTE: we don't advance the queue tail here unless we * find out that we can use it below */ switch (ARGV_TYPE(match_p->ar_type)) { case ARGV_SHORT: case ARGV_INT: case ARGV_LONG: case ARGV_FLOAT: case ARGV_DOUBLE: string_to_value(*arg_p, match_p->ar_variable, match_p->ar_type); char_c = len; /* we actually used it so we advance the queue tail position */ (*queue_tail_p)++; continue; break; } } /* create an error string */ if (argv_error_stream != NULL) { (void)fprintf(argv_error_stream, "%s: %s, unknown short option '%s%c'.\n", argv_program, USAGE_ERROR_NAME, SHORT_PREFIX, (*arg_p)[SHORT_PREFIX_LENGTH + char_c]); } *okay_bp = ARGV_FALSE; continue; } do_arg(grid, match_p, close_p, queue_list, queue_head_p, okay_bp); } continue; } /* could this be a value? */ if (grid->ar_short_arg != ARGV_LAST && *queue_head_p > *queue_tail_p) { /* pull the variable waiting for a value from the queue */ match_p = queue_list[*queue_tail_p]; (*queue_tail_p)++; /* HACK: is this the file argument */ if (match_p == NULL) { file_args(*arg_p, grid, queue_list, queue_head_p, queue_tail_p, okay_bp); } else { if (string_to_value(*arg_p, match_p->ar_variable, match_p->ar_type) != NOERROR) { *okay_bp = ARGV_FALSE; } } continue; } /* process mandatory args if some left to process */ for (grid_p = grid; grid_p->ar_short_arg != ARGV_LAST; grid_p++) { if (grid_p->ar_short_arg == ARGV_MAND && ((! (grid_p->ar_type & ARGV_FLAG_USED)) || grid_p->ar_type & ARGV_FLAG_ARRAY)) { break; } } if (grid_p->ar_short_arg != ARGV_LAST) { /* absorb another mand. arg */ if (string_to_value(*arg_p, grid_p->ar_variable, grid_p->ar_type) != NOERROR) { *okay_bp = ARGV_FALSE; } grid_p->ar_type |= ARGV_FLAG_USED; continue; } /* process maybe args if some left to process */ for (grid_p = grid; grid_p->ar_short_arg != ARGV_LAST; grid_p++) { if (grid_p->ar_short_arg == ARGV_MAYBE && ((! (grid_p->ar_type & ARGV_FLAG_USED)) || grid_p->ar_type & ARGV_FLAG_ARRAY)) { break; } } if (grid_p->ar_short_arg != ARGV_LAST) { /* absorb another maybe arg */ if (string_to_value(*arg_p, grid_p->ar_variable, grid_p->ar_type) != NOERROR) { *okay_bp = ARGV_FALSE; } grid_p->ar_type |= ARGV_FLAG_USED; continue; } /* default is an error */ unwant_c++; *okay_bp = ARGV_FALSE; } if (unwant_c > 0 && argv_error_stream != NULL) { (void)fprintf(argv_error_stream, "%s: %s, %d unwanted additional argument%s\n", argv_program, USAGE_ERROR_NAME, unwant_c, (unwant_c == 1 ? "" : "s")); } } /****************************** env processing *******************************/ /* * static char ** preprocess_env_args * * DESCRIPTION: * * Preprocess the environmental variable arguments, if any. * * RETURNS: * * A (char *) array that contains the arguments which needs to be * freed later. If it returns NULL then there are no env args. * * Faulure - -1 * * ARGUMENTS: * * environ_pp - Pointer to a (char *) which will be set with the env * string that will need to be freed after the env vector is processed. * * env_np - Pointer to an integer which is set with the number * of environ arguments. */ static char **preprocess_env_args(char **environ_pp, int *env_np) { char env_name[1024], *environ_p; *env_np = 0; /* create the env variable */ LOC_SNPRINTF(SNP_ARG(env_name, sizeof(env_name)), ENVIRON_FORMAT, argv_program); /* NOTE: by default the env name is all uppercase */ for (environ_p = env_name; *environ_p != '\0'; environ_p++) { if (islower((unsigned char) *environ_p)) { *environ_p = toupper((unsigned char) *environ_p); } } environ_p = getenv(env_name); if (environ_p == NULL) { return NULL; } /* break the list into tokens and do the list */ environ_p = string_copy(environ_p); if (environ_p == NULL) { (void)fprintf(argv_error_stream, "%s: problems allocating memory for environmental arguments\n", argv_program); return NULL; } *environ_pp = environ_p; return vectorize(environ_p, " \t", env_np); } /* * static int process_env * * DESCRIPTION: * * Process the global env variables. * * RETURNS: * * Success - 0 * * Faulure - -1 * * ARGUMENTS: * * None. */ static int process_env(void) { static int done_b = ARGV_FALSE; char *env_val, *tok_p, *env_p; int len; /* make sure we only do this once */ if (done_b) { return NOERROR; } done_b = ARGV_TRUE; /* get the argv information */ env_val = getenv(GLOBAL_NAME); if (env_val == NULL) { return NOERROR; } /* save a copy of it */ env_val = string_copy(env_val); if (env_val == NULL) { return ERROR; } env_p = env_val; for (;;) { tok_p = strsep(&env_p, " \t,:"); if (tok_p == NULL) { break; } /* skip any empty tokens */ if (*tok_p == '\0') { continue; } len = strlen(GLOBAL_CLOSE); if (strncmp(GLOBAL_CLOSE, tok_p, len) == 0) { tok_p += len; if (strcmp(tok_p, "disable") == 0 || strcmp(tok_p, "off") == 0 || strcmp(tok_p, "no") == 0 || strcmp(tok_p, "0") == 0) { argv_close_enable_b = 0; } else if (strcmp(tok_p, "enable") == 0 || strcmp(tok_p, "on") == 0 || strcmp(tok_p, "yes") == 0 || strcmp(tok_p, "1") == 0) { argv_close_enable_b = 1; } else { if (argv_error_stream != NULL) { (void)fprintf(argv_error_stream, "%s: illegal env variable '%s' '%s' argument '%s'\n", __FILE__, GLOBAL_NAME, GLOBAL_CLOSE, tok_p); } } continue; } len = strlen(GLOBAL_LASTTOG); if (strncmp(GLOBAL_LASTTOG, tok_p, len) == 0) { tok_p += len; if (strcmp(tok_p, "disable") == 0 || strcmp(tok_p, "off") == 0 || strcmp(tok_p, "no") == 0 || strcmp(tok_p, "0") == 0) { argv_last_toggle_b = 0; } else if (strcmp(tok_p, "enable") == 0 || strcmp(tok_p, "on") == 0 || strcmp(tok_p, "yes") == 0 || strcmp(tok_p, "1") == 0) { argv_last_toggle_b = 1; } else { if (argv_error_stream != NULL) { (void)fprintf(argv_error_stream, "%s: illegal env variable '%s' '%s' argument '%s'\n", __FILE__, GLOBAL_NAME, GLOBAL_LASTTOG, tok_p); } } continue; } len = strlen(GLOBAL_ENV); if (strncmp(GLOBAL_ENV, tok_p, len) == 0) { tok_p += len; if (strcmp(tok_p, "none") == 0) { argv_process_env_b = 0; argv_env_after_b = 0; } else if (strcmp(tok_p, "before") == 0) { argv_process_env_b = 1; argv_env_after_b = 0; } else if (strcmp(tok_p, "after") == 0) { argv_process_env_b = 1; argv_env_after_b = 1; } else { if (argv_error_stream != NULL) { (void)fprintf(argv_error_stream, "%s: illegal env variable '%s' '%s' argument '%s'\n", __FILE__, GLOBAL_NAME, GLOBAL_ENV, tok_p); } } continue; } len = strlen(GLOBAL_ERROR); if (strncmp(GLOBAL_ERROR, tok_p, len) == 0) { tok_p += len; if (strcmp(tok_p, "none") == 0) { argv_error_type = ARGV_USAGE_NONE; } else if (strcmp(tok_p, "see") == 0) { argv_error_type = ARGV_USAGE_SEE; } else if (strcmp(tok_p, "short") == 0) { argv_error_type = ARGV_USAGE_SHORT; } else if (strcmp(tok_p, "shortrem") == 0) { argv_error_type = ARGV_USAGE_SHORT_REM; } else if (strcmp(tok_p, "long") == 0) { argv_error_type = ARGV_USAGE_LONG; } else if (strcmp(tok_p, "all") == 0) { argv_error_type = ARGV_USAGE_ALL; } else { if (argv_error_stream != NULL) { (void)fprintf(argv_error_stream, "%s: illegal env variable '%s' '%s' argument '%s'\n", __FILE__, GLOBAL_NAME, GLOBAL_ERROR, tok_p); } } continue; } len = strlen(GLOBAL_MULTI); if (strncmp(GLOBAL_MULTI, tok_p, len) == 0) { tok_p += len; if (strcmp(tok_p, "reject") == 0) { argv_multi_accept_b = 0; } else if (strcmp(tok_p, "accept") == 0) { argv_multi_accept_b = 1; } else { if (argv_error_stream != NULL) { (void)fprintf(argv_error_stream, "%s: illegal env variable '%s' '%s' argument '%s'\n", __FILE__, GLOBAL_NAME, GLOBAL_MULTI, tok_p); } } continue; } len = strlen(GLOBAL_USAGE); if (strncmp(GLOBAL_USAGE, tok_p, len) == 0) { tok_p += len; if (strcmp(tok_p, "short") == 0) { argv_usage_type = ARGV_USAGE_SHORT; } else if (strcmp(tok_p, "shortrem") == 0) { argv_usage_type = ARGV_USAGE_SHORT_REM; } else if (strcmp(tok_p, "long") == 0) { argv_usage_type = ARGV_USAGE_LONG; } else if (strcmp(tok_p, "all") == 0) { argv_usage_type = ARGV_USAGE_ALL; } else { if (argv_error_stream != NULL) { (void)fprintf(argv_error_stream, "%s: illegal env variable '%s' '%s' argument '%s'\n", __FILE__, GLOBAL_NAME, GLOBAL_USAGE, tok_p); } } continue; } if (argv_error_stream != NULL) { (void)fprintf(argv_error_stream, "%s: illegal env variable '%s' setting '%s'\n", __FILE__, GLOBAL_NAME, tok_p); } } free(env_val); return NOERROR; } /***************************** exported routines *****************************/ /* * int argv_process_no_env * * DESCRIPTION: * * Process the user arguments with an argv_t structure array. Like * argv_process_args but without the processing of the argv * environmental variables. * * RETURNS: * * Success - 0 * * Failure - -1 * * ARGUMENTS: * * args - Array of argv_t structures. * * arg_n - Number of arguments in the argv array. * * argv - Array of character pointers terminated by 0L. */ int argv_process_no_env(argv_t *args, const int arg_n, char **argv) { int entry_c; const char *prog_p; char **env_vect_p, *environ_p = NULL; int okay_b = ARGV_TRUE; argv_t *arg_p; argv_t **queue_list = NULL; int queue_head = 0, queue_tail = 0, env_n, total_arg_n; if (args == NULL) { args = empty; } if (arg_n < 0) { if (argv_error_stream != NULL) { (void)fprintf(argv_error_stream, "%s: %s, argc argument to argv_process is %d\n", __FILE__, INTERNAL_ERROR_NAME, arg_n); } if (argv_interactive) { (void)exit(argv_error_code); } return ERROR; } if (argv == NULL) { if (argv_error_stream != NULL) { (void)fprintf(argv_error_stream, "%s: %s, argv argument to argv_process is NULL\n", __FILE__, INTERNAL_ERROR_NAME); } if (argv_interactive) { (void)exit(argv_error_code); } return ERROR; } /* set global variables */ argv_argv = argv; argv_argc = arg_n; /* build the program name from the argv[0] path */ { const char *tmp_p; prog_p = *argv; for (tmp_p = *argv; *tmp_p != '\0'; tmp_p++) { if (*tmp_p == '/') { prog_p = tmp_p + 1; } } } /* so we can step on the environmental space */ (void)strncpy(argv_program, prog_p, PROGRAM_NAME); /* count the args */ entry_c = 0; for (arg_p = args; arg_p->ar_short_arg != ARGV_LAST; arg_p++) { entry_c++; } /* verify the argument array */ if (preprocess_array(args, entry_c) != NOERROR) { return ERROR; } /* preprocess environmental args */ env_vect_p = preprocess_env_args(&environ_p, &env_n); total_arg_n = arg_n + env_n; /* allocate our value queue */ if (total_arg_n > 0) { /* allocate our argument queue */ queue_list = (argv_t **)malloc(sizeof(argv_t *) * total_arg_n); if (queue_list == NULL) { return ERROR; } queue_head = 0; queue_tail = 0; } /* do the env args before? */ if (argv_process_env_b && (! argv_env_after_b) && env_vect_p != NULL) { do_list(args, env_n, env_vect_p, queue_list, &queue_head, &queue_tail, &okay_b); free(env_vect_p); free(environ_p); env_vect_p = NULL; } /* do the external args */ do_list(args, arg_n - 1, argv + 1, queue_list, &queue_head, &queue_tail, &okay_b); /* DO the env args after? */ if (argv_process_env_b && argv_env_after_b && env_vect_p != NULL) { do_list(args, env_n, env_vect_p, queue_list, &queue_head, &queue_tail, &okay_b); free(env_vect_p); free(environ_p); env_vect_p = NULL; } /* make sure the XOR and MAND args and argument-options are okay */ if (check_mand(args) != NOERROR) { okay_b = ARGV_FALSE; } if (check_opt(queue_head, queue_tail) != NOERROR) { okay_b = ARGV_FALSE; } if (check_xor(args) != NOERROR) { okay_b = ARGV_FALSE; } /* if we allocated the space then free it */ if (arg_n > 0) { free(queue_list); } /* was there an error? */ if (! okay_b) { if (argv_error_stream != NULL) { do_usage(args, argv_error_type); } if (argv_interactive) { (void)exit(argv_error_code); } return ERROR; } return NOERROR; } /* * int argv_process * * DESCRIPTION: * * Processes a number of arguments depending on the argument array. * This routine will not modify the argv array in any way. * * NOTE: it will modify the args array by setting various flags in the * type field. returns 0 if no error else -1. * * ARGUMENTS: * * args - Array of argv_t structures that we are using to process the * user argument array. If null then an empty array is used. * * argc - Number of arguments in the argv argument array. * * argv - Array of character pointer arguments terminated by a 0L. */ int argv_process(argv_t *args, const int argc, char **argv) { if (! enabled_b) { argv_startup(); } /* we only process env variables here */ if (process_env() != NOERROR) { return ERROR; } if (argv_process_no_env(args, argc, argv) == NOERROR) { return NOERROR; } else { return ERROR; } } /* * int argv_usage * * DESCRIPTION: * * Print the standard usage messages for our argument array. You can * specify whether you want to see a short or long usage messages. * * NOTE: if this is called before argv_process then the program name * may be invalid. * * RETURNS: * * Success - 0 * * Failure - -1 * * ARGUMENTS: * * args - Our argument array to print the usage messages about. If * null then an empty array is used. * * which - Either ARGV_USAGE_SHORT (for short usage messages), * ARGV_USAGE_LONG (for long usage messages), or ARGV_USAGE_DEFAULT * (the user's default either long or short). */ int argv_usage(const argv_t *args, const int which) { if (! enabled_b) { argv_startup(); } if (process_env() != NOERROR) { return ERROR; } if (args == NULL) { args = empty; } if (which == ARGV_USAGE_SHORT || which == ARGV_USAGE_LONG || which == ARGV_USAGE_ALL) { do_usage(args, which); } else { /* default/env settings */ do_usage(args, argv_usage_type); } return NOERROR; } /* * int argv_was_used * * DESCRIPTION: * * See if an argument was used in a previous call to argv_process. * * RETURNS: * * 1 if yes it was used, else 0 if not. * * ARGUMENTS: * * args - Argument list to search. * * short_arg - Short argument to see if it was used. */ int argv_was_used(const argv_t *args, const char short_arg) { const argv_t *arg_p; if (! enabled_b) { argv_startup(); } for (arg_p = args; arg_p->ar_short_arg != ARGV_LAST; arg_p++) { if (arg_p->ar_short_arg == short_arg) { if (arg_p->ar_type & ARGV_FLAG_USED) { return 1; } else { return 0; } } } return 0; } /* * int argv_long_was_used * * DESCRIPTION: * * See if a long argument was used in a previous call to argv_process. * * RETURNS: * * 1 if yes it was used, else 0 if not. * * ARGUMENTS: * * args - Argument list to search. * * long_arg - Long argument to see if it was used. */ int argv_long_was_used(const argv_t *args, const char *long_arg) { const argv_t *arg_p; if (! enabled_b) { argv_startup(); } for (arg_p = args; arg_p->ar_short_arg != ARGV_LAST; arg_p++) { if (arg_p->ar_long_arg == long_arg) { if (arg_p->ar_type & ARGV_FLAG_USED) { return 1; } else { return 0; } } } return 0; } /* * int argv_entry_was_used * * DESCRIPTION: * * See if an entry in the argument array was used in a previous call * to argv_process. * * RETURNS: * * 1 if yes it was used, else 0 if not. * * ARGUMENTS: * * argv_entry_p - Pointer to an entry in a argv_t list. */ int argv_entry_was_used(const argv_t *argv_entry_p) { if (argv_entry_p->ar_type & ARGV_FLAG_USED) { return 1; } else { return 0; } } /* * void argv_cleanup * * DESCRIPTION: * * Frees up any allocations associated with the argument array during * argv_process. This should be done at the end of the program or * after all the arguments have been referenced. * * RETURNS: * * None. * * ARGUMENTS: * * args - Argument array we are cleaning up. */ void argv_cleanup(const argv_t *args) { const argv_t *arg_p; int entry_c; if (! enabled_b) { argv_startup(); } if (args == NULL) { return; } /* run through the argument structure */ for (arg_p = args; arg_p->ar_short_arg != ARGV_LAST; arg_p++) { /* handle any arrays */ if (arg_p->ar_type & ARGV_FLAG_ARRAY) { argv_array_t *arr_p = (argv_array_t *)arg_p->ar_variable; /* free any entries */ if (arr_p->aa_entry_n > 0) { if (ARGV_TYPE(arg_p->ar_type) == ARGV_CHAR_P) { for (entry_c = 0; entry_c < arr_p->aa_entry_n; entry_c++) { free(ARGV_ARRAY_ENTRY(*arr_p, char *, entry_c)); } } free(arr_p->aa_entries); } arr_p->aa_entries = NULL; arr_p->aa_entry_n = 0; continue; } /* handle individual charps */ if (arg_p->ar_type & ARGV_FLAG_USED && ARGV_TYPE(arg_p->ar_type) == ARGV_CHAR_P) { free(*(char **)arg_p->ar_variable); continue; } } } /* * int argv_copy_args * * DESCRIPTION: * * Copy all the arguements (not including the 0th) one after the other * into the user specified buffer. * * NOTE: you can get the 0th argument from argv_argv[0] or * argv_program. * * RETURNS: * * Success - 0 * * Failure - -1 * * ARGUMENTS: * * buf - Buffer to copy all of the user arguments into. * * buf_size - Size of the buffer. */ int argv_copy_args(char *buf, const int buf_size) { char **argv_p, *buf_p = buf, *arg_p; int arg_c, size_c = buf_size; if (! enabled_b) { argv_startup(); } if (buf_size <= 0) { return NOERROR; } *buf_p = '\0'; if (argv_argv == NULL || buf_size == 1) { return NOERROR; } for (argv_p = argv_argv + 1, arg_c = 1; arg_c < argv_argc; argv_p++, arg_c++) { /* we compare against 2 for the ' ' and the \0 */ if (size_c < 2) { break; } if (argv_p > argv_argv + 1) { *buf_p++ = ' '; size_c--; } /* we always compare against 2 to include the \0 */ for (arg_p = *argv_p; *arg_p != '\0' && size_c >= 2; size_c--) { *buf_p++ = *arg_p++; } } *buf_p = '\0'; return NOERROR; } /* * int argv_value_string * * DESCRIPTION: * * Convert the value of a RC entry to its string equivalent in the * buffer provided. * * RETURNS: * * Length of bytes copied into the buffer. * * ARGUMENTS: * * argv_entry_p - Pointer to an entry in a argv_t list. * * buf - Buffer to convert the value into. * * buf_size - Size of the buffer. */ int argv_value_string(const argv_t *argv_entry_p, char *buf, const int buf_size) { argv_array_t *arr_p; int ret, len; char details[128]; if (! enabled_b) { argv_startup(); } /* do we have an array here? */ if (argv_entry_p->ar_type & ARGV_FLAG_ARRAY) { /* if we have an array, then */ arr_p = (argv_array_t *)argv_entry_p->ar_variable; if (arr_p->aa_entry_n == 0) { strncpy(buf, "0 array entries", buf_size); buf[buf_size - 1] = '\0'; ret = strlen(buf); } else { len = value_to_string(arr_p->aa_entries, ARGV_TYPE(argv_entry_p->ar_type), buf, buf_size); if (arr_p->aa_entry_n == 1) { ret = len; } else { LOC_SNPRINTF(SNP_ARG(details, sizeof(details)), " (1st of %d entries)", arr_p->aa_entry_n); strncpy(buf + len, details, buf_size - len); buf[buf_size - 1] = '\0'; ret = strlen(buf); } } } else { ret = value_to_string(argv_entry_p->ar_variable, argv_entry_p->ar_type, buf, buf_size); } return ret; } /* * int argv_type_info * * DESCRIPTION: * * Get internal information about the type of the argument. * * RETURNS: * * The name of the type. * * ARGUMENTS: * * type - Number of argument type. * * size_p - Pointer to an unsigned integer which, if not NULL, will be * set with the size of the type. * * desc_p - Pointer to a constant character pointer which, if not * NULL, will be pointed to a description of the type. */ const char *argv_type_info(const unsigned int type, unsigned int *size_p, const char **desc_p) { unsigned int val_type; argv_type_t *type_p; val_type = ARGV_TYPE(type); for (type_p = argv_types; type_p->at_value != 0; type_p++) { if (type_p->at_value == val_type) { if (size_p != NULL) { *size_p = type_p->at_size; } if (desc_p != NULL) { *desc_p = type_p->at_desc; } return type_p->at_name; } } if (size_p != NULL) { *size_p = 0; } if (desc_p != NULL) { *desc_p = "Unknown type"; } return "(unknown type)"; } mtbl-0.5/libmy/argv.h000066400000000000000000000357411227302760400145550ustar00rootroot00000000000000/* * Defines for a generic argv and argc processor... * * Copyright 2000 by Gray Watson * * This file is part of the argv library. * * Permission to use, copy, modify, and distribute this software for * any purpose and without fee is hereby granted, provided that the * above copyright notice and this permission notice appear in all * copies, and that the name of Gray Watson not be used in advertising * or publicity pertaining to distribution of the document or software * without specific, written prior permission. * * Gray Watson makes no representations about the suitability of the * software described herein for any purpose. It is provided "as is" * without express or implied warranty. * * The author may be contacted via http://256.com/gray/ * * $Id: argv.h.1,v 1.22 2010/02/15 13:59:37 gray Exp $ */ #ifndef __ARGV_H__ #define __ARGV_H__ #ifdef __cplusplus extern "C" { #if 0 } #endif #endif /* * Version string for the library * * NOTE to gray: whenever this is changed, corresponding Changlog and * NEWS entries *must* be entered and argv.texi updated. * * ARGV LIBRARY VERSION -- 2.7.0 */ /* argv version defines */ #define ARGV_VERSION_MAJOR 2 /* X.0.0 */ #define ARGV_VERSION_MINOR 7 /* 0.X.0 */ #define ARGV_VERSION_PATCH 0 /* 0.0.X */ /* produced by configure, inserted into argv.h */ /* used to handle the const operator */ /* const is available */ /* NOTE: start of $Id: argv.h.3,v 1.33 2007/01/08 20:32:57 gray Exp $ */ /* * Generic and standardized argument processor. You describe the arguments * that you are looking for along with their types and these routines do the * work to convert them into values. * * These routines also provide standardized error and usage messages as well * as good usage documentation and long and short options. */ #include /* have to for FILE * below */ /* this defines what type the standard void memory-pointer is */ #if (defined(__STDC__) && __STDC__ == 1) || defined(__cplusplus) #define ARGV_PNT void * #else #define ARGV_PNT char * #endif /* * argument information structure. this specifies the allowable options * and some information about each one. * * { 'O', "optimize", ARGV_BOOL, &optimize, NULL, "turn on optimization" } * { 'c', "config", ARGV_CHAR_P, &config, "file", "configuration file" } */ typedef struct { char ar_short_arg; /* short argument, 'd' if '-d' */ const char *ar_long_arg; /* long version of arg, '--delete' */ unsigned int ar_type; /* type of option, see values below */ ARGV_PNT ar_variable; /* address of associated variable */ const char *ar_var_label; /* label for variable description */ const char *ar_comment; /* comment for usage message */ } argv_t; /* * Argument array type. when ARGV_FLAG_ARRAY is |'d with the ar_type * in the above structure then multiple instances of the option are * allowed and each instance is stored into the following structure * that MUST be in ar_variable in the above arg_t structure. * * NOTE: after the arguments have been processed, if aa_entryn is > 0 * then aa_entries needs to be free'd by user. argv_cleanup() can be * used for this */ typedef struct { int aa_entry_n; /* number of elements in aa_entrees */ ARGV_PNT aa_entries; /* entry list specified */ } argv_array_t; /* extract the count of the elements from an argv ARRAY */ #define ARGV_ARRAY_COUNT(array) ((array).aa_entry_n) /* extract WHICH entry of TYPE from an argv ARRAY */ #define ARGV_ARRAY_ENTRY(array, type, which) \ (((type *)(array).aa_entries)[which]) /* extract a pointer to WHICH entry of TYPE from an argv ARRAY */ #define ARGV_ARRAY_ENTRY_P(array, type, which) \ (((type *)(array).aa_entries) + which) /* special ar_short_arg value to mark the last entry in the argument array */ #define ARGV_LAST ((char)255) /* * special ar_short_arg value to mark mandatory arguments (i.e. arguments that * *must* be specified. for arguments that are not optional like [-b]. * to have a variable number of mandatory args then make the last MAND * entry be a ARG_ARRAY type. */ #define ARGV_MAND ((char)254) /* * special ar_short_arg value to mark that there is the possibility of * a mandatory argument here if one is specified. */ #define ARGV_MAYBE ((char)253) /* * special ar_short_arg value to say that the previous and next arguments in * the list should not be used together. * {'a'...}, {ARG_OR}, {'b'...}, {ARG_OR}, {'c'...} means * the user should only specific -a or -b or -c but not 2 or more. */ #define ARGV_OR ((char)252) /* * special ar_short_arg value that is the same as ARGV_OR but one of the args * must be used. * {'a'...}, {ARG_ONE_OF}, {'b'...}, {ARG_ONE_OF}, {'c'...} means * the user must specify one of -a or -b or -c but not 2 or more. * ARGV_XOR is there for compatibility with older versions. */ #define ARGV_ONE_OF ((char)251) #define ARGV_XOR ((char)251) /* * ar_type values of arg_t * NOTE: if this list is changed, some defines in argv_loc need to be changed */ #define ARGV_BOOL 1 /* boolean type, sets to ARGV_TRUE */ #define ARGV_BOOL_NEG 2 /* like bool but sets to ARGV_FALSE */ #define ARGV_BOOL_ARG 3 /* like bool but takes a yes/no arg */ #define ARGV_CHAR 4 /* single character */ #define ARGV_CHAR_P 5 /* same as STRING */ #define ARGV_SHORT 6 /* short integer number */ #define ARGV_U_SHORT 7 /* unsigned short integer number */ #define ARGV_INT 8 /* integer number */ #define ARGV_U_INT 9 /* unsigned integer number */ #define ARGV_LONG 10 /* long integer number */ #define ARGV_U_LONG 11 /* unsinged long integer number */ #define ARGV_FLOAT 12 /* floating pointer number */ #define ARGV_DOUBLE 13 /* double floating pointer number */ #define ARGV_BIN 14 /* binary number (0s and 1s) */ #define ARGV_OCT 15 /* octal number, (base 8) */ #define ARGV_HEX 16 /* hexadecimal number, (base 16) */ #define ARGV_INCR 17 /* int arg which gets ++ each time */ #define ARGV_SIZE 18 /* long arg which knows mMbBkKgG */ #define ARGV_U_SIZE 19 /* u_long arg which knows mMbBkKgG */ #define ARGV_BOOL_INT 20 /* like bool but takes an integer var*/ #define ARGV_BOOL_INT_NEG 21 /* like bool-neg but with an integer */ #define ARGV_BOOL_INT_ARG 22 /* like bool-arg but with an integer */ #define ARGV_TYPE(t) ((t) & 0x3F) /* strip off all but the var type */ #define ARGV_FLAG_ARRAY (1 << 14) /* OR with type to indicate array */ #define ARGV_FLAG_MAND (1 << 13) /* OR with type to mark mandatory */ /* NOTE: other internal flags defined in argv_loc.h */ /* argv_usage which argument values */ #define ARGV_USAGE_NONE 0 /* no usage messages -- special */ #define ARGV_USAGE_SHORT 1 /* print short usage messages */ #define ARGV_USAGE_LONG 2 /* print long-format usage messages */ #define ARGV_USAGE_DEFAULT 3 /* default usage messages */ #define ARGV_USAGE_SEE 4 /* say see --usage for more info */ #define ARGV_USAGE_SHORT_REM 5 /* short + reminder how to get long */ #define ARGV_USAGE_ALL 6 /* all usage information */ /* boolean type settings */ #define ARGV_FALSE 0 #define ARGV_TRUE 1 /*<<<<<<<<<< The below prototypes are auto-generated by fillproto */ /* This is a processed version of argv[0], pre-path removed: /bin/ls -> ls */ extern char argv_program[/* PROGRAM_NAME + 1 */]; /* A global value of argv from main after argv_process has been called */ extern char **argv_argv; /* A global value of argc from main after argv_process has been called */ extern int argv_argc; /* This should be set externally to provide general program help to user */ extern char *argv_help_string; /* This should be set externally to provide version information to the user */ extern char *argv_version_string; /* * Are we running interactively? This will exit on errors. Set to * false to return error codes instead. */ extern int argv_interactive; /* * The FILE stream that argv out_puts all its errors. Set to NULL to * not dump any error messages. Default is stderr. */ extern FILE *argv_error_stream; /* * This is the error code to exit with when we have a usage error and * we are in interactive mode. */ extern int argv_error_code; /* * Set to 1 (the default) to enable the handling of -l=foo or * --logfile=foo type of arguments. Set to 0 to disable. This allows * you to specifically assign a value to an argument. */ extern int argv_close_enable_b; /* * If the library sees a "--" argument, it will turn off further * argument process. Set to 1 to enable the ability of specifying * additional "--" arguments to reenable (basically toggle on then * off) argument processing. Set to 0 (the default) to disable this * behavior. */ extern int argv_last_toggle_b; /* * Set to 1 (the default) to have the library accept multiple usage of * the same argument. Set to 0 to have the library generate an error * if you use an argument twice. */ extern int argv_multi_accept_b; /* * Set to one of the ARGV_USAGE_ defines in the argv.h file. This * tell the library what usage information to display when --usage is * specified by the user. Default is ARGV_USAGE_LONG. */ extern int argv_usage_type; /* * Set to one of the ARGV_USAGE_ defines in the argv.h file. This * tell the library what usage information to display when an error is * encountered. The usage information accompanies the error message. * Default is ARGV_USAGE_SEE. */ extern int argv_error_type; /* * Set to 1 (the default) if you want the library look for associated * arguments from the associated program's environmental variable. If * set the 0 then no environmental variable will be used. If you are * running program foo then the library will look for the * environmental variable ARGV_foo and will add those to the argument * list specified on the command line. By default they will be * inserted in front of those on the command line unless the * argv_env_after_b is set to 1. * * NOTE: this is set by argv_process automatically. If you do not * want this behavior, you should use argv_process_no_env. */ extern int argv_process_env_b; /* * Set to 1 if you want the library to append the arguments from the * program's environmental variable after those specified on the * command line. If set the 0 (the default) then they will be * inserted before those specified on the command line. See * argv_process_env_b for more information. */ extern int argv_env_after_b; /* * int argv_process_no_env * * DESCRIPTION: * * Process the user arguments with an argv_t structure array. Like * argv_process_args but without the processing of the argv * environmental variables. * * RETURNS: * * Success - 0 * * Failure - -1 * * ARGUMENTS: * * args - Array of argv_t structures. * * arg_c - Number of arguments in the argv array. * * argv - Array of character pointers terminated by 0L. */ extern int argv_process_no_env(argv_t *args, const int arg_c, char **argv); /* * int argv_process * * DESCRIPTION: * * Processes a number of arguments depending on the argument array. * This routine will not modify the argv array in any way. * * NOTE: it will modify the args array by setting various flags in the * type field. returns 0 if no error else -1. * * ARGUMENTS: * * args - Array of argv_t structures that we are using to process the * user argument array. If null then an empty array is used. * * argc - Number of arguments in the argv argument array. * * argv - Array of character pointer arguments terminated by a 0L. */ extern int argv_process(argv_t *args, const int argc, char **argv); /* * int argv_usage * * DESCRIPTION: * * Print the standard usage messages for our argument array. You can * specify whether you want to see a short or long usage messages. * * NOTE: if this is called before argv_process then the program name * may be invalid. * * RETURNS: * * Success - 0 * * Failure - -1 * * ARGUMENTS: * * args - Our argument array to print the usage messages about. If * null then an empty array is used. * * which - Either ARGV_USAGE_SHORT (for short usage messages), * ARGV_USAGE_LONG (for long usage messages), or ARGV_USAGE_DEFAULT * (the user's default either long or short). */ extern int argv_usage(const argv_t *args, const int which); /* * int argv_was_used * * DESCRIPTION: * * See if an argument was used in a previous call to argv_process. * * RETURNS: * * 1 if yes it was used, else 0 if not. * * ARGUMENTS: * * args - Argument list to search. * * short_arg - Short argument to see if it was used. */ extern int argv_was_used(const argv_t *args, const char short_arg); /* * int argv_long_was_used * * DESCRIPTION: * * See if a long argument was used in a previous call to argv_process. * * RETURNS: * * 1 if yes it was used, else 0 if not. * * ARGUMENTS: * * args - Argument list to search. * * long_arg - Long argument to see if it was used. */ extern int argv_long_was_used(const argv_t *args, const char *long_arg); /* * int argv_entry_was_used * * DESCRIPTION: * * See if an entry in the argument array was used in a previous call * to argv_process. * * RETURNS: * * 1 if yes it was used, else 0 if not. * * ARGUMENTS: * * argv_entry_p - Pointer to an entry in a argv_t list. */ extern int argv_entry_was_used(const argv_t *argv_entry_p); /* * void argv_cleanup * * DESCRIPTION: * * Frees up any allocations associated with the argument array during * argv_process. This should be done at the end of the program or * after all the arguments have been referenced. * * RETURNS: * * None. * * ARGUMENTS: * * args - Argument array we are cleaning up. */ extern void argv_cleanup(const argv_t *args); /* * int argv_copy_args * * DESCRIPTION: * * Copy all the arguements (not including the 0th) one after the other * into the user specified buffer. * * NOTE: you can get the 0th argument from argv_argv[0] or * argv_program. * * RETURNS: * * Success - 0 * * Failure - -1 * * ARGUMENTS: * * buf - Buffer to copy all of the user arguments into. * * buf_size - Size of the buffer. */ extern int argv_copy_args(char *buf, const int buf_size); /* * int argv_value_string * * DESCRIPTION: * * Convert the value of a RC entry to its string equivalent in the * buffer provided. * * RETURNS: * * Length of bytes copied into the buffer. * * ARGUMENTS: * * argv_entry_p - Pointer to an entry in a argv_t list. * * buf - Buffer to convert the value into. * * buf_size - Size of the buffer. */ extern int argv_value_string(const argv_t *argv_entry_p, char *buf, const int buf_size); /* * int argv_type_info * * DESCRIPTION: * * Get internal information about the type of the argument. * * RETURNS: * * The name of the type. * * ARGUMENTS: * * type - Number of argument type. * * size_p - Pointer to an unsigned integer which, if not NULL, will be * set with the size of the type. * * desc_p - Pointer to a constant character pointer which, if not * NULL, will be pointed to a description of the type. */ extern const char *argv_type_info(const unsigned int type, unsigned int *size_p, const char **desc_p); /*<<<<<<<<<< This is end of the auto-generated output from fillproto. */ #ifdef __cplusplus #if 0 { #endif } #endif #endif /* ! __ARGV_H__ */ mtbl-0.5/libmy/argv_loc.h000066400000000000000000000171131227302760400154030ustar00rootroot00000000000000/* * Local defines for the argv module * * Copyright 2000 by Gray Watson * * This file is part of the argv library. * * Permission to use, copy, modify, and distribute this software for * any purpose and without fee is hereby granted, provided that the * above copyright notice and this permission notice appear in all * copies, and that the name of Gray Watson not be used in advertising * or publicity pertaining to distribution of the document or software * without specific, written prior permission. * * Gray Watson makes no representations about the suitability of the * software described herein for any purpose. It is provided "as is" * without express or implied warranty. * * The author may be contacted via http://256.com/gray/ * * $Id: argv_loc.h,v 1.50 2007/01/08 20:32:57 gray Exp $ */ #ifndef __ARGV_LOC_H__ #define __ARGV_LOC_H__ #ifdef __cplusplus extern "C" { #if 0 } #endif #endif #include "argv.h" /* to get the types */ /* * some compilation options */ /* to include RCS ids in the code */ #ifndef INCLUDE_RCS_IDS #define INCLUDE_RCS_IDS 1 #endif /* * generic constants */ /* special consideration for NULL. some compilers bitch if I redefine it */ #ifndef NULL #define NULL 0L #endif #undef MIN #define MIN(a,b) ((a) < (b) ? (a) : (b)) #undef NOERROR #define NOERROR 0 #undef ERROR #define ERROR (-1) #undef STDIN #define STDIN 0 /* * local argv defines */ #define ERROR_STREAM_INIT (FILE *)0x1 /* hack to init stderr FILE* */ #define NO_VALUE (-1) /* no mandatory args value */ #define ARRAY_INCR 10 /* increment by 10 every 10 */ #define SCREEN_WIDTH 79 /* width of screen till wrap */ #define BITS_IN_BYTE 8 /* bits in a byte */ #define SPECIAL_CHARS "e\033^^\"\"''\\\\n\nr\rt\tb\bf\fa\007" #define DUMP_SPACE_BUF 128 /* space for memory dump */ #define ARG_MALLOC_INCR 20 /* alloc in 10 increments */ #define FILE_LINE_SIZE 1024 /* max size of file lines */ /* internal flags set in the ar_type field */ /* NOTE: other external flags defined in argv.h */ #define ARGV_FLAG_USED (1 << 12) /* if arg has been specified */ /* error messages */ #define USAGE_ERROR_NAME "usage problem" #define INTERNAL_ERROR_NAME "internal argv error" /* * settings to vary the output of the argument routines */ #define PROGRAM_NAME 256 /* max size of program name */ #define EXIT_CODE 1 /* argv exit code for errors */ /* global env settings */ #define GLOBAL_NAME "GLOBAL_ARGV" /* global argv env name */ #define GLOBAL_CLOSE "close=" /* close arg setting */ #define GLOBAL_ENV "env=" /* env setting */ #define GLOBAL_ERROR "error=" /* error setting */ #define GLOBAL_MULTI "multi=" /* multi setting */ #define GLOBAL_USAGE "usage=" /* usage setting */ #define GLOBAL_LASTTOG "lasttog=" /* last-arg toggle */ /* special argument definitions */ #define LAST_ARG "--" /* arg to mark end of args */ #define LONG_PREFIX "--" /* prefix for long args */ #define SHORT_PREFIX "-" /* prefix for short args */ #define UNKNOWN_ARG "??" /* unknown argument output */ #define ARG_EQUALS '=' /* to assign value to option */ #define NUMBER_ARG_CHARS "0123456789+-." /* characters in numbers */ /* how to produce the env var using sprintf and the argv_program variable */ #define ENVIRON_FORMAT "ARGV_%s" /* special short-argument strings */ #define USAGE_CHAR_ARG '?' /* default short-opt usage */ /* special long-argument strings */ #define DISPLAY_ARG "argv-display" /* display arg variable vals */ #define FILE_ARG "argv-file" /* read args from file */ #define HELP_ARG "help" /* default help option */ #define USAGE_ARG "usage" /* default usage option */ #define USAGE_SHORT_ARG "usage-short" /* default usage-short opt */ #define USAGE_LONG_ARG "usage-long" /* default usage-long opt */ #define USAGE_ALL_ARG "usage-all" /* default usage-all opt */ #define VERSION_ARG "version" /* default version option */ /* spacing on line for usage messages */ #define SHORT_COLUMN 0 /* spaces to indent for short-args */ #define LONG_COLUMN 18 /* column for long options */ #define COMMENT_COLUMN 34 /* column for comments */ /* some in-line "labels" for comments */ #define USAGE_LABEL "Usage: " /* usage message start */ #define LONG_LABEL "or " /* put before long-option */ #define COMMENT_LABEL "" /* put before comments */ #define ARRAY_LABEL " array" /* put after displayed type */ #define BOOL_ARG_LABEL "yes|no" /* label for bool-arg arg */ /* some sizeof defines */ #define SHORT_PREFIX_LENGTH (sizeof(SHORT_PREFIX) - 1) #define LONG_PREFIX_LENGTH (sizeof(LONG_PREFIX) - 1) #define USAGE_LABEL_LENGTH (sizeof(USAGE_LABEL) - 1) #define COMMENT_LABEL_LENGTH (sizeof(COMMENT_LABEL) - 1) #define LONG_LABEL_LENGTH (sizeof(LONG_LABEL) - 1) #define UNKNOWN_ARG_LENGTH (sizeof(UNKNOWN_ARG) - 1) #define BOOL_ARG_LENGTH (sizeof(BOOL_ARG_LABEL) - 1) #define HAS_ARG(type) (! (ARGV_TYPE(type) == ARGV_BOOL \ || ARGV_TYPE(type) == ARGV_BOOL_NEG \ || ARGV_TYPE(type) == ARGV_INCR \ || ARGV_TYPE(type) == ARGV_BOOL_INT \ || ARGV_TYPE(type) == ARGV_BOOL_INT_NEG)) /******************************** argv types *********************************/ /* strcture defining argv types */ typedef struct { unsigned int at_value; /* value of the type */ const char *at_name; /* name of the type */ unsigned int at_size; /* size of type */ const char *at_desc; /* description of the type */ } argv_type_t; static argv_type_t argv_types[] = { { ARGV_BOOL, "flag", sizeof(char), "if option used, set variable to 1" }, { ARGV_BOOL_NEG, "negative flag", sizeof(int), "if option used, set variable to 0" }, { ARGV_BOOL_ARG, "flag with arg", sizeof(char), "like boolean but with an argument, true/yes/1 sets var to 1" }, { ARGV_CHAR, "character", sizeof(char), "single character" }, { ARGV_CHAR_P, "string", sizeof(char *), "multiple characters terminated with a '\\0'" }, { ARGV_SHORT, "short integer", sizeof(short), "decimal short-sized integer value" }, { ARGV_U_SHORT, "unsigned short integer", sizeof(unsigned short), "decimal unsigned short-sized integer value" }, { ARGV_INT, "integer", sizeof(int), "decimal integer value" }, { ARGV_U_INT, "unsigned integer", sizeof(unsigned int), "decimal unsigned integer value" }, { ARGV_LONG, "long integer", sizeof(long), "decimal long-sized integer value" }, { ARGV_U_LONG, "unsigned long", sizeof(unsigned long), "decimal unsigned long-sized integer value" }, { ARGV_FLOAT, "floating point", sizeof(float), "real number with decimal point" }, { ARGV_DOUBLE, "double floating point", sizeof(double), "double precision real number with decimal point" }, { ARGV_BIN, "binary", sizeof(int), "base 2 value with digits of 0 or 1" }, { ARGV_OCT, "octal", sizeof(int), "base 8 value with digits from 0-7" }, { ARGV_HEX, "hexadecimal", sizeof(int), "base 16 value with digits from 0-9, A-F" }, { ARGV_INCR, "increment", sizeof(int), "increment variable each time option used" }, { ARGV_SIZE, "long size", sizeof(long), "size as long int + [bkmg] b=byte,k=kilo,m=meg,g=gig" }, { ARGV_U_SIZE, "unsigned long size", sizeof(unsigned long), "size as unsigned long int + [bkmg] b=byte,k=kilo,m=meg,g=gig" }, { ARGV_BOOL_INT, "integer boolean", sizeof(int), "if option used, set integer variable to 1" }, { ARGV_BOOL_INT_NEG, "integer boolean", sizeof(int), "if option used, set integer variable to 0" }, { ARGV_BOOL_INT_ARG, "integer boolean", sizeof(int), "like boolean but with an argument, true/yes/1 sets integer var to 1" }, { 0, NULL, 0, NULL } }; #ifdef __cplusplus #if 0 { #endif } #endif #endif /* ! __ARGV_LOC_H__ */ mtbl-0.5/libmy/atomic.h000066400000000000000000000057401227302760400150660ustar00rootroot00000000000000#ifndef MY_ATOMIC_H #define MY_ATOMIC_H /* public domain, from http://golubenco.org/2007/06/14/atomic-operations/ */ /** * Atomic type. */ typedef struct { volatile int counter; } atomic_t; #define ATOMIC_INIT(i) { (i) } /** * Read atomic variable * @param v pointer of type atomic_t * * Atomically reads the value of @v. */ #define atomic_read(v) ((v)->counter) /** * Set atomic variable * @param v pointer of type atomic_t * @param i required value */ #define atomic_set(v, i) (((v)->counter) = (i)) /** * Add to the atomic variable * @param i integer value to add * @param v pointer of type atomic_t */ static inline void atomic_add(int i, atomic_t *v) { (void)__sync_add_and_fetch(&v->counter, i); } /** * Subtract the atomic variable * @param i integer value to subtract * @param v pointer of type atomic_t * * Atomically subtracts @i from @v. */ static inline void atomic_sub(int i, atomic_t *v) { (void)__sync_sub_and_fetch(&v->counter, i); } /** * Read atomic variable and reset to zero. * * @param v pointer of type atomic_t */ static inline int atomic_zero(atomic_t *v) { return (__sync_fetch_and_and(&v->counter, 0)); } /** * Subtract value from variable and test result * @param i integer value to subtract * @param v pointer of type atomic_t * * Atomically subtracts @i from @v and returns * true if the result is zero, or false for all * other cases. */ static inline int atomic_sub_and_test(int i, atomic_t *v) { return !(__sync_sub_and_fetch(&v->counter, i)); } /** * Increment atomic variable * @param v pointer of type atomic_t * * Atomically increments @v by 1. */ static inline void atomic_inc(atomic_t *v) { (void)__sync_fetch_and_add(&v->counter, 1); } /** * @brief decrement atomic variable * @param v: pointer of type atomic_t * * Atomically decrements @v by 1. Note that the guaranteed * useful range of an atomic_t is only 24 bits. */ static inline void atomic_dec(atomic_t *v) { (void)__sync_fetch_and_sub(&v->counter, 1); } /** * @brief Decrement and test * @param v pointer of type atomic_t * * Atomically decrements @v by 1 and * returns true if the result is 0, or false for all other * cases. */ static inline int atomic_dec_and_test(atomic_t *v) { return !(__sync_sub_and_fetch(&v->counter, 1)); } /** * @brief Increment and test * @param v pointer of type atomic_t * * Atomically increments @v by 1 * and returns true if the result is zero, or false for all * other cases. */ static inline int atomic_inc_and_test(atomic_t *v) { return !(__sync_add_and_fetch(&v->counter, 1)); } /** * @brief add and test if negative * @param v pointer of type atomic_t * @param i integer value to add * * Atomically adds @i to @v and returns true * if the result is negative, or false when * result is greater than or equal to zero. */ static inline int atomic_add_negative(int i, atomic_t *v) { return (__sync_add_and_fetch(&v->counter, i) < 0); } #endif /* MY_ATOMIC_H */ mtbl-0.5/libmy/atomic_64.h000066400000000000000000000061121227302760400153710ustar00rootroot00000000000000#ifndef MY_ATOMIC_64_H #define MY_ATOMIC_64_H #include /* public domain, from http://golubenco.org/2007/06/14/atomic-operations/ */ /** * Atomic 64-bit type. */ typedef struct { volatile int64_t counter; } atomic64_t; #define ATOMIC64_INIT(i) { (i) } /** * Read atomic variable * @param v pointer of type atomic64_t * * Atomically reads the value of @v. */ #define atomic64_read(v) ((v)->counter) /** * Set atomic variable * @param v pointer of type atomic64_t * @param i required value */ #define atomic64_set(v, i) (((v)->counter) = (i)) /** * Add to the atomic variable * @param i integer value to add * @param v pointer of type atomic64_t */ static inline void atomic64_add(int i, atomic64_t *v) { (void)__sync_add_and_fetch(&v->counter, i); } /** * Subtract the atomic variable * @param i integer value to subtract * @param v pointer of type atomic64_t * * Atomically subtracts @i from @v. */ static inline void atomic64_sub(int i, atomic64_t *v) { (void)__sync_sub_and_fetch(&v->counter, i); } /** * Read atomic variable and reset to zero. * * @param v pointer of type atomic64_t */ static inline int atomic64_zero(atomic64_t *v) { return (__sync_fetch_and_and(&v->counter, 0)); } /** * Subtract value from variable and test result * @param i integer value to subtract * @param v pointer of type atomic64_t * * Atomically subtracts @i from @v and returns * true if the result is zero, or false for all * other cases. */ static inline int atomic64_sub_and_test(int i, atomic64_t *v) { return !(__sync_sub_and_fetch(&v->counter, i)); } /** * Increment atomic variable * @param v pointer of type atomic64_t * * Atomically increments @v by 1. */ static inline void atomic64_inc(atomic64_t *v) { (void)__sync_fetch_and_add(&v->counter, 1); } /** * @brief decrement atomic variable * @param v: pointer of type atomic64_t * * Atomically decrements @v by 1. Note that the guaranteed * useful range of an atomic64_t is only 24 bits. */ static inline void atomic64_dec(atomic64_t *v) { (void)__sync_fetch_and_sub(&v->counter, 1); } /** * @brief Decrement and test * @param v pointer of type atomic64_t * * Atomically decrements @v by 1 and * returns true if the result is 0, or false for all other * cases. */ static inline int atomic64_dec_and_test(atomic64_t *v) { return !(__sync_sub_and_fetch(&v->counter, 1)); } /** * @brief Increment and test * @param v pointer of type atomic64_t * * Atomically increments @v by 1 * and returns true if the result is zero, or false for all * other cases. */ static inline int atomic64_inc_and_test(atomic64_t *v) { return !(__sync_add_and_fetch(&v->counter, 1)); } /** * @brief add and test if negative * @param v pointer of type atomic64_t * @param i integer value to add * * Atomically adds @i to @v and returns true * if the result is negative, or false when * result is greater than or equal to zero. */ static inline int atomic64_add_negative(int i, atomic64_t *v) { return (__sync_add_and_fetch(&v->counter, i) < 0); } #endif /* MY_ATOMIC_H */ mtbl-0.5/libmy/atomic_ptr.h000066400000000000000000000006471227302760400157540ustar00rootroot00000000000000#ifndef MY_ATOMIC_PTR_H #define MY_ATOMIC_PTR_H typedef struct { volatile void *ptr; } atomic_ptr_t; static inline void * atomic_ptr_get(atomic_ptr_t *v) { return ((void *) (__sync_fetch_and_add(&v->ptr, 0))); } static inline bool atomic_ptr_set(atomic_ptr_t *v, void *new_ptr) { void *old_ptr = atomic_ptr_get(v); return (__sync_bool_compare_and_swap(&v->ptr, old_ptr, new_ptr)); } #endif /* MY_ATOMIC_PTR_H */ mtbl-0.5/libmy/crc32c-slicing.c000066400000000000000000001515731227302760400163200ustar00rootroot00000000000000/* * Copyright (c) 2008-2011 WiredTiger, Inc. * * This is free and unencumbered software released into the public domain. * * Anyone is free to copy, modify, publish, use, compile, sell, or * distribute this software, either in source code form or as a compiled * binary, for any purpose, commercial or non-commercial, and by any * means. * * In jurisdictions that recognize copyright laws, the author or authors * of this software dedicate any and all copyright interest in the * software to the public domain. We make this dedication for the benefit * of the public at large and to the detriment of our heirs and * successors. We intend this dedication to be an overt act of * relinquishment in perpetuity of all present and future rights to this * software under copyright law. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ #include #include uint32_t my_crc32c_slicing(const uint8_t *, size_t); static const uint32_t g_crc_slicing[8][256] = { #ifdef WORDS_BIGENDIAN /* * Big endian tables have entries that are byte reversed from little * endian tables. */ { 0x00000000, 0x03836bf2, 0xf7703be1, 0xf4f35013, 0x1f979ac7, 0x1c14f135, 0xe8e7a126, 0xeb64cad4, 0xcf58d98a, 0xccdbb278, 0x3828e26b, 0x3bab8999, 0xd0cf434d, 0xd34c28bf, 0x27bf78ac, 0x243c135e, 0x6fc75e10, 0x6c4435e2, 0x98b765f1, 0x9b340e03, 0x7050c4d7, 0x73d3af25, 0x8720ff36, 0x84a394c4, 0xa09f879a, 0xa31cec68, 0x57efbc7b, 0x546cd789, 0xbf081d5d, 0xbc8b76af, 0x487826bc, 0x4bfb4d4e, 0xde8ebd20, 0xdd0dd6d2, 0x29fe86c1, 0x2a7ded33, 0xc11927e7, 0xc29a4c15, 0x36691c06, 0x35ea77f4, 0x11d664aa, 0x12550f58, 0xe6a65f4b, 0xe52534b9, 0x0e41fe6d, 0x0dc2959f, 0xf931c58c, 0xfab2ae7e, 0xb149e330, 0xb2ca88c2, 0x4639d8d1, 0x45bab323, 0xaede79f7, 0xad5d1205, 0x59ae4216, 0x5a2d29e4, 0x7e113aba, 0x7d925148, 0x8961015b, 0x8ae26aa9, 0x6186a07d, 0x6205cb8f, 0x96f69b9c, 0x9575f06e, 0xbc1d7b41, 0xbf9e10b3, 0x4b6d40a0, 0x48ee2b52, 0xa38ae186, 0xa0098a74, 0x54fada67, 0x5779b195, 0x7345a2cb, 0x70c6c939, 0x8435992a, 0x87b6f2d8, 0x6cd2380c, 0x6f5153fe, 0x9ba203ed, 0x9821681f, 0xd3da2551, 0xd0594ea3, 0x24aa1eb0, 0x27297542, 0xcc4dbf96, 0xcfced464, 0x3b3d8477, 0x38beef85, 0x1c82fcdb, 0x1f019729, 0xebf2c73a, 0xe871acc8, 0x0315661c, 0x00960dee, 0xf4655dfd, 0xf7e6360f, 0x6293c661, 0x6110ad93, 0x95e3fd80, 0x96609672, 0x7d045ca6, 0x7e873754, 0x8a746747, 0x89f70cb5, 0xadcb1feb, 0xae487419, 0x5abb240a, 0x59384ff8, 0xb25c852c, 0xb1dfeede, 0x452cbecd, 0x46afd53f, 0x0d549871, 0x0ed7f383, 0xfa24a390, 0xf9a7c862, 0x12c302b6, 0x11406944, 0xe5b33957, 0xe63052a5, 0xc20c41fb, 0xc18f2a09, 0x357c7a1a, 0x36ff11e8, 0xdd9bdb3c, 0xde18b0ce, 0x2aebe0dd, 0x29688b2f, 0x783bf682, 0x7bb89d70, 0x8f4bcd63, 0x8cc8a691, 0x67ac6c45, 0x642f07b7, 0x90dc57a4, 0x935f3c56, 0xb7632f08, 0xb4e044fa, 0x401314e9, 0x43907f1b, 0xa8f4b5cf, 0xab77de3d, 0x5f848e2e, 0x5c07e5dc, 0x17fca892, 0x147fc360, 0xe08c9373, 0xe30ff881, 0x086b3255, 0x0be859a7, 0xff1b09b4, 0xfc986246, 0xd8a47118, 0xdb271aea, 0x2fd44af9, 0x2c57210b, 0xc733ebdf, 0xc4b0802d, 0x3043d03e, 0x33c0bbcc, 0xa6b54ba2, 0xa5362050, 0x51c57043, 0x52461bb1, 0xb922d165, 0xbaa1ba97, 0x4e52ea84, 0x4dd18176, 0x69ed9228, 0x6a6ef9da, 0x9e9da9c9, 0x9d1ec23b, 0x767a08ef, 0x75f9631d, 0x810a330e, 0x828958fc, 0xc97215b2, 0xcaf17e40, 0x3e022e53, 0x3d8145a1, 0xd6e58f75, 0xd566e487, 0x2195b494, 0x2216df66, 0x062acc38, 0x05a9a7ca, 0xf15af7d9, 0xf2d99c2b, 0x19bd56ff, 0x1a3e3d0d, 0xeecd6d1e, 0xed4e06ec, 0xc4268dc3, 0xc7a5e631, 0x3356b622, 0x30d5ddd0, 0xdbb11704, 0xd8327cf6, 0x2cc12ce5, 0x2f424717, 0x0b7e5449, 0x08fd3fbb, 0xfc0e6fa8, 0xff8d045a, 0x14e9ce8e, 0x176aa57c, 0xe399f56f, 0xe01a9e9d, 0xabe1d3d3, 0xa862b821, 0x5c91e832, 0x5f1283c0, 0xb4764914, 0xb7f522e6, 0x430672f5, 0x40851907, 0x64b90a59, 0x673a61ab, 0x93c931b8, 0x904a5a4a, 0x7b2e909e, 0x78adfb6c, 0x8c5eab7f, 0x8fddc08d, 0x1aa830e3, 0x192b5b11, 0xedd80b02, 0xee5b60f0, 0x053faa24, 0x06bcc1d6, 0xf24f91c5, 0xf1ccfa37, 0xd5f0e969, 0xd673829b, 0x2280d288, 0x2103b97a, 0xca6773ae, 0xc9e4185c, 0x3d17484f, 0x3e9423bd, 0x756f6ef3, 0x76ec0501, 0x821f5512, 0x819c3ee0, 0x6af8f434, 0x697b9fc6, 0x9d88cfd5, 0x9e0ba427, 0xba37b779, 0xb9b4dc8b, 0x4d478c98, 0x4ec4e76a, 0xa5a02dbe, 0xa623464c, 0x52d0165f, 0x51537dad },{ 0x00000000, 0x7798a213, 0xee304527, 0x99a8e734, 0xdc618a4e, 0xabf9285d, 0x3251cf69, 0x45c96d7a, 0xb8c3149d, 0xcf5bb68e, 0x56f351ba, 0x216bf3a9, 0x64a29ed3, 0x133a3cc0, 0x8a92dbf4, 0xfd0a79e7, 0x81f1c53f, 0xf669672c, 0x6fc18018, 0x1859220b, 0x5d904f71, 0x2a08ed62, 0xb3a00a56, 0xc438a845, 0x3932d1a2, 0x4eaa73b1, 0xd7029485, 0xa09a3696, 0xe5535bec, 0x92cbf9ff, 0x0b631ecb, 0x7cfbbcd8, 0x02e38b7f, 0x757b296c, 0xecd3ce58, 0x9b4b6c4b, 0xde820131, 0xa91aa322, 0x30b24416, 0x472ae605, 0xba209fe2, 0xcdb83df1, 0x5410dac5, 0x238878d6, 0x664115ac, 0x11d9b7bf, 0x8871508b, 0xffe9f298, 0x83124e40, 0xf48aec53, 0x6d220b67, 0x1abaa974, 0x5f73c40e, 0x28eb661d, 0xb1438129, 0xc6db233a, 0x3bd15add, 0x4c49f8ce, 0xd5e11ffa, 0xa279bde9, 0xe7b0d093, 0x90287280, 0x098095b4, 0x7e1837a7, 0x04c617ff, 0x735eb5ec, 0xeaf652d8, 0x9d6ef0cb, 0xd8a79db1, 0xaf3f3fa2, 0x3697d896, 0x410f7a85, 0xbc050362, 0xcb9da171, 0x52354645, 0x25ade456, 0x6064892c, 0x17fc2b3f, 0x8e54cc0b, 0xf9cc6e18, 0x8537d2c0, 0xf2af70d3, 0x6b0797e7, 0x1c9f35f4, 0x5956588e, 0x2ecefa9d, 0xb7661da9, 0xc0febfba, 0x3df4c65d, 0x4a6c644e, 0xd3c4837a, 0xa45c2169, 0xe1954c13, 0x960dee00, 0x0fa50934, 0x783dab27, 0x06259c80, 0x71bd3e93, 0xe815d9a7, 0x9f8d7bb4, 0xda4416ce, 0xaddcb4dd, 0x347453e9, 0x43ecf1fa, 0xbee6881d, 0xc97e2a0e, 0x50d6cd3a, 0x274e6f29, 0x62870253, 0x151fa040, 0x8cb74774, 0xfb2fe567, 0x87d459bf, 0xf04cfbac, 0x69e41c98, 0x1e7cbe8b, 0x5bb5d3f1, 0x2c2d71e2, 0xb58596d6, 0xc21d34c5, 0x3f174d22, 0x488fef31, 0xd1270805, 0xa6bfaa16, 0xe376c76c, 0x94ee657f, 0x0d46824b, 0x7ade2058, 0xf9fac3fb, 0x8e6261e8, 0x17ca86dc, 0x605224cf, 0x259b49b5, 0x5203eba6, 0xcbab0c92, 0xbc33ae81, 0x4139d766, 0x36a17575, 0xaf099241, 0xd8913052, 0x9d585d28, 0xeac0ff3b, 0x7368180f, 0x04f0ba1c, 0x780b06c4, 0x0f93a4d7, 0x963b43e3, 0xe1a3e1f0, 0xa46a8c8a, 0xd3f22e99, 0x4a5ac9ad, 0x3dc26bbe, 0xc0c81259, 0xb750b04a, 0x2ef8577e, 0x5960f56d, 0x1ca99817, 0x6b313a04, 0xf299dd30, 0x85017f23, 0xfb194884, 0x8c81ea97, 0x15290da3, 0x62b1afb0, 0x2778c2ca, 0x50e060d9, 0xc94887ed, 0xbed025fe, 0x43da5c19, 0x3442fe0a, 0xadea193e, 0xda72bb2d, 0x9fbbd657, 0xe8237444, 0x718b9370, 0x06133163, 0x7ae88dbb, 0x0d702fa8, 0x94d8c89c, 0xe3406a8f, 0xa68907f5, 0xd111a5e6, 0x48b942d2, 0x3f21e0c1, 0xc22b9926, 0xb5b33b35, 0x2c1bdc01, 0x5b837e12, 0x1e4a1368, 0x69d2b17b, 0xf07a564f, 0x87e2f45c, 0xfd3cd404, 0x8aa47617, 0x130c9123, 0x64943330, 0x215d5e4a, 0x56c5fc59, 0xcf6d1b6d, 0xb8f5b97e, 0x45ffc099, 0x3267628a, 0xabcf85be, 0xdc5727ad, 0x999e4ad7, 0xee06e8c4, 0x77ae0ff0, 0x0036ade3, 0x7ccd113b, 0x0b55b328, 0x92fd541c, 0xe565f60f, 0xa0ac9b75, 0xd7343966, 0x4e9cde52, 0x39047c41, 0xc40e05a6, 0xb396a7b5, 0x2a3e4081, 0x5da6e292, 0x186f8fe8, 0x6ff72dfb, 0xf65fcacf, 0x81c768dc, 0xffdf5f7b, 0x8847fd68, 0x11ef1a5c, 0x6677b84f, 0x23bed535, 0x54267726, 0xcd8e9012, 0xba163201, 0x471c4be6, 0x3084e9f5, 0xa92c0ec1, 0xdeb4acd2, 0x9b7dc1a8, 0xece563bb, 0x754d848f, 0x02d5269c, 0x7e2e9a44, 0x09b63857, 0x901edf63, 0xe7867d70, 0xa24f100a, 0xd5d7b219, 0x4c7f552d, 0x3be7f73e, 0xc6ed8ed9, 0xb1752cca, 0x28ddcbfe, 0x5f4569ed, 0x1a8c0497, 0x6d14a684, 0xf4bc41b0, 0x8324e3a3 },{ 0x00000000, 0x7e9241a5, 0x0d526f4f, 0x73c02eea, 0x1aa4de9e, 0x64369f3b, 0x17f6b1d1, 0x6964f074, 0xc53e5138, 0xbbac109d, 0xc86c3e77, 0xb6fe7fd2, 0xdf9a8fa6, 0xa108ce03, 0xd2c8e0e9, 0xac5aa14c, 0x8a7da270, 0xf4efe3d5, 0x872fcd3f, 0xf9bd8c9a, 0x90d97cee, 0xee4b3d4b, 0x9d8b13a1, 0xe3195204, 0x4f43f348, 0x31d1b2ed, 0x42119c07, 0x3c83dda2, 0x55e72dd6, 0x2b756c73, 0x58b54299, 0x2627033c, 0x14fb44e1, 0x6a690544, 0x19a92bae, 0x673b6a0b, 0x0e5f9a7f, 0x70cddbda, 0x030df530, 0x7d9fb495, 0xd1c515d9, 0xaf57547c, 0xdc977a96, 0xa2053b33, 0xcb61cb47, 0xb5f38ae2, 0xc633a408, 0xb8a1e5ad, 0x9e86e691, 0xe014a734, 0x93d489de, 0xed46c87b, 0x8422380f, 0xfab079aa, 0x89705740, 0xf7e216e5, 0x5bb8b7a9, 0x252af60c, 0x56ead8e6, 0x28789943, 0x411c6937, 0x3f8e2892, 0x4c4e0678, 0x32dc47dd, 0xd98065c7, 0xa7122462, 0xd4d20a88, 0xaa404b2d, 0xc324bb59, 0xbdb6fafc, 0xce76d416, 0xb0e495b3, 0x1cbe34ff, 0x622c755a, 0x11ec5bb0, 0x6f7e1a15, 0x061aea61, 0x7888abc4, 0x0b48852e, 0x75dac48b, 0x53fdc7b7, 0x2d6f8612, 0x5eafa8f8, 0x203de95d, 0x49591929, 0x37cb588c, 0x440b7666, 0x3a9937c3, 0x96c3968f, 0xe851d72a, 0x9b91f9c0, 0xe503b865, 0x8c674811, 0xf2f509b4, 0x8135275e, 0xffa766fb, 0xcd7b2126, 0xb3e96083, 0xc0294e69, 0xbebb0fcc, 0xd7dfffb8, 0xa94dbe1d, 0xda8d90f7, 0xa41fd152, 0x0845701e, 0x76d731bb, 0x05171f51, 0x7b855ef4, 0x12e1ae80, 0x6c73ef25, 0x1fb3c1cf, 0x6121806a, 0x47068356, 0x3994c2f3, 0x4a54ec19, 0x34c6adbc, 0x5da25dc8, 0x23301c6d, 0x50f03287, 0x2e627322, 0x8238d26e, 0xfcaa93cb, 0x8f6abd21, 0xf1f8fc84, 0x989c0cf0, 0xe60e4d55, 0x95ce63bf, 0xeb5c221a, 0x4377278b, 0x3de5662e, 0x4e2548c4, 0x30b70961, 0x59d3f915, 0x2741b8b0, 0x5481965a, 0x2a13d7ff, 0x864976b3, 0xf8db3716, 0x8b1b19fc, 0xf5895859, 0x9ceda82d, 0xe27fe988, 0x91bfc762, 0xef2d86c7, 0xc90a85fb, 0xb798c45e, 0xc458eab4, 0xbacaab11, 0xd3ae5b65, 0xad3c1ac0, 0xdefc342a, 0xa06e758f, 0x0c34d4c3, 0x72a69566, 0x0166bb8c, 0x7ff4fa29, 0x16900a5d, 0x68024bf8, 0x1bc26512, 0x655024b7, 0x578c636a, 0x291e22cf, 0x5ade0c25, 0x244c4d80, 0x4d28bdf4, 0x33bafc51, 0x407ad2bb, 0x3ee8931e, 0x92b23252, 0xec2073f7, 0x9fe05d1d, 0xe1721cb8, 0x8816eccc, 0xf684ad69, 0x85448383, 0xfbd6c226, 0xddf1c11a, 0xa36380bf, 0xd0a3ae55, 0xae31eff0, 0xc7551f84, 0xb9c75e21, 0xca0770cb, 0xb495316e, 0x18cf9022, 0x665dd187, 0x159dff6d, 0x6b0fbec8, 0x026b4ebc, 0x7cf90f19, 0x0f3921f3, 0x71ab6056, 0x9af7424c, 0xe46503e9, 0x97a52d03, 0xe9376ca6, 0x80539cd2, 0xfec1dd77, 0x8d01f39d, 0xf393b238, 0x5fc91374, 0x215b52d1, 0x529b7c3b, 0x2c093d9e, 0x456dcdea, 0x3bff8c4f, 0x483fa2a5, 0x36ade300, 0x108ae03c, 0x6e18a199, 0x1dd88f73, 0x634aced6, 0x0a2e3ea2, 0x74bc7f07, 0x077c51ed, 0x79ee1048, 0xd5b4b104, 0xab26f0a1, 0xd8e6de4b, 0xa6749fee, 0xcf106f9a, 0xb1822e3f, 0xc24200d5, 0xbcd04170, 0x8e0c06ad, 0xf09e4708, 0x835e69e2, 0xfdcc2847, 0x94a8d833, 0xea3a9996, 0x99fab77c, 0xe768f6d9, 0x4b325795, 0x35a01630, 0x466038da, 0x38f2797f, 0x5196890b, 0x2f04c8ae, 0x5cc4e644, 0x2256a7e1, 0x0471a4dd, 0x7ae3e578, 0x0923cb92, 0x77b18a37, 0x1ed57a43, 0x60473be6, 0x1387150c, 0x6d1554a9, 0xc14ff5e5, 0xbfddb440, 0xcc1d9aaa, 0xb28fdb0f, 0xdbeb2b7b, 0xa5796ade, 0xd6b94434, 0xa82b0591 },{ 0x00000000, 0xb8aa45dd, 0x812367bf, 0x39892262, 0xf331227b, 0x4b9b67a6, 0x721245c4, 0xcab80019, 0xe66344f6, 0x5ec9012b, 0x67402349, 0xdfea6694, 0x1552668d, 0xadf82350, 0x94710132, 0x2cdb44ef, 0x3db164e9, 0x851b2134, 0xbc920356, 0x0438468b, 0xce804692, 0x762a034f, 0x4fa3212d, 0xf70964f0, 0xdbd2201f, 0x637865c2, 0x5af147a0, 0xe25b027d, 0x28e30264, 0x904947b9, 0xa9c065db, 0x116a2006, 0x8b1425d7, 0x33be600a, 0x0a374268, 0xb29d07b5, 0x782507ac, 0xc08f4271, 0xf9066013, 0x41ac25ce, 0x6d776121, 0xd5dd24fc, 0xec54069e, 0x54fe4343, 0x9e46435a, 0x26ec0687, 0x1f6524e5, 0xa7cf6138, 0xb6a5413e, 0x0e0f04e3, 0x37862681, 0x8f2c635c, 0x45946345, 0xfd3e2698, 0xc4b704fa, 0x7c1d4127, 0x50c605c8, 0xe86c4015, 0xd1e56277, 0x694f27aa, 0xa3f727b3, 0x1b5d626e, 0x22d4400c, 0x9a7e05d1, 0xe75fa6ab, 0x5ff5e376, 0x667cc114, 0xded684c9, 0x146e84d0, 0xacc4c10d, 0x954de36f, 0x2de7a6b2, 0x013ce25d, 0xb996a780, 0x801f85e2, 0x38b5c03f, 0xf20dc026, 0x4aa785fb, 0x732ea799, 0xcb84e244, 0xdaeec242, 0x6244879f, 0x5bcda5fd, 0xe367e020, 0x29dfe039, 0x9175a5e4, 0xa8fc8786, 0x1056c25b, 0x3c8d86b4, 0x8427c369, 0xbdaee10b, 0x0504a4d6, 0xcfbca4cf, 0x7716e112, 0x4e9fc370, 0xf63586ad, 0x6c4b837c, 0xd4e1c6a1, 0xed68e4c3, 0x55c2a11e, 0x9f7aa107, 0x27d0e4da, 0x1e59c6b8, 0xa6f38365, 0x8a28c78a, 0x32828257, 0x0b0ba035, 0xb3a1e5e8, 0x7919e5f1, 0xc1b3a02c, 0xf83a824e, 0x4090c793, 0x51fae795, 0xe950a248, 0xd0d9802a, 0x6873c5f7, 0xa2cbc5ee, 0x1a618033, 0x23e8a251, 0x9b42e78c, 0xb799a363, 0x0f33e6be, 0x36bac4dc, 0x8e108101, 0x44a88118, 0xfc02c4c5, 0xc58be6a7, 0x7d21a37a, 0x3fc9a052, 0x8763e58f, 0xbeeac7ed, 0x06408230, 0xccf88229, 0x7452c7f4, 0x4ddbe596, 0xf571a04b, 0xd9aae4a4, 0x6100a179, 0x5889831b, 0xe023c6c6, 0x2a9bc6df, 0x92318302, 0xabb8a160, 0x1312e4bd, 0x0278c4bb, 0xbad28166, 0x835ba304, 0x3bf1e6d9, 0xf149e6c0, 0x49e3a31d, 0x706a817f, 0xc8c0c4a2, 0xe41b804d, 0x5cb1c590, 0x6538e7f2, 0xdd92a22f, 0x172aa236, 0xaf80e7eb, 0x9609c589, 0x2ea38054, 0xb4dd8585, 0x0c77c058, 0x35fee23a, 0x8d54a7e7, 0x47eca7fe, 0xff46e223, 0xc6cfc041, 0x7e65859c, 0x52bec173, 0xea1484ae, 0xd39da6cc, 0x6b37e311, 0xa18fe308, 0x1925a6d5, 0x20ac84b7, 0x9806c16a, 0x896ce16c, 0x31c6a4b1, 0x084f86d3, 0xb0e5c30e, 0x7a5dc317, 0xc2f786ca, 0xfb7ea4a8, 0x43d4e175, 0x6f0fa59a, 0xd7a5e047, 0xee2cc225, 0x568687f8, 0x9c3e87e1, 0x2494c23c, 0x1d1de05e, 0xa5b7a583, 0xd89606f9, 0x603c4324, 0x59b56146, 0xe11f249b, 0x2ba72482, 0x930d615f, 0xaa84433d, 0x122e06e0, 0x3ef5420f, 0x865f07d2, 0xbfd625b0, 0x077c606d, 0xcdc46074, 0x756e25a9, 0x4ce707cb, 0xf44d4216, 0xe5276210, 0x5d8d27cd, 0x640405af, 0xdcae4072, 0x1616406b, 0xaebc05b6, 0x973527d4, 0x2f9f6209, 0x034426e6, 0xbbee633b, 0x82674159, 0x3acd0484, 0xf075049d, 0x48df4140, 0x71566322, 0xc9fc26ff, 0x5382232e, 0xeb2866f3, 0xd2a14491, 0x6a0b014c, 0xa0b30155, 0x18194488, 0x219066ea, 0x993a2337, 0xb5e167d8, 0x0d4b2205, 0x34c20067, 0x8c6845ba, 0x46d045a3, 0xfe7a007e, 0xc7f3221c, 0x7f5967c1, 0x6e3347c7, 0xd699021a, 0xef102078, 0x57ba65a5, 0x9d0265bc, 0x25a82061, 0x1c210203, 0xa48b47de, 0x88500331, 0x30fa46ec, 0x0973648e, 0xb1d92153, 0x7b61214a, 0xc3cb6497, 0xfa4246f5, 0x42e80328 },{ 0x00000000, 0xac6f1138, 0x58df2270, 0xf4b03348, 0xb0be45e0, 0x1cd154d8, 0xe8616790, 0x440e76a8, 0x910b67c5, 0x3d6476fd, 0xc9d445b5, 0x65bb548d, 0x21b52225, 0x8dda331d, 0x796a0055, 0xd505116d, 0xd361228f, 0x7f0e33b7, 0x8bbe00ff, 0x27d111c7, 0x63df676f, 0xcfb07657, 0x3b00451f, 0x976f5427, 0x426a454a, 0xee055472, 0x1ab5673a, 0xb6da7602, 0xf2d400aa, 0x5ebb1192, 0xaa0b22da, 0x066433e2, 0x57b5a81b, 0xfbdab923, 0x0f6a8a6b, 0xa3059b53, 0xe70bedfb, 0x4b64fcc3, 0xbfd4cf8b, 0x13bbdeb3, 0xc6becfde, 0x6ad1dee6, 0x9e61edae, 0x320efc96, 0x76008a3e, 0xda6f9b06, 0x2edfa84e, 0x82b0b976, 0x84d48a94, 0x28bb9bac, 0xdc0ba8e4, 0x7064b9dc, 0x346acf74, 0x9805de4c, 0x6cb5ed04, 0xc0dafc3c, 0x15dfed51, 0xb9b0fc69, 0x4d00cf21, 0xe16fde19, 0xa561a8b1, 0x090eb989, 0xfdbe8ac1, 0x51d19bf9, 0xae6a5137, 0x0205400f, 0xf6b57347, 0x5ada627f, 0x1ed414d7, 0xb2bb05ef, 0x460b36a7, 0xea64279f, 0x3f6136f2, 0x930e27ca, 0x67be1482, 0xcbd105ba, 0x8fdf7312, 0x23b0622a, 0xd7005162, 0x7b6f405a, 0x7d0b73b8, 0xd1646280, 0x25d451c8, 0x89bb40f0, 0xcdb53658, 0x61da2760, 0x956a1428, 0x39050510, 0xec00147d, 0x406f0545, 0xb4df360d, 0x18b02735, 0x5cbe519d, 0xf0d140a5, 0x046173ed, 0xa80e62d5, 0xf9dff92c, 0x55b0e814, 0xa100db5c, 0x0d6fca64, 0x4961bccc, 0xe50eadf4, 0x11be9ebc, 0xbdd18f84, 0x68d49ee9, 0xc4bb8fd1, 0x300bbc99, 0x9c64ada1, 0xd86adb09, 0x7405ca31, 0x80b5f979, 0x2cdae841, 0x2abedba3, 0x86d1ca9b, 0x7261f9d3, 0xde0ee8eb, 0x9a009e43, 0x366f8f7b, 0xc2dfbc33, 0x6eb0ad0b, 0xbbb5bc66, 0x17daad5e, 0xe36a9e16, 0x4f058f2e, 0x0b0bf986, 0xa764e8be, 0x53d4dbf6, 0xffbbcace, 0x5cd5a26e, 0xf0bab356, 0x040a801e, 0xa8659126, 0xec6be78e, 0x4004f6b6, 0xb4b4c5fe, 0x18dbd4c6, 0xcddec5ab, 0x61b1d493, 0x9501e7db, 0x396ef6e3, 0x7d60804b, 0xd10f9173, 0x25bfa23b, 0x89d0b303, 0x8fb480e1, 0x23db91d9, 0xd76ba291, 0x7b04b3a9, 0x3f0ac501, 0x9365d439, 0x67d5e771, 0xcbbaf649, 0x1ebfe724, 0xb2d0f61c, 0x4660c554, 0xea0fd46c, 0xae01a2c4, 0x026eb3fc, 0xf6de80b4, 0x5ab1918c, 0x0b600a75, 0xa70f1b4d, 0x53bf2805, 0xffd0393d, 0xbbde4f95, 0x17b15ead, 0xe3016de5, 0x4f6e7cdd, 0x9a6b6db0, 0x36047c88, 0xc2b44fc0, 0x6edb5ef8, 0x2ad52850, 0x86ba3968, 0x720a0a20, 0xde651b18, 0xd80128fa, 0x746e39c2, 0x80de0a8a, 0x2cb11bb2, 0x68bf6d1a, 0xc4d07c22, 0x30604f6a, 0x9c0f5e52, 0x490a4f3f, 0xe5655e07, 0x11d56d4f, 0xbdba7c77, 0xf9b40adf, 0x55db1be7, 0xa16b28af, 0x0d043997, 0xf2bff359, 0x5ed0e261, 0xaa60d129, 0x060fc011, 0x4201b6b9, 0xee6ea781, 0x1ade94c9, 0xb6b185f1, 0x63b4949c, 0xcfdb85a4, 0x3b6bb6ec, 0x9704a7d4, 0xd30ad17c, 0x7f65c044, 0x8bd5f30c, 0x27bae234, 0x21ded1d6, 0x8db1c0ee, 0x7901f3a6, 0xd56ee29e, 0x91609436, 0x3d0f850e, 0xc9bfb646, 0x65d0a77e, 0xb0d5b613, 0x1cbaa72b, 0xe80a9463, 0x4465855b, 0x006bf3f3, 0xac04e2cb, 0x58b4d183, 0xf4dbc0bb, 0xa50a5b42, 0x09654a7a, 0xfdd57932, 0x51ba680a, 0x15b41ea2, 0xb9db0f9a, 0x4d6b3cd2, 0xe1042dea, 0x34013c87, 0x986e2dbf, 0x6cde1ef7, 0xc0b10fcf, 0x84bf7967, 0x28d0685f, 0xdc605b17, 0x700f4a2f, 0x766b79cd, 0xda0468f5, 0x2eb45bbd, 0x82db4a85, 0xc6d53c2d, 0x6aba2d15, 0x9e0a1e5d, 0x32650f65, 0xe7601e08, 0x4b0f0f30, 0xbfbf3c78, 0x13d02d40, 0x57de5be8, 0xfbb14ad0, 0x0f017998, 0xa36e68a0 },{ 0x00000000, 0x196b30ef, 0xc3a08cdb, 0xdacbbc34, 0x7737f5b2, 0x6e5cc55d, 0xb4977969, 0xadfc4986, 0x1f180660, 0x0673368f, 0xdcb88abb, 0xc5d3ba54, 0x682ff3d2, 0x7144c33d, 0xab8f7f09, 0xb2e44fe6, 0x3e300cc0, 0x275b3c2f, 0xfd90801b, 0xe4fbb0f4, 0x4907f972, 0x506cc99d, 0x8aa775a9, 0x93cc4546, 0x21280aa0, 0x38433a4f, 0xe288867b, 0xfbe3b694, 0x561fff12, 0x4f74cffd, 0x95bf73c9, 0x8cd44326, 0x8d16f485, 0x947dc46a, 0x4eb6785e, 0x57dd48b1, 0xfa210137, 0xe34a31d8, 0x39818dec, 0x20eabd03, 0x920ef2e5, 0x8b65c20a, 0x51ae7e3e, 0x48c54ed1, 0xe5390757, 0xfc5237b8, 0x26998b8c, 0x3ff2bb63, 0xb326f845, 0xaa4dc8aa, 0x7086749e, 0x69ed4471, 0xc4110df7, 0xdd7a3d18, 0x07b1812c, 0x1edab1c3, 0xac3efe25, 0xb555ceca, 0x6f9e72fe, 0x76f54211, 0xdb090b97, 0xc2623b78, 0x18a9874c, 0x01c2b7a3, 0xeb5b040e, 0xf23034e1, 0x28fb88d5, 0x3190b83a, 0x9c6cf1bc, 0x8507c153, 0x5fcc7d67, 0x46a74d88, 0xf443026e, 0xed283281, 0x37e38eb5, 0x2e88be5a, 0x8374f7dc, 0x9a1fc733, 0x40d47b07, 0x59bf4be8, 0xd56b08ce, 0xcc003821, 0x16cb8415, 0x0fa0b4fa, 0xa25cfd7c, 0xbb37cd93, 0x61fc71a7, 0x78974148, 0xca730eae, 0xd3183e41, 0x09d38275, 0x10b8b29a, 0xbd44fb1c, 0xa42fcbf3, 0x7ee477c7, 0x678f4728, 0x664df08b, 0x7f26c064, 0xa5ed7c50, 0xbc864cbf, 0x117a0539, 0x081135d6, 0xd2da89e2, 0xcbb1b90d, 0x7955f6eb, 0x603ec604, 0xbaf57a30, 0xa39e4adf, 0x0e620359, 0x170933b6, 0xcdc28f82, 0xd4a9bf6d, 0x587dfc4b, 0x4116cca4, 0x9bdd7090, 0x82b6407f, 0x2f4a09f9, 0x36213916, 0xecea8522, 0xf581b5cd, 0x4765fa2b, 0x5e0ecac4, 0x84c576f0, 0x9dae461f, 0x30520f99, 0x29393f76, 0xf3f28342, 0xea99b3ad, 0xd6b7081c, 0xcfdc38f3, 0x151784c7, 0x0c7cb428, 0xa180fdae, 0xb8ebcd41, 0x62207175, 0x7b4b419a, 0xc9af0e7c, 0xd0c43e93, 0x0a0f82a7, 0x1364b248, 0xbe98fbce, 0xa7f3cb21, 0x7d387715, 0x645347fa, 0xe88704dc, 0xf1ec3433, 0x2b278807, 0x324cb8e8, 0x9fb0f16e, 0x86dbc181, 0x5c107db5, 0x457b4d5a, 0xf79f02bc, 0xeef43253, 0x343f8e67, 0x2d54be88, 0x80a8f70e, 0x99c3c7e1, 0x43087bd5, 0x5a634b3a, 0x5ba1fc99, 0x42cacc76, 0x98017042, 0x816a40ad, 0x2c96092b, 0x35fd39c4, 0xef3685f0, 0xf65db51f, 0x44b9faf9, 0x5dd2ca16, 0x87197622, 0x9e7246cd, 0x338e0f4b, 0x2ae53fa4, 0xf02e8390, 0xe945b37f, 0x6591f059, 0x7cfac0b6, 0xa6317c82, 0xbf5a4c6d, 0x12a605eb, 0x0bcd3504, 0xd1068930, 0xc86db9df, 0x7a89f639, 0x63e2c6d6, 0xb9297ae2, 0xa0424a0d, 0x0dbe038b, 0x14d53364, 0xce1e8f50, 0xd775bfbf, 0x3dec0c12, 0x24873cfd, 0xfe4c80c9, 0xe727b026, 0x4adbf9a0, 0x53b0c94f, 0x897b757b, 0x90104594, 0x22f40a72, 0x3b9f3a9d, 0xe15486a9, 0xf83fb646, 0x55c3ffc0, 0x4ca8cf2f, 0x9663731b, 0x8f0843f4, 0x03dc00d2, 0x1ab7303d, 0xc07c8c09, 0xd917bce6, 0x74ebf560, 0x6d80c58f, 0xb74b79bb, 0xae204954, 0x1cc406b2, 0x05af365d, 0xdf648a69, 0xc60fba86, 0x6bf3f300, 0x7298c3ef, 0xa8537fdb, 0xb1384f34, 0xb0faf897, 0xa991c878, 0x735a744c, 0x6a3144a3, 0xc7cd0d25, 0xdea63dca, 0x046d81fe, 0x1d06b111, 0xafe2fef7, 0xb689ce18, 0x6c42722c, 0x752942c3, 0xd8d50b45, 0xc1be3baa, 0x1b75879e, 0x021eb771, 0x8ecaf457, 0x97a1c4b8, 0x4d6a788c, 0x54014863, 0xf9fd01e5, 0xe096310a, 0x3a5d8d3e, 0x2336bdd1, 0x91d2f237, 0x88b9c2d8, 0x52727eec, 0x4b194e03, 0xe6e50785, 0xff8e376a, 0x25458b5e, 0x3c2ebbb1 },{ 0x00000000, 0xc82c0368, 0x905906d0, 0x587505b8, 0xd1c5e0a5, 0x19e9e3cd, 0x419ce675, 0x89b0e51d, 0x53fd2d4e, 0x9bd12e26, 0xc3a42b9e, 0x0b8828f6, 0x8238cdeb, 0x4a14ce83, 0x1261cb3b, 0xda4dc853, 0xa6fa5b9c, 0x6ed658f4, 0x36a35d4c, 0xfe8f5e24, 0x773fbb39, 0xbf13b851, 0xe766bde9, 0x2f4abe81, 0xf50776d2, 0x3d2b75ba, 0x655e7002, 0xad72736a, 0x24c29677, 0xecee951f, 0xb49b90a7, 0x7cb793cf, 0xbd835b3d, 0x75af5855, 0x2dda5ded, 0xe5f65e85, 0x6c46bb98, 0xa46ab8f0, 0xfc1fbd48, 0x3433be20, 0xee7e7673, 0x2652751b, 0x7e2770a3, 0xb60b73cb, 0x3fbb96d6, 0xf79795be, 0xafe29006, 0x67ce936e, 0x1b7900a1, 0xd35503c9, 0x8b200671, 0x430c0519, 0xcabce004, 0x0290e36c, 0x5ae5e6d4, 0x92c9e5bc, 0x48842def, 0x80a82e87, 0xd8dd2b3f, 0x10f12857, 0x9941cd4a, 0x516dce22, 0x0918cb9a, 0xc134c8f2, 0x7a07b77a, 0xb22bb412, 0xea5eb1aa, 0x2272b2c2, 0xabc257df, 0x63ee54b7, 0x3b9b510f, 0xf3b75267, 0x29fa9a34, 0xe1d6995c, 0xb9a39ce4, 0x718f9f8c, 0xf83f7a91, 0x301379f9, 0x68667c41, 0xa04a7f29, 0xdcfdece6, 0x14d1ef8e, 0x4ca4ea36, 0x8488e95e, 0x0d380c43, 0xc5140f2b, 0x9d610a93, 0x554d09fb, 0x8f00c1a8, 0x472cc2c0, 0x1f59c778, 0xd775c410, 0x5ec5210d, 0x96e92265, 0xce9c27dd, 0x06b024b5, 0xc784ec47, 0x0fa8ef2f, 0x57ddea97, 0x9ff1e9ff, 0x16410ce2, 0xde6d0f8a, 0x86180a32, 0x4e34095a, 0x9479c109, 0x5c55c261, 0x0420c7d9, 0xcc0cc4b1, 0x45bc21ac, 0x8d9022c4, 0xd5e5277c, 0x1dc92414, 0x617eb7db, 0xa952b4b3, 0xf127b10b, 0x390bb263, 0xb0bb577e, 0x78975416, 0x20e251ae, 0xe8ce52c6, 0x32839a95, 0xfaaf99fd, 0xa2da9c45, 0x6af69f2d, 0xe3467a30, 0x2b6a7958, 0x731f7ce0, 0xbb337f88, 0xf40e6ef5, 0x3c226d9d, 0x64576825, 0xac7b6b4d, 0x25cb8e50, 0xede78d38, 0xb5928880, 0x7dbe8be8, 0xa7f343bb, 0x6fdf40d3, 0x37aa456b, 0xff864603, 0x7636a31e, 0xbe1aa076, 0xe66fa5ce, 0x2e43a6a6, 0x52f43569, 0x9ad83601, 0xc2ad33b9, 0x0a8130d1, 0x8331d5cc, 0x4b1dd6a4, 0x1368d31c, 0xdb44d074, 0x01091827, 0xc9251b4f, 0x91501ef7, 0x597c1d9f, 0xd0ccf882, 0x18e0fbea, 0x4095fe52, 0x88b9fd3a, 0x498d35c8, 0x81a136a0, 0xd9d43318, 0x11f83070, 0x9848d56d, 0x5064d605, 0x0811d3bd, 0xc03dd0d5, 0x1a701886, 0xd25c1bee, 0x8a291e56, 0x42051d3e, 0xcbb5f823, 0x0399fb4b, 0x5becfef3, 0x93c0fd9b, 0xef776e54, 0x275b6d3c, 0x7f2e6884, 0xb7026bec, 0x3eb28ef1, 0xf69e8d99, 0xaeeb8821, 0x66c78b49, 0xbc8a431a, 0x74a64072, 0x2cd345ca, 0xe4ff46a2, 0x6d4fa3bf, 0xa563a0d7, 0xfd16a56f, 0x353aa607, 0x8e09d98f, 0x4625dae7, 0x1e50df5f, 0xd67cdc37, 0x5fcc392a, 0x97e03a42, 0xcf953ffa, 0x07b93c92, 0xddf4f4c1, 0x15d8f7a9, 0x4dadf211, 0x8581f179, 0x0c311464, 0xc41d170c, 0x9c6812b4, 0x544411dc, 0x28f38213, 0xe0df817b, 0xb8aa84c3, 0x708687ab, 0xf93662b6, 0x311a61de, 0x696f6466, 0xa143670e, 0x7b0eaf5d, 0xb322ac35, 0xeb57a98d, 0x237baae5, 0xaacb4ff8, 0x62e74c90, 0x3a924928, 0xf2be4a40, 0x338a82b2, 0xfba681da, 0xa3d38462, 0x6bff870a, 0xe24f6217, 0x2a63617f, 0x721664c7, 0xba3a67af, 0x6077affc, 0xa85bac94, 0xf02ea92c, 0x3802aa44, 0xb1b24f59, 0x799e4c31, 0x21eb4989, 0xe9c74ae1, 0x9570d92e, 0x5d5cda46, 0x0529dffe, 0xcd05dc96, 0x44b5398b, 0x8c993ae3, 0xd4ec3f5b, 0x1cc03c33, 0xc68df460, 0x0ea1f708, 0x56d4f2b0, 0x9ef8f1d8, 0x174814c5, 0xdf6417ad, 0x87111215, 0x4f3d117d },{ 0x00000000, 0x277d3c49, 0x4efa7892, 0x698744db, 0x6d821d21, 0x4aff2168, 0x237865b3, 0x040559fa, 0xda043b42, 0xfd79070b, 0x94fe43d0, 0xb3837f99, 0xb7862663, 0x90fb1a2a, 0xf97c5ef1, 0xde0162b8, 0xb4097684, 0x93744acd, 0xfaf30e16, 0xdd8e325f, 0xd98b6ba5, 0xfef657ec, 0x97711337, 0xb00c2f7e, 0x6e0d4dc6, 0x4970718f, 0x20f73554, 0x078a091d, 0x038f50e7, 0x24f26cae, 0x4d752875, 0x6a08143c, 0x9965000d, 0xbe183c44, 0xd79f789f, 0xf0e244d6, 0xf4e71d2c, 0xd39a2165, 0xba1d65be, 0x9d6059f7, 0x43613b4f, 0x641c0706, 0x0d9b43dd, 0x2ae67f94, 0x2ee3266e, 0x099e1a27, 0x60195efc, 0x476462b5, 0x2d6c7689, 0x0a114ac0, 0x63960e1b, 0x44eb3252, 0x40ee6ba8, 0x679357e1, 0x0e14133a, 0x29692f73, 0xf7684dcb, 0xd0157182, 0xb9923559, 0x9eef0910, 0x9aea50ea, 0xbd976ca3, 0xd4102878, 0xf36d1431, 0x32cb001a, 0x15b63c53, 0x7c317888, 0x5b4c44c1, 0x5f491d3b, 0x78342172, 0x11b365a9, 0x36ce59e0, 0xe8cf3b58, 0xcfb20711, 0xa63543ca, 0x81487f83, 0x854d2679, 0xa2301a30, 0xcbb75eeb, 0xecca62a2, 0x86c2769e, 0xa1bf4ad7, 0xc8380e0c, 0xef453245, 0xeb406bbf, 0xcc3d57f6, 0xa5ba132d, 0x82c72f64, 0x5cc64ddc, 0x7bbb7195, 0x123c354e, 0x35410907, 0x314450fd, 0x16396cb4, 0x7fbe286f, 0x58c31426, 0xabae0017, 0x8cd33c5e, 0xe5547885, 0xc22944cc, 0xc62c1d36, 0xe151217f, 0x88d665a4, 0xafab59ed, 0x71aa3b55, 0x56d7071c, 0x3f5043c7, 0x182d7f8e, 0x1c282674, 0x3b551a3d, 0x52d25ee6, 0x75af62af, 0x1fa77693, 0x38da4ada, 0x515d0e01, 0x76203248, 0x72256bb2, 0x555857fb, 0x3cdf1320, 0x1ba22f69, 0xc5a34dd1, 0xe2de7198, 0x8b593543, 0xac24090a, 0xa82150f0, 0x8f5c6cb9, 0xe6db2862, 0xc1a6142b, 0x64960134, 0x43eb3d7d, 0x2a6c79a6, 0x0d1145ef, 0x09141c15, 0x2e69205c, 0x47ee6487, 0x609358ce, 0xbe923a76, 0x99ef063f, 0xf06842e4, 0xd7157ead, 0xd3102757, 0xf46d1b1e, 0x9dea5fc5, 0xba97638c, 0xd09f77b0, 0xf7e24bf9, 0x9e650f22, 0xb918336b, 0xbd1d6a91, 0x9a6056d8, 0xf3e71203, 0xd49a2e4a, 0x0a9b4cf2, 0x2de670bb, 0x44613460, 0x631c0829, 0x671951d3, 0x40646d9a, 0x29e32941, 0x0e9e1508, 0xfdf30139, 0xda8e3d70, 0xb30979ab, 0x947445e2, 0x90711c18, 0xb70c2051, 0xde8b648a, 0xf9f658c3, 0x27f73a7b, 0x008a0632, 0x690d42e9, 0x4e707ea0, 0x4a75275a, 0x6d081b13, 0x048f5fc8, 0x23f26381, 0x49fa77bd, 0x6e874bf4, 0x07000f2f, 0x207d3366, 0x24786a9c, 0x030556d5, 0x6a82120e, 0x4dff2e47, 0x93fe4cff, 0xb48370b6, 0xdd04346d, 0xfa790824, 0xfe7c51de, 0xd9016d97, 0xb086294c, 0x97fb1505, 0x565d012e, 0x71203d67, 0x18a779bc, 0x3fda45f5, 0x3bdf1c0f, 0x1ca22046, 0x7525649d, 0x525858d4, 0x8c593a6c, 0xab240625, 0xc2a342fe, 0xe5de7eb7, 0xe1db274d, 0xc6a61b04, 0xaf215fdf, 0x885c6396, 0xe25477aa, 0xc5294be3, 0xacae0f38, 0x8bd33371, 0x8fd66a8b, 0xa8ab56c2, 0xc12c1219, 0xe6512e50, 0x38504ce8, 0x1f2d70a1, 0x76aa347a, 0x51d70833, 0x55d251c9, 0x72af6d80, 0x1b28295b, 0x3c551512, 0xcf380123, 0xe8453d6a, 0x81c279b1, 0xa6bf45f8, 0xa2ba1c02, 0x85c7204b, 0xec406490, 0xcb3d58d9, 0x153c3a61, 0x32410628, 0x5bc642f3, 0x7cbb7eba, 0x78be2740, 0x5fc31b09, 0x36445fd2, 0x1139639b, 0x7b3177a7, 0x5c4c4bee, 0x35cb0f35, 0x12b6337c, 0x16b36a86, 0x31ce56cf, 0x58491214, 0x7f342e5d, 0xa1354ce5, 0x864870ac, 0xefcf3477, 0xc8b2083e, 0xccb751c4, 0xebca6d8d, 0x824d2956, 0xa530151f } #else { 0x00000000, 0xf26b8303, 0xe13b70f7, 0x1350f3f4, 0xc79a971f, 0x35f1141c, 0x26a1e7e8, 0xd4ca64eb, 0x8ad958cf, 0x78b2dbcc, 0x6be22838, 0x9989ab3b, 0x4d43cfd0, 0xbf284cd3, 0xac78bf27, 0x5e133c24, 0x105ec76f, 0xe235446c, 0xf165b798, 0x030e349b, 0xd7c45070, 0x25afd373, 0x36ff2087, 0xc494a384, 0x9a879fa0, 0x68ec1ca3, 0x7bbcef57, 0x89d76c54, 0x5d1d08bf, 0xaf768bbc, 0xbc267848, 0x4e4dfb4b, 0x20bd8ede, 0xd2d60ddd, 0xc186fe29, 0x33ed7d2a, 0xe72719c1, 0x154c9ac2, 0x061c6936, 0xf477ea35, 0xaa64d611, 0x580f5512, 0x4b5fa6e6, 0xb93425e5, 0x6dfe410e, 0x9f95c20d, 0x8cc531f9, 0x7eaeb2fa, 0x30e349b1, 0xc288cab2, 0xd1d83946, 0x23b3ba45, 0xf779deae, 0x05125dad, 0x1642ae59, 0xe4292d5a, 0xba3a117e, 0x4851927d, 0x5b016189, 0xa96ae28a, 0x7da08661, 0x8fcb0562, 0x9c9bf696, 0x6ef07595, 0x417b1dbc, 0xb3109ebf, 0xa0406d4b, 0x522bee48, 0x86e18aa3, 0x748a09a0, 0x67dafa54, 0x95b17957, 0xcba24573, 0x39c9c670, 0x2a993584, 0xd8f2b687, 0x0c38d26c, 0xfe53516f, 0xed03a29b, 0x1f682198, 0x5125dad3, 0xa34e59d0, 0xb01eaa24, 0x42752927, 0x96bf4dcc, 0x64d4cecf, 0x77843d3b, 0x85efbe38, 0xdbfc821c, 0x2997011f, 0x3ac7f2eb, 0xc8ac71e8, 0x1c661503, 0xee0d9600, 0xfd5d65f4, 0x0f36e6f7, 0x61c69362, 0x93ad1061, 0x80fde395, 0x72966096, 0xa65c047d, 0x5437877e, 0x4767748a, 0xb50cf789, 0xeb1fcbad, 0x197448ae, 0x0a24bb5a, 0xf84f3859, 0x2c855cb2, 0xdeeedfb1, 0xcdbe2c45, 0x3fd5af46, 0x7198540d, 0x83f3d70e, 0x90a324fa, 0x62c8a7f9, 0xb602c312, 0x44694011, 0x5739b3e5, 0xa55230e6, 0xfb410cc2, 0x092a8fc1, 0x1a7a7c35, 0xe811ff36, 0x3cdb9bdd, 0xceb018de, 0xdde0eb2a, 0x2f8b6829, 0x82f63b78, 0x709db87b, 0x63cd4b8f, 0x91a6c88c, 0x456cac67, 0xb7072f64, 0xa457dc90, 0x563c5f93, 0x082f63b7, 0xfa44e0b4, 0xe9141340, 0x1b7f9043, 0xcfb5f4a8, 0x3dde77ab, 0x2e8e845f, 0xdce5075c, 0x92a8fc17, 0x60c37f14, 0x73938ce0, 0x81f80fe3, 0x55326b08, 0xa759e80b, 0xb4091bff, 0x466298fc, 0x1871a4d8, 0xea1a27db, 0xf94ad42f, 0x0b21572c, 0xdfeb33c7, 0x2d80b0c4, 0x3ed04330, 0xccbbc033, 0xa24bb5a6, 0x502036a5, 0x4370c551, 0xb11b4652, 0x65d122b9, 0x97baa1ba, 0x84ea524e, 0x7681d14d, 0x2892ed69, 0xdaf96e6a, 0xc9a99d9e, 0x3bc21e9d, 0xef087a76, 0x1d63f975, 0x0e330a81, 0xfc588982, 0xb21572c9, 0x407ef1ca, 0x532e023e, 0xa145813d, 0x758fe5d6, 0x87e466d5, 0x94b49521, 0x66df1622, 0x38cc2a06, 0xcaa7a905, 0xd9f75af1, 0x2b9cd9f2, 0xff56bd19, 0x0d3d3e1a, 0x1e6dcdee, 0xec064eed, 0xc38d26c4, 0x31e6a5c7, 0x22b65633, 0xd0ddd530, 0x0417b1db, 0xf67c32d8, 0xe52cc12c, 0x1747422f, 0x49547e0b, 0xbb3ffd08, 0xa86f0efc, 0x5a048dff, 0x8ecee914, 0x7ca56a17, 0x6ff599e3, 0x9d9e1ae0, 0xd3d3e1ab, 0x21b862a8, 0x32e8915c, 0xc083125f, 0x144976b4, 0xe622f5b7, 0xf5720643, 0x07198540, 0x590ab964, 0xab613a67, 0xb831c993, 0x4a5a4a90, 0x9e902e7b, 0x6cfbad78, 0x7fab5e8c, 0x8dc0dd8f, 0xe330a81a, 0x115b2b19, 0x020bd8ed, 0xf0605bee, 0x24aa3f05, 0xd6c1bc06, 0xc5914ff2, 0x37faccf1, 0x69e9f0d5, 0x9b8273d6, 0x88d28022, 0x7ab90321, 0xae7367ca, 0x5c18e4c9, 0x4f48173d, 0xbd23943e, 0xf36e6f75, 0x0105ec76, 0x12551f82, 0xe03e9c81, 0x34f4f86a, 0xc69f7b69, 0xd5cf889d, 0x27a40b9e, 0x79b737ba, 0x8bdcb4b9, 0x988c474d, 0x6ae7c44e, 0xbe2da0a5, 0x4c4623a6, 0x5f16d052, 0xad7d5351 },{ 0x00000000, 0x13a29877, 0x274530ee, 0x34e7a899, 0x4e8a61dc, 0x5d28f9ab, 0x69cf5132, 0x7a6dc945, 0x9d14c3b8, 0x8eb65bcf, 0xba51f356, 0xa9f36b21, 0xd39ea264, 0xc03c3a13, 0xf4db928a, 0xe7790afd, 0x3fc5f181, 0x2c6769f6, 0x1880c16f, 0x0b225918, 0x714f905d, 0x62ed082a, 0x560aa0b3, 0x45a838c4, 0xa2d13239, 0xb173aa4e, 0x859402d7, 0x96369aa0, 0xec5b53e5, 0xfff9cb92, 0xcb1e630b, 0xd8bcfb7c, 0x7f8be302, 0x6c297b75, 0x58ced3ec, 0x4b6c4b9b, 0x310182de, 0x22a31aa9, 0x1644b230, 0x05e62a47, 0xe29f20ba, 0xf13db8cd, 0xc5da1054, 0xd6788823, 0xac154166, 0xbfb7d911, 0x8b507188, 0x98f2e9ff, 0x404e1283, 0x53ec8af4, 0x670b226d, 0x74a9ba1a, 0x0ec4735f, 0x1d66eb28, 0x298143b1, 0x3a23dbc6, 0xdd5ad13b, 0xcef8494c, 0xfa1fe1d5, 0xe9bd79a2, 0x93d0b0e7, 0x80722890, 0xb4958009, 0xa737187e, 0xff17c604, 0xecb55e73, 0xd852f6ea, 0xcbf06e9d, 0xb19da7d8, 0xa23f3faf, 0x96d89736, 0x857a0f41, 0x620305bc, 0x71a19dcb, 0x45463552, 0x56e4ad25, 0x2c896460, 0x3f2bfc17, 0x0bcc548e, 0x186eccf9, 0xc0d23785, 0xd370aff2, 0xe797076b, 0xf4359f1c, 0x8e585659, 0x9dface2e, 0xa91d66b7, 0xbabffec0, 0x5dc6f43d, 0x4e646c4a, 0x7a83c4d3, 0x69215ca4, 0x134c95e1, 0x00ee0d96, 0x3409a50f, 0x27ab3d78, 0x809c2506, 0x933ebd71, 0xa7d915e8, 0xb47b8d9f, 0xce1644da, 0xddb4dcad, 0xe9537434, 0xfaf1ec43, 0x1d88e6be, 0x0e2a7ec9, 0x3acdd650, 0x296f4e27, 0x53028762, 0x40a01f15, 0x7447b78c, 0x67e52ffb, 0xbf59d487, 0xacfb4cf0, 0x981ce469, 0x8bbe7c1e, 0xf1d3b55b, 0xe2712d2c, 0xd69685b5, 0xc5341dc2, 0x224d173f, 0x31ef8f48, 0x050827d1, 0x16aabfa6, 0x6cc776e3, 0x7f65ee94, 0x4b82460d, 0x5820de7a, 0xfbc3faf9, 0xe861628e, 0xdc86ca17, 0xcf245260, 0xb5499b25, 0xa6eb0352, 0x920cabcb, 0x81ae33bc, 0x66d73941, 0x7575a136, 0x419209af, 0x523091d8, 0x285d589d, 0x3bffc0ea, 0x0f186873, 0x1cbaf004, 0xc4060b78, 0xd7a4930f, 0xe3433b96, 0xf0e1a3e1, 0x8a8c6aa4, 0x992ef2d3, 0xadc95a4a, 0xbe6bc23d, 0x5912c8c0, 0x4ab050b7, 0x7e57f82e, 0x6df56059, 0x1798a91c, 0x043a316b, 0x30dd99f2, 0x237f0185, 0x844819fb, 0x97ea818c, 0xa30d2915, 0xb0afb162, 0xcac27827, 0xd960e050, 0xed8748c9, 0xfe25d0be, 0x195cda43, 0x0afe4234, 0x3e19eaad, 0x2dbb72da, 0x57d6bb9f, 0x447423e8, 0x70938b71, 0x63311306, 0xbb8de87a, 0xa82f700d, 0x9cc8d894, 0x8f6a40e3, 0xf50789a6, 0xe6a511d1, 0xd242b948, 0xc1e0213f, 0x26992bc2, 0x353bb3b5, 0x01dc1b2c, 0x127e835b, 0x68134a1e, 0x7bb1d269, 0x4f567af0, 0x5cf4e287, 0x04d43cfd, 0x1776a48a, 0x23910c13, 0x30339464, 0x4a5e5d21, 0x59fcc556, 0x6d1b6dcf, 0x7eb9f5b8, 0x99c0ff45, 0x8a626732, 0xbe85cfab, 0xad2757dc, 0xd74a9e99, 0xc4e806ee, 0xf00fae77, 0xe3ad3600, 0x3b11cd7c, 0x28b3550b, 0x1c54fd92, 0x0ff665e5, 0x759baca0, 0x663934d7, 0x52de9c4e, 0x417c0439, 0xa6050ec4, 0xb5a796b3, 0x81403e2a, 0x92e2a65d, 0xe88f6f18, 0xfb2df76f, 0xcfca5ff6, 0xdc68c781, 0x7b5fdfff, 0x68fd4788, 0x5c1aef11, 0x4fb87766, 0x35d5be23, 0x26772654, 0x12908ecd, 0x013216ba, 0xe64b1c47, 0xf5e98430, 0xc10e2ca9, 0xd2acb4de, 0xa8c17d9b, 0xbb63e5ec, 0x8f844d75, 0x9c26d502, 0x449a2e7e, 0x5738b609, 0x63df1e90, 0x707d86e7, 0x0a104fa2, 0x19b2d7d5, 0x2d557f4c, 0x3ef7e73b, 0xd98eedc6, 0xca2c75b1, 0xfecbdd28, 0xed69455f, 0x97048c1a, 0x84a6146d, 0xb041bcf4, 0xa3e32483 },{ 0x00000000, 0xa541927e, 0x4f6f520d, 0xea2ec073, 0x9edea41a, 0x3b9f3664, 0xd1b1f617, 0x74f06469, 0x38513ec5, 0x9d10acbb, 0x773e6cc8, 0xd27ffeb6, 0xa68f9adf, 0x03ce08a1, 0xe9e0c8d2, 0x4ca15aac, 0x70a27d8a, 0xd5e3eff4, 0x3fcd2f87, 0x9a8cbdf9, 0xee7cd990, 0x4b3d4bee, 0xa1138b9d, 0x045219e3, 0x48f3434f, 0xedb2d131, 0x079c1142, 0xa2dd833c, 0xd62de755, 0x736c752b, 0x9942b558, 0x3c032726, 0xe144fb14, 0x4405696a, 0xae2ba919, 0x0b6a3b67, 0x7f9a5f0e, 0xdadbcd70, 0x30f50d03, 0x95b49f7d, 0xd915c5d1, 0x7c5457af, 0x967a97dc, 0x333b05a2, 0x47cb61cb, 0xe28af3b5, 0x08a433c6, 0xade5a1b8, 0x91e6869e, 0x34a714e0, 0xde89d493, 0x7bc846ed, 0x0f382284, 0xaa79b0fa, 0x40577089, 0xe516e2f7, 0xa9b7b85b, 0x0cf62a25, 0xe6d8ea56, 0x43997828, 0x37691c41, 0x92288e3f, 0x78064e4c, 0xdd47dc32, 0xc76580d9, 0x622412a7, 0x880ad2d4, 0x2d4b40aa, 0x59bb24c3, 0xfcfab6bd, 0x16d476ce, 0xb395e4b0, 0xff34be1c, 0x5a752c62, 0xb05bec11, 0x151a7e6f, 0x61ea1a06, 0xc4ab8878, 0x2e85480b, 0x8bc4da75, 0xb7c7fd53, 0x12866f2d, 0xf8a8af5e, 0x5de93d20, 0x29195949, 0x8c58cb37, 0x66760b44, 0xc337993a, 0x8f96c396, 0x2ad751e8, 0xc0f9919b, 0x65b803e5, 0x1148678c, 0xb409f5f2, 0x5e273581, 0xfb66a7ff, 0x26217bcd, 0x8360e9b3, 0x694e29c0, 0xcc0fbbbe, 0xb8ffdfd7, 0x1dbe4da9, 0xf7908dda, 0x52d11fa4, 0x1e704508, 0xbb31d776, 0x511f1705, 0xf45e857b, 0x80aee112, 0x25ef736c, 0xcfc1b31f, 0x6a802161, 0x56830647, 0xf3c29439, 0x19ec544a, 0xbcadc634, 0xc85da25d, 0x6d1c3023, 0x8732f050, 0x2273622e, 0x6ed23882, 0xcb93aafc, 0x21bd6a8f, 0x84fcf8f1, 0xf00c9c98, 0x554d0ee6, 0xbf63ce95, 0x1a225ceb, 0x8b277743, 0x2e66e53d, 0xc448254e, 0x6109b730, 0x15f9d359, 0xb0b84127, 0x5a968154, 0xffd7132a, 0xb3764986, 0x1637dbf8, 0xfc191b8b, 0x595889f5, 0x2da8ed9c, 0x88e97fe2, 0x62c7bf91, 0xc7862def, 0xfb850ac9, 0x5ec498b7, 0xb4ea58c4, 0x11abcaba, 0x655baed3, 0xc01a3cad, 0x2a34fcde, 0x8f756ea0, 0xc3d4340c, 0x6695a672, 0x8cbb6601, 0x29faf47f, 0x5d0a9016, 0xf84b0268, 0x1265c21b, 0xb7245065, 0x6a638c57, 0xcf221e29, 0x250cde5a, 0x804d4c24, 0xf4bd284d, 0x51fcba33, 0xbbd27a40, 0x1e93e83e, 0x5232b292, 0xf77320ec, 0x1d5de09f, 0xb81c72e1, 0xccec1688, 0x69ad84f6, 0x83834485, 0x26c2d6fb, 0x1ac1f1dd, 0xbf8063a3, 0x55aea3d0, 0xf0ef31ae, 0x841f55c7, 0x215ec7b9, 0xcb7007ca, 0x6e3195b4, 0x2290cf18, 0x87d15d66, 0x6dff9d15, 0xc8be0f6b, 0xbc4e6b02, 0x190ff97c, 0xf321390f, 0x5660ab71, 0x4c42f79a, 0xe90365e4, 0x032da597, 0xa66c37e9, 0xd29c5380, 0x77ddc1fe, 0x9df3018d, 0x38b293f3, 0x7413c95f, 0xd1525b21, 0x3b7c9b52, 0x9e3d092c, 0xeacd6d45, 0x4f8cff3b, 0xa5a23f48, 0x00e3ad36, 0x3ce08a10, 0x99a1186e, 0x738fd81d, 0xd6ce4a63, 0xa23e2e0a, 0x077fbc74, 0xed517c07, 0x4810ee79, 0x04b1b4d5, 0xa1f026ab, 0x4bdee6d8, 0xee9f74a6, 0x9a6f10cf, 0x3f2e82b1, 0xd50042c2, 0x7041d0bc, 0xad060c8e, 0x08479ef0, 0xe2695e83, 0x4728ccfd, 0x33d8a894, 0x96993aea, 0x7cb7fa99, 0xd9f668e7, 0x9557324b, 0x3016a035, 0xda386046, 0x7f79f238, 0x0b899651, 0xaec8042f, 0x44e6c45c, 0xe1a75622, 0xdda47104, 0x78e5e37a, 0x92cb2309, 0x378ab177, 0x437ad51e, 0xe63b4760, 0x0c158713, 0xa954156d, 0xe5f54fc1, 0x40b4ddbf, 0xaa9a1dcc, 0x0fdb8fb2, 0x7b2bebdb, 0xde6a79a5, 0x3444b9d6, 0x91052ba8 },{ 0x00000000, 0xdd45aab8, 0xbf672381, 0x62228939, 0x7b2231f3, 0xa6679b4b, 0xc4451272, 0x1900b8ca, 0xf64463e6, 0x2b01c95e, 0x49234067, 0x9466eadf, 0x8d665215, 0x5023f8ad, 0x32017194, 0xef44db2c, 0xe964b13d, 0x34211b85, 0x560392bc, 0x8b463804, 0x924680ce, 0x4f032a76, 0x2d21a34f, 0xf06409f7, 0x1f20d2db, 0xc2657863, 0xa047f15a, 0x7d025be2, 0x6402e328, 0xb9474990, 0xdb65c0a9, 0x06206a11, 0xd725148b, 0x0a60be33, 0x6842370a, 0xb5079db2, 0xac072578, 0x71428fc0, 0x136006f9, 0xce25ac41, 0x2161776d, 0xfc24ddd5, 0x9e0654ec, 0x4343fe54, 0x5a43469e, 0x8706ec26, 0xe524651f, 0x3861cfa7, 0x3e41a5b6, 0xe3040f0e, 0x81268637, 0x5c632c8f, 0x45639445, 0x98263efd, 0xfa04b7c4, 0x27411d7c, 0xc805c650, 0x15406ce8, 0x7762e5d1, 0xaa274f69, 0xb327f7a3, 0x6e625d1b, 0x0c40d422, 0xd1057e9a, 0xaba65fe7, 0x76e3f55f, 0x14c17c66, 0xc984d6de, 0xd0846e14, 0x0dc1c4ac, 0x6fe34d95, 0xb2a6e72d, 0x5de23c01, 0x80a796b9, 0xe2851f80, 0x3fc0b538, 0x26c00df2, 0xfb85a74a, 0x99a72e73, 0x44e284cb, 0x42c2eeda, 0x9f874462, 0xfda5cd5b, 0x20e067e3, 0x39e0df29, 0xe4a57591, 0x8687fca8, 0x5bc25610, 0xb4868d3c, 0x69c32784, 0x0be1aebd, 0xd6a40405, 0xcfa4bccf, 0x12e11677, 0x70c39f4e, 0xad8635f6, 0x7c834b6c, 0xa1c6e1d4, 0xc3e468ed, 0x1ea1c255, 0x07a17a9f, 0xdae4d027, 0xb8c6591e, 0x6583f3a6, 0x8ac7288a, 0x57828232, 0x35a00b0b, 0xe8e5a1b3, 0xf1e51979, 0x2ca0b3c1, 0x4e823af8, 0x93c79040, 0x95e7fa51, 0x48a250e9, 0x2a80d9d0, 0xf7c57368, 0xeec5cba2, 0x3380611a, 0x51a2e823, 0x8ce7429b, 0x63a399b7, 0xbee6330f, 0xdcc4ba36, 0x0181108e, 0x1881a844, 0xc5c402fc, 0xa7e68bc5, 0x7aa3217d, 0x52a0c93f, 0x8fe56387, 0xedc7eabe, 0x30824006, 0x2982f8cc, 0xf4c75274, 0x96e5db4d, 0x4ba071f5, 0xa4e4aad9, 0x79a10061, 0x1b838958, 0xc6c623e0, 0xdfc69b2a, 0x02833192, 0x60a1b8ab, 0xbde41213, 0xbbc47802, 0x6681d2ba, 0x04a35b83, 0xd9e6f13b, 0xc0e649f1, 0x1da3e349, 0x7f816a70, 0xa2c4c0c8, 0x4d801be4, 0x90c5b15c, 0xf2e73865, 0x2fa292dd, 0x36a22a17, 0xebe780af, 0x89c50996, 0x5480a32e, 0x8585ddb4, 0x58c0770c, 0x3ae2fe35, 0xe7a7548d, 0xfea7ec47, 0x23e246ff, 0x41c0cfc6, 0x9c85657e, 0x73c1be52, 0xae8414ea, 0xcca69dd3, 0x11e3376b, 0x08e38fa1, 0xd5a62519, 0xb784ac20, 0x6ac10698, 0x6ce16c89, 0xb1a4c631, 0xd3864f08, 0x0ec3e5b0, 0x17c35d7a, 0xca86f7c2, 0xa8a47efb, 0x75e1d443, 0x9aa50f6f, 0x47e0a5d7, 0x25c22cee, 0xf8878656, 0xe1873e9c, 0x3cc29424, 0x5ee01d1d, 0x83a5b7a5, 0xf90696d8, 0x24433c60, 0x4661b559, 0x9b241fe1, 0x8224a72b, 0x5f610d93, 0x3d4384aa, 0xe0062e12, 0x0f42f53e, 0xd2075f86, 0xb025d6bf, 0x6d607c07, 0x7460c4cd, 0xa9256e75, 0xcb07e74c, 0x16424df4, 0x106227e5, 0xcd278d5d, 0xaf050464, 0x7240aedc, 0x6b401616, 0xb605bcae, 0xd4273597, 0x09629f2f, 0xe6264403, 0x3b63eebb, 0x59416782, 0x8404cd3a, 0x9d0475f0, 0x4041df48, 0x22635671, 0xff26fcc9, 0x2e238253, 0xf36628eb, 0x9144a1d2, 0x4c010b6a, 0x5501b3a0, 0x88441918, 0xea669021, 0x37233a99, 0xd867e1b5, 0x05224b0d, 0x6700c234, 0xba45688c, 0xa345d046, 0x7e007afe, 0x1c22f3c7, 0xc167597f, 0xc747336e, 0x1a0299d6, 0x782010ef, 0xa565ba57, 0xbc65029d, 0x6120a825, 0x0302211c, 0xde478ba4, 0x31035088, 0xec46fa30, 0x8e647309, 0x5321d9b1, 0x4a21617b, 0x9764cbc3, 0xf54642fa, 0x2803e842 },{ 0x00000000, 0x38116fac, 0x7022df58, 0x4833b0f4, 0xe045beb0, 0xd854d11c, 0x906761e8, 0xa8760e44, 0xc5670b91, 0xfd76643d, 0xb545d4c9, 0x8d54bb65, 0x2522b521, 0x1d33da8d, 0x55006a79, 0x6d1105d5, 0x8f2261d3, 0xb7330e7f, 0xff00be8b, 0xc711d127, 0x6f67df63, 0x5776b0cf, 0x1f45003b, 0x27546f97, 0x4a456a42, 0x725405ee, 0x3a67b51a, 0x0276dab6, 0xaa00d4f2, 0x9211bb5e, 0xda220baa, 0xe2336406, 0x1ba8b557, 0x23b9dafb, 0x6b8a6a0f, 0x539b05a3, 0xfbed0be7, 0xc3fc644b, 0x8bcfd4bf, 0xb3debb13, 0xdecfbec6, 0xe6ded16a, 0xaeed619e, 0x96fc0e32, 0x3e8a0076, 0x069b6fda, 0x4ea8df2e, 0x76b9b082, 0x948ad484, 0xac9bbb28, 0xe4a80bdc, 0xdcb96470, 0x74cf6a34, 0x4cde0598, 0x04edb56c, 0x3cfcdac0, 0x51eddf15, 0x69fcb0b9, 0x21cf004d, 0x19de6fe1, 0xb1a861a5, 0x89b90e09, 0xc18abefd, 0xf99bd151, 0x37516aae, 0x0f400502, 0x4773b5f6, 0x7f62da5a, 0xd714d41e, 0xef05bbb2, 0xa7360b46, 0x9f2764ea, 0xf236613f, 0xca270e93, 0x8214be67, 0xba05d1cb, 0x1273df8f, 0x2a62b023, 0x625100d7, 0x5a406f7b, 0xb8730b7d, 0x806264d1, 0xc851d425, 0xf040bb89, 0x5836b5cd, 0x6027da61, 0x28146a95, 0x10050539, 0x7d1400ec, 0x45056f40, 0x0d36dfb4, 0x3527b018, 0x9d51be5c, 0xa540d1f0, 0xed736104, 0xd5620ea8, 0x2cf9dff9, 0x14e8b055, 0x5cdb00a1, 0x64ca6f0d, 0xccbc6149, 0xf4ad0ee5, 0xbc9ebe11, 0x848fd1bd, 0xe99ed468, 0xd18fbbc4, 0x99bc0b30, 0xa1ad649c, 0x09db6ad8, 0x31ca0574, 0x79f9b580, 0x41e8da2c, 0xa3dbbe2a, 0x9bcad186, 0xd3f96172, 0xebe80ede, 0x439e009a, 0x7b8f6f36, 0x33bcdfc2, 0x0badb06e, 0x66bcb5bb, 0x5eadda17, 0x169e6ae3, 0x2e8f054f, 0x86f90b0b, 0xbee864a7, 0xf6dbd453, 0xcecabbff, 0x6ea2d55c, 0x56b3baf0, 0x1e800a04, 0x269165a8, 0x8ee76bec, 0xb6f60440, 0xfec5b4b4, 0xc6d4db18, 0xabc5decd, 0x93d4b161, 0xdbe70195, 0xe3f66e39, 0x4b80607d, 0x73910fd1, 0x3ba2bf25, 0x03b3d089, 0xe180b48f, 0xd991db23, 0x91a26bd7, 0xa9b3047b, 0x01c50a3f, 0x39d46593, 0x71e7d567, 0x49f6bacb, 0x24e7bf1e, 0x1cf6d0b2, 0x54c56046, 0x6cd40fea, 0xc4a201ae, 0xfcb36e02, 0xb480def6, 0x8c91b15a, 0x750a600b, 0x4d1b0fa7, 0x0528bf53, 0x3d39d0ff, 0x954fdebb, 0xad5eb117, 0xe56d01e3, 0xdd7c6e4f, 0xb06d6b9a, 0x887c0436, 0xc04fb4c2, 0xf85edb6e, 0x5028d52a, 0x6839ba86, 0x200a0a72, 0x181b65de, 0xfa2801d8, 0xc2396e74, 0x8a0ade80, 0xb21bb12c, 0x1a6dbf68, 0x227cd0c4, 0x6a4f6030, 0x525e0f9c, 0x3f4f0a49, 0x075e65e5, 0x4f6dd511, 0x777cbabd, 0xdf0ab4f9, 0xe71bdb55, 0xaf286ba1, 0x9739040d, 0x59f3bff2, 0x61e2d05e, 0x29d160aa, 0x11c00f06, 0xb9b60142, 0x81a76eee, 0xc994de1a, 0xf185b1b6, 0x9c94b463, 0xa485dbcf, 0xecb66b3b, 0xd4a70497, 0x7cd10ad3, 0x44c0657f, 0x0cf3d58b, 0x34e2ba27, 0xd6d1de21, 0xeec0b18d, 0xa6f30179, 0x9ee26ed5, 0x36946091, 0x0e850f3d, 0x46b6bfc9, 0x7ea7d065, 0x13b6d5b0, 0x2ba7ba1c, 0x63940ae8, 0x5b856544, 0xf3f36b00, 0xcbe204ac, 0x83d1b458, 0xbbc0dbf4, 0x425b0aa5, 0x7a4a6509, 0x3279d5fd, 0x0a68ba51, 0xa21eb415, 0x9a0fdbb9, 0xd23c6b4d, 0xea2d04e1, 0x873c0134, 0xbf2d6e98, 0xf71ede6c, 0xcf0fb1c0, 0x6779bf84, 0x5f68d028, 0x175b60dc, 0x2f4a0f70, 0xcd796b76, 0xf56804da, 0xbd5bb42e, 0x854adb82, 0x2d3cd5c6, 0x152dba6a, 0x5d1e0a9e, 0x650f6532, 0x081e60e7, 0x300f0f4b, 0x783cbfbf, 0x402dd013, 0xe85bde57, 0xd04ab1fb, 0x9879010f, 0xa0686ea3 },{ 0x00000000, 0xef306b19, 0xdb8ca0c3, 0x34bccbda, 0xb2f53777, 0x5dc55c6e, 0x697997b4, 0x8649fcad, 0x6006181f, 0x8f367306, 0xbb8ab8dc, 0x54bad3c5, 0xd2f32f68, 0x3dc34471, 0x097f8fab, 0xe64fe4b2, 0xc00c303e, 0x2f3c5b27, 0x1b8090fd, 0xf4b0fbe4, 0x72f90749, 0x9dc96c50, 0xa975a78a, 0x4645cc93, 0xa00a2821, 0x4f3a4338, 0x7b8688e2, 0x94b6e3fb, 0x12ff1f56, 0xfdcf744f, 0xc973bf95, 0x2643d48c, 0x85f4168d, 0x6ac47d94, 0x5e78b64e, 0xb148dd57, 0x370121fa, 0xd8314ae3, 0xec8d8139, 0x03bdea20, 0xe5f20e92, 0x0ac2658b, 0x3e7eae51, 0xd14ec548, 0x570739e5, 0xb83752fc, 0x8c8b9926, 0x63bbf23f, 0x45f826b3, 0xaac84daa, 0x9e748670, 0x7144ed69, 0xf70d11c4, 0x183d7add, 0x2c81b107, 0xc3b1da1e, 0x25fe3eac, 0xcace55b5, 0xfe729e6f, 0x1142f576, 0x970b09db, 0x783b62c2, 0x4c87a918, 0xa3b7c201, 0x0e045beb, 0xe13430f2, 0xd588fb28, 0x3ab89031, 0xbcf16c9c, 0x53c10785, 0x677dcc5f, 0x884da746, 0x6e0243f4, 0x813228ed, 0xb58ee337, 0x5abe882e, 0xdcf77483, 0x33c71f9a, 0x077bd440, 0xe84bbf59, 0xce086bd5, 0x213800cc, 0x1584cb16, 0xfab4a00f, 0x7cfd5ca2, 0x93cd37bb, 0xa771fc61, 0x48419778, 0xae0e73ca, 0x413e18d3, 0x7582d309, 0x9ab2b810, 0x1cfb44bd, 0xf3cb2fa4, 0xc777e47e, 0x28478f67, 0x8bf04d66, 0x64c0267f, 0x507ceda5, 0xbf4c86bc, 0x39057a11, 0xd6351108, 0xe289dad2, 0x0db9b1cb, 0xebf65579, 0x04c63e60, 0x307af5ba, 0xdf4a9ea3, 0x5903620e, 0xb6330917, 0x828fc2cd, 0x6dbfa9d4, 0x4bfc7d58, 0xa4cc1641, 0x9070dd9b, 0x7f40b682, 0xf9094a2f, 0x16392136, 0x2285eaec, 0xcdb581f5, 0x2bfa6547, 0xc4ca0e5e, 0xf076c584, 0x1f46ae9d, 0x990f5230, 0x763f3929, 0x4283f2f3, 0xadb399ea, 0x1c08b7d6, 0xf338dccf, 0xc7841715, 0x28b47c0c, 0xaefd80a1, 0x41cdebb8, 0x75712062, 0x9a414b7b, 0x7c0eafc9, 0x933ec4d0, 0xa7820f0a, 0x48b26413, 0xcefb98be, 0x21cbf3a7, 0x1577387d, 0xfa475364, 0xdc0487e8, 0x3334ecf1, 0x0788272b, 0xe8b84c32, 0x6ef1b09f, 0x81c1db86, 0xb57d105c, 0x5a4d7b45, 0xbc029ff7, 0x5332f4ee, 0x678e3f34, 0x88be542d, 0x0ef7a880, 0xe1c7c399, 0xd57b0843, 0x3a4b635a, 0x99fca15b, 0x76ccca42, 0x42700198, 0xad406a81, 0x2b09962c, 0xc439fd35, 0xf08536ef, 0x1fb55df6, 0xf9fab944, 0x16cad25d, 0x22761987, 0xcd46729e, 0x4b0f8e33, 0xa43fe52a, 0x90832ef0, 0x7fb345e9, 0x59f09165, 0xb6c0fa7c, 0x827c31a6, 0x6d4c5abf, 0xeb05a612, 0x0435cd0b, 0x308906d1, 0xdfb96dc8, 0x39f6897a, 0xd6c6e263, 0xe27a29b9, 0x0d4a42a0, 0x8b03be0d, 0x6433d514, 0x508f1ece, 0xbfbf75d7, 0x120cec3d, 0xfd3c8724, 0xc9804cfe, 0x26b027e7, 0xa0f9db4a, 0x4fc9b053, 0x7b757b89, 0x94451090, 0x720af422, 0x9d3a9f3b, 0xa98654e1, 0x46b63ff8, 0xc0ffc355, 0x2fcfa84c, 0x1b736396, 0xf443088f, 0xd200dc03, 0x3d30b71a, 0x098c7cc0, 0xe6bc17d9, 0x60f5eb74, 0x8fc5806d, 0xbb794bb7, 0x544920ae, 0xb206c41c, 0x5d36af05, 0x698a64df, 0x86ba0fc6, 0x00f3f36b, 0xefc39872, 0xdb7f53a8, 0x344f38b1, 0x97f8fab0, 0x78c891a9, 0x4c745a73, 0xa344316a, 0x250dcdc7, 0xca3da6de, 0xfe816d04, 0x11b1061d, 0xf7fee2af, 0x18ce89b6, 0x2c72426c, 0xc3422975, 0x450bd5d8, 0xaa3bbec1, 0x9e87751b, 0x71b71e02, 0x57f4ca8e, 0xb8c4a197, 0x8c786a4d, 0x63480154, 0xe501fdf9, 0x0a3196e0, 0x3e8d5d3a, 0xd1bd3623, 0x37f2d291, 0xd8c2b988, 0xec7e7252, 0x034e194b, 0x8507e5e6, 0x6a378eff, 0x5e8b4525, 0xb1bb2e3c },{ 0x00000000, 0x68032cc8, 0xd0065990, 0xb8057558, 0xa5e0c5d1, 0xcde3e919, 0x75e69c41, 0x1de5b089, 0x4e2dfd53, 0x262ed19b, 0x9e2ba4c3, 0xf628880b, 0xebcd3882, 0x83ce144a, 0x3bcb6112, 0x53c84dda, 0x9c5bfaa6, 0xf458d66e, 0x4c5da336, 0x245e8ffe, 0x39bb3f77, 0x51b813bf, 0xe9bd66e7, 0x81be4a2f, 0xd27607f5, 0xba752b3d, 0x02705e65, 0x6a7372ad, 0x7796c224, 0x1f95eeec, 0xa7909bb4, 0xcf93b77c, 0x3d5b83bd, 0x5558af75, 0xed5dda2d, 0x855ef6e5, 0x98bb466c, 0xf0b86aa4, 0x48bd1ffc, 0x20be3334, 0x73767eee, 0x1b755226, 0xa370277e, 0xcb730bb6, 0xd696bb3f, 0xbe9597f7, 0x0690e2af, 0x6e93ce67, 0xa100791b, 0xc90355d3, 0x7106208b, 0x19050c43, 0x04e0bcca, 0x6ce39002, 0xd4e6e55a, 0xbce5c992, 0xef2d8448, 0x872ea880, 0x3f2bddd8, 0x5728f110, 0x4acd4199, 0x22ce6d51, 0x9acb1809, 0xf2c834c1, 0x7ab7077a, 0x12b42bb2, 0xaab15eea, 0xc2b27222, 0xdf57c2ab, 0xb754ee63, 0x0f519b3b, 0x6752b7f3, 0x349afa29, 0x5c99d6e1, 0xe49ca3b9, 0x8c9f8f71, 0x917a3ff8, 0xf9791330, 0x417c6668, 0x297f4aa0, 0xe6ecfddc, 0x8eefd114, 0x36eaa44c, 0x5ee98884, 0x430c380d, 0x2b0f14c5, 0x930a619d, 0xfb094d55, 0xa8c1008f, 0xc0c22c47, 0x78c7591f, 0x10c475d7, 0x0d21c55e, 0x6522e996, 0xdd279cce, 0xb524b006, 0x47ec84c7, 0x2fefa80f, 0x97eadd57, 0xffe9f19f, 0xe20c4116, 0x8a0f6dde, 0x320a1886, 0x5a09344e, 0x09c17994, 0x61c2555c, 0xd9c72004, 0xb1c40ccc, 0xac21bc45, 0xc422908d, 0x7c27e5d5, 0x1424c91d, 0xdbb77e61, 0xb3b452a9, 0x0bb127f1, 0x63b20b39, 0x7e57bbb0, 0x16549778, 0xae51e220, 0xc652cee8, 0x959a8332, 0xfd99affa, 0x459cdaa2, 0x2d9ff66a, 0x307a46e3, 0x58796a2b, 0xe07c1f73, 0x887f33bb, 0xf56e0ef4, 0x9d6d223c, 0x25685764, 0x4d6b7bac, 0x508ecb25, 0x388de7ed, 0x808892b5, 0xe88bbe7d, 0xbb43f3a7, 0xd340df6f, 0x6b45aa37, 0x034686ff, 0x1ea33676, 0x76a01abe, 0xcea56fe6, 0xa6a6432e, 0x6935f452, 0x0136d89a, 0xb933adc2, 0xd130810a, 0xccd53183, 0xa4d61d4b, 0x1cd36813, 0x74d044db, 0x27180901, 0x4f1b25c9, 0xf71e5091, 0x9f1d7c59, 0x82f8ccd0, 0xeafbe018, 0x52fe9540, 0x3afdb988, 0xc8358d49, 0xa036a181, 0x1833d4d9, 0x7030f811, 0x6dd54898, 0x05d66450, 0xbdd31108, 0xd5d03dc0, 0x8618701a, 0xee1b5cd2, 0x561e298a, 0x3e1d0542, 0x23f8b5cb, 0x4bfb9903, 0xf3feec5b, 0x9bfdc093, 0x546e77ef, 0x3c6d5b27, 0x84682e7f, 0xec6b02b7, 0xf18eb23e, 0x998d9ef6, 0x2188ebae, 0x498bc766, 0x1a438abc, 0x7240a674, 0xca45d32c, 0xa246ffe4, 0xbfa34f6d, 0xd7a063a5, 0x6fa516fd, 0x07a63a35, 0x8fd9098e, 0xe7da2546, 0x5fdf501e, 0x37dc7cd6, 0x2a39cc5f, 0x423ae097, 0xfa3f95cf, 0x923cb907, 0xc1f4f4dd, 0xa9f7d815, 0x11f2ad4d, 0x79f18185, 0x6414310c, 0x0c171dc4, 0xb412689c, 0xdc114454, 0x1382f328, 0x7b81dfe0, 0xc384aab8, 0xab878670, 0xb66236f9, 0xde611a31, 0x66646f69, 0x0e6743a1, 0x5daf0e7b, 0x35ac22b3, 0x8da957eb, 0xe5aa7b23, 0xf84fcbaa, 0x904ce762, 0x2849923a, 0x404abef2, 0xb2828a33, 0xda81a6fb, 0x6284d3a3, 0x0a87ff6b, 0x17624fe2, 0x7f61632a, 0xc7641672, 0xaf673aba, 0xfcaf7760, 0x94ac5ba8, 0x2ca92ef0, 0x44aa0238, 0x594fb2b1, 0x314c9e79, 0x8949eb21, 0xe14ac7e9, 0x2ed97095, 0x46da5c5d, 0xfedf2905, 0x96dc05cd, 0x8b39b544, 0xe33a998c, 0x5b3fecd4, 0x333cc01c, 0x60f48dc6, 0x08f7a10e, 0xb0f2d456, 0xd8f1f89e, 0xc5144817, 0xad1764df, 0x15121187, 0x7d113d4f },{ 0x00000000, 0x493c7d27, 0x9278fa4e, 0xdb448769, 0x211d826d, 0x6821ff4a, 0xb3657823, 0xfa590504, 0x423b04da, 0x0b0779fd, 0xd043fe94, 0x997f83b3, 0x632686b7, 0x2a1afb90, 0xf15e7cf9, 0xb86201de, 0x847609b4, 0xcd4a7493, 0x160ef3fa, 0x5f328edd, 0xa56b8bd9, 0xec57f6fe, 0x37137197, 0x7e2f0cb0, 0xc64d0d6e, 0x8f717049, 0x5435f720, 0x1d098a07, 0xe7508f03, 0xae6cf224, 0x7528754d, 0x3c14086a, 0x0d006599, 0x443c18be, 0x9f789fd7, 0xd644e2f0, 0x2c1de7f4, 0x65219ad3, 0xbe651dba, 0xf759609d, 0x4f3b6143, 0x06071c64, 0xdd439b0d, 0x947fe62a, 0x6e26e32e, 0x271a9e09, 0xfc5e1960, 0xb5626447, 0x89766c2d, 0xc04a110a, 0x1b0e9663, 0x5232eb44, 0xa86bee40, 0xe1579367, 0x3a13140e, 0x732f6929, 0xcb4d68f7, 0x827115d0, 0x593592b9, 0x1009ef9e, 0xea50ea9a, 0xa36c97bd, 0x782810d4, 0x31146df3, 0x1a00cb32, 0x533cb615, 0x8878317c, 0xc1444c5b, 0x3b1d495f, 0x72213478, 0xa965b311, 0xe059ce36, 0x583bcfe8, 0x1107b2cf, 0xca4335a6, 0x837f4881, 0x79264d85, 0x301a30a2, 0xeb5eb7cb, 0xa262caec, 0x9e76c286, 0xd74abfa1, 0x0c0e38c8, 0x453245ef, 0xbf6b40eb, 0xf6573dcc, 0x2d13baa5, 0x642fc782, 0xdc4dc65c, 0x9571bb7b, 0x4e353c12, 0x07094135, 0xfd504431, 0xb46c3916, 0x6f28be7f, 0x2614c358, 0x1700aeab, 0x5e3cd38c, 0x857854e5, 0xcc4429c2, 0x361d2cc6, 0x7f2151e1, 0xa465d688, 0xed59abaf, 0x553baa71, 0x1c07d756, 0xc743503f, 0x8e7f2d18, 0x7426281c, 0x3d1a553b, 0xe65ed252, 0xaf62af75, 0x9376a71f, 0xda4ada38, 0x010e5d51, 0x48322076, 0xb26b2572, 0xfb575855, 0x2013df3c, 0x692fa21b, 0xd14da3c5, 0x9871dee2, 0x4335598b, 0x0a0924ac, 0xf05021a8, 0xb96c5c8f, 0x6228dbe6, 0x2b14a6c1, 0x34019664, 0x7d3deb43, 0xa6796c2a, 0xef45110d, 0x151c1409, 0x5c20692e, 0x8764ee47, 0xce589360, 0x763a92be, 0x3f06ef99, 0xe44268f0, 0xad7e15d7, 0x572710d3, 0x1e1b6df4, 0xc55fea9d, 0x8c6397ba, 0xb0779fd0, 0xf94be2f7, 0x220f659e, 0x6b3318b9, 0x916a1dbd, 0xd856609a, 0x0312e7f3, 0x4a2e9ad4, 0xf24c9b0a, 0xbb70e62d, 0x60346144, 0x29081c63, 0xd3511967, 0x9a6d6440, 0x4129e329, 0x08159e0e, 0x3901f3fd, 0x703d8eda, 0xab7909b3, 0xe2457494, 0x181c7190, 0x51200cb7, 0x8a648bde, 0xc358f6f9, 0x7b3af727, 0x32068a00, 0xe9420d69, 0xa07e704e, 0x5a27754a, 0x131b086d, 0xc85f8f04, 0x8163f223, 0xbd77fa49, 0xf44b876e, 0x2f0f0007, 0x66337d20, 0x9c6a7824, 0xd5560503, 0x0e12826a, 0x472eff4d, 0xff4cfe93, 0xb67083b4, 0x6d3404dd, 0x240879fa, 0xde517cfe, 0x976d01d9, 0x4c2986b0, 0x0515fb97, 0x2e015d56, 0x673d2071, 0xbc79a718, 0xf545da3f, 0x0f1cdf3b, 0x4620a21c, 0x9d642575, 0xd4585852, 0x6c3a598c, 0x250624ab, 0xfe42a3c2, 0xb77edee5, 0x4d27dbe1, 0x041ba6c6, 0xdf5f21af, 0x96635c88, 0xaa7754e2, 0xe34b29c5, 0x380faeac, 0x7133d38b, 0x8b6ad68f, 0xc256aba8, 0x19122cc1, 0x502e51e6, 0xe84c5038, 0xa1702d1f, 0x7a34aa76, 0x3308d751, 0xc951d255, 0x806daf72, 0x5b29281b, 0x1215553c, 0x230138cf, 0x6a3d45e8, 0xb179c281, 0xf845bfa6, 0x021cbaa2, 0x4b20c785, 0x906440ec, 0xd9583dcb, 0x613a3c15, 0x28064132, 0xf342c65b, 0xba7ebb7c, 0x4027be78, 0x091bc35f, 0xd25f4436, 0x9b633911, 0xa777317b, 0xee4b4c5c, 0x350fcb35, 0x7c33b612, 0x866ab316, 0xcf56ce31, 0x14124958, 0x5d2e347f, 0xe54c35a1, 0xac704886, 0x7734cfef, 0x3e08b2c8, 0xc451b7cc, 0x8d6dcaeb, 0x56294d82, 0x1f1530a5 } #endif }; /* * Return a checksum for a chunk of memory. * * Slicing-by-8 algorithm by Michael E. Kounavis and Frank L. Berry from * Intel Corp.: * http://www.intel.com/technology/comms/perfnet/download/CRC_generators.pdf * * Based on Peter Kanowski's posting: * http://www.strchr.com/crc32_popcnt * * The big endian version calculates the same result at each step, except the * value of the crc is byte reversed from what it would be at that step for * little endian. */ uint32_t my_crc32c_slicing(const uint8_t *chunk, size_t len) { uint32_t crc, next; size_t nqwords; const uint8_t *p; crc = 0xffffffff; /* Checksum one byte at a time to the first 4B boundary. */ for (p = chunk; ((uintptr_t)p & (sizeof(uint32_t) - 1)) != 0 && len > 0; ++p, --len) { #ifdef WORDS_BIGENDIAN crc = g_crc_slicing[0][((crc >> 24) ^ *p) & 0xFF] ^ (crc << 8); #else crc = g_crc_slicing[0][(crc ^ *p) & 0xFF] ^ (crc >> 8); #endif } /* Checksum in 8B chunks. */ for (nqwords = len / sizeof(uint64_t); nqwords; nqwords--) { crc ^= *(uint32_t *)p; p += sizeof(uint32_t); next = *(uint32_t *)p; p += sizeof(uint32_t); crc = #ifdef WORDS_BIGENDIAN g_crc_slicing[4][(crc ) & 0xFF] ^ g_crc_slicing[5][(crc >> 8) & 0xFF] ^ g_crc_slicing[6][(crc >> 16) & 0xFF] ^ g_crc_slicing[7][(crc >> 24)] ^ g_crc_slicing[0][(next ) & 0xFF] ^ g_crc_slicing[1][(next >> 8) & 0xFF] ^ g_crc_slicing[2][(next >> 16) & 0xFF] ^ g_crc_slicing[3][(next >> 24)]; #else g_crc_slicing[7][(crc ) & 0xFF] ^ g_crc_slicing[6][(crc >> 8) & 0xFF] ^ g_crc_slicing[5][(crc >> 16) & 0xFF] ^ g_crc_slicing[4][(crc >> 24)] ^ g_crc_slicing[3][(next ) & 0xFF] ^ g_crc_slicing[2][(next >> 8) & 0xFF] ^ g_crc_slicing[1][(next >> 16) & 0xFF] ^ g_crc_slicing[0][(next >> 24)]; #endif } /* Checksum trailing bytes one byte at a time. */ #ifdef WORDS_BIGENDIAN for (len &= 0x7; len > 0; ++p, len--) crc = g_crc_slicing[0][((crc >> 24) ^ *p) & 0xFF] ^ (crc << 8); /* Do final byte swap to produce a result identical to little endian */ crc = ((crc << 24) & 0xFF000000) | ((crc << 8) & 0x00FF0000) | ((crc >> 8) & 0x0000FF00) | ((crc >> 24) & 0x000000FF); #else for (len &= 0x7; len > 0; ++p, len--) crc = g_crc_slicing[0][(crc ^ *p) & 0xFF] ^ (crc >> 8); #endif return (~crc); } mtbl-0.5/libmy/crc32c-sse42.c000066400000000000000000000117661227302760400156270ustar00rootroot00000000000000/* * Copyright (c) 2013 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* Copyright (c) 2008,2009,2010 Massachusetts Institute of Technology. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of the Massachusetts Institute of Technology nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* * Copyright (c) 2004-2006 Intel Corporation - All Rights Reserved * * This software program is licensed subject to the BSD License, * available at http://www.opensource.org/licenses/bsd-license.html */ // Copyright 2010 Google Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #if __GNUC__ >= 3 && defined(__x86_64__) #include #include #include #define CPUID_FEATURES (1) #define SSE42_FEATURE_BIT (1 << 20) bool my_crc32c_sse42_supported(void); uint32_t my_crc32c_sse42(const uint8_t *, size_t); static uint32_t cpuid(uint32_t level) { uint32_t eax, ebx, ecx, edx; asm("cpuid" : "=a" (eax), "=b" (ebx), "=c" (ecx), "=d" (edx) : "a" (level)); return (ecx); } bool my_crc32c_sse42_supported(void) { return (cpuid(CPUID_FEATURES) & SSE42_FEATURE_BIT); } static inline uint64_t my_asm_crc32_u64(uint64_t crc, uint64_t value) { asm("crc32q %[value], %[crc]\n" : [crc] "+r" (crc) : [value] "rm" (value)); return crc; } static inline uint32_t my_asm_crc32_u32(uint32_t crc, uint32_t value) { asm("crc32l %[value], %[crc]\n" : [crc] "+r" (crc) : [value] "rm" (value)); return crc; } static inline uint32_t my_asm_crc32_u16(uint32_t crc, uint16_t value) { asm("crc32w %[value], %[crc]\n" : [crc] "+r" (crc) : [value] "rm" (value)); return crc; } static inline uint32_t my_asm_crc32_u8(uint32_t crc, uint8_t value) { asm("crc32b %[value], %[crc]\n" : [crc] "+r" (crc) : [value] "rm" (value)); return crc; } uint32_t my_crc32c_sse42(const uint8_t *buf, size_t len) { const uint8_t *p = buf; uint64_t crc64bit = 0xFFFFFFFF; for (size_t i = 0; i < len / sizeof(uint64_t); i++) { crc64bit = my_asm_crc32_u64(crc64bit, *(uint64_t *) p); p += sizeof(uint64_t); } uint32_t crc32bit = (uint32_t) crc64bit; len &= sizeof(uint64_t) - 1; switch (len) { case 7: crc32bit = my_asm_crc32_u8(crc32bit, *p++); case 6: crc32bit = my_asm_crc32_u16(crc32bit, *(uint16_t *) p); p += 2; case 4: crc32bit = my_asm_crc32_u32(crc32bit, *(uint32_t *) p); break; case 3: crc32bit = my_asm_crc32_u8(crc32bit, *p++); case 2: crc32bit = my_asm_crc32_u16(crc32bit, *(uint16_t *) p); break; case 5: crc32bit = my_asm_crc32_u32(crc32bit, *(uint32_t *) p); p += 4; case 1: crc32bit = my_asm_crc32_u8(crc32bit, *p); break; case 0: break; default: break; } return ~crc32bit; } #endif /* __GNUC__ >= 3 && defined(__x86_64__) */ mtbl-0.5/libmy/crc32c.c000066400000000000000000000027331227302760400146630ustar00rootroot00000000000000/* * Copyright (c) 2013 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include "crc32c.h" static uint32_t my_crc32c_first(const uint8_t *buf, size_t len); my_crc32c_fp my_crc32c = my_crc32c_first; /* crc32c-slicing.c */ uint32_t my_crc32c_slicing(const uint8_t *, size_t); /* crc32c-sse42.c */ #if __GNUC__ >= 3 && defined(__x86_64__) bool my_crc32c_sse42_supported(void); uint32_t my_crc32c_sse42(const uint8_t *, size_t); #endif #if defined(__GNUC__) __attribute__((constructor)) #endif static void my_crc32c_runtime_detection(void) { #if __GNUC__ >= 3 && defined(__x86_64__) if (my_crc32c_sse42_supported()) { my_crc32c = my_crc32c_sse42; } else { my_crc32c = my_crc32c_slicing; } #else my_crc32c = my_crc32c_slicing; #endif } static uint32_t my_crc32c_first(const uint8_t *buf, size_t len) { my_crc32c_runtime_detection(); return my_crc32c(buf, len); } mtbl-0.5/libmy/crc32c.h000066400000000000000000000003001227302760400146540ustar00rootroot00000000000000#ifndef MY_CRC32C_H #define MY_CRC32C_H #include typedef uint32_t (*my_crc32c_fp)(const uint8_t *buffer, size_t length); extern my_crc32c_fp my_crc32c; #endif /* MY_CRC32C_H */ mtbl-0.5/libmy/crc32c_test.c000066400000000000000000000137041227302760400157220ustar00rootroot00000000000000#include #include #include #include "crc32c.h" /* CRC32C test vectors, adapted from linux crypto/testmgr.h and leveldb crc32c_test */ struct crc32c_testvec { uint8_t plaintext[240]; size_t psize; uint32_t value; }; static struct crc32c_testvec crc32c_testvectors[] = { { .plaintext = { 0x61 }, .psize = 1, .value = 0xc1d04330 }, { .plaintext = { 0x66, 0x6f, 0x6f }, .psize = 3, .value = 0xcfc4ae1d }, { .plaintext = { 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x77, 0x6f, 0x72, 0x6c, 0x64 }, .psize = 11, .value = 0xc99465aa }, { .plaintext = { 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x20 }, .psize = 6, .value = 0x7e627e58 }, { .plaintext = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, .psize = 32, .value = 0x8a9136aa }, { .plaintext = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, .psize = 32, .value = 0x62a8ab43 }, { .plaintext = { 0x1f, 0x1e, 0x1d, 0x1c, 0x1b, 0x1a, 0x19, 0x18, 0x17, 0x16, 0x15, 0x14, 0x13, 0x12, 0x11, 0x10, 0x0f, 0x0e, 0x0d, 0x0c, 0x0b, 0x0a, 0x09, 0x08, 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00 }, .psize = 32, .value = 0x113fdb5c }, { .plaintext = { 0x01, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x18, 0x28, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, .psize = 48, .value = 0xd9963a56 }, { .plaintext = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f }, .psize = 32, .value = 0x46dd794e }, { .plaintext = { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28 }, .psize = 40, .value = 0x0e2c157f }, { .plaintext = { 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50 }, .psize = 40, .value = 0xe980ebf6 }, { .plaintext = { 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78 }, .psize = 40, .value = 0xde74bded }, { .plaintext = { 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0 }, .psize = 40, .value = 0xd579c862 }, { .plaintext = { 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8 }, .psize = 40, .value = 0xba979ad0 }, { .plaintext = { 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0 }, .psize = 40, .value = 0x2b29d913 }, { .plaintext = { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0 }, .psize = 240, .value = 0x24c5d375 }, { .psize = 0 }, }; int main(void) { struct crc32c_testvec *tv; int rc = EXIT_SUCCESS; for (tv = &crc32c_testvectors[0]; tv->psize != 0; tv++) { uint32_t crc = my_crc32c(tv->plaintext, tv->psize); fprintf(stderr, "crc32c_test: %s: [actual=%08x, expected=%08x]\n", crc == tv->value ? "PASS" : "FAIL", tv->value, crc); if (crc != tv->value) rc = EXIT_FAILURE; } return (rc); } mtbl-0.5/libmy/getenv_int.h000066400000000000000000000005471227302760400157540ustar00rootroot00000000000000#ifndef MY_GETENV_INT_H #define MY_GETENV_INT_H #include #include #include static inline bool getenv_int(const char *name, uint64_t *value) { char *s, *t; s = getenv(name); if (s == NULL) return (false); *value = strtoul(s, &t, 0); if (*t != '\0') return (false); return (true); } #endif /* MY_GETENV_INT_H */ mtbl-0.5/libmy/heap.c000066400000000000000000000065141227302760400145220ustar00rootroot00000000000000/* * Copyright (c) 2012 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include "my_alloc.h" #include "heap.h" #include "vector.h" VECTOR_GENERATE(ptrvec, void *); struct heap { ptrvec *vec; heap_compare_func cmp; }; static inline int cmp_wrapper(heap_compare_func cmp, const void *a, const void *b) { return ((cmp(a, b) < 0) ? 1 : 0); } struct heap * heap_init(heap_compare_func cmp) { struct heap *h = my_calloc(1, sizeof(*h)); h->cmp = cmp; h->vec = ptrvec_init(1); return (h); } void heap_destroy(struct heap **h) { if (*h != NULL) { ptrvec_destroy(&(*h)->vec); free(*h); *h = NULL; } } static int siftdown(struct heap *h, size_t startpos, size_t pos) { assert(pos < ptrvec_size(h->vec)); void *newitem = ptrvec_value(h->vec, pos); while (pos > startpos) { size_t parentpos = (pos - 1) >> 1; void *parent = ptrvec_value(h->vec, parentpos); int cmp = cmp_wrapper(h->cmp, newitem, parent); if (cmp == -1) return (-1); if (cmp == 0) break; ptrvec_data(h->vec)[pos] = parent; pos = parentpos; } ptrvec_data(h->vec)[pos] = newitem; return (0); } static int siftup(struct heap *h, size_t pos) { assert(pos < ptrvec_size(h->vec)); void *newitem = ptrvec_value(h->vec, pos); size_t endpos = ptrvec_size(h->vec); size_t startpos = pos; size_t childpos = 2 * pos + 1; while (childpos < endpos) { size_t rightpos = childpos + 1; if (rightpos < endpos) { int cmp = cmp_wrapper(h->cmp, ptrvec_value(h->vec, childpos), ptrvec_value(h->vec, rightpos)); if (cmp == -1) return (-1); if (cmp == 0) childpos = rightpos; } ptrvec_data(h->vec)[pos] = ptrvec_value(h->vec, childpos); pos = childpos; childpos = 2 * pos + 1; } ptrvec_data(h->vec)[pos] = newitem; return (siftdown(h, startpos, pos)); } void heap_push(struct heap *h, void *item) { ptrvec_add(h->vec, item); siftdown(h, 0, ptrvec_size(h->vec) - 1); } void * heap_pop(struct heap *h) { if (ptrvec_size(h->vec) < 1) return (NULL); void *returnitem; void *lastelt = ptrvec_value(h->vec, ptrvec_size(h->vec) - 1); ptrvec_clip(h->vec, ptrvec_size(h->vec) - 1); if (ptrvec_size(h->vec) > 0) { returnitem = ptrvec_value(h->vec, 0); ptrvec_data(h->vec)[0] = lastelt; siftup(h, 0); } else { returnitem = lastelt; } return (returnitem); } void * heap_replace(struct heap *h, void *item) { if (ptrvec_size(h->vec) < 1) return (NULL); void *returnitem = ptrvec_value(h->vec, 0); ptrvec_data(h->vec)[0] = item; siftup(h, 0); return (returnitem); } void * heap_peek(struct heap *h) { if (ptrvec_size(h->vec) < 1) return (NULL); return ptrvec_data(h->vec)[0]; } void * heap_get(struct heap *h, size_t i) { if (i > ptrvec_size(h->vec) - 1) return (NULL); return (ptrvec_value(h->vec, i)); } size_t heap_size(struct heap *h) { return (ptrvec_size(h->vec)); } mtbl-0.5/libmy/heap.h000066400000000000000000000006631227302760400145260ustar00rootroot00000000000000#ifndef MY_HEAP_H #define MY_HEAP_H struct heap; typedef int (*heap_compare_func)(const void *a, const void *b); struct heap *heap_init(heap_compare_func); void heap_destroy(struct heap **); void heap_push(struct heap *, void *); void *heap_pop(struct heap *); void *heap_replace(struct heap *, void *); void *heap_peek(struct heap *); void *heap_get(struct heap *, size_t); size_t heap_size(struct heap *); #endif /* MY_HEAP_H */ mtbl-0.5/libmy/hex_decode.h000066400000000000000000000032051227302760400156730ustar00rootroot00000000000000/* * Copyright (c) 2012 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef MY_HEX_DECODE_H #define MY_HEX_DECODE_H #include #include #include #include #include "my_alloc.h" static inline bool hex_to_int(char hex, uint8_t *val) { hex = toupper(hex); switch (hex) { case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': *val = (hex - '0'); return (true); case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': *val = (hex - 55); return (true); default: return (false); } } static inline bool hex_decode(const char *hex, uint8_t **raw, size_t *len) { size_t hexlen = strlen(hex); uint8_t *p; if (hexlen == 0 || (hexlen % 2) != 0) return (false); *len = hexlen / 2; p = *raw = my_malloc(*len); while (hexlen != 0) { uint8_t val[2]; if (!hex_to_int(*hex, &val[0])) goto err; hex++; if (!hex_to_int(*hex, &val[1])) goto err; hex++; *p = (val[0] << 4) | val[1]; p++; hexlen -= 2; } return (true); err: free(*raw); return (false); } #endif /* MY_HEX_DECODE_H */ mtbl-0.5/libmy/ip_arith.h000066400000000000000000000065171227302760400154140ustar00rootroot00000000000000/* * Copyright (c) 2012 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef MY_IP_ARITH_H #define MY_IP_ARITH_H #ifdef HAVE_ENDIAN_H # include #else # ifdef HAVE_SYS_ENDIAN_H # include # endif #endif #include #include /* given big endian IPv4 prefix in 'address'/'prefix_len', * write lowest IP address ("network address") into 'out' */ static inline void ip4_lower(const void *address, unsigned prefix_len, void *out) { uint32_t addr; memcpy(&addr, address, 4); addr &= htobe32(~((1 << (32 - prefix_len)) - 1)); memcpy(out, &addr, 4); } /* given big endian IPv4 prefix in 'address'/'prefix_len', * write highest IP address ("broadcast address") into 'out' */ static inline void ip4_upper(const void *address, unsigned prefix_len, void *out) { uint32_t addr; memcpy(&addr, address, 4); addr |= htobe32((1 << (32 - prefix_len)) - 1); memcpy(out, &addr, 4); } /* increment IPv4 address by 1 */ static inline void ip4_incr(void *address) { uint32_t addr; memcpy(&addr, address, 4); addr = be32toh(addr) + 1; addr = htobe32(addr); memcpy(address, &addr, 4); } /* given big endian IPv6 prefix in 'address'/'prefix_len', * write lowest IP address ("network address") into 'lower' */ static inline void ip6_lower(const void *address, unsigned prefix_len, void *lower) { uint64_t addr[2], mask[2]; if (prefix_len < 64) { mask[0] = htobe64(((1ULL << prefix_len) - 1ULL) << (64ULL - prefix_len)); mask[1] = htobe64(0); } else if (prefix_len < 128) { prefix_len -= 64; mask[0] = htobe64(UINT64_MAX); mask[1] = htobe64(((1ULL << prefix_len) - 1ULL) << (64ULL - prefix_len)); } else { memset(mask, 0xFF, sizeof(mask)); } memcpy(addr, address, 16); addr[0] &= mask[0]; addr[1] &= mask[1]; memcpy(lower, addr, 16); } /* given big endian IPv6 prefix in 'address'/'prefix_len', * write highest IP address ("broadcast address") into 'upper' */ static inline void ip6_upper(const void *address, unsigned prefix_len, void *upper) { uint64_t addr[2], mask[2]; if (prefix_len < 64) { mask[0] = htobe64(~(((1ULL << prefix_len) - 1ULL) << (64ULL - prefix_len))); mask[1] = htobe64(UINT64_MAX); } else if (prefix_len < 128) { prefix_len -= 64; mask[0] = htobe64(0); mask[1] = htobe64(~(((1ULL << prefix_len) - 1ULL) << (64ULL - prefix_len))); } else { memset(&mask, 0, sizeof(mask)); } memcpy(addr, address, 16); addr[0] |= mask[0]; addr[1] |= mask[1]; memcpy(upper, addr, 16); } /* increment IPv6 address by 1 */ static inline void ip6_incr(void *address) { uint64_t addr[2]; memcpy(addr, address, 16); addr[0] = be64toh(addr[0]); addr[1] = be64toh(addr[1]); if (addr[1] == UINT64_MAX) { addr[0] += 1; addr[1] = 0; } else { addr[1] += 1; } addr[0] = htobe64(addr[0]); addr[1] = htobe64(addr[1]); memcpy(address, addr, 16); } #endif /* MY_IP_ARITH_H */ mtbl-0.5/libmy/list.h000066400000000000000000000131311227302760400145560ustar00rootroot00000000000000/* * Copyright (C) 2004, 2006, 2007 Internet Systems Consortium, Inc. ("ISC") * Copyright (C) 1997-2002 Internet Software Consortium. * * Permission to use, copy, modify, and/or distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY * AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR * PERFORMANCE OF THIS SOFTWARE. */ #ifndef ISC_LIST_H #define ISC_LIST_H 1 #define ISC_INSIST(x) #define ISC_LINK_INSIST(x) #define ISC_LIST(type) struct { type *head, *tail; } #define ISC_LIST_INIT(list) \ do { (list).head = NULL; (list).tail = NULL; } while (0) #define ISC_LINK(type) struct { type *prev, *next; } #define ISC_LINK_INIT_TYPE(elt, link, type) \ do { \ (elt)->link.prev = (type *)(-1); \ (elt)->link.next = (type *)(-1); \ } while (0) #define ISC_LINK_INIT(elt, link) \ ISC_LINK_INIT_TYPE(elt, link, void) #define ISC_LINK_LINKED(elt, link) ((void *)((elt)->link.prev) != (void *)(-1)) #define ISC_LIST_HEAD(list) ((list).head) #define ISC_LIST_TAIL(list) ((list).tail) #define ISC_LIST_EMPTY(list) ((list.head) == NULL ? 1 : 0) #define __ISC_LIST_PREPENDUNSAFE(list, elt, link) \ do { \ if ((list).head != NULL) \ (list).head->link.prev = (elt); \ else \ (list).tail = (elt); \ (elt)->link.prev = NULL; \ (elt)->link.next = (list).head; \ (list).head = (elt); \ } while (0) #define ISC_LIST_PREPEND(list, elt, link) \ do { \ ISC_LINK_INSIST(!ISC_LINK_LINKED(elt, link)); \ __ISC_LIST_PREPENDUNSAFE(list, elt, link); \ } while (0) #define ISC_LIST_INITANDPREPEND(list, elt, link) \ __ISC_LIST_PREPENDUNSAFE(list, elt, link) #define __ISC_LIST_APPENDUNSAFE(list, elt, link) \ do { \ if ((list).tail != NULL) \ (list).tail->link.next = (elt); \ else \ (list).head = (elt); \ (elt)->link.prev = (list).tail; \ (elt)->link.next = NULL; \ (list).tail = (elt); \ } while (0) #define ISC_LIST_APPEND(list, elt, link) \ do { \ ISC_LINK_INSIST(!ISC_LINK_LINKED(elt, link)); \ __ISC_LIST_APPENDUNSAFE(list, elt, link); \ } while (0) #define ISC_LIST_INITANDAPPEND(list, elt, link) \ __ISC_LIST_APPENDUNSAFE(list, elt, link) #define __ISC_LIST_UNLINKUNSAFE_TYPE(list, elt, link, type) \ do { \ if ((elt)->link.next != NULL) \ (elt)->link.next->link.prev = (elt)->link.prev; \ else { \ ISC_INSIST((list).tail == (elt)); \ (list).tail = (elt)->link.prev; \ } \ if ((elt)->link.prev != NULL) \ (elt)->link.prev->link.next = (elt)->link.next; \ else { \ ISC_INSIST((list).head == (elt)); \ (list).head = (elt)->link.next; \ } \ (elt)->link.prev = (type *)(-1); \ (elt)->link.next = (type *)(-1); \ } while (0) #define __ISC_LIST_UNLINKUNSAFE(list, elt, link) \ __ISC_LIST_UNLINKUNSAFE_TYPE(list, elt, link, void) #define ISC_LIST_UNLINK_TYPE(list, elt, link, type) \ do { \ ISC_LINK_INSIST(ISC_LINK_LINKED(elt, link)); \ __ISC_LIST_UNLINKUNSAFE_TYPE(list, elt, link, type); \ } while (0) #define ISC_LIST_UNLINK(list, elt, link) \ ISC_LIST_UNLINK_TYPE(list, elt, link, void) #define ISC_LIST_PREV(elt, link) ((elt)->link.prev) #define ISC_LIST_NEXT(elt, link) ((elt)->link.next) #define __ISC_LIST_INSERTBEFOREUNSAFE(list, before, elt, link) \ do { \ if ((before)->link.prev == NULL) \ ISC_LIST_PREPEND(list, elt, link); \ else { \ (elt)->link.prev = (before)->link.prev; \ (before)->link.prev = (elt); \ (elt)->link.prev->link.next = (elt); \ (elt)->link.next = (before); \ } \ } while (0) #define ISC_LIST_INSERTBEFORE(list, before, elt, link) \ do { \ ISC_LINK_INSIST(ISC_LINK_LINKED(before, link)); \ ISC_LINK_INSIST(!ISC_LINK_LINKED(elt, link)); \ __ISC_LIST_INSERTBEFOREUNSAFE(list, before, elt, link); \ } while (0) #define __ISC_LIST_INSERTAFTERUNSAFE(list, after, elt, link) \ do { \ if ((after)->link.next == NULL) \ ISC_LIST_APPEND(list, elt, link); \ else { \ (elt)->link.next = (after)->link.next; \ (after)->link.next = (elt); \ (elt)->link.next->link.prev = (elt); \ (elt)->link.prev = (after); \ } \ } while (0) #define ISC_LIST_INSERTAFTER(list, after, elt, link) \ do { \ ISC_LINK_INSIST(ISC_LINK_LINKED(after, link)); \ ISC_LINK_INSIST(!ISC_LINK_LINKED(elt, link)); \ __ISC_LIST_INSERTAFTERUNSAFE(list, after, elt, link); \ } while (0) #define ISC_LIST_APPENDLIST(list1, list2, link) \ do { \ if (ISC_LIST_EMPTY(list1)) \ (list1) = (list2); \ else if (!ISC_LIST_EMPTY(list2)) { \ (list1).tail->link.next = (list2).head; \ (list2).head->link.prev = (list1).tail; \ (list1).tail = (list2).tail; \ } \ (list2).head = NULL; \ (list2).tail = NULL; \ } while (0) #define ISC_LIST_ENQUEUE(list, elt, link) ISC_LIST_APPEND(list, elt, link) #define __ISC_LIST_ENQUEUEUNSAFE(list, elt, link) \ __ISC_LIST_APPENDUNSAFE(list, elt, link) #define ISC_LIST_DEQUEUE(list, elt, link) \ ISC_LIST_UNLINK_TYPE(list, elt, link, void) #define ISC_LIST_DEQUEUE_TYPE(list, elt, link, type) \ ISC_LIST_UNLINK_TYPE(list, elt, link, type) #define __ISC_LIST_DEQUEUEUNSAFE(list, elt, link) \ __ISC_LIST_UNLINKUNSAFE_TYPE(list, elt, link, void) #define __ISC_LIST_DEQUEUEUNSAFE_TYPE(list, elt, link, type) \ __ISC_LIST_UNLINKUNSAFE_TYPE(list, elt, link, type) #endif /* ISC_LIST_H */ mtbl-0.5/libmy/lookup3.c000066400000000000000000001052421227302760400151770ustar00rootroot00000000000000/* ------------------------------------------------------------------------------- lookup3.c, by Bob Jenkins, May 2006, Public Domain. These are functions for producing 32-bit hashes for hash table lookup. hashword(), hashlittle(), hashlittle2(), hashbig(), mix(), and final() are externally useful functions. Routines to test the hash are included if SELF_TEST is defined. You can use this free for any purpose. It's in the public domain. It has no warranty. You probably want to use hashlittle(). hashlittle() and hashbig() hash byte arrays. hashlittle() is is faster than hashbig() on little-endian machines. Intel and AMD are little-endian machines. On second thought, you probably want hashlittle2(), which is identical to hashlittle() except it returns two 32-bit hashes for the price of one. You could implement hashbig2() if you wanted but I haven't bothered here. If you want to find a hash of, say, exactly 7 integers, do a = i1; b = i2; c = i3; mix(a,b,c); a += i4; b += i5; c += i6; mix(a,b,c); a += i7; final(a,b,c); then use c as the hash value. If you have a variable length array of 4-byte integers to hash, use hashword(). If you have a byte array (like a character string), use hashlittle(). If you have several byte arrays, or a mix of things, see the comments above hashlittle(). Why is this so big? I read 12 bytes at a time into 3 4-byte integers, then mix those integers. This is fast (you can do a lot more thorough mixing with 12*3 instructions on 3 integers than you can with 3 instructions on 1 byte), but shoehorning those bytes into integers efficiently is messy. ------------------------------------------------------------------------------- */ //#define SELF_TEST 1 #include /* defines printf for tests */ #include /* defines time_t for timings in the test */ #include /* defines uint32_t etc */ #include /* attempt to define endianness */ #ifdef HAVE_ENDIAN_H # include #else # ifdef HAVE_SYS_ENDIAN_H # include # endif #endif #include "lookup3.h" /* * My best guess at if you are big-endian or little-endian. This may * need adjustment. */ #if (defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && \ __BYTE_ORDER == __LITTLE_ENDIAN) || \ (defined(i386) || defined(__i386__) || defined(__i486__) || \ defined(__i586__) || defined(__i686__) || defined(vax) || defined(MIPSEL)) # define HASH_LITTLE_ENDIAN 1 # define HASH_BIG_ENDIAN 0 #elif (defined(__BYTE_ORDER) && defined(__BIG_ENDIAN) && \ __BYTE_ORDER == __BIG_ENDIAN) || \ (defined(sparc) || defined(POWERPC) || defined(mc68000) || defined(sel)) # define HASH_LITTLE_ENDIAN 0 # define HASH_BIG_ENDIAN 1 #else # define HASH_LITTLE_ENDIAN 0 # define HASH_BIG_ENDIAN 0 #endif #define hashsize(n) ((uint32_t)1<<(n)) #define hashmask(n) (hashsize(n)-1) #define rot(x,k) (((x)<<(k)) | ((x)>>(32-(k)))) /* ------------------------------------------------------------------------------- mix -- mix 3 32-bit values reversibly. This is reversible, so any information in (a,b,c) before mix() is still in (a,b,c) after mix(). If four pairs of (a,b,c) inputs are run through mix(), or through mix() in reverse, there are at least 32 bits of the output that are sometimes the same for one pair and different for another pair. This was tested for: * pairs that differed by one bit, by two bits, in any combination of top bits of (a,b,c), or in any combination of bottom bits of (a,b,c). * "differ" is defined as +, -, ^, or ~^. For + and -, I transformed the output delta to a Gray code (a^(a>>1)) so a string of 1's (as is commonly produced by subtraction) look like a single 1-bit difference. * the base values were pseudorandom, all zero but one bit set, or all zero plus a counter that starts at zero. Some k values for my "a-=c; a^=rot(c,k); c+=b;" arrangement that satisfy this are 4 6 8 16 19 4 9 15 3 18 27 15 14 9 3 7 17 3 Well, "9 15 3 18 27 15" didn't quite get 32 bits diffing for "differ" defined as + with a one-bit base and a two-bit delta. I used http://burtleburtle.net/bob/hash/avalanche.html to choose the operations, constants, and arrangements of the variables. This does not achieve avalanche. There are input bits of (a,b,c) that fail to affect some output bits of (a,b,c), especially of a. The most thoroughly mixed value is c, but it doesn't really even achieve avalanche in c. This allows some parallelism. Read-after-writes are good at doubling the number of bits affected, so the goal of mixing pulls in the opposite direction as the goal of parallelism. I did what I could. Rotates seem to cost as much as shifts on every machine I could lay my hands on, and rotates are much kinder to the top and bottom bits, so I used rotates. ------------------------------------------------------------------------------- */ #define mix(a,b,c) \ { \ a -= c; a ^= rot(c, 4); c += b; \ b -= a; b ^= rot(a, 6); a += c; \ c -= b; c ^= rot(b, 8); b += a; \ a -= c; a ^= rot(c,16); c += b; \ b -= a; b ^= rot(a,19); a += c; \ c -= b; c ^= rot(b, 4); b += a; \ } /* ------------------------------------------------------------------------------- final -- final mixing of 3 32-bit values (a,b,c) into c Pairs of (a,b,c) values differing in only a few bits will usually produce values of c that look totally different. This was tested for * pairs that differed by one bit, by two bits, in any combination of top bits of (a,b,c), or in any combination of bottom bits of (a,b,c). * "differ" is defined as +, -, ^, or ~^. For + and -, I transformed the output delta to a Gray code (a^(a>>1)) so a string of 1's (as is commonly produced by subtraction) look like a single 1-bit difference. * the base values were pseudorandom, all zero but one bit set, or all zero plus a counter that starts at zero. These constants passed: 14 11 25 16 4 14 24 12 14 25 16 4 14 24 and these came close: 4 8 15 26 3 22 24 10 8 15 26 3 22 24 11 8 15 26 3 22 24 ------------------------------------------------------------------------------- */ #define final(a,b,c) \ { \ c ^= b; c -= rot(b,14); \ a ^= c; a -= rot(c,11); \ b ^= a; b -= rot(a,25); \ c ^= b; c -= rot(b,16); \ a ^= c; a -= rot(c,4); \ b ^= a; b -= rot(a,14); \ c ^= b; c -= rot(b,24); \ } /* -------------------------------------------------------------------- This works on all machines. To be useful, it requires -- that the key be an array of uint32_t's, and -- that the length be the number of uint32_t's in the key The function hashword() is identical to hashlittle() on little-endian machines, and identical to hashbig() on big-endian machines, except that the length has to be measured in uint32_ts rather than in bytes. hashlittle() is more complicated than hashword() only because hashlittle() has to dance around fitting the key bytes into registers. -------------------------------------------------------------------- */ uint32_t my_hashword( const uint32_t *k, /* the key, an array of uint32_t values */ size_t length, /* the length of the key, in uint32_ts */ uint32_t initval) /* the previous hash, or an arbitrary value */ { uint32_t a,b,c; /* Set up the internal state */ a = b = c = 0xdeadbeef + (((uint32_t)length)<<2) + initval; /*------------------------------------------------- handle most of the key */ while (length > 3) { a += k[0]; b += k[1]; c += k[2]; mix(a,b,c); length -= 3; k += 3; } /*------------------------------------------- handle the last 3 uint32_t's */ switch(length) /* all the case statements fall through */ { case 3 : c+=k[2]; case 2 : b+=k[1]; case 1 : a+=k[0]; final(a,b,c); case 0: /* case 0: nothing left to add */ break; } /*------------------------------------------------------ report the result */ return c; } /* -------------------------------------------------------------------- hashword2() -- same as hashword(), but take two seeds and return two 32-bit values. pc and pb must both be nonnull, and *pc and *pb must both be initialized with seeds. If you pass in (*pb)==0, the output (*pc) will be the same as the return value from hashword(). -------------------------------------------------------------------- */ void my_hashword2 ( const uint32_t *k, /* the key, an array of uint32_t values */ size_t length, /* the length of the key, in uint32_ts */ uint32_t *pc, /* IN: seed OUT: primary hash value */ uint32_t *pb) /* IN: more seed OUT: secondary hash value */ { uint32_t a,b,c; /* Set up the internal state */ a = b = c = 0xdeadbeef + ((uint32_t)(length<<2)) + *pc; c += *pb; /*------------------------------------------------- handle most of the key */ while (length > 3) { a += k[0]; b += k[1]; c += k[2]; mix(a,b,c); length -= 3; k += 3; } /*------------------------------------------- handle the last 3 uint32_t's */ switch(length) /* all the case statements fall through */ { case 3 : c+=k[2]; case 2 : b+=k[1]; case 1 : a+=k[0]; final(a,b,c); case 0: /* case 0: nothing left to add */ break; } /*------------------------------------------------------ report the result */ *pc=c; *pb=b; } /* ------------------------------------------------------------------------------- hashlittle() -- hash a variable-length key into a 32-bit value k : the key (the unaligned variable-length array of bytes) length : the length of the key, counting by bytes initval : can be any 4-byte value Returns a 32-bit value. Every bit of the key affects every bit of the return value. Two keys differing by one or two bits will have totally different hash values. The best hash table sizes are powers of 2. There is no need to do mod a prime (mod is sooo slow!). If you need less than 32 bits, use a bitmask. For example, if you need only 10 bits, do h = (h & hashmask(10)); In which case, the hash table should have hashsize(10) elements. If you are hashing n strings (uint8_t **)k, do it like this: for (i=0, h=0; i 12) { a += k[0]; b += k[1]; c += k[2]; mix(a,b,c); length -= 12; k += 3; } /*----------------------------- handle the last (probably partial) block */ /* * "k[2]&0xffffff" actually reads beyond the end of the string, but * then masks off the part it's not allowed to read. Because the * string is aligned, the masked-off tail is in the same word as the * rest of the string. Every machine with memory protection I've seen * does it on word boundaries, so is OK with this. But VALGRIND will * still catch it and complain. The masking trick does make the hash * noticably faster for short strings (like English words). */ #ifndef VALGRIND switch(length) { case 12: c+=k[2]; b+=k[1]; a+=k[0]; break; case 11: c+=k[2]&0xffffff; b+=k[1]; a+=k[0]; break; case 10: c+=k[2]&0xffff; b+=k[1]; a+=k[0]; break; case 9 : c+=k[2]&0xff; b+=k[1]; a+=k[0]; break; case 8 : b+=k[1]; a+=k[0]; break; case 7 : b+=k[1]&0xffffff; a+=k[0]; break; case 6 : b+=k[1]&0xffff; a+=k[0]; break; case 5 : b+=k[1]&0xff; a+=k[0]; break; case 4 : a+=k[0]; break; case 3 : a+=k[0]&0xffffff; break; case 2 : a+=k[0]&0xffff; break; case 1 : a+=k[0]&0xff; break; case 0 : return c; /* zero length strings require no mixing */ } #else /* make valgrind happy */ k8 = (const uint8_t *)k; switch(length) { case 12: c+=k[2]; b+=k[1]; a+=k[0]; break; case 11: c+=((uint32_t)k8[10])<<16; /* fall through */ case 10: c+=((uint32_t)k8[9])<<8; /* fall through */ case 9 : c+=k8[8]; /* fall through */ case 8 : b+=k[1]; a+=k[0]; break; case 7 : b+=((uint32_t)k8[6])<<16; /* fall through */ case 6 : b+=((uint32_t)k8[5])<<8; /* fall through */ case 5 : b+=k8[4]; /* fall through */ case 4 : a+=k[0]; break; case 3 : a+=((uint32_t)k8[2])<<16; /* fall through */ case 2 : a+=((uint32_t)k8[1])<<8; /* fall through */ case 1 : a+=k8[0]; break; case 0 : return c; } #endif /* !valgrind */ } else if (HASH_LITTLE_ENDIAN && ((u.i & 0x1) == 0)) { const uint16_t *k = (const uint16_t *)key; /* read 16-bit chunks */ const uint8_t *k8; /*--------------- all but last block: aligned reads and different mixing */ while (length > 12) { a += k[0] + (((uint32_t)k[1])<<16); b += k[2] + (((uint32_t)k[3])<<16); c += k[4] + (((uint32_t)k[5])<<16); mix(a,b,c); length -= 12; k += 6; } /*----------------------------- handle the last (probably partial) block */ k8 = (const uint8_t *)k; switch(length) { case 12: c+=k[4]+(((uint32_t)k[5])<<16); b+=k[2]+(((uint32_t)k[3])<<16); a+=k[0]+(((uint32_t)k[1])<<16); break; case 11: c+=((uint32_t)k8[10])<<16; /* fall through */ case 10: c+=k[4]; b+=k[2]+(((uint32_t)k[3])<<16); a+=k[0]+(((uint32_t)k[1])<<16); break; case 9 : c+=k8[8]; /* fall through */ case 8 : b+=k[2]+(((uint32_t)k[3])<<16); a+=k[0]+(((uint32_t)k[1])<<16); break; case 7 : b+=((uint32_t)k8[6])<<16; /* fall through */ case 6 : b+=k[2]; a+=k[0]+(((uint32_t)k[1])<<16); break; case 5 : b+=k8[4]; /* fall through */ case 4 : a+=k[0]+(((uint32_t)k[1])<<16); break; case 3 : a+=((uint32_t)k8[2])<<16; /* fall through */ case 2 : a+=k[0]; break; case 1 : a+=k8[0]; break; case 0 : return c; /* zero length requires no mixing */ } } else { /* need to read the key one byte at a time */ const uint8_t *k = (const uint8_t *)key; /*--------------- all but the last block: affect some 32 bits of (a,b,c) */ while (length > 12) { a += k[0]; a += ((uint32_t)k[1])<<8; a += ((uint32_t)k[2])<<16; a += ((uint32_t)k[3])<<24; b += k[4]; b += ((uint32_t)k[5])<<8; b += ((uint32_t)k[6])<<16; b += ((uint32_t)k[7])<<24; c += k[8]; c += ((uint32_t)k[9])<<8; c += ((uint32_t)k[10])<<16; c += ((uint32_t)k[11])<<24; mix(a,b,c); length -= 12; k += 12; } /*-------------------------------- last block: affect all 32 bits of (c) */ switch(length) /* all the case statements fall through */ { case 12: c+=((uint32_t)k[11])<<24; case 11: c+=((uint32_t)k[10])<<16; case 10: c+=((uint32_t)k[9])<<8; case 9 : c+=k[8]; case 8 : b+=((uint32_t)k[7])<<24; case 7 : b+=((uint32_t)k[6])<<16; case 6 : b+=((uint32_t)k[5])<<8; case 5 : b+=k[4]; case 4 : a+=((uint32_t)k[3])<<24; case 3 : a+=((uint32_t)k[2])<<16; case 2 : a+=((uint32_t)k[1])<<8; case 1 : a+=k[0]; break; case 0 : return c; } } final(a,b,c); return c; } /* * hashlittle2: return 2 32-bit hash values * * This is identical to hashlittle(), except it returns two 32-bit hash * values instead of just one. This is good enough for hash table * lookup with 2^^64 buckets, or if you want a second hash if you're not * happy with the first, or if you want a probably-unique 64-bit ID for * the key. *pc is better mixed than *pb, so use *pc first. If you want * a 64-bit value do something like "*pc + (((uint64_t)*pb)<<32)". */ void my_hashlittle2( const void *key, /* the key to hash */ size_t length, /* length of the key */ uint32_t *pc, /* IN: primary initval, OUT: primary hash */ uint32_t *pb) /* IN: secondary initval, OUT: secondary hash */ { uint32_t a,b,c; /* internal state */ union { const void *ptr; size_t i; } u; /* needed for Mac Powerbook G4 */ /* Set up the internal state */ a = b = c = 0xdeadbeef + ((uint32_t)length) + *pc; c += *pb; u.ptr = key; if (HASH_LITTLE_ENDIAN && ((u.i & 0x3) == 0)) { const uint32_t *k = (const uint32_t *)key; /* read 32-bit chunks */ #ifdef VALGRIND const uint8_t *k8; #endif /*------ all but last block: aligned reads and affect 32 bits of (a,b,c) */ while (length > 12) { a += k[0]; b += k[1]; c += k[2]; mix(a,b,c); length -= 12; k += 3; } /*----------------------------- handle the last (probably partial) block */ /* * "k[2]&0xffffff" actually reads beyond the end of the string, but * then masks off the part it's not allowed to read. Because the * string is aligned, the masked-off tail is in the same word as the * rest of the string. Every machine with memory protection I've seen * does it on word boundaries, so is OK with this. But VALGRIND will * still catch it and complain. The masking trick does make the hash * noticably faster for short strings (like English words). */ #ifndef VALGRIND switch(length) { case 12: c+=k[2]; b+=k[1]; a+=k[0]; break; case 11: c+=k[2]&0xffffff; b+=k[1]; a+=k[0]; break; case 10: c+=k[2]&0xffff; b+=k[1]; a+=k[0]; break; case 9 : c+=k[2]&0xff; b+=k[1]; a+=k[0]; break; case 8 : b+=k[1]; a+=k[0]; break; case 7 : b+=k[1]&0xffffff; a+=k[0]; break; case 6 : b+=k[1]&0xffff; a+=k[0]; break; case 5 : b+=k[1]&0xff; a+=k[0]; break; case 4 : a+=k[0]; break; case 3 : a+=k[0]&0xffffff; break; case 2 : a+=k[0]&0xffff; break; case 1 : a+=k[0]&0xff; break; case 0 : *pc=c; *pb=b; return; /* zero length strings require no mixing */ } #else /* make valgrind happy */ k8 = (const uint8_t *)k; switch(length) { case 12: c+=k[2]; b+=k[1]; a+=k[0]; break; case 11: c+=((uint32_t)k8[10])<<16; /* fall through */ case 10: c+=((uint32_t)k8[9])<<8; /* fall through */ case 9 : c+=k8[8]; /* fall through */ case 8 : b+=k[1]; a+=k[0]; break; case 7 : b+=((uint32_t)k8[6])<<16; /* fall through */ case 6 : b+=((uint32_t)k8[5])<<8; /* fall through */ case 5 : b+=k8[4]; /* fall through */ case 4 : a+=k[0]; break; case 3 : a+=((uint32_t)k8[2])<<16; /* fall through */ case 2 : a+=((uint32_t)k8[1])<<8; /* fall through */ case 1 : a+=k8[0]; break; case 0 : *pc=c; *pb=b; return; /* zero length strings require no mixing */ } #endif /* !valgrind */ } else if (HASH_LITTLE_ENDIAN && ((u.i & 0x1) == 0)) { const uint16_t *k = (const uint16_t *)key; /* read 16-bit chunks */ const uint8_t *k8; /*--------------- all but last block: aligned reads and different mixing */ while (length > 12) { a += k[0] + (((uint32_t)k[1])<<16); b += k[2] + (((uint32_t)k[3])<<16); c += k[4] + (((uint32_t)k[5])<<16); mix(a,b,c); length -= 12; k += 6; } /*----------------------------- handle the last (probably partial) block */ k8 = (const uint8_t *)k; switch(length) { case 12: c+=k[4]+(((uint32_t)k[5])<<16); b+=k[2]+(((uint32_t)k[3])<<16); a+=k[0]+(((uint32_t)k[1])<<16); break; case 11: c+=((uint32_t)k8[10])<<16; /* fall through */ case 10: c+=k[4]; b+=k[2]+(((uint32_t)k[3])<<16); a+=k[0]+(((uint32_t)k[1])<<16); break; case 9 : c+=k8[8]; /* fall through */ case 8 : b+=k[2]+(((uint32_t)k[3])<<16); a+=k[0]+(((uint32_t)k[1])<<16); break; case 7 : b+=((uint32_t)k8[6])<<16; /* fall through */ case 6 : b+=k[2]; a+=k[0]+(((uint32_t)k[1])<<16); break; case 5 : b+=k8[4]; /* fall through */ case 4 : a+=k[0]+(((uint32_t)k[1])<<16); break; case 3 : a+=((uint32_t)k8[2])<<16; /* fall through */ case 2 : a+=k[0]; break; case 1 : a+=k8[0]; break; case 0 : *pc=c; *pb=b; return; /* zero length strings require no mixing */ } } else { /* need to read the key one byte at a time */ const uint8_t *k = (const uint8_t *)key; /*--------------- all but the last block: affect some 32 bits of (a,b,c) */ while (length > 12) { a += k[0]; a += ((uint32_t)k[1])<<8; a += ((uint32_t)k[2])<<16; a += ((uint32_t)k[3])<<24; b += k[4]; b += ((uint32_t)k[5])<<8; b += ((uint32_t)k[6])<<16; b += ((uint32_t)k[7])<<24; c += k[8]; c += ((uint32_t)k[9])<<8; c += ((uint32_t)k[10])<<16; c += ((uint32_t)k[11])<<24; mix(a,b,c); length -= 12; k += 12; } /*-------------------------------- last block: affect all 32 bits of (c) */ switch(length) /* all the case statements fall through */ { case 12: c+=((uint32_t)k[11])<<24; case 11: c+=((uint32_t)k[10])<<16; case 10: c+=((uint32_t)k[9])<<8; case 9 : c+=k[8]; case 8 : b+=((uint32_t)k[7])<<24; case 7 : b+=((uint32_t)k[6])<<16; case 6 : b+=((uint32_t)k[5])<<8; case 5 : b+=k[4]; case 4 : a+=((uint32_t)k[3])<<24; case 3 : a+=((uint32_t)k[2])<<16; case 2 : a+=((uint32_t)k[1])<<8; case 1 : a+=k[0]; break; case 0 : *pc=c; *pb=b; return; /* zero length strings require no mixing */ } } final(a,b,c); *pc=c; *pb=b; } /* * hashbig(): * This is the same as hashword() on big-endian machines. It is different * from hashlittle() on all machines. hashbig() takes advantage of * big-endian byte ordering. */ uint32_t my_hashbig( const void *key, size_t length, uint32_t initval) { uint32_t a,b,c; union { const void *ptr; size_t i; } u; /* to cast key to (size_t) happily */ /* Set up the internal state */ a = b = c = 0xdeadbeef + ((uint32_t)length) + initval; u.ptr = key; if (HASH_BIG_ENDIAN && ((u.i & 0x3) == 0)) { const uint32_t *k = (const uint32_t *)key; /* read 32-bit chunks */ #ifdef VALGRIND const uint8_t *k8; #endif /*------ all but last block: aligned reads and affect 32 bits of (a,b,c) */ while (length > 12) { a += k[0]; b += k[1]; c += k[2]; mix(a,b,c); length -= 12; k += 3; } /*----------------------------- handle the last (probably partial) block */ /* * "k[2]<<8" actually reads beyond the end of the string, but * then shifts out the part it's not allowed to read. Because the * string is aligned, the illegal read is in the same word as the * rest of the string. Every machine with memory protection I've seen * does it on word boundaries, so is OK with this. But VALGRIND will * still catch it and complain. The masking trick does make the hash * noticably faster for short strings (like English words). */ #ifndef VALGRIND switch(length) { case 12: c+=k[2]; b+=k[1]; a+=k[0]; break; case 11: c+=k[2]&0xffffff00; b+=k[1]; a+=k[0]; break; case 10: c+=k[2]&0xffff0000; b+=k[1]; a+=k[0]; break; case 9 : c+=k[2]&0xff000000; b+=k[1]; a+=k[0]; break; case 8 : b+=k[1]; a+=k[0]; break; case 7 : b+=k[1]&0xffffff00; a+=k[0]; break; case 6 : b+=k[1]&0xffff0000; a+=k[0]; break; case 5 : b+=k[1]&0xff000000; a+=k[0]; break; case 4 : a+=k[0]; break; case 3 : a+=k[0]&0xffffff00; break; case 2 : a+=k[0]&0xffff0000; break; case 1 : a+=k[0]&0xff000000; break; case 0 : return c; /* zero length strings require no mixing */ } #else /* make valgrind happy */ k8 = (const uint8_t *)k; switch(length) /* all the case statements fall through */ { case 12: c+=k[2]; b+=k[1]; a+=k[0]; break; case 11: c+=((uint32_t)k8[10])<<8; /* fall through */ case 10: c+=((uint32_t)k8[9])<<16; /* fall through */ case 9 : c+=((uint32_t)k8[8])<<24; /* fall through */ case 8 : b+=k[1]; a+=k[0]; break; case 7 : b+=((uint32_t)k8[6])<<8; /* fall through */ case 6 : b+=((uint32_t)k8[5])<<16; /* fall through */ case 5 : b+=((uint32_t)k8[4])<<24; /* fall through */ case 4 : a+=k[0]; break; case 3 : a+=((uint32_t)k8[2])<<8; /* fall through */ case 2 : a+=((uint32_t)k8[1])<<16; /* fall through */ case 1 : a+=((uint32_t)k8[0])<<24; break; case 0 : return c; } #endif /* !VALGRIND */ } else { /* need to read the key one byte at a time */ const uint8_t *k = (const uint8_t *)key; /*--------------- all but the last block: affect some 32 bits of (a,b,c) */ while (length > 12) { a += ((uint32_t)k[0])<<24; a += ((uint32_t)k[1])<<16; a += ((uint32_t)k[2])<<8; a += ((uint32_t)k[3]); b += ((uint32_t)k[4])<<24; b += ((uint32_t)k[5])<<16; b += ((uint32_t)k[6])<<8; b += ((uint32_t)k[7]); c += ((uint32_t)k[8])<<24; c += ((uint32_t)k[9])<<16; c += ((uint32_t)k[10])<<8; c += ((uint32_t)k[11]); mix(a,b,c); length -= 12; k += 12; } /*-------------------------------- last block: affect all 32 bits of (c) */ switch(length) /* all the case statements fall through */ { case 12: c+=k[11]; case 11: c+=((uint32_t)k[10])<<8; case 10: c+=((uint32_t)k[9])<<16; case 9 : c+=((uint32_t)k[8])<<24; case 8 : b+=k[7]; case 7 : b+=((uint32_t)k[6])<<8; case 6 : b+=((uint32_t)k[5])<<16; case 5 : b+=((uint32_t)k[4])<<24; case 4 : a+=k[3]; case 3 : a+=((uint32_t)k[2])<<8; case 2 : a+=((uint32_t)k[1])<<16; case 1 : a+=((uint32_t)k[0])<<24; break; case 0 : return c; } } final(a,b,c); return c; } #ifdef SELF_TEST /* used for timings */ void driver1() { uint8_t buf[256]; uint32_t i; uint32_t h=0; time_t a,z; time(&a); for (i=0; i<256; ++i) buf[i] = 'x'; for (i=0; i<1; ++i) { h = hashlittle(&buf[0],1,h); } time(&z); if (z-a > 0) printf("time %d %.8x\n", z-a, h); } /* check that every input bit changes every output bit half the time */ #define HASHSTATE 1 #define HASHLEN 1 #define MAXPAIR 60 #define MAXLEN 70 void driver2() { uint8_t qa[MAXLEN+1], qb[MAXLEN+2], *a = &qa[0], *b = &qb[1]; uint32_t c[HASHSTATE], d[HASHSTATE], i=0, j=0, k, l, m=0, z; uint32_t e[HASHSTATE],f[HASHSTATE],g[HASHSTATE],h[HASHSTATE]; uint32_t x[HASHSTATE],y[HASHSTATE]; uint32_t hlen; printf("No more than %d trials should ever be needed \n",MAXPAIR/2); for (hlen=0; hlen < MAXLEN; ++hlen) { z=0; for (i=0; i>(8-j)); c[0] = hashlittle(a, hlen, m); b[i] ^= ((k+1)<>(8-j)); d[0] = hashlittle(b, hlen, m); /* check every bit is 1, 0, set, and not set at least once */ for (l=0; lz) z=k; if (k==MAXPAIR) { printf("Some bit didn't change: "); printf("%.8x %.8x %.8x %.8x %.8x %.8x ", e[0],f[0],g[0],h[0],x[0],y[0]); printf("i %d j %d m %d len %d\n", i, j, m, hlen); } if (z==MAXPAIR) goto done; } } } done: if (z < MAXPAIR) { printf("Mix success %2d bytes %2d initvals ",i,m); printf("required %d trials\n", z/2); } } printf("\n"); } /* Check for reading beyond the end of the buffer and alignment problems */ void driver3() { uint8_t buf[MAXLEN+20], *b; uint32_t len; uint8_t q[] = "This is the time for all good men to come to the aid of their country..."; uint32_t h; uint8_t qq[] = "xThis is the time for all good men to come to the aid of their country..."; uint32_t i; uint8_t qqq[] = "xxThis is the time for all good men to come to the aid of their country..."; uint32_t j; uint8_t qqqq[] = "xxxThis is the time for all good men to come to the aid of their country..."; uint32_t ref,x,y; uint8_t *p; printf("Endianness. These lines should all be the same (for values filled in):\n"); printf("%.8x %.8x %.8x\n", hashword((const uint32_t *)q, (sizeof(q)-1)/4, 13), hashword((const uint32_t *)q, (sizeof(q)-5)/4, 13), hashword((const uint32_t *)q, (sizeof(q)-9)/4, 13)); p = q; printf("%.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x\n", hashlittle(p, sizeof(q)-1, 13), hashlittle(p, sizeof(q)-2, 13), hashlittle(p, sizeof(q)-3, 13), hashlittle(p, sizeof(q)-4, 13), hashlittle(p, sizeof(q)-5, 13), hashlittle(p, sizeof(q)-6, 13), hashlittle(p, sizeof(q)-7, 13), hashlittle(p, sizeof(q)-8, 13), hashlittle(p, sizeof(q)-9, 13), hashlittle(p, sizeof(q)-10, 13), hashlittle(p, sizeof(q)-11, 13), hashlittle(p, sizeof(q)-12, 13)); p = &qq[1]; printf("%.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x\n", hashlittle(p, sizeof(q)-1, 13), hashlittle(p, sizeof(q)-2, 13), hashlittle(p, sizeof(q)-3, 13), hashlittle(p, sizeof(q)-4, 13), hashlittle(p, sizeof(q)-5, 13), hashlittle(p, sizeof(q)-6, 13), hashlittle(p, sizeof(q)-7, 13), hashlittle(p, sizeof(q)-8, 13), hashlittle(p, sizeof(q)-9, 13), hashlittle(p, sizeof(q)-10, 13), hashlittle(p, sizeof(q)-11, 13), hashlittle(p, sizeof(q)-12, 13)); p = &qqq[2]; printf("%.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x\n", hashlittle(p, sizeof(q)-1, 13), hashlittle(p, sizeof(q)-2, 13), hashlittle(p, sizeof(q)-3, 13), hashlittle(p, sizeof(q)-4, 13), hashlittle(p, sizeof(q)-5, 13), hashlittle(p, sizeof(q)-6, 13), hashlittle(p, sizeof(q)-7, 13), hashlittle(p, sizeof(q)-8, 13), hashlittle(p, sizeof(q)-9, 13), hashlittle(p, sizeof(q)-10, 13), hashlittle(p, sizeof(q)-11, 13), hashlittle(p, sizeof(q)-12, 13)); p = &qqqq[3]; printf("%.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x %.8x\n", hashlittle(p, sizeof(q)-1, 13), hashlittle(p, sizeof(q)-2, 13), hashlittle(p, sizeof(q)-3, 13), hashlittle(p, sizeof(q)-4, 13), hashlittle(p, sizeof(q)-5, 13), hashlittle(p, sizeof(q)-6, 13), hashlittle(p, sizeof(q)-7, 13), hashlittle(p, sizeof(q)-8, 13), hashlittle(p, sizeof(q)-9, 13), hashlittle(p, sizeof(q)-10, 13), hashlittle(p, sizeof(q)-11, 13), hashlittle(p, sizeof(q)-12, 13)); printf("\n"); /* check that hashlittle2 and hashlittle produce the same results */ i=47; j=0; hashlittle2(q, sizeof(q), &i, &j); if (hashlittle(q, sizeof(q), 47) != i) printf("hashlittle2 and hashlittle mismatch\n"); /* check that hashword2 and hashword produce the same results */ len = 0xdeadbeef; i=47, j=0; hashword2(&len, 1, &i, &j); if (hashword(&len, 1, 47) != i) printf("hashword2 and hashword mismatch %x %x\n", i, hashword(&len, 1, 47)); /* check hashlittle doesn't read before or after the ends of the string */ for (h=0, b=buf+1; h<8; ++h, ++b) { for (i=0; i uint32_t my_hashword(const uint32_t *key, size_t length, uint32_t initval); uint32_t my_hashlittle(const void *key, size_t length, uint32_t initval); uint32_t my_hashbig(const void *key, size_t length, uint32_t initval); void my_hashword2(const uint32_t *key, size_t length, uint32_t *pc, uint32_t *pb); void my_hashlittle2(const void *key, size_t length, uint32_t *pc, uint32_t *pb); #endif /* MY_LOOKUP3_H */ mtbl-0.5/libmy/m4/000077500000000000000000000000001227302760400137535ustar00rootroot00000000000000mtbl-0.5/libmy/m4/ax_compare_version.m4000066400000000000000000000146521227302760400201100ustar00rootroot00000000000000# =========================================================================== # http://www.gnu.org/software/autoconf-archive/ax_compare_version.html # =========================================================================== # # SYNOPSIS # # AX_COMPARE_VERSION(VERSION_A, OP, VERSION_B, [ACTION-IF-TRUE], [ACTION-IF-FALSE]) # # DESCRIPTION # # This macro compares two version strings. Due to the various number of # minor-version numbers that can exist, and the fact that string # comparisons are not compatible with numeric comparisons, this is not # necessarily trivial to do in a autoconf script. This macro makes doing # these comparisons easy. # # The six basic comparisons are available, as well as checking equality # limited to a certain number of minor-version levels. # # The operator OP determines what type of comparison to do, and can be one # of: # # eq - equal (test A == B) # ne - not equal (test A != B) # le - less than or equal (test A <= B) # ge - greater than or equal (test A >= B) # lt - less than (test A < B) # gt - greater than (test A > B) # # Additionally, the eq and ne operator can have a number after it to limit # the test to that number of minor versions. # # eq0 - equal up to the length of the shorter version # ne0 - not equal up to the length of the shorter version # eqN - equal up to N sub-version levels # neN - not equal up to N sub-version levels # # When the condition is true, shell commands ACTION-IF-TRUE are run, # otherwise shell commands ACTION-IF-FALSE are run. The environment # variable 'ax_compare_version' is always set to either 'true' or 'false' # as well. # # Examples: # # AX_COMPARE_VERSION([3.15.7],[lt],[3.15.8]) # AX_COMPARE_VERSION([3.15],[lt],[3.15.8]) # # would both be true. # # AX_COMPARE_VERSION([3.15.7],[eq],[3.15.8]) # AX_COMPARE_VERSION([3.15],[gt],[3.15.8]) # # would both be false. # # AX_COMPARE_VERSION([3.15.7],[eq2],[3.15.8]) # # would be true because it is only comparing two minor versions. # # AX_COMPARE_VERSION([3.15.7],[eq0],[3.15]) # # would be true because it is only comparing the lesser number of minor # versions of the two values. # # Note: The characters that separate the version numbers do not matter. An # empty string is the same as version 0. OP is evaluated by autoconf, not # configure, so must be a string, not a variable. # # The author would like to acknowledge Guido Draheim whose advice about # the m4_case and m4_ifvaln functions make this macro only include the # portions necessary to perform the specific comparison specified by the # OP argument in the final configure script. # # LICENSE # # Copyright (c) 2008 Tim Toolan # # Copying and distribution of this file, with or without modification, are # permitted in any medium without royalty provided the copyright notice # and this notice are preserved. This file is offered as-is, without any # warranty. #serial 11 dnl ######################################################################### AC_DEFUN([AX_COMPARE_VERSION], [ AC_REQUIRE([AC_PROG_AWK]) # Used to indicate true or false condition ax_compare_version=false # Convert the two version strings to be compared into a format that # allows a simple string comparison. The end result is that a version # string of the form 1.12.5-r617 will be converted to the form # 0001001200050617. In other words, each number is zero padded to four # digits, and non digits are removed. AS_VAR_PUSHDEF([A],[ax_compare_version_A]) A=`echo "$1" | sed -e 's/\([[0-9]]*\)/Z\1Z/g' \ -e 's/Z\([[0-9]]\)Z/Z0\1Z/g' \ -e 's/Z\([[0-9]][[0-9]]\)Z/Z0\1Z/g' \ -e 's/Z\([[0-9]][[0-9]][[0-9]]\)Z/Z0\1Z/g' \ -e 's/[[^0-9]]//g'` AS_VAR_PUSHDEF([B],[ax_compare_version_B]) B=`echo "$3" | sed -e 's/\([[0-9]]*\)/Z\1Z/g' \ -e 's/Z\([[0-9]]\)Z/Z0\1Z/g' \ -e 's/Z\([[0-9]][[0-9]]\)Z/Z0\1Z/g' \ -e 's/Z\([[0-9]][[0-9]][[0-9]]\)Z/Z0\1Z/g' \ -e 's/[[^0-9]]//g'` dnl # In the case of le, ge, lt, and gt, the strings are sorted as necessary dnl # then the first line is used to determine if the condition is true. dnl # The sed right after the echo is to remove any indented white space. m4_case(m4_tolower($2), [lt],[ ax_compare_version=`echo "x$A x$B" | sed 's/^ *//' | sort -r | sed "s/x${A}/false/;s/x${B}/true/;1q"` ], [gt],[ ax_compare_version=`echo "x$A x$B" | sed 's/^ *//' | sort | sed "s/x${A}/false/;s/x${B}/true/;1q"` ], [le],[ ax_compare_version=`echo "x$A x$B" | sed 's/^ *//' | sort | sed "s/x${A}/true/;s/x${B}/false/;1q"` ], [ge],[ ax_compare_version=`echo "x$A x$B" | sed 's/^ *//' | sort -r | sed "s/x${A}/true/;s/x${B}/false/;1q"` ],[ dnl Split the operator from the subversion count if present. m4_bmatch(m4_substr($2,2), [0],[ # A count of zero means use the length of the shorter version. # Determine the number of characters in A and B. ax_compare_version_len_A=`echo "$A" | $AWK '{print(length)}'` ax_compare_version_len_B=`echo "$B" | $AWK '{print(length)}'` # Set A to no more than B's length and B to no more than A's length. A=`echo "$A" | sed "s/\(.\{$ax_compare_version_len_B\}\).*/\1/"` B=`echo "$B" | sed "s/\(.\{$ax_compare_version_len_A\}\).*/\1/"` ], [[0-9]+],[ # A count greater than zero means use only that many subversions A=`echo "$A" | sed "s/\(\([[0-9]]\{4\}\)\{m4_substr($2,2)\}\).*/\1/"` B=`echo "$B" | sed "s/\(\([[0-9]]\{4\}\)\{m4_substr($2,2)\}\).*/\1/"` ], [.+],[ AC_WARNING( [illegal OP numeric parameter: $2]) ],[]) # Pad zeros at end of numbers to make same length. ax_compare_version_tmp_A="$A`echo $B | sed 's/./0/g'`" B="$B`echo $A | sed 's/./0/g'`" A="$ax_compare_version_tmp_A" # Check for equality or inequality as necessary. m4_case(m4_tolower(m4_substr($2,0,2)), [eq],[ test "x$A" = "x$B" && ax_compare_version=true ], [ne],[ test "x$A" != "x$B" && ax_compare_version=true ],[ AC_WARNING([illegal OP parameter: $2]) ]) ]) AS_VAR_POPDEF([A])dnl AS_VAR_POPDEF([B])dnl dnl # Execute ACTION-IF-TRUE / ACTION-IF-FALSE. if test "$ax_compare_version" = "true" ; then m4_ifvaln([$4],[$4],[:])dnl m4_ifvaln([$5],[else $5])dnl fi ]) dnl AX_COMPARE_VERSION mtbl-0.5/libmy/m4/ax_define_dir.m4000066400000000000000000000035251227302760400170020ustar00rootroot00000000000000# =========================================================================== # http://www.gnu.org/software/autoconf-archive/ax_define_dir.html # =========================================================================== # # OBSOLETE MACRO # # Deprecated because it does not comply with the GNU Coding Standards. See # the autoconf manual section "Defining Directories" for alternatives. # # SYNOPSIS # # AX_DEFINE_DIR(VARNAME, DIR [, DESCRIPTION]) # # DESCRIPTION # # This macro sets VARNAME to the expansion of the DIR variable, taking # care of fixing up ${prefix} and such. # # VARNAME is then offered as both an output variable and a C preprocessor # symbol. # # Example: # # AX_DEFINE_DIR([DATADIR], [datadir], [Where data are placed to.]) # # LICENSE # # Copyright (c) 2008 Stepan Kasal # Copyright (c) 2008 Andreas Schwab # Copyright (c) 2008 Guido U. Draheim # Copyright (c) 2008 Alexandre Oliva # # Copying and distribution of this file, with or without modification, are # permitted in any medium without royalty provided the copyright notice # and this notice are preserved. This file is offered as-is, without any # warranty. #serial 8 AU_ALIAS([AC_DEFINE_DIR], [AX_DEFINE_DIR]) AC_DEFUN([AX_DEFINE_DIR], [ prefix_NONE= exec_prefix_NONE= test "x$prefix" = xNONE && prefix_NONE=yes && prefix=$ac_default_prefix test "x$exec_prefix" = xNONE && exec_prefix_NONE=yes && exec_prefix=$prefix dnl In Autoconf 2.60, ${datadir} refers to ${datarootdir}, which in turn dnl refers to ${prefix}. Thus we have to use `eval' twice. eval ax_define_dir="\"[$]$2\"" eval ax_define_dir="\"$ax_define_dir\"" AC_SUBST($1, "$ax_define_dir") AC_DEFINE_UNQUOTED($1, "$ax_define_dir", [$3]) test "$prefix_NONE" && prefix=NONE test "$exec_prefix_NONE" && exec_prefix=NONE ]) mtbl-0.5/libmy/m4/ax_prog_xsltproc.m4000066400000000000000000000066731227302760400176260ustar00rootroot00000000000000# =========================================================================== # http://www.gnu.org/software/autoconf-archive/ax_prog_xsltproc.html # =========================================================================== # # SYNOPSIS # # AX_PROG_XSLTPROC([default-flags]) # # DESCRIPTION # # Find an xsltproc executable. # # Input: # # "default-flags" is the default $XSLTPROC_FLAGS, which will be overridden # if the user specifies --with-xsltproc-flags. # # Output: # # $XSLTPROC contains the path to xsltproc, or is empty if none was found # or the user specified --without-xsltproc. $XSLTPROC_FLAGS contains the # flags to use with xsltproc. # # LICENSE # # Copyright (c) 2008,2009 Zmanda Inc. # Copyright (c) 2008,2009 Dustin J. Mitchell # # This program is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the # Free Software Foundation; either version 2 of the License, or (at your # option) any later version. # # This program is distributed in the hope that it will be useful, but # WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General # Public License for more details. # # You should have received a copy of the GNU General Public License along # with this program. If not, see . # # As a special exception, the respective Autoconf Macro's copyright owner # gives unlimited permission to copy, distribute and modify the configure # scripts that are the output of Autoconf when processing the Macro. You # need not follow the terms of the GNU General Public License when using # or distributing such scripts, even though portions of the text of the # Macro appear in them. The GNU General Public License (GPL) does govern # all other use of the material that constitutes the Autoconf Macro. # # This special exception to the GPL applies to versions of the Autoconf # Macro released by the Autoconf Archive. When you make and distribute a # modified version of the Autoconf Macro, you may extend this special # exception to the GPL to apply to your modified version as well. #serial 5 AU_ALIAS([AC_PROG_XSLTPROC], [AX_PROG_XSLTPROC]) AC_DEFUN([AX_PROG_XSLTPROC], [ XSLTPROC_FLAGS="$1" AC_SUBST(XSLTPROC_FLAGS) # The (lack of) whitespace and overquoting here are all necessary for # proper formatting. AC_ARG_WITH(xsltproc, AS_HELP_STRING([--with-xsltproc[[[[[=PATH]]]]]], [Use the xsltproc binary in PATH.]), [ ac_with_xsltproc=$withval; ], [ ac_with_xsltproc=maybe; ]) AC_ARG_WITH(xsltproc-flags, AS_HELP_STRING([ --with-xsltproc-flags=FLAGS], [Flags to pass to xsltproc (default $1)]), [ if test "x$withval" == "xno"; then XSLTPROC_FLAGS='' else if test "x$withval" != "xyes"; then XSLTPROC_FLAGS="$withval" fi fi ]) # search for xsltproc if it wasn't specified if test "$ac_with_xsltproc" = "yes" -o "$ac_with_xsltproc" = "maybe"; then AC_PATH_PROGS(XSLTPROC,xsltproc) else if test "$ac_with_xsltproc" != "no"; then if test -x "$ac_with_xsltproc"; then XSLTPROC="$ac_with_xsltproc"; else AC_MSG_WARN([Specified xsltproc of $ac_with_xsltproc isn't]) AC_MSG_WARN([executable; searching for an alternative.]) AC_PATH_PROGS(XSLTPROC,xsltproc) fi fi fi ]) mtbl-0.5/libmy/m4/ax_pthread.m4000066400000000000000000000312671227302760400163450ustar00rootroot00000000000000# =========================================================================== # http://www.gnu.org/software/autoconf-archive/ax_pthread.html # =========================================================================== # # SYNOPSIS # # AX_PTHREAD([ACTION-IF-FOUND[, ACTION-IF-NOT-FOUND]]) # # DESCRIPTION # # This macro figures out how to build C programs using POSIX threads. It # sets the PTHREAD_LIBS output variable to the threads library and linker # flags, and the PTHREAD_CFLAGS output variable to any special C compiler # flags that are needed. (The user can also force certain compiler # flags/libs to be tested by setting these environment variables.) # # Also sets PTHREAD_CC to any special C compiler that is needed for # multi-threaded programs (defaults to the value of CC otherwise). (This # is necessary on AIX to use the special cc_r compiler alias.) # # NOTE: You are assumed to not only compile your program with these flags, # but also link it with them as well. e.g. you should link with # $PTHREAD_CC $CFLAGS $PTHREAD_CFLAGS $LDFLAGS ... $PTHREAD_LIBS $LIBS # # If you are only building threads programs, you may wish to use these # variables in your default LIBS, CFLAGS, and CC: # # LIBS="$PTHREAD_LIBS $LIBS" # CFLAGS="$CFLAGS $PTHREAD_CFLAGS" # CC="$PTHREAD_CC" # # In addition, if the PTHREAD_CREATE_JOINABLE thread-attribute constant # has a nonstandard name, defines PTHREAD_CREATE_JOINABLE to that name # (e.g. PTHREAD_CREATE_UNDETACHED on AIX). # # Also HAVE_PTHREAD_PRIO_INHERIT is defined if pthread is found and the # PTHREAD_PRIO_INHERIT symbol is defined when compiling with # PTHREAD_CFLAGS. # # ACTION-IF-FOUND is a list of shell commands to run if a threads library # is found, and ACTION-IF-NOT-FOUND is a list of commands to run it if it # is not found. If ACTION-IF-FOUND is not specified, the default action # will define HAVE_PTHREAD. # # Please let the authors know if this macro fails on any platform, or if # you have any other suggestions or comments. This macro was based on work # by SGJ on autoconf scripts for FFTW (http://www.fftw.org/) (with help # from M. Frigo), as well as ac_pthread and hb_pthread macros posted by # Alejandro Forero Cuervo to the autoconf macro repository. We are also # grateful for the helpful feedback of numerous users. # # Updated for Autoconf 2.68 by Daniel Richard G. # # LICENSE # # Copyright (c) 2008 Steven G. Johnson # Copyright (c) 2011 Daniel Richard G. # # This program is free software: you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the # Free Software Foundation, either version 3 of the License, or (at your # option) any later version. # # This program is distributed in the hope that it will be useful, but # WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General # Public License for more details. # # You should have received a copy of the GNU General Public License along # with this program. If not, see . # # As a special exception, the respective Autoconf Macro's copyright owner # gives unlimited permission to copy, distribute and modify the configure # scripts that are the output of Autoconf when processing the Macro. You # need not follow the terms of the GNU General Public License when using # or distributing such scripts, even though portions of the text of the # Macro appear in them. The GNU General Public License (GPL) does govern # all other use of the material that constitutes the Autoconf Macro. # # This special exception to the GPL applies to versions of the Autoconf # Macro released by the Autoconf Archive. When you make and distribute a # modified version of the Autoconf Macro, you may extend this special # exception to the GPL to apply to your modified version as well. #serial 20 AU_ALIAS([ACX_PTHREAD], [AX_PTHREAD]) AC_DEFUN([AX_PTHREAD], [ AC_REQUIRE([AC_CANONICAL_HOST]) AC_LANG_PUSH([C]) ax_pthread_ok=no # We used to check for pthread.h first, but this fails if pthread.h # requires special compiler flags (e.g. on True64 or Sequent). # It gets checked for in the link test anyway. # First of all, check if the user has set any of the PTHREAD_LIBS, # etcetera environment variables, and if threads linking works using # them: if test x"$PTHREAD_LIBS$PTHREAD_CFLAGS" != x; then save_CFLAGS="$CFLAGS" CFLAGS="$CFLAGS $PTHREAD_CFLAGS" save_LIBS="$LIBS" LIBS="$PTHREAD_LIBS $LIBS" AC_MSG_CHECKING([for pthread_join in LIBS=$PTHREAD_LIBS with CFLAGS=$PTHREAD_CFLAGS]) AC_TRY_LINK_FUNC(pthread_join, ax_pthread_ok=yes) AC_MSG_RESULT($ax_pthread_ok) if test x"$ax_pthread_ok" = xno; then PTHREAD_LIBS="" PTHREAD_CFLAGS="" fi LIBS="$save_LIBS" CFLAGS="$save_CFLAGS" fi # We must check for the threads library under a number of different # names; the ordering is very important because some systems # (e.g. DEC) have both -lpthread and -lpthreads, where one of the # libraries is broken (non-POSIX). # Create a list of thread flags to try. Items starting with a "-" are # C compiler flags, and other items are library names, except for "none" # which indicates that we try without any flags at all, and "pthread-config" # which is a program returning the flags for the Pth emulation library. ax_pthread_flags="pthreads none -Kthread -kthread lthread -pthread -pthreads -mthreads pthread --thread-safe -mt pthread-config" # The ordering *is* (sometimes) important. Some notes on the # individual items follow: # pthreads: AIX (must check this before -lpthread) # none: in case threads are in libc; should be tried before -Kthread and # other compiler flags to prevent continual compiler warnings # -Kthread: Sequent (threads in libc, but -Kthread needed for pthread.h) # -kthread: FreeBSD kernel threads (preferred to -pthread since SMP-able) # lthread: LinuxThreads port on FreeBSD (also preferred to -pthread) # -pthread: Linux/gcc (kernel threads), BSD/gcc (userland threads) # -pthreads: Solaris/gcc # -mthreads: Mingw32/gcc, Lynx/gcc # -mt: Sun Workshop C (may only link SunOS threads [-lthread], but it # doesn't hurt to check since this sometimes defines pthreads too; # also defines -D_REENTRANT) # ... -mt is also the pthreads flag for HP/aCC # pthread: Linux, etcetera # --thread-safe: KAI C++ # pthread-config: use pthread-config program (for GNU Pth library) case ${host_os} in solaris*) # On Solaris (at least, for some versions), libc contains stubbed # (non-functional) versions of the pthreads routines, so link-based # tests will erroneously succeed. (We need to link with -pthreads/-mt/ # -lpthread.) (The stubs are missing pthread_cleanup_push, or rather # a function called by this macro, so we could check for that, but # who knows whether they'll stub that too in a future libc.) So, # we'll just look for -pthreads and -lpthread first: ax_pthread_flags="-pthreads pthread -mt -pthread $ax_pthread_flags" ;; darwin*) ax_pthread_flags="-pthread $ax_pthread_flags" ;; esac if test x"$ax_pthread_ok" = xno; then for flag in $ax_pthread_flags; do case $flag in none) AC_MSG_CHECKING([whether pthreads work without any flags]) ;; -*) AC_MSG_CHECKING([whether pthreads work with $flag]) PTHREAD_CFLAGS="$flag" ;; pthread-config) AC_CHECK_PROG(ax_pthread_config, pthread-config, yes, no) if test x"$ax_pthread_config" = xno; then continue; fi PTHREAD_CFLAGS="`pthread-config --cflags`" PTHREAD_LIBS="`pthread-config --ldflags` `pthread-config --libs`" ;; *) AC_MSG_CHECKING([for the pthreads library -l$flag]) PTHREAD_LIBS="-l$flag" ;; esac save_LIBS="$LIBS" save_CFLAGS="$CFLAGS" LIBS="$PTHREAD_LIBS $LIBS" CFLAGS="$CFLAGS $PTHREAD_CFLAGS" # Check for various functions. We must include pthread.h, # since some functions may be macros. (On the Sequent, we # need a special flag -Kthread to make this header compile.) # We check for pthread_join because it is in -lpthread on IRIX # while pthread_create is in libc. We check for pthread_attr_init # due to DEC craziness with -lpthreads. We check for # pthread_cleanup_push because it is one of the few pthread # functions on Solaris that doesn't have a non-functional libc stub. # We try pthread_create on general principles. AC_LINK_IFELSE([AC_LANG_PROGRAM([#include static void routine(void *a) { a = 0; } static void *start_routine(void *a) { return a; }], [pthread_t th; pthread_attr_t attr; pthread_create(&th, 0, start_routine, 0); pthread_join(th, 0); pthread_attr_init(&attr); pthread_cleanup_push(routine, 0); pthread_cleanup_pop(0) /* ; */])], [ax_pthread_ok=yes], []) LIBS="$save_LIBS" CFLAGS="$save_CFLAGS" AC_MSG_RESULT($ax_pthread_ok) if test "x$ax_pthread_ok" = xyes; then break; fi PTHREAD_LIBS="" PTHREAD_CFLAGS="" done fi # Various other checks: if test "x$ax_pthread_ok" = xyes; then save_LIBS="$LIBS" LIBS="$PTHREAD_LIBS $LIBS" save_CFLAGS="$CFLAGS" CFLAGS="$CFLAGS $PTHREAD_CFLAGS" # Detect AIX lossage: JOINABLE attribute is called UNDETACHED. AC_MSG_CHECKING([for joinable pthread attribute]) attr_name=unknown for attr in PTHREAD_CREATE_JOINABLE PTHREAD_CREATE_UNDETACHED; do AC_LINK_IFELSE([AC_LANG_PROGRAM([#include ], [int attr = $attr; return attr /* ; */])], [attr_name=$attr; break], []) done AC_MSG_RESULT($attr_name) if test "$attr_name" != PTHREAD_CREATE_JOINABLE; then AC_DEFINE_UNQUOTED(PTHREAD_CREATE_JOINABLE, $attr_name, [Define to necessary symbol if this constant uses a non-standard name on your system.]) fi AC_MSG_CHECKING([if more special flags are required for pthreads]) flag=no case ${host_os} in aix* | freebsd* | darwin*) flag="-D_THREAD_SAFE";; osf* | hpux*) flag="-D_REENTRANT";; solaris*) if test "$GCC" = "yes"; then flag="-D_REENTRANT" else flag="-mt -D_REENTRANT" fi ;; esac AC_MSG_RESULT(${flag}) if test "x$flag" != xno; then PTHREAD_CFLAGS="$flag $PTHREAD_CFLAGS" fi AC_CACHE_CHECK([for PTHREAD_PRIO_INHERIT], ax_cv_PTHREAD_PRIO_INHERIT, [ AC_LINK_IFELSE([ AC_LANG_PROGRAM([[#include ]], [[int i = PTHREAD_PRIO_INHERIT;]])], [ax_cv_PTHREAD_PRIO_INHERIT=yes], [ax_cv_PTHREAD_PRIO_INHERIT=no]) ]) AS_IF([test "x$ax_cv_PTHREAD_PRIO_INHERIT" = "xyes"], AC_DEFINE([HAVE_PTHREAD_PRIO_INHERIT], 1, [Have PTHREAD_PRIO_INHERIT.])) LIBS="$save_LIBS" CFLAGS="$save_CFLAGS" # More AIX lossage: compile with *_r variant if test "x$GCC" != xyes; then case $host_os in aix*) AS_CASE(["x/$CC"], [x*/c89|x*/c89_128|x*/c99|x*/c99_128|x*/cc|x*/cc128|x*/xlc|x*/xlc_v6|x*/xlc128|x*/xlc128_v6], [#handle absolute path differently from PATH based program lookup AS_CASE(["x$CC"], [x/*], [AS_IF([AS_EXECUTABLE_P([${CC}_r])],[PTHREAD_CC="${CC}_r"])], [AC_CHECK_PROGS([PTHREAD_CC],[${CC}_r],[$CC])])]) ;; esac fi fi test -n "$PTHREAD_CC" || PTHREAD_CC="$CC" AC_SUBST(PTHREAD_LIBS) AC_SUBST(PTHREAD_CFLAGS) AC_SUBST(PTHREAD_CC) # Finally, execute ACTION-IF-FOUND/ACTION-IF-NOT-FOUND: if test x"$ax_pthread_ok" = xyes; then ifelse([$1],,AC_DEFINE(HAVE_PTHREAD,1,[Define if you have POSIX threads libraries and header files.]),[$1]) : else ax_pthread_ok=no $2 fi AC_LANG_POP ])dnl AX_PTHREAD mtbl-0.5/libmy/m4/my_check_docbook_ns_xslt_min.m4000066400000000000000000000101521227302760400221130ustar00rootroot00000000000000# SYNOPSIS # # MY_CHECK_DOCBOOK_NS_XSLT_MIN(min-xslt-version) # # DESCRIPTION # # Check that the 'current' version of docbook is at least version # min-xslt-version. If the test is successful, # $DOCBOOK_NS_XSLT_CURRENT_VERSION will be set to the current docbook # version; if not, it will be set to 'no'. # # Example: # # MY_CHECK_DOCBOOK_NS_XSLT_MIN(1.72.0) # if test "x$DOCBOOK_NS_XSLT_CURRENT_VERSION" = "xno"; then # ... # # LICENSE # # Copyright (c) 2008 Zmanda Inc. # Copyright (c) 2008 Dustin J. Mitchell # # This program is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the # Free Software Foundation; either version 2 of the License, or (at your # option) any later version. # # This program is distributed in the hope that it will be useful, but # WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General # Public License for more details. # # You should have received a copy of the GNU General Public License along # with this program. If not, see . # # As a special exception, the respective Autoconf Macro's copyright owner # gives unlimited permission to copy, distribute and modify the configure # scripts that are the output of Autoconf when processing the Macro. You # need not follow the terms of the GNU General Public License when using # or distributing such scripts, even though portions of the text of the # Macro appear in them. The GNU General Public License (GPL) does govern # all other use of the material that constitutes the Autoconf Macro. # # This special exception to the GPL applies to versions of the Autoconf # Macro released by the Autoconf Archive. When you make and distribute a # modified version of the Autoconf Macro, you may extend this special # exception to the GPL to apply to your modified version as well. #serial 5 AC_DEFUN([MY_CHECK_DOCBOOK_NS_XSLT_MIN], [ AC_REQUIRE([AX_PROG_XSLTPROC]) AC_CACHE_CHECK([for current DocBook-NS XSLT version], [ac_cv_docbook_ns_xslt_current_version], [ ac_cv_docbook_ns_xslt_current_version=no if test -n "$XSLTPROC"; then cat >conftest.xsl < EOF echo "Trying '$XSLTPROC $XSLTPROC_FLAGS http://docbook.sourceforge.net/release/xsl-ns/current/VERSION' with input:" >&AS_MESSAGE_LOG_FD echo "====" >&AS_MESSAGE_LOG_FD cat conftest.xsl >&AS_MESSAGE_LOG_FD echo "====" >&AS_MESSAGE_LOG_FD ac_cv_docbook_ns_xslt_current_version=`$XSLTPROC $XSLTPROC_FLAGS conftest.xsl http://docbook.sourceforge.net/release/xsl-ns/current/VERSION 2>&AS_MESSAGE_LOG_FD` if test "$?" != 0; then ac_cv_docbook_ns_xslt_current_version='no' fi rm conftest.xsl fi ]) DOCBOOK_NS_XSLT_CURRENT_VERSION="$ac_cv_docbook_ns_xslt_current_version" AC_MSG_CHECKING([whether DocBook-NS XSLT version is $1 or newer]) if test x"$DOCBOOK_NS_XSLT_CURRENT_VERSION" = x"no"; then AC_MSG_RESULT([no]) else AX_COMPARE_VERSION([$DOCBOOK_NS_XSLT_CURRENT_VERSION], [lt], [$1], [ # version is less than required, so mark it as "no" DOCBOOK_NS_XSLT_CURRENT_VERSION=no ]) if test x"$DOCBOOK_NS_XSLT_CURRENT_VERSION" = x"no"; then AC_MSG_RESULT([no]) else AC_MSG_RESULT([yes ($DOCBOOK_NS_XSLT_CURRENT_VERSION)]) fi fi ]) mtbl-0.5/libmy/m4/pcap.m4000066400000000000000000000025261227302760400151450ustar00rootroot00000000000000AC_DEFUN([MY_CHECK_LIBPCAP], [ libpcap_CFLAGS="" libpcap_LIBS="-lpcap" AC_ARG_WITH( [libpcap], AC_HELP_STRING([--with-libpcap=DIR], [libpcap installation path]), [], [withval="yes"] ) if test "$withval" = "yes"; then withval="/usr /usr/local" fi libpcap_dir="" AC_MSG_CHECKING([for libpcap headers]) for dir in $withval; do if test -f "$dir/include/pcap.h"; then libpcap_dir="$dir" if test "$dir" != "/usr"; then libpcap_CFLAGS="-I$dir/include" fi break fi done if test -n "$libpcap_dir"; then AC_MSG_RESULT([$libpcap_dir]) else AC_MSG_ERROR([cannot find pcap.h in $withval]) fi save_LDFLAGS="$LDFLAGS" save_LIBS="$LIBS" if test "$libpcap_dir" != "/usr"; then libpcap_LIBS="$libpcap_LIBS -L$libpcap_dir/lib" LDFLAGS="-L$libpcap_dir/lib" fi AC_CHECK_LIB( [pcap], [pcap_open_offline], [], [AC_MSG_ERROR([required library not found])] ) AC_SEARCH_LIBS( [pcap_create], [pcap], AC_DEFINE([HAVE_PCAP_CREATE], [1], [Define to 1 if pcap_create() is available.]) ) LDFLAGS="$save_LDFLAGS" LIBS="$save_LIBS" AC_SUBST([libpcap_CFLAGS]) AC_SUBST([libpcap_LIBS]) ]) mtbl-0.5/libmy/m4/pkg.m4000066400000000000000000000171671227302760400150120ustar00rootroot00000000000000# pkg.m4 - Macros to locate and utilise pkg-config. -*- Autoconf -*- # serial 1 (pkg-config-0.24) # # Copyright © 2004 Scott James Remnant . # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, but # WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # # As a special exception to the GNU General Public License, if you # distribute this file as part of a program that contains a # configuration script generated by Autoconf, you may include it under # the same distribution terms that you use for the rest of that program. # PKG_PROG_PKG_CONFIG([MIN-VERSION]) # ---------------------------------- AC_DEFUN([PKG_PROG_PKG_CONFIG], [m4_pattern_forbid([^_?PKG_[A-Z_]+$]) m4_pattern_allow([^PKG_CONFIG(_(PATH|LIBDIR|SYSROOT_DIR|ALLOW_SYSTEM_(CFLAGS|LIBS)))?$]) m4_pattern_allow([^PKG_CONFIG_(DISABLE_UNINSTALLED|TOP_BUILD_DIR|DEBUG_SPEW)$]) AC_ARG_VAR([PKG_CONFIG], [path to pkg-config utility]) AC_ARG_VAR([PKG_CONFIG_PATH], [directories to add to pkg-config's search path]) AC_ARG_VAR([PKG_CONFIG_LIBDIR], [path overriding pkg-config's built-in search path]) if test "x$ac_cv_env_PKG_CONFIG_set" != "xset"; then AC_PATH_TOOL([PKG_CONFIG], [pkg-config]) fi if test -n "$PKG_CONFIG"; then _pkg_min_version=m4_default([$1], [0.9.0]) AC_MSG_CHECKING([pkg-config is at least version $_pkg_min_version]) if $PKG_CONFIG --atleast-pkgconfig-version $_pkg_min_version; then AC_MSG_RESULT([yes]) else AC_MSG_RESULT([no]) PKG_CONFIG="" fi fi[]dnl ])# PKG_PROG_PKG_CONFIG # PKG_CHECK_EXISTS(MODULES, [ACTION-IF-FOUND], [ACTION-IF-NOT-FOUND]) # # Check to see whether a particular set of modules exists. Similar # to PKG_CHECK_MODULES(), but does not set variables or print errors. # # Please remember that m4 expands AC_REQUIRE([PKG_PROG_PKG_CONFIG]) # only at the first occurence in configure.ac, so if the first place # it's called might be skipped (such as if it is within an "if", you # have to call PKG_CHECK_EXISTS manually # -------------------------------------------------------------- AC_DEFUN([PKG_CHECK_EXISTS], [AC_REQUIRE([PKG_PROG_PKG_CONFIG])dnl if test -n "$PKG_CONFIG" && \ AC_RUN_LOG([$PKG_CONFIG --exists --print-errors "$1"]); then m4_default([$2], [:]) m4_ifvaln([$3], [else $3])dnl fi]) # _PKG_CONFIG([VARIABLE], [COMMAND], [MODULES]) # --------------------------------------------- m4_define([_PKG_CONFIG], [if test -n "$$1"; then pkg_cv_[]$1="$$1" elif test -n "$PKG_CONFIG"; then PKG_CHECK_EXISTS([$3], [pkg_cv_[]$1=`$PKG_CONFIG --[]$2 "$3" 2>/dev/null` test "x$?" != "x0" && pkg_failed=yes ], [pkg_failed=yes]) else pkg_failed=untried fi[]dnl ])# _PKG_CONFIG # _PKG_SHORT_ERRORS_SUPPORTED # ----------------------------- AC_DEFUN([_PKG_SHORT_ERRORS_SUPPORTED], [AC_REQUIRE([PKG_PROG_PKG_CONFIG]) if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then _pkg_short_errors_supported=yes else _pkg_short_errors_supported=no fi[]dnl ])# _PKG_SHORT_ERRORS_SUPPORTED # PKG_CHECK_MODULES(VARIABLE-PREFIX, MODULES, [ACTION-IF-FOUND], # [ACTION-IF-NOT-FOUND]) # # # Note that if there is a possibility the first call to # PKG_CHECK_MODULES might not happen, you should be sure to include an # explicit call to PKG_PROG_PKG_CONFIG in your configure.ac # # # -------------------------------------------------------------- AC_DEFUN([PKG_CHECK_MODULES], [AC_REQUIRE([PKG_PROG_PKG_CONFIG])dnl AC_ARG_VAR([$1][_CFLAGS], [C compiler flags for $1, overriding pkg-config])dnl AC_ARG_VAR([$1][_LIBS], [linker flags for $1, overriding pkg-config])dnl pkg_failed=no AC_MSG_CHECKING([for $1]) _PKG_CONFIG([$1][_CFLAGS], [cflags], [$2]) _PKG_CONFIG([$1][_LIBS], [libs], [$2]) m4_define([_PKG_TEXT], [Alternatively, you may set the environment variables $1[]_CFLAGS and $1[]_LIBS to avoid the need to call pkg-config. See the pkg-config man page for more details.]) if test $pkg_failed = yes; then AC_MSG_RESULT([no]) _PKG_SHORT_ERRORS_SUPPORTED if test $_pkg_short_errors_supported = yes; then $1[]_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "$2" 2>&1` else $1[]_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "$2" 2>&1` fi # Put the nasty error message in config.log where it belongs echo "$$1[]_PKG_ERRORS" >&AS_MESSAGE_LOG_FD m4_default([$4], [AC_MSG_ERROR( [Package requirements ($2) were not met: $$1_PKG_ERRORS Consider adjusting the PKG_CONFIG_PATH environment variable if you installed software in a non-standard prefix. _PKG_TEXT])[]dnl ]) elif test $pkg_failed = untried; then AC_MSG_RESULT([no]) m4_default([$4], [AC_MSG_FAILURE( [The pkg-config script could not be found or is too old. Make sure it is in your PATH or set the PKG_CONFIG environment variable to the full path to pkg-config. _PKG_TEXT To get pkg-config, see .])[]dnl ]) else $1[]_CFLAGS=$pkg_cv_[]$1[]_CFLAGS $1[]_LIBS=$pkg_cv_[]$1[]_LIBS AC_MSG_RESULT([yes]) $3 fi[]dnl ])# PKG_CHECK_MODULES # PKG_INSTALLDIR(DIRECTORY) # ------------------------- # Substitutes the variable pkgconfigdir as the location where a module # should install pkg-config .pc files. By default the directory is # $libdir/pkgconfig, but the default can be changed by passing # DIRECTORY. The user can override through the --with-pkgconfigdir # parameter. AC_DEFUN([PKG_INSTALLDIR], [m4_pushdef([pkg_default], [m4_default([$1], ['${libdir}/pkgconfig'])]) m4_pushdef([pkg_description], [pkg-config installation directory @<:@]pkg_default[@:>@]) AC_ARG_WITH([pkgconfigdir], [AS_HELP_STRING([--with-pkgconfigdir], pkg_description)],, [with_pkgconfigdir=]pkg_default) AC_SUBST([pkgconfigdir], [$with_pkgconfigdir]) m4_popdef([pkg_default]) m4_popdef([pkg_description]) ]) dnl PKG_INSTALLDIR # PKG_NOARCH_INSTALLDIR(DIRECTORY) # ------------------------- # Substitutes the variable noarch_pkgconfigdir as the location where a # module should install arch-independent pkg-config .pc files. By # default the directory is $datadir/pkgconfig, but the default can be # changed by passing DIRECTORY. The user can override through the # --with-noarch-pkgconfigdir parameter. AC_DEFUN([PKG_NOARCH_INSTALLDIR], [m4_pushdef([pkg_default], [m4_default([$1], ['${datadir}/pkgconfig'])]) m4_pushdef([pkg_description], [pkg-config arch-independent installation directory @<:@]pkg_default[@:>@]) AC_ARG_WITH([noarch-pkgconfigdir], [AS_HELP_STRING([--with-noarch-pkgconfigdir], pkg_description)],, [with_noarch_pkgconfigdir=]pkg_default) AC_SUBST([noarch_pkgconfigdir], [$with_noarch_pkgconfigdir]) m4_popdef([pkg_default]) m4_popdef([pkg_description]) ]) dnl PKG_NOARCH_INSTALLDIR # PKG_CHECK_VAR(VARIABLE, MODULE, CONFIG-VARIABLE, # [ACTION-IF-FOUND], [ACTION-IF-NOT-FOUND]) # ------------------------------------------- # Retrieves the value of the pkg-config variable for the given module. AC_DEFUN([PKG_CHECK_VAR], [AC_REQUIRE([PKG_PROG_PKG_CONFIG])dnl AC_ARG_VAR([$1], [value of $3 for $2, overriding pkg-config])dnl _PKG_CONFIG([$1], [variable="][$3]["], [$2]) AS_VAR_COPY([$1], [pkg_cv_][$1]) AS_VAR_IF([$1], [""], [$5], [$4])dnl ])# PKG_CHECK_VAR mtbl-0.5/libmy/m4/protobuf-c.m4000066400000000000000000000033071227302760400163000ustar00rootroot00000000000000AC_DEFUN([MY_CHECK_LIBPROTOBUF_C], [ libprotobuf_c_CFLAGS="" libprotobuf_c_LIBS="-lprotobuf-c" AC_ARG_WITH( [libprotobuf_c], AC_HELP_STRING([--with-libprotobuf_c=DIR], [libprotobuf-c installation path]), [], [withval="yes"] ) if test "$withval" = "yes"; then withval="/usr /usr/local" fi libprotobuf_c_dir="" AC_MSG_CHECKING([for libprotobuf-c headers]) for dir in $withval; do if test -f "$dir/include/protobuf-c/protobuf-c.h"; then libprotobuf_c_dir="$dir" if test "$dir" != "/usr"; then libprotobuf_c_CFLAGS="-I$dir/include" fi break elif test -f "$dir/include/google/protobuf-c/protobuf-c.h"; then libprotobuf_c_dir="$dir" libprotobuf_c_CFLAGS="-I$dir/include/google" break fi done if test -n "$libprotobuf_c_dir"; then AC_MSG_RESULT([$libprotobuf_c_dir]) else AC_MSG_ERROR([cannot find protobuf-c.h in $withval]) fi save_LDFLAGS="$LDFLAGS" save_LIBS="$LIBS" if test "$libprotobuf_c_dir" != "/usr"; then libprotobuf_c_LIBS="$libprotobuf_c_LIBS -L$libprotobuf_c_dir/lib" LDFLAGS="-L$libprotobuf_c_dir/lib" fi AC_CHECK_LIB( [protobuf-c], [protobuf_c_message_pack], [], [AC_MSG_ERROR([required library not found])] ) LDFLAGS="$save_LDFLAGS" LIBS="$save_LIBS" AC_SUBST([libprotobuf_c_CFLAGS]) AC_SUBST([libprotobuf_c_LIBS]) AC_PATH_PROG([PROTOC_C], [protoc-c]) if test -z "$PROTOC_C"; then AC_MSG_ERROR([The protoc-c program was not found. Please install the protobuf-c compiler!]) fi ]) mtbl-0.5/libmy/my_alloc.h000066400000000000000000000011401227302760400153770ustar00rootroot00000000000000#ifndef MY_ALLOC_H #define MY_ALLOC_H #include #include #include static inline void * my_calloc(size_t nmemb, size_t size) { void *ptr = calloc(nmemb, size); assert(ptr != NULL); return (ptr); } static inline void * my_malloc(size_t size) { void *ptr = malloc(size); assert(ptr != NULL); return (ptr); } static inline void * my_realloc(void *ptr, size_t size) { ptr = realloc(ptr, size); assert(ptr != NULL); return (ptr); } static inline char * my_strdup(const char *s) { char *ptr = strdup(s); assert(ptr != NULL); return (ptr); } #endif /* MY_ALLOC_H */ mtbl-0.5/libmy/my_byteorder.h000066400000000000000000000013271227302760400163130ustar00rootroot00000000000000#ifndef MY_BYTEORDER_H #define MY_BYTEORDER_H #include "config.h" #ifdef HAVE_ENDIAN_H # include #else # ifdef HAVE_SYS_ENDIAN_H # include # endif #endif #if HAVE_DECL_HTOLE32 # define my_htole32 htole32 #else # if defined(WORDS_BIGENDIAN) # define my_htole32 my_bswap32 # else # define my_htole32(x) (x) # endif #endif #if HAVE_DECL_LE32TOH # define my_le32toh le32toh #else # if defined(WORDS_BIGENDIAN) # define my_le32toh my_bswap32 # else # define my_le32toh(x) (x) # endif #endif static inline uint32_t my_bswap32(uint32_t x) { return ((x << 24) & 0xff000000 ) | ((x << 8) & 0x00ff0000 ) | ((x >> 8) & 0x0000ff00 ) | ((x >> 24) & 0x000000ff ); } #endif /* MY_BYTEORDER_H */ mtbl-0.5/libmy/my_fileset.c000066400000000000000000000120531227302760400157400ustar00rootroot00000000000000/* * Copyright (c) 2012 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #include #include #include "my_alloc.h" #include "my_fileset.h" #include "ubuf.h" #include "vector.h" struct fileset_entry { bool keep; char *fname; void *ptr; }; VECTOR_GENERATE(entry_vec, struct fileset_entry *); struct my_fileset { ino_t last_ino; time_t last_mtime; char *setfile; char *setdir; my_fileset_load_func load; my_fileset_unload_func unload; void *user; entry_vec *entries; }; static bool path_exists(const char *path) { struct stat sb; int ret; ret = stat(path, &sb); if (ret < 0) return (false); return (true); } static bool setfile_updated(struct my_fileset *fs) { struct stat ss; int ret; ret = stat(fs->setfile, &ss); if (ret < 0) { fprintf(stderr, "%s: stat('%s') failed: %s\n", __func__, fs->setfile, strerror(errno)); return (false); } if (fs->last_ino != ss.st_ino || fs->last_mtime != ss.st_mtime) { fs->last_ino = ss.st_ino; fs->last_mtime = ss.st_mtime; return (true); } return (false); } static int cmp_fileset_entry(const void *a, const void *b) { struct fileset_entry *fs0 = *((struct fileset_entry **) a); struct fileset_entry *fs1 = *((struct fileset_entry **) b); assert(a != NULL); assert(b != NULL); assert(fs0 != NULL); assert(fs1 != NULL); assert(fs0->fname != NULL); assert(fs1->fname != NULL); return (strcmp(fs0->fname, fs1->fname)); } static struct fileset_entry ** fetch_entry(entry_vec *entries, char *fname) { struct fileset_entry **ent; struct fileset_entry key = { .fname = fname, .ptr = NULL }; struct fileset_entry *pkey = &key; ent = bsearch(&pkey, entry_vec_data(entries), entry_vec_size(entries), sizeof(void *), cmp_fileset_entry); return (ent); } struct my_fileset * my_fileset_init( const char *setfile, my_fileset_load_func load, my_fileset_unload_func unload, void *user) { assert(path_exists(setfile)); struct my_fileset *fs = my_calloc(1, sizeof(*fs)); char *t = my_strdup(setfile); fs->setdir = my_strdup(dirname(t)); free(t); fs->setfile = my_strdup(setfile); fs->load = load; fs->unload = unload; fs->user = user; fs->entries = entry_vec_init(1); return (fs); } void my_fileset_destroy(struct my_fileset **fs) { if (*fs != NULL) { for (size_t i = 0; i < entry_vec_size((*fs)->entries); i++) { struct fileset_entry *ent = entry_vec_value((*fs)->entries, i); if ((*fs)->unload) (*fs)->unload(*fs, ent->fname, ent->ptr); free(ent->fname); free(ent); } entry_vec_destroy(&(*fs)->entries); free((*fs)->setdir); free((*fs)->setfile); free(*fs); *fs = NULL; } } void * my_fileset_user(struct my_fileset *fs) { return (fs->user); } void my_fileset_reload(struct my_fileset *fs) { assert(fs != NULL); struct fileset_entry *ent, **entptr; entry_vec *new_entries; FILE *fp; char *fname, *line = NULL; size_t len = 0; ubuf *u; if (!setfile_updated(fs)) return; u = ubuf_init(64); new_entries = entry_vec_init(1); fp = fopen(fs->setfile, "r"); if (fp == NULL) return; while (getline(&line, &len, fp) != -1) { ubuf_clip(u, 0); ubuf_add_cstr(u, fs->setdir); ubuf_add(u, '/'); ubuf_add_cstr(u, line); ubuf_rstrip(u, '\n'); fname = ubuf_cstr(u); if (path_exists(fname)) { entptr = fetch_entry(fs->entries, fname); if (entptr == NULL) { ent = my_calloc(1, sizeof(*ent)); ent->fname = my_strdup(fname); if (fs->load) ent->ptr = fs->load(fs, fname); entry_vec_add(new_entries, ent); } else { ent = my_calloc(1, sizeof(*ent)); ent->fname = my_strdup(fname); ent->ptr = (*entptr)->ptr; (*entptr)->keep = true; entry_vec_add(new_entries, ent); } } } free(line); fclose(fp); qsort(entry_vec_data(new_entries), entry_vec_size(new_entries), sizeof(void *), cmp_fileset_entry); for (size_t i = 0; i < entry_vec_size(fs->entries); i++) { ent = entry_vec_value(fs->entries, i); assert(ent != NULL); if (ent->keep == false && fs->unload) fs->unload(fs, ent->fname, ent->ptr); free(ent->fname); free(ent); } entry_vec_destroy(&fs->entries); fs->entries = new_entries; ubuf_destroy(&u); } bool my_fileset_get( struct my_fileset *fs, size_t i, const char **fname_out, void **ptr_out) { if (i < entry_vec_size(fs->entries)) { *fname_out = entry_vec_value(fs->entries, i)->fname; *ptr_out = entry_vec_value(fs->entries, i)->ptr; return (true); } return (false); } mtbl-0.5/libmy/my_fileset.h000066400000000000000000000011421227302760400157420ustar00rootroot00000000000000#ifndef MY_FILESET_H #define MY_FILESET_H #include struct my_fileset; typedef void *(*my_fileset_load_func)(struct my_fileset *, const char *fname); typedef void (*my_fileset_unload_func)(struct my_fileset *, const char *fname, void *); struct my_fileset *my_fileset_init( const char *setfile, my_fileset_load_func, my_fileset_unload_func, void *user); void my_fileset_destroy(struct my_fileset **); void *my_fileset_user(struct my_fileset *); void my_fileset_reload(struct my_fileset *); bool my_fileset_get(struct my_fileset *, size_t, const char **, void **); #endif /* MY_FILESET_H */ mtbl-0.5/libmy/my_memory_barrier.h000066400000000000000000000010631227302760400173270ustar00rootroot00000000000000#ifndef MY_MEMORY_BARRIER_H #define MY_MEMORY_BARRIER_H #if defined(__GNUC__) # if defined(__x86_64__) # define MY_HAVE_MEMORY_BARRIERS 1 # define smp_mb() asm volatile("mfence" ::: "memory") # define smp_rmb() asm volatile("" ::: "memory") # define smp_wmb() asm volatile("" ::: "memory") # elif defined(__ia64__) # define MY_HAVE_MEMORY_BARRIERS 1 # define smp_mb() asm volatile ("mf" ::: "memory") # define smp_rmb() asm volatile ("mf" ::: "memory") # define smp_wmb() asm volatile ("mf" ::: "memory") # endif #endif #endif /* MY_MEMORY_BARRIER_H */ mtbl-0.5/libmy/my_queue.c000066400000000000000000000023071227302760400154320ustar00rootroot00000000000000/* * Copyright (c) 2013, 2014 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "my_memory_barrier.h" #ifdef MY_HAVE_MEMORY_BARRIERS # define my_queue_mb_init my_queue_init # define my_queue_mb_destroy my_queue_destroy # define my_queue_mb_impl_type my_queue_impl_type # define my_queue_mb_insert my_queue_insert # define my_queue_mb_remove my_queue_remove # include "my_queue_mb.c" #else # define my_queue_mutex_init my_queue_init # define my_queue_mutex_destroy my_queue_destroy # define my_queue_mutex_impl_type my_queue_impl_type # define my_queue_mutex_insert my_queue_insert # define my_queue_mutex_remove my_queue_remove # include "my_queue_mutex.c" #endif mtbl-0.5/libmy/my_queue.h000066400000000000000000000055671227302760400154520ustar00rootroot00000000000000#ifndef MY_QUEUE_H #define MY_QUEUE_H #include /** * \file * * Fixed-size single-producer / single-consumer queue. * * This is a generic queue that supports a single producer thread and a * single consumer thread. The implementation uses a fixed power-of-2 size * circular buffer. * * The my_queue_insert() and my_queue_remove() functions are "non-blocking"; * that is, the policies for queue full / queue empty conditions are left to * the caller. These functions return a boolean indicating whether the queue * operation succeeded or not. For example, a producer that spins until an * element is successfully enqueued might look like: * * void *item; * produce_item(&item); * while (!my_queue_insert(q, item, NULL)); * * And a consumer that spins until an element is successfully dequeued * might look like: * * void *item; * while (!my_queue_remove(q, &item, NULL)); * consume_item(item); * * The my_queue_insert() and my_queue_remove() functions take an optional third * parameter for returning the spaces remaining in the queue or the count of * elements remaining, respectively. This allows for more complicated * coordination between producer and consumer, for instance a consumer thread * that sleeps when the queue is empty and is woken by the producer when it * adds an element to an empty queue. */ struct my_queue; /** * Initialize a new queue. * * \param[in] num_entries Number of elements in the queue. Must be >=2, and a power-of-2. * \param[in] size_entry Size in bytes of each queue entry. * \return Opaque pointer that is NULL on failure or non-NULL on success. */ struct my_queue * my_queue_init(unsigned num_entries, unsigned size_entry); /** * Destroy a queue. */ void my_queue_destroy(struct my_queue **q); /** * Describe the queue implementation type. */ const char * my_queue_impl_type(void); /** * Insert an element into the queue. * * \param[in] q Queue object. * \param[in] elem Element object. * \param[out] space If non-NULL, pointer to store the number of remaining * spaces in the queue. * \return true if the element was inserted into the queue, * false if the queue is full. */ bool my_queue_insert(struct my_queue *q, void *elem, unsigned *space); /** * Remove an element from the queue. * * \param[in] q Queue object. * \param[out] elem Where the element object will be copied. * \param[out] count If non-NULL, pointer to store the count of elements * remaining in the queue. * \return true if an element was removed from the queue, * false if the queue is empty. */ bool my_queue_remove(struct my_queue *q, void *elem, unsigned *count); struct my_queue_ops { struct my_queue *(*init)(unsigned, unsigned); void (*destroy)(struct my_queue **); const char *(*impl_type)(void); bool (*insert)(struct my_queue *, void *, unsigned *); bool (*remove)(struct my_queue *, void *, unsigned *); }; #endif /* MY_QUEUE_H */ mtbl-0.5/libmy/my_queue_mb.c000066400000000000000000000061251227302760400161120ustar00rootroot00000000000000/* * Copyright (c) 2013, 2014 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "my_memory_barrier.h" #ifdef MY_HAVE_MEMORY_BARRIERS #include #include #include #include "my_alloc.h" #include "my_queue.h" #define MY_ACCESS_ONCE(x) (*(volatile typeof(x) *)&(x)) struct my_queue * my_queue_mb_init(unsigned, unsigned); void my_queue_mb_destroy(struct my_queue **); const char * my_queue_mb_impl_type(void); bool my_queue_mb_insert(struct my_queue *, void *, unsigned *); bool my_queue_mb_remove(struct my_queue *, void *, unsigned *); struct my_queue { uint8_t *data; unsigned num_elems; unsigned sizeof_elem; unsigned head; unsigned tail; }; struct my_queue * my_queue_mb_init(unsigned num_elems, unsigned sizeof_elem) { struct my_queue *q; if (num_elems < 2 || ((num_elems - 1) & num_elems) != 0) return (NULL); q = my_calloc(1, sizeof(*q)); q->num_elems = num_elems; q->sizeof_elem = sizeof_elem; q->data = my_calloc(q->num_elems, q->sizeof_elem); return (q); } void my_queue_mb_destroy(struct my_queue **q) { if (*q) { free((*q)->data); free(*q); *q = NULL; } } const char * my_queue_mb_impl_type(void) { return ("memory barrier"); } static inline unsigned q_space(unsigned head, unsigned tail, unsigned size) { return ((tail - (head + 1)) & (size - 1)); } static inline unsigned q_count(unsigned head, unsigned tail, unsigned size) { return ((head - tail) & (size - 1)); } bool my_queue_mb_insert(struct my_queue *q, void *item, unsigned *pspace) { bool res = false; unsigned head = q->head; unsigned tail = MY_ACCESS_ONCE(q->tail); unsigned space = q_space(head, tail, q->num_elems); if (space >= 1) { memcpy(&q->data[head * q->sizeof_elem], item, q->sizeof_elem); smp_wmb(); q->head = (head + 1) & (q->num_elems - 1); smp_wmb(); res = true; space--; } if (pspace != NULL) *pspace = space; return (res); } bool my_queue_mb_remove(struct my_queue *q, void *item, unsigned *pcount) { bool res = false; unsigned head = MY_ACCESS_ONCE(q->head); unsigned tail = q->tail; unsigned count = q_count(head, tail, q->num_elems); if (count >= 1) { memcpy(item, &q->data[tail * q->sizeof_elem], q->sizeof_elem); smp_mb(); q->tail = (tail + 1) & (q->num_elems - 1); res = true; count--; } if (pcount != NULL) *pcount = count; return (res); } const struct my_queue_ops my_queue_mb_ops = { .init = my_queue_mb_init, .destroy = my_queue_mb_destroy, .impl_type = my_queue_mb_impl_type, .insert = my_queue_mb_insert, .remove = my_queue_mb_remove, }; #endif /* MY_HAVE_MEMORY_BARRIERS */ mtbl-0.5/libmy/my_queue_mutex.c000066400000000000000000000066651227302760400166670ustar00rootroot00000000000000/* * Copyright (c) 2013, 2014 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include "my_alloc.h" #include "my_queue.h" #if defined(__GNUC__) # define _aligned __attribute__((aligned(64))) #else # define _aligned #endif struct my_queue { uint8_t *data; unsigned num_elems; unsigned sizeof_elem; unsigned head; unsigned tail; pthread_mutex_t lock _aligned; }; struct my_queue * my_queue_mutex_init(unsigned, unsigned); void my_queue_mutex_destroy(struct my_queue **); const char * my_queue_mutex_impl_type(void); bool my_queue_mutex_insert(struct my_queue *, void *, unsigned *); bool my_queue_mutex_remove(struct my_queue *, void *, unsigned *); struct my_queue * my_queue_mutex_init(unsigned num_elems, unsigned sizeof_elem) { struct my_queue *q; if (num_elems < 2 || ((num_elems - 1) & num_elems) != 0) return (NULL); q = my_calloc(1, sizeof(*q)); q->num_elems = num_elems; q->sizeof_elem = sizeof_elem; q->data = my_calloc(q->num_elems, q->sizeof_elem); int rc = pthread_mutex_init(&q->lock, NULL); assert(rc == 0); return (q); } void my_queue_mutex_destroy(struct my_queue **q) { if (*q) { pthread_mutex_destroy(&(*q)->lock); free((*q)->data); free(*q); *q = NULL; } } const char * my_queue_mutex_impl_type(void) { return ("pthread mutex"); } static inline void q_lock(struct my_queue *q) { int rc = pthread_mutex_lock(&q->lock); assert(rc == 0); } static inline void q_unlock(struct my_queue *q) { int rc = pthread_mutex_unlock(&q->lock); assert(rc == 0); } static inline unsigned q_space(unsigned head, unsigned tail, unsigned size) { return ((tail - (head + 1)) & (size - 1)); } static inline unsigned q_count(unsigned head, unsigned tail, unsigned size) { return ((head - tail) & (size - 1)); } bool my_queue_mutex_insert(struct my_queue *q, void *item, unsigned *pspace) { q_lock(q); bool res = false; unsigned head = q->head; unsigned tail = q->tail; unsigned space = q_space(head, tail, q->num_elems); if (space >= 1) { memcpy(&q->data[head * q->sizeof_elem], item, q->sizeof_elem); q->head = (head + 1) & (q->num_elems - 1); res = true; space--; } q_unlock(q); if (pspace) *pspace = space; return (res); } bool my_queue_mutex_remove(struct my_queue *q, void *item, unsigned *pcount) { q_lock(q); bool res = false; unsigned head = q->head; unsigned tail = q->tail; unsigned count = q_count(head, tail, q->num_elems); if (count >= 1) { memcpy(item, &q->data[tail * q->sizeof_elem], q->sizeof_elem); q->tail = (tail + 1) & (q->num_elems - 1); res = true; count--; } q_unlock(q); if (pcount) *pcount = count; return (res); } const struct my_queue_ops my_queue_mutex_ops = { .init = my_queue_mutex_init, .destroy = my_queue_mutex_destroy, .impl_type = my_queue_mutex_impl_type, .insert = my_queue_mutex_insert, .remove = my_queue_mutex_remove, }; mtbl-0.5/libmy/my_time.h000066400000000000000000000022241227302760400152470ustar00rootroot00000000000000#ifndef MY_TIME_H #define MY_TIME_H #include #include #include #include static inline void my_gettime(clockid_t clk_id, struct timespec *ts) { int res; res = clock_gettime(clk_id, ts); assert(res == 0); } static inline void my_timespec_add(const struct timespec *a, struct timespec *b) { b->tv_sec += a->tv_sec; b->tv_nsec += a->tv_nsec; while (b->tv_nsec >= 1000000000) { b->tv_sec += 1; b->tv_nsec -= 1000000000; } } static inline void my_timespec_sub(const struct timespec *a, struct timespec *b) { b->tv_sec -= a->tv_sec; b->tv_nsec -= a->tv_nsec; if (b->tv_nsec < 0) { b->tv_sec -= 1; b->tv_nsec += 1000000000; } } static inline double my_timespec_to_double(const struct timespec *ts) { return (ts->tv_sec + ts->tv_nsec / 1E9); } static inline void my_timespec_from_double(double seconds, struct timespec *ts) { ts->tv_sec = (time_t) seconds; ts->tv_nsec = (long) ((seconds - ((int) seconds)) * 1E9); } static inline void my_nanosleep(const struct timespec *ts) { struct timespec rqt, rmt; for (rqt = *ts; nanosleep(&rqt, &rmt) < 0 && errno == EINTR; rqt = rmt) ; } #endif /* MY_TIME_H */ mtbl-0.5/libmy/print_string.h000066400000000000000000000021011227302760400163200ustar00rootroot00000000000000/* * Copyright (c) 2012 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef MY_PRINT_STRING_H #define MY_PRINT_STRING_H #include #include #include static inline void print_string(const void *data, size_t len, FILE *out) { uint8_t *str = (uint8_t *) data; fputc('"', out); while (len-- != 0) { unsigned c = *(str++); if (isprint(c)) { if (c == '"') fputs("\\\"", out); else fputc(c, out); } else { fprintf(out, "\\x%02x", c); } } fputc('"', out); } #endif /* MY_PRINT_STRING_H */ mtbl-0.5/libmy/rate.c000066400000000000000000000101621227302760400145320ustar00rootroot00000000000000/* * Copyright (c) 2008, 2009, 2012, 2013 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include "my_alloc.h" #include "my_time.h" #include "rate.h" struct rate { struct timespec next_tick, period, start; uint64_t count; unsigned adj_rate, rate, freq; }; static inline int64_t ts_nanos(struct timespec *ts) { return (ts->tv_sec * 1000000000 + ts->tv_nsec); } static inline struct timespec calc_next_tick(const struct timespec *t, const struct timespec *m) { struct timespec res; res = *t; if (m->tv_sec > 0) { res.tv_sec -= (res.tv_sec % m->tv_sec); res.tv_sec += m->tv_sec; } if (m->tv_nsec > 0) { res.tv_nsec -= (res.tv_nsec % m->tv_nsec); res.tv_nsec += m->tv_nsec; } else { res.tv_nsec = 0; } while (res.tv_nsec >= 1000000000) { res.tv_sec += 1; res.tv_nsec -= 1000000000; } return (res); } static inline void adjust_rate(struct rate *r, struct timespec *now) { struct timespec elapsed; double ratio; unsigned actual_rate; /* amount of time elapsed since first sleep */ elapsed = *now; my_timespec_sub(&r->start, &elapsed); /* the average event rate that has been maintained over the * lifespan of this rate-limiter. */ actual_rate = r->count / (ts_nanos(&elapsed) / (1000000000 + 0.0)); /* simple ratio of nominal event rate and average event rate */ ratio = r->rate / (actual_rate + 0.0); /* clamp this ratio to a small interval */ if (ratio < 0.99) ratio = 0.99; if (ratio > 1.01) ratio = 1.01; /* calculate a new, adjusted rate based on this ratio */ r->adj_rate *= ratio; /* calculate a new tick period based on the adjusted rate */ const double period = 1.0 / (r->adj_rate + 0.0); my_timespec_from_double(period, &r->period); } struct rate * rate_init(unsigned rate, unsigned freq) { struct rate *r; r = calloc(1, sizeof(*r)); if (r == NULL) return (NULL); r->adj_rate = rate; r->rate = rate; r->freq = freq; /* calculate the tick period */ const double period = 1.0 / (r->rate + 0.0); my_timespec_from_double(period, &r->period); return (r); } void rate_destroy(struct rate **r) { if (*r != NULL) { free(*r); *r = NULL; } } void rate_sleep(struct rate *r) { struct timespec now, til; /* what clock to use depends on whether clock_nanosleep() is available */ #if HAVE_CLOCK_NANOSLEEP static const clockid_t rate_clock = CLOCK_MONOTONIC; #else static const clockid_t rate_clock = CLOCK_REALTIME; #endif if (r == NULL) return; /* update the event counter */ r->count += 1; /* fetch the current time */ clock_gettime(rate_clock, &now); /* special case: if this is the first call to rate_sleep(), * calculate when the next tick will be. this is a little bit more * accurate than calculating it in rate_init(). */ if (r->count == 1) { r->start = now; r->next_tick = calc_next_tick(&now, &r->period); } /* adjust the rate and period every 'freq' events. * skip the first window of 'freq' events. * disabled if 'freq' is 0. */ if (r->freq != 0 && (r->count % r->freq) == 0 && r->count > r->freq) adjust_rate(r, &now); /* 'til', amount of time remaining until the next tick */ til = r->next_tick; my_timespec_sub(&now, &til); /* if 'til' is in the past, don't bother sleeping */ if (ts_nanos(&til) > 0) { /* do the sleep */ #if HAVE_CLOCK_NANOSLEEP clock_nanosleep(rate_clock, TIMER_ABSTIME, &r->next_tick, NULL); #else struct timespec rel; rel = r->next_tick; my_timespec_sub(&now, &rel); my_nanosleep(&rel); #endif /* re-fetch the current time */ clock_gettime(rate_clock, &now); } /* calculate the next tick */ r->next_tick = calc_next_tick(&now, &r->period); } mtbl-0.5/libmy/rate.h000066400000000000000000000014711227302760400145420ustar00rootroot00000000000000/* * Copyright (c) 2008, 2009, 2013 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef MY_RATE_H #define MY_RATE_H struct rate; struct rate * rate_init(unsigned rate, unsigned freq); void rate_destroy(struct rate **); void rate_sleep(struct rate *); #endif /* MY_RATE_H */ mtbl-0.5/libmy/spooldir.c000066400000000000000000000101461227302760400154340ustar00rootroot00000000000000/* * Copyright (c) 2012 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #include #include #include #include #include "my_alloc.h" #include "spooldir.h" #include "ubuf.h" #define UBUFSZ 128 struct spooldir { pthread_mutex_t lock; DIR *dir; int dir_fd; ubuf *fname; ubuf *dname_active; ubuf *dname_incoming; }; static bool path_exists(const char *path) { struct stat sb; int ret; ret = stat(path, &sb); if (ret < 0) return (false); return (true); } static bool path_isdir(const char *path) { struct stat sb; int ret; ret = stat(path, &sb); if (ret < 0) return (false); if (S_ISDIR(sb.st_mode)) return (true); return (false); } static bool path_mkdir(const char *path, mode_t mode) { int ret; if (path_isdir(path)) { return (true); } else { ret = mkdir(path, mode); if (ret < 0) { perror("mkdir"); return (false); } } return (true); } struct spooldir * spooldir_init(const char *path) { struct spooldir *s = my_calloc(1, sizeof(*s)); bool res; char *dname; pthread_mutex_init(&s->lock, NULL); dname = realpath(path, NULL); assert(dname != NULL); assert(path_isdir(dname)); s->dname_active = ubuf_init(UBUFSZ); ubuf_add_cstr(s->dname_active, dname); ubuf_add_cstr(s->dname_active, "/active"); res = path_mkdir(ubuf_cstr(s->dname_active), 0755); assert(res); s->dname_incoming = ubuf_init(UBUFSZ); ubuf_add_cstr(s->dname_incoming, dname); ubuf_add_cstr(s->dname_incoming, "/incoming"); res = path_mkdir(ubuf_cstr(s->dname_incoming), 0755); assert(res); free(dname); s->dir = opendir(ubuf_cstr(s->dname_incoming)); assert(s->dir != NULL); s->dir_fd = dirfd(s->dir); assert(s->dir_fd != -1); s->fname = ubuf_init(UBUFSZ); return (s); } void spooldir_destroy(struct spooldir **s) { if (*s != NULL) { pthread_mutex_destroy(&(*s)->lock); closedir((*s)->dir); ubuf_destroy(&(*s)->fname); ubuf_destroy(&(*s)->dname_active); ubuf_destroy(&(*s)->dname_incoming); free(*s); *s = NULL; } } char * spooldir_next(struct spooldir *s) { struct stat sb; struct dirent *de; char *ret = NULL; size_t retsz; char *fname = NULL; ubuf *src_fname; pthread_mutex_lock(&s->lock); while (fname == NULL) { while ((de = readdir(s->dir)) != NULL) { if (de->d_name[0] == '.') continue; if (fstatat(s->dir_fd, de->d_name, &sb, 0) == -1) { fprintf(stderr, "%s: fstatat() failed: %s\n", __func__, strerror(errno)); continue; } if (!S_ISREG(sb.st_mode)) continue; fname = de->d_name; break; } if (fname == NULL) { rewinddir(s->dir); usleep(500*1000); pthread_mutex_unlock(&s->lock); usleep(500*1000); return (NULL); } } assert(fname != NULL); src_fname = ubuf_init(UBUFSZ); ubuf_extend(src_fname, s->dname_incoming); ubuf_add_fmt(src_fname, "/%s", fname); ubuf_cterm(src_fname); ubuf_clip(s->fname, 0); ubuf_extend(s->fname, s->dname_active); ubuf_add_fmt(s->fname, "/%s", fname); ubuf_cterm(s->fname); if (path_exists(ubuf_cstr(s->fname))) { fprintf(stderr, "%s: WARNING: unlinking destination path %s\n", __func__, ubuf_cstr(s->fname)); unlink(ubuf_cstr(s->fname)); } int rename_ret = rename(ubuf_cstr(src_fname), ubuf_cstr(s->fname)); if (rename_ret != 0) { fprintf(stderr, "rename(%s, %s): %s\n", ubuf_cstr(src_fname), ubuf_cstr(s->fname), strerror(errno)); goto out; } ubuf_detach(s->fname, (uint8_t **) &ret, &retsz); out: ubuf_destroy(&src_fname); pthread_mutex_unlock(&s->lock); return (ret); } mtbl-0.5/libmy/spooldir.h000066400000000000000000000003401227302760400154340ustar00rootroot00000000000000#ifndef MY_SPOOLDIR_H #define MY_SPOOLDIR_H struct spooldir; struct spooldir *spooldir_init(const char *path); void spooldir_destroy(struct spooldir **); char *spooldir_next(struct spooldir *); #endif /* MY_SPOOLDIR_H */ mtbl-0.5/libmy/string_replace.h000066400000000000000000000026311227302760400166070ustar00rootroot00000000000000/* * Copyright (c) 2012 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef MY_STRING_REPLACE_H #define MY_STRING_REPLACE_H #include #include "ubuf.h" static inline char * string_replace(const char *str, const char *old, const char *new) { const char *end; char *ret; size_t retsz; if (strstr(str, old) == NULL) { char *s = strdup(str); assert(s != NULL); return (s); } ubuf *u = ubuf_new(); end = str + strlen(str) + 1; while ((ret = strstr(str, old)) != NULL) { size_t offset = (size_t) (ret - str); ubuf_append(u, (uint8_t *) str, offset); ubuf_append(u, (uint8_t *) new, strlen(new)); str += offset; if (str + strlen(old) >= end) break; str += strlen(old); } ubuf_append(u, (uint8_t *) str, strlen(str)); ubuf_cterm(u); ubuf_detach(u, (uint8_t **) &ret, &retsz); ubuf_destroy(&u); return (ret); } #endif /* MY_STRING_REPLACE_H */ mtbl-0.5/libmy/threadnum.c000066400000000000000000000005511227302760400155670ustar00rootroot00000000000000#include #include "threadnum.h" static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER; static int next_threadnum = 0; static __thread int threadnum; void my_threadnum_register(void) { pthread_mutex_lock(&lock); threadnum = next_threadnum; next_threadnum += 1; pthread_mutex_unlock(&lock); } int my_threadnum(void) { return (threadnum); } mtbl-0.5/libmy/threadnum.h000066400000000000000000000002061227302760400155710ustar00rootroot00000000000000#ifndef MY_THREADNUM_H #define MY_THREADNUM_H void my_threadnum_register(void); int my_threadnum(void); #endif /* MY_THREADNUM_H */ mtbl-0.5/libmy/tree.h000066400000000000000000000607661227302760400145620ustar00rootroot00000000000000/* $NetBSD: tree.h,v 1.16 2008/03/21 13:07:15 ad Exp $ */ /* $OpenBSD: tree.h,v 1.7 2002/10/17 21:51:54 art Exp $ */ /* * Copyright 2002 Niels Provos * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _SYS_TREE_H_ #define _SYS_TREE_H_ /* * This file defines data structures for different types of trees: * splay trees and red-black trees. * * A splay tree is a self-organizing data structure. Every operation * on the tree causes a splay to happen. The splay moves the requested * node to the root of the tree and partly rebalances it. * * This has the benefit that request locality causes faster lookups as * the requested nodes move to the top of the tree. On the other hand, * every lookup causes memory writes. * * The Balance Theorem bounds the total access time for m operations * and n inserts on an initially empty tree as O((m + n)lg n). The * amortized cost for a sequence of m accesses to a splay tree is O(lg n); * * A red-black tree is a binary search tree with the node color as an * extra attribute. It fulfills a set of conditions: * - every search path from the root to a leaf consists of the * same number of black nodes, * - each red node (except for the root) has a black parent, * - each leaf node is black. * * Every operation on a red-black tree is bounded as O(lg n). * The maximum height of a red-black tree is 2lg (n+1). */ #define SPLAY_HEAD(name, type) \ struct name { \ struct type *sph_root; /* root of the tree */ \ } #define SPLAY_INITIALIZER(root) \ { NULL } #define SPLAY_INIT(root) do { \ (root)->sph_root = NULL; \ } while (/*CONSTCOND*/ 0) #define SPLAY_ENTRY(type) \ struct { \ struct type *spe_left; /* left element */ \ struct type *spe_right; /* right element */ \ } #define SPLAY_LEFT(elm, field) (elm)->field.spe_left #define SPLAY_RIGHT(elm, field) (elm)->field.spe_right #define SPLAY_ROOT(head) (head)->sph_root #define SPLAY_EMPTY(head) (SPLAY_ROOT(head) == NULL) /* SPLAY_ROTATE_{LEFT,RIGHT} expect that tmp hold SPLAY_{RIGHT,LEFT} */ #define SPLAY_ROTATE_RIGHT(head, tmp, field) do { \ SPLAY_LEFT((head)->sph_root, field) = SPLAY_RIGHT(tmp, field); \ SPLAY_RIGHT(tmp, field) = (head)->sph_root; \ (head)->sph_root = tmp; \ } while (/*CONSTCOND*/ 0) #define SPLAY_ROTATE_LEFT(head, tmp, field) do { \ SPLAY_RIGHT((head)->sph_root, field) = SPLAY_LEFT(tmp, field); \ SPLAY_LEFT(tmp, field) = (head)->sph_root; \ (head)->sph_root = tmp; \ } while (/*CONSTCOND*/ 0) #define SPLAY_LINKLEFT(head, tmp, field) do { \ SPLAY_LEFT(tmp, field) = (head)->sph_root; \ tmp = (head)->sph_root; \ (head)->sph_root = SPLAY_LEFT((head)->sph_root, field); \ } while (/*CONSTCOND*/ 0) #define SPLAY_LINKRIGHT(head, tmp, field) do { \ SPLAY_RIGHT(tmp, field) = (head)->sph_root; \ tmp = (head)->sph_root; \ (head)->sph_root = SPLAY_RIGHT((head)->sph_root, field); \ } while (/*CONSTCOND*/ 0) #define SPLAY_ASSEMBLE(head, node, left, right, field) do { \ SPLAY_RIGHT(left, field) = SPLAY_LEFT((head)->sph_root, field); \ SPLAY_LEFT(right, field) = SPLAY_RIGHT((head)->sph_root, field);\ SPLAY_LEFT((head)->sph_root, field) = SPLAY_RIGHT(node, field); \ SPLAY_RIGHT((head)->sph_root, field) = SPLAY_LEFT(node, field); \ } while (/*CONSTCOND*/ 0) /* Generates prototypes and inline functions */ #define SPLAY_PROTOTYPE(name, type, field, cmp) \ void name##_SPLAY(struct name *, struct type *); \ void name##_SPLAY_MINMAX(struct name *, int); \ struct type *name##_SPLAY_INSERT(struct name *, struct type *); \ struct type *name##_SPLAY_REMOVE(struct name *, struct type *); \ \ /* Finds the node with the same key as elm */ \ static __inline struct type * \ name##_SPLAY_FIND(struct name *head, struct type *elm) \ { \ if (SPLAY_EMPTY(head)) \ return(NULL); \ name##_SPLAY(head, elm); \ if ((cmp)(elm, (head)->sph_root) == 0) \ return (head->sph_root); \ return (NULL); \ } \ \ static __inline struct type * \ name##_SPLAY_NEXT(struct name *head, struct type *elm) \ { \ name##_SPLAY(head, elm); \ if (SPLAY_RIGHT(elm, field) != NULL) { \ elm = SPLAY_RIGHT(elm, field); \ while (SPLAY_LEFT(elm, field) != NULL) { \ elm = SPLAY_LEFT(elm, field); \ } \ } else \ elm = NULL; \ return (elm); \ } \ \ static __inline struct type * \ name##_SPLAY_MIN_MAX(struct name *head, int val) \ { \ name##_SPLAY_MINMAX(head, val); \ return (SPLAY_ROOT(head)); \ } /* Main splay operation. * Moves node close to the key of elm to top */ #define SPLAY_GENERATE(name, type, field, cmp) \ struct type * \ name##_SPLAY_INSERT(struct name *head, struct type *elm) \ { \ if (SPLAY_EMPTY(head)) { \ SPLAY_LEFT(elm, field) = SPLAY_RIGHT(elm, field) = NULL; \ } else { \ int __comp; \ name##_SPLAY(head, elm); \ __comp = (cmp)(elm, (head)->sph_root); \ if(__comp < 0) { \ SPLAY_LEFT(elm, field) = SPLAY_LEFT((head)->sph_root, field);\ SPLAY_RIGHT(elm, field) = (head)->sph_root; \ SPLAY_LEFT((head)->sph_root, field) = NULL; \ } else if (__comp > 0) { \ SPLAY_RIGHT(elm, field) = SPLAY_RIGHT((head)->sph_root, field);\ SPLAY_LEFT(elm, field) = (head)->sph_root; \ SPLAY_RIGHT((head)->sph_root, field) = NULL; \ } else \ return ((head)->sph_root); \ } \ (head)->sph_root = (elm); \ return (NULL); \ } \ \ struct type * \ name##_SPLAY_REMOVE(struct name *head, struct type *elm) \ { \ struct type *__tmp; \ if (SPLAY_EMPTY(head)) \ return (NULL); \ name##_SPLAY(head, elm); \ if ((cmp)(elm, (head)->sph_root) == 0) { \ if (SPLAY_LEFT((head)->sph_root, field) == NULL) { \ (head)->sph_root = SPLAY_RIGHT((head)->sph_root, field);\ } else { \ __tmp = SPLAY_RIGHT((head)->sph_root, field); \ (head)->sph_root = SPLAY_LEFT((head)->sph_root, field);\ name##_SPLAY(head, elm); \ SPLAY_RIGHT((head)->sph_root, field) = __tmp; \ } \ return (elm); \ } \ return (NULL); \ } \ \ void \ name##_SPLAY(struct name *head, struct type *elm) \ { \ struct type __node, *__left, *__right, *__tmp; \ int __comp; \ \ SPLAY_LEFT(&__node, field) = SPLAY_RIGHT(&__node, field) = NULL;\ __left = __right = &__node; \ \ while ((__comp = (cmp)(elm, (head)->sph_root)) != 0) { \ if (__comp < 0) { \ __tmp = SPLAY_LEFT((head)->sph_root, field); \ if (__tmp == NULL) \ break; \ if ((cmp)(elm, __tmp) < 0){ \ SPLAY_ROTATE_RIGHT(head, __tmp, field); \ if (SPLAY_LEFT((head)->sph_root, field) == NULL)\ break; \ } \ SPLAY_LINKLEFT(head, __right, field); \ } else if (__comp > 0) { \ __tmp = SPLAY_RIGHT((head)->sph_root, field); \ if (__tmp == NULL) \ break; \ if ((cmp)(elm, __tmp) > 0){ \ SPLAY_ROTATE_LEFT(head, __tmp, field); \ if (SPLAY_RIGHT((head)->sph_root, field) == NULL)\ break; \ } \ SPLAY_LINKRIGHT(head, __left, field); \ } \ } \ SPLAY_ASSEMBLE(head, &__node, __left, __right, field); \ } \ \ /* Splay with either the minimum or the maximum element \ * Used to find minimum or maximum element in tree. \ */ \ void name##_SPLAY_MINMAX(struct name *head, int __comp) \ { \ struct type __node, *__left, *__right, *__tmp; \ \ SPLAY_LEFT(&__node, field) = SPLAY_RIGHT(&__node, field) = NULL;\ __left = __right = &__node; \ \ while (1) { \ if (__comp < 0) { \ __tmp = SPLAY_LEFT((head)->sph_root, field); \ if (__tmp == NULL) \ break; \ if (__comp < 0){ \ SPLAY_ROTATE_RIGHT(head, __tmp, field); \ if (SPLAY_LEFT((head)->sph_root, field) == NULL)\ break; \ } \ SPLAY_LINKLEFT(head, __right, field); \ } else if (__comp > 0) { \ __tmp = SPLAY_RIGHT((head)->sph_root, field); \ if (__tmp == NULL) \ break; \ if (__comp > 0) { \ SPLAY_ROTATE_LEFT(head, __tmp, field); \ if (SPLAY_RIGHT((head)->sph_root, field) == NULL)\ break; \ } \ SPLAY_LINKRIGHT(head, __left, field); \ } \ } \ SPLAY_ASSEMBLE(head, &__node, __left, __right, field); \ } #define SPLAY_NEGINF -1 #define SPLAY_INF 1 #define SPLAY_INSERT(name, x, y) name##_SPLAY_INSERT(x, y) #define SPLAY_REMOVE(name, x, y) name##_SPLAY_REMOVE(x, y) #define SPLAY_FIND(name, x, y) name##_SPLAY_FIND(x, y) #define SPLAY_NEXT(name, x, y) name##_SPLAY_NEXT(x, y) #define SPLAY_MIN(name, x) (SPLAY_EMPTY(x) ? NULL \ : name##_SPLAY_MIN_MAX(x, SPLAY_NEGINF)) #define SPLAY_MAX(name, x) (SPLAY_EMPTY(x) ? NULL \ : name##_SPLAY_MIN_MAX(x, SPLAY_INF)) #define SPLAY_FOREACH(x, name, head) \ for ((x) = SPLAY_MIN(name, head); \ (x) != NULL; \ (x) = SPLAY_NEXT(name, head, x)) /* Macros that define a red-black tree */ #define RB_HEAD(name, type) \ struct name { \ struct type *rbh_root; /* root of the tree */ \ } #define RB_INITIALIZER(root) \ { NULL } #define RB_INIT(root) do { \ (root)->rbh_root = NULL; \ } while (/*CONSTCOND*/ 0) #define RB_BLACK 0 #define RB_RED 1 #define RB_ENTRY(type) \ struct { \ struct type *rbe_left; /* left element */ \ struct type *rbe_right; /* right element */ \ struct type *rbe_parent; /* parent element */ \ int rbe_color; /* node color */ \ } #define RB_LEFT(elm, field) (elm)->field.rbe_left #define RB_RIGHT(elm, field) (elm)->field.rbe_right #define RB_PARENT(elm, field) (elm)->field.rbe_parent #define RB_COLOR(elm, field) (elm)->field.rbe_color #define RB_ROOT(head) (head)->rbh_root #define RB_EMPTY(head) (RB_ROOT(head) == NULL) #define RB_SET(elm, parent, field) do { \ RB_PARENT(elm, field) = parent; \ RB_LEFT(elm, field) = RB_RIGHT(elm, field) = NULL; \ RB_COLOR(elm, field) = RB_RED; \ } while (/*CONSTCOND*/ 0) #define RB_SET_BLACKRED(black, red, field) do { \ RB_COLOR(black, field) = RB_BLACK; \ RB_COLOR(red, field) = RB_RED; \ } while (/*CONSTCOND*/ 0) #ifndef RB_AUGMENT #define RB_AUGMENT(x) (void)(x) #endif #define RB_ROTATE_LEFT(head, elm, tmp, field) do { \ (tmp) = RB_RIGHT(elm, field); \ if ((RB_RIGHT(elm, field) = RB_LEFT(tmp, field)) != NULL) { \ RB_PARENT(RB_LEFT(tmp, field), field) = (elm); \ } \ RB_AUGMENT(elm); \ if ((RB_PARENT(tmp, field) = RB_PARENT(elm, field)) != NULL) { \ if ((elm) == RB_LEFT(RB_PARENT(elm, field), field)) \ RB_LEFT(RB_PARENT(elm, field), field) = (tmp); \ else \ RB_RIGHT(RB_PARENT(elm, field), field) = (tmp); \ } else \ (head)->rbh_root = (tmp); \ RB_LEFT(tmp, field) = (elm); \ RB_PARENT(elm, field) = (tmp); \ RB_AUGMENT(tmp); \ if ((RB_PARENT(tmp, field))) \ RB_AUGMENT(RB_PARENT(tmp, field)); \ } while (/*CONSTCOND*/ 0) #define RB_ROTATE_RIGHT(head, elm, tmp, field) do { \ (tmp) = RB_LEFT(elm, field); \ if ((RB_LEFT(elm, field) = RB_RIGHT(tmp, field)) != NULL) { \ RB_PARENT(RB_RIGHT(tmp, field), field) = (elm); \ } \ RB_AUGMENT(elm); \ if ((RB_PARENT(tmp, field) = RB_PARENT(elm, field)) != NULL) { \ if ((elm) == RB_LEFT(RB_PARENT(elm, field), field)) \ RB_LEFT(RB_PARENT(elm, field), field) = (tmp); \ else \ RB_RIGHT(RB_PARENT(elm, field), field) = (tmp); \ } else \ (head)->rbh_root = (tmp); \ RB_RIGHT(tmp, field) = (elm); \ RB_PARENT(elm, field) = (tmp); \ RB_AUGMENT(tmp); \ if ((RB_PARENT(tmp, field))) \ RB_AUGMENT(RB_PARENT(tmp, field)); \ } while (/*CONSTCOND*/ 0) /* Generates prototypes and inline functions */ #define RB_PROTOTYPE(name, type, field, cmp) \ RB_PROTOTYPE_INTERNAL(name, type, field, cmp,) #define RB_PROTOTYPE_STATIC(name, type, field, cmp) \ RB_PROTOTYPE_INTERNAL(name, type, field, cmp, __unused static) #define RB_PROTOTYPE_INTERNAL(name, type, field, cmp, attr) \ attr void name##_RB_INSERT_COLOR(struct name *, struct type *); \ attr void name##_RB_REMOVE_COLOR(struct name *, struct type *, struct type *);\ attr struct type *name##_RB_REMOVE(struct name *, struct type *); \ attr struct type *name##_RB_INSERT(struct name *, struct type *); \ attr struct type *name##_RB_FIND(struct name *, struct type *); \ attr struct type *name##_RB_NFIND(struct name *, struct type *); \ attr struct type *name##_RB_NEXT(struct type *); \ attr struct type *name##_RB_PREV(struct type *); \ attr struct type *name##_RB_MINMAX(struct name *, int); \ \ /* Main rb operation. * Moves node close to the key of elm to top */ #define RB_GENERATE(name, type, field, cmp) \ RB_GENERATE_INTERNAL(name, type, field, cmp,) #define RB_GENERATE_STATIC(name, type, field, cmp) \ RB_GENERATE_INTERNAL(name, type, field, cmp, __unused static) #define RB_GENERATE_INTERNAL(name, type, field, cmp, attr) \ attr void \ name##_RB_INSERT_COLOR(struct name *head, struct type *elm) \ { \ struct type *parent, *gparent, *tmp; \ while ((parent = RB_PARENT(elm, field)) != NULL && \ RB_COLOR(parent, field) == RB_RED) { \ gparent = RB_PARENT(parent, field); \ if (parent == RB_LEFT(gparent, field)) { \ tmp = RB_RIGHT(gparent, field); \ if (tmp && RB_COLOR(tmp, field) == RB_RED) { \ RB_COLOR(tmp, field) = RB_BLACK; \ RB_SET_BLACKRED(parent, gparent, field);\ elm = gparent; \ continue; \ } \ if (RB_RIGHT(parent, field) == elm) { \ RB_ROTATE_LEFT(head, parent, tmp, field);\ tmp = parent; \ parent = elm; \ elm = tmp; \ } \ RB_SET_BLACKRED(parent, gparent, field); \ RB_ROTATE_RIGHT(head, gparent, tmp, field); \ } else { \ tmp = RB_LEFT(gparent, field); \ if (tmp && RB_COLOR(tmp, field) == RB_RED) { \ RB_COLOR(tmp, field) = RB_BLACK; \ RB_SET_BLACKRED(parent, gparent, field);\ elm = gparent; \ continue; \ } \ if (RB_LEFT(parent, field) == elm) { \ RB_ROTATE_RIGHT(head, parent, tmp, field);\ tmp = parent; \ parent = elm; \ elm = tmp; \ } \ RB_SET_BLACKRED(parent, gparent, field); \ RB_ROTATE_LEFT(head, gparent, tmp, field); \ } \ } \ RB_COLOR(head->rbh_root, field) = RB_BLACK; \ } \ \ attr void \ name##_RB_REMOVE_COLOR(struct name *head, struct type *parent, struct type *elm) \ { \ struct type *tmp; \ while ((elm == NULL || RB_COLOR(elm, field) == RB_BLACK) && \ elm != RB_ROOT(head)) { \ if (RB_LEFT(parent, field) == elm) { \ tmp = RB_RIGHT(parent, field); \ if (RB_COLOR(tmp, field) == RB_RED) { \ RB_SET_BLACKRED(tmp, parent, field); \ RB_ROTATE_LEFT(head, parent, tmp, field);\ tmp = RB_RIGHT(parent, field); \ } \ if ((RB_LEFT(tmp, field) == NULL || \ RB_COLOR(RB_LEFT(tmp, field), field) == RB_BLACK) &&\ (RB_RIGHT(tmp, field) == NULL || \ RB_COLOR(RB_RIGHT(tmp, field), field) == RB_BLACK)) {\ RB_COLOR(tmp, field) = RB_RED; \ elm = parent; \ parent = RB_PARENT(elm, field); \ } else { \ if (RB_RIGHT(tmp, field) == NULL || \ RB_COLOR(RB_RIGHT(tmp, field), field) == RB_BLACK) {\ struct type *oleft; \ if ((oleft = RB_LEFT(tmp, field)) \ != NULL) \ RB_COLOR(oleft, field) = RB_BLACK;\ RB_COLOR(tmp, field) = RB_RED; \ RB_ROTATE_RIGHT(head, tmp, oleft, field);\ tmp = RB_RIGHT(parent, field); \ } \ RB_COLOR(tmp, field) = RB_COLOR(parent, field);\ RB_COLOR(parent, field) = RB_BLACK; \ if (RB_RIGHT(tmp, field)) \ RB_COLOR(RB_RIGHT(tmp, field), field) = RB_BLACK;\ RB_ROTATE_LEFT(head, parent, tmp, field);\ elm = RB_ROOT(head); \ break; \ } \ } else { \ tmp = RB_LEFT(parent, field); \ if (RB_COLOR(tmp, field) == RB_RED) { \ RB_SET_BLACKRED(tmp, parent, field); \ RB_ROTATE_RIGHT(head, parent, tmp, field);\ tmp = RB_LEFT(parent, field); \ } \ if ((RB_LEFT(tmp, field) == NULL || \ RB_COLOR(RB_LEFT(tmp, field), field) == RB_BLACK) &&\ (RB_RIGHT(tmp, field) == NULL || \ RB_COLOR(RB_RIGHT(tmp, field), field) == RB_BLACK)) {\ RB_COLOR(tmp, field) = RB_RED; \ elm = parent; \ parent = RB_PARENT(elm, field); \ } else { \ if (RB_LEFT(tmp, field) == NULL || \ RB_COLOR(RB_LEFT(tmp, field), field) == RB_BLACK) {\ struct type *oright; \ if ((oright = RB_RIGHT(tmp, field)) \ != NULL) \ RB_COLOR(oright, field) = RB_BLACK;\ RB_COLOR(tmp, field) = RB_RED; \ RB_ROTATE_LEFT(head, tmp, oright, field);\ tmp = RB_LEFT(parent, field); \ } \ RB_COLOR(tmp, field) = RB_COLOR(parent, field);\ RB_COLOR(parent, field) = RB_BLACK; \ if (RB_LEFT(tmp, field)) \ RB_COLOR(RB_LEFT(tmp, field), field) = RB_BLACK;\ RB_ROTATE_RIGHT(head, parent, tmp, field);\ elm = RB_ROOT(head); \ break; \ } \ } \ } \ if (elm) \ RB_COLOR(elm, field) = RB_BLACK; \ } \ \ attr struct type * \ name##_RB_REMOVE(struct name *head, struct type *elm) \ { \ struct type *child, *parent, *old = elm; \ int color; \ if (RB_LEFT(elm, field) == NULL) \ child = RB_RIGHT(elm, field); \ else if (RB_RIGHT(elm, field) == NULL) \ child = RB_LEFT(elm, field); \ else { \ struct type *left; \ elm = RB_RIGHT(elm, field); \ while ((left = RB_LEFT(elm, field)) != NULL) \ elm = left; \ child = RB_RIGHT(elm, field); \ parent = RB_PARENT(elm, field); \ color = RB_COLOR(elm, field); \ if (child) \ RB_PARENT(child, field) = parent; \ if (parent) { \ if (RB_LEFT(parent, field) == elm) \ RB_LEFT(parent, field) = child; \ else \ RB_RIGHT(parent, field) = child; \ RB_AUGMENT(parent); \ } else \ RB_ROOT(head) = child; \ if (RB_PARENT(elm, field) == old) \ parent = elm; \ (elm)->field = (old)->field; \ if (RB_PARENT(old, field)) { \ if (RB_LEFT(RB_PARENT(old, field), field) == old)\ RB_LEFT(RB_PARENT(old, field), field) = elm;\ else \ RB_RIGHT(RB_PARENT(old, field), field) = elm;\ RB_AUGMENT(RB_PARENT(old, field)); \ } else \ RB_ROOT(head) = elm; \ RB_PARENT(RB_LEFT(old, field), field) = elm; \ if (RB_RIGHT(old, field)) \ RB_PARENT(RB_RIGHT(old, field), field) = elm; \ if (parent) { \ left = parent; \ do { \ RB_AUGMENT(left); \ } while ((left = RB_PARENT(left, field)) != NULL); \ } \ goto color; \ } \ parent = RB_PARENT(elm, field); \ color = RB_COLOR(elm, field); \ if (child) \ RB_PARENT(child, field) = parent; \ if (parent) { \ if (RB_LEFT(parent, field) == elm) \ RB_LEFT(parent, field) = child; \ else \ RB_RIGHT(parent, field) = child; \ RB_AUGMENT(parent); \ } else \ RB_ROOT(head) = child; \ color: \ if (color == RB_BLACK) \ name##_RB_REMOVE_COLOR(head, parent, child); \ return (old); \ } \ \ /* Inserts a node into the RB tree */ \ attr struct type * \ name##_RB_INSERT(struct name *head, struct type *elm) \ { \ struct type *tmp; \ struct type *parent = NULL; \ int comp = 0; \ tmp = RB_ROOT(head); \ while (tmp) { \ parent = tmp; \ comp = (cmp)(elm, parent); \ if (comp < 0) \ tmp = RB_LEFT(tmp, field); \ else if (comp > 0) \ tmp = RB_RIGHT(tmp, field); \ else \ return (tmp); \ } \ RB_SET(elm, parent, field); \ if (parent != NULL) { \ if (comp < 0) \ RB_LEFT(parent, field) = elm; \ else \ RB_RIGHT(parent, field) = elm; \ RB_AUGMENT(parent); \ } else \ RB_ROOT(head) = elm; \ name##_RB_INSERT_COLOR(head, elm); \ return (NULL); \ } \ \ /* Finds the node with the same key as elm */ \ attr struct type * \ name##_RB_FIND(struct name *head, struct type *elm) \ { \ struct type *tmp = RB_ROOT(head); \ int comp; \ while (tmp) { \ comp = cmp(elm, tmp); \ if (comp < 0) \ tmp = RB_LEFT(tmp, field); \ else if (comp > 0) \ tmp = RB_RIGHT(tmp, field); \ else \ return (tmp); \ } \ return (NULL); \ } \ \ /* Finds the first node greater than or equal to the search key */ \ attr struct type * \ name##_RB_NFIND(struct name *head, struct type *elm) \ { \ struct type *tmp = RB_ROOT(head); \ struct type *res = NULL; \ int comp; \ while (tmp) { \ comp = cmp(elm, tmp); \ if (comp < 0) { \ res = tmp; \ tmp = RB_LEFT(tmp, field); \ } \ else if (comp > 0) \ tmp = RB_RIGHT(tmp, field); \ else \ return (tmp); \ } \ return (res); \ } \ \ /* ARGSUSED */ \ attr struct type * \ name##_RB_NEXT(struct type *elm) \ { \ if (RB_RIGHT(elm, field)) { \ elm = RB_RIGHT(elm, field); \ while (RB_LEFT(elm, field)) \ elm = RB_LEFT(elm, field); \ } else { \ if (RB_PARENT(elm, field) && \ (elm == RB_LEFT(RB_PARENT(elm, field), field))) \ elm = RB_PARENT(elm, field); \ else { \ while (RB_PARENT(elm, field) && \ (elm == RB_RIGHT(RB_PARENT(elm, field), field)))\ elm = RB_PARENT(elm, field); \ elm = RB_PARENT(elm, field); \ } \ } \ return (elm); \ } \ \ /* ARGSUSED */ \ attr struct type * \ name##_RB_PREV(struct type *elm) \ { \ if (RB_LEFT(elm, field)) { \ elm = RB_LEFT(elm, field); \ while (RB_RIGHT(elm, field)) \ elm = RB_RIGHT(elm, field); \ } else { \ if (RB_PARENT(elm, field) && \ (elm == RB_RIGHT(RB_PARENT(elm, field), field))) \ elm = RB_PARENT(elm, field); \ else { \ while (RB_PARENT(elm, field) && \ (elm == RB_LEFT(RB_PARENT(elm, field), field)))\ elm = RB_PARENT(elm, field); \ elm = RB_PARENT(elm, field); \ } \ } \ return (elm); \ } \ \ attr struct type * \ name##_RB_MINMAX(struct name *head, int val) \ { \ struct type *tmp = RB_ROOT(head); \ struct type *parent = NULL; \ while (tmp) { \ parent = tmp; \ if (val < 0) \ tmp = RB_LEFT(tmp, field); \ else \ tmp = RB_RIGHT(tmp, field); \ } \ return (parent); \ } #define RB_NEGINF -1 #define RB_INF 1 #define RB_INSERT(name, x, y) name##_RB_INSERT(x, y) #define RB_REMOVE(name, x, y) name##_RB_REMOVE(x, y) #define RB_FIND(name, x, y) name##_RB_FIND(x, y) #define RB_NFIND(name, x, y) name##_RB_NFIND(x, y) #define RB_NEXT(name, x, y) name##_RB_NEXT(y) #define RB_PREV(name, x, y) name##_RB_PREV(y) #define RB_MIN(name, x) name##_RB_MINMAX(x, RB_NEGINF) #define RB_MAX(name, x) name##_RB_MINMAX(x, RB_INF) #define RB_FOREACH(x, name, head) \ for ((x) = RB_MIN(name, head); \ (x) != NULL; \ (x) = name##_RB_NEXT(x)) #define RB_FOREACH_REVERSE(x, name, head) \ for ((x) = RB_MAX(name, head); \ (x) != NULL; \ (x) = name##_RB_PREV(x)) #endif /* _SYS_TREE_H_ */ mtbl-0.5/libmy/ubuf-pb.h000066400000000000000000000021271227302760400151460ustar00rootroot00000000000000/* * Copyright (c) 2013 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef MY_UBUF_PB_H #define MY_UBUF_PB_H #include #include "ubuf.h" struct ubuf_protobuf_c_buffer { ProtobufCBuffer base; ubuf *u; }; static inline void ubuf_protobuf_c_buffer_append(ProtobufCBuffer *buffer, size_t len, const uint8_t *data) { ubuf *u = ((struct ubuf_protobuf_c_buffer *) buffer)->u; ubuf_append(u, data, len); } #define UBUF_PROTOBUF_C_BUFFER_INIT(u) { { ubuf_protobuf_c_buffer_append }, u } #endif /* MY_UBUF_PB_H */ mtbl-0.5/libmy/ubuf.h000066400000000000000000000043601227302760400145500ustar00rootroot00000000000000/* * Copyright (c) 2012 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef MY_UBUF_H #define MY_UBUF_H #include #include #include #include #include #include #include "vector.h" VECTOR_GENERATE(ubuf, uint8_t); static inline ubuf * ubuf_new(void) { return (ubuf_init(64)); } static inline ubuf * ubuf_dup_cstr(const char *s) { size_t len = strlen(s); ubuf *u = ubuf_init(len + 1); ubuf_append(u, (const uint8_t *) s, len); return (u); } static inline void ubuf_add_cstr(ubuf *u, const char *s) { if (ubuf_size(u) > 0 && ubuf_value(u, ubuf_size(u) - 1) == '\x00') ubuf_clip(u, ubuf_size(u) - 1); ubuf_append(u, (const uint8_t *) s, strlen(s)); } static inline void ubuf_cterm(ubuf *u) { if (ubuf_size(u) == 0 || (ubuf_size(u) > 0 && ubuf_value(u, ubuf_size(u) - 1) != '\x00')) { ubuf_append(u, (const uint8_t *) "\x00", 1); } } static inline char * ubuf_cstr(ubuf *u) { ubuf_cterm(u); return ((char *) ubuf_data(u)); } static inline void ubuf_add_fmt(ubuf *u, const char *fmt, ...) { va_list args, args_copy; int status, needed; if (ubuf_size(u) > 0 && ubuf_value(u, ubuf_size(u) - 1) == '\x00') ubuf_clip(u, ubuf_size(u) - 1); va_start(args, fmt); va_copy(args_copy, args); needed = vsnprintf(NULL, 0, fmt, args_copy); assert(needed >= 0); va_end(args_copy); ubuf_reserve(u, ubuf_size(u) + needed + 1); status = vsnprintf((char *) ubuf_ptr(u), needed + 1, fmt, args); assert(status >= 0); ubuf_advance(u, needed); va_end(args); } static inline void ubuf_rstrip(ubuf *u, char s) { if (ubuf_size(u) > 0 && ubuf_value(u, ubuf_size(u) - 1) == ((uint8_t) s)) { ubuf_clip(u, ubuf_size(u) - 1); } } #endif /* MY_UBUF_H */ mtbl-0.5/libmy/varint.c000066400000000000000000000131501227302760400151020ustar00rootroot00000000000000/* * Copyright (c) 2012, 2013 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // Copyright (c) 2011 The LevelDB Authors. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /* * Copyright (c) 2008, Dave Benson. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are * met: * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * * Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following disclaimer * in the documentation and/or other materials provided with the * distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "varint.h" unsigned varint_length(uint64_t v) { unsigned len = 1; while (v >= 128) { v >>= 7; len++; } return (len); } unsigned varint_length_packed(const uint8_t *data, size_t len_data) { unsigned i = 0; size_t len = len_data; while (len--) { if ((data[i] & 0x80) == 0) break; i++; } if (i == len_data) return (0); return (i + 1); } size_t varint_encode32(uint8_t *src_ptr, uint32_t v) { static const unsigned B = 128; uint8_t *ptr = src_ptr; if (v < (1 << 7)) { *(ptr++) = v; } else if (v < (1 << 14)) { *(ptr++) = v | B; *(ptr++) = v >> 7; } else if (v < (1 << 21)) { *(ptr++) = v | B; *(ptr++) = (v >> 7) | B; *(ptr++) = v >> 14; } else if (v < (1 << 28)) { *(ptr++) = v | B; *(ptr++) = (v >> 7) | B; *(ptr++) = (v >> 14) | B; *(ptr++) = v >> 21; } else { *(ptr++) = v | B; *(ptr++) = (v >> 7) | B; *(ptr++) = (v >> 14) | B; *(ptr++) = (v >> 21) | B; *(ptr++) = v >> 28; } return ((size_t) (ptr - src_ptr)); } size_t varint_encode64(uint8_t *src_ptr, uint64_t v) { static const unsigned B = 128; uint8_t *ptr = src_ptr; while (v >= B) { *(ptr++) = (v & (B - 1)) | B; v >>= 7; } *(ptr++) = (uint8_t) v; return ((size_t) (ptr - src_ptr)); } size_t varint_decode32(const uint8_t *data, uint32_t *value) { unsigned len = varint_length_packed(data, 5); uint32_t val = data[0] & 0x7f; if (len > 1) { val |= ((data[1] & 0x7f) << 7); if (len > 2) { val |= ((data[2] & 0x7f) << 14); if (len > 3) { val |= ((data[3] & 0x7f) << 21); if (len > 4) val |= (data[4] << 28); } } } *value = val; return ((size_t) len); } size_t varint_decode64(const uint8_t *data, uint64_t *value) { unsigned shift, i; unsigned len = varint_length_packed(data, 10); uint64_t val; if (len < 5) { size_t tmp_len; uint32_t tmp; tmp_len = varint_decode32(data, &tmp); *value = tmp; return (tmp_len); } val = ((data[0] & 0x7f)) | ((data[1] & 0x7f) << 7) | ((data[2] & 0x7f) << 14) | ((data[3] & 0x7f) << 21); shift = 28; for (i = 4; i < len; i++) { val |= (((uint64_t)(data[i] & 0x7f)) << shift); shift += 7; } *value = val; return ((size_t) len); } mtbl-0.5/libmy/varint.h000066400000000000000000000006711227302760400151130ustar00rootroot00000000000000#ifndef MY_VARINT_H #define MY_VARINT_H #include #include unsigned varint_length(uint64_t v); unsigned varint_length_packed(const uint8_t *buf, size_t len_buf); size_t varint_encode32(uint8_t *ptr, uint32_t value); size_t varint_encode64(uint8_t *ptr, uint64_t value); size_t varint_decode32(const uint8_t *ptr, uint32_t *value); size_t varint_decode64(const uint8_t *ptr, uint64_t *value); #endif /* MY_VARINT_H */ mtbl-0.5/libmy/vector.h000066400000000000000000000106251227302760400151120ustar00rootroot00000000000000/* * Copyright (c) 2012, 2013 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include "my_alloc.h" #define VECTOR_GENERATE(name, type) \ typedef struct name##__vector { \ type * _v; \ type * _p; \ size_t _n, _n_alloced, _hint; \ } name; \ static inline name * \ name##_init(unsigned hint) \ { \ name *vec; \ vec = my_calloc(1, sizeof(name)); \ if (hint == 0) hint = 1; \ vec->_hint = vec->_n_alloced = hint; \ vec->_v = my_malloc(vec->_n_alloced * sizeof(type)); \ vec->_p = &(vec->_v[0]); \ return (vec); \ } \ static inline void \ name##_reinit(unsigned hint, name *vec) \ { \ if (hint == 0) hint = 1; \ vec->_hint = vec->_n_alloced = hint; \ vec->_n = 0; \ vec->_v = my_malloc(vec->_n_alloced * sizeof(type)); \ vec->_p = &(vec->_v[0]); \ } \ static inline void \ name##_detach(name *vec, type **out, size_t *outsz) \ { \ *(out) = (vec)->_v; \ *(outsz) = (vec)->_n; \ (vec)->_n = 0; \ (vec)->_n_alloced = (vec)->_hint; \ (vec)->_v = my_malloc((vec)->_n_alloced * sizeof(type)); \ (vec)->_p = &(vec->_v[0]); \ } \ static inline void \ name##_destroy(name **vec) \ { \ if (*vec) { \ free((*vec)->_v); \ free((*vec)); \ *vec = NULL; \ } \ } \ static inline void \ name##_reserve(name *vec, size_t n_elems) \ { \ while ((n_elems) > ((vec)->_n_alloced - (vec)->_n)) { \ (vec)->_n_alloced *= 2; \ (vec)->_v = my_realloc((vec)->_v, (vec)->_n_alloced \ * sizeof(type)); \ (vec)->_p = &((vec)->_v[(vec)->_n]); \ } \ } \ static inline void \ name##_add(name *vec, type elem) \ { \ while ((vec)->_n + 1 > (vec)->_n_alloced) { \ (vec)->_n_alloced *= 2; \ (vec)->_v = my_realloc((vec)->_v, (vec)->_n_alloced \ * sizeof(type)); \ (vec)->_p = &((vec)->_v[(vec)->_n]); \ } \ (vec)->_v[(vec)->_n] = elem; \ (vec)->_n += 1; \ (vec)->_p = &((vec)->_v[(vec)->_n]); \ } \ static inline void \ name##_append(name *vec, type const *elems, size_t n_elems) \ { \ name##_reserve(vec, n_elems); \ memcpy((vec)->_v + (vec)->_n, elems, (n_elems) * sizeof(type)); \ (vec)->_n += (n_elems); \ (vec)->_p = &((vec)->_v[(vec)->_n]); \ } \ static inline void \ name##_extend(name *vec0, name *vec1) \ { \ name##_append(vec0, (vec1)->_v, (vec1)->_n); \ } \ static inline void \ name##_reset(name *vec) \ { \ (vec)->_n = 0; \ if ((vec)->_n_alloced > (vec)->_hint) { \ (vec)->_n_alloced = (vec)->_hint; \ (vec)->_v = my_realloc((vec)->_v, (vec)->_n_alloced \ * sizeof(type)); \ } \ (vec)->_p = &(vec->_v[0]); \ } \ static inline void \ name##_clip(name *vec, size_t n_elems) \ { \ if (n_elems < (vec)->_n) { \ (vec)->_n = n_elems; \ (vec)->_p = &((vec)->_v[(vec)->_n]); \ } \ } \ static inline size_t \ name##_bytes(name *vec) \ { \ return ((vec)->_n * sizeof(type)); \ } \ static inline size_t \ name##_size(name *vec) \ { \ return ((vec)->_n); \ } \ static inline type \ name##_value(name *vec, size_t i) \ { \ assert(i < (vec)->_n); \ return ((vec)->_v[i]); \ } \ static inline type * \ name##_ptr(name *vec) \ { \ return ((vec)->_p); \ } \ static inline type * \ name##_data(name *vec) \ { \ return ((vec)->_v); \ } \ static inline void \ name##_advance(name *vec, size_t x) \ { \ assert(x <= ((vec)->_n_alloced - (vec)->_n)); \ (vec)->_n += x; \ (vec)->_p = &((vec)->_v[(vec)->_n]); \ } mtbl-0.5/libmy/zonefile.c000066400000000000000000000065721227302760400154240ustar00rootroot00000000000000#include #include #include #include #include #include #include "my_alloc.h" #include "ubuf.h" #include "zonefile.h" struct zonefile { FILE *fp; bool is_pipe; bool valid; ldns_rdf *domain; ldns_rdf *origin; ldns_rdf *prev; ldns_rr *rr_soa; uint32_t ttl; size_t count; }; static ldns_status read_soa(struct zonefile *z) { ldns_rr *rr; ldns_status status; for (;;) { status = ldns_rr_new_frm_fp_l(&rr, z->fp, &z->ttl, &z->origin, &z->prev, NULL); switch (status) { case LDNS_STATUS_OK: goto out; case LDNS_STATUS_SYNTAX_EMPTY: case LDNS_STATUS_SYNTAX_TTL: case LDNS_STATUS_SYNTAX_ORIGIN: status = LDNS_STATUS_OK; break; default: goto out; } } out: if (status != LDNS_STATUS_OK) { z->valid = false; return (LDNS_STATUS_ERR); } if (ldns_rr_get_type(rr) != LDNS_RR_TYPE_SOA) { ldns_rr_free(rr); z->valid = false; return (LDNS_STATUS_ERR); } z->count = 1; z->domain = ldns_rdf_clone(ldns_rr_owner(rr)); z->origin = ldns_rdf_clone(ldns_rr_owner(rr)); z->rr_soa = rr; return (LDNS_STATUS_OK); } struct zonefile * zonefile_init_fname(const char *fname) { struct zonefile *z = my_calloc(1, sizeof(struct zonefile)); size_t len_fname = strlen(fname); if (len_fname >= 3 && fname[len_fname - 3] == '.' && fname[len_fname - 2] == 'g' && fname[len_fname - 1] == 'z') { ubuf *u = ubuf_new(); ubuf_add_cstr(u, "zcat "); ubuf_add_cstr(u, fname); z->fp = popen(ubuf_cstr(u), "r"); z->is_pipe = true; ubuf_destroy(&u); } else { z->fp = fopen(fname, "r"); } if (z->fp == NULL) return (NULL); z->valid = true; if (read_soa(z) != LDNS_STATUS_OK) zonefile_destroy(&z); return (z); } void zonefile_destroy(struct zonefile **z) { if (*z) { if ((*z)->fp) { if ((*z)->is_pipe) pclose((*z)->fp); else fclose((*z)->fp); } if ((*z)->origin) ldns_rdf_deep_free((*z)->origin); if ((*z)->prev) ldns_rdf_deep_free((*z)->prev); if ((*z)->domain) ldns_rdf_deep_free((*z)->domain); if ((*z)->rr_soa) ldns_rr_free((*z)->rr_soa); free(*z); *z = NULL; } } const ldns_rdf * zonefile_get_domain(struct zonefile *z) { return (z->domain); } size_t zonefile_get_count(struct zonefile *z) { return (z->count); } uint32_t zonefile_get_serial(struct zonefile *z) { ldns_rdf *rdf = ldns_rr_rdf(z->rr_soa, 2); assert(rdf != NULL); return (ldns_rdf2native_int32(rdf)); } ldns_status zonefile_read(struct zonefile *z, ldns_rr **out) { ldns_rr *rr; ldns_status status = LDNS_STATUS_OK; if (!z->valid) return (LDNS_STATUS_ERR); if (z->count == 1 && z->rr_soa != NULL) { *out = z->rr_soa; z->rr_soa = NULL; return (LDNS_STATUS_OK); } for (;;) { if (feof(z->fp)) { *out = NULL; if (z->is_pipe) pclose(z->fp); else fclose(z->fp); z->fp = NULL; return (LDNS_STATUS_OK); } status = ldns_rr_new_frm_fp_l(&rr, z->fp, &z->ttl, &z->origin, &z->prev, NULL); switch (status) { case LDNS_STATUS_OK: if (ldns_rr_get_type(rr) == LDNS_RR_TYPE_SOA) { ldns_rr_free(rr); z->valid = false; status = LDNS_STATUS_ERR; goto out; } z->count++; goto out; case LDNS_STATUS_SYNTAX_EMPTY: case LDNS_STATUS_SYNTAX_TTL: case LDNS_STATUS_SYNTAX_ORIGIN: status = LDNS_STATUS_OK; break; default: goto out; } } out: if (status != LDNS_STATUS_OK) return (status); *out = rr; return (status); } mtbl-0.5/libmy/zonefile.h000066400000000000000000000006571227302760400154270ustar00rootroot00000000000000#ifndef MY_ZONEFILE_H #define MY_ZONEFILE_H #include struct zonefile; struct zonefile * zonefile_init_fname(const char *fname); void zonefile_destroy(struct zonefile **); const ldns_rdf * zonefile_get_domain(struct zonefile *); size_t zonefile_get_count(struct zonefile *); uint32_t zonefile_get_serial(struct zonefile *); ldns_status zonefile_read(struct zonefile *, ldns_rr **); #endif /* MY_ZONEFILE_H */ mtbl-0.5/m4/000077500000000000000000000000001227302760400126375ustar00rootroot00000000000000mtbl-0.5/m4/.gitignore000066400000000000000000000000771227302760400146330ustar00rootroot00000000000000libtool.m4 ltoptions.m4 ltsugar.m4 ltversion.m4 lt~obsolete.m4 mtbl-0.5/m4/pkg.m4000077700000000000000000000000001227302760400165472../libmy/m4/pkg.m4ustar00rootroot00000000000000mtbl-0.5/man/000077500000000000000000000000001227302760400130725ustar00rootroot00000000000000mtbl-0.5/man/asciidoc.conf000066400000000000000000000003251227302760400155170ustar00rootroot00000000000000ifdef::backend-docbook[] [link-inlinemacro] {0%{target}} {0#} {0#{target}{0}} {0#} endif::backend-docbook[] [quotes] ^=strong *= mtbl-0.5/man/mtbl.7000066400000000000000000000076041227302760400141270ustar00rootroot00000000000000'\" t .\" Title: mtbl .\" Author: [FIXME: author] [see http://docbook.sf.net/el/author] .\" Generator: DocBook XSL Stylesheets v1.78.1 .\" Date: 01/31/2014 .\" Manual: \ \& .\" Source: \ \& .\" Language: English .\" .TH "MTBL" "7" "01/31/2014" "\ \&" "\ \&" .\" ----------------------------------------------------------------- .\" * Define some portability stuff .\" ----------------------------------------------------------------- .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .\" http://bugs.debian.org/507673 .\" http://lists.gnu.org/archive/html/groff/2009-02/msg00013.html .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .ie \n(.g .ds Aq \(aq .el .ds Aq ' .\" ----------------------------------------------------------------- .\" * set default formatting .\" ----------------------------------------------------------------- .\" disable hyphenation .nh .\" disable justification (adjust text to left margin only) .ad l .\" ----------------------------------------------------------------- .\" * MAIN CONTENT STARTS HERE * .\" ----------------------------------------------------------------- .SH "NAME" mtbl \- immutable sorted string library .SH "SYNOPSIS" .sp \fB#include \fR .sp \fBgcc [\fR\fB\fIflags\fR\fR\fB] \fR\fB\fIfiles\fR\fR\fB \-lmtbl [\fR\fB\fIlibraries\fR\fR\fB]\fR .SH "DESCRIPTION" .sp The mtbl library provides interfaces for creating, searching, and merging Sorted String Table (SSTable) files in the \fIMTBL\fR format, which provide an immutable mapping of keys to values\&. Sorted String Tables are compact and provide fast random access to keys and key ranges\&. Keys and values are arbitrary byte arrays, and MTBL SSTables may not contain duplicate keys\&. .sp The six main interfaces provided by the mtbl library are: .PP \fBmtbl_iter\fR(3) .RS 4 Iterator objects provide a consistent interface for iterating over the key\-value entries returned by other interfaces\&. .RE .PP \fBmtbl_source\fR(3) .RS 4 Source objects provide functions for obtaining iterators from an underlying data source\&. The \fBmtbl_reader\fR and \fBmtbl_merger\fR interfaces provide functions for obtaining references to a source object\&. The source methods return an \fBmtbl_iter\fR object\&. .RE .PP \fBmtbl_reader\fR(3) .RS 4 Reader objects provide read\-only access to \fIMTBL\fR files\&. .RE .PP \fBmtbl_writer\fR(3) .RS 4 Writer objects initialize a new \fIMTBL\fR file from a sequence of key\-value entries provided by the caller\&. Keys must be in sorted order based on lexicographical byte value, and keys may not be duplicated\&. .RE .PP \fBmtbl_merger\fR(3) .RS 4 Merger objects receive multiple sequences of key\-value entries from one or more \fBmtbl_source\fR objects and combine them into a single, sorted sequence\&. The combined, merged output sequence is provided via the \fBmtbl_source\fR interface\&. .RE .PP \fBmtbl_sorter\fR(3) .RS 4 Sorter objects receive a sequence of key\-value entries provided by the caller and return them in sorted order\&. The caller must provide a callback function to merge values in the case of entries with duplicate keys\&. The sorted output sequence may be retrieved via the \fBmtbl_iter\fR interface or be dumped to an \fBmtbl_writer\fR object\&. .RE .PP \fBmtbl_fileset\fR(3) .RS 4 Fileset objects automatically maintain an \fBmtbl_source\fR built on top of the \fBmtbl_merger\fR and \fBmtbl_reader\fR interfaces\&. The set of underlying \fBmtbl_reader\fR objects is kept synchronized with a "setfile" on disk listing \fIMTBL\fR files\&. .RE .sp Additionally, several utility interfaces are provided: .PP \fBmtbl_crc32c\fR(3) .RS 4 Calculates the CRC32C checksum of a byte array\&. .RE .PP \fBmtbl_fixed\fR(3) .RS 4 Functions for fixed\-width encoding and decoding of 32 and 64 bit integers\&. .RE .PP \fBmtbl_varint\fR(3) .RS 4 Functions for varint encoding and decoding of 32 and 64 bit integers\&. .RE mtbl-0.5/man/mtbl.7.txt000066400000000000000000000046761227302760400147530ustar00rootroot00000000000000= mtbl(7) = == NAME == mtbl - immutable sorted string library == SYNOPSIS == ^#include ^ ^gcc ['flags'] 'files' -lmtbl ['libraries']^ == DESCRIPTION == The mtbl library provides interfaces for creating, searching, and merging Sorted String Table (SSTable) files in the _MTBL_ format, which provide an immutable mapping of keys to values. Sorted String Tables are compact and provide fast random access to keys and key ranges. Keys and values are arbitrary byte arrays, and MTBL SSTables may not contain duplicate keys. The six main interfaces provided by the mtbl library are: link:mtbl_iter[3]:: Iterator objects provide a consistent interface for iterating over the key-value entries returned by other interfaces. link:mtbl_source[3]:: Source objects provide functions for obtaining iterators from an underlying data source. The ^mtbl_reader^ and ^mtbl_merger^ interfaces provide functions for obtaining references to a source object. The source methods return an ^mtbl_iter^ object. link:mtbl_reader[3]:: Reader objects provide read-only access to _MTBL_ files. link:mtbl_writer[3]:: Writer objects initialize a new _MTBL_ file from a sequence of key-value entries provided by the caller. Keys must be in sorted order based on lexicographical byte value, and keys may not be duplicated. link:mtbl_merger[3]:: Merger objects receive multiple sequences of key-value entries from one or more ^mtbl_source^ objects and combine them into a single, sorted sequence. The combined, merged output sequence is provided via the ^mtbl_source^ interface. link:mtbl_sorter[3]:: Sorter objects receive a sequence of key-value entries provided by the caller and return them in sorted order. The caller must provide a callback function to merge values in the case of entries with duplicate keys. The sorted output sequence may be retrieved via the ^mtbl_iter^ interface or be dumped to an ^mtbl_writer^ object. link:mtbl_fileset[3]:: Fileset objects automatically maintain an ^mtbl_source^ built on top of the ^mtbl_merger^ and ^mtbl_reader^ interfaces. The set of underlying ^mtbl_reader^ objects is kept synchronized with a "setfile" on disk listing _MTBL_ files. Additionally, several utility interfaces are provided: link:mtbl_crc32c[3]:: Calculates the CRC32C checksum of a byte array. link:mtbl_fixed[3]:: Functions for fixed-width encoding and decoding of 32 and 64 bit integers. link:mtbl_varint[3]:: Functions for varint encoding and decoding of 32 and 64 bit integers. mtbl-0.5/man/mtbl_crc32c.3000066400000000000000000000031721227302760400152560ustar00rootroot00000000000000'\" t .\" Title: mtbl_crc32c .\" Author: [FIXME: author] [see http://docbook.sf.net/el/author] .\" Generator: DocBook XSL Stylesheets v1.78.1 .\" Date: 01/31/2014 .\" Manual: \ \& .\" Source: \ \& .\" Language: English .\" .TH "MTBL_CRC32C" "3" "01/31/2014" "\ \&" "\ \&" .\" ----------------------------------------------------------------- .\" * Define some portability stuff .\" ----------------------------------------------------------------- .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .\" http://bugs.debian.org/507673 .\" http://lists.gnu.org/archive/html/groff/2009-02/msg00013.html .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .ie \n(.g .ds Aq \(aq .el .ds Aq ' .\" ----------------------------------------------------------------- .\" * set default formatting .\" ----------------------------------------------------------------- .\" disable hyphenation .nh .\" disable justification (adjust text to left margin only) .ad l .\" ----------------------------------------------------------------- .\" * MAIN CONTENT STARTS HERE * .\" ----------------------------------------------------------------- .SH "NAME" mtbl_crc32c \- calculate CRC32C checksum .SH "SYNOPSIS" .sp \fB#include \fR .sp \fBuint32_t mtbl_crc32c(const uint8_t *\fR\fB\fIbuffer\fR\fR\fB, size_t \fR\fB\fIlength\fR\fR\fB);\fR .SH "DESCRIPTION" .sp The \fBmtbl_crc32c\fR() function calculates the CRC32C checksum of a sequence of bytes of length \fIlength\fR\&. The \fIbuffer\fR argument points to the start of the sequence\&. .SH "RETURN VALUE" .sp The CRC32C checksum\&. mtbl-0.5/man/mtbl_crc32c.3.txt000066400000000000000000000006201227302760400160670ustar00rootroot00000000000000= mtbl_crc32c(3) = == NAME == mtbl_crc32c - calculate CRC32C checksum == SYNOPSIS == ^#include ^ ^uint32_t mtbl_crc32c(const uint8_t *'buffer', size_t 'length');^ == DESCRIPTION == The ^mtbl_crc32c^() function calculates the CRC32C checksum of a sequence of bytes of length _length_. The _buffer_ argument points to the start of the sequence. == RETURN VALUE == The CRC32C checksum. mtbl-0.5/man/mtbl_dump.1000066400000000000000000000032361227302760400151430ustar00rootroot00000000000000'\" t .\" Title: mtbl_dump .\" Author: [FIXME: author] [see http://docbook.sf.net/el/author] .\" Generator: DocBook XSL Stylesheets v1.78.1 .\" Date: 01/31/2014 .\" Manual: \ \& .\" Source: \ \& .\" Language: English .\" .TH "MTBL_DUMP" "1" "01/31/2014" "\ \&" "\ \&" .\" ----------------------------------------------------------------- .\" * Define some portability stuff .\" ----------------------------------------------------------------- .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .\" http://bugs.debian.org/507673 .\" http://lists.gnu.org/archive/html/groff/2009-02/msg00013.html .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .ie \n(.g .ds Aq \(aq .el .ds Aq ' .\" ----------------------------------------------------------------- .\" * set default formatting .\" ----------------------------------------------------------------- .\" disable hyphenation .nh .\" disable justification (adjust text to left margin only) .ad l .\" ----------------------------------------------------------------- .\" * MAIN CONTENT STARTS HERE * .\" ----------------------------------------------------------------- .SH "NAME" mtbl_dump \- print key\-value entries from an MTBL file .SH "SYNOPSIS" .sp \fBmtbl_dump\fR \fIFILE\fR .SH "DESCRIPTION" .sp \fBmtbl_dump\fR(1) prints all key\-value entries from an MTBL file to stdout, in file order\&. Each entry is printed on its own line, with double quotes surrounding the key and the value, with a single space character separating the two\&. Unprintable characters and embedded double quote characters are escaped using the Python string literal syntax\&. mtbl-0.5/man/mtbl_dump.1.txt000066400000000000000000000007431227302760400157610ustar00rootroot00000000000000= mtbl_dump(1) = == NAME == mtbl_dump - print key-value entries from an MTBL file == SYNOPSIS == ^mtbl_dump^ 'FILE' == DESCRIPTION == ^mtbl_dump^(1) prints all key-value entries from an MTBL file to stdout, in file order. Each entry is printed on its own line, with double quotes surrounding the key and the value, with a single space character separating the two. Unprintable characters and embedded double quote characters are escaped using the Python string literal syntax. mtbl-0.5/man/mtbl_fileset.3000066400000000000000000000106341227302760400156330ustar00rootroot00000000000000'\" t .\" Title: mtbl_fileset .\" Author: [FIXME: author] [see http://docbook.sf.net/el/author] .\" Generator: DocBook XSL Stylesheets v1.78.1 .\" Date: 01/31/2014 .\" Manual: \ \& .\" Source: \ \& .\" Language: English .\" .TH "MTBL_FILESET" "3" "01/31/2014" "\ \&" "\ \&" .\" ----------------------------------------------------------------- .\" * Define some portability stuff .\" ----------------------------------------------------------------- .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .\" http://bugs.debian.org/507673 .\" http://lists.gnu.org/archive/html/groff/2009-02/msg00013.html .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .ie \n(.g .ds Aq \(aq .el .ds Aq ' .\" ----------------------------------------------------------------- .\" * set default formatting .\" ----------------------------------------------------------------- .\" disable hyphenation .nh .\" disable justification (adjust text to left margin only) .ad l .\" ----------------------------------------------------------------- .\" * MAIN CONTENT STARTS HERE * .\" ----------------------------------------------------------------- .SH "NAME" mtbl_fileset \- automatic multiple MTBL data file merger .SH "SYNOPSIS" .sp \fB#include \fR .sp Fileset objects: .sp .nf \fBstruct mtbl_fileset * mtbl_fileset_init(const char *\fR\fB\fIfname\fR\fR\fB, const struct mtbl_fileset_options *\fR\fB\fIfopt\fR\fR\fB);\fR .fi .sp .nf \fBvoid mtbl_fileset_destroy(struct mtbl_fileset **\fR\fB\fIf\fR\fR\fB);\fR .fi .sp .nf \fBvoid mtbl_fileset_reload(struct mtbl_fileset *\fR\fB\fIf\fR\fR\fB);\fR .fi .sp .nf \fBconst struct mtbl_source * mtbl_fileset_source(struct mtbl_fileset *\fR\fB\fIf\fR\fR\fB);\fR .fi .sp Fileset options: .sp .nf \fBstruct mtbl_fileset_options * mtbl_fileset_options_init(void);\fR .fi .sp .nf \fBvoid mtbl_fileset_options_destroy(struct mtbl_fileset_options **\fR\fB\fIfopt\fR\fR\fB);\fR .fi .sp .nf \fBvoid mtbl_fileset_options_set_merge_func( struct mtbl_fileset_options *\fR\fB\fIfopt\fR\fR\fB, mtbl_merge_func \fR\fB\fIfp\fR\fR\fB, void *\fR\fB\fIclos\fR\fR\fB);\fR .fi .sp .nf \fBvoid mtbl_fileset_options_set_reload_frequency( struct mtbl_fileset_options *\fR\fB\fIfopt\fR\fR\fB, uint32_t \fR\fB\fIreload_frequency\fR\fR\fB);\fR .fi .SH "DESCRIPTION" .sp The \fBmtbl_fileset\fR is a convenience interface for automatically maintaining a merged view of a set of MTBL data files\&. The merged entries may be consumed via the \fBmtbl_source\fR(3) and \fBmtbl_iter\fR(3) interfaces\&. .sp \fBmtbl_fileset\fR objects are initialized from a "setfile", which specifies a list of filenames of MTBL data files, one per line\&. Internally, an \fBmtbl_reader\fR object is initialized from each filename and added to an \fBmtbl_merger\fR object\&. The setfile is watched for changes and the addition or removal of filenames from the setfile will result in the corresponding addition or removal of \fBmtbl_reader\fR objects\&. .sp Because the MTBL format does not allow duplicate keys, the caller must provide a function which will accept a key and two conflicting values for that key and return a replacement value\&. This function may be called multiple times for the same key if the same key is inserted more than twice\&. See \fBmtbl_merger\fR(3) for further details about the merge function\&. .sp \fBmtbl_fileset\fR objects are created with the \fBmtbl_fileset_init\fR() function, which requires the path to a "setfile", \fIfname\fR, and a non\-NULL \fIfopt\fR argument which has been configured with a merge function \fIfp\fR\&. \fBmtbl_fileset_source\fR() should then be called in order to consume output via the \fBmtbl_source\fR(3) interface\&. .sp Accesses via the \fBmtbl_source\fR(3) interface will implicitly check for updates to the setfile\&. However, it may be necessary to explicitly call the \fBmtbl_fileset_reload\fR() function in order to check for updates, especially if files are being removed from the setfile\&. .SS "Fileset options" .sp .it 1 an-trap .nr an-no-space-flag 1 .nr an-break-flag 1 .br .ps +1 \fBmerge_func\fR .RS 4 .sp See \fBmtbl_merger\fR(3)\&. An \fBmtbl_merger\fR object is used internally for the external sort\&. .RE .sp .it 1 an-trap .nr an-no-space-flag 1 .nr an-break-flag 1 .br .ps +1 \fBreload_interval\fR .RS 4 .sp Specifies the interval between checks for updates to the setfile, in seconds\&. Defaults to 60 seconds\&. .RE mtbl-0.5/man/mtbl_fileset.3.txt000066400000000000000000000053461227302760400164550ustar00rootroot00000000000000= mtbl_fileset(3) = == NAME == mtbl_fileset - automatic multiple MTBL data file merger == SYNOPSIS == ^#include ^ Fileset objects: [verse] ^struct mtbl_fileset * mtbl_fileset_init(const char *'fname', const struct mtbl_fileset_options *'fopt');^ [verse] ^void mtbl_fileset_destroy(struct mtbl_fileset **'f');^ [verse] ^void mtbl_fileset_reload(struct mtbl_fileset *'f');^ [verse] ^const struct mtbl_source * mtbl_fileset_source(struct mtbl_fileset *'f');^ Fileset options: [verse] ^struct mtbl_fileset_options * mtbl_fileset_options_init(void);^ [verse] ^void mtbl_fileset_options_destroy(struct mtbl_fileset_options **'fopt');^ [verse] ^void mtbl_fileset_options_set_merge_func( struct mtbl_fileset_options *'fopt', mtbl_merge_func 'fp', void *'clos');^ [verse] ^void mtbl_fileset_options_set_reload_frequency( struct mtbl_fileset_options *'fopt', uint32_t 'reload_frequency');^ == DESCRIPTION == The ^mtbl_fileset^ is a convenience interface for automatically maintaining a merged view of a set of MTBL data files. The merged entries may be consumed via the ^mtbl_source^(3) and ^mtbl_iter^(3) interfaces. ^mtbl_fileset^ objects are initialized from a "setfile", which specifies a list of filenames of MTBL data files, one per line. Internally, an ^mtbl_reader^ object is initialized from each filename and added to an ^mtbl_merger^ object. The setfile is watched for changes and the addition or removal of filenames from the setfile will result in the corresponding addition or removal of ^mtbl_reader^ objects. Because the MTBL format does not allow duplicate keys, the caller must provide a function which will accept a key and two conflicting values for that key and return a replacement value. This function may be called multiple times for the same key if the same key is inserted more than twice. See ^mtbl_merger^(3) for further details about the merge function. ^mtbl_fileset^ objects are created with the ^mtbl_fileset_init^() function, which requires the path to a "setfile", _fname_, and a non-NULL _fopt_ argument which has been configured with a merge function _fp_. ^mtbl_fileset_source^() should then be called in order to consume output via the ^mtbl_source^(3) interface. Accesses via the ^mtbl_source^(3) interface will implicitly check for updates to the setfile. However, it may be necessary to explicitly call the ^mtbl_fileset_reload^() function in order to check for updates, especially if files are being removed from the setfile. === Fileset options === ==== merge_func ==== See ^mtbl_merger^(3). An ^mtbl_merger^ object is used internally for the external sort. ==== reload_interval ==== Specifies the interval between checks for updates to the setfile, in seconds. Defaults to 60 seconds. mtbl-0.5/man/mtbl_fixed.3000066400000000000000000000047501227302760400153010ustar00rootroot00000000000000'\" t .\" Title: mtbl_fixed .\" Author: [FIXME: author] [see http://docbook.sf.net/el/author] .\" Generator: DocBook XSL Stylesheets v1.78.1 .\" Date: 01/31/2014 .\" Manual: \ \& .\" Source: \ \& .\" Language: English .\" .TH "MTBL_FIXED" "3" "01/31/2014" "\ \&" "\ \&" .\" ----------------------------------------------------------------- .\" * Define some portability stuff .\" ----------------------------------------------------------------- .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .\" http://bugs.debian.org/507673 .\" http://lists.gnu.org/archive/html/groff/2009-02/msg00013.html .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .ie \n(.g .ds Aq \(aq .el .ds Aq ' .\" ----------------------------------------------------------------- .\" * set default formatting .\" ----------------------------------------------------------------- .\" disable hyphenation .nh .\" disable justification (adjust text to left margin only) .ad l .\" ----------------------------------------------------------------- .\" * MAIN CONTENT STARTS HERE * .\" ----------------------------------------------------------------- .SH "NAME" mtbl_fixed \- Fixed\-width encoding and decoding of 32 and 64 bit integers .SH "SYNOPSIS" .sp \fB#include \fR .sp \fBsize_t mtbl_fixed_encode32(uint8_t *\fR\fB\fIdst\fR\fR\fB, uint32_t \fR\fB\fIvalue\fR\fR\fB);\fR .sp \fBsize_t mtbl_fixed_encode64(uint8_t *\fR\fB\fIdst\fR\fR\fB, uint64_t \fR\fB\fIvalue\fR\fR\fB);\fR .sp \fBuint32_t mtbl_fixed_decode32(const uint8_t *\fR\fB\fIptr\fR\fR\fB);\fR .sp \fBuint64_t mtbl_fixed_decode64(const uint8_t *\fR\fB\fIptr\fR\fR\fB);\fR .SH "DESCRIPTION" .sp \fBmtbl_fixed_encode32\fR() and \fBmtbl_fixed_encode64\fR() write the 32 or 64 bit quantity, respectively, in the argument \fIvalue\fR to the buffer in the argument \fIdst\fR\&. The quantity will be written in little endian order, regardless of host architecture\&. .sp \fBmtbl_fixed_decode32\fR() and \fBmtbl_fixed_decode64\fR() read and return the 32 or 64 bit quantity, respectively, in the argument \fIptr\fR\&. The quantity will be read in little endian order, regardless of host architecture\&. .sp Bounds checking must be performed by the caller\&. .SH "RETURN VALUE" .sp \fBmtbl_fixed_encode32\fR() and \fBmtbl_fixed_encode64\fR() return the number of bytes written to the buffer\&. (4 or 8, respectively\&.) .sp \fBmtbl_fixed_decode32\fR() and \fBmtbl_fixed_decode64\fR() return the decoded quantity\&. mtbl-0.5/man/mtbl_fixed.3.txt000066400000000000000000000021661227302760400161160ustar00rootroot00000000000000= mtbl_fixed(3) = == NAME == mtbl_fixed - Fixed-width encoding and decoding of 32 and 64 bit integers == SYNOPSIS == ^#include ^ ^size_t mtbl_fixed_encode32(uint8_t *'dst', uint32_t 'value');^ ^size_t mtbl_fixed_encode64(uint8_t *'dst', uint64_t 'value');^ ^uint32_t mtbl_fixed_decode32(const uint8_t *'ptr');^ ^uint64_t mtbl_fixed_decode64(const uint8_t *'ptr');^ == DESCRIPTION == ^mtbl_fixed_encode32^() and ^mtbl_fixed_encode64^() write the 32 or 64 bit quantity, respectively, in the argument _value_ to the buffer in the argument _dst_. The quantity will be written in little endian order, regardless of host architecture. ^mtbl_fixed_decode32^() and ^mtbl_fixed_decode64^() read and return the 32 or 64 bit quantity, respectively, in the argument _ptr_. The quantity will be read in little endian order, regardless of host architecture. Bounds checking must be performed by the caller. == RETURN VALUE == ^mtbl_fixed_encode32^() and ^mtbl_fixed_encode64^() return the number of bytes written to the buffer. (4 or 8, respectively.) ^mtbl_fixed_decode32^() and ^mtbl_fixed_decode64^() return the decoded quantity. mtbl-0.5/man/mtbl_info.1000066400000000000000000000054461227302760400151360ustar00rootroot00000000000000'\" t .\" Title: mtbl_info .\" Author: [FIXME: author] [see http://docbook.sf.net/el/author] .\" Generator: DocBook XSL Stylesheets v1.78.1 .\" Date: 01/31/2014 .\" Manual: \ \& .\" Source: \ \& .\" Language: English .\" .TH "MTBL_INFO" "1" "01/31/2014" "\ \&" "\ \&" .\" ----------------------------------------------------------------- .\" * Define some portability stuff .\" ----------------------------------------------------------------- .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .\" http://bugs.debian.org/507673 .\" http://lists.gnu.org/archive/html/groff/2009-02/msg00013.html .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .ie \n(.g .ds Aq \(aq .el .ds Aq ' .\" ----------------------------------------------------------------- .\" * set default formatting .\" ----------------------------------------------------------------- .\" disable hyphenation .nh .\" disable justification (adjust text to left margin only) .ad l .\" ----------------------------------------------------------------- .\" * MAIN CONTENT STARTS HERE * .\" ----------------------------------------------------------------- .SH "NAME" mtbl_info \- display information about an MTBL file .SH "SYNOPSIS" .sp \fBmtbl_info\fR \fIFILE\fR [\fIFILE\fR]\&... .SH "DESCRIPTION" .sp \fBmtbl_info\fR(1) displays the following information about the MTBL files specified on the command line\&. .sp \fIfile name\fR \(em the name of the MTBL file\&. .sp \fIfile size\fR \(em the total size of the MTBL file, in bytes\&. .sp \fIindex bytes\fR \(em the total number of bytes and proportion of the total file size consumed by the index\&. .sp \fIdata block bytes\fR \(em the total number of bytes and proportion of the total file size consumed by data blocks\&. .sp \fIdata block size\fR \(em the maximum size of an uncompressed data block\&. .sp \fIdata block count\fR \(em the total number of data blocks\&. .sp \fIentry count\fR \(em the total number of key\-value entries\&. .sp \fIkey bytes\fR \(em the total number of bytes that all keys in the file would occupy if stored end\-to\-end in a byte array with no delimiters\&. .sp \fIvalue bytes\fR \(em the total number of bytes that all values in the file would occupy if stored end\-to\-end in a byte array with no delimiters\&. .sp \fIcompression algorithm\fR \(em the algorithm used to compress data blocks\&. Possible values are "none", "snappy" and "zlib"\&. .sp \fIcompactness\fR \(em a rough metric comparing the total number of bytes in the key\-value entries with the total size of the MTBL file\&. It is calculated as (file size) / (key bytes + value bytes), and thus takes into account the gains of data block compression and prefix key compression against the overhead of the index, trailer, and data block offset arrays\&. mtbl-0.5/man/mtbl_info.1.txt000066400000000000000000000027501227302760400157470ustar00rootroot00000000000000= mtbl_info(1) = == NAME == mtbl_info - display information about an MTBL file == SYNOPSIS == ^mtbl_info^ 'FILE' ['FILE']... == DESCRIPTION == ^mtbl_info^(1) displays the following information about the MTBL files specified on the command line. 'file name' -- the name of the MTBL file. 'file size' -- the total size of the MTBL file, in bytes. 'index bytes' -- the total number of bytes and proportion of the total file size consumed by the index. 'data block bytes' -- the total number of bytes and proportion of the total file size consumed by data blocks. 'data block size' -- the maximum size of an uncompressed data block. 'data block count' -- the total number of data blocks. 'entry count' -- the total number of key-value entries. 'key bytes' -- the total number of bytes that all keys in the file would occupy if stored end-to-end in a byte array with no delimiters. 'value bytes' -- the total number of bytes that all values in the file would occupy if stored end-to-end in a byte array with no delimiters. 'compression algorithm' -- the algorithm used to compress data blocks. Possible values are "none", "snappy" and "zlib". 'compactness' -- a rough metric comparing the total number of bytes in the key-value entries with the total size of the MTBL file. It is calculated as (file size) / (key bytes + value bytes), and thus takes into account the gains of data block compression and prefix key compression against the overhead of the index, trailer, and data block offset arrays. mtbl-0.5/man/mtbl_iter.3000066400000000000000000000045721227302760400151470ustar00rootroot00000000000000'\" t .\" Title: mtbl_iter .\" Author: [FIXME: author] [see http://docbook.sf.net/el/author] .\" Generator: DocBook XSL Stylesheets v1.78.1 .\" Date: 01/31/2014 .\" Manual: \ \& .\" Source: \ \& .\" Language: English .\" .TH "MTBL_ITER" "3" "01/31/2014" "\ \&" "\ \&" .\" ----------------------------------------------------------------- .\" * Define some portability stuff .\" ----------------------------------------------------------------- .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .\" http://bugs.debian.org/507673 .\" http://lists.gnu.org/archive/html/groff/2009-02/msg00013.html .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .ie \n(.g .ds Aq \(aq .el .ds Aq ' .\" ----------------------------------------------------------------- .\" * set default formatting .\" ----------------------------------------------------------------- .\" disable hyphenation .nh .\" disable justification (adjust text to left margin only) .ad l .\" ----------------------------------------------------------------- .\" * MAIN CONTENT STARTS HERE * .\" ----------------------------------------------------------------- .SH "NAME" mtbl_iter \- iterate over a sequence of key\-value pairs .SH "SYNOPSIS" .sp \fB#include \fR .sp .nf \fBmtbl_res mtbl_iter_next(struct mtbl_iter *\fR\fB\fIit\fR\fR\fB, const uint8_t * *\fR\fB\fIkey\fR\fR\fB, size_t *\fR\fB\fIlen_key\fR\fR\fB, const uint8_t * *\fR\fB\fIval\fR\fR\fB, size_t *\fR\fB\fIlen_val\fR\fR\fB);\fR .fi .sp .nf \fBvoid mtbl_iter_destroy(struct mtbl_iter **\fR\fB\fIit\fR\fR\fB);\fR .fi .SH "DESCRIPTION" .sp The \fBmtbl_iter\fR interface is used to return a sequence of one or more key\-value pairs\&. Once the caller obtains an \fBmtbl_iter\fR object, \fBmtbl_iter_next\fR() should be repeatedly called on it until there are no more key\-value entries to retrieve, at which point the iterator object must be freed by calling \fBmtbl_iter_destroy\fR()\&. .SH "RETURN VALUE" .sp \fBmtbl_iter_next\fR() returns \fBmtbl_res_success\fR if a key\-value entry was successfully retrieved, in which case \fIkey\fR and \fIval\fR will point to buffers of length \fIlen_key\fR and \fIlen_val\fR respectively\&. The value \fBmtbl_res_failure\fR is returned if there are no more entries to read, or if the \fIit\fR argument is NULL\&. .SH "SEE ALSO" .sp \fBmtbl_source\fR(3) mtbl-0.5/man/mtbl_iter.3.txt000066400000000000000000000020361227302760400157560ustar00rootroot00000000000000= mtbl_iter(3) = == NAME == mtbl_iter - iterate over a sequence of key-value pairs == SYNOPSIS == ^#include ^ [verse] ^mtbl_res mtbl_iter_next(struct mtbl_iter *'it', const uint8_t * *'key', size_t *'len_key', const uint8_t * *'val', size_t *'len_val');^ [verse] ^void mtbl_iter_destroy(struct mtbl_iter **'it');^ == DESCRIPTION == The ^mtbl_iter^ interface is used to return a sequence of one or more key-value pairs. Once the caller obtains an ^mtbl_iter^ object, ^mtbl_iter_next^() should be repeatedly called on it until there are no more key-value entries to retrieve, at which point the iterator object must be freed by calling ^mtbl_iter_destroy^(). == RETURN VALUE == ^mtbl_iter_next^() returns ^mtbl_res_success^ if a key-value entry was successfully retrieved, in which case _key_ and _val_ will point to buffers of length _len_key_ and _len_val_ respectively. The value ^mtbl_res_failure^ is returned if there are no more entries to read, or if the _it_ argument is NULL. == SEE ALSO == link:mtbl_source[3] mtbl-0.5/man/mtbl_merge.1000066400000000000000000000070131227302760400152720ustar00rootroot00000000000000'\" t .\" Title: mtbl_merge .\" Author: [FIXME: author] [see http://docbook.sf.net/el/author] .\" Generator: DocBook XSL Stylesheets v1.78.1 .\" Date: 01/31/2014 .\" Manual: \ \& .\" Source: \ \& .\" Language: English .\" .TH "MTBL_MERGE" "1" "01/31/2014" "\ \&" "\ \&" .\" ----------------------------------------------------------------- .\" * Define some portability stuff .\" ----------------------------------------------------------------- .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .\" http://bugs.debian.org/507673 .\" http://lists.gnu.org/archive/html/groff/2009-02/msg00013.html .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .ie \n(.g .ds Aq \(aq .el .ds Aq ' .\" ----------------------------------------------------------------- .\" * set default formatting .\" ----------------------------------------------------------------- .\" disable hyphenation .nh .\" disable justification (adjust text to left margin only) .ad l .\" ----------------------------------------------------------------- .\" * MAIN CONTENT STARTS HERE * .\" ----------------------------------------------------------------- .SH "NAME" mtbl_merge \- merge MTBL data from multiple input files into a single output file .SH "SYNOPSIS" .sp User\-provided functions: .sp .nf \fBtypedef void * (*mtbl_merge_init_func)(void);\fR .fi .sp .nf \fBtypedef void (*mtbl_merge_free_func)(void *clos);\fR .fi .sp .nf \fBtypedef void (*mtbl_merge_func)(void *\fR\fB\fIclos\fR\fR\fB, const uint8_t *\fR\fB\fIkey\fR\fR\fB, size_t \fR\fB\fIlen_key\fR\fR\fB, const uint8_t *\fR\fB\fIval0\fR\fR\fB, size_t \fR\fB\fIlen_val0\fR\fR\fB, const uint8_t *\fR\fB\fIval1\fR\fR\fB, size_t \fR\fB\fIlen_val1\fR\fR\fB, uint8_t **\fR\fB\fImerged_val\fR\fR\fB, size_t *\fR\fB\fIlen_merged_val\fR\fR\fB);\fR .fi .sp Command line tool: .sp .nf \fBexport MTBL_MERGE_DSO="\fR\fB\fIlibexample\&.so\&.0\fR\fR\fB"\fR \fBexport MTBL_MERGE_FUNC_PREFIX="\fR\fB\fIexample_merge\fR\fR\fB"\fR \fBmtbl_merge\fR \fIINPUT\fR [\fIINPUT\fR]\&... \fIOUTPUT\fR .fi .SH "DESCRIPTION" .sp \fBmtbl_merge\fR(1) is a command\-line driver for the \fBmtbl_merger\fR(3) interface\&. The \fBmtbl_merger\fR(3) interface requires a user\-provided merge function, which is loaded from a shared object whose filename is specified in the environment variable \fIMTBL_MERGE_DSO\fR\&. .sp The user\-provided merge function must have the same type as the \fImtbl_merge_func\fR function type given above in the synopsis\&. The symbol name of the merge function to be loaded from the user\-provided DSO will be constructed by appending "_func" to the string provided in the \fIMTBL_MERGE_FUNC_PREFIX\fR environment variable, which must be non\-empty\&. .sp Additionally, two optional functions may be provided: an "init" function whose symbol name is "_init" appended to the function prefix, and a "free" function whose symbol name is "_free" appended to the function prefix\&. If the "init" function exists, it will be called at the beginning, before any calls to the merge function, and the return value from the init function will be passed as the first argument to the merge function\&. If the "free" function exists, it will be called at the end, after any calls to the merge function, and its argument will be the return value of the "init" function\&. .sp The environment variable \fIMTBL_MERGE_BLOCK_SIZE\fR may optionally be set in order to configure the MTBL block size (in bytes) of the output file\&. .SH "SEE ALSO" .sp \fBmtbl_merger\fR(3) mtbl-0.5/man/mtbl_merge.1.txt000066400000000000000000000040711227302760400161110ustar00rootroot00000000000000= mtbl_merge(1) = == NAME == mtbl_merge - merge MTBL data from multiple input files into a single output file == SYNOPSIS == User-provided functions: [verse] ^typedef void * (*mtbl_merge_init_func)(void);^ [verse] ^typedef void (*mtbl_merge_free_func)(void *clos);^ [verse] ^typedef void (*mtbl_merge_func)(void *'clos', const uint8_t *'key', size_t 'len_key', const uint8_t *'val0', size_t 'len_val0', const uint8_t *'val1', size_t 'len_val1', uint8_t **'merged_val', size_t *'len_merged_val');^ Command line tool: [verse] ^export MTBL_MERGE_DSO="'libexample.so.0'"^ ^export MTBL_MERGE_FUNC_PREFIX="'example_merge'"^ ^mtbl_merge^ 'INPUT' ['INPUT']... 'OUTPUT' == DESCRIPTION == ^mtbl_merge^(1) is a command-line driver for the ^mtbl_merger^(3) interface. The ^mtbl_merger^(3) interface requires a user-provided merge function, which is loaded from a shared object whose filename is specified in the environment variable 'MTBL_MERGE_DSO'. The user-provided merge function must have the same type as the 'mtbl_merge_func' function type given above in the synopsis. The symbol name of the merge function to be loaded from the user-provided DSO will be constructed by appending "_func" to the string provided in the 'MTBL_MERGE_FUNC_PREFIX' environment variable, which must be non-empty. Additionally, two optional functions may be provided: an "init" function whose symbol name is "_init" appended to the function prefix, and a "free" function whose symbol name is "_free" appended to the function prefix. If the "init" function exists, it will be called at the beginning, before any calls to the merge function, and the return value from the init function will be passed as the first argument to the merge function. If the "free" function exists, it will be called at the end, after any calls to the merge function, and its argument will be the return value of the "init" function. The environment variable 'MTBL_MERGE_BLOCK_SIZE' may optionally be set in order to configure the MTBL block size (in bytes) of the output file. == SEE ALSO == ^mtbl_merger^(3) mtbl-0.5/man/mtbl_merger.3000066400000000000000000000130221227302760400154530ustar00rootroot00000000000000'\" t .\" Title: mtbl_merger .\" Author: [FIXME: author] [see http://docbook.sf.net/el/author] .\" Generator: DocBook XSL Stylesheets v1.78.1 .\" Date: 01/31/2014 .\" Manual: \ \& .\" Source: \ \& .\" Language: English .\" .TH "MTBL_MERGER" "3" "01/31/2014" "\ \&" "\ \&" .\" ----------------------------------------------------------------- .\" * Define some portability stuff .\" ----------------------------------------------------------------- .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .\" http://bugs.debian.org/507673 .\" http://lists.gnu.org/archive/html/groff/2009-02/msg00013.html .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .ie \n(.g .ds Aq \(aq .el .ds Aq ' .\" ----------------------------------------------------------------- .\" * set default formatting .\" ----------------------------------------------------------------- .\" disable hyphenation .nh .\" disable justification (adjust text to left margin only) .ad l .\" ----------------------------------------------------------------- .\" * MAIN CONTENT STARTS HERE * .\" ----------------------------------------------------------------- .SH "NAME" mtbl_merger \- merge multiple MTBL data sources into a single output .SH "SYNOPSIS" .sp \fB#include \fR .sp Merger objects: .sp .nf \fBstruct mtbl_merger * mtbl_merger_init(const struct mtbl_merger_options *\fR\fB\fImopt\fR\fR\fB);\fR .fi .sp .nf \fBvoid mtbl_merger_destroy(struct mtbl_merger **\fR\fB\fIm\fR\fR\fB);\fR .fi .sp .nf \fBvoid mtbl_merger_add_source(struct mtbl_merger *\fR\fB\fIm\fR\fR\fB, const struct mtbl_source *\fR\fB\fIs\fR\fR\fB);\fR .fi .sp .nf \fBconst struct mtbl_source * mtbl_merger_source(struct mtbl_merger *\fR\fB\fIm\fR\fR\fB);\fR .fi .sp Merger options: .sp .nf \fBstruct mtbl_merger_options * mtbl_merger_options_init(void);\fR .fi .sp .nf \fBvoid mtbl_merger_options_destroy(struct mtbl_merger_options **\fR\fB\fImopt\fR\fR\fB);\fR .fi .sp .nf \fBvoid mtbl_merger_options_set_merge_func( struct mtbl_merger_options *\fR\fB\fImopt\fR\fR\fB, mtbl_merge_func \fR\fB\fIfp\fR\fR\fB, void *\fR\fB\fIclos\fR\fR\fB);\fR .fi .sp .nf \fBtypedef void (*mtbl_merge_func)(void *\fR\fB\fIclos\fR\fR\fB, const uint8_t *\fR\fB\fIkey\fR\fR\fB, size_t \fR\fB\fIlen_key\fR\fR\fB, const uint8_t *\fR\fB\fIval0\fR\fR\fB, size_t \fR\fB\fIlen_val0\fR\fR\fB, const uint8_t *\fR\fB\fIval1\fR\fR\fB, size_t \fR\fB\fIlen_val1\fR\fR\fB, uint8_t **\fR\fB\fImerged_val\fR\fR\fB, size_t *\fR\fB\fIlen_merged_val\fR\fR\fB);\fR .fi .SH "DESCRIPTION" .sp Multiple MTBL data sources may be merged together using the \fBmtbl_merger\fR interface, which reads key\-value entries from one or more sources and provides these entries in sorted order\&. The sorted entries may be consumed via the \fBmtbl_source\fR(3) and \fBmtbl_iter\fR(3) interfaces\&. .sp Because the MTBL format does not allow duplicate keys, the caller must provide a function which will accept a key and two conflicting values for that key and return a replacement value\&. This function may be called multiple times for the same key if more than two sources are being merged\&. .sp \fBmtbl_merger\fR objects are created with the \fBmtbl_merger_init\fR() function, which requires a non\-NULL \fImopt\fR argument which has been configured with a merge function \fIfp\fR\&. .sp One or more \fBmtbl_reader\fR objects must be provided as input to the \fBmtbl_merger\fR object by calling \fBmtbl_merger_add_source\fR()\&. After the desired sources have been configured, \fBmtbl_merger_source\fR() should be called in order to consume the merged output via the \fBmtbl_source\fR(3) interface\&. .SS "Merger options" .sp .it 1 an-trap .nr an-no-space-flag 1 .nr an-break-flag 1 .br .ps +1 \fBmerge_func\fR .RS 4 .sp This option specifies a merge function callback, consisting of a function pointer \fIfp\fR and a pointer to user data \fIclos\fR which will be passed as the first argument to \fIfp\fR\&. The merge function callback will be used during iteration over the \fBmtbl_merger\fR object to merge entries with duplicate keys in the input sources\&. .sp The remaining arguments to the merge function are: .sp \fIkey\fR \(em pointer to the key for which there exist duplicate values\&. .sp \fIlen_key\fR \(em length of the key\&. .sp \fIval0\fR \(em pointer to the first value\&. .sp \fIlen_val_0\fR \(em length of the first value\&. .sp \fIval1\fR \(em pointer to the second value\&. .sp \fIlen_val_1\fR \(em length of the second value\&. .sp \fImerged_val\fR \(em pointer to where the callee should place its merged value\&. .sp \fIlen_merged_val\fR \(em pointer to where the callee should place the length of its merged value\&. .sp \fImerged_val\fR must be allocated with the system allocator, and the \fBmtbl_merger\fR interface takes responsibility for free()ing the value once it is no longer needed\&. .sp The callee may provide an empty value as the merged value, in which case \fImerged_val\fR must still contain an allocated, non\-NULL value and \fIlen_merged_val\fR must contain the value 0\&. .sp The callee may indicate an error by returning NULL in the \fImerged_val\fR argument, which will abort iteration over the \fBmtbl_merger\fR object\&. .RE .SH "RETURN VALUE" .sp If the merge function callback is unable to provide a merged value (that is, it fails to return a non\-NULL value in its \fImerged_val\fR argument), the merge process will be aborted, and any iterators over the \fBmtbl_merger\fR object (via the \fBmtbl_source\fR(3) interface) will return \fBmtbl_res_failure\fR\&. mtbl-0.5/man/mtbl_merger.3.txt000066400000000000000000000073051227302760400163000ustar00rootroot00000000000000= mtbl_merger(3) = == NAME == mtbl_merger - merge multiple MTBL data sources into a single output == SYNOPSIS == ^#include ^ Merger objects: [verse] ^struct mtbl_merger * mtbl_merger_init(const struct mtbl_merger_options *'mopt');^ [verse] ^void mtbl_merger_destroy(struct mtbl_merger **'m');^ [verse] ^void mtbl_merger_add_source(struct mtbl_merger *'m', const struct mtbl_source *'s');^ [verse] ^const struct mtbl_source * mtbl_merger_source(struct mtbl_merger *'m');^ Merger options: [verse] ^struct mtbl_merger_options * mtbl_merger_options_init(void);^ [verse] ^void mtbl_merger_options_destroy(struct mtbl_merger_options **'mopt');^ [verse] ^void mtbl_merger_options_set_merge_func( struct mtbl_merger_options *'mopt', mtbl_merge_func 'fp', void *'clos');^ [verse] ^typedef void (*mtbl_merge_func)(void *'clos', const uint8_t *'key', size_t 'len_key', const uint8_t *'val0', size_t 'len_val0', const uint8_t *'val1', size_t 'len_val1', uint8_t **'merged_val', size_t *'len_merged_val');^ == DESCRIPTION == Multiple MTBL data sources may be merged together using the ^mtbl_merger^ interface, which reads key-value entries from one or more sources and provides these entries in sorted order. The sorted entries may be consumed via the ^mtbl_source^(3) and ^mtbl_iter^(3) interfaces. Because the MTBL format does not allow duplicate keys, the caller must provide a function which will accept a key and two conflicting values for that key and return a replacement value. This function may be called multiple times for the same key if more than two sources are being merged. ^mtbl_merger^ objects are created with the ^mtbl_merger_init^() function, which requires a non-NULL _mopt_ argument which has been configured with a merge function _fp_. One or more ^mtbl_reader^ objects must be provided as input to the ^mtbl_merger^ object by calling ^mtbl_merger_add_source^(). After the desired sources have been configured, ^mtbl_merger_source^() should be called in order to consume the merged output via the ^mtbl_source^(3) interface. === Merger options === ==== ^merge_func^ ==== This option specifies a merge function callback, consisting of a function pointer 'fp' and a pointer to user data 'clos' which will be passed as the first argument to 'fp'. The merge function callback will be used during iteration over the ^mtbl_merger^ object to merge entries with duplicate keys in the input sources. The remaining arguments to the merge function are: 'key' -- pointer to the key for which there exist duplicate values. 'len_key' -- length of the key. 'val0' -- pointer to the first value. 'len_val_0' -- length of the first value. 'val1' -- pointer to the second value. 'len_val_1' -- length of the second value. 'merged_val' -- pointer to where the callee should place its merged value. 'len_merged_val' -- pointer to where the callee should place the length of its merged value. 'merged_val' must be allocated with the system allocator, and the ^mtbl_merger^ interface takes responsibility for free()ing the value once it is no longer needed. The callee may provide an empty value as the merged value, in which case 'merged_val' must still contain an allocated, non-NULL value and 'len_merged_val' must contain the value 0. The callee may indicate an error by returning NULL in the 'merged_val' argument, which will abort iteration over the ^mtbl_merger^ object. == RETURN VALUE == If the merge function callback is unable to provide a merged value (that is, it fails to return a non-NULL value in its _merged_val_ argument), the merge process will be aborted, and any iterators over the ^mtbl_merger^ object (via the ^mtbl_source^(3) interface) will return ^mtbl_res_failure^. mtbl-0.5/man/mtbl_reader.3000066400000000000000000000071431227302760400154430ustar00rootroot00000000000000'\" t .\" Title: mtbl_reader .\" Author: [FIXME: author] [see http://docbook.sf.net/el/author] .\" Generator: DocBook XSL Stylesheets v1.78.1 .\" Date: 01/31/2014 .\" Manual: \ \& .\" Source: \ \& .\" Language: English .\" .TH "MTBL_READER" "3" "01/31/2014" "\ \&" "\ \&" .\" ----------------------------------------------------------------- .\" * Define some portability stuff .\" ----------------------------------------------------------------- .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .\" http://bugs.debian.org/507673 .\" http://lists.gnu.org/archive/html/groff/2009-02/msg00013.html .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .ie \n(.g .ds Aq \(aq .el .ds Aq ' .\" ----------------------------------------------------------------- .\" * set default formatting .\" ----------------------------------------------------------------- .\" disable hyphenation .nh .\" disable justification (adjust text to left margin only) .ad l .\" ----------------------------------------------------------------- .\" * MAIN CONTENT STARTS HERE * .\" ----------------------------------------------------------------- .SH "NAME" mtbl_reader \- read an MTBL file .SH "SYNOPSIS" .sp \fB#include \fR .sp Reader objects: .sp .nf \fBstruct mtbl_reader * mtbl_reader_init(const char *\fR\fB\fIfname\fR\fR\fB, const struct mtbl_reader_options *\fR\fB\fIropt\fR\fR\fB);\fR .fi .sp .nf \fBstruct mtbl_reader * mtbl_reader_init_fd(int \fR\fB\fIfd\fR\fR\fB, const struct mtbl_reader_options *\fR\fB\fIropt\fR\fR\fB);\fR .fi .sp .nf \fBvoid mtbl_reader_destroy(struct mtbl_reader **\fR\fB\fIr\fR\fR\fB);\fR .fi .sp .nf \fBconst struct mtbl_source * mtbl_reader_source(struct mtbl_reader *\fR\fB\fIr\fR\fR\fB);\fR .fi .sp Reader options: .sp .nf \fBstruct mtbl_reader_options * mtbl_reader_options_init(void);\fR .fi .sp .nf \fBvoid mtbl_reader_options_destroy(struct mtbl_reader_options **\fR\fB\fIropt\fR\fR\fB);\fR .fi .sp .nf \fBvoid mtbl_reader_options_set_verify_checksums( struct mtbl_reader_options *\fR\fB\fIropt\fR\fR\fB, bool \fR\fB\fIverify_checksums\fR\fR\fB);\fR .fi .SH "DESCRIPTION" .sp MTBL files are accessed by creating an \fBmtbl_reader\fR object, calling \fBmtbl_reader_source\fR() to obtain an \fBmtbl_source\fR handle, and using the \fBmtbl_source\fR(3) interface to read entries\&. .sp \fBmtbl_reader\fR objects may be created by calling \fBmtbl_reader_init\fR() with an \fIfname\fR argument specifying the filename to be opened, or \fBmtbl_reader_init_fd\fR() may be called with an \fIfd\fR argument specifying an open, readable file descriptor\&. Since MTBL files are immutable, the same MTBL file may be opened and read from concurrently by independent threads or processes\&. .sp If the \fIropt\fR parameter to \fBmtbl_reader_init\fR() or \fBmtbl_reader_init_fd\fR() is non\-NULL, the parameters specified in the \fBmtbl_reader_options\fR object will be configured into the \fBmtbl_reader\fR object\&. .SS "Reader options" .sp .it 1 an-trap .nr an-no-space-flag 1 .nr an-break-flag 1 .br .ps +1 \fBverify_checksums\fR .RS 4 .sp Specifies whether or not the CRC32C checksum on each data block should be verified or not\&. If \fIverify_checksums\fR is enabled, a checksum mismatch will cause a runtime error\&. Note that the checksum on the index block is always verified, since the overhead of doing this once when the reader object is instantiated is minimal\&. The default is to not verify data block checksums\&. .RE .SH "RETURN VALUE" .sp \fBmtbl_reader_init\fR() and \fBmtbl_reader_init_fd\fR() return NULL on failure, and non\-NULL on success\&. mtbl-0.5/man/mtbl_reader.3.txt000066400000000000000000000041001227302760400162470ustar00rootroot00000000000000= MTBL_READER(3) = == NAME == mtbl_reader - read an MTBL file == SYNOPSIS == ^#include ^ Reader objects: [verse] ^struct mtbl_reader * mtbl_reader_init(const char *'fname', const struct mtbl_reader_options *'ropt');^ [verse] ^struct mtbl_reader * mtbl_reader_init_fd(int 'fd', const struct mtbl_reader_options *'ropt');^ [verse] ^void mtbl_reader_destroy(struct mtbl_reader **'r');^ [verse] ^const struct mtbl_source * mtbl_reader_source(struct mtbl_reader *'r');^ Reader options: [verse] ^struct mtbl_reader_options * mtbl_reader_options_init(void);^ [verse] ^void mtbl_reader_options_destroy(struct mtbl_reader_options **'ropt');^ [verse] ^void mtbl_reader_options_set_verify_checksums( struct mtbl_reader_options *'ropt', bool 'verify_checksums');^ == DESCRIPTION == MTBL files are accessed by creating an ^mtbl_reader^ object, calling ^mtbl_reader_source^() to obtain an ^mtbl_source^ handle, and using the ^mtbl_source^(3) interface to read entries. ^mtbl_reader^ objects may be created by calling ^mtbl_reader_init^() with an _fname_ argument specifying the filename to be opened, or ^mtbl_reader_init_fd^() may be called with an _fd_ argument specifying an open, readable file descriptor. Since MTBL files are immutable, the same MTBL file may be opened and read from concurrently by independent threads or processes. If the _ropt_ parameter to ^mtbl_reader_init^() or ^mtbl_reader_init_fd^() is non-NULL, the parameters specified in the ^mtbl_reader_options^ object will be configured into the ^mtbl_reader^ object. === Reader options === ==== verify_checksums ==== Specifies whether or not the CRC32C checksum on each data block should be verified or not. If _verify_checksums_ is enabled, a checksum mismatch will cause a runtime error. Note that the checksum on the index block is always verified, since the overhead of doing this once when the reader object is instantiated is minimal. The default is to not verify data block checksums. == RETURN VALUE == ^mtbl_reader_init^() and ^mtbl_reader_init_fd^() return NULL on failure, and non-NULL on success. mtbl-0.5/man/mtbl_sorter.3000066400000000000000000000136461227302760400155240ustar00rootroot00000000000000'\" t .\" Title: mtbl_sorter .\" Author: [FIXME: author] [see http://docbook.sf.net/el/author] .\" Generator: DocBook XSL Stylesheets v1.78.1 .\" Date: 01/31/2014 .\" Manual: \ \& .\" Source: \ \& .\" Language: English .\" .TH "MTBL_SORTER" "3" "01/31/2014" "\ \&" "\ \&" .\" ----------------------------------------------------------------- .\" * Define some portability stuff .\" ----------------------------------------------------------------- .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .\" http://bugs.debian.org/507673 .\" http://lists.gnu.org/archive/html/groff/2009-02/msg00013.html .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .ie \n(.g .ds Aq \(aq .el .ds Aq ' .\" ----------------------------------------------------------------- .\" * set default formatting .\" ----------------------------------------------------------------- .\" disable hyphenation .nh .\" disable justification (adjust text to left margin only) .ad l .\" ----------------------------------------------------------------- .\" * MAIN CONTENT STARTS HERE * .\" ----------------------------------------------------------------- .SH "NAME" mtbl_sorter \- sort a sequence of unordered key\-value pairs .SH "SYNOPSIS" .sp \fB#include \fR .sp Sorter objects: .sp .nf \fBstruct mtbl_sorter * mtbl_sorter_init(const struct mtbl_sorter_options *\fR\fB\fIsopt\fR\fR\fB);\fR .fi .sp .nf \fBvoid mtbl_sorter_destroy(struct mtbl_sorter **\fR\fB\fIs\fR\fR\fB);\fR .fi .sp .nf \fBmtbl_res mtbl_sorter_add(struct mtbl_sorter *\fR\fB\fIs\fR\fR\fB, const uint8_t *\fR\fB\fIkey\fR\fR\fB, size_t \fR\fB\fIlen_key\fR\fR\fB, const uint8_t *\fR\fB\fIval\fR\fR\fB, size_t \fR\fB\fIlen_val\fR\fR\fB);\fR .fi .sp .nf \fBmtbl_res mtbl_sorter_write(struct mtbl_sorter *\fR\fB\fIs\fR\fR\fB, struct mtbl_writer *\fR\fB\fIw\fR\fR\fB);\fR .fi .sp .nf \fBstruct mtbl_iter * mtbl_sorter_iter(struct mtbl_sorter *\fR\fB\fIs\fR\fR\fB);\fR .fi .sp Sorter options: .sp .nf \fBstruct mtbl_sorter_options * mtbl_sorter_options_init(void);\fR .fi .sp .nf \fBvoid mtbl_sorter_options_destroy(struct mtbl_sorter_options **\fR\fB\fIsopt\fR\fR\fB);\fR .fi .sp .nf \fBvoid mtbl_sorter_options_set_merge_func( struct mtbl_sorter_options *\fR\fB\fIsopt\fR\fR\fB, mtbl_merge_func \fR\fB\fIfp\fR\fR\fB, void *\fR\fB\fIclos\fR\fR\fB);\fR .fi .sp .nf \fBvoid mtbl_sorter_options_set_temp_dir( struct mtbl_sorter_options *\fR\fB\fIsopt\fR\fR\fB, const char *\fR\fB\fItemp_dir\fR\fR\fB);\fR .fi .sp .nf \fBvoid mtbl_sorter_options_set_max_memory( struct mtbl_sorter_options *\fR\fB\fIsopt\fR\fR\fB, size_t \fR\fB\fImax_memory\fR\fR\fB);\fR .fi .SH "DESCRIPTION" .sp The \fBmtbl_sorter\fR interface accepts a sequence of key\-value pairs with keys in arbitrary order and provides these entries in sorted order\&. The sorted entries may be consumed via the \fBmtbl_iter\fR interface using the \fBmtbl_sorter_iter\fR() function, or they may be dumped to an \fBmtbl_writer\fR object using the \fBmtbl_sorter_write\fR() function\&. The \fBmtbl_sorter\fR implementation buffers entries in memory up to a configurable limit before sorting them and writing them to disk in chunks\&. When the caller has finishing adding entries and requests the sorted output, entries from these sorted chunks are then read back and merged\&. (Thus, \fBmtbl_sorter\fR(3) is an "external sorting" implementation\&.) .sp Because the MTBL format does not allow duplicate keys, the caller must provide a function which will accept a key and two conflicting values for that key and return a replacement value\&. This function may be called multiple times for the same key if the same key is inserted more than twice\&. See \fBmtbl_merger\fR(3) for further details about the merge function\&. .sp \fBmtbl_sorter\fR objects are created with the \fBmtbl_sorter_init\fR() function, which requires a non\-NULL \fIsopt\fR argument which has been configured with a merge function \fIfp\fR\&. .sp \fBmtbl_sorter_add\fR() copies key\-value pairs from the caller into the \fBmtbl_sorter\fR object\&. Keys are specified as a pointer to a buffer, \fIkey\fR, and the length of that buffer, \fIlen_key\fR\&. Values are specified as a pointer to a buffer, \fIval\fR, and the length of that buffer, \fIlen_val\fR\&. .sp Once the caller has finished adding entries to the \fBmtbl_sorter\fR object, either \fBmtbl_sorter_write\fR() or \fBmtbl_sorter_iter\fR() should be called in order to consume the sorted output\&. It is a runtime error to call \fBmtbl_sorter_add\fR() on an \fBmtbl_sorter\fR object after iteration has begun, and once the sorted output has been consumed, it is also a runtime error to call any other function but \fBmtbl_sorter_destroy\fR() on the depleted \fBmtbl_sorter\fR object\&. .SS "Sorter options" .sp .it 1 an-trap .nr an-no-space-flag 1 .nr an-break-flag 1 .br .ps +1 \fBtemp_dir\fR .RS 4 .sp Specifies the temporary directory to use\&. Defaults to /var/tmp\&. .RE .sp .it 1 an-trap .nr an-no-space-flag 1 .nr an-break-flag 1 .br .ps +1 \fBmax_memory\fR .RS 4 .sp Specifies the maximum amount of memory to use for in\-memory sorting, in bytes\&. Defaults to 1 Gigabyte\&. This specifies a limit on the total number of bytes allocated for key\-value entries and does not include any allocation overhead\&. .RE .sp .it 1 an-trap .nr an-no-space-flag 1 .nr an-break-flag 1 .br .ps +1 \fBmerge_func\fR .RS 4 .sp See \fBmtbl_merger\fR(3)\&. An \fBmtbl_merger\fR object is used internally for the external sort\&. .RE .SH "RETURN VALUE" .sp If the merge function callback is unable to provide a merged value (that is, it fails to return a non\-NULL value in its \fImerged_val\fR argument), the sort process will be aborted, and \fBmtbl_sorter_write\fR() or \fBmtbl_iter_next\fR() will return \fBmtbl_res_failure\fR\&. .sp \fBmtbl_sorter_write\fR() returns \fBmtbl_res_success\fR if the sorted output was successfully written, and \fBmtbl_res_failure\fR otherwise\&. mtbl-0.5/man/mtbl_sorter.3.txt000066400000000000000000000077151227302760400163420ustar00rootroot00000000000000= mtbl_sorter(3) = == NAME == mtbl_sorter - sort a sequence of unordered key-value pairs == SYNOPSIS == ^#include ^ Sorter objects: [verse] ^struct mtbl_sorter * mtbl_sorter_init(const struct mtbl_sorter_options *'sopt');^ [verse] ^void mtbl_sorter_destroy(struct mtbl_sorter **'s');^ [verse] ^mtbl_res mtbl_sorter_add(struct mtbl_sorter *'s', const uint8_t *'key', size_t 'len_key', const uint8_t *'val', size_t 'len_val');^ [verse] ^mtbl_res mtbl_sorter_write(struct mtbl_sorter *'s', struct mtbl_writer *'w');^ [verse] ^struct mtbl_iter * mtbl_sorter_iter(struct mtbl_sorter *'s');^ Sorter options: [verse] ^struct mtbl_sorter_options * mtbl_sorter_options_init(void);^ [verse] ^void mtbl_sorter_options_destroy(struct mtbl_sorter_options **'sopt');^ [verse] ^void mtbl_sorter_options_set_merge_func( struct mtbl_sorter_options *'sopt', mtbl_merge_func 'fp', void *'clos');^ [verse] ^void mtbl_sorter_options_set_temp_dir( struct mtbl_sorter_options *'sopt', const char *'temp_dir');^ [verse] ^void mtbl_sorter_options_set_max_memory( struct mtbl_sorter_options *'sopt', size_t 'max_memory');^ == DESCRIPTION == The ^mtbl_sorter^ interface accepts a sequence of key-value pairs with keys in arbitrary order and provides these entries in sorted order. The sorted entries may be consumed via the ^mtbl_iter^ interface using the ^mtbl_sorter_iter^() function, or they may be dumped to an ^mtbl_writer^ object using the ^mtbl_sorter_write^() function. The ^mtbl_sorter^ implementation buffers entries in memory up to a configurable limit before sorting them and writing them to disk in chunks. When the caller has finishing adding entries and requests the sorted output, entries from these sorted chunks are then read back and merged. (Thus, ^mtbl_sorter^(3) is an "external sorting" implementation.) Because the MTBL format does not allow duplicate keys, the caller must provide a function which will accept a key and two conflicting values for that key and return a replacement value. This function may be called multiple times for the same key if the same key is inserted more than twice. See ^mtbl_merger^(3) for further details about the merge function. ^mtbl_sorter^ objects are created with the ^mtbl_sorter_init^() function, which requires a non-NULL _sopt_ argument which has been configured with a merge function _fp_. ^mtbl_sorter_add^() copies key-value pairs from the caller into the ^mtbl_sorter^ object. Keys are specified as a pointer to a buffer, _key_, and the length of that buffer, _len_key_. Values are specified as a pointer to a buffer, _val_, and the length of that buffer, _len_val_. Once the caller has finished adding entries to the ^mtbl_sorter^ object, either ^mtbl_sorter_write^() or ^mtbl_sorter_iter^() should be called in order to consume the sorted output. It is a runtime error to call ^mtbl_sorter_add^() on an ^mtbl_sorter^ object after iteration has begun, and once the sorted output has been consumed, it is also a runtime error to call any other function but ^mtbl_sorter_destroy^() on the depleted ^mtbl_sorter^ object. === Sorter options === ==== temp_dir ==== Specifies the temporary directory to use. Defaults to /var/tmp. ==== max_memory ==== Specifies the maximum amount of memory to use for in-memory sorting, in bytes. Defaults to 1 Gigabyte. This specifies a limit on the total number of bytes allocated for key-value entries and does not include any allocation overhead. ==== merge_func ==== See ^mtbl_merger^(3). An ^mtbl_merger^ object is used internally for the external sort. == RETURN VALUE == If the merge function callback is unable to provide a merged value (that is, it fails to return a non-NULL value in its _merged_val_ argument), the sort process will be aborted, and ^mtbl_sorter_write^() or ^mtbl_iter_next^() will return ^mtbl_res_failure^. ^mtbl_sorter_write^() returns ^mtbl_res_success^ if the sorted output was successfully written, and ^mtbl_res_failure^ otherwise. mtbl-0.5/man/mtbl_source.3000066400000000000000000000071711227302760400155020ustar00rootroot00000000000000'\" t .\" Title: mtbl_source .\" Author: [FIXME: author] [see http://docbook.sf.net/el/author] .\" Generator: DocBook XSL Stylesheets v1.78.1 .\" Date: 01/31/2014 .\" Manual: \ \& .\" Source: \ \& .\" Language: English .\" .TH "MTBL_SOURCE" "3" "01/31/2014" "\ \&" "\ \&" .\" ----------------------------------------------------------------- .\" * Define some portability stuff .\" ----------------------------------------------------------------- .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .\" http://bugs.debian.org/507673 .\" http://lists.gnu.org/archive/html/groff/2009-02/msg00013.html .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .ie \n(.g .ds Aq \(aq .el .ds Aq ' .\" ----------------------------------------------------------------- .\" * set default formatting .\" ----------------------------------------------------------------- .\" disable hyphenation .nh .\" disable justification (adjust text to left margin only) .ad l .\" ----------------------------------------------------------------- .\" * MAIN CONTENT STARTS HERE * .\" ----------------------------------------------------------------- .SH "NAME" mtbl_source \- obtain key\-value entries from a data source .SH "SYNOPSIS" .sp \fB#include \fR .sp .nf \fBstruct mtbl_iter * mtbl_source_iter(const struct mtbl_source *\fR\fB\fIs\fR\fR\fB);\fR .fi .sp .nf \fBstruct mtbl_iter * mtbl_source_get(const struct mtbl_source *\fR\fB\fIs\fR\fR\fB, const uint8_t *\fR\fB\fIkey\fR\fR\fB, size_t \fR\fB\fIlen_key\fR\fR\fB);\fR .fi .sp .nf \fBstruct mtbl_iter * mtbl_source_get_prefix( const struct mtbl_source *\fR\fB\fIs\fR\fR\fB, const uint8_t *\fR\fB\fIprefix\fR\fR\fB, size_t \fR\fB\fIlen_prefix\fR\fR\fB);\fR .fi .sp .nf \fBstruct mtbl_iter * mtbl_source_get_range( const struct mtbl_source *\fR\fB\fIs\fR\fR\fB, const uint8_t *\fR\fB\fIkey0\fR\fR\fB, size_t \fR\fB\fIlen_key0\fR\fR\fB, const uint8_t *\fR\fB\fIkey1\fR\fR\fB, size_t \fR\fB\fIlen_key1\fR\fR\fB);\fR .fi .sp .nf \fBmtbl_res mtbl_source_write(const struct mtbl_source *\fR\fB\fIs\fR\fR\fB, struct mtbl_writer *\fR\fB\fIw\fR\fR\fB);\fR .fi .sp .nf \fBvoid mtbl_source_destroy(struct mtbl_source **\fR\fB\fIs\fR\fR\fB);\fR .fi .SH "DESCRIPTION" .sp The \fBmtbl_source\fR iterface provides an abstraction for reading key\-value entries from mtbl data sources\&. .sp \fBmtbl_source_iter\fR() provides an iterator over all of the entries in the data source\&. .sp \fBmtbl_source_get\fR() provides an exact match iterator which returns all entries whose key matches the key provided in the arguments \fIkey\fR and \fIlen_key\fR\&. .sp \fBmtbl_source_get_prefix\fR() provides a prefix iterator which returns all entries whose keys start with \fIprefix\fR and are at least \fIlen_prefix\fR bytes long\&. .sp \fBmtbl_source_get_range\fR() provides a range iterator which returns all entries whose keys are between \fIkey0\fR and \fIkey1\fR inclusive\&. .sp \fBmtbl_source_write\fR() is a convenience function for reading all of the entries from a source and writing them to an \fBmtbl_writer\fR object\&. It is equivalent to calling \fBmtbl_writer_add\fR() on all of the entries returned from \fBmtbl_source_iter\fR()\&. .SH "RETURN VALUE" .sp \fBmtbl_source_iter\fR(), \fBmtbl_source_get\fR(), \fBmtbl_source_get_prefix\fR(), and \fBmtbl_source_get_range\fR() return \fBmtbl_iter\fR objects\&. .sp \fBmtbl_source_write\fR() returns \fBmtbl_res_success\fR if all of the entries in the data source were successfully written to the \fBmtbl_writer\fR argument, and \fBmtbl_res_failure\fR otherwise\&. .SH "SEE ALSO" .sp \fBmtbl_iter\fR(3) mtbl-0.5/man/mtbl_source.3.txt000066400000000000000000000040451227302760400163150ustar00rootroot00000000000000= mtbl_source(3) = == NAME == mtbl_source - obtain key-value entries from a data source == SYNOPSIS == ^#include ^ [verse] ^struct mtbl_iter * mtbl_source_iter(const struct mtbl_source *'s');^ [verse] ^struct mtbl_iter * mtbl_source_get(const struct mtbl_source *'s', const uint8_t *'key', size_t 'len_key');^ [verse] ^struct mtbl_iter * mtbl_source_get_prefix( const struct mtbl_source *'s', const uint8_t *'prefix', size_t 'len_prefix');^ [verse] ^struct mtbl_iter * mtbl_source_get_range( const struct mtbl_source *'s', const uint8_t *'key0', size_t 'len_key0', const uint8_t *'key1', size_t 'len_key1');^ [verse] ^mtbl_res mtbl_source_write(const struct mtbl_source *'s', struct mtbl_writer *'w');^ [verse] ^void mtbl_source_destroy(struct mtbl_source **'s');^ == DESCRIPTION == The ^mtbl_source^ iterface provides an abstraction for reading key-value entries from mtbl data sources. ^mtbl_source_iter^() provides an iterator over all of the entries in the data source. ^mtbl_source_get^() provides an exact match iterator which returns all entries whose key matches the key provided in the arguments _key_ and _len_key_. ^mtbl_source_get_prefix^() provides a prefix iterator which returns all entries whose keys start with _prefix_ and are at least _len_prefix_ bytes long. ^mtbl_source_get_range^() provides a range iterator which returns all entries whose keys are between _key0_ and _key1_ inclusive. ^mtbl_source_write^() is a convenience function for reading all of the entries from a source and writing them to an ^mtbl_writer^ object. It is equivalent to calling ^mtbl_writer_add^() on all of the entries returned from ^mtbl_source_iter^(). == RETURN VALUE == ^mtbl_source_iter^(), ^mtbl_source_get^(), ^mtbl_source_get_prefix^(), and ^mtbl_source_get_range^() return ^mtbl_iter^ objects. ^mtbl_source_write^() returns ^mtbl_res_success^ if all of the entries in the data source were successfully written to the ^mtbl_writer^ argument, and ^mtbl_res_failure^ otherwise. == SEE ALSO == link:mtbl_iter[3] mtbl-0.5/man/mtbl_varint.3000066400000000000000000000062531227302760400155050ustar00rootroot00000000000000'\" t .\" Title: mtbl_varint .\" Author: [FIXME: author] [see http://docbook.sf.net/el/author] .\" Generator: DocBook XSL Stylesheets v1.78.1 .\" Date: 01/31/2014 .\" Manual: \ \& .\" Source: \ \& .\" Language: English .\" .TH "MTBL_VARINT" "3" "01/31/2014" "\ \&" "\ \&" .\" ----------------------------------------------------------------- .\" * Define some portability stuff .\" ----------------------------------------------------------------- .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .\" http://bugs.debian.org/507673 .\" http://lists.gnu.org/archive/html/groff/2009-02/msg00013.html .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .ie \n(.g .ds Aq \(aq .el .ds Aq ' .\" ----------------------------------------------------------------- .\" * set default formatting .\" ----------------------------------------------------------------- .\" disable hyphenation .nh .\" disable justification (adjust text to left margin only) .ad l .\" ----------------------------------------------------------------- .\" * MAIN CONTENT STARTS HERE * .\" ----------------------------------------------------------------- .SH "NAME" mtbl_varint \- Variable\-width encoding and decoding of 32 and 64 bit integers .SH "SYNOPSIS" .sp \fB#include \fR .sp \fBunsigned mtbl_varint_length(uint64_t \fR\fB\fIvalue\fR\fR\fB);\fR .sp \fBunsigned mtbl_varint_length_packed(const uint8_t *\fR\fB\fIbuf\fR\fR\fB, size_t \fR\fB\fIlen_buf\fR\fR\fB);\fR .sp \fBsize_t mtbl_varint_encode32(uint8_t *\fR\fB\fIptr\fR\fR\fB, uint32_t \fR\fB\fIvalue\fR\fR\fB);\fR .sp \fBsize_t mtbl_varint_encode64(uint8_t *\fR\fB\fIptr\fR\fR\fB, uint64_t \fR\fB\fIvalue\fR\fR\fB);\fR .sp \fBsize_t mtbl_varint_decode32(const uint8_t *\fR\fB\fIptr\fR\fR\fB, uint32_t *\fR\fB\fIvalue\fR\fR\fB);\fR .sp \fBsize_t mtbl_varint_decode64(const uint8_t *\fR\fB\fIptr\fR\fR\fB, uint64_t *\fR\fB\fIvalue\fR\fR\fB);\fR .SH "DESCRIPTION" .sp \fBmtbl_varint_encode32\fR() and \fBmtbl_varint_encode64\fR() write the 32 or 64 bit quantity, respectively, in the argument \fIvalue\fR to the buffer in the argument \fIdst\fR\&. The quantity will be written in using a variable\-width encoding that uses at most 5 bytes for a 32 bit quantity or 10 bytes for a 64 bit quantity\&. .sp \fBmtbl_varint_decode32\fR() and \fBmtbl_varint_decode64\fR() read the 32 or 64 bit varint quantity, respectively, in the argument \fIptr\fR\&. The quantity read will be placed in the argument \fIvalue\fR\&. .sp Bounds checking must be performed by the caller\&. .SH "RETURN VALUE" .sp \fBmtbl_varint_encode32\fR() and \fBmtbl_varint_encode64\fR() return the number of bytes written to \fIdst\fR\&. .sp \fBmtbl_varint_decode32\fR() and \fBmtbl_varint_decode64\fR() return the number of bytes read from \fIptr\fR\&. .sp \fBmtbl_varint_length\fR() returns the number of bytes that its argument \fIvalue\fR would require in the variable\-width encoding\&. .sp \fBmtbl_varint_length_packed\fR() returns the number of bytes consumed by the variable\-width encoded quantity at its argument \fIdata\fR\&. It will read at most \fIlen_buf\fR bytes from \fIdata\fR\&. The value 0 is returned if a valid varint is not present\&. mtbl-0.5/man/mtbl_varint.3.txt000066400000000000000000000032471227302760400163230ustar00rootroot00000000000000= mtbl_varint(3) = == NAME == mtbl_varint - Variable-width encoding and decoding of 32 and 64 bit integers == SYNOPSIS == ^#include ^ ^unsigned mtbl_varint_length(uint64_t 'value');^ ^unsigned mtbl_varint_length_packed(const uint8_t *'buf', size_t 'len_buf');^ ^size_t mtbl_varint_encode32(uint8_t *'ptr', uint32_t 'value');^ ^size_t mtbl_varint_encode64(uint8_t *'ptr', uint64_t 'value');^ ^size_t mtbl_varint_decode32(const uint8_t *'ptr', uint32_t *'value');^ ^size_t mtbl_varint_decode64(const uint8_t *'ptr', uint64_t *'value');^ == DESCRIPTION == ^mtbl_varint_encode32^() and ^mtbl_varint_encode64^() write the 32 or 64 bit quantity, respectively, in the argument _value_ to the buffer in the argument _dst_. The quantity will be written in using a variable-width encoding that uses at most 5 bytes for a 32 bit quantity or 10 bytes for a 64 bit quantity. ^mtbl_varint_decode32^() and ^mtbl_varint_decode64^() read the 32 or 64 bit varint quantity, respectively, in the argument _ptr_. The quantity read will be placed in the argument _value_. Bounds checking must be performed by the caller. == RETURN VALUE == ^mtbl_varint_encode32^() and ^mtbl_varint_encode64^() return the number of bytes written to _dst_. ^mtbl_varint_decode32^() and ^mtbl_varint_decode64^() return the number of bytes read from _ptr_. ^mtbl_varint_length^() returns the number of bytes that its argument _value_ would require in the variable-width encoding. ^mtbl_varint_length_packed^() returns the number of bytes consumed by the variable-width encoded quantity at its argument _data_. It will read at most _len_buf_ bytes from _data_. The value 0 is returned if a valid varint is not present. mtbl-0.5/man/mtbl_writer.3000066400000000000000000000120671227302760400155160ustar00rootroot00000000000000'\" t .\" Title: mtbl_writer .\" Author: [FIXME: author] [see http://docbook.sf.net/el/author] .\" Generator: DocBook XSL Stylesheets v1.78.1 .\" Date: 01/31/2014 .\" Manual: \ \& .\" Source: \ \& .\" Language: English .\" .TH "MTBL_WRITER" "3" "01/31/2014" "\ \&" "\ \&" .\" ----------------------------------------------------------------- .\" * Define some portability stuff .\" ----------------------------------------------------------------- .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .\" http://bugs.debian.org/507673 .\" http://lists.gnu.org/archive/html/groff/2009-02/msg00013.html .\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .ie \n(.g .ds Aq \(aq .el .ds Aq ' .\" ----------------------------------------------------------------- .\" * set default formatting .\" ----------------------------------------------------------------- .\" disable hyphenation .nh .\" disable justification (adjust text to left margin only) .ad l .\" ----------------------------------------------------------------- .\" * MAIN CONTENT STARTS HERE * .\" ----------------------------------------------------------------- .SH "NAME" mtbl_writer \- create an MTBL file .SH "SYNOPSIS" .sp \fB#include \fR .sp Writer objects: .sp .nf \fBstruct mtbl_writer * mtbl_writer_init(const char *\fR\fB\fIfname\fR\fR\fB, const struct mtbl_writer_options *\fR\fB\fIwopt\fR\fR\fB);\fR .fi .sp .nf \fBstruct mtbl_writer * mtbl_writer_init_fd(int \fR\fB\fIfd\fR\fR\fB, const struct mtbl_writer_options *\fR\fB\fIwopt\fR\fR\fB);\fR .fi .sp .nf \fBvoid mtbl_writer_destroy(struct mtbl_writer **\fR\fB\fIw\fR\fR\fB);\fR .fi .sp .nf \fBmtbl_res mtbl_writer_add(struct mtbl_writer *\fR\fB\fIw\fR\fR\fB, const uint8_t *\fR\fB\fIkey\fR\fR\fB, size_t \fR\fB\fIlen_key\fR\fR\fB, const uint8_t *\fR\fB\fIval\fR\fR\fB, size_t \fR\fB\fIlen_val\fR\fR\fB);\fR .fi .sp Writer options: .sp .nf \fBstruct mtbl_writer_options * mtbl_writer_options_init(void);\fR .fi .sp .nf \fBvoid mtbl_writer_options_destroy(struct mtbl_writer_options **\fR\fB\fIwopt\fR\fR\fB);\fR .fi .sp .nf \fBvoid mtbl_writer_options_set_compression( struct mtbl_writer_options *\fR\fB\fIwopt\fR\fR\fB, mtbl_compression_type \fR\fB\fIcompression_type\fR\fR\fB);\fR .fi .sp .nf \fBvoid mtbl_writer_options_set_block_size( struct mtbl_writer_options *\fR\fB\fIwopt\fR\fR\fB, size_t \fR\fB\fIblock_size\fR\fR\fB);\fR .fi .sp .nf \fBvoid mtbl_writer_options_set_block_restart_interval( struct mtbl_writer_options *\fR\fB\fIwopt\fR\fR\fB, size_t \fR\fB\fIblock_restart_interval\fR\fR\fB);\fR .fi .SH "DESCRIPTION" .sp MTBL files are written to disk by creating an \fBmtbl_writer\fR object, calling \fBmtbl_writer_add\fR() for each key\-value entry, and then calling \fBmtbl_writer_destroy\fR()\&. .sp \fBmtbl_writer_add\fR() copies key\-value pairs from the caller into the \fBmtbl_writer\fR object\&. Keys are specified as a pointer to a buffer, \fIkey\fR, and the length of that buffer, \fIlen_key\fR\&. Values are specified as a pointer to a buffer, \fIval\fR, and the length of that buffer, \fIlen_val\fR\&. .sp Keys must be in sorted, lexicographical byte order\&. The same key may not be added to an \fBmtbl_writer\fR more than once\&. If the input entries are not sorted or may contain duplicate keys, then the \fBmtbl_sorter\fR(3) interface should be used instead\&. .sp \fBmtbl_writer\fR objects may be created by calling \fBmtbl_writer_init\fR() with an \fIfname\fR argument specifying a filename to be created\&. The filename must not already exist on the filesystem\&. Or, \fBmtbl_writer_init_fd\fR() may be called with an \fIfd\fR argument specifying an open, writable file descriptor\&. No data may have been written to the file descriptor\&. .sp If the \fIwopt\fR parameter to \fBmtbl_writer_init\fR() or \fBmtbl_writer_init_fd\fR() is non\-NULL, the parameters specified in the \fBmtbl_writer_options\fR object will be configured into the \fBmtbl_writer\fR object\&. .SS "Writer options" .sp .it 1 an-trap .nr an-no-space-flag 1 .nr an-break-flag 1 .br .ps +1 \fBcompression\fR .RS 4 .sp Specifies the compression algorithm to use on data blocks\&. Possible values are \fBMTBL_COMPRESSION_NONE\fR, \fBMTBL_COMPRESSION_SNAPPY\fR, or \fBMTBL_COMPRESSION_ZLIB\fR (the default)\&. .RE .sp .it 1 an-trap .nr an-no-space-flag 1 .nr an-break-flag 1 .br .ps +1 \fBblock_size\fR .RS 4 .sp The maximum size of uncompressed data blocks, specified in bytes\&. The default is 8 kilobytes\&. .RE .sp .it 1 an-trap .nr an-no-space-flag 1 .nr an-break-flag 1 .br .ps +1 \fBblock_restart_interval\fR .RS 4 .sp How frequently to restart intra\-block key prefix compression\&. The default is every 16 keys\&. .RE .SH "RETURN VALUE" .sp \fBmtbl_writer_init\fR() and \fBmtbl_writer_init_fd\fR() return NULL on failure, and non\-NULL on success\&. .sp \fBmtbl_writer_add\fR() returns \fBmtbl_res_success\fR if the key\-value entry was successfully copied into the \fBmtbl_writer\fR object, and \fBmtbl_res_failure\fR if not, for instance if there has been a key\-ordering violation\&. mtbl-0.5/man/mtbl_writer.3.txt000066400000000000000000000062131227302760400163300ustar00rootroot00000000000000= mtbl_writer(3) = == NAME == mtbl_writer - create an MTBL file == SYNOPSIS == ^#include ^ Writer objects: [verse] ^struct mtbl_writer * mtbl_writer_init(const char *'fname', const struct mtbl_writer_options *'wopt');^ [verse] ^struct mtbl_writer * mtbl_writer_init_fd(int 'fd', const struct mtbl_writer_options *'wopt');^ [verse] ^void mtbl_writer_destroy(struct mtbl_writer **'w');^ [verse] ^mtbl_res mtbl_writer_add(struct mtbl_writer *'w', const uint8_t *'key', size_t 'len_key', const uint8_t *'val', size_t 'len_val');^ Writer options: [verse] ^struct mtbl_writer_options * mtbl_writer_options_init(void);^ [verse] ^void mtbl_writer_options_destroy(struct mtbl_writer_options **'wopt');^ [verse] ^void mtbl_writer_options_set_compression( struct mtbl_writer_options *'wopt', mtbl_compression_type 'compression_type');^ [verse] ^void mtbl_writer_options_set_block_size( struct mtbl_writer_options *'wopt', size_t 'block_size');^ [verse] ^void mtbl_writer_options_set_block_restart_interval( struct mtbl_writer_options *'wopt', size_t 'block_restart_interval');^ == DESCRIPTION == MTBL files are written to disk by creating an ^mtbl_writer^ object, calling ^mtbl_writer_add^() for each key-value entry, and then calling ^mtbl_writer_destroy^(). ^mtbl_writer_add^() copies key-value pairs from the caller into the ^mtbl_writer^ object. Keys are specified as a pointer to a buffer, _key_, and the length of that buffer, _len_key_. Values are specified as a pointer to a buffer, _val_, and the length of that buffer, _len_val_. Keys must be in sorted, lexicographical byte order. The same key may not be added to an ^mtbl_writer^ more than once. If the input entries are not sorted or may contain duplicate keys, then the ^mtbl_sorter^(3) interface should be used instead. ^mtbl_writer^ objects may be created by calling ^mtbl_writer_init^() with an _fname_ argument specifying a filename to be created. The filename must not already exist on the filesystem. Or, ^mtbl_writer_init_fd^() may be called with an _fd_ argument specifying an open, writable file descriptor. No data may have been written to the file descriptor. If the _wopt_ parameter to ^mtbl_writer_init^() or ^mtbl_writer_init_fd^() is non-NULL, the parameters specified in the ^mtbl_writer_options^ object will be configured into the ^mtbl_writer^ object. === Writer options === ==== compression ==== Specifies the compression algorithm to use on data blocks. Possible values are ^MTBL_COMPRESSION_NONE^, ^MTBL_COMPRESSION_SNAPPY^, or ^MTBL_COMPRESSION_ZLIB^ (the default). ==== block_size ==== The maximum size of uncompressed data blocks, specified in bytes. The default is 8 kilobytes. ==== block_restart_interval ==== How frequently to restart intra-block key prefix compression. The default is every 16 keys. == RETURN VALUE == ^mtbl_writer_init^() and ^mtbl_writer_init_fd^() return NULL on failure, and non-NULL on success. ^mtbl_writer_add^() returns ^mtbl_res_success^ if the key-value entry was successfully copied into the ^mtbl_writer^ object, and ^mtbl_res_failure^ if not, for instance if there has been a key-ordering violation. mtbl-0.5/mtbl/000077500000000000000000000000001227302760400132555ustar00rootroot00000000000000mtbl-0.5/mtbl/.gitignore000066400000000000000000000000131227302760400152370ustar00rootroot00000000000000libmtbl.pc mtbl-0.5/mtbl/block.c000066400000000000000000000204471227302760400145220ustar00rootroot00000000000000/* * Copyright (c) 2012, 2014 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // Copyright (c) 2011 The LevelDB Authors. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "mtbl-private.h" #include "libmy/ubuf.h" struct block { uint8_t *data; size_t size; uint32_t restart_offset; bool needs_free; }; struct block_iter { struct block *block; uint8_t *data; uint32_t restarts; uint32_t num_restarts; uint32_t current; uint32_t restart_index; uint8_t *next; ubuf *key; uint8_t *val; uint32_t val_len; }; static inline uint32_t num_restarts(struct block *b) { assert(b->size >= 2*sizeof(uint32_t)); return (mtbl_fixed_decode32(b->data + b->size - sizeof(uint32_t))); } static inline uint8_t * decode_entry(uint8_t *p, uint8_t *limit, uint32_t *shared, uint32_t *non_shared, uint32_t *value_length) { if (limit - p < 3) return (NULL); *shared = p[0]; *non_shared = p[1]; *value_length = p[2]; if ((*shared | *non_shared | *value_length) < 128) { /* fast path */ p += 3; } else { p += mtbl_varint_decode32(p, shared); p += mtbl_varint_decode32(p, non_shared); p += mtbl_varint_decode32(p, value_length); assert (p <= limit); } assert(!((limit - p) < (*non_shared + *value_length))); return (p); } struct block * block_init(uint8_t *data, size_t size, bool needs_free) { struct block *b = my_calloc(1, sizeof(*b)); b->data = data; b->size = size; if (size < sizeof(uint32_t)) { b->size = 0; } else { b->restart_offset = size - (1 + num_restarts(b)) * sizeof(uint32_t); if (b->restart_offset > size - sizeof(uint32_t)) { b->size = 0; } } b->needs_free = needs_free; return (b); } void block_destroy(struct block **b) { if (*b != NULL) { if ((*b)->needs_free) free((*b)->data); free(*b); *b = NULL; } } struct block_iter * block_iter_init(struct block *b) { assert(b->size >= 2 * sizeof(uint32_t)); struct block_iter *bi = my_calloc(1, sizeof(*bi)); bi->block = b; bi->data = b->data; bi->restarts = b->restart_offset; bi->num_restarts = num_restarts(b); bi->current = bi->restarts; bi->restart_index = bi->num_restarts; assert(bi->num_restarts > 0); bi->key = ubuf_init(64); return (bi); } void block_iter_destroy(struct block_iter **bi) { if (*bi != NULL) { ubuf_destroy(&(*bi)->key); free(*bi); *bi = NULL; } } static inline uint32_t next_entry_offset(struct block_iter *bi) { /* return the offset in ->data just past the end of the current entry */ return (bi->next - bi->data); } static inline uint32_t get_restart_point(struct block_iter *bi, uint32_t idx) { assert(idx < bi->num_restarts); return (mtbl_fixed_decode32(bi->data + bi->restarts + idx * sizeof(uint32_t))); } static inline void seek_to_restart_point(struct block_iter *bi, uint32_t idx) { ubuf_reset(bi->key); bi->restart_index = idx; uint32_t offset = get_restart_point(bi, idx); bi->next = bi->data + offset; } static bool parse_next_key(struct block_iter *bi) { bi->current = next_entry_offset(bi); uint8_t *p = bi->data + bi->current; uint8_t *limit = bi->data + bi->restarts; if (p >= limit) { /* no more entries to return, mark as invalid */ bi->current = bi->restarts; bi->restart_index = bi->num_restarts; return (false); } /* decode next entry */ uint32_t shared, non_shared, value_length; p = decode_entry(p, limit, &shared, &non_shared, &value_length); assert(!(p == NULL || ubuf_size(bi->key) < shared)); ubuf_clip(bi->key, shared); ubuf_append(bi->key, p, non_shared); bi->next = p + non_shared + value_length; bi->val = p + non_shared; bi->val_len = value_length; while (bi->restart_index + 1 < bi->num_restarts && get_restart_point(bi, bi->restart_index + 1) < bi->current) { bi->restart_index += 1; } return (true); } bool block_iter_valid(const struct block_iter *bi) { return (bi->current < bi->restarts); } void block_iter_seek_to_first(struct block_iter *bi) { seek_to_restart_point(bi, 0); parse_next_key(bi); } void block_iter_seek_to_last(struct block_iter *bi) { seek_to_restart_point(bi, bi->num_restarts - 1); while (parse_next_key(bi) && next_entry_offset(bi) < bi->restarts) { /* keep skipping */ } } void block_iter_seek(struct block_iter *bi, const uint8_t *target, size_t target_len) { /* binary search in restart array to find the first restart point * with a key >= target */ uint32_t left = 0; uint32_t right = bi->num_restarts - 1; while (left < right) { uint32_t mid = (left + right + 1) / 2; uint32_t region_offset = get_restart_point(bi, mid); uint32_t shared, non_shared, value_length; const uint8_t *key_ptr = decode_entry(bi->data + region_offset, bi->data + bi->restarts, &shared, &non_shared, &value_length); if (key_ptr == NULL || (shared != 0)) { /* corruption */ return; } if (bytes_compare(key_ptr, non_shared, target, target_len) < 0) { /* key at "mid" is smaller than "target", therefore all * keys before "mid" are uninteresting */ left = mid; } else { /* key at "mid" is larger than "target", therefore all * keys at or before "mid" are uninteresting */ right = mid - 1; } } /* linear search within restart block for first key >= target */ seek_to_restart_point(bi, left); for (;;) { if (!parse_next_key(bi)) return; if (bytes_compare(ubuf_data(bi->key), ubuf_size(bi->key), target, target_len) >= 0) { return; } } } bool block_iter_next(struct block_iter *bi) { if (!block_iter_valid(bi)) return (false); parse_next_key(bi); return (block_iter_valid(bi)); } void block_iter_prev(struct block_iter *bi) { assert(block_iter_valid(bi)); const uint32_t original = bi->current; while (get_restart_point(bi, bi->restart_index) >= original) { if (bi->restart_index == 0) { /* no more entries */ bi->current = bi->restarts; bi->restart_index = bi->num_restarts; return; } bi->restart_index -= 1; } seek_to_restart_point(bi, bi->restart_index); do { /* loop until end of current entry hits the start of original entry */ } while (parse_next_key(bi) && next_entry_offset(bi) < original); } bool block_iter_get(struct block_iter *bi, const uint8_t **key, size_t *key_len, const uint8_t **val, size_t *val_len) { if (!block_iter_valid(bi)) return (false); if (key) { *key = ubuf_data(bi->key); *key_len = ubuf_size(bi->key); } if (val) { *val = bi->val; *val_len = bi->val_len; } return (true); } mtbl-0.5/mtbl/block_builder.c000066400000000000000000000110601227302760400162170ustar00rootroot00000000000000/* * Copyright (c) 2012, 2014 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "mtbl-private.h" #include "libmy/ubuf.h" VECTOR_GENERATE(uint32_vec, uint32_t); struct block_builder { size_t block_restart_interval; ubuf *buf; ubuf *last_key; uint32_vec *restarts; bool finished; size_t counter; }; struct block_builder * block_builder_init(size_t block_restart_interval) { struct block_builder *b; b = my_calloc(1, sizeof(*b)); b->block_restart_interval = block_restart_interval; b->buf = ubuf_init(65536); b->last_key = ubuf_init(256); b->restarts = uint32_vec_init(64); uint32_vec_add(b->restarts, 0); return (b); } void block_builder_destroy(struct block_builder **b) { if (*b) { uint32_vec_destroy(&((*b)->restarts)); ubuf_destroy(&((*b)->buf)); ubuf_destroy(&((*b)->last_key)); free((*b)); *b = NULL; } } void block_builder_reset(struct block_builder *b) { ubuf_reset(b->buf); ubuf_reset(b->last_key); uint32_vec_reset(b->restarts); uint32_vec_add(b->restarts, 0); b->counter = 0; b->finished = false; } bool block_builder_empty(struct block_builder *b) { return (ubuf_size(b->buf) == 0); } size_t block_builder_current_size_estimate(struct block_builder *b) { return (ubuf_bytes(b->buf) + uint32_vec_bytes(b->restarts) + sizeof(uint32_t)); } void block_builder_finish(struct block_builder *b, uint8_t **buf, size_t *bufsz) { ubuf_reserve(b->buf, uint32_vec_bytes(b->restarts) + sizeof(uint32_t)); for (size_t i = 0; i < uint32_vec_size(b->restarts); i++) { //fprintf(stderr, "%s: writing restart value: %u\n", __func__, //(unsigned) uint32_vec_value(b->restarts, i)); mtbl_fixed_encode32(ubuf_ptr(b->buf), uint32_vec_value(b->restarts, i)); ubuf_advance(b->buf, sizeof(uint32_t)); } //fprintf(stderr, "%s: writing number of restarts: %u\n", __func__, //(unsigned) uint32_vec_size(b->restarts)); mtbl_fixed_encode32(ubuf_ptr(b->buf), uint32_vec_size(b->restarts)); ubuf_advance(b->buf, sizeof(uint32_t)); b->finished = true; ubuf_detach(b->buf, buf, bufsz); } void block_builder_add(struct block_builder *b, const uint8_t *key, size_t len_key, const uint8_t *val, size_t len_val) { assert(b->counter <= b->block_restart_interval); assert(b->finished == false); /* fprintf(stderr, "----------------------------\n" "%s: writing key= '%s' (%zd) val= '%s' (%zd)\n", __func__, (char *) key, len_key, (char *) val, len_val); */ size_t shared = 0; /* see how much sharing to do with previous key */ if (b->counter < b->block_restart_interval) { const size_t min_length = (ubuf_size(b->last_key) > len_key) ? (len_key) : (ubuf_size(b->last_key)); while ((shared < min_length) && (ubuf_value(b->last_key, shared) == key[shared])) shared++; } else { /* restart compression */ //fprintf(stderr, "%s: doing restart\n", __func__); uint32_vec_add(b->restarts, (uint32_t) ubuf_bytes(b->buf)); b->counter = 0; } const size_t non_shared = len_key - shared; /* ensure enough buffer space is available */ ubuf_reserve(b->buf, 5*3 + non_shared + len_val); /* add "[shared][non-shared][value length]" to buffer */ //fprintf(stderr, "%s: writing value %u (shared)\n", __func__, (unsigned) shared); ubuf_advance(b->buf, mtbl_varint_encode32(ubuf_ptr(b->buf), shared)); //fprintf(stderr, "%s: writing value %u (non-shared)\n", __func__, (unsigned) non_shared); ubuf_advance(b->buf, mtbl_varint_encode32(ubuf_ptr(b->buf), non_shared)); //fprintf(stderr, "%s: writing value %u (value length)\n", __func__, (unsigned) len_val); ubuf_advance(b->buf, mtbl_varint_encode32(ubuf_ptr(b->buf), len_val)); /* add key suffix to buffer followed by value */ //fprintf(stderr, "%s: writing %u bytes (key suffix)\n", __func__, (unsigned) non_shared); memcpy(ubuf_ptr(b->buf), key + shared, non_shared); ubuf_advance(b->buf, non_shared); //fprintf(stderr, "%s: writing %u bytes (value)\n", __func__, (unsigned) len_val); memcpy(ubuf_ptr(b->buf), val, len_val); ubuf_advance(b->buf, len_val); /* update state */ ubuf_reset(b->last_key); ubuf_append(b->last_key, key, len_key); b->counter += 1; } mtbl-0.5/mtbl/bytes.h000066400000000000000000000024061227302760400145560ustar00rootroot00000000000000#include #include #include "libmy/ubuf.h" static inline void bytes_shortest_separator(ubuf *start, const uint8_t *limit, size_t len_limit) { size_t min_length = ubuf_size(start) < len_limit ? ubuf_size(start) : len_limit; size_t diff_index = 0; while ((diff_index < min_length) && (ubuf_data(start)[diff_index] == limit[diff_index])) { diff_index++; } if (diff_index >= min_length) return; uint8_t diff_byte = ubuf_data(start)[diff_index]; if (diff_byte < 0xFF && diff_byte + 1 < limit[diff_index]) { ubuf_data(start)[diff_index]++; ubuf_clip(start, diff_index + 1); } else if (diff_index < min_length - sizeof(uint16_t)) { /* awww yeah, big endian arithmetic on strings */ uint16_t u_start, u_limit, u_between; memcpy(&u_start, &ubuf_data(start)[diff_index], sizeof(u_start)); memcpy(&u_limit, &limit[diff_index], sizeof(u_limit)); u_start = be16toh(u_start); u_limit = be16toh(u_limit); u_between = u_start + 1; if (u_start <= u_between && u_between <= u_limit) { u_between = htobe16(u_between); memcpy(&ubuf_data(start)[diff_index], &u_between, sizeof(u_between)); ubuf_clip(start, diff_index + sizeof(uint16_t)); } } assert(bytes_compare(ubuf_data(start), ubuf_size(start), limit, len_limit) < 0); } mtbl-0.5/mtbl/crc32c_wrap.c000066400000000000000000000013731227302760400155350ustar00rootroot00000000000000/* * Copyright (c) 2012, 2014 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "mtbl-private.h" #include "libmy/crc32c.h" uint32_t mtbl_crc32c(const uint8_t *buf, size_t size) { return (my_crc32c(buf, size)); } mtbl-0.5/mtbl/fileset.c000066400000000000000000000116501227302760400150570ustar00rootroot00000000000000/* * Copyright (c) 2012-2014 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include "mtbl-private.h" #include "libmy/my_fileset.h" struct mtbl_fileset_options { size_t reload_interval; mtbl_merge_func merge; void *merge_clos; }; struct mtbl_fileset { uint32_t reload_interval; size_t n_loaded, n_unloaded; struct timespec last; struct my_fileset *fs; struct mtbl_merger *merger; struct mtbl_merger_options *mopt; struct mtbl_source *source; }; static struct mtbl_iter * fileset_source_iter(void *clos) { struct mtbl_fileset *f = (struct mtbl_fileset *) clos; mtbl_fileset_reload(f); return mtbl_source_iter(mtbl_merger_source(f->merger)); } static struct mtbl_iter * fileset_source_get(void *clos, const uint8_t *key, size_t len_key) { struct mtbl_fileset *f = (struct mtbl_fileset *) clos; mtbl_fileset_reload(f); return mtbl_source_get(mtbl_merger_source(f->merger), key, len_key); } static struct mtbl_iter * fileset_source_get_prefix(void *clos, const uint8_t *key, size_t len_key) { struct mtbl_fileset *f = (struct mtbl_fileset *) clos; mtbl_fileset_reload(f); return mtbl_source_get_prefix(mtbl_merger_source(f->merger), key, len_key); } static struct mtbl_iter * fileset_source_get_range(void *clos, const uint8_t *key0, size_t len_key0, const uint8_t *key1, size_t len_key1) { struct mtbl_fileset *f = (struct mtbl_fileset *) clos; mtbl_fileset_reload(f); return mtbl_source_get_range(mtbl_merger_source(f->merger), key0, len_key0, key1, len_key1); } struct mtbl_fileset_options * mtbl_fileset_options_init(void) { struct mtbl_fileset_options *opt; opt = my_calloc(1, sizeof(*opt)); opt->reload_interval = DEFAULT_FILESET_RELOAD_INTERVAL; return (opt); } void mtbl_fileset_options_destroy(struct mtbl_fileset_options **opt) { if (*opt) { free(*opt); *opt = NULL; } } void mtbl_fileset_options_set_merge_func(struct mtbl_fileset_options *opt, mtbl_merge_func merge, void *clos) { opt->merge = merge; opt->merge_clos = clos; } void mtbl_fileset_options_set_reload_interval(struct mtbl_fileset_options *opt, uint32_t reload_interval) { opt->reload_interval = reload_interval; } static void * fs_load(struct my_fileset *fs, const char *fname) { struct mtbl_fileset *f = (struct mtbl_fileset *) my_fileset_user(fs); f->n_loaded++; return (mtbl_reader_init(fname, NULL)); } static void fs_unload(struct my_fileset *fs, const char *fname, void *ptr) { struct mtbl_fileset *f = (struct mtbl_fileset *) my_fileset_user(fs); struct mtbl_reader *r = (struct mtbl_reader *) ptr; f->n_unloaded++; mtbl_reader_destroy(&r); } struct mtbl_fileset * mtbl_fileset_init(const char *fname, const struct mtbl_fileset_options *opt) { assert(opt != NULL); assert(opt->merge != NULL); struct mtbl_fileset *f = my_calloc(1, sizeof(*f)); f->reload_interval = opt->reload_interval; f->mopt = mtbl_merger_options_init(); mtbl_merger_options_set_merge_func(f->mopt, opt->merge, opt->merge_clos); f->merger = mtbl_merger_init(f->mopt); f->fs = my_fileset_init(fname, fs_load, fs_unload, f); assert(f->fs != NULL); f->source = mtbl_source_init(fileset_source_iter, fileset_source_get, fileset_source_get_prefix, fileset_source_get_range, NULL, f); mtbl_fileset_reload(f); return (f); } void mtbl_fileset_destroy(struct mtbl_fileset **f) { if (*f) { my_fileset_destroy(&(*f)->fs); mtbl_merger_destroy(&(*f)->merger); mtbl_merger_options_destroy(&(*f)->mopt); mtbl_source_destroy(&(*f)->source); free(*f); *f = NULL; } } const struct mtbl_source * mtbl_fileset_source(struct mtbl_fileset *f) { assert(f != NULL); assert(f->source != NULL); return (f->source); } static void fs_reinit_merger(struct mtbl_fileset *f) { const char *fname; struct mtbl_reader *reader; size_t i = 0; if (f->merger) { mtbl_merger_destroy(&f->merger); f->merger = mtbl_merger_init(f->mopt); } while (my_fileset_get(f->fs, i++, &fname, (void **) &reader)) mtbl_merger_add_source(f->merger, mtbl_reader_source(reader)); } void mtbl_fileset_reload(struct mtbl_fileset *f) { assert(f != NULL); struct timespec now; int res; res = clock_gettime(CLOCK_MONOTONIC, &now); assert(res == 0); if (now.tv_sec - f->last.tv_sec > f->reload_interval) { f->n_loaded = 0; f->n_unloaded = 0; assert(f->fs != NULL); my_fileset_reload(f->fs); if (f->n_loaded > 0 || f->n_unloaded > 0) fs_reinit_merger(f); f->last = now; } } mtbl-0.5/mtbl/fixed.c000066400000000000000000000053501227302760400145230ustar00rootroot00000000000000/* * Copyright (c) 2012 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // Copyright (c) 2011 The LevelDB Authors. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "mtbl-private.h" size_t mtbl_fixed_encode32(uint8_t *dst, uint32_t value) { value = htole32(value); memcpy(dst, &value, sizeof(value)); return (sizeof(uint32_t)); } size_t mtbl_fixed_encode64(uint8_t *dst, uint64_t value) { value = htole64(value); memcpy(dst, &value, sizeof(value)); return (sizeof(uint64_t)); } uint32_t mtbl_fixed_decode32(const uint8_t *ptr) { uint32_t result; memcpy(&result, ptr, sizeof(result)); return (le32toh(result)); } uint64_t mtbl_fixed_decode64(const uint8_t *ptr) { uint64_t result; memcpy(&result, ptr, sizeof(result)); return (le64toh(result)); } mtbl-0.5/mtbl/iter.c000066400000000000000000000026511227302760400143700ustar00rootroot00000000000000/* * Copyright (c) 2012 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "mtbl-private.h" struct mtbl_iter { mtbl_iter_next_func iter_next; mtbl_iter_free_func iter_free; void *clos; }; struct mtbl_iter * mtbl_iter_init(mtbl_iter_next_func iter_next, mtbl_iter_free_func iter_free, void *clos) { assert(iter_next != NULL); struct mtbl_iter *it = my_calloc(1, sizeof(*it)); it->iter_next = iter_next; it->iter_free = iter_free; it->clos = clos; return (it); } void mtbl_iter_destroy(struct mtbl_iter **it) { if (*it) { if ((*it)->iter_free != NULL) (*it)->iter_free((*it)->clos); free(*it); *it = NULL; } } mtbl_res mtbl_iter_next(struct mtbl_iter *it, const uint8_t **key, size_t *len_key, const uint8_t **val, size_t *len_val) { if (it == NULL) return (mtbl_res_failure); return (it->iter_next(it->clos, key, len_key, val, len_val)); } mtbl-0.5/mtbl/libmtbl.pc.in000066400000000000000000000003461227302760400156360ustar00rootroot00000000000000prefix=@prefix@ exec_prefix=@exec_prefix@ libdir=@libdir@ includedir=@includedir@ Name: libmtbl Description: immutable sorted string table library Version: @VERSION@ Libs: -L${libdir} -lmtbl Libs.private: Cflags: -I${includedir} mtbl-0.5/mtbl/merger.c000066400000000000000000000211121227302760400146770ustar00rootroot00000000000000/* * Copyright (c) 2012, 2014 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "mtbl-private.h" #include "libmy/heap.h" #include "libmy/ubuf.h" struct entry { struct mtbl_iter *it; ubuf *key; ubuf *val; }; VECTOR_GENERATE(entry_vec, struct entry *); VECTOR_GENERATE(source_vec, const struct mtbl_source *); struct merger_iter { struct mtbl_merger *m; struct heap *h; entry_vec *entries; ubuf *cur_key; ubuf *cur_val; bool finished; bool pending; }; struct mtbl_merger_options { mtbl_merge_func merge; void *merge_clos; }; struct mtbl_merger { source_vec *sources; struct mtbl_source *source; struct mtbl_merger_options opt; }; static struct mtbl_iter * merger_iter(void *); static struct mtbl_iter * merger_get(void *, const uint8_t *, size_t); static struct mtbl_iter * merger_get_prefix(void *, const uint8_t *, size_t); static struct mtbl_iter * merger_get_range(void *, const uint8_t *, size_t, const uint8_t *, size_t); struct mtbl_merger_options * mtbl_merger_options_init(void) { return (my_calloc(1, sizeof(struct mtbl_merger_options))); } void mtbl_merger_options_destroy(struct mtbl_merger_options **opt) { if (*opt) { free(*opt); *opt = NULL; } } void mtbl_merger_options_set_merge_func(struct mtbl_merger_options *opt, mtbl_merge_func merge, void *clos) { opt->merge = merge; opt->merge_clos = clos; } struct mtbl_merger * mtbl_merger_init(const struct mtbl_merger_options *opt) { struct mtbl_merger *m; m = my_calloc(1, sizeof(*m)); m->sources = source_vec_init(0); assert(opt != NULL); assert(opt->merge != NULL); memcpy(&m->opt, opt, sizeof(*opt)); m->source = mtbl_source_init(merger_iter, merger_get, merger_get_prefix, merger_get_range, NULL, m); return (m); } void mtbl_merger_destroy(struct mtbl_merger **m) { if (*m) { source_vec_destroy(&(*m)->sources); mtbl_source_destroy(&(*m)->source); free(*m); *m = NULL; } } const struct mtbl_source * mtbl_merger_source(struct mtbl_merger *m) { return (m->source); } void mtbl_merger_add_source(struct mtbl_merger *m, const struct mtbl_source *s) { source_vec_add(m->sources, s); } static int _mtbl_merger_compare(const void *va, const void *vb) { const struct entry *a = (const struct entry *) va; const struct entry *b = (const struct entry *) vb; if (a->key == NULL && b->key == NULL) return (0); if (a->key == NULL) return (1); if (b->key == NULL) return (-1); return (bytes_compare(ubuf_data(a->key), ubuf_size(a->key), ubuf_data(b->key), ubuf_size(b->key))); } static mtbl_res entry_fill(struct entry *ent) { const uint8_t *key, *val; size_t len_key, len_val; mtbl_res res; ubuf_clip(ent->key, 0); ubuf_clip(ent->val, 0); res = mtbl_iter_next(ent->it, &key, &len_key, &val, &len_val); if (res == mtbl_res_success) { ubuf_append(ent->key, key, len_key); ubuf_append(ent->val, val, len_val); } else { mtbl_iter_destroy(&ent->it); } return (res); } static mtbl_res merger_iter_next(void *v, const uint8_t **out_key, size_t *out_len_key, const uint8_t **out_val, size_t *out_len_val) { struct merger_iter *it = (struct merger_iter *) v; struct entry *e; mtbl_res res; if (it->finished) return (mtbl_res_failure); ubuf_clip(it->cur_key, 0); ubuf_clip(it->cur_val, 0); for (;;) { for (;;) { e = heap_peek(it->h); if (e == NULL) { it->finished = true; break; } if (e->it == NULL) heap_pop(it->h); else break; } if (it->finished) break; if (ubuf_size(it->cur_key) == 0) { ubuf_clip(it->cur_val, 0); ubuf_extend(it->cur_key, e->key); ubuf_extend(it->cur_val, e->val); it->pending = true; res = entry_fill(e); if (res == mtbl_res_success) heap_replace(it->h, e); continue; } if (bytes_compare(ubuf_data(it->cur_key), ubuf_size(it->cur_key), ubuf_data(e->key), ubuf_size(e->key)) == 0) { uint8_t *merged_val = NULL; size_t len_merged_val = 0; it->m->opt.merge(it->m->opt.merge_clos, ubuf_data(it->cur_key), ubuf_size(it->cur_key), ubuf_data(it->cur_val), ubuf_size(it->cur_val), ubuf_data(e->val), ubuf_size(e->val), &merged_val, &len_merged_val); if (merged_val == NULL) return (mtbl_res_failure); ubuf_clip(it->cur_val, 0); ubuf_append(it->cur_val, merged_val, len_merged_val); free(merged_val); res = entry_fill(e); if (res == mtbl_res_success) heap_replace(it->h, e); } else { break; } } if (it->pending) { it->pending = false; *out_key = ubuf_data(it->cur_key); *out_val = ubuf_data(it->cur_val); *out_len_key = ubuf_size(it->cur_key); *out_len_val = ubuf_size(it->cur_val); return (mtbl_res_success); } else { return (mtbl_res_failure); } } static void merger_iter_free(void *v) { struct merger_iter *it = (struct merger_iter *) v; if (it != NULL) { heap_destroy(&it->h); for (size_t i = 0; i < entry_vec_size(it->entries); i++) { struct entry *ent = entry_vec_value(it->entries, i); ubuf_destroy(&ent->key); ubuf_destroy(&ent->val); mtbl_iter_destroy(&ent->it); free(ent); } entry_vec_destroy(&it->entries); ubuf_destroy(&it->cur_key); ubuf_destroy(&it->cur_val); free(it); } } static struct merger_iter * merger_iter_init(struct mtbl_merger *m) { struct merger_iter *it = my_calloc(1, sizeof(*it)); it->m = m; it->h = heap_init(_mtbl_merger_compare); it->entries = entry_vec_init(source_vec_size(m->sources)); it->cur_key = ubuf_init(256); it->cur_val = ubuf_init(256); return (it); } static void merger_iter_add_entry(struct merger_iter *it, struct mtbl_iter *ent_it) { struct entry *ent = my_calloc(1, sizeof(*ent)); ent->key = ubuf_init(256); ent->val = ubuf_init(256); ent->it = ent_it; entry_fill(ent); if (ent->it == NULL) { ubuf_destroy(&ent->key); ubuf_destroy(&ent->val); free(ent); } else { heap_push(it->h, ent); entry_vec_add(it->entries, ent); } } static struct mtbl_iter * merger_iter(void *clos) { struct mtbl_merger *m = (struct mtbl_merger *) clos; struct merger_iter *it = merger_iter_init(m); for (size_t i = 0; i < source_vec_size(m->sources); i++) { const struct mtbl_source *s = source_vec_value(m->sources, i); merger_iter_add_entry(it, mtbl_source_iter(s)); } return (mtbl_iter_init(merger_iter_next, merger_iter_free, it)); } static struct mtbl_iter * merger_get(void *clos, const uint8_t *key, size_t len_key) { struct mtbl_merger *m = (struct mtbl_merger *) clos; struct merger_iter *it = merger_iter_init(m); for (size_t i = 0; i < source_vec_size(m->sources); i++) { const struct mtbl_source *s = source_vec_value(m->sources, i); struct mtbl_iter *s_it = mtbl_source_get_range(s, key, len_key, key, len_key); if (s_it != NULL) merger_iter_add_entry(it, s_it); } if (entry_vec_size(it->entries) == 0) { merger_iter_free(it); return (NULL); } return (mtbl_iter_init(merger_iter_next, merger_iter_free, it)); } static struct mtbl_iter * merger_get_range(void *clos, const uint8_t *key0, size_t len_key0, const uint8_t *key1, size_t len_key1) { struct mtbl_merger *m = (struct mtbl_merger *) clos; struct merger_iter *it = merger_iter_init(m); for (size_t i = 0; i < source_vec_size(m->sources); i++) { const struct mtbl_source *s = source_vec_value(m->sources, i); struct mtbl_iter *s_it = mtbl_source_get_range(s, key0, len_key0, key1, len_key1); if (s_it != NULL) merger_iter_add_entry(it, s_it); } if (entry_vec_size(it->entries) == 0) { merger_iter_free(it); return (NULL); } return (mtbl_iter_init(merger_iter_next, merger_iter_free, it)); } static struct mtbl_iter * merger_get_prefix(void *clos, const uint8_t *key, size_t len_key) { struct mtbl_merger *m = (struct mtbl_merger *) clos; struct merger_iter *it = merger_iter_init(m); for (size_t i = 0; i < source_vec_size(m->sources); i++) { const struct mtbl_source *s = source_vec_value(m->sources, i); struct mtbl_iter *s_it = mtbl_source_get_prefix(s, key, len_key); if (s_it != NULL) merger_iter_add_entry(it, s_it); } if (entry_vec_size(it->entries) == 0) { merger_iter_free(it); return (NULL); } return (mtbl_iter_init(merger_iter_next, merger_iter_free, it)); } mtbl-0.5/mtbl/mtbl-private.h000066400000000000000000000071441227302760400160420ustar00rootroot00000000000000/* * Copyright (c) 2012, 2014 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef MTBL_PRIVATE_H #define MTBL_PRIVATE_H #ifdef HAVE_ENDIAN_H # include #else # ifdef HAVE_SYS_ENDIAN_H # include # endif #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "mtbl.h" #include #include #include "libmy/my_alloc.h" #define MTBL_MAGIC 0x77846676 #define MTBL_TRAILER_SIZE 512 #define DEFAULT_COMPRESSION_TYPE MTBL_COMPRESSION_ZLIB #define DEFAULT_BLOCK_RESTART_INTERVAL 16 #define DEFAULT_BLOCK_SIZE 8192 #define MIN_BLOCK_SIZE 1024 #define DEFAULT_SORTER_TEMP_DIR "/var/tmp" #define DEFAULT_SORTER_MEMORY 1073741824 #define MIN_SORTER_MEMORY 10485760 #define INITIAL_SORTER_VEC_SIZE 131072 #define DEFAULT_FILESET_RELOAD_INTERVAL 60 /* types */ struct block; struct block_builder; struct block_iter; struct trailer; /* block */ struct block *block_init(uint8_t *data, size_t size, bool needs_free); void block_destroy(struct block **); struct block_iter *block_iter_init(struct block *); void block_iter_destroy(struct block_iter **); bool block_iter_valid(const struct block_iter *); void block_iter_seek_to_first(struct block_iter *); void block_iter_seek_to_last(struct block_iter *); void block_iter_seek(struct block_iter *, const uint8_t *key, size_t key_len); bool block_iter_next(struct block_iter *); void block_iter_prev(struct block_iter *); bool block_iter_get(struct block_iter *, const uint8_t **key, size_t *key_len, const uint8_t **val, size_t *val_len); /* block builder */ struct block_builder *block_builder_init(size_t block_restart_interval); size_t block_builder_current_size_estimate(struct block_builder *); void block_builder_destroy(struct block_builder **); void block_builder_finish(struct block_builder *, uint8_t **buf, size_t *bufsz); void block_builder_reset(struct block_builder *); void block_builder_add(struct block_builder *, const uint8_t *key, size_t len_key, const uint8_t *val, size_t len_val); bool block_builder_empty(struct block_builder *); /* trailer */ struct trailer { uint64_t index_block_offset; uint64_t data_block_size; uint64_t compression_algorithm; uint64_t count_entries; uint64_t count_data_blocks; uint64_t bytes_data_blocks; uint64_t bytes_index_block; uint64_t bytes_keys; uint64_t bytes_values; }; void trailer_write(struct trailer *t, uint8_t *buf); bool trailer_read(const uint8_t *buf, struct trailer *t); /* misc */ static inline int bytes_compare(const uint8_t *a, size_t len_a, const uint8_t *b, size_t len_b) { size_t len = len_a > len_b ? len_b : len_a; int ret = memcmp(a, b, len); if (ret == 0) { if (len_a < len_b) { return (-1); } else if (len_a == len_b) { return (0); } else if (len_a > len_b) { return (1); } } return (ret); } #endif /* MTBL_PRIVATE_H */ mtbl-0.5/mtbl/mtbl.h000066400000000000000000000165421227302760400143740ustar00rootroot00000000000000/* * Copyright (c) 2012 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef MTBL_H #define MTBL_H #ifdef __cplusplus extern "C" { #endif #include #include #include typedef enum { MTBL_COMPRESSION_NONE = 0, MTBL_COMPRESSION_SNAPPY = 1, MTBL_COMPRESSION_ZLIB = 2 } mtbl_compression_type; typedef enum { mtbl_res_failure = 0, mtbl_res_success = 1 } mtbl_res; /* exported types */ struct mtbl_iter; struct mtbl_source; struct mtbl_reader; struct mtbl_reader_options; struct mtbl_writer; struct mtbl_writer_options; struct mtbl_merger; struct mtbl_merger_options; struct mtbl_fileset; struct mtbl_fileset_options; struct mtbl_sorter; struct mtbl_sorter_options; typedef void (*mtbl_merge_func)(void *clos, const uint8_t *key, size_t len_key, const uint8_t *val0, size_t len_val0, const uint8_t *val1, size_t len_val1, uint8_t **merged_val, size_t *len_merged_val); typedef void * (*mtbl_merge_init_func)(void); typedef void (*mtbl_merge_free_func)(void *clos); /* iter */ typedef mtbl_res (*mtbl_iter_next_func)( void *, const uint8_t **key, size_t *len_key, const uint8_t **val, size_t *len_val); typedef void (*mtbl_iter_free_func)(void *); struct mtbl_iter * mtbl_iter_init(mtbl_iter_next_func, mtbl_iter_free_func, void *clos); void mtbl_iter_destroy(struct mtbl_iter **); mtbl_res mtbl_iter_next( struct mtbl_iter *, const uint8_t **key, size_t *len_key, const uint8_t **val, size_t *len_val) __attribute__((warn_unused_result)); /* source */ typedef struct mtbl_iter * (*mtbl_source_iter_func)(void *); typedef struct mtbl_iter * (*mtbl_source_get_func)(void *, const uint8_t *key, size_t len_key); typedef struct mtbl_iter * (*mtbl_source_get_prefix_func)(void *, const uint8_t *key, size_t len_key); typedef struct mtbl_iter * (*mtbl_source_get_range_func)( void *, const uint8_t *key0, size_t len_key0, const uint8_t *key1, size_t len_key1); typedef void (*mtbl_source_free_func)(void *); struct mtbl_source * mtbl_source_init( mtbl_source_iter_func, mtbl_source_get_func, mtbl_source_get_prefix_func, mtbl_source_get_range_func, mtbl_source_free_func, void *clos); void mtbl_source_destroy(struct mtbl_source **); struct mtbl_iter * mtbl_source_iter(const struct mtbl_source *); struct mtbl_iter * mtbl_source_get(const struct mtbl_source *, const uint8_t *key, size_t len_key); struct mtbl_iter * mtbl_source_get_prefix(const struct mtbl_source *, const uint8_t *key, size_t len_key); struct mtbl_iter * mtbl_source_get_range( const struct mtbl_source *, const uint8_t *key0, size_t len_key0, const uint8_t *key1, size_t len_key1); mtbl_res mtbl_source_write(const struct mtbl_source *, struct mtbl_writer *) __attribute__((warn_unused_result)); /* writer */ struct mtbl_writer * mtbl_writer_init(const char *fname, const struct mtbl_writer_options *); struct mtbl_writer * mtbl_writer_init_fd(int fd, const struct mtbl_writer_options *); void mtbl_writer_destroy(struct mtbl_writer **); mtbl_res mtbl_writer_add( struct mtbl_writer *, const uint8_t *key, size_t len_key, const uint8_t *val, size_t len_val) __attribute__((warn_unused_result)); /* writer options */ struct mtbl_writer_options * mtbl_writer_options_init(void); void mtbl_writer_options_destroy(struct mtbl_writer_options **); void mtbl_writer_options_set_compression( struct mtbl_writer_options *, mtbl_compression_type); void mtbl_writer_options_set_block_size( struct mtbl_writer_options *, size_t); void mtbl_writer_options_set_block_restart_interval( struct mtbl_writer_options *, size_t); /* reader */ struct mtbl_reader * mtbl_reader_init(const char *fname, const struct mtbl_reader_options *); struct mtbl_reader * mtbl_reader_init_fd(int fd, const struct mtbl_reader_options *); void mtbl_reader_destroy(struct mtbl_reader **); const struct mtbl_source * mtbl_reader_source(struct mtbl_reader *); /* reader options */ struct mtbl_reader_options * mtbl_reader_options_init(void); void mtbl_reader_options_destroy(struct mtbl_reader_options **); void mtbl_reader_options_set_verify_checksums(struct mtbl_reader_options *, bool); /* merger */ struct mtbl_merger * mtbl_merger_init(const struct mtbl_merger_options *); void mtbl_merger_destroy(struct mtbl_merger **); void mtbl_merger_add_source(struct mtbl_merger *, const struct mtbl_source *); const struct mtbl_source * mtbl_merger_source(struct mtbl_merger *); /* merger options */ struct mtbl_merger_options * mtbl_merger_options_init(void); void mtbl_merger_options_destroy(struct mtbl_merger_options **); void mtbl_merger_options_set_merge_func( struct mtbl_merger_options *, mtbl_merge_func, void *clos); /* fileset */ struct mtbl_fileset * mtbl_fileset_init(const char *fname, const struct mtbl_fileset_options *); void mtbl_fileset_destroy(struct mtbl_fileset **); void mtbl_fileset_reload(struct mtbl_fileset *); const struct mtbl_source * mtbl_fileset_source(struct mtbl_fileset *); /* fileset options */ struct mtbl_fileset_options * mtbl_fileset_options_init(void); void mtbl_fileset_options_destroy(struct mtbl_fileset_options **); void mtbl_fileset_options_set_merge_func( struct mtbl_fileset_options *, mtbl_merge_func, void *clos); void mtbl_fileset_options_set_reload_interval( struct mtbl_fileset_options *, uint32_t reload_interval); /* sorter */ struct mtbl_sorter * mtbl_sorter_init(const struct mtbl_sorter_options *); void mtbl_sorter_destroy(struct mtbl_sorter **); mtbl_res mtbl_sorter_add(struct mtbl_sorter *, const uint8_t *key, size_t len_key, const uint8_t *val, size_t len_val) __attribute__((warn_unused_result)); mtbl_res mtbl_sorter_write(struct mtbl_sorter *, struct mtbl_writer *) __attribute__((warn_unused_result)); struct mtbl_iter * mtbl_sorter_iter(struct mtbl_sorter *s); /* sorter options */ struct mtbl_sorter_options * mtbl_sorter_options_init(void); void mtbl_sorter_options_destroy(struct mtbl_sorter_options **); void mtbl_sorter_options_set_merge_func( struct mtbl_sorter_options *, mtbl_merge_func merge_fp, void *clos); void mtbl_sorter_options_set_temp_dir( struct mtbl_sorter_options *, const char *); void mtbl_sorter_options_set_max_memory( struct mtbl_sorter_options *, size_t); /* crc32c */ uint32_t mtbl_crc32c(const uint8_t *buffer, size_t length); /* fixed */ size_t mtbl_fixed_encode32(uint8_t *dst, uint32_t value); size_t mtbl_fixed_encode64(uint8_t *dst, uint64_t value); uint32_t mtbl_fixed_decode32(const uint8_t *ptr); uint64_t mtbl_fixed_decode64(const uint8_t *ptr); /* varint */ unsigned mtbl_varint_length(uint64_t v); unsigned mtbl_varint_length_packed(const uint8_t *buf, size_t len_buf); size_t mtbl_varint_encode32(uint8_t *ptr, uint32_t value); size_t mtbl_varint_encode64(uint8_t *ptr, uint64_t value); size_t mtbl_varint_decode32(const uint8_t *ptr, uint32_t *value); size_t mtbl_varint_decode64(const uint8_t *ptr, uint64_t *value); #ifdef __cplusplus } #endif #endif /* MTBL_H */ mtbl-0.5/mtbl/reader.c000066400000000000000000000240551227302760400146710ustar00rootroot00000000000000/* * Copyright (c) 2012-2014 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "mtbl-private.h" #include "libmy/ubuf.h" typedef enum { READER_ITER_TYPE_ITER, READER_ITER_TYPE_GET, READER_ITER_TYPE_GET_PREFIX, READER_ITER_TYPE_GET_RANGE, } reader_iter_type; struct reader_iter { struct mtbl_reader *r; struct block *b; struct block_iter *bi; struct block_iter *index_iter; ubuf *k; bool first; bool valid; reader_iter_type it_type; }; struct mtbl_reader_options { bool verify_checksums; }; struct mtbl_reader { int fd; struct trailer t; uint8_t *data; size_t len_data; struct mtbl_reader_options opt; struct block *index; struct mtbl_source *source; }; static mtbl_res reader_iter_next(void *, const uint8_t **, size_t *, const uint8_t **, size_t *); static void reader_iter_free(void *); static struct mtbl_iter * reader_iter(void *); static struct mtbl_iter * reader_get(void *, const uint8_t *, size_t); static struct mtbl_iter * reader_get_prefix(void *, const uint8_t *, size_t); static struct mtbl_iter * reader_get_range(void *, const uint8_t *, size_t, const uint8_t *, size_t); struct mtbl_reader_options * mtbl_reader_options_init(void) { return (my_calloc(1, sizeof(struct mtbl_reader_options))); } void mtbl_reader_options_destroy(struct mtbl_reader_options **opt) { if (*opt) { free(*opt); *opt = NULL; } } void mtbl_reader_options_set_verify_checksums(struct mtbl_reader_options *opt, bool verify_checksums) { opt->verify_checksums = verify_checksums; } struct mtbl_reader * mtbl_reader_init_fd(int orig_fd, const struct mtbl_reader_options *opt) { struct mtbl_reader *r; struct stat ss; size_t trailer_offset; int fd; size_t index_len; uint32_t index_crc; uint8_t *index_data; assert(orig_fd >= 0); fd = dup(orig_fd); assert(fd >= 0); int ret = fstat(fd, &ss); assert(ret == 0); if (ss.st_size < MTBL_TRAILER_SIZE) return (NULL); r = my_calloc(1, sizeof(*r)); if (opt != NULL) memcpy(&r->opt, opt, sizeof(*opt)); r->fd = fd; r->len_data = ss.st_size; r->data = mmap(NULL, r->len_data, PROT_READ, MAP_PRIVATE, r->fd, 0); if (r->data == MAP_FAILED) { close(r->fd); free(r); return (NULL); } trailer_offset = r->len_data - MTBL_TRAILER_SIZE; if (!trailer_read(r->data + trailer_offset, &r->t)) { mtbl_reader_destroy(&r); return (NULL); } index_len = mtbl_fixed_decode32(r->data + r->t.index_block_offset + 0); index_crc = mtbl_fixed_decode32(r->data + r->t.index_block_offset + sizeof(uint32_t)); index_data = r->data + r->t.index_block_offset + 2 * sizeof(uint32_t); assert(index_crc == mtbl_crc32c(index_data, index_len)); r->index = block_init(index_data, index_len, false); r->source = mtbl_source_init(reader_iter, reader_get, reader_get_prefix, reader_get_range, NULL, r); return (r); } struct mtbl_reader * mtbl_reader_init(const char *fname, const struct mtbl_reader_options *opt) { struct mtbl_reader *r; int fd; fd = open(fname, O_RDONLY); if (fd < 0) return (NULL); r = mtbl_reader_init_fd(fd, opt); close(fd); return (r); } void mtbl_reader_destroy(struct mtbl_reader **r) { if (*r != NULL) { block_destroy(&(*r)->index); munmap((*r)->data, (*r)->len_data); close((*r)->fd); mtbl_source_destroy(&(*r)->source); free(*r); *r = NULL; } } const struct mtbl_source * mtbl_reader_source(struct mtbl_reader *r) { assert(r != NULL); return (r->source); } static struct block * get_block(struct mtbl_reader *r, uint64_t offset) { bool needs_free = false; uint8_t *block_contents = NULL, *raw_contents = NULL; size_t block_contents_size = 0, raw_contents_size = 0; snappy_status res; int zret; z_stream zs; assert(offset < r->len_data); raw_contents_size = mtbl_fixed_decode32(&r->data[offset + 0]); raw_contents = &r->data[offset + 2 * sizeof(uint32_t)]; if (r->opt.verify_checksums) { uint32_t block_crc, calc_crc; block_crc = mtbl_fixed_decode32(&r->data[offset + sizeof(uint32_t)]); calc_crc = mtbl_crc32c(raw_contents, raw_contents_size); assert(block_crc == calc_crc); } switch (r->t.compression_algorithm) { case MTBL_COMPRESSION_NONE: block_contents = raw_contents; block_contents_size = raw_contents_size; break; case MTBL_COMPRESSION_SNAPPY: needs_free = true; block_contents_size = 2 * r->t.data_block_size; block_contents = my_calloc(1, block_contents_size); res = snappy_uncompress((const char *)raw_contents, raw_contents_size, (char *) block_contents, &block_contents_size); assert(res == SNAPPY_OK); break; case MTBL_COMPRESSION_ZLIB: needs_free = true; block_contents_size = 2 * r->t.data_block_size; zs.zalloc = Z_NULL; zs.zfree = Z_NULL; zs.opaque = Z_NULL; zs.avail_in = 0; zs.next_in = Z_NULL; memset(&zs, 0, sizeof(zs)); zret = inflateInit(&zs); assert(zret == Z_OK); zs.avail_in = raw_contents_size; zs.next_in = raw_contents; zs.avail_out = block_contents_size; zs.next_out = block_contents = my_calloc(1, block_contents_size); zret = inflate(&zs, Z_NO_FLUSH); assert(zret == Z_STREAM_END); block_contents_size = zs.total_out; inflateEnd(&zs); break; } return (block_init(block_contents, block_contents_size, needs_free)); } static struct block * get_block_at_index(struct mtbl_reader *r, struct block_iter *index_iter) { const uint8_t *ikey, *ival; size_t len_ikey, len_ival; if (block_iter_get(index_iter, &ikey, &len_ikey, &ival, &len_ival)) { struct block *b; uint64_t offset; mtbl_varint_decode64(ival, &offset); b = get_block(r, offset); return (b); } return (NULL); } static struct mtbl_iter * reader_iter(void *clos) { struct mtbl_reader *r = (struct mtbl_reader *) clos; struct reader_iter *it = my_calloc(1, sizeof(*it)); it->r = r; it->index_iter = block_iter_init(r->index); block_iter_seek_to_first(it->index_iter); it->b = get_block_at_index(r, it->index_iter); if (it->b == NULL) { block_iter_destroy(&it->index_iter); block_destroy(&it->b); free(it); return (NULL); } it->bi = block_iter_init(it->b); block_iter_seek_to_first(it->bi); it->first = true; it->valid = true; it->it_type = READER_ITER_TYPE_ITER; return (mtbl_iter_init(reader_iter_next, reader_iter_free, it)); } static struct reader_iter * reader_iter_init(struct mtbl_reader *r, const uint8_t *key, size_t len_key) { struct reader_iter *it = my_calloc(1, sizeof(*it)); it->r = r; it->index_iter = block_iter_init(r->index); block_iter_seek(it->index_iter, key, len_key); it->b = get_block_at_index(r, it->index_iter); if (it->b == NULL) { block_iter_destroy(&it->index_iter); block_destroy(&it->b); free(it); return (NULL); } it->bi = block_iter_init(it->b); block_iter_seek(it->bi, key, len_key); it->first = true; it->valid = true; return (it); } static struct mtbl_iter * reader_get(void *clos, const uint8_t *key, size_t len_key) { struct mtbl_reader *r = (struct mtbl_reader *) clos; struct reader_iter *it = reader_iter_init(r, key, len_key); if (it == NULL) return (NULL); it->k = ubuf_init(len_key); ubuf_append(it->k, key, len_key); it->it_type = READER_ITER_TYPE_GET; return (mtbl_iter_init(reader_iter_next, reader_iter_free, it)); } static struct mtbl_iter * reader_get_prefix(void *clos, const uint8_t *key, size_t len_key) { struct mtbl_reader *r = (struct mtbl_reader *) clos; struct reader_iter *it = reader_iter_init(r, key, len_key); if (it == NULL) return (NULL); it->k = ubuf_init(len_key); ubuf_append(it->k, key, len_key); it->it_type = READER_ITER_TYPE_GET_PREFIX; return (mtbl_iter_init(reader_iter_next, reader_iter_free, it)); } static struct mtbl_iter * reader_get_range(void *clos, const uint8_t *key0, size_t len_key0, const uint8_t *key1, size_t len_key1) { struct mtbl_reader *r = (struct mtbl_reader *) clos; struct reader_iter *it = reader_iter_init(r, key0, len_key0); if (it == NULL) return (NULL); it->k = ubuf_init(len_key1); ubuf_append(it->k, key1, len_key1); it->it_type = READER_ITER_TYPE_GET_RANGE; return (mtbl_iter_init(reader_iter_next, reader_iter_free, it)); } static void reader_iter_free(void *v) { struct reader_iter *it = (struct reader_iter *) v; if (it) { ubuf_destroy(&it->k); block_destroy(&it->b); block_iter_destroy(&it->bi); block_iter_destroy(&it->index_iter); free(it); } } static mtbl_res reader_iter_next(void *v, const uint8_t **key, size_t *len_key, const uint8_t **val, size_t *len_val) { struct reader_iter *it = (struct reader_iter *) v; if (!it->valid) return (mtbl_res_failure); if (!it->first) block_iter_next(it->bi); it->first = false; it->valid = block_iter_get(it->bi, key, len_key, val, len_val); if (!it->valid) { block_destroy(&it->b); block_iter_destroy(&it->bi); if (!block_iter_next(it->index_iter)) return (mtbl_res_failure); it->b = get_block_at_index(it->r, it->index_iter); it->bi = block_iter_init(it->b); block_iter_seek_to_first(it->bi); it->valid = block_iter_get(it->bi, key, len_key, val, len_val); if (!it->valid) return (mtbl_res_failure); } switch (it->it_type) { case READER_ITER_TYPE_ITER: break; case READER_ITER_TYPE_GET: if (bytes_compare(*key, *len_key, ubuf_data(it->k), ubuf_size(it->k)) != 0) it->valid = false; break; case READER_ITER_TYPE_GET_PREFIX: if (!(ubuf_size(it->k) <= *len_key && memcmp(ubuf_data(it->k), *key, ubuf_size(it->k)) == 0)) { it->valid = false; } break; case READER_ITER_TYPE_GET_RANGE: if (bytes_compare(*key, *len_key, ubuf_data(it->k), ubuf_size(it->k)) > 0) it->valid = false; break; default: assert(0); } if (it->valid) return (mtbl_res_success); return (mtbl_res_failure); } mtbl-0.5/mtbl/sorter.c000066400000000000000000000210751227302760400147440ustar00rootroot00000000000000/* * Copyright (c) 2012, 2014 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "mtbl-private.h" #include "libmy/ubuf.h" VECTOR_GENERATE(reader_vec, struct mtbl_reader *); struct sorter_iter { reader_vec *readers; struct mtbl_merger *m; struct mtbl_iter *m_iter; }; struct entry { uint32_t len_key; uint32_t len_val; uint8_t data[]; }; #define entry_key(e) ((e)->data) #define entry_val(e) ((e)->data + (e)->len_key) VECTOR_GENERATE(entry_vec, struct entry *); struct chunk { int fd; }; VECTOR_GENERATE(chunk_vec, struct chunk *); struct mtbl_sorter_options { size_t max_memory; char *tmp_dname; mtbl_merge_func merge; void *merge_clos; }; struct mtbl_sorter { chunk_vec *chunks; entry_vec *vec; size_t entry_bytes; bool iterating; struct mtbl_sorter_options opt; }; struct mtbl_sorter_options * mtbl_sorter_options_init(void) { struct mtbl_sorter_options *opt; opt = my_calloc(1, sizeof(*opt)); opt->max_memory = DEFAULT_SORTER_MEMORY; mtbl_sorter_options_set_temp_dir(opt, DEFAULT_SORTER_TEMP_DIR); return (opt); } void mtbl_sorter_options_destroy(struct mtbl_sorter_options **opt) { if (*opt) { free((*opt)->tmp_dname); free(*opt); *opt = NULL; } } void mtbl_sorter_options_set_merge_func(struct mtbl_sorter_options *opt, mtbl_merge_func merge, void *clos) { opt->merge = merge; opt->merge_clos = clos; } void mtbl_sorter_options_set_temp_dir(struct mtbl_sorter_options *opt, const char *temp_dir) { free(opt->tmp_dname); opt->tmp_dname = strdup(temp_dir); } void mtbl_sorter_options_set_max_memory(struct mtbl_sorter_options *opt, size_t max_memory) { if (max_memory < MIN_SORTER_MEMORY) max_memory = MIN_SORTER_MEMORY; opt->max_memory = max_memory; } struct mtbl_sorter * mtbl_sorter_init(const struct mtbl_sorter_options *opt) { struct mtbl_sorter *s; s = my_calloc(1, sizeof(*s)); if (opt != NULL) { memcpy(&s->opt, opt, sizeof(*opt)); s->opt.tmp_dname = strdup(opt->tmp_dname); } s->vec = entry_vec_init(INITIAL_SORTER_VEC_SIZE); s->chunks = chunk_vec_init(1); return (s); } void mtbl_sorter_destroy(struct mtbl_sorter **s) { if (*s) { for (unsigned i = 0; i < entry_vec_size((*s)->vec); i++) { struct entry *ent = entry_vec_value((*s)->vec, i); free(ent); } entry_vec_destroy(&((*s)->vec)); for (unsigned i = 0; i < chunk_vec_size((*s)->chunks); i++) { struct chunk *c = chunk_vec_value((*s)->chunks, i); close(c->fd); free(c); } chunk_vec_destroy(&((*s)->chunks)); free((*s)->opt.tmp_dname); free(*s); *s = NULL; } } static int _mtbl_sorter_compare(const void *va, const void *vb) { const struct entry *a = *((const struct entry **) va); const struct entry *b = *((const struct entry **) vb); return (bytes_compare(entry_key(a), a->len_key, entry_key(b), b->len_key)); } static mtbl_res _mtbl_sorter_write_chunk(struct mtbl_sorter *s) { mtbl_res res = mtbl_res_success; assert(!s->iterating); struct chunk *c = my_calloc(1, sizeof(*c)); char template[64]; sprintf(template, "/.mtbl.%ld.XXXXXX", (long)getpid()); ubuf *tmp_fname = ubuf_init(strlen(s->opt.tmp_dname) + strlen(template) + 1); ubuf_append(tmp_fname, (uint8_t *) s->opt.tmp_dname, strlen(s->opt.tmp_dname)); ubuf_append(tmp_fname, (uint8_t *) template, strlen(template)); ubuf_append(tmp_fname, (const uint8_t *) "\x00", 1); c->fd = mkstemp((char *) ubuf_data(tmp_fname)); assert(c->fd >= 0); int unlink_ret = unlink((char *) ubuf_data(tmp_fname)); assert(unlink_ret == 0); ubuf_destroy(&tmp_fname); struct mtbl_writer_options *wopt = mtbl_writer_options_init(); mtbl_writer_options_set_compression(wopt, MTBL_COMPRESSION_SNAPPY); struct mtbl_writer *w = mtbl_writer_init_fd(c->fd, wopt); mtbl_writer_options_destroy(&wopt); size_t entries_written = 0; struct entry **array = entry_vec_data(s->vec); qsort(array, entry_vec_size(s->vec), sizeof(void *), _mtbl_sorter_compare); for (unsigned i = 0; i < entry_vec_size(s->vec); i++) { struct entry *ent = entry_vec_value(s->vec, i); if (i + 1 < entry_vec_size(s->vec)) { struct entry *next_ent = entry_vec_value(s->vec, i + 1); struct entry *merge_ent = NULL; if (_mtbl_sorter_compare(&ent, &next_ent) == 0) { assert(s->opt.merge != NULL); uint8_t *merge_val = NULL; size_t len_merge_val = 0; s->opt.merge(s->opt.merge_clos, entry_key(ent), ent->len_key, entry_val(ent), ent->len_val, entry_val(next_ent), next_ent->len_val, &merge_val, &len_merge_val); if (merge_val == NULL) { mtbl_writer_destroy(&w); return (mtbl_res_failure); } size_t len = sizeof(struct entry) + ent->len_key + len_merge_val; merge_ent = my_malloc(len); merge_ent->len_key = ent->len_key; merge_ent->len_val = len_merge_val; memcpy(entry_key(merge_ent), entry_key(ent), ent->len_key); memcpy(entry_val(merge_ent), merge_val, len_merge_val); free(merge_val); free(ent); free(next_ent); entry_vec_data(s->vec)[i + 1] = merge_ent; continue; } } res = mtbl_writer_add(w, entry_key(ent), ent->len_key, entry_val(ent), ent->len_val); free(ent); entries_written += 1; if (res != mtbl_res_success) break; } mtbl_writer_destroy(&w); entry_vec_destroy(&s->vec); s->vec = entry_vec_init(INITIAL_SORTER_VEC_SIZE); s->entry_bytes = 0; chunk_vec_add(s->chunks, c); return (res); } mtbl_res mtbl_sorter_write(struct mtbl_sorter *s, struct mtbl_writer *w) { if (s->iterating) return (mtbl_res_failure); struct mtbl_iter *it = mtbl_sorter_iter(s); const uint8_t *key, *val; size_t len_key, len_val; mtbl_res res = mtbl_res_success; if (it == NULL) return (mtbl_res_failure); while (mtbl_iter_next(it, &key, &len_key, &val, &len_val) == mtbl_res_success) { res = mtbl_writer_add(w, key, len_key, val, len_val); if (res != mtbl_res_success) break; } mtbl_iter_destroy(&it); return (res); } mtbl_res mtbl_sorter_add(struct mtbl_sorter *s, const uint8_t *key, size_t len_key, const uint8_t *val, size_t len_val) { mtbl_res res = mtbl_res_success; if (s->iterating) return (mtbl_res_failure); assert(len_key <= UINT_MAX); assert(len_val <= UINT_MAX); struct entry *ent; size_t entry_bytes; entry_bytes = sizeof(*ent) + len_key + len_val; ent = my_malloc(entry_bytes); ent->len_key = len_key; ent->len_val = len_val; memcpy(entry_key(ent), key, len_key); memcpy(entry_val(ent), val, len_val); entry_vec_append(s->vec, &ent, 1); s->entry_bytes += entry_bytes; if (s->entry_bytes + entry_vec_bytes(s->vec) >= s->opt.max_memory) res = _mtbl_sorter_write_chunk(s); return (res); } static mtbl_res sorter_iter_next(void *v, const uint8_t **key, size_t *len_key, const uint8_t **val, size_t *len_val) { struct sorter_iter *it = (struct sorter_iter *) v; return (mtbl_iter_next(it->m_iter, key, len_key, val, len_val)); } static void sorter_iter_free(void *v) { struct sorter_iter *it = (struct sorter_iter *) v; if (it) { mtbl_iter_destroy(&it->m_iter); mtbl_merger_destroy(&it->m); for (size_t i = 0; i < reader_vec_size(it->readers); i++) { struct mtbl_reader *r = reader_vec_value(it->readers, i); mtbl_reader_destroy(&r); } reader_vec_destroy(&it->readers); free(it); } } struct mtbl_iter * mtbl_sorter_iter(struct mtbl_sorter *s) { mtbl_res res; struct sorter_iter *it = my_calloc(1, sizeof(*it)); it->readers = reader_vec_init(0); struct mtbl_merger_options *mopt = mtbl_merger_options_init(); mtbl_merger_options_set_merge_func(mopt, s->opt.merge, s->opt.merge_clos); it->m = mtbl_merger_init(mopt); mtbl_merger_options_destroy(&mopt); if (entry_vec_size(s->vec) > 0) { res = _mtbl_sorter_write_chunk(s); if (res != mtbl_res_success) return (NULL); } for (unsigned i = 0; i < chunk_vec_size(s->chunks); i++) { struct chunk *c = chunk_vec_value(s->chunks, i); struct mtbl_reader *r; r = mtbl_reader_init_fd(c->fd, NULL); mtbl_merger_add_source(it->m, mtbl_reader_source(r)); reader_vec_add(it->readers, r); } it->m_iter = mtbl_source_iter(mtbl_merger_source(it->m)); s->iterating = true; return (mtbl_iter_init(sorter_iter_next, sorter_iter_free, it)); } mtbl-0.5/mtbl/source.c000066400000000000000000000054471227302760400147330ustar00rootroot00000000000000/* * Copyright (c) 2012 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "mtbl-private.h" struct mtbl_source { mtbl_source_iter_func source_iter; mtbl_source_get_func source_get; mtbl_source_get_prefix_func source_get_prefix; mtbl_source_get_range_func source_get_range; mtbl_source_free_func source_free; void *clos; }; struct mtbl_source * mtbl_source_init(mtbl_source_iter_func source_iter, mtbl_source_get_func source_get, mtbl_source_get_prefix_func source_get_prefix, mtbl_source_get_range_func source_get_range, mtbl_source_free_func source_free, void *clos) { assert(source_iter != NULL); assert(source_get != NULL); assert(source_get_prefix != NULL); assert(source_get_range != NULL); struct mtbl_source *s = my_calloc(1, sizeof(*s)); s->source_iter = source_iter; s->source_get = source_get; s->source_get_prefix = source_get_prefix; s->source_get_range = source_get_range; s->source_free = source_free; s->clos = clos; return (s); } void mtbl_source_destroy(struct mtbl_source **s) { if (*s) { if ((*s)->source_free != NULL) (*s)->source_free((*s)->clos); free(*s); *s = NULL; } } struct mtbl_iter * mtbl_source_iter(const struct mtbl_source *s) { return (s->source_iter(s->clos)); } struct mtbl_iter * mtbl_source_get(const struct mtbl_source *s, const uint8_t *key, size_t len_key) { return (s->source_get(s->clos, key, len_key)); } struct mtbl_iter * mtbl_source_get_prefix(const struct mtbl_source *s, const uint8_t *key, size_t len_key) { return (s->source_get_prefix(s->clos, key, len_key)); } struct mtbl_iter * mtbl_source_get_range(const struct mtbl_source *s, const uint8_t *key0, size_t len_key0, const uint8_t *key1, size_t len_key1) { return (s->source_get_range(s->clos, key0, len_key0, key1, len_key1)); } mtbl_res mtbl_source_write(const struct mtbl_source *s, struct mtbl_writer *w) { const uint8_t *key, *val; size_t len_key, len_val; struct mtbl_iter *it = mtbl_source_iter(s); mtbl_res res = mtbl_res_success; if (it == NULL) return (mtbl_res_failure); while (mtbl_iter_next(it, &key, &len_key, &val, &len_val) == mtbl_res_success) { res = mtbl_writer_add(w, key, len_key, val, len_val); if (res != mtbl_res_success) break; } mtbl_iter_destroy(&it); return (res); } mtbl-0.5/mtbl/trailer.c000066400000000000000000000040671227302760400150720ustar00rootroot00000000000000/* * Copyright (c) 2012 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "mtbl-private.h" void trailer_write(struct trailer *t, uint8_t *buf) { size_t padding; uint8_t *p = buf; p += mtbl_fixed_encode64(p, t->index_block_offset); p += mtbl_fixed_encode64(p, t->data_block_size); p += mtbl_fixed_encode64(p, t->compression_algorithm); p += mtbl_fixed_encode64(p, t->count_entries); p += mtbl_fixed_encode64(p, t->count_data_blocks); p += mtbl_fixed_encode64(p, t->bytes_data_blocks); p += mtbl_fixed_encode64(p, t->bytes_index_block); p += mtbl_fixed_encode64(p, t->bytes_keys); p += mtbl_fixed_encode64(p, t->bytes_values); padding = MTBL_TRAILER_SIZE - (p - buf) - sizeof(uint32_t); while (padding-- != 0) *(p++) = '\0'; mtbl_fixed_encode32(buf + MTBL_TRAILER_SIZE - sizeof(uint32_t), MTBL_MAGIC); } bool trailer_read(const uint8_t *buf, struct trailer *t) { uint32_t magic; const uint8_t *p = buf; magic = mtbl_fixed_decode32(buf + MTBL_TRAILER_SIZE - sizeof(uint32_t)); if (magic != MTBL_MAGIC) return (false); t->index_block_offset = mtbl_fixed_decode64(p); p += 8; t->data_block_size = mtbl_fixed_decode64(p); p += 8; t->compression_algorithm = mtbl_fixed_decode64(p); p += 8; t->count_entries = mtbl_fixed_decode64(p); p += 8; t->count_data_blocks = mtbl_fixed_decode64(p); p += 8; t->bytes_data_blocks = mtbl_fixed_decode64(p); p += 8; t->bytes_index_block = mtbl_fixed_decode64(p); p += 8; t->bytes_keys = mtbl_fixed_decode64(p); p += 8; t->bytes_values = mtbl_fixed_decode64(p); p += 8; return (true); } mtbl-0.5/mtbl/varint.c000066400000000000000000000116111227302760400147240ustar00rootroot00000000000000/* * Copyright (c) 2012 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // Copyright (c) 2011 The LevelDB Authors. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /* * Copyright 2008, Dave Benson. * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You may * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0. * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. */ #include "mtbl-private.h" unsigned mtbl_varint_length(uint64_t v) { unsigned len = 1; while (v >= 128) { v >>= 7; len++; } return (len); } unsigned mtbl_varint_length_packed(const uint8_t *data, size_t len_data) { unsigned i = 0; size_t len = len_data; while (len--) { if ((data[i] & 0x80) == 0) break; i++; } if (i == len_data) return (0); return (i + 1); } size_t mtbl_varint_encode32(uint8_t *src_ptr, uint32_t v) { static const unsigned B = 128; uint8_t *ptr = src_ptr; if (v < (1 << 7)) { *(ptr++) = v; } else if (v < (1 << 14)) { *(ptr++) = v | B; *(ptr++) = v >> 7; } else if (v < (1 << 21)) { *(ptr++) = v | B; *(ptr++) = (v >> 7) | B; *(ptr++) = v >> 14; } else if (v < (1 << 28)) { *(ptr++) = v | B; *(ptr++) = (v >> 7) | B; *(ptr++) = (v >> 14) | B; *(ptr++) = v >> 21; } else { *(ptr++) = v | B; *(ptr++) = (v >> 7) | B; *(ptr++) = (v >> 14) | B; *(ptr++) = (v >> 21) | B; *(ptr++) = v >> 28; } return ((size_t) (ptr - src_ptr)); } size_t mtbl_varint_encode64(uint8_t *src_ptr, uint64_t v) { static const unsigned B = 128; uint8_t *ptr = src_ptr; while (v >= B) { *(ptr++) = (v & (B - 1)) | B; v >>= 7; } *(ptr++) = (uint8_t) v; return ((size_t) (ptr - src_ptr)); } size_t mtbl_varint_decode32(const uint8_t *data, uint32_t *value) { unsigned len = mtbl_varint_length_packed(data, 5); uint32_t val = data[0] & 0x7f; if (len > 1) { val |= ((data[1] & 0x7f) << 7); if (len > 2) { val |= ((data[2] & 0x7f) << 14); if (len > 3) { val |= ((data[3] & 0x7f) << 21); if (len > 4) val |= (data[4] << 28); } } } *value = val; return ((size_t) len); } size_t mtbl_varint_decode64(const uint8_t *data, uint64_t *value) { unsigned shift, i; unsigned len = mtbl_varint_length_packed(data, 10); uint64_t val; if (len < 5) { size_t tmp_len; uint32_t tmp; tmp_len = mtbl_varint_decode32(data, &tmp); *value = tmp; return (tmp_len); } val = ((data[0] & 0x7f)) | ((data[1] & 0x7f) << 7) | ((data[2] & 0x7f) << 14) | ((data[3] & 0x7f) << 21); shift = 28; for (i = 4; i < len; i++) { val |= (((uint64_t)(data[i] & 0x7f)) << shift); shift += 7; } *value = val; return ((size_t) len); } mtbl-0.5/mtbl/writer.c000066400000000000000000000212741227302760400147430ustar00rootroot00000000000000/* * Copyright (c) 2012, 2014 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "mtbl-private.h" #include "bytes.h" #include "libmy/ubuf.h" struct mtbl_writer_options { mtbl_compression_type compression_type; size_t block_size; size_t block_restart_interval; }; struct mtbl_writer { int fd; struct trailer t; struct block_builder *data; struct block_builder *index; struct mtbl_writer_options opt; ubuf *last_key; uint64_t last_offset; bool closed; bool pending_index_entry; uint64_t pending_offset; }; static void _mtbl_writer_finish(struct mtbl_writer *); static void _mtbl_writer_flush(struct mtbl_writer *); static void _write_all(int fd, const uint8_t *, size_t); static size_t _mtbl_writer_writeblock( struct mtbl_writer *, struct block_builder *, mtbl_compression_type); struct mtbl_writer_options * mtbl_writer_options_init(void) { struct mtbl_writer_options *opt; opt = my_calloc(1, sizeof(*opt)); opt->compression_type = DEFAULT_COMPRESSION_TYPE; opt->block_size = DEFAULT_BLOCK_SIZE; opt->block_restart_interval = DEFAULT_BLOCK_RESTART_INTERVAL; return (opt); } void mtbl_writer_options_destroy(struct mtbl_writer_options **opt) { if (*opt) { free(*opt); *opt = NULL; } } void mtbl_writer_options_set_compression(struct mtbl_writer_options *opt, mtbl_compression_type compression_type) { assert(compression_type == MTBL_COMPRESSION_NONE || compression_type == MTBL_COMPRESSION_SNAPPY || compression_type == MTBL_COMPRESSION_ZLIB); opt->compression_type = compression_type; } void mtbl_writer_options_set_block_size(struct mtbl_writer_options *opt, size_t block_size) { if (block_size < MIN_BLOCK_SIZE) block_size = MIN_BLOCK_SIZE; opt->block_size = block_size; } void mtbl_writer_options_set_block_restart_interval(struct mtbl_writer_options *opt, size_t block_restart_interval) { opt->block_restart_interval = block_restart_interval; } struct mtbl_writer * mtbl_writer_init_fd(int orig_fd, const struct mtbl_writer_options *opt) { struct mtbl_writer *w; int fd; fd = dup(orig_fd); assert(fd >= 0); w = my_calloc(1, sizeof(*w)); if (opt == NULL) { w->opt.compression_type = DEFAULT_COMPRESSION_TYPE; w->opt.block_size = DEFAULT_BLOCK_SIZE; w->opt.block_restart_interval = DEFAULT_BLOCK_RESTART_INTERVAL; } else { memcpy(&w->opt, opt, sizeof(*opt)); } w->fd = fd; w->last_key = ubuf_init(256); w->t.compression_algorithm = w->opt.compression_type; w->t.data_block_size = w->opt.block_size; w->data = block_builder_init(w->opt.block_restart_interval); w->index = block_builder_init(w->opt.block_restart_interval); return (w); } struct mtbl_writer * mtbl_writer_init(const char *fname, const struct mtbl_writer_options *opt) { struct mtbl_writer *w; int fd; fd = open(fname, O_WRONLY | O_CREAT | O_TRUNC | O_EXCL, 0644); if (fd < 0) return (NULL); w = mtbl_writer_init_fd(fd, opt); close(fd); return (w); } void mtbl_writer_destroy(struct mtbl_writer **w) { if (*w != NULL) { if (!(*w)->closed) { _mtbl_writer_finish(*w); close((*w)->fd); } block_builder_destroy(&((*w)->data)); block_builder_destroy(&((*w)->index)); ubuf_destroy(&(*w)->last_key); free(*w); *w = NULL; } } mtbl_res mtbl_writer_add(struct mtbl_writer *w, const uint8_t *key, size_t len_key, const uint8_t *val, size_t len_val) { assert(!w->closed); if (w->t.count_entries > 0) { if (!(bytes_compare(key, len_key, ubuf_data(w->last_key), ubuf_size(w->last_key)) > 0)) { return (mtbl_res_failure); } } size_t estimated_block_size = block_builder_current_size_estimate(w->data); estimated_block_size += 3*5 + len_key + len_val; if (estimated_block_size >= w->opt.block_size) _mtbl_writer_flush(w); if (w->pending_index_entry) { uint8_t enc[10]; size_t len_enc; assert(block_builder_empty(w->data)); bytes_shortest_separator(w->last_key, key, len_key); len_enc = mtbl_varint_encode64(enc, w->last_offset); /* fprintf(stderr, "%s: writing index entry, key= '%s' (%zd) val= %" PRIu64 "\n", __func__, ubuf_data(w->last_key), ubuf_size(w->last_key), w->last_offset); */ block_builder_add(w->index, ubuf_data(w->last_key), ubuf_size(w->last_key), enc, len_enc); w->pending_index_entry = false; } ubuf_reset(w->last_key); ubuf_append(w->last_key, key, len_key); w->t.count_entries += 1; w->t.bytes_keys += len_key; w->t.bytes_values += len_val; block_builder_add(w->data, key, len_key, val, len_val); return (mtbl_res_success); } static void _mtbl_writer_finish(struct mtbl_writer *w) { uint8_t tbuf[MTBL_TRAILER_SIZE]; _mtbl_writer_flush(w); assert(!w->closed); w->closed = true; if (w->pending_index_entry) { /* XXX use short successor */ uint8_t enc[10]; size_t len_enc; len_enc = mtbl_varint_encode64(enc, w->last_offset); /* fprintf(stderr, "%s: writing index entry, key= '%s' (%zd) val= %" PRIu64 "\n", __func__, ubuf_data(w->last_key), ubuf_size(w->last_key), w->last_offset); */ block_builder_add(w->index, ubuf_data(w->last_key), ubuf_size(w->last_key), enc, len_enc); w->pending_index_entry = false; } w->t.index_block_offset = w->pending_offset; w->t.bytes_index_block = _mtbl_writer_writeblock(w, w->index, MTBL_COMPRESSION_NONE); trailer_write(&w->t, tbuf); _write_all(w->fd, tbuf, sizeof(tbuf)); } static void _mtbl_writer_flush(struct mtbl_writer *w) { assert(!w->closed); if (block_builder_empty(w->data)) return; assert(!w->pending_index_entry); w->t.bytes_data_blocks += _mtbl_writer_writeblock(w, w->data, w->opt.compression_type); w->t.count_data_blocks += 1; w->pending_index_entry = true; } static size_t _mtbl_writer_writeblock(struct mtbl_writer *w, struct block_builder *b, mtbl_compression_type compression_type) { uint8_t *raw_contents = NULL, *block_contents = NULL, *comp_contents = NULL; size_t raw_contents_size = 0, block_contents_size = 0, comp_contents_size = 0; snappy_status res; int zret; z_stream zs; block_builder_finish(b, &raw_contents, &raw_contents_size); switch (compression_type) { case MTBL_COMPRESSION_NONE: block_contents = raw_contents; block_contents_size = raw_contents_size; break; case MTBL_COMPRESSION_SNAPPY: comp_contents_size = snappy_max_compressed_length(raw_contents_size); comp_contents = my_malloc(comp_contents_size); res = snappy_compress((const char *) raw_contents, raw_contents_size, (char *) comp_contents, &comp_contents_size); assert(res == SNAPPY_OK); block_contents = comp_contents; block_contents_size = comp_contents_size; break; case MTBL_COMPRESSION_ZLIB: comp_contents_size = 2 * raw_contents_size; comp_contents = my_malloc(comp_contents_size); memset(&zs, 0, sizeof(zs)); zs.zalloc = Z_NULL; zs.zfree = Z_NULL; zs.opaque = Z_NULL; zret = deflateInit(&zs, Z_DEFAULT_COMPRESSION); assert(zret == Z_OK); zs.avail_in = raw_contents_size; zs.next_in = raw_contents; zs.avail_out = comp_contents_size; zs.next_out = comp_contents; zret = deflate(&zs, Z_FINISH); assert(zret == Z_STREAM_END); assert(zs.avail_in == 0); comp_contents_size = zs.total_out; zret = deflateEnd(&zs); assert(zret == Z_OK); block_contents = comp_contents; block_contents_size = comp_contents_size; break; } assert(block_contents_size < UINT_MAX); const uint32_t crc = htole32(mtbl_crc32c(block_contents, block_contents_size)); const uint32_t len = htole32(block_contents_size); _write_all(w->fd, (const uint8_t *) &len, sizeof(len)); _write_all(w->fd, (const uint8_t *) &crc, sizeof(crc)); _write_all(w->fd, block_contents, block_contents_size); const size_t bytes_written = (sizeof(len) + sizeof(crc) + block_contents_size); w->last_offset = w->pending_offset; w->pending_offset += bytes_written; block_builder_reset(b); free(raw_contents); free(comp_contents); return (bytes_written); } static void _write_all(int fd, const uint8_t *buf, size_t size) { assert(size > 0); while (size) { ssize_t bytes_written; bytes_written = write(fd, buf, size); if (bytes_written < 0 && errno == EINTR) continue; if (bytes_written <= 0) { fprintf(stderr, "%s: write() failed: %s\n", __func__, strerror(errno)); assert(bytes_written > 0); } buf += bytes_written; size -= bytes_written; } } mtbl-0.5/src/000077500000000000000000000000001227302760400131065ustar00rootroot00000000000000mtbl-0.5/src/.gitignore000066400000000000000000000000371227302760400150760ustar00rootroot00000000000000mtbl_dump mtbl_info mtbl_merge mtbl-0.5/src/mtbl_dump.c000066400000000000000000000030031227302760400152310ustar00rootroot00000000000000/* * Copyright (c) 2012, 2014 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include "libmy/print_string.h" static bool dump(const char *fname) { const uint8_t *key, *val; size_t len_key, len_val; struct mtbl_reader *r; struct mtbl_iter *it; r = mtbl_reader_init(fname, NULL); if (r == NULL) { fprintf(stderr, "Error: mtbl_reader_init() on %s failed\n", fname); return (false); } it = mtbl_source_iter(mtbl_reader_source(r)); while (mtbl_iter_next(it, &key, &len_key, &val, &len_val)) { print_string(key, len_key, stdout); fputc(' ', stdout); print_string(val, len_val, stdout); fputc('\n', stdout); } mtbl_iter_destroy(&it); mtbl_reader_destroy(&r); return (true); } int main(int argc, char **argv) { char *fname; if (argc != 2) { fprintf(stderr, "Usage: %s \n", argv[0]); exit(EXIT_FAILURE); } fname = argv[1]; if (!dump(fname)) return (EXIT_FAILURE); return (EXIT_SUCCESS); } mtbl-0.5/src/mtbl_info.c000066400000000000000000000061571227302760400152340ustar00rootroot00000000000000/* * Copyright (c) 2012 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #include #include #include #include #include "trailer.c" static void dump(const char *fname) { int fd, ret; size_t trailer_offset; struct stat ss; struct trailer t; uint8_t buf[MTBL_TRAILER_SIZE]; fd = open(fname, O_RDONLY); if (fd < 0) { fprintf(stderr, "Error: unable to open file %s: %s\n", fname, strerror(errno)); exit(EXIT_FAILURE); } ret = fstat(fd, &ss); if (ret < 0) { perror("Error: fstat"); exit(EXIT_FAILURE); } trailer_offset = ss.st_size - MTBL_TRAILER_SIZE; off_t o = lseek(fd, trailer_offset, SEEK_SET); if (o == (off_t) -1) { fprintf(stderr, "Error: lseek() failed: %s\n", strerror(errno)); exit(EXIT_FAILURE); } ret = read(fd, buf, sizeof(buf)); if (ret != sizeof(buf)) { fprintf(stderr, "Error: short read\n"); exit(EXIT_FAILURE); } close(fd); if (!trailer_read(buf, &t)) { fprintf(stderr, "Error: unable to read trailer\n"); exit(EXIT_FAILURE); } double p_data = 100.0 * t.bytes_data_blocks / ss.st_size; double p_index = 100.0 * t.bytes_index_block / ss.st_size; double compactness = 100.0 * ss.st_size / (t.bytes_keys + t.bytes_values); printf("file name: %s\n", fname); printf("file size: %'zd\n", (size_t) ss.st_size); printf("index bytes: %'" PRIu64 " (%'.2f%%)\n", t.bytes_index_block, p_index); printf("data block bytes %'" PRIu64 " (%'.2f%%)\n", t.bytes_data_blocks, p_data); printf("data block size: %'" PRIu64 "\n", t.data_block_size); printf("data block count %'" PRIu64 "\n", t.count_data_blocks); printf("entry count: %'" PRIu64 "\n", t.count_entries); printf("key bytes: %'" PRIu64 "\n", t.bytes_keys); printf("value bytes: %'" PRIu64 "\n", t.bytes_values); printf("compression algorithm: "); if (t.compression_algorithm == MTBL_COMPRESSION_NONE) { puts("none"); } else if (t.compression_algorithm == MTBL_COMPRESSION_SNAPPY) { puts("snappy"); } else if (t.compression_algorithm == MTBL_COMPRESSION_ZLIB) { puts("zlib"); } else { printf("%" PRIu64 "\n", t.compression_algorithm); } printf("compactness: %'.2f%%\n", compactness); putchar('\n'); } int main(int argc, char **argv) { setlocale(LC_ALL, ""); if (argc < 2) { fprintf(stderr, "Usage: %s [...]\n", argv[0]); exit(EXIT_FAILURE); } for (int i = 1; i < argc; i++) dump(argv[i]); return (EXIT_SUCCESS); } mtbl-0.5/src/mtbl_merge.c000066400000000000000000000154541227302760400154000ustar00rootroot00000000000000/* * Copyright (c) 2012, 2014 by Farsight Security, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #include #include #include #include "mtbl-private.h" #include "libmy/getenv_int.h" #include "libmy/ubuf.h" #define STATS_INTERVAL 1000000 static const char *program_name; static const char *mtbl_output_fname; static const char *merge_dso_path; static const char *merge_dso_prefix; static mtbl_merge_init_func user_func_init; static mtbl_merge_free_func user_func_free; static mtbl_merge_func user_func_merge; static void *user_clos; static struct mtbl_merger *merger; static struct mtbl_writer *writer; static struct timespec start_time; static uint64_t count; static uint64_t count_merged; static void usage(void) { fprintf(stderr, "Usage: %s [...] \n" "Merges one or more MTBL input files into a single output file.\n" "Requires a merge function provided by the user at runtime via a DSO.\n" "See mtbl_merge(1) for details.\n", program_name ); exit(EXIT_FAILURE); } static void my_timespec_get(struct timespec *now) { struct timeval tv; (void) gettimeofday(&tv, NULL); now->tv_sec = tv.tv_sec; now->tv_nsec = tv.tv_usec * 1000; } static void my_timespec_sub(const struct timespec *a, struct timespec *b) { b->tv_sec -= a->tv_sec; b->tv_nsec -= a->tv_nsec; if (b->tv_nsec < 0) { b->tv_sec -= 1; b->tv_nsec += 1000000000; } } static double my_timespec_to_double(const struct timespec *ts) { return (ts->tv_sec + ts->tv_nsec / 1E9); } static void print_stats(void) { struct timespec dur; double t_dur; my_timespec_get(&dur); my_timespec_sub(&start_time, &dur); t_dur = my_timespec_to_double(&dur); fprintf(stderr, "%s: wrote %'" PRIu64 " entries (%'" PRIu64 " merged) " "in %'.2f sec, %'d ent/sec, %'d merge/sec\n", program_name, count, count_merged, t_dur, (int) (count / t_dur), (int) (count_merged / t_dur) ); } static void merge_func(void *clos, const uint8_t *key, size_t len_key, const uint8_t *val0, size_t len_val0, const uint8_t *val1, size_t len_val1, uint8_t **merged_val, size_t *len_merged_val) { user_func_merge(clos, key, len_key, val0, len_val0, val1, len_val1, merged_val, len_merged_val); count_merged += 1; } static void merge(void) { const uint8_t *key, *val; size_t len_key, len_val; struct mtbl_iter *it; it = mtbl_source_iter(mtbl_merger_source(merger)); assert(it != NULL); while (mtbl_iter_next(it, &key, &len_key, &val, &len_val) == mtbl_res_success) { mtbl_res res = mtbl_writer_add(writer, key, len_key, val, len_val); assert(res == mtbl_res_success); if ((++count % STATS_INTERVAL) == 0) print_stats(); } mtbl_iter_destroy(&it); mtbl_merger_destroy(&merger); mtbl_writer_destroy(&writer); } static void init_dso(void) { merge_dso_path = getenv("MTBL_MERGE_DSO"); merge_dso_prefix = getenv("MTBL_MERGE_FUNC_PREFIX"); if (merge_dso_path == NULL) { fprintf(stderr, "Error: MTBL_MERGE_DSO environment variable not set.\n\n"); usage(); } if (merge_dso_prefix == NULL) { fprintf(stderr, "Error: MTBL_MERGE_FUNC_PREFIX environment variable not set.\n\n"); usage(); } dlerror(); void *handle = dlopen(merge_dso_path, RTLD_NOW); if (handle == NULL) { fprintf(stderr, "Error: dlopen() failed: %s\n", dlerror()); exit(EXIT_FAILURE); } /* merge func */ ubuf *func_name = ubuf_init(0); ubuf_append(func_name, (const uint8_t *) merge_dso_prefix, strlen(merge_dso_prefix)); ubuf_append(func_name, (const uint8_t *) "_func", sizeof("_func")); user_func_merge = dlsym(handle, (const char *) ubuf_data(func_name)); if (user_func_merge == NULL) { fprintf(stderr, "Error: user merge function required but not found in DSO.\n\n"); usage(); } /* init func */ ubuf_clip(func_name, 0); ubuf_append(func_name, (const uint8_t *) merge_dso_prefix, strlen(merge_dso_prefix)); ubuf_append(func_name, (const uint8_t *) "_init_func", sizeof("_init_func")); user_func_init = dlsym(handle, (const char *) ubuf_data(func_name)); if (user_func_init != NULL) user_clos = user_func_init(); /* free func */ ubuf_clip(func_name, 0); ubuf_append(func_name, (const uint8_t *) merge_dso_prefix, strlen(merge_dso_prefix)); ubuf_append(func_name, (const uint8_t *) "_free_func", sizeof("_free_func")); user_func_free = dlsym(handle, (const char *) ubuf_data(func_name)); } static size_t get_block_size(void) { uint64_t sz; if (getenv_int("MTBL_MERGE_BLOCK_SIZE", &sz)) return ((size_t) sz); return (DEFAULT_BLOCK_SIZE); } static void init_mtbl(void) { struct mtbl_merger_options *mopt; struct mtbl_writer_options *wopt; mopt = mtbl_merger_options_init(); wopt = mtbl_writer_options_init(); mtbl_merger_options_set_merge_func(mopt, merge_func, user_clos); mtbl_writer_options_set_compression(wopt, MTBL_COMPRESSION_ZLIB); mtbl_writer_options_set_block_size(wopt, get_block_size()); merger = mtbl_merger_init(mopt); assert(merger != NULL); fprintf(stderr, "%s: opening output file %s\n", program_name, mtbl_output_fname); writer = mtbl_writer_init(mtbl_output_fname, wopt); if (writer == NULL) { fprintf(stderr, "Error: mtbl_writer_init() failed.\n\n"); usage(); } mtbl_merger_options_destroy(&mopt); mtbl_writer_options_destroy(&wopt); } int main(int argc, char **argv) { setlocale(LC_ALL, ""); program_name = argv[0]; if (argc < 3) usage(); mtbl_output_fname = argv[argc - 1]; /* open user dso */ init_dso(); /* open merger, writer */ init_mtbl(); /* open readers */ struct mtbl_reader *readers[argc-2]; for (int i = 0; i < argc - 2; i++) { const char *fname = argv[i+1]; fprintf(stderr, "%s: opening input file %s\n", program_name, fname); readers[i] = mtbl_reader_init(fname, NULL); if (readers[i] == NULL) { fprintf(stderr, "Error: mtbl_reader_init() failed.\n\n"); usage(); } mtbl_merger_add_source(merger, mtbl_reader_source(readers[i])); } /* do merge */ my_timespec_get(&start_time); merge(); /* cleanup readers */ for (int i = 0; i < argc - 2; i++) mtbl_reader_destroy(&readers[i]); /* call user cleanup */ if (user_func_free != NULL) user_func_free(user_clos); print_stats(); return (EXIT_SUCCESS); } mtbl-0.5/t/000077500000000000000000000000001227302760400125625ustar00rootroot00000000000000mtbl-0.5/t/.gitignore000066400000000000000000000001171227302760400145510ustar00rootroot00000000000000test-block_builder test-crc32c test-fixed test-trailer test-varint test-vector mtbl-0.5/t/test-block_builder.c000066400000000000000000000076321227302760400165130ustar00rootroot00000000000000#include #include #include #include #include #include #include #include #include "block_builder.c" #define NAME "test-block_builder" #include static void print_string(const uint8_t *data, size_t len, FILE *out) { unsigned c; fputc('\'', out); while (len-- != 0) { c = *(data++); if (isprint(c)) fputc(c, out); else fprintf(out, "\\x%02x", c); } fputc('\'', out); fputc('\n', out); } static int test1(void) { int ret = 0; struct block_builder *b; uint8_t *buf; size_t bufsz; b = block_builder_init(16); assert(b != NULL); fprintf(stderr, "block_builder_current_size_estimate(): %zd\n", block_builder_current_size_estimate(b)); block_builder_add(b, (uint8_t *) "foobar1", 7, (uint8_t *) "42", 2); block_builder_add(b, (uint8_t *) "foobar2", 7, (uint8_t *) "43", 2); block_builder_add(b, (uint8_t *) "foobar2ZZ", 9, (uint8_t *) "44", 2); block_builder_add(b, (uint8_t *) "foobar3", 7, (uint8_t *) "45", 2); block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "46", 2); block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "47", 2); block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "48", 2); block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "49", 2); block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "50", 2); block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "51", 2); block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "52", 2); block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "53", 2); block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "54", 2); block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "55", 2); block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "56", 2); block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "57", 2); block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "58", 2); block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "59", 2); block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "60", 2); block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "61", 2); block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "62", 2); block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "63", 2); block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "64", 2); block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "ZA", 2); block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "ZB", 2); block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "ZC", 2); block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "ZD", 2); block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "ZE", 2); block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "ZF", 2); block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "ZG", 2); block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "ZH", 2); block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "ZI", 2); block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "ZJ", 2); block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "ZK", 2); block_builder_add(b, (uint8_t *) "foobar1Y", 8, (uint8_t *) "ZM", 2); fprintf(stderr, "block_builder_current_size_estimate(): %zd\n", block_builder_current_size_estimate(b)); block_builder_finish(b, &buf, &bufsz); fprintf(stderr, "buf= %p bufsz= %zd\n", buf, bufsz); print_string(buf, bufsz, stderr); block_builder_reset(b); block_builder_destroy(&b); return (ret); } static int check(int ret, const char *s) { if (ret == 0) fprintf(stderr, NAME ": PASS: %s\n", s); else fprintf(stderr, NAME ": FAIL: %s\n", s); return (ret); } int main(int argc, char **argv) { int ret = 0; ret |= check(test1(), "test1"); if (ret) return (EXIT_FAILURE); return (EXIT_SUCCESS); } mtbl-0.5/t/test-crc32c.c000066400000000000000000000137251227302760400147720ustar00rootroot00000000000000#include #include #include #include /* CRC32C test vectors, adapted from linux crypto/testmgr.h and leveldb crc32c_test */ struct crc32c_testvec { uint8_t plaintext[240]; size_t psize; uint32_t value; }; static struct crc32c_testvec crc32c_testvectors[] = { { .plaintext = { 0x61 }, .psize = 1, .value = 0xc1d04330 }, { .plaintext = { 0x66, 0x6f, 0x6f }, .psize = 3, .value = 0xcfc4ae1d }, { .plaintext = { 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x77, 0x6f, 0x72, 0x6c, 0x64 }, .psize = 11, .value = 0xc99465aa }, { .plaintext = { 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x20 }, .psize = 6, .value = 0x7e627e58 }, { .plaintext = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, .psize = 32, .value = 0x8a9136aa }, { .plaintext = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, .psize = 32, .value = 0x62a8ab43 }, { .plaintext = { 0x1f, 0x1e, 0x1d, 0x1c, 0x1b, 0x1a, 0x19, 0x18, 0x17, 0x16, 0x15, 0x14, 0x13, 0x12, 0x11, 0x10, 0x0f, 0x0e, 0x0d, 0x0c, 0x0b, 0x0a, 0x09, 0x08, 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00 }, .psize = 32, .value = 0x113fdb5c }, { .plaintext = { 0x01, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x18, 0x28, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, .psize = 48, .value = 0xd9963a56 }, { .plaintext = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f }, .psize = 32, .value = 0x46dd794e }, { .plaintext = { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28 }, .psize = 40, .value = 0x0e2c157f }, { .plaintext = { 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50 }, .psize = 40, .value = 0xe980ebf6 }, { .plaintext = { 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78 }, .psize = 40, .value = 0xde74bded }, { .plaintext = { 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0 }, .psize = 40, .value = 0xd579c862 }, { .plaintext = { 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8 }, .psize = 40, .value = 0xba979ad0 }, { .plaintext = { 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0 }, .psize = 40, .value = 0x2b29d913 }, { .plaintext = { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0 }, .psize = 240, .value = 0x24c5d375 }, { .psize = 0 }, }; int main(int argc, char **argv) { struct crc32c_testvec *tv; int rc = EXIT_SUCCESS; for (tv = &crc32c_testvectors[0]; tv->psize != 0; tv++) { uint32_t crc = mtbl_crc32c(tv->plaintext, tv->psize); fprintf(stderr, "test-crc32c: %s: [actual=%08x, expected=%08x]\n", crc == tv->value ? "PASS" : "FAIL", tv->value, crc); if (crc != tv->value) rc = EXIT_FAILURE; } return (rc); } mtbl-0.5/t/test-deb716628.data000066400000000000000000000000301227302760400156130ustar00rootroot00000000000000AAAAAAAAAAAAAAAAAAAAvf„wmtbl-0.5/t/test-deb716628.sh000077500000000000000000000005501227302760400153260ustar00rootroot00000000000000#!/bin/sh if [ -z "${top_srcdir}" ]; then echo "top_srcdir variable not set" exit 1 fi if [ -z "${top_builddir}" ]; then echo "top_builddir variable not set" exit 1 fi echo "The following line should be an error message." "${top_builddir}/src/mtbl_dump" "${top_srcdir}/t/deb716628.data" 2>&1 if [ "$?" -eq "1" ]; then exit 0 fi exit 1 mtbl-0.5/t/test-fixed.c000066400000000000000000000033211227302760400150010ustar00rootroot00000000000000#include #include #include #include #include #include #define NAME "test-fixed" static int do_64(uint64_t val) { uint8_t buf[8]; mtbl_fixed_encode64(buf, val); fprintf(stderr, "%s: buf= 0x%02x%02x%02x%02x%02x%02x%02x%02x val= %" PRIu64 "\n", __func__, buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6], buf[7], val); if (mtbl_fixed_decode64(buf) != val) { fprintf(stderr, "%s: val= %" PRIu64 " failed\n", __func__, val); return (0); } return (1); } static int test2(void) { int ret = 0; unsigned count = 0; uint64_t val; if (!do_64(0) || !do_64(1) || !do_64(0xffffffffffffffffU)) ret |= 1; for (uint64_t i = 1; i < UINT64_MAX; i *= 17) { val = i++; if (!do_64(val)) ret |= 1; if (count++ > 10) break; } return (ret); } static int do_32(uint32_t val) { uint8_t buf[4]; mtbl_fixed_encode32(buf, val); fprintf(stderr, "%s: buf= 0x%02x%02x%02x%02x val= %u\n", __func__, buf[0], buf[1], buf[2], buf[3], val); if (mtbl_fixed_decode32(buf) != val) { fprintf(stderr, "%s: val= %u failed\n", __func__, val); return (0); } return (1); } static int test1(void) { int ret = 0; if (!do_32(0) || !do_32(1) || !do_32(0xffffffffU)) ret |= 1; for (uint64_t i = 1; i < UINT32_MAX; i *= 23) { uint32_t val = (uint32_t) i++; if (!do_32(val)) ret |= 1; } return (ret); } static int check(int ret, const char *s) { if (ret == 0) fprintf(stderr, NAME ": PASS: %s\n", s); else fprintf(stderr, NAME ": FAIL: %s\n", s); return (ret); } int main(int argc, char **argv) { int ret = 0; ret |= check(test1(), "test1"); ret |= check(test2(), "test2"); if (ret) return (EXIT_FAILURE); return (EXIT_SUCCESS); } mtbl-0.5/t/test-trailer.c000066400000000000000000000026361227302760400153540ustar00rootroot00000000000000#include #include #include #include #include #include #include #include "trailer.c" #define NAME "test-trailer" static int test2(void) { int ret = 0; struct trailer t; uint8_t tbuf[MTBL_TRAILER_SIZE]; memset(tbuf, 0, sizeof(tbuf)); if (trailer_read(tbuf, &t) != false) { fprintf(stderr, NAME ": trailer_read() should have failed but didn't\n"); ret |= 1; } return (ret); } static int test1(void) { int ret = 0; struct trailer t1, t2; uint8_t tbuf[MTBL_TRAILER_SIZE]; memset(tbuf, 0, sizeof(tbuf)); t1.index_block_offset = 123; t1.data_block_size = 65536; t1.compression_algorithm = 1; t1.count_entries = 2; t1.count_data_blocks = 3; t1.bytes_data_blocks = 4; t1.bytes_index_block = 5; t1.bytes_keys = 6; t1.bytes_values = 7; trailer_write(&t1, tbuf); if (!trailer_read(tbuf, &t2)) { fprintf(stderr, NAME ": trailer_read() failed\n"); ret |= 1; } if (memcmp(&t1, &t2, sizeof(struct trailer) != 0)) { fprintf(stderr, NAME ": t1 != t2\n"); ret |= 1; } return (ret); } static int check(int ret, const char *s) { if (ret == 0) fprintf(stderr, NAME ": PASS: %s\n", s); else fprintf(stderr, NAME ": FAIL: %s\n", s); return (ret); } int main(int argc, char **argv) { int ret = 0; ret |= check(test1(), "test1"); ret |= check(test2(), "test2"); if (ret) return (EXIT_FAILURE); return (EXIT_SUCCESS); } mtbl-0.5/t/test-varint.c000066400000000000000000000043771227302760400152210ustar00rootroot00000000000000#include #include #include #include #include #include #define NAME "test-varint" static int test2(void) { int ret = 0; uint8_t buf[8192]; uint8_t *end; uint8_t *p = buf; for (uint32_t i = 0; i < 64; i++) { const uint64_t power = 1ull << i; p += mtbl_varint_encode64(p, power); p += mtbl_varint_encode64(p, power - 1); p += mtbl_varint_encode64(p, power + 1); } end = p; p = buf; for (uint32_t i = 0; i < 64; i++) { const uint64_t power = 1ull << i; uint64_t actual, expected; p += mtbl_varint_decode64(p, &actual); expected = power; if (actual != expected) { ret |= 1; fprintf(stderr, "expected= %" PRIu64 ", actual= %" PRIu64 ", res= %u\n", expected, actual, actual == expected); } p += mtbl_varint_decode64(p, &actual); expected = power - 1; if (actual != expected) { ret |= 1; fprintf(stderr, "expected= %" PRIu64 ", actual= %" PRIu64 ", res= %u\n", expected, actual, actual == expected); } p += mtbl_varint_decode64(p, &actual); expected = power + 1; if (actual != expected) { ret |= 1; fprintf(stderr, "expected= %" PRIu64 ", actual= %" PRIu64 ", res= %u\n", expected, actual, actual == expected); } } if (p != end) { ret |= 1; fprintf(stderr, "p = %p, end = %p\n", p, end); } return (ret); } static int test1(void) { int ret = 0; uint8_t buf[8192]; uint8_t *end; uint8_t *p = buf; for (uint32_t i = 0; i < (32 * 32); i++) { uint32_t v = (i / 32) << (i % 32); p += mtbl_varint_encode32(p, v); } end = p; p = buf; for (uint32_t i = 0; i < (32 * 32); i++) { uint32_t expected = (i / 32) << (i % 32); uint32_t actual; p += mtbl_varint_decode32(p, &actual); if (expected != actual) { ret |= 1; fprintf(stderr, "expected= %u, actual= %u\n", expected, actual); } } if (p != end) { ret |= 1; fprintf(stderr, "p = %p, end = %p\n", p, end); } return (ret); } static int check(int ret, const char *s) { if (ret == 0) fprintf(stderr, NAME ": PASS: %s\n", s); else fprintf(stderr, NAME ": FAIL: %s\n", s); return (ret); } int main(int argc, char **argv) { int ret = 0; ret |= check(test1(), "test1"); ret |= check(test2(), "test2"); if (ret) return (EXIT_FAILURE); return (EXIT_SUCCESS); } mtbl-0.5/t/test-vector.c000066400000000000000000000067231227302760400152150ustar00rootroot00000000000000#include #include #include #include #include #include #include #include #include "mtbl-private.h" #include "libmy/ubuf.h" #define NAME "test-vector" VECTOR_GENERATE(uint32_vec, uint32_t); const uint32_t u32_array[] = { 23, 17, 1, 2, 3, 4, 5, 4, 3, 2, 1, 17, 23 }; #define NUM_U32_ARRAY 13 const char str_array[] = "`1234567890[]',.PYFGCRL/=AOEUIDHTNS-;QJKXBMWVZ"; static int test6(void) { int ret = 0; ubuf *v = ubuf_init(16); ubuf_append(v, (uint8_t *) str_array, sizeof(str_array)); ubuf_clip(v, 17); if (ubuf_size(v) != 17) ret |= 1; return (ret); } static int test5(void) { int ret = 0; uint32_vec *v; v = uint32_vec_init(0); for (unsigned i = 0; i < 16; i++) uint32_vec_append(v, u32_array, sizeof(u32_array) / sizeof(uint32_t)); for (unsigned i = 0; i < uint32_vec_size(v); i++) { //fprintf(stderr, "v->v[%d] = %u\n", i, uint32_vec_value(v, i)); if (uint32_vec_value(v, i) != u32_array[i % NUM_U32_ARRAY]) ret |= 1; } uint32_vec_reset(v); for (uint32_t i = 0; i < 1024; i++) uint32_vec_add(v, i); for (uint32_t i = 0; i < 1024; i++) { //fprintf(stderr, "v->v[%d] = %u\n", i, v->v[i]); if (uint32_vec_value(v, i) != i) ret |= 1; } uint32_vec_destroy(&v); return (ret); } static int test4(void) { int ret = 0; ubuf *v; v = ubuf_init(0); for (unsigned i = 0; i < 1024; i++) ubuf_append(v, (uint8_t *) str_array, sizeof(str_array)); for (unsigned i = 0; i < ubuf_size(v); i++) { unsigned j = i % sizeof(str_array); /* fprintf(stderr, "v->v[%d] = '%c', str_array[%d] = '%c'\n", i, v->v[i], j, str_array[j]); */ if (ubuf_value(v, i) != str_array[j]) ret |= 1; } ubuf_destroy(&v); return (ret); } static int test3(void) { int ret = 0; ubuf *v; v = ubuf_init(16); ubuf_append(v, (uint8_t *) str_array, sizeof(str_array)); for (unsigned i = 0; i < ubuf_size(v); i++) { //fprintf(stderr, "v->v[%d] = '%c'\n", i, v->v[i]); if (ubuf_value(v, i) != str_array[i]) ret |= 1; } ubuf_destroy(&v); return (ret); } static int test2(void) { int ret = 0; uint32_vec *v; v = uint32_vec_init(0); uint32_vec_append(v, u32_array, sizeof(u32_array) / sizeof(uint32_t)); for (uint32_t i = 0; i < uint32_vec_size(v); i++) { if (uint32_vec_value(v, i) != u32_array[i]) ret |= 1; //fprintf(stderr, "v->v[%d] = %u\n", i, v->v[i]); } if (uint32_vec_size(v) != NUM_U32_ARRAY) ret |= 1; uint32_vec_destroy(&v); return (ret); } static int test1(void) { int ret = 0; uint32_vec *v; v = uint32_vec_init(0); for (uint32_t i = 0; i < 1024; i++) uint32_vec_add(v, i); if (uint32_vec_bytes(v) != uint32_vec_size(v) * sizeof(uint32_t)) { fprintf(stderr, "uint32_vec_bytes (%zd) != v->n * sizeof(uint32_t) (%zd)\n", uint32_vec_bytes(v), uint32_vec_size(v) * sizeof(uint32_t)); ret |= 1; } for (uint32_t i = 0; i < 1024; i++) { //fprintf(stderr, "v->v[%d] = %u\n", i, v->v[i]); if (uint32_vec_value(v, i) != i) ret |= 1; } uint32_vec_destroy(&v); return (ret); } static int check(int ret, const char *s) { if (ret == 0) fprintf(stderr, NAME ": PASS: %s\n", s); else fprintf(stderr, NAME ": FAIL: %s\n", s); return (ret); } int main(int argc, char **argv) { int ret = 0; ret |= check(test1(), "test1"); ret |= check(test2(), "test2"); ret |= check(test3(), "test3"); ret |= check(test4(), "test4"); ret |= check(test5(), "test5"); ret |= check(test6(), "test6"); if (ret) return (EXIT_FAILURE); return (EXIT_SUCCESS); }